Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/build-daemon.yml
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ jobs:
os: macos-15-intel
platform: darwin-x64
- target: aarch64-apple-darwin
os: macos-14
os: macos-15
platform: darwin-arm64
runs-on: ${{ matrix.os }}
# Nothing in this job has a bounded runtime of its own, and GitHub's default
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/publish.yml
Original file line number Diff line number Diff line change
Expand Up @@ -842,7 +842,7 @@ jobs:
platform: linux-arm64
- os: macos-15-intel
platform: darwin-x64
- os: macos-14
- os: macos-15
platform: darwin-arm64
runs-on: ${{ matrix.os }}
timeout-minutes: 20
Expand Down
20 changes: 20 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,25 @@
# Changelog

## 1.0.14-beta.0 — 2026-11-10

### Fixes

- Keep telemetry retryable through prolonged network and server outages, retain interrupted acknowledgements, and preserve permanent rejections for inspection. (#997)
- Build the macOS ARM daemon on the supported macos-15 hosted runner. (#997)
- Count partial acknowledgements separately from outage attempts so a recovered endpoint can finish delivery. (#997)
- Keep credential-refused batches recoverable after an outage without exhausting the skipped-acknowledgement budget. (#997)
- Run the ARM installation check on macos-15 so a release can complete. (#997)
- Keep CLI delivery-status filename parsing aligned with the independent retry counters. (#997)

### Docs

- Clarify ingestion retries and give every public OpenAPI operation a unique ID. (#997)
- Explain manual OTLP credential-refusal recovery within the original organisation. (#997)

### Dependencies

- Update the Python SDK test dependency oauthlib to 4.0.1 to fix GHSA-3jp2-3mrg-2vxc and unblock the security gate. (#997)

## 1.0.14-beta.0 — 2026-11-02

### Docs
Expand Down
17 changes: 17 additions & 0 deletions __tests__/hooks/delivery-health.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,23 @@ describe("hooks/delivery-health parseParkedName", () => {
expect(parsed.base).toBe("claude-0");
});

it.each([
["claude-0.a99.s1.r2.c401.jsonl", 99, 401, false],
["claude-0.a5.s1.jsonl", 5, undefined, false],
["claude-0.a2.r1.c403.jsonl", 2, 403, false],
["claude-0.a9.s3.r2.c422.jsonl.poison", 9, 422, true],
])("reads independent retry budgets in %s", (name, attempt, clientStatus, poison) => {
expect(parseParkedName(name as string)).toEqual({ base: "claude-0", attempt, clientStatus, poison });
});

it.each(["claude-0.a5.soon.jsonl", "claude-0.a5.r-1.jsonl", "claude-0.a5.r4294967296.jsonl"])(
"leaves an invalid retry suffix in %s alone",
(name) => {
expect(parseParkedName(name).base).toBe(name.slice(0, -".jsonl".length));
expect(parseParkedName(name).attempt).toBe(0);
},
);

it("leaves a non-numeric .c/.a in the base alone", () => {
// Mirrors the Rust comment: parsing rather than merely matching the prefix
// is what keeps a real spool name safe.
Expand Down
8 changes: 4 additions & 4 deletions crates/fpai-collect/src/delivery.rs
Original file line number Diff line number Diff line change
Expand Up @@ -628,7 +628,7 @@ mod tests {
.await;
let failed = tmpdir("redrive-failed");
let spool = tmpdir("redrive-spool");
let refused_again = failed.join("a-0.a1.c401.jsonl");
let refused_again = failed.join("a-0.a1.r1.c401.jsonl");
let resendable = failed.join("b-0.a1.c403.jsonl");
let not_credential = failed.join("c-0.a1.c404.jsonl");
std::fs::write(&refused_again, "{\"refused\":1}\n").unwrap();
Expand Down Expand Up @@ -656,12 +656,12 @@ mod tests {
"refused again by that key: renamed into another park"
);
assert_eq!(
std::fs::read(failed.join("a-0.a2.c401.jsonl")).unwrap(),
std::fs::read(failed.join("a-0.a2.r2.c401.jsonl")).unwrap(),
b"{\"refused\":1}\n"
);
delivery.redrive_refused().await;
assert_eq!(
std::fs::read(failed.join("a-0.a3.c401.jsonl.poison")).unwrap(),
std::fs::read(failed.join("a-0.a3.r3.c401.jsonl.poison")).unwrap(),
b"{\"refused\":1}\n",
"exhausting retries must preserve the last local copy too"
);
Expand Down Expand Up @@ -770,7 +770,7 @@ mod tests {
));
delivery.deliver(fresh).await;
assert!(old.exists(), "a key that is refused re-sends nothing");
assert!(failed.join("hooks-a-1-0.a1.c401.jsonl").exists());
assert!(failed.join("hooks-a-1-0.a1.r1.c401.jsonl").exists());
std::fs::remove_dir_all(failed).unwrap();
std::fs::remove_dir_all(spool).unwrap();
}
Expand Down
160 changes: 152 additions & 8 deletions crates/fpai-collect/src/otlp.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,14 @@
//! Opaque OTLP/HTTP relay. No translation, decompression or event-uploader reuse.
//! Each batch is a JSON metadata line followed by the exact request bytes.
//! SDK exporters publish the same format; only atomic .jsonl files are read.
//!
//! Manual recovery of credential refusals (401/403): stop the daemon and verify
//! a working `events:add` key for the SAME Cloud origin and organization. Back up
//! the affected files under `<FAILPROOFAI_HOME>/state/failed/otlp/`, then move only
//! those batches into `state/spool-otlp/` with fresh `otlp-<unique>.jsonl` names,
//! never overwriting another file or changing their metadata/body bytes. Restart
//! the daemon and confirm delivery before removing the backup. Never recover
//! queues archived by `org_boundary` into a different organization's spool.
use std::fs::{self, File, OpenOptions};
use std::io::{self, Write};
use std::net::{Ipv4Addr, SocketAddr};
Expand Down Expand Up @@ -175,9 +183,21 @@ impl Relay {
// Bound response size too; an ingest proxy need not be trusted.
let mut response = response;
let mut result = Vec::new();
while let Ok(Some(chunk)) = response.chunk().await {
loop {
let chunk = match response.chunk().await {
Ok(Some(chunk)) => chunk,
Ok(None) => break,
Err(_) => {
self.report(
None,
Some("response interrupted; retained".into()),
);
return Outcome::Unreachable;
}
};
if result.len() + chunk.len() > 64 * 1024 {
break;
self.park(path, Some(code)).await;
return Outcome::Settled;
}
result.extend_from_slice(&chunk);
}
Expand All @@ -190,7 +210,7 @@ impl Relay {
return Outcome::Settled;
}
self.report(None, Some(format!("{code} parked")));
if !matches!(code, 429 | 502 | 503 | 504) {
if !retryable_status(code) {
break;
}
delay = retry_after(response.headers()).unwrap_or(delay);
Expand Down Expand Up @@ -219,16 +239,23 @@ impl Relay {
return;
};
let mut parked = ParkedName::parse(name);
parked.attempt += 1;
parked.client_status = status.filter(|c| (400..500).contains(c) && *c != 429);
parked.poison = parked.attempt >= 3;
parked.attempt = parked.attempt.saturating_add(1);
parked.client_status = status.filter(|c| !retryable_status(*c));
if !status.is_some_and(retryable_status) {
parked.client_attempts = parked.client_attempts.saturating_add(1);
}
parked.poison = !status.is_some_and(retryable_status) && parked.client_attempts >= 3;
let target = self.failed.join(parked.render());
if tokio::fs::rename(path, target).await.is_ok() {
let _ = File::open(&self.failed).and_then(|f| f.sync_all());
}
}
}

fn retryable_status(code: u16) -> bool {
matches!(code, 408 | 429 | 500..=599)
}

fn retry_after(headers: &HeaderMap) -> Option<Duration> {
let value = headers.get("retry-after")?.to_str().ok()?;
if let Ok(seconds) = value.parse::<u64>() {
Expand Down Expand Up @@ -454,7 +481,10 @@ async fn deliver_loop(relay: Arc<Relay>, sd: Shutdown) -> Result<(), TaskError>
let mut delivered = 0;
while let Ok(Some(entry)) = entries.next_entry().await {
let name = entry.file_name().to_string_lossy().to_string();
if !name.starts_with("otlp-") || !name.ends_with(".jsonl") {
if !name.starts_with("otlp-")
|| !name.ends_with(".jsonl")
|| !ParkedName::parse(&name).is_auto_retryable()
{
continue;
}
if entry.file_type().await.is_ok_and(|t| t.is_file()) {
Expand Down Expand Up @@ -679,13 +709,127 @@ mod tests {
parked
.to_str()
.unwrap()
.contains(&format!(".a1.c{code}.jsonl"))
.contains(&format!(".a1.r1.c{code}.jsonl"))
);
assert_eq!(fs::read(&parked).unwrap(), original);
assert!(
!ParkedName::parse(parked.file_name().unwrap().to_str().unwrap())
.is_auto_retryable()
);
fs::remove_dir_all(home).unwrap();
}
}

#[tokio::test]
async fn temporary_http_failures_remain_retryable_after_four_parks() {
for status in [429, 500, 503] {
let server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(status))
.expect(20)
.mount(&server)
.await;
let home = temp();
let mut relay = Relay::new(
&home,
&Ingest {
url: server.uri(),
key: "test".into(),
},
)
.unwrap();
relay.retry_base = Duration::from_millis(1);
let mut file = batch(&relay, b"kept");
let original = fs::read(&file).unwrap();
for _ in 0..4 {
relay.deliver(&file, &shutdown()).await;
file = fs::read_dir(&relay.failed)
.unwrap()
.next()
.unwrap()
.unwrap()
.path();
assert!(
ParkedName::parse(file.file_name().unwrap().to_str().unwrap())
.is_auto_retryable()
);
assert_eq!(fs::read(&file).unwrap(), original);
}
server.verify().await;
server.reset().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(401))
.expect(1)
.mount(&server)
.await;
relay.deliver(&file, &shutdown()).await;
file = fs::read_dir(&relay.failed)
.unwrap()
.next()
.unwrap()
.unwrap()
.path();
let state = ParkedName::parse(file.file_name().unwrap().to_str().unwrap());
assert_eq!(state.client_status, Some(401));
assert!(!state.poison);
assert!(!state.is_auto_retryable());
assert_eq!(fs::read(&file).unwrap(), original);
server.verify().await;
server.reset().await;
Mock::given(method("POST"))
.and(header("authorization", "Bearer replacement"))
.respond_with(ResponseTemplate::new(200))
.expect(1)
.mount(&server)
.await;
// Follow the documented manual requeue after fixing same-org credentials.
relay.key = "replacement".into();
let requeued = relay.spool.join(format!("otlp-{}.jsonl", new_request_id()));
fs::rename(&file, &requeued).unwrap();
assert_eq!(fs::read(&requeued).unwrap(), original);
assert!(
ParkedName::parse(requeued.file_name().unwrap().to_str().unwrap())
.is_auto_retryable()
);
file = requeued;
relay.deliver(&file, &shutdown()).await;
assert!(!file.exists());
server.verify().await;
fs::remove_dir_all(home).unwrap();
}
}

#[tokio::test]
async fn interrupted_success_response_keeps_the_spool_batch() {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let url = format!("http://{}", listener.local_addr().unwrap());
let server = tokio::spawn(async move {
let (mut stream, _) = listener.accept().await.unwrap();
let mut request = [0; 4096];
assert!(stream.read(&mut request).await.unwrap() > 0);
stream.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nContent-Type: application/json\r\nConnection: close\r\n\r\n{").await.unwrap();
});
let home = temp();
let relay = Relay::new(
&home,
&Ingest {
url,
key: "key".into(),
},
)
.unwrap();
let file = batch(&relay, b"kept");
let original = fs::read(&file).unwrap();
assert!(matches!(
relay.deliver(&file, &shutdown()).await,
Outcome::Unreachable
));
assert_eq!(fs::read(&file).unwrap(), original);
server.await.unwrap();
fs::remove_dir_all(home).unwrap();
}

#[tokio::test]
async fn transport_failures_never_poison_and_the_batch_delivers_once_reachable() {
let reserved = std::net::TcpListener::bind((Ipv4Addr::LOCALHOST, 0)).unwrap();
Expand Down
Loading
Loading