From 9758487373756c7604dbf8b19077aec057ffee6d Mon Sep 17 00:00:00 2001 From: Tyagiquamar Date: Thu, 19 Nov 2026 14:45:02 +0530 Subject: [PATCH] fix(compression): dedupe large blocks by full content --- src/compression/layers/observation.test.ts | 32 ++++++++++++++++++++++ src/compression/layers/observation.ts | 6 ++-- 2 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 src/compression/layers/observation.test.ts diff --git a/src/compression/layers/observation.test.ts b/src/compression/layers/observation.test.ts new file mode 100644 index 00000000..5794f88f --- /dev/null +++ b/src/compression/layers/observation.test.ts @@ -0,0 +1,32 @@ +import { describe, expect, it } from "vitest"; + +import { compressObservations } from "./observation.js"; +import type { NormalizedMessage } from "../types.js"; + +describe("compressObservations large block deduplication", () => { + it("keeps distinct messages that share only a leading 200 characters", () => { + const sharedPrefix = "The quick brown fox jumps over the lazy dog. ".repeat(5); + const firstContent = `${sharedPrefix}\nALPHA_UNIQUE ${"a".repeat(400)}`; + const secondContent = `${sharedPrefix}\nBETA_UNIQUE ${"b".repeat(400)}`; + const first: NormalizedMessage = { role: "assistant", content: firstContent }; + const second: NormalizedMessage = { role: "assistant", content: secondContent }; + + expect(firstContent.slice(0, 200)).toBe(secondContent.slice(0, 200)); + + const result = compressObservations([first, second]); + + expect(result.messages[1].content).toContain("BETA_UNIQUE"); + expect(result.messages[1].content).not.toContain("same content"); + }); + + it("still drops a genuinely repeated large block", () => { + const sharedPrefix = "The quick brown fox jumps over the lazy dog. ".repeat(5); + const content = `${sharedPrefix}\nIDENTICAL ${"c".repeat(400)}`; + const first: NormalizedMessage = { role: "assistant", content }; + const second: NormalizedMessage = { role: "assistant", content }; + + const result = compressObservations([first, second]); + + expect(result.messages[1].content).toContain("same content"); + }); +}); diff --git a/src/compression/layers/observation.ts b/src/compression/layers/observation.ts index ccb93d1e..aceebbc0 100644 --- a/src/compression/layers/observation.ts +++ b/src/compression/layers/observation.ts @@ -112,8 +112,10 @@ function deduplicateLargeBlocks(messages: NormalizedMessage[]): { return msg; } - // Hash first 200 chars as block identifier - const blockKey = msg.content.slice(0, 200); + // Use the full content as the block identifier. A 200-char prefix can match + // two messages whose tails differ, which would drop distinct information and + // emit a false "same content" pointer for the second message. + const blockKey = msg.content; if (blockHashes.has(blockKey)) { const firstIdx = blockHashes.get(blockKey)!;