From 0605aaf5b125bd5c85a0683970aecd87e536b415 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 15:46:05 +0100 Subject: [PATCH 1/2] A PDF cut short loses the share of its pages asked for A PDF a damage cuts splits its padding comment around its objects: part of it right after the header lines, in front of the first object, and the rest behind the trailer, so the cut keeps the share of the objects it keeps of the whole file. Before, all of the padding stood behind the trailer, and a cut of any share left every page. Every offset in the cross reference table counts the comment in front, and each is ten digits, so only startxref grows - a digit at a power of ten, which the padding behind gives up. Each side is nothing or at least two bytes. Objects stop where ten digits do (ISO 32000-1 section 7.5.4). Documents nothing cuts keep their bytes. The reader of maps takes an ASCII comment block between the header and the first object as padding. pdf leaves the share ratchet and joins the formats whose end closes the file, an MP4 is the manifest guard's witness of a cut in the padding behind, and the reach check of the smallest cut names the ZIP formats. A new guard writes consecutive sizes across the digit startxref gains and where the padding behind runs out, and plans a 20 GiB document. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 + internal/format/pdf/cut.go | 62 ++++++++ internal/format/pdf/document.go | 22 ++- internal/format/pdf/minimum.go | 2 +- internal/format/pdf/padding.go | 3 +- internal/format/pdf/pdf.go | 42 +++-- internal/guard/cutmanifest_test.go | 18 ++- internal/guard/cutshare_test.go | 4 +- internal/guard/mutationcoverage_test.go | 4 + internal/guard/pdfcut_test.go | 197 ++++++++++++++++++++++++ internal/oracle/layout.py | 21 ++- 11 files changed, 350 insertions(+), 33 deletions(-) create mode 100644 internal/format/pdf/cut.go create mode 100644 internal/guard/pdfcut_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 440f06e4..9b86db3d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,14 @@ because it turns other people's test suites red. the unexpected end. When the cut is shorter than the end of the archive - 1% of 64 KiB - it falls there and every file is kept, and the manifest says `in: end`. Files nothing cuts are unchanged. +- **`truncate` cuts the pages of a PDF by the percentage you set.** + `--damage truncate:missing=50` on a 1 MB PDF used to cut only padding, and + viewers showed every page. Now it removes half of the document and the + index at its end, as an interrupted download does. pdf.js (Firefox), + PDFium (Chrome, Edge), Poppler and the Windows PDF viewer refuse it, and + MuPDF and Ghostscript show the part that arrived. A cut of a few percent of + a small PDF can fall in the index, keeping every page - the manifest says + `in: end`. Files nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/pdf/cut.go b/internal/format/pdf/cut.go new file mode 100644 index 00000000..e4b6d516 --- /dev/null +++ b/internal/format/pdf/cut.go @@ -0,0 +1,62 @@ +package pdf + +// The layout of a document a damage will cut: its padding split around the +// objects. + +import "github.com/donislawdev/TestingFilesGenerator/internal/format" + +// largestOffset is the largest offset a cross reference entry can carry. ISO +// 32000-1 section 7.5.4 writes each one in ten digits. +const largestOffset = 9_999_999_999 + +// splitForACut is how much of the padding m has behind its trailer goes in +// front of its objects instead, for a document a damage will cut after its +// first cutAt bytes of whole, so that the cut keeps the share of the objects it +// keeps of the whole file (format.Request.CutAt). It returns the padding in +// front and the padding left behind, and none in front for a document nothing +// cuts. +// +// With all of the padding behind the trailer, as a document nothing cuts has +// it, a large document cut at any share kept every object and lost only +// padding. A comment may stand between the header lines and the first object as +// well, and the readers measured take it there as they take it behind the +// trailer (docs/TRUNCATE-2026-11-11.md section 12). +// +// The padding in front moves every object, so the offset startxref carries +// grows with it and can gain a digit, which the padding behind gives up. Each +// side is nothing or a comment of at least minComment bytes. Where the sides +// cannot land exactly the front takes a byte more, which cuts a byte more of +// the objects - the "at least" missing promises - or, where a digit gained +// leaves that no room, a byte less. Where the end is longer than the part the +// cut takes, all of the padding goes in front, and the cut falls in the end. +// An object can start no later than largestOffset, so in a document of more +// than about 9.3 GiB the front stops there and the cut keeps more of the +// objects than the share. +func splitForACut(m memo, whole, cutAt int64) (front, back int64) { + if m.padLen == 0 || cutAt <= 0 || cutAt >= whole { + return 0, m.padLen + } + want := format.PaddingInFront(m.at.objects, m.at.xref-m.at.objects, cutAt, whole) + if want <= 0 { + return 0, m.padLen + } + // padding is all of the padding with front of it in front: the closing + // lines carry the offset front moves. + padding := func(front int64) int64 { + return whole - int64(len(m.prefix)) - int64(len(closing(m.at.xref+front))) + } + fits := func(front int64) bool { + back := padding(front) - front + return front >= minComment && m.at.xref+front <= largestOffset && (back == 0 || back >= minComment) + } + front = min(max(want, minComment), largestOffset-m.at.xref) + if padding(front) < front { + front = padding(front) + } + for _, f := range []int64{front, front + 1, front - 1} { + if fits(f) { + return f, padding(f) - f + } + } + return 0, m.padLen +} diff --git a/internal/format/pdf/document.go b/internal/format/pdf/document.go index 6c58a75f..6dac67fe 100644 --- a/internal/format/pdf/document.go +++ b/internal/format/pdf/document.go @@ -14,12 +14,17 @@ type places struct { objects, xref int64 } -// document builds everything up to the trailer, and the closing lines. +// document builds everything up to the trailer, and the closing lines, of a +// document with front bytes of padding between its header lines and its first +// object (splitForACut). // // The two are returned apart because the padding goes between them. The // closing lines carry the offset of the cross reference table, which sits in -// the first part and therefore does not move. -func document(m memo) (prefix, suffix []byte, at places) { +// the first part and therefore does not move. The front padding is not in the +// first part - the writer puts it in at at.objects - but every offset counts +// it. The cross reference table writes each offset in ten digits, so the first +// part is the same length whatever front is. +func document(m memo, front int64) (prefix, suffix []byte, at places) { var body bytes.Buffer // The version is the one asked for and nothing else follows from it: the // file uses nothing newer than 1.4, so every version offered here is true @@ -73,10 +78,15 @@ func document(m memo) (prefix, suffix []byte, at places) { fmt.Fprintf(&body, "xref\n0 %d\n", len(objects)+1) body.WriteString("0000000000 65535 f \n") for _, off := range offsets { - fmt.Fprintf(&body, "%010d 00000 n \n", off) + fmt.Fprintf(&body, "%010d 00000 n \n", int64(off)+front) } fmt.Fprintf(&body, "trailer\n<>\n", len(objects)+1) - suffix = []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF\n", xrefPos)) - return body.Bytes(), suffix, at + return body.Bytes(), closing(at.xref + front), at +} + +// closing is startxref, the offset of the cross reference table it points at, +// and the end of file marker. +func closing(xref int64) []byte { + return []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF\n", xref)) } diff --git a/internal/format/pdf/minimum.go b/internal/format/pdf/minimum.go index e9967762..91f2fb81 100644 --- a/internal/format/pdf/minimum.go +++ b/internal/format/pdf/minimum.go @@ -63,6 +63,6 @@ func minimumBytes() int64 { if err != nil { panic("pdf: the default settings are refused: " + err.Error()) } - prefix, suffix, _ := document(memo{opts: opts}) + prefix, suffix, _ := document(memo{opts: opts}, 0) return int64(len(prefix) + len(suffix)) } diff --git a/internal/format/pdf/padding.go b/internal/format/pdf/padding.go index 5b6352e9..1ac92923 100644 --- a/internal/format/pdf/padding.go +++ b/internal/format/pdf/padding.go @@ -21,7 +21,8 @@ const ( ) // The padding channel is a comment block placed after the trailer and before -// startxref. +// startxref. A document a damage will cut has part of it in front of the +// objects instead (splitForACut). // // The obvious place, and the one this project's own notes assumed, is between // startxref and %%EOF. Measured, that is wrong twice over. diff --git a/internal/format/pdf/pdf.go b/internal/format/pdf/pdf.go index d1e132b2..09136514 100644 --- a/internal/format/pdf/pdf.go +++ b/internal/format/pdf/pdf.go @@ -51,9 +51,12 @@ type memo struct { // not depend on the padding, because the offset it carries points at the // cross reference table, which sits before the padding. suffix []byte - // at is where the objects and the cross reference table start. - at places - padLen int64 + // at is where the objects and the cross reference table start in prefix. + at places + // front is the padding between the header lines and the first object, + // which only a document a damage will cut has (splitForACut), and padLen + // the padding behind the trailer. + front, padLen int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -68,7 +71,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } m := memo{opts: opts, seed: r.Seed, label: label} - m.prefix, m.suffix, m.at = document(m) + m.prefix, m.suffix, m.at = document(m, 0) // One number answers both questions: how much padding this file needs, and // what is too small to make. Every line of body text is the same width @@ -108,6 +111,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) { default: m.padLen = r.Bytes - bare } + if m.front, m.padLen = splitForACut(m, r.Bytes, r.CutAt); m.front > 0 { + m.prefix, m.suffix, _ = document(m, m.front) + } p.Memo = m return p, nil @@ -170,18 +176,28 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } - // The map, all of it known from the plan: the header lines, the objects - - // the pages and what they draw with - the cross reference table and the + // The map, all of it known from the plan: the header lines, the comment + // in front of the objects a document a damage cuts has, the objects - the + // pages and what they draw with - the cross reference table and the // trailer, the comment that makes up the size, and startxref with the - // closing marker. The padding stands between two parts of the end. - head, end := int64(len(m.prefix)), int64(len(m.prefix))+m.padLen + // closing marker. The padding behind stands between two parts of the end. + objects := m.at.objects + m.front + table := m.at.xref + m.front + behind := int64(len(m.prefix)) + m.front format.Mark(w, format.RoleStructure, 0, m.at.objects) - format.Mark(w, format.RoleContent, m.at.objects, m.at.xref-m.at.objects) - format.Mark(w, format.RoleEnd, m.at.xref, head-m.at.xref) - format.Mark(w, format.RolePadding, head, m.padLen) - format.Mark(w, format.RoleEnd, end, int64(len(m.suffix))) + format.Mark(w, format.RolePadding, m.at.objects, m.front) + format.Mark(w, format.RoleContent, objects, table-objects) + format.Mark(w, format.RoleEnd, table, behind-table) + format.Mark(w, format.RolePadding, behind, m.padLen) + format.Mark(w, format.RoleEnd, behind+m.padLen, int64(len(m.suffix))) - if _, err := w.Write(m.prefix); err != nil { + if _, err := w.Write(m.prefix[:m.at.objects]); err != nil { + return err + } + if err := writeComment(ctx, w, m.seed, m.front); err != nil { + return err + } + if _, err := w.Write(m.prefix[m.at.objects:]); err != nil { return err } if err := writeComment(ctx, w, m.seed, m.padLen); err != nil { diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 53b2ebc0..2e11a0c8 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -3,6 +3,7 @@ package guard import ( "fmt" "os" + "slices" "strconv" "strings" "testing" @@ -36,8 +37,8 @@ type cutCase struct { // the content is left, and gets a note when the cut missed the content. // // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). -// A PDF of 1 MiB is nearly all padding after its pages, so 90% missing still -// leaves every page. A PNG splits its padding around the picture +// An MP4 of 1 MiB is nearly all padding after its video, so 90% missing still +// leaves all of it. A PNG splits its padding around the picture // for a cut, so the cut takes the share asked for, and so does a tar.gz. A // text is its words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. @@ -46,8 +47,9 @@ type cutCase struct { // one that claims content it does not have. func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ - // A GIF was this case until it split its padding for the cut. - {name: "behind", format: "pdf", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // A GIF and then a PDF were this case until each split its padding + // for the cut. + {name: "behind", format: "mp4", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, // 4 KiB on disk is 40 KiB whole. It was the case of a cut in the padding // in front of the content, with an icon and a JPEG before it, until it @@ -178,7 +180,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("behind", "pdf", 1<<20, "90"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "mp4", 1<<20, "90"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) @@ -262,7 +264,11 @@ func TestACutAtTheSmallestSizeSaysHowLongTheContentWas(t *testing.T) { } cut := *m.Damage[0].Cut whole := wholeMap(t, plannedOf[m.TargetID]) - if _, zipped := endClosesTheFile[m.TargetID]; zipped && (paddingInFront(whole) || cut.In != string(format.RoleContent)) { + // Named rather than read off endClosesTheFile, which a PDF and a + // tar.gz joined: a PDF at its smallest has room for its comment in + // front, as it should. + zipped := slices.Contains([]string{"zip", "docx", "xlsx", "pptx"}, m.TargetID) + if zipped && (paddingInFront(whole) || cut.In != string(format.RoleContent)) { t.Errorf("%s: at %d B the cut fell in the %s with padding in front of the content: %v - the layout too short for a padding part in front is not reached", m.TargetID, m.Bytes, cut.In, paddingInFront(whole)) } diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 35be63d4..dadb21aa 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -36,7 +36,6 @@ var notYetCutAtTheShare = map[string]notYet{ "jxl": {paddingBehind, 64 << 10, 50}, "mp4": {paddingBehind, 64 << 10, 50}, "webm": {paddingBehind, 64 << 10, 50}, - "pdf": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, @@ -51,7 +50,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "webp", "tiff"} +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. @@ -211,6 +210,7 @@ var endClosesTheFile = map[string]string{ "xlsx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", "pptx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", "targz": "POSIX tar puts the end of archive marker after every entry, and RFC 1952 closes a gzip member with its trailer", + "pdf": "ISO 32000-1 section 7.5 puts the cross reference table and the trailer after the body, and the end of file marker last", } // onlyTheEndBehind is whether nothing but the end stands behind the content of diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index e86b7f0a..a5029e07 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -159,6 +159,10 @@ var provenByProbe = map[string]string{ "fixed codes, must inflate to its bytes and read as those runs - its reach check, every frame of the marker met - and red on " + "2026-11-13 for a frame a byte short (the plan refuses at 65541 B) and for an empty block's BTYPE bit one place off " + "(compress/flate refuses at 65537 B), where the first passed the share and the map guards.", + "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canary - a table that sends one object a byte off " + + "must be refused - its reach check, every layout of the split met, and red on 2026-11-13 without the candidate of a " + + "byte more in front (that layout never met), with the length of startxref taken before the padding moved it (10212 B " + + "came to 10213 B) and without the stop at ten digits (20 GiB, objects ending at 19327353153).", "TestATarGzCutNearItsSmallestComesToEverySize": "its reach check - the frame met, and without a label the size after its floor " + "that keeps the layout of an archive nothing cuts, which with a label is due never - and red on 2026-11-13 without the " + "case of one byte over the floor (an internal defect at 9757 B) and with that case taken for a label too (label true).", diff --git a/internal/guard/pdfcut_test.go b/internal/guard/pdfcut_test.go new file mode 100644 index 00000000..4be7c378 --- /dev/null +++ b/internal/guard/pdfcut_test.go @@ -0,0 +1,197 @@ +package guard + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" +) + +// A PDF laid out for a cut comes to every size asked for, every offset in it +// lands, and the cut keeps the share of its objects. +// +// The padding in front of the objects moves every one of them, so the offset +// startxref carries grows with it and gains a digit at a power of ten, which +// the padding behind gives up (internal/format/pdf/cut.go). Each side is +// nothing or a comment of two bytes or more, so where the padding behind would +// come to one byte the front takes a byte more, and where that byte gains the +// digit the front takes a byte less instead. No other guard asks for the sizes +// where that happens. So this writes consecutive sizes cut at their last byte +// across the size where startxref gains its fifth digit, and at 1% and 50% +// where the padding behind runs out, holds each to its size, its offsets and +// the share, and asserts that it met the layouts that follow: all of the +// padding in front with four digits and with five, a few bytes left behind +// where the digit is gained, the share to the byte, and a byte more in front. +// +// A document of more than ten gigabytes is planned and not written: its +// objects stop where the cross reference table's ten digits do. +// +// Canary: the reader of offsets refuses a document whose table sends one +// object a byte off. +func TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand(t *testing.T) { + desc, err := format.Get("pdf") + if err != nil { + t.Fatal(err) + } + met := map[string]bool{} + sweep := func(from, to int64, cutAt func(size int64) int64) { + for size := from; size < to; size++ { + var below *format.BelowMinimumError + if _, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true}); errors.As(err, &below) { + // The one size in the gap just above the bare document. + continue + } + data, regions := writePDF(t, desc, size, cutAt(size)) + layout, err := pdfLayoutOf(data, regions, size, cutAt(size)) + if err != nil { + t.Errorf("%d B cut at %d: %v", size, cutAt(size), err) + continue + } + met[layout] = true + } + } + sweep(9_900, 10_400, func(size int64) int64 { return size - 1 }) + sweep(22_000, 24_600, func(size int64) int64 { return size - (size+99)/100 }) + smallest := desc.SmallestAccepted(format.Request{Label: true}) + sweep(smallest, smallest+600, func(size int64) int64 { return size / 2 }) + for _, layout := range []string{"all in front, startxref of 4 digits", "all in front, startxref of 5 digits", + "a few bytes behind where startxref gains a digit", "the share", "a byte more in front"} { + if !met[layout] { + t.Errorf("no size met the layout %q - met %v", layout, met) + } + } + + pdfTooLargeForTheTable(t, desc) + + // Canary: one object a byte off. + data, _ := writePDF(t, desc, 64<<10, 32<<10) + at := bytes.Index(data, []byte(" 00000 n \n")) - 1 + broken := bytes.Clone(data) + broken[at]++ + if err := pdfOffsetsLand(broken); err == nil { + t.Errorf("canary: a table that sends an object a byte off passed") + } +} + +// writePDF writes a PDF of size bytes laid out for a cut after cutAt, whole, +// and the regions it marked. +func writePDF(t *testing.T, desc format.Descriptor, size, cutAt int64) ([]byte, []format.Region) { + t.Helper() + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: cutAt}) + if err != nil { + t.Fatalf("%d B: planning: %v", size, err) + } + var out bytes.Buffer + rec := &format.MapRecorder{W: &out} + if err := desc.Generator.Write(t.Context(), rec, p); err != nil { + t.Fatalf("%d B: writing: %v", size, err) + } + return out.Bytes(), rec.Map(size, true) +} + +// pdfLayoutOf holds a PDF laid out for a cut to its size, its offsets and the +// share the cut keeps of its objects, and names the layout it took. +func pdfLayoutOf(data []byte, regions []format.Region, size, cutAt int64) (string, error) { + if int64(len(data)) != size { + return "", fmt.Errorf("came to %d B", len(data)) + } + if err := pdfOffsetsLand(data); err != nil { + return "", err + } + roles := []format.Role{format.RoleStructure, format.RolePadding, format.RoleContent, format.RoleEnd, format.RolePadding, format.RoleEnd} + if len(regions) != len(roles) { + return "", fmt.Errorf("marked %d regions", len(regions)) + } + for i, r := range regions { + if r.Role != roles[i] { + return "", fmt.Errorf("region %d is %s where %s was due", i, r.Role, roles[i]) + } + } + head, front, content, back := regions[0].Length, regions[1].Length, regions[2].Length, regions[4].Length + if front == 1 || back == 1 { + return "", fmt.Errorf("a comment of one byte: %d in front, %d behind", front, back) + } + want := format.PaddingInFront(head, content, cutAt, size) + kept, share := cutAt-head-front, content*cutAt/size + switch { + case front == 0: + return "nothing in front", nil + case front == want && kept == share: + return "the share", nil + case front == want+1 && kept == share-1: + return "a byte more in front", nil + case front == want-1 && kept == share+1: + return "a byte less in front", nil + case front < want && back == 0: + return fmt.Sprintf("all in front, startxref of %d digits", len(strconv.FormatInt(regions[3].At, 10))), nil + case front < want && back <= 3: + return "a few bytes behind where startxref gains a digit", nil + } + return "", fmt.Errorf("%d B in front where the share wants %d, keeping %d B of the objects where the share is %d", front, want, kept, share) +} + +// pdfOffsetsLand reads a PDF's cross reference table from the offset +// startxref gives and requires every object it lists to start where it says. +func pdfOffsetsLand(data []byte) error { + if !bytes.HasSuffix(data, []byte("\n%%EOF\n")) { + return errors.New("the file does not end with the end of file marker") + } + sx := bytes.LastIndex(data, []byte("startxref\n")) + if sx < 0 { + return errors.New("no startxref") + } + line := data[sx+len("startxref\n") : len(data)-len("%%EOF\n")-1] + xref, err := strconv.ParseInt(string(line), 10, 64) + if err != nil || xref < 0 || xref >= int64(len(data)) || !bytes.HasPrefix(data[xref:], []byte("xref\n0 ")) { + return fmt.Errorf("startxref gives %q, where no cross reference table starts", line) + } + table := data[xref+int64(len("xref\n0 ")):] + end := bytes.IndexByte(table, '\n') + count, err := strconv.Atoi(string(table[:max(end, 0)])) + if err != nil { + return fmt.Errorf("the table counts %q", table[:max(end, 0)]) + } + entries := table[end+1+20:] + for i := 1; i < count; i++ { + if len(entries) < 20 { + return fmt.Errorf("the table ends before object %d", i) + } + at, err := strconv.ParseInt(string(entries[:10]), 10, 64) + obj := fmt.Sprintf("%d 0 obj\n", i) + if err != nil || at < 0 || at >= int64(len(data)) || !bytes.HasPrefix(data[at:], []byte(obj)) { + return fmt.Errorf("the table sends object %d to %q, where it does not start", i, entries[:10]) + } + entries = entries[20:] + } + return nil +} + +// pdfTooLargeForTheTable plans a PDF of 20 GiB cut at 90% and holds its +// objects to where an offset of ten digits reaches (ISO 32000-1 section +// 7.5.4): the padding in front stops there, short of the share. It writes +// nothing past the marks, which come before the first byte. +func pdfTooLargeForTheTable(t *testing.T, desc format.Descriptor) { + t.Helper() + const size, largestOffset = 20 << 30, 9_999_999_999 + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: size / 10 * 9}) + if err != nil { + t.Fatalf("planning %d B: %v", int64(size), err) + } + rec := &format.MapRecorder{W: failingWriter{}} + if err := desc.Generator.Write(t.Context(), rec, p); err == nil { + t.Fatalf("writing %d B went through a writer that refuses everything", int64(size)) + } + regions := rec.Map(size, true) + if len(regions) != 6 { + t.Fatalf("%d B: marked %d regions", int64(size), len(regions)) + } + front, objects := regions[1].Length, regions[2] + if front == 0 || objects.At+objects.Length != largestOffset { + t.Errorf("%d B cut at 90%%: %d B in front and the objects end at %d, where the table's ten digits stop them at %d", + int64(size), front, objects.At+objects.Length, int64(largestOffset)) + } +} diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 9569770d..c20c6199 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -199,21 +199,34 @@ def layout_mp4(data): def layout_pdf(data): - """The header lines, the objects up to the cross reference table startxref - points at, the table with the trailer, any comment after the trailer as - padding, and startxref with the closing marker - the end in two parts.""" + """The header lines, any comment between them and the first object as + padding, the objects up to the cross reference table startxref points at, + the table with the trailer, any comment after the trailer as padding, and + startxref with the closing marker - the end in two parts. + + The header is the %PDF- line and the comment of bytes above 127 that marks + the file as binary (ISO 32000-1 section 7.5.2). A comment line after them + is padding only when it is ASCII, so the binary line is never read as it.""" if not data.startswith(b"%PDF-"): raise NoMap("no PDF header") + head = data.index(b"\n") + 1 + line = data[head:data.index(b"\n", head) + 1] + if line.startswith(b"%") and any(b > 127 for b in line): + head += len(line) first = re.search(rb"\d+ 0 obj\n", data) sx = data.rfind(b"startxref\n") if first is None or sx < 0: raise NoMap("no object or no startxref") + front = data[head:first.start()] + if front and not re.fullmatch(rb"(%[\x20-\x7e]*\n)+", front): + raise NoMap(f"between the header and the first object at {first.start()} stands something other than comments") xref = int(data[sx + 10:data.index(b"\n", sx + 10)]) trailer = data.find(b"trailer\n", xref) if data[xref:xref + 5] != b"xref\n" or trailer < 0: raise NoMap(f"startxref points at {xref}, where no cross reference table starts") closed = data.index(b">>\n", trailer) + 3 - return [[STRUCTURE, 0, first.start()], [CONTENT, first.start(), xref - first.start()], + return [[STRUCTURE, 0, head], [PADDING, head, first.start() - head], + [CONTENT, first.start(), xref - first.start()], [END, xref, closed - xref], [PADDING, closed, sx - closed], [END, sx, len(data) - sx]] From 9da8f9ca5f96c71293a48b46bcdb0ed246448c44 Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 18:32:27 +0100 Subject: [PATCH 2/2] A PDF guard holds nothing in front to the share, the changelog names the limit The guard of a PDF laid out for a cut took a document with nothing in front of its objects as one layout among the others, before it asked what the cut kept. A split that dropped the padding in front at some sizes passed it, as long as other sizes met every layout the reach check asks for. Nothing in front now passes only where the document has no padding to move, which the reach check asks for at the smallest size, or where the share wants none. A second canary holds a document nothing cuts, all of its padding behind, to a cut at its half, and requires it to fail. The changelog entry names the second cut that keeps every page: one that keeps ten billion bytes or more, past which the cross reference table's ten digits cannot point. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 15 +++++++------ internal/guard/mutationcoverage_test.go | 6 +++-- internal/guard/pdfcut_test.go | 29 +++++++++++++++++++------ 3 files changed, 34 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b86db3d..2ddf84fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -108,13 +108,14 @@ because it turns other people's test suites red. 1% of 64 KiB - it falls there and every file is kept, and the manifest says `in: end`. Files nothing cuts are unchanged. - **`truncate` cuts the pages of a PDF by the percentage you set.** - `--damage truncate:missing=50` on a 1 MB PDF used to cut only padding, and - viewers showed every page. Now it removes half of the document and the - index at its end, as an interrupted download does. pdf.js (Firefox), - PDFium (Chrome, Edge), Poppler and the Windows PDF viewer refuse it, and - MuPDF and Ghostscript show the part that arrived. A cut of a few percent of - a small PDF can fall in the index, keeping every page - the manifest says - `in: end`. Files nothing cuts are unchanged. + A cut used to take only padding, leaving every page. Now + `--damage truncate:missing=50` removes half of the document and its closing + index, as an interrupted download does. pdf.js (Firefox), PDFium + (Chrome, Edge), Poppler and the Windows PDF viewer refuse it, and MuPDF and + Ghostscript show the part that arrived. Two cuts keep every page, and the + manifest says so: a few percent of a small PDF, falling in the index, and + one keeping 10 billion bytes or more, which the index cannot reach. Files + nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index a5029e07..f75f8d12 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -159,8 +159,10 @@ var provenByProbe = map[string]string{ "fixed codes, must inflate to its bytes and read as those runs - its reach check, every frame of the marker met - and red on " + "2026-11-13 for a frame a byte short (the plan refuses at 65541 B) and for an empty block's BTYPE bit one place off " + "(compress/flate refuses at 65537 B), where the first passed the share and the map guards.", - "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canary - a table that sends one object a byte off " + - "must be refused - its reach check, every layout of the split met, and red on 2026-11-13 without the candidate of a " + + "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canaries - a table that sends one object a byte off " + + "must be refused, and the layout of a document nothing cuts must fail a cut at its half - its reach check, every " + + "layout of the split met, red on 2026-11-13 with nothing in front of a document under 400 B of padding (398 sizes, " + + "where the guard before the review passed), and red without the candidate of a " + "byte more in front (that layout never met), with the length of startxref taken before the padding moved it (10212 B " + "came to 10213 B) and without the stop at ten digits (20 GiB, objects ending at 19327353153).", "TestATarGzCutNearItsSmallestComesToEverySize": "its reach check - the frame met, and without a label the size after its floor " + diff --git a/internal/guard/pdfcut_test.go b/internal/guard/pdfcut_test.go index 4be7c378..23f13775 100644 --- a/internal/guard/pdfcut_test.go +++ b/internal/guard/pdfcut_test.go @@ -23,15 +23,19 @@ import ( // where that happens. So this writes consecutive sizes cut at their last byte // across the size where startxref gains its fifth digit, and at 1% and 50% // where the padding behind runs out, holds each to its size, its offsets and -// the share, and asserts that it met the layouts that follow: all of the -// padding in front with four digits and with five, a few bytes left behind -// where the digit is gained, the share to the byte, and a byte more in front. +// the share, and asserts that it met the layouts that follow: no padding to +// move at the smallest size, all of the padding in front with four digits and +// with five, a few bytes left behind where the digit is gained, the share to +// the byte, and a byte more in front. Nothing in front passes only where +// there is no padding or the share wants none - on 2026-11-13 the smallest +// size was the one size of the sweep with nothing in front. // // A document of more than ten gigabytes is planned and not written: its // objects stop where the cross reference table's ten digits do. // -// Canary: the reader of offsets refuses a document whose table sends one -// object a byte off. +// Canaries: the reader of offsets refuses a document whose table sends one +// object a byte off, and the layout of a document nothing cuts, all of its +// padding behind, fails a cut at its half. func TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand(t *testing.T) { desc, err := format.Get("pdf") if err != nil { @@ -58,7 +62,7 @@ func TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand(t *testing.T) { sweep(22_000, 24_600, func(size int64) int64 { return size - (size+99)/100 }) smallest := desc.SmallestAccepted(format.Request{Label: true}) sweep(smallest, smallest+600, func(size int64) int64 { return size / 2 }) - for _, layout := range []string{"all in front, startxref of 4 digits", "all in front, startxref of 5 digits", + for _, layout := range []string{"no padding to move", "all in front, startxref of 4 digits", "all in front, startxref of 5 digits", "a few bytes behind where startxref gains a digit", "the share", "a byte more in front"} { if !met[layout] { t.Errorf("no size met the layout %q - met %v", layout, met) @@ -75,6 +79,13 @@ func TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand(t *testing.T) { if err := pdfOffsetsLand(broken); err == nil { t.Errorf("canary: a table that sends an object a byte off passed") } + + // Canary: a document nothing cuts has all of its padding behind, so held to + // a cut at its half it keeps every object. + whole, regions := writePDF(t, desc, 64<<10, 0) + if layout, err := pdfLayoutOf(whole, regions, 64<<10, 32<<10); err == nil { + t.Errorf("canary: the padding all behind passed a cut at the half as %q", layout) + } } // writePDF writes a PDF of size bytes laid out for a cut after cutAt, whole, @@ -118,8 +129,12 @@ func pdfLayoutOf(data []byte, regions []format.Region, size, cutAt int64) (strin want := format.PaddingInFront(head, content, cutAt, size) kept, share := cutAt-head-front, content*cutAt/size switch { + case front == 0 && back == 0: + return "no padding to move", nil + case front == 0 && want <= 0: + return "the share wants nothing in front", nil case front == 0: - return "nothing in front", nil + return "", fmt.Errorf("nothing in front and %d B behind, where the share wants %d B in front", back, want) case front == want && kept == share: return "the share", nil case front == want+1 && kept == share-1: