diff --git a/CHANGELOG.md b/CHANGELOG.md index 440f06e4..2ddf84fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -107,6 +107,15 @@ because it turns other people's test suites red. the unexpected end. When the cut is shorter than the end of the archive - 1% of 64 KiB - it falls there and every file is kept, and the manifest says `in: end`. Files nothing cuts are unchanged. +- **`truncate` cuts the pages of a PDF by the percentage you set.** + A cut used to take only padding, leaving every page. Now + `--damage truncate:missing=50` removes half of the document and its closing + index, as an interrupted download does. pdf.js (Firefox), PDFium + (Chrome, Edge), Poppler and the Windows PDF viewer refuse it, and MuPDF and + Ghostscript show the part that arrived. Two cuts keep every page, and the + manifest says so: a few percent of a small PDF, falling in the index, and + one keeping 10 billion bytes or more, which the index cannot reach. Files + nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/pdf/cut.go b/internal/format/pdf/cut.go new file mode 100644 index 00000000..e4b6d516 --- /dev/null +++ b/internal/format/pdf/cut.go @@ -0,0 +1,62 @@ +package pdf + +// The layout of a document a damage will cut: its padding split around the +// objects. + +import "github.com/donislawdev/TestingFilesGenerator/internal/format" + +// largestOffset is the largest offset a cross reference entry can carry. ISO +// 32000-1 section 7.5.4 writes each one in ten digits. +const largestOffset = 9_999_999_999 + +// splitForACut is how much of the padding m has behind its trailer goes in +// front of its objects instead, for a document a damage will cut after its +// first cutAt bytes of whole, so that the cut keeps the share of the objects it +// keeps of the whole file (format.Request.CutAt). It returns the padding in +// front and the padding left behind, and none in front for a document nothing +// cuts. +// +// With all of the padding behind the trailer, as a document nothing cuts has +// it, a large document cut at any share kept every object and lost only +// padding. A comment may stand between the header lines and the first object as +// well, and the readers measured take it there as they take it behind the +// trailer (docs/TRUNCATE-2026-11-11.md section 12). +// +// The padding in front moves every object, so the offset startxref carries +// grows with it and can gain a digit, which the padding behind gives up. Each +// side is nothing or a comment of at least minComment bytes. Where the sides +// cannot land exactly the front takes a byte more, which cuts a byte more of +// the objects - the "at least" missing promises - or, where a digit gained +// leaves that no room, a byte less. Where the end is longer than the part the +// cut takes, all of the padding goes in front, and the cut falls in the end. +// An object can start no later than largestOffset, so in a document of more +// than about 9.3 GiB the front stops there and the cut keeps more of the +// objects than the share. +func splitForACut(m memo, whole, cutAt int64) (front, back int64) { + if m.padLen == 0 || cutAt <= 0 || cutAt >= whole { + return 0, m.padLen + } + want := format.PaddingInFront(m.at.objects, m.at.xref-m.at.objects, cutAt, whole) + if want <= 0 { + return 0, m.padLen + } + // padding is all of the padding with front of it in front: the closing + // lines carry the offset front moves. + padding := func(front int64) int64 { + return whole - int64(len(m.prefix)) - int64(len(closing(m.at.xref+front))) + } + fits := func(front int64) bool { + back := padding(front) - front + return front >= minComment && m.at.xref+front <= largestOffset && (back == 0 || back >= minComment) + } + front = min(max(want, minComment), largestOffset-m.at.xref) + if padding(front) < front { + front = padding(front) + } + for _, f := range []int64{front, front + 1, front - 1} { + if fits(f) { + return f, padding(f) - f + } + } + return 0, m.padLen +} diff --git a/internal/format/pdf/document.go b/internal/format/pdf/document.go index 6c58a75f..6dac67fe 100644 --- a/internal/format/pdf/document.go +++ b/internal/format/pdf/document.go @@ -14,12 +14,17 @@ type places struct { objects, xref int64 } -// document builds everything up to the trailer, and the closing lines. +// document builds everything up to the trailer, and the closing lines, of a +// document with front bytes of padding between its header lines and its first +// object (splitForACut). // // The two are returned apart because the padding goes between them. The // closing lines carry the offset of the cross reference table, which sits in -// the first part and therefore does not move. -func document(m memo) (prefix, suffix []byte, at places) { +// the first part and therefore does not move. The front padding is not in the +// first part - the writer puts it in at at.objects - but every offset counts +// it. The cross reference table writes each offset in ten digits, so the first +// part is the same length whatever front is. +func document(m memo, front int64) (prefix, suffix []byte, at places) { var body bytes.Buffer // The version is the one asked for and nothing else follows from it: the // file uses nothing newer than 1.4, so every version offered here is true @@ -73,10 +78,15 @@ func document(m memo) (prefix, suffix []byte, at places) { fmt.Fprintf(&body, "xref\n0 %d\n", len(objects)+1) body.WriteString("0000000000 65535 f \n") for _, off := range offsets { - fmt.Fprintf(&body, "%010d 00000 n \n", off) + fmt.Fprintf(&body, "%010d 00000 n \n", int64(off)+front) } fmt.Fprintf(&body, "trailer\n<>\n", len(objects)+1) - suffix = []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF\n", xrefPos)) - return body.Bytes(), suffix, at + return body.Bytes(), closing(at.xref + front), at +} + +// closing is startxref, the offset of the cross reference table it points at, +// and the end of file marker. +func closing(xref int64) []byte { + return []byte(fmt.Sprintf("startxref\n%d\n%%%%EOF\n", xref)) } diff --git a/internal/format/pdf/minimum.go b/internal/format/pdf/minimum.go index e9967762..91f2fb81 100644 --- a/internal/format/pdf/minimum.go +++ b/internal/format/pdf/minimum.go @@ -63,6 +63,6 @@ func minimumBytes() int64 { if err != nil { panic("pdf: the default settings are refused: " + err.Error()) } - prefix, suffix, _ := document(memo{opts: opts}) + prefix, suffix, _ := document(memo{opts: opts}, 0) return int64(len(prefix) + len(suffix)) } diff --git a/internal/format/pdf/padding.go b/internal/format/pdf/padding.go index 5b6352e9..1ac92923 100644 --- a/internal/format/pdf/padding.go +++ b/internal/format/pdf/padding.go @@ -21,7 +21,8 @@ const ( ) // The padding channel is a comment block placed after the trailer and before -// startxref. +// startxref. A document a damage will cut has part of it in front of the +// objects instead (splitForACut). // // The obvious place, and the one this project's own notes assumed, is between // startxref and %%EOF. Measured, that is wrong twice over. diff --git a/internal/format/pdf/pdf.go b/internal/format/pdf/pdf.go index d1e132b2..09136514 100644 --- a/internal/format/pdf/pdf.go +++ b/internal/format/pdf/pdf.go @@ -51,9 +51,12 @@ type memo struct { // not depend on the padding, because the offset it carries points at the // cross reference table, which sits before the padding. suffix []byte - // at is where the objects and the cross reference table start. - at places - padLen int64 + // at is where the objects and the cross reference table start in prefix. + at places + // front is the padding between the header lines and the first object, + // which only a document a damage will cut has (splitForACut), and padLen + // the padding behind the trailer. + front, padLen int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -68,7 +71,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { } m := memo{opts: opts, seed: r.Seed, label: label} - m.prefix, m.suffix, m.at = document(m) + m.prefix, m.suffix, m.at = document(m, 0) // One number answers both questions: how much padding this file needs, and // what is too small to make. Every line of body text is the same width @@ -108,6 +111,9 @@ func (generator) Plan(r format.Request) (format.Plan, error) { default: m.padLen = r.Bytes - bare } + if m.front, m.padLen = splitForACut(m, r.Bytes, r.CutAt); m.front > 0 { + m.prefix, m.suffix, _ = document(m, m.front) + } p.Memo = m return p, nil @@ -170,18 +176,28 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { default: } - // The map, all of it known from the plan: the header lines, the objects - - // the pages and what they draw with - the cross reference table and the + // The map, all of it known from the plan: the header lines, the comment + // in front of the objects a document a damage cuts has, the objects - the + // pages and what they draw with - the cross reference table and the // trailer, the comment that makes up the size, and startxref with the - // closing marker. The padding stands between two parts of the end. - head, end := int64(len(m.prefix)), int64(len(m.prefix))+m.padLen + // closing marker. The padding behind stands between two parts of the end. + objects := m.at.objects + m.front + table := m.at.xref + m.front + behind := int64(len(m.prefix)) + m.front format.Mark(w, format.RoleStructure, 0, m.at.objects) - format.Mark(w, format.RoleContent, m.at.objects, m.at.xref-m.at.objects) - format.Mark(w, format.RoleEnd, m.at.xref, head-m.at.xref) - format.Mark(w, format.RolePadding, head, m.padLen) - format.Mark(w, format.RoleEnd, end, int64(len(m.suffix))) + format.Mark(w, format.RolePadding, m.at.objects, m.front) + format.Mark(w, format.RoleContent, objects, table-objects) + format.Mark(w, format.RoleEnd, table, behind-table) + format.Mark(w, format.RolePadding, behind, m.padLen) + format.Mark(w, format.RoleEnd, behind+m.padLen, int64(len(m.suffix))) - if _, err := w.Write(m.prefix); err != nil { + if _, err := w.Write(m.prefix[:m.at.objects]); err != nil { + return err + } + if err := writeComment(ctx, w, m.seed, m.front); err != nil { + return err + } + if _, err := w.Write(m.prefix[m.at.objects:]); err != nil { return err } if err := writeComment(ctx, w, m.seed, m.padLen); err != nil { diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 53b2ebc0..2e11a0c8 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -3,6 +3,7 @@ package guard import ( "fmt" "os" + "slices" "strconv" "strings" "testing" @@ -36,8 +37,8 @@ type cutCase struct { // the content is left, and gets a note when the cut missed the content. // // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). -// A PDF of 1 MiB is nearly all padding after its pages, so 90% missing still -// leaves every page. A PNG splits its padding around the picture +// An MP4 of 1 MiB is nearly all padding after its video, so 90% missing still +// leaves all of it. A PNG splits its padding around the picture // for a cut, so the cut takes the share asked for, and so does a tar.gz. A // text is its words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. @@ -46,8 +47,9 @@ type cutCase struct { // one that claims content it does not have. func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ - // A GIF was this case until it split its padding for the cut. - {name: "behind", format: "pdf", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // A GIF and then a PDF were this case until each split its padding + // for the cut. + {name: "behind", format: "mp4", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, // 4 KiB on disk is 40 KiB whole. It was the case of a cut in the padding // in front of the content, with an icon and a JPEG before it, until it @@ -178,7 +180,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("behind", "pdf", 1<<20, "90"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "mp4", 1<<20, "90"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) @@ -262,7 +264,11 @@ func TestACutAtTheSmallestSizeSaysHowLongTheContentWas(t *testing.T) { } cut := *m.Damage[0].Cut whole := wholeMap(t, plannedOf[m.TargetID]) - if _, zipped := endClosesTheFile[m.TargetID]; zipped && (paddingInFront(whole) || cut.In != string(format.RoleContent)) { + // Named rather than read off endClosesTheFile, which a PDF and a + // tar.gz joined: a PDF at its smallest has room for its comment in + // front, as it should. + zipped := slices.Contains([]string{"zip", "docx", "xlsx", "pptx"}, m.TargetID) + if zipped && (paddingInFront(whole) || cut.In != string(format.RoleContent)) { t.Errorf("%s: at %d B the cut fell in the %s with padding in front of the content: %v - the layout too short for a padding part in front is not reached", m.TargetID, m.Bytes, cut.In, paddingInFront(whole)) } diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 35be63d4..dadb21aa 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -36,7 +36,6 @@ var notYetCutAtTheShare = map[string]notYet{ "jxl": {paddingBehind, 64 << 10, 50}, "mp4": {paddingBehind, 64 << 10, 50}, "webm": {paddingBehind, 64 << 10, 50}, - "pdf": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, @@ -51,7 +50,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "pdf", "webp", "tiff"} +var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. @@ -211,6 +210,7 @@ var endClosesTheFile = map[string]string{ "xlsx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", "pptx": "a ZIP package, laid out as APPNOTE 4.3.6 lays out a ZIP: the central directory after every part", "targz": "POSIX tar puts the end of archive marker after every entry, and RFC 1952 closes a gzip member with its trailer", + "pdf": "ISO 32000-1 section 7.5 puts the cross reference table and the trailer after the body, and the end of file marker last", } // onlyTheEndBehind is whether nothing but the end stands behind the content of diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index e86b7f0a..f75f8d12 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -159,6 +159,12 @@ var provenByProbe = map[string]string{ "fixed codes, must inflate to its bytes and read as those runs - its reach check, every frame of the marker met - and red on " + "2026-11-13 for a frame a byte short (the plan refuses at 65541 B) and for an empty block's BTYPE bit one place off " + "(compress/flate refuses at 65537 B), where the first passed the share and the map guards.", + "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canaries - a table that sends one object a byte off " + + "must be refused, and the layout of a document nothing cuts must fail a cut at its half - its reach check, every " + + "layout of the split met, red on 2026-11-13 with nothing in front of a document under 400 B of padding (398 sizes, " + + "where the guard before the review passed), and red without the candidate of a " + + "byte more in front (that layout never met), with the length of startxref taken before the padding moved it (10212 B " + + "came to 10213 B) and without the stop at ten digits (20 GiB, objects ending at 19327353153).", "TestATarGzCutNearItsSmallestComesToEverySize": "its reach check - the frame met, and without a label the size after its floor " + "that keeps the layout of an archive nothing cuts, which with a label is due never - and red on 2026-11-13 without the " + "case of one byte over the floor (an internal defect at 9757 B) and with that case taken for a label too (label true).", diff --git a/internal/guard/pdfcut_test.go b/internal/guard/pdfcut_test.go new file mode 100644 index 00000000..23f13775 --- /dev/null +++ b/internal/guard/pdfcut_test.go @@ -0,0 +1,212 @@ +package guard + +import ( + "bytes" + "errors" + "fmt" + "strconv" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" +) + +// A PDF laid out for a cut comes to every size asked for, every offset in it +// lands, and the cut keeps the share of its objects. +// +// The padding in front of the objects moves every one of them, so the offset +// startxref carries grows with it and gains a digit at a power of ten, which +// the padding behind gives up (internal/format/pdf/cut.go). Each side is +// nothing or a comment of two bytes or more, so where the padding behind would +// come to one byte the front takes a byte more, and where that byte gains the +// digit the front takes a byte less instead. No other guard asks for the sizes +// where that happens. So this writes consecutive sizes cut at their last byte +// across the size where startxref gains its fifth digit, and at 1% and 50% +// where the padding behind runs out, holds each to its size, its offsets and +// the share, and asserts that it met the layouts that follow: no padding to +// move at the smallest size, all of the padding in front with four digits and +// with five, a few bytes left behind where the digit is gained, the share to +// the byte, and a byte more in front. Nothing in front passes only where +// there is no padding or the share wants none - on 2026-11-13 the smallest +// size was the one size of the sweep with nothing in front. +// +// A document of more than ten gigabytes is planned and not written: its +// objects stop where the cross reference table's ten digits do. +// +// Canaries: the reader of offsets refuses a document whose table sends one +// object a byte off, and the layout of a document nothing cuts, all of its +// padding behind, fails a cut at its half. +func TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand(t *testing.T) { + desc, err := format.Get("pdf") + if err != nil { + t.Fatal(err) + } + met := map[string]bool{} + sweep := func(from, to int64, cutAt func(size int64) int64) { + for size := from; size < to; size++ { + var below *format.BelowMinimumError + if _, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true}); errors.As(err, &below) { + // The one size in the gap just above the bare document. + continue + } + data, regions := writePDF(t, desc, size, cutAt(size)) + layout, err := pdfLayoutOf(data, regions, size, cutAt(size)) + if err != nil { + t.Errorf("%d B cut at %d: %v", size, cutAt(size), err) + continue + } + met[layout] = true + } + } + sweep(9_900, 10_400, func(size int64) int64 { return size - 1 }) + sweep(22_000, 24_600, func(size int64) int64 { return size - (size+99)/100 }) + smallest := desc.SmallestAccepted(format.Request{Label: true}) + sweep(smallest, smallest+600, func(size int64) int64 { return size / 2 }) + for _, layout := range []string{"no padding to move", "all in front, startxref of 4 digits", "all in front, startxref of 5 digits", + "a few bytes behind where startxref gains a digit", "the share", "a byte more in front"} { + if !met[layout] { + t.Errorf("no size met the layout %q - met %v", layout, met) + } + } + + pdfTooLargeForTheTable(t, desc) + + // Canary: one object a byte off. + data, _ := writePDF(t, desc, 64<<10, 32<<10) + at := bytes.Index(data, []byte(" 00000 n \n")) - 1 + broken := bytes.Clone(data) + broken[at]++ + if err := pdfOffsetsLand(broken); err == nil { + t.Errorf("canary: a table that sends an object a byte off passed") + } + + // Canary: a document nothing cuts has all of its padding behind, so held to + // a cut at its half it keeps every object. + whole, regions := writePDF(t, desc, 64<<10, 0) + if layout, err := pdfLayoutOf(whole, regions, 64<<10, 32<<10); err == nil { + t.Errorf("canary: the padding all behind passed a cut at the half as %q", layout) + } +} + +// writePDF writes a PDF of size bytes laid out for a cut after cutAt, whole, +// and the regions it marked. +func writePDF(t *testing.T, desc format.Descriptor, size, cutAt int64) ([]byte, []format.Region) { + t.Helper() + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: cutAt}) + if err != nil { + t.Fatalf("%d B: planning: %v", size, err) + } + var out bytes.Buffer + rec := &format.MapRecorder{W: &out} + if err := desc.Generator.Write(t.Context(), rec, p); err != nil { + t.Fatalf("%d B: writing: %v", size, err) + } + return out.Bytes(), rec.Map(size, true) +} + +// pdfLayoutOf holds a PDF laid out for a cut to its size, its offsets and the +// share the cut keeps of its objects, and names the layout it took. +func pdfLayoutOf(data []byte, regions []format.Region, size, cutAt int64) (string, error) { + if int64(len(data)) != size { + return "", fmt.Errorf("came to %d B", len(data)) + } + if err := pdfOffsetsLand(data); err != nil { + return "", err + } + roles := []format.Role{format.RoleStructure, format.RolePadding, format.RoleContent, format.RoleEnd, format.RolePadding, format.RoleEnd} + if len(regions) != len(roles) { + return "", fmt.Errorf("marked %d regions", len(regions)) + } + for i, r := range regions { + if r.Role != roles[i] { + return "", fmt.Errorf("region %d is %s where %s was due", i, r.Role, roles[i]) + } + } + head, front, content, back := regions[0].Length, regions[1].Length, regions[2].Length, regions[4].Length + if front == 1 || back == 1 { + return "", fmt.Errorf("a comment of one byte: %d in front, %d behind", front, back) + } + want := format.PaddingInFront(head, content, cutAt, size) + kept, share := cutAt-head-front, content*cutAt/size + switch { + case front == 0 && back == 0: + return "no padding to move", nil + case front == 0 && want <= 0: + return "the share wants nothing in front", nil + case front == 0: + return "", fmt.Errorf("nothing in front and %d B behind, where the share wants %d B in front", back, want) + case front == want && kept == share: + return "the share", nil + case front == want+1 && kept == share-1: + return "a byte more in front", nil + case front == want-1 && kept == share+1: + return "a byte less in front", nil + case front < want && back == 0: + return fmt.Sprintf("all in front, startxref of %d digits", len(strconv.FormatInt(regions[3].At, 10))), nil + case front < want && back <= 3: + return "a few bytes behind where startxref gains a digit", nil + } + return "", fmt.Errorf("%d B in front where the share wants %d, keeping %d B of the objects where the share is %d", front, want, kept, share) +} + +// pdfOffsetsLand reads a PDF's cross reference table from the offset +// startxref gives and requires every object it lists to start where it says. +func pdfOffsetsLand(data []byte) error { + if !bytes.HasSuffix(data, []byte("\n%%EOF\n")) { + return errors.New("the file does not end with the end of file marker") + } + sx := bytes.LastIndex(data, []byte("startxref\n")) + if sx < 0 { + return errors.New("no startxref") + } + line := data[sx+len("startxref\n") : len(data)-len("%%EOF\n")-1] + xref, err := strconv.ParseInt(string(line), 10, 64) + if err != nil || xref < 0 || xref >= int64(len(data)) || !bytes.HasPrefix(data[xref:], []byte("xref\n0 ")) { + return fmt.Errorf("startxref gives %q, where no cross reference table starts", line) + } + table := data[xref+int64(len("xref\n0 ")):] + end := bytes.IndexByte(table, '\n') + count, err := strconv.Atoi(string(table[:max(end, 0)])) + if err != nil { + return fmt.Errorf("the table counts %q", table[:max(end, 0)]) + } + entries := table[end+1+20:] + for i := 1; i < count; i++ { + if len(entries) < 20 { + return fmt.Errorf("the table ends before object %d", i) + } + at, err := strconv.ParseInt(string(entries[:10]), 10, 64) + obj := fmt.Sprintf("%d 0 obj\n", i) + if err != nil || at < 0 || at >= int64(len(data)) || !bytes.HasPrefix(data[at:], []byte(obj)) { + return fmt.Errorf("the table sends object %d to %q, where it does not start", i, entries[:10]) + } + entries = entries[20:] + } + return nil +} + +// pdfTooLargeForTheTable plans a PDF of 20 GiB cut at 90% and holds its +// objects to where an offset of ten digits reaches (ISO 32000-1 section +// 7.5.4): the padding in front stops there, short of the share. It writes +// nothing past the marks, which come before the first byte. +func pdfTooLargeForTheTable(t *testing.T, desc format.Descriptor) { + t.Helper() + const size, largestOffset = 20 << 30, 9_999_999_999 + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: size / 10 * 9}) + if err != nil { + t.Fatalf("planning %d B: %v", int64(size), err) + } + rec := &format.MapRecorder{W: failingWriter{}} + if err := desc.Generator.Write(t.Context(), rec, p); err == nil { + t.Fatalf("writing %d B went through a writer that refuses everything", int64(size)) + } + regions := rec.Map(size, true) + if len(regions) != 6 { + t.Fatalf("%d B: marked %d regions", int64(size), len(regions)) + } + front, objects := regions[1].Length, regions[2] + if front == 0 || objects.At+objects.Length != largestOffset { + t.Errorf("%d B cut at 90%%: %d B in front and the objects end at %d, where the table's ten digits stop them at %d", + int64(size), front, objects.At+objects.Length, int64(largestOffset)) + } +} diff --git a/internal/oracle/layout.py b/internal/oracle/layout.py index 9569770d..c20c6199 100644 --- a/internal/oracle/layout.py +++ b/internal/oracle/layout.py @@ -199,21 +199,34 @@ def layout_mp4(data): def layout_pdf(data): - """The header lines, the objects up to the cross reference table startxref - points at, the table with the trailer, any comment after the trailer as - padding, and startxref with the closing marker - the end in two parts.""" + """The header lines, any comment between them and the first object as + padding, the objects up to the cross reference table startxref points at, + the table with the trailer, any comment after the trailer as padding, and + startxref with the closing marker - the end in two parts. + + The header is the %PDF- line and the comment of bytes above 127 that marks + the file as binary (ISO 32000-1 section 7.5.2). A comment line after them + is padding only when it is ASCII, so the binary line is never read as it.""" if not data.startswith(b"%PDF-"): raise NoMap("no PDF header") + head = data.index(b"\n") + 1 + line = data[head:data.index(b"\n", head) + 1] + if line.startswith(b"%") and any(b > 127 for b in line): + head += len(line) first = re.search(rb"\d+ 0 obj\n", data) sx = data.rfind(b"startxref\n") if first is None or sx < 0: raise NoMap("no object or no startxref") + front = data[head:first.start()] + if front and not re.fullmatch(rb"(%[\x20-\x7e]*\n)+", front): + raise NoMap(f"between the header and the first object at {first.start()} stands something other than comments") xref = int(data[sx + 10:data.index(b"\n", sx + 10)]) trailer = data.find(b"trailer\n", xref) if data[xref:xref + 5] != b"xref\n" or trailer < 0: raise NoMap(f"startxref points at {xref}, where no cross reference table starts") closed = data.index(b">>\n", trailer) + 3 - return [[STRUCTURE, 0, first.start()], [CONTENT, first.start(), xref - first.start()], + return [[STRUCTURE, 0, head], [PADDING, head, first.start() - head], + [CONTENT, first.start(), xref - first.start()], [END, xref, closed - xref], [PADDING, closed, sx - closed], [END, sx, len(data) - sx]]