diff --git a/CHANGELOG.md b/CHANGELOG.md index 2ddf84fc..04f1d2b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,13 @@ because it turns other people's test suites red. padding, which busybox reads, and the padding file inside the archive takes the rest, as it did for larger archives. Those archives are not byte for byte what 0.5.0 wrote. Every other tar.gz is unchanged. +- **A JPEG XL now opens in ffmpeg.** ffmpeg 7.1 never finished one, and + ffmpeg 9 took one below about 3.8 KB for a video and refused it, because of + the padding box behind the picture. The padding now stands in front of the + picture, in a box of the kind ffmpeg passes over, and nothing follows the + picture. Pillow, ImageMagick, Windows, Chrome and libjxl show the same + picture as before. Sizes are unchanged, but no JPEG XL is byte for byte what + 0.5.0 wrote. ### Added @@ -116,6 +123,14 @@ because it turns other people's test suites red. manifest says so: a few percent of a small PDF, falling in the index, and one keeping 10 billion bytes or more, which the index cannot reach. Files nothing cuts are unchanged. +- **`truncate` cuts the picture of an AVIF or a JPEG XL by the percentage you + set.** A cut used to take only padding, leaving the whole picture. Now + `--damage truncate:missing=50` removes half of the coded picture, as an + interrupted download does. Chrome, Firefox, Windows, libavif and Pillow + refuse the cut AVIF. Half of a JPEG XL still shows the whole picture at + lower detail in Chrome and in libjxl with partial files allowed, because the + format sends a coarse picture first. A cut of 90% shows nothing. An AVIF + nothing cuts is unchanged. Every JPEG XL changes (see Breaking). - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/avif/avif.go b/internal/format/avif/avif.go index 814131b1..6945018f 100644 --- a/internal/format/avif/avif.go +++ b/internal/format/avif/avif.go @@ -35,7 +35,8 @@ // sixty four bit box length that files above four gigabytes need. There is no // dead zone above the minimum. Pillow reads every one of them and refuses a // truncated or corrupted file, which is what makes it a witness. ffprobe reads -// the broken ones too, so it is not. +// the broken ones too, so it is not. A file a damage will cut has part of its +// free boxes in front of the media data instead (cut.go). package avif import ( @@ -166,6 +167,9 @@ type memo struct { // which is what makes every size above the minimum reachable a byte at a // time. total int64 + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). + cutAt int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -187,7 +191,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err := checkFits(r.Bytes, m); err != nil { return format.Plan{}, err } - m.total = r.Bytes + m.total, m.cutAt = r.Bytes, r.CutAt p := format.Plan{ Bytes: r.Bytes, @@ -244,7 +248,7 @@ const holdsPerPixel = 8 func checkFits(want int64, m memo) error { floor := int64(minimumBytes) reason := core.Says("avif.MinimumReason", - "the smallest picture this format draws codes to %d B at worst, and the file always carries a free box, which costs %d B even when it holds nothing", + "the smallest picture this format draws codes to %d B at worst, and the file always carries a padding box, which costs %d B even when it holds nothing", core.A("Coded", minimumBytes-boxHeader), core.A("Box", boxHeader)) // A picture named by hand can be larger than the smallest one, and then its @@ -254,7 +258,7 @@ func checkFits(want int64, m memo) error { if own := int64(len(m.coded)) + boxHeader; own > floor { floor = own reason = core.Says("avif.MinimumReasonOwn", - "a %dx%d picture at quality %d codes to %d B, and the file always carries a free box, which costs %d B even when it holds nothing", + "a %dx%d picture at quality %d codes to %d B, and the file always carries a padding box, which costs %d B even when it holds nothing", core.A("Width", m.width), core.A("Height", m.height), core.A("Quality", m.quality), core.A("Coded", len(m.coded)), core.A("Box", boxHeader)) } } @@ -491,6 +495,12 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { m.width, m.height, len(coded), m.total)) } + if m.cutAt > 0 { + if done, err := writeForACut(ctx, w, m, coded); done || err != nil { + return err + } + } + // The map: the coder's boxes, its media data the content, and the free // boxes after them that make up the size. markBoxes(w, coded) diff --git a/internal/format/avif/cut.go b/internal/format/avif/cut.go new file mode 100644 index 00000000..aef62c22 --- /dev/null +++ b/internal/format/avif/cut.go @@ -0,0 +1,342 @@ +package avif + +import ( + "context" + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/donislawdev/TestingFilesGenerator/internal/core" + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff" +) + +// writeForACut writes a file a damage will cut after its first m.cutAt bytes, +// with the padding split around the picture so that the cut keeps the share of +// the picture it keeps of the whole file (format.Request.CutAt): the coder's +// boxes up to the media data, free boxes, the media data, and free boxes +// behind it. +// +// With all of the padding behind the picture, as an uncut file has it, a large +// file cut at any share kept the whole picture and lost only padding. A free +// box may stand anywhere among the top level boxes, and readers skip it +// (docs/TRUNCATE-2026-11-11.md section 13). The one thing a box in front of the +// media data changes is where the picture starts, and the item locations in +// the meta box say that in bytes from the start of the file, so they move by +// as much. Their offsets are four bytes long as the coder writes them. One +// that the move would take past 4 GiB is widened to eight bytes, which the +// item location box allows, and the meta box grows by four bytes for each +// offset - only then, so every file that does not need it keeps the coder's +// meta box but for the numbers. +// +// A file whose share wants nothing in front, or too little for a free box, is +// written as one nothing cuts. +func writeForACut(ctx context.Context, w io.Writer, m memo, coded []byte) (bool, error) { + head, mdat, err := splitAtTheMediaData(coded) + if err != nil { + return false, err + } + pad := m.total - int64(len(coded)) + front := format.SplitFront(pad, int64(len(head)), int64(len(mdat)), m.cutAt, m.total, boxHeader) + if front == 0 { + return false, nil + } + locs, err := readItemLocations(head) + if err != nil { + return false, err + } + wide := locs.furthest+front > math.MaxUint32 + if wide { + grown := locs.growth() + front = format.SplitFront(pad-grown, int64(len(head))+grown, int64(len(mdat)), m.cutAt, m.total, boxHeader) + if front == 0 { + return false, nil + } + } + moved, err := locs.moved(head, front, wide) + if err != nil { + return false, err + } + back := m.total - int64(len(moved)) - front - int64(len(mdat)) + if back < 0 || back > 0 && back < boxHeader { + return false, core.Defect(fmt.Errorf("avif: a split of %d B in front of the picture leaves %d B behind it", front, back)) + } + + markBoxes(w, moved) + at := int64(len(moved)) + format.Mark(w, format.RolePadding, at, front) + format.Mark(w, format.RoleContent, at+front, int64(len(mdat))) + if back > 0 { + format.Mark(w, format.RolePadding, at+front+int64(len(mdat)), back) + } + + // Both sides draw their filler from the file's seed. Nobody reads padding, + // and the side behind the picture lies past any cut. + if _, err := w.Write(moved); err != nil { + return true, err + } + if err := isobmff.WritePadding(ctx, w, m.seed, front); err != nil { + return true, err + } + if _, err := w.Write(mdat); err != nil { + return true, err + } + return true, isobmff.WritePadding(ctx, w, m.seed, back) +} + +// splitAtTheMediaData parts the coder's file into the boxes in front of its +// media data and the media data box, which has to be the last one: that is +// where a box in front of the picture goes, and nothing behind it would move. +func splitAtTheMediaData(coded []byte) (head, mdat []byte, err error) { + for pos := 0; pos+boxHeader <= len(coded); { + size := int(binary.BigEndian.Uint32(coded[pos:])) + if size < boxHeader || pos+size > len(coded) { + break + } + mdat := string(coded[pos+4:pos+boxHeader]) == "mdat" + if mdat && pos+size == len(coded) { + return coded[:pos], coded[pos:], nil + } + if mdat { + break + } + pos += size + } + return nil, nil, core.Defect(fmt.Errorf("avif: the coder's file does not end with one media data box behind its other boxes")) +} + +// itemLocations is the item location box in a meta box, read far enough to +// move the places it gives. +type itemLocations struct { + // metaAt and ilocAt are where the meta box and the item location box start + // in the head, ilocLen how long the item location box is. + metaAt, ilocAt, ilocLen int + version byte + // offsetSize, lengthSize, baseSize and indexSize are the four field sizes + // in bytes, as the box gives them. + offsetSize, lengthSize, baseSize, indexSize int + // moving is how many fields hold a place in the file - the base offset of + // each item when the box has one, each extent's offset otherwise. + moving int + // furthest is the largest place any of them gives. + furthest int64 +} + +// countSize is how long the item count and each item's number are: two bytes, +// and four from version 2 on. +func (l itemLocations) countSize() int { + if l.version == 2 { + return 4 + } + return 2 +} + +// growth is how much the item location box, and so the meta box, grows when +// the fields that hold a place are widened from four bytes to eight. +func (l itemLocations) growth() int64 { return int64(4 * l.moving) } + +// readItemLocations finds the item location box in the meta box of head and +// reads the places it gives. Only places in this file (construction method 0) +// are what the coder writes, and only those are read. +func readItemLocations(head []byte) (itemLocations, error) { + l, why := locateItemLocations(head) + if why == "" { + why = l.readPlaces(head[l.ilocAt+boxHeader : l.ilocAt+l.ilocLen]) + } + if why != "" { + return itemLocations{}, core.Defect(fmt.Errorf("avif: the coder's item locations %s", why)) + } + return l, nil +} + +// locateItemLocations finds the meta box among the top level boxes of head and +// the item location box in it, or says why it cannot. +func locateItemLocations(head []byte) (itemLocations, string) { + l := itemLocations{metaAt: -1} + for pos := 0; pos+boxHeader <= len(head); { + size := int(binary.BigEndian.Uint32(head[pos:])) + if size < boxHeader || pos+size > len(head) { + return l, "stand in a box this cannot walk past" + } + why := "" + if string(head[pos+4:pos+boxHeader]) == "meta" { + l.metaAt = pos + why = l.findIn(head, pos, size) + } + if why != "" { + return l, why + } + pos += size + } + if l.metaAt < 0 || l.ilocLen == 0 { + return l, "are missing" + } + return l, "" +} + +// findIn finds the item location box among the boxes of the meta box that +// starts at pos. meta is a full box: its version and flags come before its +// boxes. +func (l *itemLocations) findIn(head []byte, pos, size int) string { + for at := pos + boxHeader + 4; at+boxHeader <= pos+size; { + n := int(binary.BigEndian.Uint32(head[at:])) + if n < boxHeader || at+n > pos+size { + return "stand in a meta box this cannot walk" + } + if string(head[at+4:at+boxHeader]) == "iloc" { + l.ilocAt, l.ilocLen = at, n + } + at += n + } + return "" +} + +// readPlaces reads the field sizes of the item location box's content b and +// every place in this file it gives, or says why it cannot. +func (l *itemLocations) readPlaces(b []byte) string { + if len(b) < 8 { + return "are cut short" + } + l.version = b[0] + if l.version > 2 { + return fmt.Sprintf("are in version %d", l.version) + } + l.offsetSize, l.lengthSize, l.baseSize = int(b[4]>>4), int(b[4]&15), int(b[5]>>4) + if l.version >= 1 { + l.indexSize = int(b[5] & 15) + } + moveBase := l.baseSize > 0 + if !moveBase && l.offsetSize == 0 { + return "give no offset to move" + } + r := reader{b: b, at: 6} + for range r.uint(l.countSize()) { + r.uint(l.countSize()) + if l.version >= 1 && r.uint(2)&15 != 0 { + return "point outside the file" + } + r.uint(2) + l.place(moveBase, r.uint(l.baseSize)) + for range r.uint(2) { + r.uint(l.indexSize) + off := r.uint(l.offsetSize) + r.uint(l.lengthSize) + l.place(!moveBase, off) + } + } + if r.short || r.at != len(b) { + return "do not read to their end" + } + return "" +} + +// place counts a field that holds a place in the file, when moves says it is +// one of the fields a move changes. +func (l *itemLocations) place(moves bool, at int64) { + if moves { + l.moving++ + l.furthest = max(l.furthest, at) + } +} + +// moved is head with every place the item locations give moved by front, and +// by what the widening adds to the meta box when wide. Widening turns the four +// byte fields that hold a place into eight byte ones. +func (l itemLocations) moved(head []byte, front int64, wide bool) ([]byte, error) { + by := front + from, to := l.offsetSize, l.offsetSize + if l.baseSize > 0 { + from, to = l.baseSize, l.baseSize + } + if wide { + if from != 4 { + return nil, core.Defect(fmt.Errorf("avif: the coder's item locations hold places in %d bytes, which this does not widen", from)) + } + to = 8 + by += l.growth() + } + + b := head[l.ilocAt+boxHeader : l.ilocAt+l.ilocLen] + out := append([]byte(nil), b[:6]...) + if l.baseSize > 0 { + out[5] = byte(to)<<4 | b[5]&15 + } else { + out[4] = byte(to)<<4 | b[4]&15 + } + r := reader{b: b, at: 6} + count := r.copy(&out, l.countSize()) + for range count { + r.copy(&out, l.countSize()) + if l.version == 1 || l.version == 2 { + r.copy(&out, 2) + } + r.copy(&out, 2) + r.move(&out, l.baseSize, l.baseSize > 0, by, to) + for range r.copy(&out, 2) { + r.copy(&out, l.indexSize) + r.move(&out, l.offsetSize, l.baseSize == 0, by, to) + r.copy(&out, l.lengthSize) + } + } + if to < 8 && by > 0 && l.furthest+by > math.MaxUint32 { + return nil, core.Defect(fmt.Errorf("avif: a place moved by %d B no longer fits four bytes", by)) + } + + grown := len(out) + boxHeader - l.ilocLen + moved := make([]byte, 0, len(head)+grown) + moved = append(moved, head[:l.ilocAt]...) + moved = binary.BigEndian.AppendUint32(moved, uint32(boxHeader+len(out))) + moved = append(moved, "iloc"...) + moved = append(moved, out...) + moved = append(moved, head[l.ilocAt+l.ilocLen:]...) + meta := binary.BigEndian.Uint32(head[l.metaAt:]) + binary.BigEndian.PutUint32(moved[l.metaAt:], meta+uint32(grown)) + return moved, nil +} + +// reader reads big endian fields of a box's content and remembers whether it +// ran out. +type reader struct { + b []byte + at int + short bool +} + +func (r *reader) uint(n int) int64 { + if r.at+n > len(r.b) { + r.short, r.at = true, len(r.b) + return 0 + } + var v int64 + for _, c := range r.b[r.at : r.at+n] { + v = v<<8 | int64(c) + } + r.at += n + return v +} + +// copy reads a field of n bytes, appends it to out as it was and returns it. +func (r *reader) copy(out *[]byte, n int) int64 { + start := r.at + v := r.uint(n) + *out = append(*out, r.b[start:r.at]...) + return v +} + +// move reads a field of n bytes and appends it to out - as it was, or, when +// moves, moved by by and written in to bytes. +func (r *reader) move(out *[]byte, n int, moves bool, by int64, to int) { + if !moves { + r.copy(out, n) + return + } + *out = appendUint(*out, r.uint(n)+by, to) +} + +func appendUint(out []byte, v int64, n int) []byte { + for i := n - 1; i >= 0; i-- { + out = append(out, byte(v>>(8*i))) + } + return out +} diff --git a/internal/format/cut.go b/internal/format/cut.go index 5cba050c..e4d54770 100644 --- a/internal/format/cut.go +++ b/internal/format/cut.go @@ -25,6 +25,38 @@ func PaddingInFront(before, content, cutAt, whole int64) int64 { return cutAt - before - int64(kept) } +// SplitFront is how much of a file's padding goes in front of its content for +// a cut after its first cutAt bytes, where the padding comes in pieces that +// cost least bytes before they carry anything - a PNG chunk, an ISO base media +// free box - so each side is nothing or at least that. before is what stands +// in front of the padding and pad all of it, both sides. +// +// The share is PaddingInFront. Where the pieces cannot land on it, the front +// grows, which moves the content later and cuts more of it - the "at least" +// missing promises - and a side too short for a piece goes to the front. The +// one way the cut keeps more is padding too short for the front the share +// wants: all of it goes in front then, and the cut keeps more than the share +// (docs/TRUNCATE-2026-11-11.md section 7.4). The manifest's cut reports what +// was kept either way. +func SplitFront(pad, before, content, cutAt, whole, least int64) int64 { + if pad == 0 || cutAt <= 0 || cutAt >= whole { + return 0 + } + front := PaddingInFront(before, content, cutAt, whole) + switch { + case front <= 0: + return 0 + case front < least: + front = least + } + if pad-front < least { + // Behind the content there is room for nothing or for less than a + // piece, so the whole padding goes in front. + return pad + } + return front +} + // AfterTheHead passes a stream on and, once its first Head bytes have gone on, // calls Insert before the rest: padding a format puts between the headers an // encoder writes first and the content that follows them. Insert gets the head diff --git a/internal/format/isobmff/isobmff.go b/internal/format/isobmff/isobmff.go index 9b60f7aa..ad8ee4a2 100644 --- a/internal/format/isobmff/isobmff.go +++ b/internal/format/isobmff/isobmff.go @@ -40,12 +40,21 @@ func WriteBoxHeader(w io.Writer, kind string, payload int64) error { // can say. The step below keeps the leftover from landing between one and // seven bytes, which no box could then carry. func WritePadding(ctx context.Context, w io.Writer, seed uint64, total int64) error { + return WriteFreeSpace(ctx, w, "free", seed, total) +} + +// WriteFreeSpace is WritePadding in boxes of kind, free or skip - the two +// names ISO 14496-12 section 8.1.2 gives a box whose content means nothing. +// They are the same to the format, and not to every reader: ffmpeg takes a +// JPEG XL container with a free box near its start for a movie, and one with +// a skip box for a picture (docs/TRUNCATE-2026-11-11.md section 13). +func WriteFreeSpace(ctx context.Context, w io.Writer, kind string, seed uint64, total int64) error { rng := core.NewRand(seed) buf := make([]byte, 32*1024) for total > 0 { payload := nextPayload(total) - if err := WriteBoxHeader(w, "free", payload); err != nil { + if err := WriteBoxHeader(w, kind, payload); err != nil { return err } if err := writeFiller(ctx, w, buf, rng, payload); err != nil { diff --git a/internal/format/jxl/cut.go b/internal/format/jxl/cut.go new file mode 100644 index 00000000..555eaef2 --- /dev/null +++ b/internal/format/jxl/cut.go @@ -0,0 +1,64 @@ +package jxl + +import ( + "context" + "io" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/isobmff" +) + +// splitForACut is how much of the padding pad goes in front of the codestream +// box of a file a damage will cut after its first cutAt bytes, so that the cut +// keeps the share of the codestream it keeps of the whole file +// (format.Request.CutAt). The rest goes behind it, past the cut. +// +// With all of the padding in front of the codestream, as an uncut file has it, +// a cut of any share would take the codestream before any padding. The +// codestream knows nothing of where it sits in the file, so nothing else +// moves (docs/TRUNCATE-2026-11-11.md section 13). Zero puts all of the padding +// behind the codestream, where the share wants nothing in front. +func splitForACut(pad, codestream, cutAt, whole int64) int64 { + return format.SplitFront(pad, int64(len(signature)+len(fileType)), codestream, cutAt, whole, boxHeader) +} + +// writeAround writes the container around the codestream with front bytes of +// skip boxes in front of the codestream box and back bytes behind it, and +// marks its map: the signature and file type boxes, the padding in front, the +// codestream box, the padding behind. A side with no padding has no box and no +// region. +// +// Skip rather than free on both sides, for ffmpeg: see the package comment. A +// file nothing cuts has nothing behind its codestream, and the side behind it +// in a file a damage cuts lies past the cut, so no file on disk carries a box +// behind its codestream. +func writeAround(ctx context.Context, w io.Writer, m memo, coded []byte, front, back int64) error { + head := int64(len(signature) + len(fileType)) + box := boxHeader + int64(len(coded)) + format.Mark(w, format.RoleStructure, 0, head) + if front > 0 { + format.Mark(w, format.RolePadding, head, front) + } + format.Mark(w, format.RoleContent, head+front, box) + if back > 0 { + format.Mark(w, format.RolePadding, head+front+box, back) + } + + if _, err := w.Write(signature[:]); err != nil { + return err + } + if _, err := w.Write(fileType[:]); err != nil { + return err + } + // Both sides draw their filler from the file's seed. Nobody reads padding. + if err := isobmff.WriteFreeSpace(ctx, w, "skip", m.seed, front); err != nil { + return err + } + if err := isobmff.WriteBoxHeader(w, "jxlc", int64(len(coded))); err != nil { + return err + } + if _, err := w.Write(coded); err != nil { + return err + } + return isobmff.WriteFreeSpace(ctx, w, "skip", m.seed, back) +} diff --git a/internal/format/jxl/jxl.go b/internal/format/jxl/jxl.go index 6d7ca328..63329091 100644 --- a/internal/format/jxl/jxl.go +++ b/internal/format/jxl/jxl.go @@ -36,20 +36,33 @@ // of the file asked for - still applies here unchanged, and this format passes // it. Owner's decision, 2026-08-31. // -// The padding channel is a free box, which is the box the container defines -// for space that means nothing. Measured on 2026-08-31 at 0, 1, 7, 64, 100, -// 101, 4096, 100 000 and 1 048 576 bytes, and again with two boxes, four -// boxes, a private box type, a skip box and the sixty four bit box length. -// libjxl through pillow-jxl-plugin reads every one of them, and refuses a -// truncated file and a corrupted one, which is what makes it a witness. The -// pure Go decoder in the encoder's own module agrees on all of it. +// The padding channel is skip boxes between the file type box and the +// codestream box, with nothing behind the codestream. A skip box is one of +// the two names the container gives a box for space that means nothing, and +// free is the other. Measured on 2026-08-31 at 0, 1, 7, 64, 100, 101, 4096, +// 100 000 and 1 048 576 bytes, and again with two boxes, four boxes, a +// private box type, a skip box and the sixty four bit box length: libjxl +// through pillow-jxl-plugin reads every one of them, and refuses a truncated +// file and a corrupted one, which is what makes it a witness. The pure Go +// decoder in the encoder's own module agrees on all of it. +// +// The place and the name are ffmpeg's, measured on 2026-11-13 +// (docs/TRUNCATE-2026-11-11.md section 13.4, O341). ffmpeg 7.1.5 with libjxl +// never finishes a JPEG XL with any box behind its codestream - free, skip or +// private, eight bytes or a thousand - and reads one with nothing behind it in +// a fifth of a second. ffmpeg 9.0.1 tells a JPEG XL by its first two +// kilobytes and takes one with a free box there for a movie, and one with a +// skip box there for a picture. With skip boxes in front, Pillow, ImageMagick, +// the Windows Imaging Component, Chrome, djxl and both ffmpeg read the same +// pixels at every size from 2 KB to 10 MB. // // Trailing bytes after a bare codestream were measured too, and they work - // both readers take them, down to a single byte, with no dead zone at all. // They are not used. They are not a structure the format defines, only // something two readers tolerate, and this project's fourth untouchable rule -// says fidelity does not drop for the implementer's convenience. The free box -// says "nothing here" in the format's own words, and it is what AVIF writes. +// says fidelity does not drop for the implementer's convenience. A file a +// damage will cut keeps part of its padding behind the codestream instead, +// where the cut never reaches (cut.go). package jxl import ( @@ -74,7 +87,7 @@ const ( // containerOverhead is everything the container costs around the // codestream: the signature box, the file type box, and the header of the - // box the codestream sits in. The free box header is counted separately, + // box the codestream sits in. The skip box header is counted separately, // because it is what makes every size above the minimum reachable. containerOverhead = len(signature) + len(fileType) + boxHeader @@ -97,7 +110,7 @@ const ( // for every seed there is. Measured across all of them by // tools/probes/jxlladder on 2026-08-31. // - // The container and the free box header are included, because this + // The container and the skip box header are included, because this // generator always writes both even when the box carries nothing. That is // what removes the unreachable band above the floor that PNG and GIF have. // @@ -149,8 +162,8 @@ func init() { // half that matters: both refuse a truncated file and a corrupted // payload, so both can say no. exiftool reads JPEG XL too and // accepts the truncated file, so it witnesses nothing. - Name: "a free box after the picture", - Where: format.PlacementEnd, + Name: "skip boxes in front of the picture", + Where: format.PlacementStart, Capacity: 0, }, Label: format.LabelVisible, @@ -202,10 +215,13 @@ type memo struct { // from the ladder without coding it and the work happens while writing. coded []byte // total is the size the file has to come to, so writing can work out how - // much free box follows the picture. There is always at least one, empty, + // much skip box goes in front of the picture. There is always at least one, empty, // which is what makes every size above the minimum reachable a byte at a // time. total int64 + // cutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). + cutAt int64 } func (generator) Plan(r format.Request) (format.Plan, error) { @@ -227,7 +243,7 @@ func (generator) Plan(r format.Request) (format.Plan, error) { if err := checkFits(r.Bytes, m); err != nil { return format.Plan{}, err } - m.total = r.Bytes + m.total, m.cutAt = r.Bytes, r.CutAt p := format.Plan{ Bytes: r.Bytes, @@ -284,7 +300,7 @@ const holdsPerPixel = 34 func checkFits(want int64, m memo) error { floor := int64(minimumBytes) reason := core.Says("jxl.MinimumReason", - "the smallest picture this format draws codes to %d B at worst, the container around it costs %d B, and the file always carries a free box, which costs %d B even when it holds nothing", + "the smallest picture this format draws codes to %d B at worst, the container around it costs %d B, and the file always carries a padding box, which costs %d B even when it holds nothing", core.A("Coded", minimumBytes-containerOverhead-boxHeader), core.A("Container", containerOverhead), core.A("Box", boxHeader)) // A picture named by hand can be larger than the smallest one, and then its @@ -294,7 +310,7 @@ func checkFits(want int64, m memo) error { if own := int64(len(m.coded) + containerOverhead + boxHeader); own > floor { floor = own reason = core.Says("jxl.MinimumReasonOwn", - "a %dx%d picture at quality %d codes to %d B, the container around it costs %d B, and the file always carries a free box, which costs %d B even when it holds nothing", + "a %dx%d picture at quality %d codes to %d B, the container around it costs %d B, and the file always carries a padding box, which costs %d B even when it holds nothing", core.A("Width", m.width), core.A("Height", m.height), core.A("Quality", m.quality), core.A("Coded", len(m.coded)), core.A("Container", containerOverhead), core.A("Box", boxHeader)) } @@ -325,7 +341,7 @@ func checkFits(want int64, m memo) error { // width and height are settings, and 40 megapixels is the ceiling. // // Ceiling is the LARGEST file that rung has ever been seen to produce, the -// container and the free box header included, and it is what lets planning +// container and the skip box header included, and it is what lets planning // choose a picture without coding one. Measured by tools/probes/jxlladder on // 2026-08-31 across all 256 seeds, four orders of magnitude of requested size - // the label carries the byte count, so its length moves with it - and with the @@ -509,29 +525,15 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if pad < boxHeader { // The ladder's ceiling for this picture was too low. Said out loud // rather than written short, because a short file is a wrong file. - return core.Defect(fmt.Errorf("jxl: a %dx%d picture coded to %d B, the container costs %d B and the file was to be %d B, which leaves no room for the free box every one of these carries", + return core.Defect(fmt.Errorf("jxl: a %dx%d picture coded to %d B, the container costs %d B and the file was to be %d B, which leaves no room for the skip box every one of these carries", m.width, m.height, len(coded), containerOverhead, m.total)) } - // The map, all of it known once the picture is coded: the signature and - // file type boxes, the box of the code stream, and the free boxes that - // make up the size. - head := int64(len(signature) + len(fileType)) - format.Mark(w, format.RoleStructure, 0, head) - format.Mark(w, format.RoleContent, head, boxHeader+int64(len(coded))) - format.Mark(w, format.RolePadding, head+boxHeader+int64(len(coded)), pad) - - if _, err := w.Write(signature[:]); err != nil { - return err - } - if _, err := w.Write(fileType[:]); err != nil { - return err - } - if err := isobmff.WriteBoxHeader(w, "jxlc", int64(len(coded))); err != nil { - return err - } - if _, err := w.Write(coded); err != nil { - return err + // The map is all known once the picture is coded, and a file nothing cuts + // has all of its padding in front of the codestream and nothing behind it. + front := pad + if m.cutAt > 0 { + front = splitForACut(pad, boxHeader+int64(len(coded)), m.cutAt, m.total) } - return isobmff.WritePadding(ctx, w, m.seed, pad) + return writeAround(ctx, w, m, coded, front, pad-front) } diff --git a/internal/format/jxl/measure.go b/internal/format/jxl/measure.go index 581553a7..d4763274 100644 --- a/internal/format/jxl/measure.go +++ b/internal/format/jxl/measure.go @@ -13,7 +13,7 @@ package jxl // FileSizeFor reports how many bytes this generator produces for one picture: -// the coded picture, the container around it, and the empty free box every +// the coded picture, the container around it, and the empty skip box every // file carries. It is the number a rung's ceiling has to be at least as large // as. // diff --git a/internal/format/mp4/mp4.go b/internal/format/mp4/mp4.go index 3aa217fd..48bb3fe0 100644 --- a/internal/format/mp4/mp4.go +++ b/internal/format/mp4/mp4.go @@ -116,7 +116,7 @@ func belowMinimum(requested, need int64, c video.Choice, s video.Settings) error Requested: requested, Minimum: need, Reason: core.Says("mp4.MinimumReasonPictures", - "a film of %s at %d frames a second with a key frame every %s and a new %dx%d picture every %s takes %d B, and the file always carries a free box, which costs %d B even when it holds nothing", + "a film of %s at %d frames a second with a key frame every %s and a new %dx%d picture every %s takes %d B, and the file always carries a padding box, which costs %d B even when it holds nothing", core.A("Duration", core.FormatDuration(s.DurationMs)), core.A("FPS", s.FPS), core.A("Interval", core.FormatDuration(s.KeyEvery*1000/int64(s.FPS))), core.A("Width", c.Width), core.A("Height", c.Height), core.A("Change", core.FormatDuration(s.ChangeMs())), diff --git a/internal/format/png/cut.go b/internal/format/png/cut.go index b68a6ecd..639ecbb4 100644 --- a/internal/format/png/cut.go +++ b/internal/format/png/cut.go @@ -38,7 +38,7 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { if pad < chunkOverhead || pad-chunkOverhead > maxChunkData { return core.Defect(fmt.Errorf("png: the picture encodes to %d B, which leaves %d B of padding for a %d B file", m.body, pad, whole)) } - front := frontPadding(pad, m.body-headSize, m.cutAt, whole) + front := format.SplitFront(pad, headSize, m.body-headSize, m.cutAt, whole, chunkOverhead) markAroundThePicture(w, m.body, front, pad-front) // Both padding chunks draw from the file's seed. Nobody reads padding, and @@ -69,36 +69,6 @@ func writeForACut(ctx context.Context, w io.Writer, m memo, whole int64) error { return err } -// frontPadding is how much of the padding, chunk headers included, goes in -// front of the picture, so that a cut after the first cutAt bytes of a whole -// file keeps content*cutAt/whole bytes of the picture's content. -// -// A chunk costs twelve bytes before any padding, so each side is nothing or at -// least that. Where the chunks cannot land exactly, the front grows, which -// moves the picture later and cuts more of it - the "at least" missing -// promises. The one way it can keep more is a tail too short for the content -// the cut should take: there all of the padding is in front, and the cut keeps -// up to a chunk's worth more than the share (docs/TRUNCATE-2026-11-11.md -// section 7.4). The manifest's cut reports what was kept either way. -func frontPadding(pad, content, cutAt, whole int64) int64 { - if pad == 0 || cutAt <= 0 || cutAt >= whole { - return 0 - } - front := format.PaddingInFront(headSize, content, cutAt, whole) - switch { - case front <= 0: - return 0 - case front < chunkOverhead: - front = chunkOverhead - } - if back := pad - front; back < chunkOverhead { - // Behind the picture there is room for nothing or for less than a - // chunk, so the whole padding goes in front. - return pad - } - return front -} - // markAroundThePicture marks the map of a file whose padding is split around // its picture: the header, the padding in front, the picture, the padding // behind and the closing chunk. A side with no padding has no chunk and no diff --git a/internal/guard/boxcut_test.go b/internal/guard/boxcut_test.go new file mode 100644 index 00000000..3f29debe --- /dev/null +++ b/internal/guard/boxcut_test.go @@ -0,0 +1,412 @@ +package guard + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "image" + "io" + "testing" + + gavif "github.com/gen2brain/gav1d/avif" + gjxl "github.com/gen2brain/jxl" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" +) + +// An AVIF or a JPEG XL laid out for a cut comes to every size asked for, its +// picture is where its boxes say, and the cut keeps the share of it. +// +// Both put free boxes in front of the picture for a cut, and a free box costs +// eight bytes before it carries anything, so each side is nothing or at least +// that: where the share wants less than a box in front, a box goes there, and +// where the padding left behind would be shorter than a box, it goes in front +// too (format.SplitFront). In an AVIF the box in front moves the media data, +// and the item locations in the meta box give its place in bytes from the +// start of the file, so they move with it. No other guard asks for the sizes +// and the cuts where those happen. So this writes consecutive sizes cut at +// their last byte and at the half, and cuts of one size a byte at a time from +// just in front of the picture, holds each to its size, to the share and, for +// an AVIF, to an item location that lands on the media data, and asserts it +// met each layout: nothing in front where the share wants nothing, a box where +// it wants less, the share to the byte, and all of the padding in front where +// what would stay behind is shorter than a box. One file of each layout is +// decoded and must give the pixels of the file nothing cuts. +// +// An AVIF of 5 GiB cut at 90% is planned and only its head is written: its +// media data starts past where four bytes reach, so the item location is +// widened to eight. +// +// Canary: an item location a byte off is refused. +func TestAnAVIFOrAJPEGXLLaidOutForACutComesToEverySizeAndReads(t *testing.T) { + for _, id := range []string{"avif", "jxl"} { + desc, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + met := map[string]bool{} + decoded := map[string]bool{} + check := func(size, cutAt int64) { + data, regions := writeBoxes(t, desc, size, cutAt) + layout, err := boxLayoutOf(id, data, regions, size, cutAt) + if err != nil { + t.Errorf("%s %d B cut at %d: %v", id, size, cutAt, err) + return + } + met[layout] = true + if !decoded[layout] { + decoded[layout] = true + whole, _ := writeBoxes(t, desc, size, 0) + if err := samePicture(id, data, whole); err != nil { + t.Errorf("%s %d B cut at %d, %s: %v", id, size, cutAt, layout, err) + } + } + } + smallest := desc.SmallestAccepted(format.Request{Label: true}) + for size := smallest; size < smallest+64; size++ { + check(size, size/2) + } + // Cuts a byte apart where the share starts to want padding in front - + // it wants cutAt times the part of the file that is not the picture, + // less the head - and near the end, where what the share leaves behind + // runs out: j bytes from the end it leaves the padding's part of j. + const size = 4_000 + _, regions := writeBoxes(t, desc, size, 0) + head, pic, pad := boxParts(t, regions) + rest := size - pic.Length + start := (head - 3) * size / rest + for cutAt := start; cutAt < start+12*size/rest; cutAt++ { + check(size, cutAt) + } + for j := int64(1); j <= 10*size/pad; j++ { + check(size, size-j) + } + for _, layout := range []string{"nothing in front, the share wants none", "a box in front, the share wants less", + "the share", "all in front, what would stay behind is shorter than a box"} { + if !met[layout] { + t.Errorf("%s: no size met the layout %q - met %v", id, layout, met) + } + } + } + + avifTooLargeForFourBytes(t) + + // Canary: an item location a byte off. + desc, _ := format.Get("avif") + data, _ := writeBoxes(t, desc, 64<<10, 32<<10) + broken := bytes.Clone(data) + at, _, err := avifItemPlace(broken) + if err != nil { + t.Fatal(err) + } + binary.BigEndian.PutUint32(broken[at:], binary.BigEndian.Uint32(broken[at:])+1) + if err := avifPlaceLands(broken); err == nil { + t.Errorf("canary: an item location a byte off passed") + } +} + +// writeBoxes writes a file of size bytes laid out for a cut after cutAt, +// whole, and the regions it marked. +func writeBoxes(t *testing.T, desc format.Descriptor, size, cutAt int64) ([]byte, []format.Region) { + t.Helper() + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: cutAt}) + if err != nil { + t.Fatalf("%s %d B: planning: %v", desc.ID, size, err) + } + var out bytes.Buffer + rec := &format.MapRecorder{W: &out} + if err := desc.Generator.Write(t.Context(), rec, p); err != nil { + t.Fatalf("%s %d B: writing: %v", desc.ID, size, err) + } + return out.Bytes(), rec.Map(size, true) +} + +// pictureRegion is the region a file marked as its content. +func pictureRegion(regions []format.Region) format.Region { + for _, r := range regions { + if r.Role == format.RoleContent { + return r + } + } + return format.Region{} +} + +// boxParts reads a file's map as what stands in front of all of its padding, +// its picture and all of its padding, wherever the padding stands. Neither +// format closes with an end region, and one would stand behind the padding, +// where head does not count it, so meeting one fails. +func boxParts(t *testing.T, regions []format.Region) (head int64, pic format.Region, pad int64) { + t.Helper() + for _, r := range regions { + switch r.Role { + case format.RoleEnd: + t.Fatalf("an end region at %d, which neither format writes", r.At) + case format.RoleStructure: + head += r.Length + case format.RolePadding: + pad += r.Length + case format.RoleContent: + pic = r + } + } + return head, pic, pad +} + +// A JPEG XL nothing cuts has nothing behind its codestream and no free box +// anywhere. ffmpeg 7.1.5 with libjxl never finishes a JPEG XL with any box +// behind its codestream, and ffmpeg 9.0.1 takes one with a free box in its +// first two kilobytes for a movie (O341, docs/TRUNCATE-2026-11-11.md section +// 13.4). So this writes the smallest sizes, where the padding is one box that +// carries nothing or a few bytes, and larger ones, and requires the codestream +// box to be the last box and no box to be a free one. +// +// Canary: the same file with an empty box behind its codestream is refused. +func TestAJPEGXLKeepsNothingBehindItsCodestream(t *testing.T) { + desc, err := format.Get("jxl") + if err != nil { + t.Fatal(err) + } + smallest := desc.SmallestAccepted(format.Request{Label: true}) + for _, size := range []int64{smallest, smallest + 1, smallest + 7, smallest + 8, smallest + 9, 2 << 10, 64 << 10, 1 << 20} { + data, _ := writeBoxes(t, desc, size, 0) + if err := jxlNothingBehind(data); err != nil { + t.Errorf("%d B: %v", size, err) + } + } + + data, _ := writeBoxes(t, desc, 2<<10, 0) + broken := append(bytes.Clone(data), 0, 0, 0, 8, 's', 'k', 'i', 'p') + if err := jxlNothingBehind(broken); err == nil { + t.Errorf("canary: a JPEG XL with an empty box behind its codestream passed") + } +} + +// jxlNothingBehind requires a JPEG XL's codestream box to be its last box, and +// none of its boxes to be a free one. +func jxlNothingBehind(data []byte) error { + last := "" + for pos := 0; pos+8 <= len(data); { + size := int(binary.BigEndian.Uint32(data[pos:])) + kind := string(data[pos+4 : pos+8]) + if size < 8 || pos+size > len(data) { + return fmt.Errorf("a %q box of %d B at %d", kind, size, pos) + } + if kind == "free" { + return fmt.Errorf("a free box at %d", pos) + } + last = kind + pos += size + } + if last != "jxlc" { + return fmt.Errorf("the last box is %q, where the codestream should end the file", last) + } + return nil +} + +// boxLayoutOf holds a file laid out for a cut to its size, its picture's place +// and the share the cut keeps of the picture, and names the layout it took. +func boxLayoutOf(id string, data []byte, regions []format.Region, size, cutAt int64) (string, error) { + if int64(len(data)) != size { + return "", fmt.Errorf("came to %d B", len(data)) + } + if id == "avif" { + if err := avifPlaceLands(data); err != nil { + return "", err + } + } else if err := jxlNoFreeAhead(data); err != nil { + return "", err + } + content := -1 + for i, r := range regions { + if r.Role == format.RoleContent { + content = i + } + } + if content < 0 { + return "", errors.New("no content marked") + } + pic := regions[content] + front, back := int64(0), int64(0) + if content > 0 && regions[content-1].Role == format.RolePadding { + front = regions[content-1].Length + } + for _, r := range regions[content+1:] { + if r.Role != format.RolePadding { + return "", fmt.Errorf("a %s region behind the picture", r.Role) + } + back += r.Length + } + if pic.At+pic.Length+back != size { + return "", fmt.Errorf("the picture ends at %d and %d B of padding follow it in a %d B file", pic.At+pic.Length, back, size) + } + if (front > 0 && front < 8) || (back > 0 && back < 8) { + return "", fmt.Errorf("a side shorter than a box: %d in front, %d behind", front, back) + } + head := pic.At - front + want := format.PaddingInFront(head, pic.Length, cutAt, size) + kept, share := cutAt-head-front, pic.Length*cutAt/size + switch { + case front == 0 && want <= 0: + return "nothing in front, the share wants none", nil + case front == 0: + return "", fmt.Errorf("nothing in front and %d B behind, where the share wants %d B in front", back, want) + case front == want && kept == share: + return "the share", nil + case front == 8 && want > 0 && want < 8: + return "a box in front, the share wants less", nil + case back == 0 && front > want && front-want < 8: + return "all in front, what would stay behind is shorter than a box", nil + } + return "", fmt.Errorf("%d B in front where the share wants %d, keeping %d B of the picture where the share is %d", front, want, kept, share) +} + +// samePicture decodes a file laid out for a cut and the file nothing cuts with +// the coder's own decoder, and requires the same pixels. +func samePicture(id string, laid, whole []byte) error { + decode := gjxl.Decode + if id == "avif" { + decode = func(r io.Reader, _ ...gjxl.Options) (image.Image, error) { return gavif.Decode(r) } + } + a, err := decode(bytes.NewReader(laid)) + if err != nil { + return fmt.Errorf("the decoder refused it: %w", err) + } + b, err := decode(bytes.NewReader(whole)) + if err != nil { + return fmt.Errorf("the decoder refused the file nothing cuts: %w", err) + } + if a.Bounds() != b.Bounds() { + return fmt.Errorf("a %v picture where nothing cut gives %v", a.Bounds(), b.Bounds()) + } + for y := a.Bounds().Min.Y; y < a.Bounds().Max.Y; y++ { + for x := a.Bounds().Min.X; x < a.Bounds().Max.X; x++ { + if a.At(x, y) != b.At(x, y) { + return fmt.Errorf("the pixel at %d,%d differs from the file nothing cuts", x, y) + } + } + } + return nil +} + +// avifItemPlace finds the one item location of an AVIF as tfg's coder writes +// it - version 0, no base offset, one item with one extent - and gives where +// its offset field is in the file and the place the field holds. +func avifItemPlace(data []byte) (field int, place int64, err error) { + for pos := 0; pos+8 <= len(data); { + size := int(binary.BigEndian.Uint32(data[pos:])) + if size < 8 || pos+size > len(data) { + break + } + if string(data[pos+4:pos+8]) == "meta" { + for at := pos + 12; at+8 <= pos+size; { + n := int(binary.BigEndian.Uint32(data[at:])) + if n < 8 { + break + } + if string(data[at+4:at+8]) == "iloc" { + b := data[at+8 : at+n] + offSize := int(b[4] >> 4) + if b[0] != 0 || b[5]>>4 != 0 || binary.BigEndian.Uint16(b[6:]) != 1 || binary.BigEndian.Uint16(b[12:]) != 1 { + return 0, 0, fmt.Errorf("an item location box this does not read: % x", b[:min(len(b), 16)]) + } + field = at + 8 + 14 + for _, c := range data[field : field+offSize] { + place = place<<8 | int64(c) + } + return field, place, nil + } + at += n + } + } + pos += size + } + return 0, 0, errors.New("no item location box") +} + +// avifPlaceLands requires the item location of an AVIF to give the first byte +// after the header of its media data box. +func avifPlaceLands(data []byte) error { + _, place, err := avifItemPlace(data) + if err != nil { + return err + } + for pos := int64(0); pos+8 <= int64(len(data)); { + size := int64(binary.BigEndian.Uint32(data[pos:])) + if size < 8 { + break + } + if string(data[pos+4:pos+8]) == "mdat" { + if place != pos+8 { + return fmt.Errorf("the item location gives %d where the media data starts at %d", place, pos+8) + } + return nil + } + pos += size + } + return errors.New("no media data box") +} + +// jxlNoFreeAhead requires no free box in front of a JPEG XL's codestream box. +// ffmpeg 9.0.1 takes a JPEG XL container with a free box in its first two +// kilobytes for a movie and refuses it, and one with a skip box there for a +// picture (docs/TRUNCATE-2026-11-11.md section 13). +func jxlNoFreeAhead(data []byte) error { + for pos := 0; pos+8 <= len(data); { + size := int(binary.BigEndian.Uint32(data[pos:])) + switch kind := string(data[pos+4 : pos+8]); { + case kind == "jxlc": + return nil + case kind == "free": + return fmt.Errorf("a free box at %d, in front of the codestream", pos) + case size < 8: + return fmt.Errorf("a box of %d B at %d", size, pos) + } + pos += size + } + return errors.New("no codestream box") +} + +// avifTooLargeForFourBytes plans an AVIF of 5 GiB cut at 90% and holds the +// head it writes to an item location of eight bytes that gives where the map +// says the media data starts. It writes nothing past the head. +func avifTooLargeForFourBytes(t *testing.T) { + t.Helper() + const size = 5 << 30 + desc, _ := format.Get("avif") + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: size / 10 * 9}) + if err != nil { + t.Fatalf("planning %d B: %v", int64(size), err) + } + head := &headOnly{keep: 1024} + rec := &format.MapRecorder{W: head} + if err := desc.Generator.Write(t.Context(), rec, p); err == nil { + t.Fatalf("writing %d B went through a writer that stops after its head", int64(size)) + } + pic := pictureRegion(rec.Map(size, true)) + field, place, err := avifItemPlace(head.got) + if err != nil { + t.Fatalf("%d B: %v", int64(size), err) + } + if width := head.got[field-14+4] >> 4; width != 8 || place != pic.At+8 || pic.At <= 1<<32 { + t.Errorf("%d B cut at 90%%: an item location of %d bytes gives %d, where the media data starts at %d", + int64(size), width, place, pic.At+8) + } +} + +// headOnly keeps the first keep bytes written to it and refuses the rest. +type headOnly struct { + keep int + got []byte +} + +func (h *headOnly) Write(p []byte) (int, error) { + if len(h.got)+len(p) > h.keep { + n := h.keep - len(h.got) + h.got = append(h.got, p[:n]...) + return n, errors.New("only the head is kept") + } + h.got = append(h.got, p...) + return len(p), nil +} diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index dadb21aa..6a9d600f 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -32,8 +32,6 @@ type notYet struct { // that leaves is held to every size and share, and so is a new format unless // it is added here with its reason. var notYetCutAtTheShare = map[string]notYet{ - "avif": {paddingBehind, 64 << 10, 50}, - "jxl": {paddingBehind, 64 << 10, 50}, "mp4": {paddingBehind, 64 << 10, 50}, "webm": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the @@ -50,7 +48,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"avif", "jxl", "mp4", "webm", "webp", "tiff"} +var listedBeforeTheRule = []string{"mp4", "webm", "webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. diff --git a/internal/guard/mutation/entries.py b/internal/guard/mutation/entries.py index 1ee94990..85a5174b 100644 --- a/internal/guard/mutation/entries.py +++ b/internal/guard/mutation/entries.py @@ -7545,7 +7545,7 @@ ), ( "the JPEG XL codestream goes into a partial codestream box without the index one needs, so no reader finds a picture", - "internal/format/jxl/jxl.go", + "internal/format/jxl/cut.go", " if err := isobmff.WriteBoxHeader(w, \"jxlc\", int64(len(coded))); err != nil {", " if err := isobmff.WriteBoxHeader(w, \"jxlp\", int64(len(coded))); err != nil {", "TestEveryFormatSurvivesItsReferenceTool", @@ -14001,8 +14001,8 @@ ( "an MP4 carries its padding in a box that is not free", "internal/format/isobmff/isobmff.go", - " if err := WriteBoxHeader(w, \"free\", payload); err != nil {", - " if err := WriteBoxHeader(w, \"skip\", payload); err != nil {", + " return WriteFreeSpace(ctx, w, \"free\", seed, total)", + " return WriteFreeSpace(ctx, w, \"skip\", seed, total)", "TestAFilmHoldsTheFramesTheManifestDeclares", ), ( diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index f75f8d12..8baf5595 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -159,6 +159,14 @@ var provenByProbe = map[string]string{ "fixed codes, must inflate to its bytes and read as those runs - its reach check, every frame of the marker met - and red on " + "2026-11-13 for a frame a byte short (the plan refuses at 65541 B) and for an empty block's BTYPE bit one place off " + "(compress/flate refuses at 65537 B), where the first passed the share and the map guards.", + "TestAJPEGXLKeepsNothingBehindItsCodestream": "its own canary - the same file with an empty box behind its " + + "codestream must be refused - and red on 2026-11-13 with the padding of a file nothing cuts put back behind the " + + "codestream (all 8 sizes).", + "TestAnAVIFOrAJPEGXLLaidOutForACutComesToEverySizeAndReads": "its own canary - an item location a byte off must be " + + "refused - its reach check, every layout of the split met, and red on 2026-11-13 with the item locations not moved " + + "(50 cuts), with the widening's four bytes left out of the move (5 GiB: 4831825054 where the media data starts at " + + "4831825058), with a side of padding allowed below a box (4000 B came to 4007 B) and with a free box rather than a " + + "skip box in front of a JPEG XL's codestream (81 cuts).", "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canaries - a table that sends one object a byte off " + "must be refused, and the layout of a document nothing cuts must fail a cut at its half - its reach check, every " + "layout of the split met, red on 2026-11-13 with nothing in front of a document under 400 B of padding (398 sizes, " + diff --git a/internal/guard/testdata/generator-golden.json b/internal/guard/testdata/generator-golden.json index 1ec1b63f..e5ed1054 100644 --- a/internal/guard/testdata/generator-golden.json +++ b/internal/guard/testdata/generator-golden.json @@ -146,18 +146,18 @@ }, "jxl_64kib": { "bytes": 65536, - "sha256": "1f7946aacc7f6a90c3b2cc9713dca2a9113d1b76d7ec61691b52935d74b26437", - "measured_on": "2026-08-31" + "sha256": "eef78830e117ca9bfca4a06e09f84f3da2fffe2f7237b4a96dca7eb77bf93f99", + "measured_on": "2026-11-13" }, "jxl_odd_size": { "bytes": 65537, - "sha256": "768e0af192480e02607d4fa6f0f343f1cb0dd8ab49d373c92245f3aa68caa017", - "measured_on": "2026-08-31" + "sha256": "5f55ef75b2e4a56d24d03b493e82561beb34260e136dca626e7f3d4cca76716c", + "measured_on": "2026-11-13" }, "jxl_quality": { "bytes": 65536, - "sha256": "c5c522aed0d2051a20f8eb52bfecd49bb13f41cf86b30170eaf3c05998323a9a", - "measured_on": "2026-08-31" + "sha256": "405418822737987df23e31d0e5927fa9472438eb5a9a10915031c1d8631b9e27", + "measured_on": "2026-11-13" }, "log_8kib": { "bytes": 8192, @@ -653,8 +653,8 @@ }, "jxl_label_cut": { "bytes": 16384, - "sha256": "47e864c31b44ecc262d3b2f9cc9cca6f404daba0bb0dbc0f252cb8d470aa3311", - "measured_on": "2026-10-27" + "sha256": "04c4b0ffa9b8f631f800f60f0436ae8b2a206266ded04bccab340b6cba75e3dc", + "measured_on": "2026-11-13" }, "mp4_label_cut": { "bytes": 65536, @@ -773,6 +773,16 @@ } }, "remeasured": [ + { + "on": "2026-11-13", + "why": "A JPEG XL carries its padding in skip boxes in front of its codestream now, where it carried a free box behind it, and nothing stands behind the codestream. ffmpeg 7.1.5 with libjxl never finishes a JPEG XL with any box behind its codestream - free, skip or private, eight bytes or a thousand - so it could not read a single file this tool wrote, and ffmpeg 9.0.1 tells a JPEG XL by its first two kilobytes and took one with a free box there for a movie, so it refused every one below about 3.8 KB (O341). With skip boxes in front, Pillow, ImageMagick, the Windows Imaging Component, Chrome, djxl and both ffmpeg read the same pixels at every size from 2 KB to 10 MB (docs/TRUNCATE-2026-11-11.md section 13.4). Sizes are unchanged, and the box costs the same eight bytes. This is a breaking change by the owner's decision of 2026-11-13. Guarded from now on by TestAJPEGXLKeepsNothingBehindItsCodestream.", + "files": [ + "jxl_64kib", + "jxl_odd_size", + "jxl_quality", + "jxl_label_cut" + ] + }, { "on": "2026-10-15", "why": "A page with no label to fill its title carries Page there now, where it carried an empty title - the smallest document and every page under --clean, in every size. The Nu HTML Checker calls an empty title an error, so every such page was invalid HTML since 2026-08-01 (O277). Page rather than the tool's name, because --clean is there to keep the tool's description out of the file. The smallest html page goes from 118 B to 122 B. The catalogue of every element names its forms article tfg-forms, and its two links to it, because DOMPurify takes out an id that shadows document.forms and the links then led nowhere (O282) - the reason the links list is tfg-links. Its hidden field is named tfg-hidden, because a name hidden shadows document.hidden and DOMPurify took that out too, leaving the field out of a submitted form. The page of a newsletter says its language, , as the html format's pages always did, written in the frame so quoted-printable gives its = as =3D. Inside it, the line in Polish and Japanese marks each half with its own lang in a span, on two lines where the Japanese half comes, so the page saying English does not make a screen reader read them as English (review of #190). A message laid out simple is unchanged. This is a breaking change by the owner's decision of 2026-10-15, docs/HTML-6-2026-10-15.md. The two placeholder pages had no golden case and have one from today. Guarded from now on by the structural checker, which refuses a document without a title with text in it and a newsletter page without its language.", diff --git a/internal/guard/testdata/screens/generate-refused.png b/internal/guard/testdata/screens/generate-refused.png index d9462bd7..e369d7f4 100644 Binary files a/internal/guard/testdata/screens/generate-refused.png and b/internal/guard/testdata/screens/generate-refused.png differ diff --git a/internal/guard/testdata/screens/generate-refused.xml b/internal/guard/testdata/screens/generate-refused.xml index 87685aab..7d86e773 100644 --- a/internal/guard/testdata/screens/generate-refused.xml +++ b/internal/guard/testdata/screens/generate-refused.xml @@ -160,8 +160,8 @@ AVIF cannot be smaller than 311 B - the smallest picture this format draws codes to 303 B at worst, and the file always - carries a free box, which costs 8 B even when it holds nothing. Requested: 1 B. Ask for 311 B or more, or set a smaller - width and height, or a lower quality + carries a padding box, which costs 8 B even when it holds nothing. Requested: 1 B. Ask for 311 B or more, or set a + smaller width and height, or a lower quality diff --git a/internal/gui/text/locale/said/en.json b/internal/gui/text/locale/said/en.json index e9d67579..3bcafcb9 100644 --- a/internal/gui/text/locale/said/en.json +++ b/internal/gui/text/locale/said/en.json @@ -341,13 +341,13 @@ }, "avif.MinimumReason": { "description": "Said by internal/format/avif - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Coded}}, {{.Box}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-97b2a4a72e3a", - "other": "the smallest picture this format draws codes to {{.Coded}} B at worst, and the file always carries a free box, which costs {{.Box}} B even when it holds nothing" + "hash": "sha256-0ce5a98f055a", + "other": "the smallest picture this format draws codes to {{.Coded}} B at worst, and the file always carries a padding box, which costs {{.Box}} B even when it holds nothing" }, "avif.MinimumReasonOwn": { "description": "Said by internal/format/avif - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Width}}, {{.Height}}, {{.Quality}}, {{.Coded}}, {{.Box}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-ff69f6fa9b64", - "other": "a {{.Width}}x{{.Height}} picture at quality {{.Quality}} codes to {{.Coded}} B, and the file always carries a free box, which costs {{.Box}} B even when it holds nothing" + "hash": "sha256-bed39f5c0c28", + "other": "a {{.Width}}x{{.Height}} picture at quality {{.Quality}} codes to {{.Coded}} B, and the file always carries a padding box, which costs {{.Box}} B even when it holds nothing" }, "checksum.AChecksumDigitsLongIsNot": { "description": "Said by internal/tool/checksum - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Digits}}, written the way the program writes them - keep each spelled exactly that way.", @@ -2322,13 +2322,13 @@ }, "jxl.MinimumReason": { "description": "Said by internal/format/jxl - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Coded}}, {{.Container}}, {{.Box}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-8031f7cf3cc6", - "other": "the smallest picture this format draws codes to {{.Coded}} B at worst, the container around it costs {{.Container}} B, and the file always carries a free box, which costs {{.Box}} B even when it holds nothing" + "hash": "sha256-0c3cd13e8d89", + "other": "the smallest picture this format draws codes to {{.Coded}} B at worst, the container around it costs {{.Container}} B, and the file always carries a padding box, which costs {{.Box}} B even when it holds nothing" }, "jxl.MinimumReasonOwn": { "description": "Said by internal/format/jxl - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Width}}, {{.Height}}, {{.Quality}}, {{.Coded}}, {{.Container}}, {{.Box}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-08ce5283cdaa", - "other": "a {{.Width}}x{{.Height}} picture at quality {{.Quality}} codes to {{.Coded}} B, the container around it costs {{.Container}} B, and the file always carries a free box, which costs {{.Box}} B even when it holds nothing" + "hash": "sha256-1d657fb59674", + "other": "a {{.Width}}x{{.Height}} picture at quality {{.Quality}} codes to {{.Coded}} B, the container around it costs {{.Container}} B, and the file always carries a padding box, which costs {{.Box}} B even when it holds nothing" }, "log.ClientAddress": { "description": "Said by internal/format/logfile - a refusal under the box it is about or at the foot of the form, or a note after a run.", @@ -2468,8 +2468,8 @@ }, "mp4.MinimumReasonPictures": { "description": "Said by internal/format/mp4 - a refusal under the box it is about or at the foot of the form, or a note after a run. Carries {{.Duration}}, {{.FPS}}, {{.Interval}}, {{.Width}}, {{.Height}}, {{.Change}}, {{.Bytes}}, {{.Free}}, written the way the program writes them - keep each spelled exactly that way.", - "hash": "sha256-2d553380ec9d", - "other": "a film of {{.Duration}} at {{.FPS}} frames a second with a key frame every {{.Interval}} and a new {{.Width}}x{{.Height}} picture every {{.Change}} takes {{.Bytes}} B, and the file always carries a free box, which costs {{.Free}} B even when it holds nothing" + "hash": "sha256-0c2a2c450da2", + "other": "a film of {{.Duration}} at {{.FPS}} frames a second with a key frame every {{.Interval}} and a new {{.Width}}x{{.Height}} picture every {{.Change}} takes {{.Bytes}} B, and the file always carries a padding box, which costs {{.Free}} B even when it holds nothing" }, "otlp.AFileHoldsWholeRecords": { "description": "Said by internal/format/otlp - a refusal under the box it is about or at the foot of the form, or a note after a run.", diff --git a/internal/gui/text/locale/said/pl.json b/internal/gui/text/locale/said/pl.json index 0affa011..3cc05771 100644 --- a/internal/gui/text/locale/said/pl.json +++ b/internal/gui/text/locale/said/pl.json @@ -67,8 +67,8 @@ "archivescan.TwoNames": { "hash": "sha256-8816d7bc260e", "other": "{{.Listed}} w spisie, {{.Local}} przy wpisie" }, "archivescan.Type": { "hash": "sha256-ea85c62d6ca3", "other": "to wpis typu {{.Type}}, którego większość programów nie rozpakowuje" }, "archivescan.WaitAndRunAgain": { "hash": "sha256-161d36039e9c", "other": "poczekaj, aż to, co go zapisuje, skończy, i uruchom to jeszcze raz" }, - "avif.MinimumReason": { "hash": "sha256-97b2a4a72e3a", "other": "najmniejszy obrazek, jaki rysuje ten format, koduje się w najgorszym razie do {{.Coded}} B, a plik zawsze niesie wolny blok, który kosztuje {{.Box}} B nawet pusty" }, - "avif.MinimumReasonOwn": { "hash": "sha256-ff69f6fa9b64", "other": "obrazek {{.Width}}x{{.Height}} w jakości {{.Quality}} koduje się do {{.Coded}} B, a plik zawsze niesie wolny blok, który kosztuje {{.Box}} B nawet pusty" }, + "avif.MinimumReason": { "hash": "sha256-0ce5a98f055a", "other": "najmniejszy obrazek, jaki rysuje ten format, koduje się w najgorszym razie do {{.Coded}} B, a plik zawsze niesie blok dopełnienia, który kosztuje {{.Box}} B nawet pusty" }, + "avif.MinimumReasonOwn": { "hash": "sha256-bed39f5c0c28", "other": "obrazek {{.Width}}x{{.Height}} w jakości {{.Quality}} koduje się do {{.Coded}} B, a plik zawsze niesie blok dopełnienia, który kosztuje {{.Box}} B nawet pusty" }, "checksum.AChecksumDigitsLongIsNot": { "hash": "sha256-0dcdc03bd2f7", "other": "suma kontrolna o długości {{.Digits}} cyfr nie jest taką, jaką to narzędzie wylicza" }, "checksum.AChecksumFileIsNeverWritten": { "hash": "sha256-49681362425b", "other": "pliku sum kontrolnych nigdy się nie nadpisuje, bo może być jedynym zapisem tego, co było w folderze" }, "checksum.AChecksumFilePromisesEveryFile": { "hash": "sha256-354be45714f3", "other": "plik sum kontrolnych obiecuje każdy plik w folderze, a taki z luką nie powiedziałby, gdzie ta luka jest" }, @@ -462,8 +462,8 @@ "html.UseOneOfThePictureFormats": { "hash": "sha256-a6669f6abe6b", "other": "Użyj jednego z: {{.Formats}}." }, "imagelabel.ThePictureIsPxTallAnd": { "hash": "sha256-238ba2886f79", "other": "Obrazek ma {{.Height}} px wysokości, a etykieta potrzebuje {{.Needs}} px, więc jej tekst jest ucięty i ten plik nie niesie czytelnej etykiety. Wciąż wskazują go jego nazwa i manifest." }, "json.HasToBeStyle": { "hash": "sha256-c1343585d922", "other": "musi to być {{.Indented}}, {{.Minified}} albo {{.RecordPerLine}}" }, - "jxl.MinimumReason": { "hash": "sha256-8031f7cf3cc6", "other": "najmniejszy obrazek, jaki rysuje ten format, koduje się w najgorszym razie do {{.Coded}} B, kontener wokół niego kosztuje {{.Container}} B, a plik zawsze niesie wolny blok, który kosztuje {{.Box}} B nawet pusty" }, - "jxl.MinimumReasonOwn": { "hash": "sha256-08ce5283cdaa", "other": "obrazek {{.Width}}x{{.Height}} w jakości {{.Quality}} koduje się do {{.Coded}} B, kontener wokół niego kosztuje {{.Container}} B, a plik zawsze niesie wolny blok, który kosztuje {{.Box}} B nawet pusty" }, + "jxl.MinimumReason": { "hash": "sha256-0c3cd13e8d89", "other": "najmniejszy obrazek, jaki rysuje ten format, koduje się w najgorszym razie do {{.Coded}} B, kontener wokół niego kosztuje {{.Container}} B, a plik zawsze niesie blok dopełnienia, który kosztuje {{.Box}} B nawet pusty" }, + "jxl.MinimumReasonOwn": { "hash": "sha256-1d657fb59674", "other": "obrazek {{.Width}}x{{.Height}} w jakości {{.Quality}} koduje się do {{.Coded}} B, kontener wokół niego kosztuje {{.Container}} B, a plik zawsze niesie blok dopełnienia, który kosztuje {{.Box}} B nawet pusty" }, "log.ClientAddress": { "hash": "sha256-df4616586839", "other": "adres klienta" }, "log.ConflictDrop": { "hash": "sha256-9d99490328fe", "other": "{{.Why}}. Usuń jedno z dwóch albo zmień drugie" }, "log.RequestMethod": { "hash": "sha256-eaa67a9c3b30", "other": "metoda żądania" }, @@ -491,7 +491,7 @@ "manifest.TwiceBecause": { "hash": "sha256-1ae6ad202098", "other": "{{.Detail}}. To narzędzie zapisuje każdy klucz raz, więc manifest zapisało albo zmieniło coś innego, a to, którą z dwóch wartości ma na myśli, to zgadywanie. Użyj manifestu, który faktycznie zapisał przebieg" }, "manifest.Unreadable": { "hash": "sha256-bb90fafec850", "other": "{{.Path}} nie daje się odczytać jako manifest: {{.Detail}}" }, "manifest.ValueTooLarge": { "hash": "sha256-65f9e8304926", "other": "{{.Path}} zawiera pojedynczą wartość dłuższą niż {{.Limit}} B, zaczynającą się mniej więcej {{.At}} B od początku, a to narzędzie czyta z manifestu naraz najwyżej tyle. Wpis manifestu zapisanego przez to narzędzie zajmuje kilobajt lub dwa, więc sprawdź, czy to manifest zapisany przez przebieg" }, - "mp4.MinimumReasonPictures": { "hash": "sha256-2d553380ec9d", "other": "film o długości {{.Duration}} przy {{.FPS}} klatkach na sekundę z klatką kluczową co {{.Interval}} i nowym obrazem {{.Width}}x{{.Height}} co {{.Change}} zajmuje {{.Bytes}} B, a plik zawsze niesie pudełko free, które kosztuje {{.Free}} B nawet puste" }, + "mp4.MinimumReasonPictures": { "hash": "sha256-0c2a2c450da2", "other": "film o długości {{.Duration}} przy {{.FPS}} klatkach na sekundę z klatką kluczową co {{.Interval}} i nowym obrazem {{.Width}}x{{.Height}} co {{.Change}} zajmuje {{.Bytes}} B, a plik zawsze niesie blok dopełnienia, który kosztuje {{.Free}} B nawet pusty" }, "otlp.AFileHoldsWholeRecords": { "hash": "sha256-8bd4f63f9cb2", "other": "plik składa się z całych rekordów, a najdłuższy, jaki te ustawienia mogą zapisać, potrzebuje tyle" }, "otlp.AFileThisLargeAddsUpMore": { "hash": "sha256-f2957a096ca2", "other": "plik tej wielkości dodałby w jednej serii {{.Metric}} więcej czasu, niż da się zapisać z dokładnością do mikrosekundy. Wybierz mniejszy plik albo temporalność przyrostowo, gdzie dany rodzaj ją ma" }, "otlp.AFileThisLargeCollectsAfter": { "hash": "sha256-a18c86e7715d", "other": "plik tej wielkości zbierany co {{.Interval}} s datowałby swoje ostatnie zbiory po roku 2262. Wybierz wcześniejszy start, krótszy odstęp albo mniejszy plik" }, diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index 4826ad63..2493ae34 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -2065,20 +2065,66 @@ def check_avif(data): if needed not in inner: fail(f"the meta box has no {needed} box, so it does not say {AVIF_MEANING[needed]}") - # Padding belongs after the picture and nowhere else. + # Padding stands after the picture data, and a file laid out for a cut puts + # part of it right in front of the picture data too + # (docs/TRUNCATE-2026-11-11.md section 13) - nowhere else. free_bytes = 0 - seen_mdat = False - for name, start, stop in top: - if name == b"mdat": - seen_mdat = True - elif name == b"free": + mdat_at = names.index("mdat") + for i, (name, start, stop) in enumerate(top): + if name == b"free": free_bytes += stop - start + 8 - if not seen_mdat: - fail("a free box sits before the picture data, and this generator puts padding after it") + if i < mdat_at and any(n != b"free" for n, _, _ in top[i + 1:mdat_at]): + fail("a free box sits in front of the picture data with other boxes between them, and this generator " + "puts padding right in front of the picture data or after it") + + # The item locations give the picture's place in bytes from the start of + # the file, so padding in front of it moves them. Each has to land inside + # the media data. + iloc = next((s, e) for n, s, e in iso_boxes(data, meta_start + 4, meta_end) if n == b"iloc") + media = [(s, e) for n, s, e in top if n == b"mdat"] + for at, length in iloc_extents(data, *iloc): + if not any(s <= at and at + length <= e for s, e in media): + fail(f"an item location gives {length} B at {at}, which is not inside the media data at {media}") ok(f"boxes {names}, brands {[b.decode('latin1') for b in brands]}, {free_bytes} B of free") +def iloc_extents(data, start, stop): + """The places in this file an item location box's payload between start and + stop gives, as (offset, length) - ISO 14496-12 section 8.11.3, versions 0 + to 2. Places in other files or in idat are left out.""" + b = data[start:stop] + if len(b) < 8 or b[0] > 2: + fail("the item location box is too short or in a version this does not read") + version = b[0] + off_size, len_size, base_size = b[4] >> 4, b[4] & 15, b[5] >> 4 + index_size = b[5] & 15 if version in (1, 2) else 0 + at = 6 + + def field(n): + nonlocal at + if at + n > len(b): + fail("the item location box ends inside a field") + v = int.from_bytes(b[at:at + n], "big") + at += n + return v + + places = [] + for _ in range(field(4 if version == 2 else 2)): + field(4 if version == 2 else 2) + method = field(2) & 15 if version in (1, 2) else 0 + field(2) + base = field(base_size) + for _ in range(field(2)): + field(index_size) + offset, length = field(off_size), field(len_size) + if method == 0: + places.append((base + offset, length)) + if at != len(b): + fail("the item location box carries bytes its fields do not account for") + return places + + JXL_SIGNATURE = b"\x00\x00\x00\x0cJXL \x0d\x0a\x87\x0a" @@ -2126,16 +2172,22 @@ def check_jxl(data): if data[code_start:code_start + 2] != b"\xff\x0a": fail("the jxlc box does not start with the codestream signature FF 0A") - # Padding belongs after the picture and nowhere else. + # Padding stands in skip boxes right in front of the codestream, and a file + # laid out for a cut puts part of it behind the codestream, past the cut + # (docs/TRUNCATE-2026-11-11.md section 13) - nowhere else. Never a free + # box: ffmpeg takes a JPEG XL with one in its first two kilobytes for a + # movie (O341). free_bytes = 0 - seen_code = False - for name, start, stop in top: - if name == b"jxlc": - seen_code = True - elif name == b"free": + code_at = names.index("jxlc") + for i, (name, start, stop) in enumerate(top): + if name == b"free": + fail("the file carries a free box, and this generator pads a JPEG XL with skip boxes, which ffmpeg " + "does not take for a movie") + if name == b"skip": free_bytes += stop - start + 8 - if not seen_code: - fail("a free box sits before the picture data, and this generator puts padding after it") + if i < code_at and any(n != b"skip" for n, _, _ in top[i + 1:code_at]): + fail("a skip box sits in front of the codestream with other boxes between them, and this generator " + "puts padding right in front of the codestream or after it") tail = len(data) - top[-1][2] if tail: