diff --git a/CHANGELOG.md b/CHANGELOG.md index 04f1d2b0..1753fda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,13 @@ because it turns other people's test suites red. lower detail in Chrome and in libjxl with partial files allowed, because the format sends a coarse picture first. A cut of 90% shows nothing. An AVIF nothing cuts is unchanged. Every JPEG XL changes (see Breaking). +- **`truncate` cuts the pictures of an MP4 or a WebM film by the percentage + you set.** A cut used to take only padding, leaving every frame. Now + `--damage truncate:missing=50` removes half of the coded pictures, as an + interrupted download does, and ffmpeg gives back the frames in front of the + cut. Chrome and Firefox stop a cut MP4 with an error and end a cut WebM + without one. The manifest of a cut WebM now says how long its pictures are, + where it said nothing. Films nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/mp4/cut.go b/internal/format/mp4/cut.go new file mode 100644 index 00000000..f342fee9 --- /dev/null +++ b/internal/format/mp4/cut.go @@ -0,0 +1,58 @@ +package mp4 + +import ( + "math" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// splitForACut settles how much of the padding goes between the movie box and +// the media data of a film a damage will cut after its first cutAt bytes, so +// that the cut keeps the share of the media data it keeps of the whole file +// (format.Request.CutAt), and gives back the padding left behind the media +// data. content is the media data's samples, as the first pass measured them. +// +// With all of the padding behind the media data, as an uncut film has it, a +// large film cut at any share kept every picture and lost only padding. A free +// box may stand anywhere among the top level boxes, and readers skip it in +// front of the media data as they do behind it - ffmpeg's faststart writes one +// there (docs/TRUNCATE-2026-11-11.md section 14.4). The one thing a box there +// changes is where every chunk starts, which stco says in bytes from the start +// of the file, so every place moves by the gap: front counts it, and the places +// are written from front. +// +// The places take four bytes unless the layout already asked for eight. One +// that the gap takes past 4 GiB widens the table to co64, as a film whose +// pictures take their whole reserve past 4 GiB has it, and the movie box grows +// by four bytes a chunk - only then. When the padding cannot give up those +// bytes and still hold a free box, the film is laid out as one nothing cuts, +// and the manifest's cut says where the cut fell. That takes a film whose +// pictures came within four bytes a chunk of their whole reserve, which the +// guard cannot reach (section 14.3). +// +// A film whose share wants nothing in front - and every film nothing cuts - is +// laid out as one nothing cuts. It sets l.gap, and l.wide when it widens. +func splitForACut(l *layout, content, total, cutAt int64) (back int64) { + before := int64(len(l.ftyp)) + l.moovLen() + mdat := l.front() - before + content + pad := total - before - mdat + l.gap = format.SplitFront(pad, before, mdat, cutAt, total, minFree) + if l.gap > 0 && !l.wide && l.front()+content > math.MaxUint32 { + grown := (wideChunk - narrowChunk) * l.chunks + l.gap = 0 + if pad-grown >= minFree { + l.wide = true + l.gap = format.SplitFront(pad-grown, before+grown, mdat, cutAt, total, minFree) + l.wide = l.gap > 0 + } + } + return total - l.front() - content +} + +// mediaData is how long the samples of the media data come to, chunk by +// chunk. +func (z sizes) mediaData(l layout) int64 { + var n int64 + l.eachChunk(func(first, samples int64) { n += z.chunk(first, samples) }) + return n +} diff --git a/internal/format/mp4/layout.go b/internal/format/mp4/layout.go index 2873623a..a08c5deb 100644 --- a/internal/format/mp4/layout.go +++ b/internal/format/mp4/layout.go @@ -43,8 +43,12 @@ type layout struct { // wide is whether the chunks' places take eight bytes each (co64), and // bigData whether the media data's length does - each only when a film // whose pictures take their whole reserve would need it, so a smaller film - // never has a field it could not fill. + // never has a field it could not fill. A film a damage will cut may widen + // its places for the padding in front of the media data (cut.go). wide, bigData bool + // gap is the padding between the movie box and the media data, nothing + // but in a film a damage will cut (cut.go). Planning counts none. + gap int64 ftyp, mvhd, tkhd, mdhd, hdlr, vmhd, dinf, stsd, stts []byte } @@ -92,13 +96,13 @@ func (l layout) boundContent() int64 { } // front is where the media data's content starts: the file type, the movie -// box and the media data's header. +// box, the padding in front of the media data and the media data's header. func (l layout) front() int64 { head := int64(isobmff.BoxHeader) if l.bigData { head += 8 } - return int64(len(l.ftyp)) + l.moovLen() + head + return int64(len(l.ftyp)) + l.moovLen() + l.gap + head } // need is the file without its padding when every picture takes its whole diff --git a/internal/format/mp4/mp4.go b/internal/format/mp4/mp4.go index 48bb3fe0..6d8841fa 100644 --- a/internal/format/mp4/mp4.go +++ b/internal/format/mp4/mp4.go @@ -21,7 +21,9 @@ // with the probe at 8, 9, 15, 16 and 1 048 576 bytes and with none. A free box // is always written, at least its eight bytes, which leaves no size above the // minimum that cannot be reached, and one larger than a box length can say is -// several (internal/format/isobmff). +// several (internal/format/isobmff). A film a damage will cut splits them +// around the media data, so the cut keeps the share of the pictures asked for +// (cut.go). package mp4 import ( @@ -138,17 +140,46 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } defer pics.Close() - // The map: the file type and the movie box, then the media data box with - // the pictures, then the free boxes that make up the size. The movie box - // is the same length whatever the pictures come to, so it is marked before - // a byte of it goes out, and the rest once the first pass has the length. - mdatAt := int64(len(l.ftyp)) + l.moovLen() - format.Mark(w, format.RoleStructure, 0, mdatAt) + // The padding behind the media data is what the first pass's lengths + // leave, not what the second pass wrote, so a second pass that came out + // different leaves the file a different size from the plan - which the + // engine refuses - rather than a movie box that describes samples it does + // not carry. A film a damage will cut has part of its padding in front of + // the media data (cut.go). + content := z.mediaData(l) + back := splitForACut(&l, content, p.Bytes, f.CutAt) + if back < 0 || back > 0 && back < minFree || l.gap+back < minFree { + return core.Defect(fmt.Errorf("mp4: the pictures came to %d B of media data and the file is %d B, which leaves no room for the free box every one of these carries", + content, p.Bytes)) + } - o := &out{w: w, buf: make([]byte, 0, 64<<10)} - content := l.writeMovie(o, z) + // The map: the file type and the movie box, the free boxes in front of the + // media data when a damage will cut the film, then the media data box with + // the pictures, then the free boxes that make up the size. The first pass + // has every length, so all of it is marked before a byte goes out. + movie := int64(len(l.ftyp)) + l.moovLen() + mdatAt := movie + l.gap + format.Mark(w, format.RoleStructure, 0, movie) + if l.gap > 0 { + format.Mark(w, format.RolePadding, movie, l.gap) + } format.Mark(w, format.RoleContent, mdatAt, l.front()-mdatAt+content) - format.Mark(w, format.RolePadding, l.front()+content, p.Bytes-l.front()-content) + if back > 0 { + format.Mark(w, format.RolePadding, l.front()+content, back) + } + + // Both sides of the padding draw their filler from the film's seed. Nobody + // reads padding, and the side behind the media data lies past any cut. + o := &out{w: w, buf: make([]byte, 0, 64<<10)} + l.writeMovie(o, z) + if l.gap > 0 { + if o.flush(); o.err != nil { + return o.err + } + if err := isobmff.WritePadding(ctx, w, f.Choice.Seed, l.gap); err != nil { + return err + } + } o.mediaHeader(l.bigData, content) if err := writeSamples(ctx, o, l, pics); err != nil { return err @@ -156,16 +187,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if o.flush(); o.err != nil { return o.err } - // The padding is what the first pass's lengths leave, not what the second - // pass wrote, so a second pass that came out different leaves the file a - // different size from the plan - which the engine refuses - rather than a - // movie box that describes samples it does not carry. - pad := p.Bytes - l.front() - content - if pad < minFree { - return core.Defect(fmt.Errorf("mp4: the pictures came to %d B of media data and the file is %d B, which leaves no room for the free box every one of these carries", - content, p.Bytes)) - } - return isobmff.WritePadding(ctx, w, f.Choice.Seed, pad) + return isobmff.WritePadding(ctx, w, f.Choice.Seed, back) } // sizes is how long each picture's two samples came out in the first pass, by @@ -232,8 +254,8 @@ func measure(ctx context.Context, f video.Film, st video.Stream) (sizes, *video. } // writeMovie writes the file type and the movie box, with the sample tables -// from the first pass's lengths, and gives back how long the media data is. -func (l layout) writeMovie(o *out, z sizes) int64 { +// from the first pass's lengths. +func (l layout) writeMovie(o *out, z sizes) { o.write(l.ftyp) o.header("moov", l.moovLen()) o.write(l.mvhd) @@ -252,7 +274,7 @@ func (l layout) writeMovie(o *out, z sizes) int64 { l.writeDependencies(o) l.writeChunkRuns(o) l.writeSizes(o, z) - return l.writeChunkPlaces(o, z) + l.writeChunkPlaces(o, z) } // writeSyncs is stss: every key frame, numbered from one. Each is a sample a @@ -315,9 +337,10 @@ func (l layout) writeSizes(o *out, z sizes) { } } -// writeChunkPlaces is stco, or co64 when the layout asked for eight bytes, and -// gives back where the last chunk ends, counted from the media data's start. -func (l layout) writeChunkPlaces(o *out, z sizes) int64 { +// writeChunkPlaces is stco, or co64 when the layout asked for eight bytes: +// each chunk's place in bytes from the start of the file, which front - and so +// any padding in front of the media data - is counted in. +func (l layout) writeChunkPlaces(o *out, z sizes) { kind := "stco" if l.wide { kind = "co64" @@ -333,7 +356,6 @@ func (l layout) writeChunkPlaces(o *out, z sizes) int64 { } content += z.chunk(first, samples) }) - return content } // writeSamples is the second pass: every frame's sample, chunk by chunk, the diff --git a/internal/format/video/plan.go b/internal/format/video/plan.go index 7af9bed1..2ea75a58 100644 --- a/internal/format/video/plan.go +++ b/internal/format/video/plan.go @@ -11,6 +11,10 @@ import ( type Film struct { Choice Choice Settings Settings + // CutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). Each container splits its padding around the + // pictures for it. + CutAt int64 } // Stream is the stream the film is written in, the one planning sized it by. @@ -50,7 +54,7 @@ func Plan(formatID string, r format.Request, need func(Stream) int64, refuse fun Exact: true, Determinism: format.DeterminismByte, Properties: Facts(c, s), - Memo: Film{Choice: c, Settings: s}, + Memo: Film{Choice: c, Settings: s, CutAt: r.CutAt}, Work: st.Work(), Holds: Holds(c.Width, c.Height), } diff --git a/internal/format/webm/cut.go b/internal/format/webm/cut.go new file mode 100644 index 00000000..a74dda5f --- /dev/null +++ b/internal/format/webm/cut.go @@ -0,0 +1,62 @@ +package webm + +import ( + "context" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// A film a damage will cut (format.Request.CutAt) has its padding split around +// the clusters, so that the cut keeps the share of the clusters it keeps of the +// whole file: a Void between the Tracks and the first cluster, and the rest in +// the Void before the Cues - none there when the share wants all of it in +// front, and the Cues follow the last cluster. +// +// With all of the padding behind the clusters, as an uncut film has it, a +// large film cut at any share kept every picture and lost only padding. A Void +// is an EBML element allowed at every level, and readers skip it in front of +// the clusters as they do behind them - mkvmerge writes one there +// (docs/TRUNCATE-2026-11-11.md section 14.4). The one thing a Void there +// changes is where every cluster starts, which the Cues say from the start of +// the segment's content, and they are written last from where the clusters +// went. The SeekHead's places stay: Info and Tracks stand in front of the +// Void, and the Cues end the file. +// +// How much goes in front depends on how long the clusters come to, and a +// cluster's length is known only once its picture is coded. So a film a damage +// will cut is coded twice, as every MP4 is: once for the lengths (measure) and +// again to write it, the second pass taking every tile the first one kept +// (video.Pictures.Again). A film nothing cuts is coded once, as before. + +// measure is the first pass of a film a damage will cut: every cluster's +// length, nothing written, so the padding can be split before the file has a +// byte. It gives back what the clusters come to and the coder of the second +// pass. The work coding is worth is reported here, and the second pass, which +// codes only what this one could not keep, reports none. +// +// first is closed on every way out, by Again or by the caller's deferred +// Close - closing it twice is harmless. +func measure(ctx context.Context, l layout, first *video.Pictures) (int64, *video.Pictures, error) { + var content uint64 + err := eachCluster(ctx, l, first, true, func(from, to int64) error { + content += elementLen(idCluster, l.clusterContent(from, to, first.SampleLens())) + return nil + }) + if err != nil { + return 0, nil, err + } + again, err := first.Again() + if err != nil { + return 0, nil, err + } + return int64(content), again, nil +} + +// splitForACut is how much of the padding pad goes in front of the clusters, +// which come to content bytes after before bytes of the file, for a cut after +// the file's first cutAt bytes of whole. Each side is nothing or a Void of at +// least its two bytes (format.SplitFront). +func splitForACut(pad, before, content, cutAt, whole int64) int64 { + return format.SplitFront(pad, before, content, cutAt, whole, minVoid) +} diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index 636674a6..190469c9 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -16,7 +16,9 @@ // depend on the pictures. The Void at the end was measured on 2026-10-06 // (section 9.6) at 2, 3, 128, 129, 16 385, 16 386 and 1 048 576 bytes and with // none. A Void is always written, at least its two bytes, which leaves no size -// above the minimum that cannot be reached. +// above the minimum that cannot be reached. A film a damage will cut splits +// the padding around the clusters, so the cut keeps the share of the pictures +// asked for, and is coded twice for it (cut.go). package webm import ( @@ -147,39 +149,116 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { l := newLayout(st) body := uint64(p.Bytes - l.outside()) cuesAt := body - l.cuesLen() + pics := f.Choice.Pictures(st) + defer func() { pics.Close() }() - // The map: the headers, the seek head, the info and the tracks, then the - // clusters - coded as they go, so their end is marked when they reach it - - // then the Void that makes up the size and the Cues at the end. - seg := l.outside() - format.Mark(w, format.RoleStructure, 0, seg+int64(l.front())) - format.Mark(w, format.RoleContent, seg+int64(l.front()), -1) + // A film a damage will cut is measured first, so part of its padding can + // stand in front of the clusters (cut.go). One nothing cuts is coded once, + // as it is written, and content stays unknown until the writing is done. + content, gap := int64(-1), int64(0) + if f.CutAt > 0 { + n, again, err := measure(ctx, l, pics) + if err != nil { + return err + } + content, pics = n, again + gap = splitForACut(int64(cuesAt-l.front())-content, l.outside()+int64(l.front()), content, f.CutAt, p.Bytes) + } + m := filmMap{w: w, l: l, cuesAt: cuesAt} + m.head(gap, content) out := &sticky{w: w} + writeHead(out, l, body, cuesAt) + // Both Voids draw their filler from the film's seed. Nobody reads padding. + if gap > 0 { + if out.err != nil { + return out.err + } + if err := writeVoid(ctx, w, gap, f.Choice.Seed); err != nil { + return err + } + } + keys, end, err := writeClusters(ctx, out, l, pics, l.front()+uint64(gap), content < 0) + if err != nil { + return err + } + void, err := voidBehind(l, cuesAt, end, gap, content) + if err != nil { + return err + } + if content < 0 { + m.tail(end) + } + if void > 0 { + if err := writeVoid(ctx, w, void, f.Choice.Seed); err != nil { + return err + } + } + writeCues(out, l, keys) + return out.err +} + +// writeHead writes everything in front of the clusters: the EBML header, the +// segment's ID and size, the seek head, the info and the tracks. +func writeHead(out *sticky, l layout, body, cuesAt uint64) { out.write(l.head) out.write(idBytes(idSegment)) out.write(sizeField(body, segmentSizeField)) out.write(seekHead(l.seekHeadLen, l.seekHeadLen+uint64(len(l.info)), cuesAt)) out.write(l.info) out.write(l.tracks) - pics := f.Choice.Pictures(st) - defer pics.Close() - keys, end, err := writeClusters(ctx, out, l, pics) - if err != nil { - return err +} + +// voidBehind is the Void between the last cluster, which ends at end, and the +// Cues at cuesAt, both from the start of the segment's content. It is refused +// when it cannot be a Void, or when the clusters of a film measured first +// came to other than the first pass measured. +func voidBehind(l layout, cuesAt, end uint64, gap, content int64) (int64, error) { + if wrote := int64(end-l.front()) - gap; content >= 0 && wrote != content { + return 0, core.Defect(fmt.Errorf("webm: the clusters came to %d B in the first pass and %d B in the second", content, wrote)) } void := int64(cuesAt) - int64(end) - if void < minVoid { - return core.Defect(fmt.Errorf("webm: the clusters came to %d B and the Cues are to start at %d B, which leaves no room for the Void every one of these carries", + if void < 0 || void > 0 && void < minVoid || gap+void < minVoid { + return 0, core.Defect(fmt.Errorf("webm: the clusters came to %d B and the Cues are to start at %d B, which leaves no room for the Void every one of these carries", end, cuesAt)) } - format.Mark(w, format.RolePadding, seg+int64(end), void) - format.Mark(w, format.RoleEnd, seg+int64(cuesAt), int64(l.cuesLen())) - if err := writeVoid(ctx, w, void, f.Choice.Seed); err != nil { - return err + return void, nil +} + +// filmMap marks a film's map (format.Mark): the headers, the seek head, the +// info and the tracks, the Void in front of the clusters when a damage will +// cut the film, the clusters, then the Void that makes up the size and the +// Cues at the end. A film measured first is marked whole before its first +// byte. One coded as it goes has the clusters' end marked when the writing +// reaches it, and a cut stops it before then. +type filmMap struct { + w io.Writer + l layout + cuesAt uint64 +} + +// head marks what goes in front of the clusters, the clusters - content +// bytes, -1 while unknown - and, when they are known, what follows them. +func (m filmMap) head(gap, content int64) { + start := m.l.outside() + int64(m.l.front()) + format.Mark(m.w, format.RoleStructure, 0, start) + if gap > 0 { + format.Mark(m.w, format.RolePadding, start, gap) + } + format.Mark(m.w, format.RoleContent, start+gap, content) + if content >= 0 { + m.tail(m.l.front() + uint64(gap+content)) } - writeCues(out, l, keys) - return out.err +} + +// tail marks what follows the clusters, which end at end from the start of +// the segment's content: the Void, when there is one, and the Cues. +func (m filmMap) tail(end uint64) { + seg := m.l.outside() + if void := int64(m.cuesAt) - int64(end); void > 0 { + format.Mark(m.w, format.RolePadding, seg+int64(end), void) + } + format.Mark(m.w, format.RoleEnd, seg+int64(m.cuesAt), int64(m.l.cuesLen())) } // sticky keeps the first error, so a long run of small writes checks once, @@ -208,40 +287,56 @@ func (s *sticky) uint(id uint32, v uint64) { } // writeClusters codes the pictures one change at a time and writes the -// clusters as it goes, giving back where each key frame's cluster starts, for -// the Cues, and where the last one ends. Every change opens a cluster, so each -// is reported as worked once - what its new tiles were worth - which adds up -// to the Work the plan counted. -func writeClusters(ctx context.Context, out *sticky, l layout, pics *video.Pictures) ([]uint64, uint64, error) { +// clusters as it goes, the first at pos from the start of the segment's +// content, giving back where each key frame's cluster starts, for the Cues, +// and where the last one ends. report says whether this pass reports the work +// - the only pass of a film nothing cuts does, the second of one a damage +// cuts does not (cut.go). +func writeClusters(ctx context.Context, out *sticky, l layout, pics *video.Pictures, pos uint64, report bool) ([]uint64, uint64, error) { s := l.stream keys := make([]uint64, 0, s.Keys()) - pos := l.front() + err := eachCluster(ctx, l, pics, report, func(first, last int64) error { + if s.IsKey(first) { + keys = append(keys, pos) + } + pos += writeCluster(out, l, pics, first, last) + return out.err + }) + if err != nil { + return nil, 0, err + } + return keys, pos, nil +} + +// eachCluster walks the film's clusters in order, by their frames [first, +// last), with the picture each carries made the current one first, and stops +// at the first error cluster gives back. Every change opens a cluster, so when +// report says so each picture is reported as worked once - what its new tiles +// were worth - which adds up to the Work the plan counted. +func eachCluster(ctx context.Context, l layout, pics *video.Pictures, report bool, cluster func(first, last int64) error) error { + s := l.stream worked := int64(-1) for first := int64(0); first < s.Frames; { if err := ctx.Err(); err != nil { - return nil, 0, err + return err } // A cluster thirty seconds into a picture asks for the picture it // already has, which At answers without coding anything. change := s.ChangeOf(first) if err := pics.At(change); err != nil { - return nil, 0, err + return err } - if change != worked { + if report && change != worked { format.Worked(ctx, pics.Work()) worked = change } - if s.IsKey(first) { - keys = append(keys, pos) - } last := l.next(first) - pos += writeCluster(out, l, pics, first, last) - if out.err != nil { - return nil, 0, out.err + if err := cluster(first, last); err != nil { + return err } first = last } - return keys, pos, nil + return nil } // writeCluster writes the cluster of frames [first, last) and returns its diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 2e11a0c8..1232ee63 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -37,8 +37,9 @@ type cutCase struct { // the content is left, and gets a note when the cut missed the content. // // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). -// An MP4 of 1 MiB is nearly all padding after its video, so 90% missing still -// leaves all of it. A PNG splits its padding around the picture +// A WebP grows its picture to fill the file and keeps what is left over +// behind it, which at 4 KiB is more than the 41 B a cut of 1% takes, so the +// cut leaves all of the picture. A PNG splits its padding around the picture // for a cut, so the cut takes the share asked for, and so does a tar.gz. A // text is its words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. @@ -47,9 +48,9 @@ type cutCase struct { // one that claims content it does not have. func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ - // A GIF and then a PDF were this case until each split its padding - // for the cut. - {name: "behind", format: "mp4", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // A GIF, a PDF and an MP4 were this case until each split its + // padding for the cut. + {name: "behind", format: "webp", size: 4 << 10, missing: "1", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, // 4 KiB on disk is 40 KiB whole. It was the case of a cut in the padding // in front of the content, with an icon and a JPEG before it, until it @@ -96,7 +97,7 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { } } - // Canary: the GIF's entry, told the cut fell in its content, and the + // Canary: the WebP's entry, told the cut fell in its content, and the // text's, told none of its content is left. behind := byTarget["behind"] if len(behind.Damage) == 1 && behind.Damage[0].Cut != nil { @@ -180,7 +181,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("behind", "mp4", 1<<20, "90"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "webp", 4<<10, "1"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 6a9d600f..132628c7 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -32,8 +32,6 @@ type notYet struct { // that leaves is held to every size and share, and so is a new format unless // it is added here with its reason. var notYetCutAtTheShare = map[string]notYet{ - "mp4": {paddingBehind, 64 << 10, 50}, - "webm": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, @@ -48,7 +46,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"mp4", "webm", "webp", "tiff"} +var listedBeforeTheRule = []string{"webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. @@ -100,7 +98,8 @@ var ( // structure by TestEveryFormatsMapIsTheOneItsStructureReadsBack. // // Canaries: a cut that kept all its content must be reported, and so must a -// format listed as not yet cutting at the share that does. +// format listed as not yet cutting at the share that does, and a file planned +// for the sweep that the manifest leaves out. func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { missingOf := map[string]int64{} var targets []engine.Target @@ -130,7 +129,9 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { plannedOf[f.Target.ID] = f } problems := map[string]string{} + inManifest := map[string]bool{} for _, m := range res.Manifest.Files { + inManifest[m.TargetID] = true missing, ok := missingOf[m.TargetID] if !ok || m.Failed { t.Errorf("%s: no file was written (%s)", m.TargetID, m.Error) @@ -154,6 +155,12 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { for _, problem := range shareVerdicts(problems, notYetCutAtTheShare) { t.Error(problem) } + for _, problem := range absentFromTheManifest(missingOf, inManifest) { + t.Error(problem) + } + if len(absentFromTheManifest(map[string]int64{"webp-4096-1": 1}, map[string]bool{})) != 1 { + t.Errorf("canary: a planned file the manifest left out passed") + } for _, problem := range listedAfterTheRule(notYetCutAtTheShare) { t.Error(problem) } @@ -193,6 +200,21 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { } } +// absentFromTheManifest is every file planned for the sweep that the manifest +// has no entry for. shareVerdicts asks only the cases a file was written for, +// so without this a format that loses one of its files to the run still +// passes on the others. +func absentFromTheManifest(missingOf map[string]int64, inManifest map[string]bool) []string { + var out []string + for id := range missingOf { + if !inManifest[id] { + out = append(out, id+": planned for the sweep and not in the manifest, so its cut was never asked") + } + } + slices.Sort(out) + return out +} + // cutInTheEnd is what shareProblem says of a cut that fell in the end a file // closes with: a central directory, a trailer. const cutInTheEnd = "the cut fell in the end" diff --git a/internal/guard/filmcut_test.go b/internal/guard/filmcut_test.go new file mode 100644 index 00000000..f350b92b --- /dev/null +++ b/internal/guard/filmcut_test.go @@ -0,0 +1,407 @@ +package guard + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" +) + +// An MP4 or a WebM laid out for a cut comes to every size asked for, every +// sample of it is where its tables say, and the cut keeps the share of its +// pictures. +// +// Both put padding in front of the pictures for a cut - free boxes between the +// movie box and the media data, a Void between the Tracks and the first +// cluster - and each side is nothing or at least a piece, eight bytes for a +// free box and two for a Void: where the share wants less than a piece in +// front, a piece goes there, and where the padding left behind would be +// shorter than one, it goes in front too (format.SplitFront). The padding in +// front moves every sample, and the chunk places of an MP4 and the Cues of a +// WebM give their places in bytes, so they move with it. A WebM is coded twice +// for it, and the first pass's lengths have to be the second's. No other guard +// asks for the sizes and the cuts where those happen. +// +// So this writes consecutive sizes cut at the half, and cuts of one size a +// byte at a time where the share starts to want padding in front and where +// the padding behind runs out, holds each to its size and to the share, and +// asserts it met each layout. Instead of decoding, every sample is read where +// the file's own tables put it and has to be the bytes of the same sample in +// the film nothing cuts - the same stream gives the same pictures in every +// decoder, which is what docs/TRUNCATE-2026-11-11.md section 14.4 measured for +// the layouts - and every cue and seek of a WebM has to land on the element it +// names. +// +// An MP4 of 5 GiB cut at 90% is planned and only its head is written: its +// media data starts past where four bytes reach, so the chunk places widen to +// co64. +// +// Canary: a chunk place a byte off is refused. +func TestAnMP4OrAWebMLaidOutForACutComesToEverySizeAndPlays(t *testing.T) { + for _, id := range []string{"mp4", "webm"} { + desc, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + least := map[string]int64{"mp4": 8, "webm": 2}[id] + wholes := map[int64][][]byte{} + met := map[string]bool{} + check := func(size, cutAt int64) { + data, regions := writeBoxes(t, desc, size, cutAt) + layout, err := filmLayoutOf(data, regions, size, cutAt, least) + if err != nil { + t.Errorf("%s %d B cut at %d: %v", id, size, cutAt, err) + return + } + met[layout] = true + if _, ok := wholes[size]; !ok { + whole, _ := writeBoxes(t, desc, size, 0) + if wholes[size], err = filmSamples(id, whole); err != nil { + t.Fatalf("%s %d B nothing cuts: %v", id, size, err) + } + } + if err := sameSamples(id, data, wholes[size]); err != nil { + t.Errorf("%s %d B cut at %d, %s: %v", id, size, cutAt, layout, err) + } + } + smallest := desc.SmallestAccepted(format.Request{Label: true}) + for size := smallest; size < smallest+6; size++ { + check(size, size/2) + } + // Cuts a byte apart where the share starts to want padding in front - + // it wants cutAt times the part of the file that is not the pictures, + // less the head - and near the end, where what the share leaves behind + // runs out: j bytes from the end it leaves j times that part, less + // what stands behind the padding (a WebM's Cues). + const size = 16_000 + _, regions := writeBoxes(t, desc, size, 0) + head, pics, end := filmParts(t, regions) + rest := size - pics.Length + start := (head - 3) * size / rest + for cutAt := start; cutAt < start+(least+5)*size/rest; cutAt++ { + check(size, cutAt) + } + for j := max(1, (end-1)*size/rest); j <= (end+least+2)*size/rest; j++ { + check(size, size-j) + } + layouts := []string{"nothing in front, the share wants none", "a piece in front, the share wants less", + "the share", "all in front, what would stay behind is shorter than a piece"} + if end > 0 { + layouts = append(layouts, "all in front, the share wants more than there is") + } + for _, layout := range layouts { + if !met[layout] { + t.Errorf("%s: no size met the layout %q - met %v", id, layout, met) + } + } + } + + mp4TooLargeForFourBytes(t) + + // Canary: a chunk place a byte off. + desc, _ := format.Get("mp4") + whole, _ := writeBoxes(t, desc, 64<<10, 0) + samples, err := filmSamples("mp4", whole) + if err != nil { + t.Fatal(err) + } + data, _ := writeBoxes(t, desc, 64<<10, 32<<10) + at := bytes.Index(data, []byte("stco")) + 12 + broken := bytes.Clone(data) + binary.BigEndian.PutUint32(broken[at:], binary.BigEndian.Uint32(broken[at:])+1) + if err := sameSamples("mp4", broken, samples); err == nil { + t.Errorf("canary: a chunk place a byte off passed") + } +} + +// filmParts reads a film's map as what stands in front of all of its padding, +// its pictures, and what stands behind all of its padding - a WebM's Cues. +func filmParts(t *testing.T, regions []format.Region) (head int64, pics format.Region, end int64) { + t.Helper() + seen := false + for _, r := range regions { + switch r.Role { + case format.RoleStructure: + head += r.Length + case format.RoleEnd: + end += r.Length + case format.RoleContent: + pics, seen = r, true + case format.RolePadding: + // What the parts around it leave of the file - filmLayoutOf + // reads the two sides of it. + } + } + if !seen || pics.Length < 0 { + t.Fatalf("the map marks the pictures as %+v", pics) + } + return head, pics, end +} + +// filmLayoutOf holds a film laid out for a cut to its size and the share the +// cut keeps of its pictures, and names the layout it took. What stands behind +// the padding behind the pictures - the Cues of a WebM - is the end region. +func filmLayoutOf(data []byte, regions []format.Region, size, cutAt, least int64) (string, error) { + if int64(len(data)) != size { + return "", fmt.Errorf("came to %d B", len(data)) + } + return filmLayoutOfMap(regions, size, cutAt, least) +} + +// filmLayoutOfMap is filmLayoutOf from the map alone, for a film whose bytes +// are not all written. +func filmLayoutOfMap(regions []format.Region, size, cutAt, least int64) (string, error) { + content := -1 + for i, r := range regions { + if r.Role == format.RoleContent { + content = i + } + } + if content < 0 { + return "", errors.New("no content marked") + } + pics := regions[content] + if pics.Length < 0 { + return "", errors.New("the pictures are marked with no length") + } + front, back, end := int64(0), int64(0), int64(0) + if content > 0 && regions[content-1].Role == format.RolePadding { + front = regions[content-1].Length + } + for _, r := range regions[content+1:] { + switch { + case r.Role == format.RolePadding && end == 0: + back += r.Length + case r.Role == format.RoleEnd: + end += r.Length + default: + return "", fmt.Errorf("a %s region at %d, behind the pictures", r.Role, r.At) + } + } + if pics.At+pics.Length+back+end != size { + return "", fmt.Errorf("the pictures end at %d, and %d B of padding and %d B of end follow them in a %d B file", + pics.At+pics.Length, back, end, size) + } + if (front > 0 && front < least) || (back > 0 && back < least) { + return "", fmt.Errorf("a side shorter than a piece: %d in front, %d behind", front, back) + } + head := pics.At - front + want := format.PaddingInFront(head, pics.Length, cutAt, size) + kept, share := cutAt-head-front, pics.Length*cutAt/size + switch { + case front == 0 && want <= 0: + return "nothing in front, the share wants none", nil + case front == 0: + return "", fmt.Errorf("nothing in front and %d B behind, where the share wants %d B in front", back, want) + case front == want && kept == share: + return "the share", nil + case front == least && want > 0 && want < least: + return "a piece in front, the share wants less", nil + case back == 0 && front > want && front-want < least: + return "all in front, what would stay behind is shorter than a piece", nil + case back == 0 && front < want && end > 0: + // A cut in the last pictures, close to the Cues, wants more in front + // than there is padding. The cut keeps more than the share then, the + // one way format.SplitFront allows, and the manifest says how much. + return "all in front, the share wants more than there is", nil + } + return "", fmt.Errorf("%d B in front where the share wants %d, keeping %d B of the pictures where the share is %d", front, want, kept, share) +} + +// sameSamples requires every sample of a film, read where its own tables put +// it, to be the bytes of the same sample of the film nothing cuts. +func sameSamples(id string, data []byte, whole [][]byte) error { + got, err := filmSamples(id, data) + if err != nil { + return err + } + if len(got) != len(whole) { + return fmt.Errorf("%d samples, where the film nothing cuts has %d", len(got), len(whole)) + } + for i := range got { + if !bytes.Equal(got[i], whole[i]) { + return fmt.Errorf("sample %d is not the bytes of the film nothing cuts", i+1) + } + } + return nil +} + +// filmSamples is every sample of a film, read where its own tables put it - +// stsz, stsc and stco or co64 for an MP4, the blocks of each cluster for a +// WebM, whose cues and seeks have to land on the elements they name. +func filmSamples(id string, data []byte) ([][]byte, error) { + if id == "webm" { + return webmSamples(data) + } + top, err := mp4Boxes(data) + if err != nil { + return nil, err + } + var moov []byte + for _, b := range top { + if b.name == "moov" { + moov = b.body + } + } + stbl, err := mp4Path(moov, "trak", "mdia", "minf", "stbl") + if err != nil { + return nil, err + } + starts, lens, err := mp4Samples(stbl) + if err != nil { + return nil, err + } + out := make([][]byte, len(starts)) + for i := range starts { + if starts[i] < 0 || starts[i]+lens[i] > int64(len(data)) { + return nil, fmt.Errorf("sample %d lies outside the file", i+1) + } + out[i] = data[starts[i] : starts[i]+lens[i]] + } + return out, nil +} + +// webmSamples walks a WebM's segment by position: the blocks of every +// cluster, and every cue and seek, which have to give the place of a cluster +// starting at the cue's time and of the element the seek names, counted from +// the start of the segment's content. +func webmSamples(data []byte) ([][]byte, error) { + var seg []byte + if err := ebmlElements(data, func(id uint64, body []byte) error { + if id == 0x18538067 { + seg = body + } + return nil + }); err != nil { + return nil, err + } + var f filmRead + ids, times := map[uint64]uint64{}, map[uint64]int64{} + var cues, seeks [][2]uint64 + for pos := 0; pos < len(seg); { + id, n, err := ebmlVint(seg[pos:], true) + if err != nil { + return nil, err + } + size, m, err := ebmlVint(seg[pos+n:], false) + if err != nil { + return nil, err + } + start := pos + n + m + if uint64(len(seg)-start) < size { + return nil, fmt.Errorf("element %#x at %d says %d B and %d remain", id, pos, size, len(seg)-start) + } + el := seg[start : start+int(size)] + ids[uint64(pos)] = id + switch id { + case 0x1F43B675: + first := len(f.blocks) + if err := walkCluster(&f, el); err != nil { + return nil, err + } + if len(f.blocks) > first { + times[uint64(pos)] = f.blocks[first].ts + } + case 0x1C53BB6B: + cues, err = webmPairs(el, 0xBB, 0xB3, 0xB7, 0xF1) + case 0x114D9B74: + seeks, err = webmPairs(el, 0x4DBB, 0x53AB, 0, 0x53AC) + } + if err != nil { + return nil, err + } + pos = start + int(size) + } + for _, c := range cues { + if ids[c[1]] != 0x1F43B675 || times[c[1]] != int64(c[0]) { + return nil, fmt.Errorf("the cue at %d ms gives %d, where no cluster starts at that time", c[0], c[1]) + } + } + for _, s := range seeks { + if ids[s[1]] != s[0] { + return nil, fmt.Errorf("the seek for %#x gives %d, where %#x starts", s[0], s[1], ids[s[1]]) + } + } + if len(cues) == 0 || len(seeks) != 3 { + return nil, fmt.Errorf("%d cues and %d seeks", len(cues), len(seeks)) + } + out := make([][]byte, len(f.blocks)) + for i, b := range f.blocks { + out[i] = b.data + } + return out, nil +} + +// webmPairs reads each element point in el as a pair: the value of key and +// the value of place, place found inside the element in, or among point's +// own when in is 0. +func webmPairs(el []byte, point, key, in, place uint64) ([][2]uint64, error) { + var out [][2]uint64 + err := ebmlElements(el, func(id uint64, body []byte) error { + if id != point { + return nil + } + var pair [2]uint64 + found := 0 + err := ebmlElements(body, func(id uint64, v []byte) error { + switch { + case id == key: + pair[0] = beUint(v) + found++ + case id == place && in == 0: + pair[1] = beUint(v) + found++ + case id == in: + return ebmlElements(v, func(id uint64, w []byte) error { + if id == place { + pair[1] = beUint(w) + found++ + } + return nil + }) + } + return nil + }) + if err == nil && found != 2 { + err = fmt.Errorf("an element %#x without its %#x or %#x", point, key, place) + } + out = append(out, pair) + return err + }) + return out, err +} + +// mp4TooLargeForFourBytes plans an MP4 of 5 GiB cut at 90% and holds the head +// it writes to chunk places of eight bytes, the first of which gives where the +// map says the samples start, and the map to the share, counted with the +// movie box the widening grew. It writes nothing past the head. +func mp4TooLargeForFourBytes(t *testing.T) { + t.Helper() + const size, cutAt = 5 << 30, 5 << 30 / 10 * 9 + desc, _ := format.Get("mp4") + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: cutAt}) + if err != nil { + t.Fatalf("planning %d B: %v", int64(size), err) + } + head := &headOnly{keep: 64 << 10} + rec := &format.MapRecorder{W: head} + if err := desc.Generator.Write(t.Context(), rec, p); err == nil { + t.Fatalf("writing %d B went through a writer that stops after its head", int64(size)) + } + regions := rec.Map(size, true) + if layout, err := filmLayoutOfMap(regions, size, cutAt, 8); layout != "the share" { + t.Errorf("%d B cut at 90%%: %q, %v", int64(size), layout, err) + } + pics := pictureRegion(regions) + at := bytes.Index(head.got, []byte("co64")) + if at < 0 || pics.At <= 1<<32 { + t.Fatalf("%d B cut at 90%%: no co64 in the head, and the media data starts at %d", int64(size), pics.At) + } + if first := int64(binary.BigEndian.Uint64(head.got[at+12:])); first != pics.At+8 { + t.Errorf("%d B cut at 90%%: the first chunk place is %d, where the samples start at %d", int64(size), first, pics.At+8) + } +} diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index 8baf5595..43cb9cd3 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -167,6 +167,12 @@ var provenByProbe = map[string]string{ "(50 cuts), with the widening's four bytes left out of the move (5 GiB: 4831825054 where the media data starts at " + "4831825058), with a side of padding allowed below a box (4000 B came to 4007 B) and with a free box rather than a " + "skip box in front of a JPEG XL's codestream (81 cuts).", + "TestAnMP4OrAWebMLaidOutForACutComesToEverySizeAndPlays": "its own canary - a chunk place a byte off must be refused " + + "- its reach check, every layout of the split met, and red on 2026-11-13 with stco not counting the free boxes in " + + "front of the media data (sample 1 not the bytes of the film nothing cuts), with the Cues not moved by the Void in " + + "front of the clusters (a cue giving 226 where no cluster starts), with a side of a WebM's padding allowed below a " + + "Void (16000 B came to 16008 B), with an MP4 that never widens to co64 (5 GiB: no co64 in the head) and with the " + + "widening's growth left out of the head the share is counted from (40 B more in front than the share).", "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canaries - a table that sends one object a byte off " + "must be refused, and the layout of a document nothing cuts must fail a cut at its half - its reach check, every " + "layout of the split met, red on 2026-11-13 with nothing in front of a document under 400 B of padding (398 sizes, " + diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index 2493ae34..f8d20f32 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -2517,11 +2517,13 @@ def check_webm(data): """A WebM film as this generator writes it, read without its code. The questions are the ones a film can get wrong while every frame still - plays: the segment ends where the file does, with its padding just before - the Cues and the Cues last (docs/WIDEO-2026-10-06.md section 15), the - SeekHead points at what it names, every block is whole AV1 with no - temporal delimiter, nothing shows a picture that may not be shown again - (section 9.2), and the Cues point at the key frames and nothing else. + plays: the segment ends where the file does, with the Cues last and its + padding in a Void right before them (docs/WIDEO-2026-10-06.md section + 15), right before the first cluster - a film laid out for a cut + (docs/TRUNCATE-2026-11-11.md section 14) - or in both, the SeekHead points + at what it names, every block is whole AV1 with no temporal delimiter, + nothing shows a picture that may not be shown again (section 9.2), and the + Cues point at the key frames and nothing else. """ top = list(ebml_elements(data, 0, len(data))) if len(top) != 2 or top[0][0] != 0x1A45DFA3 or top[1][0] != 0x18538067: @@ -2532,8 +2534,13 @@ def check_webm(data): seg_start, seg_end = top[1][2], top[1][3] children = list(ebml_elements(data, seg_start, seg_end)) - if len(children) < 2 or children[-1][0] != 0x1C53BB6B or children[-2][0] != 0xEC: - fail("the segment does not end in a Void and then the Cues, which is where this generator pads") + ids = [c[0] for c in children] + first_cluster = ids.index(0x1F43B675) if 0x1F43B675 in ids else len(ids) + padded = [i for i, ident in enumerate(ids) if ident == 0xEC] + if not ids or ids[-1] != 0x1C53BB6B or not padded or \ + any(i != first_cluster - 1 and i != len(ids) - 2 for i in padded): + fail("the segment does not end in the Cues with a Void right before them, right before the first cluster " + "or both, which is where this generator pads") at_offset = {at - seg_start: ident for ident, at, _, _ in children} blocks, cues, seeks = [], [], 0 @@ -2591,7 +2598,7 @@ def check_webm(data): keys = [(ts, cluster) for ts, key, _, cluster in blocks if key] if [ts for ts, _ in keys] != [ts for ts, _ in cues] or [c for _, c in keys] != [p for _, p in cues]: fail(f"the Cues name {len(cues)} points and the film has {len(keys)} key frames, not at the same places") - ok(f"{len(blocks)} frames, {len(keys)} key frames, every one in the Cues, Void before them") + ok(f"{len(blocks)} frames, {len(keys)} key frames, every one in the Cues, padding in {len(padded)} Void") def iso_child(data, start, end, name, depth="the movie box"): @@ -2673,7 +2680,9 @@ def check_mp4(data): The questions a film can get wrong while every frame still plays (docs/MP4-2026-10-07.md): the boxes cover the file with the free boxes - last, the sample tables agree with the media data byte for byte - the + last, right in front of the media data - a film laid out for a cut + (docs/TRUNCATE-2026-11-11.md section 14) - or both, and with at least + one, the sample tables agree with the media data byte for byte - the samples follow one another from its first byte to its last - every sync sample opens with the sequence header and a shown key frame and every key frame is a sync sample (AV1 binding section 2.4), sdtp says which samples @@ -2682,13 +2691,16 @@ def check_mp4(data): """ top = list(iso_boxes(data, 0, len(data))) names = [n for n, _, _ in top] - if names[:3] != [b"ftyp", b"moov", b"mdat"] or any(n != b"free" for n in names[3:]) or len(names) < 4: - fail(f"the file is {[n.decode('latin1') for n in names]} and this generator writes ftyp, moov, mdat, then free boxes") + media = names.index(b"mdat") if b"mdat" in names else -1 + if names[:2] != [b"ftyp", b"moov"] or media < 2 or b"free" not in names or \ + any(n != b"free" for i, n in enumerate(names[2:], 2) if i != media): + fail(f"the file is {[n.decode('latin1') for n in names]} and this generator writes ftyp, moov and mdat with " + "free boxes in front of mdat, behind it or both") _, fb, fe = top[0] brands = [data[at:at + 4] for at in range(fb + 8, fe, 4)] if b"av01" not in brands: fail("av01 is not among the compatible brands, which the AV1 binding requires (section 2.1)") - _, mb, me = top[2] + _, mb, me = top[media] trak = iso_child(data, top[1][1], top[1][2], b"trak") mdia = iso_child(data, *trak, b"mdia", "the track")