From 4931768c8db60870d69e7964afbb6a863dc2f5ee Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 23:36:10 +0100 Subject: [PATCH 1/2] An MP4 or a WebM cut short loses the share of its pictures asked for --damage truncate cut only the padding of a film of 1 MB or more, because both containers kept all of it behind the pictures. A film a damage will cut now splits its padding around them: free boxes between moov and mdat in an MP4, its chunk places moved with them and widened to co64 only when the media data then passes 4 GiB, and a Void between the Tracks and the first cluster in a WebM, its Cues moved with it. ffmpeg's faststart and mkvmerge write the same layouts. A WebM a damage will cut is coded twice, once for the lengths of its clusters, as every MP4 is. So its map, and the manifest's cut, gives the length of its pictures, which it left empty. Films nothing cuts are byte for byte what they were. The structural checker allows both layouts and still requires the tables to land. Both formats leave the ratchet of formats that do not cut at the share. The new guard holds every size and cut it sweeps to the share, and every sample to the bytes of the same sample in the film nothing cuts. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 7 + internal/format/mp4/cut.go | 58 ++++ internal/format/mp4/layout.go | 10 +- internal/format/mp4/mp4.go | 76 +++-- internal/format/video/plan.go | 6 +- internal/format/webm/cut.go | 62 ++++ internal/format/webm/webm.go | 167 +++++++--- internal/guard/cutshare_test.go | 4 +- internal/guard/filmcut_test.go | 404 ++++++++++++++++++++++++ internal/guard/mutationcoverage_test.go | 6 + internal/oracle/strict.py | 36 ++- 11 files changed, 754 insertions(+), 82 deletions(-) create mode 100644 internal/format/mp4/cut.go create mode 100644 internal/format/webm/cut.go create mode 100644 internal/guard/filmcut_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 04f1d2b0..1753fda6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,13 @@ because it turns other people's test suites red. lower detail in Chrome and in libjxl with partial files allowed, because the format sends a coarse picture first. A cut of 90% shows nothing. An AVIF nothing cuts is unchanged. Every JPEG XL changes (see Breaking). +- **`truncate` cuts the pictures of an MP4 or a WebM film by the percentage + you set.** A cut used to take only padding, leaving every frame. Now + `--damage truncate:missing=50` removes half of the coded pictures, as an + interrupted download does, and ffmpeg gives back the frames in front of the + cut. Chrome and Firefox stop a cut MP4 with an error and end a cut WebM + without one. The manifest of a cut WebM now says how long its pictures are, + where it said nothing. Films nothing cuts are unchanged. - **tfg is on npm, for tests in Node.** `npm install --save-dev testing-files-generator` gives Playwright, Vitest diff --git a/internal/format/mp4/cut.go b/internal/format/mp4/cut.go new file mode 100644 index 00000000..f342fee9 --- /dev/null +++ b/internal/format/mp4/cut.go @@ -0,0 +1,58 @@ +package mp4 + +import ( + "math" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" +) + +// splitForACut settles how much of the padding goes between the movie box and +// the media data of a film a damage will cut after its first cutAt bytes, so +// that the cut keeps the share of the media data it keeps of the whole file +// (format.Request.CutAt), and gives back the padding left behind the media +// data. content is the media data's samples, as the first pass measured them. +// +// With all of the padding behind the media data, as an uncut film has it, a +// large film cut at any share kept every picture and lost only padding. A free +// box may stand anywhere among the top level boxes, and readers skip it in +// front of the media data as they do behind it - ffmpeg's faststart writes one +// there (docs/TRUNCATE-2026-11-11.md section 14.4). The one thing a box there +// changes is where every chunk starts, which stco says in bytes from the start +// of the file, so every place moves by the gap: front counts it, and the places +// are written from front. +// +// The places take four bytes unless the layout already asked for eight. One +// that the gap takes past 4 GiB widens the table to co64, as a film whose +// pictures take their whole reserve past 4 GiB has it, and the movie box grows +// by four bytes a chunk - only then. When the padding cannot give up those +// bytes and still hold a free box, the film is laid out as one nothing cuts, +// and the manifest's cut says where the cut fell. That takes a film whose +// pictures came within four bytes a chunk of their whole reserve, which the +// guard cannot reach (section 14.3). +// +// A film whose share wants nothing in front - and every film nothing cuts - is +// laid out as one nothing cuts. It sets l.gap, and l.wide when it widens. +func splitForACut(l *layout, content, total, cutAt int64) (back int64) { + before := int64(len(l.ftyp)) + l.moovLen() + mdat := l.front() - before + content + pad := total - before - mdat + l.gap = format.SplitFront(pad, before, mdat, cutAt, total, minFree) + if l.gap > 0 && !l.wide && l.front()+content > math.MaxUint32 { + grown := (wideChunk - narrowChunk) * l.chunks + l.gap = 0 + if pad-grown >= minFree { + l.wide = true + l.gap = format.SplitFront(pad-grown, before+grown, mdat, cutAt, total, minFree) + l.wide = l.gap > 0 + } + } + return total - l.front() - content +} + +// mediaData is how long the samples of the media data come to, chunk by +// chunk. +func (z sizes) mediaData(l layout) int64 { + var n int64 + l.eachChunk(func(first, samples int64) { n += z.chunk(first, samples) }) + return n +} diff --git a/internal/format/mp4/layout.go b/internal/format/mp4/layout.go index 2873623a..a08c5deb 100644 --- a/internal/format/mp4/layout.go +++ b/internal/format/mp4/layout.go @@ -43,8 +43,12 @@ type layout struct { // wide is whether the chunks' places take eight bytes each (co64), and // bigData whether the media data's length does - each only when a film // whose pictures take their whole reserve would need it, so a smaller film - // never has a field it could not fill. + // never has a field it could not fill. A film a damage will cut may widen + // its places for the padding in front of the media data (cut.go). wide, bigData bool + // gap is the padding between the movie box and the media data, nothing + // but in a film a damage will cut (cut.go). Planning counts none. + gap int64 ftyp, mvhd, tkhd, mdhd, hdlr, vmhd, dinf, stsd, stts []byte } @@ -92,13 +96,13 @@ func (l layout) boundContent() int64 { } // front is where the media data's content starts: the file type, the movie -// box and the media data's header. +// box, the padding in front of the media data and the media data's header. func (l layout) front() int64 { head := int64(isobmff.BoxHeader) if l.bigData { head += 8 } - return int64(len(l.ftyp)) + l.moovLen() + head + return int64(len(l.ftyp)) + l.moovLen() + l.gap + head } // need is the file without its padding when every picture takes its whole diff --git a/internal/format/mp4/mp4.go b/internal/format/mp4/mp4.go index 48bb3fe0..6d8841fa 100644 --- a/internal/format/mp4/mp4.go +++ b/internal/format/mp4/mp4.go @@ -21,7 +21,9 @@ // with the probe at 8, 9, 15, 16 and 1 048 576 bytes and with none. A free box // is always written, at least its eight bytes, which leaves no size above the // minimum that cannot be reached, and one larger than a box length can say is -// several (internal/format/isobmff). +// several (internal/format/isobmff). A film a damage will cut splits them +// around the media data, so the cut keeps the share of the pictures asked for +// (cut.go). package mp4 import ( @@ -138,17 +140,46 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { } defer pics.Close() - // The map: the file type and the movie box, then the media data box with - // the pictures, then the free boxes that make up the size. The movie box - // is the same length whatever the pictures come to, so it is marked before - // a byte of it goes out, and the rest once the first pass has the length. - mdatAt := int64(len(l.ftyp)) + l.moovLen() - format.Mark(w, format.RoleStructure, 0, mdatAt) + // The padding behind the media data is what the first pass's lengths + // leave, not what the second pass wrote, so a second pass that came out + // different leaves the file a different size from the plan - which the + // engine refuses - rather than a movie box that describes samples it does + // not carry. A film a damage will cut has part of its padding in front of + // the media data (cut.go). + content := z.mediaData(l) + back := splitForACut(&l, content, p.Bytes, f.CutAt) + if back < 0 || back > 0 && back < minFree || l.gap+back < minFree { + return core.Defect(fmt.Errorf("mp4: the pictures came to %d B of media data and the file is %d B, which leaves no room for the free box every one of these carries", + content, p.Bytes)) + } - o := &out{w: w, buf: make([]byte, 0, 64<<10)} - content := l.writeMovie(o, z) + // The map: the file type and the movie box, the free boxes in front of the + // media data when a damage will cut the film, then the media data box with + // the pictures, then the free boxes that make up the size. The first pass + // has every length, so all of it is marked before a byte goes out. + movie := int64(len(l.ftyp)) + l.moovLen() + mdatAt := movie + l.gap + format.Mark(w, format.RoleStructure, 0, movie) + if l.gap > 0 { + format.Mark(w, format.RolePadding, movie, l.gap) + } format.Mark(w, format.RoleContent, mdatAt, l.front()-mdatAt+content) - format.Mark(w, format.RolePadding, l.front()+content, p.Bytes-l.front()-content) + if back > 0 { + format.Mark(w, format.RolePadding, l.front()+content, back) + } + + // Both sides of the padding draw their filler from the film's seed. Nobody + // reads padding, and the side behind the media data lies past any cut. + o := &out{w: w, buf: make([]byte, 0, 64<<10)} + l.writeMovie(o, z) + if l.gap > 0 { + if o.flush(); o.err != nil { + return o.err + } + if err := isobmff.WritePadding(ctx, w, f.Choice.Seed, l.gap); err != nil { + return err + } + } o.mediaHeader(l.bigData, content) if err := writeSamples(ctx, o, l, pics); err != nil { return err @@ -156,16 +187,7 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { if o.flush(); o.err != nil { return o.err } - // The padding is what the first pass's lengths leave, not what the second - // pass wrote, so a second pass that came out different leaves the file a - // different size from the plan - which the engine refuses - rather than a - // movie box that describes samples it does not carry. - pad := p.Bytes - l.front() - content - if pad < minFree { - return core.Defect(fmt.Errorf("mp4: the pictures came to %d B of media data and the file is %d B, which leaves no room for the free box every one of these carries", - content, p.Bytes)) - } - return isobmff.WritePadding(ctx, w, f.Choice.Seed, pad) + return isobmff.WritePadding(ctx, w, f.Choice.Seed, back) } // sizes is how long each picture's two samples came out in the first pass, by @@ -232,8 +254,8 @@ func measure(ctx context.Context, f video.Film, st video.Stream) (sizes, *video. } // writeMovie writes the file type and the movie box, with the sample tables -// from the first pass's lengths, and gives back how long the media data is. -func (l layout) writeMovie(o *out, z sizes) int64 { +// from the first pass's lengths. +func (l layout) writeMovie(o *out, z sizes) { o.write(l.ftyp) o.header("moov", l.moovLen()) o.write(l.mvhd) @@ -252,7 +274,7 @@ func (l layout) writeMovie(o *out, z sizes) int64 { l.writeDependencies(o) l.writeChunkRuns(o) l.writeSizes(o, z) - return l.writeChunkPlaces(o, z) + l.writeChunkPlaces(o, z) } // writeSyncs is stss: every key frame, numbered from one. Each is a sample a @@ -315,9 +337,10 @@ func (l layout) writeSizes(o *out, z sizes) { } } -// writeChunkPlaces is stco, or co64 when the layout asked for eight bytes, and -// gives back where the last chunk ends, counted from the media data's start. -func (l layout) writeChunkPlaces(o *out, z sizes) int64 { +// writeChunkPlaces is stco, or co64 when the layout asked for eight bytes: +// each chunk's place in bytes from the start of the file, which front - and so +// any padding in front of the media data - is counted in. +func (l layout) writeChunkPlaces(o *out, z sizes) { kind := "stco" if l.wide { kind = "co64" @@ -333,7 +356,6 @@ func (l layout) writeChunkPlaces(o *out, z sizes) int64 { } content += z.chunk(first, samples) }) - return content } // writeSamples is the second pass: every frame's sample, chunk by chunk, the diff --git a/internal/format/video/plan.go b/internal/format/video/plan.go index 7af9bed1..2ea75a58 100644 --- a/internal/format/video/plan.go +++ b/internal/format/video/plan.go @@ -11,6 +11,10 @@ import ( type Film struct { Choice Choice Settings Settings + // CutAt is where a damage will cut the file, zero when nothing does + // (format.Request.CutAt). Each container splits its padding around the + // pictures for it. + CutAt int64 } // Stream is the stream the film is written in, the one planning sized it by. @@ -50,7 +54,7 @@ func Plan(formatID string, r format.Request, need func(Stream) int64, refuse fun Exact: true, Determinism: format.DeterminismByte, Properties: Facts(c, s), - Memo: Film{Choice: c, Settings: s}, + Memo: Film{Choice: c, Settings: s, CutAt: r.CutAt}, Work: st.Work(), Holds: Holds(c.Width, c.Height), } diff --git a/internal/format/webm/cut.go b/internal/format/webm/cut.go new file mode 100644 index 00000000..a74dda5f --- /dev/null +++ b/internal/format/webm/cut.go @@ -0,0 +1,62 @@ +package webm + +import ( + "context" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + "github.com/donislawdev/TestingFilesGenerator/internal/format/video" +) + +// A film a damage will cut (format.Request.CutAt) has its padding split around +// the clusters, so that the cut keeps the share of the clusters it keeps of the +// whole file: a Void between the Tracks and the first cluster, and the rest in +// the Void before the Cues - none there when the share wants all of it in +// front, and the Cues follow the last cluster. +// +// With all of the padding behind the clusters, as an uncut film has it, a +// large film cut at any share kept every picture and lost only padding. A Void +// is an EBML element allowed at every level, and readers skip it in front of +// the clusters as they do behind them - mkvmerge writes one there +// (docs/TRUNCATE-2026-11-11.md section 14.4). The one thing a Void there +// changes is where every cluster starts, which the Cues say from the start of +// the segment's content, and they are written last from where the clusters +// went. The SeekHead's places stay: Info and Tracks stand in front of the +// Void, and the Cues end the file. +// +// How much goes in front depends on how long the clusters come to, and a +// cluster's length is known only once its picture is coded. So a film a damage +// will cut is coded twice, as every MP4 is: once for the lengths (measure) and +// again to write it, the second pass taking every tile the first one kept +// (video.Pictures.Again). A film nothing cuts is coded once, as before. + +// measure is the first pass of a film a damage will cut: every cluster's +// length, nothing written, so the padding can be split before the file has a +// byte. It gives back what the clusters come to and the coder of the second +// pass. The work coding is worth is reported here, and the second pass, which +// codes only what this one could not keep, reports none. +// +// first is closed on every way out, by Again or by the caller's deferred +// Close - closing it twice is harmless. +func measure(ctx context.Context, l layout, first *video.Pictures) (int64, *video.Pictures, error) { + var content uint64 + err := eachCluster(ctx, l, first, true, func(from, to int64) error { + content += elementLen(idCluster, l.clusterContent(from, to, first.SampleLens())) + return nil + }) + if err != nil { + return 0, nil, err + } + again, err := first.Again() + if err != nil { + return 0, nil, err + } + return int64(content), again, nil +} + +// splitForACut is how much of the padding pad goes in front of the clusters, +// which come to content bytes after before bytes of the file, for a cut after +// the file's first cutAt bytes of whole. Each side is nothing or a Void of at +// least its two bytes (format.SplitFront). +func splitForACut(pad, before, content, cutAt, whole int64) int64 { + return format.SplitFront(pad, before, content, cutAt, whole, minVoid) +} diff --git a/internal/format/webm/webm.go b/internal/format/webm/webm.go index 636674a6..190469c9 100644 --- a/internal/format/webm/webm.go +++ b/internal/format/webm/webm.go @@ -16,7 +16,9 @@ // depend on the pictures. The Void at the end was measured on 2026-10-06 // (section 9.6) at 2, 3, 128, 129, 16 385, 16 386 and 1 048 576 bytes and with // none. A Void is always written, at least its two bytes, which leaves no size -// above the minimum that cannot be reached. +// above the minimum that cannot be reached. A film a damage will cut splits +// the padding around the clusters, so the cut keeps the share of the pictures +// asked for, and is coded twice for it (cut.go). package webm import ( @@ -147,39 +149,116 @@ func (generator) Write(ctx context.Context, w io.Writer, p format.Plan) error { l := newLayout(st) body := uint64(p.Bytes - l.outside()) cuesAt := body - l.cuesLen() + pics := f.Choice.Pictures(st) + defer func() { pics.Close() }() - // The map: the headers, the seek head, the info and the tracks, then the - // clusters - coded as they go, so their end is marked when they reach it - - // then the Void that makes up the size and the Cues at the end. - seg := l.outside() - format.Mark(w, format.RoleStructure, 0, seg+int64(l.front())) - format.Mark(w, format.RoleContent, seg+int64(l.front()), -1) + // A film a damage will cut is measured first, so part of its padding can + // stand in front of the clusters (cut.go). One nothing cuts is coded once, + // as it is written, and content stays unknown until the writing is done. + content, gap := int64(-1), int64(0) + if f.CutAt > 0 { + n, again, err := measure(ctx, l, pics) + if err != nil { + return err + } + content, pics = n, again + gap = splitForACut(int64(cuesAt-l.front())-content, l.outside()+int64(l.front()), content, f.CutAt, p.Bytes) + } + m := filmMap{w: w, l: l, cuesAt: cuesAt} + m.head(gap, content) out := &sticky{w: w} + writeHead(out, l, body, cuesAt) + // Both Voids draw their filler from the film's seed. Nobody reads padding. + if gap > 0 { + if out.err != nil { + return out.err + } + if err := writeVoid(ctx, w, gap, f.Choice.Seed); err != nil { + return err + } + } + keys, end, err := writeClusters(ctx, out, l, pics, l.front()+uint64(gap), content < 0) + if err != nil { + return err + } + void, err := voidBehind(l, cuesAt, end, gap, content) + if err != nil { + return err + } + if content < 0 { + m.tail(end) + } + if void > 0 { + if err := writeVoid(ctx, w, void, f.Choice.Seed); err != nil { + return err + } + } + writeCues(out, l, keys) + return out.err +} + +// writeHead writes everything in front of the clusters: the EBML header, the +// segment's ID and size, the seek head, the info and the tracks. +func writeHead(out *sticky, l layout, body, cuesAt uint64) { out.write(l.head) out.write(idBytes(idSegment)) out.write(sizeField(body, segmentSizeField)) out.write(seekHead(l.seekHeadLen, l.seekHeadLen+uint64(len(l.info)), cuesAt)) out.write(l.info) out.write(l.tracks) - pics := f.Choice.Pictures(st) - defer pics.Close() - keys, end, err := writeClusters(ctx, out, l, pics) - if err != nil { - return err +} + +// voidBehind is the Void between the last cluster, which ends at end, and the +// Cues at cuesAt, both from the start of the segment's content. It is refused +// when it cannot be a Void, or when the clusters of a film measured first +// came to other than the first pass measured. +func voidBehind(l layout, cuesAt, end uint64, gap, content int64) (int64, error) { + if wrote := int64(end-l.front()) - gap; content >= 0 && wrote != content { + return 0, core.Defect(fmt.Errorf("webm: the clusters came to %d B in the first pass and %d B in the second", content, wrote)) } void := int64(cuesAt) - int64(end) - if void < minVoid { - return core.Defect(fmt.Errorf("webm: the clusters came to %d B and the Cues are to start at %d B, which leaves no room for the Void every one of these carries", + if void < 0 || void > 0 && void < minVoid || gap+void < minVoid { + return 0, core.Defect(fmt.Errorf("webm: the clusters came to %d B and the Cues are to start at %d B, which leaves no room for the Void every one of these carries", end, cuesAt)) } - format.Mark(w, format.RolePadding, seg+int64(end), void) - format.Mark(w, format.RoleEnd, seg+int64(cuesAt), int64(l.cuesLen())) - if err := writeVoid(ctx, w, void, f.Choice.Seed); err != nil { - return err + return void, nil +} + +// filmMap marks a film's map (format.Mark): the headers, the seek head, the +// info and the tracks, the Void in front of the clusters when a damage will +// cut the film, the clusters, then the Void that makes up the size and the +// Cues at the end. A film measured first is marked whole before its first +// byte. One coded as it goes has the clusters' end marked when the writing +// reaches it, and a cut stops it before then. +type filmMap struct { + w io.Writer + l layout + cuesAt uint64 +} + +// head marks what goes in front of the clusters, the clusters - content +// bytes, -1 while unknown - and, when they are known, what follows them. +func (m filmMap) head(gap, content int64) { + start := m.l.outside() + int64(m.l.front()) + format.Mark(m.w, format.RoleStructure, 0, start) + if gap > 0 { + format.Mark(m.w, format.RolePadding, start, gap) + } + format.Mark(m.w, format.RoleContent, start+gap, content) + if content >= 0 { + m.tail(m.l.front() + uint64(gap+content)) } - writeCues(out, l, keys) - return out.err +} + +// tail marks what follows the clusters, which end at end from the start of +// the segment's content: the Void, when there is one, and the Cues. +func (m filmMap) tail(end uint64) { + seg := m.l.outside() + if void := int64(m.cuesAt) - int64(end); void > 0 { + format.Mark(m.w, format.RolePadding, seg+int64(end), void) + } + format.Mark(m.w, format.RoleEnd, seg+int64(m.cuesAt), int64(m.l.cuesLen())) } // sticky keeps the first error, so a long run of small writes checks once, @@ -208,40 +287,56 @@ func (s *sticky) uint(id uint32, v uint64) { } // writeClusters codes the pictures one change at a time and writes the -// clusters as it goes, giving back where each key frame's cluster starts, for -// the Cues, and where the last one ends. Every change opens a cluster, so each -// is reported as worked once - what its new tiles were worth - which adds up -// to the Work the plan counted. -func writeClusters(ctx context.Context, out *sticky, l layout, pics *video.Pictures) ([]uint64, uint64, error) { +// clusters as it goes, the first at pos from the start of the segment's +// content, giving back where each key frame's cluster starts, for the Cues, +// and where the last one ends. report says whether this pass reports the work +// - the only pass of a film nothing cuts does, the second of one a damage +// cuts does not (cut.go). +func writeClusters(ctx context.Context, out *sticky, l layout, pics *video.Pictures, pos uint64, report bool) ([]uint64, uint64, error) { s := l.stream keys := make([]uint64, 0, s.Keys()) - pos := l.front() + err := eachCluster(ctx, l, pics, report, func(first, last int64) error { + if s.IsKey(first) { + keys = append(keys, pos) + } + pos += writeCluster(out, l, pics, first, last) + return out.err + }) + if err != nil { + return nil, 0, err + } + return keys, pos, nil +} + +// eachCluster walks the film's clusters in order, by their frames [first, +// last), with the picture each carries made the current one first, and stops +// at the first error cluster gives back. Every change opens a cluster, so when +// report says so each picture is reported as worked once - what its new tiles +// were worth - which adds up to the Work the plan counted. +func eachCluster(ctx context.Context, l layout, pics *video.Pictures, report bool, cluster func(first, last int64) error) error { + s := l.stream worked := int64(-1) for first := int64(0); first < s.Frames; { if err := ctx.Err(); err != nil { - return nil, 0, err + return err } // A cluster thirty seconds into a picture asks for the picture it // already has, which At answers without coding anything. change := s.ChangeOf(first) if err := pics.At(change); err != nil { - return nil, 0, err + return err } - if change != worked { + if report && change != worked { format.Worked(ctx, pics.Work()) worked = change } - if s.IsKey(first) { - keys = append(keys, pos) - } last := l.next(first) - pos += writeCluster(out, l, pics, first, last) - if out.err != nil { - return nil, 0, out.err + if err := cluster(first, last); err != nil { + return err } first = last } - return keys, pos, nil + return nil } // writeCluster writes the cluster of frames [first, last) and returns its diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index 6a9d600f..f8690562 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -32,8 +32,6 @@ type notYet struct { // that leaves is held to every size and share, and so is a new format unless // it is added here with its reason. var notYetCutAtTheShare = map[string]notYet{ - "mp4": {paddingBehind, 64 << 10, 50}, - "webm": {paddingBehind, 64 << 10, 50}, // Both take the share at 64 KiB and above. At 4 KiB what stands behind the // picture is more than a small share, so a cut of 1% keeps all of it. "webp": {smallBehind, 4 << 10, 1}, @@ -48,7 +46,7 @@ var notYetCutAtTheShare = map[string]notYet{ // guard for it, and it stays red until the format splits its padding for // format.Request.CutAt. A format that leaves the ratchet leaves this list too, // so it cannot go back. -var listedBeforeTheRule = []string{"mp4", "webm", "webp", "tiff"} +var listedBeforeTheRule = []string{"webp", "tiff"} // listedAfterTheRule is a problem for every format notYetAt names that // listedBeforeTheRule does not. diff --git a/internal/guard/filmcut_test.go b/internal/guard/filmcut_test.go new file mode 100644 index 00000000..85d813f6 --- /dev/null +++ b/internal/guard/filmcut_test.go @@ -0,0 +1,404 @@ +package guard + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "testing" + + "github.com/donislawdev/TestingFilesGenerator/internal/format" + _ "github.com/donislawdev/TestingFilesGenerator/internal/format/all" +) + +// An MP4 or a WebM laid out for a cut comes to every size asked for, every +// sample of it is where its tables say, and the cut keeps the share of its +// pictures. +// +// Both put padding in front of the pictures for a cut - free boxes between the +// movie box and the media data, a Void between the Tracks and the first +// cluster - and each side is nothing or at least a piece, eight bytes for a +// free box and two for a Void: where the share wants less than a piece in +// front, a piece goes there, and where the padding left behind would be +// shorter than one, it goes in front too (format.SplitFront). The padding in +// front moves every sample, and the chunk places of an MP4 and the Cues of a +// WebM give their places in bytes, so they move with it. A WebM is coded twice +// for it, and the first pass's lengths have to be the second's. No other guard +// asks for the sizes and the cuts where those happen. +// +// So this writes consecutive sizes cut at the half, and cuts of one size a +// byte at a time where the share starts to want padding in front and where +// the padding behind runs out, holds each to its size and to the share, and +// asserts it met each layout. Instead of decoding, every sample is read where +// the file's own tables put it and has to be the bytes of the same sample in +// the film nothing cuts - the same stream gives the same pictures in every +// decoder, which is what docs/TRUNCATE-2026-11-11.md section 14.4 measured for +// the layouts - and every cue and seek of a WebM has to land on the element it +// names. +// +// An MP4 of 5 GiB cut at 90% is planned and only its head is written: its +// media data starts past where four bytes reach, so the chunk places widen to +// co64. +// +// Canary: a chunk place a byte off is refused. +func TestAnMP4OrAWebMLaidOutForACutComesToEverySizeAndPlays(t *testing.T) { + for _, id := range []string{"mp4", "webm"} { + desc, err := format.Get(id) + if err != nil { + t.Fatal(err) + } + least := map[string]int64{"mp4": 8, "webm": 2}[id] + wholes := map[int64][][]byte{} + met := map[string]bool{} + check := func(size, cutAt int64) { + data, regions := writeBoxes(t, desc, size, cutAt) + layout, err := filmLayoutOf(data, regions, size, cutAt, least) + if err != nil { + t.Errorf("%s %d B cut at %d: %v", id, size, cutAt, err) + return + } + met[layout] = true + if _, ok := wholes[size]; !ok { + whole, _ := writeBoxes(t, desc, size, 0) + if wholes[size], err = filmSamples(id, whole); err != nil { + t.Fatalf("%s %d B nothing cuts: %v", id, size, err) + } + } + if err := sameSamples(id, data, wholes[size]); err != nil { + t.Errorf("%s %d B cut at %d, %s: %v", id, size, cutAt, layout, err) + } + } + smallest := desc.SmallestAccepted(format.Request{Label: true}) + for size := smallest; size < smallest+6; size++ { + check(size, size/2) + } + // Cuts a byte apart where the share starts to want padding in front - + // it wants cutAt times the part of the file that is not the pictures, + // less the head - and near the end, where what the share leaves behind + // runs out: j bytes from the end it leaves j times that part, less + // what stands behind the padding (a WebM's Cues). + const size = 16_000 + _, regions := writeBoxes(t, desc, size, 0) + head, pics, end := filmParts(t, regions) + rest := size - pics.Length + start := (head - 3) * size / rest + for cutAt := start; cutAt < start+(least+5)*size/rest; cutAt++ { + check(size, cutAt) + } + for j := max(1, (end-1)*size/rest); j <= (end+least+2)*size/rest; j++ { + check(size, size-j) + } + layouts := []string{"nothing in front, the share wants none", "a piece in front, the share wants less", + "the share", "all in front, what would stay behind is shorter than a piece"} + if end > 0 { + layouts = append(layouts, "all in front, the share wants more than there is") + } + for _, layout := range layouts { + if !met[layout] { + t.Errorf("%s: no size met the layout %q - met %v", id, layout, met) + } + } + } + + mp4TooLargeForFourBytes(t) + + // Canary: a chunk place a byte off. + desc, _ := format.Get("mp4") + whole, _ := writeBoxes(t, desc, 64<<10, 0) + samples, err := filmSamples("mp4", whole) + if err != nil { + t.Fatal(err) + } + data, _ := writeBoxes(t, desc, 64<<10, 32<<10) + at := bytes.Index(data, []byte("stco")) + 12 + broken := bytes.Clone(data) + binary.BigEndian.PutUint32(broken[at:], binary.BigEndian.Uint32(broken[at:])+1) + if err := sameSamples("mp4", broken, samples); err == nil { + t.Errorf("canary: a chunk place a byte off passed") + } +} + +// filmParts reads a film's map as what stands in front of all of its padding, +// its pictures, and what stands behind all of its padding - a WebM's Cues. +func filmParts(t *testing.T, regions []format.Region) (head int64, pics format.Region, end int64) { + t.Helper() + seen := false + for _, r := range regions { + switch r.Role { + case format.RoleStructure: + head += r.Length + case format.RoleEnd: + end += r.Length + case format.RoleContent: + pics, seen = r, true + } + } + if !seen || pics.Length < 0 { + t.Fatalf("the map marks the pictures as %+v", pics) + } + return head, pics, end +} + +// filmLayoutOf holds a film laid out for a cut to its size and the share the +// cut keeps of its pictures, and names the layout it took. What stands behind +// the padding behind the pictures - the Cues of a WebM - is the end region. +func filmLayoutOf(data []byte, regions []format.Region, size, cutAt, least int64) (string, error) { + if int64(len(data)) != size { + return "", fmt.Errorf("came to %d B", len(data)) + } + return filmLayoutOfMap(regions, size, cutAt, least) +} + +// filmLayoutOfMap is filmLayoutOf from the map alone, for a film whose bytes +// are not all written. +func filmLayoutOfMap(regions []format.Region, size, cutAt, least int64) (string, error) { + content := -1 + for i, r := range regions { + if r.Role == format.RoleContent { + content = i + } + } + if content < 0 { + return "", errors.New("no content marked") + } + pics := regions[content] + if pics.Length < 0 { + return "", errors.New("the pictures are marked with no length") + } + front, back, end := int64(0), int64(0), int64(0) + if content > 0 && regions[content-1].Role == format.RolePadding { + front = regions[content-1].Length + } + for _, r := range regions[content+1:] { + switch { + case r.Role == format.RolePadding && end == 0: + back += r.Length + case r.Role == format.RoleEnd: + end += r.Length + default: + return "", fmt.Errorf("a %s region at %d, behind the pictures", r.Role, r.At) + } + } + if pics.At+pics.Length+back+end != size { + return "", fmt.Errorf("the pictures end at %d, and %d B of padding and %d B of end follow them in a %d B file", + pics.At+pics.Length, back, end, size) + } + if (front > 0 && front < least) || (back > 0 && back < least) { + return "", fmt.Errorf("a side shorter than a piece: %d in front, %d behind", front, back) + } + head := pics.At - front + want := format.PaddingInFront(head, pics.Length, cutAt, size) + kept, share := cutAt-head-front, pics.Length*cutAt/size + switch { + case front == 0 && want <= 0: + return "nothing in front, the share wants none", nil + case front == 0: + return "", fmt.Errorf("nothing in front and %d B behind, where the share wants %d B in front", back, want) + case front == want && kept == share: + return "the share", nil + case front == least && want > 0 && want < least: + return "a piece in front, the share wants less", nil + case back == 0 && front > want && front-want < least: + return "all in front, what would stay behind is shorter than a piece", nil + case back == 0 && front < want && end > 0: + // A cut in the last pictures, close to the Cues, wants more in front + // than there is padding. The cut keeps more than the share then, the + // one way format.SplitFront allows, and the manifest says how much. + return "all in front, the share wants more than there is", nil + } + return "", fmt.Errorf("%d B in front where the share wants %d, keeping %d B of the pictures where the share is %d", front, want, kept, share) +} + +// sameSamples requires every sample of a film, read where its own tables put +// it, to be the bytes of the same sample of the film nothing cuts. +func sameSamples(id string, data []byte, whole [][]byte) error { + got, err := filmSamples(id, data) + if err != nil { + return err + } + if len(got) != len(whole) { + return fmt.Errorf("%d samples, where the film nothing cuts has %d", len(got), len(whole)) + } + for i := range got { + if !bytes.Equal(got[i], whole[i]) { + return fmt.Errorf("sample %d is not the bytes of the film nothing cuts", i+1) + } + } + return nil +} + +// filmSamples is every sample of a film, read where its own tables put it - +// stsz, stsc and stco or co64 for an MP4, the blocks of each cluster for a +// WebM, whose cues and seeks have to land on the elements they name. +func filmSamples(id string, data []byte) ([][]byte, error) { + if id == "webm" { + return webmSamples(data) + } + top, err := mp4Boxes(data) + if err != nil { + return nil, err + } + var moov []byte + for _, b := range top { + if b.name == "moov" { + moov = b.body + } + } + stbl, err := mp4Path(moov, "trak", "mdia", "minf", "stbl") + if err != nil { + return nil, err + } + starts, lens, err := mp4Samples(stbl) + if err != nil { + return nil, err + } + out := make([][]byte, len(starts)) + for i := range starts { + if starts[i] < 0 || starts[i]+lens[i] > int64(len(data)) { + return nil, fmt.Errorf("sample %d lies outside the file", i+1) + } + out[i] = data[starts[i] : starts[i]+lens[i]] + } + return out, nil +} + +// webmSamples walks a WebM's segment by position: the blocks of every +// cluster, and every cue and seek, which have to give the place of a cluster +// starting at the cue's time and of the element the seek names, counted from +// the start of the segment's content. +func webmSamples(data []byte) ([][]byte, error) { + var seg []byte + if err := ebmlElements(data, func(id uint64, body []byte) error { + if id == 0x18538067 { + seg = body + } + return nil + }); err != nil { + return nil, err + } + var f filmRead + ids, times := map[uint64]uint64{}, map[uint64]int64{} + var cues, seeks [][2]uint64 + for pos := 0; pos < len(seg); { + id, n, err := ebmlVint(seg[pos:], true) + if err != nil { + return nil, err + } + size, m, err := ebmlVint(seg[pos+n:], false) + if err != nil { + return nil, err + } + start := pos + n + m + if uint64(len(seg)-start) < size { + return nil, fmt.Errorf("element %#x at %d says %d B and %d remain", id, pos, size, len(seg)-start) + } + el := seg[start : start+int(size)] + ids[uint64(pos)] = id + switch id { + case 0x1F43B675: + first := len(f.blocks) + if err := walkCluster(&f, el); err != nil { + return nil, err + } + if len(f.blocks) > first { + times[uint64(pos)] = f.blocks[first].ts + } + case 0x1C53BB6B: + cues, err = webmPairs(el, 0xBB, 0xB3, 0xB7, 0xF1) + case 0x114D9B74: + seeks, err = webmPairs(el, 0x4DBB, 0x53AB, 0, 0x53AC) + } + if err != nil { + return nil, err + } + pos = start + int(size) + } + for _, c := range cues { + if ids[c[1]] != 0x1F43B675 || times[c[1]] != int64(c[0]) { + return nil, fmt.Errorf("the cue at %d ms gives %d, where no cluster starts at that time", c[0], c[1]) + } + } + for _, s := range seeks { + if ids[s[1]] != s[0] { + return nil, fmt.Errorf("the seek for %#x gives %d, where %#x starts", s[0], s[1], ids[s[1]]) + } + } + if len(cues) == 0 || len(seeks) != 3 { + return nil, fmt.Errorf("%d cues and %d seeks", len(cues), len(seeks)) + } + out := make([][]byte, len(f.blocks)) + for i, b := range f.blocks { + out[i] = b.data + } + return out, nil +} + +// webmPairs reads each element point in el as a pair: the value of key and +// the value of place, place found inside the element in, or among point's +// own when in is 0. +func webmPairs(el []byte, point, key, in, place uint64) ([][2]uint64, error) { + var out [][2]uint64 + err := ebmlElements(el, func(id uint64, body []byte) error { + if id != point { + return nil + } + var pair [2]uint64 + found := 0 + err := ebmlElements(body, func(id uint64, v []byte) error { + switch { + case id == key: + pair[0] = beUint(v) + found++ + case id == place && in == 0: + pair[1] = beUint(v) + found++ + case id == in: + return ebmlElements(v, func(id uint64, w []byte) error { + if id == place { + pair[1] = beUint(w) + found++ + } + return nil + }) + } + return nil + }) + if err == nil && found != 2 { + err = fmt.Errorf("an element %#x without its %#x or %#x", point, key, place) + } + out = append(out, pair) + return err + }) + return out, err +} + +// mp4TooLargeForFourBytes plans an MP4 of 5 GiB cut at 90% and holds the head +// it writes to chunk places of eight bytes, the first of which gives where the +// map says the samples start, and the map to the share, counted with the +// movie box the widening grew. It writes nothing past the head. +func mp4TooLargeForFourBytes(t *testing.T) { + t.Helper() + const size, cutAt = 5 << 30, 5 << 30 / 10 * 9 + desc, _ := format.Get("mp4") + p, err := desc.Generator.Plan(format.Request{Bytes: size, Label: true, Seed: 7, CutAt: cutAt}) + if err != nil { + t.Fatalf("planning %d B: %v", int64(size), err) + } + head := &headOnly{keep: 64 << 10} + rec := &format.MapRecorder{W: head} + if err := desc.Generator.Write(t.Context(), rec, p); err == nil { + t.Fatalf("writing %d B went through a writer that stops after its head", int64(size)) + } + regions := rec.Map(size, true) + if layout, err := filmLayoutOfMap(regions, size, cutAt, 8); layout != "the share" { + t.Errorf("%d B cut at 90%%: %q, %v", int64(size), layout, err) + } + pics := pictureRegion(regions) + at := bytes.Index(head.got, []byte("co64")) + if at < 0 || pics.At <= 1<<32 { + t.Fatalf("%d B cut at 90%%: no co64 in the head, and the media data starts at %d", int64(size), pics.At) + } + if first := int64(binary.BigEndian.Uint64(head.got[at+12:])); first != pics.At+8 { + t.Errorf("%d B cut at 90%%: the first chunk place is %d, where the samples start at %d", int64(size), first, pics.At+8) + } +} diff --git a/internal/guard/mutationcoverage_test.go b/internal/guard/mutationcoverage_test.go index 8baf5595..43cb9cd3 100644 --- a/internal/guard/mutationcoverage_test.go +++ b/internal/guard/mutationcoverage_test.go @@ -167,6 +167,12 @@ var provenByProbe = map[string]string{ "(50 cuts), with the widening's four bytes left out of the move (5 GiB: 4831825054 where the media data starts at " + "4831825058), with a side of padding allowed below a box (4000 B came to 4007 B) and with a free box rather than a " + "skip box in front of a JPEG XL's codestream (81 cuts).", + "TestAnMP4OrAWebMLaidOutForACutComesToEverySizeAndPlays": "its own canary - a chunk place a byte off must be refused " + + "- its reach check, every layout of the split met, and red on 2026-11-13 with stco not counting the free boxes in " + + "front of the media data (sample 1 not the bytes of the film nothing cuts), with the Cues not moved by the Void in " + + "front of the clusters (a cue giving 226 where no cluster starts), with a side of a WebM's padding allowed below a " + + "Void (16000 B came to 16008 B), with an MP4 that never widens to co64 (5 GiB: no co64 in the head) and with the " + + "widening's growth left out of the head the share is counted from (40 B more in front than the share).", "TestAPDFLaidOutForACutComesToEverySizeAndItsOffsetsLand": "its own canaries - a table that sends one object a byte off " + "must be refused, and the layout of a document nothing cuts must fail a cut at its half - its reach check, every " + "layout of the split met, red on 2026-11-13 with nothing in front of a document under 400 B of padding (398 sizes, " + diff --git a/internal/oracle/strict.py b/internal/oracle/strict.py index 2493ae34..f8d20f32 100644 --- a/internal/oracle/strict.py +++ b/internal/oracle/strict.py @@ -2517,11 +2517,13 @@ def check_webm(data): """A WebM film as this generator writes it, read without its code. The questions are the ones a film can get wrong while every frame still - plays: the segment ends where the file does, with its padding just before - the Cues and the Cues last (docs/WIDEO-2026-10-06.md section 15), the - SeekHead points at what it names, every block is whole AV1 with no - temporal delimiter, nothing shows a picture that may not be shown again - (section 9.2), and the Cues point at the key frames and nothing else. + plays: the segment ends where the file does, with the Cues last and its + padding in a Void right before them (docs/WIDEO-2026-10-06.md section + 15), right before the first cluster - a film laid out for a cut + (docs/TRUNCATE-2026-11-11.md section 14) - or in both, the SeekHead points + at what it names, every block is whole AV1 with no temporal delimiter, + nothing shows a picture that may not be shown again (section 9.2), and the + Cues point at the key frames and nothing else. """ top = list(ebml_elements(data, 0, len(data))) if len(top) != 2 or top[0][0] != 0x1A45DFA3 or top[1][0] != 0x18538067: @@ -2532,8 +2534,13 @@ def check_webm(data): seg_start, seg_end = top[1][2], top[1][3] children = list(ebml_elements(data, seg_start, seg_end)) - if len(children) < 2 or children[-1][0] != 0x1C53BB6B or children[-2][0] != 0xEC: - fail("the segment does not end in a Void and then the Cues, which is where this generator pads") + ids = [c[0] for c in children] + first_cluster = ids.index(0x1F43B675) if 0x1F43B675 in ids else len(ids) + padded = [i for i, ident in enumerate(ids) if ident == 0xEC] + if not ids or ids[-1] != 0x1C53BB6B or not padded or \ + any(i != first_cluster - 1 and i != len(ids) - 2 for i in padded): + fail("the segment does not end in the Cues with a Void right before them, right before the first cluster " + "or both, which is where this generator pads") at_offset = {at - seg_start: ident for ident, at, _, _ in children} blocks, cues, seeks = [], [], 0 @@ -2591,7 +2598,7 @@ def check_webm(data): keys = [(ts, cluster) for ts, key, _, cluster in blocks if key] if [ts for ts, _ in keys] != [ts for ts, _ in cues] or [c for _, c in keys] != [p for _, p in cues]: fail(f"the Cues name {len(cues)} points and the film has {len(keys)} key frames, not at the same places") - ok(f"{len(blocks)} frames, {len(keys)} key frames, every one in the Cues, Void before them") + ok(f"{len(blocks)} frames, {len(keys)} key frames, every one in the Cues, padding in {len(padded)} Void") def iso_child(data, start, end, name, depth="the movie box"): @@ -2673,7 +2680,9 @@ def check_mp4(data): The questions a film can get wrong while every frame still plays (docs/MP4-2026-10-07.md): the boxes cover the file with the free boxes - last, the sample tables agree with the media data byte for byte - the + last, right in front of the media data - a film laid out for a cut + (docs/TRUNCATE-2026-11-11.md section 14) - or both, and with at least + one, the sample tables agree with the media data byte for byte - the samples follow one another from its first byte to its last - every sync sample opens with the sequence header and a shown key frame and every key frame is a sync sample (AV1 binding section 2.4), sdtp says which samples @@ -2682,13 +2691,16 @@ def check_mp4(data): """ top = list(iso_boxes(data, 0, len(data))) names = [n for n, _, _ in top] - if names[:3] != [b"ftyp", b"moov", b"mdat"] or any(n != b"free" for n in names[3:]) or len(names) < 4: - fail(f"the file is {[n.decode('latin1') for n in names]} and this generator writes ftyp, moov, mdat, then free boxes") + media = names.index(b"mdat") if b"mdat" in names else -1 + if names[:2] != [b"ftyp", b"moov"] or media < 2 or b"free" not in names or \ + any(n != b"free" for i, n in enumerate(names[2:], 2) if i != media): + fail(f"the file is {[n.decode('latin1') for n in names]} and this generator writes ftyp, moov and mdat with " + "free boxes in front of mdat, behind it or both") _, fb, fe = top[0] brands = [data[at:at + 4] for at in range(fb + 8, fe, 4)] if b"av01" not in brands: fail("av01 is not among the compatible brands, which the AV1 binding requires (section 2.1)") - _, mb, me = top[2] + _, mb, me = top[media] trak = iso_child(data, top[1][1], top[1][2], b"trak") mdia = iso_child(data, *trak, b"mdia", "the track") From 823d60402732778290e8682b834a9138187fb1ea Mon Sep 17 00:00:00 2001 From: DonislawDev Date: Fri, 13 Nov 2026 23:57:52 +0100 Subject: [PATCH 2/2] The cut guards reach every case they plan and keep a witness behind the content An MP4 no longer cuts in the padding behind its pictures, and it was the case that showed the manifest's note for a cut there. The note's case moves to the one format that still cuts there: a WebP of 4 KiB cut by 1%, whose picture grows to fill the file and leaves more behind it than the cut takes. The share sweep asked only the files its manifest listed, so a format that lost one planned file passed on the others. A planned file the manifest leaves out is now an error, with a canary. A switch over the roles of a film's map names its padding. Co-Authored-By: Claude Opus 5.5 --- internal/guard/cutmanifest_test.go | 15 ++++++++------- internal/guard/cutshare_test.go | 26 +++++++++++++++++++++++++- internal/guard/filmcut_test.go | 3 +++ 3 files changed, 36 insertions(+), 8 deletions(-) diff --git a/internal/guard/cutmanifest_test.go b/internal/guard/cutmanifest_test.go index 2e11a0c8..1232ee63 100644 --- a/internal/guard/cutmanifest_test.go +++ b/internal/guard/cutmanifest_test.go @@ -37,8 +37,9 @@ type cutCase struct { // the content is left, and gets a note when the cut missed the content. // // The shapes a cut takes (docs/TRUNCATE-2026-11-11.md sections 2, 7 and 8). -// An MP4 of 1 MiB is nearly all padding after its video, so 90% missing still -// leaves all of it. A PNG splits its padding around the picture +// A WebP grows its picture to fill the file and keeps what is left over +// behind it, which at 4 KiB is more than the 41 B a cut of 1% takes, so the +// cut leaves all of the picture. A PNG splits its padding around the picture // for a cut, so the cut takes the share asked for, and so does a tar.gz. A // text is its words, so the cut falls in them. Two cuts in a row end the file where the second one does, and // the cut is said of that one. A damage that cuts nothing has no cut to report. @@ -47,9 +48,9 @@ type cutCase struct { // one that claims content it does not have. func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { cases := []cutCase{ - // A GIF and then a PDF were this case until each split its padding - // for the cut. - {name: "behind", format: "mp4", size: 1 << 20, missing: "90", in: "padding", kept: "all", note: "after the content"}, + // A GIF, a PDF and an MP4 were this case until each split its + // padding for the cut. + {name: "behind", format: "webp", size: 4 << 10, missing: "1", in: "padding", kept: "all", note: "after the content"}, {name: "picture", format: "png", size: 1 << 20, missing: "90", in: "content", kept: "share"}, // 4 KiB on disk is 40 KiB whole. It was the case of a cut in the padding // in front of the content, with an icon and a JPEG before it, until it @@ -96,7 +97,7 @@ func TestACutFileSaysWhereTheCutFellAndWhatIsLeft(t *testing.T) { } } - // Canary: the GIF's entry, told the cut fell in its content, and the + // Canary: the WebP's entry, told the cut fell in its content, and the // text's, told none of its content is left. behind := byTarget["behind"] if len(behind.Damage) == 1 && behind.Damage[0].Cut != nil { @@ -180,7 +181,7 @@ func TestTheRecordOfACutRunIsNeverLongerThanItsForecast(t *testing.T) { } opt := engine.Options{OutDir: t.TempDir(), Seed: 7, Command: "test"} planned, err := engine.Plan([]engine.Target{ - cut("picture", "png", 1<<20, "90"), cut("behind", "mp4", 1<<20, "90"), cut("text", "txt", 1000, "50"), + cut("picture", "png", 1<<20, "90"), cut("behind", "webp", 4<<10, "1"), cut("text", "txt", 1000, "50"), }, opt) if err != nil { t.Fatalf("planning: %v", err) diff --git a/internal/guard/cutshare_test.go b/internal/guard/cutshare_test.go index f8690562..132628c7 100644 --- a/internal/guard/cutshare_test.go +++ b/internal/guard/cutshare_test.go @@ -98,7 +98,8 @@ var ( // structure by TestEveryFormatsMapIsTheOneItsStructureReadsBack. // // Canaries: a cut that kept all its content must be reported, and so must a -// format listed as not yet cutting at the share that does. +// format listed as not yet cutting at the share that does, and a file planned +// for the sweep that the manifest leaves out. func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { missingOf := map[string]int64{} var targets []engine.Target @@ -128,7 +129,9 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { plannedOf[f.Target.ID] = f } problems := map[string]string{} + inManifest := map[string]bool{} for _, m := range res.Manifest.Files { + inManifest[m.TargetID] = true missing, ok := missingOf[m.TargetID] if !ok || m.Failed { t.Errorf("%s: no file was written (%s)", m.TargetID, m.Error) @@ -152,6 +155,12 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { for _, problem := range shareVerdicts(problems, notYetCutAtTheShare) { t.Error(problem) } + for _, problem := range absentFromTheManifest(missingOf, inManifest) { + t.Error(problem) + } + if len(absentFromTheManifest(map[string]int64{"webp-4096-1": 1}, map[string]bool{})) != 1 { + t.Errorf("canary: a planned file the manifest left out passed") + } for _, problem := range listedAfterTheRule(notYetCutAtTheShare) { t.Error(problem) } @@ -191,6 +200,21 @@ func TestACutTakesTheShareOfTheContentItIsAskedFor(t *testing.T) { } } +// absentFromTheManifest is every file planned for the sweep that the manifest +// has no entry for. shareVerdicts asks only the cases a file was written for, +// so without this a format that loses one of its files to the run still +// passes on the others. +func absentFromTheManifest(missingOf map[string]int64, inManifest map[string]bool) []string { + var out []string + for id := range missingOf { + if !inManifest[id] { + out = append(out, id+": planned for the sweep and not in the manifest, so its cut was never asked") + } + } + slices.Sort(out) + return out +} + // cutInTheEnd is what shareProblem says of a cut that fell in the end a file // closes with: a central directory, a trailer. const cutInTheEnd = "the cut fell in the end" diff --git a/internal/guard/filmcut_test.go b/internal/guard/filmcut_test.go index 85d813f6..f350b92b 100644 --- a/internal/guard/filmcut_test.go +++ b/internal/guard/filmcut_test.go @@ -131,6 +131,9 @@ func filmParts(t *testing.T, regions []format.Region) (head int64, pics format.R end += r.Length case format.RoleContent: pics, seen = r, true + case format.RolePadding: + // What the parts around it leave of the file - filmLayoutOf + // reads the two sides of it. } } if !seen || pics.Length < 0 {