diff --git a/CHANGELOG.md b/CHANGELOG.md index dce8d1c..64615d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ Versions after 2.17 are `MAJOR.MINOR.PATCH` (2.17 is followed by 2.17.1 or 2.18. `docs/testing.md`, "Versions and releases"); the headings below keep the numbers as they were released. +## 2.18.2 — 2026-10-20 + +- Single view: the decoder is set up for the stream's size from its SPS, so streams above + 1080p (e.g. 2304×1296) play on hardware decoders that cannot find the size themselves; a + hardware decoder that gives no frame at a size it does not declare hands the stream to a + software one after 5 s. `/stats.json` shows `size` only while a picture is shown, and adds + `stream_size` and `decoder` (PR #220, #166). + ## 2.18.1 — 2026-10-20 - Snapshot (`/snapshot.jpg`, the debug page): the single view's picture keeps its aspect ratio diff --git a/android/app/build.gradle b/android/app/build.gradle index caca9eb..ccfb652 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 44 - versionName "2.18.1" + versionCode 45 + versionName "2.18.2" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index 521cdf3..562afeb 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -222,6 +222,8 @@ protected void onCreate(Bundle savedInstanceState) { @Override public void onDecoderStarted() { removeClearOverlay(); } @Override public void onDecoderFailed() {} }); + // Single view: streams above 1080p on decoders that need to know it up front (#166) + videoDecoder.allowLargeFrames(true); audioManager = new AudioManager(); rtspClient = new RtspClient(framePool, nalAssembler, nalQueue); rtspClient.setListener(new RtspClient.Listener() { @@ -1618,11 +1620,19 @@ private Map debugState() { rtspClient != null ? rtspClient.transportInUse() : null, mTransports[mActive])); single.put("transport_mode", mTransports[mActive] != null ? mTransports[mActive].name() : null); - single.put("video", mResumed && "connected".equals(singleState) - && singleRenderedWithin(SNAPSHOT_LIVE_MS)); + boolean live = mResumed && "connected".equals(singleState) + && singleRenderedWithin(SNAPSHOT_LIVE_MS); + single.put("video", live); single.put("status", statusText != null && statusText.getVisibility() == View.VISIBLE ? statusText.getText().toString() : singleState); - if (mVideoWidth > 0) single.put("size", mVideoWidth + "x" + mVideoHeight); + // The decoded picture's size, only while it is shown: otherwise it is the size of + // an earlier stream (#166) + if (live && mVideoWidth > 0) single.put("size", mVideoWidth + "x" + mVideoHeight); + // What the camera sends, from the SPS, and the decoder taking it (#166) + int[] streamSize = videoDecoder.streamSize(); + if (streamSize != null) single.put("stream_size", streamSize[0] + "x" + streamSize[1]); + String decoderName = videoDecoder.decoderName(); + if (decoderName != null) single.put("decoder", decoderName); single.put("codec", rtspClient != null && rtspClient.isJpeg() ? "MJPEG" : codecH265 ? "H.265" : "H.264"); single.put("audio", audioCodec); diff --git a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java index f66ce7d..6d8f9ec 100644 --- a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java +++ b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java @@ -56,6 +56,41 @@ class MediaCodecManager { // Surface set by the owning activity private volatile Surface surface; + /** + * Single view (#166): the decoder is configured for the stream's size from its SPS, and + * KEY_MAX_WIDTH/HEIGHT covers it. Configured at 1280×720, some hardware decoders (Rockchip + * OMX) never give a frame for streams above 1080p, e.g. 2304×1296. Off in the grid: nine + * decoders with large buffers would not fit in memory. + */ + private volatile boolean largeFrames; + /** Picture size from the last SPS (largeFrames only); 0 while unknown. */ + private volatile int streamWidth; + private volatile int streamHeight; + /** An SPS arrived (decoderLock): with largeFrames no decoder is created before one. */ + private boolean spsSeen; + /** The last H.265 VPS (decoderLock): a decoder created at the SPS needs it first. */ + private byte[] lastVps; + /** Largest picture the current decoder was set up for; 0 when not known. */ + private int confWidth; + private int confHeight; + /** + * The hardware decoder runs at a size it does not declare: without a frame by + * {@link #probationUntilMs} the stream goes to a software decoder (#166). + */ + private boolean probation; + /** "mime WxH" the decoder on probation was set up for (Qodo on #220). */ + private String probationKey; + private long probationUntilMs; + private int probationFrames; + /** How long a decoder on probation may go without a frame once it has its parameter sets. */ + static final long PROBATION_MS = 5_000; + /** "mime WxH" of streams the hardware decoder gave no frame for: software from then on. */ + private final java.util.Set softwareFor = java.util.Collections.synchronizedSet( + new java.util.LinkedHashSet<>()); + /** Sizes kept in {@link #softwareFor}: a camera switches between a few at most. */ + static final int SOFTWARE_SIZES = 8; + /** Name of the current decoder, for /stats.json. */ + private volatile String decoderName; /** Degrees counter-clockwise the decoder turns the picture (#201); 0 for none. */ private volatile int rotation; /** The turn the current decoder applies: {@link #rotation} where Android 6+ can, else 0. */ @@ -111,6 +146,13 @@ boolean swapsSides() { return t == 90 || t == 270; } boolean hasDecoder() { return decoder != null; } + /** Picture size from the stream's SPS ({width, height}), or null while unknown (#166). */ + int[] streamSize() { + int w = streamWidth, h = streamHeight; + return w > 0 && h > 0 ? new int[]{w, h} : null; + } + /** Name of the decoder in use, or null. */ + String decoderName() { return decoder != null ? decoderName : null; } /** * Decoders alive in this process per MIME type, against the declared limit of the codec @@ -270,6 +312,43 @@ void setSurface(Surface s) { this.surface = s; } + /** See {@link #largeFrames}; takes effect with the next decoder. */ + void allowLargeFrames(boolean on) { + largeFrames = on; + } + + /** Largest supported size up to 3840×2160 (a few steps down), or null if none above 1080p. */ + static int[] maxFrameSize(MediaCodec codec, String mime) { + try { + MediaCodecInfo.VideoCapabilities video = + codec.getCodecInfo().getCapabilitiesForType(mime).getVideoCapabilities(); + if (video == null) return null; + int[][] sizes = {{3840, 2160}, {2560, 1920}, {2560, 1440}, {2304, 1296}}; + for (int[] size : sizes) { + if (video.isSizeSupported(size[0], size[1])) return size; + } + } catch (Exception ignored) { + // Capabilities unknown: keep the decoder's defaults + } + return null; + } + + /** The codec declares the size; unknown capabilities count as declared. */ + static boolean declaresSize(MediaCodec codec, String mime, int width, int height) { + try { + MediaCodecInfo.VideoCapabilities video = + codec.getCodecInfo().getCapabilitiesForType(mime).getVideoCapabilities(); + return video == null || video.isSizeSupported(width, height); + } catch (Exception e) { + return true; + } + } + + /** Above 1080p: a size some hardware decoders cannot be left to discover (#166). */ + static boolean aboveFullHd(int width, int height) { + return (long) width * height > 1920L * 1088; + } + /** See {@link #rotation}; takes effect with the next decoder. */ void setRotation(int degreesCcw) { rotation = ((degreesCcw % 360) + 360) % 360; @@ -305,15 +384,16 @@ private static void releaseMirror(GlMirror g) { */ void setEnabled(boolean enabled) { this.enabled = enabled; - lifecycle++; + final int gen = ++lifecycle; if (enabled) return; final MediaCodec target = decoder; - if (target == null) return; new Thread(() -> { String slot; GlMirror gl; synchronized (decoderLock) { - if (decoder != target) return; // already replaced or released + // The stream that comes after the pause sends its own SPS (Qodo on #220) + if (gen == lifecycle && !this.enabled) forgetStream(); + if (target == null || decoder != target) return; // already replaced or released decoder = null; slot = takeSlot(); gl = takeMirror(); @@ -357,6 +437,8 @@ private void closeDecoderLocked() { MediaCodec codec = decoder; // The next stream must not look live because of this one's last frame lastFrameMs = 0; + // Nor be set up for this one's size (#166) + forgetStream(); if (codec == null) return; decoder = null; String slot = takeSlot(); @@ -373,6 +455,16 @@ private void closeDecoderLocked() { } } + /** Stream size and parameter sets of the last stream (decoderLock held). */ + private void forgetStream() { + spsSeen = false; + streamWidth = 0; + streamHeight = 0; + lastVps = null; + probation = false; + probationKey = null; + } + private static long nowMs() { return System.nanoTime() / 1_000_000; } @@ -401,12 +493,35 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { String type = codecH265 ? "video/hevc" : "video/avc"; - MediaFormat format = MediaFormat.createVideoFormat(type, 1280, 720); + // Single view: the stream's own size from its SPS (#166) + final int width = largeFrames ? streamWidth : 0; + final int height = largeFrames ? streamHeight : 0; + final boolean sized = width > 0 && height > 0; + MediaFormat format = MediaFormat.createVideoFormat(type, + sized ? width : 1280, sized ? height : 720); // Must hold the largest NAL unit the assembler can deliver format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_NAL_SIZE); String softwareName = null; - if (!InstanceBudget.tryAcquire(type)) { + boolean noHardwareFrame = false; + final String key = sizeKey(type, width, height); + if (sized && softwareFor.contains(key)) { + // The hardware decoder gave no frame at this size: a software one, once it has room + String sw = softwareDecoder(type); + if (sw != null && InstanceBudget.tryAcquire(softwareSlot(type))) { + softwareName = sw; + noHardwareFrame = true; + } else if (sw != null) { + // Not the hardware one again: it showed nothing at this size (Qodo on #220) + synchronized (decoderLock) { + retry.onFailure(nowMs()); + } + Log.w(TAG, "No room for a software decoder at " + width + "×" + height + + ", retrying"); + return; + } + } + if (softwareName == null && !InstanceBudget.tryAcquire(type)) { int limit = InstanceBudget.limit(type); softwareName = limit >= 0 && allowSoftware ? softwareDecoder(type) : null; // Software decoders have their own limit: beyond it, the hardware one is reported @@ -428,18 +543,41 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { final String slot = software ? softwareSlot(type) : type; MediaCodec local; + boolean onProbation = false; + int maxW = 0, maxH = 0; GlMirror gl = null; int turnApplied = 0; try { if (software) { - Log.i(TAG, "Start software video decoder " + softwareName - + " (hardware limit reached, key frames only)"); + Log.i(TAG, "Start software video decoder " + softwareName + (noHardwareFrame + ? " (no frame from the hardware one at " + width + "×" + height + ")" + : " (hardware limit reached, key frames only)")); local = MediaCodec.createByCodecName(softwareName); } else { Log.i(TAG, "Start video decoder (" + type + ")"); local = MediaCodec.createDecoderByType(type); learnLimits(local, type); } + if (largeFrames) { + // Room for the stream and for the largest declared size, so the picture may + // still grow without a new decoder + int[] max = maxFrameSize(local, type); + maxW = Math.max(sized ? width : 0, max != null ? max[0] : 0); + maxH = Math.max(sized ? height : 0, max != null ? max[1] : 0); + if (maxW > 0 && maxH > 0) { + format.setInteger(MediaFormat.KEY_MAX_WIDTH, maxW); + format.setInteger(MediaFormat.KEY_MAX_HEIGHT, maxH); + } + boolean declared = !sized || declaresSize(local, type, width, height); + // Probation only where a software decoder can take over, and once per size + onProbation = !software && sized && !declared && aboveFullHd(width, height) + && !softwareFor.contains(key) + && softwareDecoder(type) != null; + Log.i(TAG, "Decoder " + local.getName() + ": stream " + + (sized ? width + "×" + height : "size unknown") + + (maxW > 0 ? ", pictures up to " + maxW + "×" + maxH : "") + + (declared ? "" : ", a size it does not declare")); + } int turn = rotation; Surface target = s; if (mirror) { @@ -481,6 +619,8 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { // A creation cancelled by a pause must not delay the next start if (gen != lifecycle || !enabled) return; retry.onFailure(nowMs()); + // The hardware decoder refused a size it does not declare: software next time + if (onProbation) markSoftware(key); } Log.e(TAG, "Cannot setup decoder (attempt " + retry.failures() + "): " + e.getMessage()); if (listener != null) listener.onDecoderFailed(); @@ -505,9 +645,20 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { softwareInUse = software; awaitingConfig = true; configSeen = 0; + decoderName = local.getName(); + confWidth = maxW; + confHeight = maxH; + probation = onProbation; + probationKey = onProbation ? key : null; + probationUntilMs = 0; + probationFrames = framesOut; } } + private static String sizeKey(String mime, int width, int height) { + return mime + " " + width + "x" + height; + } + /** * Decode one NAL frame. Returns true if there is no decoder and one may be created now: * the caller then creates it and passes the same frame again. @@ -527,68 +678,170 @@ boolean decodeFrame(Frame buffer, boolean codecH265, long watchdogRef) { flag = MediaCodec.BUFFER_FLAG_CODEC_CONFIG; } + boolean isSps = fragment == (codecH265 ? H265_NAL_SPS : H264_NAL_SPS); + boolean isVps = codecH265 && fragment == H265_NAL_VPS; + // Single view: the stream's size, before any decoder is set up for it (#166) + int[] spsSize = largeFrames && isSps + ? SpsParser.size(buffer.data(), 4, buffer.length() - 4, codecH265) : null; + boolean needCreate = false; synchronized (decoderLock) { MediaCodec codec = decoder; + if (largeFrames) { + if (isSps) { + spsSeen = true; + if (spsSize != null) { + streamWidth = spsSize[0]; + streamHeight = spsSize[1]; + } + } else if (isVps) { + lastVps = java.util.Arrays.copyOf(buffer.data(), buffer.length()); + } + } if (!enabled) { // Paused: the decoder is being released, feed nothing return false; } else if (codec == null) { + // Single view: the decoder waits for the SPS, which gives its size (#166) + if (largeFrames && !spsSeen) return false; needCreate = retry.mayCreate(nowMs()); } else if (awaitingConfig && !isConfigNal) { // Frames before the full parameter set only produce errors in a new decoder return false; + } else if (largeFrames && spsSize != null && confWidth > 0 + && (spsSize[0] > confWidth || spsSize[1] > confHeight)) { + // The picture outgrew the decoder: a new one set up for it takes this SPS + Log.i(TAG, "Stream " + spsSize[0] + "×" + spsSize[1] + " is larger than " + + confWidth + "×" + confHeight + ": new decoder"); + dropDecoder(codec); + return retry.mayCreate(nowMs()); + } else if (probation && probationExpired()) { + // The size the decoder was set up for, not a later SPS's (Qodo on #220) + Log.w(TAG, "No frame from " + decoderName + " (" + probationKey + ") in " + + PROBATION_MS / 1000 + " s: software decoder"); + markSoftware(probationKey); + dropDecoder(codec); + // The next decoder starts at the next SPS, which comes with a key frame + return false; } else { if (isConfigNal) { - configSeen |= 1 << fragment; int needed = codecH265 ? (1 << H265_NAL_VPS) | (1 << H265_NAL_SPS) | (1 << H265_NAL_PPS) : (1 << H264_NAL_SPS) | (1 << H264_NAL_PPS); - if ((configSeen & needed) == needed) awaitingConfig = false; - } - try { - // No free input buffer at once: give decoded frames back (the decoder may - // be waiting for output space) and try again, for up to INPUT_WAIT_MS. - // Dropping a NAL unit corrupts the picture until the next key frame (#89) - int inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); - long waitUntil = nowMs() + INPUT_WAIT_MS; - // A close or a pause waiting for the lock ends the wait at once (UI thread) - while (inputBufferId < 0 && nowMs() < waitUntil && enabled - && closeWaiting.get() == 0) { - drainOutput(codec); - inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); - } - if (inputBufferId >= 0) { - ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); - if (inputBuffer != null) { - inputBuffer.clear(); - inputBuffer.put(buffer.data(), 0, buffer.length()); - codec.queueInputBuffer(inputBufferId, 0, - buffer.length(), System.nanoTime() / 1000, flag); + // A decoder created at the SPS has missed the VPS before it (#166) + if (isSps && awaitingConfig && (configSeen & (1 << H265_NAL_VPS)) == 0 + && codecH265 && lastVps != null) { + configSeen |= 1 << H265_NAL_VPS; + try { + feed(codec, lastVps, lastVps.length, MediaCodec.BUFFER_FLAG_CODEC_CONFIG); + } catch (Exception e) { + failDecoder(codec, e); + return false; } - } else if (enabled && closeWaiting.get() == 0) { - // Only a full wait is a decoder loss; a close or pause cut it short - inputDropped++; } - drainOutput(codec); + configSeen |= 1 << fragment; + if ((configSeen & needed) == needed && awaitingConfig) { + awaitingConfig = false; + // The probation runs from the first key frame on + if (probation) probationUntilMs = nowMs() + PROBATION_MS; + } + } + try { + feed(codec, buffer.data(), buffer.length(), flag); } catch (Exception e) { - decoder = null; - String slot = takeSlot(); - GlMirror gl = takeMirror(); - retry.onFailure(nowMs()); - Log.e(TAG, "Codec exception, retry in " - + DecoderRetry.delayMs(retry.failures()) + " ms: " + e.getMessage()); - try { codec.stop(); } catch (Exception ignored) {} - try { codec.release(); } catch (Exception ignored) {} - releaseMirror(gl); - releaseSlot(slot); - if (listener != null) listener.onDecoderFailed(); + failDecoder(codec, e); } } } return needCreate; } + /** + * Queue one NAL unit (decoderLock held). No free input buffer at once: give decoded frames + * back (the decoder may be waiting for output space) and try again, for up to + * INPUT_WAIT_MS. Dropping a NAL unit corrupts the picture until the next key frame (#89). + */ + private void feed(MediaCodec codec, byte[] data, int length, int flag) { + int inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); + long waitUntil = nowMs() + INPUT_WAIT_MS; + // A close or a pause waiting for the lock ends the wait at once (UI thread) + while (inputBufferId < 0 && nowMs() < waitUntil && enabled + && closeWaiting.get() == 0) { + drainOutput(codec); + inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); + } + if (inputBufferId >= 0) { + ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); + if (inputBuffer != null) { + inputBuffer.clear(); + inputBuffer.put(data, 0, length); + codec.queueInputBuffer(inputBufferId, 0, length, System.nanoTime() / 1000, flag); + } + } else if (enabled && closeWaiting.get() == 0) { + // Only a full wait is a decoder loss; a close or pause cut it short + inputDropped++; + } + drainOutput(codec); + } + + /** The decoder failed while decoding (decoderLock held): release it, back off. */ + private void failDecoder(MediaCodec codec, Exception e) { + // A hardware decoder that fails at a size it does not declare: software next time + if (probation && probationKey != null) markSoftware(probationKey); + probation = false; + decoder = null; + String slot = takeSlot(); + GlMirror gl = takeMirror(); + retry.onFailure(nowMs()); + Log.e(TAG, "Codec exception, retry in " + + DecoderRetry.delayMs(retry.failures()) + " ms: " + e.getMessage()); + releaseNow(codec); + // After the decoder: its output must be gone before GL leaves the surface + releaseMirror(gl); + releaseSlot(slot); + if (listener != null) listener.onDecoderFailed(); + } + + /** Release a working decoder to replace it (decoderLock held): no back-off. */ + private void dropDecoder(MediaCodec codec) { + decoder = null; + probation = false; + String slot = takeSlot(); + GlMirror gl = takeMirror(); + releaseNow(codec); + releaseMirror(gl); + releaseSlot(slot); + } + + /** + * The hardware decoder gave no frame at this size (decoderLock held): software from now + * on, for this size among the last {@link #SOFTWARE_SIZES} (Qodo on #220). The next + * decoder waits for a fresh SPS, so it starts with the full parameter set and the key + * frame after it; the stream's size is kept for it. + */ + private void markSoftware(String key) { + synchronized (softwareFor) { + softwareFor.remove(key); + softwareFor.add(key); + java.util.Iterator it = softwareFor.iterator(); + while (softwareFor.size() > SOFTWARE_SIZES && it.hasNext()) { + it.next(); + it.remove(); + } + } + spsSeen = false; + } + + /** The decoder on probation has its parameter sets but no frame past the deadline. */ + private boolean probationExpired() { + if (framesOut != probationFrames) { + // A frame came: the hardware decoder manages the size + probation = false; + return false; + } + return probationUntilMs > 0 && nowMs() > probationUntilMs; + } + /** * The picture's size: the crop rectangle when the decoder gives one. KEY_WIDTH / KEY_HEIGHT * are the buffer, padded to the decoder's alignment (1080 → 1088; one H.265 decoder made diff --git a/android/app/src/main/java/com/openipc/decoder/SpsParser.java b/android/app/src/main/java/com/openipc/decoder/SpsParser.java new file mode 100644 index 0000000..fe964b4 --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/SpsParser.java @@ -0,0 +1,196 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * SpsParser.java — picture size from an H.264 / H.265 sequence parameter set + * + */ + +package com.openipc.decoder; + +/** + * Reads the visible picture size (after the cropping / conformance window) from an SPS, so a + * decoder can be configured for the stream's real size before the first frame (#166). + */ +final class SpsParser { + private SpsParser() {} + + /** + * Size {width, height} from one NAL unit (no start code, header included), or null if it + * is not an SPS or cannot be read. + */ + static int[] size(byte[] nal, int offset, int length, boolean h265) { + if (nal == null || length < (h265 ? 3 : 2) || offset < 0 + || offset + length > nal.length) return null; + try { + Bits b = new Bits(unescape(nal, offset, length)); + return h265 ? h265(b) : h264(b); + } catch (RuntimeException e) { + // Truncated or malformed: the size stays unknown + return null; + } + } + + private static int[] h264(Bits b) { + if ((b.u(8) & 0x1f) != MediaCodecManager.H264_NAL_SPS) return null; + int profile = b.u(8); + b.u(8); // constraint flags + b.u(8); // level + b.ue(); // seq_parameter_set_id + int chroma = 1; + boolean separatePlanes = false; + if (profile == 100 || profile == 110 || profile == 122 || profile == 244 + || profile == 44 || profile == 83 || profile == 86 || profile == 118 + || profile == 128 || profile == 138 || profile == 139 || profile == 134 + || profile == 135) { + chroma = b.ue(); + if (chroma == 3) separatePlanes = b.u(1) == 1; + b.ue(); // bit_depth_luma_minus8 + b.ue(); // bit_depth_chroma_minus8 + b.u(1); // qpprime_y_zero_transform_bypass_flag + if (b.u(1) == 1) { + for (int i = 0; i < (chroma != 3 ? 8 : 12); i++) { + if (b.u(1) == 1) skipScalingList(b, i < 6 ? 16 : 64); + } + } + } + b.ue(); // log2_max_frame_num_minus4 + int pocType = b.ue(); + if (pocType == 0) { + b.ue(); // log2_max_pic_order_cnt_lsb_minus4 + } else if (pocType == 1) { + b.u(1); // delta_pic_order_always_zero_flag + b.se(); // offset_for_non_ref_pic + b.se(); // offset_for_top_to_bottom_field + int n = b.ue(); + for (int i = 0; i < n; i++) b.se(); + } + b.ue(); // max_num_ref_frames + b.u(1); // gaps_in_frame_num_value_allowed_flag + // In long: a malformed code must not wrap round to a small size (Qodo on #220) + long widthMbs = b.ueLong() + 1; + long heightUnits = b.ueLong() + 1; + int frameMbsOnly = b.u(1); + if (frameMbsOnly == 0) b.u(1); // mb_adaptive_frame_field_flag + b.u(1); // direct_8x8_inference_flag + long width = widthMbs * 16; + long height = (2 - frameMbsOnly) * heightUnits * 16; + if (b.u(1) == 1) { + long left = b.ueLong(), right = b.ueLong(), top = b.ueLong(), bottom = b.ueLong(); + int arrayType = separatePlanes ? 0 : chroma; + int cropX = arrayType == 1 || arrayType == 2 ? 2 : 1; + int cropY = (arrayType == 1 ? 2 : 1) * (2 - frameMbsOnly); + width -= cropX * (left + right); + height -= cropY * (top + bottom); + } + return valid(width, height); + } + + private static void skipScalingList(Bits b, int size) { + int last = 8, next = 8; + for (int j = 0; j < size; j++) { + if (next != 0) next = (last + b.se() + 256) % 256; + last = next == 0 ? last : next; + } + } + + private static int[] h265(Bits b) { + if (((b.u(8) >> 1) & 0x3f) != MediaCodecManager.H265_NAL_SPS) return null; + b.u(8); // rest of the NAL header + b.u(4); // sps_video_parameter_set_id + int subLayers = b.u(3); // sps_max_sub_layers_minus1 + b.u(1); // sps_temporal_id_nesting_flag + // profile_tier_level(1, subLayers): general part, 96 bits + b.skip(96); + boolean[] profile = new boolean[subLayers]; + boolean[] level = new boolean[subLayers]; + for (int i = 0; i < subLayers; i++) { + profile[i] = b.u(1) == 1; + level[i] = b.u(1) == 1; + } + if (subLayers > 0) b.skip(2 * (8 - subLayers)); + for (int i = 0; i < subLayers; i++) { + if (profile[i]) b.skip(88); + if (level[i]) b.skip(8); + } + b.ue(); // sps_seq_parameter_set_id + int chroma = b.ue(); + boolean separatePlanes = chroma == 3 && b.u(1) == 1; + long width = b.ueLong(); + long height = b.ueLong(); + if (b.u(1) == 1) { + long left = b.ueLong(), right = b.ueLong(), top = b.ueLong(), bottom = b.ueLong(); + int arrayType = separatePlanes ? 0 : chroma; + int subW = arrayType == 1 || arrayType == 2 ? 2 : 1; + int subH = arrayType == 1 ? 2 : 1; + width -= subW * (left + right); + height -= subH * (top + bottom); + } + return valid(width, height); + } + + private static int[] valid(long width, long height) { + // Beyond 8K, empty or cropped below nothing: not a size worth configuring a decoder for + return width > 0 && height > 0 && width <= 8192 && height <= 8192 + ? new int[]{(int) width, (int) height} : null; + } + + /** The NAL unit without its emulation prevention bytes (00 00 03 → 00 00). */ + static byte[] unescape(byte[] data, int offset, int length) { + byte[] out = new byte[length]; + int n = 0, zeros = 0; + for (int i = offset; i < offset + length; i++) { + int v = data[i] & 0xff; + if (zeros >= 2 && v == 3) { + zeros = 0; + continue; + } + zeros = v == 0 ? zeros + 1 : 0; + out[n++] = (byte) v; + } + return java.util.Arrays.copyOf(out, n); + } + + /** MSB-first bit reader with Exp-Golomb codes; reading past the end throws. */ + private static final class Bits { + private final byte[] data; + private int pos; + + Bits(byte[] data) { this.data = data; } + + int u(int n) { + int v = 0; + for (int i = 0; i < n; i++) { + if (pos >= data.length * 8) throw new IllegalStateException("SPS truncated"); + v = (v << 1) | ((data[pos >> 3] >> (7 - (pos & 7))) & 1); + pos++; + } + return v; + } + + void skip(int n) { + if (pos + n > data.length * 8) throw new IllegalStateException("SPS truncated"); + pos += n; + } + + /** Unsigned Exp-Golomb code, 0 … 2^32 − 2. */ + long ueLong() { + int zeros = 0; + while (u(1) == 0) { + if (++zeros > 31) throw new IllegalStateException("bad Exp-Golomb code"); + } + return zeros == 0 ? 0 : (1L << zeros) - 1 + (u(zeros) & 0xffffffffL); + } + + /** {@link #ueLong()} where the syntax needs an int; larger values are malformed. */ + int ue() { + long v = ueLong(); + if (v > Integer.MAX_VALUE) throw new IllegalStateException("Exp-Golomb code too large"); + return (int) v; + } + + int se() { + int k = ue(); + return (k & 1) == 1 ? (k + 1) / 2 : -(k / 2); + } + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java b/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java new file mode 100644 index 0000000..75547b3 --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java @@ -0,0 +1,137 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * SpsParserTest.java — picture size from H.264 / H.265 SPS (#166) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +public class SpsParserTest { + // SPS from ffmpeg 6.1 (libx264 / libx265), testsrc, emulation prevention bytes included + private static final String H264_2304 = "67640032acd9402400a3b011000003000100000300320f183196"; + private static final String H264_1080 = "674d4028eca03c0113f2e022000003000200000300641e30632c"; + private static final String H264_720 = "6742c01fd9005005bb011000000300100000030320f1832480"; + private static final String H264_444 = "67f40028919b280f0044fc4e0220000003002000000641e30632c0"; + private static final String H264_INTERLACED = + "67640028acd94078044fde0220000003002000000643e2c5b2c0"; + private static final String H265_2304 = "420101016000000300900000030000030096a0012020051165959a" + + "4932bc05a02000000300200000030321"; + private static final String H265_1080 = "420101016000000300900000030000030078a003c08010e5965669" + + "24caf0168080000003008000000c84"; + + private static int[] size(String hex, boolean h265) { + byte[] b = hex(hex); + return SpsParser.size(b, 0, b.length, h265); + } + + private static byte[] hex(String s) { + byte[] b = new byte[s.length() / 2]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(s.substring(2 * i, 2 * i + 2), 16); + } + return b; + } + + @Test + public void h264Sizes() { + // High, 2304×1296: 144×81 macroblocks, no cropping + assertArrayEquals(new int[]{2304, 1296}, size(H264_2304, false)); + // Main, 1080p: 1088 coded, cropped to 1080 + assertArrayEquals(new int[]{1920, 1080}, size(H264_1080, false)); + assertArrayEquals(new int[]{1280, 720}, size(H264_720, false)); + // High 4:4:4: crop units of one line + assertArrayEquals(new int[]{1920, 1080}, size(H264_444, false)); + // Interlaced: map units are field macroblock pairs + assertArrayEquals(new int[]{1920, 1080}, size(H264_INTERLACED, false)); + } + + @Test + public void h265Sizes() { + assertArrayEquals(new int[]{2304, 1296}, size(H265_2304, true)); + // Coded 1920×1088 with a conformance window + assertArrayEquals(new int[]{1920, 1080}, size(H265_1080, true)); + } + + @Test + public void notAnSpsOrBroken() { + // PPS + assertNull(size("68ebe3cb22c0", false)); + // H.265 VPS + assertNull(size("40010c01ffff016000000300900000030000030078959809", true)); + // Truncated + assertNull(size(H264_2304.substring(0, 16), false)); + assertNull(size(H265_2304.substring(0, 40), true)); + // Codec mixed up + assertNull(size(H264_1080, true)); + assertNull(SpsParser.size(null, 0, 0, false)); + byte[] b = hex(H264_720); + assertNull(SpsParser.size(b, 4, b.length, false)); + } + + /** A baseline H.264 SPS with the given size fields (Exp-Golomb values as written). */ + private static int[] h264Sps(long widthMbsMinus1, long heightMinus1, long[] crop) { + StringBuilder bits = new StringBuilder(); + bits.append(bin(0x67, 8)).append(bin(66, 8)).append(bin(0, 8)).append(bin(31, 8)); + ue(bits, 0); // seq_parameter_set_id + ue(bits, 0); // log2_max_frame_num_minus4 + ue(bits, 2); // pic_order_cnt_type + ue(bits, 1); // max_num_ref_frames + bits.append('0'); // gaps_in_frame_num_value_allowed_flag + ue(bits, widthMbsMinus1); + ue(bits, heightMinus1); + bits.append("11"); // frame_mbs_only_flag, direct_8x8_inference_flag + if (crop != null) { + bits.append('1'); + for (long c : crop) ue(bits, c); + } else { + bits.append('0'); + } + bits.append("01"); // no VUI, then the stop bit + while (bits.length() % 8 != 0) bits.append('0'); + byte[] b = new byte[bits.length() / 8]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(bits.substring(8 * i, 8 * i + 8), 2); + } + return SpsParser.size(b, 0, b.length, false); + } + + private static String bin(long v, int n) { + StringBuilder s = new StringBuilder(Long.toBinaryString(v)); + while (s.length() < n) s.insert(0, '0'); + return s.toString(); + } + + private static void ue(StringBuilder bits, long v) { + String code = Long.toBinaryString(v + 1); + for (int i = 1; i < code.length(); i++) bits.append('0'); + bits.append(code); + } + + @Test + public void malformedSizesDoNotWrapRound() { + // The writer itself: 1280×720 + assertArrayEquals(new int[]{1280, 720}, h264Sps(79, 44, null)); + // 2^28 + 1 macroblocks wide is 2^32 + 16 pixels: 16 in 32-bit arithmetic (Qodo on #220) + assertNull(h264Sps(1L << 28, 44, null)); + // Crops larger than the picture, and one that overflows an int when doubled + assertNull(h264Sps(79, 44, new long[]{700, 0, 0, 0})); + assertNull(h264Sps(79, 44, new long[]{1L << 31, 0, 0, 0})); + // A sane crop still works: 1920×1088 coded, 1080 shown + assertArrayEquals(new int[]{1920, 1080}, h264Sps(119, 67, new long[]{0, 0, 0, 4})); + } + + @Test + public void emulationPreventionBytesAreRemoved() { + assertArrayEquals(new byte[]{0, 0, 1, 0, 0, 0}, + SpsParser.unescape(new byte[]{0, 0, 3, 1, 0, 0, 3, 0}, 0, 8)); + // 00 00 03 03: only the first 03 goes + assertArrayEquals(new byte[]{0, 0, 3}, + SpsParser.unescape(new byte[]{0, 0, 3, 3}, 0, 4)); + } +} diff --git a/docs/debug.md b/docs/debug.md index 58de744..86cfc63 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -122,7 +122,9 @@ One entry per video decoder and MIME type (encoders are left out). | `transport_mode` | the slot's setting: `TCP`, `UDP`, `AUTO`, `MULTICAST` | | `video` | a frame was shown within the last 2 s | | `status` | the status text on screen, or the session state | -| `size` | `WxH`, once known | +| `size` | `WxH` of the picture shown, only while `video` is true | +| `stream_size` | `WxH` the camera sends, from the SPS (H.264, H.265) | +| `decoder` | the decoder in use, e.g. `OMX.rk.video_decoder.avc` | | `codec` | `H.264`, `H.265` or `MJPEG` | | `audio` | audio codec: `NONE`, `AAC`, `LATM`, `OPUS`, `PCMU`, `PCMA`, `PCM` | | `out_fps` | frames shown in the last second |