From 7942938121be7ee5608e096949bcf221af58edc8 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 13:02:44 +0000 Subject: [PATCH 1/9] fix: single view tells the decoder the largest picture size, for streams above 1080p (#166) --- .../java/com/openipc/decoder/Decoder.java | 2 ++ .../openipc/decoder/MediaCodecManager.java | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index c43c93a0..6bfa8e29 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -213,6 +213,8 @@ protected void onCreate(Bundle savedInstanceState) { @Override public void onDecoderStarted() { removeClearOverlay(); } @Override public void onDecoderFailed() {} }); + // Single view: streams above 1080p on decoders that need to know it up front (#166) + videoDecoder.allowLargeFrames(true); audioManager = new AudioManager(); rtspClient = new RtspClient(framePool, nalAssembler, nalQueue); rtspClient.setListener(new RtspClient.Listener() { diff --git a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java index f75d6ca0..4c5acbec 100644 --- a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java +++ b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java @@ -56,6 +56,13 @@ class MediaCodecManager { // Surface set by the owning activity private volatile Surface surface; + /** + * Single view (#166): announce the largest picture the decoder supports (up to 4K) as + * KEY_MAX_WIDTH/HEIGHT. Configured at 1280×720 without it, some hardware decoders + * (Rockchip OMX) never give a frame for streams above 1080p, e.g. 2304×1296. Off in the + * grid: nine decoders with 4K buffers would not fit in memory. + */ + private volatile boolean largeFrames; /** Frames rendered so far (#82): with the frames received this gives the real decode rate. */ private volatile int framesOut; @@ -255,6 +262,27 @@ void setSurface(Surface s) { this.surface = s; } + /** See {@link #largeFrames}; takes effect with the next decoder. */ + void allowLargeFrames(boolean on) { + largeFrames = on; + } + + /** Largest supported size up to 3840×2160 (a few steps down), or null if none above 1080p. */ + static int[] maxFrameSize(MediaCodec codec, String mime) { + try { + MediaCodecInfo.VideoCapabilities video = + codec.getCodecInfo().getCapabilitiesForType(mime).getVideoCapabilities(); + if (video == null) return null; + int[][] sizes = {{3840, 2160}, {2560, 1920}, {2560, 1440}, {2304, 1296}}; + for (int[] size : sizes) { + if (video.isSizeSupported(size[0], size[1])) return size; + } + } catch (Exception ignored) { + // Capabilities unknown: keep the decoder's defaults + } + return null; + } + /** Enable or disable decoding; disabling also releases the decoder. */ /** * Enable or disable decoding. Never blocks the caller (the UI thread): disabling only @@ -374,6 +402,13 @@ void createDecoder(boolean codecH265) { Log.i(TAG, "Start video decoder (" + type + ")"); local = MediaCodec.createDecoderByType(type); learnLimits(local, type); + int[] max = largeFrames ? maxFrameSize(local, type) : null; + if (max != null) { + format.setInteger(MediaFormat.KEY_MAX_WIDTH, max[0]); + format.setInteger(MediaFormat.KEY_MAX_HEIGHT, max[1]); + Log.i(TAG, "Decoder " + local.getName() + ": pictures up to " + + max[0] + "×" + max[1]); + } } try { local.configure(format, s, null, 0); From 6dd18a2887f867bbf4f6486e18f9f2eb87bc472e Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 13:02:44 +0000 Subject: [PATCH 2/9] chore: version 2.14 --- android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index 8fadf408..1e402507 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 38 - versionName "2.13" + versionCode 39 + versionName "2.14" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From 23e06caf802ae7586518d1963c04fb270b07ce7a Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 16:15:48 +0000 Subject: [PATCH 3/9] chore: version 2.16 (2.15 went to #215) --- android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index 53d4226c..a45a4439 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 40 - versionName "2.15" + versionCode 41 + versionName "2.16" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From 0109a574e8366d37595b817b61426964553693e4 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Fri, 16 Oct 2026 19:16:42 +0000 Subject: [PATCH 4/9] chore: version 2.17 (2.16 went to #224) --- android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index a45a4439..f6bbe65f 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 41 - versionName "2.16" + versionCode 42 + versionName "2.17" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From f736851cba116cea59f4af94a1e66686c56195f9 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Sat, 17 Oct 2026 17:13:11 +0000 Subject: [PATCH 5/9] chore: version 2.18 (2.17 went to #225) --- android/app/build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/android/app/build.gradle b/android/app/build.gradle index f6bbe65f..d572c7df 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -23,8 +23,8 @@ android { applicationId "com.openipc.decoder" minSdk 21 targetSdk 36 - versionCode 42 - versionName "2.17" + versionCode 43 + versionName "2.18" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From 3fed00849b8cf7905b2f7e257c311eee5cca176e Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 19 Oct 2026 21:13:42 +0000 Subject: [PATCH 6/9] fix: single view sets the decoder up for the stream's size from its SPS (#166) The test build of PR #220 did not help on the Rockchip HY300: its OMX decoders declare no size above 1080p, so KEY_MAX_WIDTH/HEIGHT was never set. Now the single view waits for the SPS, reads the picture size from it (SpsParser, H.264 and H.265, cropping included) and configures the decoder for that size, KEY_MAX_* covering it. A larger SPS mid-stream gets a new decoder. A hardware decoder at a size it does not declare that gives no frame within 5 s after its parameter sets hands the stream to the software decoder. /stats.json: size only while a picture is shown (it kept the size of an earlier stream), plus stream_size (from the SPS) and decoder. --- .../java/com/openipc/decoder/Decoder.java | 14 +- .../openipc/decoder/MediaCodecManager.java | 272 ++++++++++++++---- .../java/com/openipc/decoder/SpsParser.java | 187 ++++++++++++ .../com/openipc/decoder/SpsParserTest.java | 85 ++++++ docs/debug.md | 4 +- 5 files changed, 509 insertions(+), 53 deletions(-) create mode 100644 android/app/src/main/java/com/openipc/decoder/SpsParser.java create mode 100644 android/app/src/test/java/com/openipc/decoder/SpsParserTest.java diff --git a/android/app/src/main/java/com/openipc/decoder/Decoder.java b/android/app/src/main/java/com/openipc/decoder/Decoder.java index b166adce..d52ccce3 100755 --- a/android/app/src/main/java/com/openipc/decoder/Decoder.java +++ b/android/app/src/main/java/com/openipc/decoder/Decoder.java @@ -1610,11 +1610,19 @@ private Map debugState() { rtspClient != null ? rtspClient.transportInUse() : null, mTransports[mActive])); single.put("transport_mode", mTransports[mActive] != null ? mTransports[mActive].name() : null); - single.put("video", mResumed && "connected".equals(singleState) - && singleRenderedWithin(SNAPSHOT_LIVE_MS)); + boolean live = mResumed && "connected".equals(singleState) + && singleRenderedWithin(SNAPSHOT_LIVE_MS); + single.put("video", live); single.put("status", statusText != null && statusText.getVisibility() == View.VISIBLE ? statusText.getText().toString() : singleState); - if (mVideoWidth > 0) single.put("size", mVideoWidth + "x" + mVideoHeight); + // The decoded picture's size, only while it is shown: otherwise it is the size of + // an earlier stream (#166) + if (live && mVideoWidth > 0) single.put("size", mVideoWidth + "x" + mVideoHeight); + // What the camera sends, from the SPS, and the decoder taking it (#166) + int[] streamSize = videoDecoder.streamSize(); + if (streamSize != null) single.put("stream_size", streamSize[0] + "x" + streamSize[1]); + String decoderName = videoDecoder.decoderName(); + if (decoderName != null) single.put("decoder", decoderName); single.put("codec", rtspClient != null && rtspClient.isJpeg() ? "MJPEG" : codecH265 ? "H.265" : "H.264"); single.put("audio", audioCodec); diff --git a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java index d6fac2d4..6f13f037 100644 --- a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java +++ b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java @@ -57,12 +57,35 @@ class MediaCodecManager { // Surface set by the owning activity private volatile Surface surface; /** - * Single view (#166): announce the largest picture the decoder supports (up to 4K) as - * KEY_MAX_WIDTH/HEIGHT. Configured at 1280×720 without it, some hardware decoders - * (Rockchip OMX) never give a frame for streams above 1080p, e.g. 2304×1296. Off in the - * grid: nine decoders with 4K buffers would not fit in memory. + * Single view (#166): the decoder is configured for the stream's size from its SPS, and + * KEY_MAX_WIDTH/HEIGHT covers it. Configured at 1280×720, some hardware decoders (Rockchip + * OMX) never give a frame for streams above 1080p, e.g. 2304×1296. Off in the grid: nine + * decoders with large buffers would not fit in memory. */ private volatile boolean largeFrames; + /** Picture size from the last SPS (largeFrames only); 0 while unknown. */ + private volatile int streamWidth; + private volatile int streamHeight; + /** An SPS arrived (decoderLock): with largeFrames no decoder is created before one. */ + private boolean spsSeen; + /** The last H.265 VPS (decoderLock): a decoder created at the SPS needs it first. */ + private byte[] lastVps; + /** Largest picture the current decoder was set up for; 0 when not known. */ + private int confWidth; + private int confHeight; + /** + * The hardware decoder runs at a size it does not declare: without a frame by + * {@link #probationUntilMs} the stream goes to a software decoder (#166). + */ + private boolean probation; + private long probationUntilMs; + private int probationFrames; + /** How long a decoder on probation may go without a frame once it has its parameter sets. */ + static final long PROBATION_MS = 5_000; + /** "mime WxH" of a stream the hardware decoder gave no frame for: software from then on. */ + private volatile String softwareFor; + /** Name of the current decoder, for /stats.json. */ + private volatile String decoderName; /** Degrees counter-clockwise the decoder turns the picture (#201); 0 for none. */ private volatile int rotation; /** The turn the current decoder applies: {@link #rotation} where Android 6+ can, else 0. */ @@ -112,6 +135,13 @@ boolean swapsSides() { return t == 90 || t == 270; } boolean hasDecoder() { return decoder != null; } + /** Picture size from the stream's SPS ({width, height}), or null while unknown (#166). */ + int[] streamSize() { + int w = streamWidth, h = streamHeight; + return w > 0 && h > 0 ? new int[]{w, h} : null; + } + /** Name of the decoder in use, or null. */ + String decoderName() { return decoder != null ? decoderName : null; } /** * Decoders alive in this process per MIME type, against the declared limit of the codec @@ -292,6 +322,22 @@ static int[] maxFrameSize(MediaCodec codec, String mime) { return null; } + /** The codec declares the size; unknown capabilities count as declared. */ + static boolean declaresSize(MediaCodec codec, String mime, int width, int height) { + try { + MediaCodecInfo.VideoCapabilities video = + codec.getCodecInfo().getCapabilitiesForType(mime).getVideoCapabilities(); + return video == null || video.isSizeSupported(width, height); + } catch (Exception e) { + return true; + } + } + + /** Above 1080p: a size some hardware decoders cannot be left to discover (#166). */ + static boolean aboveFullHd(int width, int height) { + return (long) width * height > 1920L * 1088; + } + /** See {@link #rotation}; takes effect with the next decoder. */ void setRotation(int degreesCcw) { rotation = ((degreesCcw % 360) + 360) % 360; @@ -347,6 +393,8 @@ private void closeDecoderLocked() { MediaCodec codec = decoder; // The next stream must not look live because of this one's last frame lastFrameMs = 0; + // Nor be set up for this one's size (#166) + forgetStream(); if (codec == null) return; decoder = null; String slot = takeSlot(); @@ -361,6 +409,15 @@ private void closeDecoderLocked() { } } + /** Stream size and parameter sets of the last stream (decoderLock held). */ + private void forgetStream() { + spsSeen = false; + streamWidth = 0; + streamHeight = 0; + lastVps = null; + probation = false; + } + private static long nowMs() { return System.nanoTime() / 1_000_000; } @@ -380,12 +437,26 @@ void createDecoder(boolean codecH265) { String type = codecH265 ? "video/hevc" : "video/avc"; - MediaFormat format = MediaFormat.createVideoFormat(type, 1280, 720); + // Single view: the stream's own size from its SPS (#166) + final int width = largeFrames ? streamWidth : 0; + final int height = largeFrames ? streamHeight : 0; + final boolean sized = width > 0 && height > 0; + MediaFormat format = MediaFormat.createVideoFormat(type, + sized ? width : 1280, sized ? height : 720); // Must hold the largest NAL unit the assembler can deliver format.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, MAX_NAL_SIZE); String softwareName = null; - if (!InstanceBudget.tryAcquire(type)) { + boolean noHardwareFrame = false; + if (sized && sizeKey(type, width, height).equals(softwareFor)) { + // The hardware decoder gave no frame at this size: a software one, if it has room + String sw = softwareDecoder(type); + if (sw != null && InstanceBudget.tryAcquire(softwareSlot(type))) { + softwareName = sw; + noHardwareFrame = true; + } + } + if (softwareName == null && !InstanceBudget.tryAcquire(type)) { int limit = InstanceBudget.limit(type); softwareName = limit >= 0 && allowSoftware ? softwareDecoder(type) : null; // Software decoders have their own limit: beyond it, the hardware one is reported @@ -407,22 +478,38 @@ void createDecoder(boolean codecH265) { final String slot = software ? softwareSlot(type) : type; MediaCodec local; + boolean onProbation = false; + int maxW = 0, maxH = 0; try { if (software) { - Log.i(TAG, "Start software video decoder " + softwareName - + " (hardware limit reached, key frames only)"); + Log.i(TAG, "Start software video decoder " + softwareName + (noHardwareFrame + ? " (no frame from the hardware one at " + width + "×" + height + ")" + : " (hardware limit reached, key frames only)")); local = MediaCodec.createByCodecName(softwareName); } else { Log.i(TAG, "Start video decoder (" + type + ")"); local = MediaCodec.createDecoderByType(type); learnLimits(local, type); - int[] max = largeFrames ? maxFrameSize(local, type) : null; - if (max != null) { - format.setInteger(MediaFormat.KEY_MAX_WIDTH, max[0]); - format.setInteger(MediaFormat.KEY_MAX_HEIGHT, max[1]); - Log.i(TAG, "Decoder " + local.getName() + ": pictures up to " - + max[0] + "×" + max[1]); + } + if (largeFrames) { + // Room for the stream and for the largest declared size, so the picture may + // still grow without a new decoder + int[] max = maxFrameSize(local, type); + maxW = Math.max(sized ? width : 0, max != null ? max[0] : 0); + maxH = Math.max(sized ? height : 0, max != null ? max[1] : 0); + if (maxW > 0 && maxH > 0) { + format.setInteger(MediaFormat.KEY_MAX_WIDTH, maxW); + format.setInteger(MediaFormat.KEY_MAX_HEIGHT, maxH); } + boolean declared = !sized || declaresSize(local, type, width, height); + // Probation only where a software decoder can take over, and once per size + onProbation = !software && sized && !declared && aboveFullHd(width, height) + && !sizeKey(type, width, height).equals(softwareFor) + && softwareDecoder(type) != null; + Log.i(TAG, "Decoder " + local.getName() + ": stream " + + (sized ? width + "×" + height : "size unknown") + + (maxW > 0 ? ", pictures up to " + maxW + "×" + maxH : "") + + (declared ? "" : ", a size it does not declare")); } int turn = rotation; if (turn != 0 && Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) { @@ -465,9 +552,19 @@ void createDecoder(boolean codecH265) { softwareInUse = software; awaitingConfig = true; configSeen = 0; + decoderName = local.getName(); + confWidth = maxW; + confHeight = maxH; + probation = onProbation; + probationUntilMs = 0; + probationFrames = framesOut; } } + private static String sizeKey(String mime, int width, int height) { + return mime + " " + width + "x" + height; + } + /** * Decode one NAL frame. Returns true if there is no decoder and one may be created now: * the caller then creates it and passes the same frame again. @@ -487,66 +584,143 @@ boolean decodeFrame(Frame buffer, boolean codecH265, long watchdogRef) { flag = MediaCodec.BUFFER_FLAG_CODEC_CONFIG; } + boolean isSps = fragment == (codecH265 ? H265_NAL_SPS : H264_NAL_SPS); + boolean isVps = codecH265 && fragment == H265_NAL_VPS; + // Single view: the stream's size, before any decoder is set up for it (#166) + int[] spsSize = largeFrames && isSps + ? SpsParser.size(buffer.data(), 4, buffer.length() - 4, codecH265) : null; + boolean needCreate = false; synchronized (decoderLock) { MediaCodec codec = decoder; + if (largeFrames) { + if (isSps) { + spsSeen = true; + if (spsSize != null) { + streamWidth = spsSize[0]; + streamHeight = spsSize[1]; + } + } else if (isVps) { + lastVps = java.util.Arrays.copyOf(buffer.data(), buffer.length()); + } + } if (!enabled) { // Paused: the decoder is being released, feed nothing return false; } else if (codec == null) { + // Single view: the decoder waits for the SPS, which gives its size (#166) + if (largeFrames && !spsSeen) return false; needCreate = retry.mayCreate(nowMs()); } else if (awaitingConfig && !isConfigNal) { // Frames before the full parameter set only produce errors in a new decoder return false; + } else if (largeFrames && spsSize != null && confWidth > 0 + && (spsSize[0] > confWidth || spsSize[1] > confHeight)) { + // The picture outgrew the decoder: a new one set up for it takes this SPS + Log.i(TAG, "Stream " + spsSize[0] + "×" + spsSize[1] + " is larger than " + + confWidth + "×" + confHeight + ": new decoder"); + dropDecoder(codec); + return retry.mayCreate(nowMs()); + } else if (probation && probationExpired()) { + Log.w(TAG, "No frame from " + decoderName + " at " + streamWidth + "×" + + streamHeight + " in " + PROBATION_MS / 1000 + " s: software decoder"); + softwareFor = sizeKey(codecH265 ? "video/hevc" : "video/avc", + streamWidth, streamHeight); + dropDecoder(codec); + // The next decoder starts at the next SPS, which comes with a key frame + return false; } else { if (isConfigNal) { - configSeen |= 1 << fragment; int needed = codecH265 ? (1 << H265_NAL_VPS) | (1 << H265_NAL_SPS) | (1 << H265_NAL_PPS) : (1 << H264_NAL_SPS) | (1 << H264_NAL_PPS); - if ((configSeen & needed) == needed) awaitingConfig = false; - } - try { - // No free input buffer at once: give decoded frames back (the decoder may - // be waiting for output space) and try again, for up to INPUT_WAIT_MS. - // Dropping a NAL unit corrupts the picture until the next key frame (#89) - int inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); - long waitUntil = nowMs() + INPUT_WAIT_MS; - // A close or a pause waiting for the lock ends the wait at once (UI thread) - while (inputBufferId < 0 && nowMs() < waitUntil && enabled - && closeWaiting.get() == 0) { - drainOutput(codec); - inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); - } - if (inputBufferId >= 0) { - ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); - if (inputBuffer != null) { - inputBuffer.clear(); - inputBuffer.put(buffer.data(), 0, buffer.length()); - codec.queueInputBuffer(inputBufferId, 0, - buffer.length(), System.nanoTime() / 1000, flag); + // A decoder created at the SPS has missed the VPS before it (#166) + if (isSps && awaitingConfig && (configSeen & (1 << H265_NAL_VPS)) == 0 + && codecH265 && lastVps != null) { + configSeen |= 1 << H265_NAL_VPS; + try { + feed(codec, lastVps, lastVps.length, MediaCodec.BUFFER_FLAG_CODEC_CONFIG); + } catch (Exception e) { + failDecoder(codec, e); + return false; } - } else if (enabled && closeWaiting.get() == 0) { - // Only a full wait is a decoder loss; a close or pause cut it short - inputDropped++; } - drainOutput(codec); + configSeen |= 1 << fragment; + if ((configSeen & needed) == needed && awaitingConfig) { + awaitingConfig = false; + // The probation runs from the first key frame on + if (probation) probationUntilMs = nowMs() + PROBATION_MS; + } + } + try { + feed(codec, buffer.data(), buffer.length(), flag); } catch (Exception e) { - decoder = null; - String slot = takeSlot(); - retry.onFailure(nowMs()); - Log.e(TAG, "Codec exception, retry in " - + DecoderRetry.delayMs(retry.failures()) + " ms: " + e.getMessage()); - try { codec.stop(); } catch (Exception ignored) {} - try { codec.release(); } catch (Exception ignored) {} - releaseSlot(slot); - if (listener != null) listener.onDecoderFailed(); + failDecoder(codec, e); } } } return needCreate; } + /** + * Queue one NAL unit (decoderLock held). No free input buffer at once: give decoded frames + * back (the decoder may be waiting for output space) and try again, for up to + * INPUT_WAIT_MS. Dropping a NAL unit corrupts the picture until the next key frame (#89). + */ + private void feed(MediaCodec codec, byte[] data, int length, int flag) { + int inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); + long waitUntil = nowMs() + INPUT_WAIT_MS; + // A close or a pause waiting for the lock ends the wait at once (UI thread) + while (inputBufferId < 0 && nowMs() < waitUntil && enabled + && closeWaiting.get() == 0) { + drainOutput(codec); + inputBufferId = codec.dequeueInputBuffer(INPUT_POLL_US); + } + if (inputBufferId >= 0) { + ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); + if (inputBuffer != null) { + inputBuffer.clear(); + inputBuffer.put(data, 0, length); + codec.queueInputBuffer(inputBufferId, 0, length, System.nanoTime() / 1000, flag); + } + } else if (enabled && closeWaiting.get() == 0) { + // Only a full wait is a decoder loss; a close or pause cut it short + inputDropped++; + } + drainOutput(codec); + } + + /** The decoder failed while decoding (decoderLock held): release it, back off. */ + private void failDecoder(MediaCodec codec, Exception e) { + decoder = null; + String slot = takeSlot(); + retry.onFailure(nowMs()); + Log.e(TAG, "Codec exception, retry in " + + DecoderRetry.delayMs(retry.failures()) + " ms: " + e.getMessage()); + releaseNow(codec); + releaseSlot(slot); + if (listener != null) listener.onDecoderFailed(); + } + + /** Release a working decoder to replace it (decoderLock held): no back-off. */ + private void dropDecoder(MediaCodec codec) { + decoder = null; + probation = false; + String slot = takeSlot(); + releaseNow(codec); + releaseSlot(slot); + } + + /** The decoder on probation has its parameter sets but no frame past the deadline. */ + private boolean probationExpired() { + if (framesOut != probationFrames) { + // A frame came: the hardware decoder manages the size + probation = false; + return false; + } + return probationUntilMs > 0 && nowMs() > probationUntilMs; + } + /** * The picture's size: the crop rectangle when the decoder gives one. KEY_WIDTH / KEY_HEIGHT * are the buffer, padded to the decoder's alignment (1080 → 1088; one H.265 decoder made diff --git a/android/app/src/main/java/com/openipc/decoder/SpsParser.java b/android/app/src/main/java/com/openipc/decoder/SpsParser.java new file mode 100644 index 00000000..2ea892db --- /dev/null +++ b/android/app/src/main/java/com/openipc/decoder/SpsParser.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * SpsParser.java — picture size from an H.264 / H.265 sequence parameter set + * + */ + +package com.openipc.decoder; + +/** + * Reads the visible picture size (after the cropping / conformance window) from an SPS, so a + * decoder can be configured for the stream's real size before the first frame (#166). + */ +final class SpsParser { + private SpsParser() {} + + /** + * Size {width, height} from one NAL unit (no start code, header included), or null if it + * is not an SPS or cannot be read. + */ + static int[] size(byte[] nal, int offset, int length, boolean h265) { + if (nal == null || length < (h265 ? 3 : 2) || offset < 0 + || offset + length > nal.length) return null; + try { + Bits b = new Bits(unescape(nal, offset, length)); + return h265 ? h265(b) : h264(b); + } catch (RuntimeException e) { + // Truncated or malformed: the size stays unknown + return null; + } + } + + private static int[] h264(Bits b) { + if ((b.u(8) & 0x1f) != MediaCodecManager.H264_NAL_SPS) return null; + int profile = b.u(8); + b.u(8); // constraint flags + b.u(8); // level + b.ue(); // seq_parameter_set_id + int chroma = 1; + boolean separatePlanes = false; + if (profile == 100 || profile == 110 || profile == 122 || profile == 244 + || profile == 44 || profile == 83 || profile == 86 || profile == 118 + || profile == 128 || profile == 138 || profile == 139 || profile == 134 + || profile == 135) { + chroma = b.ue(); + if (chroma == 3) separatePlanes = b.u(1) == 1; + b.ue(); // bit_depth_luma_minus8 + b.ue(); // bit_depth_chroma_minus8 + b.u(1); // qpprime_y_zero_transform_bypass_flag + if (b.u(1) == 1) { + for (int i = 0; i < (chroma != 3 ? 8 : 12); i++) { + if (b.u(1) == 1) skipScalingList(b, i < 6 ? 16 : 64); + } + } + } + b.ue(); // log2_max_frame_num_minus4 + int pocType = b.ue(); + if (pocType == 0) { + b.ue(); // log2_max_pic_order_cnt_lsb_minus4 + } else if (pocType == 1) { + b.u(1); // delta_pic_order_always_zero_flag + b.se(); // offset_for_non_ref_pic + b.se(); // offset_for_top_to_bottom_field + int n = b.ue(); + for (int i = 0; i < n; i++) b.se(); + } + b.ue(); // max_num_ref_frames + b.u(1); // gaps_in_frame_num_value_allowed_flag + int widthMbs = b.ue() + 1; + int heightUnits = b.ue() + 1; + int frameMbsOnly = b.u(1); + if (frameMbsOnly == 0) b.u(1); // mb_adaptive_frame_field_flag + b.u(1); // direct_8x8_inference_flag + int width = widthMbs * 16; + int height = (2 - frameMbsOnly) * heightUnits * 16; + if (b.u(1) == 1) { + int left = b.ue(), right = b.ue(), top = b.ue(), bottom = b.ue(); + int arrayType = separatePlanes ? 0 : chroma; + int cropX = arrayType == 1 || arrayType == 2 ? 2 : 1; + int cropY = (arrayType == 1 ? 2 : 1) * (2 - frameMbsOnly); + width -= cropX * (left + right); + height -= cropY * (top + bottom); + } + return valid(width, height); + } + + private static void skipScalingList(Bits b, int size) { + int last = 8, next = 8; + for (int j = 0; j < size; j++) { + if (next != 0) next = (last + b.se() + 256) % 256; + last = next == 0 ? last : next; + } + } + + private static int[] h265(Bits b) { + if (((b.u(8) >> 1) & 0x3f) != MediaCodecManager.H265_NAL_SPS) return null; + b.u(8); // rest of the NAL header + b.u(4); // sps_video_parameter_set_id + int subLayers = b.u(3); // sps_max_sub_layers_minus1 + b.u(1); // sps_temporal_id_nesting_flag + // profile_tier_level(1, subLayers): general part, 96 bits + b.skip(96); + boolean[] profile = new boolean[subLayers]; + boolean[] level = new boolean[subLayers]; + for (int i = 0; i < subLayers; i++) { + profile[i] = b.u(1) == 1; + level[i] = b.u(1) == 1; + } + if (subLayers > 0) b.skip(2 * (8 - subLayers)); + for (int i = 0; i < subLayers; i++) { + if (profile[i]) b.skip(88); + if (level[i]) b.skip(8); + } + b.ue(); // sps_seq_parameter_set_id + int chroma = b.ue(); + boolean separatePlanes = chroma == 3 && b.u(1) == 1; + int width = b.ue(); + int height = b.ue(); + if (b.u(1) == 1) { + int left = b.ue(), right = b.ue(), top = b.ue(), bottom = b.ue(); + int arrayType = separatePlanes ? 0 : chroma; + int subW = arrayType == 1 || arrayType == 2 ? 2 : 1; + int subH = arrayType == 1 ? 2 : 1; + width -= subW * (left + right); + height -= subH * (top + bottom); + } + return valid(width, height); + } + + private static int[] valid(int width, int height) { + // Beyond 8K or empty: not a size worth configuring a decoder for + return width > 0 && height > 0 && width <= 8192 && height <= 8192 + ? new int[]{width, height} : null; + } + + /** The NAL unit without its emulation prevention bytes (00 00 03 → 00 00). */ + static byte[] unescape(byte[] data, int offset, int length) { + byte[] out = new byte[length]; + int n = 0, zeros = 0; + for (int i = offset; i < offset + length; i++) { + int v = data[i] & 0xff; + if (zeros >= 2 && v == 3) { + zeros = 0; + continue; + } + zeros = v == 0 ? zeros + 1 : 0; + out[n++] = (byte) v; + } + return java.util.Arrays.copyOf(out, n); + } + + /** MSB-first bit reader with Exp-Golomb codes; reading past the end throws. */ + private static final class Bits { + private final byte[] data; + private int pos; + + Bits(byte[] data) { this.data = data; } + + int u(int n) { + int v = 0; + for (int i = 0; i < n; i++) { + if (pos >= data.length * 8) throw new IllegalStateException("SPS truncated"); + v = (v << 1) | ((data[pos >> 3] >> (7 - (pos & 7))) & 1); + pos++; + } + return v; + } + + void skip(int n) { + if (pos + n > data.length * 8) throw new IllegalStateException("SPS truncated"); + pos += n; + } + + int ue() { + int zeros = 0; + while (u(1) == 0) { + if (++zeros > 31) throw new IllegalStateException("bad Exp-Golomb code"); + } + return zeros == 0 ? 0 : (int) ((1L << zeros) - 1 + u(zeros)); + } + + int se() { + int k = ue(); + return (k & 1) == 1 ? (k + 1) / 2 : -(k / 2); + } + } +} diff --git a/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java b/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java new file mode 100644 index 00000000..feeace1b --- /dev/null +++ b/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java @@ -0,0 +1,85 @@ +/* + * Copyright (c) OpenIPC https://openipc.org Prosperity Public License 3.0.0 + * + * SpsParserTest.java — picture size from H.264 / H.265 SPS (#166) + * + */ + +package com.openipc.decoder; + +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +public class SpsParserTest { + // SPS from ffmpeg 6.1 (libx264 / libx265), testsrc, emulation prevention bytes included + private static final String H264_2304 = "67640032acd9402400a3b011000003000100000300320f183196"; + private static final String H264_1080 = "674d4028eca03c0113f2e022000003000200000300641e30632c"; + private static final String H264_720 = "6742c01fd9005005bb011000000300100000030320f1832480"; + private static final String H264_444 = "67f40028919b280f0044fc4e0220000003002000000641e30632c0"; + private static final String H264_INTERLACED = + "67640028acd94078044fde0220000003002000000643e2c5b2c0"; + private static final String H265_2304 = "420101016000000300900000030000030096a0012020051165959a" + + "4932bc05a02000000300200000030321"; + private static final String H265_1080 = "420101016000000300900000030000030078a003c08010e5965669" + + "24caf0168080000003008000000c84"; + + private static int[] size(String hex, boolean h265) { + byte[] b = hex(hex); + return SpsParser.size(b, 0, b.length, h265); + } + + private static byte[] hex(String s) { + byte[] b = new byte[s.length() / 2]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(s.substring(2 * i, 2 * i + 2), 16); + } + return b; + } + + @Test + public void h264Sizes() { + // High, 2304×1296: 144×81 macroblocks, no cropping + assertArrayEquals(new int[]{2304, 1296}, size(H264_2304, false)); + // Main, 1080p: 1088 coded, cropped to 1080 + assertArrayEquals(new int[]{1920, 1080}, size(H264_1080, false)); + assertArrayEquals(new int[]{1280, 720}, size(H264_720, false)); + // High 4:4:4: crop units of one line + assertArrayEquals(new int[]{1920, 1080}, size(H264_444, false)); + // Interlaced: map units are field macroblock pairs + assertArrayEquals(new int[]{1920, 1080}, size(H264_INTERLACED, false)); + } + + @Test + public void h265Sizes() { + assertArrayEquals(new int[]{2304, 1296}, size(H265_2304, true)); + // Coded 1920×1088 with a conformance window + assertArrayEquals(new int[]{1920, 1080}, size(H265_1080, true)); + } + + @Test + public void notAnSpsOrBroken() { + // PPS + assertNull(size("68ebe3cb22c0", false)); + // H.265 VPS + assertNull(size("40010c01ffff016000000300900000030000030078959809", true)); + // Truncated + assertNull(size(H264_2304.substring(0, 16), false)); + assertNull(size(H265_2304.substring(0, 40), true)); + // Codec mixed up + assertNull(size(H264_1080, true)); + assertNull(SpsParser.size(null, 0, 0, false)); + byte[] b = hex(H264_720); + assertNull(SpsParser.size(b, 4, b.length, false)); + } + + @Test + public void emulationPreventionBytesAreRemoved() { + assertArrayEquals(new byte[]{0, 0, 1, 0, 0, 0}, + SpsParser.unescape(new byte[]{0, 0, 3, 1, 0, 0, 3, 0}, 0, 8)); + // 00 00 03 03: only the first 03 goes + assertArrayEquals(new byte[]{0, 0, 3}, + SpsParser.unescape(new byte[]{0, 0, 3, 3}, 0, 4)); + } +} diff --git a/docs/debug.md b/docs/debug.md index 083d0efd..f4d03fed 100644 --- a/docs/debug.md +++ b/docs/debug.md @@ -122,7 +122,9 @@ One entry per video decoder and MIME type (encoders are left out). | `transport_mode` | the slot's setting: `TCP`, `UDP`, `AUTO`, `MULTICAST` | | `video` | a frame was shown within the last 2 s | | `status` | the status text on screen, or the session state | -| `size` | `WxH`, once known | +| `size` | `WxH` of the picture shown, only while `video` is true | +| `stream_size` | `WxH` the camera sends, from the SPS (H.264, H.265) | +| `decoder` | the decoder in use, e.g. `OMX.rk.video_decoder.avc` | | `codec` | `H.264`, `H.265` or `MJPEG` | | `audio` | audio codec: `NONE`, `AAC`, `LATM`, `OPUS`, `PCMU`, `PCMA`, `PCM` | | `out_fps` | frames shown in the last second | From 35f2bc892db31ffeff8889ce69994d9aacdf2e0e Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Mon, 19 Oct 2026 21:13:43 +0000 Subject: [PATCH 7/9] chore: version 2.17.1 --- CHANGELOG.md | 8 ++++++++ android/app/build.gradle | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2375ab38..62fd2c19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,14 @@ Versions after 2.17 are `MAJOR.MINOR.PATCH` (2.17 is followed by 2.17.1 or 2.18. `docs/testing.md`, "Versions and releases"); the headings below keep the numbers as they were released. +## 2.17.1 — 2026-10-19 + +- Single view: the decoder is set up for the stream's size from its SPS, so streams above + 1080p (e.g. 2304×1296) play on hardware decoders that cannot find the size themselves; a + hardware decoder that gives no frame at a size it does not declare hands the stream to a + software one after 5 s. `/stats.json` shows `size` only while a picture is shown, and adds + `stream_size` and `decoder` (PR #220, #166). + ## 2.17 — 2026-10-17 - Turn in the camera settings (Auto, 0°, 90°, 180°, 270°), stored per slot; H.264/H.265 are diff --git a/android/app/build.gradle b/android/app/build.gradle index d572c7df..d64621de 100755 --- a/android/app/build.gradle +++ b/android/app/build.gradle @@ -24,7 +24,7 @@ android { minSdk 21 targetSdk 36 versionCode 43 - versionName "2.18" + versionName "2.17.1" buildConfigField "String", "GIT_HASH", "\"${getGitHash()}\"" } From 4046f6bec4b90682a67a412a6cf0f8eb9fc83d0f Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 20 Oct 2026 18:01:00 +0000 Subject: [PATCH 8/9] =?UTF-8?q?fix:=20software=20fallback=20review=20fixes?= =?UTF-8?q?=20=E2=80=94=20failures,=20no=20room,=20size=20key,=20pause,=20?= =?UTF-8?q?SPS=20bounds=20(#166)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A hardware decoder that fails setup or decoding at a size it does not declare marks that size for the software decoder, like a probation that ran out. - A size marked for software waits for a free software decoder instead of going back to the hardware one that showed nothing. - The probation remembers the size it started at; a later SPS does not change which size is marked. - A pause forgets the stream's size and parameter sets: the next stream sends its own SPS before a decoder is made. - SpsParser computes sizes and crops in long and rejects codes that would wrap round; tests for malformed SPS. --- .../openipc/decoder/MediaCodecManager.java | 39 ++++++++++---- .../java/com/openipc/decoder/SpsParser.java | 35 ++++++++----- .../com/openipc/decoder/SpsParserTest.java | 52 +++++++++++++++++++ 3 files changed, 103 insertions(+), 23 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java index d98c43b1..f836ff22 100644 --- a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java +++ b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java @@ -78,6 +78,8 @@ class MediaCodecManager { * {@link #probationUntilMs} the stream goes to a software decoder (#166). */ private boolean probation; + /** "mime WxH" the decoder on probation was set up for (Qodo on #220). */ + private String probationKey; private long probationUntilMs; private int probationFrames; /** How long a decoder on probation may go without a frame once it has its parameter sets. */ @@ -379,15 +381,16 @@ private static void releaseMirror(GlMirror g) { */ void setEnabled(boolean enabled) { this.enabled = enabled; - lifecycle++; + final int gen = ++lifecycle; if (enabled) return; final MediaCodec target = decoder; - if (target == null) return; new Thread(() -> { String slot; GlMirror gl; synchronized (decoderLock) { - if (decoder != target) return; // already replaced or released + // The stream that comes after the pause sends its own SPS (Qodo on #220) + if (gen == lifecycle && !this.enabled) forgetStream(); + if (target == null || decoder != target) return; // already replaced or released decoder = null; slot = takeSlot(); gl = takeMirror(); @@ -456,6 +459,7 @@ private void forgetStream() { streamHeight = 0; lastVps = null; probation = false; + probationKey = null; } private static long nowMs() { @@ -497,12 +501,21 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { String softwareName = null; boolean noHardwareFrame = false; - if (sized && sizeKey(type, width, height).equals(softwareFor)) { - // The hardware decoder gave no frame at this size: a software one, if it has room + final String key = sizeKey(type, width, height); + if (sized && key.equals(softwareFor)) { + // The hardware decoder gave no frame at this size: a software one, once it has room String sw = softwareDecoder(type); if (sw != null && InstanceBudget.tryAcquire(softwareSlot(type))) { softwareName = sw; noHardwareFrame = true; + } else if (sw != null) { + // Not the hardware one again: it showed nothing at this size (Qodo on #220) + synchronized (decoderLock) { + retry.onFailure(nowMs()); + } + Log.w(TAG, "No room for a software decoder at " + width + "×" + height + + ", retrying"); + return; } } if (softwareName == null && !InstanceBudget.tryAcquire(type)) { @@ -555,7 +568,7 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { boolean declared = !sized || declaresSize(local, type, width, height); // Probation only where a software decoder can take over, and once per size onProbation = !software && sized && !declared && aboveFullHd(width, height) - && !sizeKey(type, width, height).equals(softwareFor) + && !key.equals(softwareFor) && softwareDecoder(type) != null; Log.i(TAG, "Decoder " + local.getName() + ": stream " + (sized ? width + "×" + height : "size unknown") @@ -603,6 +616,8 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { // A creation cancelled by a pause must not delay the next start if (gen != lifecycle || !enabled) return; retry.onFailure(nowMs()); + // The hardware decoder refused a size it does not declare: software next time + if (onProbation) softwareFor = key; } Log.e(TAG, "Cannot setup decoder (attempt " + retry.failures() + "): " + e.getMessage()); if (listener != null) listener.onDecoderFailed(); @@ -631,6 +646,7 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { confWidth = maxW; confHeight = maxH; probation = onProbation; + probationKey = onProbation ? key : null; probationUntilMs = 0; probationFrames = framesOut; } @@ -697,10 +713,10 @@ boolean decodeFrame(Frame buffer, boolean codecH265, long watchdogRef) { dropDecoder(codec); return retry.mayCreate(nowMs()); } else if (probation && probationExpired()) { - Log.w(TAG, "No frame from " + decoderName + " at " + streamWidth + "×" - + streamHeight + " in " + PROBATION_MS / 1000 + " s: software decoder"); - softwareFor = sizeKey(codecH265 ? "video/hevc" : "video/avc", - streamWidth, streamHeight); + // The size the decoder was set up for, not a later SPS's (Qodo on #220) + Log.w(TAG, "No frame from " + decoderName + " (" + probationKey + ") in " + + PROBATION_MS / 1000 + " s: software decoder"); + softwareFor = probationKey; dropDecoder(codec); // The next decoder starts at the next SPS, which comes with a key frame return false; @@ -767,6 +783,9 @@ private void feed(MediaCodec codec, byte[] data, int length, int flag) { /** The decoder failed while decoding (decoderLock held): release it, back off. */ private void failDecoder(MediaCodec codec, Exception e) { + // A hardware decoder that fails at a size it does not declare: software next time + if (probation && probationKey != null) softwareFor = probationKey; + probation = false; decoder = null; String slot = takeSlot(); GlMirror gl = takeMirror(); diff --git a/android/app/src/main/java/com/openipc/decoder/SpsParser.java b/android/app/src/main/java/com/openipc/decoder/SpsParser.java index 2ea892db..fe964b40 100644 --- a/android/app/src/main/java/com/openipc/decoder/SpsParser.java +++ b/android/app/src/main/java/com/openipc/decoder/SpsParser.java @@ -66,15 +66,16 @@ private static int[] h264(Bits b) { } b.ue(); // max_num_ref_frames b.u(1); // gaps_in_frame_num_value_allowed_flag - int widthMbs = b.ue() + 1; - int heightUnits = b.ue() + 1; + // In long: a malformed code must not wrap round to a small size (Qodo on #220) + long widthMbs = b.ueLong() + 1; + long heightUnits = b.ueLong() + 1; int frameMbsOnly = b.u(1); if (frameMbsOnly == 0) b.u(1); // mb_adaptive_frame_field_flag b.u(1); // direct_8x8_inference_flag - int width = widthMbs * 16; - int height = (2 - frameMbsOnly) * heightUnits * 16; + long width = widthMbs * 16; + long height = (2 - frameMbsOnly) * heightUnits * 16; if (b.u(1) == 1) { - int left = b.ue(), right = b.ue(), top = b.ue(), bottom = b.ue(); + long left = b.ueLong(), right = b.ueLong(), top = b.ueLong(), bottom = b.ueLong(); int arrayType = separatePlanes ? 0 : chroma; int cropX = arrayType == 1 || arrayType == 2 ? 2 : 1; int cropY = (arrayType == 1 ? 2 : 1) * (2 - frameMbsOnly); @@ -114,10 +115,10 @@ private static int[] h265(Bits b) { b.ue(); // sps_seq_parameter_set_id int chroma = b.ue(); boolean separatePlanes = chroma == 3 && b.u(1) == 1; - int width = b.ue(); - int height = b.ue(); + long width = b.ueLong(); + long height = b.ueLong(); if (b.u(1) == 1) { - int left = b.ue(), right = b.ue(), top = b.ue(), bottom = b.ue(); + long left = b.ueLong(), right = b.ueLong(), top = b.ueLong(), bottom = b.ueLong(); int arrayType = separatePlanes ? 0 : chroma; int subW = arrayType == 1 || arrayType == 2 ? 2 : 1; int subH = arrayType == 1 ? 2 : 1; @@ -127,10 +128,10 @@ private static int[] h265(Bits b) { return valid(width, height); } - private static int[] valid(int width, int height) { - // Beyond 8K or empty: not a size worth configuring a decoder for + private static int[] valid(long width, long height) { + // Beyond 8K, empty or cropped below nothing: not a size worth configuring a decoder for return width > 0 && height > 0 && width <= 8192 && height <= 8192 - ? new int[]{width, height} : null; + ? new int[]{(int) width, (int) height} : null; } /** The NAL unit without its emulation prevention bytes (00 00 03 → 00 00). */ @@ -171,12 +172,20 @@ void skip(int n) { pos += n; } - int ue() { + /** Unsigned Exp-Golomb code, 0 … 2^32 − 2. */ + long ueLong() { int zeros = 0; while (u(1) == 0) { if (++zeros > 31) throw new IllegalStateException("bad Exp-Golomb code"); } - return zeros == 0 ? 0 : (int) ((1L << zeros) - 1 + u(zeros)); + return zeros == 0 ? 0 : (1L << zeros) - 1 + (u(zeros) & 0xffffffffL); + } + + /** {@link #ueLong()} where the syntax needs an int; larger values are malformed. */ + int ue() { + long v = ueLong(); + if (v > Integer.MAX_VALUE) throw new IllegalStateException("Exp-Golomb code too large"); + return (int) v; } int se() { diff --git a/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java b/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java index feeace1b..75547b3a 100644 --- a/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java +++ b/android/app/src/test/java/com/openipc/decoder/SpsParserTest.java @@ -74,6 +74,58 @@ public void notAnSpsOrBroken() { assertNull(SpsParser.size(b, 4, b.length, false)); } + /** A baseline H.264 SPS with the given size fields (Exp-Golomb values as written). */ + private static int[] h264Sps(long widthMbsMinus1, long heightMinus1, long[] crop) { + StringBuilder bits = new StringBuilder(); + bits.append(bin(0x67, 8)).append(bin(66, 8)).append(bin(0, 8)).append(bin(31, 8)); + ue(bits, 0); // seq_parameter_set_id + ue(bits, 0); // log2_max_frame_num_minus4 + ue(bits, 2); // pic_order_cnt_type + ue(bits, 1); // max_num_ref_frames + bits.append('0'); // gaps_in_frame_num_value_allowed_flag + ue(bits, widthMbsMinus1); + ue(bits, heightMinus1); + bits.append("11"); // frame_mbs_only_flag, direct_8x8_inference_flag + if (crop != null) { + bits.append('1'); + for (long c : crop) ue(bits, c); + } else { + bits.append('0'); + } + bits.append("01"); // no VUI, then the stop bit + while (bits.length() % 8 != 0) bits.append('0'); + byte[] b = new byte[bits.length() / 8]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) Integer.parseInt(bits.substring(8 * i, 8 * i + 8), 2); + } + return SpsParser.size(b, 0, b.length, false); + } + + private static String bin(long v, int n) { + StringBuilder s = new StringBuilder(Long.toBinaryString(v)); + while (s.length() < n) s.insert(0, '0'); + return s.toString(); + } + + private static void ue(StringBuilder bits, long v) { + String code = Long.toBinaryString(v + 1); + for (int i = 1; i < code.length(); i++) bits.append('0'); + bits.append(code); + } + + @Test + public void malformedSizesDoNotWrapRound() { + // The writer itself: 1280×720 + assertArrayEquals(new int[]{1280, 720}, h264Sps(79, 44, null)); + // 2^28 + 1 macroblocks wide is 2^32 + 16 pixels: 16 in 32-bit arithmetic (Qodo on #220) + assertNull(h264Sps(1L << 28, 44, null)); + // Crops larger than the picture, and one that overflows an int when doubled + assertNull(h264Sps(79, 44, new long[]{700, 0, 0, 0})); + assertNull(h264Sps(79, 44, new long[]{1L << 31, 0, 0, 0})); + // A sane crop still works: 1920×1088 coded, 1080 shown + assertArrayEquals(new int[]{1920, 1080}, h264Sps(119, 67, new long[]{0, 0, 0, 4})); + } + @Test public void emulationPreventionBytesAreRemoved() { assertArrayEquals(new byte[]{0, 0, 1, 0, 0, 0}, From 0fe880783730c46a0d9c31696893f37986d5b071 Mon Sep 17 00:00:00 2001 From: Compot Developer Date: Tue, 20 Oct 2026 18:12:18 +0000 Subject: [PATCH 9/9] fix: software fallback keeps every failed size and starts at a fresh SPS (#166) softwareFor holds the last 8 sizes the hardware decoder gave no frame for, not only the latest. Marking a size makes the next decoder wait for a new SPS, so it gets the whole parameter set before the next key frame. --- .../openipc/decoder/MediaCodecManager.java | 36 +++++++++++++++---- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java index f836ff22..6d8f9ece 100644 --- a/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java +++ b/android/app/src/main/java/com/openipc/decoder/MediaCodecManager.java @@ -84,8 +84,11 @@ class MediaCodecManager { private int probationFrames; /** How long a decoder on probation may go without a frame once it has its parameter sets. */ static final long PROBATION_MS = 5_000; - /** "mime WxH" of a stream the hardware decoder gave no frame for: software from then on. */ - private volatile String softwareFor; + /** "mime WxH" of streams the hardware decoder gave no frame for: software from then on. */ + private final java.util.Set softwareFor = java.util.Collections.synchronizedSet( + new java.util.LinkedHashSet<>()); + /** Sizes kept in {@link #softwareFor}: a camera switches between a few at most. */ + static final int SOFTWARE_SIZES = 8; /** Name of the current decoder, for /stats.json. */ private volatile String decoderName; /** Degrees counter-clockwise the decoder turns the picture (#201); 0 for none. */ @@ -502,7 +505,7 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { String softwareName = null; boolean noHardwareFrame = false; final String key = sizeKey(type, width, height); - if (sized && key.equals(softwareFor)) { + if (sized && softwareFor.contains(key)) { // The hardware decoder gave no frame at this size: a software one, once it has room String sw = softwareDecoder(type); if (sw != null && InstanceBudget.tryAcquire(softwareSlot(type))) { @@ -568,7 +571,7 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { boolean declared = !sized || declaresSize(local, type, width, height); // Probation only where a software decoder can take over, and once per size onProbation = !software && sized && !declared && aboveFullHd(width, height) - && !key.equals(softwareFor) + && !softwareFor.contains(key) && softwareDecoder(type) != null; Log.i(TAG, "Decoder " + local.getName() + ": stream " + (sized ? width + "×" + height : "size unknown") @@ -617,7 +620,7 @@ void createDecoder(boolean codecH265, JpegCodec.Current current) { if (gen != lifecycle || !enabled) return; retry.onFailure(nowMs()); // The hardware decoder refused a size it does not declare: software next time - if (onProbation) softwareFor = key; + if (onProbation) markSoftware(key); } Log.e(TAG, "Cannot setup decoder (attempt " + retry.failures() + "): " + e.getMessage()); if (listener != null) listener.onDecoderFailed(); @@ -716,7 +719,7 @@ boolean decodeFrame(Frame buffer, boolean codecH265, long watchdogRef) { // The size the decoder was set up for, not a later SPS's (Qodo on #220) Log.w(TAG, "No frame from " + decoderName + " (" + probationKey + ") in " + PROBATION_MS / 1000 + " s: software decoder"); - softwareFor = probationKey; + markSoftware(probationKey); dropDecoder(codec); // The next decoder starts at the next SPS, which comes with a key frame return false; @@ -784,7 +787,7 @@ private void feed(MediaCodec codec, byte[] data, int length, int flag) { /** The decoder failed while decoding (decoderLock held): release it, back off. */ private void failDecoder(MediaCodec codec, Exception e) { // A hardware decoder that fails at a size it does not declare: software next time - if (probation && probationKey != null) softwareFor = probationKey; + if (probation && probationKey != null) markSoftware(probationKey); probation = false; decoder = null; String slot = takeSlot(); @@ -810,6 +813,25 @@ private void dropDecoder(MediaCodec codec) { releaseSlot(slot); } + /** + * The hardware decoder gave no frame at this size (decoderLock held): software from now + * on, for this size among the last {@link #SOFTWARE_SIZES} (Qodo on #220). The next + * decoder waits for a fresh SPS, so it starts with the full parameter set and the key + * frame after it; the stream's size is kept for it. + */ + private void markSoftware(String key) { + synchronized (softwareFor) { + softwareFor.remove(key); + softwareFor.add(key); + java.util.Iterator it = softwareFor.iterator(); + while (softwareFor.size() > SOFTWARE_SIZES && it.hasNext()) { + it.next(); + it.remove(); + } + } + spsSeen = false; + } + /** The decoder on probation has its parameter sets but no frame past the deadline. */ private boolean probationExpired() { if (framesOut != probationFrames) {