From 189858de2d48248dba755eaf7d530d8220ce2a73 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 11 Nov 2026 10:24:58 +0200 Subject: [PATCH] Android: scan dense codes at 1440x1080 instead of 640x480 A camera session opened with no preview size -- every Vision scanner -- analysed frames at 640x480 on Android. That is too few pixels per module for a dense code such as a PDF417: it read on a phone with a sharp lens and failed on others. The iOS port had the same default until #6159. - An NV21 session with no preview size now asks CameraX for 1440x1080, turned to the display's orientation. 4:3 on purpose: the Preview use case is 4:3, so the analysed field stays the one on screen. Other sessions keep 640x480, because each of their frames is compressed to JPEG on the analysis thread. - Half a request (one dimension zero) used to be paired with one default dimension; it now gets the whole default. - The NV21 array is handed back to the converter for the next frame, so the port allocates none per frame at the larger size, and the luma plane is copied a row at a time. Co-Authored-By: Claude Fable 5.2 --- .../android/AndroidCameraFrameConverter.java | 63 ++++++++++++++++- .../impl/android/AndroidCameraImpl.java | 62 ++++++++++++++-- .../AndroidCameraFrameConverterTest.java | 70 +++++++++++++++++++ ...ndroidCameraThreadingHarness.java.template | 6 ++ scripts/camera-regression/extract-android.py | 1 + 5 files changed, 193 insertions(+), 9 deletions(-) diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidCameraFrameConverter.java b/Ports/Android/src/com/codename1/impl/android/AndroidCameraFrameConverter.java index 1b82f96a6a1..a923907fbfe 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidCameraFrameConverter.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidCameraFrameConverter.java @@ -24,21 +24,69 @@ import java.nio.ByteBuffer; -/** Packs camera YUV_420_888 planes without assuming a device-specific layout. */ +/** + * The camera port's device-independent arithmetic: packing YUV_420_888 planes + * without assuming a device-specific layout, and choosing the analysis size. + */ final class AndroidCameraFrameConverter { private AndroidCameraFrameConverter() { } + /** Long side of the analysis size an NV21 session gets when it asks for none. */ + static final int NV21_ANALYSIS_LONG_SIDE = 1440; + /** Short side of the analysis size an NV21 session gets when it asks for none. */ + static final int NV21_ANALYSIS_SHORT_SIDE = 1080; + /** Long side of the analysis size any other session gets when it asks for none. */ + static final int OTHER_ANALYSIS_LONG_SIDE = 640; + /** Short side of the analysis size any other session gets when it asks for none. */ + static final int OTHER_ANALYSIS_SHORT_SIDE = 480; + + /** + * Picks the frame analysis size as {width, height}. A request naming both + * dimensions is returned unchanged. Anything else gets the default for the + * kind of frame asked for, turned to the display's orientation: half a + * request (one dimension zero) cannot be completed without knowing the + * aspect ratio the caller had in mind, and pairing it with one default + * dimension produced sizes such as 1920x480. + */ + static int[] analysisSize(int requestedWidth, int requestedHeight, + boolean nv21Frames, boolean portrait) { + if (requestedWidth > 0 && requestedHeight > 0) { + return new int[] {requestedWidth, requestedHeight}; + } + int longSide = nv21Frames ? NV21_ANALYSIS_LONG_SIDE : OTHER_ANALYSIS_LONG_SIDE; + int shortSide = nv21Frames ? NV21_ANALYSIS_SHORT_SIDE : OTHER_ANALYSIS_SHORT_SIDE; + return portrait ? new int[] {shortSide, longSide} : new int[] {longSide, shortSide}; + } + static byte[] toNV21(int width, int height, ByteBuffer y, int yRowStride, int yPixelStride, ByteBuffer u, int uRowStride, int uPixelStride, ByteBuffer v, int vRowStride, int vPixelStride) { + return toNV21(width, height, y, yRowStride, yPixelStride, + u, uRowStride, uPixelStride, v, vRowStride, vPixelStride, null); + } + + /** + * As above, writing into {@code reuse} when it is exactly the size the + * frame needs and allocating otherwise. A 1440x1080 frame is 2.3 MB, so a + * session hands back the array it was given last time and the port itself + * allocates none per frame; a consumer that keeps the pixels still makes + * its own copy. Every byte of the result is written, so nothing of the + * previous frame is left. + */ + static byte[] toNV21(int width, int height, + ByteBuffer y, int yRowStride, int yPixelStride, + ByteBuffer u, int uRowStride, int uPixelStride, + ByteBuffer v, int vRowStride, int vPixelStride, + byte[] reuse) { long pixels = (long) width * height; if (width <= 0 || height <= 0 || (width & 1) != 0 || (height & 1) != 0 || pixels + pixels / 2 > Integer.MAX_VALUE) { throw new IllegalArgumentException("Invalid YUV dimensions"); } - byte[] result = new byte[(int) (pixels + pixels / 2)]; + int length = (int) (pixels + pixels / 2); + byte[] result = reuse != null && reuse.length == length ? reuse : new byte[length]; copyPlane(y, yRowStride, yPixelStride, width, height, result, 0, 1); // NV21 interleaves V then U. Planes can be planar, overlapping, padded, // sliced, or read-only; absolute reads preserve their positions. @@ -59,6 +107,17 @@ private static void copyPlane(ByteBuffer plane, int rowStride, int pixelStride, throw new IllegalArgumentException("Truncated or invalid YUV plane"); } int base = plane.position(); + if (pixelStride == 1 && step == 1) { + // The luma plane, two thirds of the frame: whole rows at a time. + // A duplicate keeps the caller's position where it was. + ByteBuffer rows = plane.duplicate(); + for (int row = 0; row < height; row++) { + rows.position(base + row * rowStride); + rows.get(out, offset, width); + offset += width; + } + return; + } for (int row = 0; row < height; row++) { int input = base + row * rowStride; for (int col = 0; col < width; col++) { diff --git a/Ports/Android/src/com/codename1/impl/android/AndroidCameraImpl.java b/Ports/Android/src/com/codename1/impl/android/AndroidCameraImpl.java index f0c1b7c78fc..a124a95c8b3 100644 --- a/Ports/Android/src/com/codename1/impl/android/AndroidCameraImpl.java +++ b/Ports/Android/src/com/codename1/impl/android/AndroidCameraImpl.java @@ -118,6 +118,13 @@ public class AndroidCameraImpl extends CameraImpl { private Object imageAnalysis; private View previewView; private final AtomicBoolean listenerBusy = new AtomicBoolean(); + /** + * The NV21 array of the last frame, handed back to the converter for the + * next one. Only touched while listenerBusy is held, which is one frame + * at a time. CameraFrame documents its arrays as valid only during + * onFrame, which is what makes handing the same one out again legal. + */ + private byte[] nv21Buffer; private volatile FrameListener frameListener; private volatile FrameFormat frameFormat = FrameFormat.JPEG; private volatile long frameIntervalNanos; @@ -232,10 +239,7 @@ public void open(String cameraId, CameraSessionOptions opts) throws IOException Object iaBuilder = clsImageAnalysisBuilder.getConstructor().newInstance(); clsImageAnalysisBuilder.getMethod("setTargetResolution", Class.forName("android.util.Size")) - .invoke(iaBuilder, - new android.util.Size( - options.getPreviewWidth() > 0 ? options.getPreviewWidth() : 640, - options.getPreviewHeight() > 0 ? options.getPreviewHeight() : 480)); + .invoke(iaBuilder, analysisTargetSize(options)); // STRATEGY_KEEP_ONLY_LATEST = 0 in CameraX try { clsImageAnalysisBuilder.getMethod("setBackpressureStrategy", int.class) @@ -471,7 +475,10 @@ private void onImageProxy(Object imageProxy) { Image img = (Image) imageProxy.getClass().getMethod("getImage").invoke(imageProxy); Object imageInfo = imageProxy.getClass().getMethod("getImageInfo").invoke(imageProxy); int rotation = (Integer) imageInfo.getClass().getMethod("getRotationDegrees").invoke(imageInfo); - frame = img == null ? null : createFrame(img, rotation, now, frameFormat); + frame = img == null ? null : createFrame(img, rotation, now, frameFormat, nv21Buffer); + if (frame != null && frame.getRawBytes() != null) { + nv21Buffer = frame.getRawBytes(); + } } catch (Throwable t) { Log.w(TAG, "Frame conversion failed: " + t.getMessage()); return; @@ -486,6 +493,46 @@ private void onImageProxy(Object imageProxy) { } } + /** + * The resolution frames are analysed at when the session names none. + * + * A session that asks for NV21 frames -- every Vision scanner -- used to + * get 640x480, which is too few pixels per module for a dense code such + * as a PDF417: it read on a phone with a sharp lens and failed on others. + * It now gets 1440x1080. That is 4:3 on purpose: the Preview use case is + * 4:3 by default, so what is analysed stays exactly what is shown and a + * rectangle a detector reports still maps onto the preview. 1080p would + * crop the analysed field to 16:9 for no more pixels across a code that + * fills the short side. This is a request, not a guarantee: CameraX picks + * the nearest size the device supports for the three use cases bound + * together, so a limited camera answers with less. + * + * Every other session keeps 640x480: JPEG, and RGBA8888, which this port + * answers with JPEG. Each of their frames is compressed on the analysis + * thread whether or not anyone reads it closely. + * + * CameraX reads the size in the display's current rotation, not the + * sensor's, so the default is turned to match. The display is asked, not + * the window: in split screen a window can be wider than tall on a + * display that is portrait. A size the application asked for is passed + * through as it always was. + */ + private android.util.Size analysisTargetSize(CameraSessionOptions options) { + boolean portrait = false; + try { + android.util.DisplayMetrics metrics = new android.util.DisplayMetrics(); + activity.getWindowManager().getDefaultDisplay().getRealMetrics(metrics); + portrait = metrics.heightPixels > metrics.widthPixels; + } catch (Throwable t) { + Log.w(TAG, "Could not read the display orientation: " + t); + } + int[] size = AndroidCameraFrameConverter.analysisSize( + options.getPreviewWidth(), options.getPreviewHeight(), + options.getFrameFormat() == FrameFormat.NV21, + portrait); + return new android.util.Size(size[0], size[1]); + } + private static void closeImageProxy(Object imageProxy) { try { imageProxy.getClass().getMethod("close").invoke(imageProxy); @@ -493,7 +540,7 @@ private static void closeImageProxy(Object imageProxy) { } private static CameraFrame createFrame(Image image, int rotation, long timestamp, - FrameFormat requested) { + FrameFormat requested, byte[] reuse) { final int w = image.getWidth(); final int h = image.getHeight(); if (image.getFormat() == ImageFormat.JPEG) { @@ -509,7 +556,8 @@ private static CameraFrame createFrame(Image image, int rotation, long timestamp final byte[] nv21 = AndroidCameraFrameConverter.toNV21(w, h, planes[0].getBuffer(), planes[0].getRowStride(), planes[0].getPixelStride(), planes[1].getBuffer(), planes[1].getRowStride(), planes[1].getPixelStride(), - planes[2].getBuffer(), planes[2].getRowStride(), planes[2].getPixelStride()); + planes[2].getBuffer(), planes[2].getRowStride(), planes[2].getPixelStride(), + reuse); CameraFrame.JpegEncoder encoder = new CameraFrame.JpegEncoder() { public byte[] encode() { YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, w, h, null); diff --git a/maven/core-unittests/src/test/java/com/codename1/camera/AndroidCameraFrameConverterTest.java b/maven/core-unittests/src/test/java/com/codename1/camera/AndroidCameraFrameConverterTest.java index 8a3b02d5e68..9f1c43cfa57 100644 --- a/maven/core-unittests/src/test/java/com/codename1/camera/AndroidCameraFrameConverterTest.java +++ b/maven/core-unittests/src/test/java/com/codename1/camera/AndroidCameraFrameConverterTest.java @@ -58,6 +58,76 @@ static void closeLoader() throws Exception { if (loader != null) loader.close(); } + private int[] analysisSize(int width, int height, boolean raw, boolean portrait) + throws Exception { + Method pick = convert.getDeclaringClass().getDeclaredMethod("analysisSize", + int.class, int.class, boolean.class, boolean.class); + pick.setAccessible(true); + return (int[]) pick.invoke(null, width, height, raw, portrait); + } + + @Test + void rawSessionWithNoPreviewSizeIsAnalysedAt1080Lines() throws Exception { + // 640x480 was the default, and too coarse for a dense PDF417. 4:3, as + // the preview is, so the analysed field is the one on screen. + assertArrayEquals(new int[] {1440, 1080}, analysisSize(0, 0, true, false)); + } + + @Test + void defaultAnalysisSizeFollowsThePortraitDisplay() throws Exception { + // CameraX reads the size in the display's orientation. + assertArrayEquals(new int[] {1080, 1440}, analysisSize(0, 0, true, true)); + assertArrayEquals(new int[] {480, 640}, analysisSize(0, 0, false, true)); + } + + @Test + void jpegSessionKeepsTheSmallDefault() throws Exception { + // Every JPEG frame is compressed on the analysis thread. + assertArrayEquals(new int[] {640, 480}, analysisSize(0, 0, false, false)); + } + + @Test + void requestedAnalysisSizeIsPassedThrough() throws Exception { + assertArrayEquals(new int[] {640, 480}, analysisSize(640, 480, true, true)); + assertArrayEquals(new int[] {1280, 720}, analysisSize(1280, 720, false, false)); + } + + @Test + void halfARequestGetsTheWholeDefault() throws Exception { + assertArrayEquals(new int[] {1440, 1080}, analysisSize(1920, 0, true, false)); + assertArrayEquals(new int[] {480, 640}, analysisSize(0, 480, false, true)); + } + + private byte[] packInto(byte[] reuse) throws Exception { + Method reusing = convert.getDeclaringClass().getDeclaredMethod("toNV21", int.class, int.class, + ByteBuffer.class, int.class, int.class, ByteBuffer.class, int.class, + int.class, ByteBuffer.class, int.class, int.class, byte[].class); + reusing.setAccessible(true); + return (byte[]) reusing.invoke(null, 4, 4, ByteBuffer.wrap(new byte[] { + 1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}), 4, 1, + ByteBuffer.wrap(new byte[] {21,22,23,24}), 2, 1, + ByteBuffer.wrap(new byte[] {31,32,33,34}), 2, 1, reuse); + } + + @Test + void arrayOfTheRightSizeIsReusedAndFullyOverwritten() throws Exception { + byte[] previous = new byte[24]; + java.util.Arrays.fill(previous, (byte) 99); + byte[] packed = packInto(previous); + assertSame(previous, packed); + assertArrayEquals(expected(), packed); + } + + @Test + void arrayOfAnotherSizeIsNotReused() throws Exception { + // The analysis size changes when a session is reopened. + byte[] previous = new byte[25]; + byte[] packed = packInto(previous); + assertNotSame(previous, packed); + assertArrayEquals(expected(), packed); + assertArrayEquals(expected(), packInto(null)); + } + private byte[] pack(ByteBuffer y, int ys, ByteBuffer u, int us, int up, ByteBuffer v, int vs, int vp) throws Exception { return (byte[]) convert.invoke(null, 4, 4, y, ys, 1, u, us, up, v, vs, vp); diff --git a/maven/core-unittests/src/test/resources/AndroidCameraThreadingHarness.java.template b/maven/core-unittests/src/test/resources/AndroidCameraThreadingHarness.java.template index 2eeb49a801b..f8b4dc196e3 100644 --- a/maven/core-unittests/src/test/resources/AndroidCameraThreadingHarness.java.template +++ b/maven/core-unittests/src/test/resources/AndroidCameraThreadingHarness.java.template @@ -35,6 +35,12 @@ public class AndroidCameraThreadingHarness { /* PRODUCTION_METHODS */ + // The production method reads the display through Android's resources; + // AndroidCameraFrameConverterTest holds the size it picks. + private android.util.Size analysisTargetSize(CameraSessionOptions options) { + return new android.util.Size(options.getPreviewWidth(), options.getPreviewHeight()); + } + private static void check(boolean ok, String message) { if (!ok) throw new AssertionError(message); } diff --git a/scripts/camera-regression/extract-android.py b/scripts/camera-regression/extract-android.py index 9d5c2d9c5ee..b7d93d299b2 100644 --- a/scripts/camera-regression/extract-android.py +++ b/scripts/camera-regression/extract-android.py @@ -48,6 +48,7 @@ def method(signature): private volatile FrameFormat frameFormat = FrameFormat.JPEG; private volatile long frameIntervalNanos; private long lastFrameNanos; + private byte[] nv21Buffer; public interface FrameListener { void onFrame(CameraFrame frame); } public void deliver(Object proxy) { onImageProxy(proxy); } ''' + methods + '\n}\n')