Android: scan dense codes at 1440x1080 instead of 640x480

Ports/Android/src/com/codename1/impl/android/AndroidCameraFrameConverter.java

@@ -24,21 +24,69 @@
 
 import java.nio.ByteBuffer;
 
-/** Packs camera YUV_420_888 planes without assuming a device-specific layout. */
+/**
+ * The camera port's device-independent arithmetic: packing YUV_420_888 planes
+ * without assuming a device-specific layout, and choosing the analysis size.
+ */
 final class AndroidCameraFrameConverter {
     private AndroidCameraFrameConverter() {
     }
 
+    /** Long side of the analysis size an NV21 session gets when it asks for none. */
+    static final int NV21_ANALYSIS_LONG_SIDE = 1440;
+    /** Short side of the analysis size an NV21 session gets when it asks for none. */
+    static final int NV21_ANALYSIS_SHORT_SIDE = 1080;
+    /** Long side of the analysis size any other session gets when it asks for none. */
+    static final int OTHER_ANALYSIS_LONG_SIDE = 640;
+    /** Short side of the analysis size any other session gets when it asks for none. */
+    static final int OTHER_ANALYSIS_SHORT_SIDE = 480;
+
+    /**
+     * Picks the frame analysis size as {width, height}. A request naming both
+     * dimensions is returned unchanged. Anything else gets the default for the
+     * kind of frame asked for, turned to the display's orientation: half a
+     * request (one dimension zero) cannot be completed without knowing the
+     * aspect ratio the caller had in mind, and pairing it with one default
+     * dimension produced sizes such as 1920x480.
+     */
+    static int[] analysisSize(int requestedWidth, int requestedHeight,
+                              boolean nv21Frames, boolean portrait) {
+        if (requestedWidth > 0 && requestedHeight > 0) {
+            return new int[] {requestedWidth, requestedHeight};
+        }
+        int longSide = nv21Frames ? NV21_ANALYSIS_LONG_SIDE : OTHER_ANALYSIS_LONG_SIDE;
+        int shortSide = nv21Frames ? NV21_ANALYSIS_SHORT_SIDE : OTHER_ANALYSIS_SHORT_SIDE;
+        return portrait ? new int[] {shortSide, longSide} : new int[] {longSide, shortSide};
+    }
+
     static byte[] toNV21(int width, int height,
                          ByteBuffer y, int yRowStride, int yPixelStride,
                          ByteBuffer u, int uRowStride, int uPixelStride,
                          ByteBuffer v, int vRowStride, int vPixelStride) {
+        return toNV21(width, height, y, yRowStride, yPixelStride,
+                u, uRowStride, uPixelStride, v, vRowStride, vPixelStride, null);
+    }
+
+    /**
+     * As above, writing into {@code reuse} when it is exactly the size the
+     * frame needs and allocating otherwise. A 1440x1080 frame is 2.3 MB, so a
+     * session hands back the array it was given last time and the port itself
+     * allocates none per frame; a consumer that keeps the pixels still makes
+     * its own copy. Every byte of the result is written, so nothing of the
+     * previous frame is left.
+     */
+    static byte[] toNV21(int width, int height,
+                         ByteBuffer y, int yRowStride, int yPixelStride,
+                         ByteBuffer u, int uRowStride, int uPixelStride,
+                         ByteBuffer v, int vRowStride, int vPixelStride,
+                         byte[] reuse) {
         long pixels = (long) width * height;
         if (width <= 0 || height <= 0 || (width & 1) != 0 || (height & 1) != 0
                 || pixels + pixels / 2 > Integer.MAX_VALUE) {
             throw new IllegalArgumentException("Invalid YUV dimensions");
         }
-        byte[] result = new byte[(int) (pixels + pixels / 2)];
+        int length = (int) (pixels + pixels / 2);
+        byte[] result = reuse != null && reuse.length == length ? reuse : new byte[length];
         copyPlane(y, yRowStride, yPixelStride, width, height, result, 0, 1);
         // NV21 interleaves V then U. Planes can be planar, overlapping, padded,
         // sliced, or read-only; absolute reads preserve their positions.
@@ -59,6 +107,17 @@ private static void copyPlane(ByteBuffer plane, int rowStride, int pixelStride,
             throw new IllegalArgumentException("Truncated or invalid YUV plane");
         }
         int base = plane.position();
+        if (pixelStride == 1 && step == 1) {
+            // The luma plane, two thirds of the frame: whole rows at a time.
+            // A duplicate keeps the caller's position where it was.
+            ByteBuffer rows = plane.duplicate();
+            for (int row = 0; row < height; row++) {
+                rows.position(base + row * rowStride);
+                rows.get(out, offset, width);
+                offset += width;
+            }
+            return;
+        }
         for (int row = 0; row < height; row++) {
             int input = base + row * rowStride;
             for (int col = 0; col < width; col++) {

Ports/Android/src/com/codename1/impl/android/AndroidCameraImpl.java

@@ -118,6 +118,13 @@ public class AndroidCameraImpl extends CameraImpl {
     private Object imageAnalysis;
     private View previewView;
     private final AtomicBoolean listenerBusy = new AtomicBoolean();
+    /**
+     * The NV21 array of the last frame, handed back to the converter for the
+     * next one. Only touched while listenerBusy is held, which is one frame
+     * at a time. CameraFrame documents its arrays as valid only during
+     * onFrame, which is what makes handing the same one out again legal.
+     */
+    private byte[] nv21Buffer;
     private volatile FrameListener frameListener;
     private volatile FrameFormat frameFormat = FrameFormat.JPEG;
     private volatile long frameIntervalNanos;
@@ -232,10 +239,7 @@ public void open(String cameraId, CameraSessionOptions opts) throws IOException
             Object iaBuilder = clsImageAnalysisBuilder.getConstructor().newInstance();
             clsImageAnalysisBuilder.getMethod("setTargetResolution",
                     Class.forName("android.util.Size"))
-                    .invoke(iaBuilder,
-                            new android.util.Size(
-                                    options.getPreviewWidth() > 0 ? options.getPreviewWidth() : 640,
-                                    options.getPreviewHeight() > 0 ? options.getPreviewHeight() : 480));
+                    .invoke(iaBuilder, analysisTargetSize(options));
             // STRATEGY_KEEP_ONLY_LATEST = 0 in CameraX
             try {
                 clsImageAnalysisBuilder.getMethod("setBackpressureStrategy", int.class)
@@ -471,7 +475,10 @@ private void onImageProxy(Object imageProxy) {
                 Image img = (Image) imageProxy.getClass().getMethod("getImage").invoke(imageProxy);
                 Object imageInfo = imageProxy.getClass().getMethod("getImageInfo").invoke(imageProxy);
                 int rotation = (Integer) imageInfo.getClass().getMethod("getRotationDegrees").invoke(imageInfo);
-                frame = img == null ? null : createFrame(img, rotation, now, frameFormat);
+                frame = img == null ? null : createFrame(img, rotation, now, frameFormat, nv21Buffer);
+                if (frame != null && frame.getRawBytes() != null) {
+                    nv21Buffer = frame.getRawBytes();
+                }
             } catch (Throwable t) {
                 Log.w(TAG, "Frame conversion failed: " + t.getMessage());
                 return;
@@ -486,14 +493,54 @@ private void onImageProxy(Object imageProxy) {
         }
     }
 
+    /**
+     * The resolution frames are analysed at when the session names none.
+     *
+     * A session that asks for NV21 frames -- every Vision scanner -- used to
+     * get 640x480, which is too few pixels per module for a dense code such
+     * as a PDF417: it read on a phone with a sharp lens and failed on others.
+     * It now gets 1440x1080. That is 4:3 on purpose: the Preview use case is
+     * 4:3 by default, so what is analysed stays exactly what is shown and a
+     * rectangle a detector reports still maps onto the preview. 1080p would
+     * crop the analysed field to 16:9 for no more pixels across a code that
+     * fills the short side. This is a request, not a guarantee: CameraX picks
+     * the nearest size the device supports for the three use cases bound
+     * together, so a limited camera answers with less.
+     *
+     * Every other session keeps 640x480: JPEG, and RGBA8888, which this port
+     * answers with JPEG. Each of their frames is compressed on the analysis
+     * thread whether or not anyone reads it closely.
+     *
+     * CameraX reads the size in the display's current rotation, not the
+     * sensor's, so the default is turned to match. The display is asked, not
+     * the window: in split screen a window can be wider than tall on a
+     * display that is portrait. A size the application asked for is passed
+     * through as it always was.
+     */
+    private android.util.Size analysisTargetSize(CameraSessionOptions options) {
+        boolean portrait = false;
+        try {
+            android.util.DisplayMetrics metrics = new android.util.DisplayMetrics();
+            activity.getWindowManager().getDefaultDisplay().getRealMetrics(metrics);
+            portrait = metrics.heightPixels > metrics.widthPixels;
+        } catch (Throwable t) {
+            Log.w(TAG, "Could not read the display orientation: " + t);
+        }
+        int[] size = AndroidCameraFrameConverter.analysisSize(
+                options.getPreviewWidth(), options.getPreviewHeight(),
+                options.getFrameFormat() == FrameFormat.NV21,
+                portrait);
+        return new android.util.Size(size[0], size[1]);
+    }
+
     private static void closeImageProxy(Object imageProxy) {
         try {
             imageProxy.getClass().getMethod("close").invoke(imageProxy);
         } catch (Throwable ignored) { }
     }
 
     private static CameraFrame createFrame(Image image, int rotation, long timestamp,
-                                           FrameFormat requested) {
+                                           FrameFormat requested, byte[] reuse) {
         final int w = image.getWidth();
         final int h = image.getHeight();
         if (image.getFormat() == ImageFormat.JPEG) {
@@ -509,7 +556,8 @@ private static CameraFrame createFrame(Image image, int rotation, long timestamp
         final byte[] nv21 = AndroidCameraFrameConverter.toNV21(w, h,
                 planes[0].getBuffer(), planes[0].getRowStride(), planes[0].getPixelStride(),
                 planes[1].getBuffer(), planes[1].getRowStride(), planes[1].getPixelStride(),
-                planes[2].getBuffer(), planes[2].getRowStride(), planes[2].getPixelStride());
+                planes[2].getBuffer(), planes[2].getRowStride(), planes[2].getPixelStride(),
+                reuse);
         CameraFrame.JpegEncoder encoder = new CameraFrame.JpegEncoder() {
             public byte[] encode() {
                 YuvImage yuv = new YuvImage(nv21, ImageFormat.NV21, w, h, null);

maven/core-unittests/src/test/java/com/codename1/camera/AndroidCameraFrameConverterTest.java

@@ -58,6 +58,76 @@ static void closeLoader() throws Exception {
         if (loader != null) loader.close();
     }
 
+    private int[] analysisSize(int width, int height, boolean raw, boolean portrait)
+            throws Exception {
+        Method pick = convert.getDeclaringClass().getDeclaredMethod("analysisSize",
+                int.class, int.class, boolean.class, boolean.class);
+        pick.setAccessible(true);
+        return (int[]) pick.invoke(null, width, height, raw, portrait);
+    }
+
+    @Test
+    void rawSessionWithNoPreviewSizeIsAnalysedAt1080Lines() throws Exception {
+        // 640x480 was the default, and too coarse for a dense PDF417. 4:3, as
+        // the preview is, so the analysed field is the one on screen.
+        assertArrayEquals(new int[] {1440, 1080}, analysisSize(0, 0, true, false));
+    }
+
+    @Test
+    void defaultAnalysisSizeFollowsThePortraitDisplay() throws Exception {
+        // CameraX reads the size in the display's orientation.
+        assertArrayEquals(new int[] {1080, 1440}, analysisSize(0, 0, true, true));
+        assertArrayEquals(new int[] {480, 640}, analysisSize(0, 0, false, true));
+    }
+
+    @Test
+    void jpegSessionKeepsTheSmallDefault() throws Exception {
+        // Every JPEG frame is compressed on the analysis thread.
+        assertArrayEquals(new int[] {640, 480}, analysisSize(0, 0, false, false));
+    }
+
+    @Test
+    void requestedAnalysisSizeIsPassedThrough() throws Exception {
+        assertArrayEquals(new int[] {640, 480}, analysisSize(640, 480, true, true));
+        assertArrayEquals(new int[] {1280, 720}, analysisSize(1280, 720, false, false));
+    }
+
+    @Test
+    void halfARequestGetsTheWholeDefault() throws Exception {
+        assertArrayEquals(new int[] {1440, 1080}, analysisSize(1920, 0, true, false));
+        assertArrayEquals(new int[] {480, 640}, analysisSize(0, 480, false, true));
+    }
+
+    private byte[] packInto(byte[] reuse) throws Exception {
+        Method reusing = convert.getDeclaringClass().getDeclaredMethod("toNV21", int.class, int.class,
+                ByteBuffer.class, int.class, int.class, ByteBuffer.class, int.class,
+                int.class, ByteBuffer.class, int.class, int.class, byte[].class);
+        reusing.setAccessible(true);
+        return (byte[]) reusing.invoke(null, 4, 4, ByteBuffer.wrap(new byte[] {
+                1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16}), 4, 1,
+                ByteBuffer.wrap(new byte[] {21,22,23,24}), 2, 1,
+                ByteBuffer.wrap(new byte[] {31,32,33,34}), 2, 1, reuse);
+    }
+
+    @Test
+    void arrayOfTheRightSizeIsReusedAndFullyOverwritten() throws Exception {
+        byte[] previous = new byte[24];
+        java.util.Arrays.fill(previous, (byte) 99);
+        byte[] packed = packInto(previous);
+        assertSame(previous, packed);
+        assertArrayEquals(expected(), packed);
+    }
+
+    @Test
+    void arrayOfAnotherSizeIsNotReused() throws Exception {
+        // The analysis size changes when a session is reopened.
+        byte[] previous = new byte[25];
+        byte[] packed = packInto(previous);
+        assertNotSame(previous, packed);
+        assertArrayEquals(expected(), packed);
+        assertArrayEquals(expected(), packInto(null));
+    }
+
     private byte[] pack(ByteBuffer y, int ys, ByteBuffer u, int us, int up,
                         ByteBuffer v, int vs, int vp) throws Exception {
         return (byte[]) convert.invoke(null, 4, 4, y, ys, 1, u, us, up, v, vs, vp);

maven/core-unittests/src/test/resources/AndroidCameraThreadingHarness.java.template

@@ -35,6 +35,12 @@ public class AndroidCameraThreadingHarness {
 
     /* PRODUCTION_METHODS */
 
+    // The production method reads the display through Android's resources;
+    // AndroidCameraFrameConverterTest holds the size it picks.
+    private android.util.Size analysisTargetSize(CameraSessionOptions options) {
+        return new android.util.Size(options.getPreviewWidth(), options.getPreviewHeight());
+    }
+
     private static void check(boolean ok, String message) {
         if (!ok) throw new AssertionError(message);
     }

scripts/camera-regression/extract-android.py

@@ -48,6 +48,7 @@ def method(signature):
     private volatile FrameFormat frameFormat = FrameFormat.JPEG;
     private volatile long frameIntervalNanos;
     private long lastFrameNanos;
+    private byte[] nv21Buffer;
     public interface FrameListener { void onFrame(CameraFrame frame); }
     public void deliver(Object proxy) { onImageProxy(proxy); }
 ''' + methods + '\n}\n')