ci(repo): install the Android NDK before builds and retry a stalled e2e start

.github/actions/install-android-ndk/action.yml

@@ -0,0 +1,54 @@
+name: 'Install Android NDK'
+description: 'Install the NDK and CMake that Flutter plugins build native code with'
+runs:
+  using: "composite"
+  steps:
+    - name: Install the NDK and CMake
+      shell: bash
+      run: |
+        # Plugins with native code (e.g. `jni`, used by `path_provider_android`)
+        # otherwise make Gradle download these mid-build. Needs Flutter set up.
+        #
+        # The NDK Flutter sets for plugins (`flutter.ndkVersion`), read from the
+        # installed Flutter so an upgrade keeps it in step.
+        ndk=$(grep -oP 'ndkVersion: String = "\K[0-9.]+' \
+          "$FLUTTER_ROOT/packages/flutter_tools/gradle/src/main/kotlin/FlutterExtension.kt" || true)
+        if [ -z "$ndk" ]; then
+          echo "::error::Could not read flutter.ndkVersion from the Flutter SDK"
+          exit 1
+        fi
+
+        # The CMake version the Android Gradle Plugin builds native code with.
+        cmake=3.22.1
+        # By full path, so the runner's own SDK is the one updated.
+        sdkmanager="$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager"
+        if [ ! -x "$sdkmanager" ]; then
+          echo "::error::sdkmanager not found at $sdkmanager"
+          exit 1
+        fi
+        ndk_dir="$ANDROID_HOME/ndk/$ndk"
+        installed() {
+          [ -f "$ndk_dir/source.properties" ] && [ -x "$ANDROID_HOME/cmake/$cmake/bin/cmake" ]
+        }
+
+        # sdkmanager unpacks the download itself. Gradle's own NDK download can
+        # arrive corrupt ("Archive is not a ZIP archive"), and then every retry
+        # of the build reuses the broken copy.
+        # A stalled sdkmanager fails its attempt instead of holding the job; three
+        # 5-minute attempts stay inside the 30-minute build job.
+        for attempt in 1 2 3; do
+          installed && break
+          if [ "$attempt" -gt 1 ]; then
+            echo "::warning::Installing NDK $ndk and CMake $cmake failed; retrying (attempt $attempt)"
+            rm -rf "$ndk_dir" "$ANDROID_HOME/cmake/$cmake" "$ANDROID_HOME/.temp"
+            sleep 15
+          fi
+          yes | timeout 300 "$sdkmanager" --install "ndk;$ndk" "cmake;$cmake" > "$RUNNER_TEMP/sdkmanager.log" 2>&1 || true
+        done
+
+        if ! installed; then
+          tail -n 30 "$RUNNER_TEMP/sdkmanager.log"
+          echo "::error::Could not install NDK $ndk and CMake $cmake"
+          exit 1
+        fi
+        echo "Installed NDK $ndk and CMake $cmake"

.github/workflows/distribute_external.yml

@@ -69,6 +69,8 @@ jobs:
 
       - uses: ./.github/actions/setup-flutter
 
+      - uses: ./.github/actions/install-android-ndk
+
       - uses: ./.github/actions/setup-ruby
 
       - name: Distribute to S3

.github/workflows/distribute_internal.yml

@@ -73,6 +73,8 @@ jobs:
 
       - uses: ./.github/actions/setup-flutter
 
+      - uses: ./.github/actions/install-android-ndk
+
       - name: "Install Tools"
         run: flutter pub global activate melos
 

.github/workflows/e2e_test.yml

@@ -110,6 +110,8 @@ jobs:
 
       - uses: ./.github/actions/setup-flutter
 
+      - uses: ./.github/actions/install-android-ndk
+
       - uses: ./.github/actions/enable-kvm
 
       - name: Bootstrap

.github/workflows/e2e_test_cron.yml

@@ -54,6 +54,8 @@ jobs:
 
       - uses: ./.github/actions/setup-flutter
 
+      - uses: ./.github/actions/install-android-ndk
+
       - uses: ./.github/actions/enable-kvm
 
       # The emulator, Gradle and flutter test together can briefly need more than the runner's

.github/workflows/stream_flutter_workflow.yml

@@ -182,6 +182,8 @@ jobs:
         if: matrix.platform == 'ios'
       - uses: ./.github/actions/cache-gradle
         if: matrix.platform == 'android'
+      - uses: ./.github/actions/install-android-ndk
+        if: matrix.platform == 'android'
       - name: "Install Tools"
         run: flutter pub global activate melos
       - name: "Bootstrap Workspace"

e2e_app/fastlane/Fastfile

@@ -240,6 +240,56 @@ def run_time_limit(target, selected, files)
   [(15 + (2 * [tests, 1].max)) * 60, 45 * 60].min
 end
 
+# How long the app may take, once its APK starts installing, to start its first
+# test. A healthy run installs and starts its first test within seconds; seen on CI,
+# an app that installed but never started a test would otherwise hold the run until
+# its time limit.
+STARTUP_LIMIT = 4 * 60
+
+# Printed by the app as each test starts and ends.
+TEST_REPORTED = /ALLURE-(RESULT|ATTACH)::|E2E-TEST::/
+
+# Runs [argv], whose output is copied to [log], and returns whether it succeeded.
+# A run whose app installs but starts no test within STARTUP_LIMIT calls
+# [on_stalled], is stopped and fails; a run that can't be started fails too.
+# [argv] runs from the app folder with no shell in between, so its process group
+# is the one `timeout` leads and stopping it reaches flutter too.
+def run_watching_startup(argv, log, on_stalled:)
+  pid =
+    begin
+      Process.spawn(*argv, pgroup: true, chdir: root_path)
+    rescue StandardError => e
+      UI.important("Could not start flutter test: #{e.message}")
+      return false
+    end
+  installed_at = nil
+  loop do
+    _, status = Process.wait2(pid, Process::WNOHANG)
+    return status.success? if status
+
+    lines = File.exist?(log) ? File.readlines(log) : []
+    install = lines.rindex { |line| line.include?('Installing build/') }
+    reported = install && lines.drop(install + 1).any? { |line| line.match?(TEST_REPORTED) }
+    # A test started: only the run's time limit applies from here.
+    return Process.wait2(pid).last.success? if reported
+
+    installed_at ||= Time.now if install
+    if installed_at && Time.now - installed_at >= STARTUP_LIMIT
+      on_stalled.call
+      # The whole group: flutter, the tee pipeline and the `timeout` guard.
+      Process.kill('TERM', -pid) rescue nil
+      begin
+        Timeout.timeout(30) { Process.wait(pid) }
+      rescue Timeout::Error
+        Process.kill('KILL', -pid) rescue nil
+        Process.wait(pid) rescue nil
+      end
+      return false
+    end
+    sleep(5)
+  end
+end
+
 def boot_simulator(udid:)
   return if `xcrun simctl list devices booted`.to_s.include?(udid)
   return unless `xcrun simctl list devices`.to_s.include?(udid)
@@ -497,14 +547,16 @@ lane :run_e2e_test do |options|
                 "{ grep --line-buffered -vE 'ALLURE-(RESULT|ATTACH)::|E2E-TEST::' || true; }"
           time_limit = run_time_limit(target, selected, files)
           # `timeout` signals its whole process group, so flutter and the tee pipeline go too.
-          guard = timeout_bin.empty? ? '' : "#{timeout_bin} --kill-after=30 #{time_limit} "
+          guard = timeout_bin.empty? ? [] : [timeout_bin, '--kill-after=30', time_limit.to_s]
           started_at = Time.now
-          ok = begin
-            sh("#{guard}bash -o pipefail -c #{Shellwords.escape(cmd)}")
-            true
-          rescue StandardError
-            false
-          end
+          ok = run_watching_startup(
+            [*guard, 'bash', '-o', 'pipefail', '-c', cmd],
+            attempt_log,
+            on_stalled: -> {
+              UI.important("#{target} (attempt #{attempt}/#{retries + 1}): the app installed but no test " \
+                           "started within #{STARTUP_LIMIT / 60} min")
+            },
+          )
           if !ok && !guard.empty? && Time.now - started_at >= time_limit
             UI.important("#{target} (attempt #{attempt}/#{retries + 1}): killed after #{time_limit / 60} min " \
                          'with no exit from flutter test; the device may have died')

e2e_app/integration_test/allure/allure.dart

@@ -7,7 +7,8 @@ const allureResultMarker = 'ALLURE-RESULT::';
 const allureAttachmentMarker = 'ALLURE-ATTACH::';
 
 /// Lightweight per-test outcome marker, emitted once per test as
-/// `E2E-TEST::<status>::<base64(name)>`. The Fastlane lane parses these to
+/// `E2E-TEST::<status>::<base64(name)>`, and as `E2E-TEST::started::…` when the
+/// test starts. The Fastlane lane parses these to
 /// retry **only** the tests that failed in an attempt (rather than re-running
 /// the whole file); the name is base64-encoded to survive any characters in
 /// the test description. Separate from [allureResultMarker] (which carries the
@@ -110,6 +111,9 @@ class Allure {
         for (final entry in labels.entries) {'name': entry.key, 'value': entry.value},
       ],
     );
+
+    // Lets the Fastlane lane tell a test that runs long from an app that never started one.
+    print('${allureTestStatusMarker}started::${base64.encode(utf8.encode(name))}');
   }
 
   /// Records a BDD step marker. The statements that run after this call, up to