diff --git a/.github/actions/install-android-ndk/action.yml b/.github/actions/install-android-ndk/action.yml new file mode 100644 index 0000000000..372ad2f88d --- /dev/null +++ b/.github/actions/install-android-ndk/action.yml @@ -0,0 +1,54 @@ +name: 'Install Android NDK' +description: 'Install the NDK and CMake that Flutter plugins build native code with' +runs: + using: "composite" + steps: + - name: Install the NDK and CMake + shell: bash + run: | + # Plugins with native code (e.g. `jni`, used by `path_provider_android`) + # otherwise make Gradle download these mid-build. Needs Flutter set up. + # + # The NDK Flutter sets for plugins (`flutter.ndkVersion`), read from the + # installed Flutter so an upgrade keeps it in step. + ndk=$(grep -oP 'ndkVersion: String = "\K[0-9.]+' \ + "$FLUTTER_ROOT/packages/flutter_tools/gradle/src/main/kotlin/FlutterExtension.kt" || true) + if [ -z "$ndk" ]; then + echo "::error::Could not read flutter.ndkVersion from the Flutter SDK" + exit 1 + fi + + # The CMake version the Android Gradle Plugin builds native code with. + cmake=3.22.1 + # By full path, so the runner's own SDK is the one updated. + sdkmanager="$ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager" + if [ ! -x "$sdkmanager" ]; then + echo "::error::sdkmanager not found at $sdkmanager" + exit 1 + fi + ndk_dir="$ANDROID_HOME/ndk/$ndk" + installed() { + [ -f "$ndk_dir/source.properties" ] && [ -x "$ANDROID_HOME/cmake/$cmake/bin/cmake" ] + } + + # sdkmanager unpacks the download itself. Gradle's own NDK download can + # arrive corrupt ("Archive is not a ZIP archive"), and then every retry + # of the build reuses the broken copy. + # A stalled sdkmanager fails its attempt instead of holding the job; three + # 5-minute attempts stay inside the 30-minute build job. + for attempt in 1 2 3; do + installed && break + if [ "$attempt" -gt 1 ]; then + echo "::warning::Installing NDK $ndk and CMake $cmake failed; retrying (attempt $attempt)" + rm -rf "$ndk_dir" "$ANDROID_HOME/cmake/$cmake" "$ANDROID_HOME/.temp" + sleep 15 + fi + yes | timeout 300 "$sdkmanager" --install "ndk;$ndk" "cmake;$cmake" > "$RUNNER_TEMP/sdkmanager.log" 2>&1 || true + done + + if ! installed; then + tail -n 30 "$RUNNER_TEMP/sdkmanager.log" + echo "::error::Could not install NDK $ndk and CMake $cmake" + exit 1 + fi + echo "Installed NDK $ndk and CMake $cmake" diff --git a/.github/workflows/distribute_external.yml b/.github/workflows/distribute_external.yml index 774032d694..aad833cc65 100644 --- a/.github/workflows/distribute_external.yml +++ b/.github/workflows/distribute_external.yml @@ -69,6 +69,8 @@ jobs: - uses: ./.github/actions/setup-flutter + - uses: ./.github/actions/install-android-ndk + - uses: ./.github/actions/setup-ruby - name: Distribute to S3 diff --git a/.github/workflows/distribute_internal.yml b/.github/workflows/distribute_internal.yml index 907f83a5dc..4fa787c17e 100644 --- a/.github/workflows/distribute_internal.yml +++ b/.github/workflows/distribute_internal.yml @@ -73,6 +73,8 @@ jobs: - uses: ./.github/actions/setup-flutter + - uses: ./.github/actions/install-android-ndk + - name: "Install Tools" run: flutter pub global activate melos diff --git a/.github/workflows/e2e_test.yml b/.github/workflows/e2e_test.yml index 326b680a84..ee40490d2f 100644 --- a/.github/workflows/e2e_test.yml +++ b/.github/workflows/e2e_test.yml @@ -110,6 +110,8 @@ jobs: - uses: ./.github/actions/setup-flutter + - uses: ./.github/actions/install-android-ndk + - uses: ./.github/actions/enable-kvm - name: Bootstrap diff --git a/.github/workflows/e2e_test_cron.yml b/.github/workflows/e2e_test_cron.yml index fe6ca4859f..09e99dc534 100644 --- a/.github/workflows/e2e_test_cron.yml +++ b/.github/workflows/e2e_test_cron.yml @@ -54,6 +54,8 @@ jobs: - uses: ./.github/actions/setup-flutter + - uses: ./.github/actions/install-android-ndk + - uses: ./.github/actions/enable-kvm # The emulator, Gradle and flutter test together can briefly need more than the runner's diff --git a/.github/workflows/stream_flutter_workflow.yml b/.github/workflows/stream_flutter_workflow.yml index 0f5c7f7884..16c061be2b 100644 --- a/.github/workflows/stream_flutter_workflow.yml +++ b/.github/workflows/stream_flutter_workflow.yml @@ -182,6 +182,8 @@ jobs: if: matrix.platform == 'ios' - uses: ./.github/actions/cache-gradle if: matrix.platform == 'android' + - uses: ./.github/actions/install-android-ndk + if: matrix.platform == 'android' - name: "Install Tools" run: flutter pub global activate melos - name: "Bootstrap Workspace" diff --git a/e2e_app/fastlane/Fastfile b/e2e_app/fastlane/Fastfile index d9eeb6c9d6..ff3b428d03 100644 --- a/e2e_app/fastlane/Fastfile +++ b/e2e_app/fastlane/Fastfile @@ -240,6 +240,56 @@ def run_time_limit(target, selected, files) [(15 + (2 * [tests, 1].max)) * 60, 45 * 60].min end +# How long the app may take, once its APK starts installing, to start its first +# test. A healthy run installs and starts its first test within seconds; seen on CI, +# an app that installed but never started a test would otherwise hold the run until +# its time limit. +STARTUP_LIMIT = 4 * 60 + +# Printed by the app as each test starts and ends. +TEST_REPORTED = /ALLURE-(RESULT|ATTACH)::|E2E-TEST::/ + +# Runs [argv], whose output is copied to [log], and returns whether it succeeded. +# A run whose app installs but starts no test within STARTUP_LIMIT calls +# [on_stalled], is stopped and fails; a run that can't be started fails too. +# [argv] runs from the app folder with no shell in between, so its process group +# is the one `timeout` leads and stopping it reaches flutter too. +def run_watching_startup(argv, log, on_stalled:) + pid = + begin + Process.spawn(*argv, pgroup: true, chdir: root_path) + rescue StandardError => e + UI.important("Could not start flutter test: #{e.message}") + return false + end + installed_at = nil + loop do + _, status = Process.wait2(pid, Process::WNOHANG) + return status.success? if status + + lines = File.exist?(log) ? File.readlines(log) : [] + install = lines.rindex { |line| line.include?('Installing build/') } + reported = install && lines.drop(install + 1).any? { |line| line.match?(TEST_REPORTED) } + # A test started: only the run's time limit applies from here. + return Process.wait2(pid).last.success? if reported + + installed_at ||= Time.now if install + if installed_at && Time.now - installed_at >= STARTUP_LIMIT + on_stalled.call + # The whole group: flutter, the tee pipeline and the `timeout` guard. + Process.kill('TERM', -pid) rescue nil + begin + Timeout.timeout(30) { Process.wait(pid) } + rescue Timeout::Error + Process.kill('KILL', -pid) rescue nil + Process.wait(pid) rescue nil + end + return false + end + sleep(5) + end +end + def boot_simulator(udid:) return if `xcrun simctl list devices booted`.to_s.include?(udid) return unless `xcrun simctl list devices`.to_s.include?(udid) @@ -497,14 +547,16 @@ lane :run_e2e_test do |options| "{ grep --line-buffered -vE 'ALLURE-(RESULT|ATTACH)::|E2E-TEST::' || true; }" time_limit = run_time_limit(target, selected, files) # `timeout` signals its whole process group, so flutter and the tee pipeline go too. - guard = timeout_bin.empty? ? '' : "#{timeout_bin} --kill-after=30 #{time_limit} " + guard = timeout_bin.empty? ? [] : [timeout_bin, '--kill-after=30', time_limit.to_s] started_at = Time.now - ok = begin - sh("#{guard}bash -o pipefail -c #{Shellwords.escape(cmd)}") - true - rescue StandardError - false - end + ok = run_watching_startup( + [*guard, 'bash', '-o', 'pipefail', '-c', cmd], + attempt_log, + on_stalled: -> { + UI.important("#{target} (attempt #{attempt}/#{retries + 1}): the app installed but no test " \ + "started within #{STARTUP_LIMIT / 60} min") + }, + ) if !ok && !guard.empty? && Time.now - started_at >= time_limit UI.important("#{target} (attempt #{attempt}/#{retries + 1}): killed after #{time_limit / 60} min " \ 'with no exit from flutter test; the device may have died') diff --git a/e2e_app/integration_test/allure/allure.dart b/e2e_app/integration_test/allure/allure.dart index 98ab1a9225..4ade9bc435 100644 --- a/e2e_app/integration_test/allure/allure.dart +++ b/e2e_app/integration_test/allure/allure.dart @@ -7,7 +7,8 @@ const allureResultMarker = 'ALLURE-RESULT::'; const allureAttachmentMarker = 'ALLURE-ATTACH::'; /// Lightweight per-test outcome marker, emitted once per test as -/// `E2E-TEST::::`. The Fastlane lane parses these to +/// `E2E-TEST::::`, and as `E2E-TEST::started::…` when the +/// test starts. The Fastlane lane parses these to /// retry **only** the tests that failed in an attempt (rather than re-running /// the whole file); the name is base64-encoded to survive any characters in /// the test description. Separate from [allureResultMarker] (which carries the @@ -110,6 +111,9 @@ class Allure { for (final entry in labels.entries) {'name': entry.key, 'value': entry.value}, ], ); + + // Lets the Fastlane lane tell a test that runs long from an app that never started one. + print('${allureTestStatusMarker}started::${base64.encode(utf8.encode(name))}'); } /// Records a BDD step marker. The statements that run after this call, up to