diff --git a/.github/actions/maestro-ios/action.yml b/.github/actions/maestro-ios/action.yml index b5e4036d6280..1259868963a3 100644 --- a/.github/actions/maestro-ios/action.yml +++ b/.github/actions/maestro-ios/action.yml @@ -26,6 +26,9 @@ inputs: required: false default: '' description: Maestro device OS name, such as iOS-26-2 + argent-label: + required: true + description: Label the calling workflow is registered under on Argent Cloud; the job logs in as it runs: using: composite @@ -33,11 +36,13 @@ runs: - name: Installing Maestro shell: bash run: export MAESTRO_VERSION=2.6.1; curl -Ls "https://get.maestro.mobile.dev" | bash - - name: Installing Maestro dependencies - shell: bash - run: | - brew tap facebook/fb - brew install facebook/fb/idb-companion + - name: Install sim-remote + uses: software-mansion-labs/argent-cloud-actions/install@v2 + - name: Acquire remote simulator + # Logs in with the job's OIDC id token; released in the post step. + uses: software-mansion-labs/argent-cloud-actions/acquire@v2 + with: + label: ${{ inputs.argent-label }} - name: Set up JDK 17 uses: actions/setup-java@v5 with: @@ -57,6 +62,8 @@ runs: cd ${{ inputs.working-directory }} yarn start & sleep 5 # to give metro time to load + - name: Install Maestro shims + uses: software-mansion-labs/argent-cloud-actions/maestro-shims@v2 - name: Run tests id: run-tests shell: bash diff --git a/.github/actions/post-pr-comment/action.yml b/.github/actions/post-pr-comment/action.yml index e289df1c48a7..fb8b44114ade 100644 --- a/.github/actions/post-pr-comment/action.yml +++ b/.github/actions/post-pr-comment/action.yml @@ -12,6 +12,10 @@ inputs: marker: description: 'HTML comment marker to identify this comment' required: true + issue-number: + description: 'Pull request to comment on; defaults to the one the workflow runs for' + required: false + default: '' runs: using: composite steps: @@ -21,6 +25,7 @@ runs: SECTIONS_INPUT: ${{ inputs.sections }} MARKER_INPUT: ${{ inputs.marker }} HEADER_INPUT: ${{ inputs.header }} + ISSUE_NUMBER_INPUT: ${{ inputs.issue-number }} with: script: | const marker = process.env.MARKER_INPUT; @@ -30,7 +35,7 @@ runs: .filter(s => s.trim()); const {owner, repo} = context.repo; - const issue_number = context.issue.number; + const issue_number = Number(process.env.ISSUE_NUMBER_INPUT) || context.issue.number; const {data: comments} = await github.rest.issues.listComments({ owner, repo, issue_number, diff --git a/.github/workflow-scripts/maestro-ios.js b/.github/workflow-scripts/maestro-ios.js index a9782ec6506f..2249081d3912 100644 --- a/.github/workflow-scripts/maestro-ios.js +++ b/.github/workflow-scripts/maestro-ios.js @@ -81,7 +81,24 @@ function installAppOnSimulator(appPath, udid) { childProcess.execSync(`xcrun simctl install "${udid}" "${appPath}"`); } +// A remote simulator's localhost is the remote host, so Metro needs a tunnel. +function exposeMetroToSimulator(udid) { + const simRemote = process.env.SIM_REMOTE_BIN; + if (!simRemote) { + return; + } + + console.log('Exposing local Metro port 8081 to the remote simulator'); + childProcess.execSync(`"${simRemote}" reverse start "${udid}" 8081`, { + stdio: 'inherit', + }); +} + function bringSimulatorInForeground() { + // Nothing to foreground on a remote simulator. + if (process.env.SIM_REMOTE_BIN) { + return; + } console.log('Bringing simulator in foreground'); childProcess.execSync('open -a simulator'); } @@ -266,6 +283,9 @@ async function main(args = process.argv.slice(2)) { launchSimulator(simulator); installAppOnSimulator(appPath, simulator.udid); bringSimulatorInForeground(); + if (isDebug) { + exposeMetroToSimulator(simulator.udid); + } await launchAppOnSimulator(appId, simulator.udid, isDebug); await executeFlows(appId, simulator.udid, maestroFlow, jsengine); console.log('Test finished'); diff --git a/.github/workflows/e2e-ios-on-demand.yml b/.github/workflows/e2e-ios-on-demand.yml new file mode 100644 index 000000000000..879d864d1457 --- /dev/null +++ b/.github/workflows/e2e-ios-on-demand.yml @@ -0,0 +1,121 @@ +name: E2E iOS on demand + +# Runs the iOS Maestro E2E suites against a pull request when a maintainer +# comments `@react-native-bot e2e-tests` on it. The apps come from the +# artifacts a run of the pull request's head already built, the Maestro flows +# from the pull request, and the workflows and scripts from the default branch. + +on: + issue_comment: + types: [created] + +permissions: + contents: read + +# Only a trigger comment shares the pull request's group, so a newer request +# supersedes a running one and any other comment cancels nothing. +concurrency: + group: e2e-ios-on-demand-${{ github.event.issue.number }}-${{ startsWith(github.event.comment.body, '@react-native-bot e2e-tests') && 'run' || github.event.comment.id }} + cancel-in-progress: true + +jobs: + request: + if: >- + github.event.issue.pull_request != null && + startsWith(github.event.comment.body, '@react-native-bot e2e-tests') && + contains(fromJson('["OWNER","MEMBER","COLLABORATOR"]'), github.event.comment.author_association) + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + pull-requests: write + outputs: + head-sha: ${{ steps.find.outputs.head-sha }} + run-id: ${{ steps.find.outputs.run-id }} + steps: + - name: Find the pull request's build + id: find + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + run: | + set -euo pipefail + sha=$(gh api "repos/$GH_REPO/pulls/$PR" --jq .head.sha) + needed="RNTesterApp-NewArch-Debug RNTesterApp-NewArch-Release RNTestProject-Debug RNTestProject-Release react-native-package" + run_id= + for id in $(gh api "repos/$GH_REPO/actions/runs?head_sha=$sha&per_page=30" --jq '.workflow_runs[].id'); do + have=$(gh api "repos/$GH_REPO/actions/runs/$id/artifacts?per_page=100" --jq '.artifacts[].name') + missing= + for name in $needed; do + grep -qx "$name" <<<"$have" || missing="$missing $name" + done + if [ -z "$missing" ]; then + run_id=$id + break + fi + done + if [ -z "$run_id" ]; then + echo "::error::no run of $sha has built all of: $needed" + exit 1 + fi + echo "head-sha=$sha" >> "$GITHUB_OUTPUT" + echo "run-id=$run_id" >> "$GITHUB_OUTPUT" + - name: Acknowledge the request + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + COMMENT: ${{ github.event.comment.id }} + run: gh api -X POST "repos/$GH_REPO/issues/comments/$COMMENT/reactions" -f content=rocket > /dev/null + + rntester: + needs: request + uses: ./.github/workflows/e2e-ios-rntester.yml + permissions: + contents: read + actions: read + id-token: write + with: + argent-label: facebook.e2e-ios-on-demand + artifact-run-id: ${{ needs.request.outputs.run-id }} + flows-ref: ${{ needs.request.outputs.head-sha }} + fail-on-error: true + + templateapp: + needs: request + uses: ./.github/workflows/e2e-ios-templateapp.yml + permissions: + contents: read + actions: read + id-token: write + with: + argent-label: facebook.e2e-ios-on-demand + artifact-run-id: ${{ needs.request.outputs.run-id }} + flows-ref: ${{ needs.request.outputs.head-sha }} + fail-on-error: true + + report: + needs: [request, rntester, templateapp] + if: always() && needs.request.result != 'skipped' + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Comment the results + env: + GH_TOKEN: ${{ secrets.REACT_NATIVE_BOT_GITHUB_TOKEN || github.token }} + GH_REPO: ${{ github.repository }} + PR: ${{ github.event.issue.number }} + REQUEST: ${{ needs.request.result }} + HEAD_SHA: ${{ needs.request.outputs.head-sha }} + RNTESTER: ${{ needs.rntester.outputs.status }} + TEMPLATEAPP: ${{ needs.templateapp.outputs.status }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + if [ "$REQUEST" != "success" ]; then + body="iOS E2E: no run of the pull request's head has built the apps yet, see [the request]($RUN_URL)." + else + body="iOS E2E on ${HEAD_SHA:0:7} ([run]($RUN_URL)): RNTester **${RNTESTER:-skipped}**, template app **${TEMPLATEAPP:-skipped}**." + fi + gh api -X POST "repos/$GH_REPO/issues/$PR/comments" -f body="$body" > /dev/null diff --git a/.github/workflows/e2e-ios-rntester.yml b/.github/workflows/e2e-ios-rntester.yml index c841c409b9ea..5e13d42c0ecd 100644 --- a/.github/workflows/e2e-ios-rntester.yml +++ b/.github/workflows/e2e-ios-rntester.yml @@ -2,6 +2,8 @@ name: E2E iOS RNTester permissions: contents: read + actions: read + id-token: write on: workflow_call: @@ -9,6 +11,18 @@ on: fail-on-error: type: boolean default: false + artifact-run-id: + description: Run to take the app artifacts from; defaults to this run + type: string + default: '' + flows-ref: + description: Ref to take the Maestro flows from; defaults to the checked-out tree + type: string + default: '' + argent-label: + description: Label the calling workflow is registered under on Argent Cloud + type: string + required: true outputs: status: description: 'The result of the E2E tests (success or failure)' @@ -16,7 +30,10 @@ on: jobs: test: - runs-on: macos-26-large + # A pull_request run has no id token to log into the remote simulators + # with; the suite runs on request from e2e-ios-on-demand.yml instead. + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest outputs: status: ${{ steps.report-status.outputs.status }} strategy: @@ -33,6 +50,20 @@ jobs: with: name: RNTesterApp-NewArch-${{ matrix.flavor }} path: /tmp/RNTesterBuild/RNTester.app + run-id: ${{ inputs.artifact-run-id || github.run_id }} + github-token: ${{ github.token }} + - name: Checkout the Maestro flows under test + if: ${{ inputs.flows-ref != '' }} + uses: actions/checkout@v6 + with: + ref: ${{ inputs.flows-ref }} + path: .flows-under-test + sparse-checkout: packages/rn-tester/.maestro + - name: Use the Maestro flows under test + if: ${{ inputs.flows-ref != '' }} + run: | + rm -rf packages/rn-tester/.maestro + mv .flows-under-test/packages/rn-tester/.maestro packages/rn-tester/.maestro - name: Check downloaded folder content run: ls -lR /tmp/RNTesterBuild - name: Run E2E Tests @@ -40,6 +71,7 @@ jobs: continue-on-error: true uses: ./.github/actions/maestro-ios with: + argent-label: ${{ inputs.argent-label }} app-path: '/tmp/RNTesterBuild/RNTester.app' app-id: com.meta.RNTester.localDevelopment maestro-flow: ./packages/rn-tester/.maestro/ @@ -59,8 +91,11 @@ jobs: status: ${{ steps.check.outputs.status }} steps: - id: check + # A skipped test job is not a failure: the tests do not run on a + # pull_request run. run: | - if [[ "${{ needs.test.result }}" != "success" || + test="${{ needs.test.result }}" + if [[ ("$test" != "success" && "$test" != "skipped") || "${{ needs.test.outputs.status }}" == "failure" ]]; then echo "status=failure" >> $GITHUB_OUTPUT else diff --git a/.github/workflows/e2e-ios-suggest.yml b/.github/workflows/e2e-ios-suggest.yml new file mode 100644 index 000000000000..2d732250b22b --- /dev/null +++ b/.github/workflows/e2e-ios-suggest.yml @@ -0,0 +1,48 @@ +name: Suggest the iOS E2E run + +# Once a pull request's Test All run has built the apps, tells maintainers how +# to run the iOS Maestro E2E suites against them (see e2e-ios-on-demand.yml). + +on: + workflow_run: + workflows: [Test All] + types: [completed] + +permissions: + contents: read + +jobs: + suggest: + if: github.event.workflow_run.event == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + actions: read + pull-requests: write + steps: + - uses: actions/checkout@v6 + - name: Find the pull request whose apps were built + id: find + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.event.workflow_run.id }} + HEAD_SHA: ${{ github.event.workflow_run.head_sha }} + run: | + set -euo pipefail + have=$(gh api "repos/$GH_REPO/actions/runs/$RUN_ID/artifacts?per_page=100" --jq '.artifacts[].name') + for name in RNTesterApp-NewArch-Debug RNTesterApp-NewArch-Release RNTestProject-Debug RNTestProject-Release react-native-package; do + if ! grep -qx "$name" <<<"$have"; then + echo "$name was not built; nothing to suggest" + exit 0 + fi + done + pr=$(gh api "repos/$GH_REPO/commits/$HEAD_SHA/pulls" --jq '.[0].number // empty') + echo "pr=$pr" >> "$GITHUB_OUTPUT" + - name: Post the hint + if: steps.find.outputs.pr != '' + uses: ./.github/actions/post-pr-comment + with: + issue-number: ${{ steps.find.outputs.pr }} + marker: '' + sections: '["Maintainers: to run the iOS Maestro E2E suites on remote simulators against this pull request, comment `@react-native-bot e2e-tests`"]' diff --git a/.github/workflows/e2e-ios-templateapp.yml b/.github/workflows/e2e-ios-templateapp.yml index e91b5ba0f72b..34073c92b248 100644 --- a/.github/workflows/e2e-ios-templateapp.yml +++ b/.github/workflows/e2e-ios-templateapp.yml @@ -2,6 +2,8 @@ name: E2E iOS Template App permissions: contents: read + actions: read + id-token: write on: workflow_call: @@ -9,13 +11,27 @@ on: fail-on-error: type: boolean default: false + artifact-run-id: + description: Run to take the app and package artifacts from; defaults to this run, which then builds them + type: string + default: '' + flows-ref: + description: Ref to take the Maestro flows from; defaults to the checked-out tree + type: string + default: '' + argent-label: + description: Label the calling workflow is registered under on Argent Cloud + type: string + required: true outputs: status: description: 'The result of the E2E tests (success or failure)' value: ${{ jobs.report.outputs.status }} jobs: + # Building needs Xcode; the tests drive a remote simulator from Linux. build: + if: inputs.artifact-run-id == '' runs-on: macos-26-large strategy: fail-fast: false @@ -99,7 +115,12 @@ jobs: test: needs: build - runs-on: macos-26-large + # A pull_request run has no id token to log into the remote simulators + # with; the suite runs on request from e2e-ios-on-demand.yml instead. + if: >- + always() && github.event_name != 'pull_request' && + (needs.build.result == 'success' || needs.build.result == 'skipped') + runs-on: ubuntu-latest outputs: status: ${{ steps.report-status.outputs.status }} strategy: @@ -118,6 +139,8 @@ jobs: with: name: RNTestProject-${{ matrix.flavor }} path: /tmp/RNTestProjectBuild/RNTestProject.app + run-id: ${{ inputs.artifact-run-id || github.run_id }} + github-token: ${{ github.token }} - name: Check downloaded folder content run: ls -l /tmp/RNTestProjectBuild/RNTestProject.app - name: Download React Native Package @@ -126,6 +149,20 @@ jobs: with: name: react-native-package path: /tmp/react-native-tmp + run-id: ${{ inputs.artifact-run-id || github.run_id }} + github-token: ${{ github.token }} + - name: Checkout the Maestro flows under test + if: ${{ inputs.flows-ref != '' }} + uses: actions/checkout@v6 + with: + ref: ${{ inputs.flows-ref }} + path: .flows-under-test + sparse-checkout: scripts/e2e/.maestro + - name: Use the Maestro flows under test + if: ${{ inputs.flows-ref != '' }} + run: | + rm -rf scripts/e2e/.maestro + mv .flows-under-test/scripts/e2e/.maestro scripts/e2e/.maestro - name: Configure git if: ${{ matrix.flavor == 'Debug' }} shell: bash @@ -152,6 +189,7 @@ jobs: continue-on-error: true uses: ./.github/actions/maestro-ios with: + argent-label: ${{ inputs.argent-label }} app-path: '/tmp/RNTestProjectBuild/RNTestProject.app' app-id: org.reactjs.native.example.RNTestProject maestro-flow: ./scripts/e2e/.maestro/ @@ -170,9 +208,13 @@ jobs: status: ${{ steps.check.outputs.status }} steps: - id: check + # Skipped jobs are not failures: the build is skipped when the apps + # come from another run, the tests on a pull_request run. run: | - if [[ "${{ needs.build.result }}" != "success" || - "${{ needs.test.result }}" != "success" || + build="${{ needs.build.result }}" + test="${{ needs.test.result }}" + if [[ ("$build" != "success" && "$build" != "skipped") || + ("$test" != "success" && "$test" != "skipped") || "${{ needs.test.outputs.status }}" == "failure" ]]; then echo "status=failure" >> $GITHUB_OUTPUT else diff --git a/.github/workflows/test-all.yml b/.github/workflows/test-all.yml index f724039a046f..4552759fab80 100644 --- a/.github/workflows/test-all.yml +++ b/.github/workflows/test-all.yml @@ -10,6 +10,8 @@ on: permissions: contents: read + actions: read + id-token: write jobs: set_release_type: @@ -193,12 +195,16 @@ jobs: needs: test_ios_rntester if: ${{ needs.test_ios_rntester.result == 'success' }} uses: ./.github/workflows/e2e-ios-rntester.yml + with: + argent-label: facebook.test-all secrets: inherit test_e2e_ios_rntester_retry_1: needs: test_e2e_ios_rntester if: ${{ always() && needs.test_e2e_ios_rntester.outputs.status == 'failure' }} uses: ./.github/workflows/e2e-ios-rntester.yml + with: + argent-label: facebook.test-all secrets: inherit test_e2e_ios_rntester_retry_2: @@ -206,6 +212,7 @@ jobs: if: ${{ always() && needs.test_e2e_ios_rntester_retry_1.outputs.status == 'failure' }} uses: ./.github/workflows/e2e-ios-rntester.yml with: + argent-label: facebook.test-all fail-on-error: true secrets: inherit @@ -220,12 +227,16 @@ jobs: needs: [build_npm_package, prebuild_apple_dependencies, check_code_changes] if: needs.check_code_changes.outputs.should_test_ios == 'true' uses: ./.github/workflows/e2e-ios-templateapp.yml + with: + argent-label: facebook.test-all secrets: inherit test_e2e_ios_templateapp_retry_1: needs: test_e2e_ios_templateapp if: ${{ always() && needs.test_e2e_ios_templateapp.outputs.status == 'failure' }} uses: ./.github/workflows/e2e-ios-templateapp.yml + with: + argent-label: facebook.test-all secrets: inherit test_e2e_ios_templateapp_retry_2: @@ -233,6 +244,7 @@ jobs: if: ${{ always() && needs.test_e2e_ios_templateapp_retry_1.outputs.status == 'failure' }} uses: ./.github/workflows/e2e-ios-templateapp.yml with: + argent-label: facebook.test-all fail-on-error: true secrets: inherit