From cf53dab44023d38f745a01c7f1b09279ee4b7bd2 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Fri, 14 Aug 2026 18:30:59 -0400 Subject: [PATCH 1/5] Add new commands to rolling upgrade suite, and trigger both in CI --- .../dist/src/main/compose/upgrade/test.sh | 10 ++++--- .../callbacks/1.3.0/callback.sh | 2 +- .../callbacks/1.4.0/callback.sh | 4 +-- .../callbacks/2.0.0/callback.sh | 4 +-- .../callbacks/common/callback.sh | 6 ++-- .../upgrades/non-rolling-upgrade/driver.sh | 2 +- .../callbacks/common/callback.sh | 6 ++-- .../upgrades/rolling-upgrade/driver.sh | 4 +-- .../check-finalization.robot | 2 +- .../upgrade/{ => non-rolling}/finalize.robot | 2 +- .../upgrade/{ => non-rolling}/lib.robot | 6 ++-- .../upgrade/rolling/check-finalization.robot | 30 +++++++++++++++++++ .../smoketest/upgrade/rolling/finalize.robot | 29 ++++++++++++++++++ 13 files changed, 84 insertions(+), 23 deletions(-) rename hadoop-ozone/dist/src/main/smoketest/upgrade/{ => non-rolling}/check-finalization.robot (98%) rename hadoop-ozone/dist/src/main/smoketest/upgrade/{ => non-rolling}/finalize.robot (96%) rename hadoop-ozone/dist/src/main/smoketest/upgrade/{ => non-rolling}/lib.robot (91%) create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/check-finalization.robot create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/finalize.robot diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh index e9d0795db1ea..d9d13c7968dc 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -31,9 +31,14 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # Upgrade tests to be run. In CI we want to run just one set, but for a release # we might advise the release manager to run the full matrix. +# Rolling upgrade test. This requires both the start and end version to support rolling upgrade. +# Since we have not yet done a release with rolling upgrade support, it is just doing a rolling +# restart in the same version for now. +run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" + # This is the version of Ozone that should use the runner image to run the # code that was built. Other versions will pull images from docker hub. -# run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION" +run_test ha non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 2.1.1 "$OZONE_CURRENT_VERSION" # run_test ha non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION" # run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION" @@ -43,9 +48,6 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # run_test ha non-rolling-upgrade 1.2.1 "$OZONE_CURRENT_VERSION" # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION" -# Rolling upgrade test -run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" - generate_report "upgrade" "$ALL_RESULT_DIR" exit "$RESULT" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh index 155395b29f9e..6d68a78a6c51 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh @@ -20,7 +20,7 @@ source "$TEST_DIR"/testlib.sh with_this_version_pre_finalized() { # New layout features were added in this version, so OM and SCM should be # pre-finalized. - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/non-rolling/check-finalization.robot # Test that EC is disabled when pre-finalized. execute_robot_test "$SCM" -N "${OUTPUT_NAME}-ec" --include pre-finalized-ec-tests ec/upgrade-ec-check.robot } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.4.0/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.4.0/callback.sh index 95e9a749aa20..1dccca15a2c8 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.4.0/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.4.0/callback.sh @@ -21,12 +21,12 @@ source "$TEST_DIR"/testlib.sh ### CALLBACKS ### with_this_version_pre_finalized() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/non-rolling/check-finalization.robot execute_robot_test "$SCM" -N "${OUTPUT_NAME}-snapshot" --include pre-finalized-snapshot-tests snapshot/upgrade-snapshot-check.robot } with_this_version_finalized() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot execute_robot_test "$SCM" -N "${OUTPUT_NAME}-snapshot" snapshot/snapshot-sh.robot } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/2.0.0/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/2.0.0/callback.sh index 6cf3593bf863..c8de504d0fae 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/2.0.0/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/2.0.0/callback.sh @@ -19,13 +19,13 @@ source "$TEST_DIR"/testlib.sh with_this_version_pre_finalized() { # New layout features were added in this version, so OM and SCM should be pre-finalized. - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/non-rolling/check-finalization.robot # Test that HSync is disabled when pre-finalized. execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" --include pre-finalized-hsync-tests hsync/upgrade-hsync-check.robot } with_this_version_finalized() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" admincli/lease-recovery.robot execute_robot_test "$SCM" -N "${OUTPUT_NAME}-freon-hsync" freon/hsync.robot } diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh index e4a419f00df1..6256c0bde8d8 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh @@ -20,7 +20,7 @@ source "$TEST_DIR"/testlib.sh ### CALLBACKS ### with_old_version() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot generate old1 "$SCM" validate old1 "$SCM" } @@ -37,7 +37,7 @@ with_this_version_pre_finalized() { } with_old_version_downgraded() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot validate old1 "$SCM" validate new1 "$SCM" @@ -50,7 +50,7 @@ with_old_version_downgraded() { } with_this_version_finalized() { - execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot validate old1 "$SCM" validate new1 "$SCM" validate old2 "$SCM" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh index ea59c8d69eea..2d7b6cce7587 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh @@ -82,5 +82,5 @@ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-4-finalized" OZONE_KEEP_RESULTS=true start_docker_env # Sends commands to finalize OM and SCM. -execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot +execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/non-rolling/finalize.robot callback with_this_version_finalized diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh index 00ffce1a6716..78dd5d805617 100644 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh @@ -32,7 +32,7 @@ with_service_restarted() { } with_old_version() { - execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/rolling/check-finalization.robot generate old1 "$CLIENT" validate old1 "$CLIENT" @@ -52,7 +52,7 @@ with_this_version_pre_finalized() { } with_old_version_downgraded() { - execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/rolling/check-finalization.robot validate old1 "$CLIENT" validate new1 "$CLIENT" @@ -64,7 +64,7 @@ with_old_version_downgraded() { } with_this_version_finalized() { - execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot + execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/rolling/check-finalization.robot validate old1 "$CLIENT" validate new1 "$CLIENT" validate old2 "$CLIENT" diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh index 8666abcc71f6..387b3475cd5a 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh @@ -139,6 +139,6 @@ create_containers "$CLIENT" echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---" OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-5-finalized" -# Sends commands to finalize OM and SCM. -execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot +# Sends command to finalize the cluster. +execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/rolling/finalize.robot callback with_this_version_finalized diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/check-finalization.robot similarity index 98% rename from hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot rename to hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/check-finalization.robot index 717a2998f794..74c8e217c00d 100644 --- a/hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/check-finalization.robot @@ -15,7 +15,7 @@ *** Settings *** Documentation Finalize Upgrade of the Ozone cluster -Resource ../commonlib.robot +Resource ../../commonlib.robot Resource lib.robot Test Timeout 10 minutes Suite Setup Get Security Enabled From Config diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/finalize.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/finalize.robot similarity index 96% rename from hadoop-ozone/dist/src/main/smoketest/upgrade/finalize.robot rename to hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/finalize.robot index 0fcf52c601b6..bdbbd8ef2797 100644 --- a/hadoop-ozone/dist/src/main/smoketest/upgrade/finalize.robot +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/finalize.robot @@ -15,7 +15,7 @@ *** Settings *** Documentation Finalize Upgrade of the Ozone cluster -Resource ../commonlib.robot +Resource ../../commonlib.robot Resource lib.robot Test Timeout 10 minutes Suite Setup Get Security Enabled From Config diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/lib.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/lib.robot similarity index 91% rename from hadoop-ozone/dist/src/main/smoketest/upgrade/lib.robot rename to hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/lib.robot index 25c71c7ff881..92803379ee3e 100644 --- a/hadoop-ozone/dist/src/main/smoketest/upgrade/lib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/lib.robot @@ -14,10 +14,10 @@ # limitations under the License. *** Settings *** -Documentation Keywords for Upgrade Tests +Documentation Keywords for Non-Rolling Upgrade Tests Library OperatingSystem -Resource ../lib/os.robot -Resource ../admincli/lib.resource +Resource ../../lib/os.robot +Resource ../../admincli/lib.resource *** Keywords *** OM Finalization Status diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/check-finalization.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/check-finalization.robot new file mode 100644 index 000000000000..ea19c8f0e12a --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/check-finalization.robot @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Check finalization status of the Ozone cluster after rolling upgrade +Resource ../../commonlib.robot +Resource ../../lib/os.robot +Resource ../../admincli/lib.resource +Test Timeout 10 minutes +Suite Setup Get Security Enabled From Config +Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + +*** Test Cases *** +Check Cluster Finalized + [Tags] finalized + ${param} = Get OM Service Param + ${result} = Execute ozone admin upgrade status --json ${param} | jq -r .clusterFinalizationStatus + Should Be Equal ${result} FINALIZED diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/finalize.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/finalize.robot new file mode 100644 index 000000000000..3b8dacf9bba9 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/finalize.robot @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Finalize rolling upgrade of the Ozone cluster +Resource ../../commonlib.robot +Resource ../../lib/os.robot +Resource ../../admincli/lib.resource +Test Timeout 10 minutes +Suite Setup Get Security Enabled From Config +Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + +*** Test Cases *** +Finalize Cluster + ${param} = Get OM Service Param + ${result} = Execute ozone admin upgrade finalize --wait ${param} + Should Contain ${result} Finalization complete. From 38abf225d7ca063bd6536642ad48d50aead70752 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Tue, 25 Aug 2026 13:49:39 -0400 Subject: [PATCH 2/5] Fix exclusions --- hadoop-ozone/dist/src/main/compose/testlib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/testlib.sh b/hadoop-ozone/dist/src/main/compose/testlib.sh index f8a9b3b8a2ea..28797c36b6af 100755 --- a/hadoop-ozone/dist/src/main/compose/testlib.sh +++ b/hadoop-ozone/dist/src/main/compose/testlib.sh @@ -303,7 +303,7 @@ reorder_om_nodes() { ## @description Create stack dump of each java process in each container create_stack_dumps() { local c pid procname - for c in $(docker-compose ps | cut -f1 -d' ' | grep -e datanode -e om -e recon -e s3g -e scm | grep -v -e prometheus); do + for c in $(docker-compose ps | cut -f1 -d' ' | grep -e datanode -e om -e recon -e s3g -e scm | grep -v -e prometheus -e s3g-haproxy); do while read -r pid procname; do echo "jstack $pid > ${RESULT_DIR}/${c}_${procname}.stack" docker exec "${c}" bash -c "jstack $pid" > "${RESULT_DIR}/${c}_${procname}.stack" From c13c4dde7ed3b4cdfed991b8a7f564fcd013c958 Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Wed, 26 Aug 2026 15:24:26 -0400 Subject: [PATCH 3/5] Add fork with the ability to choose whether downgrade is rolling or not --- .../upgrades/rolling-upgrade/driver.sh | 46 +++++++++++++++++-- 1 file changed, 42 insertions(+), 4 deletions(-) diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh index 387b3475cd5a..7f3827ba9480 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh @@ -34,6 +34,13 @@ echo "--- RUNNING ROLLING UPGRADE TEST FROM $OZONE_UPGRADE_FROM TO $OZONE_UPGRAD source "$TEST_DIR"/testlib.sh +# The first upgrade (old -> new) is always exercised as a rolling (zero-downtime) upgrade, +# since that is the path this suite exists to prove. The downgrade and the subsequent +# re-upgrade are run NON-ROLLING (all servers restarts at once) to save CI time. The +# rolling versions of those two phases are switched off here. Set this to "true" to run all +# three phases as rolling (e.g. when debugging the full zero-downtime downgrade path). +DOWNGRADE_AND_REUPGRADE_ROLLING=false + ## @description Restart one service with the target image. ## @param name of the service ## @param stage prefix, used for the data prefix @@ -104,6 +111,29 @@ rolling_restart_all_services() { fi } +## @description Restart all server services at once (non-rolling) with the target image. +## @param stage prefix, used for OUTPUT_NAME +## @param target image +non_rolling_restart_all_services() { + local stage_prefix="$1" + local target_image="$2" + local services=(scm1 scm2 scm3 recon dn1 dn2 dn3 dn4 dn5 om1 om2 om3 s3g1 s3g2 s3g3) + + OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}" + + prepare_for_image "${target_image}" + echo "--- PREPARED ${target_image} IMAGE ---" + + echo "--- STOPPING ALL SERVER SERVICES ---" + stop_containers "${services[@]}" + + echo "--- STARTING ALL SERVER SERVICES WITH IMAGE ${target_image} ---" + create_containers "${services[@]}" + + wait_for_safemode_exit + wait_for_om_leader +} + echo "--- SETTING UP OLD VERSION $OZONE_UPGRADE_FROM ---" OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-1-original" prepare_for_image "$OZONE_UPGRADE_FROM" @@ -119,14 +149,22 @@ rolling_restart_all_services "2-upgrade" "$OZONE_UPGRADE_TO" OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-pre-finalized" callback with_this_version_pre_finalized -echo "--- ROLLING DOWNGRADE TO $OZONE_UPGRADE_FROM ---" -rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM" "reverse" +echo "--- DOWNGRADE TO $OZONE_UPGRADE_FROM ---" +if [[ "$DOWNGRADE_AND_REUPGRADE_ROLLING" == "true" ]]; then + rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM" "reverse" +else + non_rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM" +fi OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-downgraded" callback with_old_version_downgraded -echo "--- ROLLING UPGRADE TO $OZONE_UPGRADE_TO ---" -rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO" +echo "--- UPGRADE TO $OZONE_UPGRADE_TO ---" +if [[ "$DOWNGRADE_AND_REUPGRADE_ROLLING" == "true" ]]; then + rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO" +else + non_rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO" +fi # Upgrade client after all server components are upgraded but before finalization, # so new client APIs can be exercised against pre-finalized servers. From eb30a736fed427face5dbcf6b9f1c7fabe5fe49d Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 27 Aug 2026 11:36:15 -0400 Subject: [PATCH 4/5] Only run non-rolling suite --- hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh index d9d13c7968dc..511a0970fa74 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/test.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/test.sh @@ -34,7 +34,7 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir # Rolling upgrade test. This requires both the start and end version to support rolling upgrade. # Since we have not yet done a release with rolling upgrade support, it is just doing a rolling # restart in the same version for now. -run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" +# run_test ha rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION" # This is the version of Ozone that should use the runner image to run the # code that was built. Other versions will pull images from docker hub. From 8aa7a0cd77a5c62a989c6bd5b884a487f4be5cae Mon Sep 17 00:00:00 2001 From: Ethan Rose Date: Thu, 27 Aug 2026 16:51:27 -0400 Subject: [PATCH 5/5] Restore prepare robot file to run against older versions --- .../upgrades/non-rolling-upgrade/driver.sh | 6 ++-- .../upgrade/non-rolling/prepare.robot | 33 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/prepare.robot diff --git a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh index 2d7b6cce7587..4018f5903cda 100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh @@ -53,7 +53,7 @@ echo "--- RUNNING WITH OLD VERSION $OZONE_UPGRADE_FROM ---" start_docker_env callback with_old_version -execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/prepare.robot +execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/non-rolling/prepare.robot stop_docker_env prepare_for_image "$OZONE_UPGRADE_TO" export OM_HA_ARGS='--upgrade' @@ -62,7 +62,7 @@ echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO PRE-FINALIZED ---" OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-pre-finalized" OZONE_KEEP_RESULTS=true start_docker_env callback with_this_version_pre_finalized -execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/prepare.robot +execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/non-rolling/prepare.robot stop_docker_env prepare_for_image "$OZONE_UPGRADE_FROM" set_downgrade_om_args @@ -72,7 +72,7 @@ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-downgraded" OZONE_KEEP_RESULTS=true start_docker_env callback with_old_version_downgraded -execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/prepare.robot +execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/non-rolling/prepare.robot stop_docker_env prepare_for_image "$OZONE_UPGRADE_TO" export OM_HA_ARGS='--upgrade' diff --git a/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/prepare.robot b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/prepare.robot new file mode 100644 index 000000000000..967862372455 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/prepare.robot @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Retained so we can test upgrades from older versions of Ozone that required +# prepare for upgrade. The current version no longer needs prepare, but keeps a +# compatibility shim so this test still passes when run against a new server. + +*** Settings *** +Documentation Prepares OMs +Resource ../../commonlib.robot +Resource lib.robot +Test Timeout 5 minutes +Suite Setup Get Security Enabled From Config +Test Setup Run Keyword if '${SECURITY_ENABLED}' == 'true' Kinit test user testuser testuser.keytab + +*** Test Cases *** +Prepare Ozone Manager + Pass Execution If '%{OZONE_UPGRADE_FROM}' == '1.1.0' OM prepare is skipped for version %{OZONE_UPGRADE_FROM} + ${service_id} = Get OM Service ID + Pass Execution If '${service_id}' == '' OM prepare skipped in non-HA + Wait Until Keyword Succeeds 3min 10sec Prepare OM