HDDS-16020. Configure upgrade acceptance tests to run on master

hadoop-ozone/dist/src/main/compose/testlib.sh

@@ -305,7 +305,7 @@ reorder_om_nodes() {
 ## @description Create stack dump of each java process in each container
 create_stack_dumps() {
   local c pid procname
-  for c in $(docker-compose ps | cut -f1 -d' ' | grep -e datanode -e om -e recon -e s3g -e scm | grep -v -e prometheus); do
+  for c in $(docker-compose ps | cut -f1 -d' ' | grep -e datanode -e om -e recon -e s3g -e scm | grep -v -e prometheus -e s3g-haproxy); do
     while read -r pid procname; do
       echo "jstack $pid > ${RESULT_DIR}/${c}_${procname}.stack"
       docker exec "${c}" bash -c "jstack $pid" > "${RESULT_DIR}/${c}_${procname}.stack"

hadoop-ozone/dist/src/main/compose/upgrade/test.sh

@@ -31,9 +31,14 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir
 # Upgrade tests to be run. In CI we want to run just one set, but for a release
 # we might advise the release manager to run the full matrix.
 
+# Rolling upgrade test. This requires both the start and end version to support rolling upgrade.
+# Since we have not yet done a release with rolling upgrade support, it is just doing a rolling
+# restart in the same version for now.
+# run_test ha     rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
+
 # This is the version of Ozone that should use the runner image to run the
 # code that was built. Other versions will pull images from docker hub.
-# run_test ha     non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION"
+run_test ha     non-rolling-upgrade 2.2.0 "$OZONE_CURRENT_VERSION"
 # run_test ha     non-rolling-upgrade 2.1.1 "$OZONE_CURRENT_VERSION"
 # run_test ha     non-rolling-upgrade 2.0.0 "$OZONE_CURRENT_VERSION"
 # run_test non-ha non-rolling-upgrade 1.4.1 "$OZONE_CURRENT_VERSION"
@@ -43,9 +48,6 @@ RESULT_DIR="$ALL_RESULT_DIR" create_results_dir
 # run_test ha non-rolling-upgrade 1.2.1 "$OZONE_CURRENT_VERSION"
 # run_test om-ha non-rolling-upgrade 1.1.0 "$OZONE_CURRENT_VERSION"
 
-# Rolling upgrade test
-run_test ha     rolling-upgrade "$OZONE_CURRENT_VERSION" "$OZONE_CURRENT_VERSION"
-
 generate_report "upgrade" "$ALL_RESULT_DIR"
 
 exit "$RESULT"

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.3.0/callback.sh

@@ -20,7 +20,7 @@ source "$TEST_DIR"/testlib.sh
 with_this_version_pre_finalized() {
   # New layout features were added in this version, so OM and SCM should be
   # pre-finalized.
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/non-rolling/check-finalization.robot
   # Test that EC is disabled when pre-finalized.
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-ec" --include pre-finalized-ec-tests ec/upgrade-ec-check.robot
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/1.4.0/callback.sh

@@ -21,12 +21,12 @@ source "$TEST_DIR"/testlib.sh
 ### CALLBACKS ###
 
 with_this_version_pre_finalized() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/non-rolling/check-finalization.robot
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-snapshot" --include pre-finalized-snapshot-tests snapshot/upgrade-snapshot-check.robot
 }
 
 with_this_version_finalized() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-snapshot" snapshot/snapshot-sh.robot
 }
 

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/2.0.0/callback.sh

@@ -19,13 +19,13 @@ source "$TEST_DIR"/testlib.sh
 
 with_this_version_pre_finalized() {
   # New layout features were added in this version, so OM and SCM should be pre-finalized.
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include pre-finalized upgrade/non-rolling/check-finalization.robot
   # Test that HSync is disabled when pre-finalized.
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" --include pre-finalized-hsync-tests hsync/upgrade-hsync-check.robot
 }
 
 with_this_version_finalized() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-hsync" admincli/lease-recovery.robot
   execute_robot_test "$SCM" -N "${OUTPUT_NAME}-freon-hsync" freon/hsync.robot
 }

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/callbacks/common/callback.sh

@@ -20,7 +20,7 @@ source "$TEST_DIR"/testlib.sh
 ### CALLBACKS ###
 
 with_old_version() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot
   generate old1 "$SCM"
   validate old1 "$SCM"
 }
@@ -37,7 +37,7 @@ with_this_version_pre_finalized() {
 }
 
 with_old_version_downgraded() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot
   validate old1 "$SCM"
   validate new1 "$SCM"
 
@@ -50,7 +50,7 @@ with_old_version_downgraded() {
 }
 
 with_this_version_finalized() {
-  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$SCM" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/non-rolling/check-finalization.robot
   validate old1 "$SCM"
   validate new1 "$SCM"
   validate old2 "$SCM"

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/driver.sh

@@ -53,7 +53,7 @@ echo "--- RUNNING WITH OLD VERSION $OZONE_UPGRADE_FROM ---"
 start_docker_env
 callback with_old_version
 
-execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/prepare.robot
+execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/non-rolling/prepare.robot
 stop_docker_env
 prepare_for_image "$OZONE_UPGRADE_TO"
 export OM_HA_ARGS='--upgrade'
@@ -62,7 +62,7 @@ echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO PRE-FINALIZED ---"
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-pre-finalized"
 OZONE_KEEP_RESULTS=true start_docker_env
 callback with_this_version_pre_finalized
-execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/prepare.robot
+execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/non-rolling/prepare.robot
 stop_docker_env
 prepare_for_image "$OZONE_UPGRADE_FROM"
 set_downgrade_om_args
@@ -72,7 +72,7 @@ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-downgraded"
 OZONE_KEEP_RESULTS=true start_docker_env
 callback with_old_version_downgraded
 
-execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/prepare.robot
+execute_robot_test "$SCM" -N "${OUTPUT_NAME}-prepare" upgrade/non-rolling/prepare.robot
 stop_docker_env
 prepare_for_image "$OZONE_UPGRADE_TO"
 export OM_HA_ARGS='--upgrade'
@@ -82,5 +82,5 @@ OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-4-finalized"
 OZONE_KEEP_RESULTS=true start_docker_env
 
 # Sends commands to finalize OM and SCM.
-execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot
+execute_robot_test "$SCM" -N "${OUTPUT_NAME}-finalize" upgrade/non-rolling/finalize.robot
 callback with_this_version_finalized

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/callbacks/common/callback.sh

@@ -32,7 +32,7 @@ with_service_restarted() {
 }
 
 with_old_version() {
-  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/rolling/check-finalization.robot
   generate old1 "$CLIENT"
   validate old1 "$CLIENT"
 
@@ -52,7 +52,7 @@ with_this_version_pre_finalized() {
 }
 
 with_old_version_downgraded() {
-  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/rolling/check-finalization.robot
   validate old1 "$CLIENT"
   validate new1 "$CLIENT"
 
@@ -64,7 +64,7 @@ with_old_version_downgraded() {
 }
 
 with_this_version_finalized() {
-  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/check-finalization.robot
+  execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-check-finalization" --include finalized upgrade/rolling/check-finalization.robot
   validate old1 "$CLIENT"
   validate new1 "$CLIENT"
   validate old2 "$CLIENT"

hadoop-ozone/dist/src/main/compose/upgrade/upgrades/rolling-upgrade/driver.sh

@@ -34,6 +34,13 @@ echo "--- RUNNING ROLLING UPGRADE TEST FROM $OZONE_UPGRADE_FROM TO $OZONE_UPGRAD
 
 source "$TEST_DIR"/testlib.sh
 
+# The first upgrade (old -> new) is always exercised as a rolling (zero-downtime) upgrade,
+# since that is the path this suite exists to prove. The downgrade and the subsequent
+# re-upgrade are run NON-ROLLING (all servers restarts at once) to save CI time. The
+# rolling versions of those two phases are switched off here. Set this to "true" to run all
+# three phases as rolling (e.g. when debugging the full zero-downtime downgrade path).
+DOWNGRADE_AND_REUPGRADE_ROLLING=false
+
 ## @description Restart one service with the target image.
 ## @param name of the service
 ## @param stage prefix, used for the data prefix
@@ -104,6 +111,29 @@ rolling_restart_all_services() {
   fi
 }
 
+## @description Restart all server services at once (non-rolling) with the target image.
+## @param stage prefix, used for OUTPUT_NAME
+## @param target image
+non_rolling_restart_all_services() {
+  local stage_prefix="$1"
+  local target_image="$2"
+  local services=(scm1 scm2 scm3 recon dn1 dn2 dn3 dn4 dn5 om1 om2 om3 s3g1 s3g2 s3g3)
+
+  OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-${stage_prefix}"
+
+  prepare_for_image "${target_image}"
+  echo "--- PREPARED ${target_image} IMAGE ---"
+
+  echo "--- STOPPING ALL SERVER SERVICES ---"
+  stop_containers "${services[@]}"
+
+  echo "--- STARTING ALL SERVER SERVICES WITH IMAGE ${target_image} ---"
+  create_containers "${services[@]}"
+
+  wait_for_safemode_exit
+  wait_for_om_leader
+}
+
 echo "--- SETTING UP OLD VERSION $OZONE_UPGRADE_FROM ---"
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-1-original"
 prepare_for_image "$OZONE_UPGRADE_FROM"
@@ -119,14 +149,22 @@ rolling_restart_all_services "2-upgrade" "$OZONE_UPGRADE_TO"
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-2-pre-finalized"
 callback with_this_version_pre_finalized
 
-echo "--- ROLLING DOWNGRADE TO $OZONE_UPGRADE_FROM ---"
-rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM" "reverse"
+echo "--- DOWNGRADE TO $OZONE_UPGRADE_FROM ---"
+if [[ "$DOWNGRADE_AND_REUPGRADE_ROLLING" == "true" ]]; then
+  rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM" "reverse"
+else
+  non_rolling_restart_all_services "3-downgrade" "$OZONE_UPGRADE_FROM"
+fi
 
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-3-downgraded"
 callback with_old_version_downgraded
 
-echo "--- ROLLING UPGRADE TO $OZONE_UPGRADE_TO ---"
-rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO"
+echo "--- UPGRADE TO $OZONE_UPGRADE_TO ---"
+if [[ "$DOWNGRADE_AND_REUPGRADE_ROLLING" == "true" ]]; then
+  rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO"
+else
+  non_rolling_restart_all_services "4-upgrade" "$OZONE_UPGRADE_TO"
+fi
 
 # Upgrade client after all server components are upgraded but before finalization,
 # so new client APIs can be exercised against pre-finalized servers.
@@ -139,6 +177,6 @@ create_containers "$CLIENT"
 echo "--- RUNNING WITH NEW VERSION $OZONE_UPGRADE_TO FINALIZED ---"
 OUTPUT_NAME="${OZONE_UPGRADE_FROM}-${OZONE_UPGRADE_TO}-5-finalized"
 
-# Sends commands to finalize OM and SCM.
-execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/finalize.robot
+# Sends command to finalize the cluster.
+execute_robot_test "$CLIENT" -N "${OUTPUT_NAME}-finalize" upgrade/rolling/finalize.robot
 callback with_this_version_finalized

hadoop-ozone/dist/src/main/smoketest/upgrade/check-finalization.robot → hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/check-finalization.robot

@@ -15,7 +15,7 @@
 
 *** Settings ***
 Documentation       Finalize Upgrade of the Ozone cluster
-Resource            ../commonlib.robot
+Resource            ../../commonlib.robot
 Resource            lib.robot
 Test Timeout        10 minutes
 Suite Setup         Get Security Enabled From Config

hadoop-ozone/dist/src/main/smoketest/upgrade/finalize.robot → hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/finalize.robot

@@ -15,7 +15,7 @@
 
 *** Settings ***
 Documentation       Finalize Upgrade of the Ozone cluster
-Resource            ../commonlib.robot
+Resource            ../../commonlib.robot
 Resource            lib.robot
 Test Timeout        10 minutes
 Suite Setup         Get Security Enabled From Config

hadoop-ozone/dist/src/main/smoketest/upgrade/lib.robot → hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/lib.robot

@@ -14,10 +14,10 @@
 # limitations under the License.
 
 *** Settings ***
-Documentation       Keywords for Upgrade Tests
+Documentation       Keywords for Non-Rolling Upgrade Tests
 Library             OperatingSystem
-Resource            ../lib/os.robot
-Resource            ../admincli/lib.resource
+Resource            ../../lib/os.robot
+Resource            ../../admincli/lib.resource
 
 *** Keywords ***
 OM Finalization Status

hadoop-ozone/dist/src/main/smoketest/upgrade/non-rolling/prepare.robot

@@ -0,0 +1,33 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Retained so we can test upgrades from older versions of Ozone that required
+# prepare for upgrade. The current version no longer needs prepare, but keeps a
+# compatibility shim so this test still passes when run against a new server.
+
+*** Settings ***
+Documentation       Prepares OMs
+Resource            ../../commonlib.robot
+Resource            lib.robot
+Test Timeout        5 minutes
+Suite Setup         Get Security Enabled From Config
+Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+
+*** Test Cases ***
+Prepare Ozone Manager
+    Pass Execution If    '%{OZONE_UPGRADE_FROM}' == '1.1.0'    OM prepare is skipped for version %{OZONE_UPGRADE_FROM}
+    ${service_id} =      Get OM Service ID
+    Pass Execution If    '${service_id}' == ''    OM prepare skipped in non-HA
+    Wait Until Keyword Succeeds      3min       10sec     Prepare OM

hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/check-finalization.robot

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Check finalization status of the Ozone cluster after rolling upgrade
+Resource            ../../commonlib.robot
+Resource            ../../lib/os.robot
+Resource            ../../admincli/lib.resource
+Test Timeout        10 minutes
+Suite Setup         Get Security Enabled From Config
+Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+
+*** Test Cases ***
+Check Cluster Finalized
+    [Tags]    finalized
+    ${param} =     Get OM Service Param
+    ${result} =    Execute      ozone admin upgrade status --json ${param} | jq -r .clusterFinalizationStatus
+    Should Be Equal    ${result}    FINALIZED

hadoop-ozone/dist/src/main/smoketest/upgrade/rolling/finalize.robot

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       Finalize rolling upgrade of the Ozone cluster
+Resource            ../../commonlib.robot
+Resource            ../../lib/os.robot
+Resource            ../../admincli/lib.resource
+Test Timeout        10 minutes
+Suite Setup         Get Security Enabled From Config
+Test Setup          Run Keyword if    '${SECURITY_ENABLED}' == 'true'    Kinit test user     testuser     testuser.keytab
+
+*** Test Cases ***
+Finalize Cluster
+    ${param} =     Get OM Service Param
+    ${result} =    Execute      ozone admin upgrade finalize --wait ${param}
+    Should Contain    ${result}    Finalization complete.