From 5a496698e9c20fd5d277d70c5cd8db2850a06912 Mon Sep 17 00:00:00 2001 From: Brad House Date: Mon, 9 Nov 2026 20:08:42 +0000 Subject: [PATCH] CKS: pin the autoscaler and CCM in the binaries ISO builder create-kubernetes-binaries-iso.sh downloaded the cluster-autoscaler manifest from kubernetes/autoscaler master and the CCM manifest from cloudstack-kubernetes-provider main, and bundled whatever images those named. The autoscaler image was apache/cloudstack-kubernetes-autoscaler :latest, a 2022 development build, for every Kubernetes version, and the CCM was whatever main held on the day of the build (v1.1.0 for a long time, which never initialises nodes). Nothing recorded what went in. - Autoscaler: use the newest cluster-autoscaler release for the ISO's Kubernetes minor version (or AUTOSCALER_VERSION), take the CloudStack example manifest from that release tag, and bundle the upstream registry.k8s.io image of the same release, which includes the CloudStack provider. Add a ClusterRole for the VolumeAttachment and DRA read access that newer releases need and the example lacks. - CCM: pin to v1.2.0 (or CCM_VERSION) and set the manifest's image to that tag. - Write versions.txt into the ISO with every version, URL and image digest. - Download with curl -f, one file per call, so a failed download stops the build instead of bundling an error page. Build in a fresh mktemp directory, and remove pulled images with --sync so containerd's garbage collection can't delete layers the next pull shares. - Upgrade: the autoscaler image now changes with the Kubernetes version, so once every node is upgraded, redeploy the autoscaler of a cluster with autoscaling enabled from the new ISO, with the cluster's min, max and id. A failure is logged and doesn't fail the upgrade. Signed-off-by: Brad House --- .../KubernetesClusterActionWorker.java | 5 + ...esClusterResourceModifierActionWorker.java | 3 +- .../KubernetesClusterUpgradeWorker.java | 28 +++ .../KubernetesClusterUpgradeWorkerTest.java | 68 +++++++ .../util/create-kubernetes-binaries-iso.sh | 192 +++++++++++++++--- 5 files changed, 263 insertions(+), 33 deletions(-) diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java index 7a06b11e3bf4..15ec3275e456 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java @@ -394,6 +394,11 @@ protected void logAndThrow(final Level logLevel, final String message, final Exc logTransitStateAndThrow(logLevel, message, null, null, ex); } + protected String getAutoscalerEnableCommand(Long minSize, Long maxSize) { + return String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", + kubernetesCluster.getUuid(), maxSize, minSize); + } + protected File getManagementServerSshPublicKeyFile() { boolean devel = Boolean.parseBoolean(configurationDao.getValue("developer")); String keyFile = String.format("%s/.ssh/id_rsa", System.getProperty("user.home")); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java index 573e752ce125..60f57c7ebf2f 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterResourceModifierActionWorker.java @@ -930,8 +930,7 @@ protected boolean autoscaleCluster(boolean enable, Long minSize, Long maxSize) { try { if (enable) { - String command = String.format("sudo /opt/bin/autoscale-kube-cluster -i %s -e -M %d -m %d", - kubernetesCluster.getUuid(), maxSize, minSize); + String command = getAutoscalerEnableCommand(minSize, maxSize); Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), pkFile, null, command, 10000, 10000, 60000); diff --git a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java index 4c2725fc2a2f..027e9d0924c1 100644 --- a/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java +++ b/plugins/integrations/kubernetes-service/src/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java @@ -156,6 +156,32 @@ private void upgradeKubernetesClusterNodes() { } } + /** + * The autoscaler on a binaries ISO matches the ISO's Kubernetes version, and the upgrade has + * copied the new ISO's autoscaler manifest to the control node. Once the nodes in this upgrade + * have imported the new ISO's images, redeploy the autoscaler from it with the cluster's + * settings. A failure leaves the previous autoscaler running, so it doesn't fail the upgrade. + */ + protected void updateAutoscaler() { + if (!kubernetesCluster.getAutoscalingEnabled()) { + return; + } + if (kubernetesCluster.getMinSize() == null || kubernetesCluster.getMaxSize() == null) { + logger.warn("Not updating the autoscaler of Kubernetes cluster {}: its minimum or maximum size is not set", kubernetesCluster); + return; + } + String command = getAutoscalerEnableCommand(kubernetesCluster.getMinSize(), kubernetesCluster.getMaxSize()); + try { + Pair result = SshHelper.sshExecute(publicIpAddress, sshPort, getControlNodeLoginUser(), + sshKeyFile, null, command, 10000, 10000, 60000); + if (!result.first()) { + logger.warn("Unable to update the autoscaler of Kubernetes cluster {}: {}", kubernetesCluster, result.second()); + } + } catch (Exception e) { + logger.warn("Unable to update the autoscaler of Kubernetes cluster {}", kubernetesCluster, e); + } + } + public boolean upgradeCluster() throws CloudRuntimeException { init(); if (logger.isInfoEnabled()) { @@ -184,6 +210,8 @@ public boolean upgradeCluster() throws CloudRuntimeException { if (!updated) { stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed); } else { + // While the cluster is still Upgrading, so autoscaling can't be changed meanwhile + updateAutoscaler(); stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded); } return updated; diff --git a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java index ff8f875d06e1..3477c1fc6838 100644 --- a/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java +++ b/plugins/integrations/kubernetes-service/src/test/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorkerTest.java @@ -22,11 +22,15 @@ import com.cloud.kubernetes.cluster.dao.KubernetesClusterVmMapDao; import com.cloud.kubernetes.version.KubernetesSupportedVersion; import com.cloud.uservm.UserVm; +import com.cloud.utils.Pair; +import com.cloud.utils.ssh.SshHelper; import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; +import org.mockito.ArgumentMatchers; import org.mockito.Mock; +import org.mockito.MockedStatic; import org.mockito.Mockito; import org.mockito.junit.MockitoJUnitRunner; @@ -80,4 +84,68 @@ public void testFilterOutManualUpgradeNodesFromClusterUpgrade() { Assert.assertTrue(ids.contains(controlNodeId) && ids.contains(workerNode1Id)); Assert.assertFalse(ids.contains(workerNode2Id)); } + + private MockedStatic mockAutoscalerSsh(KubernetesClusterUpgradeWorker spyWorker, Pair result) { + Mockito.doReturn("cloud").when(spyWorker).getControlNodeLoginUser(); + MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class); + sshHelper.when(() -> SshHelper.sshExecute(ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt())).thenReturn(result); + return sshHelper; + } + + @Test + public void testUpdateAutoscalerSkippedWhenAutoscalingDisabled() { + Mockito.when(kubernetesCluster.getAutoscalingEnabled()).thenReturn(false); + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + worker.updateAutoscaler(); + sshHelper.verifyNoInteractions(); + } + } + + @Test + public void testUpdateAutoscalerSkippedWithoutMinMaxSize() { + Mockito.when(kubernetesCluster.getAutoscalingEnabled()).thenReturn(true); + Mockito.when(kubernetesCluster.getMinSize()).thenReturn(null); + try (MockedStatic sshHelper = Mockito.mockStatic(SshHelper.class)) { + worker.updateAutoscaler(); + sshHelper.verifyNoInteractions(); + } + } + + @Test + public void testUpdateAutoscalerRedeploysWithClusterSettings() { + Mockito.when(kubernetesCluster.getAutoscalingEnabled()).thenReturn(true); + Mockito.when(kubernetesCluster.getUuid()).thenReturn("cluster-uuid"); + Mockito.when(kubernetesCluster.getMinSize()).thenReturn(1L); + Mockito.when(kubernetesCluster.getMaxSize()).thenReturn(5L); + KubernetesClusterUpgradeWorker spyWorker = Mockito.spy(worker); + try (MockedStatic sshHelper = mockAutoscalerSsh(spyWorker, new Pair<>(true, ""))) { + spyWorker.updateAutoscaler(); + sshHelper.verify(() -> SshHelper.sshExecute(ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.eq("cloud"), + ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.eq("sudo /opt/bin/autoscale-kube-cluster -i cluster-uuid -e -M 5 -m 1"), + ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt())); + } + } + + @Test + public void testUpdateAutoscalerDoesNotThrowOnFailure() { + Mockito.when(kubernetesCluster.getAutoscalingEnabled()).thenReturn(true); + Mockito.when(kubernetesCluster.getUuid()).thenReturn("cluster-uuid"); + Mockito.when(kubernetesCluster.getMinSize()).thenReturn(1L); + Mockito.when(kubernetesCluster.getMaxSize()).thenReturn(5L); + KubernetesClusterUpgradeWorker spyWorker = Mockito.spy(worker); + try (MockedStatic sshHelper = mockAutoscalerSsh(spyWorker, new Pair<>(false, "autoscale-kube-cluster: not found"))) { + spyWorker.updateAutoscaler(); + sshHelper.when(() -> SshHelper.sshExecute(ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(), ArgumentMatchers.any(), ArgumentMatchers.anyString(), ArgumentMatchers.anyInt(), + ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt())).thenThrow(new RuntimeException("ssh failed")); + spyWorker.updateAutoscaler(); + sshHelper.verify(() -> SshHelper.sshExecute(ArgumentMatchers.any(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyString(), + ArgumentMatchers.any(), ArgumentMatchers.any(), + ArgumentMatchers.eq("sudo /opt/bin/autoscale-kube-cluster -i cluster-uuid -e -M 5 -m 1"), + ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt(), ArgumentMatchers.anyInt()), Mockito.times(2)); + } + } } diff --git a/scripts/util/create-kubernetes-binaries-iso.sh b/scripts/util/create-kubernetes-binaries-iso.sh index ebaf072771c5..736388d01eb6 100755 --- a/scripts/util/create-kubernetes-binaries-iso.sh +++ b/scripts/util/create-kubernetes-binaries-iso.sh @@ -18,9 +18,33 @@ set -e +# Default cloudstack-kubernetes-provider (CCM) release. v1.2.0 is the first +# release that initialises nodes (sets their providerID and zone) correctly. +DEFAULT_CCM_VERSION="v1.2.0" +CSI_DRIVER_VERSION="v3.0.0" + +# Fail on HTTP errors instead of saving the error page as the file +download() { + curl -fsSL --retry 3 "$1" -o "$2" +} + +# Point the only container image in manifest $1 at image $2 +set_manifest_image() { + local count + count=$(grep -c -E "^[[:space:]-]*image:" "$1" || true) + if [ "${count}" != "1" ]; then + echo "ERROR: expected one image in $1, found ${count}" + exit 1 + fi + sed -i -E "s|^([[:space:]-]*image:).*|\1 $2|" "$1" +} + if [ $# -lt 6 ]; then echo "Invalid input. Valid usage: ./create-kubernetes-binaries-iso.sh OUTPUT_PATH KUBERNETES_VERSION CNI_VERSION CRICTL_VERSION WEAVENET_NETWORK_YAML_CONFIG DASHBOARD_YAML_CONFIG BUILD_NAME [ARCH] [ETCD_VERSION]" echo "eg: ./create-kubernetes-binaries-iso.sh ./ 1.11.4 0.7.1 1.11.1 https://github.com/weaveworks/weave/releases/download/latest_release/weave-daemonset-k8s-1.11.yaml https://raw.githubusercontent.com/kubernetes/dashboard/v1.10.0/src/deploy/recommended/kubernetes-dashboard.yaml setup-v1.11.4 amd64" + echo "Optional environment variables:" + echo " AUTOSCALER_VERSION cluster-autoscaler release, e.g. 1.33.6 (default: the newest release for the Kubernetes minor version)" + echo " CCM_VERSION cloudstack-kubernetes-provider release, e.g. v1.2.0 (default: ${DEFAULT_CCM_VERSION})" exit 1 fi @@ -43,7 +67,10 @@ RELEASE="v${2}" VAL="1.18.0" output_dir="${1}" start_dir="$PWD" -iso_dir="/tmp/iso" +# A fresh directory, so files left by a failed build don't end up in this ISO +iso_dir=$(mktemp -d) +chmod 755 "${iso_dir}" +trap 'rm -rf "${iso_dir}"' EXIT working_dir="${iso_dir}/" mkdir -p "${working_dir}" build_name="${7}-${ARCH_SUFFIX}.iso" @@ -53,69 +80,139 @@ CNI_VERSION="v${3}" echo "Downloading CNI ${CNI_VERSION}..." cni_dir="${working_dir}/cni/" mkdir -p "${cni_dir}" -cni_status_code=$(curl -L --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-${ARCH}.tgz") -if [[ ${cni_status_code} -eq 404 ]] ; then - curl -L --write-out "%{http_code}\n" "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-${ARCH}-${CNI_VERSION}.tgz" -o "${cni_dir}/cni-plugins-${ARCH}.tgz" +# Older CNI releases name the archive without "linux-" +CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-${ARCH}-${CNI_VERSION}.tgz" +if ! download "${CNI_URL}" "${cni_dir}/cni-plugins-${ARCH}.tgz"; then + CNI_URL="https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-${ARCH}-${CNI_VERSION}.tgz" + download "${CNI_URL}" "${cni_dir}/cni-plugins-${ARCH}.tgz" fi CRICTL_VERSION="v${4}" echo "Downloading CRI tools ${CRICTL_VERSION}..." crictl_dir="${working_dir}/cri-tools/" mkdir -p "${crictl_dir}" -curl -L "https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" -o "${crictl_dir}/crictl-linux-${ARCH}.tar.gz" +CRICTL_URL="https://github.com/kubernetes-incubator/cri-tools/releases/download/${CRICTL_VERSION}/crictl-${CRICTL_VERSION}-linux-${ARCH}.tar.gz" +download "${CRICTL_URL}" "${crictl_dir}/crictl-linux-${ARCH}.tar.gz" echo "Downloading Kubernetes tools ${RELEASE}..." k8s_dir="${working_dir}/k8s" mkdir -p "${k8s_dir}" cd "${k8s_dir}" -curl -L --remote-name-all https://dl.k8s.io/release/${RELEASE}/bin/linux/${ARCH}/{kubeadm,kubelet,kubectl} +for binary in kubeadm kubelet kubectl; do + download "https://dl.k8s.io/release/${RELEASE}/bin/linux/${ARCH}/${binary}" "${binary}" +done kubeadm_file_permissions=`stat --format '%a' kubeadm` chmod +x kubeadm echo "Downloading kubelet.service ${RELEASE}..." cd "${start_dir}" kubelet_service_file="${working_dir}/kubelet.service" -touch "${kubelet_service_file}" if [[ `echo "${2} $VAL" | awk '{print ($1 < $2)}'` == 1 ]]; then - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} + KUBELET_SERVICE_URL="https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/kubelet.service" else - curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/kubelet.service" | sed "s:/usr/bin:/opt/bin:g" > ${kubelet_service_file} + KUBELET_SERVICE_URL="https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/kubelet.service" fi +download "${KUBELET_SERVICE_URL}" "${kubelet_service_file}" +sed -i "s:/usr/bin:/opt/bin:g" "${kubelet_service_file}" echo "Downloading 10-kubeadm.conf ${RELEASE}..." kubeadm_conf_file="${working_dir}/10-kubeadm.conf" -touch "${kubeadm_conf_file}" if [[ `echo "${2} $val" | awk '{print ($1 < $2)}'` == 1 ]]; then - curl -sSL "https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} + KUBEADM_CONF_URL="https://raw.githubusercontent.com/kubernetes/kubernetes/${RELEASE}/build/debs/10-kubeadm.conf" else - curl -sSL "https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/10-kubeadm.conf" | sed "s:/usr/bin:/opt/bin:g" > ${kubeadm_conf_file} + KUBEADM_CONF_URL="https://raw.githubusercontent.com/shapeblue/cloudstack-nonoss/main/cks/10-kubeadm.conf" fi +download "${KUBEADM_CONF_URL}" "${kubeadm_conf_file}" +sed -i "s:/usr/bin:/opt/bin:g" "${kubeadm_conf_file}" NETWORK_CONFIG_URL="${5}" echo "Downloading network config ${NETWORK_CONFIG_URL}" network_conf_file="${working_dir}/network.yaml" -curl -sSL ${NETWORK_CONFIG_URL} -o ${network_conf_file} +download "${NETWORK_CONFIG_URL}" "${network_conf_file}" DASHBORAD_CONFIG_URL="${6}" echo "Downloading dashboard config ${DASHBORAD_CONFIG_URL}" dashboard_conf_file="${working_dir}/dashboard.yaml" -curl -sSL ${DASHBORAD_CONFIG_URL} -o ${dashboard_conf_file} +download "${DASHBORAD_CONFIG_URL}" "${dashboard_conf_file}" -# TODO : Change the url once merged -AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" +# The cluster-autoscaler only supports the Kubernetes minor version it was +# released with, so use the newest release for this minor version unless +# AUTOSCALER_VERSION is set. +KUBERNETES_MINOR=$(echo "${2}" | cut -d. -f1-2) +if [ -z "${AUTOSCALER_VERSION}" ]; then + AUTOSCALER_VERSION=$(git ls-remote --tags --refs https://github.com/kubernetes/autoscaler.git "cluster-autoscaler-${KUBERNETES_MINOR}.*" \ + | sed -n -E 's|.*refs/tags/cluster-autoscaler-([0-9]+\.[0-9]+\.[0-9]+)$|\1|p' | sort -V | tail -n 1) + if [ -z "${AUTOSCALER_VERSION}" ]; then + echo "ERROR: could not find a cluster-autoscaler release for Kubernetes ${KUBERNETES_MINOR} (is git installed and github.com reachable?). Set AUTOSCALER_VERSION to choose one." + exit 1 + fi +fi +AUTOSCALER_VERSION="${AUTOSCALER_VERSION#v}" +if [ "$(echo "${AUTOSCALER_VERSION}" | cut -d. -f1-2)" != "${KUBERNETES_MINOR}" ]; then + echo "WARNING: cluster-autoscaler ${AUTOSCALER_VERSION} is not built for Kubernetes ${KUBERNETES_MINOR}" +fi +# The example manifest names an image that is not versioned, so use the +# upstream image (it includes the CloudStack provider) of the same release. +AUTOSCALER_IMAGE="registry.k8s.io/autoscaling/cluster-autoscaler:v${AUTOSCALER_VERSION}" +AUTOSCALER_URL="https://raw.githubusercontent.com/kubernetes/autoscaler/cluster-autoscaler-${AUTOSCALER_VERSION}/cluster-autoscaler/cloudprovider/cloudstack/examples/cluster-autoscaler-standard.yaml" echo "Downloading kubernetes cluster autoscaler ${AUTOSCALER_URL}" autoscaler_conf_file="${working_dir}/autoscaler.yaml" -curl -sSL ${AUTOSCALER_URL} -o ${autoscaler_conf_file} +download "${AUTOSCALER_URL}" "${autoscaler_conf_file}" +set_manifest_image "${autoscaler_conf_file}" "${AUTOSCALER_IMAGE}" +# The example manifest's ClusterRole lacks read access that newer releases +# need: VolumeAttachments (1.32 and later) and DRA objects (1.35 and later). +# Without it the autoscaler never finishes syncing its caches and never +# scales, so grant it in an additional ClusterRole. +cat >> "${autoscaler_conf_file}" < "${working_dir}/versions.txt" <> "${working_dir}/versions.txt" -rm -rf "${iso_dir}" +mkisofs -o "${output_dir}/${build_name}" -J -R -l "${iso_dir}"