chore: Update versions for SDP 26.11

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file.
 
 ### Added
 
+- Support for Spark `4.1.3` and `4.2.0` ([#790]).
 - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field
   ([#773]).
 - The history server and Spark Connect server StatefulSets now carry the
@@ -43,6 +44,12 @@ All notable changes to this project will be documented in this file.
   identifiers (matching `[a-zA-Z_][a-zA-Z0-9_]*`) and are now rejected if they are not ([#761]).
 - Make operations infallible where dependent on static inputs ([#766], [#769]).
 - Support Vector 0.58.0 ([#784], [#788], [#789]).
+- Deprecate support for Spark `4.1.2` ([#790]).
+- ClusterRoles for Spark driver and Spark Connect now allow managing `networkpolicies`, which Spark 4.2 creates for executors by default ([#790]).
+
+### Removed
+
+- Support for Spark `3.5.8` and `4.1.1` ([#790]).
 
 ### Fixed
 
@@ -98,6 +105,7 @@ All notable changes to this project will be documented in this file.
 [#787]: https://github.com/stackabletech/spark-k8s-operator/pull/787
 [#788]: https://github.com/stackabletech/spark-k8s-operator/pull/788
 [#789]: https://github.com/stackabletech/spark-k8s-operator/pull/789
+[#790]: https://github.com/stackabletech/spark-k8s-operator/pull/790
 
 ## [26.7.0] - 2026-07-21
 

deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml

@@ -27,6 +27,15 @@ rules:
       - patch
       - update
       - watch
+  # Spark 4.2 and later apply (server-side) a NetworkPolicy for the executor pods. It is owned by
+  # the driver pod and thereby garbage collected, so no delete is needed.
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+    verbs:
+      - create
+      - patch
 {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
   - apiGroups:
       - security.openshift.io

deploy/helm/spark-k8s-operator/templates/spark-connect-clusterrole.yaml

@@ -27,6 +27,15 @@ rules:
       - patch
       - update
       - watch
+  # Spark 4.2 and later apply (server-side) a NetworkPolicy for the executor pods. It is owned by
+  # the driver pod and thereby garbage collected, so no delete is needed.
+  - apiGroups:
+      - networking.k8s.io
+    resources:
+      - networkpolicies
+    verbs:
+      - create
+      - patch
 {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
   - apiGroups:
       - security.openshift.io

docs/modules/spark-k8s/examples/example-spark-connect.yaml

@@ -5,10 +5,10 @@ metadata:
   name: spark-connect # <1>
 spec:
   image:
-    productVersion: "3.5.8" # <2>
+    productVersion: "4.1.3" # <2>
     pullPolicy: IfNotPresent
   args:
-    - "--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.11.0" # <3>
+    - "--packages org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0" # <3>
   server:
     podOverrides:
       spec:

docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc

@@ -38,7 +38,7 @@ metadata:
     spark-application.template.applyStrategy: "enforce" # <4>
 spec: # <5>
   sparkImage:
-    productVersion: "4.1.1"
+    productVersion: "4.2.0"
   mode: cluster
   mainClass: com.example.Main
   mainApplicationFile: "/examples.jar"
@@ -60,7 +60,7 @@ metadata:
   name: app-template # <2>
 spec:
   sparkImage:
-    productVersion: "4.1.1"
+    productVersion: "4.2.0"
     pullPolicy: IfNotPresent
   mode: cluster
   mainClass: com.example.Main
@@ -104,7 +104,7 @@ metadata:
     spark-application.template.2.name: "app-template-2"
 spec: # <3>
   sparkImage:
-    productVersion: "4.1.1"
+    productVersion: "4.2.0"
   mode: cluster
   mainClass: com.example.Main
   mainApplicationFile: "/examples.jar"

docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc

@@ -29,7 +29,7 @@ Below is an example of a custom image that includes a JDBC driver:
 
 [source, Dockerfile]
 ----
-FROM oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev # <1>
+FROM oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev # <1>
 
 RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" # <2>
 ----
@@ -41,8 +41,8 @@ Build your custom image and push it to your container registry.
 
 [source, bash]
 ----
-docker build -t my-registry/spark-k8s:3.5.8-psql .
-docker push my-registry/spark-k8s:3.5.8-psql
+docker build -t my-registry/spark-k8s:4.2.0-psql .
+docker push my-registry/spark-k8s:4.2.0-psql
 ----
 
 And the following snippet showcases an application that uses the custom image:
@@ -55,8 +55,8 @@ metadata:
   name: spark-jdbc
 spec:
   sparkImage:
-    custom: "my-registry/spark-k8s:3.5.8-psql" # <1>
-    productVersion: "3.5.8" # <2>
+    custom: "my-registry/spark-k8s:4.2.0-psql" # <1>
+    productVersion: "4.2.0" # <2>
 ...
 ----
 

docs/modules/spark-k8s/pages/usage-guide/operations/pod-placement.adoc

@@ -22,7 +22,7 @@ spec:
   mode: cluster
   mainApplicationFile: app.jar
   sparkImage:
-    productVersion: 4.1.2
+    productVersion: 4.2.0
   job:
     config:
       affinity: # <2>

docs/modules/spark-k8s/pages/usage-guide/spark-connect.adoc

@@ -69,7 +69,7 @@ metadata:
   name: spark-connect
 spec:
   image:
-    productVersion: 4.1.2
+    productVersion: 4.2.0
   connectors:
     s3connection:
       reference: s3-connection
@@ -87,7 +87,7 @@ metadata:
   name: spark-connect
 spec:
   image:
-    productVersion: 4.1.2
+    productVersion: 4.2.0
   connectors:
     s3buckets:
       - reference: aws-cust-1-ingest
@@ -140,7 +140,7 @@ spec:
       spec:
         initContainers:
           - name: kinit
-            image: oci.stackable.tech/sdp/spark-k8s:4.1.1-stackable0.0.0-dev
+            image: oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev
             command: ["/bin/bash", "-euo", "pipefail", "-c"]
             args:
               - kinit -kt /stackable/kerberos/keytab spark-connect/spark-connect.example.svc.cluster.local@EXAMPLE.COM

docs/modules/spark-k8s/partials/supported-versions.adoc

@@ -3,10 +3,10 @@
 // Stackable Platform documentation.
 // Please sort the versions in descending order (newest first)
 
-- 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS)
-- 4.1.1 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated)
-- 3.5.8 (HBase 2.6.6, Hadoop 3.4.3, Scala 2.12, Python 3.11, Java 17) (Deprecated)
+- 4.2.0 (Hadoop 3.5.0, Scala 2.13, Python 3.12, Java 21)
+- 4.1.3 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS)
+- 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated)
 
-Apache Spark 4.1.1 & 4.1.2 have the following known issues (as of July 2026):
+Apache Spark 4.x has the following known issues (as of July 2026):
 
 - Missing HBase compatibility (See: https://issues.apache.org/jira/browse/HBASE-30178)

tests/templates/kuttl/product-config-compat/12-assert.yaml

@@ -13,7 +13,9 @@ commands:
         jq_expr="$4"
 
         expected=$(sed "s|__NAMESPACE__|$NAMESPACE|g" "$fixture_path" | jq -S .)
-        actual=$(kubectl -n "$NAMESPACE" get "$kind" "$name" -o json | jq "$jq_expr" | jq -S .)
+        # Owner reference UIDs embedded in pod templates differ on every run.
+        actual=$(kubectl -n "$NAMESPACE" get "$kind" "$name" -o json | jq "$jq_expr" | jq -S . \
+          | sed -E 's/uid: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/uid: __UID__/g')
 
         expected_file=$(mktemp)
         actual_file=$(mktemp)

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json

@@ -2,5 +2,5 @@
   "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-env.sh": "",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    prometheus.io/scrape: 'true'\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n    prometheus.io/scrape: 'true'\n    stackable.tech/vendor: Stackable\n  name: spark\n  ownerReferences:\n  - apiVersion: spark.stackable.tech/v1alpha1\n    kind: SparkApplication\n    name: pyspark-pi\n    uid: __UID__\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n    image: oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n    seccompProfile:\n      type: RuntimeDefault\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-driver-pod-template\n    name: config\n"
 }

tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json

@@ -2,5 +2,5 @@
   "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE",
   "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n",
   "spark-env.sh": "",
-  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n    image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: config\n"
+  "template.yaml": "metadata:\n  labels:\n    app.kubernetes.io/component: spark\n    app.kubernetes.io/instance: pyspark-pi\n    app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n    app.kubernetes.io/name: spark-k8s\n    app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n    stackable.tech/vendor: Stackable\n  name: spark\nspec:\n  affinity: {}\n  containers:\n  - env:\n    - name: CONTAINERDEBUG_LOG_DIRECTORY\n      value: /stackable/log/containerdebug\n    - name: _STACKABLE_PRE_HOOK\n      value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n    image: oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev\n    imagePullPolicy: IfNotPresent\n    name: spark\n    resources:\n      limits:\n        cpu: '2'\n        memory: 1Gi\n      requests:\n        cpu: '1'\n        memory: 1Gi\n    volumeMounts:\n    - mountPath: /stackable/spark/conf\n      name: config\n    - mountPath: /stackable/log_config\n      name: log-config\n    - mountPath: /stackable/log\n      name: log\n  enableServiceLinks: false\n  securityContext:\n    fsGroup: 1000\n    runAsNonRoot: true\n    seccompProfile:\n      type: RuntimeDefault\n  serviceAccountName: pyspark-pi\n  volumes:\n  - emptyDir:\n      sizeLimit: 39Mi\n    name: log\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: log-config\n  - configMap:\n      name: pyspark-pi-executor-pod-template\n    name: config\n"
 }