diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b8696f2..f56a0fdd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ All notable changes to this project will be documented in this file. ### Added +- Support for Spark `4.1.3` and `4.2.0` ([#790]). - Support floating tags for product images via the new `spec.image.stackableVersionPolicy` field ([#773]). - The history server and Spark Connect server StatefulSets now carry the @@ -43,6 +44,12 @@ All notable changes to this project will be documented in this file. identifiers (matching `[a-zA-Z_][a-zA-Z0-9_]*`) and are now rejected if they are not ([#761]). - Make operations infallible where dependent on static inputs ([#766], [#769]). - Support Vector 0.58.0 ([#784], [#788], [#789]). +- Deprecate support for Spark `4.1.2` ([#790]). +- ClusterRoles for Spark driver and Spark Connect now allow managing `networkpolicies`, which Spark 4.2 creates for executors by default ([#790]). + +### Removed + +- Support for Spark `3.5.8` and `4.1.1` ([#790]). ### Fixed @@ -98,6 +105,7 @@ All notable changes to this project will be documented in this file. [#787]: https://github.com/stackabletech/spark-k8s-operator/pull/787 [#788]: https://github.com/stackabletech/spark-k8s-operator/pull/788 [#789]: https://github.com/stackabletech/spark-k8s-operator/pull/789 +[#790]: https://github.com/stackabletech/spark-k8s-operator/pull/790 ## [26.7.0] - 2026-07-21 diff --git a/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml b/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml index 5bb81e19..ea908c9c 100644 --- a/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml +++ b/deploy/helm/spark-k8s-operator/templates/spark-clusterrole.yaml @@ -27,6 +27,15 @@ rules: - patch - update - watch + # Spark 4.2 and later apply (server-side) a NetworkPolicy for the executor pods. It is owned by + # the driver pod and thereby garbage collected, so no delete is needed. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - patch {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} - apiGroups: - security.openshift.io diff --git a/deploy/helm/spark-k8s-operator/templates/spark-connect-clusterrole.yaml b/deploy/helm/spark-k8s-operator/templates/spark-connect-clusterrole.yaml index 434f132b..01ccfc94 100644 --- a/deploy/helm/spark-k8s-operator/templates/spark-connect-clusterrole.yaml +++ b/deploy/helm/spark-k8s-operator/templates/spark-connect-clusterrole.yaml @@ -27,6 +27,15 @@ rules: - patch - update - watch + # Spark 4.2 and later apply (server-side) a NetworkPolicy for the executor pods. It is owned by + # the driver pod and thereby garbage collected, so no delete is needed. + - apiGroups: + - networking.k8s.io + resources: + - networkpolicies + verbs: + - create + - patch {{ if .Capabilities.APIVersions.Has "security.openshift.io/v1" }} - apiGroups: - security.openshift.io diff --git a/docs/modules/spark-k8s/examples/example-spark-connect.yaml b/docs/modules/spark-k8s/examples/example-spark-connect.yaml index c05e1705..19be5f61 100644 --- a/docs/modules/spark-k8s/examples/example-spark-connect.yaml +++ b/docs/modules/spark-k8s/examples/example-spark-connect.yaml @@ -5,10 +5,10 @@ metadata: name: spark-connect # <1> spec: image: - productVersion: "3.5.8" # <2> + productVersion: "4.1.3" # <2> pullPolicy: IfNotPresent args: - - "--packages org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.11.0" # <3> + - "--packages org.apache.iceberg:iceberg-spark-runtime-4.1_2.13:1.11.0" # <3> server: podOverrides: spec: diff --git a/docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc b/docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc index a10884cd..f23ecf99 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/app_templates.adoc @@ -38,7 +38,7 @@ metadata: spark-application.template.applyStrategy: "enforce" # <4> spec: # <5> sparkImage: - productVersion: "4.1.1" + productVersion: "4.2.0" mode: cluster mainClass: com.example.Main mainApplicationFile: "/examples.jar" @@ -60,7 +60,7 @@ metadata: name: app-template # <2> spec: sparkImage: - productVersion: "4.1.1" + productVersion: "4.2.0" pullPolicy: IfNotPresent mode: cluster mainClass: com.example.Main @@ -104,7 +104,7 @@ metadata: spark-application.template.2.name: "app-template-2" spec: # <3> sparkImage: - productVersion: "4.1.1" + productVersion: "4.2.0" mode: cluster mainClass: com.example.Main mainApplicationFile: "/examples.jar" diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index 235fe5f5..478101b3 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -29,7 +29,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev # <1> +FROM oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" # <2> ---- @@ -41,8 +41,8 @@ Build your custom image and push it to your container registry. [source, bash] ---- -docker build -t my-registry/spark-k8s:3.5.8-psql . -docker push my-registry/spark-k8s:3.5.8-psql +docker build -t my-registry/spark-k8s:4.2.0-psql . +docker push my-registry/spark-k8s:4.2.0-psql ---- And the following snippet showcases an application that uses the custom image: @@ -55,8 +55,8 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "my-registry/spark-k8s:3.5.8-psql" # <1> - productVersion: "3.5.8" # <2> + custom: "my-registry/spark-k8s:4.2.0-psql" # <1> + productVersion: "4.2.0" # <2> ... ---- diff --git a/docs/modules/spark-k8s/pages/usage-guide/operations/pod-placement.adoc b/docs/modules/spark-k8s/pages/usage-guide/operations/pod-placement.adoc index 9ea874b9..8781d76e 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/operations/pod-placement.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/operations/pod-placement.adoc @@ -22,7 +22,7 @@ spec: mode: cluster mainApplicationFile: app.jar sparkImage: - productVersion: 4.1.2 + productVersion: 4.2.0 job: config: affinity: # <2> diff --git a/docs/modules/spark-k8s/pages/usage-guide/spark-connect.adoc b/docs/modules/spark-k8s/pages/usage-guide/spark-connect.adoc index ec86c01b..bc44bcf7 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/spark-connect.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/spark-connect.adoc @@ -69,7 +69,7 @@ metadata: name: spark-connect spec: image: - productVersion: 4.1.2 + productVersion: 4.2.0 connectors: s3connection: reference: s3-connection @@ -87,7 +87,7 @@ metadata: name: spark-connect spec: image: - productVersion: 4.1.2 + productVersion: 4.2.0 connectors: s3buckets: - reference: aws-cust-1-ingest @@ -140,7 +140,7 @@ spec: spec: initContainers: - name: kinit - image: oci.stackable.tech/sdp/spark-k8s:4.1.1-stackable0.0.0-dev + image: oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev command: ["/bin/bash", "-euo", "pipefail", "-c"] args: - kinit -kt /stackable/kerberos/keytab spark-connect/spark-connect.example.svc.cluster.local@EXAMPLE.COM diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index 494d69e6..4213ff6a 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,10 +3,10 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS) -- 4.1.1 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated) -- 3.5.8 (HBase 2.6.6, Hadoop 3.4.3, Scala 2.12, Python 3.11, Java 17) (Deprecated) +- 4.2.0 (Hadoop 3.5.0, Scala 2.13, Python 3.12, Java 21) +- 4.1.3 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (LTS) +- 4.1.2 (Hadoop 3.4.3, Scala 2.13, Python 3.12, Java 21) (Deprecated) -Apache Spark 4.1.1 & 4.1.2 have the following known issues (as of July 2026): +Apache Spark 4.x has the following known issues (as of July 2026): - Missing HBase compatibility (See: https://issues.apache.org/jira/browse/HBASE-30178) diff --git a/tests/templates/kuttl/product-config-compat/12-assert.yaml b/tests/templates/kuttl/product-config-compat/12-assert.yaml index 269b674e..25ded920 100644 --- a/tests/templates/kuttl/product-config-compat/12-assert.yaml +++ b/tests/templates/kuttl/product-config-compat/12-assert.yaml @@ -13,7 +13,9 @@ commands: jq_expr="$4" expected=$(sed "s|__NAMESPACE__|$NAMESPACE|g" "$fixture_path" | jq -S .) - actual=$(kubectl -n "$NAMESPACE" get "$kind" "$name" -o json | jq "$jq_expr" | jq -S .) + # Owner reference UIDs embedded in pod templates differ on every run. + actual=$(kubectl -n "$NAMESPACE" get "$kind" "$name" -o json | jq "$jq_expr" | jq -S . \ + | sed -E 's/uid: [0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/uid: __UID__/g') expected_file=$(mktemp) actual_file=$(mktemp) diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json index d3b356ec..c8664e09 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-driver-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n prometheus.io/scrape: 'true'\n stackable.tech/vendor: Stackable\n name: spark\n ownerReferences:\n - apiVersion: spark.stackable.tech/v1alpha1\n kind: SparkApplication\n name: pyspark-pi\n uid: __UID__\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n image: oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-driver-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json index 7d15d473..0a6347f5 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-executor-pod-template-data.json @@ -2,5 +2,5 @@ "log4j2.properties": "appenders = FILE, CONSOLE\n\nappender.CONSOLE.type = Console\nappender.CONSOLE.name = CONSOLE\nappender.CONSOLE.target = SYSTEM_ERR\nappender.CONSOLE.layout.type = PatternLayout\nappender.CONSOLE.layout.pattern = %d{ISO8601} %p [%t] %c - %m%n\nappender.CONSOLE.filter.threshold.type = ThresholdFilter\nappender.CONSOLE.filter.threshold.level = INFO\n\nappender.FILE.type = RollingFile\nappender.FILE.name = FILE\nappender.FILE.fileName = /stackable/log/spark/spark.log4j2.xml\nappender.FILE.filePattern = /stackable/log/spark/spark.log4j2.xml.%i\nappender.FILE.layout.type = XMLLayout\nappender.FILE.policies.type = Policies\nappender.FILE.policies.size.type = SizeBasedTriggeringPolicy\nappender.FILE.policies.size.size = 5MB\nappender.FILE.strategy.type = DefaultRolloverStrategy\nappender.FILE.strategy.max = 1\nappender.FILE.filter.threshold.type = ThresholdFilter\nappender.FILE.filter.threshold.level = INFO\n\n\nrootLogger.level=INFO\nrootLogger.appenderRefs = CONSOLE, FILE\nrootLogger.appenderRef.CONSOLE.ref = CONSOLE\nrootLogger.appenderRef.FILE.ref = FILE", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", "spark-env.sh": "", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n image: oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: spark\n app.kubernetes.io/instance: pyspark-pi\n app.kubernetes.io/managed-by: spark.stackable.tech_sparkapplication\n app.kubernetes.io/name: spark-k8s\n app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\n name: spark\nspec:\n affinity: {}\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n - name: _STACKABLE_PRE_HOOK\n value: containerdebug --output=/stackable/log/containerdebug-state.json --loop & . /stackable/spark/conf/spark-env.sh; export SPARK_ENV_LOADED=1\n image: oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev\n imagePullPolicy: IfNotPresent\n name: spark\n resources:\n limits:\n cpu: '2'\n memory: 1Gi\n requests:\n cpu: '1'\n memory: 1Gi\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/log\n name: log\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n serviceAccountName: pyspark-pi\n volumes:\n - emptyDir:\n sizeLimit: 39Mi\n name: log\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: log-config\n - configMap:\n name: pyspark-pi-executor-pod-template\n name: config\n" } diff --git a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json index 9b7757fc..108065db 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/pyspark-pi-job-template-spec.json @@ -3,7 +3,7 @@ "containers": [ { "args": [ - "containerdebug --output=/stackable/log/containerdebug-state.json --loop & /stackable/spark/bin/spark-submit --verbose --master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} --deploy-mode cluster --name pyspark-pi --conf spark.kubernetes.driver.podTemplateFile=/stackable/spark/driver-pod-templates/template.yaml --conf spark.kubernetes.executor.podTemplateFile=/stackable/spark/executor-pod-templates/template.yaml --conf spark.kubernetes.driver.podTemplateContainerName=spark --conf spark.kubernetes.executor.podTemplateContainerName=spark --conf spark.kubernetes.namespace=__NAMESPACE__ --conf spark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev --conf spark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev --conf spark.driver.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.driver.extraClassPath=/stackable/spark/extra-jars/* --conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.executor.extraClassPath=/stackable/spark/extra-jars/* --conf spark.driver.extraJavaOptions=\"-Djava.security.properties=/stackable/spark/conf/security.properties\" --conf spark.executor.extraJavaOptions=\"-Djava.security.properties=/stackable/spark/conf/security.properties\" --conf spark.metrics.conf.\\*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet --conf spark.metrics.conf.\\*.sink.prometheusServlet.path=/metrics/prometheus --conf spark.ui.prometheus.enabled=true --conf spark.sql.streaming.metricsEnabled=true --conf \"spark.driver.cores=2\" --conf \"spark.driver.memory=640m\" --conf \"spark.executor.cores=2\" --conf \"spark.executor.instances=1\" --conf \"spark.executor.memory=640m\" --conf \"spark.kubernetes.container.image.pullPolicy=IfNotPresent\" --conf \"spark.kubernetes.driver.limit.cores=2\" --conf \"spark.kubernetes.driver.request.cores=1\" --conf \"spark.kubernetes.executor.limit.cores=2\" --conf \"spark.kubernetes.executor.request.cores=1\" --conf \"spark.kubernetes.memoryOverheadFactor=0.0\" local:///stackable/spark/examples/src/main/python/pi.py" + "containerdebug --output=/stackable/log/containerdebug-state.json --loop & /stackable/spark/bin/spark-submit --verbose --master k8s://https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT_HTTPS} --deploy-mode cluster --name pyspark-pi --conf spark.kubernetes.driver.podTemplateFile=/stackable/spark/driver-pod-templates/template.yaml --conf spark.kubernetes.executor.podTemplateFile=/stackable/spark/executor-pod-templates/template.yaml --conf spark.kubernetes.driver.podTemplateContainerName=spark --conf spark.kubernetes.executor.podTemplateContainerName=spark --conf spark.kubernetes.namespace=__NAMESPACE__ --conf spark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev --conf spark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev --conf spark.driver.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.driver.extraClassPath=/stackable/spark/extra-jars/* --conf spark.executor.defaultJavaOptions=-Dlog4j.configurationFile=/stackable/log_config/log4j2.properties --conf spark.executor.extraClassPath=/stackable/spark/extra-jars/* --conf spark.driver.extraJavaOptions=\"-Djava.security.properties=/stackable/spark/conf/security.properties\" --conf spark.executor.extraJavaOptions=\"-Djava.security.properties=/stackable/spark/conf/security.properties\" --conf spark.metrics.conf.\\*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet --conf spark.metrics.conf.\\*.sink.prometheusServlet.path=/metrics/prometheus --conf spark.ui.prometheus.enabled=true --conf spark.sql.streaming.metricsEnabled=true --conf \"spark.driver.cores=2\" --conf \"spark.driver.memory=640m\" --conf \"spark.executor.cores=2\" --conf \"spark.executor.instances=1\" --conf \"spark.executor.memory=640m\" --conf \"spark.kubernetes.container.image.pullPolicy=IfNotPresent\" --conf \"spark.kubernetes.driver.limit.cores=2\" --conf \"spark.kubernetes.driver.request.cores=1\" --conf \"spark.kubernetes.executor.limit.cores=2\" --conf \"spark.kubernetes.executor.request.cores=1\" --conf \"spark.kubernetes.memoryOverheadFactor=0.0\" local:///stackable/spark/examples/src/main/python/pi.py" ], "command": [ "/bin/bash", @@ -30,7 +30,7 @@ "value": "containerdebug --output=/stackable/log/containerdebug-state.json --loop &" } ], - "image": "oci.stackable.tech/sdp/spark-k8s:3.5.8-stackable0.0.0-dev", + "image": "oci.stackable.tech/sdp/spark-k8s:4.2.0-stackable0.0.0-dev", "imagePullPolicy": "IfNotPresent", "name": "spark-submit", "resources": { @@ -70,7 +70,10 @@ "schedulerName": "default-scheduler", "securityContext": { "fsGroup": 1000, - "runAsNonRoot": true + "runAsNonRoot": true, + "seccompProfile": { + "type": "RuntimeDefault" + } }, "serviceAccount": "pyspark-pi", "serviceAccountName": "pyspark-pi", diff --git a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json index 015a970d..dc889b4e 100644 --- a/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json +++ b/tests/templates/kuttl/product-config-compat/fixtures/spark-connect-server-data.json @@ -1,6 +1,6 @@ { "metrics.properties": "*.sink.prometheusServlet.class=org.apache.spark.metrics.sink.PrometheusServlet\n*.sink.prometheusServlet.path=/metrics/prometheus\n", "security.properties": "networkaddress.cache.negative.ttl=0\nnetworkaddress.cache.ttl=30\n", - "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-3.5.8.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.container.image.pullPolicy=Always\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:3.5.8-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n", - "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 3.5.8-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/truststore\n name: stackable-truststore\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - configMap:\n name: spark-connect-log-config\n name: log-config\n - emptyDir: {}\n name: stackable-truststore\n" + "spark-defaults.conf": "spark.driver.cores=3\nspark.driver.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\\ -Dmy.custom.jvm.arg\\=customValue\nspark.driver.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.driver.host=spark-connect-server-headless\nspark.executor.defaultJavaOptions=-Djava.security.properties\\=/stackable/spark/conf/security.properties\\ -Dlog4j.configurationFile\\=/stackable/log_config/log4j2.properties\nspark.executor.extraClassPath=/stackable/spark/extra-jars/*\\:/stackable/spark/connect/spark-connect-4.2.0.jar\nspark.executor.instances=3\nspark.executor.memory=1024M\nspark.executor.memoryOverhead=1m\nspark.kubernetes.authenticate.driver.serviceAccountName=spark-connect-serviceaccount\nspark.kubernetes.container.image.pullPolicy=Always\nspark.kubernetes.driver.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.driver.pod.name=${env\\:HOSTNAME}\nspark.kubernetes.executor.container.image=oci.stackable.tech/sdp/spark-k8s\\:4.2.0-stackable0.0.0-dev\nspark.kubernetes.executor.limit.cores=1\nspark.kubernetes.executor.podTemplateContainerName=spark\nspark.kubernetes.executor.podTemplateFile=/stackable/spark/conf/template.yaml\nspark.kubernetes.executor.request.cores=1\nspark.kubernetes.namespace=__NAMESPACE__\nspark.metrics.conf=/stackable/spark/conf/metrics.properties\nspark.ui.prometheus.enabled=true\n", + "template.yaml": "metadata:\n labels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/managed-by: spark.stackable.tech_connect\n app.kubernetes.io/name: spark-connect\n app.kubernetes.io/version: 4.2.0-stackable0.0.0-dev\n stackable.tech/vendor: Stackable\nspec:\n affinity:\n podAntiAffinity:\n preferredDuringSchedulingIgnoredDuringExecution:\n - podAffinityTerm:\n labelSelector:\n matchLabels:\n app.kubernetes.io/component: executor\n app.kubernetes.io/instance: spark-connect\n app.kubernetes.io/name: spark-connect\n topologyKey: kubernetes.io/hostname\n weight: 70\n containers:\n - env:\n - name: CONTAINERDEBUG_LOG_DIRECTORY\n value: /stackable/log/containerdebug\n name: spark\n volumeMounts:\n - mountPath: /stackable/spark/conf\n name: config\n - mountPath: /stackable/log\n name: log\n - mountPath: /stackable/log_config\n name: log-config\n - mountPath: /stackable/truststore\n name: stackable-truststore\n enableServiceLinks: false\n securityContext:\n fsGroup: 1000\n runAsNonRoot: true\n seccompProfile:\n type: RuntimeDefault\n volumes:\n - emptyDir:\n sizeLimit: 30Mi\n name: log\n - configMap:\n name: spark-connect-executor\n name: config\n - configMap:\n name: spark-connect-log-config\n name: log-config\n - emptyDir: {}\n name: stackable-truststore\n" } diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 9247d55c..bfd18f7a 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -5,34 +5,34 @@ dimensions: - "false" - name: spark values: - - 3.5.8 - - 4.1.1 - 4.1.2 + - 4.1.3 + - 4.2.0 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-logging values: - - 3.5.8 - - 4.1.1 - 4.1.2 - - name: spark-hbase-connector - values: - - 3.5.8 - # No hbase-connector release with support for Spark 4 yet. - # - 4.1.1 + - 4.1.3 + - 4.2.0 + # No hbase-connector release with support for Spark 4 yet, so the hbase-connector test is disabled + # until one is available. + # - name: spark-hbase-connector + # values: + # - 3.5.8 - name: spark-delta-lake values: - - 3.5.8 - - 4.1.1 - 4.1.2 + - 4.1.3 + - 4.2.0 # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev # Note: We keep this list, as new Spark versions are only supported delayed by Iceberg - name: spark-iceberg values: - - 3.5.8 - - 4.1.1 - 4.1.2 + - 4.1.3 + # - 4.2.0 # No iceberg-spark-runtime-4.2 release yet - name: iceberg-latest values: - 1.11.0 @@ -41,13 +41,13 @@ dimensions: - 4.0.0 - name: spark-connect values: - - 3.5.8 - - 4.1.1 - 4.1.2 + - 4.1.3 + # - 4.2.0 # No iceberg-spark-runtime-4.2 release yet # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-regression values: - - 3.5.8 + - 4.2.0 - name: krb5 values: - 1.21.1 @@ -136,13 +136,13 @@ tests: dimensions: - spark-delta-lake - openshift - - name: hbase-connector - dimensions: - - spark-hbase-connector - - hbase - - hdfs-latest - - zookeeper-latest - - openshift + # - name: hbase-connector + # dimensions: + # - spark-hbase-connector + # - hbase + # - hdfs-latest + # - zookeeper-latest + # - openshift - name: custom-log-directory dimensions: - spark