From d0abd58239642382c23830a281019c914bbe5d3a Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 2 Nov 2026 08:58:35 -0800 Subject: [PATCH 01/18] fix(launchers): Keep resolved input values out of error messages A dataclass prints every field in its generated repr, so interpolating an InputArgument into a launcher error message published the resolved secret to the log, the error reporter, the database and the execution-log API alike. repr=False on the value field removes it from all 19 of those messages at once, while leaving the field itself unchanged. --- .../launchers/interfaces.py | 5 +- tests/launchers/test_input_argument_repr.py | 108 ++++++++++++++++++ 2 files changed, 112 insertions(+), 1 deletion(-) create mode 100644 tests/launchers/test_input_argument_repr.py diff --git a/cloud_pipelines_backend/launchers/interfaces.py b/cloud_pipelines_backend/launchers/interfaces.py index 433e09dc..365ddafa 100644 --- a/cloud_pipelines_backend/launchers/interfaces.py +++ b/cloud_pipelines_backend/launchers/interfaces.py @@ -51,7 +51,10 @@ class LaunchedContainerNotFoundError(LauncherError): class InputArgument: total_size: int is_dir: bool - value: str | None = None + # A secret consumed as a value arrives here in plaintext, and the generated repr of this + # class is interpolated into launcher error messages, which are logged, reported and + # stored. `repr=False` keeps the value out of every one of those strings. + value: str | None = dataclasses.field(default=None, repr=False) uri: str | None = None staging_uri: str dynamic_data: str | dict[str, Any] | None = None diff --git a/tests/launchers/test_input_argument_repr.py b/tests/launchers/test_input_argument_repr.py new file mode 100644 index 00000000..6978f74a --- /dev/null +++ b/tests/launchers/test_input_argument_repr.py @@ -0,0 +1,108 @@ +"""The resolved input argument must not print its value. + +A secret consumed as a value is held in `InputArgument.value`. Launcher error messages +interpolate the whole argument -- `f"... {input_argument=}"` -- and the orchestrator logs +that message, reports it, stores it on the execution node and serves it through the +execution-log API, so a printed value reaches several durable places at once. +""" + +import dataclasses +import json +import pathlib +import traceback +from typing import Final + +from cloud_pipelines_backend.launchers import interfaces + +_SECRET_VALUE: Final[str] = "secret-value-authenticates-nothing" + +_LAUNCHER_MODULES: Final[tuple[str, ...]] = ( + "kubernetes_launchers.py", + "local_docker_launchers.py", + "huggingface_launchers.py", +) + + +def _input_argument() -> interfaces.InputArgument: + return interfaces.InputArgument( + total_size=len(_SECRET_VALUE), + is_dir=False, + value=_SECRET_VALUE, + uri=None, + staging_uri="memory://staging/inputs/token/data", + ) + + +class TestInputArgumentDoesNotPrintItsValue: + def test_every_way_of_stringifying_it_omits_the_value(self) -> None: + argument = _input_argument() + for rendered in ( + repr(argument), + str(argument), + f"{argument}", + f"{argument!r}", + f"{argument=}", + "{}".format(argument), + "%s" % (argument,), + ): + assert _SECRET_VALUE not in rendered, rendered + + def test_the_other_fields_still_print(self) -> None: + """The messages are written to identify an input, so they must keep doing that.""" + rendered = repr(_input_argument()) + assert "total_size=" in rendered + assert "is_dir=" in rendered + assert "staging_uri=" in rendered + + def test_the_value_is_still_readable_and_still_a_field(self) -> None: + """Hidden from printing, not removed: the launcher passes it to the container.""" + argument = _input_argument() + assert argument.value == _SECRET_VALUE + assert "value" in {field.name for field in dataclasses.fields(argument)} + + def test_a_traceback_built_the_way_the_launchers_build_one_is_clean(self) -> None: + """The message shape used at every raise site that reports an input argument.""" + argument = _input_argument() + input_name = "token" + try: + raise interfaces.LauncherError( + f"Artifact is too big to consume as value. Consume it as file instead." + f" {input_name=}, {argument=}" + ) + except interfaces.LauncherError: + captured = traceback.format_exc() + assert _SECRET_VALUE not in captured + assert "input_name='token'" in captured + + +class TestNoLauncherMessageReachesAroundTheRepr: + """`repr=False` is defeated by a message that interpolates the field directly. + + This guards the fix rather than the symptom: a new `{input_argument=}` is safe, while + an `{input_argument.value}` would print the secret again. + """ + + def test_no_launcher_formats_an_argument_value_into_a_string(self) -> None: + launchers_dir = pathlib.Path(interfaces.__file__).resolve().parent + offenders: list[str] = [] + for module_name in _LAUNCHER_MODULES: + source = (launchers_dir / module_name).read_text() + for forbidden in ( + "{input_argument.value", + "{artifact_data.value", + "{ia.value", + ): + if forbidden in source: + offenders.append(f"{module_name}: {forbidden}") + assert not offenders, offenders + + def test_the_value_is_not_serialized_into_a_dict_anywhere(self) -> None: + """`dataclasses.asdict` ignores `repr=False`, so no launcher may use it here.""" + launchers_dir = pathlib.Path(interfaces.__file__).resolve().parent + for module_name in _LAUNCHER_MODULES: + source = (launchers_dir / module_name).read_text() + assert "asdict(input_argument" not in source, module_name + + def test_asdict_would_have_leaked_it(self) -> None: + """Pins why the test above exists, rather than asserting a bare convention.""" + assert _SECRET_VALUE in json.dumps(dataclasses.asdict(_input_argument())) From ab18e5d20b56a0eade0f14a63a9df427cb688e10 Mon Sep 17 00:00:00 2001 From: Alexey Volkov Date: Mon, 9 Nov 2026 11:45:47 -0800 Subject: [PATCH 02/18] feat: Kubernetes launcher - Secrets - Deliver secret inputs through a Kubernetes Secret Carries #431 unchanged on the current master; the import block conflict is resolved by keeping re, threading and time. --- .../launchers/kubernetes_launchers.py | 110 ++++++++++++++++++ 1 file changed, 110 insertions(+) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 14d05ece..0777d0f6 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -8,6 +8,7 @@ import logging import os import pathlib +import re import threading import time import typing @@ -551,6 +552,105 @@ def __init__( _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) + def _handle_secrets_before_preparing_pod( + self, + *, + input_arguments: dict[str, interfaces.InputArgument], + ) -> tuple[dict[str, str], dict[str, str]]: + # TODO!: Handle secrets consumed as files by mounting the required Secret keys as volumes. + secret_name_to_secret_value: dict[str, str] = {} + secret_env_name_to_secret_name: dict[str, str] = {} + for input_name, input_argument in list(input_arguments.items()): + if not input_argument.dynamic_data: + continue + dynamic_data_kind, dynamic_data_properties = ( + container_component_utils.parse_dynamic_data_argument( + input_argument.dynamic_data + ) + ) + # Hiding secrets + _DYNAMIC_DATA_SECRET_KEY = "secret" + _DYNAMIC_DATA_SECRET_NAME_KEY = "name" + if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: + secret_name = (dynamic_data_properties or {}).get( + _DYNAMIC_DATA_SECRET_NAME_KEY + ) + if not input_argument.value: + raise interfaces.LauncherError( + f"Orchestrator should have filled secret value for {input_name=}, but got: {input_argument=}" + ) + if not secret_name: + raise interfaces.LauncherError( + f"Secret does not have name: {input_argument.dynamic_data=}" + ) + + env_var_name = "__TANGLE_SECRET_" + re.sub( + "[^_0-9a-zA-Z]+", "_", input_name.upper() + ) + secret_name_to_secret_value[secret_name] = input_argument.value + secret_env_name_to_secret_name[env_var_name] = secret_name + # Hiding the secret value by replacing it with env variable sourced from a Kubernetes Secret + input_argument.value = f"$({secret_name})" + continue + return secret_name_to_secret_value, secret_env_name_to_secret_name + + def _handle_secrets_after_preparing_pod( + self, + *, + pod: k8s_client_lib.V1Pod, + secret_name_to_secret_value: dict[str, str], + secret_env_name_to_secret_name: dict[str, str], + annotations: dict[str, Any] | None = None, + ): + # Create secrets + if not secret_name_to_secret_value or not secret_env_name_to_secret_name: + return + + assert pod.metadata + # TODO: Stop supporting changing namespace in _pod_postprocessor + namespace = pod.metadata.namespace or self._namespace + + created_by = (annotations or {}).get( + common_annotations.PIPELINE_RUN_CREATED_BY_ANNOTATION_KEY + ) or "anonymous" + # Kubernetes' Secrets can contain multiple key-value pairs. + # So we store all user's secret in a single Kubernetes Secret object + kubernetes_secret_name = naming_utils.sanitize_kubernetes_resource_name( + created_by.replace("@", "-at-") + ) + core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) + try: + core_api_client.create_namespaced_secret( + namespace=namespace, + body=k8s_client_lib.V1Secret( + metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), + data=secret_name_to_secret_value, + ), + ) + except Exception: + # TODO: Verify that such patching does not delete other secret key-value pairs. + core_api_client.patch_namespaced_secret( + name=kubernetes_secret_name, + namespace=namespace, + body=k8s_client_lib.V1Secret(data=secret_name_to_secret_value), + ) + + # TODO: properly select the main container + main_container = pod.spec.containers[0] + for env_var_name, secret_name in secret_env_name_to_secret_name.items(): + main_container.env = main_container.env or [] + main_container.env.append( + k8s_client_lib.V1EnvVar( + name=env_var_name, + value_from=k8s_client_lib.V1EnvVarSource( + secret_key_ref=k8s_client_lib.V1SecretKeySelector( + name=kubernetes_secret_name, + key=secret_name, + ) + ), + ) + ) + def launch_container_task( self, *, @@ -580,6 +680,9 @@ def launch_container_task( _MULTI_NODE_NODE_0_ADDRESS_DYNAMIC_DATA_KEY: "localhost", _MULTI_NODE_ALL_NODE_ADDRESSES_DYNAMIC_DATA_KEY: "localhost", } + secret_name_to_secret_value, secret_env_name_to_secret_name = ( + self._handle_secrets_before_preparing_pod(input_arguments=input_arguments) + ) for input_name, input_argument in list(input_arguments.items()): if input_argument.value is not None or input_argument.uri is not None: continue @@ -614,6 +717,13 @@ def launch_container_task( if self._pod_postprocessor: pod = self._pod_postprocessor(pod=pod, annotations=annotations) + self._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=secret_name_to_secret_value, + secret_env_name_to_secret_name=secret_env_name_to_secret_name, + annotations=annotations, + ) + core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) try: created_pod: k8s_client_lib.V1Pod = core_api_client.create_namespaced_pod( From d1ee4016efbec839885cb910fa47a42968a1b73e Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 12:14:33 -0800 Subject: [PATCH 03/18] test: Kubernetes launcher - Secrets - Cover secret delivery through a Kubernetes Secret --- .../kubernetes/test_kubernetes_secrets.py | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) create mode 100644 tests/launchers/kubernetes/test_kubernetes_secrets.py diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py new file mode 100644 index 00000000..70b22589 --- /dev/null +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -0,0 +1,190 @@ +"""Tests for delivering secret inputs to a container through a Kubernetes Secret.""" + +from __future__ import annotations + +from typing import Final + +import pytest +from kubernetes import client as k8s_client_lib + +from cloud_pipelines_backend.launchers import ( + common_annotations, + interfaces, + kubernetes_launchers, +) + +_SECRET_VALUE: Final[str] = "FAKE-tok-123" +_CREATED_BY_ALICE: Final[dict[str, str]] = { + common_annotations.PIPELINE_RUN_CREATED_BY_ANNOTATION_KEY: "alice@example.com" +} + + +class _SecretsApi: + """Stands in for the Kubernetes API and records the Secrets the launcher writes.""" + + def __init__(self) -> None: + self.created: list[tuple[str, k8s_client_lib.V1Secret]] = [] + self.patched: list[tuple[str, k8s_client_lib.V1Secret]] = [] + self.existing: set[str] = set() + + def create_namespaced_secret( + self, *, namespace: str, body: k8s_client_lib.V1Secret + ) -> None: + if body.metadata.name in self.existing: + raise k8s_client_lib.ApiException(status=409, reason="Conflict") + self.created.append((namespace, body)) + + def patch_namespaced_secret( + self, *, name: str, namespace: str, body: k8s_client_lib.V1Secret + ) -> None: + del namespace + self.patched.append((name, body)) + + +@pytest.fixture +def secrets_api(monkeypatch: pytest.MonkeyPatch) -> _SecretsApi: + api = _SecretsApi() + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda *, api_client: api) + return api + + +def _make_launcher() -> kubernetes_launchers._KubernetesPodLauncher: + # Skips __init__, which contacts the cluster; the secret handling reads only these two. + launcher = object.__new__(kubernetes_launchers._KubernetesPodLauncher) + launcher._api_client = None + launcher._namespace = "default" + return launcher + + +def _make_secret_argument() -> interfaces.InputArgument: + return interfaces.InputArgument( + total_size=len(_SECRET_VALUE), + is_dir=False, + value=_SECRET_VALUE, + staging_uri="gs://bucket/token", + dynamic_data={"secret": {"name": "hf-token"}}, + ) + + +def _make_pod(*, args: list[str]) -> k8s_client_lib.V1Pod: + return k8s_client_lib.V1Pod( + metadata=k8s_client_lib.V1ObjectMeta(name="test-pod", namespace="tangle-jobs"), + spec=k8s_client_lib.V1PodSpec( + containers=[k8s_client_lib.V1Container(name="main", args=args)] + ), + ) + + +def _launch( + *, + launcher: kubernetes_launchers._KubernetesPodLauncher, + annotations: dict[str, str] | None = None, +) -> k8s_client_lib.V1Pod: + """Runs both secret phases around a pod whose only argument is the secret input.""" + argument = _make_secret_argument() + values, env_names = launcher._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + pod = _make_pod(args=[argument.value]) + launcher._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=values, + secret_env_name_to_secret_name=env_names, + annotations=annotations, + ) + return pod + + +def test_secret_value_is_replaced_by_a_placeholder() -> None: + argument = _make_secret_argument() + + values, env_names = _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + + assert _SECRET_VALUE not in argument.value + assert values == {"hf-token": _SECRET_VALUE} + assert env_names == {"__TANGLE_SECRET_TOKEN": "hf-token"} + + +def test_inputs_without_secrets_are_left_alone() -> None: + argument = interfaces.InputArgument( + total_size=5, is_dir=False, value="hello", staging_uri="gs://bucket/greeting" + ) + + values, env_names = _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"greeting": argument} + ) + + assert (values, env_names) == ({}, {}) + assert argument.value == "hello" + + +@pytest.mark.parametrize( + "value, dynamic_data", + [ + pytest.param(None, {"secret": {"name": "hf-token"}}, id="no value"), + pytest.param(_SECRET_VALUE, {"secret": {}}, id="no secret name"), + ], +) +def test_incomplete_secret_argument_is_a_launcher_error( + value: str | None, dynamic_data: dict[str, dict[str, str]] +) -> None: + argument = interfaces.InputArgument( + total_size=0, + is_dir=False, + value=value, + staging_uri="gs://bucket/token", + dynamic_data=dynamic_data, + ) + + with pytest.raises(interfaces.LauncherError): + _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + + +def test_secret_is_created_in_the_pod_namespace_and_named_after_the_run_creator( + secrets_api: _SecretsApi, +) -> None: + _launch(launcher=_make_launcher(), annotations=_CREATED_BY_ALICE) + + [(namespace, secret)] = secrets_api.created + assert namespace == "tangle-jobs" + assert secret.metadata.name == "alice-at-example-com" + + +def test_run_without_a_creator_uses_the_anonymous_secret( + secrets_api: _SecretsApi, +) -> None: + _launch(launcher=_make_launcher()) + + [(_, secret)] = secrets_api.created + assert secret.metadata.name == "anonymous" + + +def test_existing_secret_is_patched(secrets_api: _SecretsApi) -> None: + secrets_api.existing.add("alice-at-example-com") + + _launch(launcher=_make_launcher(), annotations=_CREATED_BY_ALICE) + + assert secrets_api.created == [] + [(name, _)] = secrets_api.patched + assert name == "alice-at-example-com" + + +@pytest.mark.usefixtures("secrets_api") +def test_container_reads_the_value_from_the_secret() -> None: + pod = _launch(launcher=_make_launcher(), annotations=_CREATED_BY_ALICE) + + assert pod.spec.containers[0].env == [ + k8s_client_lib.V1EnvVar( + name="__TANGLE_SECRET_TOKEN", + value_from=k8s_client_lib.V1EnvVarSource( + secret_key_ref=k8s_client_lib.V1SecretKeySelector( + name="alice-at-example-com", key="hf-token" + ) + ), + ) + ] + assert _SECRET_VALUE not in pod.spec.containers[0].args[0] From 0456d4a996837ada5e9d02b3d8bd8ffbd1536d35 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 13 Nov 2026 01:24:27 -0800 Subject: [PATCH 04/18] fix: Kubernetes launcher - Secrets - Refuse two secret inputs on one env var name Input names like api-key and api_key both become __TANGLE_SECRET_API_KEY; the second used to overwrite the first, so one input silently received the other's value. The launch now fails with LauncherError. --- .../launchers/kubernetes_launchers.py | 4 ++++ tests/launchers/kubernetes/test_kubernetes_secrets.py | 10 ++++++++++ 2 files changed, 14 insertions(+) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 0777d0f6..42495fdf 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -587,6 +587,10 @@ def _handle_secrets_before_preparing_pod( env_var_name = "__TANGLE_SECRET_" + re.sub( "[^_0-9a-zA-Z]+", "_", input_name.upper() ) + if env_var_name in secret_env_name_to_secret_name: + raise interfaces.LauncherError( + f"Two secret inputs map to the environment variable {env_var_name}; rename {input_name!r} or the other one" + ) secret_name_to_secret_value[secret_name] = input_argument.value secret_env_name_to_secret_name[env_var_name] = secret_name # Hiding the secret value by replacing it with env variable sourced from a Kubernetes Secret diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 70b22589..4827b48f 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -188,3 +188,13 @@ def test_container_reads_the_value_from_the_secret() -> None: ) ] assert _SECRET_VALUE not in pod.spec.containers[0].args[0] + + +def test_two_secret_inputs_on_one_env_var_name_are_a_launcher_error() -> None: + first, second = _make_secret_argument(), _make_secret_argument() + second.dynamic_data = {"secret": {"name": "anthropic-key"}} + + with pytest.raises(interfaces.LauncherError, match="__TANGLE_SECRET_API_KEY"): + _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"api-key": first, "api_key": second} + ) From 5245e97f5d5d3a47fcc12baae5893e8c8b79c1e2 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 11:59:46 -0800 Subject: [PATCH 05/18] fix: Kubernetes launcher - Secrets - Point the placeholder at the environment variable --- .../launchers/kubernetes_launchers.py | 2 +- tests/launchers/kubernetes/test_kubernetes_secrets.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 42495fdf..f924b8a4 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -594,7 +594,7 @@ def _handle_secrets_before_preparing_pod( secret_name_to_secret_value[secret_name] = input_argument.value secret_env_name_to_secret_name[env_var_name] = secret_name # Hiding the secret value by replacing it with env variable sourced from a Kubernetes Secret - input_argument.value = f"$({secret_name})" + input_argument.value = f"$({env_var_name})" continue return secret_name_to_secret_value, secret_env_name_to_secret_name diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 4827b48f..015f33ca 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -198,3 +198,13 @@ def test_two_secret_inputs_on_one_env_var_name_are_a_launcher_error() -> None: _make_launcher()._handle_secrets_before_preparing_pod( input_arguments={"api-key": first, "api_key": second} ) + + +@pytest.mark.usefixtures("secrets_api") +def test_placeholder_names_the_environment_variable_the_pod_defines() -> None: + pod = _launch(launcher=_make_launcher()) + + container = pod.spec.containers[0] + assert [env.name for env in container.env] == ["__TANGLE_SECRET_TOKEN"] + assert container.args == ["$(__TANGLE_SECRET_TOKEN)"] + assert _SECRET_VALUE not in container.args[0] From e6ba67a1ee15821d1e1c8f8780df94581a6e3e33 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 12:18:03 -0800 Subject: [PATCH 06/18] fix: Kubernetes launcher - Secrets - Write values through stringData, not the base64 data field --- .../launchers/kubernetes_launchers.py | 4 ++-- tests/launchers/kubernetes/test_kubernetes_secrets.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index f924b8a4..33887c7e 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -628,7 +628,7 @@ def _handle_secrets_after_preparing_pod( namespace=namespace, body=k8s_client_lib.V1Secret( metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), - data=secret_name_to_secret_value, + string_data=secret_name_to_secret_value, ), ) except Exception: @@ -636,7 +636,7 @@ def _handle_secrets_after_preparing_pod( core_api_client.patch_namespaced_secret( name=kubernetes_secret_name, namespace=namespace, - body=k8s_client_lib.V1Secret(data=secret_name_to_secret_value), + body=k8s_client_lib.V1Secret(string_data=secret_name_to_secret_value), ) # TODO: properly select the main container diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 015f33ca..1028a4a7 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -208,3 +208,14 @@ def test_placeholder_names_the_environment_variable_the_pod_defines() -> None: assert [env.name for env in container.env] == ["__TANGLE_SECRET_TOKEN"] assert container.args == ["$(__TANGLE_SECRET_TOKEN)"] assert _SECRET_VALUE not in container.args[0] + + +def test_secret_carries_the_plain_value_in_string_data( + secrets_api: _SecretsApi, +) -> None: + # `data` must be base64, so the API server would decode the plain value into other bytes. + _launch(launcher=_make_launcher()) + + [(_, secret)] = secrets_api.created + assert secret.string_data == {"hf-token": _SECRET_VALUE} + assert secret.data is None From 068e553eb85a5ea2a0db902baca9b2b7c0e21093 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 12:21:48 -0800 Subject: [PATCH 07/18] fix: Kubernetes launcher - Secrets - Add the environment entry to the main container --- .../launchers/kubernetes_launchers.py | 7 +++++-- .../kubernetes/test_kubernetes_secrets.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 33887c7e..f37080de 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -639,8 +639,11 @@ def _handle_secrets_after_preparing_pod( body=k8s_client_lib.V1Secret(string_data=secret_name_to_secret_value), ) - # TODO: properly select the main container - main_container = pod.spec.containers[0] + main_container = next( + container + for container in pod.spec.containers + if container.name == _MAIN_CONTAINER_NAME + ) for env_var_name, secret_name in secret_env_name_to_secret_name.items(): main_container.env = main_container.env or [] main_container.env.append( diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 1028a4a7..7d7d9ba1 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -219,3 +219,21 @@ def test_secret_carries_the_plain_value_in_string_data( [(_, secret)] = secrets_api.created assert secret.string_data == {"hf-token": _SECRET_VALUE} assert secret.data is None + + +@pytest.mark.usefixtures("secrets_api") +def test_environment_entry_goes_to_the_main_container_not_the_first() -> None: + sidecar = k8s_client_lib.V1Container(name="sidecar") + pod = _make_pod(args=[]) + pod.spec.containers.insert(0, sidecar) + + _make_launcher()._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value={"hf-token": _SECRET_VALUE}, + secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, + annotations=None, + ) + + [_, main] = pod.spec.containers + assert sidecar.env is None + assert [env.name for env in main.env] == ["__TANGLE_SECRET_TOKEN"] From 4356b33327b239dec6053acd81d74dd13938a5c7 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 12:24:54 -0800 Subject: [PATCH 08/18] refactor: Kubernetes launcher - Secrets - Move the secret phases to the shared launcher base --- .../launchers/kubernetes_launchers.py | 212 +++++++++--------- 1 file changed, 106 insertions(+), 106 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index f37080de..83bee068 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -241,6 +241,112 @@ def __init__( # Fall back from using informer cache to using API calls. self._pod_informer = None + def _handle_secrets_before_preparing_pod( + self, + *, + input_arguments: dict[str, interfaces.InputArgument], + ) -> tuple[dict[str, str], dict[str, str]]: + # TODO!: Handle secrets consumed as files by mounting the required Secret keys as volumes. + secret_name_to_secret_value: dict[str, str] = {} + secret_env_name_to_secret_name: dict[str, str] = {} + for input_name, input_argument in list(input_arguments.items()): + if not input_argument.dynamic_data: + continue + dynamic_data_kind, dynamic_data_properties = ( + container_component_utils.parse_dynamic_data_argument( + input_argument.dynamic_data + ) + ) + # Hiding secrets + _DYNAMIC_DATA_SECRET_KEY = "secret" + _DYNAMIC_DATA_SECRET_NAME_KEY = "name" + if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: + secret_name = (dynamic_data_properties or {}).get( + _DYNAMIC_DATA_SECRET_NAME_KEY + ) + if not input_argument.value: + raise interfaces.LauncherError( + f"Orchestrator should have filled secret value for {input_name=}, but got: {input_argument=}" + ) + if not secret_name: + raise interfaces.LauncherError( + f"Secret does not have name: {input_argument.dynamic_data=}" + ) + + env_var_name = "__TANGLE_SECRET_" + re.sub( + "[^_0-9a-zA-Z]+", "_", input_name.upper() + ) + if env_var_name in secret_env_name_to_secret_name: + raise interfaces.LauncherError( + f"Two secret inputs map to the environment variable {env_var_name}; rename {input_name!r} or the other one" + ) + secret_name_to_secret_value[secret_name] = input_argument.value + secret_env_name_to_secret_name[env_var_name] = secret_name + # Hiding the secret value by replacing it with env variable sourced from a Kubernetes Secret + input_argument.value = f"$({env_var_name})" + continue + return secret_name_to_secret_value, secret_env_name_to_secret_name + + def _handle_secrets_after_preparing_pod( + self, + *, + pod: k8s_client_lib.V1Pod, + secret_name_to_secret_value: dict[str, str], + secret_env_name_to_secret_name: dict[str, str], + annotations: dict[str, Any] | None = None, + ): + # Create secrets + if not secret_name_to_secret_value or not secret_env_name_to_secret_name: + return + + assert pod.metadata + # TODO: Stop supporting changing namespace in _pod_postprocessor + namespace = pod.metadata.namespace or self._namespace + + created_by = (annotations or {}).get( + common_annotations.PIPELINE_RUN_CREATED_BY_ANNOTATION_KEY + ) or "anonymous" + # Kubernetes' Secrets can contain multiple key-value pairs. + # So we store all user's secret in a single Kubernetes Secret object + kubernetes_secret_name = naming_utils.sanitize_kubernetes_resource_name( + created_by.replace("@", "-at-") + ) + core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) + try: + core_api_client.create_namespaced_secret( + namespace=namespace, + body=k8s_client_lib.V1Secret( + metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), + string_data=secret_name_to_secret_value, + ), + ) + except Exception: + # TODO: Verify that such patching does not delete other secret key-value pairs. + core_api_client.patch_namespaced_secret( + name=kubernetes_secret_name, + namespace=namespace, + body=k8s_client_lib.V1Secret(string_data=secret_name_to_secret_value), + ) + + main_container = next( + container + for container in pod.spec.containers + if container.name == _MAIN_CONTAINER_NAME + ) + for env_var_name, secret_name in secret_env_name_to_secret_name.items(): + main_container.env = main_container.env or [] + main_container.env.append( + k8s_client_lib.V1EnvVar( + name=env_var_name, + value_from=k8s_client_lib.V1EnvVarSource( + secret_key_ref=k8s_client_lib.V1SecretKeySelector( + name=kubernetes_secret_name, + key=secret_name, + ) + ), + ) + ) + def _prepare_kubernetes_pod( self, *, @@ -552,112 +658,6 @@ def __init__( _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) - def _handle_secrets_before_preparing_pod( - self, - *, - input_arguments: dict[str, interfaces.InputArgument], - ) -> tuple[dict[str, str], dict[str, str]]: - # TODO!: Handle secrets consumed as files by mounting the required Secret keys as volumes. - secret_name_to_secret_value: dict[str, str] = {} - secret_env_name_to_secret_name: dict[str, str] = {} - for input_name, input_argument in list(input_arguments.items()): - if not input_argument.dynamic_data: - continue - dynamic_data_kind, dynamic_data_properties = ( - container_component_utils.parse_dynamic_data_argument( - input_argument.dynamic_data - ) - ) - # Hiding secrets - _DYNAMIC_DATA_SECRET_KEY = "secret" - _DYNAMIC_DATA_SECRET_NAME_KEY = "name" - if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: - secret_name = (dynamic_data_properties or {}).get( - _DYNAMIC_DATA_SECRET_NAME_KEY - ) - if not input_argument.value: - raise interfaces.LauncherError( - f"Orchestrator should have filled secret value for {input_name=}, but got: {input_argument=}" - ) - if not secret_name: - raise interfaces.LauncherError( - f"Secret does not have name: {input_argument.dynamic_data=}" - ) - - env_var_name = "__TANGLE_SECRET_" + re.sub( - "[^_0-9a-zA-Z]+", "_", input_name.upper() - ) - if env_var_name in secret_env_name_to_secret_name: - raise interfaces.LauncherError( - f"Two secret inputs map to the environment variable {env_var_name}; rename {input_name!r} or the other one" - ) - secret_name_to_secret_value[secret_name] = input_argument.value - secret_env_name_to_secret_name[env_var_name] = secret_name - # Hiding the secret value by replacing it with env variable sourced from a Kubernetes Secret - input_argument.value = f"$({env_var_name})" - continue - return secret_name_to_secret_value, secret_env_name_to_secret_name - - def _handle_secrets_after_preparing_pod( - self, - *, - pod: k8s_client_lib.V1Pod, - secret_name_to_secret_value: dict[str, str], - secret_env_name_to_secret_name: dict[str, str], - annotations: dict[str, Any] | None = None, - ): - # Create secrets - if not secret_name_to_secret_value or not secret_env_name_to_secret_name: - return - - assert pod.metadata - # TODO: Stop supporting changing namespace in _pod_postprocessor - namespace = pod.metadata.namespace or self._namespace - - created_by = (annotations or {}).get( - common_annotations.PIPELINE_RUN_CREATED_BY_ANNOTATION_KEY - ) or "anonymous" - # Kubernetes' Secrets can contain multiple key-value pairs. - # So we store all user's secret in a single Kubernetes Secret object - kubernetes_secret_name = naming_utils.sanitize_kubernetes_resource_name( - created_by.replace("@", "-at-") - ) - core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) - try: - core_api_client.create_namespaced_secret( - namespace=namespace, - body=k8s_client_lib.V1Secret( - metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), - string_data=secret_name_to_secret_value, - ), - ) - except Exception: - # TODO: Verify that such patching does not delete other secret key-value pairs. - core_api_client.patch_namespaced_secret( - name=kubernetes_secret_name, - namespace=namespace, - body=k8s_client_lib.V1Secret(string_data=secret_name_to_secret_value), - ) - - main_container = next( - container - for container in pod.spec.containers - if container.name == _MAIN_CONTAINER_NAME - ) - for env_var_name, secret_name in secret_env_name_to_secret_name.items(): - main_container.env = main_container.env or [] - main_container.env.append( - k8s_client_lib.V1EnvVar( - name=env_var_name, - value_from=k8s_client_lib.V1EnvVarSource( - secret_key_ref=k8s_client_lib.V1SecretKeySelector( - name=kubernetes_secret_name, - key=secret_name, - ) - ), - ) - ) - def launch_container_task( self, *, From 98779350a759ee79dc6d3078cdb44c4d169ca435 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 12:33:12 -0800 Subject: [PATCH 09/18] feat: Kubernetes launcher - Secrets - Deliver secrets through a Kubernetes Secret on the Job launcher --- .../launchers/kubernetes_launchers.py | 10 +++ .../kubernetes/test_kubernetes_secrets.py | 89 ++++++++++++++++++- 2 files changed, 98 insertions(+), 1 deletion(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 83bee068..a5fa5b40 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -1360,6 +1360,9 @@ def launch_container_task( _MULTI_NODE_NODE_0_ADDRESS_DYNAMIC_DATA_KEY: node_0_address, _MULTI_NODE_ALL_NODE_ADDRESSES_DYNAMIC_DATA_KEY: all_node_addresses_str, } + secret_name_to_secret_value, secret_env_name_to_secret_name = ( + self._handle_secrets_before_preparing_pod(input_arguments=input_arguments) + ) for input_name, input_argument in list(input_arguments.items()): if input_argument.value is not None or input_argument.uri is not None: @@ -1402,6 +1405,13 @@ def launch_container_task( # we should prohibit/ignore changing pod namespace in the pod post-processor. namespace = pod.metadata.namespace + self._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=secret_name_to_secret_value, + secret_env_name_to_secret_name=secret_env_name_to_secret_name, + annotations=annotations, + ) + service: k8s_client_lib.V1Service | None = None if enable_multi_node: main_container_spec = pod.spec.containers[0] diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 7d7d9ba1..91ad1e92 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -2,11 +2,13 @@ from __future__ import annotations +import pathlib from typing import Final import pytest from kubernetes import client as k8s_client_lib +from cloud_pipelines_backend import component_structures as structures from cloud_pipelines_backend.launchers import ( common_annotations, interfaces, @@ -40,11 +42,44 @@ def patch_namespaced_secret( del namespace self.patched.append((name, body)) + def create_namespaced_service( + self, *, namespace: str, body: k8s_client_lib.V1Service, _request_timeout: int + ) -> k8s_client_lib.V1Service: + return body + + +class _JobsApi: + """Stands in for the Kubernetes batch API and records the Jobs the launcher creates.""" + + def __init__(self) -> None: + self.created: list[k8s_client_lib.V1Job] = [] + + def create_namespaced_job( + self, *, namespace: str, body: k8s_client_lib.V1Job, _request_timeout: int + ) -> k8s_client_lib.V1Job: + body.metadata.namespace = namespace + body.metadata.uid = "job-uid" + self.created.append(body) + return body + @pytest.fixture def secrets_api(monkeypatch: pytest.MonkeyPatch) -> _SecretsApi: api = _SecretsApi() - monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda *, api_client: api) + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda api_client: api) + return api + + +@pytest.fixture +def jobs_api(monkeypatch: pytest.MonkeyPatch) -> _JobsApi: + api = _JobsApi() + monkeypatch.setattr(k8s_client_lib, "BatchV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib.VersionApi, "get_code", lambda self, **_: None) + + def _no_informer(**kwargs: object) -> None: + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) return api @@ -237,3 +272,55 @@ def test_environment_entry_goes_to_the_main_container_not_the_first() -> None: [_, main] = pod.spec.containers assert sidecar.env is None assert [env.name for env in main.env] == ["__TANGLE_SECRET_TOKEN"] + + +def _launch_job(*, annotations: dict[str, str], log_uri: str) -> None: + """Launches a Job whose only argument is the secret input.""" + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher( + api_client=k8s_client_lib.ApiClient() + ) + launcher.launch_container_task( + component_spec=structures.ComponentSpec( + name="test", + inputs=[structures.InputSpec(name="token")], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec( + image="alpine", + args=[structures.InputValuePlaceholder(input_name="token")], + ) + ), + ), + input_arguments={"token": _make_secret_argument()}, + output_uris={}, + log_uri=log_uri, + annotations={ + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: "123", + **annotations, + }, + ) + + +@pytest.mark.parametrize( + "annotations", + [ + pytest.param({}, id="single node"), + pytest.param( + {kubernetes_launchers.MULTI_NODE_NUMBER_OF_NODES_ANNOTATION_KEY: "2"}, + id="two nodes", + ), + ], +) +def test_job_container_reads_the_value_from_the_secret( + annotations: dict[str, str], + secrets_api: _SecretsApi, + jobs_api: _JobsApi, + tmp_path: pathlib.Path, +) -> None: + _launch_job(annotations=annotations, log_uri=str(tmp_path / "log.txt")) + + [job] = jobs_api.created + [main] = job.spec.template.spec.containers + assert "__TANGLE_SECRET_TOKEN" in [env.name for env in main.env] + assert main.args == ["$(__TANGLE_SECRET_TOKEN)"] + [(_, secret)] = secrets_api.created + assert secret.string_data == {"hf-token": _SECRET_VALUE} From d2790e14ed4415c3487923e3f2edf1dcb96f1cae Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 15:19:52 -0800 Subject: [PATCH 10/18] feat: Kubernetes launcher - Secrets - One Secret per container execution, owned by its Pod or Job --- .../launchers/kubernetes_launchers.py | 102 ++++++---- .../kubernetes/test_kubernetes_secrets.py | 183 ++++++++++++------ 2 files changed, 192 insertions(+), 93 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index a5fa5b40..0f3d9ffc 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -12,6 +12,7 @@ import threading import time import typing +import uuid from typing import Any, Final, Optional import kubernetes.client.exceptions @@ -294,39 +295,15 @@ def _handle_secrets_after_preparing_pod( secret_name_to_secret_value: dict[str, str], secret_env_name_to_secret_name: dict[str, str], annotations: dict[str, Any] | None = None, - ): - # Create secrets + ) -> k8s_client_lib.V1Secret | None: if not secret_name_to_secret_value or not secret_env_name_to_secret_name: - return + return None - assert pod.metadata - # TODO: Stop supporting changing namespace in _pod_postprocessor - namespace = pod.metadata.namespace or self._namespace - - created_by = (annotations or {}).get( - common_annotations.PIPELINE_RUN_CREATED_BY_ANNOTATION_KEY - ) or "anonymous" - # Kubernetes' Secrets can contain multiple key-value pairs. - # So we store all user's secret in a single Kubernetes Secret object - kubernetes_secret_name = naming_utils.sanitize_kubernetes_resource_name( - created_by.replace("@", "-at-") - ) - core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) - try: - core_api_client.create_namespaced_secret( - namespace=namespace, - body=k8s_client_lib.V1Secret( - metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), - string_data=secret_name_to_secret_value, - ), - ) - except Exception: - # TODO: Verify that such patching does not delete other secret key-value pairs. - core_api_client.patch_namespaced_secret( - name=kubernetes_secret_name, - namespace=namespace, - body=k8s_client_lib.V1Secret(string_data=secret_name_to_secret_value), - ) + # One Secret per container execution, created once its Pod or Job exists. + container_execution_id = (annotations or {}).get( + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + ) or uuid.uuid4().hex[:8] + kubernetes_secret_name = "tangle-ce-" + container_execution_id main_container = next( container @@ -346,6 +323,37 @@ def _handle_secrets_after_preparing_pod( ), ) ) + return k8s_client_lib.V1Secret( + metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), + string_data=secret_name_to_secret_value, + ) + + def _create_secret_or_delete_pod_or_job( + self, + *, + secret: k8s_client_lib.V1Secret, + pod_or_job: k8s_client_lib.V1OwnerReference, + namespace: str, + delete_pod_or_job: typing.Callable[[], Any], + ) -> None: + """Creates the Secret owned by the Pod or Job, deleting the Pod or Job if that fails.""" + secret.metadata.owner_references = [pod_or_job] + try: + k8s_client_lib.CoreV1Api( + api_client=self._api_client + ).create_namespaced_secret( + namespace=namespace, body=secret, _request_timeout=self._request_timeout + ) + except Exception as ex: + try: + delete_pod_or_job() + except Exception: + _logger.exception( + f"Failed to roll back {pod_or_job.kind} {pod_or_job.name} after Secret creation failed." + ) + raise interfaces.LauncherError( + f"Failed to create Kubernetes Secret {secret.metadata.name}" + ) from ex def _prepare_kubernetes_pod( self, @@ -724,7 +732,7 @@ def launch_container_task( if self._pod_postprocessor: pod = self._pod_postprocessor(pod=pod, annotations=annotations) - self._handle_secrets_after_preparing_pod( + secret = self._handle_secrets_after_preparing_pod( pod=pod, secret_name_to_secret_value=secret_name_to_secret_value, secret_env_name_to_secret_name=secret_env_name_to_secret_name, @@ -746,6 +754,20 @@ def launch_container_task( pod_name: str = created_pod.metadata.name pod_namespace: str = created_pod.metadata.namespace _logger.info(f"Created pod {pod_name} in namespace {pod_namespace}") + if secret: + self._create_secret_or_delete_pod_or_job( + secret=secret, + pod_or_job=k8s_client_lib.V1OwnerReference( + api_version="v1", + kind="Pod", + name=pod_name, + uid=created_pod.metadata.uid, + ), + namespace=pod_namespace, + delete_pod_or_job=lambda: core_api_client.delete_namespaced_pod( + name=pod_name, namespace=pod_namespace, grace_period_seconds=0 + ), + ) launched_kubernetes_container = LaunchedKubernetesContainer( pod_name=pod_name, @@ -1405,7 +1427,7 @@ def launch_container_task( # we should prohibit/ignore changing pod namespace in the pod post-processor. namespace = pod.metadata.namespace - self._handle_secrets_after_preparing_pod( + secret = self._handle_secrets_after_preparing_pod( pod=pod, secret_name_to_secret_value=secret_name_to_secret_value, secret_env_name_to_secret_name=secret_env_name_to_secret_name, @@ -1522,6 +1544,20 @@ def launch_container_task( job_namespace: str = created_job.metadata.namespace job_uid: str = created_job.metadata.uid _logger.info(f"Created Kubernetes Job {job_name} in namespace {job_namespace}") + if secret: + self._create_secret_or_delete_pod_or_job( + secret=secret, + pod_or_job=k8s_client_lib.V1OwnerReference( + api_version="batch/v1", kind="Job", name=job_name, uid=job_uid + ), + namespace=job_namespace, + delete_pod_or_job=lambda: batch_api_client.delete_namespaced_job( + name=job_name, + namespace=job_namespace, + grace_period_seconds=0, + propagation_policy="Background", + ), + ) if enable_multi_node: assert service diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 91ad1e92..5e3ef4d2 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -16,31 +16,40 @@ ) _SECRET_VALUE: Final[str] = "FAKE-tok-123" -_CREATED_BY_ALICE: Final[dict[str, str]] = { - common_annotations.PIPELINE_RUN_CREATED_BY_ANNOTATION_KEY: "alice@example.com" +_EXECUTION_123: Final[dict[str, str]] = { + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: "123" } class _SecretsApi: - """Stands in for the Kubernetes API and records the Secrets the launcher writes.""" + """Stands in for the Kubernetes API and records the Secrets and Pods the launcher writes.""" def __init__(self) -> None: self.created: list[tuple[str, k8s_client_lib.V1Secret]] = [] - self.patched: list[tuple[str, k8s_client_lib.V1Secret]] = [] - self.existing: set[str] = set() + self.deleted_pods: list[str] = [] + self.fail_secrets = False def create_namespaced_secret( - self, *, namespace: str, body: k8s_client_lib.V1Secret + self, *, namespace: str, body: k8s_client_lib.V1Secret, _request_timeout: int ) -> None: - if body.metadata.name in self.existing: - raise k8s_client_lib.ApiException(status=409, reason="Conflict") + if self.fail_secrets: + raise k8s_client_lib.ApiException( + status=500, reason="Internal Server Error" + ) self.created.append((namespace, body)) - def patch_namespaced_secret( - self, *, name: str, namespace: str, body: k8s_client_lib.V1Secret + def create_namespaced_pod( + self, *, namespace: str, body: k8s_client_lib.V1Pod, _request_timeout: int + ) -> k8s_client_lib.V1Pod: + body.metadata.name = "task-pod" + body.metadata.namespace = namespace + body.metadata.uid = "pod-uid" + return body + + def delete_namespaced_pod( + self, *, name: str, namespace: str, grace_period_seconds: int ) -> None: - del namespace - self.patched.append((name, body)) + self.deleted_pods.append(name) def create_namespaced_service( self, *, namespace: str, body: k8s_client_lib.V1Service, _request_timeout: int @@ -53,6 +62,7 @@ class _JobsApi: def __init__(self) -> None: self.created: list[k8s_client_lib.V1Job] = [] + self.deleted: list[str] = [] def create_namespaced_job( self, *, namespace: str, body: k8s_client_lib.V1Job, _request_timeout: int @@ -62,11 +72,27 @@ def create_namespaced_job( self.created.append(body) return body + def delete_namespaced_job( + self, + *, + name: str, + namespace: str, + grace_period_seconds: int, + propagation_policy: str, + ) -> None: + self.deleted.append(name) + @pytest.fixture def secrets_api(monkeypatch: pytest.MonkeyPatch) -> _SecretsApi: api = _SecretsApi() monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib.VersionApi, "get_code", lambda self, **_: None) + + def _no_informer(**kwargs: object) -> None: + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) return api @@ -74,12 +100,6 @@ def secrets_api(monkeypatch: pytest.MonkeyPatch) -> _SecretsApi: def jobs_api(monkeypatch: pytest.MonkeyPatch) -> _JobsApi: api = _JobsApi() monkeypatch.setattr(k8s_client_lib, "BatchV1Api", lambda api_client: api) - monkeypatch.setattr(k8s_client_lib.VersionApi, "get_code", lambda self, **_: None) - - def _no_informer(**kwargs: object) -> None: - raise RuntimeError("Informers are disabled in tests.") - - monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) return api @@ -179,45 +199,16 @@ def test_incomplete_secret_argument_is_a_launcher_error( ) -def test_secret_is_created_in_the_pod_namespace_and_named_after_the_run_creator( - secrets_api: _SecretsApi, -) -> None: - _launch(launcher=_make_launcher(), annotations=_CREATED_BY_ALICE) - - [(namespace, secret)] = secrets_api.created - assert namespace == "tangle-jobs" - assert secret.metadata.name == "alice-at-example-com" - - -def test_run_without_a_creator_uses_the_anonymous_secret( - secrets_api: _SecretsApi, -) -> None: - _launch(launcher=_make_launcher()) - - [(_, secret)] = secrets_api.created - assert secret.metadata.name == "anonymous" - - -def test_existing_secret_is_patched(secrets_api: _SecretsApi) -> None: - secrets_api.existing.add("alice-at-example-com") - - _launch(launcher=_make_launcher(), annotations=_CREATED_BY_ALICE) - - assert secrets_api.created == [] - [(name, _)] = secrets_api.patched - assert name == "alice-at-example-com" - - @pytest.mark.usefixtures("secrets_api") def test_container_reads_the_value_from_the_secret() -> None: - pod = _launch(launcher=_make_launcher(), annotations=_CREATED_BY_ALICE) + pod = _launch(launcher=_make_launcher(), annotations=_EXECUTION_123) assert pod.spec.containers[0].env == [ k8s_client_lib.V1EnvVar( name="__TANGLE_SECRET_TOKEN", value_from=k8s_client_lib.V1EnvVarSource( secret_key_ref=k8s_client_lib.V1SecretKeySelector( - name="alice-at-example-com", key="hf-token" + name="tangle-ce-123", key="hf-token" ) ), ) @@ -245,13 +236,14 @@ def test_placeholder_names_the_environment_variable_the_pod_defines() -> None: assert _SECRET_VALUE not in container.args[0] -def test_secret_carries_the_plain_value_in_string_data( - secrets_api: _SecretsApi, -) -> None: +def test_secret_carries_the_plain_value_in_string_data() -> None: # `data` must be base64, so the API server would decode the plain value into other bytes. - _launch(launcher=_make_launcher()) + secret = _make_launcher()._handle_secrets_after_preparing_pod( + pod=_make_pod(args=[]), + secret_name_to_secret_value={"hf-token": _SECRET_VALUE}, + secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, + ) - [(_, secret)] = secrets_api.created assert secret.string_data == {"hf-token": _SECRET_VALUE} assert secret.data is None @@ -274,11 +266,38 @@ def test_environment_entry_goes_to_the_main_container_not_the_first() -> None: assert [env.name for env in main.env] == ["__TANGLE_SECRET_TOKEN"] -def _launch_job(*, annotations: dict[str, str], log_uri: str) -> None: - """Launches a Job whose only argument is the secret input.""" - launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher( - api_client=k8s_client_lib.ApiClient() - ) +_JOB_LAUNCHER: Final = ( + kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher +) +_POD_OR_JOB: Final = pytest.mark.parametrize( + "launcher_class, pod_or_job", + [ + pytest.param( + kubernetes_launchers.KubernetesWithHostPathContainerLauncher, + k8s_client_lib.V1OwnerReference( + api_version="v1", kind="Pod", name="task-pod", uid="pod-uid" + ), + id="pod", + ), + pytest.param( + _JOB_LAUNCHER, + k8s_client_lib.V1OwnerReference( + api_version="batch/v1", kind="Job", name="tangle-ce-123", uid="job-uid" + ), + id="job", + ), + ], +) + + +def _launch_task( + *, + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + annotations: dict[str, str], + log_uri: str, +) -> None: + """Launches a task whose only argument is the secret input.""" + launcher = launcher_class(api_client=k8s_client_lib.ApiClient()) launcher.launch_container_task( component_spec=structures.ComponentSpec( name="test", @@ -316,7 +335,11 @@ def test_job_container_reads_the_value_from_the_secret( jobs_api: _JobsApi, tmp_path: pathlib.Path, ) -> None: - _launch_job(annotations=annotations, log_uri=str(tmp_path / "log.txt")) + _launch_task( + launcher_class=_JOB_LAUNCHER, + annotations=annotations, + log_uri=str(tmp_path / "log.txt"), + ) [job] = jobs_api.created [main] = job.spec.template.spec.containers @@ -324,3 +347,43 @@ def test_job_container_reads_the_value_from_the_secret( assert main.args == ["$(__TANGLE_SECRET_TOKEN)"] [(_, secret)] = secrets_api.created assert secret.string_data == {"hf-token": _SECRET_VALUE} + + +@_POD_OR_JOB +@pytest.mark.usefixtures("jobs_api") +def test_secret_is_named_after_the_execution_and_owned_by_its_pod_or_job( + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + pod_or_job: k8s_client_lib.V1OwnerReference, + secrets_api: _SecretsApi, + tmp_path: pathlib.Path, +) -> None: + _launch_task( + launcher_class=launcher_class, + annotations=_EXECUTION_123, + log_uri=str(tmp_path / "log.txt"), + ) + + [(namespace, secret)] = secrets_api.created + assert namespace == "default" + assert secret.metadata.name == "tangle-ce-123" + assert secret.metadata.owner_references == [pod_or_job] + + +@_POD_OR_JOB +def test_failed_secret_deletes_its_pod_or_job( + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + pod_or_job: k8s_client_lib.V1OwnerReference, + secrets_api: _SecretsApi, + jobs_api: _JobsApi, + tmp_path: pathlib.Path, +) -> None: + secrets_api.fail_secrets = True + + with pytest.raises(interfaces.LauncherError): + _launch_task( + launcher_class=launcher_class, + annotations=_EXECUTION_123, + log_uri=str(tmp_path / "log.txt"), + ) + + assert secrets_api.deleted_pods + jobs_api.deleted == [pod_or_job.name] From 49cf46399d3582e3dc6a82e13f4d6fddce788414 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 13 Nov 2026 01:25:43 -0800 Subject: [PATCH 11/18] fix: Kubernetes launcher - Secrets - Key the Secret by env var name Kubernetes refuses Secret keys outside [-._a-zA-Z0-9], which Tangle secret names may use (e.g. "hf token"). The env var name is derived from the input name, always valid, and unique within the execution's own Secret, so the Secret and its secretKeyRef use it. Done here rather than earlier in the stack: with one Secret per user, keying by env var name would let two runs that share an input name overwrite each other's value. --- .../launchers/kubernetes_launchers.py | 12 ++++++--- .../kubernetes/test_kubernetes_secrets.py | 27 ++++++++++++++++--- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 0f3d9ffc..1a39631a 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -304,13 +304,19 @@ def _handle_secrets_after_preparing_pod( common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY ) or uuid.uuid4().hex[:8] kubernetes_secret_name = "tangle-ce-" + container_execution_id + # Keyed by env var name, unique within this execution's Secret: Kubernetes + # refuses keys outside [-._a-zA-Z0-9], which secret names may use. + secret_data = { + env_var_name: secret_name_to_secret_value[secret_name] + for env_var_name, secret_name in secret_env_name_to_secret_name.items() + } main_container = next( container for container in pod.spec.containers if container.name == _MAIN_CONTAINER_NAME ) - for env_var_name, secret_name in secret_env_name_to_secret_name.items(): + for env_var_name in secret_env_name_to_secret_name: main_container.env = main_container.env or [] main_container.env.append( k8s_client_lib.V1EnvVar( @@ -318,14 +324,14 @@ def _handle_secrets_after_preparing_pod( value_from=k8s_client_lib.V1EnvVarSource( secret_key_ref=k8s_client_lib.V1SecretKeySelector( name=kubernetes_secret_name, - key=secret_name, + key=env_var_name, ) ), ) ) return k8s_client_lib.V1Secret( metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), - string_data=secret_name_to_secret_value, + string_data=secret_data, ) def _create_secret_or_delete_pod_or_job( diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 5e3ef4d2..229336c8 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -208,7 +208,7 @@ def test_container_reads_the_value_from_the_secret() -> None: name="__TANGLE_SECRET_TOKEN", value_from=k8s_client_lib.V1EnvVarSource( secret_key_ref=k8s_client_lib.V1SecretKeySelector( - name="tangle-ce-123", key="hf-token" + name="tangle-ce-123", key="__TANGLE_SECRET_TOKEN" ) ), ) @@ -244,7 +244,7 @@ def test_secret_carries_the_plain_value_in_string_data() -> None: secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, ) - assert secret.string_data == {"hf-token": _SECRET_VALUE} + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} assert secret.data is None @@ -346,7 +346,7 @@ def test_job_container_reads_the_value_from_the_secret( assert "__TANGLE_SECRET_TOKEN" in [env.name for env in main.env] assert main.args == ["$(__TANGLE_SECRET_TOKEN)"] [(_, secret)] = secrets_api.created - assert secret.string_data == {"hf-token": _SECRET_VALUE} + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} @_POD_OR_JOB @@ -387,3 +387,24 @@ def test_failed_secret_deletes_its_pod_or_job( ) assert secrets_api.deleted_pods + jobs_api.deleted == [pod_or_job.name] + + +def test_secret_is_keyed_by_the_env_var_name_so_any_secret_name_works() -> None: + # Kubernetes refuses "hf token" as a Secret key; the input-derived name is always valid. + argument = _make_secret_argument() + argument.dynamic_data = {"secret": {"name": "hf token"}} + launcher = _make_launcher() + values, env_names = launcher._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + pod = _make_pod(args=[argument.value]) + + secret = launcher._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=values, + secret_env_name_to_secret_name=env_names, + ) + + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} + [env] = pod.spec.containers[0].env + assert env.value_from.secret_key_ref.key == "__TANGLE_SECRET_TOKEN" From 1b3122cbe5e9bd8c15d4e3154054623baa73fd6a Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 13 Nov 2026 01:53:35 -0800 Subject: [PATCH 12/18] refactor: Kubernetes launcher - Secrets - Rename pod_or_job to owner_reference --- .../launchers/kubernetes_launchers.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 1a39631a..96458db0 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -338,12 +338,12 @@ def _create_secret_or_delete_pod_or_job( self, *, secret: k8s_client_lib.V1Secret, - pod_or_job: k8s_client_lib.V1OwnerReference, + owner_reference: k8s_client_lib.V1OwnerReference, namespace: str, delete_pod_or_job: typing.Callable[[], Any], ) -> None: """Creates the Secret owned by the Pod or Job, deleting the Pod or Job if that fails.""" - secret.metadata.owner_references = [pod_or_job] + secret.metadata.owner_references = [owner_reference] try: k8s_client_lib.CoreV1Api( api_client=self._api_client @@ -355,7 +355,7 @@ def _create_secret_or_delete_pod_or_job( delete_pod_or_job() except Exception: _logger.exception( - f"Failed to roll back {pod_or_job.kind} {pod_or_job.name} after Secret creation failed." + f"Failed to roll back {owner_reference.kind} {owner_reference.name} after Secret creation failed." ) raise interfaces.LauncherError( f"Failed to create Kubernetes Secret {secret.metadata.name}" @@ -763,7 +763,7 @@ def launch_container_task( if secret: self._create_secret_or_delete_pod_or_job( secret=secret, - pod_or_job=k8s_client_lib.V1OwnerReference( + owner_reference=k8s_client_lib.V1OwnerReference( api_version="v1", kind="Pod", name=pod_name, @@ -1553,7 +1553,7 @@ def launch_container_task( if secret: self._create_secret_or_delete_pod_or_job( secret=secret, - pod_or_job=k8s_client_lib.V1OwnerReference( + owner_reference=k8s_client_lib.V1OwnerReference( api_version="batch/v1", kind="Job", name=job_name, uid=job_uid ), namespace=job_namespace, From 78f9f11fd287f1c2177b499767c6ab9234ecae96 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 15:29:06 -0800 Subject: [PATCH 13/18] feat: Kubernetes launcher - Secrets - Mount secrets consumed as files from the Kubernetes Secret --- .../launchers/kubernetes_launchers.py | 62 ++++++++++++++----- .../kubernetes/test_kubernetes_secrets.py | 62 ++++++++++++++++++- 2 files changed, 106 insertions(+), 18 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 96458db0..aee390b5 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -85,6 +85,8 @@ # dynamicData argument keys _MULTI_NODE_NUMBER_OF_NODES_DYNAMIC_DATA_KEY = "system/multi_node/number_of_nodes" _MULTI_NODE_NODE_INDEX_DYNAMIC_DATA_KEY = "system/multi_node/node_index" +_DYNAMIC_DATA_SECRET_KEY = "secret" +_DYNAMIC_DATA_SECRET_NAME_KEY = "name" _MULTI_NODE_NODE_0_ADDRESS_DYNAMIC_DATA_KEY = "system/multi_node/node_0_address" _MULTI_NODE_ALL_NODE_ADDRESSES_DYNAMIC_DATA_KEY = "system/multi_node/all_node_addresses" @@ -97,6 +99,11 @@ _CONTAINER_FILE_NAME = "data" +def _secret_env_var_name(*, input_name: str) -> str: + """Returns the environment variable, also the Secret key, that carries a secret input.""" + return "__TANGLE_SECRET_" + re.sub("[^_0-9a-zA-Z]+", "_", input_name.upper()) + + def _create_volume_and_volume_mount_host_path( container_path: str, artifact_uri: str, @@ -247,7 +254,6 @@ def _handle_secrets_before_preparing_pod( *, input_arguments: dict[str, interfaces.InputArgument], ) -> tuple[dict[str, str], dict[str, str]]: - # TODO!: Handle secrets consumed as files by mounting the required Secret keys as volumes. secret_name_to_secret_value: dict[str, str] = {} secret_env_name_to_secret_name: dict[str, str] = {} for input_name, input_argument in list(input_arguments.items()): @@ -259,8 +265,6 @@ def _handle_secrets_before_preparing_pod( ) ) # Hiding secrets - _DYNAMIC_DATA_SECRET_KEY = "secret" - _DYNAMIC_DATA_SECRET_NAME_KEY = "name" if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: secret_name = (dynamic_data_properties or {}).get( _DYNAMIC_DATA_SECRET_NAME_KEY @@ -274,9 +278,7 @@ def _handle_secrets_before_preparing_pod( f"Secret does not have name: {input_argument.dynamic_data=}" ) - env_var_name = "__TANGLE_SECRET_" + re.sub( - "[^_0-9a-zA-Z]+", "_", input_name.upper() - ) + env_var_name = _secret_env_var_name(input_name=input_name) if env_var_name in secret_env_name_to_secret_name: raise interfaces.LauncherError( f"Two secret inputs map to the environment variable {env_var_name}; rename {input_name!r} or the other one" @@ -316,7 +318,11 @@ def _handle_secrets_after_preparing_pod( for container in pod.spec.containers if container.name == _MAIN_CONTAINER_NAME ) + command_line = (main_container.command or []) + (main_container.args or []) for env_var_name in secret_env_name_to_secret_name: + # An input consumed as a file reads the Secret volume, not the environment. + if not any(f"$({env_var_name})" in part for part in command_line): + continue main_container.env = main_container.env or [] main_container.env.append( k8s_client_lib.V1EnvVar( @@ -329,6 +335,9 @@ def _handle_secrets_after_preparing_pod( ), ) ) + for volume in pod.spec.volumes or []: + if volume.secret and not volume.secret.secret_name: + volume.secret.secret_name = kubernetes_secret_name return k8s_client_lib.V1Secret( metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), string_data=secret_data, @@ -433,10 +442,18 @@ def get_input_value(input_name: str) -> str: def get_input_path(input_name: str) -> str: input_argument = input_arguments[input_name] uri = input_argument.uri + container_path = ( + container_inputs_root + / naming_utils.sanitize_file_name(input_name) + / _CONTAINER_FILE_NAME + ).as_posix() + volume_name = naming_utils.sanitize_kubernetes_resource_name( + "inputs-" + input_name + ) if not uri: # Prohibiting consuming some kinds of dynamic data as file if input_argument.dynamic_data: - dynamic_data_kind, _ = ( + dynamic_data_kind, dynamic_data_properties = ( container_component_utils.parse_dynamic_data_argument( input_argument.dynamic_data ) @@ -445,6 +462,29 @@ def get_input_path(input_name: str) -> str: raise interfaces.LauncherError( f"It's not possible to consume this dynamic data argument as a file: {input_name=}, {input_argument=}" ) + if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: + # The second secret phase fills in the Secret's name. + volume_map[volume_name] = k8s_client_lib.V1Volume( + name=volume_name, + secret=k8s_client_lib.V1SecretVolumeSource( + items=[ + k8s_client_lib.V1KeyToPath( + key=_secret_env_var_name(input_name=input_name), + path=_CONTAINER_FILE_NAME, + ) + ] + ), + ) + volume_mounts.append( + k8s_client_lib.V1VolumeMount( + name=volume_name, + mount_path=pathlib.PurePosixPath( + container_path + ).parent.as_posix(), + read_only=True, + ) + ) + return container_path if input_argument.value is None: if input_argument.dynamic_data: raise NotImplementedError( @@ -466,14 +506,6 @@ def get_input_path(input_name: str) -> str: # Updating the input_arguments with the URI of the uploaded value input_argument.uri = uri - container_path = ( - container_inputs_root - / naming_utils.sanitize_file_name(input_name) - / _CONTAINER_FILE_NAME - ).as_posix() - volume_name = naming_utils.sanitize_kubernetes_resource_name( - "inputs-" + input_name - ) volume, volume_mount = self._create_volume_and_volume_mount( container_path=container_path, artifact_uri=uri, diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py index 229336c8..3ed2d81b 100644 --- a/tests/launchers/kubernetes/test_kubernetes_secrets.py +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -26,6 +26,7 @@ class _SecretsApi: def __init__(self) -> None: self.created: list[tuple[str, k8s_client_lib.V1Secret]] = [] + self.created_pods: list[k8s_client_lib.V1Pod] = [] self.deleted_pods: list[str] = [] self.fail_secrets = False @@ -44,6 +45,7 @@ def create_namespaced_pod( body.metadata.name = "task-pod" body.metadata.namespace = namespace body.metadata.uid = "pod-uid" + self.created_pods.append(body) return body def delete_namespaced_pod( @@ -251,7 +253,7 @@ def test_secret_carries_the_plain_value_in_string_data() -> None: @pytest.mark.usefixtures("secrets_api") def test_environment_entry_goes_to_the_main_container_not_the_first() -> None: sidecar = k8s_client_lib.V1Container(name="sidecar") - pod = _make_pod(args=[]) + pod = _make_pod(args=["$(__TANGLE_SECRET_TOKEN)"]) pod.spec.containers.insert(0, sidecar) _make_launcher()._handle_secrets_after_preparing_pod( @@ -266,6 +268,20 @@ def test_environment_entry_goes_to_the_main_container_not_the_first() -> None: assert [env.name for env in main.env] == ["__TANGLE_SECRET_TOKEN"] +@pytest.mark.usefixtures("secrets_api") +def test_placeholder_in_the_command_gets_its_environment_entry() -> None: + pod = _make_pod(args=[]) + pod.spec.containers[0].command = ["$(__TANGLE_SECRET_TOKEN)"] + + _make_launcher()._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value={"hf-token": _SECRET_VALUE}, + secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, + ) + + assert [env.name for env in pod.spec.containers[0].env] == ["__TANGLE_SECRET_TOKEN"] + + _JOB_LAUNCHER: Final = ( kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher ) @@ -295,8 +311,12 @@ def _launch_task( launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], annotations: dict[str, str], log_uri: str, + as_file: bool = False, ) -> None: - """Launches a task whose only argument is the secret input.""" + """Launches a task whose only argument is the secret input, as a value or as a file.""" + placeholder = ( + structures.InputPathPlaceholder if as_file else structures.InputValuePlaceholder + ) launcher = launcher_class(api_client=k8s_client_lib.ApiClient()) launcher.launch_container_task( component_spec=structures.ComponentSpec( @@ -305,7 +325,7 @@ def _launch_task( implementation=structures.ContainerImplementation( container=structures.ContainerSpec( image="alpine", - args=[structures.InputValuePlaceholder(input_name="token")], + args=[placeholder(input_name="token")], ) ), ), @@ -408,3 +428,39 @@ def test_secret_is_keyed_by_the_env_var_name_so_any_secret_name_works() -> None: assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} [env] = pod.spec.containers[0].env assert env.value_from.secret_key_ref.key == "__TANGLE_SECRET_TOKEN" + + +@_POD_OR_JOB +def test_file_input_reads_the_secret_volume( + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + pod_or_job: k8s_client_lib.V1OwnerReference, + secrets_api: _SecretsApi, + jobs_api: _JobsApi, + tmp_path: pathlib.Path, +) -> None: + _launch_task( + launcher_class=launcher_class, + annotations={}, + log_uri=str(tmp_path / "log.txt"), + as_file=True, + ) + + [pod_spec] = [pod.spec for pod in secrets_api.created_pods] + [ + job.spec.template.spec for job in jobs_api.created + ] + [main] = pod_spec.containers + assert main.args == ["/tmp/inputs/token/data"] + assert "__TANGLE_SECRET_TOKEN" not in [env.name for env in main.env or []] + [volume] = [volume for volume in pod_spec.volumes if volume.secret] + assert volume.secret == k8s_client_lib.V1SecretVolumeSource( + secret_name="tangle-ce-123", + items=[k8s_client_lib.V1KeyToPath(key="__TANGLE_SECRET_TOKEN", path="data")], + ) + assert ( + k8s_client_lib.V1VolumeMount( + name=volume.name, mount_path="/tmp/inputs/token", read_only=True + ) + in main.volume_mounts + ) + [(_, secret)] = secrets_api.created + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} From 5f292b2290721ae867e5c5399e1fbe5cd70f4da1 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 13 Nov 2026 01:58:32 -0800 Subject: [PATCH 14/18] docs: Kubernetes launcher - Secrets - Say why secret volumes start unnamed --- cloud_pipelines_backend/launchers/kubernetes_launchers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index aee390b5..926c6bef 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -335,6 +335,7 @@ def _handle_secrets_after_preparing_pod( ), ) ) + # get_input_path leaves secret volumes unnamed; the name is chosen here, once, since its fallback is random. for volume in pod.spec.volumes or []: if volume.secret and not volume.secret.secret_name: volume.secret.secret_name = kubernetes_secret_name From b46d014c1cd36892b7f9a5bdfa74238425ee79c3 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 15:50:20 -0800 Subject: [PATCH 15/18] feat: Kubernetes launcher - Namespace - Let callers choose the namespace through a hook --- .../launchers/google_kubernetes_launchers.py | 4 + .../launchers/kubernetes_launchers.py | 24 ++- .../kubernetes/test_kubernetes_namespace.py | 190 ++++++++++++++++++ 3 files changed, 217 insertions(+), 1 deletion(-) create mode 100644 tests/launchers/kubernetes/test_kubernetes_namespace.py diff --git a/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py b/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py index 57f959f8..4311bf9a 100644 --- a/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py @@ -26,6 +26,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: kubernetes_launchers.PodPostProcessor | None = None, + namespace_chooser: kubernetes_launchers.NamespaceChooser | None = None, ): pod_postprocessors = [ kubernetes_launchers._google_kubernetes_engine_accelerator_pod_postprocessor @@ -44,6 +45,7 @@ def __init__( pod_labels=pod_labels, pod_annotations={"gke-gcsfuse/volumes": "true"} | (pod_annotations or {}), pod_postprocessor=final_pod_postporocessor, + namespace_chooser=namespace_chooser, _storage_provider=google_cloud_storage.GoogleCloudStorageProvider( gcs_client ), @@ -67,6 +69,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: kubernetes_launchers.PodPostProcessor | None = None, + namespace_chooser: kubernetes_launchers.NamespaceChooser | None = None, always_launch_jobs: bool = False, ): pod_postprocessors = [ @@ -86,6 +89,7 @@ def __init__( pod_labels=pod_labels, pod_annotations={"gke-gcsfuse/volumes": "true"} | (pod_annotations or {}), pod_postprocessor=final_pod_postporocessor, + namespace_chooser=namespace_chooser, always_launch_jobs=always_launch_jobs, _storage_provider=google_cloud_storage.GoogleCloudStorageProvider( gcs_client diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 14d05ece..55f3eb0d 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -172,6 +172,10 @@ def __call__( ) -> k8s_client_lib.V1Pod: ... +class NamespaceChooser(typing.Protocol): + def __call__(self, *, annotations: dict[str, str] | None = None) -> str | None: ... + + class _KubernetesContainerLauncherBase: """Launcher that launches container using Kubernetes""" @@ -186,6 +190,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ [str, str, str, bool], @@ -203,6 +208,7 @@ def __init__( pod_annotations or {} ) self._pod_postprocessor = pod_postprocessor + self._namespace_chooser = namespace_chooser self._create_volume_and_volume_mount = _create_volume_and_volume_mount try: @@ -511,7 +517,8 @@ def _choose_namespace( Override this method to choose namespace dynamically. """ - del annotations + if self._namespace_chooser: + return self._namespace_chooser(annotations=annotations) or self._namespace return self._namespace @@ -532,6 +539,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ [str, str, str, bool], @@ -547,6 +555,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _storage_provider=_storage_provider, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -738,6 +747,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, ): super().__init__( namespace=namespace, @@ -749,6 +759,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _create_volume_and_volume_mount=_create_volume_and_volume_mount_host_path, ) @@ -768,6 +779,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, ): pod_postprocessors = [_google_kubernetes_engine_accelerator_pod_postprocessor] if pod_postprocessor: @@ -788,6 +800,7 @@ def __init__( pod_labels=pod_labels, pod_annotations={"gke-gcsfuse/volumes": "true"} | (pod_annotations or {}), pod_postprocessor=final_pod_postporocessor, + namespace_chooser=namespace_chooser, _create_volume_and_volume_mount=_create_volume_and_volume_mount_google_cloud_storage, ) @@ -1115,6 +1128,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ [str, str, str, bool], @@ -1131,6 +1145,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -2036,6 +2051,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, always_launch_jobs: bool = False, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ @@ -2052,6 +2068,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _storage_provider=_storage_provider, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -2063,6 +2080,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _storage_provider=_storage_provider, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -2143,6 +2161,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, ): super().__init__( namespace=namespace, @@ -2152,6 +2171,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _storage_provider=local_storage.LocalStorageProvider(), _create_volume_and_volume_mount=_create_volume_and_volume_mount_host_path, ) @@ -2170,6 +2190,7 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, + namespace_chooser: NamespaceChooser | None = None, ): super().__init__( namespace=namespace, @@ -2179,6 +2200,7 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, + namespace_chooser=namespace_chooser, _storage_provider=local_storage.LocalStorageProvider(), _create_volume_and_volume_mount=_create_volume_and_volume_mount_host_path, ) diff --git a/tests/launchers/kubernetes/test_kubernetes_namespace.py b/tests/launchers/kubernetes/test_kubernetes_namespace.py new file mode 100644 index 00000000..47051751 --- /dev/null +++ b/tests/launchers/kubernetes/test_kubernetes_namespace.py @@ -0,0 +1,190 @@ +"""Tests for choosing a launch's namespace through the launcher's `namespace_chooser`.""" + +from __future__ import annotations + +import importlib +import pathlib +import sys +import types +from unittest import mock + +import pytest +from kubernetes import client as k8s_client_lib + +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend.launchers import common_annotations, kubernetes_launchers + +_RETRIES_KEY = kubernetes_launchers.RETRIES_MAX_RETRIES_ANNOTATION_KEY + + +class _Cluster: + """Stands in for the Kubernetes API and records the namespace of each Pod and Job.""" + + def __init__(self) -> None: + self.namespaces: list[tuple[str, str]] = [] + + def create_namespaced_pod( + self, *, namespace: str, body: k8s_client_lib.V1Pod, _request_timeout: int + ) -> k8s_client_lib.V1Pod: + self.namespaces.append(("Pod", namespace)) + body.metadata.name = "task-pod" + body.metadata.namespace = namespace + return body + + def create_namespaced_job( + self, *, namespace: str, body: k8s_client_lib.V1Job, _request_timeout: int + ) -> k8s_client_lib.V1Job: + self.namespaces.append(("Job", namespace)) + body.metadata.namespace = namespace + body.metadata.uid = "job-uid" + return body + + def create_namespaced_service( + self, *, namespace: str, body: k8s_client_lib.V1Service, _request_timeout: int + ) -> k8s_client_lib.V1Service: + return body + + +@pytest.fixture +def cluster(monkeypatch: pytest.MonkeyPatch) -> _Cluster: + api = _Cluster() + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib, "BatchV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib.VersionApi, "get_code", lambda self, **_: None) + + def _no_informer(**kwargs: object) -> None: + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) + return api + + +def _choose_team_namespace(*, annotations: dict[str, str] | None = None) -> str | None: + return "team-namespace" if (annotations or {}).get("team") else None + + +def _launch( + *, + launcher: kubernetes_launchers._KubernetesContainerLauncherBase, + annotations: dict[str, str], + log_uri: str, +) -> None: + launcher.launch_container_task( + component_spec=structures.ComponentSpec( + name="test", + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="alpine") + ), + ), + input_arguments={}, + output_uris={}, + log_uri=log_uri, + annotations={ + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: "123", + **annotations, + }, + ) + + +@pytest.mark.parametrize( + "annotations, namespace", + [ + pytest.param({"team": "yes"}, "team-namespace", id="chooser picks"), + pytest.param({}, "default", id="chooser declines"), + ], +) +def test_job_is_created_in_the_namespace_the_chooser_picks( + annotations: dict[str, str], + namespace: str, + cluster: _Cluster, + tmp_path: pathlib.Path, +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher( + api_client=k8s_client_lib.ApiClient(), + namespace_chooser=_choose_team_namespace, + ) + + _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [("Job", namespace)] + + +@pytest.mark.parametrize( + "annotations, kind", + [ + pytest.param({"team": "yes"}, "Pod", id="pod"), + pytest.param({"team": "yes", _RETRIES_KEY: "1"}, "Job", id="job"), + ], +) +def test_router_hands_the_chooser_to_both_launchers( + annotations: dict[str, str], + kind: str, + cluster: _Cluster, + tmp_path: pathlib.Path, +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + namespace_chooser=_choose_team_namespace, + ) + + _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [(kind, "team-namespace")] + + +@pytest.fixture +def google_kubernetes_launchers(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + """Imports the GKE launchers with a stand-in for google-cloud-storage, which is not installed.""" + import google.cloud + + storage = mock.MagicMock() + monkeypatch.setitem(sys.modules, "google.cloud.storage", storage) + monkeypatch.setattr(google.cloud, "storage", storage, raising=False) + for name in [ + "cloud_pipelines.orchestration.storage_providers.google_cloud_storage", + "cloud_pipelines_backend.launchers.google_kubernetes_launchers", + ]: + monkeypatch.setitem(sys.modules, name, None) + monkeypatch.delitem(sys.modules, name) + return importlib.import_module( + "cloud_pipelines_backend.launchers.google_kubernetes_launchers" + ) + + +@pytest.mark.parametrize( + "module_name, class_name", + [ + ("kubernetes_launchers", "KubernetesWithHostPathContainerLauncher"), + ("kubernetes_launchers", "GoogleKubernetesEngineLauncher"), + ( + "google_kubernetes_launchers", + "GoogleKubernetesEngine_UsingGoogleCloudStorage_KubernetesJobLauncher", + ), + ( + "google_kubernetes_launchers", + "GoogleKubernetesEngine_UsingGoogleCloudStorage_KubernetesPodOrJobLauncher", + ), + ], +) +def test_launcher_hands_the_chooser_on( + module_name: str, + class_name: str, + cluster: _Cluster, + google_kubernetes_launchers: types.ModuleType, + tmp_path: pathlib.Path, +) -> None: + module = { + "kubernetes_launchers": kubernetes_launchers, + "google_kubernetes_launchers": google_kubernetes_launchers, + }[module_name] + launcher = getattr(module, class_name)( + api_client=k8s_client_lib.ApiClient(), + namespace_chooser=_choose_team_namespace, + ) + + _launch( + launcher=launcher, annotations={"team": "yes"}, log_uri=str(tmp_path / "log") + ) + + [(_, namespace)] = cluster.namespaces + assert namespace == "team-namespace" From 80df164335db360fbbe7930358875dcbabafeb29 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 13 Nov 2026 02:10:19 -0800 Subject: [PATCH 16/18] feat: Kubernetes launcher - Namespace - Accept the chooser as the namespace argument --- .../launchers/google_kubernetes_launchers.py | 8 +--- .../launchers/kubernetes_launchers.py | 47 +++++++----------- .../kubernetes/test_kubernetes_namespace.py | 48 ++++++++++++++++--- 3 files changed, 59 insertions(+), 44 deletions(-) diff --git a/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py b/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py index 4311bf9a..04e6529e 100644 --- a/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py @@ -19,14 +19,13 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | kubernetes_launchers.NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, gcs_client: "storage.Client | None" = None, pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: kubernetes_launchers.PodPostProcessor | None = None, - namespace_chooser: kubernetes_launchers.NamespaceChooser | None = None, ): pod_postprocessors = [ kubernetes_launchers._google_kubernetes_engine_accelerator_pod_postprocessor @@ -45,7 +44,6 @@ def __init__( pod_labels=pod_labels, pod_annotations={"gke-gcsfuse/volumes": "true"} | (pod_annotations or {}), pod_postprocessor=final_pod_postporocessor, - namespace_chooser=namespace_chooser, _storage_provider=google_cloud_storage.GoogleCloudStorageProvider( gcs_client ), @@ -62,14 +60,13 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | kubernetes_launchers.NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, gcs_client: "storage.Client | None" = None, pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: kubernetes_launchers.PodPostProcessor | None = None, - namespace_chooser: kubernetes_launchers.NamespaceChooser | None = None, always_launch_jobs: bool = False, ): pod_postprocessors = [ @@ -89,7 +86,6 @@ def __init__( pod_labels=pod_labels, pod_annotations={"gke-gcsfuse/volumes": "true"} | (pod_annotations or {}), pod_postprocessor=final_pod_postporocessor, - namespace_chooser=namespace_chooser, always_launch_jobs=always_launch_jobs, _storage_provider=google_cloud_storage.GoogleCloudStorageProvider( gcs_client diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 55f3eb0d..a6bba3c1 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -173,7 +173,7 @@ def __call__( class NamespaceChooser(typing.Protocol): - def __call__(self, *, annotations: dict[str, str] | None = None) -> str | None: ... + def __call__(self, *, annotations: dict[str, str] | None = None) -> str: ... class _KubernetesContainerLauncherBase: @@ -183,14 +183,13 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ [str, str, str, bool], @@ -208,7 +207,6 @@ def __init__( pod_annotations or {} ) self._pod_postprocessor = pod_postprocessor - self._namespace_chooser = namespace_chooser self._create_volume_and_volume_mount = _create_volume_and_volume_mount try: @@ -233,7 +231,8 @@ def __init__( # The SharedInformer class can only either watch all namespaces or a single namespace. # So, we're using our custom `MultiNamespaceInformer` class that supports watching multiple namespaces. list_func=core_api_client.list_namespaced_pod, - initial_namespaces=[namespace], + # A chooser's namespaces are watched as launches reach them. + initial_namespaces=[namespace] if isinstance(namespace, str) else [], # TODO: Use label selector so that only Tangle pods/jobs get into the cache. # But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change. # label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true", @@ -517,9 +516,9 @@ def _choose_namespace( Override this method to choose namespace dynamically. """ - if self._namespace_chooser: - return self._namespace_chooser(annotations=annotations) or self._namespace - return self._namespace + if isinstance(self._namespace, str): + return self._namespace + return self._namespace(annotations=annotations) class _KubernetesPodLauncher( @@ -532,14 +531,13 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ [str, str, str, bool], @@ -555,7 +553,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _storage_provider=_storage_provider, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -740,14 +737,13 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, ): super().__init__( namespace=namespace, @@ -759,7 +755,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _create_volume_and_volume_mount=_create_volume_and_volume_mount_host_path, ) @@ -771,7 +766,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", @@ -779,7 +774,6 @@ def __init__( pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, ): pod_postprocessors = [_google_kubernetes_engine_accelerator_pod_postprocessor] if pod_postprocessor: @@ -800,7 +794,6 @@ def __init__( pod_labels=pod_labels, pod_annotations={"gke-gcsfuse/volumes": "true"} | (pod_annotations or {}), pod_postprocessor=final_pod_postporocessor, - namespace_chooser=namespace_chooser, _create_volume_and_volume_mount=_create_volume_and_volume_mount_google_cloud_storage, ) @@ -1122,13 +1115,12 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ [str, str, str, bool], @@ -1145,7 +1137,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -1160,7 +1151,8 @@ def __init__( # The SharedInformer class can only either watch all namespaces or a single namespace. # So, we're using our custom `MultiNamespaceInformer` class that supports watching multiple namespaces. list_func=batch_api_client.list_namespaced_job, - initial_namespaces=[namespace], + # A chooser's namespaces are watched as launches reach them. + initial_namespaces=[namespace] if isinstance(namespace, str) else [], # list_func=batch_api_client.list_job_for_all_namespaces, # namespace=None, # TODO: Use label selector so that only Tangle pods/jobs get into the cache. @@ -2045,13 +2037,12 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, always_launch_jobs: bool = False, _storage_provider: storage_provider_interfaces.StorageProvider, _create_volume_and_volume_mount: typing.Callable[ @@ -2068,7 +2059,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _storage_provider=_storage_provider, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -2080,7 +2070,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _storage_provider=_storage_provider, _create_volume_and_volume_mount=_create_volume_and_volume_mount, ) @@ -2154,14 +2143,13 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, # job_name_prefix: str = "task-", pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, ): super().__init__( namespace=namespace, @@ -2171,7 +2159,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _storage_provider=local_storage.LocalStorageProvider(), _create_volume_and_volume_mount=_create_volume_and_volume_mount_host_path, ) @@ -2184,13 +2171,12 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_labels: dict[str, str] | None = None, pod_annotations: dict[str, str] | None = None, pod_postprocessor: PodPostProcessor | None = None, - namespace_chooser: NamespaceChooser | None = None, ): super().__init__( namespace=namespace, @@ -2200,7 +2186,6 @@ def __init__( pod_labels=pod_labels, pod_annotations=pod_annotations, pod_postprocessor=pod_postprocessor, - namespace_chooser=namespace_chooser, _storage_provider=local_storage.LocalStorageProvider(), _create_volume_and_volume_mount=_create_volume_and_volume_mount_host_path, ) diff --git a/tests/launchers/kubernetes/test_kubernetes_namespace.py b/tests/launchers/kubernetes/test_kubernetes_namespace.py index 47051751..ab3a0c22 100644 --- a/tests/launchers/kubernetes/test_kubernetes_namespace.py +++ b/tests/launchers/kubernetes/test_kubernetes_namespace.py @@ -1,4 +1,4 @@ -"""Tests for choosing a launch's namespace through the launcher's `namespace_chooser`.""" +"""Tests for choosing a launch's namespace with a chooser passed as the launcher's `namespace`.""" from __future__ import annotations @@ -59,8 +59,8 @@ def _no_informer(**kwargs: object) -> None: return api -def _choose_team_namespace(*, annotations: dict[str, str] | None = None) -> str | None: - return "team-namespace" if (annotations or {}).get("team") else None +def _choose_team_namespace(*, annotations: dict[str, str] | None = None) -> str: + return "team-namespace" if (annotations or {}).get("team") else "default" def _launch( @@ -90,7 +90,7 @@ def _launch( "annotations, namespace", [ pytest.param({"team": "yes"}, "team-namespace", id="chooser picks"), - pytest.param({}, "default", id="chooser declines"), + pytest.param({}, "default", id="chooser falls back"), ], ) def test_job_is_created_in_the_namespace_the_chooser_picks( @@ -101,7 +101,7 @@ def test_job_is_created_in_the_namespace_the_chooser_picks( ) -> None: launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher( api_client=k8s_client_lib.ApiClient(), - namespace_chooser=_choose_team_namespace, + namespace=_choose_team_namespace, ) _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) @@ -124,7 +124,7 @@ def test_router_hands_the_chooser_to_both_launchers( ) -> None: launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( api_client=k8s_client_lib.ApiClient(), - namespace_chooser=_choose_team_namespace, + namespace=_choose_team_namespace, ) _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) @@ -179,7 +179,7 @@ def test_launcher_hands_the_chooser_on( }[module_name] launcher = getattr(module, class_name)( api_client=k8s_client_lib.ApiClient(), - namespace_chooser=_choose_team_namespace, + namespace=_choose_team_namespace, ) _launch( @@ -188,3 +188,37 @@ def test_launcher_hands_the_chooser_on( [(_, namespace)] = cluster.namespaces assert namespace == "team-namespace" + + +@pytest.mark.parametrize( + "namespace, watched", + [ + pytest.param("team-namespace", ["team-namespace"], id="string"), + pytest.param(_choose_team_namespace, [], id="chooser"), + ], +) +@pytest.mark.usefixtures("cluster") +def test_informers_start_on_a_namespace_string_only( + namespace: str | kubernetes_launchers.NamespaceChooser, + watched: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[list[str]] = [] + + class _Informer: + def __init__(self, *, initial_namespaces: list[str], **kwargs: object) -> None: + seen.append(initial_namespaces) + + def start(self) -> None: + pass + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _Informer) + for api in ["CoreV1Api", "BatchV1Api"]: + monkeypatch.setattr(k8s_client_lib, api, lambda api_client: mock.MagicMock()) + + kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), namespace=namespace + ) + + # The Pod launcher's pod informer, and the Job launcher's pod and Job informers. + assert seen == [watched] * 3 From 1363b0cb039ca242961a9bfd3cc43a7690a69cfa Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Mon, 9 Nov 2026 17:27:36 -0800 Subject: [PATCH 17/18] feat: Kubernetes launcher - Namespace - Ignore namespace changes made by the pod post-processor --- .../launchers/kubernetes_launchers.py | 16 ++++-- .../kubernetes/test_kubernetes_namespace.py | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 5 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index a6bba3c1..73975735 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -619,11 +619,12 @@ def launch_container_task( # Applying the pod post-processor if self._pod_postprocessor: pod = self._pod_postprocessor(pod=pod, annotations=annotations) + _keep_chosen_namespace(pod=pod, namespace=namespace) core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) try: created_pod: k8s_client_lib.V1Pod = core_api_client.create_namespaced_pod( - namespace=pod.metadata.namespace or namespace, + namespace=namespace, body=pod, _request_timeout=self._request_timeout, ) @@ -730,6 +731,14 @@ def _post_processor( return _post_processor +def _keep_chosen_namespace(*, pod: k8s_client_lib.V1Pod, namespace: str) -> None: + if pod.metadata.namespace != namespace: + _logger.warning( + f"Ignoring the pod post-processor's namespace {pod.metadata.namespace!r}; using {namespace!r}. Pass a namespace chooser as namespace instead." + ) + pod.metadata.namespace = namespace + + class KubernetesWithHostPathContainerLauncher(_KubernetesPodLauncher): """Launcher that uses single-node Kubernetes (uses hostPath for data passing)""" @@ -1287,10 +1296,7 @@ def launch_container_task( assert pod.spec assert pod.spec.containers - # Changing the namespace to the final outcome from the pod_postprocessor. - # TODO: In the future (once consumers have migrated to _choose_namespace) - # we should prohibit/ignore changing pod namespace in the pod post-processor. - namespace = pod.metadata.namespace + _keep_chosen_namespace(pod=pod, namespace=namespace) service: k8s_client_lib.V1Service | None = None if enable_multi_node: diff --git a/tests/launchers/kubernetes/test_kubernetes_namespace.py b/tests/launchers/kubernetes/test_kubernetes_namespace.py index ab3a0c22..c1fad39f 100644 --- a/tests/launchers/kubernetes/test_kubernetes_namespace.py +++ b/tests/launchers/kubernetes/test_kubernetes_namespace.py @@ -26,6 +26,8 @@ def __init__(self) -> None: def create_namespaced_pod( self, *, namespace: str, body: k8s_client_lib.V1Pod, _request_timeout: int ) -> k8s_client_lib.V1Pod: + if body.metadata.namespace not in (None, namespace): + raise k8s_client_lib.ApiException(status=400, reason="namespace mismatch") self.namespaces.append(("Pod", namespace)) body.metadata.name = "task-pod" body.metadata.namespace = namespace @@ -222,3 +224,57 @@ def start(self) -> None: # The Pod launcher's pod informer, and the Job launcher's pod and Job informers. assert seen == [watched] * 3 + + +def _move_to_another_namespace( + *, pod: k8s_client_lib.V1Pod, annotations: dict[str, str] | None = None +) -> k8s_client_lib.V1Pod: + pod.metadata.namespace = "another-namespace" + return pod + + +def _leave_the_namespace( + *, pod: k8s_client_lib.V1Pod, annotations: dict[str, str] | None = None +) -> k8s_client_lib.V1Pod: + return pod + + +@pytest.mark.parametrize( + "annotations, kind", + [ + pytest.param({}, "Pod", id="pod"), + pytest.param({_RETRIES_KEY: "1"}, "Job", id="job"), + ], +) +def test_namespace_change_by_the_post_processor_is_ignored_with_a_warning( + annotations: dict[str, str], + kind: str, + cluster: _Cluster, + tmp_path: pathlib.Path, + caplog: pytest.LogCaptureFixture, +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + pod_postprocessor=_move_to_another_namespace, + ) + + _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [(kind, "default")] + assert ( + "Ignoring the pod post-processor's namespace 'another-namespace'" in caplog.text + ) + + +def test_post_processor_that_keeps_the_namespace_logs_no_warning( + cluster: _Cluster, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + pod_postprocessor=_leave_the_namespace, + ) + + _launch(launcher=launcher, annotations={}, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [("Pod", "default")] + assert "Ignoring the pod post-processor's namespace" not in caplog.text From cd8748baffa459087c04d28bff5860ecec099032 Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Fri, 13 Nov 2026 02:28:54 -0800 Subject: [PATCH 18/18] fix: Kubernetes launcher - Namespace - Keep the chosen namespace after the Job transform hook --- .../launchers/kubernetes_launchers.py | 4 +++- .../test_kubernetes_job_service_lifecycle.py | 13 ++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 73975735..279ccaf4 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -1391,11 +1391,13 @@ def launch_container_task( ) job = self._transform_job_before_launching(job=job, annotations=annotations) + # The namespace chosen above is final; a hook that moves the Job is undone. + job.metadata.namespace = namespace batch_api_client = k8s_client_lib.BatchV1Api(api_client=self._api_client) try: created_job: k8s_client_lib.V1Job = batch_api_client.create_namespaced_job( - namespace=job.metadata.namespace or namespace, + namespace=namespace, body=job, _request_timeout=self._request_timeout, ) diff --git a/tests/launchers/test_kubernetes_job_service_lifecycle.py b/tests/launchers/test_kubernetes_job_service_lifecycle.py index 45f91406..45f44565 100644 --- a/tests/launchers/test_kubernetes_job_service_lifecycle.py +++ b/tests/launchers/test_kubernetes_job_service_lifecycle.py @@ -15,6 +15,8 @@ def _launcher(monkeypatch, *, service_error=None): def create_job(namespace, body, _request_timeout): events.append("job") + # Kubernetes rejects a Job whose namespace differs from the request's. + assert body.metadata.namespace in (None, namespace) body.metadata.namespace = namespace body.metadata.uid = "job-uid" batch.created.append((namespace, body, _request_timeout)) @@ -129,8 +131,8 @@ def test_service_creation_failure_rolls_back_the_job(monkeypatch): ] -def test_service_is_created_in_the_namespace_the_job_landed_in(monkeypatch): - launcher, _, core, _ = _launcher(monkeypatch) +def test_hook_cannot_move_the_job_or_its_service(monkeypatch): + launcher, batch, core, _ = _launcher(monkeypatch) def move_job(job, annotations): job.metadata.namespace = "other-jobs" @@ -140,10 +142,11 @@ def move_job(job, annotations): launched = _launch(launcher) + assert batch.created[0][0] == "tangle-jobs" namespace, service, _ = core.created[0] - assert namespace == "other-jobs" - assert service.metadata.namespace == "other-jobs" - assert launched._namespace == "other-jobs" + assert namespace == "tangle-jobs" + assert service.metadata.namespace == "tangle-jobs" + assert launched._namespace == "tangle-jobs" def _launched_job(monkeypatch, *, service, service_delete_status=None):