diff --git a/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py b/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py index 57f959f8..04e6529e 100644 --- a/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/google_kubernetes_launchers.py @@ -19,7 +19,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | kubernetes_launchers.NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, gcs_client: "storage.Client | None" = None, @@ -60,7 +60,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | kubernetes_launchers.NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, gcs_client: "storage.Client | None" = None, diff --git a/cloud_pipelines_backend/launchers/interfaces.py b/cloud_pipelines_backend/launchers/interfaces.py index 433e09dc..365ddafa 100644 --- a/cloud_pipelines_backend/launchers/interfaces.py +++ b/cloud_pipelines_backend/launchers/interfaces.py @@ -51,7 +51,10 @@ class LaunchedContainerNotFoundError(LauncherError): class InputArgument: total_size: int is_dir: bool - value: str | None = None + # A secret consumed as a value arrives here in plaintext, and the generated repr of this + # class is interpolated into launcher error messages, which are logged, reported and + # stored. `repr=False` keeps the value out of every one of those strings. + value: str | None = dataclasses.field(default=None, repr=False) uri: str | None = None staging_uri: str dynamic_data: str | dict[str, Any] | None = None diff --git a/cloud_pipelines_backend/launchers/kubernetes_launchers.py b/cloud_pipelines_backend/launchers/kubernetes_launchers.py index 14d05ece..a9d305ef 100644 --- a/cloud_pipelines_backend/launchers/kubernetes_launchers.py +++ b/cloud_pipelines_backend/launchers/kubernetes_launchers.py @@ -8,9 +8,11 @@ import logging import os import pathlib +import re import threading import time import typing +import uuid from typing import Any, Final, Optional import kubernetes.client.exceptions @@ -83,6 +85,8 @@ # dynamicData argument keys _MULTI_NODE_NUMBER_OF_NODES_DYNAMIC_DATA_KEY = "system/multi_node/number_of_nodes" _MULTI_NODE_NODE_INDEX_DYNAMIC_DATA_KEY = "system/multi_node/node_index" +_DYNAMIC_DATA_SECRET_KEY = "secret" +_DYNAMIC_DATA_SECRET_NAME_KEY = "name" _MULTI_NODE_NODE_0_ADDRESS_DYNAMIC_DATA_KEY = "system/multi_node/node_0_address" _MULTI_NODE_ALL_NODE_ADDRESSES_DYNAMIC_DATA_KEY = "system/multi_node/all_node_addresses" @@ -95,6 +99,11 @@ _CONTAINER_FILE_NAME = "data" +def _secret_env_var_name(*, input_name: str) -> str: + """Returns the environment variable, also the Secret key, that carries a secret input.""" + return "__TANGLE_SECRET_" + re.sub("[^_0-9a-zA-Z]+", "_", input_name.upper()) + + def _create_volume_and_volume_mount_host_path( container_path: str, artifact_uri: str, @@ -172,6 +181,10 @@ def __call__( ) -> k8s_client_lib.V1Pod: ... +class NamespaceChooser(typing.Protocol): + def __call__(self, *, annotations: dict[str, str] | None = None) -> str: ... + + class _KubernetesContainerLauncherBase: """Launcher that launches container using Kubernetes""" @@ -179,7 +192,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", @@ -227,7 +240,8 @@ def __init__( # The SharedInformer class can only either watch all namespaces or a single namespace. # So, we're using our custom `MultiNamespaceInformer` class that supports watching multiple namespaces. list_func=core_api_client.list_namespaced_pod, - initial_namespaces=[namespace], + # A chooser's namespaces are watched as launches reach them. + initial_namespaces=[namespace] if isinstance(namespace, str) else [], # TODO: Use label selector so that only Tangle pods/jobs get into the cache. # But Tangle has only recently started labeling it's pods/jobs, so if we add filtering too soon, this will become a breaking change. # label_selector=f"{_TANGLE_KUBERNETES_LABEL_KEY}=true", @@ -240,6 +254,128 @@ def __init__( # Fall back from using informer cache to using API calls. self._pod_informer = None + def _handle_secrets_before_preparing_pod( + self, + *, + input_arguments: dict[str, interfaces.InputArgument], + ) -> tuple[dict[str, str], dict[str, str]]: + secret_name_to_secret_value: dict[str, str] = {} + secret_env_name_to_secret_name: dict[str, str] = {} + for input_name, input_argument in list(input_arguments.items()): + if not input_argument.dynamic_data: + continue + dynamic_data_kind, dynamic_data_properties = ( + container_component_utils.parse_dynamic_data_argument( + input_argument.dynamic_data + ) + ) + # Hiding secrets + if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: + secret_name = (dynamic_data_properties or {}).get( + _DYNAMIC_DATA_SECRET_NAME_KEY + ) + if not input_argument.value: + raise interfaces.LauncherError( + f"Orchestrator should have filled secret value for {input_name=}, but got: {input_argument=}" + ) + if not secret_name: + raise interfaces.LauncherError( + f"Secret does not have name: {input_argument.dynamic_data=}" + ) + + env_var_name = _secret_env_var_name(input_name=input_name) + if env_var_name in secret_env_name_to_secret_name: + raise interfaces.LauncherError( + f"Two secret inputs map to the environment variable {env_var_name}; rename {input_name!r} or the other one" + ) + secret_name_to_secret_value[secret_name] = input_argument.value + secret_env_name_to_secret_name[env_var_name] = secret_name + # Hiding the secret value by replacing it with env variable sourced from a Kubernetes Secret + input_argument.value = f"$({env_var_name})" + continue + return secret_name_to_secret_value, secret_env_name_to_secret_name + + def _handle_secrets_after_preparing_pod( + self, + *, + pod: k8s_client_lib.V1Pod, + secret_name_to_secret_value: dict[str, str], + secret_env_name_to_secret_name: dict[str, str], + annotations: dict[str, Any] | None = None, + ) -> k8s_client_lib.V1Secret | None: + if not secret_name_to_secret_value or not secret_env_name_to_secret_name: + return None + + # One Secret per container execution, created once its Pod or Job exists. + container_execution_id = (annotations or {}).get( + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY + ) or uuid.uuid4().hex[:8] + kubernetes_secret_name = "tangle-ce-" + container_execution_id + # Keyed by env var name, unique within this execution's Secret: Kubernetes + # refuses keys outside [-._a-zA-Z0-9], which secret names may use. + secret_data = { + env_var_name: secret_name_to_secret_value[secret_name] + for env_var_name, secret_name in secret_env_name_to_secret_name.items() + } + + main_container = next( + container + for container in pod.spec.containers + if container.name == _MAIN_CONTAINER_NAME + ) + command_line = (main_container.command or []) + (main_container.args or []) + for env_var_name in secret_env_name_to_secret_name: + # An input consumed as a file reads the Secret volume, not the environment. + if not any(f"$({env_var_name})" in part for part in command_line): + continue + main_container.env = main_container.env or [] + main_container.env.append( + k8s_client_lib.V1EnvVar( + name=env_var_name, + value_from=k8s_client_lib.V1EnvVarSource( + secret_key_ref=k8s_client_lib.V1SecretKeySelector( + name=kubernetes_secret_name, + key=env_var_name, + ) + ), + ) + ) + # get_input_path leaves secret volumes unnamed; the name is chosen here, once, since its fallback is random. + for volume in pod.spec.volumes or []: + if volume.secret and not volume.secret.secret_name: + volume.secret.secret_name = kubernetes_secret_name + return k8s_client_lib.V1Secret( + metadata=k8s_client_lib.V1ObjectMeta(name=kubernetes_secret_name), + string_data=secret_data, + ) + + def _create_secret_or_delete_pod_or_job( + self, + *, + secret: k8s_client_lib.V1Secret, + owner_reference: k8s_client_lib.V1OwnerReference, + namespace: str, + delete_pod_or_job: typing.Callable[[], Any], + ) -> None: + """Creates the Secret owned by the Pod or Job, deleting the Pod or Job if that fails.""" + secret.metadata.owner_references = [owner_reference] + try: + k8s_client_lib.CoreV1Api( + api_client=self._api_client + ).create_namespaced_secret( + namespace=namespace, body=secret, _request_timeout=self._request_timeout + ) + except Exception as ex: + try: + delete_pod_or_job() + except Exception: + _logger.exception( + f"Failed to roll back {owner_reference.kind} {owner_reference.name} after Secret creation failed." + ) + raise interfaces.LauncherError( + f"Failed to create Kubernetes Secret {secret.metadata.name}" + ) from ex + def _prepare_kubernetes_pod( self, *, @@ -312,10 +448,18 @@ def get_input_value(input_name: str) -> str: def get_input_path(input_name: str) -> str: input_argument = input_arguments[input_name] uri = input_argument.uri + container_path = ( + container_inputs_root + / naming_utils.sanitize_file_name(input_name) + / _CONTAINER_FILE_NAME + ).as_posix() + volume_name = naming_utils.sanitize_kubernetes_resource_name( + "inputs-" + input_name + ) if not uri: # Prohibiting consuming some kinds of dynamic data as file if input_argument.dynamic_data: - dynamic_data_kind, _ = ( + dynamic_data_kind, dynamic_data_properties = ( container_component_utils.parse_dynamic_data_argument( input_argument.dynamic_data ) @@ -324,6 +468,29 @@ def get_input_path(input_name: str) -> str: raise interfaces.LauncherError( f"It's not possible to consume this dynamic data argument as a file: {input_name=}, {input_argument=}" ) + if dynamic_data_kind == _DYNAMIC_DATA_SECRET_KEY: + # The second secret phase fills in the Secret's name. + volume_map[volume_name] = k8s_client_lib.V1Volume( + name=volume_name, + secret=k8s_client_lib.V1SecretVolumeSource( + items=[ + k8s_client_lib.V1KeyToPath( + key=_secret_env_var_name(input_name=input_name), + path=_CONTAINER_FILE_NAME, + ) + ] + ), + ) + volume_mounts.append( + k8s_client_lib.V1VolumeMount( + name=volume_name, + mount_path=pathlib.PurePosixPath( + container_path + ).parent.as_posix(), + read_only=True, + ) + ) + return container_path if input_argument.value is None: if input_argument.dynamic_data: raise NotImplementedError( @@ -345,14 +512,6 @@ def get_input_path(input_name: str) -> str: # Updating the input_arguments with the URI of the uploaded value input_argument.uri = uri - container_path = ( - container_inputs_root - / naming_utils.sanitize_file_name(input_name) - / _CONTAINER_FILE_NAME - ).as_posix() - volume_name = naming_utils.sanitize_kubernetes_resource_name( - "inputs-" + input_name - ) volume, volume_mount = self._create_volume_and_volume_mount( container_path=container_path, artifact_uri=uri, @@ -511,8 +670,9 @@ def _choose_namespace( Override this method to choose namespace dynamically. """ - del annotations - return self._namespace + if isinstance(self._namespace, str): + return self._namespace + return self._namespace(annotations=annotations) class _KubernetesPodLauncher( @@ -525,7 +685,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", @@ -580,6 +740,9 @@ def launch_container_task( _MULTI_NODE_NODE_0_ADDRESS_DYNAMIC_DATA_KEY: "localhost", _MULTI_NODE_ALL_NODE_ADDRESSES_DYNAMIC_DATA_KEY: "localhost", } + secret_name_to_secret_value, secret_env_name_to_secret_name = ( + self._handle_secrets_before_preparing_pod(input_arguments=input_arguments) + ) for input_name, input_argument in list(input_arguments.items()): if input_argument.value is not None or input_argument.uri is not None: continue @@ -613,11 +776,19 @@ def launch_container_task( # Applying the pod post-processor if self._pod_postprocessor: pod = self._pod_postprocessor(pod=pod, annotations=annotations) + _keep_chosen_namespace(pod=pod, namespace=namespace) + + secret = self._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=secret_name_to_secret_value, + secret_env_name_to_secret_name=secret_env_name_to_secret_name, + annotations=annotations, + ) core_api_client = k8s_client_lib.CoreV1Api(api_client=self._api_client) try: created_pod: k8s_client_lib.V1Pod = core_api_client.create_namespaced_pod( - namespace=pod.metadata.namespace or namespace, + namespace=namespace, body=pod, _request_timeout=self._request_timeout, ) @@ -629,6 +800,20 @@ def launch_container_task( pod_name: str = created_pod.metadata.name pod_namespace: str = created_pod.metadata.namespace _logger.info(f"Created pod {pod_name} in namespace {pod_namespace}") + if secret: + self._create_secret_or_delete_pod_or_job( + secret=secret, + owner_reference=k8s_client_lib.V1OwnerReference( + api_version="v1", + kind="Pod", + name=pod_name, + uid=created_pod.metadata.uid, + ), + namespace=pod_namespace, + delete_pod_or_job=lambda: core_api_client.delete_namespaced_pod( + name=pod_name, namespace=pod_namespace, grace_period_seconds=0 + ), + ) launched_kubernetes_container = LaunchedKubernetesContainer( pod_name=pod_name, @@ -724,6 +909,14 @@ def _post_processor( return _post_processor +def _keep_chosen_namespace(*, pod: k8s_client_lib.V1Pod, namespace: str) -> None: + if pod.metadata.namespace != namespace: + _logger.warning( + f"Ignoring the pod post-processor's namespace {pod.metadata.namespace!r}; using {namespace!r}. Pass a namespace chooser as namespace instead." + ) + pod.metadata.namespace = namespace + + class KubernetesWithHostPathContainerLauncher(_KubernetesPodLauncher): """Launcher that uses single-node Kubernetes (uses hostPath for data passing)""" @@ -731,7 +924,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", @@ -760,7 +953,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_name_prefix: str = "task-pod-", @@ -1109,7 +1302,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_labels: dict[str, str] | None = None, @@ -1145,7 +1338,8 @@ def __init__( # The SharedInformer class can only either watch all namespaces or a single namespace. # So, we're using our custom `MultiNamespaceInformer` class that supports watching multiple namespaces. list_func=batch_api_client.list_namespaced_job, - initial_namespaces=[namespace], + # A chooser's namespaces are watched as launches reach them. + initial_namespaces=[namespace] if isinstance(namespace, str) else [], # list_func=batch_api_client.list_job_for_all_namespaces, # namespace=None, # TODO: Use label selector so that only Tangle pods/jobs get into the cache. @@ -1243,6 +1437,9 @@ def launch_container_task( _MULTI_NODE_NODE_0_ADDRESS_DYNAMIC_DATA_KEY: node_0_address, _MULTI_NODE_ALL_NODE_ADDRESSES_DYNAMIC_DATA_KEY: all_node_addresses_str, } + secret_name_to_secret_value, secret_env_name_to_secret_name = ( + self._handle_secrets_before_preparing_pod(input_arguments=input_arguments) + ) for input_name, input_argument in list(input_arguments.items()): if input_argument.value is not None or input_argument.uri is not None: @@ -1280,10 +1477,14 @@ def launch_container_task( assert pod.spec assert pod.spec.containers - # Changing the namespace to the final outcome from the pod_postprocessor. - # TODO: In the future (once consumers have migrated to _choose_namespace) - # we should prohibit/ignore changing pod namespace in the pod post-processor. - namespace = pod.metadata.namespace + _keep_chosen_namespace(pod=pod, namespace=namespace) + + secret = self._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=secret_name_to_secret_value, + secret_env_name_to_secret_name=secret_env_name_to_secret_name, + annotations=annotations, + ) service: k8s_client_lib.V1Service | None = None if enable_multi_node: @@ -1378,11 +1579,13 @@ def launch_container_task( ) job = self._transform_job_before_launching(job=job, annotations=annotations) + # The namespace chosen above is final; a hook that moves the Job is undone. + job.metadata.namespace = namespace batch_api_client = k8s_client_lib.BatchV1Api(api_client=self._api_client) try: created_job: k8s_client_lib.V1Job = batch_api_client.create_namespaced_job( - namespace=job.metadata.namespace or namespace, + namespace=namespace, body=job, _request_timeout=self._request_timeout, ) @@ -1395,6 +1598,20 @@ def launch_container_task( job_namespace: str = created_job.metadata.namespace job_uid: str = created_job.metadata.uid _logger.info(f"Created Kubernetes Job {job_name} in namespace {job_namespace}") + if secret: + self._create_secret_or_delete_pod_or_job( + secret=secret, + owner_reference=k8s_client_lib.V1OwnerReference( + api_version="batch/v1", kind="Job", name=job_name, uid=job_uid + ), + namespace=job_namespace, + delete_pod_or_job=lambda: batch_api_client.delete_namespaced_job( + name=job_name, + namespace=job_namespace, + grace_period_seconds=0, + propagation_policy="Background", + ), + ) if enable_multi_node: assert service @@ -2030,7 +2247,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_labels: dict[str, str] | None = None, @@ -2136,7 +2353,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, # job_name_prefix: str = "task-", @@ -2164,7 +2381,7 @@ def __init__( self, *, api_client: k8s_client_lib.ApiClient, - namespace: str = "default", + namespace: str | NamespaceChooser = "default", service_account_name: str | None = None, request_timeout: int | tuple[int, int] = 10, pod_labels: dict[str, str] | None = None, diff --git a/tests/launchers/kubernetes/test_kubernetes_namespace.py b/tests/launchers/kubernetes/test_kubernetes_namespace.py new file mode 100644 index 00000000..c1fad39f --- /dev/null +++ b/tests/launchers/kubernetes/test_kubernetes_namespace.py @@ -0,0 +1,280 @@ +"""Tests for choosing a launch's namespace with a chooser passed as the launcher's `namespace`.""" + +from __future__ import annotations + +import importlib +import pathlib +import sys +import types +from unittest import mock + +import pytest +from kubernetes import client as k8s_client_lib + +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend.launchers import common_annotations, kubernetes_launchers + +_RETRIES_KEY = kubernetes_launchers.RETRIES_MAX_RETRIES_ANNOTATION_KEY + + +class _Cluster: + """Stands in for the Kubernetes API and records the namespace of each Pod and Job.""" + + def __init__(self) -> None: + self.namespaces: list[tuple[str, str]] = [] + + def create_namespaced_pod( + self, *, namespace: str, body: k8s_client_lib.V1Pod, _request_timeout: int + ) -> k8s_client_lib.V1Pod: + if body.metadata.namespace not in (None, namespace): + raise k8s_client_lib.ApiException(status=400, reason="namespace mismatch") + self.namespaces.append(("Pod", namespace)) + body.metadata.name = "task-pod" + body.metadata.namespace = namespace + return body + + def create_namespaced_job( + self, *, namespace: str, body: k8s_client_lib.V1Job, _request_timeout: int + ) -> k8s_client_lib.V1Job: + self.namespaces.append(("Job", namespace)) + body.metadata.namespace = namespace + body.metadata.uid = "job-uid" + return body + + def create_namespaced_service( + self, *, namespace: str, body: k8s_client_lib.V1Service, _request_timeout: int + ) -> k8s_client_lib.V1Service: + return body + + +@pytest.fixture +def cluster(monkeypatch: pytest.MonkeyPatch) -> _Cluster: + api = _Cluster() + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib, "BatchV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib.VersionApi, "get_code", lambda self, **_: None) + + def _no_informer(**kwargs: object) -> None: + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) + return api + + +def _choose_team_namespace(*, annotations: dict[str, str] | None = None) -> str: + return "team-namespace" if (annotations or {}).get("team") else "default" + + +def _launch( + *, + launcher: kubernetes_launchers._KubernetesContainerLauncherBase, + annotations: dict[str, str], + log_uri: str, +) -> None: + launcher.launch_container_task( + component_spec=structures.ComponentSpec( + name="test", + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec(image="alpine") + ), + ), + input_arguments={}, + output_uris={}, + log_uri=log_uri, + annotations={ + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: "123", + **annotations, + }, + ) + + +@pytest.mark.parametrize( + "annotations, namespace", + [ + pytest.param({"team": "yes"}, "team-namespace", id="chooser picks"), + pytest.param({}, "default", id="chooser falls back"), + ], +) +def test_job_is_created_in_the_namespace_the_chooser_picks( + annotations: dict[str, str], + namespace: str, + cluster: _Cluster, + tmp_path: pathlib.Path, +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher( + api_client=k8s_client_lib.ApiClient(), + namespace=_choose_team_namespace, + ) + + _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [("Job", namespace)] + + +@pytest.mark.parametrize( + "annotations, kind", + [ + pytest.param({"team": "yes"}, "Pod", id="pod"), + pytest.param({"team": "yes", _RETRIES_KEY: "1"}, "Job", id="job"), + ], +) +def test_router_hands_the_chooser_to_both_launchers( + annotations: dict[str, str], + kind: str, + cluster: _Cluster, + tmp_path: pathlib.Path, +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + namespace=_choose_team_namespace, + ) + + _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [(kind, "team-namespace")] + + +@pytest.fixture +def google_kubernetes_launchers(monkeypatch: pytest.MonkeyPatch) -> types.ModuleType: + """Imports the GKE launchers with a stand-in for google-cloud-storage, which is not installed.""" + import google.cloud + + storage = mock.MagicMock() + monkeypatch.setitem(sys.modules, "google.cloud.storage", storage) + monkeypatch.setattr(google.cloud, "storage", storage, raising=False) + for name in [ + "cloud_pipelines.orchestration.storage_providers.google_cloud_storage", + "cloud_pipelines_backend.launchers.google_kubernetes_launchers", + ]: + monkeypatch.setitem(sys.modules, name, None) + monkeypatch.delitem(sys.modules, name) + return importlib.import_module( + "cloud_pipelines_backend.launchers.google_kubernetes_launchers" + ) + + +@pytest.mark.parametrize( + "module_name, class_name", + [ + ("kubernetes_launchers", "KubernetesWithHostPathContainerLauncher"), + ("kubernetes_launchers", "GoogleKubernetesEngineLauncher"), + ( + "google_kubernetes_launchers", + "GoogleKubernetesEngine_UsingGoogleCloudStorage_KubernetesJobLauncher", + ), + ( + "google_kubernetes_launchers", + "GoogleKubernetesEngine_UsingGoogleCloudStorage_KubernetesPodOrJobLauncher", + ), + ], +) +def test_launcher_hands_the_chooser_on( + module_name: str, + class_name: str, + cluster: _Cluster, + google_kubernetes_launchers: types.ModuleType, + tmp_path: pathlib.Path, +) -> None: + module = { + "kubernetes_launchers": kubernetes_launchers, + "google_kubernetes_launchers": google_kubernetes_launchers, + }[module_name] + launcher = getattr(module, class_name)( + api_client=k8s_client_lib.ApiClient(), + namespace=_choose_team_namespace, + ) + + _launch( + launcher=launcher, annotations={"team": "yes"}, log_uri=str(tmp_path / "log") + ) + + [(_, namespace)] = cluster.namespaces + assert namespace == "team-namespace" + + +@pytest.mark.parametrize( + "namespace, watched", + [ + pytest.param("team-namespace", ["team-namespace"], id="string"), + pytest.param(_choose_team_namespace, [], id="chooser"), + ], +) +@pytest.mark.usefixtures("cluster") +def test_informers_start_on_a_namespace_string_only( + namespace: str | kubernetes_launchers.NamespaceChooser, + watched: list[str], + monkeypatch: pytest.MonkeyPatch, +) -> None: + seen: list[list[str]] = [] + + class _Informer: + def __init__(self, *, initial_namespaces: list[str], **kwargs: object) -> None: + seen.append(initial_namespaces) + + def start(self) -> None: + pass + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _Informer) + for api in ["CoreV1Api", "BatchV1Api"]: + monkeypatch.setattr(k8s_client_lib, api, lambda api_client: mock.MagicMock()) + + kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), namespace=namespace + ) + + # The Pod launcher's pod informer, and the Job launcher's pod and Job informers. + assert seen == [watched] * 3 + + +def _move_to_another_namespace( + *, pod: k8s_client_lib.V1Pod, annotations: dict[str, str] | None = None +) -> k8s_client_lib.V1Pod: + pod.metadata.namespace = "another-namespace" + return pod + + +def _leave_the_namespace( + *, pod: k8s_client_lib.V1Pod, annotations: dict[str, str] | None = None +) -> k8s_client_lib.V1Pod: + return pod + + +@pytest.mark.parametrize( + "annotations, kind", + [ + pytest.param({}, "Pod", id="pod"), + pytest.param({_RETRIES_KEY: "1"}, "Job", id="job"), + ], +) +def test_namespace_change_by_the_post_processor_is_ignored_with_a_warning( + annotations: dict[str, str], + kind: str, + cluster: _Cluster, + tmp_path: pathlib.Path, + caplog: pytest.LogCaptureFixture, +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + pod_postprocessor=_move_to_another_namespace, + ) + + _launch(launcher=launcher, annotations=annotations, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [(kind, "default")] + assert ( + "Ignoring the pod post-processor's namespace 'another-namespace'" in caplog.text + ) + + +def test_post_processor_that_keeps_the_namespace_logs_no_warning( + cluster: _Cluster, tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +) -> None: + launcher = kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesPodOrJobLauncher( + api_client=k8s_client_lib.ApiClient(), + pod_postprocessor=_leave_the_namespace, + ) + + _launch(launcher=launcher, annotations={}, log_uri=str(tmp_path / "log")) + + assert cluster.namespaces == [("Pod", "default")] + assert "Ignoring the pod post-processor's namespace" not in caplog.text diff --git a/tests/launchers/kubernetes/test_kubernetes_secrets.py b/tests/launchers/kubernetes/test_kubernetes_secrets.py new file mode 100644 index 00000000..3ed2d81b --- /dev/null +++ b/tests/launchers/kubernetes/test_kubernetes_secrets.py @@ -0,0 +1,466 @@ +"""Tests for delivering secret inputs to a container through a Kubernetes Secret.""" + +from __future__ import annotations + +import pathlib +from typing import Final + +import pytest +from kubernetes import client as k8s_client_lib + +from cloud_pipelines_backend import component_structures as structures +from cloud_pipelines_backend.launchers import ( + common_annotations, + interfaces, + kubernetes_launchers, +) + +_SECRET_VALUE: Final[str] = "FAKE-tok-123" +_EXECUTION_123: Final[dict[str, str]] = { + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: "123" +} + + +class _SecretsApi: + """Stands in for the Kubernetes API and records the Secrets and Pods the launcher writes.""" + + def __init__(self) -> None: + self.created: list[tuple[str, k8s_client_lib.V1Secret]] = [] + self.created_pods: list[k8s_client_lib.V1Pod] = [] + self.deleted_pods: list[str] = [] + self.fail_secrets = False + + def create_namespaced_secret( + self, *, namespace: str, body: k8s_client_lib.V1Secret, _request_timeout: int + ) -> None: + if self.fail_secrets: + raise k8s_client_lib.ApiException( + status=500, reason="Internal Server Error" + ) + self.created.append((namespace, body)) + + def create_namespaced_pod( + self, *, namespace: str, body: k8s_client_lib.V1Pod, _request_timeout: int + ) -> k8s_client_lib.V1Pod: + body.metadata.name = "task-pod" + body.metadata.namespace = namespace + body.metadata.uid = "pod-uid" + self.created_pods.append(body) + return body + + def delete_namespaced_pod( + self, *, name: str, namespace: str, grace_period_seconds: int + ) -> None: + self.deleted_pods.append(name) + + def create_namespaced_service( + self, *, namespace: str, body: k8s_client_lib.V1Service, _request_timeout: int + ) -> k8s_client_lib.V1Service: + return body + + +class _JobsApi: + """Stands in for the Kubernetes batch API and records the Jobs the launcher creates.""" + + def __init__(self) -> None: + self.created: list[k8s_client_lib.V1Job] = [] + self.deleted: list[str] = [] + + def create_namespaced_job( + self, *, namespace: str, body: k8s_client_lib.V1Job, _request_timeout: int + ) -> k8s_client_lib.V1Job: + body.metadata.namespace = namespace + body.metadata.uid = "job-uid" + self.created.append(body) + return body + + def delete_namespaced_job( + self, + *, + name: str, + namespace: str, + grace_period_seconds: int, + propagation_policy: str, + ) -> None: + self.deleted.append(name) + + +@pytest.fixture +def secrets_api(monkeypatch: pytest.MonkeyPatch) -> _SecretsApi: + api = _SecretsApi() + monkeypatch.setattr(k8s_client_lib, "CoreV1Api", lambda api_client: api) + monkeypatch.setattr(k8s_client_lib.VersionApi, "get_code", lambda self, **_: None) + + def _no_informer(**kwargs: object) -> None: + raise RuntimeError("Informers are disabled in tests.") + + monkeypatch.setattr(kubernetes_launchers, "MultiNamespaceInformer", _no_informer) + return api + + +@pytest.fixture +def jobs_api(monkeypatch: pytest.MonkeyPatch) -> _JobsApi: + api = _JobsApi() + monkeypatch.setattr(k8s_client_lib, "BatchV1Api", lambda api_client: api) + return api + + +def _make_launcher() -> kubernetes_launchers._KubernetesPodLauncher: + # Skips __init__, which contacts the cluster; the secret handling reads only these two. + launcher = object.__new__(kubernetes_launchers._KubernetesPodLauncher) + launcher._api_client = None + launcher._namespace = "default" + return launcher + + +def _make_secret_argument() -> interfaces.InputArgument: + return interfaces.InputArgument( + total_size=len(_SECRET_VALUE), + is_dir=False, + value=_SECRET_VALUE, + staging_uri="gs://bucket/token", + dynamic_data={"secret": {"name": "hf-token"}}, + ) + + +def _make_pod(*, args: list[str]) -> k8s_client_lib.V1Pod: + return k8s_client_lib.V1Pod( + metadata=k8s_client_lib.V1ObjectMeta(name="test-pod", namespace="tangle-jobs"), + spec=k8s_client_lib.V1PodSpec( + containers=[k8s_client_lib.V1Container(name="main", args=args)] + ), + ) + + +def _launch( + *, + launcher: kubernetes_launchers._KubernetesPodLauncher, + annotations: dict[str, str] | None = None, +) -> k8s_client_lib.V1Pod: + """Runs both secret phases around a pod whose only argument is the secret input.""" + argument = _make_secret_argument() + values, env_names = launcher._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + pod = _make_pod(args=[argument.value]) + launcher._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=values, + secret_env_name_to_secret_name=env_names, + annotations=annotations, + ) + return pod + + +def test_secret_value_is_replaced_by_a_placeholder() -> None: + argument = _make_secret_argument() + + values, env_names = _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + + assert _SECRET_VALUE not in argument.value + assert values == {"hf-token": _SECRET_VALUE} + assert env_names == {"__TANGLE_SECRET_TOKEN": "hf-token"} + + +def test_inputs_without_secrets_are_left_alone() -> None: + argument = interfaces.InputArgument( + total_size=5, is_dir=False, value="hello", staging_uri="gs://bucket/greeting" + ) + + values, env_names = _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"greeting": argument} + ) + + assert (values, env_names) == ({}, {}) + assert argument.value == "hello" + + +@pytest.mark.parametrize( + "value, dynamic_data", + [ + pytest.param(None, {"secret": {"name": "hf-token"}}, id="no value"), + pytest.param(_SECRET_VALUE, {"secret": {}}, id="no secret name"), + ], +) +def test_incomplete_secret_argument_is_a_launcher_error( + value: str | None, dynamic_data: dict[str, dict[str, str]] +) -> None: + argument = interfaces.InputArgument( + total_size=0, + is_dir=False, + value=value, + staging_uri="gs://bucket/token", + dynamic_data=dynamic_data, + ) + + with pytest.raises(interfaces.LauncherError): + _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + + +@pytest.mark.usefixtures("secrets_api") +def test_container_reads_the_value_from_the_secret() -> None: + pod = _launch(launcher=_make_launcher(), annotations=_EXECUTION_123) + + assert pod.spec.containers[0].env == [ + k8s_client_lib.V1EnvVar( + name="__TANGLE_SECRET_TOKEN", + value_from=k8s_client_lib.V1EnvVarSource( + secret_key_ref=k8s_client_lib.V1SecretKeySelector( + name="tangle-ce-123", key="__TANGLE_SECRET_TOKEN" + ) + ), + ) + ] + assert _SECRET_VALUE not in pod.spec.containers[0].args[0] + + +def test_two_secret_inputs_on_one_env_var_name_are_a_launcher_error() -> None: + first, second = _make_secret_argument(), _make_secret_argument() + second.dynamic_data = {"secret": {"name": "anthropic-key"}} + + with pytest.raises(interfaces.LauncherError, match="__TANGLE_SECRET_API_KEY"): + _make_launcher()._handle_secrets_before_preparing_pod( + input_arguments={"api-key": first, "api_key": second} + ) + + +@pytest.mark.usefixtures("secrets_api") +def test_placeholder_names_the_environment_variable_the_pod_defines() -> None: + pod = _launch(launcher=_make_launcher()) + + container = pod.spec.containers[0] + assert [env.name for env in container.env] == ["__TANGLE_SECRET_TOKEN"] + assert container.args == ["$(__TANGLE_SECRET_TOKEN)"] + assert _SECRET_VALUE not in container.args[0] + + +def test_secret_carries_the_plain_value_in_string_data() -> None: + # `data` must be base64, so the API server would decode the plain value into other bytes. + secret = _make_launcher()._handle_secrets_after_preparing_pod( + pod=_make_pod(args=[]), + secret_name_to_secret_value={"hf-token": _SECRET_VALUE}, + secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, + ) + + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} + assert secret.data is None + + +@pytest.mark.usefixtures("secrets_api") +def test_environment_entry_goes_to_the_main_container_not_the_first() -> None: + sidecar = k8s_client_lib.V1Container(name="sidecar") + pod = _make_pod(args=["$(__TANGLE_SECRET_TOKEN)"]) + pod.spec.containers.insert(0, sidecar) + + _make_launcher()._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value={"hf-token": _SECRET_VALUE}, + secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, + annotations=None, + ) + + [_, main] = pod.spec.containers + assert sidecar.env is None + assert [env.name for env in main.env] == ["__TANGLE_SECRET_TOKEN"] + + +@pytest.mark.usefixtures("secrets_api") +def test_placeholder_in_the_command_gets_its_environment_entry() -> None: + pod = _make_pod(args=[]) + pod.spec.containers[0].command = ["$(__TANGLE_SECRET_TOKEN)"] + + _make_launcher()._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value={"hf-token": _SECRET_VALUE}, + secret_env_name_to_secret_name={"__TANGLE_SECRET_TOKEN": "hf-token"}, + ) + + assert [env.name for env in pod.spec.containers[0].env] == ["__TANGLE_SECRET_TOKEN"] + + +_JOB_LAUNCHER: Final = ( + kubernetes_launchers.Local_Kubernetes_UsingHostPathStorage_KubernetesJobLauncher +) +_POD_OR_JOB: Final = pytest.mark.parametrize( + "launcher_class, pod_or_job", + [ + pytest.param( + kubernetes_launchers.KubernetesWithHostPathContainerLauncher, + k8s_client_lib.V1OwnerReference( + api_version="v1", kind="Pod", name="task-pod", uid="pod-uid" + ), + id="pod", + ), + pytest.param( + _JOB_LAUNCHER, + k8s_client_lib.V1OwnerReference( + api_version="batch/v1", kind="Job", name="tangle-ce-123", uid="job-uid" + ), + id="job", + ), + ], +) + + +def _launch_task( + *, + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + annotations: dict[str, str], + log_uri: str, + as_file: bool = False, +) -> None: + """Launches a task whose only argument is the secret input, as a value or as a file.""" + placeholder = ( + structures.InputPathPlaceholder if as_file else structures.InputValuePlaceholder + ) + launcher = launcher_class(api_client=k8s_client_lib.ApiClient()) + launcher.launch_container_task( + component_spec=structures.ComponentSpec( + name="test", + inputs=[structures.InputSpec(name="token")], + implementation=structures.ContainerImplementation( + container=structures.ContainerSpec( + image="alpine", + args=[placeholder(input_name="token")], + ) + ), + ), + input_arguments={"token": _make_secret_argument()}, + output_uris={}, + log_uri=log_uri, + annotations={ + common_annotations.CONTAINER_EXECUTION_ID_ANNOTATION_KEY: "123", + **annotations, + }, + ) + + +@pytest.mark.parametrize( + "annotations", + [ + pytest.param({}, id="single node"), + pytest.param( + {kubernetes_launchers.MULTI_NODE_NUMBER_OF_NODES_ANNOTATION_KEY: "2"}, + id="two nodes", + ), + ], +) +def test_job_container_reads_the_value_from_the_secret( + annotations: dict[str, str], + secrets_api: _SecretsApi, + jobs_api: _JobsApi, + tmp_path: pathlib.Path, +) -> None: + _launch_task( + launcher_class=_JOB_LAUNCHER, + annotations=annotations, + log_uri=str(tmp_path / "log.txt"), + ) + + [job] = jobs_api.created + [main] = job.spec.template.spec.containers + assert "__TANGLE_SECRET_TOKEN" in [env.name for env in main.env] + assert main.args == ["$(__TANGLE_SECRET_TOKEN)"] + [(_, secret)] = secrets_api.created + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} + + +@_POD_OR_JOB +@pytest.mark.usefixtures("jobs_api") +def test_secret_is_named_after_the_execution_and_owned_by_its_pod_or_job( + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + pod_or_job: k8s_client_lib.V1OwnerReference, + secrets_api: _SecretsApi, + tmp_path: pathlib.Path, +) -> None: + _launch_task( + launcher_class=launcher_class, + annotations=_EXECUTION_123, + log_uri=str(tmp_path / "log.txt"), + ) + + [(namespace, secret)] = secrets_api.created + assert namespace == "default" + assert secret.metadata.name == "tangle-ce-123" + assert secret.metadata.owner_references == [pod_or_job] + + +@_POD_OR_JOB +def test_failed_secret_deletes_its_pod_or_job( + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + pod_or_job: k8s_client_lib.V1OwnerReference, + secrets_api: _SecretsApi, + jobs_api: _JobsApi, + tmp_path: pathlib.Path, +) -> None: + secrets_api.fail_secrets = True + + with pytest.raises(interfaces.LauncherError): + _launch_task( + launcher_class=launcher_class, + annotations=_EXECUTION_123, + log_uri=str(tmp_path / "log.txt"), + ) + + assert secrets_api.deleted_pods + jobs_api.deleted == [pod_or_job.name] + + +def test_secret_is_keyed_by_the_env_var_name_so_any_secret_name_works() -> None: + # Kubernetes refuses "hf token" as a Secret key; the input-derived name is always valid. + argument = _make_secret_argument() + argument.dynamic_data = {"secret": {"name": "hf token"}} + launcher = _make_launcher() + values, env_names = launcher._handle_secrets_before_preparing_pod( + input_arguments={"token": argument} + ) + pod = _make_pod(args=[argument.value]) + + secret = launcher._handle_secrets_after_preparing_pod( + pod=pod, + secret_name_to_secret_value=values, + secret_env_name_to_secret_name=env_names, + ) + + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} + [env] = pod.spec.containers[0].env + assert env.value_from.secret_key_ref.key == "__TANGLE_SECRET_TOKEN" + + +@_POD_OR_JOB +def test_file_input_reads_the_secret_volume( + launcher_class: type[kubernetes_launchers._KubernetesContainerLauncherBase], + pod_or_job: k8s_client_lib.V1OwnerReference, + secrets_api: _SecretsApi, + jobs_api: _JobsApi, + tmp_path: pathlib.Path, +) -> None: + _launch_task( + launcher_class=launcher_class, + annotations={}, + log_uri=str(tmp_path / "log.txt"), + as_file=True, + ) + + [pod_spec] = [pod.spec for pod in secrets_api.created_pods] + [ + job.spec.template.spec for job in jobs_api.created + ] + [main] = pod_spec.containers + assert main.args == ["/tmp/inputs/token/data"] + assert "__TANGLE_SECRET_TOKEN" not in [env.name for env in main.env or []] + [volume] = [volume for volume in pod_spec.volumes if volume.secret] + assert volume.secret == k8s_client_lib.V1SecretVolumeSource( + secret_name="tangle-ce-123", + items=[k8s_client_lib.V1KeyToPath(key="__TANGLE_SECRET_TOKEN", path="data")], + ) + assert ( + k8s_client_lib.V1VolumeMount( + name=volume.name, mount_path="/tmp/inputs/token", read_only=True + ) + in main.volume_mounts + ) + [(_, secret)] = secrets_api.created + assert secret.string_data == {"__TANGLE_SECRET_TOKEN": _SECRET_VALUE} diff --git a/tests/launchers/test_input_argument_repr.py b/tests/launchers/test_input_argument_repr.py new file mode 100644 index 00000000..6978f74a --- /dev/null +++ b/tests/launchers/test_input_argument_repr.py @@ -0,0 +1,108 @@ +"""The resolved input argument must not print its value. + +A secret consumed as a value is held in `InputArgument.value`. Launcher error messages +interpolate the whole argument -- `f"... {input_argument=}"` -- and the orchestrator logs +that message, reports it, stores it on the execution node and serves it through the +execution-log API, so a printed value reaches several durable places at once. +""" + +import dataclasses +import json +import pathlib +import traceback +from typing import Final + +from cloud_pipelines_backend.launchers import interfaces + +_SECRET_VALUE: Final[str] = "secret-value-authenticates-nothing" + +_LAUNCHER_MODULES: Final[tuple[str, ...]] = ( + "kubernetes_launchers.py", + "local_docker_launchers.py", + "huggingface_launchers.py", +) + + +def _input_argument() -> interfaces.InputArgument: + return interfaces.InputArgument( + total_size=len(_SECRET_VALUE), + is_dir=False, + value=_SECRET_VALUE, + uri=None, + staging_uri="memory://staging/inputs/token/data", + ) + + +class TestInputArgumentDoesNotPrintItsValue: + def test_every_way_of_stringifying_it_omits_the_value(self) -> None: + argument = _input_argument() + for rendered in ( + repr(argument), + str(argument), + f"{argument}", + f"{argument!r}", + f"{argument=}", + "{}".format(argument), + "%s" % (argument,), + ): + assert _SECRET_VALUE not in rendered, rendered + + def test_the_other_fields_still_print(self) -> None: + """The messages are written to identify an input, so they must keep doing that.""" + rendered = repr(_input_argument()) + assert "total_size=" in rendered + assert "is_dir=" in rendered + assert "staging_uri=" in rendered + + def test_the_value_is_still_readable_and_still_a_field(self) -> None: + """Hidden from printing, not removed: the launcher passes it to the container.""" + argument = _input_argument() + assert argument.value == _SECRET_VALUE + assert "value" in {field.name for field in dataclasses.fields(argument)} + + def test_a_traceback_built_the_way_the_launchers_build_one_is_clean(self) -> None: + """The message shape used at every raise site that reports an input argument.""" + argument = _input_argument() + input_name = "token" + try: + raise interfaces.LauncherError( + f"Artifact is too big to consume as value. Consume it as file instead." + f" {input_name=}, {argument=}" + ) + except interfaces.LauncherError: + captured = traceback.format_exc() + assert _SECRET_VALUE not in captured + assert "input_name='token'" in captured + + +class TestNoLauncherMessageReachesAroundTheRepr: + """`repr=False` is defeated by a message that interpolates the field directly. + + This guards the fix rather than the symptom: a new `{input_argument=}` is safe, while + an `{input_argument.value}` would print the secret again. + """ + + def test_no_launcher_formats_an_argument_value_into_a_string(self) -> None: + launchers_dir = pathlib.Path(interfaces.__file__).resolve().parent + offenders: list[str] = [] + for module_name in _LAUNCHER_MODULES: + source = (launchers_dir / module_name).read_text() + for forbidden in ( + "{input_argument.value", + "{artifact_data.value", + "{ia.value", + ): + if forbidden in source: + offenders.append(f"{module_name}: {forbidden}") + assert not offenders, offenders + + def test_the_value_is_not_serialized_into_a_dict_anywhere(self) -> None: + """`dataclasses.asdict` ignores `repr=False`, so no launcher may use it here.""" + launchers_dir = pathlib.Path(interfaces.__file__).resolve().parent + for module_name in _LAUNCHER_MODULES: + source = (launchers_dir / module_name).read_text() + assert "asdict(input_argument" not in source, module_name + + def test_asdict_would_have_leaked_it(self) -> None: + """Pins why the test above exists, rather than asserting a bare convention.""" + assert _SECRET_VALUE in json.dumps(dataclasses.asdict(_input_argument())) diff --git a/tests/launchers/test_kubernetes_job_service_lifecycle.py b/tests/launchers/test_kubernetes_job_service_lifecycle.py index 45f91406..45f44565 100644 --- a/tests/launchers/test_kubernetes_job_service_lifecycle.py +++ b/tests/launchers/test_kubernetes_job_service_lifecycle.py @@ -15,6 +15,8 @@ def _launcher(monkeypatch, *, service_error=None): def create_job(namespace, body, _request_timeout): events.append("job") + # Kubernetes rejects a Job whose namespace differs from the request's. + assert body.metadata.namespace in (None, namespace) body.metadata.namespace = namespace body.metadata.uid = "job-uid" batch.created.append((namespace, body, _request_timeout)) @@ -129,8 +131,8 @@ def test_service_creation_failure_rolls_back_the_job(monkeypatch): ] -def test_service_is_created_in_the_namespace_the_job_landed_in(monkeypatch): - launcher, _, core, _ = _launcher(monkeypatch) +def test_hook_cannot_move_the_job_or_its_service(monkeypatch): + launcher, batch, core, _ = _launcher(monkeypatch) def move_job(job, annotations): job.metadata.namespace = "other-jobs" @@ -140,10 +142,11 @@ def move_job(job, annotations): launched = _launch(launcher) + assert batch.created[0][0] == "tangle-jobs" namespace, service, _ = core.created[0] - assert namespace == "other-jobs" - assert service.metadata.namespace == "other-jobs" - assert launched._namespace == "other-jobs" + assert namespace == "tangle-jobs" + assert service.metadata.namespace == "tangle-jobs" + assert launched._namespace == "tangle-jobs" def _launched_job(monkeypatch, *, service, service_delete_status=None):