Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion cloud_pipelines_backend/orchestrator_sql.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
from . import backend_types_sql as bts
from . import component_structures as structures
from . import filter_query_sql
from . import secret_value_store
from .launchers import common_annotations
from .launchers import interfaces as launcher_interfaces
from .instrumentation import bugsnag_instrumentation
Expand Down Expand Up @@ -70,6 +71,7 @@ def __init__(
sleep_seconds_between_queue_sweeps: float = 1.0,
output_data_purge_duration: datetime.timedelta = None,
*,
secret_value_store: secret_value_store.SecretValueStore | None = None,
# Internal/experimental:
_max_container_execution_refresh_error_retries: int = 3,
_max_queue_batch_size: int = 1,
Expand All @@ -94,6 +96,7 @@ def __init__(
self._max_queue_batch_size = _max_queue_batch_size
self._max_queue_batch_duration = _max_queue_batch_duration
self._queued_execution_interceptor = queued_execution_interceptor
self._secret_value_store = secret_value_store

def run_loop(self):
while True:
Expand Down Expand Up @@ -713,7 +716,9 @@ def generate_execution_log_uri(
raise OrchestratorError(
f"{execution.id=}: User error: Error resolving a secret argument for {input_name=}: User {user_id} does not have secret {secret_name}."
)
secret_value = secret.secret_value
secret_value = secret_value_store.read_secret_value(
secret=secret, store=self._secret_value_store
)
input_artifact_data[input_name] = bts.ArtifactData(
total_size=len(secret_value.encode("utf-8")),
is_dir=False,
Expand Down
22 changes: 22 additions & 0 deletions cloud_pipelines_backend/secret_value_store.py
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,28 @@ def raise_if_storage_type_mismatch(
)


def read_secret_value(*, secret: bts.Secret, store: SecretValueStore | None) -> str:
"""Returns the secret's value for a launch: from the row, or through the store that holds it.

Raises when the value is in a store other than this server's, so the launch fails instead of
passing the row's placeholder to the task as if it were the value.
"""
storage_type = get_storage_type(secret=secret)
if storage_type is None:
return secret.secret_value
configured = store.type if store else None
if storage_type != configured:
raise errors.SecretStorageTypeMismatchError(
f"Secret '{secret.secret_name}' is kept in the {storage_type!r} store, "
f"but this server's secret store is {configured!r}; it cannot be read for this launch."
)
return store.get(
storage_details=get_storage_details(secret=secret),
user_id=secret.user_id,
secret_name=secret.secret_name,
)


def retire_secret_value(
*,
store: SecretValueStore,
Expand Down
121 changes: 120 additions & 1 deletion tests/test_secret_value_store.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import logging
import pathlib
from typing import Any
from unittest import mock

import fastapi
import pytest
Expand All @@ -11,8 +12,11 @@
from cloud_pipelines_backend import api_router
from cloud_pipelines_backend import api_server_sql
from cloud_pipelines_backend import backend_types_sql as bts
from cloud_pipelines_backend import component_structures as structures
from cloud_pipelines_backend import database_ops
from cloud_pipelines_backend import errors
from cloud_pipelines_backend import orchestrator_sql
from cloud_pipelines_backend.launchers import interfaces as launcher_interfaces


class _FakeStore:
Expand All @@ -24,6 +28,7 @@ def __init__(
type: str = "fake",
engine: sqlalchemy.Engine | None = None,
fail_delete: bool = False,
fail_get: bool = False,
) -> None:
self.type = type
self.versions: dict[str, str] = {}
Expand All @@ -32,6 +37,7 @@ def __init__(
self._engine = engine
self.row_version_at_delete: list[str | None] = []
self._fail_delete = fail_delete
self._fail_get = fail_get

def save(
self, *, user_id: str, secret_name: str, value: str
Expand All @@ -44,7 +50,9 @@ def save(
def get(
self, *, storage_details: dict[str, Any], user_id: str, secret_name: str
) -> str:
"""Returns the saved version."""
"""Returns the saved version, or fails when told to."""
if self._fail_get:
raise RuntimeError("store unavailable")
return self.versions[storage_details["version"]]

def delete(
Expand Down Expand Up @@ -254,3 +262,114 @@ def test_a_refusal_is_a_503_naming_both_stores(tmp_path: pathlib.Path):
assert "'fake' store" in response.json()["message"]
assert "None" in response.json()["message"]
assert _row(engine=engine) is not None


def _launch_token(
*, tmp_path: pathlib.Path, read_store: _FakeStore | None
) -> tuple[mock.MagicMock, bts.ExecutionNode]:
"""Runs a one-task pipeline that takes alice's TOKEN as an input, with `read_store` as the orchestrator's store."""
engine = _engine(tmp_path=tmp_path)
session_factory = lambda: orm.Session(bind=engine)
component = structures.ComponentSpec(
inputs=[structures.InputSpec(name="token")],
implementation=structures.ContainerImplementation(
container=structures.ContainerSpec(image="python")
),
)
task = structures.TaskSpec(
component_ref=structures.ComponentReference(spec=component),
arguments={
"token": structures.DynamicDataArgument(
dynamic_data={"secret": {"name": "TOKEN"}}
)
},
)
pipeline = structures.ComponentSpec(
implementation=structures.GraphImplementation(
graph=structures.GraphSpec(tasks={"task": task})
)
)
api_server_sql.PipelineRunsApiService_Sql().create(
session=session_factory(),
root_task=structures.TaskSpec(
component_ref=structures.ComponentReference(spec=pipeline)
),
created_by="alice",
)
launch = mock.MagicMock(
return_value=mock.MagicMock(
status=launcher_interfaces.ContainerStatus.PENDING, to_dict=lambda: {}
)
)
orchestrator_sql.OrchestratorService_Sql(
session_factory=session_factory,
launcher=mock.MagicMock(launch_container_task=launch),
storage_provider=mock.MagicMock(),
data_root_uri="file:///tmp/artifacts",
logs_root_uri="file:///tmp/logs",
secret_value_store=read_store,
).process_each_queue_once()
execution = (
session_factory()
.query(bts.ExecutionNode)
.filter(bts.ExecutionNode.container_execution_status.is_not(None))
.one()
)
return launch, execution


def _launched_value(*, launch: mock.MagicMock) -> str:
"""The value the launcher was given for the `token` input."""
return launch.call_args.kwargs["input_arguments"]["token"].value


def _system_error(*, execution: bts.ExecutionNode) -> str:
"""Asserts the launch failed as a system error, and returns its message."""
assert (
execution.container_execution_status
== bts.ContainerExecutionStatus.SYSTEM_ERROR
)
return execution.extra_data[
bts.EXECUTION_NODE_EXTRA_DATA_SYSTEM_ERROR_EXCEPTION_MESSAGE_KEY
]


@pytest.mark.parametrize("read_store", [None, _FakeStore()], ids=["no-store", "store"])
def test_launch_passes_a_plain_row_as_is(
tmp_path: pathlib.Path, read_store: _FakeStore | None
):
_save(engine=_engine(tmp_path=tmp_path), store=None, value="s3cret")
launch, _ = _launch_token(tmp_path=tmp_path, read_store=read_store)
assert _launched_value(launch=launch) == "s3cret"


def test_launch_reads_a_stored_row_through_the_store(tmp_path: pathlib.Path):
store = _FakeStore()
_save(engine=_engine(tmp_path=tmp_path), store=store, value="s3cret")
launch, _ = _launch_token(tmp_path=tmp_path, read_store=store)
assert _launched_value(launch=launch) == "s3cret"


@pytest.mark.parametrize(
"read_store, configured",
[(None, "None"), (_FakeStore(type="other"), "'other'")],
ids=["no-store", "other-store"],
)
def test_launch_fails_when_the_store_does_not_match(
tmp_path: pathlib.Path, read_store: _FakeStore | None, configured: str
):
_save(engine=_engine(tmp_path=tmp_path), store=_FakeStore(), value="s3cret")
launch, execution = _launch_token(tmp_path=tmp_path, read_store=read_store)
launch.assert_not_called()
message = _system_error(execution=execution)
assert "kept in the 'fake' store" in message
assert f"this server's secret store is {configured}" in message


def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path):
_save(engine=_engine(tmp_path=tmp_path), store=_FakeStore(), value="s3cret")
launch, execution = _launch_token(
tmp_path=tmp_path, read_store=_FakeStore(fail_get=True)
)
launch.assert_not_called()
assert "store unavailable" in _system_error(execution=execution)
Loading