From 64f54dd7978073e77c40c75fc2ffcd97d190da0e Mon Sep 17 00:00:00 2001 From: Yue Chao Qin Date: Tue, 17 Nov 2026 17:24:43 -0800 Subject: [PATCH] feat: GSM Secrets - Save secret values through an optional store hook --- cloud_pipelines_backend/api_router.py | 17 +- cloud_pipelines_backend/api_server_sql.py | 38 ++- cloud_pipelines_backend/errors.py | 4 + cloud_pipelines_backend/secret_value_store.py | 126 +++++++++ tests/test_secret_value_store.py | 256 ++++++++++++++++++ 5 files changed, 438 insertions(+), 3 deletions(-) create mode 100644 cloud_pipelines_backend/secret_value_store.py create mode 100644 tests/test_secret_value_store.py diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index 0e8fa41a..5614afb0 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -18,6 +18,7 @@ from .instrumentation import contextual_logging if typing.TYPE_CHECKING: + from . import secret_value_store from .launchers import interfaces as launcher_interfaces @@ -40,6 +41,7 @@ def setup_routes( container_launcher_for_log_streaming: "launcher_interfaces.ContainerTaskLauncher[launcher_interfaces.LaunchedContainer] | None" = None, default_component_library_owner_username: str = "admin", do_skip_backfill: bool = False, + secret_value_store: "secret_value_store.SecretValueStore | None" = None, ): def get_session(): with orm.Session(autocommit=False, autoflush=False, bind=db_engine) as session: @@ -76,6 +78,7 @@ async def lifespan(app: fastapi.FastAPI): user_details_getter=user_details_getter, get_launcher=get_launcher, lifespan=lifespan, + secret_value_store=secret_value_store, ) @@ -93,6 +96,7 @@ def _setup_routes_internal( | typing.Callable[..., abc.Iterator[typing.Any]] | None ) = None, + secret_value_store: "secret_value_store.SecretValueStore | None" = None, ): # We request `app: fastapi.FastAPI` instead of just returning the router # because we want to add exception handler which is only supported for `FastAPI`. @@ -150,6 +154,15 @@ def handle_api_validation_error( content={"detail": str(exc)}, ) + @app.exception_handler(errors.SecretStorageTypeMismatchError) + def handle_secret_storage_type_mismatch_error( + request: fastapi.Request, exc: errors.SecretStorageTypeMismatchError + ): + return fastapi.responses.JSONResponse( + status_code=fastapi.status.HTTP_503_SERVICE_UNAVAILABLE, + content={"message": str(exc)}, + ) + @app.exception_handler(NotImplementedError) def handle_not_implemented_error( request: fastapi.Request, exc: NotImplementedError @@ -462,7 +475,9 @@ def get_current_user( ) ### Secrets routes - secrets_service = api_server_sql.SecretsApiService() + secrets_service = api_server_sql.SecretsApiService( + secret_value_store=secret_value_store + ) router.get("/api/secrets/", tags=["secrets"], **default_config)( inject_session_dependency( diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index 8277606e..c48aba51 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -13,6 +13,7 @@ from .access_control import checks as access_control_checks from .access_control import runs as access_control_runs from . import filter_query_sql +from . import secret_value_store if typing.TYPE_CHECKING: from cloud_pipelines.orchestration.storage_providers import ( @@ -1256,6 +1257,13 @@ class ListSecretsResponse: class SecretsApiService: + def __init__( + self, + *, + secret_value_store: secret_value_store.SecretValueStore | None = None, + ): + self._secret_value_store = secret_value_store + def create_secret( self, *, @@ -1318,7 +1326,9 @@ def _create_or_update_secret( raise errors.ItemAlreadyExistsError( f"Secret with name '{secret_name}' already exists." ) - secret.secret_value = secret_value + secret_value_store.raise_if_storage_type_mismatch( + secret=secret, store=self._secret_value_store + ) secret.updated_at = current_time else: if raise_if_not_exists: @@ -1328,17 +1338,30 @@ def _create_or_update_secret( secret = bts.Secret( user_id=user_id, secret_name=secret_name, - secret_value=secret_value, + # Set below, through the store when there is one. + secret_value="", created_at=current_time, updated_at=current_time, ) session.add(secret) + # The new value is saved before the row commits, and the one it replaces is retired after, + # so the row never names a value that is already gone. + previous_storage_details = secret_value_store.store_secret_value( + secret=secret, value=secret_value, store=self._secret_value_store + ) if description: secret.description = description if expires_at: secret.expires_at = expires_at response = SecretInfoResponse.from_db(secret) session.commit() + if previous_storage_details is not None: + secret_value_store.retire_secret_value( + store=self._secret_value_store, + storage_details=previous_storage_details, + user_id=user_id, + secret_name=secret_name, + ) return response def delete_secret( @@ -1353,8 +1376,19 @@ def delete_secret( raise errors.ItemNotFoundError( f"Secret with name '{secret_name}' does not exist." ) + secret_value_store.raise_if_storage_type_mismatch( + secret=secret, store=self._secret_value_store + ) + storage_details = secret_value_store.get_storage_details(secret=secret) session.delete(secret) session.commit() + if storage_details is not None: + secret_value_store.retire_secret_value( + store=self._secret_value_store, + storage_details=storage_details, + user_id=user_id, + secret_name=secret_name, + ) def list_secrets( self, diff --git a/cloud_pipelines_backend/errors.py b/cloud_pipelines_backend/errors.py index dee86527..9c684fea 100644 --- a/cloud_pipelines_backend/errors.py +++ b/cloud_pipelines_backend/errors.py @@ -22,6 +22,10 @@ class PermissionError(Exception): pass +class SecretStorageTypeMismatchError(Exception): + """A secret's value is in a store of another type than this server's, or the server has none, so it cannot be changed.""" + + class ApiValidationError(Exception): """Base for all filter/annotation validation errors -> 422.""" diff --git a/cloud_pipelines_backend/secret_value_store.py b/cloud_pipelines_backend/secret_value_store.py new file mode 100644 index 00000000..254ef451 --- /dev/null +++ b/cloud_pipelines_backend/secret_value_store.py @@ -0,0 +1,126 @@ +import logging +import typing +from typing import Any, Final + +from . import backend_types_sql as bts +from . import errors + +_logger = logging.getLogger(__name__) + +EXTRA_DATA_STORAGE_KEY: Final[str] = "storage" +STORAGE_TYPE_KEY: Final[str] = "type" +STORAGE_DETAILS_KEY: Final[str] = "details" + + +class SecretValueStore(typing.Protocol): + """Keeps secret values outside the database row, for example in a cloud secret manager.""" + + # Recorded in the row's `extra_data`, so later reads and writes know which store holds the value. + type: str + + def save( + self, *, user_id: str, secret_name: str, value: str + ) -> tuple[str, dict[str, Any]]: + """Saves the value; returns what the row's `secret_value` keeps, and the storage details for `get` and `delete`. + + Example, a store that keeps numbered versions: `save(user_id="alice", secret_name="HF_TOKEN", + value="hf_FAKE123")` returns `("", {"version": "4"})` — the row keeps `""`, and `{"version": "4"}` + is what `get` and `delete` are later given as `storage_details`. + """ + ... + + def get( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> str: + """Returns the value that `save` saved with these storage details.""" + ... + + def delete( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> None: + """Retires the value that `save` saved with these storage details.""" + ... + + +def get_storage_type(*, secret: bts.Secret) -> str | None: + """Returns the type of the store holding the row's value, or None when the value is plain in the row. + + Example: with `extra_data = {"storage": {"type": "gsm", "details": {"version": "4"}}}` it returns `"gsm"`. + """ + storage = (secret.extra_data or {}).get(EXTRA_DATA_STORAGE_KEY) + return storage[STORAGE_TYPE_KEY] if storage else None + + +def get_storage_details(*, secret: bts.Secret) -> dict[str, Any] | None: + """Returns what the store needs to find the row's value, or None when the value is plain in the row. + + Example: with `extra_data = {"storage": {"type": "gsm", "details": {"version": "4"}}}` it returns + `{"version": "4"}`. + """ + storage = (secret.extra_data or {}).get(EXTRA_DATA_STORAGE_KEY) + return storage[STORAGE_DETAILS_KEY] if storage else None + + +def store_secret_value( + *, secret: bts.Secret, value: str, store: SecretValueStore | None +) -> dict[str, Any] | None: + """Sets the row's value, through the store if one is given. + + Returns the storage details of the value it replaces in the store, or None when there was none. + """ + if store is None: + secret.secret_value = value + return None + previous_storage_details = get_storage_details(secret=secret) + secret.secret_value, storage_details = store.save( + user_id=secret.user_id, secret_name=secret.secret_name, value=value + ) + # A merge: the copy keeps every other key, so only the storage entry changes. + secret.extra_data = { + **(secret.extra_data or {}), + EXTRA_DATA_STORAGE_KEY: { + STORAGE_TYPE_KEY: store.type, + STORAGE_DETAILS_KEY: storage_details, + }, + } + return previous_storage_details + + +def raise_if_storage_type_mismatch( + *, secret: bts.Secret, store: SecretValueStore | None +) -> None: + """Raises when the row's value is in a store other than this server's, leaving the row unchanged. + + Writing plain text over it, or deleting the row, would leave the stored value behind and still readable. + """ + storage_type = get_storage_type(secret=secret) + if storage_type is None: + return + configured = store.type if store else None + if storage_type != configured: + raise errors.SecretStorageTypeMismatchError( + f"Secret '{secret.secret_name}' is kept in the {storage_type!r} store, " + f"but this server's secret store is {configured!r}; it cannot be changed or deleted here." + ) + + +def retire_secret_value( + *, + store: SecretValueStore, + storage_details: dict[str, Any], + user_id: str, + secret_name: str, +) -> None: + """Retires a stored value the row no longer names, logging a failure instead of raising it. + + A helper for after the row commits: the change is already saved, so raising would report a successful save + as a failure, and the only harm of a failed retire is an old value left enabled in the store. + """ + try: + store.delete( + storage_details=storage_details, user_id=user_id, secret_name=secret_name + ) + except Exception: + _logger.exception( + f"Could not retire the previous stored value of secret '{secret_name}'." + ) diff --git a/tests/test_secret_value_store.py b/tests/test_secret_value_store.py new file mode 100644 index 00000000..9ff1c6f8 --- /dev/null +++ b/tests/test_secret_value_store.py @@ -0,0 +1,256 @@ +import logging +import pathlib +from typing import Any + +import fastapi +import pytest +import sqlalchemy +from sqlalchemy import orm +from starlette.testclient import TestClient + +from cloud_pipelines_backend import api_router +from cloud_pipelines_backend import api_server_sql +from cloud_pipelines_backend import backend_types_sql as bts +from cloud_pipelines_backend import database_ops +from cloud_pipelines_backend import errors + + +class _FakeStore: + """Keeps values in memory, one numbered version per `save`, like a cloud secret manager.""" + + def __init__( + self, + *, + type: str = "fake", + engine: sqlalchemy.Engine | None = None, + fail_delete: bool = False, + ) -> None: + self.type = type + self.versions: dict[str, str] = {} + self.retired: list[str] = [] + # With an engine, `delete` records which version the committed row names at that moment. + self._engine = engine + self.row_version_at_delete: list[str | None] = [] + self._fail_delete = fail_delete + + def save( + self, *, user_id: str, secret_name: str, value: str + ) -> tuple[str, dict[str, Any]]: + """Saves the value as the next version.""" + version = str(len(self.versions) + 1) + self.versions[version] = value + return "", {"version": version} + + def get( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> str: + """Returns the saved version.""" + return self.versions[storage_details["version"]] + + def delete( + self, *, storage_details: dict[str, Any], user_id: str, secret_name: str + ) -> None: + """Retires the version, or fails when told to.""" + if self._engine is not None: + with orm.Session(bind=self._engine) as session: + row = session.get(bts.Secret, (user_id, secret_name)) + self.row_version_at_delete.append( + row.extra_data["storage"]["details"]["version"] if row else None + ) + if self._fail_delete: + raise RuntimeError("store unavailable") + self.retired.append(storage_details["version"]) + + +def _engine(*, tmp_path: pathlib.Path) -> sqlalchemy.Engine: + """Returns a file database, so a second session sees only committed rows.""" + return database_ops.create_db_engine_and_migrate_db( + database_uri=f"sqlite:///{tmp_path / 'db.sqlite'}" + ) + + +def _save( + *, + engine: sqlalchemy.Engine, + store: _FakeStore | None, + value: str, + update: bool = False, +) -> None: + """Creates or updates alice's TOKEN through the service.""" + service = api_server_sql.SecretsApiService(secret_value_store=store) + method = service.update_secret if update else service.create_secret + with orm.Session(bind=engine) as session: + method( + session=session, user_id="alice", secret_name="TOKEN", secret_value=value + ) + + +def _row(*, engine: sqlalchemy.Engine) -> bts.Secret | None: + """Reads alice's TOKEN as committed.""" + with orm.Session(bind=engine, expire_on_commit=False) as session: + return session.get(bts.Secret, ("alice", "TOKEN")) + + +def test_without_store_keeps_the_value_in_the_row(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=None, value="s3cret") + row = _row(engine=engine) + assert row.secret_value == "s3cret" + assert row.extra_data is None + + +def test_without_store_update_and_delete_work_as_before(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=None, value="s3cret") + _save(engine=engine, store=None, value="n3w", update=True) + row = _row(engine=engine) + assert (row.secret_value, row.extra_data) == ("n3w", None) + with orm.Session(bind=engine) as session: + api_server_sql.SecretsApiService().delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + assert _row(engine=engine) is None + + +def test_with_store_the_row_keeps_only_what_save_returned(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _save(engine=engine, store=store, value="s3cret") + row = _row(engine=engine) + assert row.secret_value == "" + assert row.extra_data == {"storage": {"type": "fake", "details": {"version": "1"}}} + assert store.versions == {"1": "s3cret"} + + +def test_update_retires_the_previous_version_after_the_row_commits( + tmp_path: pathlib.Path, +): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore(engine=engine) + _save(engine=engine, store=store, value="v1") + _save(engine=engine, store=store, value="v2", update=True) + assert _row(engine=engine).extra_data["storage"]["details"] == {"version": "2"} + assert store.retired == ["1"] + # When version 1 was retired, the committed row already named version 2. + assert store.row_version_at_delete == ["2"] + + +def test_delete_retires_the_version_after_the_row_is_gone(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore(engine=engine) + _save(engine=engine, store=store, value="v1") + with orm.Session(bind=engine) as session: + api_server_sql.SecretsApiService(secret_value_store=store).delete_secret( + session=session, user_id="alice", secret_name="TOKEN" + ) + assert _row(engine=engine) is None + assert store.retired == ["1"] + assert store.row_version_at_delete == [None] + + +def test_a_plain_row_moves_into_the_store_on_update(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _save(engine=engine, store=None, value="plain") + _save(engine=engine, store=store, value="v2", update=True) + row = _row(engine=engine) + assert row.secret_value == "" + assert row.extra_data["storage"]["details"] == {"version": "1"} + assert store.retired == [] + + +def test_other_extra_data_keys_survive(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=None, value="plain") + with orm.Session(bind=engine) as session: + session.get(bts.Secret, ("alice", "TOKEN")).extra_data = {"owner_team": "data"} + session.commit() + _save(engine=engine, store=_FakeStore(), value="v2", update=True) + extra_data = _row(engine=engine).extra_data + assert extra_data["owner_team"] == "data" + assert "storage" in extra_data + + +@pytest.mark.parametrize("server_store", [None, _FakeStore(type="other")]) +def test_save_and_delete_refused_when_the_row_is_in_another_store( + tmp_path: pathlib.Path, server_store: _FakeStore | None +): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=_FakeStore(), value="v1") + before = _row(engine=engine) + with pytest.raises(errors.SecretStorageTypeMismatchError, match="'fake' store"): + _save(engine=engine, store=server_store, value="plain", update=True) + with orm.Session(bind=engine) as session: + with pytest.raises(errors.SecretStorageTypeMismatchError): + api_server_sql.SecretsApiService( + secret_value_store=server_store + ).delete_secret(session=session, user_id="alice", secret_name="TOKEN") + after = _row(engine=engine) + assert (after.secret_value, after.extra_data) == ( + before.secret_value, + before.extra_data, + ) + if server_store is not None: + assert server_store.versions == {} + assert server_store.retired == [] + + +def test_create_of_an_existing_name_saves_nothing_in_the_store( + tmp_path: pathlib.Path, +): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + _save(engine=engine, store=store, value="v1") + with pytest.raises(errors.ItemAlreadyExistsError): + _save(engine=engine, store=store, value="v2") + assert store.versions == {"1": "v1"} + + +def test_a_failed_retire_is_logged_and_the_update_stands( + tmp_path: pathlib.Path, caplog: pytest.LogCaptureFixture +): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore(fail_delete=True) + _save(engine=engine, store=store, value="v1") + with caplog.at_level(logging.ERROR): + _save(engine=engine, store=store, value="v2", update=True) + assert _row(engine=engine).extra_data["storage"]["details"] == {"version": "2"} + assert "Could not retire the previous stored value of secret 'TOKEN'" in caplog.text + assert "v1" not in caplog.text and "v2" not in caplog.text + + +def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: + """Returns an API client for alice, with the given store passed to setup_routes.""" + app = fastapi.FastAPI() + api_router.setup_routes( + app=app, + db_engine=engine, + user_details_getter=lambda: api_router.UserDetails( + name="alice", permissions={"read": True, "write": True, "admin": False} + ), + secret_value_store=store, + ) + return TestClient(app) + + +def test_setup_routes_passes_the_store_to_secret_routes(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + store = _FakeStore() + response = _client(engine=engine, store=store).post( + "/api/secrets/", + params={"secret_name": "TOKEN"}, + json={"secret_value": "s3cret"}, + ) + assert response.status_code == 200, response.text + assert _row(engine=engine).secret_value == "" + assert store.versions == {"1": "s3cret"} + + +def test_a_refusal_is_a_503_naming_both_stores(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=_FakeStore(), value="v1") + response = _client(engine=engine, store=None).delete("/api/secrets/TOKEN") + assert response.status_code == 503, response.text + assert "'fake' store" in response.json()["message"] + assert "None" in response.json()["message"] + assert _row(engine=engine) is not None