diff --git a/cloud_pipelines_backend/api_router.py b/cloud_pipelines_backend/api_router.py index 5614afb..22c24ef 100644 --- a/cloud_pipelines_backend/api_router.py +++ b/cloud_pipelines_backend/api_router.py @@ -634,6 +634,22 @@ def admin_set_execution_node_status( execution_node = session.get(backend_types_sql.ExecutionNode, id) execution_node.container_execution_status = status + @router.post( + "/api/admin/secrets/backfill_storage", + tags=["admin"], + dependencies=[ensure_admin_user_dependency], + **default_config, + ) + def admin_backfill_secret_storage( + session: typing.Annotated[orm.Session, fastapi.Depends(get_session)], + limit: typing.Annotated[int, fastapi.Query(ge=1, le=1000)] = 50, + user_id: str | None = None, + secret_name: str | None = None, + ) -> dict[str, int]: + return secrets_service.backfill_secret_storage( + session=session, limit=limit, user_id=user_id, secret_name=secret_name + ) + @router.get("/api/admin/sql_engine_connection_pool_status") async def get_sql_engine_connection_pool_status( session: typing.Annotated[orm.Session, fastapi.Depends(get_session)], diff --git a/cloud_pipelines_backend/api_server_sql.py b/cloud_pipelines_backend/api_server_sql.py index c48aba5..48ac65e 100644 --- a/cloud_pipelines_backend/api_server_sql.py +++ b/cloud_pipelines_backend/api_server_sql.py @@ -1364,6 +1364,61 @@ def _create_or_update_secret( ) return response + def backfill_secret_storage( + self, + *, + session: orm.Session, + limit: int, + user_id: str | None = None, + secret_name: str | None = None, + ) -> dict[str, int]: + """Moves up to `limit` plain-text secrets into the store, committing each row, and reports how many are left. + + `user_id` and `secret_name`, when given, move only matching rows; both together pick one secret. + """ + if not self._secret_value_store: + raise errors.ApiValidationError("No secret value store is configured.") + is_plaintext = sql.func.json_extract( + bts.Secret.extra_data, f"$.{secret_value_store.EXTRA_DATA_STORAGE_KEY}" + ).is_(None) + secrets_for_backfill = session.execute( + sql.select(bts.Secret.user_id, bts.Secret.secret_name) + .where(is_plaintext) + .filter_by( + **({"user_id": user_id} if user_id is not None else {}), + **({"secret_name": secret_name} if secret_name is not None else {}), + ) + .order_by(bts.Secret.user_id, bts.Secret.secret_name) + .limit(limit) + ).all() + moved_to_store = 0 + for user_id, secret_name in secrets_for_backfill: + # Re-read under a row lock: an update or another backfill may have moved or deleted it since. + secret = session.scalars( + sql.select(bts.Secret) + .where( + bts.Secret.user_id == user_id, + bts.Secret.secret_name == secret_name, + is_plaintext, + ) + .with_for_update() + ).one_or_none() + if secret is None: + continue + secret_value_store.store_secret_value( + secret=secret, value=secret.secret_value, store=self._secret_value_store + ) + # Each value is already saved in the store, so its row commits before the next is touched. + session.commit() + moved_to_store += 1 + count = sql.select(sql.func.count()).select_from(bts.Secret) + return { + "moved_to_store": moved_to_store, + "already_stored": session.scalar(count.where(~is_plaintext)) + - moved_to_store, + "still_plaintext": session.scalar(count.where(is_plaintext)), + } + def delete_secret( self, *, diff --git a/tests/test_secret_value_store.py b/tests/test_secret_value_store.py index 2c3503d..8ba2638 100644 --- a/tests/test_secret_value_store.py +++ b/tests/test_secret_value_store.py @@ -29,6 +29,7 @@ def __init__( engine: sqlalchemy.Engine | None = None, fail_delete: bool = False, fail_get: bool = False, + fail_save_value: str | None = None, ) -> None: self.type = type self.versions: dict[str, str] = {} @@ -38,11 +39,14 @@ def __init__( self.row_version_at_delete: list[str | None] = [] self._fail_delete = fail_delete self._fail_get = fail_get + self._fail_save_value = fail_save_value def save( self, *, user_id: str, secret_name: str, value: str ) -> tuple[str, dict[str, Any]]: - """Saves the value as the next version.""" + """Saves the value as the next version, or fails for `fail_save_value`.""" + if value == self._fail_save_value: + raise RuntimeError("store unavailable") version = str(len(self.versions) + 1) self.versions[version] = value return "", {"version": version} @@ -227,14 +231,16 @@ def test_a_failed_retire_is_logged_and_the_update_stands( assert "v1" not in caplog.text and "v2" not in caplog.text -def _client(*, engine: sqlalchemy.Engine, store: _FakeStore | None) -> TestClient: +def _client( + *, engine: sqlalchemy.Engine, store: _FakeStore | None, admin: bool = False +) -> TestClient: """Returns an API client for alice, with the given store passed to setup_routes.""" app = fastapi.FastAPI() api_router.setup_routes( app=app, db_engine=engine, user_details_getter=lambda: api_router.UserDetails( - name="alice", permissions={"read": True, "write": True, "admin": False} + name="alice", permissions={"read": True, "write": True, "admin": admin} ), secret_value_store=store, ) @@ -373,3 +379,176 @@ def test_launch_fails_when_the_store_cannot_read(tmp_path: pathlib.Path): ) launch.assert_not_called() assert "store unavailable" in _system_error(execution=execution) + + +def _plain_rows( + *, engine: sqlalchemy.Engine, names: list[str], user_id: str = "alice" +) -> None: + """Creates the owner's secrets as plain-text rows, each valued as its name in lower case.""" + service = api_server_sql.SecretsApiService() + with orm.Session(bind=engine) as session: + for name in names: + service.create_secret( + session=session, + user_id=user_id, + secret_name=name, + secret_value=name.lower(), + ) + + +def _backfill( + *, + engine: sqlalchemy.Engine, + store: _FakeStore | None, + limit: int = 50, + **filters: str, +) -> dict[str, int]: + """Runs one backfill call through the service, narrowed by any `user_id` or `secret_name` given.""" + service = api_server_sql.SecretsApiService(secret_value_store=store) + with orm.Session(bind=engine) as session: + return service.backfill_secret_storage(session=session, limit=limit, **filters) + + +def _rows(*, engine: sqlalchemy.Engine) -> dict[str, tuple[str, Any]]: + """Reads every committed row as name -> (secret_value, extra_data).""" + with orm.Session(bind=engine) as session: + return { + row.secret_name: (row.secret_value, row.extra_data) + for row in session.scalars(sqlalchemy.select(bts.Secret)) + } + + +def _stored(version: str, type: str = "fake") -> tuple[str, Any]: + """Returns how a row looks once its value is in the store.""" + return "", {"storage": {"type": type, "details": {"version": version}}} + + +def test_backfill_moves_plain_rows_into_the_store(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + store = _FakeStore() + result = _backfill(engine=engine, store=store) + assert result == {"moved_to_store": 2, "already_stored": 0, "still_plaintext": 0} + assert _rows(engine=engine) == {"A": _stored("1"), "B": _stored("2")} + assert store.versions == {"1": "a", "2": "b"} + + +def test_backfill_rerun_skips_moved_rows(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + store = _FakeStore() + _backfill(engine=engine, store=store) + result = _backfill(engine=engine, store=store) + assert result == {"moved_to_store": 0, "already_stored": 2, "still_plaintext": 0} + assert store.versions == {"1": "a", "2": "b"} + + +def test_backfill_stops_at_the_limit(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B", "C"]) + store = _FakeStore() + first = _backfill(engine=engine, store=store, limit=2) + assert first == {"moved_to_store": 2, "already_stored": 0, "still_plaintext": 1} + second = _backfill(engine=engine, store=store, limit=2) + assert second == {"moved_to_store": 1, "already_stored": 2, "still_plaintext": 0} + + +def test_a_failed_save_stops_the_backfill_and_keeps_earlier_rows( + tmp_path: pathlib.Path, +): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B", "C"]) + with pytest.raises(RuntimeError, match="store unavailable"): + _backfill(engine=engine, store=_FakeStore(fail_save_value="b")) + assert _rows(engine=engine) == { + "A": _stored("1"), + "B": ("b", None), + "C": ("c", None), + } + + +def test_backfill_leaves_rows_in_another_store_alone(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _save(engine=engine, store=_FakeStore(type="other"), value="s3cret") + _plain_rows(engine=engine, names=["A"]) + result = _backfill(engine=engine, store=_FakeStore()) + assert result == {"moved_to_store": 1, "already_stored": 1, "still_plaintext": 0} + assert _rows(engine=engine) == { + "A": _stored("1"), + "TOKEN": _stored("1", type="other"), + } + + +def test_backfill_without_a_store_is_refused(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A"]) + with pytest.raises(errors.ApiValidationError, match="No secret value store"): + _backfill(engine=engine, store=None) + assert _rows(engine=engine) == {"A": ("a", None)} + + +@pytest.mark.parametrize("admin", [False, True], ids=["user", "admin"]) +def test_the_backfill_route_is_for_admins_only(tmp_path: pathlib.Path, admin: bool): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A"]) + response = _client(engine=engine, store=_FakeStore(), admin=admin).post( + "/api/admin/secrets/backfill_storage", params={"limit": 10} + ) + if admin: + assert response.status_code == 200, response.text + assert response.json() == { + "moved_to_store": 1, + "already_stored": 0, + "still_plaintext": 0, + } + assert _rows(engine=engine) == {"A": _stored("1")} + else: + assert response.status_code == 403, response.text + assert _rows(engine=engine) == {"A": ("a", None)} + + +def _stored_keys(*, engine: sqlalchemy.Engine) -> set[tuple[str, str]]: + """Returns (user_id, secret_name) of every committed row whose value is in a store.""" + with orm.Session(bind=engine) as session: + return { + (row.user_id, row.secret_name) + for row in session.scalars(sqlalchemy.select(bts.Secret)) + if row.extra_data + } + + +@pytest.mark.parametrize( + ("filters", "moved"), + [ + ({"user_id": "alice", "secret_name": "A"}, {("alice", "A")}), + ({"user_id": "alice"}, {("alice", "A"), ("alice", "B")}), + ({"secret_name": "A"}, {("alice", "A"), ("bob", "A")}), + ], + ids=["one-secret", "one-owner", "one-name"], +) +def test_backfill_moves_only_the_matching_rows( + tmp_path: pathlib.Path, filters: dict[str, str], moved: set[tuple[str, str]] +): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + _plain_rows(engine=engine, names=["A"], user_id="bob") + result = _backfill(engine=engine, store=_FakeStore(), **filters) + assert result["moved_to_store"] == len(moved) + assert _stored_keys(engine=engine) == moved + + +def test_the_backfill_route_passes_the_filters(tmp_path: pathlib.Path): + engine = _engine(tmp_path=tmp_path) + _plain_rows(engine=engine, names=["A", "B"]) + _plain_rows(engine=engine, names=["A"], user_id="bob") + response = _client(engine=engine, store=_FakeStore(), admin=True).post( + "/api/admin/secrets/backfill_storage", + params={"user_id": "alice", "secret_name": "A"}, + ) + assert response.status_code == 200, response.text + assert response.json() == { + "moved_to_store": 1, + "already_stored": 0, + "still_plaintext": 2, + } + assert _stored_keys(engine=engine) == {("alice", "A")}