From b9714a9e799f0bfb887eb62bd54bf3f54d63d7f4 Mon Sep 17 00:00:00 2001 From: Will Mobley Date: Fri, 20 Nov 2026 15:40:10 -0600 Subject: [PATCH] Replace a piece's contracts in the same transaction as its sync upsert The catalog sync deleted the contracts of every piece first, then upserted the pieces one request at a time. For the ~30 s that took, a flow check found pieces with no inputs or outputs and refused runs with FLOW_WIRE_TARGET, FLOW_WIRE_SOURCE and FLOW_OUTPUT. Each piece's delete and upsert are now one mutation, which Hasura runs as one transaction. --- svo-adapter-service/app/hasura.py | 42 +++++++++++++++++++ svo-adapter-service/app/mint_sync.py | 36 +++++++--------- svo-adapter-service/app/store.py | 7 ++++ .../tests/test_etl_piece_ownership.py | 2 +- .../tests/test_etl_sync_roundtrip.py | 17 ++++---- 5 files changed, 75 insertions(+), 29 deletions(-) diff --git a/svo-adapter-service/app/hasura.py b/svo-adapter-service/app/hasura.py index e6da38c9..943e6e61 100755 --- a/svo-adapter-service/app/hasura.py +++ b/svo-adapter-service/app/hasura.py @@ -320,6 +320,48 @@ async def execute(self, query: str, variables: dict[str, Any] | None = None) -> # The sync re-inserts a spec's contracts on every run (nested insert under an # upserted spec appends), so it clears them first. Edges cascade and are # recomputed after the sync. +# Replace one registry row and its contracts in one request. Hasura runs the root +# fields of a mutation in order inside one transaction, so the row is never seen +# without contracts: a flow checked during a sync no longer finds a piece with no +# inputs or outputs. `$ids` is the row's own id when its contracts are replaced, +# else empty (an `_in` of nothing deletes nothing). +REPLACE_TRANSFORM_SPEC_MUTATION = """ +mutation ReplaceMintSyncSpec($ids: [String!]!, $obj: adapter_transform_spec_insert_input!) { + delete_adapter_transform_contract(where: { transform_spec_id: { _in: $ids } }) { + affected_rows + } + insert_adapter_transform_spec_one( + object: $obj + on_conflict: { + constraint: transform_spec_mint_model_config_id_key + update_columns: [ + name description transform_type method tapis_app_id app_version + stage env_from_args file_inputs mint_synced_at + ] + } + ) { id mint_model_config_id } +} +""" + +REPLACE_ETL_TRANSFORM_SPEC_MUTATION = """ +mutation ReplaceMintETLSpec($ids: [String!]!, $obj: adapter_transform_spec_insert_input!) { + delete_adapter_transform_contract(where: { transform_spec_id: { _in: $ids } }) { + affected_rows + } + insert_adapter_transform_spec_one( + object: $obj + on_conflict: { + constraint: transform_spec_pkey + update_columns: [ + name description transform_type method tapis_app_id app_version + stage parameters_schema_json env_from_args file_inputs mint_synced_at + owner_username visibility mint_etl_process_id + ] + } + ) { id } +} +""" + DELETE_SPEC_CONTRACTS_MUTATION = """ mutation DeleteSpecContracts($ids: [String!]!) { delete_adapter_transform_contract(where: { transform_spec_id: { _in: $ids } }) { diff --git a/svo-adapter-service/app/mint_sync.py b/svo-adapter-service/app/mint_sync.py index 48d30cf3..48c6ecf1 100755 --- a/svo-adapter-service/app/mint_sync.py +++ b/svo-adapter-service/app/mint_sync.py @@ -440,8 +440,7 @@ async def sync_one_etl_process(hasura_client: Any, process: dict[str, Any]) -> d when the process cannot be a registry row. ``hasura_client`` is the adapter's own (admin) client: it writes the adapter schema. An approval already recorded for the piece is kept; it still has to match the piece to count.""" - from .hasura import ( - DELETE_SPEC_CONTRACTS_MUTATION, GET_APPROVAL_SPEC_QUERY, UPSERT_ETL_TRANSFORM_SPEC_MUTATION) + from .hasura import GET_APPROVAL_SPEC_QUERY, REPLACE_ETL_TRANSFORM_SPEC_MUTATION row = etl_process_to_spec_row(process, []) if row is None: @@ -453,11 +452,12 @@ async def sync_one_etl_process(hasura_client: Any, process: dict[str, Any]) -> d if existing and existing.get("mint_etl_process_id") != row["id"]: raise SpecIdCollision(f"a registry spec with id {row['id']} already exists and is not this piece") contracts = row.pop("contracts", None) - await hasura_client.execute(DELETE_SPEC_CONTRACTS_MUTATION, {"ids": [row["id"]]}) row["mint_synced_at"] = "now()" if contracts: row["contracts"] = contracts - await hasura_client.execute(UPSERT_ETL_TRANSFORM_SPEC_MUTATION, {"obj": row}) + # The old contracts go in the same transaction as the upsert (see + # REPLACE_ETL_TRANSFORM_SPEC_MUTATION), so the piece is never seen without them. + await hasura_client.execute(REPLACE_ETL_TRANSFORM_SPEC_MUTATION, {"ids": [row["id"]], "obj": row}) return row @@ -477,12 +477,11 @@ async def sync_mint_to_adapter( 5. Return a SyncResult with counts + warnings """ from .hasura import ( - UPSERT_TRANSFORM_SPEC_MUTATION, - UPSERT_ETL_TRANSFORM_SPEC_MUTATION, + REPLACE_TRANSFORM_SPEC_MUTATION, + REPLACE_ETL_TRANSFORM_SPEC_MUTATION, DELETE_OBSOLETE_MINT_SPECS_MUTATION, DELETE_OBSOLETE_ETL_SPECS_MUTATION, GET_MINT_SPEC_IDS_QUERY, - DELETE_SPEC_CONTRACTS_MUTATION, ) if mint_client is None: @@ -544,27 +543,22 @@ async def sync_mint_to_adapter( } existing_ids = set(spec_id_by_config) - # Replace, don't append: clear the contracts of every spec this sync is - # about to re-insert them for. - replace_ids = sorted({ - str(row["id"]) if "mint_model_config_id" not in row - else str(spec_id_by_config.get(row["mint_model_config_id"]) or "") - for row in rows_to_upsert if row.get("contracts") - } - {""}) - if replace_ids: - await hasura_client.execute(DELETE_SPEC_CONTRACTS_MUTATION, {"ids": replace_ids}) - for row in rows_to_upsert: contracts = row.pop("contracts", None) is_etl = "mint_model_config_id" not in row is_update = row.get("mint_model_config_id") in existing_ids if not is_etl else False - # Upsert on mint_model_config_id: if the row exists, update all fields; - # contracts are deleted and re-inserted via the nested mutation. + # Replace, don't append: a row whose contracts are re-inserted has its old + # ones deleted in the same transaction as the upsert, so no reader ever + # sees the row without contracts. A row synced without contracts keeps + # the ones it has. + spec_id = (str(row["id"]) if is_etl + else str(spec_id_by_config.get(row["mint_model_config_id"]) or "")) + replace_ids = [spec_id] if contracts and spec_id else [] row["mint_synced_at"] = "now()" if contracts: row["contracts"] = contracts - mutation = UPSERT_ETL_TRANSFORM_SPEC_MUTATION if is_etl else UPSERT_TRANSFORM_SPEC_MUTATION - await hasura_client.execute(mutation, {"obj": row}) + mutation = REPLACE_ETL_TRANSFORM_SPEC_MUTATION if is_etl else REPLACE_TRANSFORM_SPEC_MUTATION + await hasura_client.execute(mutation, {"ids": replace_ids, "obj": row}) if is_update: result.updated += 1 else: diff --git a/svo-adapter-service/app/store.py b/svo-adapter-service/app/store.py index 472b35d7..5de955e0 100755 --- a/svo-adapter-service/app/store.py +++ b/svo-adapter-service/app/store.py @@ -437,6 +437,13 @@ def _op_UpsertMintETLSpec(self, v): def _op_DeleteSpecContracts(self, v): return {"delete_adapter_transform_contract": {"affected_rows": 0}} + # The replace mutations: delete the row's contracts, then upsert it, as one. + def _op_ReplaceMintSyncSpec(self, v): + return {**self._op_DeleteSpecContracts(v), **self._op_UpsertMintSyncSpec(v)} + + def _op_ReplaceMintETLSpec(self, v): + return {**self._op_DeleteSpecContracts(v), **self._op_UpsertMintETLSpec(v)} + def _op_GetMintSpecIds(self, v): mint_rows = [ {"mint_model_config_id": r["mint_model_config_id"]} diff --git a/svo-adapter-service/tests/test_etl_piece_ownership.py b/svo-adapter-service/tests/test_etl_piece_ownership.py index 4f400ce8..f56114e7 100644 --- a/svo-adapter-service/tests/test_etl_piece_ownership.py +++ b/svo-adapter-service/tests/test_etl_piece_ownership.py @@ -84,7 +84,7 @@ def test_built_in_process_stays_public_and_ownerless(): def test_sync_upserts_ownership_and_never_writes_approval(): fake = _FakeHasura([_process()]) asyncio.run(sync_mint_to_adapter(fake)) - (upsert,) = fake.variables_of("UpsertMintETLSpec") + (upsert,) = fake.variables_of("ReplaceMintETLSpec") assert upsert["obj"]["owner_username"] == "alice" assert upsert["obj"]["visibility"] == "private" assert "approved_source_sha256" not in upsert["obj"] diff --git a/svo-adapter-service/tests/test_etl_sync_roundtrip.py b/svo-adapter-service/tests/test_etl_sync_roundtrip.py index 5f8180e5..b10bc96b 100644 --- a/svo-adapter-service/tests/test_etl_sync_roundtrip.py +++ b/svo-adapter-service/tests/test_etl_sync_roundtrip.py @@ -16,7 +16,7 @@ ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) -from app.hasura import DELETE_SPEC_CONTRACTS_MUTATION # noqa: E402 +from app.hasura import DELETE_SPEC_CONTRACTS_MUTATION, REPLACE_ETL_TRANSFORM_SPEC_MUTATION # noqa: E402 from app.mint_sync import etl_process_to_spec_row, sync_mint_to_adapter # noqa: E402 from app.planner import build_plan_json # noqa: E402 @@ -95,13 +95,16 @@ async def execute(self, query: str, variables: dict | None = None) -> dict: return {"insert_adapter_transform_spec_one": {"id": "x"}} -def test_resync_clears_contracts_before_reinserting(): +def test_resync_replaces_contracts_in_the_same_request_as_the_upsert(): + # One request per piece deletes its contracts and upserts it, which Hasura runs + # as one transaction: a flow checked mid-sync never sees a piece without them. aoi = next(s for s in _hosted_specs() if s.get("id") == "ts-aoi-from-boundary") fake = _FakeHasura([_as_catalog_row(register.to_object(aoi))], existing=[]) asyncio.run(sync_mint_to_adapter(fake, dry_run=False)) - kinds = [q for q, _ in fake.calls] - delete_at = next(i for i, q in enumerate(kinds) if q == DELETE_SPEC_CONTRACTS_MUTATION) - upsert_at = next(i for i, q in enumerate(kinds) if "UpsertMintETLSpec" in q) - assert delete_at < upsert_at - assert fake.calls[delete_at][1] == {"ids": ["ts-aoi-from-boundary"]} + assert not any(q == DELETE_SPEC_CONTRACTS_MUTATION for q, _ in fake.calls) + (query, variables), = [(q, v) for q, v in fake.calls if "ReplaceMintETLSpec" in q] + assert query == REPLACE_ETL_TRANSFORM_SPEC_MUTATION + assert query.index("delete_adapter_transform_contract") < query.index("insert_adapter_transform_spec_one") + assert variables["ids"] == ["ts-aoi-from-boundary"] + assert variables["obj"]["contracts"]