Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions svo-adapter-service/app/hasura.py
Original file line number Diff line number Diff line change
Expand Up @@ -320,6 +320,48 @@ async def execute(self, query: str, variables: dict[str, Any] | None = None) ->
# The sync re-inserts a spec's contracts on every run (nested insert under an
# upserted spec appends), so it clears them first. Edges cascade and are
# recomputed after the sync.
# Replace one registry row and its contracts in one request. Hasura runs the root
# fields of a mutation in order inside one transaction, so the row is never seen
# without contracts: a flow checked during a sync no longer finds a piece with no
# inputs or outputs. `$ids` is the row's own id when its contracts are replaced,
# else empty (an `_in` of nothing deletes nothing).
REPLACE_TRANSFORM_SPEC_MUTATION = """
mutation ReplaceMintSyncSpec($ids: [String!]!, $obj: adapter_transform_spec_insert_input!) {
delete_adapter_transform_contract(where: { transform_spec_id: { _in: $ids } }) {
affected_rows
}
insert_adapter_transform_spec_one(
object: $obj
on_conflict: {
constraint: transform_spec_mint_model_config_id_key
update_columns: [
name description transform_type method tapis_app_id app_version
stage env_from_args file_inputs mint_synced_at
]
}
) { id mint_model_config_id }
}
"""

REPLACE_ETL_TRANSFORM_SPEC_MUTATION = """
mutation ReplaceMintETLSpec($ids: [String!]!, $obj: adapter_transform_spec_insert_input!) {
delete_adapter_transform_contract(where: { transform_spec_id: { _in: $ids } }) {
affected_rows
}
insert_adapter_transform_spec_one(
object: $obj
on_conflict: {
constraint: transform_spec_pkey
update_columns: [
name description transform_type method tapis_app_id app_version
stage parameters_schema_json env_from_args file_inputs mint_synced_at
owner_username visibility mint_etl_process_id
]
}
) { id }
}
"""

DELETE_SPEC_CONTRACTS_MUTATION = """
mutation DeleteSpecContracts($ids: [String!]!) {
delete_adapter_transform_contract(where: { transform_spec_id: { _in: $ids } }) {
Expand Down
36 changes: 15 additions & 21 deletions svo-adapter-service/app/mint_sync.py
Original file line number Diff line number Diff line change
Expand Up @@ -440,8 +440,7 @@ async def sync_one_etl_process(hasura_client: Any, process: dict[str, Any]) -> d
when the process cannot be a registry row. ``hasura_client`` is the adapter's
own (admin) client: it writes the adapter schema. An approval already
recorded for the piece is kept; it still has to match the piece to count."""
from .hasura import (
DELETE_SPEC_CONTRACTS_MUTATION, GET_APPROVAL_SPEC_QUERY, UPSERT_ETL_TRANSFORM_SPEC_MUTATION)
from .hasura import GET_APPROVAL_SPEC_QUERY, REPLACE_ETL_TRANSFORM_SPEC_MUTATION

row = etl_process_to_spec_row(process, [])
if row is None:
Expand All @@ -453,11 +452,12 @@ async def sync_one_etl_process(hasura_client: Any, process: dict[str, Any]) -> d
if existing and existing.get("mint_etl_process_id") != row["id"]:
raise SpecIdCollision(f"a registry spec with id {row['id']} already exists and is not this piece")
contracts = row.pop("contracts", None)
await hasura_client.execute(DELETE_SPEC_CONTRACTS_MUTATION, {"ids": [row["id"]]})
row["mint_synced_at"] = "now()"
if contracts:
row["contracts"] = contracts
await hasura_client.execute(UPSERT_ETL_TRANSFORM_SPEC_MUTATION, {"obj": row})
# The old contracts go in the same transaction as the upsert (see
# REPLACE_ETL_TRANSFORM_SPEC_MUTATION), so the piece is never seen without them.
await hasura_client.execute(REPLACE_ETL_TRANSFORM_SPEC_MUTATION, {"ids": [row["id"]], "obj": row})
return row


Expand All @@ -477,12 +477,11 @@ async def sync_mint_to_adapter(
5. Return a SyncResult with counts + warnings
"""
from .hasura import (
UPSERT_TRANSFORM_SPEC_MUTATION,
UPSERT_ETL_TRANSFORM_SPEC_MUTATION,
REPLACE_TRANSFORM_SPEC_MUTATION,
REPLACE_ETL_TRANSFORM_SPEC_MUTATION,
DELETE_OBSOLETE_MINT_SPECS_MUTATION,
DELETE_OBSOLETE_ETL_SPECS_MUTATION,
GET_MINT_SPEC_IDS_QUERY,
DELETE_SPEC_CONTRACTS_MUTATION,
)

if mint_client is None:
Expand Down Expand Up @@ -544,27 +543,22 @@ async def sync_mint_to_adapter(
}
existing_ids = set(spec_id_by_config)

# Replace, don't append: clear the contracts of every spec this sync is
# about to re-insert them for.
replace_ids = sorted({
str(row["id"]) if "mint_model_config_id" not in row
else str(spec_id_by_config.get(row["mint_model_config_id"]) or "")
for row in rows_to_upsert if row.get("contracts")
} - {""})
if replace_ids:
await hasura_client.execute(DELETE_SPEC_CONTRACTS_MUTATION, {"ids": replace_ids})

for row in rows_to_upsert:
contracts = row.pop("contracts", None)
is_etl = "mint_model_config_id" not in row
is_update = row.get("mint_model_config_id") in existing_ids if not is_etl else False
# Upsert on mint_model_config_id: if the row exists, update all fields;
# contracts are deleted and re-inserted via the nested mutation.
# Replace, don't append: a row whose contracts are re-inserted has its old
# ones deleted in the same transaction as the upsert, so no reader ever
# sees the row without contracts. A row synced without contracts keeps
# the ones it has.
spec_id = (str(row["id"]) if is_etl
else str(spec_id_by_config.get(row["mint_model_config_id"]) or ""))
replace_ids = [spec_id] if contracts and spec_id else []
row["mint_synced_at"] = "now()"
if contracts:
row["contracts"] = contracts
mutation = UPSERT_ETL_TRANSFORM_SPEC_MUTATION if is_etl else UPSERT_TRANSFORM_SPEC_MUTATION
await hasura_client.execute(mutation, {"obj": row})
mutation = REPLACE_ETL_TRANSFORM_SPEC_MUTATION if is_etl else REPLACE_TRANSFORM_SPEC_MUTATION
await hasura_client.execute(mutation, {"ids": replace_ids, "obj": row})
if is_update:
result.updated += 1
else:
Expand Down
7 changes: 7 additions & 0 deletions svo-adapter-service/app/store.py
Original file line number Diff line number Diff line change
Expand Up @@ -437,6 +437,13 @@ def _op_UpsertMintETLSpec(self, v):
def _op_DeleteSpecContracts(self, v):
return {"delete_adapter_transform_contract": {"affected_rows": 0}}

# The replace mutations: delete the row's contracts, then upsert it, as one.
def _op_ReplaceMintSyncSpec(self, v):
return {**self._op_DeleteSpecContracts(v), **self._op_UpsertMintSyncSpec(v)}

def _op_ReplaceMintETLSpec(self, v):
return {**self._op_DeleteSpecContracts(v), **self._op_UpsertMintETLSpec(v)}

def _op_GetMintSpecIds(self, v):
mint_rows = [
{"mint_model_config_id": r["mint_model_config_id"]}
Expand Down
2 changes: 1 addition & 1 deletion svo-adapter-service/tests/test_etl_piece_ownership.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def test_built_in_process_stays_public_and_ownerless():
def test_sync_upserts_ownership_and_never_writes_approval():
fake = _FakeHasura([_process()])
asyncio.run(sync_mint_to_adapter(fake))
(upsert,) = fake.variables_of("UpsertMintETLSpec")
(upsert,) = fake.variables_of("ReplaceMintETLSpec")
assert upsert["obj"]["owner_username"] == "alice"
assert upsert["obj"]["visibility"] == "private"
assert "approved_source_sha256" not in upsert["obj"]
Expand Down
17 changes: 10 additions & 7 deletions svo-adapter-service/tests/test_etl_sync_roundtrip.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from app.hasura import DELETE_SPEC_CONTRACTS_MUTATION # noqa: E402
from app.hasura import DELETE_SPEC_CONTRACTS_MUTATION, REPLACE_ETL_TRANSFORM_SPEC_MUTATION # noqa: E402
from app.mint_sync import etl_process_to_spec_row, sync_mint_to_adapter # noqa: E402
from app.planner import build_plan_json # noqa: E402

Expand Down Expand Up @@ -95,13 +95,16 @@ async def execute(self, query: str, variables: dict | None = None) -> dict:
return {"insert_adapter_transform_spec_one": {"id": "x"}}


def test_resync_clears_contracts_before_reinserting():
def test_resync_replaces_contracts_in_the_same_request_as_the_upsert():
# One request per piece deletes its contracts and upserts it, which Hasura runs
# as one transaction: a flow checked mid-sync never sees a piece without them.
aoi = next(s for s in _hosted_specs() if s.get("id") == "ts-aoi-from-boundary")
fake = _FakeHasura([_as_catalog_row(register.to_object(aoi))], existing=[])
asyncio.run(sync_mint_to_adapter(fake, dry_run=False))

kinds = [q for q, _ in fake.calls]
delete_at = next(i for i, q in enumerate(kinds) if q == DELETE_SPEC_CONTRACTS_MUTATION)
upsert_at = next(i for i, q in enumerate(kinds) if "UpsertMintETLSpec" in q)
assert delete_at < upsert_at
assert fake.calls[delete_at][1] == {"ids": ["ts-aoi-from-boundary"]}
assert not any(q == DELETE_SPEC_CONTRACTS_MUTATION for q, _ in fake.calls)
(query, variables), = [(q, v) for q, v in fake.calls if "ReplaceMintETLSpec" in q]
assert query == REPLACE_ETL_TRANSFORM_SPEC_MUTATION
assert query.index("delete_adapter_transform_contract") < query.index("insert_adapter_transform_spec_one")
assert variables["ids"] == ["ts-aoi-from-boundary"]
assert variables["obj"]["contracts"]
Loading