fix(harness): clean every trace span before it leaves for Observe (TH-9504)

.github/workflows/sdk-smoke.yml

@@ -89,6 +89,7 @@ jobs:
           tests/harness/test_simulation_mark_bootstrap.py
           tests/harness/test_process_preflight.py
           tests/providers
+          tests/harness/test_observability.py
           tests/harness/test_scenario_source_provision_agent.py
           tests/harness/test_framework_detection.py
           tests/harness/test_hosted_entrypoint.py

src/fi/alk/harness/hosted_authoring_entrypoint.py

@@ -71,6 +71,27 @@ def _platform_simulator_values(values: dict[str, str]) -> dict[str, str]:
     }
 
 
+# In secrets.json only names a customer can't declare may be kept: SIMULATOR_* and FI_* are the
+# platform's. A customer may declare GOOGLE_CLOUD_PROJECT, so under that name it is masked there.
+_PLATFORM_ONLY_SETTINGS = frozenset(
+    name
+    for name in observability.NON_SECRET_SETTINGS
+    if name.startswith(("SIMULATOR_", "FI_"))
+)
+
+
+def _span_secret_values(
+    all_values: dict[str, str], gateway_values: dict[str, str]
+) -> tuple[str, ...]:
+    """The values spans replace whole: everything in both files, except the platform settings a
+    span keeps. Each file is read on its own and the tuples joined, so a name in both masks both
+    values."""
+    return (
+        *observability.declared_values(all_values, _PLATFORM_ONLY_SETTINGS),
+        *observability.declared_values(gateway_values),
+    )
+
+
 def _configure_generation_environment(values: dict[str, str]) -> None:
     for name in _PASSTHROUGH:
         if values.get(name):
@@ -194,7 +215,13 @@ def main(argv: list[str] | None = None) -> int:
         os.environ["ALK_FOREGROUND_COORDINATOR"] = "1"
     job_id, run_id, telemetry = _authoring_job_context(forwarded)
     if job_id:
-        observability.begin(job_id, run_id, telemetry)
+        # Spans carry tool output, and a tool can print a key.
+        observability.begin(
+            job_id,
+            run_id,
+            telemetry,
+            secret_values=_span_secret_values(all_values, gateway_values),
+        )
         observability.stage("authoring")
     try:
         from .authoring_runtime_validation import RuntimeValidationError

src/fi/alk/harness/hosted_entrypoint.py

@@ -2272,17 +2272,27 @@ async def run_job(
     )
     scenarios_client.configure_offline_control(events_spool.root / "control")
 
+    # Read once, at boot: the runtime deletes secrets.json on its first provision().
+    customer_secret_values = deps.peek_secret_values()
+    declared_secret_values = (
+        *customer_secret_values,
+        *tuple(simulator_secret_values.values()),
+    )
+    # Spans: the customer's values go whole (no names come with them here). The simulator channel
+    # goes whole too, apart from the platform settings named in NON_SECRET_SETTINGS, which a span
+    # keeps.
+    span_secret_values = (
+        *customer_secret_values,
+        *observability.declared_values(simulator_secret_values),
+    )
     adapter = OutboundAdapter(
         capabilities,
         events_spool=events_spool,
         events_client=events_client,
         results_client=results_client,
         artifacts_client=artifacts_client,
         channel_state=channel_state,
-        extra_secret_values=(
-            *deps.peek_secret_values(),
-            *tuple(simulator_secret_values.values()),
-        ),
+        extra_secret_values=declared_secret_values,
         clock=deps.clock,
         flush_window_seconds=deps.flush_window_seconds,
     )
@@ -2491,8 +2501,12 @@ async def _fail(
         if broker_socket is not None:
             os.environ["ALK_COMPOSE_BROKER_SOCKET"] = str(broker_socket)
 
+        # Spans are a channel out too (span_secret_values says which values they replace).
         observability.begin(
-            job.job_id, job.run_id, (job.metadata or {}).get("telemetry")
+            job.job_id,
+            job.run_id,
+            (job.metadata or {}).get("telemetry"),
+            secret_values=span_secret_values,
         )
 
         if job.seed is None: