From a094dbf105b9dda6ee980a0f1cbe82bb4c59ec73 Mon Sep 17 00:00:00 2001 From: Maxwell Date: Tue, 27 Oct 2026 20:53:06 +0200 Subject: [PATCH 1/2] feat(platform): add read-only inventory collection contracts --- .github/workflows/ci.yml | 18 + .gitignore | 3 + Makefile | 18 + docs/platform/current-state.md | 2 + docs/platform/live-inventory.md | 115 +++ infra/pulumi/shared/environment.schema.json | 262 +++++ infra/pulumi/shared/inventory.schema.json | 387 +++++++ tests/platform/test_inventory.py | 466 +++++++++ tools/README.md | 15 + tools/platform/inventory.py | 1010 +++++++++++++++++++ 10 files changed, 2296 insertions(+) create mode 100644 Makefile create mode 100644 docs/platform/live-inventory.md create mode 100644 infra/pulumi/shared/environment.schema.json create mode 100644 infra/pulumi/shared/inventory.schema.json create mode 100644 tests/platform/test_inventory.py create mode 100755 tools/platform/inventory.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b66cacc5..7c8b6db0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,6 +29,24 @@ jobs: echo "No compose files found" fi + platform-inventory: + name: Platform inventory validation + needs: [lint] + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v7 + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: '3.15' + - name: Install inventory test dependencies + run: python -m pip install -r tools/requirements.txt pytest==8.4.2 + - name: Validate read-only inventory and safety contracts + run: python -m pytest -q tests/platform/test_inventory.py + trivy-scan: needs: [lint] name: Filesystem scan diff --git a/.gitignore b/.gitignore index 9d387d4e..77ed3b0c 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ __pycache__/ # Doco-CD bootstrap secrets doco-cd/secrets/ + +# Normalized host inventory artifacts remain local and private. +.artifacts/platform/ diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e69e11b7 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +ROOT := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))) +PYTHON ?= python3 +PLATFORM_ARTIFACT_ROOT ?= $(ROOT)/.artifacts/platform + +.PHONY: platform-inventory platform-inventory-validate + +platform-inventory: + @$(PYTHON) "$(ROOT)/tools/platform/inventory.py" collect \ + --environment "$(ENV)" \ + --repo-root "$(ROOT)" \ + --artifact-root "$(PLATFORM_ARTIFACT_ROOT)" \ + $(if $(strip $(DOCKER_CONTEXT)),--context "$(DOCKER_CONTEXT)",) + +platform-inventory-validate: + @$(PYTHON) "$(ROOT)/tools/platform/inventory.py" validate \ + --environment "$(ENV)" \ + --artifact-root "$(PLATFORM_ARTIFACT_ROOT)" \ + $(if $(strip $(INVENTORY_FILE)),--input "$(INVENTORY_FILE)",) diff --git a/docs/platform/current-state.md b/docs/platform/current-state.md index 49f09d8b..924d6644 100644 --- a/docs/platform/current-state.md +++ b/docs/platform/current-state.md @@ -58,6 +58,8 @@ Required environment inputs: verified Hetzner project/location, target architect Raw docker inspect, process environments, APISIX consumer exports and database data can contain secrets. Store them only in restricted evidence storage. Commit sanitized facts and opaque evidence references, never raw dumps. +The sanitized runtime observation and unresolved migration gate are recorded in [live-inventory.md](live-inventory.md). The selected Docker engine is a source-host candidate, not a confirmed production identity; repository facts above remain pinned to the baseline commit. + ## Historical references Closed issues [#475](https://github.com/AniTrend/local-stack/issues/475), [#500](https://github.com/AniTrend/local-stack/issues/500), [#503](https://github.com/AniTrend/local-stack/issues/503), [#504](https://github.com/AniTrend/local-stack/issues/504), [#505](https://github.com/AniTrend/local-stack/issues/505), [#533](https://github.com/AniTrend/local-stack/issues/533), [#534](https://github.com/AniTrend/local-stack/issues/534), [#535](https://github.com/AniTrend/local-stack/issues/535), [#536](https://github.com/AniTrend/local-stack/issues/536) and [#546](https://github.com/AniTrend/local-stack/issues/546) explain earlier Unleash, Swarm, rendering and deployment decisions. Reuse working code and tests; confirm current behaviour before duplicating or reviving old work. diff --git a/docs/platform/live-inventory.md b/docs/platform/live-inventory.md new file mode 100644 index 00000000..6f467b58 --- /dev/null +++ b/docs/platform/live-inventory.md @@ -0,0 +1,115 @@ +# Live inventory snapshot + +**Gate:** G0 blocked. **Docker snapshot:** 2026-10-25T12:00:34Z. **Read-only host probes:** 2026-10-25T12:01:54Z. **Collector run:** `20261025T120034Z-4318e7a6`. + +This is a sanitized snapshot of the selected Docker engine and repository service definitions. The engine is a source-host candidate, not an owner-confirmed production identity. No containers, volumes, networks, DNS records, credentials, or cloud resources were changed. Database probes below were read-only. The private normalized artifact is ignored at `.artifacts/platform/20261025T120034Z-4318e7a6/inventory.json`; it has directory mode `0700` and file mode `0600` and is not committed. No approved external evidence store was configured. + +Repository service facts remain pinned to [`e80a48a8b8d7d4d9732840b618e73a2f4b58a1c3`](https://github.com/AniTrend/local-stack/commit/e80a48a8b8d7d4d9732840b618e73a2f4b58a1c3). The collector ran from documentation commit `d7e0add6242b88d02443fb5ddb8ba584f23149e5` with an uncommitted worktree, so this is not a clean-candidate release record. + +## Host and runtime + +| **Observed** | **Value** | +| --- | --- | +| Docker context | `default`, local Unix endpoint | +| Docker client and server | `29.9.1` | +| Docker host OS and architecture | Pop!_OS 24.04 LTS, `x86_64` | +| Docker engine mode | Swarm inactive, not a Swarm manager | +| Host CPU and memory reported by Docker | 4 CPUs, 16,704,286,720 bytes | +| Container and volume counts | 34 containers total, 28 running, 22 local volumes | +| Repository service matches | 16 of 24 Compose service definitions | +| Other runtime containers | 18 unmatched to this repository; names are intentionally omitted | +| Host systemd timers | 13 observed; unit names are private | +| Tailscale daemon | `tailscaled.service` active; ACLs and device grants not inspected | +| Doco-CD controller | `doco-cd.service` inactive; no Doco-CD container observed on this engine | +| Portainer | Container running; deployment authority was not verified | + +`docker stack ls` is unavailable because this engine is not a Swarm manager. Compose labels identify service/project matches, not the person or controller that performs updates. The source `.doco-cd.yml` points at `dev` and enables reconciliation; that configuration is not evidence of a running controller. Portainer remains a possible controller because the authored service has Docker socket access. + +The runtime query found no repository or selected-engine service named `anitrend-relations`. A backend on another host or outside this Docker context remains unknown. + +## Repository-matched services + +Docker `RepoDigests` are the observed image manifest identities. Resource usage is a single `docker stats --no-stream` sample at observation time, not a capacity test. `unknown` health means Docker reported no health state; it is not a pass. All 16 matched containers reported zero Docker memory and CPU limits. + +All 16 observed images reported `linux/amd64`, matching the engine's `x86_64` architecture. + +| Service | Runtime image | Observed digest | State / health | CPU / memory | +| --- | --- | --- | --- | --- | +| `beszel` | `henrygd/beszel:0.21.0` | `sha256:6b39a8b64f9b15d00a25e7683e31a93c46161182a54f62713525107064e10599` | running / unknown | 0.01% / 24 MiB | +| `beszel-agent` | `henrygd/beszel-agent-intel:0.21.0` | `sha256:4ca051605018b4877d438102bc975f46f8e9318c94a1afbdcb71f12107abe535` | running / unknown | 0.00% / 16 MiB | +| `grafana` | `grafana/grafana-oss:13.0.1` | `sha256:0f86bada30d65ef9d0183b90c1e2682ac92d53d95da8bed322b984ea78a4a73a` | running / unknown | 1.34% / 120 MiB | +| `growthbook` | `growthbook/growthbook:4.4.0` | `sha256:4e65f53c7fc11906f3346ba3b9d9b9f86a973c14e1139290a77f3b6d9a5e79ce` | running / healthy | 1.93% / 903 MiB | +| `growthbook-proxy` | `growthbook/proxy:v1.3.5` | `sha256:b698e9345aaa8ef264aedc71437035b340a37044977fddcc96794a8391a9fbb9` | running / unhealthy | 0.59% / 69 MiB | +| `loki` | `grafana/loki:3.7.2` | `sha256:191d4fdfb7264f16989f0a57f320872620a5a7c2ceeec6229212c4190ec49b86` | exited | not sampled | +| `memcached` | `memcached:1.6.41` | `sha256:f7a252e7ba3fbbe9672c483354c5081d02b780122c3bb97bd311d5662b54d0ad` | running / unknown | 0.02% / 1 MiB | +| `mongo` | `mongo:8.3.4` | `sha256:48a009d2d8007e92d6d7e8baa31713cd11c48c06e827e856240e5a1d319b49d9` | running / unknown | 1.37% / 125 MiB | +| `otel-collector` | `otel/opentelemetry-collector-contrib:0.152.1` | `sha256:fa1f6ea8dabd3042fabf4411eed7fa52f253edd8940140bec89a573e62a24eb7` | exited | not sampled | +| `portainer` | `portainer/portainer-ce:2.43.0` | `sha256:707366c811956fe077135a6633af67e698529612869711cdb24896c892b28feb` | running / unknown | 0.00% / 36 MiB | +| `postgres` | `postgres:18` | `sha256:f7ce845ee6873dd84be93c9828fe0d1fab0f9707dc9ac569694657398b290bce` | running / unknown | 0.00% / 23 MiB | +| `prometheus` | `prom/prometheus:v3.11.3` | `sha256:e4254400b85610324913f0dc4acf92603d9984e7519414c5a12811aa6146acc3` | restarting | not sampled | +| `redis` | `redis:8.6-alpine` | `sha256:d146f83b1e0f02fc27c26a50cee39338c736674c5959db84363e6ae3cd9e02d2` | running / unknown | 0.41% / 13 MiB | +| `tempo` | `grafana/tempo:2.10.5` | `sha256:ee21727732c7a7199cb71c3eee9153bbf23f9b0b87619f0555a0cf21a67f1a33` | exited | not sampled | +| `traefik` | `traefik:v3.7` | `sha256:6b9cbca6fac42ab0075f5437d8dc1685cfd188626d8d515839ea94f8b6271c42` | running / unknown | 0.14% / 94 MiB | +| `website` | `ghcr.io/anitrend/website:0.4.2` | `sha256:4ec9c3d27c0f21b0211895a47052e34ffe5a2cc7a33ab2410e36f57eabc21d92` | running / unknown | 0.00% / 9 MiB | + +The 8 repository service definitions not observed on this engine are `apisix_dashboard`, `apisix_gateway`, `etcd`, `local-stack-deployer`, `doco-cd`, `edge-graphql`, `anitrend-edge`, and `unleash`. This means not observed on the selected engine, not proven absent from every host. + +Several runtime tags differ from the pinned repository snapshot, including GrowthBook, GrowthBook Proxy, MongoDB, Redis, Grafana, Loki, OpenTelemetry Collector, Prometheus, Tempo, Portainer, Memcached, and the Beszel agent image variant. No image upgrade or service repair was performed as part of phase 0. + +## Data and persistence observations + +| Service | Read-only runtime evidence | Storage observation | Migration constraint | +| --- | --- | --- | --- | +| PostgreSQL | `18.4 (Debian 18.4-1.pgdg13+1)`; data directory `/var/lib/postgresql/18/docker` | `postgres-data`, mounted at `/var/lib/postgresql`, measured at 209,700,000 bytes | Keep PostgreSQL 18. The source Compose comment still describes a PG17 layout and declares `/var/lib/postgresql/data`; actual server data directory and mount differ from that comment/source path. No restore was run. | +| MongoDB | `mongod --version` reports `8.3.4`, `x86_64` | `mongo_mongo-data`, measured at 744,800,000 bytes, mounted at `/data`, `/data/configdb`, and `/data/db` | The source snapshot declares only `/data/configdb` and `/data/db`. Runtime topology needs owner review and a tested database-aware restore. | +| Redis | `redis-server` reports `8.6.3`; `CONFIG GET` reports `appendonly=yes`, `dir=/data`, `dbfilename=dump.rdb` | No Docker mount observed for the Redis container | **Inferred:** AOF data is in the container writable layer and may be lost when the container is removed. Do not classify Redis as disposable cache until the writer/data owner confirms. | + +The inspected database values and mount metadata are configuration/runtime facts only. No database rows, keys, credentials, or raw process environments were read or committed. + +## Network and health boundary + +| Service | Host binding observed | +| --- | --- | +| `traefik` | TCP 80 and 443, wildcard IPv4 and IPv6 | +| `mongo` | TCP 27017, wildcard IPv4 and IPv6 | +| `postgres` | TCP 5432, wildcard IPv4 and IPv6 | +| `redis` | TCP 6379, wildcard IPv4 and IPv6 | + +Wildcard bindings establish Docker host-port publication, not internet reachability. No outside-tailnet, Host/SNI, direct-port, or IPv6 probe was run. They are required before any private-ingress gate can pass. The observed unhealthy GrowthBook Proxy, restarting Prometheus, and exited Loki, OTel Collector, and Tempo were left unchanged. + +## Phase 0 decisions and blockers + +The collector records 24 repository service units, 18 unmatched runtime containers, per-service `unresolved` dispositions, and a schema-valid production environment contract with 37 unresolved inputs. Each service record has explicit fields for source identity, disposition, owner, ingress, health, configuration/secret references, dependencies, mounts, database facts, writer/job mapping, external dependencies, backup/restore method, and runtime evidence. The contract schemas are `infra/pulumi/shared/inventory.schema.json` and `infra/pulumi/shared/environment.schema.json`. + +G0 remains blocked. The source host role, runtime source commit, per-service owners and freeze procedures, writer/job schedule, external API providers, database backup/restore paths, APISIX route/consumer/plugin export, Hetzner project/location/server type, DNS provider/zone/records/TTL, private hostnames/certificate strategy, Tailscale grants, Pulumi backend, backup destination/retention/recovery references, RPO/RTO, maintenance window, downtime limit, and rollback thresholds lack owner-backed evidence. The selected engine does not run APISIX, so its live policy export is unavailable here. No service has a final migration disposition. + +The required inventory targets ran with an isolated interpreter installed from `tools/requirements.txt`: + +```sh +make PYTHON=/tmp/omp-platform-inventory-653-testenv/bin/python platform-inventory ENV=legacy +make PYTHON=/tmp/omp-platform-inventory-653-testenv/bin/python platform-inventory-validate ENV=legacy +/tmp/omp-platform-inventory-653-testenv/bin/python -m pytest -q tests/platform/test_inventory.py +``` + +Results: `schema=valid reconciliation=blocked`, `schema=valid gate=blocked`, and the complete inventory test file passed 6 tests. The system `python3` lacks PyYAML and jsonschema; install the documented tool requirements before using Make with its default interpreter. + +Read-only host probes: + +```sh +docker exec --user postgres postgres sh -c 'exec psql --no-psqlrc --tuples-only --no-align --set=ON_ERROR_STOP=1 --username="$POSTGRES_USER" --dbname="${POSTGRES_DB:-$POSTGRES_USER}" --command="BEGIN READ ONLY; SHOW server_version; SHOW data_directory; COMMIT;"' +docker exec mongo mongod --version +docker exec redis redis-server --version +docker exec redis redis-cli --raw CONFIG GET appendonly dir dbfilename +systemctl is-active doco-cd.service +systemctl is-active tailscaled.service +``` + +The PostgreSQL query at 2026-10-25T12:01:54Z returned version 18.4 and `/var/lib/postgresql/18/docker`; MongoDB and Redis returned 8.3.4 and 8.6.3. Redis reported AOF enabled, directory `/data`, and `dump.rdb` filename. The Doco-CD unit was inactive and Tailscale active at that probe time. + +The automated collector uses a synthetic Docker CLI fixture in tests. The selected real Docker engine has active workloads, so no disposable Docker engine was designated for the phase-0 fixture acceptance check. External Host/SNI, direct-port, and IPv6 probes, backup/restore, cloud provisioning, and cutover were not run. This report does not claim those gates passed. + +## Collector publication checks + +The collector draft was checked again before publication on 2026-10-27 without changing the historical observations above. The inventory regression file passed 6 tests under the existing isolated platform Python environment. A new explicit `ENV=lab DOCKER_CONTEXT=default` read-only collection produced run `20261027T183959Z-fcf5c1ad`; schema validation returned `schema=valid gate=blocked`. The private JSON remains ignored. This later lab collection is not a new production inventory, does not resolve G0, and did not mutate the Docker daemon. + +The CI `Platform inventory validation` job runs the synthetic Docker CLI regression suite with read-only repository permissions and no cloud credentials. The real disposable-engine fixture acceptance and owner-backed live reconciliation still remain to be performed. diff --git a/infra/pulumi/shared/environment.schema.json b/infra/pulumi/shared/environment.schema.json new file mode 100644 index 00000000..414bd745 --- /dev/null +++ b/infra/pulumi/shared/environment.schema.json @@ -0,0 +1,262 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:anitrend:platform:environment:v1", + "title": "AniTrend platform environment intent", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "target_environment", "inputs"], + "properties": { + "schema_version": { "const": "anitrend.platform.environment.v1" }, + "target_environment": { "const": "production" }, + "inputs": { + "type": "object", + "additionalProperties": false, + "required": [ + "provider", + "allowed_docker_contexts", + "safety_policy", + "source_host_identity_ref", + "deployment_owner_map_ref", + "deployment_freeze_steps_ref", + "service_dependency_map_ref", + "database_runtime_facts_ref", + "gateway_policy_export_ref", + "external_provider_map_ref", + "backup_restore_matrix_ref", + "hetzner_project_ref", + "region", + "server_type", + "architecture", + "os_image", + "network_cidrs", + "volumes", + "public_routes", + "private_routes", + "dns_provider_ref", + "dns_zone_ref", + "dns_records", + "private_hostnames", + "certificate_strategy_ref", + "tailnet_grants_ref", + "state_backend_ref", + "backup_destination_ref", + "backup_retention_days", + "credential_refs", + "recovery_credential_refs", + "rpo_hours", + "rto_minutes", + "maintenance_window_ref", + "maximum_downtime_minutes", + "rollback_thresholds", + "writer_job_map_ref" + ], + "properties": { + "provider": { "$ref": "#/$defs/providerInput" }, + "allowed_docker_contexts": { "$ref": "#/$defs/stringListInput" }, + "safety_policy": { "$ref": "#/$defs/safetyPolicyInput" }, + "source_host_identity_ref": { "$ref": "#/$defs/stringInput" }, + "deployment_owner_map_ref": { "$ref": "#/$defs/stringInput" }, + "deployment_freeze_steps_ref": { "$ref": "#/$defs/stringInput" }, + "service_dependency_map_ref": { "$ref": "#/$defs/stringInput" }, + "database_runtime_facts_ref": { "$ref": "#/$defs/stringInput" }, + "gateway_policy_export_ref": { "$ref": "#/$defs/stringInput" }, + "external_provider_map_ref": { "$ref": "#/$defs/stringInput" }, + "backup_restore_matrix_ref": { "$ref": "#/$defs/stringInput" }, + "hetzner_project_ref": { "$ref": "#/$defs/stringInput" }, + "region": { "$ref": "#/$defs/stringInput" }, + "server_type": { "$ref": "#/$defs/stringInput" }, + "architecture": { "$ref": "#/$defs/stringInput" }, + "os_image": { "$ref": "#/$defs/stringInput" }, + "network_cidrs": { "$ref": "#/$defs/networkListInput" }, + "volumes": { "$ref": "#/$defs/volumeListInput" }, + "public_routes": { "$ref": "#/$defs/routeListInput" }, + "private_routes": { "$ref": "#/$defs/routeListInput" }, + "dns_provider_ref": { "$ref": "#/$defs/stringInput" }, + "dns_zone_ref": { "$ref": "#/$defs/stringInput" }, + "dns_records": { "$ref": "#/$defs/dnsRecordListInput" }, + "private_hostnames": { "$ref": "#/$defs/stringListInput" }, + "certificate_strategy_ref": { "$ref": "#/$defs/stringInput" }, + "tailnet_grants_ref": { "$ref": "#/$defs/stringInput" }, + "state_backend_ref": { "$ref": "#/$defs/stringInput" }, + "backup_destination_ref": { "$ref": "#/$defs/stringInput" }, + "backup_retention_days": { "$ref": "#/$defs/integerInput" }, + "credential_refs": { "$ref": "#/$defs/credentialListInput" }, + "recovery_credential_refs": { "$ref": "#/$defs/credentialListInput" }, + "rpo_hours": { "$ref": "#/$defs/integerInput" }, + "rto_minutes": { "$ref": "#/$defs/integerInput" }, + "maintenance_window_ref": { "$ref": "#/$defs/stringInput" }, + "maximum_downtime_minutes": { "$ref": "#/$defs/integerInput" }, + "rollback_thresholds": { "$ref": "#/$defs/stringListInput" }, + "writer_job_map_ref": { "$ref": "#/$defs/stringInput" } + } + } + }, + "$defs": { + "inputBase": { + "type": "object", + "additionalProperties": false, + "required": ["status", "value", "evidence_ref"], + "properties": { + "status": { "enum": ["confirmed", "unresolved", "not_applicable"] }, + "value": {}, + "evidence_ref": { "type": ["string", "null"], "minLength": 1 } + }, + "allOf": [ + { + "if": { "properties": { "status": { "const": "unresolved" } }, "required": ["status"] }, + "then": { "properties": { "value": { "type": "null" }, "evidence_ref": { "type": "null" } } } + }, + { + "if": { "properties": { "status": { "const": "confirmed" } }, "required": ["status"] }, + "then": { "properties": { "value": { "not": { "type": "null" } }, "evidence_ref": { "type": "string", "minLength": 1 } } } + }, + { + "if": { "properties": { "status": { "const": "not_applicable" } }, "required": ["status"] }, + "then": { "properties": { "value": { "type": "null" }, "evidence_ref": { "type": "string", "minLength": 1 } } } + } + ] + }, + "stringInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { "properties": { "value": { "type": ["string", "null"], "minLength": 1 } } } + ] + }, + "providerInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { "properties": { "value": { "enum": ["hcloud", null] } } } + ] + }, + "safetyPolicyInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { + "properties": { + "value": { + "type": ["object", "null"], + "additionalProperties": false, + "required": ["protect_resources", "allow_destroy", "require_environment_approval", "deny_pull_request_mutations"], + "properties": { + "protect_resources": { "type": "boolean" }, + "allow_destroy": { "type": "boolean" }, + "require_environment_approval": { "type": "boolean" }, + "deny_pull_request_mutations": { "type": "boolean" } + } + } + } + } + ] + }, + "integerInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { "properties": { "value": { "type": ["integer", "null"], "minimum": 0 } } } + ] + }, + "stringListInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { "properties": { "value": { "type": ["array", "null"], "items": { "type": "string", "minLength": 1 } } } } + ] + }, + "credentialListInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { "properties": { "value": { "type": ["array", "null"], "items": { "type": "string", "pattern": "^[A-Za-z0-9_.:/-]+$" } } } } + ] + }, + "networkListInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { + "properties": { + "value": { + "type": ["array", "null"], + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "cidr", "purpose"], + "properties": { + "name": { "type": "string", "minLength": 1 }, + "cidr": { "type": "string", "minLength": 1 }, + "purpose": { "type": "string", "minLength": 1 } + } + } + } + } + } + ] + }, + "volumeListInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { + "properties": { + "value": { + "type": ["array", "null"], + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id_ref", "size_gb", "mount_path", "data_classification", "protected"], + "properties": { + "id_ref": { "type": "string", "minLength": 1 }, + "size_gb": { "type": "number", "exclusiveMinimum": 0 }, + "mount_path": { "type": "string", "minLength": 1 }, + "data_classification": { "type": "string", "minLength": 1 }, + "protected": { "type": "boolean" } + } + } + } + } + } + ] + }, + "routeListInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { + "properties": { + "value": { + "type": ["array", "null"], + "items": { + "type": "object", + "additionalProperties": false, + "required": ["hostname", "service_ref", "port", "protocol"], + "properties": { + "hostname": { "type": "string", "minLength": 1 }, + "service_ref": { "type": "string", "minLength": 1 }, + "port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "protocol": { "enum": ["http", "https", "tcp", "udp"] } + } + } + } + } + } + ] + }, + "dnsRecordListInput": { + "allOf": [ + { "$ref": "#/$defs/inputBase" }, + { + "properties": { + "value": { + "type": ["array", "null"], + "items": { + "type": "object", + "additionalProperties": false, + "required": ["hostname", "record_type", "target_ref", "ttl_seconds"], + "properties": { + "hostname": { "type": "string", "minLength": 1 }, + "record_type": { "enum": ["A", "AAAA", "CNAME"] }, + "target_ref": { "type": "string", "minLength": 1 }, + "ttl_seconds": { "type": "integer", "minimum": 1 } + } + } + } + } + } + ] + } + } +} diff --git a/infra/pulumi/shared/inventory.schema.json b/infra/pulumi/shared/inventory.schema.json new file mode 100644 index 00000000..d9bfa202 --- /dev/null +++ b/infra/pulumi/shared/inventory.schema.json @@ -0,0 +1,387 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "urn:anitrend:platform:inventory:v1", + "title": "AniTrend platform runtime inventory", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "environment", + "observed_at_utc", + "collector_commit", + "collector_worktree_clean", + "docker", + "repository_services", + "unmatched_containers", + "collection_issues", + "environment_contract", + "reconciliation" + ], + "properties": { + "schema_version": { "const": "anitrend.platform.inventory.v1" }, + "run_id": { "type": "string", "pattern": "^[0-9]{8}T[0-9]{6}Z-[0-9a-f]{8}$" }, + "environment": { "enum": ["lab", "legacy", "rehearsal", "production"] }, + "observed_at_utc": { "type": "string", "format": "date-time" }, + "collector_commit": { "type": "string", "pattern": "^(?:[0-9a-f]{40}|unknown)$" }, + "collector_worktree_clean": { "type": "boolean" }, + "docker": { + "type": "object", + "additionalProperties": false, + "required": [ + "context_name", + "endpoint_kind", + "client_version", + "server_version", + "operating_system", + "architecture", + "swarm_state", + "swarm_manager", + "cpu_count", + "memory_total_bytes", + "container_count", + "running_count", + "resource_usage_status", + "containers", + "volumes", + "volume_inventory_status", + "host_timers" + ], + "properties": { + "context_name": { "type": "string", "minLength": 1 }, + "endpoint_kind": { "enum": ["local_unix", "local_npipe", "remote_ssh", "remote_tcp", "unknown"] }, + "client_version": { "type": "string" }, + "server_version": { "type": "string" }, + "operating_system": { "type": "string" }, + "architecture": { "type": "string" }, + "swarm_state": { "type": "string" }, + "swarm_manager": { "type": "boolean" }, + "cpu_count": { "type": "integer", "minimum": 0 }, + "memory_total_bytes": { "type": "integer", "minimum": 0 }, + "container_count": { "type": "integer", "minimum": 0 }, + "running_count": { "type": "integer", "minimum": 0 }, + "resource_usage_status": { "enum": ["collected", "partial", "unavailable", "not_applicable"] }, + "containers": { + "type": "array", + "items": { "$ref": "#/$defs/container" } + }, + "volumes": { + "type": "array", + "items": { "$ref": "#/$defs/volume" } + }, + "volume_inventory_status": { "enum": ["collected", "unavailable"] }, + "host_timers": { "$ref": "#/$defs/timerInventory" } + } + }, + "repository_services": { + "type": "array", + "items": { "$ref": "#/$defs/repositoryService" } + }, + "unmatched_containers": { + "type": "array", + "items": { "$ref": "#/$defs/unmatchedContainer" } + }, + "collection_issues": { "type": "array", "items": { "type": "string" } }, + "environment_contract": { + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "target_environment", "inputs"], + "properties": { + "schema_version": { "const": "anitrend.platform.environment.v1" }, + "target_environment": { "const": "production" }, + "inputs": { "type": "object" } + } + }, + "reconciliation": { + "type": "object", + "additionalProperties": false, + "required": ["decision", "blocked_reasons"], + "properties": { + "decision": { "enum": ["pass", "fail", "blocked", "skipped"] }, + "blocked_reasons": { "type": "array", "items": { "type": "string" } } + } + } + }, + "$defs": { + "container": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "name", + "image_ref", + "image_id", + "repo_digests", + "image_os", + "image_architecture", + "state", + "health", + "compose_project", + "compose_service", + "source_service_id", + "ports", + "networks", + "mounts", + "resource_limits", + "resource_usage", + "container_layer_size_bytes", + "rootfs_size_bytes", + "restart_policy" + ], + "properties": { + "id": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "name": { "type": "string" }, + "image_ref": { "type": "string" }, + "image_id": { "type": "string", "pattern": "^sha256:[0-9a-f]{64}$" }, + "repo_digests": { "type": "array", "items": { "type": "string" } }, + "image_os": { "type": ["string", "null"] }, + "image_architecture": { "type": ["string", "null"] }, + "state": { "type": "string" }, + "health": { "enum": ["healthy", "unhealthy", "starting", "unknown"] }, + "compose_project": { "type": ["string", "null"] }, + "compose_service": { "type": ["string", "null"] }, + "source_service_id": { "type": ["string", "null"] }, + "ports": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["container_port", "protocol", "host_port", "host_address_scope", "address_family"], + "properties": { + "container_port": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "protocol": { "type": "string" }, + "host_port": { "type": ["integer", "null"], "minimum": 1, "maximum": 65535 }, + "host_address_scope": { "enum": ["wildcard", "loopback", "private", "public", "other", "unknown"] }, + "address_family": { "enum": ["ipv4", "ipv6", "unknown"] } + } + } + }, + "networks": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "aliases"], + "properties": { + "name": { "type": "string" }, + "aliases": { "type": "array", "items": { "type": "string" } } + } + } + }, + "mounts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["type", "name", "source_path", "destination", "read_only", "size_bytes", "size_status"], + "properties": { + "type": { "type": "string" }, + "name": { "type": ["string", "null"] }, + "source_path": { "type": ["string", "null"] }, + "destination": { "type": "string" }, + "read_only": { "type": "boolean" }, + "size_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "size_status": { "enum": ["measured", "unknown"] } + } + } + }, + "resource_limits": { + "type": "object", + "additionalProperties": false, + "required": ["memory_bytes", "nano_cpus"], + "properties": { + "memory_bytes": { "type": "integer", "minimum": 0 }, + "nano_cpus": { "type": "integer", "minimum": 0 } + } + }, + "resource_usage": { + "type": ["object", "null"], + "additionalProperties": false, + "required": [ + "cpu_percent", + "memory_used_bytes", + "memory_limit_bytes", + "memory_percent", + "network_received_bytes", + "network_sent_bytes", + "block_read_bytes", + "block_written_bytes", + "pid_count" + ], + "properties": { + "cpu_percent": { "type": ["number", "null"], "minimum": 0 }, + "memory_used_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "memory_limit_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "memory_percent": { "type": ["number", "null"], "minimum": 0 }, + "network_received_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "network_sent_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "block_read_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "block_written_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "pid_count": { "type": ["integer", "null"], "minimum": 0 } + } + }, + "container_layer_size_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "rootfs_size_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "restart_policy": { "type": "string" } + } + }, + "volume": { + "type": "object", + "additionalProperties": false, + "required": ["name", "driver", "scope", "links", "size_bytes", "size_status"], + "properties": { + "name": { "type": "string" }, + "driver": { "type": "string" }, + "scope": { "type": "string" }, + "links": { "type": ["integer", "null"], "minimum": 0 }, + "size_bytes": { "type": ["integer", "null"], "minimum": 0 }, + "size_status": { "enum": ["measured", "unknown"] } + } + }, + "timerInventory": { + "type": "object", + "additionalProperties": false, + "required": ["status", "units"], + "properties": { + "status": { "enum": ["collected", "not_local_linux", "unavailable"] }, + "units": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["unit", "next", "last", "activates"], + "properties": { + "unit": { "type": "string" }, + "next": { "type": ["string", "null"] }, + "last": { "type": ["string", "null"] }, + "activates": { "type": ["string", "null"] } + } + } + } + } + }, + "repositoryService": { + "type": "object", + "additionalProperties": false, + "required": [ + "id", + "source_repository", + "source_path", + "compose_project", + "service", + "image_ref", + "container_name", + "depends_on", + "declared_mounts", + "configuration_refs", + "restart_policy", + "stack_extension", + "runtime_match_status", + "runtime_match_method", + "runtime_type", + "runtime_container_ids", + "current_deployment_owner", + "target_deployment_owner", + "disposition", + "ingress_class", + "data_classification", + "health_contract", + "database_runtime", + "writer_job_map_ref", + "external_provider_map_ref", + "backup_restore_method_ref", + "evidence_refs" + ], + "properties": { + "id": { "type": "string" }, + "source_repository": { "const": "AniTrend/local-stack" }, + "source_path": { "type": "string" }, + "compose_project": { "type": "string" }, + "service": { "type": "string" }, + "image_ref": { "type": ["string", "null"] }, + "container_name": { "type": ["string", "null"] }, + "depends_on": { "type": "array", "items": { "type": "string" } }, + "declared_mounts": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["type", "source", "target", "read_only"], + "properties": { + "type": { "type": "string" }, + "source": { "type": "string" }, + "target": { "type": "string" }, + "read_only": { "type": "boolean" } + } + } + }, + "configuration_refs": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["kind", "ref"], + "properties": { + "kind": { "enum": ["env_file", "secret", "config"] }, + "ref": { "type": "string" } + } + } + }, + "restart_policy": { "type": ["string", "null"] }, + "stack_extension": { "type": ["string", "null"] }, + "runtime_match_status": { "enum": ["matched", "not_observed", "ambiguous"] }, + "runtime_match_method": { "enum": ["compose-labels", "container-name", "not-observed", "ambiguous"] }, + "runtime_type": { "enum": ["docker-compose", "docker-container", "not_observed", "ambiguous"] }, + "runtime_container_ids": { "type": "array", "items": { "type": "string" } }, + "current_deployment_owner": { "type": "string" }, + "target_deployment_owner": { "type": "string" }, + "disposition": { "enum": ["migrate", "retain", "retire", "absent", "unresolved"] }, + "ingress_class": { "enum": ["public", "private", "internal", "none", "unresolved"] }, + "data_classification": { "type": "string" }, + "health_contract": { + "type": "object", + "additionalProperties": false, + "required": ["status", "source_healthcheck_declared", "runtime_health_states", "evidence_ref"], + "properties": { + "status": { "enum": ["confirmed", "unresolved", "not_applicable"] }, + "source_healthcheck_declared": { "type": "boolean" }, + "runtime_health_states": { + "type": "array", + "items": { "enum": ["healthy", "unhealthy", "starting", "unknown"] } + }, + "evidence_ref": { "type": ["string", "null"] } + } + }, + "database_runtime": { + "type": "object", + "additionalProperties": false, + "required": ["status", "server_version", "topology_ref", "data_paths"], + "properties": { + "status": { "enum": ["confirmed", "unresolved", "not_applicable"] }, + "server_version": { "type": ["string", "null"] }, + "topology_ref": { "type": ["string", "null"] }, + "data_paths": { "type": "array", "items": { "type": "string" } } + } + }, + "writer_job_map_ref": { "type": ["string", "null"] }, + "external_provider_map_ref": { "type": ["string", "null"] }, + "backup_restore_method_ref": { "type": ["string", "null"] }, + "evidence_refs": { "type": "array", "items": { "type": "string" } } + } + }, + "unmatchedContainer": { + "type": "object", + "additionalProperties": false, + "required": ["container_id", "name", "compose_project", "compose_service", "disposition", "deployment_owner", "evidence_ref"], + "properties": { + "container_id": { "type": "string" }, + "name": { "type": "string" }, + "compose_project": { "type": ["string", "null"] }, + "compose_service": { "type": ["string", "null"] }, + "disposition": { "enum": ["migrate", "retain", "retire", "absent", "unresolved"] }, + "deployment_owner": { "type": "string" }, + "evidence_ref": { "type": "string" } + } + } + } +} \ No newline at end of file diff --git a/tests/platform/test_inventory.py b/tests/platform/test_inventory.py new file mode 100644 index 00000000..8254c3c9 --- /dev/null +++ b/tests/platform/test_inventory.py @@ -0,0 +1,466 @@ +from __future__ import annotations + +import json +import os +from pathlib import Path +import subprocess +import sys +import textwrap +from typing import Optional, Tuple + +import pytest + + +REPO_ROOT = Path(__file__).resolve().parents[2] +INVENTORY_CLI = REPO_ROOT / "tools" / "platform" / "inventory.py" +CONTAINER_ID = "a" * 64 +IMAGE_ID = "sha256:" + "b" * 64 +IMAGE_DIGEST = "example/web@sha256:" + "c" * 64 +SECRET_SENTINEL = "fixture-only-not-a-real-secret" + + +def make_toolchain(tmp_path: Path, endpoint: str) -> Tuple[Path, Path]: + bin_dir = tmp_path / "bin" + bin_dir.mkdir(parents=True) + log_path = tmp_path / "commands.jsonl" + docker = bin_dir / "docker" + docker.write_text( + textwrap.dedent( + f"""\ + #!/usr/bin/env python3 + import json + import os + import sys + + args = sys.argv[1:] + if args[:1] == ["--context"]: + args = args[2:] + with open(os.environ["DOCKER_TEST_LOG"], "a", encoding="utf-8") as log: + log.write(json.dumps(args) + "\\n") + + container_id = {CONTAINER_ID!r} + image_id = {IMAGE_ID!r} + if args[:2] == ["context", "show"]: + print("fixture") + elif args[:2] == ["context", "inspect"]: + print(json.dumps({{"Host": os.environ["DOCKER_TEST_ENDPOINT"]}})) + elif args[:1] == ["version"]: + print(json.dumps({{"client_version": "29.9.1", "server_version": "29.9.1"}})) + elif args[:1] == ["info"]: + print(json.dumps({{ + "operating_system": "Ubuntu 24.04 LTS", + "architecture": "x86_64", + "swarm_state": "inactive", + "swarm_manager": False, + "cpu_count": 4, + "memory_total_bytes": 8589934592, + }})) + elif args[:1] == ["ps"]: + if "--quiet" in args: + print(container_id) + else: + print(container_id + "\\tweb\\tUp 20 minutes (unhealthy)") + elif args[:1] == ["inspect"]: + print(json.dumps({{ + "id": container_id, + "name": "/web", + "image_ref": "example/web:1.2.3", + "image_id": image_id, + "state": "running", + "project": "sample", + "service": "web", + "ports": {{ + "80/tcp": [{{"HostIp": "0.0.0.0", "HostPort": "18080"}}], + "443/tcp": [{{"HostIp": "::", "HostPort": "18443"}}], + }}, + "mounts": [{{ + "Type": "volume", + "Name": "fixture-data", + "Source": "/private/docker/volumes/fixture-data/_data", + "Destination": "/data", + "RW": True, + }}], + "networks": {{ + "sample_default": {{ + "Aliases": ["web"], + "IPAddress": "10.2.0.7", + "GlobalIPv6Address": "2001:db8::7", + }} + }}, + "memory_limit_bytes": 0, + "nano_cpus": 0, + "size_rw_bytes": 4096, + "size_rootfs_bytes": 1048576, + }})) + elif args[:2] == ["image", "inspect"]: + print(json.dumps({{ + "id": image_id, + "os": "linux", + "architecture": "amd64", + "repo_digests": [{IMAGE_DIGEST!r}], + }})) + elif args[:1] == ["stats"]: + print(json.dumps({{ + "Container": container_id[:12], + "ID": container_id[:12], + "Name": "web", + "CPUPerc": "1.25%", + "MemUsage": "24MiB / 1GiB", + "MemPerc": "2.34%", + "NetIO": "10kB / 20kB", + "BlockIO": "30kB / 40kB", + "PIDs": "5", + }})) + elif args[:2] == ["system", "df"]: + print(json.dumps({{ + "Volumes": [ + {{"Name": "fixture-data", "Driver": "local", "Scope": "local", "Links": 1, "Size": "744MB", "Mountpoint": "/private/docker/root", "Labels": {{"secret": {SECRET_SENTINEL!r}}}}}, + {{"Name": "fixture-binary", "Driver": "local", "Scope": "local", "Links": 0, "Size": "1.5GiB", "Mountpoint": "/private/docker/root2", "Labels": {{}}}}, + {{"Name": "fixture-empty", "Driver": "local", "Scope": "local", "Links": 0, "Size": "0B", "Mountpoint": "/private/docker/root3", "Labels": {{}}}}, + ] + }})) + else: + print("unexpected Docker command", file=sys.stderr) + sys.exit(64) + """ + ), + encoding="utf-8", + ) + docker.chmod(0o755) + + systemctl = bin_dir / "systemctl" + systemctl.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env python3 + import json + print(json.dumps([ + {"unit": "backup.timer", "next": "2026-10-26 02:00:00 UTC", "last": "2026-10-25 02:00:00 UTC", "activates": "backup.service"} + ])) + """ + ), + encoding="utf-8", + ) + systemctl.chmod(0o755) + return bin_dir, log_path + + +def run_cli( + tmp_path: Path, + *, + environment: Optional[str], + endpoint: str = "unix:///var/run/docker.sock", + repo_root: Optional[Path] = None, + artifact_root: Optional[Path] = None, + command: str = "collect", + input_file: Optional[Path] = None, +) -> subprocess.CompletedProcess: + bin_dir, log_path = make_toolchain(tmp_path, endpoint) + env = os.environ.copy() + env["PATH"] = str(bin_dir) + os.pathsep + env["PATH"] + env["DOCKER_TEST_ENDPOINT"] = endpoint + env["DOCKER_TEST_LOG"] = str(log_path) + args = [sys.executable, str(INVENTORY_CLI), command] + if environment is not None: + args.extend(["--environment", environment]) + if repo_root is not None: + args.extend(["--repo-root", str(repo_root)]) + if artifact_root is not None: + args.extend(["--artifact-root", str(artifact_root)]) + if input_file is not None: + args.extend(["--input", str(input_file)]) + result = subprocess.run(args, text=True, capture_output=True, env=env, check=False) + result.test_log_path = log_path + return result + + +def make_compose_fixture(root: Path) -> None: + service_dir = root / "sample" + service_dir.mkdir(parents=True) + (service_dir / "docker-compose.yml").write_text( + textwrap.dedent( + """\ + services: + web: + image: example/web:1.2.3 + env_file: .env + depends_on: + db: + condition: service_healthy + volumes: + - fixture-data:/data + volumes: + fixture-data: {} + """ + ), + encoding="utf-8", + ) + (service_dir / ".env").write_text( + "UNUSED_CANARY=" + SECRET_SENTINEL + "\n", + encoding="utf-8", + ) + + +def latest_inventory(artifact_root: Path, environment: str) -> Path: + matches = [] + for candidate in artifact_root.glob("*/inventory.json"): + inventory = json.loads(candidate.read_text(encoding="utf-8")) + if inventory["environment"] == environment: + matches.append(candidate) + assert len(matches) == 1 + return matches[0] + + +def test_collect_requires_an_explicit_environment_before_running_docker(tmp_path: Path): + result = run_cli(tmp_path, environment=None) + + assert result.returncode == 2 + assert "--environment" in result.stderr + assert not result.test_log_path.exists() + + +def test_inventory_normalizes_runtime_evidence_without_reading_env_or_network_secrets(tmp_path: Path): + repo_root = tmp_path / "fixture-repository" + repo_root.mkdir() + make_compose_fixture(repo_root) + artifact_root = tmp_path / "artifacts" + + result = run_cli( + tmp_path / "commands", + environment="legacy", + repo_root=repo_root, + artifact_root=artifact_root, + ) + + assert result.returncode == 0, result.stderr + inventory_path = latest_inventory(artifact_root, "legacy") + serialized = inventory_path.read_text(encoding="utf-8") + inventory = json.loads(serialized) + + assert inventory["docker"]["endpoint_kind"] == "local_unix" + assert inventory["docker"]["swarm_state"] == "inactive" + assert inventory["docker"]["resource_usage_status"] == "collected" + assert inventory["docker"]["containers"][0]["health"] == "unhealthy" + assert inventory["docker"]["containers"][0]["source_service_id"] == "sample/docker-compose.yml:web" + service_record = next( + service for service in inventory["repository_services"] + if service["id"] == "sample/docker-compose.yml:web" + ) + assert service_record["source_repository"] == "AniTrend/local-stack" + assert service_record["runtime_type"] == "docker-compose" + assert service_record["ingress_class"] == "unresolved" + assert service_record["configuration_refs"] == [{"kind": "env_file", "ref": ".env"}] + assert service_record["health_contract"] == { + "status": "unresolved", + "source_healthcheck_declared": False, + "runtime_health_states": ["unhealthy"], + "evidence_ref": "docker-container:" + CONTAINER_ID, + } + assert service_record["database_runtime"] == { + "status": "unresolved", + "server_version": None, + "topology_ref": None, + "data_paths": [], + } + assert service_record["writer_job_map_ref"] is None + assert service_record["external_provider_map_ref"] is None + assert service_record["backup_restore_method_ref"] is None + assert "source:sample/docker-compose.yml" in service_record["evidence_refs"] + assert "docker-container:" + CONTAINER_ID in service_record["evidence_refs"] + assert inventory["docker"]["containers"][0]["ports"] == [ + { + "container_port": 80, + "protocol": "tcp", + "host_port": 18080, + "host_address_scope": "wildcard", + "address_family": "ipv4", + }, + { + "container_port": 443, + "protocol": "tcp", + "host_port": 18443, + "host_address_scope": "wildcard", + "address_family": "ipv6", + }, + ] + container = inventory["docker"]["containers"][0] + assert container["mounts"][0]["size_bytes"] == 744_000_000 + volumes = {volume["name"]: volume for volume in inventory["docker"]["volumes"]} + assert volumes["fixture-binary"]["size_bytes"] == 1_610_612_736 + assert volumes["fixture-empty"]["size_bytes"] == 0 + assert container["resource_usage"]["cpu_percent"] == 1.25 + assert container["resource_usage"]["memory_used_bytes"] == 24 * 1024 * 1024 + assert container["repo_digests"] == [IMAGE_DIGEST] + assert inventory["docker"]["host_timers"]["units"][0]["unit"] == "backup.timer" + assert inventory["reconciliation"]["decision"] == "blocked" + assert SECRET_SENTINEL not in serialized + assert "10.2.0.7" not in serialized + assert "2001:db8::7" not in serialized + assert "/private/docker/root" not in serialized + + validation = run_cli( + tmp_path / "validation", + environment="legacy", + artifact_root=artifact_root, + command="validate", + input_file=inventory_path, + ) + assert validation.returncode == 0, validation.stderr + assert "schema=valid" in validation.stdout + assert "gate=blocked" in validation.stdout + + +def test_lab_inventory_refuses_remote_docker_before_enumerating_containers(tmp_path: Path): + result = run_cli( + tmp_path, + environment="lab", + endpoint="ssh://ops:private@example.invalid", + repo_root=tmp_path, + artifact_root=tmp_path / "artifacts", + ) + + assert result.returncode != 0 + assert "local Docker endpoint" in result.stderr + assert "ops:private" not in result.stderr + assert not (tmp_path / "artifacts").exists() + commands = [json.loads(line) for line in result.test_log_path.read_text(encoding="utf-8").splitlines()] + assert all(command[0] == "context" for command in commands) + + +def test_validator_rejects_unapproved_raw_container_fields_without_echoing_values(tmp_path: Path): + repo_root = tmp_path / "fixture-repository" + repo_root.mkdir() + make_compose_fixture(repo_root) + artifact_root = tmp_path / "artifacts" + result = run_cli( + tmp_path / "collection", + environment="legacy", + repo_root=repo_root, + artifact_root=artifact_root, + ) + assert result.returncode == 0, result.stderr + inventory_path = latest_inventory(artifact_root, "legacy") + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + inventory["docker"]["containers"][0]["environment"] = {"PRIVATE_TOKEN": SECRET_SENTINEL} + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + + validation = run_cli( + tmp_path / "validation", + environment="legacy", + artifact_root=artifact_root, + command="validate", + input_file=inventory_path, + ) + + assert validation.returncode != 0 + assert "additional properties" in validation.stderr.lower() + assert SECRET_SENTINEL not in validation.stderr + + +def test_validator_rejects_pass_when_required_inventory_inputs_are_unresolved(tmp_path: Path): + repo_root = tmp_path / "fixture-repository" + repo_root.mkdir() + make_compose_fixture(repo_root) + artifact_root = tmp_path / "artifacts" + collection = run_cli( + tmp_path / "collection", + environment="legacy", + repo_root=repo_root, + artifact_root=artifact_root, + ) + assert collection.returncode == 0, collection.stderr + inventory_path = latest_inventory(artifact_root, "legacy") + inventory = json.loads(inventory_path.read_text(encoding="utf-8")) + inventory["reconciliation"]["decision"] = "pass" + inventory["reconciliation"]["blocked_reasons"] = [] + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + + validation = run_cli( + tmp_path / "validation", + environment="legacy", + artifact_root=artifact_root, + command="validate", + input_file=inventory_path, + ) + + assert validation.returncode != 0 + assert "cannot pass" in validation.stderr.lower() + + +def test_inventory_contains_a_typed_non_secret_environment_contract(tmp_path: Path): + repo_root = tmp_path / "fixture-repository" + repo_root.mkdir() + make_compose_fixture(repo_root) + artifact_root = tmp_path / "artifacts" + result = run_cli( + tmp_path / "collection", + environment="legacy", + repo_root=repo_root, + artifact_root=artifact_root, + ) + + assert result.returncode == 0, result.stderr + inventory = json.loads(latest_inventory(artifact_root, "legacy").read_text(encoding="utf-8")) + contract = inventory["environment_contract"] + + assert contract["schema_version"] == "anitrend.platform.environment.v1" + assert contract["target_environment"] == "production" + assert set(contract["inputs"]) == { + "provider", + "allowed_docker_contexts", + "safety_policy", + "source_host_identity_ref", + "deployment_owner_map_ref", + "deployment_freeze_steps_ref", + "service_dependency_map_ref", + "database_runtime_facts_ref", + "gateway_policy_export_ref", + "external_provider_map_ref", + "backup_restore_matrix_ref", + "hetzner_project_ref", + "region", + "server_type", + "architecture", + "os_image", + "network_cidrs", + "volumes", + "public_routes", + "private_routes", + "dns_provider_ref", + "dns_zone_ref", + "dns_records", + "private_hostnames", + "certificate_strategy_ref", + "tailnet_grants_ref", + "state_backend_ref", + "backup_destination_ref", + "backup_retention_days", + "credential_refs", + "recovery_credential_refs", + "rpo_hours", + "rto_minutes", + "maintenance_window_ref", + "maximum_downtime_minutes", + "rollback_thresholds", + "writer_job_map_ref", + } + assert all(value == {"status": "unresolved", "value": None, "evidence_ref": None} for value in contract["inputs"].values()) + inventory["environment_contract"]["inputs"]["credential_refs"] = { + "status": "confirmed", + "value": ["password=" + SECRET_SENTINEL], + "evidence_ref": "restricted://fixture-reference", + } + inventory_path = latest_inventory(artifact_root, "legacy") + inventory_path.write_text(json.dumps(inventory), encoding="utf-8") + + validation = run_cli( + tmp_path / "validation", + environment="legacy", + artifact_root=artifact_root, + command="validate", + input_file=inventory_path, + ) + + assert validation.returncode != 0 + assert "environment contract schema rejected" in validation.stderr + assert SECRET_SENTINEL not in validation.stderr diff --git a/tools/README.md b/tools/README.md index 00dbdea7..d459e685 100644 --- a/tools/README.md +++ b/tools/README.md @@ -41,3 +41,18 @@ docker stack deploy -c .rendered/docker-compose.infrastructure.rendered.yml infr - This is a preprocessing step; paths remain relative to the original file location. - It does not evaluate command substitutions or complex expressions, only variable substitution. + +## Platform inventory + +Use the read-only collector for a selected Docker context: + +```sh +make platform-inventory ENV=legacy +make platform-inventory-validate ENV=legacy +``` + +`ENV` is required. `lab` collection rejects remote Docker endpoints. The collector never runs `docker exec`, reads `.env` contents, or emits container environment values. It stores a normalized machine report under the ignored `.artifacts/platform//` directory with owner-only file permissions. The report includes Docker image, port, network, mount, disk, timer, and resource metadata, so keep it local or in approved restricted evidence storage. + +Schema validity is separate from migration readiness. Validation prints the reconciliation gate state; `gate=blocked` is not a completed inventory or a production pass. The public report belongs in `docs/platform/live-inventory.md` and must omit private host paths, addresses, unrelated workload names, and raw runtime exports. + +The versioned inventory and environment intent schemas live in `infra/pulumi/shared/`. The environment contract uses typed values, explicit resolution status, and opaque evidence references; credential fields accept reference names only. diff --git a/tools/platform/inventory.py b/tools/platform/inventory.py new file mode 100755 index 00000000..07e2b268 --- /dev/null +++ b/tools/platform/inventory.py @@ -0,0 +1,1010 @@ +#!/usr/bin/env python3 +"""Collect a normalized, secret-safe snapshot of Docker and repository metadata.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +from decimal import Decimal, InvalidOperation +import ipaddress +import json +import os +from pathlib import Path +import re +import shutil +import subprocess +import sys +from typing import Any, Dict, List, Optional, Sequence, Tuple +import uuid + +import jsonschema +import yaml + + +SCHEMA_VERSION = "anitrend.platform.inventory.v1" +ENVIRONMENTS = ("lab", "legacy", "rehearsal", "production") +DEFAULT_ARTIFACT_ROOT = ".artifacts/platform" +EXCLUDED_SOURCE_PARTS = {".git", ".rendered", ".venv", "node_modules", "stacks", "__pycache__"} + +CONTAINER_INSPECT_FORMAT = ( + r'{{printf "{\"id\":%s,\"name\":%s,\"image_ref\":%s,' + r'\"image_id\":%s,\"state\":%s,\"project\":%s,\"service\":%s,' + r'\"ports\":%s,\"mounts\":%s,\"networks\":%s,' + r'\"memory_limit_bytes\":%v,\"nano_cpus\":%v,' + r'\"size_rw_bytes\":%v,\"size_rootfs_bytes\":%v,\"restart_policy\":%s}"' + r' (json .Id) (json .Name) (json .Config.Image) (json .Image)' + r' (json .State.Status) (json (index .Config.Labels "com.docker.compose.project"))' + r' (json (index .Config.Labels "com.docker.compose.service"))' + r' (json .NetworkSettings.Ports) (json .Mounts) (json .NetworkSettings.Networks)' + r' (json .HostConfig.Memory) (json .HostConfig.NanoCpus)' + r' (json .SizeRw) (json .SizeRootFs) (json .HostConfig.RestartPolicy.Name)}}' +) +IMAGE_INSPECT_FORMAT = ( + r'{{printf "{\"id\":%s,\"os\":%s,\"architecture\":%s,\"repo_digests\":%s}"' + r' (json .Id) (json .Os) (json .Architecture) (json .RepoDigests)}}' +) +DOCKER_VERSION_FORMAT = ( + r'{{printf "{\"client_version\":%s,\"server_version\":%s}"' + r' (json .Client.Version) (json .Server.Version)}}' +) +DOCKER_INFO_FORMAT = ( + r'{{printf "{\"operating_system\":%s,\"architecture\":%s,' + r'\"swarm_state\":%s,\"swarm_manager\":%v,\"cpu_count\":%v,' + r'\"memory_total_bytes\":%v}"' + r' (json .OperatingSystem) (json .Architecture) (json .Swarm.LocalNodeState)' + r' (json .Swarm.ControlAvailable) (json .NCPU) (json .MemTotal)}}' +) + +SIZE_UNITS = { + "B": 1, + "KB": 1000, + "MB": 1000**2, + "GB": 1000**3, + "TB": 1000**4, + "KIB": 1024, + "MIB": 1024**2, + "GIB": 1024**3, + "TIB": 1024**4, +} +SIZE_PATTERN = re.compile(r"^\s*([0-9]+(?:\.[0-9]+)?)\s*([A-Za-z]+)\s*$") +MEMORY_USAGE_PATTERN = re.compile(r"^\s*(.*?)\s*/\s*(.*?)\s*$") + + +class InventoryError(Exception): + """An expected, user-actionable inventory failure without sensitive output.""" + + +def _run_command(command: Sequence[str], timeout: int = 30) -> str: + try: + result = subprocess.run( + list(command), + text=True, + capture_output=True, + check=False, + timeout=timeout, + ) + except FileNotFoundError as error: + executable = Path(command[0]).name if command else "command" + raise InventoryError("required executable is unavailable: " + executable) from error + except subprocess.TimeoutExpired as error: + executable = Path(command[0]).name if command else "command" + raise InventoryError(executable + " exceeded the inventory timeout") from error + if result.returncode != 0: + executable = Path(command[0]).name if command else "command" + raise InventoryError(executable + " failed with exit code " + str(result.returncode)) + return result.stdout + + +def _run_docker(arguments: Sequence[str], context: Optional[str] = None, timeout: int = 30) -> str: + command = ["docker"] + if context: + command.extend(["--context", context]) + command.extend(arguments) + try: + return _run_command(command, timeout=timeout) + except InventoryError as error: + action = arguments[0] if arguments else "command" + raise InventoryError("Docker " + action + ": " + str(error)) from error + + +def _parse_json_records(output: str, label: str) -> List[Dict[str, Any]]: + if not output.strip(): + return [] + try: + parsed = json.loads(output) + records = parsed if isinstance(parsed, list) else [parsed] + except json.JSONDecodeError: + records = [] + for line in output.splitlines(): + if not line.strip(): + continue + try: + records.append(json.loads(line)) + except json.JSONDecodeError as error: + raise InventoryError(label + " returned invalid JSON") from error + if not all(isinstance(record, dict) for record in records): + raise InventoryError(label + " returned an unexpected JSON shape") + return records + + +def _endpoint_kind(endpoint: str) -> str: + if endpoint.startswith("unix://"): + return "local_unix" + if endpoint.startswith("npipe://"): + return "local_npipe" + if endpoint.startswith("ssh://"): + return "remote_ssh" + if endpoint.startswith(("tcp://", "http://", "https://")): + return "remote_tcp" + return "unknown" + + +def _docker_context(requested_context: Optional[str]) -> Tuple[str, str]: + context = requested_context + if context is None: + context = _run_docker(["context", "show"]).strip() + if not context: + raise InventoryError("Docker did not report a selected context") + output = _run_docker(["context", "inspect", "--format", "{{json .Endpoints.docker}}", context]) + try: + endpoint_data = json.loads(output) + except json.JSONDecodeError as error: + raise InventoryError("Docker context endpoint metadata is invalid") from error + endpoint = endpoint_data.get("Host", "") if isinstance(endpoint_data, dict) else "" + return context, _endpoint_kind(endpoint) + + +def _parse_size_bytes(value: Any) -> Optional[int]: + if not isinstance(value, str): + return None + match = SIZE_PATTERN.match(value) + if not match: + return None + unit = match.group(2).upper() + multiplier = SIZE_UNITS.get(unit) + if multiplier is None: + return None + try: + return int(Decimal(match.group(1)) * multiplier) + except InvalidOperation: + return None + + +def _address_classification(value: Any) -> Tuple[str, str]: + if not isinstance(value, str) or not value: + return "unknown", "unknown" + try: + address = ipaddress.ip_address(value) + except ValueError: + return "unknown", "unknown" + family = "ipv4" if address.version == 4 else "ipv6" + if address.is_unspecified: + scope = "wildcard" + elif address.is_loopback: + scope = "loopback" + elif address.is_private: + scope = "private" + elif address.is_global: + scope = "public" + else: + scope = "other" + return family, scope + + +def _normalize_ports(raw_ports: Any) -> List[Dict[str, Any]]: + normalized = [] + if not isinstance(raw_ports, dict): + return normalized + for container_port, bindings in raw_ports.items(): + port_value, separator, protocol = str(container_port).rpartition("/") + if not separator: + continue + try: + container_port_number = int(port_value) + except ValueError: + continue + if not isinstance(bindings, list): + continue + for binding in bindings: + if not isinstance(binding, dict): + continue + host_ip = binding.get("HostIp", "") + family, scope = _address_classification(host_ip) + host_port = binding.get("HostPort") + try: + host_port_number = int(host_port) if host_port else None + except (TypeError, ValueError): + host_port_number = None + normalized.append( + { + "container_port": container_port_number, + "protocol": protocol, + "host_port": host_port_number, + "host_address_scope": scope, + "address_family": family, + } + ) + return sorted(normalized, key=lambda item: (item["container_port"], item["protocol"], item["host_port"] or 0)) + + +def _normalize_mounts(raw_mounts: Any, volume_sizes: Dict[str, Optional[int]]) -> List[Dict[str, Any]]: + normalized = [] + if not isinstance(raw_mounts, list): + return normalized + for mount in raw_mounts: + if not isinstance(mount, dict): + continue + name = mount.get("Name") or None + size_bytes = volume_sizes.get(name) if name else None + normalized.append( + { + "type": mount.get("Type") or "unknown", + "name": name, + "source_path": mount.get("Source") or None, + "destination": mount.get("Destination") or "", + "read_only": not bool(mount.get("RW", False)), + "size_bytes": size_bytes, + "size_status": "measured" if size_bytes is not None else "unknown", + } + ) + return sorted(normalized, key=lambda item: item["destination"]) + + +def _parse_memory_pair(value: Any) -> Tuple[Optional[int], Optional[int]]: + if not isinstance(value, str): + return None, None + match = MEMORY_USAGE_PATTERN.match(value) + if not match: + return None, None + return _parse_size_bytes(match.group(1)), _parse_size_bytes(match.group(2)) + + +def _parse_io_pair(value: Any) -> Tuple[Optional[int], Optional[int]]: + return _parse_memory_pair(value) + + +def _normalize_stats(raw: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: + if raw is None: + return None + cpu_text = raw.get("CPUPerc", "") + try: + cpu_percent = float(str(cpu_text).strip().rstrip("%")) + except ValueError: + cpu_percent = None + memory_used, memory_limit = _parse_memory_pair(raw.get("MemUsage")) + network_received, network_sent = _parse_io_pair(raw.get("NetIO")) + block_read, block_written = _parse_io_pair(raw.get("BlockIO")) + try: + pid_count = int(raw.get("PIDs")) + except (TypeError, ValueError): + pid_count = None + try: + memory_percent = float(str(raw.get("MemPerc", "")).strip().rstrip("%")) + except ValueError: + memory_percent = None + return { + "cpu_percent": cpu_percent, + "memory_used_bytes": memory_used, + "memory_limit_bytes": memory_limit, + "memory_percent": memory_percent, + "network_received_bytes": network_received, + "network_sent_bytes": network_sent, + "block_read_bytes": block_read, + "block_written_bytes": block_written, + "pid_count": pid_count, + } + + +def _collect_timers(endpoint_kind: str) -> Dict[str, Any]: + if endpoint_kind != "local_unix" or not sys.platform.startswith("linux"): + return {"status": "not_local_linux", "units": []} + if shutil.which("systemctl") is None: + return {"status": "unavailable", "units": []} + command = ["systemctl", "list-timers", "--all", "--no-pager", "--no-legend", "--output=json"] + try: + output = _run_command(command, timeout=15) + timers = json.loads(output) if output.strip() else [] + except (InventoryError, json.JSONDecodeError): + return {"status": "unavailable", "units": []} + if not isinstance(timers, list): + return {"status": "unavailable", "units": []} + units = [] + for timer in timers: + if not isinstance(timer, dict) or not isinstance(timer.get("unit"), str): + continue + units.append( + { + "unit": timer["unit"], + "next": timer.get("next") if isinstance(timer.get("next"), str) else None, + "last": timer.get("last") if isinstance(timer.get("last"), str) else None, + "activates": timer.get("activates") if isinstance(timer.get("activates"), str) else None, + } + ) + return {"status": "collected", "units": sorted(units, key=lambda item: item["unit"])} + + + +def _dependency_names(value: Any) -> List[str]: + if isinstance(value, dict): + return sorted(str(name) for name in value.keys()) + if isinstance(value, list): + return sorted(str(name) for name in value) + if isinstance(value, str): + return [value] + return [] + + +def _env_file_refs(value: Any) -> List[str]: + refs = [value] if isinstance(value, str) else value if isinstance(value, list) else [] + return sorted(str(ref) for ref in refs if isinstance(ref, str)) + + +def _secret_refs(value: Any) -> List[str]: + if not isinstance(value, list): + return [] + refs = [] + for secret in value: + if isinstance(secret, str): + refs.append(secret) + elif isinstance(secret, dict) and isinstance(secret.get("source"), str): + refs.append(secret["source"]) + return sorted(set(refs)) + + +def _source_mounts(value: Any) -> List[Dict[str, Any]]: + if not isinstance(value, list): + return [] + mounts = [] + for mount in value: + if isinstance(mount, dict): + mounts.append( + { + "type": str(mount.get("type", "unknown")), + "source": str(mount.get("source", "")), + "target": str(mount.get("target", "")), + "read_only": bool(mount.get("read_only", False)), + } + ) + elif isinstance(mount, str): + pieces = mount.split(":") + if len(pieces) >= 2: + mounts.append( + { + "type": "declared", + "source": pieces[0], + "target": pieces[1], + "read_only": "ro" in pieces[2:], + } + ) + return mounts + + +def _source_service_records(repo_root: Path) -> Tuple[List[Dict[str, Any]], List[str]]: + services = [] + errors = [] + compose_paths = [] + for pattern in ("docker-compose*.yml", "docker-compose*.yaml"): + compose_paths.extend(repo_root.rglob(pattern)) + for path in sorted(set(compose_paths)): + relative = path.relative_to(repo_root) + if EXCLUDED_SOURCE_PARTS.intersection(relative.parts): + continue + try: + document = yaml.safe_load(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, yaml.YAMLError): + errors.append(relative.as_posix() + ": compose source could not be parsed") + continue + if not isinstance(document, dict) or not isinstance(document.get("services"), dict): + continue + stack_extension = document.get("x-stack") + for name, service in sorted(document["services"].items()): + if not isinstance(name, str) or not isinstance(service, dict): + errors.append(relative.as_posix() + ": service declaration is malformed") + continue + image = service.get("image") + container_name = service.get("container_name") + configuration_refs = [] + configuration_refs.extend( + {"kind": "env_file", "ref": ref} for ref in _env_file_refs(service.get("env_file")) + ) + configuration_refs.extend( + {"kind": "secret", "ref": ref} for ref in _secret_refs(service.get("secrets")) + ) + configuration_refs.extend( + {"kind": "config", "ref": ref} for ref in _secret_refs(service.get("configs")) + ) + services.append( + { + "id": relative.as_posix() + ":" + name, + "source_repository": "AniTrend/local-stack", + "source_path": relative.as_posix(), + "compose_project": path.parent.name, + "service": name, + "image_ref": image if isinstance(image, str) else None, + "container_name": container_name if isinstance(container_name, str) else None, + "depends_on": _dependency_names(service.get("depends_on")), + "declared_mounts": _source_mounts(service.get("volumes")), + "configuration_refs": sorted(configuration_refs, key=lambda item: (item["kind"], item["ref"])), + "health_contract": { + "status": "unresolved", + "source_healthcheck_declared": isinstance(service.get("healthcheck"), dict), + "runtime_health_states": [], + "evidence_ref": None, + }, + "database_runtime": { + "status": "unresolved", + "server_version": None, + "topology_ref": None, + "data_paths": [], + }, + "writer_job_map_ref": None, + "external_provider_map_ref": None, + "backup_restore_method_ref": None, + "ingress_class": "unresolved", + "runtime_type": "not_observed", + "evidence_refs": ["source:" + relative.as_posix()], + "restart_policy": service.get("restart") if isinstance(service.get("restart"), str) else None, + "stack_extension": stack_extension if isinstance(stack_extension, str) else None, + "runtime_match_status": "not_observed", + "runtime_match_method": "not-observed", + "runtime_container_ids": [], + "current_deployment_owner": "unknown", + "target_deployment_owner": "unresolved", + "disposition": "unresolved", + "data_classification": "unresolved", + } + ) + return services, errors + + +def _collect_volume_rows(context: str) -> Tuple[List[Dict[str, Any]], Dict[str, Optional[int]], str]: + try: + output = _run_docker(["system", "df", "--verbose", "--format", "json"], context, timeout=120) + records = _parse_json_records(output, "Docker disk usage") + except InventoryError: + return [], {}, "unavailable" + payload = records[0] if records else {} + raw_volumes = payload.get("Volumes", []) if isinstance(payload, dict) else [] + if not isinstance(raw_volumes, list): + return [], {}, "unavailable" + volumes = [] + sizes: Dict[str, Optional[int]] = {} + for volume in raw_volumes: + if not isinstance(volume, dict) or not isinstance(volume.get("Name"), str): + continue + name = volume["Name"] + size_bytes = _parse_size_bytes(volume.get("Size")) + sizes[name] = size_bytes + volumes.append( + { + "name": name, + "driver": volume.get("Driver") if isinstance(volume.get("Driver"), str) else "unknown", + "scope": volume.get("Scope") if isinstance(volume.get("Scope"), str) else "unknown", + "links": volume.get("Links") if isinstance(volume.get("Links"), int) else None, + "size_bytes": size_bytes, + "size_status": "measured" if size_bytes is not None else "unknown", + } + ) + return sorted(volumes, key=lambda item: item["name"]), sizes, "collected" + + +def _collect_stats(context: str) -> Dict[str, Dict[str, Any]]: + try: + output = _run_docker(["stats", "--no-stream", "--format", "json"], context, timeout=60) + records = _parse_json_records(output, "Docker resource statistics") + except InventoryError: + return {} + stats = {} + for record in records: + identifier = record.get("ID") + if isinstance(identifier, str): + stats[identifier] = record + return stats + + +def _collect_images(image_ids: Sequence[str], context: str) -> Dict[str, Dict[str, Any]]: + images = {} + for image_id in sorted(set(image_ids)): + if not image_id: + continue + try: + output = _run_docker(["image", "inspect", "--format", IMAGE_INSPECT_FORMAT, image_id], context) + records = _parse_json_records(output, "Docker image metadata") + except InventoryError: + continue + if not records: + continue + record = records[0] + identifier = record.get("id") + if isinstance(identifier, str): + repo_digests = record.get("repo_digests") + images[identifier] = { + "os": record.get("os") if isinstance(record.get("os"), str) else None, + "architecture": record.get("architecture") if isinstance(record.get("architecture"), str) else None, + "repo_digests": sorted(item for item in repo_digests if isinstance(item, str)) if isinstance(repo_digests, list) else [], + } + return images + + +def _status_rows(output: str) -> Dict[str, str]: + statuses = {} + for line in output.splitlines(): + fields = line.split("\t", 2) + if len(fields) == 3: + statuses[fields[0]] = fields[2] + return statuses + + +def _health_status(status: str) -> str: + normalized = status.lower() + if "(healthy)" in normalized: + return "healthy" + if "(unhealthy)" in normalized: + return "unhealthy" + if "(health: starting)" in normalized or "(starting)" in normalized: + return "starting" + return "unknown" + + +def _network_names(raw_networks: Any) -> List[Dict[str, Any]]: + networks = [] + if not isinstance(raw_networks, dict): + return networks + for name, endpoint in sorted(raw_networks.items()): + aliases = endpoint.get("Aliases", []) if isinstance(endpoint, dict) else [] + networks.append( + { + "name": str(name), + "aliases": sorted(alias for alias in aliases if isinstance(alias, str)) if isinstance(aliases, list) else [], + } + ) + return networks + + +def _normalize_container( + raw: Dict[str, Any], + status_text: str, + image_metadata: Dict[str, Dict[str, Any]], + stats_rows: Dict[str, Dict[str, Any]], + volume_sizes: Dict[str, Optional[int]], +) -> Dict[str, Any]: + identifier = str(raw.get("id", "")) + image_id = str(raw.get("image_id", "")) + image = image_metadata.get(image_id, {}) + raw_stats = None + for short_id, stats in stats_rows.items(): + if identifier.startswith(short_id): + raw_stats = stats + break + name = raw.get("name", "") + return { + "id": identifier, + "name": str(name).lstrip("/"), + "image_ref": raw.get("image_ref") if isinstance(raw.get("image_ref"), str) else "unknown", + "image_id": image_id, + "repo_digests": image.get("repo_digests", []), + "image_os": image.get("os"), + "image_architecture": image.get("architecture"), + "state": raw.get("state") if isinstance(raw.get("state"), str) else "unknown", + "health": _health_status(status_text), + "compose_project": raw.get("project") if isinstance(raw.get("project"), str) else None, + "compose_service": raw.get("service") if isinstance(raw.get("service"), str) else None, + "source_service_id": None, + "ports": _normalize_ports(raw.get("ports")), + "networks": _network_names(raw.get("networks")), + "mounts": _normalize_mounts(raw.get("mounts"), volume_sizes), + "resource_limits": { + "memory_bytes": raw.get("memory_limit_bytes") if isinstance(raw.get("memory_limit_bytes"), int) else 0, + "nano_cpus": raw.get("nano_cpus") if isinstance(raw.get("nano_cpus"), int) else 0, + }, + "resource_usage": _normalize_stats(raw_stats), + "container_layer_size_bytes": raw.get("size_rw_bytes") if isinstance(raw.get("size_rw_bytes"), int) else None, + "rootfs_size_bytes": raw.get("size_rootfs_bytes") if isinstance(raw.get("size_rootfs_bytes"), int) else None, + "restart_policy": raw.get("restart_policy") if isinstance(raw.get("restart_policy"), str) else "unknown", + } + + +def _match_source_services( + containers: List[Dict[str, Any]], source_services: List[Dict[str, Any]] +) -> List[Dict[str, Any]]: + by_labels: Dict[Tuple[Optional[str], Optional[str]], List[Dict[str, Any]]] = {} + by_container_name: Dict[str, List[Dict[str, Any]]] = {} + for service in source_services: + by_labels.setdefault((service["compose_project"], service["service"]), []).append(service) + if service["container_name"]: + by_container_name.setdefault(service["container_name"], []).append(service) + for container in containers: + match_method = "compose-labels" + matches = by_labels.get((container["compose_project"], container["compose_service"]), []) + if not matches: + matches = by_container_name.get(container["name"], []) + match_method = "container-name" + if len(matches) == 1: + service = matches[0] + container["source_service_id"] = service["id"] + service["runtime_match_status"] = "matched" + service["runtime_match_method"] = match_method + service["runtime_type"] = "docker-compose" if match_method == "compose-labels" else "docker-container" + service["current_deployment_owner"] = "Compose labels observed; deployment operator unknown" if match_method == "compose-labels" else "container name matched; deployment operator unknown" + service["runtime_container_ids"].append(container["id"]) + service["health_contract"]["runtime_health_states"].append(container["health"]) + service["health_contract"]["evidence_ref"] = "docker-container:" + container["id"] + service["evidence_refs"].append("docker-container:" + container["id"]) + elif len(matches) > 1: + for service in matches: + service["runtime_match_status"] = "ambiguous" + service["runtime_match_method"] = "ambiguous" + service["runtime_type"] = "ambiguous" + else: + container["source_service_id"] = None + for service in source_services: + if service["runtime_match_status"] == "not_observed": + service["runtime_match_method"] = "not-observed" + service["runtime_type"] = "not_observed" + service["current_deployment_owner"] = "not observed on selected Docker engine" + service["health_contract"]["runtime_health_states"] = sorted( + set(service["health_contract"]["runtime_health_states"]) + ) + return source_services + + +def _collect_docker(environment: str, requested_context: Optional[str]) -> Tuple[Dict[str, Any], List[str]]: + context, endpoint_kind = _docker_context(requested_context) + if environment == "lab" and endpoint_kind not in ("local_unix", "local_npipe"): + raise InventoryError("ENV=lab requires an explicitly selected local Docker endpoint") + version_records = _parse_json_records( + _run_docker(["version", "--format", DOCKER_VERSION_FORMAT], context), "Docker version" + ) + info_records = _parse_json_records( + _run_docker(["info", "--format", DOCKER_INFO_FORMAT], context), "Docker host information" + ) + if not version_records or not info_records: + raise InventoryError("Docker did not return required host metadata") + info = info_records[0] + listing = _run_docker( + ["ps", "--all", "--no-trunc", "--format", "{{.ID}}\t{{.Names}}\t{{.Status}}"], context + ) + status_texts = _status_rows(listing) + container_ids = list(status_texts.keys()) + raw_containers = [] + if container_ids: + inspect_output = _run_docker( + ["inspect", "--size", "--type", "container", "--format", CONTAINER_INSPECT_FORMAT, *container_ids], + context, + timeout=120, + ) + raw_containers = _parse_json_records(inspect_output, "Docker container metadata") + volume_rows, volume_sizes, volume_status = _collect_volume_rows(context) + running_containers = [item for item in raw_containers if item.get("state") == "running"] + stats_rows = _collect_stats(context) if running_containers else {} + stats_observed = sum( + 1 + for item in running_containers + if any(str(item.get("id", "")).startswith(short_id) for short_id in stats_rows) + ) + if not running_containers: + resource_usage_status = "not_applicable" + elif stats_observed == len(running_containers): + resource_usage_status = "collected" + elif stats_observed: + resource_usage_status = "partial" + else: + resource_usage_status = "unavailable" + errors = [] + if resource_usage_status in ("partial", "unavailable"): + errors.append("Docker resource statistics are partial or unavailable") + image_metadata = _collect_images( + [str(item.get("image_id", "")) for item in raw_containers], context + ) + containers = [] + for raw in raw_containers: + identifier = str(raw.get("id", "")) + containers.append( + _normalize_container( + raw, + status_texts.get(identifier, ""), + image_metadata, + stats_rows, + volume_sizes, + ) + ) + timers = _collect_timers(endpoint_kind) + if volume_status != "collected": + errors.append("Docker volume sizes could not be collected") + if any(not container["repo_digests"] for container in containers): + errors.append("one or more container images have no observed repository digest") + running_count = sum(1 for container in containers if container["state"] == "running") + docker = { + "context_name": context, + "endpoint_kind": endpoint_kind, + "client_version": version_records[0].get("client_version", "unknown"), + "server_version": version_records[0].get("server_version", "unknown"), + "operating_system": info.get("operating_system", "unknown"), + "architecture": info.get("architecture", "unknown"), + "swarm_state": info.get("swarm_state", "unknown"), + "swarm_manager": bool(info.get("swarm_manager", False)), + "cpu_count": info.get("cpu_count") if isinstance(info.get("cpu_count"), int) else 0, + "memory_total_bytes": info.get("memory_total_bytes") if isinstance(info.get("memory_total_bytes"), int) else 0, + "container_count": len(containers), + "running_count": running_count, + "resource_usage_status": resource_usage_status, + "containers": sorted(containers, key=lambda item: item["name"]), + "volumes": volume_rows, + "volume_inventory_status": volume_status, + "host_timers": timers, + } + return docker, errors + + +def _reconcile_containers(containers: List[Dict[str, Any]], source_services: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + source_ids = {service["id"] for service in source_services} + unmatched = [] + for container in containers: + source_id = container.get("source_service_id") + if source_id is None or source_id not in source_ids: + unmatched.append( + { + "container_id": container["id"], + "name": container["name"], + "compose_project": container["compose_project"], + "compose_service": container["compose_service"], + "disposition": "unresolved", + "deployment_owner": "unknown", + "evidence_ref": "docker-container:" + container["id"], + } + ) + return unmatched + + +def _empty_environment_contract(schema_path: Path) -> Dict[str, Any]: + try: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + inputs = schema["properties"]["inputs"]["properties"] + except (OSError, UnicodeError, json.JSONDecodeError, KeyError, TypeError) as error: + raise InventoryError("environment contract schema could not be loaded") from error + if not isinstance(inputs, dict) or not inputs: + raise InventoryError("environment contract schema has no input definitions") + return { + "schema_version": "anitrend.platform.environment.v1", + "target_environment": "production", + "inputs": { + name: {"status": "unresolved", "value": None, "evidence_ref": None} + for name in inputs + }, + } + + + +def _git_metadata(repo_root: Path) -> Tuple[str, bool]: + try: + revision = _run_command(["git", "-C", str(repo_root), "rev-parse", "HEAD"], timeout=5).strip() + dirty = bool(_run_command(["git", "-C", str(repo_root), "status", "--porcelain"], timeout=5).strip()) + except InventoryError: + return "unknown", False + if not re.fullmatch(r"[0-9a-f]{40}", revision): + return "unknown", dirty + return revision, dirty + + +def _reconciliation_validation_error(payload: Dict[str, Any]) -> Optional[str]: + reconciliation = payload["reconciliation"] + decision = reconciliation["decision"] + reasons = reconciliation["blocked_reasons"] + if decision != "pass": + if decision in ("blocked", "fail", "skipped") and not reasons: + return "non-passing reconciliation states require a reason" + return None + if payload["environment"] == "lab": + return "reconciliation gate cannot pass from ENV=lab inventory" + + unresolved = not payload["collector_worktree_clean"] or payload["collector_commit"] == "unknown" + unresolved = unresolved or bool(payload["collection_issues"]) or bool(reasons) + unresolved = unresolved or payload["docker"]["volume_inventory_status"] != "collected" + if payload["docker"]["running_count"] and payload["docker"]["resource_usage_status"] != "collected": + unresolved = True + for item in payload["environment_contract"]["inputs"].values(): + if item["status"] not in ("confirmed", "not_applicable") or not item["evidence_ref"]: + unresolved = True + if item["status"] == "confirmed" and item["value"] is None: + unresolved = True + if item["status"] == "not_applicable" and item["value"] is not None: + unresolved = True + inputs = payload["environment_contract"]["inputs"] + if inputs["provider"]["status"] != "confirmed" or inputs["provider"]["value"] != "hcloud": + unresolved = True + required_safety_policy = { + "protect_resources": True, + "allow_destroy": False, + "require_environment_approval": True, + "deny_pull_request_mutations": True, + } + if inputs["safety_policy"]["status"] != "confirmed" or inputs["safety_policy"]["value"] != required_safety_policy: + unresolved = True + for service in payload["repository_services"]: + if service["disposition"] == "unresolved" or service["data_classification"] == "unresolved": + unresolved = True + if service["runtime_match_status"] == "ambiguous": + unresolved = True + for field in ("current_deployment_owner", "target_deployment_owner"): + if any(token in service[field].lower() for token in ("unknown", "unresolved", "pending")): + unresolved = True + if ( + service["ingress_class"] == "unresolved" + or service["health_contract"]["status"] == "unresolved" + or service["database_runtime"]["status"] == "unresolved" + or not service["writer_job_map_ref"] + or not service["external_provider_map_ref"] + or not service["backup_restore_method_ref"] + or not service["evidence_refs"] + ): + unresolved = True + for container in payload["unmatched_containers"]: + if container["disposition"] == "unresolved" or "unknown" in container["deployment_owner"].lower(): + unresolved = True + if unresolved: + return "reconciliation gate cannot pass while required inventory fields are unresolved" + return None + +def _validate_payload(payload: Dict[str, Any], schema_path: Path) -> Optional[str]: + environment_schema_path = schema_path.with_name("environment.schema.json") + try: + schema = json.loads(schema_path.read_text(encoding="utf-8")) + environment_schema = json.loads(environment_schema_path.read_text(encoding="utf-8")) + jsonschema.Draft202012Validator.check_schema(schema) + jsonschema.Draft202012Validator.check_schema(environment_schema) + validator = jsonschema.Draft202012Validator(schema, format_checker=jsonschema.FormatChecker()) + error = next(iter(validator.iter_errors(payload)), None) + environment_validator = jsonschema.Draft202012Validator( + environment_schema, format_checker=jsonschema.FormatChecker() + ) + environment_error = next( + iter(environment_validator.iter_errors(payload.get("environment_contract", {}))), None + ) + except (OSError, UnicodeError, json.JSONDecodeError, jsonschema.SchemaError): + return "inventory or environment schema could not be loaded or validated" + if error is not None: + location = ".".join(str(part) for part in error.absolute_path) or "$" + description = "additional properties" if error.validator == "additionalProperties" else "invalid field" + return "inventory schema rejected " + location + ": " + description + if environment_error is not None: + location = ".".join(str(part) for part in environment_error.absolute_path) or "$" + description = "additional properties" if environment_error.validator == "additionalProperties" else "invalid field" + return "environment contract schema rejected " + location + ": " + description + return _reconciliation_validation_error(payload) + + +def _collect_inventory( + environment: str, + repo_root: Path, + artifact_root: Path, + schema_path: Path, + requested_context: Optional[str], +) -> Tuple[Dict[str, Any], Path]: + docker, collection_errors = _collect_docker(environment, requested_context) + source_services, source_errors = _source_service_records(repo_root) + collection_errors.extend(source_errors) + source_services = _match_source_services(docker["containers"], source_services) + unmatched = _reconcile_containers(docker["containers"], source_services) + revision, dirty = _git_metadata(repo_root) + observed = datetime.now(timezone.utc).replace(microsecond=0) + run_id = observed.strftime("%Y%m%dT%H%M%SZ") + "-" + uuid.uuid4().hex[:8] + for service in source_services: + if not service["runtime_container_ids"]: + service["evidence_refs"].append("docker-snapshot:" + run_id) + service["evidence_refs"] = sorted(set(service["evidence_refs"])) + payload = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "environment": environment, + "observed_at_utc": observed.isoformat().replace("+00:00", "Z"), + "collector_commit": revision, + "collector_worktree_clean": not dirty, + "docker": docker, + "repository_services": source_services, + "unmatched_containers": unmatched, + "collection_issues": sorted(collection_errors), + "environment_contract": _empty_environment_contract( + schema_path.with_name("environment.schema.json") + ), + "reconciliation": { + "decision": "blocked", + "blocked_reasons": [ + "The source host role and production identity are not confirmed by owner-supplied evidence.", + "Service dispositions, ownership, writer/job schedules, external providers, and recovery paths require reconciliation.", + "Required target, network, credential-reference, backup, and cutover inputs remain unresolved.", + ], + }, + } + error = _validate_payload(payload, schema_path) + if error: + raise InventoryError(error) + output_dir = artifact_root / run_id + try: + output_dir.mkdir(mode=0o700, parents=True, exist_ok=False) + output_path = output_dir / "inventory.json" + descriptor = os.open(output_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as output: + json.dump(payload, output, indent=2, sort_keys=True) + output.write("\n") + except OSError as error: + raise InventoryError("private inventory artifact could not be written") from error + return payload, output_path + + +def _latest_inventory(artifact_root: Path, environment: str) -> Path: + candidates = [] + for path in artifact_root.glob("*/inventory.json"): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + continue + if isinstance(payload, dict) and payload.get("environment") == environment: + candidates.append(path) + if not candidates: + raise InventoryError("no inventory artifact exists for the requested ENV") + return max(candidates, key=lambda path: path.stat().st_mtime) + + +def _validate_inventory(environment: str, input_path: Optional[Path], artifact_root: Path, schema_path: Path) -> str: + path = input_path if input_path is not None else _latest_inventory(artifact_root, environment) + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as error: + raise InventoryError("inventory file could not be read as JSON") from error + if not isinstance(payload, dict): + raise InventoryError("inventory JSON must contain an object") + if payload.get("environment") != environment: + raise InventoryError("inventory ENV does not match the requested ENV") + error = _validate_payload(payload, schema_path) + if error: + raise InventoryError(error) + decision = payload.get("reconciliation", {}).get("decision", "blocked") + return "schema=valid gate=" + str(decision) + + +def _repository_root() -> Path: + return Path(__file__).resolve().parents[2] + + +def _argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Read-only, secret-safe platform inventory collection") + subparsers = parser.add_subparsers(dest="command", required=True) + collect = subparsers.add_parser("collect", help="collect Docker and Compose metadata without changing the daemon") + collect.add_argument("--environment", required=True, choices=ENVIRONMENTS) + collect.add_argument("--repo-root", type=Path, default=_repository_root()) + collect.add_argument("--artifact-root", type=Path, default=_repository_root() / DEFAULT_ARTIFACT_ROOT) + collect.add_argument("--context", help="explicit Docker context; omitted uses the selected context without changing it") + validate = subparsers.add_parser("validate", help="validate the latest or selected inventory schema") + validate.add_argument("--environment", required=True, choices=ENVIRONMENTS) + validate.add_argument("--artifact-root", type=Path, default=_repository_root() / DEFAULT_ARTIFACT_ROOT) + validate.add_argument("--input", type=Path) + return parser + + +def main(arguments: Optional[Sequence[str]] = None) -> int: + parser = _argument_parser() + options = parser.parse_args(arguments) + repo_root = _repository_root() + schema_path = repo_root / "infra" / "pulumi" / "shared" / "inventory.schema.json" + try: + if options.command == "collect": + payload, path = _collect_inventory( + options.environment, + options.repo_root.resolve(), + options.artifact_root.resolve(), + schema_path, + options.context, + ) + print( + "inventory=" + path.relative_to(options.artifact_root.resolve()).as_posix() + + " schema=valid reconciliation=" + payload["reconciliation"]["decision"] + ) + return 0 + print(_validate_inventory(options.environment, options.input, options.artifact_root, schema_path)) + return 0 + except InventoryError as error: + print("error: " + str(error), file=sys.stderr) + return 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From eb48f84bb584b53e9d939ac17d6f93793bce2e6f Mon Sep 17 00:00:00 2001 From: Maxwell Date: Tue, 27 Oct 2026 21:26:57 +0200 Subject: [PATCH 2/2] fix(platform): require complete inventory reconciliation evidence --- .github/workflows/ci.yml | 2 +- docs/platform/live-inventory.md | 6 +- infra/pulumi/shared/inventory.schema.json | 11 +- tests/platform/test_inventory.py | 17 +- .../platform/test_inventory_reconciliation.py | 146 ++++++++++++++++++ tools/README.md | 4 + tools/platform/inventory.py | 123 +++++++++++++-- 7 files changed, 280 insertions(+), 29 deletions(-) create mode 100644 tests/platform/test_inventory_reconciliation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7c8b6db0..bc424593 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Install inventory test dependencies run: python -m pip install -r tools/requirements.txt pytest==8.4.2 - name: Validate read-only inventory and safety contracts - run: python -m pytest -q tests/platform/test_inventory.py + run: python -m pytest -q tests/platform trivy-scan: needs: [lint] diff --git a/docs/platform/live-inventory.md b/docs/platform/live-inventory.md index 6f467b58..8ec7f603 100644 --- a/docs/platform/live-inventory.md +++ b/docs/platform/live-inventory.md @@ -18,7 +18,7 @@ Repository service facts remain pinned to [`e80a48a8b8d7d4d9732840b618e73a2f4b58 | Container and volume counts | 34 containers total, 28 running, 22 local volumes | | Repository service matches | 16 of 24 Compose service definitions | | Other runtime containers | 18 unmatched to this repository; names are intentionally omitted | -| Host systemd timers | 13 observed; unit names are private | +| Collector-host systemd timers | 13 observed on the collector's host, not proof of the Docker engine host's jobs; unit names are private | | Tailscale daemon | `tailscaled.service` active; ACLs and device grants not inspected | | Doco-CD controller | `doco-cd.service` inactive; no Doco-CD container observed on this engine | | Portainer | Container running; deployment authority was not verified | @@ -113,3 +113,7 @@ The automated collector uses a synthetic Docker CLI fixture in tests. The select The collector draft was checked again before publication on 2026-10-27 without changing the historical observations above. The inventory regression file passed 6 tests under the existing isolated platform Python environment. A new explicit `ENV=lab DOCKER_CONTEXT=default` read-only collection produced run `20261027T183959Z-fcf5c1ad`; schema validation returned `schema=valid gate=blocked`. The private JSON remains ignored. This later lab collection is not a new production inventory, does not resolve G0, and did not mutate the Docker daemon. The CI `Platform inventory validation` job runs the synthetic Docker CLI regression suite with read-only repository permissions and no cloud credentials. The real disposable-engine fixture acceptance and owner-backed live reconciliation still remain to be performed. + +Review repairs added direct reconciliation regressions and realistic Docker/systemd formats. The complete platform inventory suite now passes 30 tests. Running the new semantic/normalization regressions against the published pre-repair collector `a094dbf` produced 23 failures and one passing semantic control. The repaired collector rejects waived required production inputs, disallowed contexts, missing/duplicate/inconsistent container mappings, unresolved ownership, and missing confirmed database/health facts. + +The repaired read-only collector produced lab run `20261027T192226Z-bf4b5c69`; schema validation again returned `schema=valid gate=blocked`. Timer observations now use `collector_host_timers` with explicit collector provenance, integer-microsecond timestamps become UTC ISO 8601 strings, unavailable timestamps remain null, and Docker's numeric-string volume reference counts are preserved. Older private artifacts carrying the previous timer field must be recollected, not edited into passing evidence. diff --git a/infra/pulumi/shared/inventory.schema.json b/infra/pulumi/shared/inventory.schema.json index d9bfa202..4e625a1a 100644 --- a/infra/pulumi/shared/inventory.schema.json +++ b/infra/pulumi/shared/inventory.schema.json @@ -45,7 +45,7 @@ "containers", "volumes", "volume_inventory_status", - "host_timers" + "collector_host_timers" ], "properties": { "context_name": { "type": "string", "minLength": 1 }, @@ -70,7 +70,7 @@ "items": { "$ref": "#/$defs/volume" } }, "volume_inventory_status": { "enum": ["collected", "unavailable"] }, - "host_timers": { "$ref": "#/$defs/timerInventory" } + "collector_host_timers": { "$ref": "#/$defs/timerInventory" } } }, "repository_services": { @@ -241,9 +241,10 @@ "timerInventory": { "type": "object", "additionalProperties": false, - "required": ["status", "units"], + "required": ["status", "host_scope", "units"], "properties": { "status": { "enum": ["collected", "not_local_linux", "unavailable"] }, + "host_scope": { "const": "collector" }, "units": { "type": "array", "items": { @@ -252,8 +253,8 @@ "required": ["unit", "next", "last", "activates"], "properties": { "unit": { "type": "string" }, - "next": { "type": ["string", "null"] }, - "last": { "type": ["string", "null"] }, + "next": { "type": ["string", "null"], "format": "date-time" }, + "last": { "type": ["string", "null"], "format": "date-time" }, "activates": { "type": ["string", "null"] } } } diff --git a/tests/platform/test_inventory.py b/tests/platform/test_inventory.py index 8254c3c9..41d3fa04 100644 --- a/tests/platform/test_inventory.py +++ b/tests/platform/test_inventory.py @@ -114,9 +114,10 @@ def make_toolchain(tmp_path: Path, endpoint: str) -> Tuple[Path, Path]: elif args[:2] == ["system", "df"]: print(json.dumps({{ "Volumes": [ - {{"Name": "fixture-data", "Driver": "local", "Scope": "local", "Links": 1, "Size": "744MB", "Mountpoint": "/private/docker/root", "Labels": {{"secret": {SECRET_SENTINEL!r}}}}}, - {{"Name": "fixture-binary", "Driver": "local", "Scope": "local", "Links": 0, "Size": "1.5GiB", "Mountpoint": "/private/docker/root2", "Labels": {{}}}}, - {{"Name": "fixture-empty", "Driver": "local", "Scope": "local", "Links": 0, "Size": "0B", "Mountpoint": "/private/docker/root3", "Labels": {{}}}}, + {{"Name": "fixture-data", "Driver": "local", "Scope": "local", "Links": "1", "Size": "744MB", "Mountpoint": "/private/docker/root", "Labels": {{"secret": {SECRET_SENTINEL!r}}}}}, + {{"Name": "fixture-binary", "Driver": "local", "Scope": "local", "Links": "0", "Size": "1.5GiB", "Mountpoint": "/private/docker/root2", "Labels": {{}}}}, + {{"Name": "fixture-empty", "Driver": "local", "Scope": "local", "Links": "0", "Size": "0B", "Mountpoint": "/private/docker/root3", "Labels": {{}}}}, + {{"Name": "fixture-unparseable", "Driver": "local", "Scope": "local", "Links": "invalid", "Size": "0B", "Mountpoint": "/private/docker/root4", "Labels": {{}}}}, ] }})) else: @@ -135,7 +136,8 @@ def make_toolchain(tmp_path: Path, endpoint: str) -> Tuple[Path, Path]: #!/usr/bin/env python3 import json print(json.dumps([ - {"unit": "backup.timer", "next": "2026-10-26 02:00:00 UTC", "last": "2026-10-25 02:00:00 UTC", "activates": "backup.service"} + {"unit": "backup.timer", "next": 1734381600000000, "last": 1734295200000000, "activates": "backup.service"}, + {"unit": "unparseable.timer", "next": "unknown", "last": "unknown", "activates": "unknown.service"} ])) """ ), @@ -288,10 +290,15 @@ def test_inventory_normalizes_runtime_evidence_without_reading_env_or_network_se volumes = {volume["name"]: volume for volume in inventory["docker"]["volumes"]} assert volumes["fixture-binary"]["size_bytes"] == 1_610_612_736 assert volumes["fixture-empty"]["size_bytes"] == 0 + assert volumes["fixture-data"]["links"] == 1 + assert volumes["fixture-binary"]["links"] == 0 + assert volumes["fixture-unparseable"]["links"] is None assert container["resource_usage"]["cpu_percent"] == 1.25 assert container["resource_usage"]["memory_used_bytes"] == 24 * 1024 * 1024 assert container["repo_digests"] == [IMAGE_DIGEST] - assert inventory["docker"]["host_timers"]["units"][0]["unit"] == "backup.timer" + assert inventory["docker"]["collector_host_timers"]["host_scope"] == "collector" + assert inventory["docker"]["collector_host_timers"]["units"][0]["unit"] == "backup.timer" + assert inventory["docker"]["collector_host_timers"]["units"][1]["next"] is None assert inventory["reconciliation"]["decision"] == "blocked" assert SECRET_SENTINEL not in serialized assert "10.2.0.7" not in serialized diff --git a/tests/platform/test_inventory_reconciliation.py b/tests/platform/test_inventory_reconciliation.py new file mode 100644 index 00000000..3706fd3c --- /dev/null +++ b/tests/platform/test_inventory_reconciliation.py @@ -0,0 +1,146 @@ +from __future__ import annotations + +import copy +import json + +import pytest + +from tools.platform import inventory + + +@pytest.fixture +def reconciled(): + # Only the fields consumed by the post-schema semantic validator are included. + # This fixture exercises semantic reconciliation after structural validation. + inputs = { + name: {"status": "confirmed", "value": "evidence-backed-value", "evidence_ref": "restricted://decision"} + for name in ( + "provider", "hetzner_project_ref", "source_host_identity_ref", "region", "architecture", + "server_type", "os_image", "writer_job_map_ref", "maintenance_window_ref", "rollback_thresholds", + ) + } + inputs["provider"]["value"] = "hcloud" + inputs["allowed_docker_contexts"] = { + "status": "confirmed", "value": ["approved-production"], "evidence_ref": "restricted://context", + } + inputs["rollback_thresholds"]["value"] = ["approved error/latency limits"] + inputs["safety_policy"] = { + "status": "confirmed", "evidence_ref": "restricted://safety", + "value": {"protect_resources": True, "allow_destroy": False, + "require_environment_approval": True, "deny_pull_request_mutations": True}, + } + service_id = "postgres/docker-compose.yaml:postgres" + matched = {"id": "a" * 64, "name": "postgres", "compose_project": "database", "compose_service": "postgres", + "source_service_id": service_id} + unmatched = {"id": "b" * 64, "name": "external-app", "compose_project": "external", "compose_service": "app", + "source_service_id": None} + return { + "environment": "production", "collector_commit": "c" * 40, "collector_worktree_clean": True, + "collection_issues": [], "reconciliation": {"decision": "pass", "blocked_reasons": []}, + "environment_contract": {"inputs": inputs}, + "docker": {"context_name": "approved-production", "volume_inventory_status": "collected", + "resource_usage_status": "collected", "running_count": 2, "containers": [matched, unmatched]}, + "repository_services": [{ + "id": service_id, "runtime_container_ids": [matched["id"]], + "disposition": "migrate", "data_classification": "durable", "runtime_match_status": "matched", + "current_deployment_owner": "legacy-operations", "target_deployment_owner": "dokploy-operations", + "ingress_class": "internal", "health_contract": {"status": "confirmed", "evidence_ref": "restricted://health"}, + "database_runtime": {"status": "confirmed", "server_version": "18.4", "topology_ref": "restricted://db", + "data_paths": ["/var/lib/postgresql/18/docker"]}, + "writer_job_map_ref": "restricted://writers", "external_provider_map_ref": "restricted://providers", + "backup_restore_method_ref": "restricted://restore", "evidence_refs": ["restricted://observations"], + }], + "unmatched_containers": [{ + "container_id": unmatched["id"], "name": unmatched["name"], "compose_project": "external", "compose_service": "app", + "disposition": "retain", "deployment_owner": "external-operations", "evidence_ref": "restricted://owner", + }], + } + + +def test_complete_reconciliation_semantics_are_accepted(reconciled): + assert inventory._reconciliation_validation_error(reconciled) is None + + +@pytest.mark.parametrize("name", ["hetzner_project_ref", "source_host_identity_ref", "architecture", "writer_job_map_ref", "maintenance_window_ref", "rollback_thresholds"]) +def test_required_production_inputs_cannot_be_waived(reconciled, name): + reconciled["environment_contract"]["inputs"][name] = { + "status": "not_applicable", "value": None, "evidence_ref": "restricted://attempted-waiver", + } + assert inventory._reconciliation_validation_error(reconciled) is not None + + +def test_unapproved_observed_context_cannot_pass(reconciled): + reconciled["docker"]["context_name"] = "unapproved-source" + assert inventory._reconciliation_validation_error(reconciled) is not None + + +@pytest.mark.parametrize("change", ["missing", "duplicate", "inconsistent"]) +def test_every_unmatched_container_requires_exact_reconciliation(reconciled, change): + if change == "missing": + reconciled["unmatched_containers"] = [] + elif change == "duplicate": + reconciled["unmatched_containers"].append(copy.deepcopy(reconciled["unmatched_containers"][0])) + else: + reconciled["unmatched_containers"][0]["compose_project"] = "another-project" + assert inventory._reconciliation_validation_error(reconciled) is not None + + +@pytest.mark.parametrize("location,owner", [("current", ""), ("target", " "), ("unmatched", "unresolved"), ("unmatched", "pending")]) +def test_deployment_owners_must_be_resolved(reconciled, location, owner): + if location == "unmatched": + reconciled["unmatched_containers"][0]["deployment_owner"] = owner + else: + reconciled["repository_services"][0][location + "_deployment_owner"] = owner + assert inventory._reconciliation_validation_error(reconciled) is not None + + +@pytest.mark.parametrize("field,value", [("server_version", None), ("topology_ref", None), ("data_paths", [])]) +def test_confirmed_database_requires_observed_facts(reconciled, field, value): + reconciled["repository_services"][0]["database_runtime"][field] = value + assert inventory._reconciliation_validation_error(reconciled) is not None + + +def test_confirmed_health_requires_evidence(reconciled): + reconciled["repository_services"][0]["health_contract"]["evidence_ref"] = None + assert inventory._reconciliation_validation_error(reconciled) is not None + + +def test_forged_service_match_cannot_remove_an_unowned_container(reconciled): + reconciled["docker"]["containers"][1]["source_service_id"] = "nonexistent-service" + reconciled["unmatched_containers"] = [] + assert inventory._reconciliation_validation_error(reconciled) is not None + + +def test_service_runtime_mapping_must_match_observed_containers(reconciled): + reconciled["repository_services"][0]["runtime_container_ids"] = [] + assert inventory._reconciliation_validation_error(reconciled) is not None + + +def test_real_systemd_microsecond_timestamps_are_preserved(monkeypatch): + monkeypatch.setattr(inventory.sys, "platform", "linux") + monkeypatch.setattr(inventory.shutil, "which", lambda name: "/usr/bin/systemctl") + monkeypatch.setattr(inventory, "_run_command", lambda *args, **kwargs: json.dumps([ + {"unit": "backup.timer", "activates": "backup.service", "next": 1700000000123456, "last": 0}, + ])) + timers = inventory._collect_timers("local_unix") + assert timers["units"][0]["next"] == "2023-11-14T22:13:20.123456Z" + assert timers["units"][0]["last"] is None + + +def test_unix_socket_timer_observations_are_not_engine_host_proof(monkeypatch): + monkeypatch.setattr(inventory.sys, "platform", "linux") + monkeypatch.setattr(inventory.shutil, "which", lambda name: "/usr/bin/systemctl") + monkeypatch.setattr(inventory, "_run_command", lambda *args, **kwargs: "[]") + timers = inventory._collect_timers("local_unix") + assert timers["host_scope"] == "collector" + + +def test_real_docker_string_reference_counts_are_preserved(monkeypatch): + monkeypatch.setattr(inventory, "_run_docker", lambda *args, **kwargs: json.dumps({"Volumes": [ + {"Name": "unused", "Links": "0", "Size": "0B"}, + {"Name": "used", "Links": "1", "Size": "1kB"}, + {"Name": "unknown", "Links": "N/A", "Size": "N/A"}, + ]})) + volumes, _, _ = inventory._collect_volume_rows("approved-production") + counts = {volume["name"]: volume["links"] for volume in volumes} + assert counts == {"unused": 0, "used": 1, "unknown": None} diff --git a/tools/README.md b/tools/README.md index d459e685..1c5b54ed 100644 --- a/tools/README.md +++ b/tools/README.md @@ -56,3 +56,7 @@ make platform-inventory-validate ENV=legacy Schema validity is separate from migration readiness. Validation prints the reconciliation gate state; `gate=blocked` is not a completed inventory or a production pass. The public report belongs in `docs/platform/live-inventory.md` and must omit private host paths, addresses, unrelated workload names, and raw runtime exports. The versioned inventory and environment intent schemas live in `infra/pulumi/shared/`. The environment contract uses typed values, explicit resolution status, and opaque evidence references; credential fields accept reference names only. + +The pass-reconciliation check requires all current production environment inputs to be confirmed with concrete values and evidence references; this contract defines no automatic `not_applicable` exemption. It also binds the observed context to its approved allowlist and cross-checks every observed container against its service or unmatched-owner record. Confirmed health and database facts need their supporting evidence, version/topology and data paths. + +`docker.collector_host_timers` explicitly describes the collector workstation, not the Docker engine host. A local Unix socket can lead to a Docker Desktop VM. The source host's actual job/writer map therefore still needs independent confirmation. Timestamp fields are UTC ISO 8601 or null for unavailable values. Recollect pre-repair artifacts rather than renaming their old timer field; schema validity and actual G0 acceptance remain separate. diff --git a/tools/platform/inventory.py b/tools/platform/inventory.py index 07e2b268..51692de2 100755 --- a/tools/platform/inventory.py +++ b/tools/platform/inventory.py @@ -25,6 +25,7 @@ DEFAULT_ARTIFACT_ROOT = ".artifacts/platform" EXCLUDED_SOURCE_PARTS = {".git", ".rendered", ".venv", "node_modules", "stacks", "__pycache__"} + CONTAINER_INSPECT_FORMAT = ( r'{{printf "{\"id\":%s,\"name\":%s,\"image_ref\":%s,' r'\"image_id\":%s,\"state\":%s,\"project\":%s,\"service\":%s,' @@ -294,19 +295,34 @@ def _normalize_stats(raw: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: } +def _systemd_timestamp(value: Any) -> Optional[str]: + """Normalize systemd's integer microseconds without losing precision.""" + if isinstance(value, bool) or not isinstance(value, int) or value <= 0 or value == 2**64 - 1: + return None + seconds, microseconds = divmod(value, 1_000_000) + try: + timestamp = datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=microseconds) + return timestamp.isoformat().replace("+00:00", "Z") + except (OSError, ValueError, OverflowError): + return None + + def _collect_timers(endpoint_kind: str) -> Dict[str, Any]: + # A local Docker Unix socket does not prove the engine shares the collector + # host (Docker Desktop for Linux runs its engine in a VM), so these timers are + # explicitly recorded as collector-host observations. if endpoint_kind != "local_unix" or not sys.platform.startswith("linux"): - return {"status": "not_local_linux", "units": []} + return {"status": "not_local_linux", "host_scope": "collector", "units": []} if shutil.which("systemctl") is None: - return {"status": "unavailable", "units": []} + return {"status": "unavailable", "host_scope": "collector", "units": []} command = ["systemctl", "list-timers", "--all", "--no-pager", "--no-legend", "--output=json"] try: output = _run_command(command, timeout=15) timers = json.loads(output) if output.strip() else [] except (InventoryError, json.JSONDecodeError): - return {"status": "unavailable", "units": []} + return {"status": "unavailable", "host_scope": "collector", "units": []} if not isinstance(timers, list): - return {"status": "unavailable", "units": []} + return {"status": "unavailable", "host_scope": "collector", "units": []} units = [] for timer in timers: if not isinstance(timer, dict) or not isinstance(timer.get("unit"), str): @@ -314,12 +330,16 @@ def _collect_timers(endpoint_kind: str) -> Dict[str, Any]: units.append( { "unit": timer["unit"], - "next": timer.get("next") if isinstance(timer.get("next"), str) else None, - "last": timer.get("last") if isinstance(timer.get("last"), str) else None, - "activates": timer.get("activates") if isinstance(timer.get("activates"), str) else None, + "next": _systemd_timestamp(timer.get("next")), + "last": _systemd_timestamp(timer.get("last")), + "activates": timer.get("activates") if isinstance(timer.get("activates"), str) else "unknown", } ) - return {"status": "collected", "units": sorted(units, key=lambda item: item["unit"])} + return { + "status": "collected", + "host_scope": "collector", + "units": sorted(units, key=lambda item: item["unit"]), + } @@ -456,6 +476,17 @@ def _source_service_records(repo_root: Path) -> Tuple[List[Dict[str, Any]], List return services, errors +def _parse_link_count(value: Any) -> Optional[int]: + """Parse Docker's string-valued volume reference counts.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return value if value >= 0 else None + if isinstance(value, str) and value.isdecimal(): + return int(value) + return None + + def _collect_volume_rows(context: str) -> Tuple[List[Dict[str, Any]], Dict[str, Optional[int]], str]: try: output = _run_docker(["system", "df", "--verbose", "--format", "json"], context, timeout=120) @@ -479,7 +510,7 @@ def _collect_volume_rows(context: str) -> Tuple[List[Dict[str, Any]], Dict[str, "name": name, "driver": volume.get("Driver") if isinstance(volume.get("Driver"), str) else "unknown", "scope": volume.get("Scope") if isinstance(volume.get("Scope"), str) else "unknown", - "links": volume.get("Links") if isinstance(volume.get("Links"), int) else None, + "links": _parse_link_count(volume.get("Links")), "size_bytes": size_bytes, "size_status": "measured" if size_bytes is not None else "unknown", } @@ -730,7 +761,7 @@ def _collect_docker(environment: str, requested_context: Optional[str]) -> Tuple "containers": sorted(containers, key=lambda item: item["name"]), "volumes": volume_rows, "volume_inventory_status": volume_status, - "host_timers": timers, + "collector_host_timers": timers, } return docker, errors @@ -801,14 +832,18 @@ def _reconciliation_validation_error(payload: Dict[str, Any]) -> Optional[str]: unresolved = unresolved or payload["docker"]["volume_inventory_status"] != "collected" if payload["docker"]["running_count"] and payload["docker"]["resource_usage_status"] != "collected": unresolved = True - for item in payload["environment_contract"]["inputs"].values(): - if item["status"] not in ("confirmed", "not_applicable") or not item["evidence_ref"]: - unresolved = True - if item["status"] == "confirmed" and item["value"] is None: + inputs = payload["environment_contract"]["inputs"] + # All current production inputs are required by G0. No exemption rule has + # been approved for this contract, so a not_applicable input cannot pass. + for item in inputs.values(): + if item["status"] != "confirmed" or not item["evidence_ref"] or not item["evidence_ref"].strip(): unresolved = True - if item["status"] == "not_applicable" and item["value"] is not None: + if item["value"] is None or (isinstance(item["value"], str) and not item["value"].strip()): unresolved = True - inputs = payload["environment_contract"]["inputs"] + observed_context = payload["docker"]["context_name"].strip() + allowed = inputs["allowed_docker_contexts"]["value"] + if not isinstance(allowed, list) or not observed_context or observed_context not in allowed: + unresolved = True if inputs["provider"]["status"] != "confirmed" or inputs["provider"]["value"] != "hcloud": unresolved = True required_safety_policy = { @@ -825,6 +860,8 @@ def _reconciliation_validation_error(payload: Dict[str, Any]) -> Optional[str]: if service["runtime_match_status"] == "ambiguous": unresolved = True for field in ("current_deployment_owner", "target_deployment_owner"): + if not service[field].strip(): + unresolved = True if any(token in service[field].lower() for token in ("unknown", "unresolved", "pending")): unresolved = True if ( @@ -837,8 +874,60 @@ def _reconciliation_validation_error(payload: Dict[str, Any]) -> Optional[str]: or not service["evidence_refs"] ): unresolved = True + if service["health_contract"]["status"] == "confirmed" and not service["health_contract"].get("evidence_ref"): + unresolved = True + if service["database_runtime"]["status"] == "confirmed": + if ( + not service["database_runtime"].get("server_version") + or not service["database_runtime"].get("topology_ref") + or not service["database_runtime"].get("data_paths") + ): + unresolved = True + observed_containers = { + container["id"]: container + for container in payload["docker"]["containers"] + if isinstance(container.get("id"), str) and container["id"] + } + if len(observed_containers) != len(payload["docker"]["containers"]): + unresolved = True + services_by_id = {service["id"]: service for service in payload["repository_services"]} + if len(services_by_id) != len(payload["repository_services"]): + unresolved = True + for observed in observed_containers.values(): + service_id = observed.get("source_service_id") + if service_id is not None and ( + service_id not in services_by_id + or observed["id"] not in services_by_id[service_id]["runtime_container_ids"] + ): + unresolved = True + for service in services_by_id.values(): + runtime_ids = service["runtime_container_ids"] + if len(runtime_ids) != len(set(runtime_ids)) or any( + container_id not in observed_containers + or observed_containers[container_id].get("source_service_id") != service["id"] + for container_id in runtime_ids + ): + unresolved = True + unmatched_ids = [container.get("container_id") for container in payload["unmatched_containers"]] + if len(unmatched_ids) != len(set(unmatched_ids)): + unresolved = True + if set(unmatched_ids) != { + container["id"] for container in observed_containers.values() if container.get("source_service_id") is None + }: + unresolved = True for container in payload["unmatched_containers"]: - if container["disposition"] == "unresolved" or "unknown" in container["deployment_owner"].lower(): + owner = str(container.get("deployment_owner") or "").strip() + if not owner or any(token in owner.lower() for token in ("unknown", "unresolved", "pending")): + unresolved = True + if container["disposition"] == "unresolved": + unresolved = True + observed = observed_containers.get(container["container_id"]) + if ( + observed is None + or observed.get("name") != container.get("name") + or observed.get("compose_project") != container.get("compose_project") + or observed.get("compose_service") != container.get("compose_service") + ): unresolved = True if unresolved: return "reconciliation gate cannot pass while required inventory fields are unresolved"