diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5783ddc..126b417 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -24,7 +24,7 @@ On a PR you can comment `@coderabbitai review` (incremental) or `@coderabbitai f ## Development guidelines -- Keep collectors **read-only** toward cloud accounts by default. The Phase 4 workload scan is the opt-in exception: it may create and delete snapshots in the account, and it stores package identifiers only +- Keep collectors **read-only** toward cloud accounts by default. `om scan workload` is the opt-in exception: it may create and delete snapshots in the account, and it stores package identifiers only - Document graph schema changes in `docs/` - Prefer small, reviewable PRs - Add tests for graph queries and normalization logic diff --git a/README.md b/README.md index 51a9b6c..fdb3f67 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ OpenSourceOM Core is the platform behind [opensourceom.org](https://opensourceom Traditional scanners flood you with CVEs and misconfigurations. OpenSourceOM connects the dots — showing which findings sit on paths from the internet to your sensitive data and privileged identities. -> **Status:** Phase 3 has shipped. Phase 4 (workload package inventory) and Phase 5 (secret and malware findings on that storage) are open. Further rule packs stay open for contributors ([#10](https://github.com/OpenSourceOM/core/issues/10)). See the [roadmap](./docs/ROADMAP.md). +> **Status:** Phase 3 has shipped. Phase 4 is open: `om scan workload` inventories AWS Linux root volumes. Phase 5 (secret and malware findings on that storage) is open. Further rule packs stay open for contributors ([#10](https://github.com/OpenSourceOM/core/issues/10)). See the [roadmap](./docs/ROADMAP.md). ## Why this exists @@ -93,6 +93,8 @@ go build -o om ./cmd/om # Or scan cloud / cluster inventory export AWS_REGION=us-east-1 ./om scan aws +# On an EC2 instance in that account, as root, after the API scan: +# sudo -E ./om scan workload ./om scan k8s # requires kubeconfig # Or ingest a custom collector (stdout is a graph batch) @@ -133,6 +135,7 @@ An API token is created only when an admin asks for one, from the console or wit ```bash ./om scan demo # sample environment, no credentials ./om scan aws +./om scan workload # root on an EC2 worker; Linux root-volume packages ./om scan azure # requires AZURE_SUBSCRIPTION_ID + az login ./om scan gcp # requires GCP_PROJECT_ID + ADC ./om scan k8s # requires kubeconfig or in-cluster credentials @@ -162,7 +165,7 @@ Full documentation: [opensourceom.org](https://opensourceom.org) (docs at [opens | **1** | Attack path queries, CVE enrichment, web UI, Azure/GCP collectors | | **2** | CSPM rules, blast radius, K8s connector, exports | | **3** | Plugin SDK, Helm, embedded rule pack, demo graph, inventory-backed CVEs, crown-jewel datastores, attack-path findings, and cloud audit context (CloudTrail, Activity Log, Admin Activity). Shipped. | -| **4** | Package inventory from persistent workload storage: root volumes, boot images, container layers, function bundles, shared filesystems. Open. | +| **4** | Package inventory from persistent workload storage. AWS Linux root volumes ship with `om scan workload`. The other clouds, boot images, container layers, function bundles, and shared filesystems are open. | | **5** | Secret and malware findings on storage Phase 4 already opens. Kind and path only. Open. | Details: [docs/ROADMAP.md](./docs/ROADMAP.md) diff --git a/collectors/README.md b/collectors/README.md index 1cc61ca..9bd1247 100644 --- a/collectors/README.md +++ b/collectors/README.md @@ -20,7 +20,7 @@ Cloud and platform ingestion plugins. Each collector normalizes provider APIs in The **demo** collector loads a fixed environment (internet-exposed web tier, production database, public/private buckets, admin vs app identities, public Kubernetes service). Use it to exercise CSPM packs without cloud credentials. -The **AWS** collector emits properties the CIS pack matches on: `open_ingress`, `imdsv2`, `public_ip`, S3 `encryption` / `versioning` / `public_access_block`, and IAM user `mfa` / `unused_access_keys`. It also records RDS and Aurora instances as datastores. S3 bucket tags and the RDS `TagList` copy `sensitivity` or `data-class` onto the datastore. `sensitivity` wins when both are set. The same scan reads CloudTrail management events from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. +The **AWS** collector emits properties the CIS pack matches on: `open_ingress`, `imdsv2`, `public_ip`, S3 `encryption` / `versioning` / `public_access_block`, and IAM user `mfa` / `unused_access_keys`. It also records RDS and Aurora instances as datastores. S3 bucket tags and the RDS `TagList` copy `sensitivity` or `data-class` onto the datastore. `sensitivity` wins when both are set. The same scan reads CloudTrail management events from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. `om scan aws` does not read instance disks. `om scan workload` is the separate command that inventories packages on Linux root volumes. The **Azure** collector records logical SQL servers. The same scan reads administrative Activity Log events from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. Entra ID sign-in logs are not part of this scan. The **GCP** collector records Cloud SQL instances. The same scan reads Admin Activity audit logs from the last 24 hours and stores the ones that name an identity and a resource on an exposed path. A failed lookup omits those events. Data Access logs are not part of this scan. A workload gets `CAN_ACCESS` only when a security group, firewall, or VPC path allows it. Azure copies the crown-jewel mark from storage-account and SQL-server tags. GCP copies it from a bucket label or a Cloud SQL user label. The keys are `sensitivity` and `data-class`. A plugin may set `sensitivity` on a datastore directly. diff --git a/deploy/helm/opensourceom/values.yaml b/deploy/helm/opensourceom/values.yaml index f481f96..be0042f 100644 --- a/deploy/helm/opensourceom/values.yaml +++ b/deploy/helm/opensourceom/values.yaml @@ -58,7 +58,8 @@ resources: # collector is enabled and credentials are set (a static key, an existing # Secret, or serviceAccountAuth). Kubernetes is off unless enabled; without # a kubeconfig Secret the chart creates a read-only ServiceAccount for this -# cluster. `om scan demo` is not scheduled. +# cluster. `om scan demo` and `om scan workload` are not scheduled. +# The workload scan mounts disks and runs on an EC2 instance in the account. scan: schedule: "0 */6 * * *" successfulJobsHistoryLimit: 3 diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 72017aa..1a5e633 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -5,7 +5,7 @@ SPDX-License-Identifier: Apache-2.0 # Architecture -> **Status:** Phase 3 has shipped. The collector plugin SDK, Helm, the embedded rule pack, and cloud audit context from CloudTrail, Activity Log, and Admin Activity logs are in the tree, alongside Phase 2 CSPM rules, blast radius, Kubernetes ingest, and exports. Phase 4 (package inventory from persistent workload storage) and Phase 5 (secret and malware findings on that storage) are open. Further rule packs stay open for contributors. +> **Status:** Phase 3 has shipped. The collector plugin SDK, Helm, the embedded rule pack, and cloud audit context from CloudTrail, Activity Log, and Admin Activity logs are in the tree, alongside Phase 2 CSPM rules, blast radius, Kubernetes ingest, and exports. Phase 4 is open: `om scan workload` inventories AWS Linux root volumes. Phase 5 (secret and malware findings on that storage) is open. Further rule packs stay open for contributors. ## Overview @@ -32,6 +32,7 @@ SPDX-License-Identifier: Apache-2.0 | Component | Location | Notes | |-----------|----------|-------| | **Collectors** | `internal/collectors/` | AWS, Azure, GCP, Kubernetes, demo; RDS, Azure SQL, and Cloud SQL are datastores; AWS emits CIS pack properties and recent CloudTrail management events; Azure emits recent Activity Log events; GCP emits recent Admin Activity audit logs | +| **Workload packages** | `internal/workload/` | `om scan workload` reads AWS Linux root volumes and sets `packages` | | **Plugin SDK** | `sdk/collector`, `internal/plugins/` | External executables; `om scan plugin` | | **Graph store** | `internal/graph/`, `migrations/` | PostgreSQL `nodes` + `edges` | | **Path queries** | `internal/graph/query.go` | Named queries including `internet-to-datastore` and `internet-to-sensitive-datastore` | @@ -51,7 +52,7 @@ SPDX-License-Identifier: Apache-2.0 Findings link to affected resources via `VIOLATES` edges. CSPM and CVE findings share the same `Finding` node type but use `finding_type` and `rule_id` / `cve_id` properties for provenance. -See [ADR 001](./adr/001-graph-schema-v0.md), [ADR 002](./adr/002-phase1-findings-ui.md), [ADR 003](./adr/003-phase2-cspm-rbac.md), [ADR 004](./adr/004-collector-plugin-sdk.md), and [ADR 005](./adr/005-console-admins.md). +See [ADR 001](./adr/001-graph-schema-v0.md), [ADR 002](./adr/002-phase1-findings-ui.md), [ADR 003](./adr/003-phase2-cspm-rbac.md), [ADR 004](./adr/004-collector-plugin-sdk.md), [ADR 005](./adr/005-console-admins.md), and [ADR 006](./adr/006-aws-root-volume-packages.md). ## Technology choices @@ -84,7 +85,9 @@ Further community rule packs (PCI and additional CIS mappings) stay open for con ## Phase 4 and Phase 5 -Planned, not in the tree. Phase 4 writes `packages` (CPE 2.3 names and package URLs) from persistent workload storage onto the workload node that enrichment already matches. The sources, in order, are AWS Linux root volumes, Azure VM and GCE root volumes, boot images, container image layers and function bundles, and shared filesystems (EFS, Azure Files, Filestore). The scan is opt-in. `om scan aws`, `om scan azure`, and `om scan gcp` stay read-only API collectors. The workload scan runs a worker in the account, and it stores package identifiers only. +Phase 4 writes `packages` (CPE 2.3 names and package URLs) from persistent workload storage onto the workload node that enrichment already matches. The sources, in order, are AWS Linux root volumes, Azure VM and GCE root volumes, boot images, container image layers and function bundles, and shared filesystems (EFS, Azure Files, Filestore). The scan is opt-in. `om scan aws`, `om scan azure`, and `om scan gcp` stay read-only API collectors. + +`om scan workload` is the AWS root-volume scan. It runs as root on an EC2 instance in the account, snapshots the root volume, mounts the copy read-only, and stores package identifiers only. A later `om scan aws` keeps `packages`. Windows, LVM, and non-root disks are skipped. The other Phase 4 sources are still open. See [ADR 006](./adr/006-aws-root-volume-packages.md). Phase 5 adds a finding for a secret pattern or a malware signature on storage that Phase 4 reader already opened. The finding stores the kind and the path. Object bodies and database rows are in neither phase. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 2b10a1b..ae20fd3 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -68,12 +68,14 @@ The scan is opt-in and separate from `om scan aws`, `om scan azure`, and `om sca The path, in order: -- [ ] **AWS Linux root volumes** — [#102](https://github.com/OpenSourceOM/core/issues/102) +- [x] **AWS Linux root volumes** — [#102](https://github.com/OpenSourceOM/core/issues/102) - [ ] **Azure VM and GCE root volumes** — the same walker, those snapshot APIs — [#103](https://github.com/OpenSourceOM/core/issues/103) - [ ] **Boot images** — an AMI and the Azure and GCE equivalents, scanned once and applied to instances launched from that image — [#104](https://github.com/OpenSourceOM/core/issues/104) - [ ] **Container image layers and function bundles** — layers for images already stored on Kubernetes workloads, plus Lambda deployment packages and the Azure and GCP equivalents. No volume attach — [#105](https://github.com/OpenSourceOM/core/issues/105) - [ ] **Shared filesystems** — EFS, Azure Files, and Filestore, when the application is installed there and a root-volume scan would miss it — [#106](https://github.com/OpenSourceOM/core/issues/106) +`om scan workload` reads Linux root volumes in the current AWS account. It runs as root on an EC2 instance in that account, snapshots the root volume, mounts the copy read-only, and writes CPE 2.3 names and package URLs onto `packages`. A later `om scan aws` leaves `packages` in place. Windows, LVM, and disks other than the root volume are skipped. + Object bodies and database rows stay out. They are customer data, not the installed software. A public bucket or database is already an exposure finding from the API scan. ## Phase 5 — Findings on inventoried storage diff --git a/docs/adr/006-aws-root-volume-packages.md b/docs/adr/006-aws-root-volume-packages.md new file mode 100644 index 0000000..ca48b9f --- /dev/null +++ b/docs/adr/006-aws-root-volume-packages.md @@ -0,0 +1,46 @@ + + +# ADR 006: AWS Linux root-volume package inventory + +**Status:** Accepted +**Date:** 2026-10-14 +**Phase:** 4 + +## Context + +`om enrich cve` writes a finding when `packages` on a workload matches a CPE 2.3 name or a versioned package URL. `om scan aws` records the instance from `DescribeInstances` and copies the AMI id into `image`. It does not read the root volume, so a package installed on the instance never reaches `packages`. + +Reading a volume means creating a snapshot and attaching a copy. That is a write to the account. The API collectors stay read-only. + +## Decision + +`om scan workload` is a separate command. It runs as root on an EC2 instance in the account and region being scanned. + +For each Linux instance with an EBS root volume, the command: + +1. Creates a snapshot of the root volume and a volume from that snapshot in the worker's availability zone. +2. Attaches the volume to the worker and mounts it read-only. +3. Reads the dpkg, apk, or rpm database and writes CPE 2.3 names and versioned package URLs onto `packages` of the existing workload node. +4. Unmounts, detaches, and deletes the volume and the snapshot, including when a later step fails. + +Windows instances, instance-store roots, LVM physical volumes, and non-root disks are skipped. The graph, logs, and exports store package identifiers. They do not store file contents. + +An inventory upsert that omits `packages` keeps the list already stored. `om scan aws` can run again without erasing the workload scan. An upsert that sets `packages`, including an empty list, replaces it. + +The IAM actions are `ec2:CreateSnapshot`, `ec2:AttachVolume`, `ec2:DeleteSnapshot`, `ec2:DeleteVolume`, the describe and tag calls around that lifecycle, and `kms:CreateGrant` plus `kms:Decrypt` (with `kms:DescribeKey`, `kms:GenerateDataKeyWithoutPlaintext`, and `kms:ReEncrypt*`) for an encrypted root volume. `om scan aws` does not gain these actions. + +Azure, GCE, boot images, container layers, function bundles, and shared filesystems stay on later Phase 4 issues. Secret and malware findings stay on Phase 5, after this reader exists. + +## Consequences + +The command has to run on an instance in the account. A process killed with SIGKILL can leave a snapshot or volume; both are tagged `opensourceom:scan=workload`. The Helm chart does not schedule this command. + +`om enrich cve` is unchanged. It attaches a CVE only when one of these identifiers matches. + +## References + +- [docs/ROADMAP.md](../ROADMAP.md) +- [Issue #102](https://github.com/OpenSourceOM/core/issues/102) diff --git a/go.mod b/go.mod index b0200eb..5cf1e49 100644 --- a/go.mod +++ b/go.mod @@ -13,14 +13,18 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v8 v8.0.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources/v3 v3.0.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage/v3 v3.0.0 + github.com/anchore/go-rpmdb v0.2.2 github.com/aws/aws-sdk-go-v2 v1.47.3 github.com/aws/aws-sdk-go-v2/config v1.33.6 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 github.com/aws/aws-sdk-go-v2/service/cloudtrail v1.65.4 github.com/aws/aws-sdk-go-v2/service/ec2 v1.338.1 github.com/aws/aws-sdk-go-v2/service/iam v1.64.4 github.com/aws/aws-sdk-go-v2/service/rds v1.130.3 github.com/aws/aws-sdk-go-v2/service/s3 v1.114.3 github.com/aws/aws-sdk-go-v2/service/sts v1.51.1 + github.com/aws/smithy-go v1.28.5 + github.com/glebarez/go-sqlite v1.23.0 github.com/jackc/pgx/v5 v5.11.0 github.com/spf13/cobra v1.10.2 golang.org/x/crypto v0.57.0 @@ -47,7 +51,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.20.6 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.20.1 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.5.6 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.8.6 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.5.6 // indirect @@ -58,10 +61,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/signin v1.10.1 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.38.1 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.43.1 // indirect - github.com/aws/smithy-go v1.28.5 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20260202195803-dba9d589def2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/envoyproxy/go-control-plane/envoy v1.37.0 // indirect github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect @@ -96,11 +99,14 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/spiffe/go-spiffe/v2 v2.8.1 // indirect github.com/x448/float16 v0.8.4 // indirect @@ -130,6 +136,10 @@ require ( k8s.io/klog/v2 v2.140.0 // indirect k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect k8s.io/utils v0.0.0-20260626114624-be93311217bd // indirect + modernc.org/libc v1.74.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.55.0 // indirect sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect diff --git a/go.sum b/go.sum index 387ac63..2238ab7 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0 github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.57.0/go.mod h1:dzcEjy1WJ0Q4u9twNR3LcLhNoYMRCrMCMafpxa0TjPQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0 h1:RoO5+d7uCmDqovLrHCr2/BuViUXvdcrNxyNM1pN9dDQ= github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.57.0/go.mod h1:YqwkQPrWSC7+byyc1VlKbWLBF5JsW5IoL6xUkemYSXk= +github.com/anchore/go-rpmdb v0.2.2 h1:44VC7mvFAxV9t5JUZYg3S9sqG+E94iZvyWACMDq9hSk= +github.com/anchore/go-rpmdb v0.2.2/go.mod h1:Y64EbetdqhryMjtk8IzXkdCQw/aNDx+feZCT9aj6rjs= github.com/aws/aws-sdk-go-v2 v1.47.3 h1:HRoT/MkNi3bjO5xF4dCzLzvVmkvRGHj3/cLINDnJejk= github.com/aws/aws-sdk-go-v2 v1.47.3/go.mod h1:xSl+OSI5ZUw6bSMupWc218zE5Pl+65MCCu3ySWFY99A= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.22 h1:oiEmPz/YMMeh/JvAE8xSxNCIsvWE6lpFe4ZPlyFykQI= @@ -111,6 +113,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= @@ -125,6 +129,8 @@ github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeO github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/glebarez/go-sqlite v1.23.0 h1:FyhIq4jqmgphQAUlY79zPldYGwISEZikaDfhiGWkkaI= +github.com/glebarez/go-sqlite v1.23.0/go.mod h1:IIYrOH3L0rHY3jb4IXOHoWdklNajSGUN2eJcvK8WrnI= github.com/go-jose/go-jose/v4 v4.1.4 h1:moDMcTHmvE6Groj34emNPLs/qtYXRVcd6S7NHbHz3kA= github.com/go-jose/go-jose/v4 v4.1.4/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -177,6 +183,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.10 h1:EMp+aOuXN6l8cE/gjF5Bt+vyZxsUuyCWe9chDWR/+uU= github.com/google/s2a-go v0.1.10/go.mod h1:pz4tyvwXvJLLbyrkh6FW1eS2zPUXMaTmyNhYtyP2tNw= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -185,6 +193,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.22 h1:NU4XpII6jD+Dxcot94 github.com/googleapis/enterprise-certificate-proxy v0.3.22/go.mod h1:L3D/IQExI6LqEjBdXcZQ1WluSgigQmSwBboFstVPM4w= github.com/googleapis/gax-go/v2 v2.26.2 h1:ydkmNXxj7bEmmeK5AihkKnWxyOyBR9TDebvp5L5izk8= github.com/googleapis/gax-go/v2 v2.26.2/go.mod h1:sMKqnMesnKH+3wiRJROcttA+cJoZoGbZl1vDQ8XYtGk= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= @@ -205,6 +215,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -213,6 +225,8 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFd github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= @@ -220,6 +234,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -268,6 +284,8 @@ go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.60.0 h1:79p50tfZlm0J9YfoDsSi639qSXNGVwEzOPLCxM2FsYU= golang.org/x/net v0.60.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.37.0 h1:JUlcxA8oAtauLfiH8FX2/FkAWHAdi0QtGCGc+hofE98= @@ -275,6 +293,7 @@ golang.org/x/oauth2 v0.37.0/go.mod h1:IxwZNxUULJmpBFf9K/9NTMSIfZZuvuTy1gGxhigP/5 golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= @@ -283,6 +302,8 @@ golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/time v0.16.0 h1:vMb6ptszcQMkcwiRTAuNNU50gom6++Q/6gY2hDM6VDE= golang.org/x/time v0.16.0/go.mod h1:rVKOqvZeKvrDKTQiAHJ7wmwP0RzleSphoEA9RcdLA0s= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= google.golang.org/api v0.299.0 h1:b3K+ydSMd0kh6TQI6bJyApRQfqQX2MfSOaVkpM59mJw= @@ -319,6 +340,34 @@ k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3d k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= +modernc.org/cc/v4 v4.29.0 h1:CXgwL8cvxmyzBQZzbSl/6xFtMCryb6u8IOqDci39cgc= +modernc.org/cc/v4 v4.29.0/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.6 h1:sBgfIwyN0TQ9C5hwIeuqyeAKyMWnbvj2fvpF4L11uzU= +modernc.org/ccgo/v4 v4.34.6/go.mod h1:SZ8YcN9NG7XVsQYdm6jYBvi8PQP1qi+kqB6OhjqI3Fk= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.4 h1:2g65LGVSmFQrXeITAw97x7hCRvZFcyE1uDP+7Vng7JI= +modernc.org/gc/v3 v3.1.4/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.74.1 h1:bdR4VTKFMC4966QSNZ05XLGI/VwzVa2kTUX51Dm0riQ= +modernc.org/libc v1.74.1/go.mod h1:uH4t5bOx3G3g9Xcmj10YKlTcVISlRDwv8VoQJG9n8Os= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.55.0 h1:hIFh0MCH0rGinQ/4KYb5/UbCkRkb+UP+OkLCVWa5MTM= +modernc.org/sqlite v1.55.0/go.mod h1:4ntCLuNmnH8+GNqjka1wNg7KJd5/Hi5FYp8K+XQ7GZw= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= diff --git a/internal/cmd/scan_workload.go b/internal/cmd/scan_workload.go new file mode 100644 index 0000000..c0ad938 --- /dev/null +++ b/internal/cmd/scan_workload.go @@ -0,0 +1,119 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "fmt" + "os" + "strings" + "time" + + awscol "github.com/OpenSourceOM/core/internal/collectors/aws" + "github.com/OpenSourceOM/core/internal/workload" + awsv "github.com/OpenSourceOM/core/internal/workload/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ec2" + "github.com/spf13/cobra" +) + +var ( + workloadTimeout time.Duration + workloadInstances []string +) + +var scanWorkloadCmd = &cobra.Command{ + Use: "workload", + Short: "Inventory packages on AWS Linux root volumes", + Long: workloadLong(), + RunE: runScanWorkload, +} + +func workloadLong() string { + return `Inventory packages on Linux root volumes in the current AWS account. + +Run it as root on an EC2 instance in that account. The command snapshots each root volume, creates a volume from the snapshot in this instance's availability zone, attaches it, and mounts it read-only. It writes CPE 2.3 names and package URLs onto the workload node from om scan aws, then deletes the snapshot and the volume, including when a read fails. File contents stay on the volume. + +om scan aws stays a read-only API scan. A later om scan aws keeps packages already stored on the workload. Snapshots and volumes are tagged opensourceom:scan=workload, which is how to find one left behind by a killed process. + +This command reads Linux root volumes. Windows instances are skipped. LVM and disks other than the root volume are skipped. Azure, GCE, boot images, container layers, function bundles, and shared filesystems are later scans. + +IAM actions: + ` + strings.Join(awsv.Permissions, "\n ") +} + +func runScanWorkload(cmd *cobra.Command, args []string) error { + cfg := loadConfig() + ctx := cmd.Context() + worker, err := awsv.LookupWorker(ctx) + if err != nil { + return err + } + if err := awsv.CheckRegion(worker.Region, cfg.AWSRegion); err != nil { + return err + } + if os.Geteuid() != 0 { + return fmt.Errorf("om scan workload mounts the snapshot and must run as root on the worker") + } + + store, err := openGraphStore(ctx, cfg) + if err != nil { + return err + } + defer store.Close() + + collector, err := awscol.NewCollector(ctx, cfg.AWSRegion) + if err != nil { + return err + } + awsCfg, err := config.LoadDefaultConfig(ctx, config.WithRegion(cfg.AWSRegion)) + if err != nil { + return fmt.Errorf("load aws config: %w", err) + } + + scanner := &awsv.Scanner{ + API: ec2.NewFromConfig(awsCfg), + Worker: awsv.Worker{ + InstanceID: worker.InstanceID, + AvailabilityZone: worker.AvailabilityZone, + Region: worker.Region, + }, + Only: workloadInstances, + Timeout: workloadTimeout, + Mount: awsv.NewGuest().MountReadOnly, + Walk: workload.Walk, + OnStart: func(instanceID, name string) { + fmt.Printf("%s: snapshotting the root volume\n", workloadLabel(instanceID, name)) + }, + } + wrote := 0 + err = scanner.Scan(ctx, func(res awsv.Result) error { + label := workloadLabel(res.InstanceID, res.Name) + if res.Skipped != "" { + fmt.Printf("%s: skipped (%s)\n", label, res.Skipped) + return nil + } + nodeID := awscol.WorkloadNodeID(collector.AccountID, cfg.AWSRegion, res.InstanceID) + if err := store.SetWorkloadPackages(ctx, nodeID, res.Packages); err != nil { + return err + } + fmt.Printf("%s: %d package identifiers\n", label, len(res.Packages)) + wrote++ + return nil + }) + fmt.Printf("Stored packages on %d workloads.\n", wrote) + return err +} + +func workloadLabel(instanceID, name string) string { + if name != "" && name != instanceID { + return instanceID + " (" + name + ")" + } + return instanceID +} + +func init() { + scanWorkloadCmd.Flags().DurationVar(&workloadTimeout, "timeout", 30*time.Minute, "time limit for each instance") + scanWorkloadCmd.Flags().StringArrayVar(&workloadInstances, "instance", nil, "EC2 instance to scan (repeat to scan several; the default is every instance in the region)") + scanCmd.AddCommand(scanWorkloadCmd) +} diff --git a/internal/cmd/scan_workload_test.go b/internal/cmd/scan_workload_test.go new file mode 100644 index 0000000..4f6dd27 --- /dev/null +++ b/internal/cmd/scan_workload_test.go @@ -0,0 +1,25 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package cmd + +import ( + "strings" + "testing" + + awsv "github.com/OpenSourceOM/core/internal/workload/aws" +) + +func TestScanWorkloadRegistered(t *testing.T) { + for _, command := range scanCmd.Commands() { + if command.Name() == "workload" { + for _, action := range awsv.Permissions { + if !strings.Contains(command.Long, action) { + t.Errorf("help missing %s", action) + } + } + return + } + } + t.Fatal("om scan workload is not registered") +} diff --git a/internal/collectors/aws/collector.go b/internal/collectors/aws/collector.go index b72784c..4ce16df 100644 --- a/internal/collectors/aws/collector.go +++ b/internal/collectors/aws/collector.go @@ -450,7 +450,15 @@ func ec2Platform(instance ec2types.Instance) string { return "linux/unix" } +// WorkloadNodeID is the graph id om scan aws stores for an EC2 instance. +func WorkloadNodeID(accountID, region, instanceID string) string { + return fmt.Sprintf("aws:%s:%s:workload:%s", accountID, region, instanceID) +} + func (c *Collector) nodeID(kind, resource string) string { + if kind == "workload" { + return WorkloadNodeID(c.AccountID, c.Region, resource) + } return fmt.Sprintf("aws:%s:%s:%s:%s", c.AccountID, c.Region, kind, resource) } diff --git a/internal/collectors/aws/collector_test.go b/internal/collectors/aws/collector_test.go index 5acccf2..558f130 100644 --- a/internal/collectors/aws/collector_test.go +++ b/internal/collectors/aws/collector_test.go @@ -679,6 +679,13 @@ func TestS3CopiesSensitivityTag(t *testing.T) { assertSensitivity("logs", "", false) } +func TestWorkloadNodeID(t *testing.T) { + c := &Collector{Region: "us-east-1", AccountID: "111122223333"} + if c.nodeID("workload", "i-1") != WorkloadNodeID("111122223333", "us-east-1", "i-1") { + t.Fatalf("id = %s", c.nodeID("workload", "i-1")) + } +} + func testSecurityGroup(id string, open bool) ec2types.SecurityGroup { sg := ec2types.SecurityGroup{ GroupId: aws.String(id), diff --git a/internal/graph/store.go b/internal/graph/store.go index 2af2851..8b93aec 100644 --- a/internal/graph/store.go +++ b/internal/graph/store.go @@ -87,6 +87,8 @@ func upsertBatch(ctx context.Context, tx pgx.Tx, batch Batch) error { if err != nil { return err } + // packages is written by the workload scan. An API scan that omits the + // key keeps the list already stored on the node. _, err = tx.Exec(ctx, ` INSERT INTO nodes (id, type, name, provider, region, account_id, properties, updated_at) VALUES ($1, $2, $3, $4, $5, $6, $7, now()) @@ -96,7 +98,12 @@ func upsertBatch(ctx context.Context, tx pgx.Tx, batch Batch) error { provider = EXCLUDED.provider, region = EXCLUDED.region, account_id = EXCLUDED.account_id, - properties = EXCLUDED.properties, + properties = CASE + WHEN jsonb_exists(EXCLUDED.properties, 'packages') + OR NOT jsonb_exists(nodes.properties, 'packages') + THEN EXCLUDED.properties + ELSE EXCLUDED.properties || jsonb_build_object('packages', nodes.properties->'packages') + END, updated_at = now() `, node.ID, node.Type, node.Name, node.Provider, node.Region, node.AccountID, props) if err != nil { @@ -386,6 +393,31 @@ func (s *Store) InternetReachableWorkloadIDs(ctx context.Context) ([]string, err return ids, rows.Err() } +// SetWorkloadPackages stores package identifiers on an existing workload. +// A later inventory upsert that omits packages keeps this list. +func (s *Store) SetWorkloadPackages(ctx context.Context, id string, packages []string) error { + if packages == nil { + packages = []string{} + } + raw, err := json.Marshal(packages) + if err != nil { + return err + } + tag, err := s.pool.Exec(ctx, ` + UPDATE nodes + SET properties = jsonb_set(COALESCE(properties, '{}'::jsonb), '{packages}', $2::jsonb, true), + updated_at = now() + WHERE id = $1 AND type = $3 + `, id, string(raw), NodeWorkload) + if err != nil { + return fmt.Errorf("set packages on %s: %w", id, err) + } + if tag.RowsAffected() == 0 { + return fmt.Errorf("workload %s is not in the graph", id) + } + return nil +} + func (s *Store) GetNode(ctx context.Context, id string) (Node, error) { var node Node var props []byte diff --git a/internal/graph/store_test.go b/internal/graph/store_test.go index deaa4c8..269d003 100644 --- a/internal/graph/store_test.go +++ b/internal/graph/store_test.go @@ -5,6 +5,7 @@ package graph_test import ( "context" + "strings" "testing" "github.com/OpenSourceOM/core/internal/graph" @@ -118,6 +119,86 @@ func TestReplaceInventoryKeepsOtherNamespaces(t *testing.T) { requireNode(t, ctx, store, dev, true) } +func TestUpsertKeepsPackagesWhenScanOmitsThem(t *testing.T) { + ctx := context.Background() + store := openQueryFixture(t) + const id = "aws:111122223333:us-east-1:workload:i-1" + original := []any{ + "cpe:2.3:a:openssl:openssl:1.1.1k:*:*:*:*:*:*:*", + "pkg:rpm/amzn/openssl@1.1.1k-7.amzn2", + } + node := graph.Node{ + ID: id, Type: graph.NodeWorkload, Name: "web", Provider: "aws", + Region: "us-east-1", AccountID: "111122223333", + Properties: map[string]any{"public_ip": true, "packages": original}, + } + if err := store.UpsertBatch(ctx, graph.Batch{Nodes: []graph.Node{node}}); err != nil { + t.Fatal(err) + } + + node.Properties = map[string]any{"public_ip": false, "image": "ami-1"} + if err := store.UpsertBatch(ctx, graph.Batch{Nodes: []graph.Node{node}}); err != nil { + t.Fatal(err) + } + got := mustNode(t, ctx, store, id) + if got.Properties["public_ip"] != false || got.Properties["image"] != "ami-1" { + t.Fatalf("properties = %#v", got.Properties) + } + if joinPackages(t, got.Properties) != "cpe:2.3:a:openssl:openssl:1.1.1k:*:*:*:*:*:*:* pkg:rpm/amzn/openssl@1.1.1k-7.amzn2" { + t.Fatalf("packages = %#v", got.Properties["packages"]) + } + + if err := store.SetWorkloadPackages(ctx, id, []string{"pkg:deb/ubuntu/openssl@1.1.1n-0ubuntu1"}); err != nil { + t.Fatal(err) + } + got = mustNode(t, ctx, store, id) + if got.Properties["image"] != "ami-1" { + t.Fatalf("image was dropped: %#v", got.Properties) + } + if joinPackages(t, got.Properties) != "pkg:deb/ubuntu/openssl@1.1.1n-0ubuntu1" { + t.Fatalf("packages = %#v", got.Properties["packages"]) + } + + node.Properties = map[string]any{"public_ip": false, "packages": []any{}} + if err := store.UpsertBatch(ctx, graph.Batch{Nodes: []graph.Node{node}}); err != nil { + t.Fatal(err) + } + got = mustNode(t, ctx, store, id) + if joinPackages(t, got.Properties) != "" { + t.Fatalf("packages = %#v", got.Properties["packages"]) + } + + if err := store.SetWorkloadPackages(ctx, "missing", []string{"pkg:deb/debian/bash@5.0"}); err == nil { + t.Fatal("expected a missing workload to fail") + } +} + +func mustNode(t *testing.T, ctx context.Context, store *graph.Store, id string) graph.Node { + t.Helper() + node, err := store.GetNode(ctx, id) + if err != nil { + t.Fatal(err) + } + return node +} + +func joinPackages(t *testing.T, props map[string]any) string { + t.Helper() + raw, ok := props["packages"].([]any) + if !ok { + t.Fatalf("packages = %#v", props["packages"]) + } + parts := make([]string, len(raw)) + for i, item := range raw { + s, ok := item.(string) + if !ok { + t.Fatalf("package %d = %#v", i, item) + } + parts[i] = s + } + return strings.Join(parts, " ") +} + func findingNode(id, resource, account, findingType string) graph.Node { return graph.Node{ ID: "finding:" + id + ":" + resource, diff --git a/internal/workload/aws/guest.go b/internal/workload/aws/guest.go new file mode 100644 index 0000000..8499888 --- /dev/null +++ b/internal/workload/aws/guest.go @@ -0,0 +1,213 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/OpenSourceOM/core/internal/workload" +) + +type guest struct { + findDevice func(ctx context.Context, volumeID string) (string, error) + partitions func(disk string) []string + isLVM func(device string) bool + mount func(device, target string) error + unmount func(target string) error + isRoot func(target string) bool +} + +// NewGuest mounts an attached EBS volume read-only on this Linux instance. +func NewGuest() *guest { + return &guest{ + findDevice: waitForDisk, + partitions: readPartitions, + isLVM: deviceIsLVM, + mount: mountReadOnly, + unmount: unmount, + isRoot: workload.HasPackageDB, + } +} + +func (g *guest) MountReadOnly(ctx context.Context, volumeID string) (string, func() error, error) { + disk, err := g.findDevice(ctx, volumeID) + if err != nil { + return "", nil, err + } + candidates := g.partitions(disk) + if len(candidates) == 0 { + candidates = []string{disk} + } + sawLVM := false + var lastMount error + for _, device := range candidates { + if g.isLVM(device) { + sawLVM = true + continue + } + target, err := os.MkdirTemp("", "om-workload-") + if err != nil { + return "", nil, err + } + if err := g.mount(device, target); err != nil { + os.RemoveAll(target) + lastMount = err + continue + } + if g.isRoot(target) { + return target, func() error { return g.cleanup(target) }, nil + } + if err := g.cleanup(target); err != nil { + return "", nil, err + } + } + if lastMount != nil { + return "", nil, lastMount + } + if sawLVM { + return "", nil, workload.ErrLVM + } + return "", nil, workload.ErrNoPackages +} + +func (g *guest) cleanup(target string) error { + err := g.unmount(target) + return errors.Join(err, os.RemoveAll(target)) +} + +func waitForDisk(ctx context.Context, volumeID string) (string, error) { + var last error + for { + path, err := lookupDisk(volumeID) + if err == nil { + return path, nil + } + last = err + timer := time.NewTimer(time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return "", fmt.Errorf("volume %s did not appear on this instance: %w", volumeID, errors.Join(ctx.Err(), last)) + case <-timer.C: + } + } +} + +func lookupDisk(volumeID string) (string, error) { + entries, err := os.ReadDir("/dev/disk/by-id") + if err != nil { + return "", err + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + name := diskByID(names, volumeID) + if name == "" { + return "", fmt.Errorf("no disk for %s", volumeID) + } + return filepath.EvalSymlinks(filepath.Join("/dev/disk/by-id", name)) +} + +func diskByID(names []string, volumeID string) string { + serial := strings.ReplaceAll(volumeID, "-", "") + for _, name := range names { + if !volumeNameMatches(name, serial) { + continue + } + return name + } + return "" +} + +func volumeNameMatches(name, serial string) bool { + if strings.Contains(name, "-part") { + return false + } + i := strings.Index(name, serial) + if i < 0 { + return false + } + end := i + len(serial) + if end < len(name) && isHex(name[end]) { + return false + } + return true +} + +func isHex(b byte) bool { + return (b >= '0' && b <= '9') || (b >= 'a' && b <= 'f') || (b >= 'A' && b <= 'F') +} + +func readPartitions(disk string) []string { + base := filepath.Base(disk) + entries, err := os.ReadDir(filepath.Join("/sys/block", base)) + if err != nil { + return nil + } + names := make([]string, 0, len(entries)) + for _, entry := range entries { + names = append(names, entry.Name()) + } + return partitionDevices(disk, names) +} + +func partitionDevices(disk string, entries []string) []string { + base := filepath.Base(disk) + var out []string + for _, name := range entries { + if name != base && strings.HasPrefix(name, base) { + out = append(out, filepath.Join("/dev", name)) + } + } + sort.Strings(out) + return out +} + +func deviceIsLVM(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + buf := make([]byte, 8) + if _, err := f.ReadAt(buf, 512); err != nil { + return false + } + return string(buf) == "LABELONE" +} + +func mountReadOnly(device, target string) error { + options := []string{ + "ro,noexec,nosuid,nodev", + "ro,norecovery,noexec,nosuid,nodev", + } + var err error + for _, opt := range options { + cmd := exec.Command("mount", "-o", opt, device, target) + out, runErr := cmd.CombinedOutput() + if runErr == nil { + return nil + } + err = fmt.Errorf("mount %s: %s", device, strings.TrimSpace(string(out))) + } + return err +} + +func unmount(target string) error { + cmd := exec.Command("umount", target) + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("umount %s: %s", target, strings.TrimSpace(string(out))) + } + return nil +} diff --git a/internal/workload/aws/guest_test.go b/internal/workload/aws/guest_test.go new file mode 100644 index 0000000..79504b5 --- /dev/null +++ b/internal/workload/aws/guest_test.go @@ -0,0 +1,102 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/OpenSourceOM/core/internal/workload" +) + +func TestGuestMountsLinuxRoot(t *testing.T) { + var mounted, unmounted []string + g := &guest{ + findDevice: func(context.Context, string) (string, error) { return "/dev/nvme1n1", nil }, + partitions: func(string) []string { return []string{"/dev/nvme1n1p1", "/dev/nvme1n1p2"} }, + isLVM: func(string) bool { return false }, + mount: func(device, target string) error { + mounted = append(mounted, device) + if device == "/dev/nvme1n1p2" { + if err := os.MkdirAll(filepath.Join(target, "var/lib/dpkg"), 0o755); err != nil { + return err + } + return os.WriteFile(filepath.Join(target, "var/lib/dpkg/status"), []byte("Package: bash\n"), 0o644) + } + return nil + }, + unmount: func(target string) error { + unmounted = append(unmounted, target) + return nil + }, + isRoot: workload.HasPackageDB, + } + root, cleanup, err := g.MountReadOnly(context.Background(), "vol-1") + if err != nil { + t.Fatal(err) + } + if !workload.HasPackageDB(root) { + t.Fatal("mounted a filesystem without a package database") + } + if err := cleanup(); err != nil { + t.Fatal(err) + } + if len(mounted) != 2 || mounted[1] != "/dev/nvme1n1p2" { + t.Fatalf("mounted %v", mounted) + } + if len(unmounted) != 2 { + t.Fatalf("unmounted %d filesystems", len(unmounted)) + } +} + +func TestGuestSkipsLVM(t *testing.T) { + g := &guest{ + findDevice: func(context.Context, string) (string, error) { return "/dev/nvme1n1", nil }, + partitions: func(string) []string { return nil }, + isLVM: func(string) bool { return true }, + mount: func(string, string) error { + t.Fatal("LVM device was mounted") + return nil + }, + unmount: func(string) error { return nil }, + isRoot: workload.HasPackageDB, + } + _, _, err := g.MountReadOnly(context.Background(), "vol-1") + if err != workload.ErrLVM { + t.Fatalf("err = %v", err) + } +} + +func TestDiskByIDSkipsPartitions(t *testing.T) { + names := []string{ + "nvme-Amazon_Elastic_Block_Store_vol0abc-part1", + "nvme-Amazon_Elastic_Block_Store_vol0abc", + "nvme-Amazon_Elastic_Block_Store_vol0abcdef", + } + if got := diskByID(names, "vol-0abc"); got != "nvme-Amazon_Elastic_Block_Store_vol0abc" { + t.Fatalf("disk = %s", got) + } +} + +func TestPartitionDevices(t *testing.T) { + got := partitionDevices("/dev/nvme1n1", []string{"nvme1n1", "nvme1n1p2", "nvme1n1p1", "queue"}) + want := "/dev/nvme1n1p1,/dev/nvme1n1p2" + if len(got) != 2 || got[0]+","+got[1] != want { + t.Fatalf("partitions = %v", got) + } +} + +func TestDeviceIsLVM(t *testing.T) { + path := filepath.Join(t.TempDir(), "disk") + buf := make([]byte, 520) + copy(buf[512:], []byte("LABELONE")) + if err := os.WriteFile(path, buf, 0o644); err != nil { + t.Fatal(err) + } + if !deviceIsLVM(path) { + t.Fatal("expected an LVM label") + } +} diff --git a/internal/workload/aws/identity.go b/internal/workload/aws/identity.go new file mode 100644 index 0000000..8a47580 --- /dev/null +++ b/internal/workload/aws/identity.go @@ -0,0 +1,64 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" +) + +// LookupWorker reads this instance's id, availability zone, and region from IMDS. +func LookupWorker(ctx context.Context) (Worker, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return Worker{}, fmt.Errorf("load aws config: %w", err) + } + client := imds.NewFromConfig(cfg) + id, err := metadata(ctx, client, "instance-id") + if err != nil { + return Worker{}, fmt.Errorf("om scan workload runs on an EC2 instance in this account: %w", err) + } + az, err := metadata(ctx, client, "placement/availability-zone") + if err != nil { + return Worker{}, fmt.Errorf("read availability zone: %w", err) + } + region, err := metadata(ctx, client, "placement/region") + if err != nil { + return Worker{}, fmt.Errorf("read region: %w", err) + } + return Worker{InstanceID: id, AvailabilityZone: az, Region: region}, nil +} + +func metadata(ctx context.Context, client *imds.Client, path string) (string, error) { + out, err := client.GetMetadata(ctx, &imds.GetMetadataInput{Path: path}) + if err != nil { + return "", err + } + defer out.Content.Close() + body, err := io.ReadAll(out.Content) + if err != nil { + return "", err + } + value := strings.TrimSpace(string(body)) + if value == "" { + return "", fmt.Errorf("empty %s", path) + } + return value, nil +} + +// CheckRegion reports when the worker's region differs from AWS_REGION. +func CheckRegion(workerRegion, configured string) error { + if workerRegion == "" || configured == "" { + return fmt.Errorf("AWS region is empty") + } + if workerRegion != configured { + return fmt.Errorf("this instance is in %s; set AWS_REGION=%s", workerRegion, workerRegion) + } + return nil +} diff --git a/internal/workload/aws/identity_test.go b/internal/workload/aws/identity_test.go new file mode 100644 index 0000000..e3fecb2 --- /dev/null +++ b/internal/workload/aws/identity_test.go @@ -0,0 +1,16 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import "testing" + +func TestCheckRegion(t *testing.T) { + if err := CheckRegion("us-east-1", "us-east-1"); err != nil { + t.Fatal(err) + } + err := CheckRegion("us-west-2", "us-east-1") + if err == nil || err.Error() != "this instance is in us-west-2; set AWS_REGION=us-west-2" { + t.Fatalf("err = %v", err) + } +} diff --git a/internal/workload/aws/scan.go b/internal/workload/aws/scan.go new file mode 100644 index 0000000..128d6b9 --- /dev/null +++ b/internal/workload/aws/scan.go @@ -0,0 +1,545 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "context" + "errors" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/aws/smithy-go" +) + +// Permissions are the IAM actions om scan workload adds. +// kms:CreateGrant and kms:Decrypt are the grant an encrypted root volume needs. +var Permissions = []string{ + "ec2:DescribeInstances", + "ec2:CreateSnapshot", + "ec2:DescribeSnapshots", + "ec2:DeleteSnapshot", + "ec2:CreateVolume", + "ec2:DescribeVolumes", + "ec2:AttachVolume", + "ec2:DetachVolume", + "ec2:DeleteVolume", + "ec2:CreateTags", + "kms:CreateGrant", + "kms:Decrypt", + "kms:DescribeKey", + "kms:GenerateDataKeyWithoutPlaintext", + "kms:ReEncryptFrom", + "kms:ReEncryptTo", +} + +// Worker is the EC2 instance the scan runs on. +type Worker struct { + InstanceID string + AvailabilityZone string + Region string +} + +// Result is one instance after the snapshot has been deleted. +type Result struct { + InstanceID string + Name string + Packages []string + Skipped string +} + +// Scanner snapshots Linux root volumes, reads them, and deletes what it created. +type Scanner struct { + API ec2API + Worker Worker + Only []string + Timeout time.Duration + Poll time.Duration + Mount func(ctx context.Context, volumeID string) (root string, cleanup func() error, err error) + Walk func(root string) ([]string, error) + OnStart func(instanceID, name string) + + cleanupErrs []error +} + +type ec2API interface { + DescribeInstances(context.Context, *ec2.DescribeInstancesInput, ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) + CreateSnapshot(context.Context, *ec2.CreateSnapshotInput, ...func(*ec2.Options)) (*ec2.CreateSnapshotOutput, error) + DescribeSnapshots(context.Context, *ec2.DescribeSnapshotsInput, ...func(*ec2.Options)) (*ec2.DescribeSnapshotsOutput, error) + CreateVolume(context.Context, *ec2.CreateVolumeInput, ...func(*ec2.Options)) (*ec2.CreateVolumeOutput, error) + DescribeVolumes(context.Context, *ec2.DescribeVolumesInput, ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) + AttachVolume(context.Context, *ec2.AttachVolumeInput, ...func(*ec2.Options)) (*ec2.AttachVolumeOutput, error) + DetachVolume(context.Context, *ec2.DetachVolumeInput, ...func(*ec2.Options)) (*ec2.DetachVolumeOutput, error) + DeleteVolume(context.Context, *ec2.DeleteVolumeInput, ...func(*ec2.Options)) (*ec2.DeleteVolumeOutput, error) + DeleteSnapshot(context.Context, *ec2.DeleteSnapshotInput, ...func(*ec2.Options)) (*ec2.DeleteSnapshotOutput, error) +} + +var _ ec2API = (*ec2.Client)(nil) + +// Scan reads each Linux root volume and calls write after that instance's +// snapshot and volume are deleted. A skip leaves packages unchanged. +func (s *Scanner) Scan(ctx context.Context, write func(Result) error) error { + if s.Worker.InstanceID == "" || s.Worker.AvailabilityZone == "" { + return fmt.Errorf("worker instance id and availability zone are required") + } + if s.API == nil || s.Mount == nil || s.Walk == nil { + return fmt.Errorf("workload scan is missing an EC2 client, mount, or package walker") + } + targets, worker, err := s.list(ctx) + if err != nil { + return err + } + timeout := s.Timeout + if timeout <= 0 { + timeout = 30 * time.Minute + } + var failed []error + for _, inst := range targets { + ictx, cancel := context.WithTimeout(ctx, timeout) + res, scanErr := s.scanInstance(ictx, inst, worker) + cancel() + if scanErr != nil { + failed = append(failed, scanErr) + continue + } + if write != nil { + if err := write(res); err != nil { + failed = append(failed, fmt.Errorf("%s: %w", res.InstanceID, err)) + } + } + } + return errors.Join(append(failed, s.cleanupErrs...)...) +} + +func (s *Scanner) list(ctx context.Context) ([]ec2types.Instance, ec2types.Instance, error) { + instances, err := s.describe(ctx, nil) + if err != nil { + return nil, ec2types.Instance{}, err + } + byID := map[string]ec2types.Instance{} + for _, inst := range instances { + byID[aws.ToString(inst.InstanceId)] = inst + } + worker, ok := byID[s.Worker.InstanceID] + if !ok { + return nil, ec2types.Instance{}, fmt.Errorf("worker %s is not in this region", s.Worker.InstanceID) + } + if len(s.Only) == 0 { + return instances, worker, nil + } + var targets []ec2types.Instance + for _, id := range s.Only { + inst, ok := byID[id] + if !ok { + return nil, ec2types.Instance{}, fmt.Errorf("instance %s is not in this region", id) + } + targets = append(targets, inst) + } + return targets, worker, nil +} + +func (s *Scanner) describe(ctx context.Context, ids []string) ([]ec2types.Instance, error) { + input := &ec2.DescribeInstancesInput{} + if len(ids) > 0 { + input.InstanceIds = ids + } + var out []ec2types.Instance + for { + page, err := s.API.DescribeInstances(ctx, input) + if err != nil { + return nil, fmt.Errorf("describe instances: %w", err) + } + for _, reservation := range page.Reservations { + out = append(out, reservation.Instances...) + } + if page.NextToken == nil || *page.NextToken == "" { + return out, nil + } + input.NextToken = page.NextToken + } +} + +func (s *Scanner) scanInstance(ctx context.Context, inst, worker ec2types.Instance) (Result, error) { + id := aws.ToString(inst.InstanceId) + res := Result{InstanceID: id, Name: instanceName(inst)} + if !scannableState(inst) { + res.Skipped = "not running" + return res, nil + } + if isWindows(inst) { + res.Skipped = "Windows" + return res, nil + } + volumeID, skip, err := rootVolume(inst) + if err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } + if skip != "" { + res.Skipped = skip + return res, nil + } + + if s.OnStart != nil { + s.OnStart(id, res.Name) + } + snapID, err := s.createSnapshot(ctx, volumeID, id) + if err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } + defer s.deleteSnapshot(snapID) + if err := s.waitSnapshot(ctx, snapID); err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } + + volID, err := s.createVolume(ctx, snapID, id) + if err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } + defer s.deleteVolume(volID) + if err := s.waitVolume(ctx, volID, ec2types.VolumeStateAvailable); err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } + + device := freeDevice(worker.BlockDeviceMappings) + if device == "" { + return Result{}, fmt.Errorf("%s: no free device name on the worker", id) + } + _, err = s.API.AttachVolume(ctx, &ec2.AttachVolumeInput{ + Device: aws.String(device), + InstanceId: aws.String(s.Worker.InstanceID), + VolumeId: aws.String(volID), + }) + if err != nil { + return Result{}, fmt.Errorf("%s: attach volume: %w", id, err) + } + defer s.detach(volID) + if err := s.waitAttached(ctx, volID); err != nil { + return Result{}, fmt.Errorf("%s: %w", id, err) + } + + root, unmount, err := s.Mount(ctx, volID) + if err != nil { + if reason, ok := skipReason(err); ok { + res.Skipped = reason + return res, nil + } + return Result{}, fmt.Errorf("%s: %w", id, err) + } + defer func() { s.note(unmount()) }() + + pkgs, err := s.Walk(root) + if err != nil { + if reason, ok := skipReason(err); ok { + res.Skipped = reason + return res, nil + } + return Result{}, fmt.Errorf("%s: %w", id, err) + } + res.Packages = pkgs + return res, nil +} + +func (s *Scanner) createSnapshot(ctx context.Context, volumeID, instanceID string) (string, error) { + out, err := s.API.CreateSnapshot(ctx, &ec2.CreateSnapshotInput{ + VolumeId: aws.String(volumeID), + Description: aws.String("OpenSourceOM workload package inventory"), + TagSpecifications: []ec2types.TagSpecification{{ + ResourceType: ec2types.ResourceTypeSnapshot, + Tags: scanTags(instanceID), + }}, + }) + if err != nil { + return "", fmt.Errorf("create snapshot: %w", err) + } + id := aws.ToString(out.SnapshotId) + if id == "" { + return "", fmt.Errorf("create snapshot returned an empty snapshot id") + } + return id, nil +} + +func (s *Scanner) createVolume(ctx context.Context, snapID, instanceID string) (string, error) { + out, err := s.API.CreateVolume(ctx, &ec2.CreateVolumeInput{ + AvailabilityZone: aws.String(s.Worker.AvailabilityZone), + SnapshotId: aws.String(snapID), + VolumeType: ec2types.VolumeTypeGp3, + TagSpecifications: []ec2types.TagSpecification{{ + ResourceType: ec2types.ResourceTypeVolume, + Tags: scanTags(instanceID), + }}, + }) + if err != nil { + return "", fmt.Errorf("create volume: %w", err) + } + id := aws.ToString(out.VolumeId) + if id == "" { + return "", fmt.Errorf("create volume returned an empty volume id") + } + return id, nil +} + +func scanTags(instanceID string) []ec2types.Tag { + return []ec2types.Tag{ + {Key: aws.String("opensourceom:scan"), Value: aws.String("workload")}, + {Key: aws.String("opensourceom:instance"), Value: aws.String(instanceID)}, + {Key: aws.String("Name"), Value: aws.String("opensourceom-workload-" + instanceID)}, + } +} + +func (s *Scanner) waitSnapshot(ctx context.Context, id string) error { + return s.poll(ctx, func() (bool, error) { + out, err := s.API.DescribeSnapshots(ctx, &ec2.DescribeSnapshotsInput{SnapshotIds: []string{id}}) + if err != nil { + return false, fmt.Errorf("describe snapshot: %w", err) + } + if len(out.Snapshots) == 0 { + return false, fmt.Errorf("snapshot %s disappeared", id) + } + switch out.Snapshots[0].State { + case ec2types.SnapshotStateCompleted: + return true, nil + case ec2types.SnapshotStateError: + return false, fmt.Errorf("snapshot %s failed", id) + default: + return false, nil + } + }) +} + +func (s *Scanner) waitVolume(ctx context.Context, id string, want ec2types.VolumeState) error { + return s.poll(ctx, func() (bool, error) { + vol, err := s.volume(ctx, id) + if err != nil { + return false, err + } + if vol.State == ec2types.VolumeStateError { + return false, fmt.Errorf("volume %s entered the error state", id) + } + return vol.State == want, nil + }) +} + +func (s *Scanner) waitAttached(ctx context.Context, id string) error { + return s.poll(ctx, func() (bool, error) { + vol, err := s.volume(ctx, id) + if err != nil { + return false, err + } + if len(vol.Attachments) == 0 { + return false, nil + } + switch vol.Attachments[0].State { + case ec2types.VolumeAttachmentStateAttached: + return true, nil + case ec2types.VolumeAttachmentStateDetached: + return false, fmt.Errorf("volume %s detached during attach", id) + default: + return false, nil + } + }) +} + +func (s *Scanner) volume(ctx context.Context, id string) (ec2types.Volume, error) { + out, err := s.API.DescribeVolumes(ctx, &ec2.DescribeVolumesInput{VolumeIds: []string{id}}) + if err != nil { + return ec2types.Volume{}, fmt.Errorf("describe volume: %w", err) + } + if len(out.Volumes) == 0 { + return ec2types.Volume{}, fmt.Errorf("volume %s disappeared", id) + } + return out.Volumes[0], nil +} + +func (s *Scanner) detach(volID string) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + _, err := s.API.DetachVolume(ctx, &ec2.DetachVolumeInput{VolumeId: aws.String(volID)}) + if isNotFound(err) { + return + } + if err != nil { + s.note(fmt.Errorf("detach volume %s: %w", volID, err)) + return + } + err = s.poll(ctx, func() (bool, error) { + vol, err := s.volume(ctx, volID) + if isNotFound(err) { + return true, nil + } + if err != nil { + return false, err + } + return vol.State == ec2types.VolumeStateAvailable, nil + }) + if err != nil { + s.note(fmt.Errorf("detach volume %s: %w", volID, err)) + } +} + +func (s *Scanner) deleteVolume(volID string) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + err := s.poll(ctx, func() (bool, error) { + vol, err := s.volume(ctx, volID) + if isNotFound(err) || (err != nil && strings.Contains(err.Error(), "disappeared")) { + return true, nil + } + if err != nil { + return false, err + } + if vol.State == ec2types.VolumeStateError { + return true, nil + } + return vol.State == ec2types.VolumeStateAvailable, nil + }) + if err != nil { + s.note(fmt.Errorf("delete volume %s: %w", volID, err)) + return + } + _, err = s.API.DeleteVolume(ctx, &ec2.DeleteVolumeInput{VolumeId: aws.String(volID)}) + if isNotFound(err) { + return + } + if err != nil { + s.note(fmt.Errorf("delete volume %s: %w", volID, err)) + } +} + +func (s *Scanner) deleteSnapshot(snapID string) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, err := s.API.DeleteSnapshot(ctx, &ec2.DeleteSnapshotInput{SnapshotId: aws.String(snapID)}) + if isNotFound(err) { + return + } + if err != nil { + s.note(fmt.Errorf("delete snapshot %s: %w", snapID, err)) + } +} + +func (s *Scanner) note(err error) { + if err != nil { + s.cleanupErrs = append(s.cleanupErrs, err) + } +} + +func (s *Scanner) poll(ctx context.Context, ready func() (bool, error)) error { + interval := s.Poll + if interval <= 0 { + interval = 5 * time.Second + } + for { + ok, err := ready() + if err != nil || ok { + return err + } + timer := time.NewTimer(interval) + select { + case <-ctx.Done(): + timer.Stop() + return ctx.Err() + case <-timer.C: + } + } +} + +func isNotFound(err error) bool { + var api smithy.APIError + if !errors.As(err, &api) { + return false + } + switch api.ErrorCode() { + case "InvalidSnapshot.NotFound", "InvalidVolume.NotFound": + return true + default: + return false + } +} + +func skipReason(err error) (string, bool) { + var reason interface{ SkipReason() string } + if err != nil && errors.As(err, &reason) { + return reason.SkipReason(), true + } + return "", false +} + +func rootVolume(inst ec2types.Instance) (string, string, error) { + if inst.RootDeviceType == ec2types.DeviceTypeInstanceStore { + return "", "instance store root", nil + } + if inst.RootDeviceType != ec2types.DeviceTypeEbs { + return "", "no EBS root volume", nil + } + root := aws.ToString(inst.RootDeviceName) + for _, mapping := range inst.BlockDeviceMappings { + if aws.ToString(mapping.DeviceName) != root || mapping.Ebs == nil { + continue + } + id := aws.ToString(mapping.Ebs.VolumeId) + if id == "" { + return "", "", fmt.Errorf("root device %s has no volume id", root) + } + return id, "", nil + } + return "", "", fmt.Errorf("root device %s is not attached", root) +} + +func freeDevice(mappings []ec2types.InstanceBlockDeviceMapping) string { + used := map[string]bool{} + for _, mapping := range mappings { + name := aws.ToString(mapping.DeviceName) + used[name] = true + used[deviceAlias(name)] = true + } + for letter := byte('f'); letter <= 'p'; letter++ { + sd := "/dev/sd" + string(letter) + if !used[sd] && !used["/dev/xvd"+string(letter)] { + return sd + } + } + return "" +} + +func deviceAlias(name string) string { + switch { + case strings.HasPrefix(name, "/dev/xvd") && len(name) == len("/dev/xvdf"): + return "/dev/sd" + name[len(name)-1:] + case strings.HasPrefix(name, "/dev/sd") && len(name) == len("/dev/sdf"): + return "/dev/xvd" + name[len(name)-1:] + default: + return name + } +} + +func isWindows(inst ec2types.Instance) bool { + if inst.Platform == ec2types.PlatformValuesWindows { + return true + } + return strings.Contains(strings.ToLower(aws.ToString(inst.PlatformDetails)), "windows") +} + +func scannableState(inst ec2types.Instance) bool { + if inst.State == nil { + return false + } + switch inst.State.Name { + case ec2types.InstanceStateNameRunning, ec2types.InstanceStateNameStopped: + return true + default: + return false + } +} + +func instanceName(inst ec2types.Instance) string { + for _, tag := range inst.Tags { + if aws.ToString(tag.Key) == "Name" && tag.Value != nil { + return aws.ToString(tag.Value) + } + } + return aws.ToString(inst.InstanceId) +} diff --git a/internal/workload/aws/scan_test.go b/internal/workload/aws/scan_test.go new file mode 100644 index 0000000..cc1530b --- /dev/null +++ b/internal/workload/aws/scan_test.go @@ -0,0 +1,479 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package aws + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/OpenSourceOM/core/internal/workload" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +func TestPermissionsCoverSnapshotLifecycle(t *testing.T) { + for _, action := range []string{ + "ec2:CreateSnapshot", + "ec2:AttachVolume", + "ec2:DeleteSnapshot", + "ec2:DeleteVolume", + "kms:CreateGrant", + "kms:Decrypt", + } { + if !slices.Contains(Permissions, action) { + t.Errorf("missing %s", action) + } + } +} + +func TestScanWritesPackagesAndDeletesSnapshot(t *testing.T) { + fake := newFake(testInstances()) + var unmounted bool + var got []Result + scanner := testScanner(t, fake, func(ctx context.Context, volumeID string) (string, func() error, error) { + if !strings.HasPrefix(volumeID, "vol-scan-") { + t.Fatalf("mounted %s", volumeID) + } + dir := t.TempDir() + writeRoot(t, dir) + return dir, func() error { + unmounted = true + return nil + }, nil + }, workload.Walk) + scanner.Only = []string{"i-web"} + + err := scanner.Scan(context.Background(), func(res Result) error { + got = append(got, res) + return nil + }) + if err != nil { + t.Fatal(err) + } + if !unmounted { + t.Fatal("mount was left in place") + } + if len(got) != 1 || got[0].Skipped != "" { + t.Fatalf("results = %#v", got) + } + if got[0].Name != "web" { + t.Fatalf("name = %s", got[0].Name) + } + joined := strings.Join(got[0].Packages, "\n") + if !strings.Contains(joined, "pkg:deb/ubuntu/openssl@1.1.1n-0ubuntu1") { + t.Fatalf("packages =\n%s", joined) + } + if strings.Contains(joined, "PRIVATE KEY") || strings.Contains(joined, "AKIA") { + t.Fatalf("file contents leaked:\n%s", joined) + } + assertOrder(t, fake.log, "CreateSnapshot", "AttachVolume", "DetachVolume", "DeleteVolume", "DeleteSnapshot") + if len(fake.snapInputs) != 1 || aws.ToString(fake.snapInputs[0].VolumeId) != "vol-root" { + t.Fatalf("snapshots = %#v", fake.snapInputs) + } + if len(fake.volInputs) != 1 { + t.Fatal("expected one volume") + } + vol := fake.volInputs[0] + if aws.ToString(vol.AvailabilityZone) != "us-east-1a" || aws.ToString(vol.SnapshotId) != "snap-1" { + t.Fatalf("volume = %#v", vol) + } + if vol.KmsKeyId != nil || vol.Encrypted != nil || vol.VolumeType != ec2types.VolumeTypeGp3 { + t.Fatalf("volume encryption or type = %#v", vol) + } + if tagValue(vol.TagSpecifications[0].Tags, "opensourceom:scan") != "workload" { + t.Fatalf("tags = %#v", vol.TagSpecifications) + } + if len(fake.attachInputs) != 1 || aws.ToString(fake.attachInputs[0].Device) != "/dev/sdg" { + t.Fatalf("attach = %#v", fake.attachInputs) + } + if len(fake.vols) != 0 || len(fake.snaps) != 0 { + t.Fatalf("leftover snaps=%d vols=%d", len(fake.snaps), len(fake.vols)) + } +} + +func TestScanSkipsWindowsAndInstanceStore(t *testing.T) { + fake := newFake(append(testInstances(), + windowsInstance("i-win"), + storeInstance("i-store"), + )) + scanner := testScanner(t, fake, func(context.Context, string) (string, func() error, error) { + t.Fatal("skipped instance was mounted") + return "", nil, nil + }, workload.Walk) + scanner.Only = []string{"i-win", "i-store"} + var got []Result + err := scanner.Scan(context.Background(), func(res Result) error { + got = append(got, res) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) != 2 || got[0].Skipped != "Windows" || got[1].Skipped != "instance store root" { + t.Fatalf("results = %#v", got) + } + if slices.Contains(fake.log, "CreateSnapshot") { + t.Fatalf("snapshot created: %v", fake.log) + } +} + +func TestScanDeletesSnapshotWhenWalkFails(t *testing.T) { + fake := newFake(testInstances()) + scanner := testScanner(t, fake, func(context.Context, string) (string, func() error, error) { + return t.TempDir(), func() error { return nil }, nil + }, func(string) ([]string, error) { + return nil, errors.New("read rpm database: corrupt") + }) + scanner.Only = []string{"i-web"} + err := scanner.Scan(context.Background(), nil) + if err == nil || !strings.Contains(err.Error(), "corrupt") { + t.Fatalf("err = %v", err) + } + assertOrder(t, fake.log, "CreateSnapshot", "AttachVolume", "DetachVolume", "DeleteVolume", "DeleteSnapshot") + if len(fake.snaps) != 0 || len(fake.vols) != 0 { + t.Fatal("snapshot or volume was left behind") + } +} + +func TestScanSkipsLVMAfterCleanup(t *testing.T) { + fake := newFake(testInstances()) + scanner := testScanner(t, fake, func(context.Context, string) (string, func() error, error) { + return "", nil, workload.ErrLVM + }, func(string) ([]string, error) { + t.Fatal("LVM root was walked") + return nil, nil + }) + scanner.Only = []string{"i-web"} + var got []Result + if err := scanner.Scan(context.Background(), func(res Result) error { + got = append(got, res) + return nil + }); err != nil { + t.Fatal(err) + } + if len(got) != 1 || got[0].Skipped != workload.ErrLVM.Error() { + t.Fatalf("results = %#v", got) + } + assertOrder(t, fake.log, "CreateSnapshot", "DeleteSnapshot") + if len(fake.snaps) != 0 { + t.Fatal("snapshot left behind") + } +} + +func TestScanWaitsForSnapshotThenDeletesOnTimeout(t *testing.T) { + fake := newFake(testInstances()) + fake.stuckSnapshot = true + scanner := testScanner(t, fake, func(context.Context, string) (string, func() error, error) { + t.Fatal("mounted a pending snapshot") + return "", nil, nil + }, workload.Walk) + scanner.Only = []string{"i-web"} + scanner.Timeout = 40 * time.Millisecond + scanner.Poll = 5 * time.Millisecond + err := scanner.Scan(context.Background(), nil) + if err == nil { + t.Fatal("expected timeout") + } + if !slices.Contains(fake.log, "DeleteSnapshot") { + t.Fatalf("log = %v", fake.log) + } + if slices.Contains(fake.log, "CreateVolume") { + t.Fatal("volume created from a pending snapshot") + } +} + +func TestScanPaginatesInstances(t *testing.T) { + fake := newFake(testInstances()) + fake.pages = 2 + scanner := testScanner(t, fake, func(context.Context, string) (string, func() error, error) { + return "", nil, workload.ErrNoPackages + }, workload.Walk) + var ids []string + if err := scanner.Scan(context.Background(), func(res Result) error { + ids = append(ids, res.InstanceID) + return nil + }); err != nil { + t.Fatal(err) + } + if !slices.Equal(ids, []string{"i-worker", "i-web"}) { + t.Fatalf("ids = %v", ids) + } + if fake.describePages < 2 { + t.Fatalf("describe pages = %d", fake.describePages) + } +} + +func TestFreeDeviceSkipsNamesInUse(t *testing.T) { + got := freeDevice([]ec2types.InstanceBlockDeviceMapping{ + {DeviceName: aws.String("/dev/xvdf")}, + {DeviceName: aws.String("/dev/sdg")}, + }) + if got != "/dev/sdh" { + t.Fatalf("device = %s", got) + } +} + +func testScanner(t *testing.T, fake *fakeEC2, mount func(context.Context, string) (string, func() error, error), walk func(string) ([]string, error)) *Scanner { + t.Helper() + return &Scanner{ + API: fake, + Worker: Worker{ + InstanceID: "i-worker", + AvailabilityZone: "us-east-1a", + }, + Poll: time.Millisecond, + Mount: mount, + Walk: walk, + } +} + +func testInstances() []ec2types.Instance { + worker := linuxInstance("i-worker", "worker", "vol-worker") + worker.BlockDeviceMappings = append(worker.BlockDeviceMappings, ec2types.InstanceBlockDeviceMapping{ + DeviceName: aws.String("/dev/sdf"), + Ebs: &ec2types.EbsInstanceBlockDevice{VolumeId: aws.String("vol-extra")}, + }) + return []ec2types.Instance{ + worker, + linuxInstance("i-web", "web", "vol-root"), + } +} + +func linuxInstance(id, name, rootVol string) ec2types.Instance { + return ec2types.Instance{ + InstanceId: aws.String(id), + State: &ec2types.InstanceState{Name: ec2types.InstanceStateNameRunning}, + RootDeviceType: ec2types.DeviceTypeEbs, + RootDeviceName: aws.String("/dev/xvda"), + PlatformDetails: aws.String("Linux/UNIX"), + Tags: []ec2types.Tag{{Key: aws.String("Name"), Value: aws.String(name)}}, + BlockDeviceMappings: []ec2types.InstanceBlockDeviceMapping{ + {DeviceName: aws.String("/dev/xvda"), Ebs: &ec2types.EbsInstanceBlockDevice{VolumeId: aws.String(rootVol)}}, + {DeviceName: aws.String("/dev/xvdb"), Ebs: &ec2types.EbsInstanceBlockDevice{VolumeId: aws.String("vol-data")}}, + }, + } +} + +func windowsInstance(id string) ec2types.Instance { + inst := linuxInstance(id, "win", "vol-win") + inst.Platform = ec2types.PlatformValuesWindows + inst.PlatformDetails = aws.String("Windows") + return inst +} + +func storeInstance(id string) ec2types.Instance { + inst := linuxInstance(id, "store", "") + inst.RootDeviceType = ec2types.DeviceTypeInstanceStore + inst.BlockDeviceMappings = nil + return inst +} + +func writeRoot(t *testing.T, root string) { + t.Helper() + status := "Package: openssl\nStatus: install ok installed\nArchitecture: amd64\nVersion: 1.1.1n-0ubuntu1\nDescription: library\n BEGIN PRIVATE KEY\n" + for _, rel := range []string{"etc/os-release", "var/lib/dpkg/status", "home/app/notes.txt"} { + if err := os.MkdirAll(filepath.Dir(filepath.Join(root, rel)), 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(root, "etc/os-release"), []byte("ID=ubuntu\nVERSION_ID=\"22.04\"\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "var/lib/dpkg/status"), []byte(status), 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(root, "home/app/notes.txt"), []byte("AKIAIOSFODNN7EXAMPLE"), 0o644); err != nil { + t.Fatal(err) + } +} + +func assertOrder(t *testing.T, log []string, steps ...string) { + t.Helper() + at := -1 + for _, step := range steps { + next := slices.Index(log[at+1:], step) + if next < 0 { + t.Fatalf("log %v missing %s after position %d", log, step, at) + } + at += next + 1 + } +} + +func tagValue(tags []ec2types.Tag, key string) string { + for _, tag := range tags { + if aws.ToString(tag.Key) == key { + return aws.ToString(tag.Value) + } + } + return "" +} + +type snapState struct { + state ec2types.SnapshotState + seen int +} + +type volRec struct { + attached bool +} + +type fakeEC2 struct { + instances []ec2types.Instance + pages int + describePages int + log []string + snapN int + volN int + snaps map[string]*snapState + vols map[string]*volRec + snapInputs []*ec2.CreateSnapshotInput + volInputs []*ec2.CreateVolumeInput + attachInputs []*ec2.AttachVolumeInput + stuckSnapshot bool + delaySnapshot bool +} + +func newFake(instances []ec2types.Instance) *fakeEC2 { + return &fakeEC2{ + instances: instances, + snaps: map[string]*snapState{}, + vols: map[string]*volRec{}, + } +} + +func (f *fakeEC2) DescribeInstances(_ context.Context, in *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { + f.log = append(f.log, "DescribeInstances") + f.describePages++ + instances := f.instances + var next *string + if f.pages == 2 { + mid := (len(instances) + 1) / 2 + if in.NextToken == nil || aws.ToString(in.NextToken) == "" { + instances = instances[:mid] + token := "2" + next = &token + } else { + instances = instances[mid:] + } + } + return &ec2.DescribeInstancesOutput{ + NextToken: next, + Reservations: []ec2types.Reservation{{Instances: instances}}, + }, nil +} + +func (f *fakeEC2) CreateSnapshot(_ context.Context, in *ec2.CreateSnapshotInput, _ ...func(*ec2.Options)) (*ec2.CreateSnapshotOutput, error) { + f.log = append(f.log, "CreateSnapshot") + f.snapInputs = append(f.snapInputs, in) + f.snapN++ + id := fmt.Sprintf("snap-%d", f.snapN) + state := ec2types.SnapshotStateCompleted + if f.stuckSnapshot || f.delaySnapshot { + state = ec2types.SnapshotStatePending + } + f.snaps[id] = &snapState{state: state} + return &ec2.CreateSnapshotOutput{SnapshotId: aws.String(id)}, nil +} + +func (f *fakeEC2) DescribeSnapshots(_ context.Context, in *ec2.DescribeSnapshotsInput, _ ...func(*ec2.Options)) (*ec2.DescribeSnapshotsOutput, error) { + f.log = append(f.log, "DescribeSnapshots") + if len(in.SnapshotIds) == 0 { + return &ec2.DescribeSnapshotsOutput{}, nil + } + st := f.snaps[in.SnapshotIds[0]] + if st == nil { + return &ec2.DescribeSnapshotsOutput{}, nil + } + if f.delaySnapshot && st.state == ec2types.SnapshotStatePending { + st.seen++ + if st.seen > 1 { + st.state = ec2types.SnapshotStateCompleted + } + } + return &ec2.DescribeSnapshotsOutput{ + Snapshots: []ec2types.Snapshot{{ + SnapshotId: aws.String(in.SnapshotIds[0]), + State: st.state, + }}, + }, nil +} + +func (f *fakeEC2) CreateVolume(_ context.Context, in *ec2.CreateVolumeInput, _ ...func(*ec2.Options)) (*ec2.CreateVolumeOutput, error) { + f.log = append(f.log, "CreateVolume") + f.volInputs = append(f.volInputs, in) + f.volN++ + id := fmt.Sprintf("vol-scan-%d", f.volN) + f.vols[id] = &volRec{} + return &ec2.CreateVolumeOutput{VolumeId: aws.String(id)}, nil +} + +func (f *fakeEC2) DescribeVolumes(_ context.Context, in *ec2.DescribeVolumesInput, _ ...func(*ec2.Options)) (*ec2.DescribeVolumesOutput, error) { + f.log = append(f.log, "DescribeVolumes") + if len(in.VolumeIds) == 0 { + return &ec2.DescribeVolumesOutput{}, nil + } + id := in.VolumeIds[0] + rec := f.vols[id] + if rec == nil { + return &ec2.DescribeVolumesOutput{}, nil + } + vol := ec2types.Volume{VolumeId: aws.String(id), State: ec2types.VolumeStateAvailable} + if rec.attached { + vol.State = ec2types.VolumeStateInUse + vol.Attachments = []ec2types.VolumeAttachment{{ + VolumeId: aws.String(id), + State: ec2types.VolumeAttachmentStateAttached, + }} + } + return &ec2.DescribeVolumesOutput{Volumes: []ec2types.Volume{vol}}, nil +} + +func (f *fakeEC2) AttachVolume(_ context.Context, in *ec2.AttachVolumeInput, _ ...func(*ec2.Options)) (*ec2.AttachVolumeOutput, error) { + f.log = append(f.log, "AttachVolume") + f.attachInputs = append(f.attachInputs, in) + rec := f.vols[aws.ToString(in.VolumeId)] + if rec == nil { + return nil, errors.New("missing volume") + } + rec.attached = true + return &ec2.AttachVolumeOutput{}, nil +} + +func (f *fakeEC2) DetachVolume(_ context.Context, in *ec2.DetachVolumeInput, _ ...func(*ec2.Options)) (*ec2.DetachVolumeOutput, error) { + f.log = append(f.log, "DetachVolume") + rec := f.vols[aws.ToString(in.VolumeId)] + if rec == nil { + return nil, errors.New("missing volume") + } + rec.attached = false + return &ec2.DetachVolumeOutput{}, nil +} + +func (f *fakeEC2) DeleteVolume(_ context.Context, in *ec2.DeleteVolumeInput, _ ...func(*ec2.Options)) (*ec2.DeleteVolumeOutput, error) { + f.log = append(f.log, "DeleteVolume") + id := aws.ToString(in.VolumeId) + rec := f.vols[id] + if rec == nil { + return &ec2.DeleteVolumeOutput{}, nil + } + if rec.attached { + return nil, errors.New("volume in use") + } + delete(f.vols, id) + return &ec2.DeleteVolumeOutput{}, nil +} + +func (f *fakeEC2) DeleteSnapshot(_ context.Context, in *ec2.DeleteSnapshotInput, _ ...func(*ec2.Options)) (*ec2.DeleteSnapshotOutput, error) { + f.log = append(f.log, "DeleteSnapshot") + delete(f.snaps, aws.ToString(in.SnapshotId)) + return &ec2.DeleteSnapshotOutput{}, nil +} diff --git a/internal/workload/id.go b/internal/workload/id.go new file mode 100644 index 0000000..f4bbc6f --- /dev/null +++ b/internal/workload/id.go @@ -0,0 +1,125 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "strings" + "unicode" +) + +type osRelease struct { + ID string + VersionID string +} + +type software struct { + ecosystem string + name string + version string + cpeVersion string + epoch string + arch string +} + +func identifiers(osr osRelease, pkg software) []string { + if pkg.name == "" || pkg.version == "" || pkg.cpeVersion == "" { + return nil + } + ns := namespace(osr, pkg.ecosystem) + cpe := "cpe:2.3:a:" + cpeEscape(strings.ToLower(pkg.name)) + ":" + cpeEscape(strings.ToLower(pkg.name)) + ":" + cpeEscape(pkg.cpeVersion) + ":*:*:*:*:*:*:*" + purl := "pkg:" + pkg.ecosystem + "/" + purlEscape(ns) + "/" + purlEscape(pkg.name) + "@" + purlEscape(pkg.version) + var query []string + if pkg.arch != "" { + query = append(query, "arch="+purlEscape(pkg.arch)) + } + if distro := distroName(osr); distro != "" { + query = append(query, "distro="+purlEscape(distro)) + } + if pkg.epoch != "" && pkg.epoch != "0" { + query = append(query, "epoch="+purlEscape(pkg.epoch)) + } + if len(query) > 0 { + purl += "?" + strings.Join(query, "&") + } + return []string{cpe, purl} +} + +func namespace(osr osRelease, ecosystem string) string { + if osr.ID != "" { + return osr.ID + } + switch ecosystem { + case "deb": + return "debian" + case "rpm": + return "rhel" + default: + return "alpine" + } +} + +func distroName(osr osRelease) string { + if osr.ID == "" { + return "" + } + if osr.VersionID == "" { + return osr.ID + } + return osr.ID + "-" + osr.VersionID +} + +func cpeEscape(s string) string { + var b strings.Builder + for _, r := range s { + switch r { + case '\\', ':', '*', '?', '"', ' ': + b.WriteByte('\\') + } + b.WriteRune(r) + } + return b.String() +} + +func purlEscape(s string) string { + return strings.NewReplacer( + "%", "%25", + "/", "%2F", + "?", "%3F", + "#", "%23", + "@", "%40", + " ", "%20", + ).Replace(s) +} + +func splitDebVersion(v string) (epoch, upstream, full string) { + full = v + if i := strings.IndexByte(v, ':'); i > 0 && digits(v[:i]) { + epoch = v[:i] + full = v[i+1:] + } + upstream = full + if i := strings.LastIndexByte(full, '-'); i > 0 { + upstream = full[:i] + } + return epoch, upstream, full +} + +func apkUpstream(v string) string { + if i := strings.LastIndex(v, "-r"); i > 0 && digits(v[i+2:]) { + return v[:i] + } + return v +} + +func digits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if !unicode.IsDigit(r) { + return false + } + } + return true +} diff --git a/internal/workload/rpm.go b/internal/workload/rpm.go new file mode 100644 index 0000000..74286a1 --- /dev/null +++ b/internal/workload/rpm.go @@ -0,0 +1,72 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + + rpmdb "github.com/anchore/go-rpmdb/pkg" + // Pure-Go SQLite driver used to open rpmdb.sqlite. + _ "github.com/glebarez/go-sqlite" +) + +var rpmDBPaths = []string{ + "usr/lib/sysimage/rpm/rpmdb.sqlite", + "var/lib/rpm/rpmdb.sqlite", + "usr/lib/sysimage/rpm/Packages.db", + "var/lib/rpm/Packages.db", + "var/lib/rpm/Packages", +} + +func readRPM(root string) ([]software, bool, error) { + for _, rel := range rpmDBPaths { + path := filepath.Join(root, rel) + info, err := os.Stat(path) + if os.IsNotExist(err) { + continue + } + if err != nil { + return nil, false, fmt.Errorf("read rpm database: %w", err) + } + if info.Size() > maxPackageDBBytes { + return nil, false, fmt.Errorf("%s is %d bytes", filepath.Base(path), info.Size()) + } + db, err := rpmdb.Open(path) + if err != nil { + return nil, false, fmt.Errorf("read rpm database: %w", err) + } + // The database lists installed files. Only the name and version are kept. + list, err := db.ListPackages() + db.Close() + if err != nil { + return nil, false, fmt.Errorf("read rpm database: %w", err) + } + var out []software + for _, pkg := range list { + if pkg == nil || pkg.Name == "" || pkg.Name == "gpg-pubkey" || pkg.Version == "" { + continue + } + version := pkg.Version + if pkg.Release != "" { + version += "-" + pkg.Release + } + sw := software{ + ecosystem: "rpm", + name: pkg.Name, + version: version, + cpeVersion: pkg.Version, + arch: pkg.Arch, + } + if pkg.Epoch != nil && *pkg.Epoch > 0 { + sw.epoch = strconv.Itoa(*pkg.Epoch) + } + out = append(out, sw) + } + return out, true, nil + } + return nil, false, nil +} diff --git a/internal/workload/skip.go b/internal/workload/skip.go new file mode 100644 index 0000000..f02cf74 --- /dev/null +++ b/internal/workload/skip.go @@ -0,0 +1,18 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package workload + +// Skip is a condition that leaves the workload's packages unchanged. +type Skip string + +func (s Skip) Error() string { return string(s) } + +func (s Skip) SkipReason() string { return string(s) } + +const ( + // ErrNoPackages means the root filesystem has no dpkg, apk, or rpm database. + ErrNoPackages Skip = "no package database" + // ErrLVM means the root volume is an LVM physical volume. + ErrLVM Skip = "LVM root volumes are not scanned" +) diff --git a/internal/workload/walk.go b/internal/workload/walk.go new file mode 100644 index 0000000..1f1048c --- /dev/null +++ b/internal/workload/walk.go @@ -0,0 +1,249 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strings" +) + +const maxPackageDBBytes = 256 << 20 + +// packageDBPaths are the package databases Walk reads. File contents elsewhere +// on the filesystem are left unread. +var packageDBPaths = []string{ + "var/lib/dpkg/status", + "lib/apk/db/installed", + "usr/lib/apk/db/installed", + "usr/lib/sysimage/rpm/rpmdb.sqlite", + "var/lib/rpm/rpmdb.sqlite", + "usr/lib/sysimage/rpm/Packages.db", + "var/lib/rpm/Packages.db", + "var/lib/rpm/Packages", +} + +// HasPackageDB reports whether root contains a dpkg, apk, or rpm database. +func HasPackageDB(root string) bool { + for _, rel := range packageDBPaths { + if _, err := os.Stat(filepath.Join(root, rel)); err == nil { + return true + } + } + return false +} + +// Walk reads package databases under root and returns CPE 2.3 names and +// versioned package URLs. It does not return file contents. +func Walk(root string) ([]string, error) { + osr := readOSRelease(root) + var ( + found bool + pkgs []software + ) + for _, read := range []func(string) ([]software, bool, error){readDeb, readAPK, readRPM} { + list, ok, err := read(root) + if err != nil { + return nil, err + } + if !ok { + continue + } + found = true + pkgs = append(pkgs, list...) + } + if !found { + return nil, ErrNoPackages + } + seen := map[string]struct{}{} + var ids []string + for _, pkg := range pkgs { + for _, id := range identifiers(osr, pkg) { + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + ids = append(ids, id) + } + } + sort.Strings(ids) + return ids, nil +} + +func readOSRelease(root string) osRelease { + for _, rel := range []string{"etc/os-release", "usr/lib/os-release"} { + f, err := os.Open(filepath.Join(root, rel)) + if err != nil { + continue + } + osr := parseOSRelease(f) + f.Close() + return osr + } + return osRelease{} +} + +func parseOSRelease(r io.Reader) osRelease { + var osr osRelease + sc := bufio.NewScanner(io.LimitReader(r, 1<<20)) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok { + continue + } + value = strings.Trim(strings.TrimSpace(value), `"'`) + switch key { + case "ID": + osr.ID = strings.ToLower(value) + case "VERSION_ID": + osr.VersionID = value + } + } + return osr +} + +func readDeb(root string) ([]software, bool, error) { + path := filepath.Join(root, "var/lib/dpkg/status") + f, err := openDB(path) + if err != nil { + if os.IsNotExist(err) { + return nil, false, nil + } + return nil, false, fmt.Errorf("read dpkg status: %w", err) + } + defer f.Close() + return parseDeb(f), true, nil +} + +func parseDeb(r io.Reader) []software { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 2*1024*1024) + var fields map[string]string + var out []software + flush := func() { + if fields == nil { + return + } + name := fields["Package"] + version := fields["Version"] + if name != "" && version != "" && debInstalled(fields["Status"]) { + epoch, upstream, full := splitDebVersion(version) + out = append(out, software{ + ecosystem: "deb", + name: name, + version: full, + cpeVersion: upstream, + epoch: epoch, + arch: fields["Architecture"], + }) + } + fields = nil + } + for sc.Scan() { + line := sc.Text() + if line == "" { + flush() + continue + } + if strings.HasPrefix(line, " ") { + continue + } + key, value, ok := strings.Cut(line, ": ") + if !ok { + continue + } + if fields == nil { + fields = map[string]string{} + } + fields[key] = value + } + flush() + return out +} + +func debInstalled(status string) bool { + parts := strings.Fields(status) + return len(parts) == 3 && parts[2] == "installed" +} + +func readAPK(root string) ([]software, bool, error) { + for _, rel := range []string{"lib/apk/db/installed", "usr/lib/apk/db/installed"} { + f, err := openDB(filepath.Join(root, rel)) + if os.IsNotExist(err) { + continue + } + if err != nil { + return nil, false, fmt.Errorf("read apk database: %w", err) + } + pkgs := parseAPK(f) + f.Close() + return pkgs, true, nil + } + return nil, false, nil +} + +func parseAPK(r io.Reader) []software { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64*1024), 2*1024*1024) + var name, version, arch string + var out []software + flush := func() { + if name != "" && version != "" { + out = append(out, software{ + ecosystem: "apk", + name: name, + version: version, + cpeVersion: apkUpstream(version), + arch: arch, + }) + } + name, version, arch = "", "", "" + } + for sc.Scan() { + line := sc.Text() + if line == "" { + flush() + continue + } + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + switch key { + case "P": + name = value + case "V": + version = value + case "A": + arch = value + } + } + flush() + return out +} + +func openDB(path string) (*os.File, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + info, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + if info.Size() > maxPackageDBBytes { + f.Close() + return nil, fmt.Errorf("%s is %d bytes", filepath.Base(path), info.Size()) + } + return f, nil +} diff --git a/internal/workload/walk_test.go b/internal/workload/walk_test.go new file mode 100644 index 0000000..b17808a --- /dev/null +++ b/internal/workload/walk_test.go @@ -0,0 +1,206 @@ +// Copyright 2026 OpenSourceOM +// SPDX-License-Identifier: Apache-2.0 + +package workload + +import ( + "database/sql" + "encoding/binary" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/OpenSourceOM/core/internal/enrichment" +) + +func TestWalkDeb(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "etc/os-release"), "ID=ubuntu\nVERSION_ID=\"22.04\"\n") + writeFile(t, filepath.Join(root, "var/lib/dpkg/status"), ` +Package: openssl +Status: install ok installed +Architecture: amd64 +Version: 1.1.1n-0ubuntu1 +Description: library + BEGIN PRIVATE KEY + +Package: bash +Status: install ok installed +Architecture: amd64 +Version: 1:2.0.5-1 + +Package: old +Status: deinstall ok config-files +Architecture: all +Version: 1.0 + +Package: empty +Status: install ok installed +Architecture: amd64 +Version: +`) + writeFile(t, filepath.Join(root, "home/app/notes.txt"), "AKIAIOSFODNN7EXAMPLE") + + ids, err := Walk(root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(ids, "\n") + for _, forbidden := range []string{"PRIVATE KEY", "AKIAIOSFODNN7EXAMPLE", "library"} { + if strings.Contains(joined, forbidden) { + t.Fatalf("identifier list contains %q\n%s", forbidden, joined) + } + } + want := []string{ + "cpe:2.3:a:bash:bash:2.0.5:*:*:*:*:*:*:*", + "cpe:2.3:a:openssl:openssl:1.1.1n:*:*:*:*:*:*:*", + "pkg:deb/ubuntu/bash@2.0.5-1?arch=amd64&distro=ubuntu-22.04&epoch=1", + "pkg:deb/ubuntu/openssl@1.1.1n-0ubuntu1?arch=amd64&distro=ubuntu-22.04", + } + if strings.Join(ids, "\n") != strings.Join(want, "\n") { + t.Fatalf("packages =\n%s\nwant\n%s", strings.Join(ids, "\n"), strings.Join(want, "\n")) + } + assertParsed(t, ids) +} + +func TestWalkAPK(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "etc/os-release"), "ID=alpine\nVERSION_ID=3.19\n") + writeFile(t, filepath.Join(root, "lib/apk/db/installed"), ` +P:openssl +V:3.1.4-r0 +A:x86_64 + +P:busybox +V:1.36.1-r15 +A:x86_64 +`) + ids, err := Walk(root) + if err != nil { + t.Fatal(err) + } + want := []string{ + "cpe:2.3:a:busybox:busybox:1.36.1:*:*:*:*:*:*:*", + "cpe:2.3:a:openssl:openssl:3.1.4:*:*:*:*:*:*:*", + "pkg:apk/alpine/busybox@1.36.1-r15?arch=x86_64&distro=alpine-3.19", + "pkg:apk/alpine/openssl@3.1.4-r0?arch=x86_64&distro=alpine-3.19", + } + if strings.Join(ids, "\n") != strings.Join(want, "\n") { + t.Fatalf("packages =\n%s\nwant\n%s", strings.Join(ids, "\n"), strings.Join(want, "\n")) + } + assertParsed(t, ids) +} + +func TestWalkRPM(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "etc/os-release"), "ID=amzn\nVERSION_ID=\"2023\"\n") + if err := os.MkdirAll(filepath.Join(root, "usr/lib/sysimage/rpm"), 0o755); err != nil { + t.Fatal(err) + } + writeSQLiteRPMDB(t, filepath.Join(root, "usr/lib/sysimage/rpm/rpmdb.sqlite")) + + ids, err := Walk(root) + if err != nil { + t.Fatal(err) + } + joined := strings.Join(ids, "\n") + if strings.Contains(joined, "gpg-pubkey") { + t.Fatalf("gpg-pubkey was stored:\n%s", joined) + } + want := []string{ + "cpe:2.3:a:openssl:openssl:1.1.1k:*:*:*:*:*:*:*", + "pkg:rpm/amzn/openssl@1.1.1k-7.amzn2023?arch=x86_64&distro=amzn-2023", + } + if joined != strings.Join(want, "\n") { + t.Fatalf("packages =\n%s\nwant\n%s", joined, strings.Join(want, "\n")) + } + assertParsed(t, ids) +} + +func TestWalkMissingDatabase(t *testing.T) { + root := t.TempDir() + writeFile(t, filepath.Join(root, "etc/os-release"), "ID=ubuntu\n") + writeFile(t, filepath.Join(root, "etc/passwd"), "root:x:0:0:root:/root:/bin/bash\n") + _, err := Walk(root) + if err != ErrNoPackages { + t.Fatalf("err = %v", err) + } +} + +func TestCPEEscape(t *testing.T) { + got := identifiers(osRelease{ID: "debian"}, software{ + ecosystem: "deb", name: "weird", version: "1.0:2", cpeVersion: "1.0:2", + }) + if len(got) != 2 || !strings.Contains(got[0], `1.0\:2`) { + t.Fatalf("ids = %#v", got) + } + assertParsed(t, got) +} + +func assertParsed(t *testing.T, ids []string) { + t.Helper() + raw := make([]any, len(ids)) + for i, id := range ids { + raw[i] = id + } + inv := enrichment.ParseInventory(map[string]any{"packages": raw}) + if len(inv.Packages) != len(ids) { + t.Fatalf("parsed %d of %d: %#v", len(inv.Packages), len(ids), ids) + } +} + +func writeFile(t *testing.T, path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } +} + +func writeSQLiteRPMDB(t *testing.T, path string) { + t.Helper() + db, err := sql.Open("sqlite", path) + if err != nil { + t.Fatal(err) + } + defer db.Close() + if _, err := db.Exec(`CREATE TABLE Packages (hnum INTEGER PRIMARY KEY, blob BLOB NOT NULL)`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec( + `INSERT INTO Packages (hnum, blob) VALUES (1, ?), (2, ?)`, + rpmHeader("openssl", "1.1.1k", "7.amzn2023", "x86_64"), + rpmHeader("gpg-pubkey", "abcd", "1234", "x86_64"), + ); err != nil { + t.Fatal(err) + } +} + +func rpmHeader(name, version, release, arch string) []byte { + fields := []struct { + tag uint32 + val string + }{ + {1000, name}, + {1001, version}, + {1002, release}, + {1022, arch}, + } + var data []byte + var entries [][4]uint32 + for _, field := range fields { + entries = append(entries, [4]uint32{field.tag, 6, uint32(len(data)), 1}) + data = append(data, append([]byte(field.val), 0)...) + } + b := binary.BigEndian.AppendUint32(nil, uint32(len(entries))) + b = binary.BigEndian.AppendUint32(b, uint32(len(data))) + for _, entry := range entries { + for _, v := range entry { + b = binary.BigEndian.AppendUint32(b, v) + } + } + return append(b, data...) +}