From 25a6636186b3c0363bb7b31b200d29f451d3093f Mon Sep 17 00:00:00 2001 From: Portll Date: Sat, 7 Nov 2026 19:13:20 +1030 Subject: [PATCH] feat(schema): bind the three manifests data files to schemas by name eol-snapshot.json, install-catalog.json and security-baseline.map.json had no schema at schema/.schema.json, the path schema-canon and spine's declared-schema binder both look up. - schema/eol-snapshot.schema.json: new. endoflife.date field names; eol is a date, true, false or null, as bin/eol-snapshot.mjs writes. - schema/install-catalog.schema.json: new. Closed per-tool keys, id-check records, release pins and the cosign fields lib/release-signing.mjs requires. - schema/manifest-map.schema.json renamed to security-baseline.map.schema.json. - registry-inventory: eol-snapshot reclassified feed to store; install-catalog named manifest.schema.json, which it fails on 4 keys. - bin/test/data-file-schemas.test.mjs: broken copies refused at the key. No data file changed. --- bin/test/data-file-schemas.test.mjs | 99 +++++++++++ monitor/registry-inventory.json | 15 +- package.json | 2 +- schema/eol-snapshot.schema.json | 32 ++++ schema/install-catalog.schema.json | 164 ++++++++++++++++++ ...json => security-baseline.map.schema.json} | 2 +- 6 files changed, 306 insertions(+), 8 deletions(-) create mode 100644 bin/test/data-file-schemas.test.mjs create mode 100644 schema/eol-snapshot.schema.json create mode 100644 schema/install-catalog.schema.json rename schema/{manifest-map.schema.json => security-baseline.map.schema.json} (97%) diff --git a/bin/test/data-file-schemas.test.mjs b/bin/test/data-file-schemas.test.mjs new file mode 100644 index 00000000..f07c16a8 --- /dev/null +++ b/bin/test/data-file-schemas.test.mjs @@ -0,0 +1,99 @@ +// The three manifests data files bind to schema/.schema.json by name, the rule +// monitor/test/schema-canon.test.mjs and spine's compile/config/declared.mjs both apply. +// Each schema refuses a broken copy at the broken key. + +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync, existsSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { validateAgainstSchema } from '../../monitor/registry.mjs'; + +const REPO = resolve(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const STEMS = ['eol-snapshot', 'install-catalog', 'security-baseline.map']; +const schemaPath = (stem) => resolve(REPO, 'schema', `${stem}.schema.json`); +const load = (stem) => JSON.parse(readFileSync(resolve(REPO, 'manifests', `${stem}.json`), 'utf8')); +const errorsFor = (stem, doc) => validateAgainstSchema(doc, { path: schemaPath(stem) }).errors; + +function refusedAt(stem, doc, where) { + const errors = errorsFor(stem, doc); + assert.ok(errors.some((e) => e.startsWith(`schema: ${where}`)), `expected a refusal at ${where}, got:\n${errors.join('\n')}`); +} + +test('each data file has a by-name schema, the inventory names it, and the shipped file satisfies it', () => { + const inventory = JSON.parse(readFileSync(resolve(REPO, 'monitor', 'registry-inventory.json'), 'utf8')); + for (const stem of STEMS) { + assert.ok(existsSync(schemaPath(stem)), `schema/${stem}.schema.json is missing`); + const row = inventory.entries.find((e) => e.path === `manifests/${stem}.json`); + assert.equal(row?.schema, `schema/${stem}.schema.json`, `inventory row for ${stem}`); + assert.deepEqual(errorsFor(stem, load(stem)), [], stem); + } +}); + +test('eol-snapshot refuses a non-ISO eol date, an unknown cycle key and an empty product', () => { + const [product] = Object.keys(load('eol-snapshot').products); + const badDate = load('eol-snapshot'); + badDate.products[product][0].eol = '2021/04/30'; + refusedAt('eol-snapshot', badDate, `products.${product}[0].eol`); + + const extra = load('eol-snapshot'); + extra.products[product][0].lts = true; + refusedAt('eol-snapshot', extra, `products.${product}[0]: unknown key 'lts'`); + + const empty = load('eol-snapshot'); + empty.products[product] = []; + refusedAt('eol-snapshot', empty, `products.${product}: needs at least 1`); +}); + +test('install-catalog refuses a misspelt manager, an unknown id state and a non-hex pin', () => { + const cat = load('install-catalog'); + const [tool] = Object.keys(cat.tools); + const typo = load('install-catalog'); + typo.tools[tool].brw = 'x'; + refusedAt('install-catalog', typo, `tools.${tool}: unknown key 'brw'`); + + const state = load('install-catalog'); + state.idChecksByManager = { brew: { state: 'verified', checkedAt: null, how: 'probed every id in one pass' } }; + refusedAt('install-catalog', state, 'idChecksByManager.brew.state'); + + const pinned = Object.keys(cat.tools).find((n) => cat.tools[n].release?.darwin?.x64); + assert.ok(pinned, 'precondition: a tool with a pinned darwin x64 release'); + const pin = load('install-catalog'); + pin.tools[pinned].release.darwin.x64.sha256 = 'ABC'; + refusedAt('install-catalog', pin, `tools.${pinned}.release.darwin.x64.sha256`); +}); + +test('install-catalog refuses a cosign block missing what lib/release-signing.mjs requires', () => { + const cat = load('install-catalog'); + const signed = Object.keys(cat.tools).find((n) => cat.tools[n].release?.signing?.kind === 'cosign-checksums'); + assert.ok(signed, 'precondition: a tool with a cosign-checksums signing block'); + for (const key of ['checksums', 'identity', 'issuer']) { + const doc = load('install-catalog'); + delete doc.tools[signed].release.signing[key]; + refusedAt('install-catalog', doc, `tools.${signed}.release.signing: matched 0 of 2 oneOf`); + } +}); + +test('install-catalog keeps both postInstall and steps shapes, and refuses an empty argv', () => { + const ok = load('install-catalog'); + const [tool] = Object.keys(ok.tools); + ok.tools[tool].postInstall = [['run', 'this'], 'a note']; + ok.tools[tool].steps = { 'check-id': 'do this' }; + assert.deepEqual(errorsFor('install-catalog', ok), []); + + const empty = load('install-catalog'); + empty.tools[tool].postInstall = [[]]; + refusedAt('install-catalog', empty, `tools.${tool}.postInstall[0]: needs at least 1`); +}); + +test('security-baseline.map refuses a string line and an unknown entry key', () => { + const map = load('security-baseline.map'); + const [id] = Object.keys(map.byId); + const line = load('security-baseline.map'); + line.byId[id].line = '12'; + refusedAt('security-baseline.map', line, `byId.${id}.line`); + + const extra = load('security-baseline.map'); + extra.byId[id].owner = 'x'; + refusedAt('security-baseline.map', extra, `byId.${id}: unknown key 'owner'`); +}); diff --git a/monitor/registry-inventory.json b/monitor/registry-inventory.json index 6256211e..acd21ecc 100644 --- a/monitor/registry-inventory.json +++ b/monitor/registry-inventory.json @@ -77,9 +77,11 @@ }, { "path": "manifests/eol-snapshot.json", - "kind": "feed", - "schema": null, - "reason": "endoflife.date release cycles, compacted to cycle, eol and codename by bin/eol-snapshot.mjs. The dates are the publisher's; bin/eol-scan.mjs reads those three fields and lists a version it cannot place as undetermined." + "kind": "store", + "schema": "schema/eol-snapshot.schema.json", + "binding": "none", + "reason": "bin/eol-snapshot.mjs writes the compact shape and bin/eol-scan.mjs reads it back, so both ends are ours. The dates are endoflife.date's.", + "note": "Was a schema-less feed until 2026-11-07. monitor/test/schema-canon.test.mjs validates it by name." }, { "path": "manifests/bola/example.json", @@ -116,8 +118,9 @@ { "path": "manifests/install-catalog.json", "kind": "declaration", - "schema": "schema/manifest.schema.json", - "binding": "none" + "schema": "schema/install-catalog.schema.json", + "binding": "none", + "note": "Named schema/manifest.schema.json until 2026-11-07, which the file fails on 4 keys. monitor/test/schema-canon.test.mjs validates it by name." }, { "path": "manifests/jvm-egress-allowlist.json", @@ -164,7 +167,7 @@ { "path": "manifests/security-baseline.map.json", "kind": "store", - "schema": "schema/manifest-map.schema.json", + "schema": "schema/security-baseline.map.schema.json", "binding": "mirrored", "note": "Generated by bin/manifest-map.mjs; --check compares it byte-for-byte against a regeneration, which subsumes shape validation." }, diff --git a/package.json b/package.json index cb2b0c38..4cac5d58 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "commitwork", - "version": "0.6.125", + "version": "0.6.126", "description": "Repository security scanner, local CI runner and scheduled monitor, with remediation and compliance-evidence tooling. Manifest-driven; runs against any git repository.", "homepage": "https://www.commitwork.online", "repository": { diff --git a/schema/eol-snapshot.schema.json b/schema/eol-snapshot.schema.json new file mode 100644 index 00000000..9e848cd5 --- /dev/null +++ b/schema/eol-snapshot.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://commitwork.online/schema/eol-snapshot.schema.json", + "title": "End-of-life snapshot", + "description": "Offline copy of endoflife.date release cycles, written by bin/eol-snapshot.mjs and read by bin/eol-scan.mjs. Field names are endoflife.date's (cycle, eol, codename); the compact shape is commitwork's. eol is a date, true (ended, date unpublished), false (not ended), or null (upstream value unreadable).", + "$comment": "OASIS OpenEoX Core (CSD01, 2026-07-13) maps eol dates to end_of_life, but has no boolean or null form, so it is not the storage shape. No $ref: monitor/registry.mjs refuses it.", + "type": "object", + "required": ["source", "fetched", "products"], + "additionalProperties": false, + "properties": { + "source": { "type": "string", "minLength": 1, "maxLength": 200 }, + "fetched": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "products": { + "type": "object", + "propertyNames": { "pattern": "^[a-z0-9][a-z0-9-]*$" }, + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["cycle", "eol"], + "additionalProperties": false, + "properties": { + "cycle": { "type": "string", "minLength": 1, "maxLength": 64 }, + "eol": { "type": ["string", "boolean", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "codename": { "type": "string", "minLength": 1, "maxLength": 64, "pattern": "^[^A-Z]*$" } + } + } + } + } + } +} diff --git a/schema/install-catalog.schema.json b/schema/install-catalog.schema.json new file mode 100644 index 00000000..c56d9538 --- /dev/null +++ b/schema/install-catalog.schema.json @@ -0,0 +1,164 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://commitwork.online/schema/install-catalog.schema.json", + "title": "Scanner install catalogue", + "description": "Hand-authored tool data for `commitwork setup` (bin/setup.mjs). Key order is print order. brew, winget, scoop, pipx, npm, apt and dnf carry a package id; cargo, gem, composer and go carry a whole command. A null id means the manager has no package. An idChecks record overrides the idChecksByManager record for one id.", + "$comment": "No open format covers a per-tool map across brew, winget, scoop, pipx and command-string managers. No $ref: monitor/registry.mjs refuses it, so the id-check record is written out twice.", + "type": "object", + "required": ["tools"], + "additionalProperties": false, + "properties": { + "note": { "type": "string" }, + "idChecksByManager": { + "type": "object", + "propertyNames": { "pattern": "^(brew|winget|scoop|pipx|npm|apt|dnf|cargo|gem|composer|go|release)$" }, + "additionalProperties": { + "type": "object", + "required": ["state", "checkedAt", "how"], + "additionalProperties": false, + "properties": { + "state": { "enum": ["resolves", "unresolved", "unprobed"] }, + "checkedAt": { "type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "how": { "type": "string", "minLength": 21 } + } + } + }, + "tools": { + "type": "object", + "propertyNames": { "pattern": "^[a-z0-9][a-z0-9.-]*$" }, + "additionalProperties": { + "type": "object", + "required": ["why"], + "additionalProperties": false, + "properties": { + "why": { "type": "string", "minLength": 1 }, + "url": { "type": "string", "pattern": "^https://" }, + "manual": { "type": "boolean" }, + "brew": { "type": ["string", "null"], "minLength": 1 }, + "winget": { "type": ["string", "null"], "minLength": 1 }, + "scoop": { "type": ["string", "null"], "minLength": 1 }, + "pipx": { "type": "string", "minLength": 1 }, + "npm": { "type": "string", "minLength": 1 }, + "apt": { "type": "string", "minLength": 1 }, + "dnf": { "type": "string", "minLength": 1 }, + "cargo": { "type": "string", "minLength": 1 }, + "gem": { "type": "string", "minLength": 1 }, + "composer": { "type": "string", "minLength": 1 }, + "go": { "type": "string", "minLength": 1 }, + "platforms": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "enum": ["darwin", "linux", "win32"] } + }, + "providedBy": { "type": "string", "minLength": 1 }, + "requiredBy": { "type": "string", "minLength": 1 }, + "postInstall": { + "description": "A string is a note and never runs. An array holds argv arrays to run and string notes.", + "type": ["string", "array"], + "minLength": 1, + "items": { + "type": ["string", "array"], + "minLength": 1, + "minItems": 1, + "items": { "type": "string", "minLength": 1 } + } + }, + "steps": { + "description": "Manual steps: a list, or an object keyed by the check id that reports the step absent.", + "type": ["array", "object"], + "items": { "type": "string", "minLength": 1 }, + "additionalProperties": { "type": "string", "minLength": 1 } + }, + "ready": { + "type": "object", + "required": ["cmd", "expect"], + "additionalProperties": false, + "properties": { + "cmd": { "type": "array", "minItems": 1, "items": { "type": "string", "minLength": 1 } }, + "expect": { "type": "string", "minLength": 1 }, + "why": { "type": "string" } + } + }, + "requiresAccount": { + "type": "object", + "required": ["vendor"], + "additionalProperties": false, + "properties": { + "vendor": { "type": "string", "minLength": 1 }, + "needs": { "type": "array", "items": { "type": "string", "minLength": 1 } }, + "freeTier": { "type": "string" }, + "why": { "type": "string" } + } + }, + "idChecks": { + "type": "object", + "propertyNames": { "pattern": "^(brew|winget|scoop|pipx|npm|apt|dnf|cargo|gem|composer|go)$" }, + "additionalProperties": { + "type": "object", + "required": ["state", "checkedAt", "how"], + "additionalProperties": false, + "properties": { + "state": { "enum": ["resolves", "unresolved", "unprobed"] }, + "checkedAt": { "type": ["string", "null"], "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "how": { "type": "string", "minLength": 21 } + } + } + }, + "release": { + "description": "A pinned GitHub release asset per platform and arch. lib/release-signing.mjs refuses a cosign-checksums block missing checksums, identity or issuer.", + "type": "object", + "required": ["repo", "tag"], + "additionalProperties": false, + "properties": { + "repo": { "type": "string", "pattern": "^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$" }, + "tag": { "type": "string", "minLength": 1 }, + "signing": { + "type": "object", + "required": ["kind"], + "additionalProperties": false, + "properties": { + "kind": { "enum": ["cosign-checksums", "none"] }, + "checksums": { "type": "string", "minLength": 1 }, + "identity": { "type": "string", "minLength": 1 }, + "issuer": { "type": "string", "pattern": "^https://" }, + "checkedAt": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "verifiedAt": { "type": "string", "pattern": "^\\d{4}-\\d{2}-\\d{2}$" }, + "note": { "type": "string" } + }, + "oneOf": [ + { "properties": { "kind": { "const": "cosign-checksums" } }, "required": ["checksums", "identity", "issuer"] }, + { "properties": { "kind": { "const": "none" } } } + ] + }, + "darwin": { + "type": "object", + "additionalProperties": false, + "properties": { + "x64": { "type": "object", "required": ["asset", "sha256"], "additionalProperties": false, "properties": { "asset": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, + "arm64": { "type": "object", "required": ["asset", "sha256"], "additionalProperties": false, "properties": { "asset": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } } + } + }, + "linux": { + "type": "object", + "additionalProperties": false, + "properties": { + "x64": { "type": "object", "required": ["asset", "sha256"], "additionalProperties": false, "properties": { "asset": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, + "arm64": { "type": "object", "required": ["asset", "sha256"], "additionalProperties": false, "properties": { "asset": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } } + } + }, + "win32": { + "type": "object", + "additionalProperties": false, + "properties": { + "x64": { "type": "object", "required": ["asset", "sha256"], "additionalProperties": false, "properties": { "asset": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } }, + "arm64": { "type": "object", "required": ["asset", "sha256"], "additionalProperties": false, "properties": { "asset": { "type": "string", "minLength": 1 }, "sha256": { "type": "string", "pattern": "^[0-9a-f]{64}$" } } } + } + } + } + } + } + } + } + } +} diff --git a/schema/manifest-map.schema.json b/schema/security-baseline.map.schema.json similarity index 97% rename from schema/manifest-map.schema.json rename to schema/security-baseline.map.schema.json index b4e7e21f..12543dcc 100644 --- a/schema/manifest-map.schema.json +++ b/schema/security-baseline.map.schema.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$id": "https://commitwork.local/schema/manifest-map.schema.json", + "$id": "https://commitwork.local/schema/security-baseline.map.schema.json", "title": "manifest map", "description": "The generated index of a check manifest, written by bin/manifest-map.mjs. `line` is the whole point of the file — a reader jumps to a check instead of scanning 257K — so it is required on every entry and nullable only where the id string could not be located literally in the raw text.", "type": "object",