diff --git a/python/understack-workflows/pyproject.toml b/python/understack-workflows/pyproject.toml index 3e122c907..eedc7b6ed 100644 --- a/python/understack-workflows/pyproject.toml +++ b/python/understack-workflows/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ ] [project.scripts] +apply-node-runbook = "understack_workflows.main.apply_node_runbook:main" bmc-kube-password = "understack_workflows.main.bmc_display_password:main" bmc-password = "understack_workflows.main.print_bmc_password:main" enroll-fw = "understack_workflows.main.enroll_fw:main" diff --git a/python/understack-workflows/tests/test_apply_node_runbook.py b/python/understack-workflows/tests/test_apply_node_runbook.py new file mode 100644 index 000000000..882b6fe74 --- /dev/null +++ b/python/understack-workflows/tests/test_apply_node_runbook.py @@ -0,0 +1,300 @@ +import argparse +from types import SimpleNamespace +from typing import cast +from unittest.mock import call + +import pytest +from ironicclient.v1.node import Node + +from understack_workflows import ironic_node +from understack_workflows.main import apply_node_runbook + +# --- ironic_node.run_runbooks_standalone (state bracketing) ------------------- + + +def _patch_client( + mocker, + *, + provision_state, + runbook_traits=("CUSTOM_DELL",), + node_traits=("CUSTOM_DELL",), +): + client = mocker.patch.object(ironic_node, "IronicClient").return_value + client.get_runbook.return_value = SimpleNamespace( + uuid="runbook-123", traits=list(runbook_traits) + ) + client.get_node_traits.return_value = list(node_traits) + mocker.patch.object( + ironic_node, + "refresh", + side_effect=[SimpleNamespace(uuid="node-123", provision_state=provision_state)] + + [SimpleNamespace(uuid="node-123", provision_state="manageable")] * 10, + ) + return client + + +def test_standalone_available_node_is_bracketed_manage_and_provide(mocker): + client = _patch_client(mocker, provision_state="available") + node = cast(Node, SimpleNamespace(uuid="node-123")) + + ironic_node.run_runbooks_standalone(node, ["bmc-configure-snmpv3-user"]) + + # manage -> clean(runbook) -> provide, in that order. + assert client.set_node_provision_state.mock_calls == [ + call( + "node-123", "manage", clean_steps=None, runbook=None, disable_ramdisk=None + ), + call( + "node-123", + "clean", + clean_steps=None, + runbook="runbook-123", + disable_ramdisk=None, + ), + call( + "node-123", "provide", clean_steps=None, runbook=None, disable_ramdisk=None + ), + ] + assert client.wait_for_node_provision_state.mock_calls == [ + call("node-123", "manageable", timeout=ironic_node.NODE_STATE_TIMEOUT_SECS), + call("node-123", "manageable", timeout=ironic_node.NODE_STATE_TIMEOUT_SECS), + call("node-123", "available", timeout=ironic_node.NODE_STATE_TIMEOUT_SECS), + ] + + +def test_standalone_manageable_node_is_not_provided(mocker): + client = _patch_client(mocker, provision_state="manageable") + node = cast(Node, SimpleNamespace(uuid="node-123")) + + ironic_node.run_runbooks_standalone(node, ["bmc-configure-snmpv3-user"]) + + # No manage, no provide: only the clean runs; node stays manageable. + assert client.set_node_provision_state.mock_calls == [ + call( + "node-123", + "clean", + clean_steps=None, + runbook="runbook-123", + disable_ramdisk=None, + ), + ] + + +@pytest.mark.parametrize("state", ["active", "deploying", "clean failed", "enroll"]) +def test_standalone_unexpected_state_is_refused(mocker, state): + client = _patch_client(mocker, provision_state=state) + node = cast(Node, SimpleNamespace(uuid="node-123")) + + with pytest.raises(ValueError, match="refusing to run runbook"): + ironic_node.run_runbooks_standalone(node, ["bmc-configure-snmpv3-user"]) + + client.set_node_provision_state.assert_not_called() + + +@pytest.mark.parametrize("runbook_traits", [[], ["CUSTOM_OTHER"], None]) +def test_standalone_rejects_ineligible_runbook(mocker, runbook_traits): + client = _patch_client( + mocker, + provision_state="manageable", + node_traits=("CUSTOM_DELL",), + ) + client.get_runbook.return_value.traits = runbook_traits + node = cast(Node, SimpleNamespace(uuid="node-123")) + + with pytest.raises(ValueError, match=r"traits metadata|matching node trait"): + ironic_node.run_runbooks_standalone(node, ["bmc-configure-snmpv3-user"]) + + client.set_node_provision_state.assert_not_called() + + +# --- CLI --------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "value", ["invalid", '"first"', "{}", "null", "[null]", "[1]", '[""]', '[" "]'] +) +def test_parse_runbooks_rejects_invalid(value): + with pytest.raises(argparse.ArgumentTypeError, match="JSON array"): + apply_node_runbook.parse_runbooks(value) + + +def test_main_single_node(mocker): + mocker.patch( + "sys.argv", + [ + "apply-node-runbook", + "--node", + "node-123", + "--runbooks", + '["first", "second"]', + ], + ) + mocker.patch.object(apply_node_runbook.helpers, "setup_logger") + client = mocker.patch.object(apply_node_runbook, "IronicClient").return_value + run = mocker.patch.object(ironic_node, "run_runbooks_standalone") + + apply_node_runbook.main() + + client.get_node.assert_called_once_with("node-123") + run.assert_called_once_with(client.get_node.return_value, ["first", "second"]) + + +def test_main_propagates_runbook_failure(mocker): + mocker.patch( + "sys.argv", + ["apply-node-runbook", "--node", "node-123", "--runbooks", '["first"]'], + ) + mocker.patch.object(apply_node_runbook.helpers, "setup_logger") + mocker.patch.object(apply_node_runbook, "IronicClient") + mocker.patch.object( + ironic_node, "run_runbooks_standalone", side_effect=RuntimeError("clean failed") + ) + + with pytest.raises(RuntimeError, match="clean failed"): + apply_node_runbook.main() + + +def test_main_requires_node(mocker, capsys): + mocker.patch("sys.argv", ["apply-node-runbook", "--runbooks", '["first"]']) + mocker.patch.object(apply_node_runbook.helpers, "setup_logger") + client = mocker.patch.object(apply_node_runbook, "IronicClient") + + with pytest.raises(SystemExit) as exc: + apply_node_runbook.main() + + assert exc.value.code == 2 + assert "--node" in capsys.readouterr().err + client.assert_not_called() + + +@pytest.mark.parametrize( + ("state", "field"), + [ + (state, field) + for state in ["available", "manageable"] + for field in [ + "maintenance", + "allocation_id", + "instance_uuid", + "reservation", + "target_provision_state", + ] + ], +) +def test_standalone_refuses_reserved_or_unhealthy_node(mocker, field, state): + client = _patch_client(mocker, provision_state=state) + mocker.patch.object( + ironic_node, + "refresh", + return_value=SimpleNamespace( + uuid="node-123", provision_state=state, **{field: True} + ), + ) + with pytest.raises(ValueError, match=rf"State '{state}', {field}=True"): + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), ["first"] + ) + client.set_node_provision_state.assert_not_called() + + +def test_sequence_brackets_once_and_stops_without_provide_on_failure(mocker): + client = _patch_client(mocker, provision_state="available") + client.wait_for_node_provision_state.side_effect = [ + None, + None, + RuntimeError("clean failed"), + ] + node = cast(Node, SimpleNamespace(uuid="node-123")) + with pytest.raises(RuntimeError, match="clean failed"): + ironic_node.run_runbooks_standalone(node, ["first", "second", "third"]) + assert [c.args[1] for c in client.set_node_provision_state.call_args_list] == [ + "manage", + "clean", + "clean", + ] + + +def test_sequence_restores_available_only_once(mocker): + client = _patch_client(mocker, provision_state="available") + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), ["first", "second"] + ) + assert [c.args[1] for c in client.set_node_provision_state.call_args_list] == [ + "manage", + "clean", + "clean", + "provide", + ] + + +def test_empty_sequence_does_not_change_state(mocker): + refresh = mocker.patch.object(ironic_node, "refresh") + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), [] + ) + refresh.assert_not_called() + + +def test_sequence_preflights_later_runbooks_before_state_change(mocker): + client = _patch_client(mocker, provision_state="available") + client.get_runbook.side_effect = [ + SimpleNamespace(uuid="r-1", traits=["CUSTOM_DELL"]), + SimpleNamespace(uuid="r-2", traits=["CUSTOM_OTHER"]), + ] + with pytest.raises(ValueError, match="matching node trait"): + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), ["first", "second"] + ) + client.set_node_provision_state.assert_not_called() + + +def test_sequence_does_not_provide_when_manage_fails(mocker): + client = _patch_client(mocker, provision_state="available") + client.wait_for_node_provision_state.side_effect = RuntimeError("manage failed") + with pytest.raises(RuntimeError, match="manage failed"): + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), ["first", "second"] + ) + assert [c.args[1] for c in client.set_node_provision_state.call_args_list] == [ + "manage" + ] + + +def test_sequence_refuses_instance_association_added_after_manage(mocker): + client = _patch_client(mocker, provision_state="available") + mocker.patch.object( + ironic_node, + "refresh", + side_effect=[ + SimpleNamespace(uuid="node-123", provision_state="available"), + SimpleNamespace( + uuid="node-123", + provision_state="manageable", + instance_uuid="instance-123", + ), + ], + ) + with pytest.raises(ValueError, match="instance_uuid='instance-123'"): + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), ["first"] + ) + assert [c.args[1] for c in client.set_node_provision_state.call_args_list] == [ + "manage" + ] + + +def test_sequence_refuses_unexpected_idle_state_before_clean(mocker): + client = _patch_client(mocker, provision_state="manageable") + mocker.patch.object( + ironic_node, + "refresh", + side_effect=[ + SimpleNamespace(uuid="node-123", provision_state=state) + for state in ["manageable", "available"] + ], + ) + with pytest.raises(ValueError, match=r"Expected manageable.*got 'available'"): + ironic_node.run_runbooks_standalone( + cast(Node, SimpleNamespace(uuid="node-123")), ["first"] + ) + client.set_node_provision_state.assert_not_called() diff --git a/python/understack-workflows/understack_workflows/ironic_node.py b/python/understack-workflows/understack_workflows/ironic_node.py index 54baa97e3..7b4d41776 100644 --- a/python/understack-workflows/understack_workflows/ironic_node.py +++ b/python/understack-workflows/understack_workflows/ironic_node.py @@ -368,6 +368,81 @@ def run_post_inspection_runbook(node: Node, name_or_id: str) -> None: ) +def validate_node_runbook(node: Node, name_or_id: str): + """Resolve a runbook and check its trait requirements without mutation.""" + client = IronicClient() + runbook = client.get_runbook(name_or_id) + traits = client.get_node_traits(node.uuid) + runbook_traits = getattr(runbook, "traits", None) + if runbook_traits is None: + raise ValueError( + f"[node:{node.uuid}] Runbook {name_or_id!r} has no traits metadata; " + "Ironic API 1.112 or newer is required" + ) + if not set(traits).intersection(runbook_traits): + raise ValueError( + f"[node:{node.uuid}] Runbook {name_or_id!r} requires at least one " + f"matching node trait from {sorted(runbook_traits)}" + ) + return runbook + + +def validate_runbook_node(node: Node) -> None: + """Require an idle, healthy node with no instance or allocation.""" + state = node.provision_state + if state not in {"available", "manageable"}: + raise ValueError( + f"[node:{node.uuid}] State {state!r}: refusing to run runbooks" + ) + for field in ( + "maintenance", + "reservation", + "target_provision_state", + "allocation_id", + "instance_uuid", + ): + value = getattr(node, field, None) + if value: + raise ValueError( + f"[node:{node.uuid}] State {state!r}, {field}={value!r}: " + "refusing to run runbooks" + ) + + +def run_runbooks_standalone(node: Node, runbooks: list[str]) -> None: + """Run idle-node runbooks in order; restore availability only on success.""" + if not runbooks: + return + current = refresh(node) + validate_runbook_node(current) + # Validate the entire sequence before removing the node from availability. + resolved = [(name, validate_node_runbook(node, name)) for name in runbooks] + restore_available = current.provision_state == "available" + if restore_available: + transition(node, target_state="manage", expected_state="manageable") + for name, runbook in resolved: + current = refresh(node) + validate_runbook_node(current) + if current.provision_state != "manageable": + raise ValueError( + f"[node:{node.uuid}] Expected manageable before {name!r}, " + f"got {current.provision_state!r}" + ) + logger.info("[node:%s] Running runbook %s", node.uuid, name) + try: + transition( + node, + target_state="clean", + expected_state="manageable", + runbook=runbook.uuid, + ) + except Exception: + logger.exception("[node:%s] Runbook %s failed", node.uuid, name) + raise + if restore_available: + transition(node, target_state="provide", expected_state="available") + + def apply_firmware_updates(node: Node) -> None: client = IronicClient() traits = client.get_node_traits(node.uuid) diff --git a/python/understack-workflows/understack_workflows/main/apply_node_runbook.py b/python/understack-workflows/understack_workflows/main/apply_node_runbook.py new file mode 100644 index 000000000..829503099 --- /dev/null +++ b/python/understack-workflows/understack_workflows/main/apply_node_runbook.py @@ -0,0 +1,57 @@ +import argparse +import json + +from understack_workflows import helpers +from understack_workflows import ironic_node +from understack_workflows.ironic.client import IronicClient + + +def main() -> None: + """Run Ironic runbooks in order against an enrolled node.""" + helpers.setup_logger() + args = argument_parser().parse_args() + node = IronicClient().get_node(args.node) + ironic_node.run_runbooks_standalone(node, args.runbooks) + + +def parse_runbooks(value: str) -> list[str]: + """Parse an ordered JSON array of nonempty runbook names or UUIDs.""" + try: + runbooks = json.loads(value) + except ValueError as exc: + raise argparse.ArgumentTypeError( + "Expected a JSON array of runbook names or UUIDs" + ) from exc + if not isinstance(runbooks, list) or any( + not isinstance(runbook, str) or not runbook.strip() for runbook in runbooks + ): + raise argparse.ArgumentTypeError( + "Expected a JSON array of nonempty runbook names or UUIDs" + ) + return runbooks + + +def argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="apply-node-runbook", + description="Run Ironic runbooks in order against an enrolled node.", + ) + parser.add_argument( + "--runbooks", + type=parse_runbooks, + required=True, + help=( + "JSON array of runbook names or UUIDs to run in order, e.g. " + "'[\"bmc-configure-snmpv3-user\"]'" + ), + ) + parser.add_argument( + "--node", + required=True, + help="Node name or UUID", + ) + return parser + + +if __name__ == "__main__": + main() diff --git a/workflows/argo-events/kustomization.yaml b/workflows/argo-events/kustomization.yaml index ac59f6bf3..fe18f8fce 100644 --- a/workflows/argo-events/kustomization.yaml +++ b/workflows/argo-events/kustomization.yaml @@ -16,6 +16,7 @@ resources: - workflowtemplates/enroll-fw.yaml - workflowtemplates/enroll-netdev.yaml - workflowtemplates/enroll-server.yaml + - workflowtemplates/workflowtemplate-bmc-configure-snmpv3-user.yaml - workflowtemplates/reclean-server.yaml - workflowtemplates/openstack-oslo-event.yaml - workflowtemplates/netapp-configure-net.yaml diff --git a/workflows/argo-events/workflowtemplates/workflowtemplate-bmc-configure-snmpv3-user.yaml b/workflows/argo-events/workflowtemplates/workflowtemplate-bmc-configure-snmpv3-user.yaml new file mode 100644 index 000000000..51ea2a788 --- /dev/null +++ b/workflows/argo-events/workflowtemplates/workflowtemplate-bmc-configure-snmpv3-user.yaml @@ -0,0 +1,51 @@ +apiVersion: argoproj.io/v1alpha1 +kind: WorkflowTemplate +metadata: + name: bmc-configure-snmpv3-user + annotations: + workflows.argoproj.io/title: Configure the SNMPv3 user on a node BMC + workflows.argoproj.io/description: | + Defined in `workflows/argo-events/workflowtemplates/workflowtemplate-bmc-configure-snmpv3-user.yaml` + + Applies the bmc-configure-snmpv3-user runbook to the specified Ironic node. + Requires an available or manageable node with no instance or allocation. +spec: + entrypoint: run + serviceAccountName: workflow + arguments: + parameters: + - name: node_uuid + templates: + - name: run + inputs: + parameters: + - name: node_uuid + synchronization: + mutexes: + - name: ironic-runbooks-{{inputs.parameters.node_uuid}} + volumes: + - name: openstack + secret: + secretName: baremetal-manage + items: + - key: clouds.yaml + path: clouds.yaml + container: + image: ghcr.io/rackerlabs/understack/ironic-nautobot-client:latest + command: + - apply-node-runbook + args: + - --node + - "{{inputs.parameters.node_uuid}}" + - --runbooks + - '["bmc-configure-snmpv3-user"]' + envFrom: + - configMapRef: + name: cluster-metadata + env: + - name: OS_CLOUD + value: understack + volumeMounts: + - name: openstack + mountPath: /etc/openstack + readOnly: true