diff --git a/README.md b/README.md index 1d08a70b..c2fa0e65 100644 --- a/README.md +++ b/README.md @@ -211,6 +211,15 @@ We can now run this model using the plugboard CLI with the command: plugboard process run my-model.yaml ``` +Process fields can be overridden at run time with repeated `--param` / `-p` flags. Use +`process.default.parameter.` for process parameters or +`component...` for component fields. Values are parsed +as YAML: + +```shell +plugboard process run my-model.yaml --param process.default.parameter.scale=2.0 -p component.a.arg.enabled=true +``` + ## 📖 Documentation For more information including a detailed API reference and step-by-step usage examples, refer to the [documentation site](https://docs.plugboard.dev). We recommend diving into the [tutorials](https://docs.plugboard.dev/latest/examples/tutorials/hello-world/) for a step-by-step guide to getting started. diff --git a/docs/examples/tutorials/hello-world.md b/docs/examples/tutorials/hello-world.md index 10bb38e9..42519826 100644 --- a/docs/examples/tutorials/hello-world.md +++ b/docs/examples/tutorials/hello-world.md @@ -63,6 +63,12 @@ We can now run this model using the plugboard CLI with the command: plugboard process run model.yaml ``` +You can override process parameters or component fields at run time without editing the YAML: + +```shell +plugboard process run model.yaml --param process.default.parameter.scale=2.0 -p component.a.arg.iters=10 +``` + You should see that an output `.txt` file has been created, showing the the model as run successfully. Congratulations - you have built and run your first Plugboard model! In the following tutorials we will build up some more complex components and processes to demonstrate the power of the framework. diff --git a/docs/examples/tutorials/more-complex-process.md b/docs/examples/tutorials/more-complex-process.md index f899abe8..bcaa6592 100644 --- a/docs/examples/tutorials/more-complex-process.md +++ b/docs/examples/tutorials/more-complex-process.md @@ -101,6 +101,17 @@ Now we can supply the `scale` value when we create the [`Process`][plugboard.pro 1. Supply a dictionary of common parameter values to the `Process` here. They are accessible from within all components. 2. If you need to override a parameter on a specific component, you can supply it via the component-level `parameters` argument. +When running a YAML-defined model from the CLI you can override process parameters or component +arguments, initial values, and parameters without editing the file. Repeat `--param` / `-p` as +`name=value` pairs, where `name` is +`process.default.parameter.` or +`component...`. Values are parsed as YAML, so numbers, +booleans, lists and mappings keep their natural types: + +```shell +plugboard process run parameters.yaml --param process.default.parameter.scale=2.0 -p component.scale.arg.factor=2 +``` + ## Next steps You've now learned how to build up complex model layouts in Plugboard. In the next tutorial we'll show how powerful a Plugboard model can be as we start to include different types of [`Component`][plugboard.component.Component]. diff --git a/plugboard-schemas/plugboard_schemas/__init__.py b/plugboard-schemas/plugboard_schemas/__init__.py index b578491e..ca4117ce 100644 --- a/plugboard-schemas/plugboard_schemas/__init__.py +++ b/plugboard-schemas/plugboard_schemas/__init__.py @@ -40,6 +40,7 @@ Status, ) from .tune import ( + BaseFieldSpec, CategoricalParameterSpec, Direction, FloatParameterSpec, @@ -50,6 +51,8 @@ TuneArgsDict, TuneArgsSpec, TuneSpec, + override_parameter, + parse_parameter_name, ) @@ -57,6 +60,7 @@ __all__ = [ + "BaseFieldSpec", "CategoricalParameterSpec", "ComponentSpec", "ComponentArgsDict", @@ -100,4 +104,6 @@ "validate_input_events", "validate_no_unresolved_cycles", "validate_process", + "override_parameter", + "parse_parameter_name", ] diff --git a/plugboard-schemas/plugboard_schemas/tune.py b/plugboard-schemas/plugboard_schemas/tune.py index 76580c86..9702f2aa 100644 --- a/plugboard-schemas/plugboard_schemas/tune.py +++ b/plugboard-schemas/plugboard_schemas/tune.py @@ -6,6 +6,7 @@ from pydantic import Field, PositiveInt, ValidationInfo, field_validator, model_validator from ._common import PlugboardBaseModel +from .process import ProcessSpec class OptunaSpec(PlugboardBaseModel): @@ -77,6 +78,73 @@ def full_name(self) -> str: ) +def parse_parameter_name(name: str) -> BaseFieldSpec: + """Parse a tunable parameter's fully-qualified name. + + Args: + name: A parameter name such as ``component.my_component.arg.scale``. + + Returns: + The corresponding field specification. + + Raises: + ValueError: If the name does not identify an overridable parameter. + """ + try: + object_type, object_name, field_type, field_name = name.split(".", maxsplit=3) + except ValueError as e: + raise ValueError( + "Parameter name must have the format " + "'component...' or " + "'process.default.parameter.'." + ) from e + + if not object_name or not field_name: + raise ValueError("Parameter names must include an object name and field name.") + if object_type not in {"component", "process"}: + raise ValueError(f"Unknown object type {object_type!r} for parameter override.") + if object_type == "process" and object_name != "default": + raise ValueError("Process parameter names must use 'process.default'.") + if (object_type == "component" and field_type not in {"arg", "initial_value", "parameter"}) or ( + object_type == "process" and field_type != "parameter" + ): + raise ValueError(f"Parameter name {name!r} does not identify an overridable parameter.") + return BaseFieldSpec( + object_type=object_type, + object_name=None if object_type == "process" else object_name, + field_type=field_type, + field_name=field_name, + ) + + +def override_parameter(process: ProcessSpec, param: BaseFieldSpec, value: _t.Any) -> None: + """Override a parameter or initial value in a process specification. + + Args: + process: The process specification to update. + param: The field to override. + value: The replacement value. + + Raises: + ValueError: If the target component is not present in the process. + """ + if param.object_type == "component": + try: + component = next(c for c in process.args.components if c.args.name == param.object_name) + except StopIteration: + raise ValueError(f"Component {param.object_name} not found in process.") + if param.field_type == "arg": + setattr(component.args, param.field_name, value) + elif param.field_type == "initial_value": + component.args.initial_values[param.field_name] = value + elif param.field_type == "parameter": + component.args.parameters[param.field_name] = value + elif param.object_type == "process": + process.args.parameters[param.field_name] = value + else: + raise ValueError(f"Unknown object type {param.object_type} for parameter override.") + + class ObjectiveSpec(BaseFieldSpec): """Specification for an objective field.""" diff --git a/plugboard/cli/ai/skills/run-process-scenario/SKILL.md b/plugboard/cli/ai/skills/run-process-scenario/SKILL.md index 1363a4d7..f0b34518 100644 --- a/plugboard/cli/ai/skills/run-process-scenario/SKILL.md +++ b/plugboard/cli/ai/skills/run-process-scenario/SKILL.md @@ -18,16 +18,16 @@ Create or update a YAML config for the requested scenario, then run it with `plu 1. Make sure a YAML config exists for the model. If the model only exists in Python, create the YAML first by using the `create-yaml-config` skill at `../create-yaml-config/SKILL.md`. 2. Ask for any missing scenario inputs before running anything. -3. Update the YAML config with the exact parameter values the user requested. +3. Prefer overriding process parameters or component fields on the CLI with repeated `--param` / `-p` `name=value` flags when only values change for a scenario. Use `process.default.parameter.` or `component...`. Edit the YAML when component structure, connectors, or defaults need to change. 4. Preserve a clear component structure that matches the real-world model. Do not collapse multiple entities into one component just to make the config shorter. -5. Validate the updated YAML against `plugboard_schemas.ConfigSpec` before running it. -6. Run: +5. Validate the YAML against `plugboard_schemas.ConfigSpec` before running it. +6. Run, for example: ```sh -plugboard process run path/to/model.yaml +plugboard process run path/to/model.yaml --param process.default.parameter.scale=2.0 -p component.simulation.arg.max_iters=10 ``` -7. Report what configuration was used, what validation was performed, what command was run, and the key outputs or generated artifacts. +7. Report what configuration was used, what validation was performed, what command was run (including any `--param` overrides), and the key outputs or generated artifacts. 8. If the scenario required new parameters, confirm that the final config remains YAML-friendly and reusable. ## Output diff --git a/plugboard/cli/process/__init__.py b/plugboard/cli/process/__init__.py index 71c9a244..837b6893 100644 --- a/plugboard/cli/process/__init__.py +++ b/plugboard/cli/process/__init__.py @@ -13,7 +13,13 @@ from plugboard.diagram import MermaidDiagram from plugboard.process import Process, ProcessBuilder -from plugboard.schemas import ConfigSpec, validate_process +from plugboard.schemas import ( + BaseFieldSpec, + ConfigSpec, + override_parameter, + parse_parameter_name, + validate_process, +) from plugboard.tune import Tuner from plugboard.utils import add_sys_path, run_coro_sync @@ -34,6 +40,63 @@ def _read_yaml(path: Path) -> ConfigSpec: return ConfigSpec.model_validate(data) +def _parse_param_override(param: str) -> tuple[BaseFieldSpec, _t.Any]: + """Parse a single ``name=value`` parameter override. + + Values are decoded as YAML scalars/collections so that numbers, booleans, + nulls, lists and mappings keep their natural types. Plain strings are left + as strings. Use quotes around a value if YAML would otherwise coerce it + (for example ``name='"yes"'``). + """ + if "=" not in param: + raise typer.BadParameter( + f"Invalid parameter override {param!r}. Expected format: name=value.", + param_hint="--param", + ) + key, _, raw_value = param.partition("=") + if not key: + raise typer.BadParameter( + f"Invalid parameter override {param!r}. Parameter name must not be empty.", + param_hint="--param", + ) + if raw_value == "": + value = "" + else: + try: + value = msgspec.yaml.decode(raw_value.encode()) + except msgspec.DecodeError as e: + raise typer.BadParameter( + f"Could not parse value for parameter {key!r}: {raw_value!r}.", + param_hint="--param", + ) from e + try: + field = ( + BaseFieldSpec( + object_type="process", + object_name=None, + field_type="parameter", + field_name=key, + ) + if "." not in key + else parse_parameter_name(key) + ) + except ValueError as e: + raise typer.BadParameter( + f"Invalid parameter name {key!r}: {e}", + param_hint="--param", + ) from e + return field, value + + +def _apply_param_overrides(config: ConfigSpec, params: list[str] | None) -> None: + """Apply CLI parameter overrides to the process configuration in place.""" + if not params: + return + for param in params: + field, value = _parse_param_override(param) + override_parameter(config.plugboard.process, field, value) + + def _build_process(config: ConfigSpec) -> Process: process = ProcessBuilder.build(config.plugboard.process) return process @@ -102,6 +165,19 @@ def run( ), ), ] = None, + param: Annotated[ + _t.Optional[list[str]], + typer.Option( + "--param", + "-p", + help=( + "Override a process or component field as name=value. Repeatable. " + "Use component... or " + "process.default.parameter.; a bare name targets a process parameter. " + "Values are parsed as YAML." + ), + ), + ] = None, ) -> None: """Run a Plugboard process.""" config_spec = _read_yaml(config) @@ -116,6 +192,12 @@ def run( process_type # type: ignore[arg-type] ) + try: + _apply_param_overrides(config_spec, param) + except typer.BadParameter as e: + stderr.print(f"[red]{e}[/red]") + raise typer.Exit(2) from e + with Progress( SpinnerColumn("arrow3"), TextColumn("[progress.description]{task.description}"), diff --git a/plugboard/schemas/__init__.py b/plugboard/schemas/__init__.py index 4ee8fc79..29b9a567 100644 --- a/plugboard/schemas/__init__.py +++ b/plugboard/schemas/__init__.py @@ -14,6 +14,7 @@ DEFAULT_STATE_BACKEND_CLS_PATH, ENTITY_ID_REGEX, RAY_STATE_BACKEND_CLS_PATH, + BaseFieldSpec, CategoricalParameterSpec, ComponentArgsDict, ComponentArgsSpec, @@ -46,6 +47,8 @@ TuneArgsDict, TuneArgsSpec, TuneSpec, + override_parameter, + parse_parameter_name, simple_cycles, validate_all_inputs_connected, validate_input_events, @@ -55,6 +58,7 @@ __all__ = [ + "BaseFieldSpec", "CategoricalParameterSpec", "ComponentSpec", "ComponentArgsDict", @@ -96,4 +100,6 @@ "validate_input_events", "validate_no_unresolved_cycles", "validate_process", + "override_parameter", + "parse_parameter_name", ] diff --git a/plugboard/tune/tune.py b/plugboard/tune/tune.py index 379b16a6..563806ba 100644 --- a/plugboard/tune/tune.py +++ b/plugboard/tune/tune.py @@ -18,6 +18,7 @@ ParameterSpec, ProcessSpec, Resource, + override_parameter, ) from plugboard.utils import DI, run_coro_sync from plugboard.utils.dependencies import depends_on_optional @@ -174,29 +175,6 @@ def _build_parameter( **parameter.model_dump(exclude={"type"}) ) - @staticmethod - def _override_parameter( - process: ProcessSpec, param: ParameterSpec, value: _t.Any - ) -> None: # pragma: no cover - if param.object_type == "component": - try: - component = next( - c for c in process.args.components if c.args.name == param.object_name - ) - except StopIteration: - raise ValueError(f"Component {param.object_name} not found in process.") - if param.field_type == "arg": - setattr(component.args, param.field_name, value) - elif param.field_type == "initial_value": - component.args.initial_values[param.field_name] = value - elif param.field_type == "parameter": - component.args.parameters[param.field_name] = value - elif param.object_type == "process": - if param.field_type == "parameter": - process.args.parameters[param.field_name] = value - else: - raise ValueError(f"Unknown object type {param.object_type} for parameter override.") - @staticmethod def _get_objective(process: Process, objective: ObjectiveSpec) -> _t.Any: # pragma: no cover if objective.object_type != "component": @@ -280,7 +258,7 @@ def fn(config: dict[str, _t.Any]) -> dict[str, _t.Any]: # pragma: no cover # Custom search spaces may include intermediate parameters not in the Tuner self._logger.warning("Parameter from config not found in Tuner", param=name) continue - self._override_parameter(spec, self._parameters_dict[name], value) + override_parameter(spec, self._parameters_dict[name], value) process = ProcessBuilder.build(spec) result = {} diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 6428e15c..ccc7da59 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -13,6 +13,7 @@ import pytest import respx +import typer from typer.testing import CliRunner from plugboard.cli import app @@ -164,6 +165,95 @@ async def test_cli_process_run_with_ray_override() -> None: assert process_spec.args.state.type == "plugboard.state.RayStateBackend" +@pytest.mark.asyncio +async def test_cli_process_run_with_param_overrides() -> None: + """Tests the process run command with generic --param / -p overrides.""" + with patch("plugboard.cli.process.ProcessBuilder") as mock_process_builder: + mock_process = AsyncMock() + mock_process_builder.build.return_value = mock_process + result = runner.invoke( + app, + [ + "process", + "run", + "tests/data/dynamic-param-process.yaml", + "--param", + "process.default.parameter.max_iters=3", + "-p", + "component.a.arg.iters=5", + "--param", + "component.d.initial_value.in_1=[1, 2]", + "-p", + "component.d.parameter.enabled=true", + ], + ) + assert result.exit_code == 0 + assert "Process complete" in result.stdout + mock_process_builder.build.assert_called_once() + process_spec = mock_process_builder.build.call_args[0][0] + assert process_spec.args.parameters == { + "max_iters": 3, + } + components = {component.args.name: component for component in process_spec.args.components} + assert components["a"].args.iters == 5 + assert components["d"].args.initial_values["in_1"] == [1, 2] + assert components["d"].args.parameters["enabled"] is True + + +@pytest.mark.asyncio +async def test_cli_process_run_with_invalid_param_override() -> None: + """Tests the process run command rejects malformed --param values.""" + with patch("plugboard.cli.process.ProcessBuilder") as mock_process_builder: + result = runner.invoke( + app, + ["process", "run", "tests/data/minimal-process.yaml", "--param", "not-a-pair"], + ) + assert result.exit_code == 2 + assert "Invalid parameter override" in result.stderr + mock_process_builder.build.assert_not_called() + + +def test_parse_param_override() -> None: + """Tests name=value parsing for generic parameter overrides.""" + from plugboard.cli.process import _parse_param_override + + field, value = _parse_param_override("component.a.arg.scale=2.0") + assert field.full_name == "component.a.arg.scale" + assert value == 2.0 + field, value = _parse_param_override("process.default.parameter.name=hello") + assert field.full_name == "process.default.parameter.name" + assert value == "hello" + field, value = _parse_param_override("max_iters=3") + assert field.full_name == "process.default.parameter.max_iters" + assert value == 3 + field, value = _parse_param_override("component.a.parameter.flag=true") + assert field.full_name == "component.a.parameter.flag" + assert value is True + field, value = _parse_param_override("component.a.initial_value.empty=") + assert field.full_name == "component.a.initial_value.empty" + assert value == "" + field, value = _parse_param_override('component.a.arg.name="yes"') + assert field.full_name == "component.a.arg.name" + assert value == "yes" + + +@pytest.mark.parametrize( + ("param", "message"), + [ + ("not-a-pair", "Expected format"), + ("=value", "Parameter name must not be empty"), + ("component.a.arg.value=[", "Could not parse value"), + ("component.a.field.value=1", "does not identify an overridable parameter"), + ], +) +def test_parse_param_override_rejects_invalid_values(param: str, message: str) -> None: + """Tests invalid generic parameter overrides produce CLI errors.""" + from plugboard.cli.process import _parse_param_override + + with pytest.raises(typer.BadParameter, match=message): + _parse_param_override(param) + + def test_cli_process_validate() -> None: """Tests the process validate command.""" result = runner.invoke(app, ["process", "validate", "tests/data/minimal-process.yaml"]) diff --git a/tests/unit/test_schemas.py b/tests/unit/test_schemas.py index e69e76d0..76edf432 100644 --- a/tests/unit/test_schemas.py +++ b/tests/unit/test_schemas.py @@ -3,7 +3,13 @@ import msgspec import pytest -from plugboard.schemas import ConfigSpec, TuneArgsSpec, TuneSpec +from plugboard.schemas import ( + ConfigSpec, + TuneArgsSpec, + TuneSpec, + override_parameter, + parse_parameter_name, +) def test_config_spec() -> None: @@ -95,3 +101,62 @@ def test_tune_spec() -> None: # Invalid objective length should raise a validation error with pytest.raises(ValueError): _ = TuneSpec(args=TuneArgsSpec.model_validate(invalid_spec)) + + +def test_parameter_override() -> None: + """Tests parsing and applying generic parameter overrides.""" + with open("tests/data/dynamic-param-process.yaml", "rb") as f: + config = ConfigSpec.model_validate(msgspec.yaml.decode(f.read())) + + process = config.plugboard.process + override_parameter(process, parse_parameter_name("process.default.parameter.max_iters"), 3) + override_parameter(process, parse_parameter_name("component.a.arg.iters"), 5) + override_parameter( + process, + parse_parameter_name("component.d.initial_value.in_1"), + [1, 2], + ) + override_parameter( + process, + parse_parameter_name("component.d.parameter.enabled"), + True, + ) + + components = {component.args.name: component for component in process.args.components} + assert process.args.parameters["max_iters"] == 3 + assert components["a"].args.model_dump()["iters"] == 5 + assert components["d"].args.initial_values["in_1"] == [1, 2] + assert components["d"].args.parameters["enabled"] is True + + with pytest.raises(ValueError, match="Unknown object type"): + parse_parameter_name("connector.my_connector.arg.value") + + +@pytest.mark.parametrize( + ("name", "message"), + [ + ("component.a", "must have the format"), + ("component..arg.value", "must include an object name"), + ("component.a.arg.", "must include an object name and field name"), + ("process.custom.parameter.value", "must use 'process.default'"), + ("component.a.field.value", "does not identify an overridable parameter"), + ("process.default.arg.value", "does not identify an overridable parameter"), + ], +) +def test_parse_parameter_name_rejects_invalid_names(name: str, message: str) -> None: + """Tests invalid generic override field names are rejected.""" + with pytest.raises(ValueError, match=message): + parse_parameter_name(name) + + +def test_parameter_override_rejects_unknown_component() -> None: + """Tests overrides cannot target components outside the process.""" + with open("tests/data/dynamic-param-process.yaml", "rb") as f: + config = ConfigSpec.model_validate(msgspec.yaml.decode(f.read())) + + with pytest.raises(ValueError, match="Component unknown not found"): + override_parameter( + config.plugboard.process, + parse_parameter_name("component.unknown.arg.value"), + 1, + )