From 843bfc2d4b894a4e293cbfe939dc3038bc8ed44c Mon Sep 17 00:00:00 2001 From: rdwr-egore Date: Mon, 19 Oct 2026 21:55:51 +0000 Subject: [PATCH] fix: align v1.6.1 installer YAML with reference settings --- README.md | 2 +- docs/dev/DEVELOPMENT_AND_RELEASE.md | 2 + .../operations/INSTALLATION_AND_OPERATIONS.md | 6 +- docs/operations/RELEASE_NOTES_V1.6.1.md | 3 +- install.sh | 8 ++ tests/test_installer_image.py | 84 ++++++++++++++++++- watchdog-config.yaml.example | 4 +- 7 files changed, 104 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index cdd8165..640e6cb 100644 --- a/README.md +++ b/README.md @@ -281,7 +281,7 @@ failures on application containers. | Version | Date | Author | Changes | |---------|------------|--------|---------| -| 1.6.1 | Pending | — | **In preparation, not published:** HA-monitoring transient transition diagnostics (without hiding safety-critical faults); remove unverified default MariaDB label exception while retaining strict built-in dump-helper handling and optional label policies. See [draft v1.6.1 release notes](docs/operations/RELEASE_NOTES_V1.6.1.md). Additional changes and validation to be recorded before release. | +| 1.6.1 | Pending | — | **In preparation, not published:** HA-monitoring transient transition diagnostics (without hiding safety-critical faults); remove unverified default MariaDB label exception while retaining strict built-in dump-helper handling and optional label policies; complete installer-generated YAML and enforce reference-template parity in tests. See [draft v1.6.1 release notes](docs/operations/RELEASE_NOTES_V1.6.1.md). Additional changes and validation to be recorded before release. | | 1.6.0 | 2026-10-15 | — | Changes since 1.5.4: PRs [#3](https://github.com/Radware/CyberController-Container-Alert/pull/3), [#4](https://github.com/Radware/CyberController-Container-Alert/pull/4), [#5](https://github.com/Radware/CyberController-Container-Alert/pull/5), [#6](https://github.com/Radware/CyberController-Container-Alert/pull/6), [#7](https://github.com/Radware/CyberController-Container-Alert/pull/7), [#8](https://github.com/Radware/CyberController-Container-Alert/pull/8), [#10](https://github.com/Radware/CyberController-Container-Alert/pull/10); corrected restart-loop/recovery lifecycle, planned-stop/exit-137 handling, event ID/name reuse, independent optional HA failover detection, Syslog and notification consistency, one complete versioned release package and wizard, revised informational alert messages, and isolated-only release smoke tests. See [complete v1.6.0 release notes](docs/operations/RELEASE_NOTES_V1.6.0.md). | | 1.5.4 | 2026-09-23 | Rahul Kumar | Versioned Docker image (`watchdog:`) driven by the root `VERSION` file; installer loads the packaged `watchdog.tar`, rejects an archive that does not carry the release tag, aborts on Compose failures, and verifies the running image; restart-loop detection now uses Docker `RestartCount` instead of counting poll observations; `container_health_checks` overrides implemented; fixed `.env` migration on root-owned files; `watchdog-config.yaml` made readable by the non-root container user | | 1.5.3 | 2026-09-16 | Rahul Kumar | Resource Limits Enforced | diff --git a/docs/dev/DEVELOPMENT_AND_RELEASE.md b/docs/dev/DEVELOPMENT_AND_RELEASE.md index c16bb4a..4327040 100644 --- a/docs/dev/DEVELOPMENT_AND_RELEASE.md +++ b/docs/dev/DEVELOPMENT_AND_RELEASE.md @@ -54,6 +54,8 @@ git diff --check The Python test environment needs the repo's test dependencies (including Docker SDK, Requests, PyYAML). Installer/package tests use fake Docker. **Never** run real failure-injection tests or `tests/release-smoke.sh` on operational/shared Docker hosts. +**YAML parity guard:** `tests/test_installer_image.py` runs the installer with a fake Docker executable, parses its generated YAML and the reference `watchdog-config.yaml.example`, and compares **all active nested configuration key paths in both directions**, plus non-wizard default values. Any future setting added to or removed from either file must be reflected in the other. Wizard-specific destinations and SNMPv3-only conditional settings are intentionally not compared for literal values. Preserve existing configuration on upgrade; never force-regenerate customer YAML as part of a parity fix. + **Step 4 — Commit, push, request review.** ```bash diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index d05c9b7..065654a 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -231,10 +231,14 @@ HA traps use the same OID and var-binds with `cwFailureType=ha-failover` and `cw | `auto_health_check.timeout_seconds` | `2` | Probe timeout | | `container_health_checks` | `{}` | Per-container `http` / `exec` probe overrides | | `excluded_containers` | `[]` | Containers excluded from alerts | -| `ignored_container_events` | `[]` (optional; not generated by installer) | Opt-in suppression of Docker events based on **verified** container labels; not required for known MariaDB dump helpers | +| `ignored_container_events` | `[]` | Installer generates an empty list; opt-in Docker-event exclusions require **verified** container labels. The known MariaDB dump-helper exception does not need a label rule | | `ha_monitor.enabled` | `false` | Optional active-node change alerts | | `ha_monitor.exporter_url` | `http://127.0.0.1:9664/metrics` | Local HA exporter address | +**Generated configuration completeness (v1.6.1):** New installer-generated `watchdog-config.yaml` files now explicitly include every active setting from the reference template, including `lifecycle_detection.enabled: true`, `lifecycle_detection.correlation_window_seconds: 3`, `restart_recovery_stability_seconds: 120`, `alert_on_recovery: true`, and an empty `ignored_container_events` list. Alert destinations and channel-specific options still reflect wizard selections; optional SNMPv3 fields are generated only when SNMPv3 is selected. + +The v1.6.0 installer did not write the first four fields, but the Python application already used those defaults, so their absence did not disable monitoring behavior. **On upgrades, the installer preserves the existing YAML** rather than injecting missing keys. To expose the options on an existing installation, add them manually after backing up the configuration; there is no need to regenerate or overwrite the file merely to retain the original defaults. + **Temporary MariaDB SQL dump helpers:** The watchdog recognizes the known CyberController syntax-check helper automatically if the container uses the expected `kvision_infra_mariadb` image, is not the primary MariaDB service, has `AutoRemove=True`, mounts `/mnt/cli/tmp/mariadb_dump.sql.gz`, and exits 137. This narrowly scoped built-in exception does **not** depend on `ignored_container_events`. The installer does not generate an assumed MariaDB role label. Operators can still add explicit `ignored_container_events` entries for other verified, intentionally excluded labelled containers; normal application failures should not be excluded. **Upgrades:** Existing `watchdog-config.yaml` files remain unchanged, including an older `com.radware.cybercontroller.role: mariadb-ha-syntax-check` rule if present. That old rule is not required for the observed dump-helper path; review it before removing it from a customer configuration rather than changing it automatically. diff --git a/docs/operations/RELEASE_NOTES_V1.6.1.md b/docs/operations/RELEASE_NOTES_V1.6.1.md index a053879..43516f1 100644 --- a/docs/operations/RELEASE_NOTES_V1.6.1.md +++ b/docs/operations/RELEASE_NOTES_V1.6.1.md @@ -8,12 +8,13 @@ |---|---|---| | HA monitoring diagnostics | During failover, tolerate up to two consecutive checks with no promoted HA Operator. On the third consecutive such reading, emit a throttled error; conflicting promoted nodes, loss of quorum and exporter failures remain immediate errors. | Reduces expected transient Syslog ERROR noise without hiding sustained or unsafe HA status. No change to HA failover/failback detection, owner-only notifications, or Docker container monitoring. | | MariaDB helper configuration clarity | Remove the unverified MariaDB syntax-check Docker label rule from Python defaults, installer-generated configuration and the example template. Retain optional explicit label-based exclusions and the existing narrow built-in dump-helper exception. | Avoids documenting a label not observed on the temporary MariaDB helper; no change to the verified built-in exit-137 suppression. Existing customer YAML remains preserved on upgrades, including older rules. | +| Installer configuration parity | Generate the previously omitted `lifecycle_detection` settings, `restart_recovery_stability_seconds`, `alert_on_recovery`, and an explicit empty `ignored_container_events` list. Add `runbook_base_url` to the reference example so all active nested YAML key paths match the installer. Add an automated bidirectional key/default parity test. | New installations expose all documented options. No runtime change to the existing Python defaults; wizard-selected destinations are preserved. Existing customer YAML remains untouched during upgrades. | **Additional changes:** To be updated as development PRs are integrated. ## Compatibility and operations -No new service, port, endpoint or privilege is required. Default HA polling remains 60 seconds; the existing 2-valid-poll failover confirmation and 5-minute repeated-error rate limit are retained. Newly generated configs omit the optional `ignored_container_events` key; the reference template defaults it to `[]`. The event-label mechanism is still available for explicit, verified labels. The strict built-in MariaDB dump-helper exception stays enabled and unchanged. Installer upgrades **preserve existing customer YAML**, so they do not automatically remove old label entries. +No new service, port, endpoint or privilege is required. Default HA polling remains 60 seconds; the existing 2-valid-poll failover confirmation and 5-minute repeated-error rate limit are retained. Newly generated configs explicitly include `ignored_container_events: []` (no rules), matching the reference template, rather than including an unverified MariaDB label rule. The event-label mechanism is still available for explicit, verified labels. The strict built-in MariaDB dump-helper exception stays enabled and unchanged. Installer upgrades **preserve existing customer YAML**, so they do not automatically remove old label entries. ## Validation (pending) diff --git a/install.sh b/install.sh index b2f3b2e..47c7b54 100755 --- a/install.sh +++ b/install.sh @@ -675,8 +675,16 @@ configure_all() { printf "cooldown_minutes: 5\n" printf "restart_threshold: 5\n" printf "restart_window_minutes: 10\n" + printf "restart_recovery_stability_seconds: 120\n" printf "unhealthy_cycles_threshold: 3\n" + printf "alert_on_recovery: true\n" + printf "\n# Correlate Docker-requested stops with delayed die events.\n" + printf "lifecycle_detection:\n" + printf " enabled: true\n" + printf " correlation_window_seconds: 3\n" printf "\nexcluded_containers: []\n" + printf "# Optional verified label-based event exclusions; no default MariaDB label rule.\n" + printf "ignored_container_events: []\n" printf "\nrunbook_base_url: \"\"\n" printf "\nlog_level: INFO\n" printf "log_file: /var/log/watchdog/watchdog.log\n" diff --git a/tests/test_installer_image.py b/tests/test_installer_image.py index 005d8d6..e380b54 100644 --- a/tests/test_installer_image.py +++ b/tests/test_installer_image.py @@ -15,6 +15,17 @@ VERSION = (ROOT / "VERSION").read_text().strip() +def config_key_paths(config: dict, prefix: str = "") -> set[str]: + """All active YAML key paths, including parent objects and empty mappings.""" + found: set[str] = set() + for key, value in config.items(): + path = f"{prefix}.{key}" if prefix else str(key) + found.add(path) + if isinstance(value, dict): + found.update(config_key_paths(value, path)) + return found + + class InstallerImageTests(unittest.TestCase): def setUp(self): self.workspace = tempfile.TemporaryDirectory(prefix="watchdog-installer-fake-") @@ -176,7 +187,78 @@ def test_generated_config_includes_optional_ha_disabled(self): self.assertEqual(config["ha_monitor"], { "enabled": False, "exporter_url": "http://127.0.0.1:9664/metrics", }) - self.assertNotIn("ignored_container_events", config) + self.assertEqual(config["ignored_container_events"], []) + self.assertEqual(self.calls(), []) + + def test_generated_yaml_matches_all_active_reference_keys_and_defaults(self): + # Regression guard: every new reference YAML field must also be + # generated by the wizard. Runs only in the fake-Docker workspace. + body = ('detect_docker_gid() { echo 999; }; ' + '_publish_file() { cp "$1" "$2"; }; ' + 'DC=docker; configure_all') + result = self.run_function( + body, input_data="\n".join(["4", "", "", "", "", "", ""]) + "\n" + ) + self.assertEqual(result.returncode, 0, result.output) + generated = yaml.safe_load((self.root / "watchdog-config.yaml").read_text()) + template = yaml.safe_load((ROOT / "watchdog-config.yaml.example").read_text()) + + # Key parity in BOTH directions, including nested settings, avoids + # silently omitting a new option (or documenting an obsolete option). + self.assertEqual(config_key_paths(generated), config_key_paths(template)) + # Wizard-picked destinations/channels necessarily differ from + # example placeholders; all non-interactive defaults must agree. + non_interactive = ( + "check_interval_seconds", "cooldown_minutes", + "restart_threshold", "restart_window_minutes", + "restart_recovery_stability_seconds", "unhealthy_cycles_threshold", + "alert_on_recovery", "lifecycle_detection", + "excluded_containers", "ignored_container_events", + "log_level", "log_file", "state_file", "boot_grace_seconds", + "runbook_base_url", "auto_health_check", "container_health_checks", + "ha_monitor", + ) + for setting in non_interactive: + with self.subTest(setting=setting): + self.assertEqual(generated[setting], template[setting]) + + self.assertEqual(generated["lifecycle_detection"], + {"enabled": True, "correlation_window_seconds": 3}) + self.assertEqual(generated["restart_recovery_stability_seconds"], 120) + self.assertIs(generated["alert_on_recovery"], True) + self.assertEqual(generated["ignored_container_events"], []) + self.assertEqual(self.calls(), []) + + def test_snmpv3_wizard_retains_reference_settings_and_keeps_secrets_in_env(self): + # Conditional SNMPv3 settings are additional to the active reference + # schema, not a reason to omit any common setting. + body = ('detect_docker_gid() { echo 999; }; ' + '_publish_file() { cp "$1" "$2"; }; ' + 'DC=docker; configure_all') + inputs = [ + "3", "nms.example.invalid", "162", "v3", + "snmpuser", "SHA", "example-auth-value", "AES", + "example-priv-value", "", "", + ] + result = self.run_function(body, input_data=chr(10).join(inputs) + chr(10)) + self.assertEqual(result.returncode, 0, result.output) + config = yaml.safe_load((self.root / "watchdog-config.yaml").read_text()) + template = yaml.safe_load((ROOT / "watchdog-config.yaml.example").read_text()) + self.assertTrue(config_key_paths(template) <= config_key_paths(config)) + self.assertEqual(config["alert_channels"], ["snmp_trap"]) + self.assertEqual(config["snmp_trap"]["version"], "v3") + self.assertEqual(config["snmp_trap"]["v3_username"], "snmpuser") + self.assertEqual(config["snmp_trap"]["v3_auth_key_env"], "SNMP_V3_AUTH_KEY") + self.assertEqual(config["snmp_trap"]["v3_priv_key_env"], "SNMP_V3_PRIV_KEY") + self.assertEqual(config["lifecycle_detection"], template["lifecycle_detection"]) + self.assertEqual(config["ignored_container_events"], []) + self.assertNotIn("example-auth-value", + (self.root / "watchdog-config.yaml").read_text()) + self.assertNotIn("example-priv-value", + (self.root / "watchdog-config.yaml").read_text()) + env = (self.root / ".env").read_text() + self.assertIn("SNMP_V3_AUTH_KEY=example-auth-value", env) + self.assertIn("SNMP_V3_PRIV_KEY=example-priv-value", env) self.assertEqual(self.calls(), []) def test_generated_config_can_enable_ha_with_custom_exporter(self): diff --git a/watchdog-config.yaml.example b/watchdog-config.yaml.example index 01915ae..b60a0c9 100644 --- a/watchdog-config.yaml.example +++ b/watchdog-config.yaml.example @@ -42,7 +42,9 @@ excluded_containers: [] # No label rule is needed for that helper. ignored_container_events: [] - +# Optional runbook URL included in alert details (leave empty when not configured). +runbook_base_url: "" + # ── Logging ─────────────────────────────────────────────────────────────────── log_level: INFO # DEBUG | INFO | WARNING | ERROR (default INFO) log_file: /var/log/watchdog/watchdog.log # bind-mounted to ./watchdog/watchdog.log on host