diff --git a/docs/dev/HA_MONITORING.md b/docs/dev/HA_MONITORING.md index 1b5ce4e..210806c 100644 --- a/docs/dev/HA_MONITORING.md +++ b/docs/dev/HA_MONITORING.md @@ -11,6 +11,8 @@ | Run HA polling in a **separate thread and HTTP session** from Docker polling | Slow or blocked Docker API calls must not delay HA observations; HA exporter failures must not interrupt container monitoring | | Require **quorum, a successful Pacemaker scrape, one local node and exactly one promoted owner** | Missing or conflicting telemetry is not proof of a role change; fail closed instead of inventing a failover | | Require **two consecutive valid observations** of a new active node | Avoid reporting transient/promoted-resource ambiguity while the cluster is transitioning | +| Allow **two consecutive no-promoted-owner observations** before logging an ERROR on the third | A failover may temporarily have no active/promoted operator. Avoid transient Syslog noise, but do not hide persistent loss of an owner | +| Report **multiple promoted owners, exporter failures and loss of quorum immediately** | These are not an ordinary empty-promotion gap and must not be silenced by the grace period | | **Only the newly active node** emits the alert | Watchdogs run on both nodes; duplicate notifications from old and new active nodes would be misleading | | Persist the confirmed active node using the existing state file | Survive watchdog restarts without replaying an old role change or creating a second state store | | Reuse the existing notification channels and SNMP MIB | Treat HA as another monitored event without adding a new alert transport or breaking container notifications | @@ -21,7 +23,7 @@ 2. It reads `ha_cluster_pacemaker_resources{resource="HAOperatorRep-Res",role="promoted",status="active"} == 1` to identify the active owner and `ha_cluster_corosync_member_votes{local="true"}` to identify the local node. The Pacemaker scrape-success and quorum metrics must both be 1. 3. The **first valid reading** establishes a silent baseline. An active-node change must appear in **two valid consecutive checks** before it is confirmed. 4. On a confirmed transition A → B, both nodes can update their saved baseline, but **only the watchdog local to B** issues one `WARNING` `ha-failover` event with previous/new node details. -5. Invalid/unavailable metrics leave the last confirmed baseline unchanged and reset a pending candidate. Exporter errors are logged at most once every five minutes; they never trigger a speculative failover alert. +5. Invalid/unavailable metrics leave the last confirmed baseline unchanged and reset a pending candidate. A valid Pacemaker scrape with quorum but **zero promoted HA Operator** matches is treated as a possible promotion gap: the first two consecutive observations are DEBUG-only; the third consecutive observation generates an ERROR. At the default 60-second interval, the third observation is about two minutes after the first. A valid observation resets the counter. **Multiple promoted owners**, failed Pacemaker scrape/lost quorum, and exporter transport failures generate an immediate ERROR. Repeated identical diagnostic errors are throttled to at most once every five minutes; a different failure reason is logged immediately. These observations never trigger a speculative failover alert. 6. HA state is kept separately from container incidents in `watchdog/state.json`. Restart with monitoring enabled restores that state. Running the watchdog with HA monitoring disabled clears its baseline, so a later re-enable starts silently. 7. During the host reboot grace interval, HA role checks are deferred. HA events are not included in container recovery or reboot-summary logic. diff --git a/docs/operations/INSTALLATION_AND_OPERATIONS.md b/docs/operations/INSTALLATION_AND_OPERATIONS.md index 05ae4bc..8202236 100644 --- a/docs/operations/INSTALLATION_AND_OPERATIONS.md +++ b/docs/operations/INSTALLATION_AND_OPERATIONS.md @@ -258,6 +258,8 @@ ha_monitor: Enable this only for HA deployments. Verify its URL read-only, then restart **only** the watchdog ([Section 4.6](#46-apply-configuration-changes)). The first valid reading sets a baseline without sending an alert. +Starting in v1.6.1, up to two consecutive HA exporter observations with **no promoted HA Operator** are treated as a possible short failover transition and do not generate Syslog ERROR noise. The third consecutive observation is an ERROR (about two minutes after the first with default 60-second polling). Multiple promoted owners, exporter failures and missing quorum remain immediate errors; the watchdog does not automatically remediate Pacemaker or change application containers. + ### 4.5 Environment variables The installer manages most `.env` values. SMTP passwords, Slack webhooks and SNMPv3 passphrases belong here, not in YAML. The SNMPv1/v2c community is configured in YAML, so protect that file too. diff --git a/ha_monitor.py b/ha_monitor.py index c2c52ed..d6b2f45 100644 --- a/ha_monitor.py +++ b/ha_monitor.py @@ -21,6 +21,10 @@ } +class NoPromotedHAOperatorError(ValueError): + """Valid HA scrape/quorum, but no uniquely promoted owner *yet*.""" + + @dataclass(frozen=True) class HAObservation: active_node: str @@ -94,8 +98,12 @@ def parse_ha_metrics(text: str) -> HAObservation: raise ValueError("Pacemaker scrape failed or cluster has no quorum") if len(local) != 1 or not local[0] or len(local[0]) > 255: raise ValueError("cannot identify unique local HA node") - if len(promoted) != 1 or not promoted[0] or len(promoted[0]) > 255: - raise ValueError("cannot identify unique promoted HA Operator") + if not promoted: + raise NoPromotedHAOperatorError("no promoted HA Operator observed") + if len(promoted) > 1: + raise ValueError("multiple promoted HA Operators observed") + if not promoted[0] or len(promoted[0]) > 255: + raise ValueError("invalid promoted HA Operator node") return HAObservation(active_node=promoted[0], local_node=local[0]) diff --git a/tests/test_ha_monitor.py b/tests/test_ha_monitor.py index d84cce7..76d0f23 100644 --- a/tests/test_ha_monitor.py +++ b/tests/test_ha_monitor.py @@ -9,7 +9,7 @@ import watchdog from ha_monitor import ( - HAObservation, HARoleTracker, MAX_METRICS_BYTES, + HAObservation, HARoleTracker, MAX_METRICS_BYTES, NoPromotedHAOperatorError, parse_ha_metrics, read_ha_observation, ) @@ -68,6 +68,22 @@ def test_fail_closed_on_invalid_state(self): with self.subTest(sample=sample[-75:]), self.assertRaises(ValueError): parse_ha_metrics(sample) + def test_zero_promoted_is_distinct_from_multiple_promoted(self): + no_promoted = metrics().replace( + 'role="promoted",resource="HAOperatorRep-Res"', + 'role="unpromoted",resource="HAOperatorRep-Res"') + with self.assertRaisesRegex(NoPromotedHAOperatorError, "no promoted"): + parse_ha_metrics(no_promoted) + + two_promoted = metrics(extras=( + f'ha_cluster_pacemaker_resources{{node="{B}",role="promoted",' + 'resource="HAOperatorRep-Res",status="active"} 1')) + with self.assertRaisesRegex(ValueError, "multiple promoted"): + parse_ha_metrics(two_promoted) + with self.assertRaises(ValueError) as raised: + parse_ha_metrics(two_promoted) + self.assertNotIsInstance(raised.exception, NoPromotedHAOperatorError) + def test_ignores_non_matching_roles_and_untrusted_other_metric(self): sample = metrics(owner=A, local=A, extras= 'ha_cluster_pacemaker_resources{node="x",resource="HAOperatorRep-Res",role="promoted",status="active"} 0\n' @@ -147,6 +163,8 @@ def create_shell(self, state_file): w._ha_enabled = True w._ha_tracker = HARoleTracker(w.state.get_ha_node()) w._ha_error_log_at = 0.0 + w._ha_last_error_reason = None + w._ha_missing_promoted_polls = 0 w._session = Mock() w._boot_time = 123 w._grace_until = 0.0 @@ -187,7 +205,7 @@ def test_ha_exporter_failure_is_error_but_throttled(self): with tempfile.TemporaryDirectory() as td: w = self.create_shell(f"{td}/state.json") with patch("watchdog.read_ha_observation", side_effect=TimeoutError("offline")), \ - patch("watchdog.time.monotonic", side_effect=[1000, 1000, 1100, 1301, 1301]), \ + patch("watchdog.time.monotonic", side_effect=[1000, 1100, 1301]), \ patch("watchdog.log.error") as err: w._poll_ha(w._session) w._poll_ha(w._session) @@ -196,6 +214,94 @@ def test_ha_exporter_failure_is_error_but_throttled(self): self.assertIn("HA check unavailable", err.call_args.args[0]) w._dispatch.assert_not_called() + def test_short_no_promoted_gap_is_quiet_and_does_not_change_state(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + w._ha_tracker = HARoleTracker(A) + missing = NoPromotedHAOperatorError("no promoted HA Operator observed") + with patch("watchdog.read_ha_observation", side_effect=[ + missing, missing, HAObservation(A, A)]), \ + patch("watchdog.log.error") as err: + for _ in range(3): + w._poll_ha(w._session) + err.assert_not_called() # no Syslog ERROR on normal short failover gap + w._dispatch.assert_not_called() + self.assertEqual(w._ha_tracker.confirmed, A) + self.assertEqual(w._ha_missing_promoted_polls, 0) + + def test_persistent_no_promoted_gap_reports_after_three_polls_and_throttles(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + w._ha_tracker = HARoleTracker(A) + missing = NoPromotedHAOperatorError("no promoted HA Operator observed") + with patch("watchdog.read_ha_observation", side_effect=missing), \ + patch("watchdog.time.monotonic", side_effect=[1120, 1180, 1421]), \ + patch("watchdog.log.error") as err: + w._poll_ha(w._session) + w._poll_ha(w._session) + err.assert_not_called() + w._poll_ha(w._session) # third consecutive no-owner observation + self.assertEqual(err.call_count, 1) + w._poll_ha(w._session) # 60s later, throttled + self.assertEqual(err.call_count, 1) + w._poll_ha(w._session) # >= 300s later, diagnostic repeated + self.assertEqual(err.call_count, 2) + self.assertIn("NoPromotedHAOperatorError", err.call_args.args) + self.assertEqual(w._ha_tracker.confirmed, A) + w._dispatch.assert_not_called() + + def test_good_observation_resets_no_promoted_counter(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + w._ha_tracker = HARoleTracker(A) + missing = NoPromotedHAOperatorError("no promoted HA Operator observed") + with patch("watchdog.read_ha_observation", side_effect=[ + missing, HAObservation(A, A), missing, missing]), \ + patch("watchdog.log.error") as err: + for _ in range(4): + w._poll_ha(w._session) + err.assert_not_called() + self.assertEqual(w._ha_missing_promoted_polls, 2) + + def test_critical_ha_faults_are_immediate_even_after_transition_gap(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + missing = NoPromotedHAOperatorError("no promoted HA Operator observed") + faults = [ + missing, missing, + ValueError("multiple promoted HA Operators observed"), + TimeoutError("exporter unavailable"), + ValueError("Pacemaker scrape failed or cluster has no quorum"), + ] + with patch("watchdog.read_ha_observation", side_effect=faults), \ + patch("watchdog.time.monotonic", return_value=1000), \ + patch("watchdog.log.error") as err: + for _ in range(2): + w._poll_ha(w._session) + err.assert_not_called() + w._poll_ha(w._session) + self.assertEqual(err.call_count, 1) + w._poll_ha(w._session) + self.assertEqual(err.call_count, 2) + w._poll_ha(w._session) + self.assertEqual(err.call_count, 3) + self.assertEqual(w._ha_missing_promoted_polls, 0) + w._dispatch.assert_not_called() + + def test_failover_after_short_gap_still_requires_two_good_polls(self): + with tempfile.TemporaryDirectory() as td: + w = self.create_shell(f"{td}/state.json") + w._ha_tracker = HARoleTracker(A) + missing = NoPromotedHAOperatorError("no promoted HA Operator observed") + with patch("watchdog.read_ha_observation", side_effect=[ + missing, missing, HAObservation(B, B), HAObservation(B, B)]), \ + patch("watchdog.log.error") as err: + for _ in range(4): + w._poll_ha(w._session) + err.assert_not_called() + w._dispatch.assert_called_once() + self.assertEqual(w.state.get_ha_node(), B) + def test_failed_exporter_does_not_overwrite_state_or_alert(self): with tempfile.TemporaryDirectory() as td: w = self.create_shell(f"{td}/state.json") diff --git a/watchdog.py b/watchdog.py index 9922d84..734367c 100644 --- a/watchdog.py +++ b/watchdog.py @@ -27,11 +27,12 @@ import yaml from lifecycle import DockerLifecycleCorrelator -from ha_monitor import HARoleTracker, read_ha_observation +from ha_monitor import HARoleTracker, NoPromotedHAOperatorError, read_ha_observation # ── Logging setup ───────────────────────────────────────────────────────────── LOG_FORMAT = "%(asctime)s [%(levelname)s] %(name)s: %(message)s" SYSLOG_FORMAT = "watchdog[%(process)d]: %(message)s" +HA_NO_PROMOTED_ERROR_POLLS = 3 # Ignore short promotion gaps, never indefinite loss. class AlertSyslogFilter(logging.Filter): @@ -1422,6 +1423,8 @@ def __init__(self, cfg: dict): self.state.save(self._boot_time) self._ha_tracker = HARoleTracker(self.state.get_ha_node()) self._ha_error_log_at = float("-inf") + self._ha_last_error_reason = None + self._ha_missing_promoted_polls = 0 # ── State restore ───────────────────────────────────────────────────────── def _restore_state(self) -> None: @@ -1882,11 +1885,28 @@ def _poll_ha(self, session: requests.Session) -> None: observation = read_ha_observation(session, url) except Exception as exc: self._ha_tracker.reset_candidate() - if time.monotonic() - self._ha_error_log_at >= 300: - self._ha_error_log_at = time.monotonic() + if isinstance(exc, NoPromotedHAOperatorError): + self._ha_missing_promoted_polls += 1 + if self._ha_missing_promoted_polls < HA_NO_PROMOTED_ERROR_POLLS: + log.debug("HA promotion pending (%d/%d): %s", + self._ha_missing_promoted_polls, + HA_NO_PROMOTED_ERROR_POLLS, exc) + return + else: + # Multiple promoters, lost quorum and exporter failures are not + # normal promotion gaps; report their first occurrence at once. + self._ha_missing_promoted_polls = 0 + + reason = (type(exc).__name__, str(exc)) + now = time.monotonic() + if reason != self._ha_last_error_reason or now - self._ha_error_log_at >= 300: + self._ha_error_log_at = now + self._ha_last_error_reason = reason log.error("HA check unavailable (%s): %s", type(exc).__name__, exc) return + self._ha_missing_promoted_polls = 0 + self._ha_last_error_reason = None changed, change = self._ha_tracker.observe(observation) if changed: self.state.set_ha_node(self._ha_tracker.confirmed)