From 168a58223551f1ff5de8d4e69db1b353d6679102 Mon Sep 17 00:00:00 2001 From: Bartosz Burda Date: Thu, 27 Aug 2026 15:21:59 +0200 Subject: [PATCH] fault_manager: document which debounce lever fits which reporter, and test it The debounce counter only moves when an event arrives, so confirmation_threshold and healing_threshold only work for a reporter that keeps sending FAILED while a condition is still there. The docs recommended confirmation_threshold: -3 with healing_threshold: 3 to everyone. For a reporter that sends one FAILED per raise and one clear per de-assert the second event never comes. The fault stops at PREFAILED and never confirms, and ListFaults with an empty status filter returns CONFIRMED only, so nobody sees it. Healing breaks the same way: it needs healing_threshold minus confirmation_threshold consecutive PASSED events and only one is sent, so a confirmed fault stays CONFIRMED until someone calls ~/clear_fault. auto_confirm_after_sec is the lever for that kind of reporter and already works. It holds the first FAILED in PREFAILED and confirms it only if it is still there when the window closes, so a condition that recovers never reaches an operator. The docs listed the parameter but never said what it is for. Add an integration test driving the real node over the real services with the event counts such a reporter sends: one FAILED per raise, one PASSED per clear. It also pins the case where a condition clears inside the window, which is what stops a config from passing by only delaying a false alarm. No production code change. --- docs/config/fault-manager.rst | 26 ++ src/ros2_medkit_fault_manager/CMakeLists.txt | 6 + src/ros2_medkit_fault_manager/README.md | 30 +++ .../test_debounce_and_healing.test.py | 249 ++++++++++++++++++ 4 files changed, 311 insertions(+) create mode 100644 src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index 6c9e6cdb3..a2428647d 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -75,6 +75,32 @@ The fault manager uses AUTOSAR DEM-style debounce filtering to prevent fault fla For immediate fault confirmation (no debounce), set ``confirmation_threshold: 0``. Faults with ``SEVERITY_CRITICAL`` always bypass debounce regardless of this setting. +.. important:: + + The counter only moves when an event arrives, so ``confirmation_threshold`` and + ``healing_threshold`` work only for a reporter that keeps sending FAILED while the condition + is still there. + + A reporter that sends one FAILED when a condition appears and one clear when it goes away + never sends the second event. ``confirmation_threshold: -3`` then leaves the fault in + PREFAILED forever, and the default fault list returns CONFIRMED only, so the fault is never + seen. ``healing_threshold: 3`` has the same problem: healing needs + ``healing_threshold - confirmation_threshold`` consecutive PASSED events, and only one is + sent, so the fault stays CONFIRMED until someone calls ``~/clear_fault``. + + For such a reporter, filter by time instead of by count: + + .. code-block:: yaml + + confirmation_threshold: -2 # first FAILED stays PREFAILED + auto_confirm_after_sec: 3.0 # confirm it if it is still there after 3 s + healing_enabled: true + healing_threshold: 0 # heal on the single PASSED + + Choose ``auto_confirm_after_sec`` from how often the reporter samples, so a condition has to + survive a few sampling cycles before it confirms. A glitch that clears in time never reaches + CONFIRMED, because the clear takes the fault out of PREFAILED before the timer fires. + Near-Miss Retention ~~~~~~~~~~~~~~~~~~~ diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 624b4d4c2..98090512e 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -192,6 +192,12 @@ if(BUILD_TESTING) medkit_add_launch_test(test_entity_thresholds_integration test/integration/test_entity_thresholds_integration.test.py TIMEOUT 60 LABELS "integration") + # Drives the debounce and healing pair with the event counts an edge-triggered + # reporter actually sends: one FAILED per raise, one PASSED per clear. Two + # cases wait out the auto-confirm window, hence the timeout. + medkit_add_launch_test(test_debounce_and_healing test/integration/test_debounce_and_healing.test.py + TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_rosbag_entity_scope test/integration/test_rosbag_entity_scope.test.py TIMEOUT 120 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 98ac04883..b68daea97 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -263,6 +263,36 @@ events to return to the default (CONFIRMED-only) list. During that window `last_ reflects the activity; `occurrence_count` does not, because it counts the edge that started the occurrence, not every report within it. +### Choosing the right lever for your reporter + +The counter only moves when an event arrives, so the count-based settings above work only for a +reporter that keeps sending FAILED while the condition is still there. A reporter that samples a +value on a timer and reports on every sample is of that kind. + +Many reporters do not work that way. They send one FAILED when the condition appears and one clear +when it goes away, and nothing in between. For such a reporter the second FAILED never arrives, so +`confirmation_threshold: -3` means the fault stays PREFAILED and never confirms. The default fault +list returns CONFIRMED only, so the fault is invisible. Healing has the same problem in reverse: +`healing_threshold: 3` needs four consecutive PASSED events after a fault confirmed at `-1`, and +only one PASSED is ever sent, so the fault stays CONFIRMED until someone calls `~/clear_fault`. + +Pick by how your reporter behaves: + +| Reporter repeats FAILED while the condition holds | Reporter sends one event per transition | +|---|---| +| `confirmation_threshold: -N` filters N noisy samples | `confirmation_threshold: -2` plus `auto_confirm_after_sec` | +| `healing_threshold: N` needs N clean samples | `healing_threshold: 0` heals on the single PASSED | + +For the second column the filtering is done by time, not by count. `confirmation_threshold: -2` +keeps the first FAILED in PREFAILED, and `auto_confirm_after_sec` confirms it if it is still there +when the timeout expires. Choose the timeout from how often the reporter samples, so a condition +has to survive a few sampling cycles before it confirms. A glitch that clears in time never +reaches CONFIRMED, because a clear takes the fault out of PREFAILED before the timer fires. + +Two settings ignore the counter completely. `SEVERITY_CRITICAL` confirms at once while +`critical_immediate_confirm` is true, which is the default. `auto_confirm_after_sec` promotes a +PREFAILED fault without changing its counter. + ### Fault Lifecycle with Debounce ``` diff --git a/src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py b/src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py new file mode 100644 index 000000000..c7770ecf6 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Debounce and healing contract for an edge-triggered fault reporter. + +A reporter is edge-triggered when it sends one FAILED event as a condition +appears and one clear as it goes away, instead of repeating FAILED on every +sample. The count-based debounce cannot filter a noisy sample for such a +reporter: the second FAILED that would move the counter never arrives. The +time-based lever does the filtering instead, so these tests pin the pair. + +Every case sends the number of events an edge-triggered reporter really sends +(one), never the number the counter would need. +""" + +import os +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.markers +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +DATABASE_PATH = os.path.join(tempfile.mkdtemp(prefix='debounce_healing_'), 'faults.db') + +# Seconds a fault stays PREFAILED before the timer confirms it. The node runs +# that timer once a second, so a confirmation lands within AUTO_CONFIRM_SEC + 1. +AUTO_CONFIRM_SEC = 3.0 + +# Every status, so a test can see a fault the default CONFIRMED-only filter hides. +ALL_STATUSES = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + + +def generate_test_description(): + """Launch fault_manager with the appliance debounce and healing settings.""" + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'sqlite', + 'database_path': DATABASE_PATH, + # Below -1, so the first FAILED lands in PREFAILED instead of + # confirming. An edge-triggered reporter never sends the second + # event, so the counter stays here and the timer below decides. + 'confirmation_threshold': -2, + 'auto_confirm_after_sec': AUTO_CONFIRM_SEC, + # A clear arrives as one PASSED event, so healing has to finish on + # that one event. Threshold 0 is what makes it reachable. + 'healing_enabled': True, + 'healing_threshold': 0, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + + return ( + LaunchDescription([ + fault_manager_node, + launch_testing.actions.ReadyToTest(), + ]), + { + 'fault_manager_node': fault_manager_node, + }, + ) + + +class TestDebounceAndHealing(unittest.TestCase): + """One failed read must not confirm, and a de-assert must heal unattended.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_debounce_healing_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + + assert cls.report_client.wait_for_service(timeout_sec=10.0), \ + 'report_fault service not available' + assert cls.list_client.wait_for_service(timeout_sec=10.0), \ + 'list_faults service not available' + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=5.0) + self.assertIsNotNone(future.result(), 'Service call timed out') + return future.result() + + def _report(self, fault_code, event_type, severity=Fault.SEVERITY_ERROR): + """Send one ReportFault event, the way an edge-triggered reporter does.""" + request = ReportFault.Request() + request.fault_code = fault_code + request.event_type = event_type + request.severity = severity + request.description = 'debounce and healing contract test' + request.source_id = '/test_plc' + response = self._call(self.report_client, request) + self.assertTrue(response.accepted, f'ReportFault rejected for {fault_code}') + + def _status_of(self, fault_code): + """Return the status of one fault, or None when the store has no such fault.""" + request = ListFaults.Request() + request.statuses = ALL_STATUSES + response = self._call(self.list_client, request) + for fault in response.faults: + if fault.fault_code == fault_code: + return fault.status + return None + + def _default_filter_codes(self): + """Fault codes an operator sees with no status filter (CONFIRMED only).""" + response = self._call(self.list_client, ListFaults.Request()) + return [fault.fault_code for fault in response.faults] + + def _wait_for_status(self, fault_code, expected, timeout_sec): + """Poll until the fault reaches expected, returning the last status seen.""" + deadline = time.time() + timeout_sec + status = self._status_of(fault_code) + while time.time() < deadline and status != expected: + time.sleep(0.25) + status = self._status_of(fault_code) + return status + + def test_01_single_failed_read_does_not_confirm(self): + """One bad sample must not raise a confirmed fault.""" + code = 'PLC_SINGLE_READ' + self._report(code, ReportFault.Request.EVENT_FAILED) + + status = self._status_of(code) + self.assertIsNotNone(status, 'fault was not recorded at all') + self.assertNotEqual( + status, Fault.STATUS_CONFIRMED, + 'a single failed read confirmed the fault immediately' + ) + self.assertEqual(status, Fault.STATUS_PREFAILED) + + def test_02_prefailed_fault_is_hidden_from_the_default_list(self): + """A not-yet-confirmed fault must not reach an operator listing.""" + code = 'PLC_HIDDEN_WHILE_PENDING' + self._report(code, ReportFault.Request.EVENT_FAILED) + + self.assertNotIn( + code, self._default_filter_codes(), + 'an unconfirmed fault is already visible in the default fault list' + ) + + def test_03_sustained_condition_confirms_with_nobody_acting(self): + """ + A real fault must still surface. + + The reporter sends its one FAILED and never repeats it, so only the + time-based lever can promote this. Without it the fault would stay + PREFAILED forever and the appliance would go quiet. + """ + code = 'PLC_SUSTAINED' + self._report(code, ReportFault.Request.EVENT_FAILED) + + status = self._wait_for_status( + code, Fault.STATUS_CONFIRMED, AUTO_CONFIRM_SEC + 5.0 + ) + self.assertEqual( + status, Fault.STATUS_CONFIRMED, + f'a sustained fault never confirmed, it is still {status}' + ) + self.assertIn(code, self._default_filter_codes()) + + def test_04_transient_that_clears_in_time_never_confirms(self): + """ + The falsifying case for the whole setting. + + A glitch that goes away before the window closes must never confirm. + If it does, the configuration only delays a false alarm rather than + filtering it. + """ + code = 'PLC_TRANSIENT' + self._report(code, ReportFault.Request.EVENT_FAILED) + self._report(code, ReportFault.Request.EVENT_PASSED) + + # Sit past the auto-confirm window and the timer tick behind it. + time.sleep(AUTO_CONFIRM_SEC + 3.0) + + status = self._status_of(code) + self.assertNotEqual( + status, Fault.STATUS_CONFIRMED, + 'a transient that already cleared was confirmed by the timer' + ) + self.assertNotIn(code, self._default_filter_codes()) + + def test_05_deasserted_alarm_heals_with_nobody_acting(self): + """ + A confirmed fault must return to healed on the reporter's single clear. + + The reporter sends exactly one PASSED, so healing has to complete on + that one event. With healing off, or with a threshold above zero, the + fault stays CONFIRMED until a human clears it. + """ + code = 'PLC_DEASSERTED' + self._report(code, ReportFault.Request.EVENT_FAILED) + + status = self._wait_for_status( + code, Fault.STATUS_CONFIRMED, AUTO_CONFIRM_SEC + 5.0 + ) + self.assertEqual( + status, Fault.STATUS_CONFIRMED, 'fault never confirmed, cannot test healing' + ) + + self._report(code, ReportFault.Request.EVENT_PASSED) + + status = self._status_of(code) + self.assertEqual( + status, Fault.STATUS_HEALED, + f'a de-asserted alarm did not heal on its single clear, it is {status}' + ) + self.assertNotIn(code, self._default_filter_codes()) + + +@launch_testing.post_shutdown_test() +class TestDebounceAndHealingShutdown(unittest.TestCase): + """Check the node exited cleanly.""" + + def test_exit_code(self, proc_info, fault_manager_node): + launch_testing.asserts.assertExitCodes( + proc_info, allowable_exit_codes=[0, -2, -15], process=fault_manager_node + )