diff --git a/docs/config/fault-manager.rst b/docs/config/fault-manager.rst index 6c9e6cdb3..a2428647d 100644 --- a/docs/config/fault-manager.rst +++ b/docs/config/fault-manager.rst @@ -75,6 +75,32 @@ The fault manager uses AUTOSAR DEM-style debounce filtering to prevent fault fla For immediate fault confirmation (no debounce), set ``confirmation_threshold: 0``. Faults with ``SEVERITY_CRITICAL`` always bypass debounce regardless of this setting. +.. important:: + + The counter only moves when an event arrives, so ``confirmation_threshold`` and + ``healing_threshold`` work only for a reporter that keeps sending FAILED while the condition + is still there. + + A reporter that sends one FAILED when a condition appears and one clear when it goes away + never sends the second event. ``confirmation_threshold: -3`` then leaves the fault in + PREFAILED forever, and the default fault list returns CONFIRMED only, so the fault is never + seen. ``healing_threshold: 3`` has the same problem: healing needs + ``healing_threshold - confirmation_threshold`` consecutive PASSED events, and only one is + sent, so the fault stays CONFIRMED until someone calls ``~/clear_fault``. + + For such a reporter, filter by time instead of by count: + + .. code-block:: yaml + + confirmation_threshold: -2 # first FAILED stays PREFAILED + auto_confirm_after_sec: 3.0 # confirm it if it is still there after 3 s + healing_enabled: true + healing_threshold: 0 # heal on the single PASSED + + Choose ``auto_confirm_after_sec`` from how often the reporter samples, so a condition has to + survive a few sampling cycles before it confirms. A glitch that clears in time never reaches + CONFIRMED, because the clear takes the fault out of PREFAILED before the timer fires. + Near-Miss Retention ~~~~~~~~~~~~~~~~~~~ diff --git a/src/ros2_medkit_fault_manager/CMakeLists.txt b/src/ros2_medkit_fault_manager/CMakeLists.txt index 624b4d4c2..98090512e 100644 --- a/src/ros2_medkit_fault_manager/CMakeLists.txt +++ b/src/ros2_medkit_fault_manager/CMakeLists.txt @@ -192,6 +192,12 @@ if(BUILD_TESTING) medkit_add_launch_test(test_entity_thresholds_integration test/integration/test_entity_thresholds_integration.test.py TIMEOUT 60 LABELS "integration") + # Drives the debounce and healing pair with the event counts an edge-triggered + # reporter actually sends: one FAILED per raise, one PASSED per clear. Two + # cases wait out the auto-confirm window, hence the timeout. + medkit_add_launch_test(test_debounce_and_healing test/integration/test_debounce_and_healing.test.py + TIMEOUT 120 LABELS "integration") + medkit_add_launch_test(test_rosbag_entity_scope test/integration/test_rosbag_entity_scope.test.py TIMEOUT 120 LABELS "integration") diff --git a/src/ros2_medkit_fault_manager/README.md b/src/ros2_medkit_fault_manager/README.md index 98ac04883..b68daea97 100644 --- a/src/ros2_medkit_fault_manager/README.md +++ b/src/ros2_medkit_fault_manager/README.md @@ -263,6 +263,36 @@ events to return to the default (CONFIRMED-only) list. During that window `last_ reflects the activity; `occurrence_count` does not, because it counts the edge that started the occurrence, not every report within it. +### Choosing the right lever for your reporter + +The counter only moves when an event arrives, so the count-based settings above work only for a +reporter that keeps sending FAILED while the condition is still there. A reporter that samples a +value on a timer and reports on every sample is of that kind. + +Many reporters do not work that way. They send one FAILED when the condition appears and one clear +when it goes away, and nothing in between. For such a reporter the second FAILED never arrives, so +`confirmation_threshold: -3` means the fault stays PREFAILED and never confirms. The default fault +list returns CONFIRMED only, so the fault is invisible. Healing has the same problem in reverse: +`healing_threshold: 3` needs four consecutive PASSED events after a fault confirmed at `-1`, and +only one PASSED is ever sent, so the fault stays CONFIRMED until someone calls `~/clear_fault`. + +Pick by how your reporter behaves: + +| Reporter repeats FAILED while the condition holds | Reporter sends one event per transition | +|---|---| +| `confirmation_threshold: -N` filters N noisy samples | `confirmation_threshold: -2` plus `auto_confirm_after_sec` | +| `healing_threshold: N` needs N clean samples | `healing_threshold: 0` heals on the single PASSED | + +For the second column the filtering is done by time, not by count. `confirmation_threshold: -2` +keeps the first FAILED in PREFAILED, and `auto_confirm_after_sec` confirms it if it is still there +when the timeout expires. Choose the timeout from how often the reporter samples, so a condition +has to survive a few sampling cycles before it confirms. A glitch that clears in time never +reaches CONFIRMED, because a clear takes the fault out of PREFAILED before the timer fires. + +Two settings ignore the counter completely. `SEVERITY_CRITICAL` confirms at once while +`critical_immediate_confirm` is true, which is the default. `auto_confirm_after_sec` promotes a +PREFAILED fault without changing its counter. + ### Fault Lifecycle with Debounce ``` diff --git a/src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py b/src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py new file mode 100644 index 000000000..c7770ecf6 --- /dev/null +++ b/src/ros2_medkit_fault_manager/test/integration/test_debounce_and_healing.test.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +# Copyright 2026 bburda +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Debounce and healing contract for an edge-triggered fault reporter. + +A reporter is edge-triggered when it sends one FAILED event as a condition +appears and one clear as it goes away, instead of repeating FAILED on every +sample. The count-based debounce cannot filter a noisy sample for such a +reporter: the second FAILED that would move the counter never arrives. The +time-based lever does the filtering instead, so these tests pin the pair. + +Every case sends the number of events an edge-triggered reporter really sends +(one), never the number the counter would need. +""" + +import os +import tempfile +import time +import unittest + +from launch import LaunchDescription +import launch_ros.actions +import launch_testing.actions +import launch_testing.markers +import rclpy +from rclpy.node import Node +from ros2_medkit_msgs.msg import Fault +from ros2_medkit_msgs.srv import ListFaults, ReportFault + +DATABASE_PATH = os.path.join(tempfile.mkdtemp(prefix='debounce_healing_'), 'faults.db') + +# Seconds a fault stays PREFAILED before the timer confirms it. The node runs +# that timer once a second, so a confirmation lands within AUTO_CONFIRM_SEC + 1. +AUTO_CONFIRM_SEC = 3.0 + +# Every status, so a test can see a fault the default CONFIRMED-only filter hides. +ALL_STATUSES = ['PREFAILED', 'PREPASSED', 'CONFIRMED', 'HEALED', 'CLEARED'] + + +def generate_test_description(): + """Launch fault_manager with the appliance debounce and healing settings.""" + fault_manager_node = launch_ros.actions.Node( + package='ros2_medkit_fault_manager', + executable='fault_manager_node', + name='fault_manager', + output='screen', + parameters=[{ + 'storage_type': 'sqlite', + 'database_path': DATABASE_PATH, + # Below -1, so the first FAILED lands in PREFAILED instead of + # confirming. An edge-triggered reporter never sends the second + # event, so the counter stays here and the timer below decides. + 'confirmation_threshold': -2, + 'auto_confirm_after_sec': AUTO_CONFIRM_SEC, + # A clear arrives as one PASSED event, so healing has to finish on + # that one event. Threshold 0 is what makes it reachable. + 'healing_enabled': True, + 'healing_threshold': 0, + }], + sigterm_timeout='30', + sigkill_timeout='15', + ) + + return ( + LaunchDescription([ + fault_manager_node, + launch_testing.actions.ReadyToTest(), + ]), + { + 'fault_manager_node': fault_manager_node, + }, + ) + + +class TestDebounceAndHealing(unittest.TestCase): + """One failed read must not confirm, and a de-assert must heal unattended.""" + + @classmethod + def setUpClass(cls): + rclpy.init() + cls.node = Node('test_debounce_healing_client') + cls.report_client = cls.node.create_client(ReportFault, '/fault_manager/report_fault') + cls.list_client = cls.node.create_client(ListFaults, '/fault_manager/list_faults') + + assert cls.report_client.wait_for_service(timeout_sec=10.0), \ + 'report_fault service not available' + assert cls.list_client.wait_for_service(timeout_sec=10.0), \ + 'list_faults service not available' + + @classmethod + def tearDownClass(cls): + cls.node.destroy_node() + rclpy.shutdown() + + def _call(self, client, request): + future = client.call_async(request) + rclpy.spin_until_future_complete(self.node, future, timeout_sec=5.0) + self.assertIsNotNone(future.result(), 'Service call timed out') + return future.result() + + def _report(self, fault_code, event_type, severity=Fault.SEVERITY_ERROR): + """Send one ReportFault event, the way an edge-triggered reporter does.""" + request = ReportFault.Request() + request.fault_code = fault_code + request.event_type = event_type + request.severity = severity + request.description = 'debounce and healing contract test' + request.source_id = '/test_plc' + response = self._call(self.report_client, request) + self.assertTrue(response.accepted, f'ReportFault rejected for {fault_code}') + + def _status_of(self, fault_code): + """Return the status of one fault, or None when the store has no such fault.""" + request = ListFaults.Request() + request.statuses = ALL_STATUSES + response = self._call(self.list_client, request) + for fault in response.faults: + if fault.fault_code == fault_code: + return fault.status + return None + + def _default_filter_codes(self): + """Fault codes an operator sees with no status filter (CONFIRMED only).""" + response = self._call(self.list_client, ListFaults.Request()) + return [fault.fault_code for fault in response.faults] + + def _wait_for_status(self, fault_code, expected, timeout_sec): + """Poll until the fault reaches expected, returning the last status seen.""" + deadline = time.time() + timeout_sec + status = self._status_of(fault_code) + while time.time() < deadline and status != expected: + time.sleep(0.25) + status = self._status_of(fault_code) + return status + + def test_01_single_failed_read_does_not_confirm(self): + """One bad sample must not raise a confirmed fault.""" + code = 'PLC_SINGLE_READ' + self._report(code, ReportFault.Request.EVENT_FAILED) + + status = self._status_of(code) + self.assertIsNotNone(status, 'fault was not recorded at all') + self.assertNotEqual( + status, Fault.STATUS_CONFIRMED, + 'a single failed read confirmed the fault immediately' + ) + self.assertEqual(status, Fault.STATUS_PREFAILED) + + def test_02_prefailed_fault_is_hidden_from_the_default_list(self): + """A not-yet-confirmed fault must not reach an operator listing.""" + code = 'PLC_HIDDEN_WHILE_PENDING' + self._report(code, ReportFault.Request.EVENT_FAILED) + + self.assertNotIn( + code, self._default_filter_codes(), + 'an unconfirmed fault is already visible in the default fault list' + ) + + def test_03_sustained_condition_confirms_with_nobody_acting(self): + """ + A real fault must still surface. + + The reporter sends its one FAILED and never repeats it, so only the + time-based lever can promote this. Without it the fault would stay + PREFAILED forever and the appliance would go quiet. + """ + code = 'PLC_SUSTAINED' + self._report(code, ReportFault.Request.EVENT_FAILED) + + status = self._wait_for_status( + code, Fault.STATUS_CONFIRMED, AUTO_CONFIRM_SEC + 5.0 + ) + self.assertEqual( + status, Fault.STATUS_CONFIRMED, + f'a sustained fault never confirmed, it is still {status}' + ) + self.assertIn(code, self._default_filter_codes()) + + def test_04_transient_that_clears_in_time_never_confirms(self): + """ + The falsifying case for the whole setting. + + A glitch that goes away before the window closes must never confirm. + If it does, the configuration only delays a false alarm rather than + filtering it. + """ + code = 'PLC_TRANSIENT' + self._report(code, ReportFault.Request.EVENT_FAILED) + self._report(code, ReportFault.Request.EVENT_PASSED) + + # Sit past the auto-confirm window and the timer tick behind it. + time.sleep(AUTO_CONFIRM_SEC + 3.0) + + status = self._status_of(code) + self.assertNotEqual( + status, Fault.STATUS_CONFIRMED, + 'a transient that already cleared was confirmed by the timer' + ) + self.assertNotIn(code, self._default_filter_codes()) + + def test_05_deasserted_alarm_heals_with_nobody_acting(self): + """ + A confirmed fault must return to healed on the reporter's single clear. + + The reporter sends exactly one PASSED, so healing has to complete on + that one event. With healing off, or with a threshold above zero, the + fault stays CONFIRMED until a human clears it. + """ + code = 'PLC_DEASSERTED' + self._report(code, ReportFault.Request.EVENT_FAILED) + + status = self._wait_for_status( + code, Fault.STATUS_CONFIRMED, AUTO_CONFIRM_SEC + 5.0 + ) + self.assertEqual( + status, Fault.STATUS_CONFIRMED, 'fault never confirmed, cannot test healing' + ) + + self._report(code, ReportFault.Request.EVENT_PASSED) + + status = self._status_of(code) + self.assertEqual( + status, Fault.STATUS_HEALED, + f'a de-asserted alarm did not heal on its single clear, it is {status}' + ) + self.assertNotIn(code, self._default_filter_codes()) + + +@launch_testing.post_shutdown_test() +class TestDebounceAndHealingShutdown(unittest.TestCase): + """Check the node exited cleanly.""" + + def test_exit_code(self, proc_info, fault_manager_node): + launch_testing.asserts.assertExitCodes( + proc_info, allowable_exit_codes=[0, -2, -15], process=fault_manager_node + )