From 04f29e106474bb6f83f95c3c4ed31e30f5fcd808 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 17 Aug 2026 21:11:51 +0800 Subject: [PATCH 01/21] Fix clipboard writers that never worked on 64-bit Windows set_clipboard_files, set_clipboard_html, set_clipboard_rtf and set_clipboard_csv raised OverflowError on every call, and the matching readers raised whenever that format was actually present. Each module declared restype but not argtypes, so ctypes passed the pointer-width memory handle as c_int. Declaring only restype fixes the value coming back and leaves the argument going in broken. The prototypes and the open/alloc/lock dance now live once in utils/clipboard/win32_clipboard_api.py, on private WinDLL handles so the declarations cannot leak into other user32 callers in the process. The pure-function tests could not catch this: the byte packing was always correct and nothing exercised the Win32 half. The new test round-trips text, HTML, RTF, CSV and file lists through the real clipboard, and adds a static check that no module calls a handle API without prototypes. --- .../utils/clipboard/win32_clipboard_api.py | 115 +++++++++++++++ .../utils/clipboard_files/clipboard_files.py | 53 +++---- .../clipboard_formats/clipboard_formats.py | 7 +- .../clipboard_rich_formats.py | 57 +++----- .../utils/rich_clipboard/rich_clipboard.py | 62 +++----- .../test_clipboard_win32_prototypes.py | 133 ++++++++++++++++++ 6 files changed, 308 insertions(+), 119 deletions(-) create mode 100644 je_auto_control/utils/clipboard/win32_clipboard_api.py create mode 100644 test/unit_test/headless/test_clipboard_win32_prototypes.py diff --git a/je_auto_control/utils/clipboard/win32_clipboard_api.py b/je_auto_control/utils/clipboard/win32_clipboard_api.py new file mode 100644 index 00000000..1e7f201a --- /dev/null +++ b/je_auto_control/utils/clipboard/win32_clipboard_api.py @@ -0,0 +1,115 @@ +"""Win32 clipboard prototypes and the open/alloc/lock dance, declared once. + +Every clipboard format — text, image, HTML, RTF, CSV, file drops — goes through +the same four Win32 calls, and every module that reimplemented them got the same +detail wrong: ``argtypes``. A memory handle is pointer-width, ctypes defaults an +undeclared parameter to ``c_int``, and so ``GlobalLock(handle)`` raises +``OverflowError: int too long to convert`` for any real handle on 64-bit +Windows. Declaring only ``restype`` fixes the value coming *back* and leaves the +argument going *in* broken, which is exactly what three modules did — their +``set_clipboard_*`` functions failed on every single call. + +Handles are private (``WinDLL``, not the process-wide cached ``windll``): +prototypes live on the function objects, so a shared handle would leak these +declarations into every other user32 caller in the process. +""" +import ctypes +import sys +from ctypes import wintypes +from typing import Optional, Tuple + +GMEM_MOVEABLE = 0x0002 +_OPEN_FAILED = "OpenClipboard failed" + + +def _require_windows() -> None: + if not sys.platform.startswith("win"): + raise RuntimeError("the Win32 clipboard API is only available on Windows") + + +def clipboard_api() -> Tuple[object, object]: + """``(user32, kernel32)`` with every clipboard prototype declared.""" + _require_windows() + user32 = ctypes.WinDLL("user32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + + user32.OpenClipboard.argtypes = [wintypes.HWND] + user32.OpenClipboard.restype = wintypes.BOOL + user32.EmptyClipboard.argtypes = [] + user32.EmptyClipboard.restype = wintypes.BOOL + user32.CloseClipboard.argtypes = [] + user32.CloseClipboard.restype = wintypes.BOOL + user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE] + user32.SetClipboardData.restype = wintypes.HANDLE + user32.GetClipboardData.argtypes = [wintypes.UINT] + user32.GetClipboardData.restype = wintypes.HANDLE + user32.RegisterClipboardFormatW.argtypes = [wintypes.LPCWSTR] + user32.RegisterClipboardFormatW.restype = wintypes.UINT + user32.EnumClipboardFormats.argtypes = [wintypes.UINT] + user32.EnumClipboardFormats.restype = wintypes.UINT + user32.GetClipboardFormatNameW.argtypes = [wintypes.UINT, wintypes.LPWSTR, + ctypes.c_int] + user32.GetClipboardFormatNameW.restype = ctypes.c_int + kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t] + kernel32.GlobalAlloc.restype = wintypes.HGLOBAL + kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalLock.restype = ctypes.c_void_p + kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalUnlock.restype = wintypes.BOOL + kernel32.GlobalSize.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalSize.restype = ctypes.c_size_t + return user32, kernel32 + + +def register_format(name: str) -> int: + """Register (or look up) a named clipboard format id.""" + user32, _kernel32 = clipboard_api() + return int(user32.RegisterClipboardFormatW(name)) + + +def set_clipboard_format(format_id: int, payload: bytes, *, + empty_first: bool = True) -> None: + """Put raw bytes on the clipboard under ``format_id``. + + The buffer is allocated and filled *before* the clipboard is opened, so a + failure part-way leaves the user's clipboard untouched rather than emptied. + On success the clipboard owns the handle and must not free it here. + """ + user32, kernel32 = clipboard_api() + handle = kernel32.GlobalAlloc(GMEM_MOVEABLE, len(payload)) + if not handle: + raise RuntimeError("GlobalAlloc failed") + pointer = kernel32.GlobalLock(handle) + if not pointer: + raise RuntimeError("GlobalLock failed") + ctypes.memmove(pointer, payload, len(payload)) + kernel32.GlobalUnlock(handle) + if not user32.OpenClipboard(None): + raise RuntimeError(_OPEN_FAILED) + try: + if empty_first: + user32.EmptyClipboard() + if not user32.SetClipboardData(int(format_id), handle): + raise RuntimeError(f"SetClipboardData({format_id}) failed") + finally: + user32.CloseClipboard() + + +def get_clipboard_format(format_id: int) -> Optional[bytes]: + """Read the clipboard's ``format_id`` payload, or ``None`` when absent.""" + user32, kernel32 = clipboard_api() + if not user32.OpenClipboard(None): + raise RuntimeError(_OPEN_FAILED) + try: + handle = user32.GetClipboardData(int(format_id)) + if not handle: + return None + pointer = kernel32.GlobalLock(handle) + if not pointer: + return None + try: + return ctypes.string_at(pointer, kernel32.GlobalSize(handle)) + finally: + kernel32.GlobalUnlock(handle) + finally: + user32.CloseClipboard() diff --git a/je_auto_control/utils/clipboard_files/clipboard_files.py b/je_auto_control/utils/clipboard_files/clipboard_files.py index a3be087d..8a1ff45b 100644 --- a/je_auto_control/utils/clipboard_files/clipboard_files.py +++ b/je_auto_control/utils/clipboard_files/clipboard_files.py @@ -66,43 +66,22 @@ def get_clipboard_files() -> Optional[List[str]]: def _win_set_hdrop(blob: bytes) -> None: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - user32.EmptyClipboard() - handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(blob)) - if not handle: - raise RuntimeError("GlobalAlloc failed") - pointer = kernel32.GlobalLock(handle) - ctypes.memmove(pointer, blob, len(blob)) - kernel32.GlobalUnlock(handle) - if not user32.SetClipboardData(_CF_HDROP, handle): - raise RuntimeError("SetClipboardData(CF_HDROP) failed") - finally: - user32.CloseClipboard() + """Delegates to the one place that declares the Win32 prototypes. + + This module used to hand-roll the calls with ``restype`` but no + ``argtypes``: a memory handle is pointer-width, ctypes defaults an + undeclared parameter to ``c_int``, and so ``GlobalLock(handle)`` raised + ``OverflowError: int too long to convert`` on 64-bit Windows — + ``set_clipboard_files`` failed on every call. + """ + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + set_clipboard_format, + ) + set_clipboard_format(_CF_HDROP, blob) def _win_get_hdrop() -> Optional[bytes]: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - user32.GetClipboardData.restype = wintypes.HANDLE - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - handle = user32.GetClipboardData(_CF_HDROP) - if not handle: - return None - pointer = kernel32.GlobalLock(handle) - size = kernel32.GlobalSize(handle) - data = ctypes.string_at(pointer, size) - kernel32.GlobalUnlock(handle) - return data - finally: - user32.CloseClipboard() + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + get_clipboard_format, + ) + return get_clipboard_format(_CF_HDROP) diff --git a/je_auto_control/utils/clipboard_formats/clipboard_formats.py b/je_auto_control/utils/clipboard_formats/clipboard_formats.py index 0527b7e2..9c7bc738 100644 --- a/je_auto_control/utils/clipboard_formats/clipboard_formats.py +++ b/je_auto_control/utils/clipboard_formats/clipboard_formats.py @@ -115,8 +115,11 @@ def list_clipboard_formats() -> List[Dict[str, Any]]: import sys if not sys.platform.startswith("win"): raise RuntimeError("list_clipboard_formats is only supported on Windows") - import ctypes - user32 = ctypes.windll.user32 + # Prototypes come from the shared module, on a *private* handle: declaring + # them on the process-wide ``ctypes.windll.user32`` would leak into every + # other caller in this process. + from je_auto_control.utils.clipboard.win32_clipboard_api import clipboard_api + user32, _kernel32 = clipboard_api() if not user32.OpenClipboard(None): raise RuntimeError("OpenClipboard failed") try: diff --git a/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py b/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py index 8e277b24..30b07010 100644 --- a/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py +++ b/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py @@ -166,53 +166,30 @@ def csv_to_rows(text: str, *, delimiter: str = ",") -> List[List[str]]: # --- Win32 clipboard I/O --------------------------------------------------- def _format_id(name: str) -> int: - import ctypes - return ctypes.windll.user32.RegisterClipboardFormatW(name) + from je_auto_control.utils.clipboard.win32_clipboard_api import register_format + return register_format(name) def _win_set_format(format_id: int, payload: bytes, *, empty_first: bool = True) -> None: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - if empty_first: - user32.EmptyClipboard() - handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(payload)) - if not handle: - raise RuntimeError("GlobalAlloc failed") - pointer = kernel32.GlobalLock(handle) - ctypes.memmove(pointer, payload, len(payload)) - kernel32.GlobalUnlock(handle) - if not user32.SetClipboardData(format_id, handle): - raise RuntimeError("SetClipboardData failed") - finally: - user32.CloseClipboard() + """Delegates to the one place that declares the Win32 prototypes. + + This used to hand-roll the calls with ``restype`` but no ``argtypes``, so + ``GlobalLock`` raised ``OverflowError`` on 64-bit Windows and + ``set_clipboard_rtf`` / ``set_clipboard_csv`` had never worked. + """ + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + set_clipboard_format, + ) + set_clipboard_format(format_id, payload, empty_first=empty_first) def _win_get_format(format_id: int) -> Optional[bytes]: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - user32.GetClipboardData.restype = wintypes.HANDLE - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - handle = user32.GetClipboardData(format_id) - if not handle: - return None - pointer = kernel32.GlobalLock(handle) - size = kernel32.GlobalSize(handle) - data = ctypes.string_at(pointer, size) - kernel32.GlobalUnlock(handle) - return data.split(b"\x00", 1)[0] - finally: - user32.CloseClipboard() + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + get_clipboard_format, + ) + data = get_clipboard_format(format_id) + return None if data is None else data.split(b"\x00", 1)[0] def _seed_plaintext(text: str) -> None: diff --git a/je_auto_control/utils/rich_clipboard/rich_clipboard.py b/je_auto_control/utils/rich_clipboard/rich_clipboard.py index 0719d486..289d450c 100644 --- a/je_auto_control/utils/rich_clipboard/rich_clipboard.py +++ b/je_auto_control/utils/rich_clipboard/rich_clipboard.py @@ -91,52 +91,34 @@ def get_clipboard_html() -> Optional[str]: def _html_format_id(): - import ctypes - return ctypes.windll.user32.RegisterClipboardFormatW(_HTML_FORMAT_NAME) + from je_auto_control.utils.clipboard.win32_clipboard_api import register_format + return register_format(_HTML_FORMAT_NAME) def _win_set_html(cf_html: bytes, fragment_plaintext: Optional[str]) -> None: - import ctypes - from ctypes import wintypes + """Put CF_HTML on the clipboard, optionally seeding a plain-text fallback. + + The Win32 dance (prototypes, alloc, lock, open) lives in + ``utils/clipboard/win32_clipboard_api.py``; this module only decides *what* + bytes go on the clipboard. It used to hand-roll those calls with ``restype`` + but no ``argtypes``, so every call raised ``OverflowError`` on 64-bit + Windows — the function had never worked. + """ from je_auto_control.utils.clipboard.clipboard import set_clipboard - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.restype = ctypes.c_void_p + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + set_clipboard_format, + ) if fragment_plaintext is not None: set_clipboard(fragment_plaintext) # seeds CF_UNICODETEXT first - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - if fragment_plaintext is None: - user32.EmptyClipboard() - handle = kernel32.GlobalAlloc(0x0002, len(cf_html) + 1) - if not handle: - raise RuntimeError("GlobalAlloc failed") - pointer = kernel32.GlobalLock(handle) - ctypes.memmove(pointer, cf_html + b"\x00", len(cf_html) + 1) - kernel32.GlobalUnlock(handle) - if not user32.SetClipboardData(_html_format_id(), handle): - raise RuntimeError("SetClipboardData(CF_HTML) failed") - finally: - user32.CloseClipboard() + # Keep that seed when there is one: emptying the clipboard here would throw + # the plain-text fallback away again. + set_clipboard_format(_html_format_id(), cf_html + b"\x00", + empty_first=fragment_plaintext is None) def _win_get_html() -> Optional[bytes]: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - user32.GetClipboardData.restype = wintypes.HANDLE - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - handle = user32.GetClipboardData(_html_format_id()) - if not handle: - return None - pointer = kernel32.GlobalLock(handle) - size = kernel32.GlobalSize(handle) - data = ctypes.string_at(pointer, size) - kernel32.GlobalUnlock(handle) - return data.split(b"\x00", 1)[0] - finally: - user32.CloseClipboard() + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + get_clipboard_format, + ) + data = get_clipboard_format(_html_format_id()) + return None if data is None else data.split(b"\x00", 1)[0] diff --git a/test/unit_test/headless/test_clipboard_win32_prototypes.py b/test/unit_test/headless/test_clipboard_win32_prototypes.py new file mode 100644 index 00000000..518bb8e9 --- /dev/null +++ b/test/unit_test/headless/test_clipboard_win32_prototypes.py @@ -0,0 +1,133 @@ +"""Every clipboard format must survive a real round-trip, and no module may +hand-roll the Win32 calls again. No Qt. + +Three ``set_clipboard_*`` functions were broken on 64-bit Windows for their +whole existence — HTML, RTF and CSV — and so was the file-drop writer. All four +failed the same way: the module declared ``restype`` but not ``argtypes``, so +ctypes passed a pointer-width memory handle as ``c_int`` and ``GlobalLock`` +raised ``OverflowError: int too long to convert``. Pure-function tests could not +see it (the byte packing was always correct) and no test called the Win32 half. + +So this file tests the half that was untested: the actual clipboard. It skips +when the clipboard cannot be opened at all — a locked workstation, a session +without a window station, or a non-Windows CI runner — rather than reporting a +failure the environment made inevitable. +""" +import sys + +import pytest + +_WINDOWS = sys.platform.startswith("win") +pytestmark = pytest.mark.skipif(not _WINDOWS, reason="Windows clipboard only") + + +def _clipboard_available() -> bool: + from je_auto_control.utils.clipboard.clipboard import get_clipboard + try: + get_clipboard() + return True + except Exception: # noqa: BLE001 - locked desktop / no window station + return False + + +@pytest.fixture() +def clipboard(): + """Skip when unusable, and put the user's clipboard back afterwards.""" + if not _clipboard_available(): + pytest.skip("clipboard cannot be opened in this session") + from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, set_clipboard, + ) + saved = get_clipboard() + try: + yield + finally: + try: + set_clipboard(saved) + except Exception: # noqa: BLE001 # nosec B110 + pass + + +def test_text_round_trip(clipboard): + from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, set_clipboard, + ) + set_clipboard("round-trip probe") + assert get_clipboard() == "round-trip probe" + + +def test_html_round_trip(clipboard): + from je_auto_control.utils.rich_clipboard.rich_clipboard import ( + get_clipboard_html, set_clipboard_html, + ) + set_clipboard_html("hi") + assert "hi" in (get_clipboard_html() or "") + + +def test_rtf_round_trip(clipboard): + from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( + build_rtf, get_clipboard_rtf, set_clipboard_rtf, + ) + set_clipboard_rtf(build_rtf("hello")) + assert "hello" in (get_clipboard_rtf() or "") + + +def test_csv_round_trip(clipboard): + from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( + get_clipboard_csv, set_clipboard_csv, + ) + set_clipboard_csv([["a", "b"], ["c", "d"]]) + assert get_clipboard_csv() == [["a", "b"], ["c", "d"]] + + +def test_file_list_round_trip(clipboard, tmp_path): + from je_auto_control.utils.clipboard_files.clipboard_files import ( + get_clipboard_files, set_clipboard_files, + ) + one = tmp_path / "one.png" + one.write_bytes(b"x") + set_clipboard_files([str(one)]) + assert get_clipboard_files() == [str(one)] + + +def test_format_enumeration_sees_what_was_written(clipboard): + from je_auto_control.utils.clipboard.clipboard import set_clipboard + from je_auto_control.utils.clipboard_formats.clipboard_formats import ( + clipboard_formats, + ) + set_clipboard("text only") + summary = clipboard_formats() + assert summary["has_text"] is True + assert summary["has_files"] is False + + +# --- static invariant ------------------------------------------------------- + +_HANDLE_CALLS = ("GlobalLock", "GlobalAlloc", "GlobalSize", "SetClipboardData", + "GetClipboardData") +_SHARED_MODULE = "win32_clipboard_api" + + +def test_no_module_hand_rolls_the_clipboard_calls_without_prototypes(): + """A handle call needs declared ``argtypes`` — in the file or via the shared module. + + This is the invariant the three broken modules violated. Any new clipboard + code either goes through ``utils/clipboard/win32_clipboard_api.py`` or + declares the prototypes itself; ``restype``-only is what shipped a function + that had never once worked. + """ + import pathlib + + root = pathlib.Path(__file__).resolve().parents[3] / "je_auto_control" + offenders = [] + for path in root.rglob("*.py"): + text = path.read_text(encoding="utf-8", errors="ignore") + if not any(call in text for call in _HANDLE_CALLS): + continue + if _SHARED_MODULE in text or "argtypes" in text: + continue + offenders.append(str(path.relative_to(root))) + assert not offenders, ( + "these modules call Win32 handle APIs without declaring argtypes and " + "without going through the shared clipboard module: " + repr(offenders) + ) From 71daedb9bb42c3da2ca3da2b5e20670701a0f3c6 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 17 Aug 2026 21:12:00 +0800 Subject: [PATCH 02/21] Save only the windows a layout can restore save_window_layout documented "every titled window" but its default lister returned all of them, while restore_window_layout addresses a window by title and skips blank ones. On a real desktop that was 28 entries saved against 15 restorable, so a caller reporting the saved count was over-promising by a factor of two. The lister now passes titled_only=True, and a round-trip test pins save and restore to the same set. --- .../utils/window_capture/window_capture.py | 10 +++++- .../unit_test/headless/test_window_capture.py | 34 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/je_auto_control/utils/window_capture/window_capture.py b/je_auto_control/utils/window_capture/window_capture.py index 56053bb2..30fdf3d2 100644 --- a/je_auto_control/utils/window_capture/window_capture.py +++ b/je_auto_control/utils/window_capture/window_capture.py @@ -71,8 +71,16 @@ def capture_window(title: str, output_path: Union[str, Path], *, def _default_lister() -> List[Tuple[int, str]]: + """Titled windows only — an untitled one cannot be restored. + + ``restore_window_layout`` addresses a window by its title, so an entry with + a blank one is skipped there; saving them anyway made the two disagree on + how many windows a layout contains. On a real desktop that is roughly half + the entries (measured here: 28 saved, 15 restorable), so a caller reporting + the saved count was over-promising by a factor of two. + """ from je_auto_control.wrapper.auto_control_window import list_windows - return list_windows() + return list_windows(titled_only=True) def save_window_layout(path: Optional[Union[str, Path]] = None, *, diff --git a/test/unit_test/headless/test_window_capture.py b/test/unit_test/headless/test_window_capture.py index e17b2bc7..c9029249 100644 --- a/test/unit_test/headless/test_window_capture.py +++ b/test/unit_test/headless/test_window_capture.py @@ -115,3 +115,37 @@ def test_snap_window_unknown_position_raises(): with pytest.raises(ValueError): snap_window("E", "diagonal", screen_size=lambda: (1000, 800), mover=lambda *a: True) + + +def test_default_lister_only_offers_windows_restore_can_address(monkeypatch): + """Saving an untitled window is a promise the restore side cannot keep. + + ``restore_window_layout`` finds a window by title and skips blank ones, so + an untitled entry inflates the saved count without ever being restored — + on a real desktop that was 28 saved against 15 restorable. + """ + from je_auto_control.utils.window_capture import window_capture as wc + + seen = {} + + def _fake_list_windows(titled_only=False): + seen["titled_only"] = titled_only + return [(1, "Editor"), (2, " "), (3, "")] + + import je_auto_control.wrapper.auto_control_window as window_api + monkeypatch.setattr(window_api, "list_windows", _fake_list_windows) + assert wc._default_lister() == [(1, "Editor"), (2, " "), (3, "")] + assert seen["titled_only"] is True + + +def test_saved_entries_are_all_restorable(): + """Round-trip: every entry save produces must survive restore.""" + layout = save_window_layout( + lister=lambda: [(1, "Editor"), (2, "Browser")], + geometry=lambda title: (0, 0, 100, 100), + ) + moved = [] + restored = restore_window_layout( + layout, mover=lambda title, *rect: moved.append(title) or True) + assert restored == len(layout) == 2 + assert moved == ["Editor", "Browser"] From 3c800c95bd30448823f12094e26720b3412af596 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 17 Aug 2026 21:12:16 +0800 Subject: [PATCH 03/21] Address windows by owner and post input where it lands Three related gaps in window handling, all of which failed silently. A window title cannot identify a multi-process application: its windows are named after whatever they display and several of its processes have no window at all. foreground_window_process_id, window_process_id, windows_for_process_id and minimize_windows_for_process answer by ownership instead. Unavailable reads as None rather than a bare 0, which a caller could otherwise match against a process list and hit the System Idle Process. Posted input went to the top-level frame, but keyboard messages are delivered to the control that has focus, and a click belongs to the child under the point in that child's client coordinates. Measured on Character Map: posting to the frame typed nothing, posting to the focused edit typed the character. post_key_to_window and post_click_to_window resolve the real target (GetGUIThreadInfo, ChildWindowFromPointEx) and report whether the messages were queued; a printable key also posts WM_CHAR, without which most edit controls type nothing. send_key_event_to_window and send_mouse_event_to_window are deprecated and delegate to those. They kept reporting success while doing nothing, so leaving them as-is meant leaving a trap. Behaviour changes: the key sender now matches the title as a substring (it required an exact title), and the mouse sender accepts a title as well as the hwnd it always took. All four surfaces are wired: facade, AC_* commands, MCP tools and Script Builder specs. --- je_auto_control/__init__.py | 10 +- .../gui/script_builder/command_schema.py | 39 +++++ .../utils/executor/action_executor.py | 54 +++++- .../utils/mcp_server/tools/_factories.py | 70 ++++++++ .../utils/mcp_server/tools/_handlers.py | 49 ++++++ .../windows/window/windows_window_manage.py | 158 +++++++++++++++++ .../wrapper/auto_control_keyboard.py | 55 +++--- je_auto_control/wrapper/auto_control_mouse.py | 69 ++++++-- .../wrapper/auto_control_window.py | 121 ++++++++++++- test/unit_test/headless/test_window_manage.py | 159 ++++++++++++++++++ 10 files changed, 741 insertions(+), 43 deletions(-) diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index f6c48481..4f6fa121 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1269,8 +1269,10 @@ # Cross-platform window manager (headless) from je_auto_control.wrapper.auto_control_window import ( close_window_by_title, find_window, focus_window, foreground_window, - list_windows, minimize_window_by_title, move_window_by_title, - show_window_by_title, wait_for_window, window_rect, + foreground_window_process_id, list_windows, minimize_window_by_title, + minimize_windows_for_process, move_window_by_title, post_click_to_window, + post_key_to_window, show_window_by_title, wait_for_window, + window_process_id, window_rect, windows_for_process_id, ) # Windows-only modules (ctypes.WINFUNCTYPE / Win32 API) — gated so # ``import je_auto_control`` keeps working on macOS / Linux. Kept last @@ -1329,7 +1331,9 @@ def start_autocontrol_gui(*args, **kwargs): "list_windows", "find_window", "focus_window", "wait_for_window", "close_window_by_title", "show_window_by_title", "minimize_window_by_title", "foreground_window", "window_rect", - "move_window_by_title", + "move_window_by_title", "foreground_window_process_id", + "window_process_id", "post_key_to_window", "post_click_to_window", + "windows_for_process_id", "minimize_windows_for_process", # Clipboard "get_clipboard", "set_clipboard", "get_clipboard_image", "set_clipboard_image", diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index f250aa11..e9ff2656 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1097,6 +1097,45 @@ def _add_window_specs(specs: List[CommandSpec]) -> None: fields=(FieldSpec("title_substring", FieldType.STRING),), description="Screen rectangle (left, top, right, bottom) of a window.", )) + specs.append(CommandSpec( + "AC_foreground_window_pid", "Window", "Foreground Window PID", + description="PID owning the foreground window (0 when unavailable).", + )) + specs.append(CommandSpec( + "AC_windows_for_pid", "Window", "Windows by Process ID", + fields=(FieldSpec("pid", FieldType.INT),), + description="Visible top-level windows owned by a process.", + )) + specs.append(CommandSpec( + "AC_minimize_windows_for_pid", "Window", "Minimize Windows by PID", + fields=(FieldSpec("pid", FieldType.INT),), + description="Minimise every window a process owns.", + )) + specs.append(CommandSpec( + "AC_post_key_to_window", "Window", "Post Key to Window", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("key", FieldType.STRING), + ), + description="Type one key into a window without focusing it " + "(best effort: some applications ignore posted input).", + )) + specs.append(CommandSpec( + "AC_post_click_to_window", "Window", "Post Click to Window", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("button", FieldType.STRING, optional=True), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + ), + description="Click inside a window without focusing it; x / y are " + "relative to the window's top-left corner.", + )) + specs.append(CommandSpec( + "AC_window_pid", "Window", "Window PID by Title", + fields=(FieldSpec("title_substring", FieldType.STRING),), + description="PID owning the first matching window (0 when none).", + )) specs.append(CommandSpec( "AC_move_window", "Window", "Move Window by Title", fields=( diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index fb919e02..d292c317 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -81,9 +81,11 @@ from je_auto_control.wrapper.auto_control_record import record, stop_record from je_auto_control.wrapper.auto_control_screen import screenshot, screen_size from je_auto_control.wrapper.auto_control_window import ( - close_window_by_title, focus_window, foreground_window, list_windows, - minimize_window_by_title, move_window_by_title, wait_for_window, - window_rect, + close_window_by_title, focus_window, foreground_window, + foreground_window_process_id, list_windows, minimize_window_by_title, + minimize_windows_for_process, move_window_by_title, post_click_to_window, + post_key_to_window, wait_for_window, window_process_id, window_rect, + windows_for_process_id, ) @@ -6657,6 +6659,46 @@ def _window_rect(title_substring: str, return {"rect": list(rect) if rect is not None else None} +def _foreground_window_process_id() -> Dict[str, Any]: + """Adapter: the PID owning the foreground window (``0`` when unknown).""" + return {"pid": foreground_window_process_id() or 0} + + +def _window_process_id(title_substring: str, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: the PID owning the first matching window (``0`` when none).""" + return {"pid": window_process_id(title_substring, bool(case_sensitive)) or 0} + + +def _windows_for_process_id(pid: int, + titled_only: bool = False) -> Dict[str, Any]: + """Adapter: every visible top-level window owned by a process.""" + found = windows_for_process_id(int(pid), bool(titled_only)) + return {"windows": [{"hwnd": hwnd, "title": title} + for hwnd, title in found]} + + +def _minimize_windows_for_process(pid: int) -> Dict[str, Any]: + """Adapter: minimise every window a process owns.""" + return {"minimized": minimize_windows_for_process(int(pid))} + + +def _post_key_to_window(title_substring: str, key: str, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: post one key to a window without focusing it.""" + return {"posted": bool(post_key_to_window(title_substring, key, + bool(case_sensitive)))} + + +def _post_click_to_window(title_substring: str, button: str = "left", + x: int = 0, y: int = 0, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: post one click into a window without focusing it.""" + return {"posted": bool(post_click_to_window(title_substring, button, + int(x), int(y), + bool(case_sensitive)))} + + def _canonicalize_url(url: str) -> Dict[str, Any]: """Adapter: opinionated canonical form of a URL, for equality checks.""" from je_auto_control.utils.url_canon import canonicalize_url @@ -7063,6 +7105,12 @@ def __init__(self): "AC_close_window": close_window_by_title, "AC_minimize_window": minimize_window_by_title, "AC_foreground_window": _foreground_window, + "AC_foreground_window_pid": _foreground_window_process_id, + "AC_window_pid": _window_process_id, + "AC_windows_for_pid": _windows_for_process_id, + "AC_minimize_windows_for_pid": _minimize_windows_for_process, + "AC_post_key_to_window": _post_key_to_window, + "AC_post_click_to_window": _post_click_to_window, "AC_window_rect": _window_rect, "AC_move_window": move_window_by_title, diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 9105f38a..17960fb9 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -320,6 +320,76 @@ def window_tools() -> List[MCPTool]: handler=h.window_rect, annotations=READ_ONLY, ), + MCPTool( + name="ac_foreground_window_pid", + description=("PID of the process owning the foreground window as " + "{pid}; 0 when unavailable. Titles are not identity — " + "use this to tell which program is in front."), + input_schema=schema({}), + handler=h.foreground_window_pid, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_windows_for_pid", + description=("Visible top-level windows owned by a process, as " + "{windows: [{hwnd, title}]}. Titles cannot address a " + "multi-process application; ownership can."), + input_schema=schema({ + "pid": {"type": "integer"}, + "titled_only": {"type": "boolean"}, + }, required=["pid"]), + handler=h.windows_for_pid, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_minimize_windows_for_pid", + description=("Minimise every window a process owns, as " + "{minimized}."), + input_schema=schema({"pid": {"type": "integer"}}, required=["pid"]), + handler=h.minimize_windows_for_pid, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_post_key_to_window", + description=("Post one key to a window without focusing it, as " + "{posted}. Goes to the control that has keyboard " + "focus. Best effort: applications reading raw input " + "or checking the foreground ignore posted messages."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "key": {"type": "string"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring", "key"]), + handler=h.post_key_to_window, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_post_click_to_window", + description=("Post one click into a window without focusing it, " + "as {posted}. x / y are relative to the window's " + "top-left corner. Same best-effort caveat as " + "ac_post_key_to_window."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "button": {"type": "string"}, + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring"]), + handler=h.post_click_to_window, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_window_pid", + description=("PID of the process owning the first matching window " + "as {pid}; 0 when nothing matched."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring"]), + handler=h.window_pid, + annotations=READ_ONLY, + ), MCPTool( name="ac_window_move", description=("Move and resize the first matching window to " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 76f2d695..b86a15d6 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -451,6 +451,55 @@ def window_rect(title_substring: str, return {"rect": list(rect) if rect is not None else None} +def foreground_window_pid() -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + foreground_window_process_id as _pid, + ) + return {"pid": _pid() or 0} + + +def window_pid(title_substring: str, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + window_process_id as _pid, + ) + return {"pid": _pid(title_substring, case_sensitive=case_sensitive) or 0} + + +def windows_for_pid(pid: int, titled_only: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + windows_for_process_id as _windows, + ) + return {"windows": [{"hwnd": hwnd, "title": title} + for hwnd, title in _windows(int(pid), titled_only)]} + + +def minimize_windows_for_pid(pid: int) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + minimize_windows_for_process as _minimize, + ) + return {"minimized": _minimize(int(pid))} + + +def post_key_to_window(title_substring: str, key: str, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + post_key_to_window as _post, + ) + return {"posted": bool(_post(title_substring, key, + case_sensitive=case_sensitive))} + + +def post_click_to_window(title_substring: str, button: str = "left", + x: int = 0, y: int = 0, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + post_click_to_window as _post, + ) + return {"posted": bool(_post(title_substring, button, int(x), int(y), + case_sensitive=case_sensitive))} + + def _resolve_window_hwnd(title_substring: str, case_sensitive: bool) -> int: from je_auto_control.wrapper.auto_control_window import find_window diff --git a/je_auto_control/windows/window/windows_window_manage.py b/je_auto_control/windows/window/windows_window_manage.py index d922fdb2..5ea10eb8 100644 --- a/je_auto_control/windows/window/windows_window_manage.py +++ b/je_auto_control/windows/window/windows_window_manage.py @@ -62,6 +62,9 @@ _user32.MoveWindow.restype = wintypes.BOOL _user32.IsIconic.argtypes = [wintypes.HWND] _user32.IsIconic.restype = wintypes.BOOL +_user32.GetWindowThreadProcessId.argtypes = [wintypes.HWND, + ctypes.POINTER(wintypes.DWORD)] +_user32.GetWindowThreadProcessId.restype = wintypes.DWORD WM_CLOSE = 0x0010 SW_RESTORE = 9 @@ -122,6 +125,27 @@ def get_window_rect(hwnd: int) -> Optional[Tuple[int, int, int, int]]: return (int(rect.left), int(rect.top), int(rect.right), int(rect.bottom)) +def get_window_process_id(hwnd: int) -> int: + """ + 視窗所屬行程的 PID，查不到回 0 + The PID of the process owning the window, or 0 when unavailable + + 這是「畫面上那個視窗是哪個程式」的唯一可靠答案：視窗標題會被程式自己改成 + 任何字串，行程名不會。有了它，呼叫端才能把前景視窗接到 psutil 之類的行程 + 資訊上。 + + This is the only reliable answer to "which program owns that window": a + title is whatever the application decides to display, a process is not. + """ + pid = wintypes.DWORD(0) + # 回傳值是 thread id（0 代表失敗）；pid 由 out 參數帶回來。 + # The return value is the thread id (0 on failure); the pid comes back + # through the out parameter. + if not _user32.GetWindowThreadProcessId(hwnd, byref(pid)): + return 0 + return int(pid.value) + + def is_window_minimized(hwnd: int) -> bool: """ 視窗目前是否被最小化 @@ -206,3 +230,137 @@ def move_window(hwnd: int, x: int, y: int, width: int, height: int, return bool(_user32.MoveWindow(int(hwnd), int(x), int(y), int(width), int(height), bool(repaint))) + + +# --- Posting input to a window without focusing it -------------------------- +# +# Keyboard and mouse messages are delivered to the *control that has focus*, not +# to the top-level window. Posting to the top-level handle is therefore a no-op +# for anything with child controls — measured on Character Map: posting to the +# frame typed nothing, posting to the focused edit typed the character. The +# helpers below resolve that target before posting. + + +class _GUIThreadInfo(ctypes.Structure): + _fields_ = [ + ("cbSize", wintypes.DWORD), + ("flags", wintypes.DWORD), + ("hwndActive", wintypes.HWND), + ("hwndFocus", wintypes.HWND), + ("hwndCapture", wintypes.HWND), + ("hwndMenuOwner", wintypes.HWND), + ("hwndMoveSize", wintypes.HWND), + ("hwndCaret", wintypes.HWND), + ("rcCaret", wintypes.RECT), + ] + + +_user32.GetGUIThreadInfo.argtypes = [wintypes.DWORD, + ctypes.POINTER(_GUIThreadInfo)] +_user32.GetGUIThreadInfo.restype = wintypes.BOOL +_user32.ChildWindowFromPointEx.argtypes = [wintypes.HWND, wintypes.POINT, + wintypes.UINT] +_user32.ChildWindowFromPointEx.restype = wintypes.HWND +_user32.ScreenToClient.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.POINT)] +_user32.ScreenToClient.restype = wintypes.BOOL +_user32.ClientToScreen.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.POINT)] +_user32.ClientToScreen.restype = wintypes.BOOL + +WM_KEYDOWN = 0x0100 +WM_KEYUP = 0x0101 +WM_CHAR = 0x0102 +_CWP_SKIPINVISIBLE = 0x0001 +_CWP_SKIPTRANSPARENT = 0x0004 +_MOUSE_MESSAGES = { + "left": (0x0201, 0x0202, 0x0001), # WM_LBUTTONDOWN / UP / MK_LBUTTON + "right": (0x0204, 0x0205, 0x0002), # WM_RBUTTON* / MK_RBUTTON + "middle": (0x0207, 0x0208, 0x0010), # WM_MBUTTON* / MK_MBUTTON +} + + +def get_focused_control(hwnd: int) -> int: + """ + 視窗執行緒目前的焦點控制項，問不到就回原本的 hwnd + The control with keyboard focus in the window's thread, else ``hwnd`` + + 走 `GetGUIThreadInfo` 而不是 `AttachThreadInput` + `GetFocus`：後者會把兩個 + 執行緒的輸入狀態接在一起，附帶影響前景與焦點——那正是這條路徑要避免的。 + + Uses ``GetGUIThreadInfo`` rather than ``AttachThreadInput`` + ``GetFocus``: + attaching joins two threads' input state and disturbs focus, which is the + one thing this code path exists to avoid. + """ + thread_id = _user32.GetWindowThreadProcessId(hwnd, None) + if not thread_id: + return int(hwnd) + info = _GUIThreadInfo() + info.cbSize = ctypes.sizeof(_GUIThreadInfo) + if _user32.GetGUIThreadInfo(thread_id, byref(info)) and info.hwndFocus: + return int(info.hwndFocus) + return int(hwnd) + + +def deepest_child_at(hwnd: int, x: int, y: int) -> int: + """ + 視窗內某個點底下最深的子控制項（座標是相對視窗左上角） + The deepest child control under a window-relative point + + 只走 `hwnd` 自己的子樹，所以目標視窗被別的程式蓋住也一樣正確—— + `WindowFromPoint` 會回到最上層那個視窗，對背景操作是錯的答案。 + """ + rect = get_window_rect(hwnd) + if rect is None: + return int(hwnd) + screen = wintypes.POINT(rect[0] + int(x), rect[1] + int(y)) + current = int(hwnd) + for _depth in range(8): # 巢狀控制項的合理上限，兼作迴圈保險 + point = wintypes.POINT(screen.x, screen.y) + _user32.ScreenToClient(current, byref(point)) + child = _user32.ChildWindowFromPointEx( + current, point, _CWP_SKIPINVISIBLE | _CWP_SKIPTRANSPARENT) + if not child or int(child) == current: + return current + current = int(child) + return current + + +def post_key(hwnd: int, keycode: int, character: str = "") -> bool: + """ + 把一次按鍵投遞給視窗（不搶焦點）；回傳訊息是否都排進佇列 + Post one key press to a window without focusing it + + 可列印字元要送 `WM_CHAR`：控制項是靠它拿到文字的，只送 `WM_KEYDOWN` 對多數 + 編輯控制項不會產生任何字。 + + A printable character also needs ``WM_CHAR``: edit controls take their text + from that message, so ``WM_KEYDOWN`` alone types nothing in most of them. + """ + target = get_focused_control(hwnd) + posted = bool(_user32.PostMessageW(target, WM_KEYDOWN, int(keycode), 0)) + if character: + posted = bool(_user32.PostMessageW( + target, WM_CHAR, ord(character[0]), 0)) and posted + posted = bool(_user32.PostMessageW(target, WM_KEYUP, int(keycode), 0)) and posted + return posted + + +def post_click(hwnd: int, button: str, x: int, y: int) -> bool: + """ + 把一次點擊投遞給視窗內 `(x, y)` 的控制項（座標相對視窗左上角，不搶焦點） + Post one click at a window-relative point without focusing the window + """ + key = str(button).lower() + if key not in _MOUSE_MESSAGES: + raise ValueError( + f"unknown mouse button {button!r}; expected one of " + f"{sorted(_MOUSE_MESSAGES)}") + down, up, flag = _MOUSE_MESSAGES[key] + rect = get_window_rect(hwnd) + if rect is None: + return False + target = deepest_child_at(hwnd, x, y) + point = wintypes.POINT(rect[0] + int(x), rect[1] + int(y)) + _user32.ScreenToClient(target, byref(point)) + l_param = ((int(point.y) & 0xFFFF) << 16) | (int(point.x) & 0xFFFF) + posted = bool(_user32.PostMessageW(target, down, flag, l_param)) + return bool(_user32.PostMessageW(target, up, 0, l_param)) and posted diff --git a/je_auto_control/wrapper/auto_control_keyboard.py b/je_auto_control/wrapper/auto_control_keyboard.py index 859cc646..00e1c225 100644 --- a/je_auto_control/wrapper/auto_control_keyboard.py +++ b/je_auto_control/wrapper/auto_control_keyboard.py @@ -1,4 +1,5 @@ import sys +import warnings from typing import Optional, Union, Tuple from je_auto_control.utils.exception.exception_tags import ( @@ -231,33 +232,41 @@ def hotkey(key_code_list: list, is_shift: bool = False) -> Optional[Tuple[str, s def send_key_event_to_window(window_title: str, keycode: Union[int, str]) -> None: """ - 將鍵盤事件送到指定視窗 - Send a key event to a specific window - - :param window_title: 視窗標題 Window title + 將鍵盤事件送到指定視窗（**已棄用**，改用 ``post_key_to_window``） + Send a key event to a specific window. **Deprecated** — use + ``je_auto_control.post_key_to_window``. + + 這支原本把訊息投遞給**頂層視窗**，但鍵盤訊息是送給**有焦點的子控制項**的， + 所以對任何有子控制項的程式都等於什麼都沒做——而且照樣回報成功。實測（字元 + 對應表）：投遞給外框，一個字都沒進去；投遞給焦點控制項，字就進去了。現在 + 轉呼叫 ``post_key_to_window``，行為因此**改變**（會真的作用），並發出 + ``DeprecationWarning``。視窗標題也跟著改成**片段比對**，與其餘視窗函式一致。 + + This posted to the top-level frame, but keyboard messages go to the control + that *has focus*: it silently did nothing in any application with child + controls while still reporting success. It now delegates to + ``post_key_to_window``, so the behaviour changes — it works — and the title + is matched as a substring like every other window function. + + :param window_title: 視窗標題片段 Window title substring :param keycode: 鍵盤代碼或字串 Keycode or string """ + warnings.warn( + "send_key_event_to_window is deprecated; use post_key_to_window. The " + "old implementation posted to the top-level frame and silently did " + "nothing for windows with child controls.", + DeprecationWarning, stacklevel=2, + ) autocontrol_logger.info(f"send_key_event_to_window, window={window_title}, keycode={keycode}") + if sys.platform == "darwin": + return + from je_auto_control.wrapper.auto_control_window import post_key_to_window try: - # macOS 不支援直接送鍵盤事件 - if sys.platform == "darwin": - return - - # 解析 keycode Resolve keycode - if isinstance(keycode, int): - get_key_code = keycode - else: - get_key_code = keyboard_keys_table.get(keycode) - if get_key_code is None: - raise AutoControlKeyboardException(f"Key not found: {keycode}") - - # 呼叫底層 API Send event - keyboard.send_key_event_to_window(window_title, keycode=get_key_code) - - # 紀錄動作 Record action - record_action_to_list("send_key_event_to_window", {"window_title": window_title, "keycode": get_key_code}) - - except (OSError, RuntimeError, AttributeError, TypeError, ValueError) as error: + posted = post_key_to_window(window_title, keycode) + record_action_to_list( + "send_key_event_to_window", + {"window_title": window_title, "keycode": keycode, "posted": posted}) + except Exception as error: # noqa: BLE001 - preserved contract: never raises record_action_to_list("send_key_event_to_window", {"window_title": window_title, "keycode": keycode}, repr(error)) autocontrol_logger.error( f"send_key_event_to_window failed, window={window_title}, keycode={keycode}, error={repr(error)}" diff --git a/je_auto_control/wrapper/auto_control_mouse.py b/je_auto_control/wrapper/auto_control_mouse.py index 7e5e0a02..ab97cb09 100644 --- a/je_auto_control/wrapper/auto_control_mouse.py +++ b/je_auto_control/wrapper/auto_control_mouse.py @@ -1,5 +1,6 @@ import ctypes import sys +import warnings from typing import Tuple, Union from je_auto_control.utils.exception.exception_tags import ( @@ -279,25 +280,67 @@ def mouse_scroll(scroll_value: int, x: int = None, y: int = None, def send_mouse_event_to_window(window, mouse_keycode: Union[int, str], x: int = None, y: int = None) -> None: """ - 將滑鼠事件送到指定視窗 - Send mouse event to a specific window - - :param window: 視窗 handle Window handle + 將滑鼠事件送到指定視窗（**已棄用**，改用 ``post_click_to_window``） + Send mouse event to a specific window. **Deprecated** — use + ``je_auto_control.post_click_to_window``. + + 原本把訊息投遞給傳進來的那個 handle，也就是**頂層視窗**；但點擊要送給座標 + 底下的**子控制項**、而且座標要換算成那個控制項的 client 座標，否則點不到 + 任何東西。這支現在轉呼叫修好的路徑：`window` 傳字串就當**標題片段**（與其餘 + 視窗函式一致），傳整數仍當 hwnd（維持舊呼叫端的型別）。 + + Posted to whatever handle it was given — the top-level frame — but a click + belongs to the child control under the point, in that control's client + coordinates. It now delegates to the fixed path: a string ``window`` is a + title substring (consistent with every other window function), an int is + still an hwnd. + + :param window: 視窗 handle 或標題片段 Window handle or title substring :param mouse_keycode: 滑鼠按鍵代碼 Mouse keycode - :param x: X 座標 X position + :param x: X 座標（相對視窗左上角）X position, relative to the window :param y: Y 座標 Y position """ + warnings.warn( + "send_mouse_event_to_window is deprecated; use post_click_to_window. " + "The old implementation posted to the top-level frame, so the click " + "landed on nothing in any window with child controls.", + DeprecationWarning, stacklevel=2, + ) autocontrol_logger.info(f"send_mouse_event_to_window, window={window}, keycode={mouse_keycode}, x={x}, y={y}") param = {"window": window, "keycode": mouse_keycode, "x": x, "y": y} + if sys.platform == "darwin": + autocontrol_logger.warning("send_mouse_event_to_window not supported on macOS") + return try: - if sys.platform == "darwin": - autocontrol_logger.warning("send_mouse_event_to_window not supported on macOS") - return - - mouse_keycode, x, y = mouse_preprocess(mouse_keycode, x, y) - mouse.send_mouse_event_to_window(window, mouse_keycode=mouse_keycode, x=x, y=y) - record_action_to_list("send_mouse_event_to_window", param) + button = _button_name_for_post(mouse_keycode) + if isinstance(window, str): + from je_auto_control.wrapper.auto_control_window import ( + post_click_to_window, + ) + posted = post_click_to_window(window, button, int(x or 0), int(y or 0)) + else: + from je_auto_control.windows.window import windows_window_manage as wm + posted = wm.post_click(int(window), button, int(x or 0), int(y or 0)) + record_action_to_list("send_mouse_event_to_window", {**param, "posted": posted}) - except (OSError, RuntimeError, AttributeError, TypeError, ValueError) as error: + except Exception as error: # noqa: BLE001 - preserved contract: never raises record_action_to_list("send_mouse_event_to_window", param, repr(error)) autocontrol_logger.error(f"send_mouse_event_to_window failed: {repr(error)}") + + +def _button_name_for_post(mouse_keycode: Union[int, str]) -> str: + """把舊介面收的按鍵代碼轉成投遞路徑用的按鍵名。 + + 舊介面同時吃名稱（``mouse_left``）與底層代碼元組；後者反查回名稱，查不到就 + 當左鍵並記一筆——這條是相容路徑，不值得為了它讓呼叫端爆掉。 + """ + if isinstance(mouse_keycode, str): + name = mouse_keycode.lower() + return name[len("mouse_"):] if name.startswith("mouse_") else name + for name, code in mouse_keys_table.items(): + if code == mouse_keycode: + return name[len("mouse_"):] + autocontrol_logger.warning( + "send_mouse_event_to_window: unknown keycode %r, assuming left", + mouse_keycode) + return "left" diff --git a/je_auto_control/wrapper/auto_control_window.py b/je_auto_control/wrapper/auto_control_window.py index 9469f727..20b20066 100644 --- a/je_auto_control/wrapper/auto_control_window.py +++ b/je_auto_control/wrapper/auto_control_window.py @@ -5,7 +5,7 @@ """ import sys import time -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union from je_auto_control.utils.exception.exceptions import AutoControlActionException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -122,6 +122,125 @@ def foreground_window() -> Optional[Tuple[int, str]]: return hwnd, titles.get(hwnd, "") +def post_key_to_window(title_substring: str, key: Union[int, str], + case_sensitive: bool = False) -> bool: + """Type one key into a window **without focusing it**. ``False`` if no match. + + Unlike :func:`send_key_event_to_window` this resolves the window by + substring (like every other function here) and posts to the control that + actually has keyboard focus. Posting to the top-level frame — what the older + function does — types nothing in any application with child controls; + measured on Character Map, the frame swallowed the key and the focused edit + accepted it. + + **This is best effort, not input.** ``PostMessage`` returning true means the + message reached a queue, not that the application acted on it: games, + anything reading raw input, and applications that check whether they are in + the foreground all ignore posted messages. Callers must say so rather than + reporting success. + """ + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + keycode, character = _resolve_key(key) + return wm.post_key(hit[0], keycode, character) + + +def post_click_to_window(title_substring: str, button: str = "left", + x: int = 0, y: int = 0, + case_sensitive: bool = False) -> bool: + """Click inside a window **without focusing it**; ``False`` if no match. + + ``x`` / ``y`` are relative to the window's top-left corner. The click is + posted to the deepest child control under that point, in that control's own + client coordinates — a click posted to the frame lands nowhere. Same + best-effort caveat as :func:`post_key_to_window`. + """ + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + return wm.post_click(hit[0], _mouse_button_name(button), int(x), int(y)) + + +def _resolve_key(key: Union[int, str]) -> Tuple[int, str]: + """``(virtual key code, character to also post as WM_CHAR)``.""" + if isinstance(key, int): + return int(key), "" + name = str(key) + from je_auto_control.wrapper.platform_wrapper import keyboard_keys_table + keycode = keyboard_keys_table.get(name) + if keycode is None: + raise AutoControlActionException(f"unknown key name: {name!r}") + # A one-character key is text: edit controls take their content from + # WM_CHAR, so posting only the virtual-key messages types nothing. + character = name if len(name) == 1 and name.isprintable() else "" + return int(keycode), character + + +def _mouse_button_name(button: str) -> str: + """Accept both plain and ``mouse_``-prefixed button names.""" + name = str(button).lower() + return name[len("mouse_"):] if name.startswith("mouse_") else name + + +def foreground_window_process_id() -> Optional[int]: + """The PID owning the foreground window, or ``None``. + + A window title is not identity: applications rewrite theirs at will, and + unrelated programs share titles like ``Settings``. Callers that need to know + *which program* the user is actually in front of — presence reporting, + activity probes, "is my automation target focused" — have to go through the + process id. + """ + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + hwnd = wm.get_foreground_window() + if not hwnd: + return None + return wm.get_window_process_id(hwnd) or None + + +def window_process_id(title_substring: str, + case_sensitive: bool = False) -> Optional[int]: + """The PID owning the first window whose title contains the substring.""" + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return None + from je_auto_control.windows.window import windows_window_manage as wm + return wm.get_window_process_id(hit[0]) or None + + +def windows_for_process_id(pid: int, + titled_only: bool = False) -> List[Tuple[int, str]]: + """Every visible top-level window owned by ``pid``. + + Titles cannot address a multi-process application: a browser's windows are + named after whatever page they show, and several of its processes have no + window at all. Ownership is the stable key. + """ + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + target = int(pid) + return [(hwnd, title) for hwnd, title in list_windows(titled_only) + if wm.get_window_process_id(hwnd) == target] + + +def minimize_windows_for_process(pid: int) -> int: + """Minimise every visible top-level window owned by ``pid``; return the count.""" + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + minimized = 0 + for hwnd, _title in windows_for_process_id(pid): + if wm.minimize_window(hwnd): + minimized += 1 + return minimized + + def window_rect(title_substring: str, case_sensitive: bool = False, ) -> Optional[Tuple[int, int, int, int]]: diff --git a/test/unit_test/headless/test_window_manage.py b/test/unit_test/headless/test_window_manage.py index 3042a734..ae13b57e 100644 --- a/test/unit_test/headless/test_window_manage.py +++ b/test/unit_test/headless/test_window_manage.py @@ -127,3 +127,162 @@ def SetForegroundWindow(self, hwnd): # noqa: N802 # Win32 name seen.clear() module.show_window(7, 3) # SW_MAXIMIZE assert seen == [("show", 7, 3), ("front", 7)] + + +# --- process id ------------------------------------------------------------ + +def test_window_process_id_asks_the_backend_for_the_match(wm, monkeypatch): + monkeypatch.setattr(wm, "get_window_process_id", lambda h: 4242) + assert w.window_process_id("Editor") == 4242 + + +def test_window_process_id_is_none_when_nothing_matches(wm): + assert w.window_process_id("no such window") is None + + +def test_foreground_window_process_id_follows_the_foreground_hwnd( + wm, monkeypatch): + seen = [] + monkeypatch.setattr(wm, "get_window_process_id", + lambda h: seen.append(h) or 99) + assert w.foreground_window_process_id() == 99 + assert seen == [13] # the hwnd get_foreground_window reported + + +def test_process_id_zero_reads_as_unknown_not_as_pid_zero(wm, monkeypatch): + """0 is the backend's "could not tell", and it is never a real user pid. + + Returning it verbatim would let a caller compare `pid == 0` against a + process list and match the System Idle Process. + """ + monkeypatch.setattr(wm, "get_window_process_id", lambda h: 0) + assert w.foreground_window_process_id() is None + assert w.window_process_id("Editor") is None + + +def test_real_foreground_window_process_id_is_a_live_pid(): + """No stubs: the ctypes prototype has to survive a real call.""" + pid = w.foreground_window_process_id() + assert pid is None or (isinstance(pid, int) and pid > 0) + + +# --- posting input without focus ------------------------------------------- + +def test_post_key_targets_the_focused_control_not_the_frame(wm, monkeypatch): + """The whole point of the fix. + + Keyboard messages are delivered to the control that has focus. Measured on + Character Map: posting to the top-level frame typed nothing, posting to the + focused edit typed the character — so a "background typing" feature that + posts to the frame silently does nothing in any app with child controls. + """ + posted = [] + monkeypatch.setattr(wm, "get_focused_control", lambda hwnd: 999) + monkeypatch.setattr(wm, "post_key", + lambda hwnd, code, char="": posted.append( + (hwnd, code, char)) or True) + assert w.post_key_to_window("Editor", "a") is True + assert posted == [(11, 65, "a")] # hwnd of the matched window + + +def test_post_key_sends_a_character_for_printable_keys_only(): + assert w._resolve_key("a") == (65, "a") # WM_CHAR carries the text + assert w._resolve_key("f5")[1] == "" # a function key has no character + assert w._resolve_key(65) == (65, "") + + +def test_post_key_rejects_an_unknown_key_name(): + from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, + ) + with pytest.raises(AutoControlActionException): + w._resolve_key("wingdings") + + +def test_post_click_accepts_both_button_spellings(wm, monkeypatch): + seen = [] + monkeypatch.setattr(wm, "post_click", + lambda hwnd, button, x, y: seen.append( + (hwnd, button, x, y)) or True) + assert w.post_click_to_window("Editor", "mouse_right", 5, 6) is True + assert w.post_click_to_window("Editor", "LEFT", 1, 2) is True + assert seen == [(11, "right", 5, 6), (11, "left", 1, 2)] + + +def test_posting_to_a_missing_window_is_false_not_an_exception(wm): + assert w.post_key_to_window("no such window", "a") is False + assert w.post_click_to_window("no such window") is False + + +def test_post_click_rejects_unknown_buttons(): + from je_auto_control.windows.window import windows_window_manage as module + with pytest.raises(ValueError): + module.post_click(0, "scroll", 0, 0) + + +def test_focused_control_falls_back_to_the_window_itself(): + """A window whose thread reports no focus must still be a usable target.""" + from je_auto_control.windows.window import windows_window_manage as module + assert module.get_focused_control(0) == 0 + + +# --- windows by owning process --------------------------------------------- + +def test_windows_for_process_id_filters_by_owner(wm, monkeypatch): + monkeypatch.setattr(wm, "get_window_process_id", + lambda hwnd: {11: 4242, 12: 7, 13: 4242}.get(hwnd, 0)) + assert w.windows_for_process_id(4242) == [(11, "Editor"), (13, "Browser")] + + +def test_windows_for_process_id_can_keep_untitled_windows(wm, monkeypatch): + """A browser's helper windows are often untitled — and still worth acting on.""" + monkeypatch.setattr(wm, "get_window_process_id", lambda hwnd: 4242) + assert len(w.windows_for_process_id(4242)) == 3 + assert len(w.windows_for_process_id(4242, titled_only=True)) == 2 + + +def test_minimize_windows_for_process_counts_only_what_it_minimised( + wm, monkeypatch): + monkeypatch.setattr(wm, "get_window_process_id", lambda hwnd: 4242) + monkeypatch.setattr(wm, "minimize_window", lambda hwnd: hwnd != 12) + assert w.minimize_windows_for_process(4242) == 2 + + +# --- the deprecated pair now delegates instead of doing nothing ------------- + +def test_deprecated_key_sender_warns_and_delegates(wm, monkeypatch): + """It used to post to the frame and silently do nothing while reporting success.""" + from je_auto_control.wrapper import auto_control_keyboard as k + + posted = [] + monkeypatch.setattr(wm, "get_focused_control", lambda hwnd: hwnd) + monkeypatch.setattr(wm, "post_key", + lambda hwnd, code, char="": posted.append( + (hwnd, code, char)) or True) + with pytest.warns(DeprecationWarning): + k.send_key_event_to_window("Editor", "a") + assert posted == [(11, 65, "a")] # resolved by title substring + + +def test_deprecated_mouse_sender_accepts_an_hwnd_and_a_title(wm, monkeypatch): + """The old signature took an hwnd; keep that working while fixing the target.""" + from je_auto_control.wrapper import auto_control_mouse as m + + seen = [] + monkeypatch.setattr(wm, "post_click", + lambda hwnd, button, x, y: seen.append( + (hwnd, button, x, y)) or True) + with pytest.warns(DeprecationWarning): + m.send_mouse_event_to_window(11, "mouse_right", 5, 6) + with pytest.warns(DeprecationWarning): + m.send_mouse_event_to_window("Editor", "mouse_left", 1, 2) + assert seen == [(11, "right", 5, 6), (11, "left", 1, 2)] + + +def test_deprecated_mouse_sender_maps_raw_keycodes_back_to_a_button(): + from je_auto_control.wrapper.auto_control_mouse import _button_name_for_post + from je_auto_control.wrapper.platform_wrapper import mouse_keys_table + + assert _button_name_for_post("mouse_middle") == "middle" + assert _button_name_for_post(mouse_keys_table["mouse_right"]) == "right" + assert _button_name_for_post(object()) == "left" # unknown → safe default From 2dec2aa75da8280f6b16db5135cd582b8059debe Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 17 Aug 2026 21:12:25 +0800 Subject: [PATCH 04/21] Declare Win32 prototypes for screen size and pixel reads MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The module declared none, so the HDC from GetDC — pointer-width — came back through ctypes' default c_int, and the same truncated value was passed on to GetPixel and ReleaseDC: a wrong colour and a leaked DC, with no symptom either way. Same trap as the HWND and HGLOBAL handles elsewhere. The module now owns private user32 / gdi32 handles so the declarations cannot leak into other callers, and its import-time SetProcessDPIAware() call is documented as what it is: process-wide and irreversible. --- .../windows/screen/win32_screen.py | 50 ++++++++++++++++--- 1 file changed, 43 insertions(+), 7 deletions(-) diff --git a/je_auto_control/windows/screen/win32_screen.py b/je_auto_control/windows/screen/win32_screen.py index 88a5769d..e0aa268a 100644 --- a/je_auto_control/windows/screen/win32_screen.py +++ b/je_auto_control/windows/screen/win32_screen.py @@ -9,11 +9,47 @@ raise AutoControlException(windows_import_error_message) import ctypes +from ctypes import wintypes -# 初始化 Win32 API 函式 Initialize Win32 API functions -_user32 = ctypes.windll.user32 -_user32.SetProcessDPIAware() # 確保 DPI 感知，避免座標偏移 -_gdi32 = ctypes.windll.gdi32 +# 這個模組持有自己的 user32 / gdi32 handle，而不是共用 `ctypes.windll`： +# argtypes/restype 是設在**函式物件**上的，共用 handle 會讓這裡的原型外溢到 +# 別的呼叫者（`utils/window_capture/` 就用自己的 RECT 呼叫 GetWindowRect）。 +# +# This module owns its user32 / gdi32 handles rather than sharing +# ``ctypes.windll``: prototypes live on the function objects, so a shared handle +# would leak these declarations into every other caller in the process. +_user32 = ctypes.WinDLL("user32", use_last_error=True) +_gdi32 = ctypes.WinDLL("gdi32", use_last_error=True) + +# HDC 是**指標寬度**的 handle。ctypes 預設把回傳值與參數當成 c_int，在 64 位元 +# Windows 上會截斷——`GetDC` 回來就已經是壞的，再傳給 `GetPixel` / `ReleaseDC` +# 只會讓錯誤沉默地擴散（顏色讀錯、DC 沒被釋放）。與 `windows_window_manage` 的 +# HWND、剪貼簿的 HGLOBAL 是同一個陷阱，所以每支都明寫原型。 +# +# An HDC is a pointer-width handle and ctypes defaults to ``c_int``, which +# truncates it on 64-bit Windows: the value is already wrong coming out of +# ``GetDC``, and passing it on silently reads the wrong colour and leaks the DC. +_user32.SetProcessDPIAware.argtypes = [] +_user32.SetProcessDPIAware.restype = wintypes.BOOL +_user32.GetSystemMetrics.argtypes = [ctypes.c_int] +_user32.GetSystemMetrics.restype = ctypes.c_int +_user32.GetDC.argtypes = [wintypes.HWND] +_user32.GetDC.restype = wintypes.HDC +_user32.ReleaseDC.argtypes = [wintypes.HWND, wintypes.HDC] +_user32.ReleaseDC.restype = ctypes.c_int +_gdi32.GetPixel.argtypes = [wintypes.HDC, ctypes.c_int, ctypes.c_int] +_gdi32.GetPixel.restype = wintypes.COLORREF + +# 確保 DPI 感知，避免座標偏移。**這是行程層級的副作用**，而它發生在 import 時： +# 一旦設定就無法還原，之後所有 Win32 座標查詢都會拿到實體像素。這正是本模組被 +# import 的理由（螢幕尺寸與取色都必須是實體座標），但呼叫端要知道它會影響整個 +# 行程——擷取與滑鼠座標的換算請走 `utils/monitor_layout`。 +# +# Process-wide and irreversible, and it happens at import time; conversions +# between physical and logical coordinates belong to ``utils/monitor_layout``. +_user32.SetProcessDPIAware() + +_CLR_INVALID = 0xFFFFFFFF def size() -> List[int]: @@ -41,8 +77,8 @@ def get_pixel(x: int, y: int, hwnd: int = 0) -> Tuple[int, int, int]: raise AutoControlException("GetDC failed") try: - pixel = _gdi32.GetPixel(dc, x, y) - if pixel == 0xFFFFFFFF: # GetPixel 失敗時回傳 -1 (0xFFFFFFFF) + pixel = int(_gdi32.GetPixel(dc, int(x), int(y))) + if pixel == _CLR_INVALID: # GetPixel 失敗時回傳 CLR_INVALID raise AutoControlException("GetPixel failed") r = pixel & 0xFF @@ -50,4 +86,4 @@ def get_pixel(x: int, y: int, hwnd: int = 0) -> Tuple[int, int, int]: b = (pixel >> 16) & 0xFF return r, g, b finally: - _user32.ReleaseDC(hwnd, dc) \ No newline at end of file + _user32.ReleaseDC(hwnd, dc) From 50092e8dd1bc751b981fd790d078e967c7bd17d7 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Mon, 17 Aug 2026 21:12:43 +0800 Subject: [PATCH 05/21] Record the counts, the fixes and the remaining decision Command / MCP-tool counts move to 773 and 676 (the doc-count guard measures them), the touched module line counts are re-measured, and the clipboard subpackage row now names the shared Win32 prototype module. Progress.md loses the settled item: the two deprecated window-input functions are no longer a decision, they delegate. --- CHANGELOG.md | 58 +++++++++++++++++++++++++++++++++++++++++ README.md | 6 ++--- README/README_zh-CN.md | 6 ++--- README/README_zh-TW.md | 6 ++--- WHATS_NEW.md | 58 +++++++++++++++++++++++++++++++++++++++++ architecture_explore.md | 30 ++++++++++----------- 6 files changed, 140 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0064d1..b9ebd26a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,34 @@ only when documented here with a migration path. `ac_foreground_window`, `ac_window_rect`). `list_windows` takes `titled_only`, and `move_window_by_title` keeps the window's current size when width/height are omitted. +- Window ownership: `foreground_window_process_id` and `window_process_id` + (`AC_foreground_window_pid`, `AC_window_pid`; `ac_foreground_window_pid`, + `ac_window_pid`; two Script Builder specs), on the Windows backend + `get_window_process_id`. A title is whatever the application decides to + display, so it cannot answer "which program is the user actually in front + of" — the process id can. Unavailable reads as `None` (`{"pid": 0}` on the + JSON surfaces) rather than a bare `0`, which a caller could otherwise match + against a process list and hit the System Idle Process. +- Windows by owning process: `windows_for_process_id` and + `minimize_windows_for_process` (`AC_windows_for_pid`, + `AC_minimize_windows_for_pid`; `ac_windows_for_pid`, + `ac_minimize_windows_for_pid`; two Script Builder specs). A multi-process + application cannot be addressed by title — its windows are named after + whatever they display and several of its processes have no window at all — + so ownership is the stable key. +- Input posted to a window without focusing it: `post_key_to_window` and + `post_click_to_window` (`AC_post_key_to_window`, `AC_post_click_to_window`; + `ac_post_key_to_window`, `ac_post_click_to_window`; two Script Builder specs), + on the Windows backend `get_focused_control`, `deepest_child_at`, `post_key` + and `post_click`. They resolve the window by title *substring* like every + other function here, and post to the control that actually has keyboard + focus — or, for a click, to the deepest child under the point, in that + child's client coordinates. Posting to the top-level frame (what the older + `send_key_event_to_window` does) types nothing in any application with child + controls: measured on Character Map, the frame swallowed the key while the + focused edit accepted it. Both return whether the messages were queued, and + posting remains best effort — applications reading raw input or checking the + foreground ignore posted messages. - `utils/url_canon` reaches its delivery surfaces: `canonicalize_url`, `normalize_url`, `urls_equal`, `build_query` and `parse_query` are exported from the facade, with `AC_canonicalize_url` / `AC_normalize_url` / @@ -71,6 +99,19 @@ only when documented here with a migration path. ### Changed +- **`send_key_event_to_window` / `send_mouse_event_to_window` now actually + reach the target.** They posted to the top-level frame, but keyboard messages + go to the control that *has focus* and a click belongs to the child under the + point in that child's client coordinates — so for any window with child + controls they did nothing at all while still reporting success. They delegate + to `post_key_to_window` / `post_click_to_window`. Two visible consequences: + the key sender now matches the window title as a *substring* (it required an + exact title before, via `FindWindowW`), and the mouse sender accepts a title + string as well as the hwnd it always took. +- `save_window_layout` now snapshots only titled windows (its documented + behaviour). Untitled entries could never be restored — `restore_window_layout` + addresses a window by title and skips blank ones — so they only inflated the + saved count, by roughly half on a real desktop. - `set_clipboard_image` accepts PNG bytes **or** a path to any Pillow-readable image, and `get_clipboard_image` / `set_clipboard_image` are now exported from `je_auto_control.utils.clipboard` and the top-level facade, with @@ -131,11 +172,28 @@ only when documented here with a migration path. ### Deprecated +- `send_key_event_to_window` and `send_mouse_event_to_window` — use + `post_key_to_window` / `post_click_to_window`. Both now emit a + `DeprecationWarning` and delegate to the working implementation; see Changed + for the behaviour that changes. + - New integrations should avoid the eager, historical top-level import surface and import stable entry points from `je_auto_control.api`. ### Fixed +- **Four clipboard writers never worked on 64-bit Windows**: + `set_clipboard_files`, `set_clipboard_html`, `set_clipboard_rtf` and + `set_clipboard_csv` all raised `OverflowError: int too long to convert` on + every call, and the matching readers failed whenever that format was actually + present. Each module declared `restype` but not `argtypes`, so ctypes passed + the pointer-width memory handle as `c_int`. The prototypes and the + open/alloc/lock dance now live once in + `je_auto_control/utils/clipboard/win32_clipboard_api.py`, on private `WinDLL` + handles so the declarations cannot leak into other user32 callers, and + `rich_clipboard`, `clipboard_rich_formats`, `clipboard_files` and + `clipboard_formats` all go through it. + - `write` failing a whole string on the first character outside the 192-entry virtual-key table — on a US layout that includes `, . / : ? ! _ + @ %` and every CJK character, so URLs and non-English text could not be typed at all. diff --git a/README.md b/README.md index 676e2b75..41039c33 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. - **One API, six platforms.** `wrapper/platform_wrapper.py` picks the backend at import time; your script does not change between Windows, macOS, X11, and Wayland. -- **Scriptable without Python.** 767 `AC_*` commands cover the whole feature set, so a +- **Scriptable without Python.** 773 `AC_*` commands cover the whole feature set, so a JSON file can do anything the library can — including loops, branches, try/catch, macros, and variables. - **Headless by default.** `import je_auto_control` never loads Qt. The GUI is an @@ -134,7 +134,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Natural-language planner | `plan_actions`, `run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`, `run_agent` | `AC_run_agent` | Computer Use | | Record & replay | `record`, `stop_record` | `AC_record`, `AC_stop_record` | Record | -| JSON scripting | `execute_action`, `execute_files` | all 767 commands | Script, Script Builder | +| JSON scripting | `execute_action`, `execute_files` | all 773 commands | Script, Script Builder | | Variables & flow control | `execute_action_with_vars` | `AC_set_var`, `AC_loop`, `AC_for_each`, `AC_try`, `AC_retry` | Variables | | Data-driven runs | — | `AC_for_each_row` (CSV / JSON / SQLite / Excel) | Data Sources | | Assertions | `assert_text`, `assert_image` | `AC_assert_text` + 20 more | Assertions | @@ -185,7 +185,7 @@ entry point still works. | Surface | Start it with | Notes | |---|---|---| -| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 670 tools for Claude Desktop / Claude Code / custom tool loops. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | +| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 676 tools for Claude Desktop / Claude Code / custom tool loops. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | | **REST API** | `je_auto_control start-rest` | Bearer token, per-IP rate limit + lockout, SQLite audit hook, `/metrics`, `/openapi.json`, `/docs` Swagger UI, `/dashboard`. | | **TCP socket server** | `je_auto_control start-server` | Newline-framed JSON action lists. Binds `127.0.0.1` by default. | | **pytest plugin** | installed automatically | Fixtures plus a Gherkin step library for pytest-bdd / behave. | diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index d23c68ca..30e193e5 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -20,7 +20,7 @@ - **一套 API，六个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 Windows、macOS、X11 与 Wayland 上都不需要改写。 -- **不写 Python 也能脚本化。** 767 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 +- **不写 Python 也能脚本化。** 773 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 能做的任何事——包含循环、分支、try/catch、宏与变量。 - **默认无头运行。** `import je_auto_control` 绝不会加载 Qt。GUI 是可选包，包在同一个无头内核之外。 - **四种定位方式。** 模板匹配、OCR、无障碍树、视觉语言模型——可通过锚点定位器与自愈回退串接组合。 @@ -128,7 +128,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 自然语言规划 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 录制与回放 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 脚本 | `execute_action`、`execute_files` | 全部 767 个命令 | Script、Script Builder | +| JSON 脚本 | `execute_action`、`execute_files` | 全部 773 个命令 | Script、Script Builder | | 变量与流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 数据驱动执行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 断言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 个 | Assertions | @@ -177,7 +177,7 @@ je_auto_control version | 接口 | 启动方式 | 说明 | |---|---|---| -| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 670 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | +| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 676 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、按 IP 限流与锁定、SQLite 审计 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 服务器** | `je_auto_control start-server` | 以换行分隔的 JSON 动作列表。默认绑定 `127.0.0.1`。 | | **pytest 插件** | 安装后自动生效 | 提供 fixture 与供 pytest-bdd／behave 使用的 Gherkin step library。 | diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index d75feed8..3699b1bd 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -20,7 +20,7 @@ - **一套 API，六個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 Windows、macOS、X11 與 Wayland 上都不需要改寫。 -- **不寫 Python 也能腳本化。** 767 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 +- **不寫 Python 也能腳本化。** 773 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 - **預設無頭執行。** `import je_auto_control` 絕不會載入 Qt。GUI 是選用套件，包在同一個無頭核心之外。 - **四種定位方式。** 樣板比對、OCR、無障礙樹、視覺語言模型——可透過錨點定位器與自癒後備串接組合。 @@ -128,7 +128,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 自然語言規劃 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 錄製與重播 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 腳本 | `execute_action`、`execute_files` | 全部 767 個指令 | Script、Script Builder | +| JSON 腳本 | `execute_action`、`execute_files` | 全部 773 個指令 | Script、Script Builder | | 變數與流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 資料驅動執行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 斷言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 個 | Assertions | @@ -177,7 +177,7 @@ je_auto_control version | 介面 | 啟動方式 | 說明 | |---|---|---| -| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 670 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | +| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 676 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、逐 IP 限流與鎖定、SQLite 稽核 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 伺服器** | `je_auto_control start-server` | 以換行分隔的 JSON 動作清單。預設綁 `127.0.0.1`。 | | **pytest 外掛** | 安裝後自動生效 | 提供 fixture 與供 pytest-bdd／behave 使用的 Gherkin step library。 | diff --git a/WHATS_NEW.md b/WHATS_NEW.md index fe09f4fc..1c084aa5 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,63 @@ # What's New — AutoControl +## What's new (2026-08-17) + +### Which Program Owns That Window + +- **`foreground_window_process_id` / `window_process_id`** (`AC_foreground_window_pid`, + `AC_window_pid`, `ac_foreground_window_pid`, `ac_window_pid`, two Script Builder + specs; Windows backend `get_window_process_id`): a window title is whatever the + application decides to display, and unrelated programs share titles like + `Settings`, so "which program is the user in front of" had no reliable answer. + It does now, and callers can join it against a process list. Unavailable reads + as `None` — never a bare `0`, which would match the System Idle Process. +- **The two older window-input functions are deprecated and now work.** + `send_key_event_to_window` / `send_mouse_event_to_window` posted to the frame, + which reaches nothing in a window with child controls; they delegate to the + new pair, warn once, and keep their old argument types working. +- **`get_pixel` no longer risks a truncated device context.** The Windows screen + backend declared no prototypes, so an HDC — pointer-width — came back through + ctypes' default `c_int`, and the same truncated value was passed on to + `GetPixel` and `ReleaseDC`. Every prototype is declared and the module owns + private DLL handles, so the declarations cannot leak into other callers. +- **Act on a process's windows, not on a title** + (`windows_for_process_id`, `minimize_windows_for_process`): a browser names + its windows after the page they show and runs a dozen processes without any + window, so "minimise that application" was previously hand-rolled Win32 in + every caller — enumerate windows, ask each one which process owns it, filter, + minimise. That loop now lives here once. +- **Typing into a window without stealing focus actually works now** + (`post_key_to_window`, `post_click_to_window`, `AC_post_key_to_window`, + `AC_post_click_to_window`, `ac_post_key_to_window`, `ac_post_click_to_window`). + The existing `send_key_event_to_window` posted to the top-level frame, and + keyboard messages go to the control that *has focus* — so it silently did + nothing in any application with child controls while still reporting success. + Measured on Character Map: posting to the frame typed nothing, posting to the + focused edit typed the character; on Windows 11 Notepad the new path types + into a background window while the foreground window keeps focus. Clicks + resolve the deepest child under the point and convert to its client + coordinates. This is still best effort by nature — games and anything reading + raw input ignore posted messages — so both functions return whether the + messages were queued rather than claiming the application acted. +- **Half the clipboard was broken on 64-bit Windows, and nothing noticed.** + `set_clipboard_files`, `set_clipboard_html`, `set_clipboard_rtf` and + `set_clipboard_csv` raised `OverflowError` on every call — four writers that + had never once worked. Each module declared `restype` but not `argtypes`, so + a pointer-width memory handle went through ctypes' default `c_int`. The + pure-function tests could not see it (the byte packing was always right) and + nothing exercised the Win32 half. The prototypes now live once in + `utils/clipboard/win32_clipboard_api.py`, every clipboard module goes through + it, and a new test round-trips text, HTML, RTF, CSV and file lists through the + real clipboard — plus a static check that no module hand-rolls those calls + without declaring prototypes again. +- **`save_window_layout` no longer records windows it cannot restore.** Its + docstring promised titled windows; the default lister returned all of them, + while `restore_window_layout` addresses a window by title and skips blank ones. + On a real desktop that was 28 entries saved against 15 restorable — a caller + reporting the saved count was over-promising by a factor of two. The lister now + passes `titled_only=True`, and a round-trip test pins save and restore to the + same set. + ## What's new (2026-08-15) ### Text Entry and On-Screen Location That Match What You See diff --git a/architecture_explore.md b/architecture_explore.md index 52cb1a4e..6df61ba8 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -22,10 +22,10 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | Python 模組總數（含周邊子專案） | 998 | | 程式碼總行數 | 133,534 | | `je_auto_control/utils/` 子套件數 | 308 | -| `AC_*` 動作指令數（`known_commands()` 實測） | 767 | +| `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,221 | | GUI 分頁數（`main_widget` 註冊） | 48 | -| MCP 工具數（`build_default_tool_registry()` 實測） | 670 | +| MCP 工具數（`build_default_tool_registry()` 實測） | 676 | | `test_*.py` 測試檔／測試函式 | 458 / 4,319 | | 範例腳本 | 27 | @@ -49,7 +49,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 │ 全部只呼叫下面這一層，不含業務邏輯 ┌───────────────────────────────▼──────────────────────────────────────────┐ │ 執行核心 Execution Core │ -│ utils/executor/action_executor.py ── Executor.event_dict（767 個 AC_*） │ +│ utils/executor/action_executor.py ── Executor.event_dict（773 個 AC_*） │ │ utils/executor/flow_control.py ── 34 個區塊指令（迴圈/分支/try/巨集） │ │ utils/script_vars ── ${var} 插值 │ utils/json ── action 檔 I/O │ └───────────────────────────────┬──────────────────────────────────────────┘ @@ -170,12 +170,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `wrapper/_platform_osx.py` | 150 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 268 | X11 後端組裝（python-Xlib + 選用 uinput）。 | | `wrapper/_platform_wayland.py` | 58 | Wayland 後端組裝（libei／ydotool／grim）。 | -| `wrapper/auto_control_mouse.py` | 304 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | -| `wrapper/auto_control_keyboard.py` | 228 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | +| `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_keyboard.py` | 273 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | | `wrapper/auto_control_screen.py` | 98 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | | `wrapper/auto_control_record.py` | 76 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | -| `wrapper/auto_control_window.py` | 94 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態（目前僅 Windows 實作）。 | +| `wrapper/auto_control_window.py` | 293 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | ### 5.3 平台後端 @@ -190,14 +190,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `keyboard/win32_ctype_keyboard_control.py` | 55 | 鍵盤事件產生。 | | `record/win32_input_hook.py` | 223 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | | `record/win32_record.py` | 126 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）。 | -| `screen/win32_screen.py` | 53 | 螢幕尺寸與像素讀取。 | -| `window/windows_window_manage.py` | 208 | 視窗列舉／聚焦／關閉／最小化／幾何（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | +| `screen/win32_screen.py` | 89 | 螢幕尺寸與像素讀取。**每支 Win32 函式都明寫 argtypes/restype**（HDC 是指標寬度，走預設的 c_int 會截斷，錯誤會沉默地擴散到 GetPixel／ReleaseDC），並持有自己的 user32／gdi32 handle。import 時呼叫 `SetProcessDPIAware()`——**行程層級且不可還原**，實體↔邏輯座標換算請走 `utils/monitor_layout`。 | +| `window/windows_window_manage.py` | 366 | 視窗列舉／聚焦／關閉／最小化／幾何／所屬行程 PID／投遞式輸入（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | | `message/window_message.py` | 97 | 直接對視窗送 `WM_*` 訊息（背景輸入）。 | | `interception/_dll.py` | 231 | `interception.dll` 的延遲 ctypes 載入與結構定義。 | | `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | | `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | -#### macOS（`osx/`，17 檔／771 行） +#### macOS（`osx/`，17 檔／773 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -268,7 +268,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/dag/` | 478 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 105 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 98 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 8,931 | **核心**。`Executor` 指令分派表（767 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/executor/` | 8,931 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 138 | action list 的單步除錯器與追蹤器 | | `utils/input_macro/` | 129 | 定時輸入事件重播與宣告式輸入序列 DSL | | `utils/json/` | 75 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | @@ -490,7 +490,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/cua_action/` | 129 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 363 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 94 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,441 | **無頭 MCP 伺服器**（16K LOC，預設註冊 670 個工具＝651 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 16,441 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 182 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 108 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 456 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | @@ -658,7 +658,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/clipboard/` | 353 | 跨平台無頭剪貼簿存取（文字 + 影像）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | +| `utils/clipboard/` | 468 | 跨平台無頭剪貼簿存取（文字 + 影像）＋ `win32_clipboard_api.py`：**所有剪貼簿格式共用的 Win32 原型與 open/alloc/lock 流程**（`argtypes` 只宣告一半曾讓四支 writer 在 64 位元上必然丟 `OverflowError`，見 CHANGELOG）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | | `utils/clipboard_files/` | 119 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | | `utils/clipboard_formats/` | 152 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | | `utils/clipboard_history/` | 111 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | @@ -683,12 +683,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `action_executor.py` | 8,042 | `Executor` 類別與 `event_dict` 分派表（767 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `action_executor.py` | 8,042 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | | `flow_control.py` | 757 | 34 個區塊指令：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`、`AC_*_to_var`）／`AC_assert_var`。`LoopBreak`／`LoopContinue` 以例外實作。 | | `action_schema.py` | 94 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。 | | `mouse_aliases.py` | 40 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,441 行，670 個工具）— 最大子系統 +#### `utils/mcp_server/`（16,441 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -1011,7 +1011,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/ocr/` | 9 | 1,105 | | `utils/usbip/` | 5 | 925 | | `utils/assertion/` | 3 | 866 | -| `osx/` | 17 | 771 | +| `osx/` | 17 | 773 | | `autocontrol-lsp/` | 8 | 752 | | `utils/hotkey/` | 7 | 734 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,047 | From 8bf9c7e22442222abcbce7576307cc732f3af874 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:14:22 +0800 Subject: [PATCH 06/21] Verify the Wayland paths against real peers instead of a mock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five container jobs replace assumptions with measurements, and none of them needs the VM three of these items had been recorded as needing: * wayland-verification — a real headless sway session, 27 checks over two output layouts, the second one negative-origin. * eis-verification — the real libeis server over a Unix socket, so the libei sender is read back off the wire, 20 checks. * portal-verification — a real dbus-daemon and real liboeffis against a portal implemented here; ConnectToEIS hands over a live fd to that same libeis, 20 checks including six refusal paths. * ydotool-verification — a real uinput device read back from /dev/input/eventN, 12 checks. * seat-verification — a wlroots session consuming the real ydotool device, cursor position read out of grim -c pixels, 14 checks over the same two layouts. A seat that consumes libinput devices is WLR_BACKENDS=headless,libinput plus LIBSEAT_BACKEND=builtin plus SEATD_VTBOUND=0, with udev up before ydotoold. A portal is whoever owns the bus name. Both were mistaken for things a container cannot do. The jobs fail loudly when uinput or evdev will not load rather than skipping, so a runner that cannot provide them says so. --- docker/.dockerignore => .dockerignore | 0 .gitattributes | 6 + .github/workflows/docker.yml | 254 +++++++ docker/Dockerfile.eis | 83 +++ docker/Dockerfile.portal | 98 +++ docker/Dockerfile.seat | 118 ++++ docker/Dockerfile.wayland | 88 +++ docker/Dockerfile.ydotool | 102 +++ docker/eis_server.py | 431 ++++++++++++ docker/eis_verify.py | 644 ++++++++++++++++++ docker/entrypoint-seat.sh | 111 +++ docker/entrypoint-wayland.sh | 81 +++ docker/libei_verify.py | 286 ++++++++ docker/portal_server.py | 454 ++++++++++++ docker/portal_verify.py | 566 +++++++++++++++ docker/seat_verify.py | 493 ++++++++++++++ docker/wayland_verify.py | 383 +++++++++++ docker/ydotool_verify.py | 404 +++++++++++ .../headless/test_docker_artifacts.py | 149 +++- 19 files changed, 4749 insertions(+), 2 deletions(-) rename docker/.dockerignore => .dockerignore (100%) create mode 100644 docker/Dockerfile.eis create mode 100644 docker/Dockerfile.portal create mode 100644 docker/Dockerfile.seat create mode 100644 docker/Dockerfile.wayland create mode 100644 docker/Dockerfile.ydotool create mode 100644 docker/eis_server.py create mode 100644 docker/eis_verify.py create mode 100644 docker/entrypoint-seat.sh create mode 100644 docker/entrypoint-wayland.sh create mode 100644 docker/libei_verify.py create mode 100644 docker/portal_server.py create mode 100644 docker/portal_verify.py create mode 100644 docker/seat_verify.py create mode 100644 docker/wayland_verify.py create mode 100644 docker/ydotool_verify.py diff --git a/docker/.dockerignore b/.dockerignore similarity index 100% rename from docker/.dockerignore rename to .dockerignore diff --git a/.gitattributes b/.gitattributes index dfe07704..e1207c57 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,8 @@ # Auto detect text files and perform LF normalization * text=auto + +# Shell scripts keep LF whatever the checkout platform. They are executed by +# /bin/sh inside a Linux container, and a CRLF shebang makes the kernel look +# for an interpreter literally named "/bin/sh\r": the image builds, then every +# container dies with "no such file or directory" on the entrypoint. +*.sh text eol=lf diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d0905b11..d76d7236 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -108,3 +108,257 @@ jobs: echo "REST health probe never succeeded" >&2 exit 1 fi + + wayland-verification: + name: Wayland backend against a real compositor + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the Wayland verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.wayland + tags: autocontrol-wayland:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # Two halves, both of which mocks structurally cannot cover. + # + # Capture: sway's headless backend needs no GPU, no seat and no display, + # so a plain ubuntu runner can host a genuine wlroots session. grim's + # argv and -g geometry, wlr-randr's undocumented output format, and the + # whole screenshot -> screen_grabber -> capture -> grab_image chain are + # checked against pixels the compositor actually painted. Twice: once + # with the outputs side by side from the origin, and once with the + # left-hand one at x=-1280, which is the layout of any desktop with a + # monitor left of the primary. The whole-screen capture then starts at + # a negative coordinate, and a size, a crop or a located hit that + # assumes (0, 0) is wrong by the width of that monitor. + # + # libei: every entry point the ctypes binding names is resolved against + # the real libei.so — a misspelled symbol passes every mock and fails + # only on a user's machine — and the fail-closed chain is driven end to + # end. It also re-checks whether ei_unref still segfaults upstream, so + # the workaround in LibeiBackend._teardown gets removed once it can be. + # + # The container exits with the number of failed checks. + - name: Verify against headless sway and the real libei + run: docker run --rm autocontrol-wayland:ci + + eis-verification: + name: libei sender against a real EIS server + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the EIS verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.eis + tags: autocontrol-eis:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # The other half of the input path, and the half no mock can reach: a + # wrong capability value, a mis-marshalled variadic bind or a wrong + # scroll unit is *accepted* by a fake symbol table and only rejected by + # something that speaks the protocol. libeis is that something — the + # server side of libei's own protocol, packaged by Debian — so + # docker/eis_server.py runs a real EIS implementation on a Unix socket + # and records what AutoControl's real sender does to it. No compositor + # and no desktop session are involved. + # + # It also re-checks whether ei_unref is still safe on a live context, + # which is what lets _teardown release instead of leaking. + # + # The container exits with the number of failed checks. + - name: Verify the libei sender against libeis + run: docker run --rm autocontrol-eis:ci + + portal-verification: + name: RemoteDesktop portal handshake against a real liboeffis + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the portal verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.portal + tags: autocontrol-portal:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # How a client gets an EIS socket on GNOME and KDE: not a path on disk + # but a file descriptor handed over D-Bus at the end of the + # org.freedesktop.portal.RemoteDesktop dance. That was recorded as + # needing a GNOME VM because xdg-desktop-portal-wlr has no RemoteDesktop + # interface — but the portal is a D-Bus interface, not a compositor + # feature, so docker/portal_server.py owns the well-known name on a + # private session bus and answers the four calls for real. + # + # Its ConnectToEIS hands back a live connection to the same real libeis + # server the eis job uses, so the whole chain is checked at once: the + # call order and predicted request paths, the device mask the user would + # be consenting to, that the descriptor carries a real EI session, and + # that input emitted through it is recorded by an independent + # implementation. + # + # And every way a portal says no — a dismissed dialog, a dialog left + # open, a withheld descriptor, a closed session, a portal too old to + # have ConnectToEIS, no portal at all — has to come back as a refusal on + # this project's own clock rather than a hang or a silent downgrade. + # + # The container exits with the number of failed checks. + - name: Verify the portal handshake against liboeffis + run: docker run --rm autocontrol-portal:ci + + seat-verification: + name: ydotool absolute move against a seat that consumes it + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the seat verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.seat + tags: autocontrol-seat:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Load the uinput and evdev kernel modules + run: | + sudo modprobe uinput + sudo modprobe evdev || true + test -e /dev/uinput || { + echo "::error::/dev/uinput is missing after modprobe; this runner's" + echo "::error::kernel cannot host the seat verification." + exit 1 + } + + # The join between the two images above, and the one every earlier note + # in this file said needed a VM. The wayland job runs a compositor that + # consumes no input; the ydotool job reads ydotool's events off the + # kernel with no compositor. Neither can say where the cursor ends up. + # + # wlroots can: WLR_BACKENDS=headless,libinput keeps the outputs virtual + # while running the real libinput backend, libseat's builtin backend + # opens the device without logind, and SEATD_VTBOUND=0 stops it + # reaching for a VT no container owns. ydotoold's device is then an + # ordinary seat device, and grim -c draws the cursor into a screenshot. + # + # That settles what --absolute is absolute *to* — the top-left of the + # output layout, not layout (0, 0), which is the translation + # linux_wayland/mouse.py now applies — and what pointer acceleration + # does to it, which is double the distance asked for under libinput's + # default profile. It runs over the same two layouts as the wayland + # job, and the negative-origin one is where an untranslated request + # lands on the wrong monitor entirely. + # + # The container exits with the number of failed checks. + - name: Verify the absolute move against a real seat + run: | + docker run --rm --device /dev/uinput --device-cgroup-rule 'c 13:* rmw' autocontrol-seat:ci + + ydotool-verification: + name: ydotool argv against a real uinput device + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the ydotool verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.ydotool + tags: autocontrol-ydotool:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # uinput is what ydotoold writes into; evdev is the handler that turns + # the device it creates into the /dev/input/eventN this verification + # reads back. Both ship with the runner's kernel as modules. The + # explicit check is here so a kernel without them fails saying so, + # rather than the container reporting an empty device list. + - name: Load the uinput and evdev kernel modules + run: | + sudo modprobe uinput + sudo modprobe evdev || true + test -e /dev/uinput || { + echo "::error::/dev/uinput is missing after modprobe; this runner's" + echo "::error::kernel cannot host the ydotool verification." + exit 1 + } + ls -l /dev/uinput + + # The half neither other image can reach. sway's headless backend + # consumes no libinput devices, so an injected event has nowhere to + # arrive there — but arriving is not what is being checked. ydotoold + # creates an ordinary uinput device and the kernel publishes it as an + # evdev node, so reading that node returns the exact input_event structs + # ydotool wrote, with no compositor and no seat in the picture. + # + # That settles the click bitmasks, the split press / release edges drag + # depends on, what --absolute really puts on the wire, and the wheel + # signs this project had assumed from the kernel's REL_WHEEL convention + # and never measured. The last check drives the backend's own functions + # rather than a hand-written argv, so the two halves meet. + # + # --device covers /dev/uinput, which exists before the container starts. + # The input node does not — ydotoold creates it afterwards — so the + # cgroup rule grants character major 13 and nothing else, which is much + # narrower than --privileged. + # + # The container exits with the number of failed checks. + - name: Verify the ydotool argv against the kernel + run: | + docker run --rm \ + --device /dev/uinput \ + --device-cgroup-rule 'c 13:* rmw' \ + autocontrol-ydotool:ci diff --git a/docker/Dockerfile.eis b/docker/Dockerfile.eis new file mode 100644 index 00000000..8a4aabae --- /dev/null +++ b/docker/Dockerfile.eis @@ -0,0 +1,83 @@ +# AutoControl EIS verification image — libei talking to libeis, no desktop. +# +# The Wayland image (docker/Dockerfile.wayland) answers the capture half and +# closes by listing what it cannot: the libei *input* path needs a peer that +# speaks the EI protocol, and xdg-desktop-portal-wlr has no RemoteDesktop +# interface to provide one. That was recorded as needing a GNOME VM. +# +# It does not. libeis is the server side of the same protocol, Debian packages +# it, and the two libraries will talk to each other over a plain Unix socket. +# So this image runs a real EIS server (docker/eis_server.py) and drives +# AutoControl's real sender against it — which settles the capability and +# event-type enum values, the variadic ei_seat_bind_capabilities call, seat +# grants, device pause/resume, whether start_emulating -> frame actually puts +# events on the wire, the discrete-scroll sign, and the absolute pointer's +# coordinate space: that region offsets are part of the coordinate, and that +# libei drops a motion landing outside every region without saying a word. +# +# What it does not cover, and where that lives instead: +# * how a client *gets* this socket on GNOME and KDE — a file descriptor +# handed over D-Bus by org.freedesktop.portal.RemoteDesktop rather than a +# path. That was recorded here as needing a GNOME VM too; it does not, +# because the portal is a D-Bus interface rather than a compositor +# feature. docker/Dockerfile.portal owns the well-known name and runs the +# real liboeffis through the real handshake, ending in a live connection +# to this same server. +# +# ydotool used to be listed here too, as needing /dev/uinput "and a seat that +# consumes it". The seat was never the requirement: docker/Dockerfile.ydotool +# reads the injected events straight off the kernel device. +# +# Build: docker build -f docker/Dockerfile.eis -t autocontrol-eis:latest . +# Run: docker run --rm autocontrol-eis:latest + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# libei1 is the client this project binds; libeis1 is the server side of the +# same protocol and the whole reason this image exists. +# +# grim / wtype / wlr-randr are not used by anything below, but the Wayland +# backend refuses to import without them and `import je_auto_control` then +# falls back to the X11 backend — which dies reaching for DISPLAY=:0 before a +# line of this verification can run. +# +# libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libei1 libeis1 \ + grim wtype wlr-randr \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/eis_server.py /opt/verify/eis_server.py +COPY docker/eis_verify.py /opt/verify/eis_verify.py + +# Runs as root on purpose: one verification, then exit. Not a service image. +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/opt/verify \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=wayland + +ENTRYPOINT ["python", "/opt/verify/eis_verify.py"] diff --git a/docker/Dockerfile.portal b/docker/Dockerfile.portal new file mode 100644 index 00000000..664309f3 --- /dev/null +++ b/docker/Dockerfile.portal @@ -0,0 +1,98 @@ +# AutoControl portal verification image — the RemoteDesktop D-Bus handshake. +# +# Three images already answer three quarters of the Linux input and capture +# story: docker/Dockerfile.wayland (a real wlroots compositor), +# docker/Dockerfile.eis (a real libeis peer) and docker/Dockerfile.ydotool (a +# real uinput device). All three close by naming the same remaining gap in the +# same words — "the portal consent dialog and the EIS fd handover — that is +# xdg-desktop-portal + GNOME/mutter, not the EI protocol itself" — and it was +# recorded as needing a GNOME VM. +# +# It does not. The portal is a *D-Bus interface*, not a compositor feature: +# whatever owns org.freedesktop.portal.Desktop and answers CreateSession -> +# SelectDevices -> Start -> ConnectToEIS is a portal as far as liboeffis is +# concerned. docker/portal_server.py is that, on a private session bus, and +# its ConnectToEIS hands back a live connection to the same real EIS server +# the eis image uses. So the real liboeffis runs the real handshake, the +# descriptor it produces carries a real EI session, and the input emitted +# through it is recorded by an independent implementation. +# +# The one thing it still does not claim is the consent dialog itself — no user +# dismisses anything here. What a dialog produces is a Response code or +# silence, and docker/portal_verify.py drives a grant, a refusal, a dialog +# left open, a withheld descriptor, a closed session, a portal too old to have +# ConnectToEIS, and no portal at all. +# +# Two interpreters on purpose. Debian's python3 has GDBus, which is what can +# pass a file descriptor over D-Bus from Python; the image's own python3.12 +# has AutoControl. The portal has to be a separate process from its client in +# any case, so the split costs nothing. +# +# Build: docker build -f docker/Dockerfile.portal -t autocontrol-portal:latest . +# Run: docker run --rm autocontrol-portal:latest + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# liboeffis1 is the subject: the portal half of the libei handshake, and the +# library je_auto_control/linux_wayland/oeffis.py binds. Note it is a separate +# binary package that libei1 does not depend on — it is packaged on trixie, +# but installing libei1 alone does not bring it, which is the difference +# between the portal fast path being available and being quietly off. +# +# libei1 / libeis1 are the two ends of the EI protocol the descriptor carries. +# dbus supplies dbus-daemon, and python3-gi supplies the GDBus bindings the +# mock portal needs to put a file descriptor on the bus. +# +# Note what is deliberately NOT installed: gdbus. The Screenshot tier used to +# shell out to it, which could never work — the portal's answer is a signal +# directed at the connection that asked, and two gdbus processes are two +# connections. portal.py speaks D-Bus itself now, so this image is also the +# proof that the tier needs no binary beyond a session bus. +# +# grim / wtype / wlr-randr are not used by anything below, but the Wayland +# backend refuses to import without them and `import je_auto_control` then +# falls back to the X11 backend — which dies reaching for DISPLAY=:0 before a +# line of this verification can run. +# +# libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + liboeffis1 libei1 libeis1 \ + dbus python3-gi gir1.2-glib-2.0 \ + grim wtype wlr-randr \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/eis_server.py /opt/verify/eis_server.py +COPY docker/portal_server.py /opt/verify/portal_server.py +COPY docker/portal_verify.py /opt/verify/portal_verify.py + +# Runs as root on purpose: one verification, then exit. Not a service image. +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/opt/verify \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=wayland + +ENTRYPOINT ["python", "/opt/verify/portal_verify.py"] diff --git a/docker/Dockerfile.seat b/docker/Dockerfile.seat new file mode 100644 index 00000000..bb9c8179 --- /dev/null +++ b/docker/Dockerfile.seat @@ -0,0 +1,118 @@ +# AutoControl seat verification image — a wlroots compositor that really +# consumes the ydotool device. +# +# The four images before this one each stop at the same wall. +# docker/Dockerfile.wayland runs a real compositor but sets +# WLR_LIBINPUT_NO_DEVICES=1, so nothing is injected into it; +# docker/Dockerfile.ydotool reads ydotool's events off the kernel with no +# compositor at all. Between them, what `mousemove --absolute` puts on the +# wire is settled and what a compositor does with it was not — and that was +# recorded in Progress.md as needing a VM running a desktop that consumes +# libinput devices. +# +# It does not. A "seat that consumes libinput devices" is three environment +# variables away from the same headless sway the capture image already runs: +# +# WLR_BACKENDS=headless,libinput virtual outputs, real input backend +# LIBSEAT_BACKEND=builtin open the devices directly, no logind +# SEATD_VTBOUND=0 and without reaching for a VT +# +# udev is the fourth requirement and the one that is easy to miss: libinput +# enumerates devices through udev, not through /dev, so udevd has to be +# running before ydotoold creates its uinput device. The entrypoint starts +# them in that order and fails loudly if `libinput list-devices` cannot see +# the result, rather than letting sway come up with an empty seat and every +# measurement below silently become a measurement of nothing. +# +# What it then answers, in layout coordinates read back from pixels the +# compositor painted (`grim -c` composites the cursor): +# * that --absolute counts from the top-left corner of the output layout, +# which is layout (0, 0) only while no monitor sits left of or above the +# primary one — the translation je_auto_control/linux_wayland/mouse.py +# now applies, measured rather than assumed; +# * that the displacement is relative motion and so is scaled by the +# compositor's pointer acceleration, which is what ydotool's own --help +# means by "You need to disable mouse speed acceleration for correct +# absolute movement"; +# * that with that acceleration off, AutoControl's own set_position lands +# on the layout pixel it was given, on both outputs, and that get_pixel +# reads the same pixel back. +# +# Build: docker build -f docker/Dockerfile.seat -t autocontrol-seat:latest . +# Run: modprobe uinput evdev +# docker run --rm --device /dev/uinput \ +# --device-cgroup-rule 'c 13:* rmw' autocontrol-seat:latest + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# ydotool 1.0.4 comes from unstable for the reason docker/Dockerfile.ydotool +# gives: trixie ships none and bookworm's 0.1.8 answers this argv with exit +# code 0 and no events. The pin keeps every other package on trixie. +# +# - sway + wlr-randr + grim: the compositor and the two tools this reads it +# with. grim's -c is what draws the cursor into the capture. +# - udev: systemd-udevd, so libinput can enumerate the device at all. +# - libinput-tools: the entrypoint's own precondition check, before sway. +# - dmz-cursor-theme: a cursor that is certain to be drawn. libwayland has a +# built-in fallback, but a theme on disk makes the pixels this measures the +# compositor's decision rather than a fallback's. +# - wtype: unused here, but the Wayland backend refuses to import without it +# and `import je_auto_control` then falls back to X11 and dies on DISPLAY. +# - libgl1 + libglib2.0-0: opencv-python's hard import-time requirements. +RUN printf 'deb http://deb.debian.org/debian sid main\n' \ + > /etc/apt/sources.list.d/sid.list \ + && printf 'Package: *\nPin: release a=unstable\nPin-Priority: 100\n' \ + > /etc/apt/preferences.d/no-sid-by-default \ + && apt-get update \ + && apt-get install -y --no-install-recommends -t sid ydotool \ + && apt-get install -y --no-install-recommends \ + sway grim wtype wlr-randr \ + udev libinput-tools dmz-cursor-theme \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/seat_verify.py /opt/verify/seat_verify.py +COPY docker/entrypoint-seat.sh /usr/local/bin/autocontrol-seat-verify +RUN chmod +x /usr/local/bin/autocontrol-seat-verify + +# Runs as root on purpose: creating the input node, opening it as a seat +# device and writing to /dev/uinput all need it, and this image exists to run +# one verification and exit. It is not a deployable service image — +# docker/Dockerfile is. +# +# The CLI input path is forced: libei would otherwise be preferred, and this +# image is entirely about the ydotool fallback. +ENV PYTHONUNBUFFERED=1 \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + YDOTOOL_SOCKET=/tmp/.ydotool_socket \ + JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli \ + WLR_BACKENDS=headless,libinput \ + WLR_LIBINPUT_NO_DEVICES=1 \ + WLR_RENDERER=pixman \ + WLR_HEADLESS_OUTPUTS=2 \ + LIBSEAT_BACKEND=builtin \ + SEATD_VTBOUND=0 + +ENTRYPOINT ["/usr/local/bin/autocontrol-seat-verify"] diff --git a/docker/Dockerfile.wayland b/docker/Dockerfile.wayland new file mode 100644 index 00000000..2750622d --- /dev/null +++ b/docker/Dockerfile.wayland @@ -0,0 +1,88 @@ +# AutoControl Wayland verification image — a real wlroots compositor, headless. +# +# The Wayland backend's CLI argv, the wlr-randr output format and the whole +# screenshot -> screen_grabber -> capture -> grab_image chain had only ever +# been exercised against mocks: no CI runner offers a Wayland session, and a +# developer machine running Windows cannot provide one either. sway's headless +# backend needs no GPU, no seat and no display, so it runs in a container and +# answers those questions for real. +# +# The entrypoint runs the verification over two output layouts: the outputs +# side by side from the origin, and the left-hand one moved to x=-1280. sway +# accepts a negative `position` and grim accepts a negative `-g`, so the +# negative-origin layout every multi-monitor desktop has is a real compositor +# answering rather than a second mock. +# +# What this image CANNOT answer, and why: +# * ydotool — needs /dev/uinput, and sway's headless backend consumes no +# libinput devices, so an injected event would have nowhere to arrive. +# Arriving turned out not to be what needed checking: docker/ +# Dockerfile.ydotool reads the events back off the kernel device ydotoold +# creates, with no compositor and no seat involved. +# * libei / the RemoteDesktop portal — xdg-desktop-portal-wlr implements +# ScreenCast and Screenshot but not RemoteDesktop, so there is no +# ConnectToEIS here. That was read as needing a GNOME (mutter) session, +# and it does not: the protocol half is docker/Dockerfile.eis and the +# portal half is docker/Dockerfile.portal, which owns the well-known +# D-Bus name itself rather than waiting for a desktop to provide one. +# +# Build: docker build -f docker/Dockerfile.wayland -t autocontrol-wayland:latest . +# Run: docker run --rm autocontrol-wayland:latest + +# Build every wheel in a throwaway stage, as docker/Dockerfile does, so the +# runtime layer installs binaries only and the dependency set is fixed here +# rather than re-resolved against PyPI at run time. +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# - sway: the wlroots compositor. WLR_BACKENDS=headless gives it a virtual +# output with no DRM device and no seat. +# - grim / wtype / wlr-randr: the three wlroots-protocol helpers the Wayland +# backend shells out to, and the actual subjects of this verification. +# - libgl1 + libglib2.0-0: opencv-python hard-requires libGL.so.1 and +# libgthread-2.0.so.0 at import, so the package cannot even load without them. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + sway grim wtype wlr-randr \ + libei1 \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/wayland_verify.py /opt/verify/wayland_verify.py +COPY docker/libei_verify.py /opt/verify/libei_verify.py +COPY docker/entrypoint-wayland.sh /usr/local/bin/autocontrol-wayland-verify +RUN chmod +x /usr/local/bin/autocontrol-wayland-verify + +# Runs as root on purpose: this image exists to run one verification and +# exit, and root removes the XDG_RUNTIME_DIR ownership fiddling that would +# otherwise be the only thing standing between here and an answer. It is not +# a deployable service image — docker/Dockerfile is. +ENV PYTHONUNBUFFERED=1 \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + WLR_BACKENDS=headless \ + WLR_LIBINPUT_NO_DEVICES=1 \ + WLR_RENDERER=pixman \ + WLR_HEADLESS_OUTPUTS=2 + +ENTRYPOINT ["/usr/local/bin/autocontrol-wayland-verify"] diff --git a/docker/Dockerfile.ydotool b/docker/Dockerfile.ydotool new file mode 100644 index 00000000..fab3098a --- /dev/null +++ b/docker/Dockerfile.ydotool @@ -0,0 +1,102 @@ +# AutoControl ydotool verification image — real /dev/uinput, no compositor. +# +# The Wayland image (docker/Dockerfile.wayland) answers the capture half and +# the EIS image (docker/Dockerfile.eis) answers libei's. Both close by naming +# the same remaining gap in the same words — ydotool "needs /dev/uinput and a +# seat that consumes it" — and that was recorded as needing a GNOME VM. +# +# It does not. A seat is what makes an injected event *arrive somewhere*; it +# is not what makes it *observable*. ydotoold creates an ordinary uinput +# device, the kernel publishes it as /dev/input/eventN, and reading that node +# returns the exact input_event structs ydotool wrote. So this image runs +# AutoControl's real mouse and keyboard backends and reads back what the +# kernel got: the click bitmasks, the split press / release edges that drag +# depends on, what --absolute actually puts on the wire, the wheel signs this +# project had only ever assumed, and numeric key codes. +# +# Why ydotool comes from unstable: 1.0 replaced the entire command line, and +# every argument this backend builds arrived in it. Trixie ships no ydotool +# at all and bookworm ships 0.1.8, which answers this argv with exit code 0 +# and no events — see je_auto_control/linux_wayland/_ydotool_cli.py. Pulling +# one package from sid, pinned so nothing else follows it, is what gets the +# CLI under test onto the same Debian base as the other two images. +# +# What this image still cannot answer, and why: +# * whether a compositor clamps the INT32_MIN reset that --absolute relies +# on. The kernel side of that is checked here; the clamp is the +# compositor's behaviour, and it remains open in Progress.md rather than +# being quietly claimed. +# The portal consent dialog and the EIS fd handover used to be listed here +# too. They are xdg-desktop-portal's, not ydotool's, and they are now covered +# by docker/Dockerfile.portal against the real liboeffis. +# +# Build: docker build -f docker/Dockerfile.ydotool -t autocontrol-ydotool:latest . +# Run: modprobe uinput evdev +# docker run --rm --device /dev/uinput \ +# --device-cgroup-rule 'c 13:* rmw' autocontrol-ydotool:latest +# +# The cgroup rule is what lets the container open the input node ydotoold +# creates *after* it starts, which --device cannot cover because the node does +# not exist yet at run time. It grants character major 13 (input) and nothing +# else, which is a good deal narrower than --privileged. + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# ydotool 1.0.4 comes from unstable; the pin keeps every other package on +# trixie, so this is one binary and its already-satisfied dependencies rather +# than a half-upgraded base. +# +# grim / wtype / wlr-randr are not used by anything below, but the Wayland +# backend refuses to import without them and `import je_auto_control` then +# falls back to the X11 backend — which dies reaching for DISPLAY=:0 before a +# line of this verification can run. +# +# libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. +RUN printf 'deb http://deb.debian.org/debian sid main\n' \ + > /etc/apt/sources.list.d/sid.list \ + && printf 'Package: *\nPin: release a=unstable\nPin-Priority: 100\n' \ + > /etc/apt/preferences.d/no-sid-by-default \ + && apt-get update \ + && apt-get install -y --no-install-recommends -t sid ydotool \ + && apt-get install -y --no-install-recommends \ + grim wtype wlr-randr \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/ydotool_verify.py /opt/verify/ydotool_verify.py + +# Runs as root on purpose: writing to /dev/uinput and creating the input node +# both need it, and this image exists to run one verification and exit. It is +# not a deployable service image — docker/Dockerfile is. +# +# The Wayland backend is selected by XDG_SESSION_TYPE alone here: there is no +# compositor, and none is needed, because every function under test shells out +# to ydotool rather than talking to a Wayland socket. The CLI path is forced +# because libei would otherwise win and this image is about the fallback. +ENV PYTHONUNBUFFERED=1 \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli + +ENTRYPOINT ["python3", "/opt/verify/ydotool_verify.py"] diff --git a/docker/eis_server.py b/docker/eis_server.py new file mode 100644 index 00000000..9a7bd6ec --- /dev/null +++ b/docker/eis_server.py @@ -0,0 +1,431 @@ +"""A minimal EIS server, bound to the real ``libeis.so`` with ctypes. + +AutoControl's libei sender had been verified as far as a peer-less container +can go: every prototype resolves, every call behaves, and the whole +fail-closed chain runs. What could not be checked without something that +speaks the protocol was the half where a wrong value is silently accepted — +the capability and event-type enums, the variadic +``ei_seat_bind_capabilities`` marshalling, seat capability grants, device +pause/resume, and whether ``start_emulating`` → event → ``frame`` actually +puts anything on the wire. + +``libeis`` is the server side of the same protocol and Debian packages it +(``libeis1`` / ``libeis-dev``), so that peer can just be built here: this +module runs a real EIS implementation on a Unix socket, in a thread, and +records everything a client does to it. No compositor, no desktop session, no +GNOME VM — the two libraries talk to each other and the recording is the +evidence. + +It is a verification fixture, not shipped library code, which is why it lives +under ``docker/`` next to the image that runs it. +""" +from __future__ import annotations + +import ctypes +import select +import threading +from typing import Dict, List, Optional, Sequence, Set, Tuple + +from je_auto_control.linux_wayland._ctypes_bind import BoundSymbols, bind + +_LIBRARY_CANDIDATES = ("eis", "libeis", "libeis.so.1", "libeis.so.0") + +# enum eis_device_capability — the server side of the same bitmask libei.py +# declares. Deliberately re-declared from libeis.h rather than imported from +# libei.py: if both sides read the same wrong constant the test passes while +# the protocol is broken, and this file exists to catch exactly that. +EIS_DEVICE_CAP_POINTER = 1 << 0 +EIS_DEVICE_CAP_POINTER_ABSOLUTE = 1 << 1 +EIS_DEVICE_CAP_KEYBOARD = 1 << 2 +EIS_DEVICE_CAP_TOUCH = 1 << 3 +EIS_DEVICE_CAP_SCROLL = 1 << 4 +EIS_DEVICE_CAP_BUTTON = 1 << 5 + +CAPABILITY_NAMES = { + EIS_DEVICE_CAP_POINTER: "POINTER", + EIS_DEVICE_CAP_POINTER_ABSOLUTE: "POINTER_ABSOLUTE", + EIS_DEVICE_CAP_KEYBOARD: "KEYBOARD", + EIS_DEVICE_CAP_TOUCH: "TOUCH", + EIS_DEVICE_CAP_SCROLL: "SCROLL", + EIS_DEVICE_CAP_BUTTON: "BUTTON", +} + +# enum eis_event_type, from libeis.h. +EIS_EVENT_CLIENT_CONNECT = 1 +EIS_EVENT_CLIENT_DISCONNECT = 2 +EIS_EVENT_SEAT_BIND = 3 +EIS_EVENT_DEVICE_CLOSED = 4 +EIS_EVENT_FRAME = 100 +EIS_EVENT_DEVICE_START_EMULATING = 200 +EIS_EVENT_DEVICE_STOP_EMULATING = 201 +EIS_EVENT_POINTER_MOTION = 300 +EIS_EVENT_POINTER_MOTION_ABSOLUTE = 400 +EIS_EVENT_BUTTON_BUTTON = 500 +EIS_EVENT_SCROLL_DISCRETE = 603 +EIS_EVENT_KEYBOARD_KEY = 700 + +#: What this server offers a seat. Wider than AutoControl asks for on +#: purpose — a client that binds a capability it never requested, or misses +#: one it did, then shows up in ``bound_capabilities``. +OFFERED_CAPABILITIES = ( + EIS_DEVICE_CAP_POINTER, EIS_DEVICE_CAP_POINTER_ABSOLUTE, + EIS_DEVICE_CAP_KEYBOARD, EIS_DEVICE_CAP_TOUCH, + EIS_DEVICE_CAP_SCROLL, EIS_DEVICE_CAP_BUTTON, +) + +#: The absolute pointer needs a region or the compositor has no coordinate +#: space to place motion in. +REGION_SIZE = (1920, 1080) + +#: Default shape of that space: one region at the origin, as ``((x, y), +#: (width, height))``. ``regions=`` overrides it, which is how the offset +#: layout that a monitor left of the primary produces gets exercised. +DEFAULT_REGIONS = (((0, 0), REGION_SIZE),) + +_VOID = ctypes.c_void_p +_U32 = ctypes.c_uint32 +_PROTOTYPES = ( + ("eis_new", _VOID, (_VOID,)), + ("eis_unref", _VOID, (_VOID,)), + ("eis_setup_backend_socket", ctypes.c_int, (_VOID, ctypes.c_char_p)), + ("eis_get_fd", ctypes.c_int, (_VOID,)), + ("eis_dispatch", None, (_VOID,)), + ("eis_get_event", _VOID, (_VOID,)), + ("eis_event_unref", _VOID, (_VOID,)), + ("eis_event_get_type", ctypes.c_int, (_VOID,)), + ("eis_event_get_client", _VOID, (_VOID,)), + ("eis_event_get_seat", _VOID, (_VOID,)), + ("eis_event_get_device", _VOID, (_VOID,)), + ("eis_event_seat_has_capability", ctypes.c_bool, (_VOID, ctypes.c_int)), + ("eis_client_connect", None, (_VOID,)), + ("eis_client_disconnect", None, (_VOID,)), + ("eis_client_is_sender", ctypes.c_bool, (_VOID,)), + ("eis_client_get_name", ctypes.c_char_p, (_VOID,)), + ("eis_client_new_seat", _VOID, (_VOID, ctypes.c_char_p)), + ("eis_seat_configure_capability", None, (_VOID, ctypes.c_int)), + ("eis_seat_add", None, (_VOID,)), + ("eis_seat_new_device", _VOID, (_VOID,)), + ("eis_device_configure_name", None, (_VOID, ctypes.c_char_p)), + ("eis_device_configure_capability", None, (_VOID, ctypes.c_int)), + ("eis_device_new_region", _VOID, (_VOID,)), + ("eis_region_set_size", None, (_VOID, _U32, _U32)), + ("eis_region_set_offset", None, (_VOID, _U32, _U32)), + ("eis_region_add", None, (_VOID,)), + ("eis_device_add", None, (_VOID,)), + ("eis_device_remove", None, (_VOID,)), + ("eis_device_pause", None, (_VOID,)), + ("eis_device_resume", None, (_VOID,)), + ("eis_device_get_name", ctypes.c_char_p, (_VOID,)), + ("eis_event_emulating_get_sequence", _U32, (_VOID,)), + ("eis_event_keyboard_get_key", _U32, (_VOID,)), + ("eis_event_keyboard_get_key_is_press", ctypes.c_bool, (_VOID,)), + ("eis_event_pointer_get_absolute_x", ctypes.c_double, (_VOID,)), + ("eis_event_pointer_get_absolute_y", ctypes.c_double, (_VOID,)), + ("eis_event_button_get_button", _U32, (_VOID,)), + ("eis_event_button_get_is_press", ctypes.c_bool, (_VOID,)), + ("eis_event_scroll_get_discrete_dx", ctypes.c_int32, (_VOID,)), + ("eis_event_scroll_get_discrete_dy", ctypes.c_int32, (_VOID,)), +) + + +class EisUnavailable(RuntimeError): + """libeis is missing, or a server cannot be brought up here.""" + + +def load_symbols() -> Optional[BoundSymbols]: + """Resolve every libeis entry point, or None if one is missing.""" + return bind(_LIBRARY_CANDIDATES, _PROTOTYPES) + + +class Recording: + """Everything one client did, as the server saw it happen.""" + + def __init__(self) -> None: + self.clients: List[str] = [] + self.sender_flags: List[bool] = [] + self.bound_capabilities: Set[int] = set() + self.seat_binds = 0 + self.devices: List[str] = [] + self.emulating_sequences: List[Tuple[str, int]] = [] + self.stopped_emulating = 0 + self.frames = 0 + self.keys: List[Tuple[int, bool]] = [] + self.absolute_motions: List[Tuple[float, float]] = [] + self.buttons: List[Tuple[int, bool]] = [] + self.scrolls: List[Tuple[int, int]] = [] + self.disconnects = 0 + #: Device labels this server asked to pause, for diagnosis. + self.paused: List[str] = [] + #: ``(event type, device name)`` for everything, in arrival order — + #: what a failing check needs to explain itself. + self.event_log: List[Tuple[int, str]] = [] + + def capability_names(self) -> List[str]: + return sorted(CAPABILITY_NAMES.get(cap, str(cap)) + for cap in self.bound_capabilities) + + +class RecordingEisServer: + """A real EIS server that grants a seat and records what arrives. + + Runs its whole libeis context on one background thread: libeis is not + thread-safe, and the client under test drives the main thread, so the two + contexts never share one. + """ + + def __init__(self, socket_path: str, + *, symbols: Optional[BoundSymbols] = None, + regions: Optional[Sequence[Tuple[Tuple[int, int], + Tuple[int, int]]]] = None, + ) -> None: + self.socket_path = socket_path + self.recording = Recording() + self.regions = tuple(DEFAULT_REGIONS if regions is None else regions) + self._symbols = symbols if symbols is not None else load_symbols() + self._eis: Optional[int] = None + self._devices: Dict[str, int] = {} + self._seat_devices_created = False + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + self._error: Optional[BaseException] = None + #: Work handed to the serving thread; a list because append and pop + #: are atomic and this needs no more locking than that. + self._pending: List[tuple] = [] + + # --- lifecycle -------------------------------------------------------- + + def start(self) -> None: + """Bring the server up and begin answering clients.""" + if self._symbols is None: + raise EisUnavailable("libeis.so.* not found on the loader path") + context = self._symbols.eis_new(None) + if not context: + raise EisUnavailable("eis_new returned NULL") + self._eis = context + code = self._symbols.eis_setup_backend_socket( + context, self.socket_path.encode("utf-8")) + if code != 0: + raise EisUnavailable(f"eis_setup_backend_socket returned {code}") + self._thread = threading.Thread(target=self._serve, daemon=True, + name="eis-server") + self._thread.start() + + def stop(self, timeout: float = 2.0) -> None: + """Ask the loop to finish and wait for it.""" + self._stop.set() + if self._thread is not None: + self._thread.join(timeout) + self._thread = None + + @property + def error(self) -> Optional[BaseException]: + """Whatever took the serving thread down, if anything did.""" + return self._error + + def pause_devices(self) -> None: + """Suspend every device, so a client must stop emulating.""" + self._on_server_thread(self._pause_all) + + def _pause_all(self) -> None: + for label, device in self._devices.items(): + name = self._symbols.eis_device_get_name(device) + self.recording.paused.append( + f"{label}={name.decode() if name else ''}") + self._symbols.eis_device_pause(device) + # libeis buffers outgoing messages; a dispatch is what puts them on + # the wire, and nothing else here would trigger one while the client + # is only *sending*. + self._symbols.eis_dispatch(self._eis) + + def resume_devices(self) -> None: + """Resume every device, so a client may emulate again.""" + self._on_server_thread( + lambda: [self._symbols.eis_device_resume(device) + for device in self._devices.values()]) + + def _on_server_thread(self, action, timeout: float = 2.0) -> None: + """Run ``action`` on the serving thread and wait for it. + + libeis is not thread-safe, and every other call into this context + already happens on that thread. Pausing a device from the caller's + thread appeared to work and then simply never reached the client — + which reads exactly like the client ignoring the event, so it is + worth not being able to make that mistake. + """ + done = threading.Event() + self._pending.append((action, done)) + if not done.wait(timeout): + raise EisUnavailable("the server thread did not run a queued action") + + # --- event loop ------------------------------------------------------- + + def _serve(self) -> None: + try: + poll_fd = int(self._symbols.eis_get_fd(self._eis)) + while not self._stop.is_set(): + self._run_pending() + ready, _, _ = select.select([poll_fd], [], [], 0.05) + if not ready: + continue + self._symbols.eis_dispatch(self._eis) + self._drain() + except BaseException as error: # noqa: BLE001 # reason: a fixture thread must report, not vanish + self._error = error + + def _run_pending(self) -> None: + while self._pending: + action, done = self._pending.pop(0) + try: + action() + finally: + done.set() + + def _drain(self) -> None: + while True: + event = self._symbols.eis_get_event(self._eis) + if not event: + return + try: + self._on_event(event) + finally: + self._symbols.eis_event_unref(event) + + def _on_event(self, event: int) -> None: + event_type = int(self._symbols.eis_event_get_type(event)) + self.recording.event_log.append((event_type, self._device_name(event))) + handler = _HANDLERS.get(event_type) + if handler is not None: + handler(self, event) + + def _device_name(self, event: int) -> str: + """The device an event belongs to, for the log. ``''`` when it has none.""" + device = self._symbols.eis_event_get_device(event) + if not device: + return "" + name = self._symbols.eis_device_get_name(device) + return name.decode("utf-8", "replace") if name else f"<{device:#x}>" + + # --- handlers --------------------------------------------------------- + + def _on_client_connect(self, event: int) -> None: + """Accept a sender client and offer it one fully-capable seat.""" + client = self._symbols.eis_event_get_client(event) + name = self._symbols.eis_client_get_name(client) + self.recording.clients.append( + name.decode("utf-8", "replace") if name else "") + self.recording.sender_flags.append( + bool(self._symbols.eis_client_is_sender(client))) + self._symbols.eis_client_connect(client) + seat = self._symbols.eis_client_new_seat(client, b"autocontrol-seat") + for capability in OFFERED_CAPABILITIES: + self._symbols.eis_seat_configure_capability(seat, capability) + self._symbols.eis_seat_add(seat) + + def _on_seat_bind(self, event: int) -> None: + """Record what the client asked this seat for, then hand it devices. + + This is the whole point of the fixture. ``eis_event_seat_has_capability`` + reads back what the client's variadic ``ei_seat_bind_capabilities`` + actually put on the wire, so a wrong enum value or a mis-marshalled + argument shows up as a missing or unexpected capability rather than as + a handshake that quietly never completes. + """ + self.recording.seat_binds += 1 + seat = self._symbols.eis_event_get_seat(event) + bound = {cap for cap in OFFERED_CAPABILITIES + if self._symbols.eis_event_seat_has_capability(event, cap)} + self.recording.bound_capabilities |= bound + if not bound or self._seat_devices_created: + return + self._seat_devices_created = True + if EIS_DEVICE_CAP_KEYBOARD in bound: + self._add_device("keyboard", seat, (EIS_DEVICE_CAP_KEYBOARD,)) + pointer_caps = tuple( + cap for cap in (EIS_DEVICE_CAP_POINTER_ABSOLUTE, + EIS_DEVICE_CAP_BUTTON, EIS_DEVICE_CAP_SCROLL) + if cap in bound) + if pointer_caps: + self._add_device("pointer", seat, pointer_caps, region=True) + + def _add_device(self, label: str, seat: int, capabilities: tuple, + *, region: bool = False) -> None: + device = self._symbols.eis_seat_new_device(seat) + self._symbols.eis_device_configure_name( + device, f"autocontrol-{label}".encode("utf-8")) + for capability in capabilities: + self._symbols.eis_device_configure_capability(device, capability) + if region: + for offset, size in self.regions: + shape = self._symbols.eis_device_new_region(device) + self._symbols.eis_region_set_offset(shape, *offset) + self._symbols.eis_region_set_size(shape, *size) + self._symbols.eis_region_add(shape) + self._symbols.eis_device_add(device) + self._symbols.eis_device_resume(device) + self._devices[label] = device + self.recording.devices.append(label) + + def _on_start_emulating(self, event: int) -> None: + device = self._symbols.eis_event_get_device(event) + name = self._symbols.eis_device_get_name(device) if device else None + self.recording.emulating_sequences.append(( + name.decode("utf-8", "replace") if name else "", + int(self._symbols.eis_event_emulating_get_sequence(event)), + )) + + def _on_stop_emulating(self, _event: int) -> None: + self.recording.stopped_emulating += 1 + + def _on_frame(self, _event: int) -> None: + self.recording.frames += 1 + + def _on_keyboard_key(self, event: int) -> None: + self.recording.keys.append(( + int(self._symbols.eis_event_keyboard_get_key(event)), + bool(self._symbols.eis_event_keyboard_get_key_is_press(event)), + )) + + def _on_absolute_motion(self, event: int) -> None: + self.recording.absolute_motions.append(( + float(self._symbols.eis_event_pointer_get_absolute_x(event)), + float(self._symbols.eis_event_pointer_get_absolute_y(event)), + )) + + def _on_button(self, event: int) -> None: + self.recording.buttons.append(( + int(self._symbols.eis_event_button_get_button(event)), + bool(self._symbols.eis_event_button_get_is_press(event)), + )) + + def _on_scroll_discrete(self, event: int) -> None: + self.recording.scrolls.append(( + int(self._symbols.eis_event_scroll_get_discrete_dx(event)), + int(self._symbols.eis_event_scroll_get_discrete_dy(event)), + )) + + def _on_client_disconnect(self, _event: int) -> None: + self.recording.disconnects += 1 + + +_HANDLERS = { + EIS_EVENT_CLIENT_CONNECT: RecordingEisServer._on_client_connect, + EIS_EVENT_CLIENT_DISCONNECT: RecordingEisServer._on_client_disconnect, + EIS_EVENT_SEAT_BIND: RecordingEisServer._on_seat_bind, + EIS_EVENT_DEVICE_START_EMULATING: RecordingEisServer._on_start_emulating, + EIS_EVENT_DEVICE_STOP_EMULATING: RecordingEisServer._on_stop_emulating, + EIS_EVENT_FRAME: RecordingEisServer._on_frame, + EIS_EVENT_KEYBOARD_KEY: RecordingEisServer._on_keyboard_key, + EIS_EVENT_POINTER_MOTION_ABSOLUTE: RecordingEisServer._on_absolute_motion, + EIS_EVENT_BUTTON_BUTTON: RecordingEisServer._on_button, + EIS_EVENT_SCROLL_DISCRETE: RecordingEisServer._on_scroll_discrete, +} + + +__all__ = [ + "CAPABILITY_NAMES", "EisUnavailable", "OFFERED_CAPABILITIES", + "DEFAULT_REGIONS", "REGION_SIZE", + "Recording", "RecordingEisServer", "load_symbols", + "EIS_DEVICE_CAP_POINTER", "EIS_DEVICE_CAP_POINTER_ABSOLUTE", + "EIS_DEVICE_CAP_KEYBOARD", "EIS_DEVICE_CAP_TOUCH", + "EIS_DEVICE_CAP_SCROLL", "EIS_DEVICE_CAP_BUTTON", +] diff --git a/docker/eis_verify.py b/docker/eis_verify.py new file mode 100644 index 00000000..b4114a50 --- /dev/null +++ b/docker/eis_verify.py @@ -0,0 +1,644 @@ +"""Verify AutoControl's libei sender against a real EIS peer. + +``libei_verify.py`` goes as far as a peer-less container can: every prototype +resolves, every call behaves, and the fail-closed chain runs end to end. It +closes by listing what it cannot answer — the capability and event-type enum +values, and whether the variadic ``ei_seat_bind_capabilities`` call is +marshalled correctly — because those need something that speaks the protocol. + +``docker/eis_server.py`` is that something. With a real EIS implementation on +the other end of the socket, every one of those questions has an answer, and +they are answered here: + + * the seat records exactly the four capabilities AutoControl asks for, so + both the ``EI_DEVICE_CAP_*`` bitmask values and the variadic call are + right — a wrong value binds the wrong capability or none at all; + * ``SEAT_ADDED`` / ``DEVICE_ADDED`` / ``DEVICE_RESUMED`` are recognised, so + the ``EI_EVENT_*`` values are right — a wrong one leaves the handshake + waiting until it times out; + * ``start_emulating`` → event → ``frame`` puts real events on the wire, with + the key codes, coordinates, button codes and scroll signs intended — + including the axis flip ``mouse.scroll()`` applies between the kernel's + ``REL_WHEEL`` frame and libei's, and the negative value that flip makes; + * tearing down a *live* context is finally testable — it is safe, which is + what lets ``_teardown`` release a completed session instead of leaking a + context and an fd per process, and this file is the sentinel for that. + +Two things it measures but cannot settle, and says so rather than claiming +either way: ``eis_device_pause()`` puts nothing on the wire for a sender +client on libeis 1.3.901, so ``DEVICE_PAUSED`` handling still has no peer to +drive it; and the ``start_emulating`` sequence number does not survive the +trip, so the client's counter cannot be read back from this side. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import faulthandler +import os +import subprocess # nosec B404 # reason: runs this interpreter to isolate a known segfault +import sys +import time +import traceback +from typing import Any, Callable, List, Tuple + +# A wrong prototype in a ctypes binding is a segfault, not an exception. +faulthandler.enable() + +_results: List[Tuple[str, bool]] = [] + +#: evdev codes the emission checks use. KEY_A, BTN_LEFT. +KEY_A = 30 +BTN_LEFT = 272 +TARGET_POSITION = (640, 400) + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _wait_for(predicate: Callable[[], bool], timeout: float, + description: str) -> None: + """Spin until the server thread has recorded something, or give up.""" + import time + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.02) + raise AssertionError(f"timed out waiting for {description}") + + +def _socket_path() -> str: + runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") # nosec B108 # reason: container fallback only + os.makedirs(runtime, exist_ok=True) + path = os.path.join(runtime, "eis-verify") + if os.path.exists(path): + os.unlink(path) + return path + + +def _connected_backend(libei, server, path): + """Drive a real handshake to completion against the recording server.""" + backend = libei.LibeiBackend() + backend.connect(timeout=5.0, socket_path=path.encode("utf-8")) + _require(backend.is_connected, + "connect() returned but the backend reports no live device") + _require(server.error is None, f"the server thread died: {server.error!r}") + return backend + + +def _check_handshake(server) -> str: + record = server.recording + _require(record.seat_binds >= 1, "the client never bound a seat") + _require(all(record.sender_flags), + "the client did not present itself as a sender") + return (f"client={record.clients[0]!r} seat binds={record.seat_binds} " + f"devices={record.devices}") + + +def _check_capabilities(server, libei) -> str: + """The enum values and the variadic bind call, read back off the wire.""" + from eis_server import CAPABILITY_NAMES + expected = set(libei._WANTED_CAPS) + bound = server.recording.bound_capabilities + _require(bound == expected, ( + "the seat was bound with " + f"{sorted(CAPABILITY_NAMES.get(c, c) for c in bound)} but AutoControl " + f"asks for {sorted(CAPABILITY_NAMES.get(c, c) for c in expected)} — " + "either an EI_DEVICE_CAP_* value is wrong or the variadic " + "ei_seat_bind_capabilities call is mis-marshalled")) + return ", ".join(server.recording.capability_names()) + + +def _check_keyboard(backend, server) -> str: + backend.press_key(KEY_A) + backend.release_key(KEY_A) + _wait_for(lambda: len(server.recording.keys) >= 2, 2.0, "two key events") + _require(server.recording.keys[:2] == [(KEY_A, True), (KEY_A, False)], + f"the server saw {server.recording.keys[:2]}") + return f"press+release of key {KEY_A} arrived in order" + + +def _check_pointer(backend, server) -> str: + backend.set_position(*TARGET_POSITION) + _wait_for(lambda: server.recording.absolute_motions, 2.0, "absolute motion") + got = server.recording.absolute_motions[-1] + _require(got == TARGET_POSITION, + f"asked for {TARGET_POSITION}, the server saw {got}") + return f"absolute motion landed on {got}" + + +def _check_button(backend, server) -> str: + backend.click_button(BTN_LEFT) + _wait_for(lambda: len(server.recording.buttons) >= 2, 2.0, "button events") + _require(server.recording.buttons[:2] == [(BTN_LEFT, True), + (BTN_LEFT, False)], + f"the server saw {server.recording.buttons[:2]}") + return f"press+release of button {BTN_LEFT} arrived in order" + + +#: libei.h: "A discrete scroll event is based logical scroll units (equivalent +#: to one mouse wheel click). The value for one scroll unit is 120." +SCROLL_UNIT = 120 + + +def _check_scroll_unit(backend, server) -> str: + """One wheel click has to arrive as one wheel click. + + This is the path Progress.md left deliberately unwired because the sign + was a guess. The sign turns out to be the smaller half of the question: + libei measures discrete scroll in 120ths of a click, so a raw detent count + is 1/120th of a scroll and libei logs it as a client bug. + """ + backend.scroll(0, 1) + _wait_for(lambda: server.recording.scrolls, 2.0, "a discrete scroll") + got = server.recording.scrolls[-1] + _require(got == (0, SCROLL_UNIT), ( + f"scroll(0, 1) — one wheel click down — reached the server as {got}, " + f"but libei measures discrete scroll in units of {SCROLL_UNIT} per " + "click, so this is 1/120th of the scroll that was asked for")) + return (f"scroll(0, 1) arrives as {got}: one click, positive on the y " + "axis, no axis swap") + + +def _check_scroll_wiring(backend, server) -> str: + """The public ``mouse.scroll()`` has to land the axis flip on the wire. + + ``_check_scroll_unit`` drives ``LibeiBackend.scroll`` directly, so it says + nothing about the frame conversion the mouse module does on top: this + repository's ``wayland_scroll_direction_*`` constants are in the kernel's + ``REL_WHEEL`` frame (positive is up, which is what ydotool writes) and + libei is in the ``wl_pointer`` frame (positive is down), so the vertical + axis is negated on the way out and the horizontal one is not. + + Running it through the real server is also the only place a *negative* + discrete value is put on the wire — the direct check only ever sent a + positive one, so a marshalling fault on the sign would have gone unseen. + """ + from unittest.mock import patch + + from je_auto_control.linux_wayland import mouse as wayland_mouse + + def _next_scroll(call) -> tuple: + before = len(server.recording.scrolls) + call() + _wait_for(lambda: len(server.recording.scrolls) > before, 2.0, + "a discrete scroll from mouse.scroll()") + return server.recording.scrolls[-1] + + with patch.object(wayland_mouse, "_try_libei", return_value=backend): + up = _next_scroll(lambda: wayland_mouse.scroll( + 1, wayland_mouse.wayland_scroll_direction_up)) + down = _next_scroll(lambda: wayland_mouse.scroll( + 1, wayland_mouse.wayland_scroll_direction_down)) + right = _next_scroll(lambda: wayland_mouse.scroll( + 1, wayland_mouse.wayland_scroll_direction_right)) + + _require(up == (0, -SCROLL_UNIT), ( + f"scrolling up reached the server as {up}, not (0, {-SCROLL_UNIT}) — " + "libei counts positive y as down, so up has to arrive negative")) + _require(down == (0, SCROLL_UNIT), + f"scrolling down reached the server as {down}") + _require(right == (SCROLL_UNIT, 0), ( + f"scrolling right reached the server as {right} — the horizontal " + "axis must not be flipped, both frames count right as positive")) + return (f"up arrives as {up}, down as {down}, right as {right}: the " + "vertical flip survives, including the negative value") + + +def _check_frames(server) -> str: + """No frame means nothing was delivered, however many events were sent.""" + frames = server.recording.frames + _require(frames >= 4, f"only {frames} frames for 9 emissions") + return f"{frames} frames — every emission was committed" + + +def _check_emulating(server) -> str: + """Every device the client kept has to open an emulation transaction. + + libei is explicit that "sending events before ei_device_start_emulating() + ... is a client bug", so a device that receives events without one is a + protocol violation this peer happens to tolerate and a stricter + compositor need not. + """ + offered = {f"autocontrol-{label}" for label in server.recording.devices} + + def _all_started() -> bool: + return {name for name, _seq in server.recording.emulating_sequences} \ + >= offered + + # The handshake returns as soon as the *required* devices are live, so the + # last device's transaction can still be in flight when this runs. + try: + _wait_for(_all_started, 2.0, "every device to start emulating") + except AssertionError: + pass + started = {name for name, _sequence in server.recording.emulating_sequences} + _require(started == offered, ( + f"devices {sorted(offered)} were handed over but only " + f"{sorted(started)} started emulating.\n server saw: " + f"{_format_log(server.recording.event_log)}")) + return f"{sorted(started)}, sequences " \ + f"{[seq for _n, seq in server.recording.emulating_sequences]}" + + +_EVENT_NAMES = { + 1: "CLIENT_CONNECT", 2: "CLIENT_DISCONNECT", 3: "SEAT_BIND", + 4: "DEVICE_CLOSED", 100: "FRAME", 200: "START_EMULATING", + 201: "STOP_EMULATING", 300: "POINTER_MOTION", 400: "MOTION_ABSOLUTE", + 500: "BUTTON", 603: "SCROLL_DISCRETE", 700: "KEYBOARD_KEY", +} + + +def _format_log(entries) -> str: + return ", ".join( + f"{_EVENT_NAMES.get(kind, kind)}{'@' + name if name else ''}" + for kind, name in entries) + + +def _check_sequence_reaches_the_server(backend, server, libei) -> str: + """Does the start/stop sequence number survive the trip at all? + + libei documents it as identifying one start→stop transaction and requires + it to rise by at least one per call. AutoControl counts correctly, but the + server reads 0 for every transaction — so either the number is not put on + the wire by this libei, or it is not read back by this libeis. Sending an + unmistakable value settles which end to blame. + """ + symbols = libei._load_symbols() + device = backend._devices[libei.EI_DEVICE_CAP_KEYBOARD] + symbols.ei_device_stop_emulating(device) + symbols.ei_device_start_emulating(device, 4242) + _wait_for(lambda: any(seq == 4242 for _n, seq + in server.recording.emulating_sequences) + or server.recording.stopped_emulating > 0, 2.0, + "the restarted emulation transaction") + seen = [seq for _n, seq in server.recording.emulating_sequences] + if 4242 in seen: + return "an explicit 4242 arrives intact, so the number is carried" + return (f"an explicit 4242 arrives as {seen[-1]} — libei {_libei_version()} " + "does not carry the sequence number, so AutoControl's counter " + "cannot be checked from this side (it satisfies the contract " + "regardless: it rises by one per call)") + + +def _libei_version() -> str: + import ctypes.util + return str(ctypes.util.find_library("ei")) + + +def _check_pause_fails_closed(backend, server) -> str: + """A paused device must take its capability out of service, not misfire. + + ``is_connected`` is not polled here on purpose: it only reads cached + state, and the client learns about a pause when it next pumps — which is + inside ``_emit``. Asking for a keystroke is therefore the only honest way + to ask whether the pause was noticed. + """ + import select + import time + symbols = libei_module()._load_symbols() + seen: List[int] = [] + original = backend._on_event + + def spy(event: int) -> None: + seen.append(int(symbols.ei_event_get_type(event))) + original(event) + + backend._on_event = spy + try: + server.pause_devices() + deadline = time.monotonic() + 3.0 + attempts = 0 + while time.monotonic() < deadline: + attempts += 1 + try: + backend.press_key(KEY_A) + except Exception as error: # noqa: BLE001 # reason: the type is the finding + _require(type(error).__name__ == "LibeiUnavailable", + f"a paused device raised {error!r}, not a fail-closed") + return (f"the keystroke {attempts} calls after the pause " + f"failed closed: {str(error)[:60]}") + time.sleep(0.05) + # Nothing arrived. Before blaming the client, ask whether anything was + # sent at all: an idle fd means the pause never left the server. + poll_fd = int(symbols.ei_get_fd(backend._ei)) + readable = bool(select.select([poll_fd], [], [], 1.0)[0]) + finally: + backend._on_event = original + _require(not readable and not seen, ( + f"the client was told about the pause (events {seen}, fd " + f"{'readable' if readable else 'idle'}) and still emitted {attempts} " + "keystrokes, so DEVICE_PAUSED is received and ignored")) + return ("UNTESTED here: libeis 1.3.901 put nothing on the wire for " + f"eis_device_pause on {len(server.recording.paused)} live devices " + "(client fd stayed idle for 4s), so the client's DEVICE_PAUSED " + "handling still has no peer to exercise it — see Progress.md") + + +def libei_module(): + from je_auto_control.linux_wayland import libei + return libei + + +def _check_resume_recovers(backend, server) -> str: + """And a device must still be usable after a pause/resume round trip.""" + before = len(server.recording.keys) + server.resume_devices() + _wait_for(lambda: server.recording.frames >= 0, 0.3, "the resume to land") + backend.press_key(KEY_A) + backend.release_key(KEY_A) + _wait_for(lambda: len(server.recording.keys) >= before + 2, 2.0, + "keys after the resume") + return "the device came back and emitted again" + + +def _live_teardown_sentinel(path: str) -> str: + """Is ``ei_unref`` safe once the handshake has actually completed? + + ``libei_verify.py`` established that it segfaults on a context whose + backend opened but never handshook. Whether a *live* context is safe was + untestable without a peer; it is testable now, and it is safe — which is + what lets ``_teardown`` release a completed session rather than leaking + its context. This is therefore a regression sentinel for that decision, + and it runs in a subprocess because the answer is a signal, not an + exception. + """ + program = ( + "import sys; sys.path.insert(0, '/opt/verify');" + "from eis_server import RecordingEisServer;" + "from je_auto_control.linux_wayland import libei;" + f"srv = RecordingEisServer({path + '-teardown'!r}); srv.start();" + "b = libei.LibeiBackend();" + f"b.connect(timeout=5.0, socket_path={(path + '-teardown').encode()!r});" + "assert b.is_connected;" + "sym = libei._load_symbols();" + "sym.ei_unref(b._ei);" + "print('survived')" + ) + finished = subprocess.run( # nosec B603 # reason: this interpreter, fixed argv + [sys.executable, "-c", program], capture_output=True, timeout=60) + if finished.returncode == 0: + return ("safe, which is what _teardown now relies on to release a " + "completed session instead of leaking its context") + if finished.returncode == -11: + print() + print(" *** REVISIT *** ei_unref now SEGFAULTS on a live") + print(" context too. LibeiBackend._teardown releases completed") + print(" sessions on the measurement that it is safe, so that") + print(" must go back to abandoning them — this is a crash in a") + print(" library that drives the user's desktop. See Progress.md.") + print() + raise AssertionError( + "ei_unref segfaults on a live context (rc=-11); _teardown's " + "release path is no longer safe on this libei") + detail = finished.stderr.decode("utf-8", "replace").strip().splitlines() + return f"inconclusive (rc={finished.returncode}): {detail[-1] if detail else ''}" + + +#: An offset region layout: the shape a compositor must advertise for a +#: desktop whose left-most monitor sits at a negative layout coordinate. +#: Region offsets are ``uint32``, so it cannot advertise the negative +#: coordinate itself — which is the whole reason the two spaces can differ. +OFFSET_REGIONS = (((0, 0), (1280, 1024)), ((1280, 0), (1920, 1080))) + + +def _offset_region_session(path, libei, server_class): + """Bring up a peer whose pointer regions are the OFFSET_REGIONS layout.""" + if os.path.exists(path): + os.unlink(path) + server = server_class(path, regions=OFFSET_REGIONS) + server.start() + backend = libei.LibeiBackend() + backend.connect(timeout=5.0, socket_path=path.encode("utf-8")) + _require(backend.is_connected, "the offset-region handshake never landed") + return backend, server + + +def _pointer_device(backend, libei): + device = backend._devices.get(libei.EI_DEVICE_CAP_POINTER_ABSOLUTE) + _require(bool(device), "no absolute pointer device was granted") + return device + + +def _check_regions_are_read_back(backend, libei) -> str: + """The client sees the regions the server configured, offsets included.""" + got = backend._device_regions(_pointer_device(backend, libei)) + expected = [(x, y, w, h) for (x, y), (w, h) in OFFSET_REGIONS] + _require(got == expected, + f"the server configured {expected}, the client reads {got}") + return f"{got} — offsets survive the trip, so they are part of the space" + + +def _check_offset_region_takes_absolute_coordinates(backend, server, + libei) -> str: + """A region at x=1280 takes 1380 for a point 100 pixels into it.""" + before = len(server.recording.absolute_motions) + backend.set_position(1380, 100) + _wait_for(lambda: len(server.recording.absolute_motions) > before, 2.0, + "absolute motion into the offset region") + got = server.recording.absolute_motions[-1] + _require(got == (1380.0, 100.0), + f"asked for (1380, 100) inside a region at x=1280, saw {got}") + return "region-space coordinates carry the offset, they are not local" + + +def _check_libei_drops_out_of_region_motion(backend, server, libei) -> str: + """The measurement the whole guard rests on. + + Called under the guard rather than through it: ``set_position`` now + refuses this point, so the raw entry point is used to find out what libei + does when nothing stops it. If some future libei clamps instead of + dropping, this is the check that says so. + """ + device = _pointer_device(backend, libei) + symbols = backend._symbols + before = list(server.recording.absolute_motions) + symbols.ei_device_pointer_motion_absolute(device, 9000.0, 9000.0) + symbols.ei_device_frame(device, symbols.ei_now(backend._ei)) + time.sleep(0.5) + after = server.recording.absolute_motions + _require(after == before, ( + "libei no longer drops an absolute motion outside every region — it " + f"delivered {after[len(before):]}. Re-read LibeiBackend._region_point: " + "its refusal exists because this was silent")) + return ("(9000, 9000) reached the server as nothing at all — no event, " + "no error, no return code: the silent no-op _region_point " + "replaces with a refusal") + + +def _check_out_of_region_move_is_refused(backend, server, libei) -> str: + """AutoControl turns that silence into something the CLI path can act on.""" + before = len(server.recording.absolute_motions) + try: + backend.set_position(9000, 9000) + except libei.LibeiUnavailable as error: + _require("outside every region" in str(error), + f"refused, but with an unhelpful message: {error}") + time.sleep(0.3) + _require(len(server.recording.absolute_motions) == before, + "the refusal still put a motion on the wire") + return f"refused with {str(error)[:60]}... — _select_input.emitted "\ + "hands it to ydotool" + raise AssertionError( + "set_position(9000, 9000) returned as though the pointer had moved; " + "libei dropped it and nobody was told") + + +def _check_negative_origin_is_normalised(backend, server, libei, + monkey) -> str: + """The layout half of the same problem, end to end against the peer. + + A monitor left of the primary puts this project's layout origin at + ``(-1280, 0)`` while the compositor's regions still start at 0. Asking + for the top-left pixel of that monitor means asking for ``(-1280, 0)``, + which no region covers — so it has to arrive as ``(0, 0)``. + """ + before = len(server.recording.absolute_motions) + with monkey(libei, "_layout_origin", lambda: (-1280, 0)): + backend.set_position(-1280, 10) + _wait_for(lambda: len(server.recording.absolute_motions) > before, 2.0, + "the normalised motion") + got = server.recording.absolute_motions[-1] + _require(got == (0.0, 10.0), + f"(-1280, 10) on a layout starting at -1280 arrived as {got}, " + "not the (0, 10) the region space calls that pixel") + return "(-1280, 10) arrives as (0, 10): input and capture name one pixel" + + +class _swap: + """Minimal context-managed attribute swap; no pytest in this image.""" + + def __init__(self, target, name, value): + self._target, self._name, self._value = target, name, value + + def __enter__(self): + self._old = getattr(self._target, self._name) + setattr(self._target, self._name, self._value) + return self + + def __exit__(self, *_exc): + setattr(self._target, self._name, self._old) + return False + + +def _run_region_checks(path, libei, server_class) -> None: + """Every region check, on a peer of their own. + + They need a differently-shaped device than the rest of the file, and a + device's regions are fixed when the compositor adds it. + """ + backend, server = _offset_region_session(path, libei, server_class) + try: + check("the client reads back the regions the compositor advertised", + lambda: _check_regions_are_read_back(backend, libei)) + check("an offset region takes absolute coordinates, not local ones", + lambda: _check_offset_region_takes_absolute_coordinates( + backend, server, libei)) + check("libei still drops out-of-region motion without a word", + lambda: _check_libei_drops_out_of_region_motion( + backend, server, libei)) + check("an out-of-region move is refused rather than silently lost", + lambda: _check_out_of_region_move_is_refused( + backend, server, libei)) + check("a negative layout origin is normalised into region space", + lambda: _check_negative_origin_is_normalised( + backend, server, libei, _swap)) + finally: + backend.disconnect() + server.stop() + + +def main() -> int: + print("=" * 72) + print("AutoControl libei sender — against a real EIS server (libeis)") + print("=" * 72) + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from eis_server import RecordingEisServer, load_symbols + from je_auto_control.linux_wayland import libei + + if load_symbols() is None: + print("libeis.so.* not found — install libeis1. Nothing to verify.") + return 1 + + path = _socket_path() + server = RecordingEisServer(path) + server.start() + print(f" EIS server listening at {path}") + print("-" * 72) + + backend = check("the handshake completes against a real EIS peer", + lambda: _connected_backend(libei, server, path)) + if backend is None: + server.stop() + print("the handshake never completed; the rest cannot run") + return 1 + sum(1 for _n, ok in _results if not ok) + + check("the client connects, binds a seat and is handed devices", + lambda: _check_handshake(server)) + check("the seat binds exactly the capabilities AutoControl asks for", + lambda: _check_capabilities(server, libei)) + check("a device starts emulating with a rising sequence number", + lambda: _check_emulating(server)) + check("press_key / release_key arrive as the right evdev code", + lambda: _check_keyboard(backend, server)) + check("set_position arrives as absolute motion at the right point", + lambda: _check_pointer(backend, server)) + check("press_button / release_button arrive as the right BTN_ code", + lambda: _check_button(backend, server)) + check("scroll() arrives as whole wheel clicks, on the right axis and sign", + lambda: _check_scroll_unit(backend, server)) + check("mouse.scroll()'s kernel-to-libei axis flip reaches the server", + lambda: _check_scroll_wiring(backend, server)) + check("every emission was committed with a frame", + lambda: _check_frames(server)) + check("the emulation sequence number, end to end", + lambda: _check_sequence_reaches_the_server(backend, server, libei)) + check("a paused device is either acted on, or never announced", + lambda: _check_pause_fails_closed(backend, server)) + check("a resumed device can emit again", + lambda: _check_resume_recovers(backend, server)) + + check("disconnect() after a live session does not crash the process", + lambda: (backend.disconnect(), backend.disconnect(), + "torn down twice")[-1]) + check("ei_unref on a live context is still safe — teardown depends on it", + lambda: _live_teardown_sentinel(path)) + + # The absolute pointer's coordinate space, on a peer whose regions are + # shaped like the desktop that made it a question: two monitors, the + # left one at a negative layout coordinate the region space cannot hold. + _run_region_checks(path + "-regions", libei, RecordingEisServer) + + server.stop() + if server.error is not None: + print(f" NOTE: the server thread ended with {server.error!r}") + + failed = [name for name, ok in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/entrypoint-seat.sh b/docker/entrypoint-seat.sh new file mode 100644 index 00000000..91c747ab --- /dev/null +++ b/docker/entrypoint-seat.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# Bring up a wlroots session that really consumes the ydotool device, and ask +# it where an absolute move lands. +# +# Three things have to be true before the question can even be asked, and each +# was once the reason this was recorded as needing a VM: +# +# * the compositor must run libinput. WLR_BACKENDS=headless,libinput keeps +# the outputs virtual — no GPU, no DRM — while the input half is the real +# libinput backend rather than nothing at all. +# * libinput must be allowed to open the device. libseat's builtin backend +# does that without logind, but it also tries to take over a VT; a +# container has none, and SEATD_VTBOUND=0 is what stops it trying. +# * udev must know the device. ydotoold's uinput node is published by the +# kernel, but libinput enumerates through udev, so udevd has to be running +# when the device appears. It is started first for that reason. +# +# The device node itself is created here: udevd inside a container does not +# get to populate /dev, so the minor number is computed from sysfs the same +# way docker/ydotool_verify.py does it. +# +# The verification runs twice, over the same two layouts as +# docker/entrypoint-wayland.sh — side by side from the origin, and with +# HEADLESS-1 moved to x=-1280. The second is the one that separates "the +# layout corner" from "layout (0, 0)"; the first is the control where the two +# coincide and every check has to pass anyway. +set -eu + +RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg}" +mkdir -p "$RUNTIME_DIR" /run/udev +chmod 700 "$RUNTIME_DIR" + +LOG=/tmp/verify.log +RC=/tmp/verify.rc +SWAY_LOG=/tmp/sway.log + +echo "starting systemd-udevd" +/usr/lib/systemd/systemd-udevd --daemon +sleep 1 + +echo "starting ydotoold" +ydotoold --socket-path="$YDOTOOL_SOCKET" --socket-own="$(id -u):$(id -g)" \ + >/tmp/ydotoold.log 2>&1 & +sleep 3 + +# Input event nodes are character major 13, minor 64 + N. +mkdir -p /dev/input +for sysfs in /sys/class/input/event*; do + [ -e "$sysfs" ] || continue + node=$(basename "$sysfs") + [ -e "/dev/input/$node" ] || \ + mknod "/dev/input/$node" c 13 $((64 + ${node#event})) +done + +if [ -z "$(ls -A /dev/input 2>/dev/null)" ]; then + echo "No input device appeared. ydotoold could not create one:" + cat /tmp/ydotoold.log + echo "The container needs --device /dev/uinput and" + echo "--device-cgroup-rule 'c 13:* rmw', and the host needs the uinput" + echo "module loaded." + exit 1 +fi +ls -l /dev/input + +if ! libinput list-devices | grep -qi ydotool; then + echo "libinput cannot see the ydotool device, so sway will not either:" + libinput list-devices || true + exit 1 +fi + +# Both outputs carry a flat colour so that anything else in a capture is the +# cursor. The values are mirrored in seat_verify.py as OUTPUT_COLOURS. +cat > /tmp/sway.cfg.in <<'SWAYCFG' +default_border none +output HEADLESS-1 position @POS1@ bg #123456 solid_color +output HEADLESS-2 position 0 0 bg #abcdef solid_color +exec sh -c 'python3 /opt/verify/seat_verify.py >/tmp/verify.log 2>&1; echo $? >/tmp/verify.rc; swaymsg exit' +SWAYCFG + +# Runs one sway session over one layout; echoes the number of failed checks. +run_session() { + label="$1" + position="$2" + rm -f "$LOG" "$RC" "$SWAY_LOG" + sed "s/@POS1@/$position/" /tmp/sway.cfg.in > /tmp/sway.cfg + + echo "========================================================================" + echo "layout: $label (HEADLESS-1 at $position)" + echo "========================================================================" + + if ! sway -c /tmp/sway.cfg >"$SWAY_LOG" 2>&1; then + echo "sway exited non-zero; its log follows:" + cat "$SWAY_LOG" + fi + + if [ -s "$LOG" ]; then + cat "$LOG" + else + echo "The verification produced no output. sway log:" + cat "$SWAY_LOG" + return 1 + fi + return "$(cat "$RC" 2>/dev/null || echo 1)" +} + +rc=0 +run_session "side by side from the origin" "1280 0" || rc=$((rc + $?)) +echo +run_session "negative origin" "-1280 0" || rc=$((rc + $?)) + +exit "$rc" diff --git a/docker/entrypoint-wayland.sh b/docker/entrypoint-wayland.sh new file mode 100644 index 00000000..120dcdc2 --- /dev/null +++ b/docker/entrypoint-wayland.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# Bring up a headless sway session and run the Wayland verification in it. +# +# The verification has to run *inside* the compositor's session, because +# WAYLAND_DISPLAY is what makes grim, wtype and wlr-randr able to talk to it +# at all — and what makes AutoControl's platform wrapper select the Wayland +# backend. sway execs the script, the script writes its log and exit status +# to files, and this wrapper reports both once sway is gone. +# +# It does that twice, over two output layouts: +# +# * side by side from the origin — sway's own default for two headless +# outputs, and what a single-monitor desktop looks like too; +# * HEADLESS-1 moved to (-1280, 0) — what a desktop looks like whenever a +# monitor sits left of (or above) the primary one. The whole-layout +# capture then starts at a negative coordinate, so every mapping between +# a pixel and a screen coordinate has to subtract that origin. sway's +# headless backend accepts a negative `position`, and grim accepts a +# negative `-g`, so this is a real compositor answering rather than a +# mock agreeing with itself. +set -eu + +RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg}" +mkdir -p "$RUNTIME_DIR" +chmod 700 "$RUNTIME_DIR" + +LOG=/tmp/verify.log +RC=/tmp/verify.rc +SWAY_LOG=/tmp/sway.log + +# The two outputs are painted different solid colours. A uniform screen would +# let a wrong region grab look right, and identical colours would let a +# red/blue swap through — these do neither. The values are mirrored in +# wayland_verify.py as OUTPUT_COLOURS. +cat > /tmp/sway.cfg.in <<'SWAYCFG' +default_border none +output HEADLESS-1 position @POS1@ bg #123456 solid_color +output HEADLESS-2 position 0 0 bg #abcdef solid_color +exec sh -c 'python3 /opt/verify/wayland_verify.py >/tmp/verify.log 2>&1; echo $? >/tmp/verify.rc; swaymsg exit' +SWAYCFG + +# Runs one sway session over one layout; echoes the number of failed checks. +run_session() { + label="$1" + position="$2" + rm -f "$LOG" "$RC" "$SWAY_LOG" + sed "s/@POS1@/$position/" /tmp/sway.cfg.in > /tmp/sway.cfg + + echo "========================================================================" + echo "layout: $label (HEADLESS-1 at $position)" + echo "========================================================================" + + # sway's own chatter goes to a file: it is only interesting when the + # session fails to come up at all, and it would otherwise bury the + # verification. + if ! sway -c /tmp/sway.cfg >"$SWAY_LOG" 2>&1; then + echo "sway exited non-zero; its log follows:" + cat "$SWAY_LOG" + fi + + if [ -s "$LOG" ]; then + cat "$LOG" + else + echo "The verification produced no output. sway log:" + cat "$SWAY_LOG" + return 1 + fi + return "$(cat "$RC" 2>/dev/null || echo 1)" +} + +rc=0 +run_session "side by side from the origin" "1280 0" || rc=$((rc + $?)) +echo +run_session "negative origin" "-1280 0" || rc=$((rc + $?)) + +# The libei half needs no compositor — it drives the binding against the real +# libei.so — so it runs after sway is gone rather than inside a session. +echo +python3 /opt/verify/libei_verify.py || rc=$((rc + $?)) + +exit "$rc" diff --git a/docker/libei_verify.py b/docker/libei_verify.py new file mode 100644 index 00000000..00a7d027 --- /dev/null +++ b/docker/libei_verify.py @@ -0,0 +1,286 @@ +"""Verify AutoControl's libei binding against the real ``libei.so``. + +No compositor is needed for this half. What the unit tests cannot check — +because they inject a fake symbol table — is whether the entry points this +binding names actually exist in the shared object, with the signatures it +declares. A single misspelled name or a wrong ``argtypes`` would sail past +every mock and only surface on a user's machine. + +So: resolve every prototype against the installed library, then drive +``connect()`` at a socket that accepts the connection but speaks no EI. The +handshake cannot complete, and that is the point — the fail-closed promise +("anything short of a live device means use the ydotool CLI") is checked +here against the real library rather than asserted about a mock. + +What this half cannot answer is anything a peer has to *agree* with: the +capability and event-type enum values, the variadic +``ei_seat_bind_capabilities`` call, and whether emission puts anything on the +wire. ``docker/eis_verify.py`` answers those by running a real libeis server +on the other end of the socket. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import ctypes.util +import faulthandler +import os +import socket +import sys +import threading +import traceback +from typing import Any, Callable, List, Tuple + +# A wrong prototype in a ctypes binding shows up as a segfault, not as an +# exception, and a segfault with no traceback is the hardest kind of bug to +# act on. faulthandler turns it into a Python stack ending at the exact call. +faulthandler.enable() + +_results: List[Tuple[str, bool]] = [] + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def serve_silent_socket(path: str) -> socket.socket: + """Accept connections at ``path`` and then say nothing at all. + + libei will connect and begin its handshake; nothing answers, so the + client has to give up on its own deadline rather than hang. + """ + if os.path.exists(path): + os.unlink(path) + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(path) + server.listen(4) + + def accept_forever() -> None: + held = [] + while True: + try: + conn, _ = server.accept() + except OSError: + return + held.append(conn) # keep it open; never write + + threading.Thread(target=accept_forever, daemon=True).start() + return server + + +def main() -> int: + print("=" * 72) + print("AutoControl libei binding — against the real libei.so") + print("=" * 72) + + resolved = ctypes.util.find_library("ei") + print(f"find_library('ei') = {resolved!r}") + print(f"find_library('oeffis') = {ctypes.util.find_library('oeffis')!r}") + print("-" * 72) + + from je_auto_control.linux_wayland import _select_input, libei, oeffis + from je_auto_control.linux_wayland import keyboard as wl_keyboard + + # --- the check the mocks structurally cannot make -------------------- + def _symbols(): + symbols = libei._load_symbols() + if symbols is None: + raise AssertionError( + "not one prototype resolved — either libei.so is absent or a " + "name in _PROTOTYPES does not exist in it") + missing = [name for name, _, _ in libei._PROTOTYPES + if not hasattr(symbols, name)] + if missing: + raise AssertionError(f"unresolved entry points: {missing}") + # The variadic one is bound separately, without argtypes. + if not hasattr(symbols, "ei_seat_bind_capabilities"): + raise AssertionError("ei_seat_bind_capabilities did not resolve") + return f"{len(libei._PROTOTYPES)} prototypes + 1 variadic, all resolved" + check("every libei entry point this binding names exists", _symbols) + + check("LibeiBackend reports the library as available", + lambda: _assert_true(libei.LibeiBackend().is_available, + "is_available was False with libei installed")) + + # --- liboeffis: packaged separately, so easily absent ---------------- + # Debian does package it (liboeffis1 on trixie), but as its own binary + # package that libei1 does not depend on — so installing libei alone + # leaves the portal route off, which is the state this image is in and + # the state a user who installed one package would be in. + available = oeffis.is_available() + print(f" liboeffis available here: {available}") + if not available: + print(" (liboeffis is not installed here, so the portal route is") + print(" unavailable and connect() falls back to the socket —") + print(" which is exactly the path exercised below. The portal") + print(" route itself is covered by docker/portal_verify.py.)") + + # --- a real sender against a socket that speaks no EI ---------------- + runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") + socket_path = os.path.join(runtime, "eis-0") + server = serve_silent_socket(socket_path) + print(f" silent EIS stand-in listening at {socket_path}") + + # --- raw, one call at a time ----------------------------------------- + # connect() is half a dozen library calls deep. Walking them by hand with + # flushed output means a crash names the call that caused it instead of + # just the function that contained it. + def _raw_walk(): + symbols = libei._load_symbols() + step = lambda msg: print(f" · {msg}", flush=True) # noqa: E731 + + step("ei_new_sender(None) ...") + handle = symbols.ei_new_sender(None) + step(f" -> {handle!r}") + if not handle: + raise AssertionError("ei_new_sender returned NULL") + + step(f"ei_setup_backend_socket(handle, {socket_path!r}) ...") + code = symbols.ei_setup_backend_socket( + handle, socket_path.encode("utf-8")) + step(f" -> {code}") + + step("ei_get_fd(handle) ...") + poll_fd = symbols.ei_get_fd(handle) + step(f" -> {poll_fd}") + + step("ei_dispatch(handle) ...") + symbols.ei_dispatch(handle) + step(" -> returned") + + step("ei_get_event(handle) ...") + event = symbols.ei_get_event(handle) + step(f" -> {event!r}") + while event: + kind = symbols.ei_event_get_type(event) + step(f" event type {kind}") + symbols.ei_event_unref(event) + event = symbols.ei_get_event(handle) + step(f" next -> {event!r}") + + # ei_unref is NOT called here: on this libei it segfaults once the + # backend is open. The sentinel below establishes that separately, + # in a subprocess, so it cannot take this run down with it. + step("(context abandoned — see the ei_unref sentinel)") + return "every call up to teardown behaves" + check("each libei call in isolation", _raw_walk) + + # --- the upstream defect this binding works around ------------------- + def _unref_sentinel(): + import subprocess + program = ( + "import ctypes, ctypes.util, os, socket, threading;" + "lib = ctypes.CDLL(ctypes.util.find_library('ei'));" + "lib.ei_new_sender.restype = ctypes.c_void_p;" + "lib.ei_new_sender.argtypes = (ctypes.c_void_p,);" + "lib.ei_setup_backend_socket.restype = ctypes.c_int;" + "lib.ei_setup_backend_socket.argtypes = " + "(ctypes.c_void_p, ctypes.c_char_p);" + "lib.ei_unref.restype = ctypes.c_void_p;" + "lib.ei_unref.argtypes = (ctypes.c_void_p,);" + f"p = {socket_path!r};" + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM);" + "s.connect(p);" + "h = lib.ei_new_sender(None);" + "rc = lib.ei_setup_backend_socket(h, p.encode());" + "assert rc == 0, rc;" + "lib.ei_unref(h)" + ) + finished = subprocess.run([sys.executable, "-c", program], + capture_output=True) + if finished.returncode == -11: + return ("still segfaults (rc=-11), so the abandon-on-teardown " + "workaround in libei.py::_teardown is still required") + print() + print(" *** REVISIT *** ei_unref no longer crashes on this") + print(" libei (rc=%s). The workaround in LibeiBackend._teardown" + % finished.returncode) + print(" can probably go; see Progress.md.") + print() + return f"no longer crashes (rc={finished.returncode}) — see above" + check("ei_unref after a successful setup — upstream state", _unref_sentinel) + + def _connect_fails_closed(): + backend = libei.LibeiBackend() + try: + backend.connect(timeout=1.0, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable as error: + return f"LibeiUnavailable: {str(error)[:90]}" + raise AssertionError( + "connect() reported success against a peer that sent nothing, so " + "the handshake is not actually gating on a live device") + check("connect() against a silent peer fails closed, not open", + _connect_fails_closed) + + def _no_crash_on_teardown(): + backend = libei.LibeiBackend() + try: + backend.connect(timeout=0.5, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable: + pass + backend.disconnect() # must be safe after a failed connect + backend.disconnect() # and idempotent + return "teardown survived a failed connect, twice" + check("teardown after a failed handshake does not crash the process", + _no_crash_on_teardown) + + # --- the fallback the whole design rests on -------------------------- + libei.reset_default_backend() + check("active_backend() gives up and hands over to the CLI", + lambda: _assert_true(_select_input.active_backend() is None, + "active_backend() returned a backend that " + "cannot emit")) + + def _keyboard_falls_back(): + # ydotool is deliberately not installed in this image, so the CLI + # path must surface its install hint — not a libei error and not a + # silent no-op. + try: + wl_keyboard.press_key(30) + except Exception as error: # noqa: BLE001 # reason: any type is informative + if "ydotool" in str(error): + return f"{type(error).__name__}: {str(error)[:60]}" + raise + raise AssertionError("press_key claimed success with no libei and no " + "ydotool") + check("press_key falls through to the ydotool CLI path", + _keyboard_falls_back) + + server.close() + + print("-" * 72) + print("Not covered here, because it needs a peer that speaks EI:") + print(" the capability / event-type enum values, the variadic") + print(" ei_seat_bind_capabilities call, seat grants, emission and") + print(" the live-context teardown. docker/eis_verify.py covers all") + print(" of that against a real libeis server — run it too.") + + failed = [name for name, ok in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/portal_server.py b/docker/portal_server.py new file mode 100644 index 00000000..5e1c0575 --- /dev/null +++ b/docker/portal_server.py @@ -0,0 +1,454 @@ +"""A mock ``org.freedesktop.portal.Desktop`` that implements RemoteDesktop. + +The libei input path has two halves. ``docker/eis_server.py`` answers the +protocol half — a real ``libeis`` peer on a Unix socket, which settles the +enum values, the variadic bind and what actually goes on the wire. The other +half is how a client *gets* that socket on GNOME and KDE: it is not a path on +disk but a file descriptor handed over D-Bus at the end of the +``org.freedesktop.portal.RemoteDesktop`` session dance, and +:mod:`je_auto_control.linux_wayland.oeffis` drives ``liboeffis`` to perform it. + +That half was recorded as needing a GNOME VM, on the grounds that +``xdg-desktop-portal-wlr`` implements ScreenCast and Screenshot but not +RemoteDesktop. What that reasoning missed is that the portal is *a D-Bus +interface*, not a compositor feature: anything that owns the well-known name +and answers the four calls is a portal as far as ``liboeffis`` is concerned. +So this module is that — a real D-Bus service on a real session bus, driving +the real ``liboeffis`` through the real handshake — and ``ConnectToEIS`` +hands back a live connection to the real EIS server next door, which makes +the whole chain end at pixels-equivalent evidence: input emitted through a +portal-obtained fd, recorded by an independent implementation. + +What it deliberately does *not* claim to be: a consent dialog. There is no +user here to dismiss anything. What a dialog produces, though — a grant, a +refusal, and a wait that never ends — are all just Response codes and silence +on the bus, so ``--behaviour`` produces each of them and the caller's +fail-closed handling is verified against all three. + +Runs on the system interpreter, not the one that has AutoControl installed: +GDBus is what can pass a file descriptor over D-Bus from Python, it comes +from Debian's ``python3-gi``, and it is bound to Debian's ``python3``. That +suits it anyway — the portal has to be a separate process from the client +that calls it. +""" +from __future__ import annotations + +import argparse +import json +import os +import socket +import struct +import sys +import zlib +from typing import Any, Dict, Optional + +import gi + +gi.require_version("Gio", "2.0") +from gi.repository import Gio, GLib # noqa: E402 # reason: gi.require_version must run first + + +BUS_NAME = "org.freedesktop.portal.Desktop" +OBJECT_PATH = "/org/freedesktop/portal/desktop" +REQUEST_INTERFACE = "org.freedesktop.portal.Request" +SESSION_INTERFACE = "org.freedesktop.portal.Session" + +#: Portal response codes, from the XDG portal specification. +RESPONSE_SUCCESS = 0 +RESPONSE_CANCELLED = 1 + +#: Everything a compositor could offer: keyboard | pointer | touchscreen. +ALL_DEVICE_TYPES = 7 + +#: ``grant`` is the happy path; the rest are the ways a real portal says no. +BEHAVIOURS = ("grant", "deny", "stall", "no-fd", "close") + +#: The same idea for the Screenshot interface, which is a different portal +#: with a different client: :mod:`je_auto_control.linux_wayland.portal` speaks +#: D-Bus itself, so what is under test there is hand-written marshalling and a +#: directed signal arriving on a path the client predicted. +SCREENSHOT_BEHAVIOURS = ("grant", "deny", "stall", "no-uri", "not-a-file") + +#: A filename with a space in it on purpose: the URI the portal returns is +#: percent-encoded, and unquoting it is a step the client has to get right. +SCREENSHOT_NAME = "autocontrol portal shot.png" + +#: What the mock paints, so the client can be checked on the bytes it got +#: rather than merely on getting some. +SHOT_SIZE = (4, 3) +SHOT_RGB = (0, 128, 255) + +PORTAL_XML = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + +REQUEST_XML = """ + + + + + + + + + +""" + +SESSION_XML = """ + + + + + + + + + +""" + + +def sender_token(sender: str) -> str: + """The client's unique bus name as the portal spec spells it in a path. + + Request and session objects live at a path the *client* predicts so it can + subscribe before it calls, which only works if both sides mangle the name + the same way: drop the leading colon, turn dots into underscores. + """ + return sender.lstrip(":").replace(".", "_") + + +def _interface(xml: str, name: str = "") -> Gio.DBusInterfaceInfo: + """One interface out of an introspection document, by name or the first.""" + interfaces = Gio.DBusNodeInfo.new_for_xml(xml).interfaces + if not name: + return interfaces[0] + return next(info for info in interfaces if info.name == name) + + +def encode_png(width: int, height: int, rgb: tuple) -> bytes: + """A valid PNG of one flat colour, with nothing but the standard library. + + The mock runs on Debian's interpreter, which has no imaging package, and + handing the client bytes that are not really a PNG would check the + transport while quietly skipping whether the result is usable. + """ + row = bytes((0,)) + bytes(rgb) * width + raw = zlib.compress(row * height, 9) + + def chunk(kind: bytes, payload: bytes) -> bytes: + body = kind + payload + return (struct.pack(">I", len(payload)) + body + + struct.pack(">I", zlib.crc32(body) & 0xFFFFFFFF)) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + + chunk(b"IDAT", raw) + chunk(b"IEND", b"")) + + +class MockPortal: + """Owns the portal bus name and answers the RemoteDesktop calls. + + :param eis_socket: the EIS server socket ``ConnectToEIS`` connects to. + :param behaviour: one of :data:`BEHAVIOURS`. + :param record_path: JSON file the driver reads to see what arrived. + :param version: what to report as the RemoteDesktop interface version; + ``ConnectToEIS`` only exists from 2, so 1 is a portal too old to use. + """ + + def __init__(self, eis_socket: str, behaviour: str, record_path: str, + version: int = 2, screenshot: str = "grant", + shot_dir: str = "/tmp") -> None: # nosec B108 # reason: container default, overridden by the driver + self.eis_socket = eis_socket + self.behaviour = behaviour + self.record_path = record_path + self.version = version + self.screenshot = screenshot + self.shot_dir = shot_dir + self.record: Dict[str, Any] = { + "calls": [], "device_types": None, "properties": [], + "session_closed_by_client": False, "shot_path": "", + } + self.connection: Optional[Gio.DBusConnection] = None + #: Registration ids are kept only so the objects stay exported. + self._exported: list = [] + self._remote_desktop_info = _interface( + PORTAL_XML, "org.freedesktop.portal.RemoteDesktop") + self._screenshot_info = _interface( + PORTAL_XML, "org.freedesktop.portal.Screenshot") + self._request_info = _interface(REQUEST_XML) + self._session_info = _interface(SESSION_XML) + + # --- recording -------------------------------------------------------- + + def _flush(self) -> None: + """Write the record out now; the driver reads it while we still run.""" + with open(self.record_path, "w", encoding="utf-8") as handle: + json.dump(self.record, handle) + + def _note(self, method: str, detail: Any = None) -> None: + self.record["calls"].append({"method": method, "detail": detail}) + self._flush() + + # --- lifecycle -------------------------------------------------------- + + def run(self) -> None: + """Take the bus name and serve until killed.""" + self.connection = Gio.bus_get_sync(Gio.BusType.SESSION, None) + for info in (self._remote_desktop_info, self._screenshot_info): + self.connection.register_object( + OBJECT_PATH, info, + self._on_method_call, self._on_get_property, None) + Gio.bus_own_name_on_connection( + self.connection, BUS_NAME, Gio.BusNameOwnerFlags.NONE, + lambda *_args: print("portal: owns " + BUS_NAME, flush=True), + lambda *_args: print("portal: lost " + BUS_NAME, flush=True)) + self._flush() + print(f"portal: ready (behaviour={self.behaviour}, " + f"screenshot={self.screenshot}, version={self.version})", + flush=True) + GLib.MainLoop().run() + + # --- properties ------------------------------------------------------- + + def _on_get_property(self, _connection: Gio.DBusConnection, _sender: str, + _path: str, _interface_name: str, + name: str) -> Optional[GLib.Variant]: + """``version`` is the first thing liboeffis reads, before any call.""" + self.record["properties"].append(name) + self._flush() + if name == "version": + return GLib.Variant("u", self.version) + if name == "AvailableDeviceTypes": + return GLib.Variant("u", ALL_DEVICE_TYPES) + return None + + # --- method dispatch -------------------------------------------------- + + def _on_method_call(self, _connection: Gio.DBusConnection, sender: str, + _path: str, _interface_name: str, method: str, + parameters: GLib.Variant, + invocation: Gio.DBusMethodInvocation) -> None: + print(f"portal: {method}{parameters}", flush=True) + handler = getattr(self, f"_do_{method}", None) + if handler is None: + invocation.return_error_literal( + Gio.dbus_error_quark(), Gio.DBusError.UNKNOWN_METHOD, method) + return + handler(sender, parameters, invocation) + + def _request_path(self, sender: str, options: Dict[str, Any]) -> str: + """The object path the client already subscribed to.""" + token = options.get("handle_token", "unnamed") + return f"{OBJECT_PATH}/request/{sender_token(sender)}/{token}" + + def _export_request(self, path: str) -> None: + """Export the request object so ``Close()`` on it is answerable.""" + self._exported.append(self.connection.register_object( + path, self._request_info, self._answer_empty, None, None)) + + def _answer_empty(self, _connection: Gio.DBusConnection, _sender: str, + _path: str, _interface_name: str, _method: str, + _parameters: GLib.Variant, + invocation: Gio.DBusMethodInvocation) -> None: + invocation.return_value(None) + + def _on_session_call(self, _connection: Gio.DBusConnection, _sender: str, + _path: str, _interface_name: str, method: str, + _parameters: GLib.Variant, + invocation: Gio.DBusMethodInvocation) -> None: + """Record whether the client ends its grant explicitly, or just leaves.""" + if method == "Close": + self.record["session_closed_by_client"] = True + self._flush() + invocation.return_value(None) + + def _respond(self, sender: str, path: str, code: int, + results: Dict[str, GLib.Variant]) -> bool: + """Emit the ``Response`` signal the whole portal protocol turns on.""" + self.connection.emit_signal( + sender, path, REQUEST_INTERFACE, "Response", + GLib.Variant("(ua{sv})", (code, results))) + return False # reason: one-shot, so GLib.idle_add does not repeat it + + # --- the four RemoteDesktop calls ------------------------------------- + + def _do_CreateSession(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + options = parameters.unpack()[0] + self._note("CreateSession", options) + request = self._request_path(sender, options) + self._export_request(request) + token = options.get("session_handle_token", "unnamed") + session = f"{OBJECT_PATH}/session/{sender_token(sender)}/{token}" + self._exported.append(self.connection.register_object( + session, self._session_info, self._on_session_call, + lambda *_args: GLib.Variant("u", 2), None)) + invocation.return_value(GLib.Variant("(o)", (request,))) + if self.behaviour == "stall": + # A consent dialog nobody answers: the handle was returned, the + # Response never comes. The client must give up on its own clock. + return + GLib.idle_add(self._respond, sender, request, RESPONSE_SUCCESS, + {"session_handle": GLib.Variant("s", session)}) + if self.behaviour == "close": + GLib.idle_add(self._close_session, sender, session) + + def _close_session(self, sender: str, session: str) -> bool: + """End the grant the way a compositor does when the session stops.""" + self.connection.emit_signal( + sender, session, SESSION_INTERFACE, "Closed", + GLib.Variant("(a{sv})", ({},))) + return False + + def _do_SelectDevices(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + session, options = parameters.unpack() + self._note("SelectDevices", {"session": session, "options": options}) + types = options.get("types") + if types is not None: + self.record["device_types"] = int(types) + self._flush() + request = self._request_path(sender, options) + self._export_request(request) + invocation.return_value(GLib.Variant("(o)", (request,))) + GLib.idle_add(self._respond, sender, request, RESPONSE_SUCCESS, {}) + + def _do_Start(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + session, parent_window, options = parameters.unpack() + self._note("Start", {"session": session, "parent": parent_window, + "options": options}) + request = self._request_path(sender, options) + self._export_request(request) + invocation.return_value(GLib.Variant("(o)", (request,))) + # ``deny`` is the consent dialog the user dismissed: the portal + # answers, and the answer is no. + denied = self.behaviour == "deny" + code = RESPONSE_CANCELLED if denied else RESPONSE_SUCCESS + results = {} if denied else {"devices": GLib.Variant("u", 3)} + GLib.idle_add(self._respond, sender, request, code, results) + + def _do_ConnectToEIS(self, _sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + session, options = parameters.unpack() + self._note("ConnectToEIS", {"session": session, "options": options}) + if self.behaviour == "no-fd": + invocation.return_error_literal( + Gio.dbus_error_quark(), Gio.DBusError.FAILED, + "this portal will not hand over an EIS fd") + return + try: + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.connect(self.eis_socket) + except OSError as error: + invocation.return_error_literal( + Gio.dbus_error_quark(), Gio.DBusError.FAILED, str(error)) + return + # GDBus dups what it appends, so the local end is closed here and the + # client gets its own descriptor — which is exactly the ownership a + # real portal hands over. + fd_list = Gio.UnixFDList.new() + index = fd_list.append(client.fileno()) + client.close() + invocation.return_value_with_unix_fd_list( + GLib.Variant("(h)", (index,)), fd_list) + + + # --- the Screenshot call ---------------------------------------------- + + def _do_Screenshot(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + """Answer the interface every desktop implements, however it captures. + + The client here is not a C library but AutoControl's own D-Bus + marshalling, in :mod:`je_auto_control.linux_wayland._dbus_client`. + Answering on the path the client predicted, with a signal directed at + the connection that called, is what that code has to cope with — and + is exactly what the ``gdbus monitor`` design it replaced could not. + """ + parent_window, options = parameters.unpack() + self._note("Screenshot", {"parent": parent_window, "options": options}) + request = self._request_path(sender, options) + self._export_request(request) + invocation.return_value(GLib.Variant("(o)", (request,))) + if self.screenshot == "stall": + return + GLib.idle_add(self._respond_screenshot, sender, request) + + def _respond_screenshot(self, sender: str, request: str) -> bool: + """Every outcome a real Screenshot portal can end on.""" + if self.screenshot == "deny": + return self._respond(sender, request, RESPONSE_CANCELLED, {}) + if self.screenshot == "no-uri": + return self._respond(sender, request, RESPONSE_SUCCESS, {}) + if self.screenshot == "not-a-file": + return self._respond(sender, request, RESPONSE_SUCCESS, + {"uri": GLib.Variant( + "s", "https://example.invalid/shot.png")}) + path = os.path.join(self.shot_dir, SCREENSHOT_NAME) + with open(path, "wb") as image: + image.write(encode_png(*SHOT_SIZE, SHOT_RGB)) + self.record["shot_path"] = path + self._flush() + uri = "file://" + GLib.uri_escape_string(path, "/", False) + return self._respond(sender, request, RESPONSE_SUCCESS, + {"uri": GLib.Variant("s", uri)}) + + +def main() -> int: + """Parse the arguments and serve until the driver kills the process.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--eis-socket", required=True, + help="EIS server socket that ConnectToEIS connects to") + parser.add_argument("--behaviour", default="grant", choices=BEHAVIOURS) + parser.add_argument("--record", required=True, + help="JSON file recording what the client asked for") + parser.add_argument("--version", type=int, default=2, + help="RemoteDesktop interface version to advertise") + parser.add_argument("--screenshot", default="grant", + choices=SCREENSHOT_BEHAVIOURS) + parser.add_argument("--shot-dir", default="/tmp", # nosec B108 # reason: container default + help="directory the mock writes its capture into") + arguments = parser.parse_args() + os.umask(0o077) + MockPortal(arguments.eis_socket, arguments.behaviour, arguments.record, + arguments.version, arguments.screenshot, + arguments.shot_dir).run() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/portal_verify.py b/docker/portal_verify.py new file mode 100644 index 00000000..6684c579 --- /dev/null +++ b/docker/portal_verify.py @@ -0,0 +1,566 @@ +"""Verify AutoControl's portal handshake against a real ``liboeffis``. + +``eis_verify.py`` drives the libei sender against a real EIS peer, but it +reaches that peer the way no GNOME or KDE desktop offers: by handing +``connect()`` a socket path. The route those desktops actually use is +``org.freedesktop.portal.RemoteDesktop`` — a D-Bus session dance ending in a +file descriptor passed over SCM_RIGHTS — and it was recorded as unverifiable +without a GNOME VM, because ``xdg-desktop-portal-wlr`` has no RemoteDesktop +interface. + +The portal is a D-Bus interface, though, not a compositor feature. +``docker/portal_server.py`` owns the well-known name and answers the four +calls, so the real ``liboeffis`` runs the real handshake here, and its +``ConnectToEIS`` returns a live connection to the real EIS server from +``eis_server.py``. That makes the whole path checkable end to end: + + * the four calls arrive in the order the specification prescribes, at the + request paths the client predicted — a mismatch there is a client that + subscribes to a signal nobody sends and then times out; + * ``SelectDevices`` receives the keyboard-and-pointer mask this project + asks for, so :data:`~je_auto_control.linux_wayland.oeffis.OEFFIS_DEVICE_DEFAULT` + is neither wider than the grant the user consents to nor the ``= 0`` + all-devices sentinel it would be if the constant were wrong; + * the descriptor that comes back carries a real EI session: the sender + completes its handshake on it and the input it emits is recorded by an + independent implementation at the far end; + * and every way a portal says no — a dismissed dialog, a dialog left open, + a refused descriptor, a closed session, a portal too old to have + ``ConnectToEIS`` at all, no portal on the bus — produces a refusal on + this project's own clock rather than a hang or a silent downgrade. + +What is still not claimed: the consent dialog itself. No user dismisses +anything here. What a dialog *produces* is a Response code or silence, and +all three outcomes are exercised; what it looks like is mutter's business. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import contextlib +import faulthandler +import json +import os +import shutil +import stat +import subprocess # nosec B404 # reason: launches dbus-daemon and the mock portal, argv lists, no shell +import sys +import threading +import time +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +# A wrong prototype in a ctypes binding is a segfault, not an exception. +faulthandler.enable() + +_results: List[Tuple[str, bool]] = [] + +#: evdev codes the emission check uses. KEY_A, BTN_LEFT. +KEY_A = 30 +BTN_LEFT = 272 +TARGET_POSITION = (640, 400) + +#: The order the XDG portal specification prescribes, and the order a client +#: that got it wrong would never complete in. +EXPECTED_CALLS = ("CreateSession", "SelectDevices", "Start", "ConnectToEIS") + +#: What the mock paints. Re-declared rather than imported from +#: ``portal_server``, which runs on the other interpreter — and which is the +#: right call anyway: two sides agreeing on one wrong constant is a test that +#: passes while the thing under it is broken. +SHOT_SIZE = (4, 3) +SHOT_RGB = (0, 128, 255) + +#: Long enough for a portal that is going to answer; short enough that the +#: ones written not to answer do not hold the image up. +GRANT_TIMEOUT = 10.0 +REFUSAL_TIMEOUT = 3.0 + + +def check(name: str, function: Callable[[], Any]) -> Any: + """Run one check, record the outcome, and never let it stop the rest.""" + try: + detail = function() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _runtime_dir() -> str: + runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") # nosec B108 # reason: container fallback only + os.makedirs(runtime, exist_ok=True) + return runtime + + +def _fresh_socket_path(name: str) -> str: + path = os.path.join(_runtime_dir(), name) + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + return path + + +class SessionBus: + """A private ``dbus-daemon`` this run owns, exported to the environment. + + liboeffis reads ``DBUS_SESSION_BUS_ADDRESS`` from the environment of the + process it is loaded into, so the address has to land in :data:`os.environ` + and not only in the child's. + """ + + def __init__(self) -> None: + self._process: Optional[subprocess.Popen] = None + self.address = "" + + def start(self) -> str: + binary = shutil.which("dbus-daemon") + if binary is None: + raise RuntimeError("dbus-daemon is not installed") + # argv is a private allow-list; no shell and no user-supplied component. + self._process = subprocess.Popen( # nosec B603 # nosemgrep + [binary, "--session", "--print-address", "--nofork"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + address = (self._process.stdout.readline() or "").strip() + if not address: + raise RuntimeError("dbus-daemon printed no bus address") + self.address = address + os.environ["DBUS_SESSION_BUS_ADDRESS"] = address + return address + + def stop(self) -> None: + if self._process is None: + return + with contextlib.suppress(OSError, ValueError): + self._process.terminate() + with contextlib.suppress(subprocess.TimeoutExpired, OSError, ValueError): + self._process.wait(timeout=5) + self._process = None + + +class Portal: + """The mock portal, running as a child process for one scenario. + + Used as a context manager so the bus name is released before the next + scenario asks for it. + """ + + def __init__(self, eis_socket: str, behaviour: str = "grant", + version: int = 2, screenshot: str = "grant") -> None: + self.eis_socket = eis_socket + self.behaviour = behaviour + self.version = version + self.screenshot = screenshot + self.record_path = os.path.join( + _runtime_dir(), + f"portal-record-{behaviour}-{screenshot}-{version}.json") + self.lines: List[str] = [] + self._process: Optional[subprocess.Popen] = None + self._pump: Optional[threading.Thread] = None + + def __enter__(self) -> "Portal": + server = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "portal_server.py") + # Debian's python3 is the one with GDBus; this interpreter is the one + # with AutoControl. The portal has to be a separate process anyway. + argv = ["/usr/bin/python3", server, + "--eis-socket", self.eis_socket, + "--behaviour", self.behaviour, + "--record", self.record_path, + "--version", str(self.version), + "--screenshot", self.screenshot, + "--shot-dir", _runtime_dir()] + # argv is a private allow-list; no shell and no user-supplied component. + self._process = subprocess.Popen( # nosec B603 # nosemgrep + argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + self._pump = threading.Thread(target=self._read, daemon=True, + name="portal-stdout") + self._pump.start() + self._await_name() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is None: + return + with contextlib.suppress(OSError, ValueError): + self._process.terminate() + with contextlib.suppress(subprocess.TimeoutExpired, OSError, ValueError): + self._process.wait(timeout=5) + if self._pump is not None: + self._pump.join(timeout=2) + self._process = None + + def _read(self) -> None: + stream = self._process.stdout if self._process else None + if stream is None: + return + with contextlib.suppress(OSError, ValueError): + for line in stream: + self.lines.append(line.rstrip()) + + def _await_name(self, timeout: float = 15.0) -> None: + """Wait until the portal owns the name, not merely until it started.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if any("owns " in line for line in list(self.lines)): + return + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + "the mock portal exited before taking the bus name:\n" + + "\n".join(self.lines)) + time.sleep(0.02) + raise RuntimeError("the mock portal never took the bus name") + + def recorded(self) -> Dict[str, Any]: + """What the portal has seen so far.""" + try: + with open(self.record_path, "r", encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError): + return {"calls": [], "device_types": None, "properties": []} + + def methods(self) -> List[str]: + return [call["method"] for call in self.recorded()["calls"]] + + def await_call(self, method: str, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if method in self.methods(): + return + time.sleep(0.02) + raise AssertionError( + f"the portal never saw {method}; it saw {self.methods()}") + + +def _wait_for(predicate: Callable[[], bool], timeout: float, + description: str) -> None: + """Spin until the server thread has recorded something, or give up.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.02) + raise AssertionError(f"timed out waiting for {description}") + + +def _refused(action: Callable[[], Any], expected: type, + budget: float) -> str: + """Assert an action fails closed, promptly, and return the reason it gave.""" + started = time.monotonic() + try: + action() + except expected as error: + elapsed = time.monotonic() - started + _require(elapsed <= budget, + f"the refusal took {elapsed:.1f}s, longer than the {budget:g}s " + f"budget — that is a hang wearing a refusal's clothes") + return f"{str(error)[:96]} (in {elapsed:.1f}s)" + raise AssertionError( + f"expected {expected.__name__}; the call returned instead") + + +# --- checks --------------------------------------------------------------- + + +def _check_liboeffis_binds(oeffis) -> str: + """Every entry point the binding names, resolved against the real library.""" + symbols = oeffis.load_symbols() + _require(symbols is not None, + "liboeffis.so.* did not resolve — install liboeffis1") + _require(oeffis.is_available(), "is_available() disagrees with load_symbols()") + _require(oeffis.OEFFIS_DEVICE_DEFAULT != oeffis.OEFFIS_DEVICE_ALL_DEVICES, + "the default device mask is the all-devices sentinel") + return f"default device mask = {oeffis.OEFFIS_DEVICE_DEFAULT}" + + +def _check_no_portal_is_refused(oeffis) -> str: + """The state of every desktop that has no RemoteDesktop portal.""" + return _refused(lambda: oeffis.connect_eis_fd(timeout=REFUSAL_TIMEOUT), + oeffis.OeffisUnavailable, REFUSAL_TIMEOUT + 3.0) + + +def _check_call_order(portal: Portal) -> str: + calls = portal.methods() + _require(tuple(calls[:len(EXPECTED_CALLS)]) == EXPECTED_CALLS, + f"the portal saw {calls}, not {list(EXPECTED_CALLS)}") + _require("version" in portal.recorded()["properties"], + "liboeffis never read the RemoteDesktop version property") + return " -> ".join(calls) + + +def _check_device_mask(portal: Portal, oeffis) -> str: + """What the user is actually asked to consent to.""" + types = portal.recorded()["device_types"] + _require(types is not None, "SelectDevices carried no `types` option") + _require(types == oeffis.OEFFIS_DEVICE_DEFAULT, + f"the portal was asked for {types}, not " + f"{oeffis.OEFFIS_DEVICE_DEFAULT}") + _require(not types & oeffis.OEFFIS_DEVICE_TOUCHSCREEN, + "the grant includes a touchscreen this backend never emits on") + return f"types={types} (keyboard|pointer)" + + +def _check_fd_is_live_and_owned(oeffis) -> str: + """The descriptor is a real socket, and it is the caller's to close. + + ``oeffis_get_eis_fd`` is documented as returning a ``dup()`` whose owner + is the caller. Everything downstream depends on that: libei takes + ownership of what it is handed and closes it when torn down, so a second + owner here would be a double close. + """ + eis_fd, session = oeffis.connect_eis_fd(timeout=GRANT_TIMEOUT) + try: + _require(eis_fd >= 0, f"the portal handed over fd {eis_fd}") + info = os.fstat(eis_fd) + _require(stat.S_ISSOCK(info.st_mode), + f"fd {eis_fd} is not a socket (mode {info.st_mode:#o})") + os.close(eis_fd) + with contextlib.suppress(OSError): + os.fstat(eis_fd) + raise AssertionError( + f"fd {eis_fd} survived close() — it was not ours to own") + finally: + session.close() + session.close() # idempotent: the teardown path calls it too + return f"fd {eis_fd} was a socket, and closing it was ours to do" + + +def _connected_through_the_portal(libei): + """A full sender handshake with no socket path — the portal's route.""" + backend = libei.LibeiBackend() + backend.connect(timeout=GRANT_TIMEOUT) + _require(backend.is_connected, + "connect() returned but the backend reports no live device") + return backend + + +def _check_session_reaches_eis(server) -> str: + record = server.recording + _wait_for(lambda: bool(record.devices), 5.0, "the seat to hand over devices") + _require(record.seat_binds >= 1, "the client never bound a seat") + _require(all(record.sender_flags), + "the client did not present itself as a sender") + return (f"client={record.clients[0]!r} seat binds={record.seat_binds} " + f"devices={record.devices}") + + +def _check_input_over_the_portal_fd(backend, server) -> str: + record = server.recording + backend.press_key(KEY_A) + backend.release_key(KEY_A) + backend.set_position(*TARGET_POSITION) + backend.press_button(BTN_LEFT) + backend.release_button(BTN_LEFT) + _wait_for(lambda: len(record.keys) >= 2, 5.0, "the key press and release") + _wait_for(lambda: bool(record.absolute_motions), 5.0, "the absolute motion") + _wait_for(lambda: len(record.buttons) >= 2, 5.0, "the button edges") + _require(record.keys[:2] == [(KEY_A, True), (KEY_A, False)], + f"the server recorded {record.keys[:2]}") + landed = record.absolute_motions[-1] + _require(tuple(int(value) for value in landed) == TARGET_POSITION, + f"the pointer landed at {landed}, not {TARGET_POSITION}") + _require(record.buttons[:2] == [(BTN_LEFT, True), (BTN_LEFT, False)], + f"the server recorded {record.buttons[:2]}") + return (f"keys={record.keys[:2]} motion={landed} " + f"buttons={record.buttons[:2]}") + + +def _check_disconnect_ends_the_grant(backend, server, portal: Portal) -> str: + """Tearing down must revoke the grant, not merely stop emitting.""" + before = server.recording.disconnects + backend.disconnect() + backend.disconnect() + _wait_for(lambda: server.recording.disconnects > before, 5.0, + "the EIS server to see the client go away") + closed = portal.recorded().get("session_closed_by_client", False) + return ("the portal session was closed explicitly" if closed else + "the D-Bus connection was dropped, which revokes the grant") + + +def _check_portal_refuses(oeffis) -> str: + return _refused(lambda: oeffis.connect_eis_fd(timeout=GRANT_TIMEOUT), + oeffis.OeffisUnavailable, GRANT_TIMEOUT) + + +def _check_open_dialog_times_out(oeffis) -> str: + """A consent dialog nobody answers must end on our clock, not never.""" + reason = _refused(lambda: oeffis.connect_eis_fd(timeout=REFUSAL_TIMEOUT), + oeffis.OeffisUnavailable, REFUSAL_TIMEOUT + 3.0) + _require("consent" in reason, + f"the timeout did not name the consent dialog: {reason}") + return reason + + +def _check_backend_surfaces_the_refusal(libei) -> str: + """A refused portal must reach the caller as this project's own error.""" + return _refused(lambda: libei.LibeiBackend().connect(timeout=GRANT_TIMEOUT), + libei.LibeiUnavailable, GRANT_TIMEOUT + 3.0) + + +def _check_capture_returns_the_portal_bytes(wayland_portal, portal: Portal) -> str: + """The Screenshot tier, end to end, on hand-marshalled D-Bus. + + Nothing here is mocked: a real ``dbus-daemon`` routes real messages that + :mod:`je_auto_control.linux_wayland._dbus_client` marshalled itself, to a + real portal that answers with a signal directed at the caller — which is + the property that made the previous ``gdbus monitor`` design unable to work + at all, and which nothing short of a real bus would have shown. + """ + import numpy + import cv2 + + payload = wayland_portal.capture_png(timeout=15.0) + _require(payload.startswith(b"\x89PNG"), + f"the portal tier returned {payload[:8]!r}, which is not a PNG") + decoded = cv2.imdecode(numpy.frombuffer(payload, dtype=numpy.uint8), + cv2.IMREAD_COLOR) + _require(decoded is not None, "the portal's PNG did not decode") + height, width = decoded.shape[:2] + _require((width, height) == SHOT_SIZE, + f"decoded {width}x{height}, expected {SHOT_SIZE}") + # OpenCV decodes to BGR; the mock painted RGB. + _require(tuple(int(value) for value in decoded[0][0]) == SHOT_RGB[::-1], + f"the first pixel is {decoded[0][0]}, not {SHOT_RGB[::-1]}") + shot = portal.recorded().get("shot_path", "") + _require(bool(shot), "the portal never recorded where it wrote the capture") + _require(not os.path.exists(shot), + f"the portal's file at {shot!r} was left behind") + return (f"{len(payload)} bytes, decoded {width}x{height}, " + f"and {os.path.basename(shot)!r} was cleaned up") + + +def _check_capture_refused(wayland_portal, needle: str, + timeout: float = 15.0) -> str: + """A capture that cannot succeed must say why, not return half an image.""" + reason = _refused(lambda: wayland_portal.capture_png(timeout=timeout), + wayland_portal.AutoControlScreenException, timeout + 3.0) + _require(needle in reason, + f"the failure did not mention {needle!r}: {reason}") + return reason + + +# --- scenarios ------------------------------------------------------------ + + +def _run_grant_scenario(portal: Portal, server, libei, oeffis, + wayland_portal) -> None: + """Everything that needs a portal which says yes.""" + backend = check("libei completes its handshake over a portal-obtained fd", + lambda: _connected_through_the_portal(libei)) + check("the portal dance runs the four calls in the prescribed order", + lambda: _check_call_order(portal)) + check("SelectDevices is asked for keyboard and pointer, nothing wider", + lambda: _check_device_mask(portal, oeffis)) + check("the EIS fd is a live socket the caller owns and must close", + lambda: _check_fd_is_live_and_owned(oeffis)) + if backend is None: + return + check("the portal's descriptor carries a real EI session", + lambda: _check_session_reaches_eis(server)) + check("input emitted through the portal path reaches the EIS server", + lambda: _check_input_over_the_portal_fd(backend, server)) + check("disconnect() revokes the grant instead of leaving it open", + lambda: _check_disconnect_ends_the_grant(backend, server, portal)) + check("the Screenshot tier reaches a real portal and returns real pixels", + lambda: _check_capture_returns_the_portal_bytes( + wayland_portal, portal)) + + +def _run_refusal_scenarios(socket_path: str, libei, oeffis) -> None: + """Every way a portal says no, and this project's answer to each.""" + with Portal(socket_path, behaviour="deny"): + check("a dismissed consent dialog is refused, not worked around", + lambda: _check_portal_refuses(oeffis)) + check("a refused portal reaches the caller as LibeiUnavailable", + lambda: _check_backend_surfaces_the_refusal(libei)) + with Portal(socket_path, behaviour="stall"): + check("a consent dialog left open times out on this project's clock", + lambda: _check_open_dialog_times_out(oeffis)) + with Portal(socket_path, behaviour="no-fd"): + check("a portal that withholds the descriptor fails closed", + lambda: _check_portal_refuses(oeffis)) + with Portal(socket_path, behaviour="close"): + check("a portal that closes the session fails closed", + lambda: _check_portal_refuses(oeffis)) + with Portal(socket_path, version=1): + check("a portal too old to have ConnectToEIS fails closed", + lambda: _check_portal_refuses(oeffis)) + + +def _run_capture_scenarios(socket_path: str, wayland_portal) -> None: + """Every way the Screenshot portal ends without an image.""" + with Portal(socket_path, screenshot="deny"): + check("a dismissed screenshot dialog is named as dismissed", + lambda: _check_capture_refused(wayland_portal, "dismissed")) + with Portal(socket_path, screenshot="stall"): + check("a screenshot dialog left open times out on our clock", + lambda: _check_capture_refused( + wayland_portal, "did not answer", REFUSAL_TIMEOUT)) + with Portal(socket_path, screenshot="no-uri"): + check("a success carrying no image URI is treated as a failure", + lambda: _check_capture_refused(wayland_portal, "no image URI")) + with Portal(socket_path, screenshot="not-a-file"): + check("a URI that is not a local file is refused", + lambda: _check_capture_refused(wayland_portal, "non-file URI")) + + +def main() -> int: + """Run every check and return the number that failed.""" + print("=" * 72) + print("AutoControl RemoteDesktop portal — against a real liboeffis") + print("=" * 72) + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from eis_server import RecordingEisServer, load_symbols + from je_auto_control.linux_wayland import libei, oeffis + from je_auto_control.linux_wayland import portal as wayland_portal + + if load_symbols() is None: + print("libeis.so.* not found — install libeis1. Nothing to verify.") + return 1 + + bus = SessionBus() + print(f" session bus at {bus.start()}") + + socket_path = _fresh_socket_path("eis-portal-verify") + server = RecordingEisServer(socket_path) + server.start() + print(f" EIS server listening at {socket_path}") + print("-" * 72) + + try: + check("liboeffis resolves and every entry point binds", + lambda: _check_liboeffis_binds(oeffis)) + check("no portal on the bus is refused rather than waited on", + lambda: _check_no_portal_is_refused(oeffis)) + with Portal(socket_path) as portal: + _run_grant_scenario(portal, server, libei, oeffis, wayland_portal) + _run_refusal_scenarios(socket_path, libei, oeffis) + _run_capture_scenarios(socket_path, wayland_portal) + finally: + server.stop() + bus.stop() + + if server.error is not None: + print(f" NOTE: the server thread ended with {server.error!r}") + + failed = [name for name, ok in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/seat_verify.py b/docker/seat_verify.py new file mode 100644 index 00000000..bf7df439 --- /dev/null +++ b/docker/seat_verify.py @@ -0,0 +1,493 @@ +"""Verify where ydotool's absolute move actually puts the cursor. + +The other four verification images each answer one half of the Wayland input +story and stop at the same wall. ``docker/Dockerfile.wayland`` runs a real +wlroots compositor but consumes no input devices; ``docker/Dockerfile.ydotool`` +reads ydotool's events straight off the kernel with no compositor at all. So +what ``mousemove --absolute`` *puts on the wire* is settled, and what a +compositor *does with it* was recorded in ``Progress.md`` as needing a VM with +"a compositor that consumes libinput devices". + +It does not need one, for the third time in this project's history. wlroots +takes ``WLR_BACKENDS=headless,libinput``: the outputs stay virtual while the +input side is the real libinput backend, and libseat's builtin backend opens +the devices directly once ``SEATD_VTBOUND=0`` stops it reaching for a VT that +a container has no business owning. ydotoold's uinput device is then an +ordinary seat device like any mouse, and ``grim -c`` composites the cursor +into a screenshot — so the compositor answers, in layout coordinates, the +question this file exists to ask: + + **Does ``ydotool mousemove --absolute -x X -y Y`` put the cursor at layout + ``(X, Y)``?** + +No, twice over: + +* Its origin is the corner the compositor clamps to, which is the top-left of + the *output layout*. That is layout ``(0, 0)`` only while every output sits + at a non-negative position; on the layout every desktop with a monitor left + of the primary one has, the two differ by the layout origin. That is the + translation :func:`je_auto_control.linux_wayland.mouse._ydotool_point` now + applies, and this file is where the number comes from. +* The displacement is relative motion, so the compositor's pointer + acceleration scales it. Under libinput's default adaptive profile the + cursor moves twice as far from that corner as asked. ydotool's own + ``--help`` says "You need to disable mouse speed acceleration for correct + absolute movement"; this measures what ignoring that costs. + +**What a screenshot can see is the cursor's image, not its hotspot**, and the +two differ by whatever offset the cursor theme declares. Nothing here depends +on that offset: every claim is a difference between two captures, a distance +that the offset cancels out of, or an output the cursor is unambiguously +inside. The one absolute reading — that a move to ``(0, 0)`` draws the cursor +flush into the layout's very first pixel — is the clamp pinning it there, +which is exactly the claim being made. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import json +import os +import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell +import sys +import time +import traceback +from typing import Any, Callable, List, Optional, Tuple + +import numpy +from PIL import Image + +_results: List[Tuple[str, bool]] = [] + +Point = Tuple[int, int] + +#: The solid colour ``entrypoint-seat.sh`` paints each output. Anything else +#: in a capture is the cursor: nothing else is ever on screen. Keyed by name +#: because sway lists its outputs in whatever order it likes. +OUTPUT_COLOURS = { + "HEADLESS-1": (0x12, 0x34, 0x56), + "HEADLESS-2": (0xAB, 0xCD, 0xEF), +} + +#: How long to let a move settle before capturing. ydotool's own start delay +#: is 100 ms; the rest is the compositor's next frame. +_SETTLE_SECONDS = 0.6 + +#: The cursor image is drawn from its hotspot by at most this many pixels on +#: either axis, so a reading is "the same pixel" within it. Only used where a +#: check compares a drawn position against a requested coordinate directly; +#: every other check is a difference, which the offset cancels out of. +_CURSOR_IMAGE_SLACK = 2 + +#: libinput's maximum acceleration factor for the adaptive profile at the +#: default speed of 0. ``--absolute`` sends both of its events in a single +#: frame, so the velocity saturates the profile and the factor lands here. +_ADAPTIVE_MAX_FACTOR = 2 + +CAPTURE = "/tmp/seat-capture.png" # nosec B108 # reason: container-only scratch + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _run(argv: List[str], *, timeout: float = 10.0) -> str: + """Run a tool from this image and return its stdout.""" + completed = subprocess.run( # nosec B603 # reason: fixed argv, no shell + argv, check=True, timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + return completed.stdout.decode("utf-8", errors="replace") + + +def _minus(left: Point, right: Point) -> Point: + return (left[0] - right[0], left[1] - right[1]) + + +def _plus(left: Point, right: Point) -> Point: + return (left[0] + right[0], left[1] + right[1]) + + +# -------------------------------------------------------------------------- +# The layout, straight out of the compositor +# -------------------------------------------------------------------------- + +class Layout: + """The output layout sway reports, as this verification's ground truth. + + Read from sway's IPC rather than from AutoControl's own ``wlr-randr`` + parser, so that agreeing with it is a check rather than a tautology. + """ + + def __init__(self, outputs: List[Tuple[str, int, int, int, int]]) -> None: + _require(bool(outputs), "sway reported no active outputs") + self.outputs = outputs + self.rects = [rect for _, *rect in outputs] + self.origin: Point = (min(x for x, _, _, _ in self.rects), + min(y for _, y, _, _ in self.rects)) + + def output_at(self, x: int, y: int) -> Optional[str]: + """Name of the output containing ``(x, y)``, or None.""" + for name, left, top, width, height in self.outputs: + if left <= x < left + width and top <= y < top + height: + return name + return None + + def far_from(self, x: int, y: int) -> Point: + """A point on another output, to park the cursor well out of the way.""" + for name, left, top, _, _ in self.outputs: + if name != self.output_at(x, y): + return (left + 600, top + 500) + return (x + 400, y + 400) + + def __str__(self) -> str: + return " + ".join(f"{name} {w}x{h}@({x},{y})" + for name, x, y, w, h in self.outputs) + + +def read_layout() -> Layout: + """Ask sway for its outputs.""" + outputs = json.loads(_run(["swaymsg", "-r", "-t", "get_outputs"])) + return Layout([ + (str(item["name"]), + int(item["rect"]["x"]), int(item["rect"]["y"]), + int(item["rect"]["width"]), int(item["rect"]["height"])) + for item in outputs if item.get("active") + ]) + + +def pointer_devices() -> List[str]: + """Names of every pointer sway's libinput backend is holding.""" + inputs = json.loads(_run(["swaymsg", "-r", "-t", "get_inputs"])) + return [item.get("name", "") for item in inputs + if item.get("type") == "pointer"] + + +def set_pointer_acceleration(profile: str, speed: str) -> None: + """Reconfigure every pointer, then let the change settle.""" + _run(["swaymsg", f"input type:pointer accel_profile {profile}"]) + _run(["swaymsg", f"input type:pointer pointer_accel {speed}"]) + time.sleep(0.2) + + +# -------------------------------------------------------------------------- +# Where the cursor was drawn, in layout coordinates +# -------------------------------------------------------------------------- + +def drawn_cursor(layout: Layout) -> Point: + """Layout coordinate of the drawn cursor's top-left pixel. + + ``grim -c`` composites the cursor into a capture of the whole layout, + whose first pixel is the layout origin. The outputs carry two flat + colours, so every other pixel belongs to the cursor. + """ + _run(["grim", "-c", CAPTURE]) + with Image.open(CAPTURE) as raw: + frame = numpy.asarray(raw.convert("RGB")) + background = numpy.zeros(frame.shape[:2], dtype=bool) + for colour in OUTPUT_COLOURS.values(): + background |= (frame == numpy.array(colour, dtype=frame.dtype)).all(-1) + rows, columns = numpy.nonzero(~background) + _require(rows.size > 0, + "no cursor was drawn: the capture is entirely background, so " + "either the compositor has no cursor theme or grim -c did not " + "composite one") + return _plus(layout.origin, (int(columns.min()), int(rows.min()))) + + +# -------------------------------------------------------------------------- +# The two ways to ask for a move +# -------------------------------------------------------------------------- + +def ydotool_absolute(x: int, y: int) -> None: + """The raw CLI call, with no translation of any kind.""" + _run(["ydotool", "mousemove", "--absolute", "-x", str(x), "-y", str(y)]) + + +def autocontrol_set_position(x: int, y: int) -> None: + """The backend's own entry point, translation included.""" + from je_auto_control.linux_wayland import mouse + mouse.set_position(x, y) + + +def land(move: Callable[[int, int], None], layout: Layout, + point: Point) -> Point: + """Ask ``move`` for ``point`` and report where the cursor was drawn.""" + move(*point) + time.sleep(_SETTLE_SECONDS) + return drawn_cursor(layout) + + +# -------------------------------------------------------------------------- +# Checks +# -------------------------------------------------------------------------- + +def _seat_holds_the_device(names: List[str]) -> str: + _require(any("ydotool" in name.lower() for name in names), + f"sway is holding no ydotool pointer; it has {names}. Without " + "one the libinput backend never picked the device up and " + "nothing below would be measuring a compositor at all.") + return ", ".join(names) + + +def _origin_is_the_layout_corner(layout: Layout) -> str: + """``--absolute 0 0`` drives the cursor into the layout's first pixel. + + The clamp holds it there and clips the image against the edge, so the + drawn position is the corner itself with no theme offset in the way. + """ + drawn = land(ydotool_absolute, layout, (0, 0)) + _require(drawn == layout.origin, + f"--absolute -x 0 -y 0 drew the cursor at {drawn}, expected it " + f"flush into the layout corner {layout.origin}") + return f"(0, 0) -> {drawn}, the layout's first pixel" + + +def _moves_one_pixel_per_pixel(layout: Layout, near: Point, + far: Point) -> str: + """With acceleration off, the displacement is the one that was asked for.""" + drawn_near = land(ydotool_absolute, layout, near) + drawn_far = land(ydotool_absolute, layout, far) + _require(_minus(drawn_far, drawn_near) == _minus(far, near), + f"asking for {near} then {far} moved the cursor " + f"{_minus(drawn_far, drawn_near)}, expected {_minus(far, near)}") + return f"{near} -> {far} moved {_minus(drawn_far, drawn_near)}" + + +def _counts_from_the_corner(layout: Layout, point: Point) -> str: + """The requested offset is measured from the corner, not from (0, 0).""" + drawn = land(ydotool_absolute, layout, point) + expected = _plus(layout.origin, point) + distance = _minus(drawn, expected) + _require(max(abs(distance[0]), abs(distance[1])) <= _CURSOR_IMAGE_SLACK, + f"--absolute -x {point[0]} -y {point[1]} drew the cursor at " + f"{drawn}, which is {distance} from the corner plus the offset " + f"{expected}") + return f"{point} -> {drawn}, corner + {point} within {distance}" + + +def _an_untranslated_zero_misses_its_monitor(layout: Layout) -> str: + """The failure this translation exists to prevent, made concrete.""" + if layout.origin == (0, 0): + return "not applicable: this layout's corner is layout (0, 0)" + drawn = land(ydotool_absolute, layout, (0, 0)) + reached = layout.output_at(*drawn) + intended = layout.output_at(0, 0) + _require(reached is not None and intended is not None + and reached != intended, + f"expected an untranslated (0, 0) to reach a different output " + f"than layout (0, 0); it drew at {drawn} on output {reached} " + f"while layout (0, 0) is on output {intended}") + return (f"reaches output {reached} at {drawn}, while layout (0, 0) is on " + f"output {intended} — {abs(layout.origin[0])} px apart") + + +def _acceleration_scales_the_move(layout: Layout, near: Point, + far: Point) -> str: + """Under the default profile the displacement is not the one asked for.""" + drawn_near = land(ydotool_absolute, layout, near) + drawn_far = land(ydotool_absolute, layout, far) + moved = _minus(drawn_far, drawn_near) + asked = _minus(far, near) + _require(moved != asked, + f"asking for {near} then {far} moved the cursor exactly {moved} " + "under the default adaptive profile, so this image is no longer " + "measuring pointer acceleration at all") + return f"asked {asked}, moved {moved}" + + +def _acceleration_factor_is_two(layout: Layout, near: Point, + far: Point) -> str: + """Pin today's number, so a libinput change is loud rather than silent.""" + drawn_near = land(ydotool_absolute, layout, near) + drawn_far = land(ydotool_absolute, layout, far) + moved = _minus(drawn_far, drawn_near) + asked = _minus(far, near) + expected = (asked[0] * _ADAPTIVE_MAX_FACTOR, + asked[1] * _ADAPTIVE_MAX_FACTOR) + _require(moved == expected, + f"expected libinput's adaptive profile to saturate at " + f"{_ADAPTIVE_MAX_FACTOR}x — {asked} becoming {expected} — but " + f"the cursor moved {moved}") + return f"{asked} became {moved}, {_ADAPTIVE_MAX_FACTOR}x on both axes" + + +def _backend_agrees_with_the_compositor(layout: Layout) -> str: + """AutoControl reads the same origin the compositor reports.""" + from je_auto_control.linux_wayland import screen + reported = screen.layout_origin() + _require(reported == layout.origin, + f"screen.layout_origin() says {reported}, sway says " + f"{layout.origin}") + return f"{reported}" + + +def _set_position_subtracts_exactly_the_origin(layout: Layout, + point: Point) -> str: + """The backend's translation is the layout origin and nothing else. + + Comparing the two paths against each other rather than against a + coordinate keeps the cursor theme out of it entirely: whatever offset the + drawn image carries is the same in both captures. + """ + through_backend = land(autocontrol_set_position, layout, point) + raw = land(ydotool_absolute, layout, _minus(point, layout.origin)) + _require(through_backend == raw, + f"set_position{point} drew at {through_backend} while the raw " + f"CLI asked for {_minus(point, layout.origin)} drew at {raw}") + return f"set_position{point} == --absolute{_minus(point, layout.origin)}" + + +def _set_position_lands_on_the_layout_pixel(layout: Layout, + point: Point) -> str: + """Ask in layout coordinates, land on that pixel.""" + drawn = land(autocontrol_set_position, layout, point) + distance = _minus(drawn, point) + _require(max(abs(distance[0]), abs(distance[1])) <= _CURSOR_IMAGE_SLACK, + f"mouse.set_position{point} drew the cursor at {drawn}, " + f"{distance} away") + _require(layout.output_at(*drawn) == layout.output_at(*point), + f"mouse.set_position{point} landed on output " + f"{layout.output_at(*drawn)}, not {layout.output_at(*point)}") + return f"{point} -> drawn at {drawn}" + + +def _software_cursor_lands_in_the_capture(layout: Layout, + point: Point) -> str: + """Record that a capture asked for no cursor contains one anyway. + + ``grim`` is invoked without ``-c`` here and everywhere else in the + project, so the request is right; what comes back is not. wlroots draws a + *software* cursor whenever the backend has no cursor plane — always on + headless, and on any DRM session where the driver refuses one or the user + set ``WLR_NO_HARDWARE_CURSORS=1`` — and a software cursor is composited + into the output buffer that ``wlr-screencopy`` then hands over. Every + locator, template match and OCR read in this project goes through that + capture, so on such a session the pointer punches a pointer-shaped hole + in whatever it is sitting on. + + Asserted the way it measures rather than the way it ought to be: a + wlroots that starts honouring ``overlay_cursor`` for software cursors + will fail this check, which is the notification this project wants. + """ + from je_auto_control.linux_wayland import screen + land(autocontrol_set_position, layout, point) + under_cursor = tuple(screen.get_pixel(*point)) + land(autocontrol_set_position, layout, layout.far_from(*point)) + uncovered = tuple(screen.get_pixel(*point)) + named = layout.output_at(*point) + _require(uncovered == OUTPUT_COLOURS[named], + f"with the pointer parked elsewhere, get_pixel{point} returned " + f"{uncovered} rather than {named}'s {OUTPUT_COLOURS[named]}") + _require(under_cursor != uncovered, + f"get_pixel{point} returned {uncovered} with the pointer both " + "on that pixel and away from it. If this compositor has started " + "honouring the no-overlay request for software cursors, the " + "caveat recorded in docs/CAPABILITY_MATRIX.md and the READMEs " + "can go, along with cursor_may_be_captured in the diagnostics " + "bundle.") + return (f"{under_cursor} under the pointer vs {uncovered} without it — " + "the capture is not cursor-free") + + +def _input_and_capture_address_one_pixel(layout: Layout, point: Point) -> str: + """``set_position`` and ``get_pixel`` have to mean the same point. + + The pointer is parked elsewhere before the pixel is read: on a compositor + drawing a software cursor the capture would otherwise return the cursor's + own colour, which is a different measurement from this one. + """ + from je_auto_control.linux_wayland import screen + drawn = land(autocontrol_set_position, layout, point) + reached = layout.output_at(*drawn) + land(autocontrol_set_position, layout, layout.far_from(*point)) + colour = tuple(screen.get_pixel(*point)) + named = layout.output_at(*point) + _require(named is not None, f"{point} is on no output") + _require(reached == named, + f"the pointer reached {reached} while the pixel was read on " + f"{named}") + _require(colour == OUTPUT_COLOURS[named], + f"get_pixel{point} returned {colour}, but {named} is painted " + f"{OUTPUT_COLOURS[named]}") + return f"both reach {named}, painted {colour}" + + +# -------------------------------------------------------------------------- + +def _interior_points(layout: Layout) -> List[Point]: + """One point per output, far enough inside that no edge can clip it.""" + return [(x + 120, y + 90) for x, y, _, _ in layout.rects] + + +def main() -> int: + os.environ.setdefault("JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", "cli") + + layout = read_layout() + print(f"layout: {layout} origin {layout.origin}\n") + + check("sway's libinput backend is holding the ydotool device", + lambda: _seat_holds_the_device(pointer_devices())) + + # Everything about the origin is measured with acceleration switched off, + # because a scaled move cannot show where it counted from. + set_pointer_acceleration("flat", "0") + + check("--absolute counts from the layout corner, not layout (0, 0)", + lambda: _origin_is_the_layout_corner(layout)) + check("with acceleration off the move is one pixel per pixel", + lambda: _moves_one_pixel_per_pixel(layout, (50, 50), (600, 400))) + check("a 500x300 request lands 500x300 from that corner", + lambda: _counts_from_the_corner(layout, (500, 300))) + check("an untranslated (0, 0) misses the monitor it names", + lambda: _an_untranslated_zero_misses_its_monitor(layout)) + + check("screen.layout_origin() matches the compositor's own layout", + lambda: _backend_agrees_with_the_compositor(layout)) + for point in _interior_points(layout): + check(f"set_position{point} subtracts exactly the layout origin", + lambda p=point: _set_position_subtracts_exactly_the_origin( + layout, p)) + check(f"set_position{point} lands on that layout pixel", + lambda p=point: _set_position_lands_on_the_layout_pixel( + layout, p)) + check("set_position and get_pixel address the same monitor", + lambda: _input_and_capture_address_one_pixel( + layout, _interior_points(layout)[0])) + check("a software cursor lands in a capture that asked for none", + lambda: _software_cursor_lands_in_the_capture( + layout, _interior_points(layout)[0])) + + # And with the compositor's default profile back, what the option name + # promises stops being true at all. + set_pointer_acceleration("adaptive", "0") + check("the default profile scales the move, so it is not absolute", + lambda: _acceleration_scales_the_move(layout, (50, 50), (300, 200))) + check("that scaling is libinput's 2x adaptive ceiling today", + lambda: _acceleration_factor_is_two(layout, (50, 50), (300, 200))) + + failed = [name for name, ok in _results if not ok] + print("\n" + "=" * 70) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" failed: {name}") + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/wayland_verify.py b/docker/wayland_verify.py new file mode 100644 index 00000000..7f4ddbf6 --- /dev/null +++ b/docker/wayland_verify.py @@ -0,0 +1,383 @@ +"""Verify AutoControl's Wayland backend against a real wlroots compositor. + +Runs inside a headless sway session (see ``Dockerfile.wayland``). Everything +the unit tests can only assert against a mock is checked here against the +compositor itself: the argv grim actually accepts, the geometry string it +actually honours, the output format wlr-randr actually prints, and whether +the whole ``screenshot()`` -> ``screen_grabber`` -> ``capture`` -> +``grab_image`` chain really returns the pixels on screen. + +Ground truth comes from ``swaymsg -t get_outputs`` — the compositor's own +report — rather than from an assumption about how sway lays two headless +outputs out. That matters: sway puts HEADLESS-2 at x=0 and HEADLESS-1 to its +right, so anything keyed on list order rather than on the reported geometry +tests the wrong pixel. The two outputs are painted different solid colours so +a region grab has something to get *wrong*: on a uniform screen any rectangle +looks correct. + +The entrypoint runs this twice, against two sway configs. The second one +moves HEADLESS-1 to ``position -1280 0``, which is the layout a desktop has +whenever a monitor sits left of (or above) the primary one: the capture then +starts at a negative coordinate, and everything that maps a pixel to a screen +coordinate has to subtract that origin rather than assume ``(0, 0)``. Nothing +below is written for one layout or the other — every coordinate is derived +from what the compositor reports. + +Exit status is the number of failed checks, so the container's exit code +says whether this passed. +""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import traceback +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Tuple + +# Painted onto the named outputs by the sway config. Deliberately asymmetric +# in every channel so a red/blue swap cannot pass. Which one ends up on the +# left is sway's business, so everything below keys on the *name*. +OUTPUT_COLOURS = { + "HEADLESS-1": (0x12, 0x34, 0x56), + "HEADLESS-2": (0xAB, 0xCD, 0xEF), +} + +_results: List[Tuple[str, bool, str]] = [] + + +@dataclass(frozen=True) +class Layout: + """The compositor's output layout, as this run found it.""" + + origin_x: int + origin_y: int + width: int + height: int + outputs: List[dict] + + @property + def left(self) -> dict: + """The left-most output.""" + return self.outputs[0] + + @property + def right(self) -> dict: + """The right-most output (the left-most one when there is only one).""" + return self.outputs[-1] + + @property + def multi(self) -> bool: + """Whether there is more than one output to tell apart.""" + return len(self.outputs) > 1 + + def colour(self, output: dict) -> Tuple[int, int, int]: + """The solid colour sway was told to paint ``output``.""" + return OUTPUT_COLOURS[output["name"]] + + def image_point(self, x: int, y: int) -> Tuple[int, int]: + """Layout coordinate ``(x, y)`` as a pixel index into a full capture.""" + return (x - self.origin_x, y - self.origin_y) + + def inside(self, output: dict, dx: int = 10, dy: int = 10) -> Tuple[int, int]: + """A layout coordinate ``(dx, dy)`` into ``output``.""" + rect = output["rect"] + return (rect["x"] + dx, rect["y"] + dy) + + +def check(name: str, fn: Callable[[], Any]) -> Any: + """Run one check, record pass/fail, and keep going either way.""" + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a failed check must not stop the rest + _results.append((name, False, traceback.format_exc(limit=3).strip())) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True, str(detail))) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def note(message: str) -> None: + print(f" {message}") + + +def sway_outputs() -> List[dict]: + """The compositor's own description of its outputs.""" + raw = subprocess.run(["swaymsg", "-t", "get_outputs", "-r"], + check=True, stdout=subprocess.PIPE).stdout + return json.loads(raw) + + +def read_layout() -> Layout: + """Describe the live layout, sorted so "left" and "right" mean what they say.""" + outputs = sorted(sway_outputs(), key=lambda o: o["rect"]["x"]) + origin_x = min(o["rect"]["x"] for o in outputs) + origin_y = min(o["rect"]["y"] for o in outputs) + width = max(o["rect"]["x"] + o["rect"]["width"] for o in outputs) - origin_x + height = max(o["rect"]["y"] + o["rect"]["height"] for o in outputs) - origin_y + return Layout(origin_x, origin_y, width, height, outputs) + + +def report_environment(layout: Layout) -> None: + """Print what session this is and what the compositor says it is showing.""" + print(f"WAYLAND_DISPLAY = {os.environ.get('WAYLAND_DISPLAY')!r}") + print(f"XDG_SESSION_TYPE = {os.environ.get('XDG_SESSION_TYPE')!r}") + print(f"DISPLAY = {os.environ.get('DISPLAY')!r} " + f"(absent means no XWayland, so nothing can silently fall back to it)") + for out in layout.outputs: + rect = out["rect"] + print(f"output {out['name']}: {rect['width']}x{rect['height']} " + f"at ({rect['x']},{rect['y']}) " + f"painted {OUTPUT_COLOURS.get(out['name'])}") + print(f"layout = {layout.width}x{layout.height} at " + f"({layout.origin_x},{layout.origin_y})") + print("-" * 72) + + +def check_detection(modules: Dict[str, Any]) -> None: + """The session is seen as Wayland and the capture tier resolves to grim.""" + detect = modules["detect"] + capture = modules["capture"] + screen_grabber = modules["screen_grabber"] + check("session detected as wayland", + lambda: _assert_eq(detect.select_display_server(), "wayland")) + check("capture tier resolves to grim", + lambda: _assert_eq(capture.available_tool(), "grim")) + check("platform wrapper publishes grab_image", + lambda: _assert_true(screen_grabber.backend_grab_image() is not None, + "backend_grab_image() returned None, so every " + "capture would have gone to Pillow/mss")) + + +def check_geometry(screen: Any, layout: Layout) -> None: + """wlr-randr's format, the reported size and the origin it implies.""" + def _wlr_randr_raw(): + raw = subprocess.run(["wlr-randr"], check=True, + stdout=subprocess.PIPE).stdout.decode() + print(" wlr-randr prints:") + for line in raw.splitlines()[:8]: + print(f" | {line}") + return "captured" + check("wlr-randr runs", _wlr_randr_raw) + check("screen.size() reports the whole layout, not one output", + lambda: _assert_eq(screen.size(), (layout.width, layout.height))) + # On the shifted layout these two differ by 1280: the size is the width + # of the bounding box, never the coordinate of its right edge. + check("layout_origin() matches what the compositor reports", + lambda: _assert_eq(screen.layout_origin(), + (layout.origin_x, layout.origin_y))) + full = check("grab_image() full frame", + lambda: _size_of(screen.grab_image())) + check("full frame matches the layout bounding box", + lambda: _assert_eq(full, (layout.width, layout.height))) + + +def check_pixels(screen: Any, layout: Layout) -> None: + """Colour fidelity, grim's -g geometry and get_pixel, in layout coordinates.""" + image = screen.grab_image() + left_rgb = layout.colour(layout.left) + right_rgb = layout.colour(layout.right) + check(f"left output {layout.left['name']} reads its own colour (RGB order)", + lambda: _assert_eq(image.getpixel( + layout.image_point(*layout.inside(layout.left))), left_rgb)) + if layout.multi: + check(f"right output {layout.right['name']} reads its own colour " + f"(proves the x offset is real)", + lambda: _assert_eq(image.getpixel( + layout.image_point(*layout.inside(layout.right))), right_rgb)) + + rx, ry = layout.inside(layout.right, 5, 5) + region = [rx, ry, rx + 100, ry + 50] + cropped = check("grab_image(region) honours grim -g size", + lambda: _size_of(screen.grab_image(region))) + check("region is 100x50 as asked", lambda: _assert_eq(cropped, (100, 50))) + check("region landed on the right-hand output, not at the origin", + lambda: _assert_eq(screen.grab_image(region).getpixel((0, 0)), + right_rgb)) + lx, ly = layout.inside(layout.left, 5, 5) + check("a region on the left-hand output reads the left colour", + lambda: _assert_eq( + screen.grab_image([lx, ly, lx + 100, ly + 50]).getpixel((0, 0)), + left_rgb)) + + check("get_pixel() on the left-hand output", + lambda: _assert_eq(screen.get_pixel(*layout.inside(layout.left)), + left_rgb)) + if layout.multi: + check("get_pixel() on the right-hand output", + lambda: _assert_eq(screen.get_pixel(*layout.inside(layout.right)), + right_rgb)) + + +def check_public_paths(modules: Dict[str, Any], layout: Layout) -> None: + """The chain every locator, recorder and remote-desktop frame goes through.""" + screen = modules["screen"] + screen_grabber = modules["screen_grabber"] + left_rgb = layout.colour(layout.left) + left_point = layout.image_point(*layout.inside(layout.left)) + + def _screenshot_file(): + from PIL import Image + path = "/tmp/shot.png" + returned = screen.screenshot(path) + _assert_eq(returned, path) + with Image.open(path) as saved: + return f"{saved.width}x{saved.height} {saved.format}" + check("screen.screenshot() writes a real PNG", _screenshot_file) + + def _public_screenshot(): + import je_auto_control as ac + frame = ac.screenshot() + _assert_eq(frame.shape, (layout.height, layout.width, 3)) + # The wrapper converts to BGR for OpenCV, so the channels reverse. + pixel = tuple(int(v) for v in frame[left_point[1]][left_point[0]]) + _assert_eq(pixel, tuple(reversed(left_rgb))) + return f"shape={frame.shape} BGR pixel={pixel}" + check("je_auto_control.screenshot() returns the real screen in BGR", + _public_screenshot) + + def _pil_screenshot(): + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + return _size_of(pil_screenshot()) + check("pil_screenshot() goes through the backend", _pil_screenshot) + + def _grab_logical(): + from je_auto_control.utils.monitor_layout.logical_frame import grab_logical + frame, ox, oy = grab_logical() + _assert_eq(frame.getpixel(left_point), left_rgb) + # The origin is what a caller adds to a hit found in the frame, so on + # the shifted layout it has to be the negative one — a locator that + # gets (0, 0) here clicks 1280 px to the right of what it matched. + _assert_eq((ox, oy), (layout.origin_x, layout.origin_y)) + return f"{frame.width}x{frame.height} origin=({ox},{oy})" + check("grab_logical() — the locator and OCR capture path", _grab_logical) + + def _mss_shim(): + with screen_grabber.mss_grabber() as sct: + monitors = sct.monitors + _assert_eq((monitors[1]["left"], monitors[1]["top"]), + (layout.origin_x, layout.origin_y)) + shot = sct.grab(monitors[1]) + _assert_eq(shot.size, (layout.width, layout.height)) + # mss consumers read .bgra; four bytes per pixel, blue first. + offset = 4 * (left_point[1] * layout.width + left_point[0]) + _assert_eq(tuple(shot.bgra[offset:offset + 3]), tuple(reversed(left_rgb))) + return f"{len(monitors)} monitors, monitor[1] at {monitors[1]['left']},{monitors[1]['top']}" + check("mss-shaped shim (recorder / WebRTC / MCP path)", _mss_shim) + + +def check_fallback_crop(modules: Dict[str, Any], layout: Layout) -> None: + """The tiers that cannot apply a region themselves, cropped by us. + + grim is the only helper that takes a geometry; gnome-screenshot, + spectacle, the portal and an operator's own command all return the whole + layout and :func:`screen.grab_image` crops afterwards. Pointing the + operator override at grim is what puts a *real* whole-layout PNG through + that crop — and on the shifted layout a crop that forgets the origin + silently returns black padding instead of the left-hand monitor. + """ + capture = modules["capture"] + screen = modules["screen"] + if not layout.multi: + return + os.environ[capture.CAPTURE_COMMAND_ENV] = "grim {output}" + try: + check("operator override tier is selected", + lambda: _assert_eq(capture.available_tool(), + f"${capture.CAPTURE_COMMAND_ENV}")) + check("override returns the whole layout, region unapplied", + lambda: _assert_eq( + (lambda c: (_size_of_png(c.data), c.region_applied))( + capture.grab_png([0, 0, 10, 10])), + ((layout.width, layout.height), False))) + for output in (layout.left, layout.right): + x, y = layout.inside(output, 5, 5) + check(f"cropped region on {output['name']} reads its own colour", + lambda x=x, y=y, output=output: _assert_eq( + screen.grab_image([x, y, x + 100, y + 50]).getpixel((0, 0)), + layout.colour(output))) + check("get_pixel() through the cropping tier", + lambda: _assert_eq(screen.get_pixel(*layout.inside(layout.left)), + layout.colour(layout.left))) + finally: + del os.environ[capture.CAPTURE_COMMAND_ENV] + + +def main() -> int: + print("=" * 72) + print("AutoControl Wayland verification — real sway session") + print("=" * 72) + + layout = read_layout() + report_environment(layout) + + from je_auto_control.linux_wayland import _detect, capture, screen + from je_auto_control.linux_wayland import keyboard as wl_keyboard + from je_auto_control.utils.cv2_utils import screen_grabber + modules = {"detect": _detect, "capture": capture, "screen": screen, + "screen_grabber": screen_grabber} + + check_detection(modules) + check_geometry(screen, layout) + check_pixels(screen, layout) + check_public_paths(modules, layout) + check_fallback_crop(modules, layout) + + def _wtype(): + # Nothing is focused, so the keystrokes go nowhere. What is being + # checked is that wtype accepts this argv and exits cleanly — the + # part the unit tests could only assert against a mock. + wl_keyboard.write("autocontrol") + return "wtype accepted `wtype -- TEXT`" + check("wtype argv is accepted by the real binary", _wtype) + + # --- what this environment cannot answer ----------------------------- + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("ydotool — needs /dev/uinput, and sway's headless backend consumes") + note(" no libinput devices, so an injected event has nowhere to land.") + note("libei / RemoteDesktop portal — xdg-desktop-portal-wlr implements") + note(" ScreenCast and Screenshot but not RemoteDesktop, so there is no") + note(" ConnectToEIS here. Covered instead by docker/eis_verify.py (the") + note(" protocol) and docker/portal_verify.py (the D-Bus handshake).") + + # --- summary --------------------------------------------------------- + failed = [name for name, ok, _ in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +def _assert_eq(actual: Any, expected: Any) -> str: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + return repr(actual) + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +def _size_of(image: Any) -> Tuple[int, int]: + return (image.width, image.height) + + +def _size_of_png(data: bytes) -> Tuple[int, int]: + from io import BytesIO + + from PIL import Image + with Image.open(BytesIO(data)) as image: + return (image.width, image.height) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/ydotool_verify.py b/docker/ydotool_verify.py new file mode 100644 index 00000000..a530a773 --- /dev/null +++ b/docker/ydotool_verify.py @@ -0,0 +1,404 @@ +"""Verify AutoControl's ydotool argv against the events the kernel really gets. + +The Wayland image answers the capture half and the EIS image answers libei's +half. Both close by naming the same remaining gap — ydotool, which "needs +/dev/uinput and a seat that consumes it" — and that gap was recorded as +needing a GNOME VM. + +It does not need one. A seat is required for an injected event to *arrive +somewhere*, but not for it to be *observed*: ydotoold creates an ordinary +uinput device, the kernel exposes it as ``/dev/input/eventN``, and reading +that node gives back the exact ``input_event`` structs ydotool wrote. No +compositor is involved, so a container with ``/dev/uinput`` and the input +device cgroup can settle every claim this backend makes about the CLI: + + * ``click`` bitmasks — ``0xc0`` / ``0xc1`` / ``0xc2`` really are BTN_LEFT / + BTN_RIGHT / BTN_MIDDLE, and the split edges ``0x40`` / ``0x80`` really do + send a press without a release and a release without a press, which is + the whole basis of :func:`press_mouse` and drag; + * ``mousemove --absolute`` — what it puts on the wire, which is not what + the option name suggests; + * ``mousemove --wheel`` signs — the direction this project assumed from the + kernel's ``REL_WHEEL`` convention and never measured, plus that the axes + are not swapped; + * ``key CODE:STATE`` — numeric evdev codes, both edges. + +It also pins the reason :mod:`je_auto_control.linux_wayland._ydotool_cli` +exists: the same argv is replayed against whatever ydotool is installed, and +the legacy 0.1.x CLI answers ``0`` while emitting nothing. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import glob +import os +import struct +import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell +import sys +import time +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +_results: List[Tuple[str, bool]] = [] + +#: ``struct input_event``: two ``long`` for the timeval, then type/code/value. +_EVENT_FORMAT = "llHHi" +_EVENT_SIZE = struct.calcsize(_EVENT_FORMAT) + +EV_SYN, EV_KEY, EV_REL, EV_ABS = 0x00, 0x01, 0x02, 0x03 + +BTN_LEFT, BTN_RIGHT, BTN_MIDDLE = 272, 273, 274 +KEY_A, KEY_LEFTCTRL = 30, 29 +REL_X, REL_Y, REL_HWHEEL, REL_WHEEL = 0x00, 0x01, 0x06, 0x08 + +#: Input event nodes are character major 13, minor 64 + N. +_INPUT_MAJOR = 13 +_INPUT_MINOR_BASE = 64 + +#: ydotoold names its device this; matched case-insensitively. +_DEVICE_NAME_FRAGMENT = "ydotool" + +#: Long enough for ydotool's own default 100 ms start delay plus slack. +_SETTLE_SECONDS = 0.45 + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +# -------------------------------------------------------------------------- +# Reading the virtual device +# -------------------------------------------------------------------------- + +def _device_names() -> Dict[str, str]: + """Map ``eventN`` -> device name, straight out of sysfs.""" + names = {} + for path in glob.glob("/sys/class/input/event*"): + try: + with open(os.path.join(path, "device", "name"), + encoding="utf-8") as handle: + names[os.path.basename(path)] = handle.read().strip() + except OSError: + continue + return names + + +class VirtualDevice: + """Every ydotoold input node, opened non-blocking and read as a group. + + ydotoold may expose more than one node (a stale daemon from an earlier + run leaves one behind), and which one carries a given event is not + something this project should depend on, so they are drained together. + """ + + def __init__(self) -> None: + self._fds: Dict[str, int] = {} + + def open_all(self) -> List[str]: + os.makedirs("/dev/input", exist_ok=True) + opened = [] + for node_name, device_name in _device_names().items(): + if _DEVICE_NAME_FRAGMENT not in device_name.lower(): + continue + if node_name in self._fds: + continue + path = f"/dev/input/{node_name}" + if not os.path.exists(path): + minor = _INPUT_MINOR_BASE + int(node_name.replace("event", "")) + os.mknod(path, 0o600 | 0o020000, + os.makedev(_INPUT_MAJOR, minor)) + self._fds[node_name] = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + opened.append(path) + return opened + + def drain(self) -> List[Tuple[int, int, int]]: + """Return every pending ``(type, code, value)``, SYN frames dropped.""" + events = [] + for fd in self._fds.values(): + while True: + try: + data = os.read(fd, _EVENT_SIZE) + except BlockingIOError: + break + except OSError: + break + if not data or len(data) < _EVENT_SIZE: + break + _, _, etype, code, value = struct.unpack(_EVENT_FORMAT, data) + if etype != EV_SYN: + events.append((etype, code, value)) + return events + + def close(self) -> None: + for fd in self._fds.values(): + try: + os.close(fd) + except OSError: + pass + self._fds.clear() + + +def _emit(device: VirtualDevice, argv: List[str], + ) -> Tuple[int, str, List[Tuple[int, int, int]]]: + """Run one ydotool command and return ``(rc, stderr, events)``.""" + device.drain() + completed = subprocess.run( # nosec B603 # nosemgrep + argv, capture_output=True, timeout=20, check=False, + ) + time.sleep(_SETTLE_SECONDS) + stderr = completed.stderr.decode("utf-8", errors="replace").strip() + return completed.returncode, stderr, device.drain() + + +def _keys(events: List[Tuple[int, int, int]]) -> List[Tuple[int, int]]: + return [(code, value) for etype, code, value in events if etype == EV_KEY] + + +def _rel(events: List[Tuple[int, int, int]]) -> List[Tuple[int, int]]: + return [(code, value) for etype, code, value in events if etype == EV_REL] + + +# -------------------------------------------------------------------------- +# The checks +# -------------------------------------------------------------------------- + +def _button_check(device: VirtualDevice, tool: str, code: str, + expected: List[Tuple[int, int]]) -> str: + rc, stderr, events = _emit(device, [tool, "click", code]) + _require(rc == 0, f"ydotool click {code} exited {rc}: {stderr}") + seen = _keys(events) + _require(seen == expected, + f"click {code} produced {seen}, expected {expected}") + return f"{code} -> {seen}" + + +def _wheel_check(device: VirtualDevice, tool: str, delta_x: int, delta_y: int, + expected: List[Tuple[int, int]]) -> str: + rc, stderr, events = _emit(device, [ + tool, "mousemove", "--wheel", "-x", str(delta_x), "-y", str(delta_y)]) + _require(rc == 0, f"ydotool mousemove --wheel exited {rc}: {stderr}") + seen = [pair for pair in _rel(events) + if pair[0] in (REL_WHEEL, REL_HWHEEL)] + _require(seen == expected, + f"wheel ({delta_x}, {delta_y}) produced {seen}, " + f"expected {expected}") + return f"(-x {delta_x} -y {delta_y}) -> {seen}" + + +def _absolute_move_check(device: VirtualDevice, tool: str) -> str: + """``--absolute`` is relative-with-a-reset, and that is worth pinning. + + ydotool 1.x has no ABS axes on its device. It fakes an absolute move by + sending ``INT32_MIN`` on both axes first — which every compositor clamps + to the top-left corner — and then the target coordinates as a relative + delta from there. So AutoControl's ``set_position`` lands on the pixel it + asked for only because of that clamp, and the reset pair has to be + present: without it the move would be relative to wherever the cursor + already was. + """ + target_x, target_y = 640, 400 + rc, stderr, events = _emit(device, [ + tool, "mousemove", "--absolute", + "-x", str(target_x), "-y", str(target_y)]) + _require(rc == 0, f"ydotool mousemove --absolute exited {rc}: {stderr}") + moves = _rel(events) + _require(len(moves) == 4, + f"expected a reset pair then the target pair, got {moves}") + reset_x, reset_y, move_x, move_y = moves + int32_min = -(2 ** 31) + _require(reset_x == (REL_X, int32_min) and reset_y == (REL_Y, int32_min), + f"expected an INT32_MIN reset on both axes, got " + f"{reset_x}, {reset_y}") + _require(move_x == (REL_X, target_x) and move_y == (REL_Y, target_y), + f"expected the target as a relative delta, got {move_x}, {move_y}") + return "reset to origin, then REL_X 640 / REL_Y 400" + + +def _key_check(device: VirtualDevice, tool: str) -> str: + rc, stderr, events = _emit(device, [tool, "key", "30:1", "30:0"]) + _require(rc == 0, f"ydotool key exited {rc}: {stderr}") + seen = _keys(events) + _require(seen == [(KEY_A, 1), (KEY_A, 0)], + f"key 30:1 30:0 produced {seen}") + return f"30:1 30:0 -> {seen}" + + +def _backend_argv_check(device: VirtualDevice) -> str: + """Drive the real backend functions, not a hand-written argv. + + Everything above proves what ydotool does with a given command line. This + proves AutoControl builds that command line — the two halves are only + worth something together. + """ + os.environ["JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND"] = "cli" + from je_auto_control.linux_wayland import keyboard, mouse + + device.drain() + mouse.click_mouse(mouse.wayland_mouse_right) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [(BTN_RIGHT, 1), (BTN_RIGHT, 0)], + "mouse.click_mouse(right) did not reach the kernel as BTN_RIGHT") + + device.drain() + mouse.press_mouse(mouse.wayland_mouse_left) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [(BTN_LEFT, 1)], + "mouse.press_mouse(left) sent something other than a lone press") + mouse.release_mouse(mouse.wayland_mouse_left) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [(BTN_LEFT, 0)], + "mouse.release_mouse(left) sent something other than a lone " + "release") + + device.drain() + mouse.scroll(1, mouse.wayland_scroll_direction_up) + time.sleep(_SETTLE_SECONDS) + _require(_rel(device.drain()) == [(REL_WHEEL, 1)], + "mouse.scroll(up) did not reach the kernel as REL_WHEEL +1") + + device.drain() + keyboard.hotkey([KEY_LEFTCTRL, KEY_A]) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [ + (KEY_LEFTCTRL, 1), (KEY_A, 1), (KEY_A, 0), (KEY_LEFTCTRL, 0)], + "keyboard.hotkey did not press in order and release in reverse") + return "click / press / release / scroll / hotkey all land as intended" + + +def _legacy_cli_detection_check(tool: str) -> str: + """The installed CLI must be classified as the modern one. + + This is the sentinel for :mod:`_ydotool_cli`: if a future ydotool changes + its usage banner, the probe starts answering ``unknown``, the guard stops + protecting anyone, and this check is what says so. + """ + from je_auto_control.linux_wayland import _ydotool_cli + + _ydotool_cli.reset_cache() + generation = _ydotool_cli.cli_generation(tool) + _require(generation == _ydotool_cli.MODERN, + f"the probe classified this ydotool as {generation!r}, not " + f"{_ydotool_cli.MODERN!r}; the 0.1.x guard is now blind") + _require(_ydotool_cli.reject_legacy_cli(tool) == tool, + "reject_legacy_cli refused a modern ydotool") + return generation + + +def _start_daemon() -> subprocess.Popen: + # Debian's ydotoold binds $XDG_RUNTIME_DIR/.ydotool_socket, and creates + # neither the directory nor a fallback: without it the daemon still comes + # up and still creates the uinput device, but every client exits 2 with an + # empty stderr. Upstream's default is /tmp/.ydotool_socket, so which path + # is in use is a packaging detail — making the directory is what keeps + # both working. + runtime_dir = os.environ.get("XDG_RUNTIME_DIR") + if runtime_dir: + os.makedirs(runtime_dir, mode=0o700, exist_ok=True) + os.chmod(runtime_dir, 0o700) + daemon = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["ydotoold"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, + ) + deadline = time.monotonic() + 15.0 + while time.monotonic() < deadline: + if any(_DEVICE_NAME_FRAGMENT in name.lower() + for name in _device_names().values()): + time.sleep(0.5) # let the node settle before the first read + return daemon + if daemon.poll() is not None: + raise RuntimeError(f"ydotoold exited {daemon.returncode}") + time.sleep(0.1) + raise RuntimeError("ydotoold created no input device within 15s") + + +def _tool_path() -> str: + from je_auto_control.linux_wayland._detect import binary_path + path = binary_path("ydotool") + if path is None: + raise RuntimeError("ydotool is not on PATH") + return path + + +def main() -> int: + print("AutoControl ydotool verification — real /dev/uinput, no compositor") + print("=" * 70) + + if not os.path.exists("/dev/uinput"): + print("FAIL /dev/uinput is missing. Run the container with " + "--device /dev/uinput (and `modprobe uinput evdev` on the host).") + return 1 + + tool = _tool_path() + daemon: Optional[subprocess.Popen] = None + device = VirtualDevice() + try: + daemon = _start_daemon() + nodes = device.open_all() + if not nodes: + print("FAIL ydotoold's device has no evdev node. The host kernel " + "needs the evdev handler (`modprobe evdev`), and the " + "container needs --device-cgroup-rule 'c 13:* rmw'.") + return 1 + print(f"reading {', '.join(nodes)}\n") + + check("click 0xc0 is BTN_LEFT down then up", + lambda: _button_check(device, tool, "0xc0", + [(BTN_LEFT, 1), (BTN_LEFT, 0)])) + check("click 0x40 is a press with no release", + lambda: _button_check(device, tool, "0x40", [(BTN_LEFT, 1)])) + check("click 0x80 is a release with no press", + lambda: _button_check(device, tool, "0x80", [(BTN_LEFT, 0)])) + check("click 0xc1 is BTN_RIGHT", + lambda: _button_check(device, tool, "0xc1", + [(BTN_RIGHT, 1), (BTN_RIGHT, 0)])) + check("click 0xc2 is BTN_MIDDLE", + lambda: _button_check(device, tool, "0xc2", + [(BTN_MIDDLE, 1), (BTN_MIDDLE, 0)])) + check("mousemove --absolute resets to the origin first", + lambda: _absolute_move_check(device, tool)) + check("wheel +y is REL_WHEEL positive (up, per the kernel convention)", + lambda: _wheel_check(device, tool, 0, 1, [(REL_WHEEL, 1)])) + check("wheel -y is REL_WHEEL negative (down)", + lambda: _wheel_check(device, tool, 0, -1, [(REL_WHEEL, -1)])) + check("wheel +x is REL_HWHEEL positive (right), axes not swapped", + lambda: _wheel_check(device, tool, 2, 0, [(REL_HWHEEL, 2)])) + check("key CODE:STATE sends numeric evdev codes", + lambda: _key_check(device, tool)) + check("the version probe recognises this CLI as modern", + lambda: _legacy_cli_detection_check(tool)) + check("the backend's own argv lands as intended", + lambda: _backend_argv_check(device)) + finally: + device.close() + if daemon is not None: + daemon.terminate() + try: + daemon.wait(timeout=5) + except subprocess.TimeoutExpired: + daemon.kill() + + failed = [name for name, ok in _results if not ok] + print("\n" + "=" * 70) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" failed: {name}") + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test/unit_test/headless/test_docker_artifacts.py b/test/unit_test/headless/test_docker_artifacts.py index 7bb24cf8..5abf85c0 100644 --- a/test/unit_test/headless/test_docker_artifacts.py +++ b/test/unit_test/headless/test_docker_artifacts.py @@ -10,7 +10,11 @@ import pytest -_DOCKER_DIR = Path(__file__).resolve().parents[3] / "docker" +_REPO_ROOT = Path(__file__).resolve().parents[3] +_DOCKER_DIR = _REPO_ROOT / "docker" +#: Spelled from ``chr`` so this file stays byte-identical under any +#: checkout normalisation that might rewrite a literal in it. +CRLF = (chr(13) + chr(10)).encode() def test_dockerfile_exists_and_uses_python_base(): @@ -43,13 +47,47 @@ def test_compose_file_declares_three_services(): assert "8765:8765" in raw +def test_dockerignore_sits_at_the_build_context_root(): + """Docker reads ``.dockerignore`` from the build *context* root only. + + Every documented build passes the repository root as the context + (``docker build -f docker/Dockerfile .``), so a copy next to the + Dockerfile is never read: the whole tree -- .git, .venv, test/ -- is + shipped to the daemon and the exclusions do nothing. + """ + assert (_REPO_ROOT / ".dockerignore").is_file(), ( + ".dockerignore must live at the repository root, not in docker/") + assert not (_DOCKER_DIR / ".dockerignore").exists(), ( + "a docker/.dockerignore is dead weight; Docker never reads it") + + def test_dockerignore_keeps_build_context_lean(): - raw = (_DOCKER_DIR / ".dockerignore").read_text(encoding="utf-8") + raw = (_REPO_ROOT / ".dockerignore").read_text(encoding="utf-8") # The biggest space-wasters should all be excluded. for line in ("test/", "docs/", "__pycache__", "*.egg-info"): assert line in raw, f".dockerignore missing {line}" +@pytest.mark.parametrize("script", [ + "entrypoint.sh", "entrypoint-xfce.sh", "entrypoint-wayland.sh", + "entrypoint-seat.sh", +]) +def test_entrypoints_keep_unix_line_endings(script): + """A CRLF shebang makes an image that builds and then cannot start. + + A carriage return at the end of the shebang sends the kernel looking + for an interpreter whose name ends in one, and every container dies + with the useless ``exec ...: no such file or directory``. + ``.gitattributes`` pins ``*.sh text eol=lf`` so a Windows checkout + cannot reintroduce it; this fails if that pin is ever dropped. + """ + path = _DOCKER_DIR / script + if not path.exists(): # optional variants stay optional + pytest.skip(f"{script} not present") + assert CRLF not in path.read_bytes(), ( + f"{script} has CRLF line endings; its container cannot exec /bin/sh") + + @pytest.mark.parametrize("expected_port", ["9939", "9940", "8765"]) def test_dockerfile_exposes_each_service_port(expected_port): raw = (_DOCKER_DIR / "Dockerfile").read_text(encoding="utf-8") @@ -94,6 +132,113 @@ def test_github_actions_docker_workflow_exists(): assert "pytest" in raw +def test_ydotool_verification_image_and_script_exist(): + """The ydotool path is only verified while these two files are wired up.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.ydotool").read_text( + encoding="utf-8") + # ydotool 1.0 replaced the CLI; trixie ships none and bookworm ships + # 0.1.8, so the image has to pull the one under test from unstable. + assert "sid" in dockerfile + assert "ydotool" in dockerfile + assert "ydotool_verify.py" in dockerfile + + script = (_DOCKER_DIR / "ydotool_verify.py").read_text(encoding="utf-8") + # Reading the kernel device back is the whole point; a version that only + # checks exit codes would pass against the CLI that exits 0 and emits + # nothing, which is the bug this exists for. + assert "/dev/input" in script + assert "input_event" in script + + +def test_ydotool_verification_job_grants_uinput_and_the_input_cgroup(): + """--device cannot cover a node ydotoold creates after the container starts.""" + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "ydotool-verification" in raw + assert "modprobe uinput" in raw + assert "--device /dev/uinput" in raw + assert "c 13:* rmw" in raw + + +def test_seat_verification_image_and_script_exist(): + """The one image where an injected event reaches a compositor. + + The other two Wayland images each hold one half still: the capture image + runs a compositor that consumes no input, the ydotool image reads the + kernel with no compositor. This one needs all four of the settings that + join them, and dropping any of them turns the verification into a + measurement of an empty seat that still passes. + """ + dockerfile = (_DOCKER_DIR / "Dockerfile.seat").read_text(encoding="utf-8") + for setting in ("WLR_BACKENDS=headless,libinput", + "LIBSEAT_BACKEND=builtin", + "SEATD_VTBOUND=0"): + assert setting in dockerfile, f"Dockerfile.seat missing {setting}" + # libinput enumerates through udev, not through /dev, so udevd has to be + # in the image or sway comes up holding nothing. + assert "udev" in dockerfile + assert "seat_verify.py" in dockerfile + + entrypoint = (_DOCKER_DIR / "entrypoint-seat.sh").read_text( + encoding="utf-8") + # Coming up with an empty seat must fail here rather than downstream. + assert "libinput list-devices" in entrypoint + # Both layouts, as the capture image does: the negative-origin one is the + # only one where the layout corner and layout (0, 0) differ. + assert "-1280 0" in entrypoint + + script = (_DOCKER_DIR / "seat_verify.py").read_text(encoding="utf-8") + # grim -c is what makes the cursor visible to a screenshot at all. + assert '"-c"' in script + # And the two findings the image exists to hold. + assert "layout corner" in script + assert "acceleration" in script + + +def test_seat_verification_job_grants_uinput_and_the_input_cgroup(): + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "seat-verification" in raw + assert "docker/Dockerfile.seat" in raw + assert "autocontrol-seat:ci" in raw + + +def test_portal_verification_image_and_scripts_exist(): + """The portal path is only verified while these three files are wired up.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.portal").read_text(encoding="utf-8") + # liboeffis is a separate binary package that libei1 does not depend on, + # so installing libei alone leaves the portal route quietly off. + assert "liboeffis1" in dockerfile + assert "portal_verify.py" in dockerfile + assert "portal_server.py" in dockerfile + # gdbus must stay out: the Screenshot tier speaks D-Bus itself now, and + # this image is the proof that it needs no binary beyond a session bus. + assert "libglib2.0-bin" not in dockerfile + + server = (_DOCKER_DIR / "portal_server.py").read_text(encoding="utf-8") + # A portal that answers only CreateSession is not a portal; the fd handover + # at the end of the dance is the whole point. + for method in ("CreateSession", "SelectDevices", "Start", "ConnectToEIS"): + assert method in server, f"mock portal missing {method}" + assert "UnixFDList" in server, "the mock must really pass a descriptor" + + verify = (_DOCKER_DIR / "portal_verify.py").read_text(encoding="utf-8") + # Refusals are half the value: a portal that says no must fail closed. + for behaviour in ("deny", "stall", "no-fd", "close"): + assert f'"{behaviour}"' in verify, f"no scenario drives {behaviour}" + + +def test_portal_verification_job_runs_the_image(): + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "portal-verification" in raw + assert "docker/Dockerfile.portal" in raw + assert "autocontrol-portal:ci" in raw + + def test_gitlab_template_covers_build_test_smoke_stages(): raw = (_REPO_ROOT / "ci_templates" / ".gitlab-ci.yml").read_text( encoding="utf-8", From de2b1d7af0b9aacfaca478e6d3899b1e45742a53 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:14:43 +0800 Subject: [PATCH 07/21] Read the whole Wayland layout, and talk to the portal directly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two capture defects that only a real compositor could show. A monitor left of or above the primary one puts the layout's top-left pixel at a negative coordinate, and three places assumed (0, 0): size() returned the layout's right edge rather than its width, so every caller composing size with a capture asked for half the desktop; the crop applied when a tier cannot take a region itself cropped in layout coordinates on a layout-origin image, returning black padding; and grab_logical() reported origin (0, 0), so a match on that monitor was reported a monitor's width to the right and the click missed. The backend now publishes layout_origin(), size() returns the bounding box, and the crop subtracts the origin. The xdg-desktop-portal tier could never have worked. A portal Response is directed at the connection that made the call, and the tier listened on a `gdbus monitor` subprocess while calling from a separate `gdbus` invocation — two connections, so the listener was never the addressee. Against a real bus every capture ran its full 30-second timeout. The tier now speaks D-Bus itself on one connection, subscribing to the request path it predicts before it calls, which also drops the gdbus binary requirement: _dbus_client is stdlib-only. --- je_auto_control/linux_wayland/_dbus_client.py | 624 ++++++++++++++++++ je_auto_control/linux_wayland/_detect.py | 18 +- je_auto_control/linux_wayland/_layout.py | 83 +++ je_auto_control/linux_wayland/capture.py | 236 +++++++ je_auto_control/linux_wayland/portal.py | 207 ++++++ je_auto_control/linux_wayland/screen.py | 295 ++++++--- .../headless/test_wayland_backend.py | 347 +++++++++- .../headless/test_wayland_capture_tiers.py | 339 ++++++++++ .../headless/test_wayland_dbus_client.py | 306 +++++++++ 9 files changed, 2328 insertions(+), 127 deletions(-) create mode 100644 je_auto_control/linux_wayland/_dbus_client.py create mode 100644 je_auto_control/linux_wayland/_layout.py create mode 100644 je_auto_control/linux_wayland/capture.py create mode 100644 je_auto_control/linux_wayland/portal.py create mode 100644 test/unit_test/headless/test_wayland_capture_tiers.py create mode 100644 test/unit_test/headless/test_wayland_dbus_client.py diff --git a/je_auto_control/linux_wayland/_dbus_client.py b/je_auto_control/linux_wayland/_dbus_client.py new file mode 100644 index 00000000..b146eede --- /dev/null +++ b/je_auto_control/linux_wayland/_dbus_client.py @@ -0,0 +1,624 @@ +"""A minimal D-Bus session-bus client, in the standard library alone. + +This exists because of one property of the XDG portal protocol: a portal call +returns only a *request handle*, and the answer arrives later as a ``Response`` +signal — **directed at the unique bus name that made the call**. The bus routes +a directed message to its destination and nowhere else, so a listener on a +second connection never receives it, whatever match rules it adds. + +That rules out the obvious shell-out. ``gdbus call`` and ``gdbus monitor`` are +two processes with two unique names, so the monitor is not the caller and the +Response is not addressed to it; measured against a real ``dbus-daemon``, it +sees the call go past and the answer never arrive. The only listener that +does see it is a full bus monitor (``dbus-monitor``, which asks the bus for +``BecomeMonitor``), and making a screenshot require permission to observe every +message on the user's session bus is a poor trade for a fallback tier. + +So the subscription and the call happen on one connection here, which is what +every portal client library does and what the protocol is designed for. The +cost is marshalling D-Bus by hand; the scope is kept to exactly what a portal +conversation needs — connect, authenticate, ``Hello``, ``AddMatch``, one method +call, and read signals until the matching one arrives. It is deliberately not a +general D-Bus binding: no properties, no introspection, no object export, no +file-descriptor passing (:mod:`je_auto_control.linux_wayland.oeffis` uses +liboeffis for the one call that needs that). + +Every failure is an :class:`AutoControlException` subclass, so the containment +boundaries that catch the family keep working. +""" +from __future__ import annotations + +import contextlib +import os +import socket +import struct +import time +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: Message types, from the D-Bus specification. +METHOD_CALL = 1 +METHOD_RETURN = 2 +ERROR = 3 +SIGNAL = 4 + +#: Header field codes. +FIELD_PATH = 1 +FIELD_INTERFACE = 2 +FIELD_MEMBER = 3 +FIELD_ERROR_NAME = 4 +FIELD_REPLY_SERIAL = 5 +FIELD_DESTINATION = 6 +FIELD_SENDER = 7 +FIELD_SIGNATURE = 8 + +#: ``NO_REPLY_EXPECTED``. Declared for completeness; nothing here uses it, +#: because every call this makes is one whose answer is worth waiting for. +FLAG_NO_REPLY_EXPECTED = 1 + +BUS_NAME = "org.freedesktop.DBus" +BUS_PATH = "/org/freedesktop/DBus" +BUS_INTERFACE = "org.freedesktop.DBus" + +_ADDRESS_ENV = "DBUS_SESSION_BUS_ADDRESS" +_MAX_MESSAGE = 128 * 1024 * 1024 # the specification's own ceiling +_ALIGNMENT = {"y": 1, "b": 4, "n": 2, "q": 2, "i": 4, "u": 4, + "x": 8, "t": 8, "d": 8, "s": 4, "o": 4, "g": 1, + "a": 4, "(": 8, "{": 8, "v": 1, "h": 4} + + +class DBusError(AutoControlException): + """The session bus is unreachable, or answered with an error.""" + + +class Variant: + """A value with an explicit D-Bus type, for the ``a{sv}`` option maps.""" + + __slots__ = ("signature", "value") + + def __init__(self, signature: str, value: Any) -> None: + self.signature = signature + self.value = value + + def __repr__(self) -> str: + return f"Variant({self.signature!r}, {self.value!r})" + + +# --- marshalling ---------------------------------------------------------- + + +class _Writer: + """Little-endian marshaller. Alignment is relative to the message start.""" + + def __init__(self, offset: int = 0) -> None: + self._parts: List[bytes] = [] + self._length = offset + + def align(self, boundary: int) -> None: + padding = (-self._length) % boundary + if padding: + self._parts.append(bytes(padding)) + self._length += padding + + def raw(self, data: bytes) -> None: + self._parts.append(data) + self._length += len(data) + + def byte(self, value: int) -> None: + self.raw(struct.pack(" None: + self.align(4) + self.raw(struct.pack(" None: + encoded = value.encode("utf-8") + self.uint32(len(encoded)) + self.raw(encoded + b"\x00") + + def signature(self, value: str) -> None: + encoded = value.encode("ascii") + self.byte(len(encoded)) + self.raw(encoded + b"\x00") + + def value(self, sig: str, value: Any) -> None: + """Write one complete value of the given single signature.""" + _write_value(self, _SignatureReader(sig), value) + + @property + def data(self) -> bytes: + return b"".join(self._parts) + + def __len__(self) -> int: + return self._length + + +class _SignatureReader: + """Walks a signature string one complete type at a time.""" + + def __init__(self, text: str) -> None: + self.text = text + self.index = 0 + + def done(self) -> bool: + return self.index >= len(self.text) + + def peek(self) -> str: + return self.text[self.index] + + def take(self) -> str: + code = self.text[self.index] + self.index += 1 + return code + + def take_complete(self) -> str: + """Take one whole type, following containers to their close.""" + start = self.index + code = self.take() + if code == "a": + self.take_complete() + elif code in "({": + closing = ")" if code == "(" else "}" + while self.peek() != closing: + self.take_complete() + self.take() + return self.text[start:self.index] + + +def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + code = reader.take() + if code == "y": + writer.byte(int(value)) + elif code == "b": + writer.uint32(1 if value else 0) + elif code == "u": + writer.uint32(int(value)) + elif code in "so": + writer.string(str(value)) + elif code == "g": + writer.signature(str(value)) + elif code == "v": + _write_variant(writer, value) + elif code == "a": + _write_array(writer, reader, value) + elif code in "({": + _write_struct(writer, reader, code, value) + else: + raise DBusError(f"cannot marshal D-Bus type {code!r}") + + +def _write_variant(writer: _Writer, value: Any) -> None: + variant = value if isinstance(value, Variant) else _guess_variant(value) + writer.signature(variant.signature) + writer.value(variant.signature, variant.value) + + +def _guess_variant(value: Any) -> Variant: + if isinstance(value, bool): + return Variant("b", value) + if isinstance(value, int): + return Variant("u", value) + if isinstance(value, str): + return Variant("s", value) + raise DBusError(f"no obvious D-Bus type for {type(value).__name__}") + + +def _write_array(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + element = reader.take_complete() + writer.align(4) + # The length prefix counts the elements only, and it is written before + # the padding that aligns the first one — so the body is built separately + # at the offset it will actually occupy. + body_start = len(writer) + 4 + padding = (-body_start) % _ALIGNMENT.get(element[0], 1) + body = _Writer(body_start + padding) + items = value.items() if isinstance(value, dict) else value + for item in items: + body.align(_ALIGNMENT.get(element[0], 1)) + _write_value(body, _SignatureReader(element), item) + writer.uint32(len(body) - body_start - padding) + writer.raw(bytes(padding)) + writer.raw(body.data) + + +def _write_struct(writer: _Writer, reader: _SignatureReader, code: str, + value: Any) -> None: + closing = ")" if code == "(" else "}" + writer.align(8) + for item in value: + if reader.peek() == closing: + raise DBusError("too many members for this struct signature") + _write_value(writer, reader, item) + if reader.peek() != closing: + raise DBusError("too few members for this struct signature") + reader.take() + + +class _Reader: + """Little-endian demarshaller over one complete message.""" + + def __init__(self, data: bytes, offset: int = 0) -> None: + self.data = data + self.offset = offset + + def align(self, boundary: int) -> None: + self.offset += (-self.offset) % boundary + + def take(self, count: int) -> bytes: + if self.offset + count > len(self.data): + raise DBusError("truncated D-Bus message") + chunk = self.data[self.offset:self.offset + count] + self.offset += count + return chunk + + def byte(self) -> int: + return self.take(1)[0] + + def uint32(self) -> int: + self.align(4) + return struct.unpack(" str: + length = self.uint32() + text = self.take(length).decode("utf-8", errors="replace") + self.take(1) + return text + + def signature(self) -> str: + length = self.byte() + text = self.take(length).decode("ascii", errors="replace") + self.take(1) + return text + + def value(self, sig: str) -> Any: + return _read_value(self, _SignatureReader(sig)) + + +def _read_value(reader: _Reader, signature: _SignatureReader) -> Any: + code = signature.take() + if code == "y": + return reader.byte() + if code == "b": + return bool(reader.uint32()) + if code == "u": + return reader.uint32() + if code in "so": + return reader.string() + if code == "g": + return reader.signature() + if code == "v": + return reader.value(reader.signature()) + if code == "a": + return _read_array(reader, signature) + if code in "({": + return _read_struct(reader, signature, code) + raise DBusError(f"cannot demarshal D-Bus type {code!r}") + + +def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: + element = signature.take_complete() + length = reader.uint32() + reader.align(_ALIGNMENT.get(element[0], 1)) + end = reader.offset + length + items = [] + while reader.offset < end: + reader.align(_ALIGNMENT.get(element[0], 1)) + items.append(_read_value(reader, _SignatureReader(element))) + if element.startswith("{"): + return {key: value for key, value in items} + return items + + +def _read_struct(reader: _Reader, signature: _SignatureReader, + code: str) -> tuple: + closing = ")" if code == "(" else "}" + reader.align(8) + members = [] + while signature.peek() != closing: + members.append(_read_value(reader, signature)) + signature.take() + return tuple(members) + + +class Message: + """One decoded D-Bus message: the header fields that matter, and the body.""" + + __slots__ = ("type", "serial", "fields", "body") + + def __init__(self, message_type: int, serial: int, + fields: Dict[int, Any], body: List[Any]) -> None: + self.type = message_type + self.serial = serial + self.fields = fields + self.body = body + + @property + def path(self) -> str: + return self.fields.get(FIELD_PATH, "") + + @property + def interface(self) -> str: + return self.fields.get(FIELD_INTERFACE, "") + + @property + def member(self) -> str: + return self.fields.get(FIELD_MEMBER, "") + + @property + def reply_serial(self) -> int: + return self.fields.get(FIELD_REPLY_SERIAL, 0) + + @property + def error_name(self) -> str: + return self.fields.get(FIELD_ERROR_NAME, "") + + +# --- the connection ------------------------------------------------------- + + +def session_address() -> Optional[str]: + """The session bus address, or None when this process has no session bus.""" + address = os.environ.get(_ADDRESS_ENV, "").strip() + return address or None + + +def is_available() -> bool: + """Whether a session bus address is set, so connecting is worth trying.""" + return session_address() is not None + + +def _socket_target(address: str) -> Tuple[str, bool]: + """Pick a connectable ``unix:`` transport out of a bus address. + + :return: the socket path and whether it is in the abstract namespace. + """ + for candidate in address.split(";"): + if not candidate.startswith("unix:"): + continue + options = dict( + part.split("=", 1) for part in candidate[len("unix:"):].split(",") + if "=" in part) + if "path" in options: + return options["path"], False + if "abstract" in options: + return options["abstract"], True + raise DBusError(f"no usable unix transport in {address!r}") + + +class SessionBus: + """An authenticated connection to the session bus, as a context manager.""" + + def __init__(self, address: Optional[str] = None) -> None: + self.address = address or session_address() + self.unique_name = "" + self._socket: Optional[socket.socket] = None + self._serial = 0 + self._buffer = b"" + #: Messages read while waiting for a method reply. A portal's Response + #: signal and its method return come from two different senders, so + #: the bus gives no ordering between them and the signal can arrive + #: first — dropping it here would be a wait that never ends. + self._queued: List[Message] = [] + + # --- lifecycle -------------------------------------------------------- + + def __enter__(self) -> "SessionBus": + self.connect() + return self + + def __exit__(self, *_exception: Any) -> None: + self.close() + + def connect(self) -> None: + """Open the socket, authenticate, and say ``Hello``.""" + if self.address is None: + raise DBusError( + f"{_ADDRESS_ENV} is not set, so there is no session bus to " + f"reach the desktop portal on", + ) + path, abstract = _socket_target(self.address) + try: + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._socket.connect(("\0" + path) if abstract else path) + except OSError as error: + self.close() + raise DBusError(f"cannot reach the session bus: {error}") from error + self._authenticate() + self.unique_name = self.call( + BUS_NAME, BUS_PATH, BUS_INTERFACE, "Hello", "", [])[0] + + def close(self) -> None: + """Drop the connection; never raise from teardown.""" + if self._socket is not None: + with contextlib.suppress(OSError): + self._socket.close() + self._socket = None + + @property + def sender_token(self) -> str: + """This connection's name as the portal spells it inside object paths.""" + return self.unique_name.lstrip(":").replace(".", "_") + + # --- authentication --------------------------------------------------- + + def _authenticate(self) -> None: + """The SASL EXTERNAL handshake, which is a uid in hex over a socket.""" + uid = str(os.getuid()).encode("ascii") + self._send_raw(b"\x00AUTH EXTERNAL " + uid.hex().encode("ascii") + + b"\r\n") + reply = self._read_line() + if not reply.startswith("OK"): + raise DBusError(f"the session bus refused authentication: {reply}") + self._send_raw(b"BEGIN\r\n") + + def _send_raw(self, data: bytes) -> None: + if self._socket is None: + raise DBusError("the session bus connection is closed") + try: + self._socket.sendall(data) + except OSError as error: + raise DBusError(f"writing to the session bus failed: {error}") from error + + def _read_line(self, timeout: float = 10.0) -> str: + """Read one CRLF-terminated line of the auth conversation.""" + deadline = time.monotonic() + timeout + while b"\r\n" not in self._buffer: + self._fill(deadline - time.monotonic()) + line, _, self._buffer = self._buffer.partition(b"\r\n") + return line.decode("utf-8", errors="replace") + + # --- reading ---------------------------------------------------------- + + def _fill(self, timeout: float) -> None: + """Read whatever is available, or fail once the deadline has passed.""" + if self._socket is None: + raise DBusError("the session bus connection is closed") + if timeout <= 0: + raise DBusError("the session bus did not answer in time") + self._socket.settimeout(timeout) + try: + chunk = self._socket.recv(65536) + except socket.timeout as error: + raise DBusError("the session bus did not answer in time") from error + except OSError as error: + raise DBusError(f"reading from the session bus failed: {error}") from error + if not chunk: + raise DBusError("the session bus closed the connection") + self._buffer += chunk + + def read_message(self, deadline: float) -> Message: + """Read one whole message, or fail when the deadline passes.""" + while len(self._buffer) < 16: + self._fill(deadline - time.monotonic()) + endian, message_type, _flags, _version = struct.unpack( + " _MAX_MESSAGE: + raise DBusError("the session bus sent an implausibly large message") + while len(self._buffer) < total: + self._fill(deadline - time.monotonic()) + raw, self._buffer = self._buffer[:total], self._buffer[total:] + return _decode(raw, message_type, serial, fields_length, header_end) + + # --- writing ---------------------------------------------------------- + + def _next_serial(self) -> int: + self._serial += 1 + return self._serial + + def send(self, message_type: int, fields: Dict[int, Tuple[str, Any]], + signature: str, body: List[Any], flags: int = 0) -> int: + """Marshal and send one message; return its serial.""" + payload = _Writer() + for code, value in body_pairs(signature, body): + payload.value(code, value) + serial = self._next_serial() + header = _Writer() + header.raw(struct.pack(" List[Any]: + """Make a method call and return the reply body.""" + fields = { + FIELD_PATH: ("o", path), + FIELD_INTERFACE: ("s", interface), + FIELD_MEMBER: ("s", member), + FIELD_DESTINATION: ("s", destination), + } + if signature: + fields[FIELD_SIGNATURE] = ("g", signature) + serial = self.send(METHOD_CALL, fields, signature, body) + deadline = time.monotonic() + timeout + while True: + message = self.read_message(deadline) + if message.reply_serial != serial: + self._queued.append(message) + continue + if message.type == ERROR: + detail = message.body[0] if message.body else "" + raise DBusError(f"{message.error_name}: {detail}".strip(": ")) + if message.type == METHOD_RETURN: + return message.body + + def add_match(self, rule: str) -> None: + """Subscribe to signals, and wait for the bus to confirm the rule. + + Waiting matters twice over: a malformed rule is reported rather than + silently never matching, and the round trip proves the subscription is + in place before the call that provokes the signal is made. + """ + self.call(BUS_NAME, BUS_PATH, BUS_INTERFACE, "AddMatch", "s", [rule]) + + def wait_for_signal(self, paths: List[str], interface: str, member: str, + timeout: float) -> List[Any]: + """Read until the signal we subscribed to arrives, or time out.""" + def matches(message: Message) -> bool: + return (message.type == SIGNAL and message.member == member + and message.interface == interface + and message.path in paths) + + for index, message in enumerate(self._queued): + if matches(message): + del self._queued[index] + return message.body + self._queued.clear() + deadline = time.monotonic() + max(0.0, timeout) + while True: + message = self.read_message(deadline) + if matches(message): + return message.body + + +def body_pairs(signature: str, body: List[Any]): + """Pair each top-level type in a signature with its argument.""" + reader = _SignatureReader(signature) + for value in body: + if reader.done(): + raise DBusError("more arguments than the signature declares") + yield reader.take_complete(), value + if not reader.done(): + raise DBusError("fewer arguments than the signature declares") + + +def _decode(raw: bytes, message_type: int, serial: int, fields_length: int, + header_end: int) -> Message: + """Turn one complete message's bytes into a :class:`Message`.""" + reader = _Reader(raw, 12) + fields: Dict[int, Any] = {} + reader.uint32() # the header array's own length, already read + end = 16 + fields_length + while reader.offset < end: + reader.align(8) + code = reader.byte() + fields[code] = reader.value(reader.signature()) + reader.offset = header_end + ((-header_end) % 8) + body: List[Any] = [] + signature = fields.get(FIELD_SIGNATURE, "") + if signature: + walker = _SignatureReader(signature) + while not walker.done(): + body.append(_read_value(reader, _SignatureReader( + walker.take_complete()))) + return Message(message_type, serial, fields, body) + + +__all__ = [ + "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", + "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", + "Variant", "is_available", "session_address", +] diff --git a/je_auto_control/linux_wayland/_detect.py b/je_auto_control/linux_wayland/_detect.py index e7e00d61..98bfff4d 100644 --- a/je_auto_control/linux_wayland/_detect.py +++ b/je_auto_control/linux_wayland/_detect.py @@ -13,7 +13,18 @@ WAYLAND_WTYPE = "wtype" WAYLAND_YDOTOOL = "ydotool" + +# Screen capture is the one place where no single tool covers Wayland: +# grim speaks wlr-screencopy (sway / Hyprland / river), GNOME answers +# through gnome-screenshot and KDE through spectacle. See +# :mod:`je_auto_control.linux_wayland.capture` for the order they are tried in. WAYLAND_GRIM = "grim" +WAYLAND_GNOME_SCREENSHOT = "gnome-screenshot" +WAYLAND_SPECTACLE = "spectacle" +WAYLAND_WLR_RANDR = "wlr-randr" +# xdg-desktop-portal needs no binary at all: je_auto_control.linux_wayland. +# portal speaks D-Bus itself, because the portal's answer is directed at the +# connection that asked and no separate CLI process can be that connection. _ENV_OVERRIDE = "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER" @@ -59,7 +70,8 @@ def missing_dependencies(required: Iterable[str]) -> List[str]: __all__ = [ - "WAYLAND_GRIM", "WAYLAND_WTYPE", "WAYLAND_YDOTOOL", - "binary_path", "is_wayland_session", "missing_dependencies", - "select_display_server", + "WAYLAND_GNOME_SCREENSHOT", "WAYLAND_GRIM", + "WAYLAND_SPECTACLE", "WAYLAND_WLR_RANDR", "WAYLAND_WTYPE", + "WAYLAND_YDOTOOL", "binary_path", "is_wayland_session", + "missing_dependencies", "select_display_server", ] diff --git a/je_auto_control/linux_wayland/_layout.py b/je_auto_control/linux_wayland/_layout.py new file mode 100644 index 00000000..fdf82dff --- /dev/null +++ b/je_auto_control/linux_wayland/_layout.py @@ -0,0 +1,83 @@ +"""The output layout's origin, for the input paths that must translate into it. + +Capture and input do not share a coordinate space on Wayland, and the gap is +the layout origin: :func:`je_auto_control.linux_wayland.screen.grab_image` +returns the whole output layout, whose top-left pixel is +:func:`~je_auto_control.linux_wayland.screen.layout_origin` and only ``(0, 0)`` +while every output sits at a non-negative position. Put a monitor left of the +primary one and the origin goes negative, while both input transports address +a space that starts at that corner: + +* libei advertises regions with ``uint32`` offsets, so it *cannot* describe a + point left of or above the origin; +* ``ydotool mousemove --absolute`` drives the cursor into the corner the + compositor clamps to and then moves relative to it, and that corner is the + layout's top-left. + +Both therefore need the same subtraction, which is why it lives here rather +than in either of them. It is a separate module because +:mod:`je_auto_control.linux_wayland.screen` imports Pillow at module scope and +the input paths must keep working on a host without it — a missing capture +tool means no correction, never a failed move. +""" +from __future__ import annotations + +import time +from typing import Dict, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: How long a reading stays good for. +#: +#: :mod:`je_auto_control.linux_wayland.screen` deliberately caches nothing — +#: an output can move between two captures and a stale layout is worse than +#: the ``shutil.which`` miss it costs. That reasoning does not survive the move +#: to the input path: the ydotool backend consults this on *every* absolute +#: move, so an uncached lookup spawns a ``wlr-randr`` process per move and +#: doubles the cost of a path that already spawns one. A window this short is +#: orders of magnitude below any real monitor rearrangement and orders of +#: magnitude above the gap between two moves in a drag. +_CACHE_SECONDS = 1.0 + +#: ``{"origin": (monotonic_stamp, (x, y))}``. A dict rather than a pair of +#: module globals so refreshing it needs no ``global`` statement. +_CACHE: Dict[str, Tuple[float, Tuple[int, int]]] = {} + + +def layout_origin() -> Tuple[int, int]: + """Return the layout coordinate of the capture's top-left pixel. + + ``(0, 0)`` whenever the answer cannot be obtained — no ``wlr-randr``, no + Pillow, no capture tool — which is also the correct answer for every + layout those hosts are likely to have, and leaves the caller sending the + coordinate it would have sent before this translation existed. + + Cached for :data:`_CACHE_SECONDS`; see there for why this one caches and + the capture path does not. + + :return: (origin_x, origin_y) + """ + now = time.monotonic() + cached = _CACHE.get("origin") + if cached is not None and now - cached[0] < _CACHE_SECONDS: + return cached[1] + origin = _read_layout_origin() + _CACHE["origin"] = (now, origin) + return origin + + +def _read_layout_origin() -> Tuple[int, int]: + """Ask the screen backend, tolerating every way it can be unavailable.""" + try: + from je_auto_control.linux_wayland import screen + return screen.layout_origin() + except (ImportError, OSError, AutoControlException): + return (0, 0) + + +def reset_cache() -> None: + """Drop the cached reading, so the next call measures again.""" + _CACHE.clear() + + +__all__ = ["layout_origin", "reset_cache"] diff --git a/je_auto_control/linux_wayland/capture.py b/je_auto_control/linux_wayland/capture.py new file mode 100644 index 00000000..a2672c80 --- /dev/null +++ b/je_auto_control/linux_wayland/capture.py @@ -0,0 +1,236 @@ +"""Wayland screen capture, tried against each compositor's own tool. + +No Wayland compositor exposes a readable root window the way X11 does, and +which helper *can* read the screen differs by desktop: wlroots compositors +(sway, Hyprland, river) implement ``wlr-screencopy``, which ``grim`` speaks; +GNOME answers through ``gnome-screenshot``; KDE through ``spectacle``. None of +them is guaranteed to be present, so ``xdg-desktop-portal`` backs them all up +(see :mod:`portal`), and an operator whose setup none of that fits can name +their own command. The tiers, in order: + +1. ``JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND`` — an explicit operator override. +2. ``grim`` · 3. ``gnome-screenshot`` · 4. ``spectacle``. +5. ``xdg-desktop-portal``, over the session bus. + +Only ``grim`` accepts a region itself. Every other tier returns the whole +screen, so :class:`Capture` reports which of the two happened and the caller +crops when it has to. +""" +from __future__ import annotations + +import contextlib +import os +import shlex +import subprocess # nosec B404 # reason: argv-list from a private allow-list, no shell +import tempfile +from dataclasses import dataclass +from typing import Callable, List, Optional, Sequence, Tuple + +from je_auto_control.linux_wayland import portal +from je_auto_control.linux_wayland._detect import ( + WAYLAND_GNOME_SCREENSHOT, WAYLAND_GRIM, WAYLAND_SPECTACLE, binary_path, +) +from je_auto_control.utils.exception.exceptions import AutoControlScreenException + + +CAPTURE_TIMEOUT = 15.0 + +#: Operator override: a full command line whose ``{output}`` placeholder is +#: replaced with a temporary PNG path. Split with ``shlex``, run without a +#: shell. Always a whole-screen capture — regions are cropped afterwards, so +#: the command never has to understand geometry. +CAPTURE_COMMAND_ENV = "JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND" + +_OVERRIDE_LABEL = f"${CAPTURE_COMMAND_ENV}" +_PORTAL_LABEL = "xdg-desktop-portal" + +_MISSING_TOOL_HINT = ( + "No Wayland screen-capture tool found. Install the one your compositor " + "supports: grim (sway / Hyprland / river, or any wlr-screencopy " + "compositor), gnome-screenshot (GNOME), or spectacle (KDE). Installing " + "none of them, the xdg-desktop-portal fallback needs only a session bus " + "and is tried automatically. Or name " + "your own capture command in " + CAPTURE_COMMAND_ENV + ", using {output} " + "for the file to write. To capture through XWayland set " + "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 — note XWayland cannot see " + "native Wayland windows, so that capture is blank for most of the desktop." +) + + +@dataclass(frozen=True) +class Capture: + """One capture's PNG bytes, plus whether the tool applied the region. + + ``region_applied`` is False whenever the helper could only grab the + whole screen, which tells the caller it still has to crop. + """ + + data: bytes + region_applied: bool + + +def run_tool(argv: List[str], *, timeout: float = CAPTURE_TIMEOUT) -> bytes: + """Run one capture helper and return its stdout. + + :param argv: absolute binary path plus arguments, never a shell string. + :param timeout: seconds before the helper is considered hung. + :return: the helper's stdout (empty bytes when it writes to a file). + """ + # argv comes from a private allow-list (grim / gnome-screenshot / + # spectacle / wlr-randr resolved through shutil.which), never user + # input; no shell=True. + try: + completed = subprocess.run( # nosec B603 # nosemgrep + argv, check=True, timeout=timeout, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + except subprocess.CalledProcessError as error: + message = (error.stderr or b"").decode("utf-8", errors="replace") + raise AutoControlScreenException( + f"{argv[0]} exited {error.returncode}: {message.strip()}", + ) from error + except subprocess.TimeoutExpired as error: + raise AutoControlScreenException( + f"{argv[0]} timed out after {timeout}s", + ) from error + return completed.stdout or b"" + + +def geometry(screen_region: Sequence[int]) -> str: + """Format ``[x1, y1, x2, y2]`` as grim's ``-g`` argument (``"x,y WxH"``).""" + x1, y1, x2, y2 = (int(value) for value in screen_region) + if x2 <= x1 or y2 <= y1: + raise AutoControlScreenException( + f"screen_region must have positive width and height; got " + f"[{x1}, {y1}, {x2}, {y2}]", + ) + return f"{x1},{y1} {x2 - x1}x{y2 - y1}" + + +def _gnome_argv(executable: str, output_path: str) -> List[str]: + return [executable, "-f", output_path] + + +def _spectacle_argv(executable: str, output_path: str) -> List[str]: + # -b background (no GUI), -n no notification, -f full screen, -o output. + return [executable, "-b", "-n", "-f", "-o", output_path] + + +# Helpers that can only write a full-screen PNG to a file, in the order they +# are tried once grim is absent. +_FILE_TOOLS: Tuple[Tuple[str, Callable[[str, str], List[str]]], ...] = ( + (WAYLAND_GNOME_SCREENSHOT, _gnome_argv), + (WAYLAND_SPECTACLE, _spectacle_argv), +) + + +def _override_template() -> str: + """The operator's capture command line, or "" when unset.""" + return (os.environ.get(CAPTURE_COMMAND_ENV) or "").strip() + + +def _override_argv(output_path: str) -> List[str]: + """Build the operator's capture command for one output path. + + ``shlex.split`` then per-token substitution — never a shell, and never + string-concatenating the path into a command line, so a path containing + spaces or quotes stays one argument. + """ + template = _override_template() + argv = shlex.split(template) + if not argv: + raise AutoControlScreenException( + f"{CAPTURE_COMMAND_ENV} is set but empty", + ) + if not any("{output}" in token for token in argv): + raise AutoControlScreenException( + f"{CAPTURE_COMMAND_ENV} must contain {{output}}, the path the " + f"command should write the capture to; got {template!r}", + ) + return [token.replace("{output}", output_path) for token in argv] + + +def available_tool() -> Optional[str]: + """Return the capture tier that would be used, or None if there is none. + + Reported by the diagnostics bundle so an operator can see *why* a capture + failed without reproducing it. + """ + if _override_template(): + return _OVERRIDE_LABEL + for name in (WAYLAND_GRIM, *(tool for tool, _ in _FILE_TOOLS)): + if binary_path(name) is not None: + return name + return _PORTAL_LABEL if portal.is_available() else None + + +def _grim_capture(executable: str, + screen_region: Optional[Sequence[int]]) -> Capture: + argv = [executable] + if screen_region is not None: + argv.extend(["-g", geometry(screen_region)]) + argv.append("-") + data = run_tool(argv) + if not data: + raise AutoControlScreenException("grim produced no output") + return Capture(data, screen_region is not None) + + +def _write_to_temp_png(write: Callable[[str], None], label: str) -> Capture: + """Let ``write`` fill a temporary PNG, then read and delete it.""" + handle, output_path = tempfile.mkstemp(prefix="je_autocontrol_", + suffix=".png") + os.close(handle) + try: + write(output_path) + with open(output_path, "rb") as captured: + data = captured.read() + finally: + with contextlib.suppress(OSError): + os.unlink(output_path) + if not data: + raise AutoControlScreenException(f"{label} produced no output") + return Capture(data, False) + + +def _file_capture(executable: str, + build_argv: Callable[[str, str], List[str]]) -> Capture: + """Run a helper that writes a PNG to a path we choose.""" + return _write_to_temp_png( + lambda output_path: run_tool(build_argv(executable, output_path)), + executable, + ) + + +def _override_capture() -> Capture: + """Run the operator's own capture command.""" + return _write_to_temp_png( + lambda output_path: run_tool(_override_argv(output_path)), + _OVERRIDE_LABEL, + ) + + +def grab_png(screen_region: Optional[Sequence[int]] = None) -> Capture: + """Capture the screen as PNG bytes using the first tier that can. + + :param screen_region: ``[x1, y1, x2, y2]`` to capture, or None for all. + :return: the PNG bytes and whether the region was applied by the tier. + """ + if _override_template(): + return _override_capture() + grim = binary_path(WAYLAND_GRIM) + if grim is not None: + return _grim_capture(grim, screen_region) + for name, build_argv in _FILE_TOOLS: + executable = binary_path(name) + if executable is not None: + return _file_capture(executable, build_argv) + if portal.is_available(): + return Capture(portal.capture_png(), False) + raise AutoControlScreenException(_MISSING_TOOL_HINT) + + +__all__ = [ + "CAPTURE_COMMAND_ENV", "CAPTURE_TIMEOUT", "Capture", "available_tool", + "geometry", "grab_png", "run_tool", +] diff --git a/je_auto_control/linux_wayland/portal.py b/je_auto_control/linux_wayland/portal.py new file mode 100644 index 00000000..bc4e446c --- /dev/null +++ b/je_auto_control/linux_wayland/portal.py @@ -0,0 +1,207 @@ +"""Last-resort screen capture through ``xdg-desktop-portal``. + +The three CLI helpers in :mod:`capture` each cover one compositor family and +none of them is guaranteed to be installed — GNOME has not shipped +``gnome-screenshot`` by default since 42. ``org.freedesktop.portal.Screenshot`` +is the one interface every modern desktop implements, so it is tried after +them all rather than not at all. + +It is awkward enough to deserve explaining. The portal does not return the +image: ``Screenshot`` returns a *request* object path and the result arrives +later as a ``Response`` signal on it, **directed at the unique bus name that +made the call**. So the subscription has to be in place before the call, and +it has to be on the same connection — a listener anywhere else is not the +destination, and the bus routes a directed message to its destination only. + +That is why this does not shell out. The obvious implementation — ``gdbus +monitor`` in one process, ``gdbus call`` in another — cannot work and did not: +each invocation opens its own connection under its own unique name, so the +monitor is never the caller. Measured against a real ``dbus-daemon``, the +monitor saw the call go past and the answer never arrived, and the capture +timed out every time. :mod:`_dbus_client` speaks the protocol directly instead, +which also drops the ``gdbus`` binary from what a desktop has to have +installed for this tier to work. + +Two consequences the caller has to live with, both documented at the tier that +uses this: the portal may show a consent dialog the first time (so the wait is +bounded by :data:`DEFAULT_TIMEOUT`, not instant), and it always captures the +whole screen, so a region is cropped afterwards. + +Every failure here is an ordinary capture failure — the caller reports the +install hint for the CLI helpers, which is the actionable advice. +""" +from __future__ import annotations + +import contextlib +import os +import threading +import urllib.parse +from typing import Any, Dict, List, Tuple + +from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.utils.exception.exceptions import AutoControlScreenException + + +PORTAL_BUS = "org.freedesktop.portal.Desktop" +PORTAL_PATH = "/org/freedesktop/portal/desktop" +SCREENSHOT_INTERFACE = "org.freedesktop.portal.Screenshot" +REQUEST_INTERFACE = "org.freedesktop.portal.Request" +PORTAL_METHOD = f"{SCREENSHOT_INTERFACE}.Screenshot" + +# The portal may put a consent dialog in front of the response the first time, +# so this is a human-scale wait, not a machine-scale one. +DEFAULT_TIMEOUT = 30.0 +_CALL_TIMEOUT = 10.0 + +# 0 success, 1 cancelled by the user, 2 ended some other way. +_RESPONSE_MEANING = { + 1: "the user dismissed the desktop portal's screenshot dialog", + 2: "the desktop portal ended the screenshot request", +} + + +def is_available() -> bool: + """Whether a session bus exists, so the portal tier is worth trying.""" + return _dbus_client.is_available() + + +def capture_png(timeout: float = DEFAULT_TIMEOUT) -> bytes: + """Ask the desktop portal for a full-screen capture; return PNG bytes. + + :param timeout: seconds to wait for the portal's Response signal, + including any time the user spends on a consent dialog. + :return: the captured image file's bytes. + """ + try: + with _dbus_client.SessionBus() as bus: + uri = _request_and_await(bus, timeout) + except _dbus_client.DBusError as error: + raise AutoControlScreenException( + f"the desktop portal could not be reached: {error}", + ) from error + return _read_and_discard(_path_from_uri(uri)) + + +def _request_and_await(bus: "_dbus_client.SessionBus", timeout: float) -> str: + """Subscribe, call, and wait — in that order, on the one connection.""" + token = _handle_token() + predicted = _request_path(bus.sender_token, token) + bus.add_match(_match_rule(predicted)) + handle = _screenshot(bus, token) + paths = [predicted] + # A portal that ignores handle_token answers on a path of its own choosing. + # The specification tells clients to follow the returned handle, and the + # subscription for it can only be added once the call has returned it — + # which is exactly why the predicted one is subscribed to first. + if handle and handle != predicted: + bus.add_match(_match_rule(handle)) + paths.append(handle) + try: + body = bus.wait_for_signal(paths, REQUEST_INTERFACE, "Response", + timeout) + except _dbus_client.DBusError as error: + raise AutoControlScreenException( + f"desktop portal did not answer within {timeout:g}s " + f"(a consent dialog may be waiting)", + ) from error + return _uri_from_response(body) + + +def _screenshot(bus: "_dbus_client.SessionBus", token: str) -> str: + """Make the Screenshot call; its return value is only a request handle.""" + options = { + "interactive": _dbus_client.Variant("b", False), + "handle_token": _dbus_client.Variant("s", token), + } + body = bus.call(PORTAL_BUS, PORTAL_PATH, SCREENSHOT_INTERFACE, + "Screenshot", "sa{sv}", ["", options], + timeout=_CALL_TIMEOUT) + return str(body[0]) if body else "" + + +#: Guards the counter below. Captures can be started from the callback +#: executor and a GUI thread at once, and ``+= 1`` is not atomic — two threads +#: reading the same value would put both portal answers on one object path. +_TOKEN_LOCK = threading.Lock() +_REQUEST_COUNT = 0 + + +def _handle_token() -> str: + """A token unique to this request, which the request path is built from. + + Only ``[A-Za-z0-9_]`` is legal in the object path element it becomes, and + reusing one across two concurrent captures would put both answers on one + path, so the process and a counter both go in. + """ + global _REQUEST_COUNT # noqa: PLW0603 # reason: one counter per process is the point + with _TOKEN_LOCK: + _REQUEST_COUNT += 1 + count = _REQUEST_COUNT + return f"je_auto_control_{os.getpid()}_{count}" + + +def _request_path(sender_token: str, handle_token: str) -> str: + """Where the portal will emit ``Response``, by the specification's rule.""" + return f"{PORTAL_PATH}/request/{sender_token}/{handle_token}" + + +def _match_rule(path: str) -> str: + """A match rule narrow enough that only this request's answer arrives.""" + return (f"type='signal',sender='{PORTAL_BUS}',path='{path}'," + f"interface='{REQUEST_INTERFACE}',member='Response'") + + +def _uri_from_response(body: List[Any]) -> str: + """Read the screenshot URI out of one ``Response`` signal body.""" + code, results = _response_parts(body) + if code != 0: + raise AutoControlScreenException( + _RESPONSE_MEANING.get(code, f"desktop portal returned {code}"), + ) + uri = results.get("uri", "") + if not uri: + raise AutoControlScreenException( + "desktop portal reported success but returned no image URI", + ) + return str(uri) + + +def _response_parts(body: List[Any]) -> Tuple[int, Dict[str, Any]]: + """Split a ``(u, a{sv})`` signal body, rejecting anything else.""" + if len(body) < 2 or not isinstance(body[1], dict): + raise AutoControlScreenException( + f"the desktop portal sent a Response this cannot read: {body!r}", + ) + return int(body[0]), body[1] + + +def _path_from_uri(uri: str) -> str: + """Convert the portal's ``file://`` URI into a local path.""" + parsed = urllib.parse.urlparse(uri) + if parsed.scheme != "file": + raise AutoControlScreenException( + f"desktop portal returned a non-file URI: {uri!r}", + ) + return urllib.parse.unquote(parsed.path) + + +def _read_and_discard(path: str) -> bytes: + """Read the portal's file, then remove it — it is ours to clean up.""" + try: + with open(path, "rb") as captured: + data = captured.read() + except OSError as error: + raise AutoControlScreenException( + f"could not read the portal's screenshot at {path!r}: {error}", + ) from error + finally: + with contextlib.suppress(OSError): + os.unlink(path) + if not data: + raise AutoControlScreenException("desktop portal wrote an empty file") + return data + + +__all__ = ["DEFAULT_TIMEOUT", "PORTAL_BUS", "PORTAL_METHOD", "PORTAL_PATH", + "REQUEST_INTERFACE", "SCREENSHOT_INTERFACE", "capture_png", + "is_available"] diff --git a/je_auto_control/linux_wayland/screen.py b/je_auto_control/linux_wayland/screen.py index 4ae4dc18..fef24f04 100644 --- a/je_auto_control/linux_wayland/screen.py +++ b/je_auto_control/linux_wayland/screen.py @@ -1,135 +1,252 @@ -"""Wayland screen backend (grim + wlr-randr CLI bridges). - -Screen capture goes through ``grim`` — the wlroots screencopy tool — -because the xdg-desktop-portal ScreenCast path needs a user-consent -dialog every call. Resolution comes from ``wlr-randr`` when present; -otherwise the GNOME ``gnome-screenshot`` fallback is consulted. +"""Wayland screen backend (compositor capture tools + wlr-randr). + +Everything here funnels through :func:`grab_image`, which is also the hook +the framework's generic capture layer looks for +(:mod:`je_auto_control.utils.cv2_utils.screen_grabber`). That indirection is +the point: Pillow's ``ImageGrab`` and ``mss`` both read the X11 root window +on Linux, and under Wayland that root belongs to XWayland, which does not +composite native Wayland windows. Pillow's own fallback to the same capture +tools only runs when the X11 grab *raises*, which it does not while XWayland +is up; ``mss`` has no fallback at all. Publishing ``grab_image`` here is what +makes locators, OCR, screenshots, recording and remote-desktop frames see the +real screen — and see it the same way regardless of whether XWayland is +running. + +Resolution comes from ``wlr-randr`` when it is present and from the capture +itself otherwise, so the call works on GNOME / KDE without extra tools. """ from __future__ import annotations import re -import subprocess # nosec B404 # reason: argv-list, no shell interpolation -from typing import List, Optional, Tuple +from io import BytesIO +from typing import List, Optional, Sequence, Tuple from PIL import Image -from je_auto_control.linux_wayland._detect import WAYLAND_GRIM, binary_path -from je_auto_control.utils.exception.exceptions import AutoControlException - +from je_auto_control.linux_wayland import capture as wayland_capture +from je_auto_control.linux_wayland._detect import WAYLAND_WLR_RANDR, binary_path +from je_auto_control.utils.exception.exceptions import AutoControlScreenException -_RESOLUTION_RE = re.compile( - # Bounded quantifiers (max 5 digits per side, more than enough for - # any real monitor resolution) keep this regex provably linear-time. - r"(\d{1,5})x(\d{1,5})", -) -_INSTALL_HINT_GRIM = ( - "grim is required for Wayland screenshots. " - "Install with your package manager (e.g. `apt install grim`)." -) +# Bounded quantifiers (max 5 digits per side, more than enough for any real +# monitor resolution) keep these regexes provably linear-time. +_MODE_RE = re.compile(r"(\d{1,5})x(\d{1,5})") +_POSITION_RE = re.compile(r"^\s*Position:\s*(-?\d{1,5}),(-?\d{1,5})") +_ENABLED_RE = re.compile(r"^\s*Enabled:\s*(\w+)") -def _require_grim() -> str: - path = binary_path(WAYLAND_GRIM) - if path is None: - raise AutoControlException(_INSTALL_HINT_GRIM) - return path - -def _run(argv: list, *, timeout: float = 10.0) -> bytes: - # argv comes from a private allow-list (grim / wlr-randr absolute - # paths via shutil.which), never user input; no shell=True. +def _validate_region(screen_region: Sequence[int]) -> Tuple[int, int, int, int]: + """Reject a region that would crop to an empty image.""" try: - completed = subprocess.run( # nosec B603 # nosemgrep - argv, check=True, timeout=timeout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) - except subprocess.CalledProcessError as error: - message = (error.stderr or b"").decode("utf-8", errors="replace") - raise AutoControlException( - f"{argv[0]} exited {error.returncode}: {message.strip()}", + x1, y1, x2, y2 = (int(value) for value in screen_region) + except (TypeError, ValueError) as error: + raise AutoControlScreenException( + f"screen_region must be 4 ints [left, top, right, bottom]; " + f"got {screen_region!r}", ) from error - except subprocess.TimeoutExpired as error: - raise AutoControlException( - f"{argv[0]} timed out after {timeout}s", - ) from error - return completed.stdout or b"" + if x2 <= x1 or y2 <= y1: + raise AutoControlScreenException( + f"screen_region must have positive width and height; got " + f"[{x1}, {y1}, {x2}, {y2}]", + ) + return x1, y1, x2, y2 + + +def grab_image(screen_region: Optional[Sequence[int]] = None) -> Image.Image: + """Capture the screen and return it as an RGB :class:`PIL.Image.Image`. + + ``screen_region`` is ``[left, top, right, bottom]`` — the same bbox + convention as ``ImageGrab.grab``, so the generic capture layer can hand + this straight to callers written against Pillow. + + Coordinates are the compositor's layout coordinates, which is what + ``grim -g`` takes and what ``wlr-randr`` reports positions in. The + whole-screen capture spans the entire output layout rather than one + monitor, so its top-left pixel is :func:`layout_origin` — ``(0, 0)`` + only while no output sits left of or above the origin. The crop below + subtracts that origin; grim applies the region itself and needs no + help, but every other tier hands back the whole layout. + + :param screen_region: region to capture, or None for the whole layout. + :return: the captured image, always in RGB mode. + """ + region = _validate_region(screen_region) if screen_region is not None else None + captured = wayland_capture.grab_png(region) + with Image.open(BytesIO(captured.data)) as raw: + # convert() reads the frame in, so the result outlives the file object. + image = raw.convert("RGB") + if region is None or captured.region_applied: + return image + origin_x, origin_y = layout_origin() + x1, y1, x2, y2 = region + return image.crop((x1 - origin_x, y1 - origin_y, + x2 - origin_x, y2 - origin_y)) def size() -> Tuple[int, int]: - """Return the primary monitor's pixel size. + """Return the size of the whole output layout, in pixels. Named ``size`` to match the backend contract the wrapper calls (``screen.size()``), as the windows / osx / x11 backends all do. - Tries ``wlr-randr`` first (sway / hyprland) then falls back to - grim's PNG header so the call still works on GNOME / KDE without - extra dependencies. + This is the size of the layout bounding box, not one monitor and not + its right/bottom edge: it has to agree with :func:`grab_image`, because + ``get_pixel`` addresses the same coordinate space and the mss-shaped + shim composes the two into its monitor list. On a layout whose left-most + output starts at a negative x the two differ — the edge is smaller than + the width by the origin — and reporting the edge would make every + consumer of ``size()`` grab a frame narrower than the screen. + + Tries ``wlr-randr`` first (sway / Hyprland) and falls back to measuring + a capture, so the call still works on GNOME / KDE. + + :return: (width, height) """ coords = _size_from_wlr_randr() if coords is not None: return coords - return _size_from_grim_capture() + image = grab_image() + return int(image.width), int(image.height) + + +def layout_origin() -> Tuple[int, int]: + """Return the layout coordinate of :func:`grab_image`'s top-left pixel. + + ``(0, 0)`` on a single-monitor layout and on any layout whose outputs + all sit at non-negative coordinates — but a monitor placed left of or + above the primary one gives the layout a negative origin, and a capture + of the whole layout then starts there rather than at the origin of the + coordinate space. + + Callers that map a position found *in* a frame back to a screen + coordinate have to add this; :func:`grab_image` subtracts it when it + crops. ``(0, 0)`` where ``wlr-randr`` cannot say (GNOME / KDE), which + is also the answer for the layouts it would otherwise report. + + :return: (origin_x, origin_y) + """ + rects = _wlr_randr_rects() + if not rects: + return (0, 0) + return (min(x for x, _, _, _ in rects), min(y for _, y, _, _ in rects)) def get_pixel(x: int, y: int) -> Tuple[int, int, int]: """Return the ``(r, g, b)`` colour at ``(x, y)``. - grim can capture an arbitrary region, so a 1x1 grab at the requested - point is the cheapest way to read a single pixel. Returns RGB to match - the x11 backend. + Captures a 1x1 region at the point — grim grabs exactly that, and the + file-based helpers grab the screen and crop. Returns RGB to match the + x11 / windows / osx backends. + + :param x: X coordinate. + :param y: Y coordinate. + :return: (R, G, B) """ - grim = _require_grim() - data = _run([grim, "-g", f"{int(x)},{int(y)} 1x1", "-"], timeout=10.0) - if not data: - raise AutoControlException("grim produced no output") - from io import BytesIO - with Image.open(BytesIO(data)) as image: - return image.convert("RGB").getpixel((0, 0)) + image = grab_image([int(x), int(y), int(x) + 1, int(y) + 1]) + return image.getpixel((0, 0)) def screenshot(file_path: Optional[str] = None, screen_region: Optional[List[int]] = None) -> Optional[str]: - """Capture the screen with ``grim``. + """Capture the screen, saving to ``file_path`` when one is given. + + ``screen_region`` is ``[x1, y1, x2, y2]``, matching the X11 backend's + calling convention. - ``screen_region`` is ``[x1, y1, x2, y2]`` (matching the X11 - backend's calling convention). When ``file_path`` is omitted the - capture is returned as PNG bytes via grim's stdout but discarded; - callers should pass an explicit path to keep the file. + :param file_path: where to write the PNG, or None to discard the capture. + :param screen_region: region to capture, or None for the whole layout. + :return: ``file_path`` unchanged, so callers can chain on it. """ - grim = _require_grim() - argv = [grim] - if screen_region is not None: - x1, y1, x2, y2 = (int(v) for v in screen_region) - argv.extend(["-g", f"{x1},{y1} {x2 - x1}x{y2 - y1}"]) - argv.append(file_path if file_path else "-") - _run(argv) + image = grab_image(screen_region) + if file_path: + try: + image.save(file_path) + except (OSError, ValueError) as error: + raise AutoControlScreenException( + f"Failed to save screenshot to {file_path!r}: {error}", + ) from error return file_path -def _size_from_wlr_randr() -> Optional[Tuple[int, int]]: - if binary_path("wlr-randr") is None: - return None +def parse_wlr_randr(text: str) -> List[Tuple[int, int, int, int]]: + """Parse ``wlr-randr`` into ``(x, y, width, height)`` per enabled output. + + An output block starts at a line with no leading whitespace and its + fields are indented under it:: + + HEADLESS-2 "Headless output 1" + Enabled: yes + Modes: + 1280x720 px (current) + Position: 0,0 + + Taking the first ``WxH`` in the whole document — which is what this used + to do — reads one monitor's mode and calls it the screen size. On any + multi-output layout that disagrees with :func:`grab_image`, which returns + the whole layout, and the two are composed by the mss-shaped shim. + """ + rects: List[Tuple[int, int, int, int]] = [] + mode: Optional[Tuple[int, int]] = None + position: Optional[Tuple[int, int]] = None + enabled = True + + def flush() -> None: + if enabled and mode is not None: + x, y = position if position is not None else (0, 0) + rects.append((x, y, mode[0], mode[1])) + + for line in text.splitlines(): + if line and not line[0].isspace(): + flush() + mode, position, enabled = None, None, True + continue + enabled_match = _ENABLED_RE.match(line) + if enabled_match: + enabled = enabled_match.group(1).lower() == "yes" + continue + position_match = _POSITION_RE.match(line) + if position_match: + position = (int(position_match.group(1)), + int(position_match.group(2))) + continue + if "current" in line: + mode_match = _MODE_RE.search(line) + if mode_match: + mode = (int(mode_match.group(1)), int(mode_match.group(2))) + flush() + return rects + + +def _wlr_randr_rects() -> List[Tuple[int, int, int, int]]: + """Enabled outputs as ``(x, y, width, height)``, empty when unknown. + + Not cached: an output can be plugged in, unplugged or moved between two + captures, and a stale layout is worse than the ``shutil.which`` miss + this costs on the desktops that have no ``wlr-randr`` at all. + """ + executable = binary_path(WAYLAND_WLR_RANDR) + if executable is None: + return [] try: - output = _run(["wlr-randr"], timeout=5.0).decode( + output = wayland_capture.run_tool([executable], timeout=5.0).decode( "utf-8", errors="replace", ) - except AutoControlException: - return None - for line in output.splitlines(): - match = _RESOLUTION_RE.search(line) - if match: - return int(match.group(1)), int(match.group(2)) - return None + except AutoControlScreenException: + return [] + return parse_wlr_randr(output) -def _size_from_grim_capture() -> Tuple[int, int]: - grim = _require_grim() - data = _run([grim, "-"], timeout=10.0) - if not data: - raise AutoControlException("grim produced no output") - from io import BytesIO - with Image.open(BytesIO(data)) as image: - return int(image.width), int(image.height) +def _size_from_wlr_randr() -> Optional[Tuple[int, int]]: + """Layout bounding box size from ``wlr-randr``, or None when it cannot say.""" + rects = _wlr_randr_rects() + if not rects: + return None + left = min(x for x, _, _, _ in rects) + top = min(y for _, y, _, _ in rects) + right = max(x + width for x, _, width, _ in rects) + bottom = max(y + height for _, y, _, height in rects) + return (right - left, bottom - top) -__all__ = ["size", "get_pixel", "screenshot"] +__all__ = ["get_pixel", "grab_image", "layout_origin", "parse_wlr_randr", + "screenshot", "size"] diff --git a/test/unit_test/headless/test_wayland_backend.py b/test/unit_test/headless/test_wayland_backend.py index c6a5ad93..dd6a6a29 100644 --- a/test/unit_test/headless/test_wayland_backend.py +++ b/test/unit_test/headless/test_wayland_backend.py @@ -12,13 +12,40 @@ import pytest from je_auto_control.linux_wayland import ( - _detect, keyboard as wayland_keyboard, mouse as wayland_mouse, + _detect, _ydotool_cli, capture as wayland_capture, + keyboard as wayland_keyboard, mouse as wayland_mouse, screen as wayland_screen, ) from je_auto_control.linux_wayland.keymap import keyboard_keys_table from je_auto_control.utils.exception.exceptions import AutoControlException +@pytest.fixture(autouse=True) +def _pin_the_cli_input_path(): + """Keep every dispatch test in this file on the wtype / ydotool argv. + + ``mouse`` and ``keyboard`` prefer libei wherever it is loadable, so on a + host that has it these tests would probe for a portal and then assert + against argv that was never produced. The libei path has its own file, + ``test_wayland_libei.py``. + + The ydotool generation cache is pre-seeded for the same reason. Before + building any argv both backends probe the installed ydotool once, because + 0.1.x answers this argv with exit code 0 and no events — and that probe is + a ``subprocess.run`` call too, so it would land in the captured argv list + of every dispatch test here. What it does instead is its own file, + ``test_wayland_ydotool_cli.py``. + """ + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + try: + with patch.object(wayland_mouse, "_try_libei", return_value=None), \ + patch.object(wayland_keyboard, "_try_libei", return_value=None): + yield + finally: + _ydotool_cli.reset_cache() + + # === Detection ============================================================== def test_is_wayland_session_reads_session_type(): @@ -227,39 +254,49 @@ def test_mouse_raises_when_ydotool_missing(): # === Screen dispatch ======================================================== -def test_screenshot_calls_grim_with_path(): +def _fake_capture(captured, png): + """Stand in for subprocess.run inside the capture module.""" + def runner(argv, **_kwargs): + captured.append(list(argv)) + return subprocess.CompletedProcess(argv, 0, png, b"") # nosemgrep + return runner + + +def test_screenshot_captures_through_grim_and_saves_with_pillow(tmp_path): + """grim writes PNG to stdout and Pillow saves it, so the same capture + can also be handed back in memory to the locators.""" captured: list = [] - # CompletedProcess constructor used to mock subprocess.run. - with patch.object(wayland_screen, "binary_path", + png = _one_pixel_png((1, 2, 3)) + target = tmp_path / "out.png" + with patch.object(wayland_capture, "binary_path", return_value="/usr/bin/grim"), \ - patch.object(wayland_screen.subprocess, "run", - side_effect=lambda argv, **kw: (captured.append(argv) - or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep - wayland_screen.screenshot("out.png") - assert captured == [["/usr/bin/grim", "out.png"]] + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture(captured, png)): + assert wayland_screen.screenshot(str(target)) == str(target) + assert captured == [["/usr/bin/grim", "-"]] + assert target.exists() def test_screenshot_passes_screen_region(): captured: list = [] - with patch.object(wayland_screen, "binary_path", + png = _one_pixel_png((1, 2, 3)) + with patch.object(wayland_capture, "binary_path", return_value="/usr/bin/grim"), \ - patch.object(wayland_screen.subprocess, "run", - side_effect=lambda argv, **kw: (captured.append(argv) - or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep - wayland_screen.screenshot("out.png", screen_region=[10, 20, 110, 220]) + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture(captured, png)): + wayland_screen.screenshot(None, screen_region=[10, 20, 110, 220]) assert captured == [[ - "/usr/bin/grim", "-g", "10,20 100x200", "out.png", + "/usr/bin/grim", "-g", "10,20 100x200", "-", ]] -@pytest.mark.parametrize("direction_name, expected_axis, expected_amount", [ - ("wayland_scroll_direction_down", "-y", "-5"), - ("wayland_scroll_direction_up", "-y", "5"), - ("wayland_scroll_direction_left", "-x", "-5"), - ("wayland_scroll_direction_right", "-x", "5"), +@pytest.mark.parametrize("direction_name, expected_x, expected_y", [ + ("wayland_scroll_direction_down", "0", "-5"), + ("wayland_scroll_direction_up", "0", "5"), + ("wayland_scroll_direction_left", "-5", "0"), + ("wayland_scroll_direction_right", "5", "0"), ]) -def test_scroll_honours_direction(direction_name, expected_axis, - expected_amount): +def test_scroll_honours_direction(direction_name, expected_x, expected_y): """Regression: scroll's signature was ``(direction, x, y)`` while the wrapper calls ``scroll(scroll_value, scroll_direction)``. The direction bound to ``x`` and was then dropped, so every direction scrolled the same @@ -272,8 +309,9 @@ def test_scroll_honours_direction(direction_name, expected_axis, patch.object(wayland_mouse.subprocess, "run", side_effect=_fake_run(captured)): wayland_mouse.scroll(5, direction) + # Both axes are always sent, per ydotool's documented example. assert captured[0][1:] == [ - "mousemove", "--wheel", expected_axis, expected_amount, + "mousemove", "--wheel", "-x", expected_x, "-y", expected_y, ] @@ -295,42 +333,281 @@ def test_get_pixel_grabs_a_one_by_one_region_at_the_point(): png = _one_pixel_png((10, 20, 30)) captured: list = [] - def _run(argv, **_kwargs): - captured.append(argv) - return subprocess.CompletedProcess(argv, 0, png, b"") # nosemgrep - - with patch.object(wayland_screen, "binary_path", + with patch.object(wayland_capture, "binary_path", return_value="/usr/bin/grim"), \ - patch.object(wayland_screen.subprocess, "run", side_effect=_run): + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture(captured, png)): assert wayland_screen.get_pixel(7, 9) == (10, 20, 30) assert "-g" in captured[0] and "7,9 1x1" in captured[0] def _one_pixel_png(rgb) -> bytes: + return _solid_png((1, 1), rgb) + + +def _solid_png(size, rgb) -> bytes: from io import BytesIO from PIL import Image buffer = BytesIO() - Image.new("RGB", (1, 1), rgb).save(buffer, format="PNG") + Image.new("RGB", size, rgb).save(buffer, format="PNG") return buffer.getvalue() +# Captured verbatim from `wlr-randr` inside the headless sway session in +# docker/Dockerfile.wayland — two outputs, the second one to the right of +# the first, no refresh rate on a headless mode. +_REAL_WLR_RANDR = """HEADLESS-2 "Headless output 1" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: 0,0 + Transform: normal + Scale: 1.000000 +HEADLESS-1 "Headless output 2" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: 1280,0 + Transform: normal + Scale: 1.000000 +""" + + +def test_wlr_randr_parses_every_output_not_just_the_first(): + """Regression, found by running against a real sway session. + + The old parser returned the first ``WxH`` anywhere in the document, so a + two-monitor layout reported one monitor's mode as the screen size — + while ``grab_image()`` returned the whole layout. The mss-shaped shim + composes the two, so it asked for a region half the size of the screen. + """ + rects = wayland_screen.parse_wlr_randr(_REAL_WLR_RANDR) + assert rects == [(0, 0, 1280, 720), (1280, 0, 1280, 720)] + + +def test_wlr_randr_skips_a_disabled_output(): + text = _REAL_WLR_RANDR.replace(" Enabled: yes\n Modes:\n" + " 1280x720 px (current)\n" + " Position: 1280,0\n", + " Enabled: no\n") + assert wayland_screen.parse_wlr_randr(text) == [(0, 0, 1280, 720)] + + +def test_wlr_randr_reads_a_mode_line_that_carries_a_refresh_rate(): + """A real monitor prints `1920x1080 px, 60.000000 Hz (preferred, current)`; + only the headless backend omits the rate.""" + text = ('DP-1 "Acme"\n Enabled: yes\n Modes:\n' + ' 1920x1080 px, 60.000000 Hz (preferred, current)\n' + ' 1280x720 px, 60.000000 Hz\n Position: 0,0\n') + assert wayland_screen.parse_wlr_randr(text) == [(0, 0, 1920, 1080)] + + def test_screen_size_uses_wlr_randr_when_available(): with patch.object(wayland_screen, "binary_path", side_effect=lambda name: "/usr/bin/" + name), \ patch.object( - wayland_screen.subprocess, "run", + wayland_capture.subprocess, "run", return_value=subprocess.CompletedProcess( # nosemgrep - ["wlr-randr"], 0, b" HDMI-A-1 1920x1080@60.000Hz\n", b"", + ["wlr-randr"], 0, _REAL_WLR_RANDR.encode(), b"", ), ): - assert wayland_screen.size() == (1920, 1080) + # The bounding box of both outputs, not the first output's mode. + assert wayland_screen.size() == (2560, 720) + + +def test_screen_size_falls_back_to_measuring_a_capture(): + """GNOME / KDE have no wlr-randr, so the size comes from the capture.""" + png = _one_pixel_png((0, 0, 0)) + with patch.object(wayland_screen, "binary_path", return_value=None), \ + patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture([], png)): + assert wayland_screen.size() == (1, 1) + + +# Captured verbatim from the same sway session with HEADLESS-1 moved to +# `position -1280 0`, which sway's headless backend accepts. A monitor +# placed left of the primary one is what puts a real desktop here, and it +# is the layout every assertion below is about. +_NEGATIVE_ORIGIN_WLR_RANDR = """HEADLESS-2 "Headless output 1" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: 0,0 + Transform: normal + Scale: 1.000000 + Adaptive Sync: disabled +HEADLESS-1 "Headless output 2" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: -1280,0 + Transform: normal + Scale: 1.000000 + Adaptive Sync: disabled +""" + + +def _with_wlr_randr(reported): + """The two seams ``wlr-randr`` is read through, as a pair of patches.""" + return ( + patch.object(wayland_screen, "binary_path", + side_effect=lambda name: "/usr/bin/" + name), + patch.object( + wayland_capture.subprocess, "run", + return_value=subprocess.CompletedProcess( # nosemgrep + ["wlr-randr"], 0, reported.encode(), b"", + ), + ), + ) -def test_screenshot_raises_when_grim_missing(): +def test_wlr_randr_parses_a_negative_output_position(): + rects = wayland_screen.parse_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + assert rects == [(0, 0, 1280, 720), (-1280, 0, 1280, 720)] + + +def test_screen_size_is_the_layout_width_not_its_right_edge(): + """Regression: ``size()`` returned ``max(x + width)``. + + On the layout above that is 1280 — the right edge — while the capture + grim returns is 2560 wide. Everything that composes the two (the + mss-shaped shim's monitor list, the recorder, the WebRTC host) then + asked for half the desktop and called it the whole screen. + """ + binary, run = _with_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + with binary, run: + assert wayland_screen.size() == (2560, 720) + + +def test_layout_origin_is_the_left_most_output(): + binary, run = _with_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + with binary, run: + assert wayland_screen.layout_origin() == (-1280, 0) + + +def test_layout_origin_is_the_origin_on_an_ordinary_layout(): + binary, run = _with_wlr_randr(_REAL_WLR_RANDR) + with binary, run: + assert wayland_screen.layout_origin() == (0, 0) + + +def test_layout_origin_is_the_origin_when_wlr_randr_is_absent(): + """GNOME / KDE ship no wlr-randr, and a guess would be worse than (0, 0).""" with patch.object(wayland_screen, "binary_path", return_value=None): - with pytest.raises(AutoControlException, match="grim"): + assert wayland_screen.layout_origin() == (0, 0) + + +def test_grab_image_crops_relative_to_the_layout_origin(): + """Regression: the crop used layout coordinates on a layout-origin image. + + Only grim applies a region itself; gnome-screenshot, spectacle, the + portal and the operator's own command all hand back the whole layout, + and its top-left pixel is the layout origin rather than (0, 0). Cropping + ``[-1275, 5, -1175, 55]`` straight off that image asks Pillow for a box + starting 1275 px to the left of the frame — which pads with black + instead of returning the left-hand monitor. + """ + from io import BytesIO + + from PIL import Image + left, right = (0x12, 0x34, 0x56), (0xAB, 0xCD, 0xEF) + layout = Image.new("RGB", (2560, 720), right) + layout.paste(Image.new("RGB", (1280, 720), left), (0, 0)) + buffer = BytesIO() + layout.save(buffer, format="PNG") + + binary, run = _with_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + whole_layout = wayland_capture.Capture(buffer.getvalue(), False) + with binary, run, patch.object(wayland_capture, "grab_png", + return_value=whole_layout): + cropped = wayland_screen.grab_image([-1275, 5, -1175, 55]) + # A region on the right-hand output still reads the right colour, + # so what changed is a shift and not a constant. + other = wayland_screen.grab_image([5, 5, 105, 55]) + assert cropped.size == (100, 50) + assert cropped.getpixel((0, 0)) == left + assert other.getpixel((0, 0)) == right + + +def test_screenshot_raises_when_no_capture_tool_is_installed(): + """The message has to name every tool that would have worked, because + which one is right depends on the compositor the operator is running.""" + with patch.object(wayland_capture, "binary_path", return_value=None): + with pytest.raises(AutoControlException) as error: wayland_screen.screenshot("out.png") + message = str(error.value) + for tool in ("grim", "gnome-screenshot", "spectacle"): + assert tool in message + + +def test_capture_falls_back_to_gnome_screenshot_when_grim_is_absent(tmp_path): + """grim only speaks wlr-screencopy; GNOME needs its own helper.""" + png = _one_pixel_png((4, 5, 6)) + captured: list = [] + + def _which(name): + return None if name == "grim" else "/usr/bin/" + name + + def _run(argv, **_kwargs): + captured.append(list(argv)) + # The helper writes a file rather than to stdout. + with open(argv[-1], "wb") as handle: + handle.write(png) + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_capture, "binary_path", side_effect=_which), \ + patch.object(wayland_capture.subprocess, "run", side_effect=_run): + image = wayland_screen.grab_image() + assert image.size == (1, 1) + assert captured[0][0] == "/usr/bin/gnome-screenshot" + + +def test_grab_image_crops_when_the_helper_cannot_take_a_region(): + """Only grim applies a region itself; the file-based helpers always + return the whole screen, so the region has to be cropped afterwards.""" + png = _solid_png((4, 2), (7, 8, 9)) + + def _which(name): + return None if name == "grim" else "/usr/bin/" + name + + def _run(argv, **_kwargs): + with open(argv[-1], "wb") as handle: + handle.write(png) + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_capture, "binary_path", side_effect=_which), \ + patch.object(wayland_capture.subprocess, "run", side_effect=_run): + image = wayland_screen.grab_image([1, 0, 3, 2]) + assert image.size == (2, 2) + + +def test_grab_image_rejects_an_empty_region(): + with pytest.raises(AutoControlException, match="positive width"): + wayland_screen.grab_image([10, 10, 10, 20]) + + +def test_available_tool_reports_the_helper_that_would_run(): + with patch.object(wayland_capture, "binary_path", + side_effect=lambda name: None if name == "grim" + else "/usr/bin/" + name): + assert wayland_capture.available_tool() == "gnome-screenshot" + with patch.object(wayland_capture, "binary_path", return_value=None): + assert wayland_capture.available_tool() is None # === Listener / record stubs =============================================== diff --git a/test/unit_test/headless/test_wayland_capture_tiers.py b/test/unit_test/headless/test_wayland_capture_tiers.py new file mode 100644 index 00000000..42d67559 --- /dev/null +++ b/test/unit_test/headless/test_wayland_capture_tiers.py @@ -0,0 +1,339 @@ +"""The capture tiers below the three compositor CLI helpers. + +No compositor helper is guaranteed to be installed — GNOME has not shipped +``gnome-screenshot`` by default since 42 — so the capture layer falls back to +``xdg-desktop-portal`` and, above everything, honours an operator-supplied +command. These tests drive both without a Wayland session: the session bus +connection is a fake that answers from a script, and the operator command is +any program that writes a file. + +The input side of the portal lives in ``test_wayland_oeffis.py``; it needs a +file descriptor rather than a file, so it cannot share this route. +""" +import subprocess +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import capture as wayland_capture +from je_auto_control.linux_wayland import portal as wayland_portal +from je_auto_control.linux_wayland import screen as wayland_screen +from je_auto_control.utils.exception.exceptions import AutoControlScreenException + + +def _png(size=(2, 2), rgb=(9, 9, 9)) -> bytes: + from io import BytesIO + + from PIL import Image + buffer = BytesIO() + Image.new("RGB", size, rgb).save(buffer, format="PNG") + return buffer.getvalue() + + +def _no_binaries(name): + return None + + +# === Operator override ===================================================== + +def test_override_command_wins_over_every_installed_tool(monkeypatch): + """An operator who names a command means it — detection does not get a + vote, or the override could not rescue a box where grim exists but is + broken.""" + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, + "/usr/bin/mycap --png {output}") + png = _png() + captured = [] + + def run(argv, **_kwargs): + captured.append(list(argv)) + with open(argv[-1], "wb") as handle: + handle.write(png) + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_capture.subprocess, "run", side_effect=run): + image = wayland_screen.grab_image() + + assert image.size == (2, 2) + assert captured[0][:2] == ["/usr/bin/mycap", "--png"] + assert captured[0][2].endswith(".png") + + +def test_override_keeps_a_spaced_path_as_one_argument(monkeypatch): + """The path is substituted per-token after shlex.split, never + concatenated into a command line, so a temp dir with a space in it + cannot split into two arguments (or be quoted into a shell).""" + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, "cap {output}") + argv = wayland_capture._override_argv("/tmp/a b/shot.png") + assert argv == ["cap", "/tmp/a b/shot.png"] + + +def test_override_without_the_output_placeholder_is_rejected(monkeypatch): + """A command with nowhere to write would silently produce an empty file; + say so at the boundary instead.""" + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, "grim -") + with pytest.raises(AutoControlScreenException, match=r"\{output\}"): + wayland_capture._override_argv("/tmp/x.png") + + +def test_override_is_reported_as_the_active_tool(monkeypatch): + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, "cap {output}") + assert wayland_capture.available_tool() == "$" + \ + wayland_capture.CAPTURE_COMMAND_ENV + + +def test_blank_override_is_ignored(monkeypatch): + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, " ") + with patch.object(wayland_capture, "binary_path", side_effect=_no_binaries), \ + patch.object(wayland_portal, "is_available", return_value=False): + assert wayland_capture.available_tool() is None + + +# === Portal tier =========================================================== + +def test_portal_runs_only_after_every_cli_helper_is_absent(monkeypatch): + monkeypatch.delenv(wayland_capture.CAPTURE_COMMAND_ENV, raising=False) + png = _png() + with patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_portal, "capture_png", + return_value=png) as portal_call, \ + patch.object(wayland_capture.subprocess, "run", + return_value=subprocess.CompletedProcess( # nosemgrep + ["grim"], 0, png, b"")): + wayland_capture.grab_png() + portal_call.assert_not_called() + + +def test_portal_is_used_when_no_helper_is_installed(monkeypatch): + monkeypatch.delenv(wayland_capture.CAPTURE_COMMAND_ENV, raising=False) + png = _png() + with patch.object(wayland_capture, "binary_path", side_effect=_no_binaries), \ + patch.object(wayland_portal, "is_available", return_value=True), \ + patch.object(wayland_portal, "capture_png", return_value=png): + result = wayland_capture.grab_png([0, 0, 1, 1]) + assert result.data == png + # The portal always captures everything, so the region must be cropped. + assert result.region_applied is False + + +def test_portal_is_reported_as_the_active_tool(monkeypatch): + monkeypatch.delenv(wayland_capture.CAPTURE_COMMAND_ENV, raising=False) + with patch.object(wayland_capture, "binary_path", side_effect=_no_binaries), \ + patch.object(wayland_portal, "is_available", return_value=True): + assert wayland_capture.available_tool() == "xdg-desktop-portal" + + +# === Portal response handling ============================================== +# +# The portal answers with a signal directed at the connection that called, so +# these drive the module the way the bus does: a fake connection that records +# the order things happened in. The real bus, the real marshalling and a real +# portal are exercised in ``docker/portal_verify.py`` — a mock cannot show +# that a directed signal never reaches a second connection, which is the +# defect this design replaced. + +_URI = "file:///run/user/1000/doc/ab/screenshot.png" + + +class _FakeBus: + """A session bus that answers from a script and records the order.""" + + sender_token = "1_9" + + def __init__(self, body=None, handle=None, error=None): + self.body = [0, {"uri": _URI}] if body is None else body + self.handle = handle + self.error = error + self.events = [] + self.rules = [] + self.awaited_paths = None + + def __enter__(self): + return self + + def __exit__(self, *_exception): + self.events.append("close") + + def add_match(self, rule): + self.events.append("add_match") + self.rules.append(rule) + + def call(self, _destination, _path, _interface, member, _signature, + _body, timeout=None): + del timeout + self.events.append(f"call:{member}") + return [self.handle] if self.handle else [] + + def wait_for_signal(self, paths, _interface, _member, _timeout): + self.events.append("wait") + self.awaited_paths = list(paths) + if self.error is not None: + raise self.error + return self.body + + +def _with_bus(bus): + """Patch the module's connection factory to hand back ``bus``.""" + return patch.object(wayland_portal._dbus_client, "SessionBus", + return_value=bus) + + +def test_portal_extracts_the_uri_from_a_response_signal(): + assert wayland_portal._uri_from_response([0, {"uri": _URI}]) == _URI + + +def test_portal_rejects_a_response_body_it_cannot_read(): + """A Response that is not ``(u, a{sv})`` must be named, not indexed into.""" + with pytest.raises(AutoControlScreenException, match="cannot read"): + wayland_portal._uri_from_response([0]) + + +def test_portal_reports_a_user_cancelled_request(): + """Response code 1 is the user dismissing the dialog; that has to read as + a cancellation, not as a parse failure.""" + with pytest.raises(AutoControlScreenException, match="dismissed"): + wayland_portal._uri_from_response([1, {}]) + + +def test_portal_rejects_a_success_with_no_uri(): + with pytest.raises(AutoControlScreenException, match="no image URI"): + wayland_portal._uri_from_response([0, {}]) + + +def test_portal_decodes_a_percent_escaped_file_uri(): + assert wayland_portal._path_from_uri( + "file:///run/user/1000/my%20shot.png", + ) == "/run/user/1000/my shot.png" + + +def test_portal_rejects_a_non_file_uri(): + with pytest.raises(AutoControlScreenException, match="non-file URI"): + wayland_portal._path_from_uri("https://example.invalid/x.png") + + +def test_portal_reads_then_deletes_the_file_it_was_handed(tmp_path): + """The portal hands over a real file; leaving one behind per capture + would fill the runtime dir over a long automation run.""" + target = tmp_path / "shot.png" + target.write_bytes(_png()) + assert wayland_portal._read_and_discard(str(target)) == _png() + assert not target.exists() + + +def test_portal_is_unavailable_without_a_session_bus(monkeypatch): + """No bus address is the one case where this tier cannot even be tried.""" + monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + assert wayland_portal.is_available() is False + with pytest.raises(AutoControlScreenException, match="session bus"): + wayland_portal.capture_png(timeout=0.1) + + +def test_portal_subscribes_before_it_calls(): + """The ordering the whole design turns on. + + The portal emits ``Response`` as soon as it has an answer, and it emits it + to the caller. Calling first and subscribing afterwards is a race that + loses the answer whenever the portal is quick — which, with no consent + dialog to show, it always is. + """ + bus = _FakeBus() + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + return_value=""), \ + patch.object(wayland_portal, "_read_and_discard", + return_value=_png()): + wayland_portal.capture_png(timeout=1.0) + assert bus.events[:3] == ["add_match", "call:Screenshot", "wait"] + + +def test_portal_subscribes_to_the_path_it_predicted(): + """The request path is built from our own unique name and handle token.""" + bus = _FakeBus() + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + return_value=""), \ + patch.object(wayland_portal, "_read_and_discard", + return_value=_png()): + wayland_portal.capture_png(timeout=1.0) + assert len(bus.awaited_paths) == 1 + predicted = bus.awaited_paths[0] + assert predicted.startswith( + "/org/freedesktop/portal/desktop/request/1_9/je_auto_control_") + assert predicted in bus.rules[0] + assert "member='Response'" in bus.rules[0] + + +def test_portal_also_follows_a_handle_that_differs_from_the_prediction(): + """A portal that ignores ``handle_token`` answers somewhere of its own. + + The specification tells clients to follow the returned handle, so both + paths have to be live: the predicted one could not be dropped (the answer + may already be on its way) and the returned one could not be subscribed + to any earlier than this. + """ + elsewhere = "/org/freedesktop/portal/desktop/request/1_9/portal_chose" + bus = _FakeBus(handle=elsewhere) + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + return_value=""), \ + patch.object(wayland_portal, "_read_and_discard", + return_value=_png()): + wayland_portal.capture_png(timeout=1.0) + assert bus.events[:4] == ["add_match", "call:Screenshot", "add_match", + "wait"] + assert elsewhere in bus.awaited_paths + assert len(bus.awaited_paths) == 2 + + +def test_portal_gives_each_request_its_own_token(): + """Two captures sharing a token would put both answers on one path.""" + first = wayland_portal._handle_token() + second = wayland_portal._handle_token() + assert first != second + for token in (first, second): + assert token.replace("_", "").isalnum(), \ + "the token becomes an object-path element, so it must be [A-Za-z0-9_]" + + +def test_portal_times_out_rather_than_waiting_forever(): + """A consent dialog nobody answers must not hang the caller's script.""" + bus = _FakeBus(error=wayland_portal._dbus_client.DBusError("no answer")) + with _with_bus(bus): + with pytest.raises(AutoControlScreenException, match="did not answer"): + wayland_portal.capture_png(timeout=0.05) + assert bus.events[-1] == "close", "the connection must not be left open" + + +def test_portal_reports_a_bus_that_went_away(): + """The bus dropping mid-request is a capture failure, not a crash.""" + bus = _FakeBus() + error = wayland_portal._dbus_client.DBusError("the session bus closed") + with _with_bus(bus), patch.object(bus, "call", side_effect=error): + with pytest.raises(AutoControlScreenException, + match="could not be reached"): + wayland_portal.capture_png(timeout=1.0) + + +def test_portal_captures_end_to_end_from_a_response_signal(tmp_path): + """The whole tier wired together: signal -> URI -> bytes -> cleanup. + + Only the URI-to-path step is stubbed, because a portal URI is an absolute + POSIX path that a Windows dev host cannot represent; that step has its own + tests above. + """ + target = tmp_path / "portal-shot.png" + target.write_bytes(_png()) + bus = _FakeBus(body=[0, {"uri": _URI}]) + seen = [] + + def resolve(value): + seen.append(value) + return str(target) + + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + side_effect=resolve): + data = wayland_portal.capture_png(timeout=5.0) + + assert seen == [_URI] + assert data == _png() + assert not target.exists() diff --git a/test/unit_test/headless/test_wayland_dbus_client.py b/test/unit_test/headless/test_wayland_dbus_client.py new file mode 100644 index 00000000..77532340 --- /dev/null +++ b/test/unit_test/headless/test_wayland_dbus_client.py @@ -0,0 +1,306 @@ +"""The hand-written D-Bus marshalling behind the xdg-desktop-portal tier. + +``je_auto_control.linux_wayland._dbus_client`` exists because a portal answers +with a signal *directed at the connection that asked*, so the subscription and +the call have to share one connection — which rules out shelling out to +``gdbus`` and leaves marshalling D-Bus by hand. + +Hand-written marshalling is exactly the kind of code that passes review and +fails on a real bus, so it is checked twice. Absolute correctness — that these +bytes are the bytes a real ``dbus-daemon`` accepts and that a real portal +answers — is settled in ``docker/portal_verify.py`` against a real bus. What +is checked here is everything a bus is not needed for: the alignment rules, +the signature walker, container types, address parsing, and that a message +this module writes is one it reads back unchanged. + +No session bus is touched: the socket is a loopback object that hands back +whatever was written to it. +""" +import struct + +import pytest + +from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.linux_wayland._dbus_client import ( + DBusError, SessionBus, Variant, _Reader, _SignatureReader, _Writer, +) + + +class _LoopbackSocket: + """Whatever is written can be read back, so a message can round-trip.""" + + def __init__(self): + self.written = b"" + self.readable = b"" + self.closed = False + + def sendall(self, data): + self.written += data + + def settimeout(self, _timeout): + return None + + def recv(self, size): + chunk, self.readable = self.readable[:size], self.readable[size:] + return chunk + + def close(self): + self.closed = True + + +def _bus(): + """A connection whose socket is loopback, so nothing leaves the process.""" + bus = SessionBus(address="unix:path=/nonexistent") + bus._socket = _LoopbackSocket() + bus.unique_name = ":1.9" + return bus + + +# === Signatures ============================================================ + +@pytest.mark.parametrize("signature, expected", [ + ("s", ["s"]), + ("sa{sv}", ["s", "a{sv}"]), + ("ua{sv}", ["u", "a{sv}"]), + ("a(yv)", ["a(yv)"]), + ("aas", ["aas"]), + ("(us)o", ["(us)", "o"]), + ("a{sa{sv}}", ["a{sa{sv}}"]), +]) +def test_signature_walker_takes_one_whole_type_at_a_time(signature, expected): + """A container has to be followed to its close, however deeply it nests.""" + reader = _SignatureReader(signature) + taken = [] + while not reader.done(): + taken.append(reader.take_complete()) + assert taken == expected + + +# === Byte-exact marshalling ================================================ + +def test_a_string_is_length_then_bytes_then_a_nul(): + writer = _Writer() + writer.value("s", "abc") + assert writer.data == struct.pack("BBBBIII", ord("B"), _dbus_client.SIGNAL, 0, 1, 0, 1, 0) + with pytest.raises(DBusError, match="big-endian"): + bus.read_message(deadline=_never()) + + +# === Addresses ============================================================= + +@pytest.mark.parametrize("address, expected", [ + ("unix:path=/run/user/1000/bus", ("/run/user/1000/bus", False)), + ("unix:path=/tmp/dbus-x,guid=deadbeef", ("/tmp/dbus-x", False)), + ("unix:abstract=/tmp/dbus-y,guid=f00", ("/tmp/dbus-y", True)), + ("tcp:host=localhost,port=1;unix:path=/run/bus", ("/run/bus", False)), +]) +def test_a_bus_address_resolves_to_a_socket(address, expected): + assert _dbus_client._socket_target(address) == expected + + +def test_an_address_with_no_unix_transport_is_refused(): + with pytest.raises(DBusError, match="no usable unix transport"): + _dbus_client._socket_target("tcp:host=localhost,port=1234") + + +def test_availability_follows_the_environment(monkeypatch): + monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + assert _dbus_client.is_available() is False + assert _dbus_client.session_address() is None + monkeypatch.setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/bus") + assert _dbus_client.is_available() is True + + +def test_connecting_without_an_address_says_which_variable_is_missing(monkeypatch): + monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + with pytest.raises(DBusError, match="DBUS_SESSION_BUS_ADDRESS"): + SessionBus().connect() + + +def test_the_sender_token_is_the_unique_name_as_a_path_element(): + """The portal builds the request path out of this, so both sides have to + mangle the name the same way or the client subscribes to the wrong path.""" + bus = SessionBus(address="unix:path=/x") + bus.unique_name = ":1.42" + assert bus.sender_token == "1_42" + + +def test_closing_twice_is_safe_and_closes_the_socket(): + bus = _bus() + socket = bus._socket + bus.close() + bus.close() + assert socket.closed + + +def test_a_dbus_error_is_part_of_the_framework_family(): + """Every containment boundary catches AutoControlException; a sibling of + it would escape all of them.""" + from je_auto_control.utils.exception.exceptions import AutoControlException + assert issubclass(DBusError, AutoControlException) + + +def _never() -> float: + """A deadline far enough away that the loopback socket always wins.""" + import time + return time.monotonic() + 60.0 From 2afc5694d9e379710c607dc78c55f4e013db0152 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:15:05 +0800 Subject: [PATCH 08/21] Send every screen read through the platform grabber MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Eleven call sites imported PIL.ImageGrab or mss directly. On Wayland both read the XWayland root, which holds none of the native windows — so OCR, smart waits, visual regression, the recorder, the WebRTC host, the monitor enumerator, the MCP monitor tools and the template cropper each captured a blank or stale screen while reporting success. screen_grabber is now the one place that decides: it hands back an ImageGrab-shaped and an mss-shaped object backed by the compositor's own capture tool under Wayland, and by Pillow / mss everywhere else. Nothing above it changes shape. The diagnostics bundle gains a screen_capture check naming the tier in use, because a Wayland session with no capture helper is the one case where the framework can still move the mouse and see nothing. It also carries cursor_may_be_captured: wlroots composites a software cursor into the buffer wlr-screencopy hands back whenever the backend has no cursor plane, so a locator can find a pointer-shaped hole in its target, and the report that explains that failure should say so. --- .../gui/remote_desktop/webrtc_panel.py | 4 +- .../gui/selector/template_cropper.py | 4 +- .../utils/cv2_utils/screen_grabber.py | 229 +++++++++++++++ je_auto_control/utils/cv2_utils/screenshot.py | 18 +- .../utils/cv2_utils/video_recording.py | 6 +- .../utils/diagnostics/diagnostics.py | 43 +++ .../utils/mcp_server/tools/_handlers.py | 10 +- .../utils/monitor_layout/logical_frame.py | 35 ++- .../utils/monitor_layout/monitor_layout.py | 4 +- je_auto_control/utils/ocr/ocr_engine.py | 13 +- .../utils/remote_desktop/webrtc_transport.py | 10 +- je_auto_control/utils/smart_waits/waits.py | 7 +- .../utils/visual_regression/compare.py | 11 +- test/unit_test/headless/test_diagnostics.py | 36 +++ .../headless/test_r3_vision_capture.py | 4 +- .../unit_test/headless/test_screen_grabber.py | 278 ++++++++++++++++++ 16 files changed, 674 insertions(+), 38 deletions(-) create mode 100644 je_auto_control/utils/cv2_utils/screen_grabber.py create mode 100644 test/unit_test/headless/test_screen_grabber.py diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel.py b/je_auto_control/gui/remote_desktop/webrtc_panel.py index d5c226cd..b4849caf 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel.py @@ -620,8 +620,8 @@ def _on_toggle_adaptive(self, value: bool) -> None: def _populate_monitor_combo(self) -> None: try: - import mss - with mss.mss() as sct: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as sct: monitors = sct.monitors for idx, mon in enumerate(monitors): if idx == 0: diff --git a/je_auto_control/gui/selector/template_cropper.py b/je_auto_control/gui/selector/template_cropper.py index 35e06426..50d0045c 100644 --- a/je_auto_control/gui/selector/template_cropper.py +++ b/je_auto_control/gui/selector/template_cropper.py @@ -4,7 +4,6 @@ import cv2 import numpy as np -from PIL import ImageGrab from PySide6.QtWidgets import QWidget from je_auto_control.gui.selector.region_selector import open_region_selector @@ -13,8 +12,9 @@ def _capture_region(region: Tuple[int, int, int, int]) -> np.ndarray: """Grab the given (x, y, w, h) region as a BGR numpy array.""" + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber x, y, w, h = region - pil_image = ImageGrab.grab(bbox=(x, y, x + w, y + h), all_screens=True) + pil_image = image_grabber().grab(bbox=(x, y, x + w, y + h), all_screens=True) return cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR) diff --git a/je_auto_control/utils/cv2_utils/screen_grabber.py b/je_auto_control/utils/cv2_utils/screen_grabber.py new file mode 100644 index 00000000..7c977ac3 --- /dev/null +++ b/je_auto_control/utils/cv2_utils/screen_grabber.py @@ -0,0 +1,229 @@ +"""One place that decides how this platform's pixels are read. + +Pillow's ``ImageGrab`` and ``mss`` both read the X11 root window on Linux. +Under a Wayland session that root belongs to XWayland, which does not +composite native Wayland windows, so what comes back is not the desktop the +user sees. + +Pillow does have a fallback — but a narrower one than it first appears. +Reading ``PIL/ImageGrab.py``: the ``gnome-screenshot`` / ``grim`` / +``spectacle`` path runs only inside ``except OSError`` around +``grabscreen_x11``, so it fires when Pillow lacks XCB or there is no X +display at all. With XWayland running — the default on GNOME, KDE and sway — +``DISPLAY`` is set and the X11 grab *succeeds*, so the fallback never +triggers and the caller gets the XWayland root. ``mss`` has no fallback in +any configuration. + +Every locator, OCR call, screenshot, recording and remote-desktop frame in +the framework reached for one of those two libraries directly, so on the +common Wayland setup none of them saw the real screen while the Wayland +backend's own capture went unused. Routing through the backend also makes +the behaviour deterministic rather than dependent on whether XWayland +happens to be running, lets ``grim`` apply a region natively, and gives a +failure an actionable message instead of a blank frame. + +A platform backend that the generic libraries cannot see through publishes +a ``grab_image(screen_region=None) -> PIL.Image.Image`` function (only +:mod:`je_auto_control.linux_wayland.screen` needs to today). This module +wraps it in whichever library shape the caller already uses, so call sites +stay as they were: + +* :func:`image_grabber` — an ``ImageGrab``-shaped object (``.grab(bbox=...)``). +* :func:`mss_grabber` — an ``mss.mss()``-shaped context manager + (``.monitors``, ``.grab(monitor)``). + +A backend may also publish ``layout_origin() -> (x, y)``, the coordinate its +whole-screen capture starts at. It is ``(0, 0)`` for most desktops and +negative on any layout with a monitor left of or above the primary one, which +is why :func:`backend_layout_origin` reports it rather than letting each +caller assume the frame begins at the origin. + +Where the backend publishes no ``grab_image`` — Windows, macOS, Linux X11 — +both functions hand back the real library untouched, so nothing about those +platforms changes. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + +Region = Sequence[int] +GrabImage = Callable[..., Any] + + +def backend_grab_image() -> Optional[GrabImage]: + """Return the active platform backend's ``grab_image``, or None. + + None means the generic libraries can see this session and should be + used as-is. + """ + try: + from je_auto_control.wrapper.platform_wrapper import screen + except (ImportError, AutoControlException): + return None + return getattr(screen, "grab_image", None) + + +class _BackendImageGrab: # pylint: disable=too-few-public-methods # reason: must expose exactly ImageGrab's surface + """``ImageGrab``-shaped adapter over a backend's ``grab_image``.""" + + def __init__(self, grab_image: GrabImage) -> None: + self._grab_image = grab_image + + def grab(self, bbox: Optional[Region] = None, **_kwargs: Any) -> Any: + """Capture ``bbox`` (left, top, right, bottom), or everything. + + ``ImageGrab``'s other keywords (``all_screens``, ``xdisplay``, + ``include_layered_windows``) are accepted and ignored: a Wayland + capture already spans the whole output layout and has no X display + to name, so honouring them would mean promising a distinction the + compositor does not offer. + """ + return self._grab_image(screen_region=list(bbox) if bbox is not None else None) + + +class _BackendShot: + """``mss.ScreenShot``-shaped view of one captured image.""" + + def __init__(self, image: Any, left: int, top: int) -> None: + self._image = image if image.mode == "RGB" else image.convert("RGB") + self.width = int(self._image.width) + self.height = int(self._image.height) + self.left = int(left) + self.top = int(top) + self.bgra = self._image.convert("RGBA").tobytes("raw", "BGRA") + # numpy reads this to build an array, the same way mss's own + # ScreenShot exposes its BGRA buffer. + self.__array_interface__ = { + "version": 3, + "shape": (self.height, self.width, 4), + "typestr": "|u1", + "data": self.bgra, + } + + @property + def size(self) -> Tuple[int, int]: + """``(width, height)``, as ``mss.ScreenShot.size`` reports it.""" + return self.width, self.height + + @property + def pos(self) -> Tuple[int, int]: + """``(left, top)`` of the captured rectangle.""" + return self.left, self.top + + @property + def rgb(self) -> bytes: + """The frame as packed RGB bytes.""" + return self._image.tobytes() + + +class _BackendMss: + """``mss.mss()``-shaped adapter over a backend's ``grab_image``. + + The compositor reports one seamless output layout rather than the + per-monitor rectangles mss enumerates, so ``monitors`` holds the + combined desktop twice: index 0 by mss's convention, and index 1 so + callers that default to "the first real screen" still get a frame. + """ + + def __init__(self, grab_image: GrabImage, + screen_size: Callable[[], Tuple[int, int]], + layout_origin: Optional[Callable[[], Tuple[int, int]]] = None) -> None: + self._grab_image = grab_image + self._screen_size = screen_size + self._layout_origin = layout_origin or (lambda: (0, 0)) + self._monitors: Optional[List[Dict[str, int]]] = None + + @property + def monitors(self) -> List[Dict[str, int]]: + """Monitor rectangles, mss-style: index 0 spans the whole desktop. + + ``left`` / ``top`` are the layout's origin rather than a hardcoded + ``(0, 0)``: mss reports a negative left for a monitor placed to the + left of the primary one, and :meth:`grab` feeds these straight back + as a capture region — so a layout with a negative origin described + as starting at ``(0, 0)`` grabs a rectangle that is off the screen + on one side and misses that much of the desktop on the other. + """ + if self._monitors is None: + width, height = self._screen_size() + left, top = self._layout_origin() + layout = {"left": int(left), "top": int(top), + "width": int(width), "height": int(height)} + self._monitors = [dict(layout), dict(layout)] + return self._monitors + + def grab(self, monitor: Dict[str, int]) -> _BackendShot: + """Capture one monitor rectangle and return an mss-shaped shot.""" + left = int(monitor["left"]) + top = int(monitor["top"]) + right = left + int(monitor["width"]) + bottom = top + int(monitor["height"]) + image = self._grab_image(screen_region=[left, top, right, bottom]) + return _BackendShot(image, left, top) + + def close(self) -> None: + """No handle to release; present so the mss shape is complete.""" + + def __enter__(self) -> "_BackendMss": + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + +def _backend_screen_size() -> Tuple[int, int]: + from je_auto_control.wrapper.platform_wrapper import screen + return screen.size() + + +def backend_layout_origin() -> Tuple[int, int]: + """Top-left of the backend's whole-screen capture, in screen coordinates. + + ``(0, 0)`` for a backend that publishes no ``layout_origin`` — which is + every backend the generic libraries can already see (Windows, macOS, + X11), and the right answer for any layout that starts at the origin + anyway. Only the origin is reported, never the size: a caller that has + the frame already knows how big it is, and asking a backend for its + size can cost a whole extra capture where no tool can report it. + """ + try: + from je_auto_control.wrapper.platform_wrapper import screen + except (ImportError, AutoControlException): + return (0, 0) + origin = getattr(screen, "layout_origin", None) + if not callable(origin): + return (0, 0) + # pylint: disable=not-callable # reason: pylint resolves the getattr to its + # None default against whichever backend this host imports, and every + # backend but Wayland publishes no layout_origin; callable() is the check. + left, top = origin() + return (int(left), int(top)) + + +def image_grabber() -> Any: + """Return an ``ImageGrab``-shaped grabber for the current platform.""" + grab_image = backend_grab_image() + if grab_image is None: + from PIL import ImageGrab + return ImageGrab + return _BackendImageGrab(grab_image) + + +def mss_grabber() -> Any: + """Return an ``mss.mss()``-shaped grabber for the current platform. + + Use in place of ``mss.mss()``; the result is a context manager either + way, so ``with mss_grabber() as sct:`` works unchanged. + """ + grab_image = backend_grab_image() + if grab_image is None: + import mss + # mss 10 deprecated the lowercase factory in favour of MSS; both + # yield the same platform class, so prefer the one that is current. + return (getattr(mss, "MSS", None) or mss.mss)() + return _BackendMss(grab_image, _backend_screen_size, + backend_layout_origin) + + +__all__ = ["backend_grab_image", "backend_layout_origin", "image_grabber", + "mss_grabber"] diff --git a/je_auto_control/utils/cv2_utils/screenshot.py b/je_auto_control/utils/cv2_utils/screenshot.py index 0c10ccbd..3c6c84a4 100644 --- a/je_auto_control/utils/cv2_utils/screenshot.py +++ b/je_auto_control/utils/cv2_utils/screenshot.py @@ -1,6 +1,7 @@ -from PIL import ImageGrab, Image +from PIL import Image from typing import List, Optional +from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber from je_auto_control.utils.exception.exceptions import AutoControlScreenException @@ -31,8 +32,14 @@ def _validate_region(screen_region: List[int]) -> None: def pil_screenshot(file_path: Optional[str] = None, screen_region: Optional[List[int]] = None) -> Image.Image: """ - Take a screenshot using PIL (Pillow). - 使用 PIL (Pillow) 擷取螢幕畫面 + Take a screenshot through the platform's capture backend. + 透過平台擷取後端擷取螢幕畫面 + + Kept named ``pil_screenshot`` (and still returning a Pillow image) + because it is public API, but the grabber is now chosen per platform: + Pillow's ``ImageGrab`` reads the X11 root window, which under Wayland + belongs to XWayland and holds none of the native Wayland windows. See + :mod:`je_auto_control.utils.cv2_utils.screen_grabber`. :param file_path: (str | None) Path to save the screenshot. If None, do not save. 螢幕截圖的存檔路徑，若為 None 則不存檔 @@ -41,11 +48,12 @@ def pil_screenshot(file_path: Optional[str] = None, screen_region: Optional[List :return: PIL.Image.Image object 擷取到的影像物件 """ # 擷取螢幕畫面 Capture screen + grabber = image_grabber() if screen_region is not None: _validate_region(screen_region) - image = ImageGrab.grab(bbox=screen_region) + image = grabber.grab(bbox=screen_region) else: - image = ImageGrab.grab() + image = grabber.grab() # 如果指定了存檔路徑，則存檔 Save if file_path is provided. # Fail fast: a swallowed save error would leave the caller believing the diff --git a/je_auto_control/utils/cv2_utils/video_recording.py b/je_auto_control/utils/cv2_utils/video_recording.py index b2efe91a..3128b8c2 100644 --- a/je_auto_control/utils/cv2_utils/video_recording.py +++ b/je_auto_control/utils/cv2_utils/video_recording.py @@ -1,8 +1,8 @@ import threading import cv2 import numpy as np -from mss import mss +from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -10,7 +10,7 @@ class RecordingThread(threading.Thread): """ RecordingThread 螢幕錄影執行緒 - - 使用 mss 擷取螢幕畫面 + - 透過平台擷取後端取得畫面 (mss，Wayland 則走合成器的擷取工具) - 使用 OpenCV VideoWriter 寫入影片檔案 """ @@ -44,7 +44,7 @@ def run(self): 執行錄影迴圈 Run recording loop """ - with mss() as sct: + with mss_grabber() as sct: resolution = sct.monitors[0] output_file = self.video_name + ".mp4" diff --git a/je_auto_control/utils/diagnostics/diagnostics.py b/je_auto_control/utils/diagnostics/diagnostics.py index ddbecb4e..4b214995 100644 --- a/je_auto_control/utils/diagnostics/diagnostics.py +++ b/je_auto_control/utils/diagnostics/diagnostics.py @@ -136,6 +136,48 @@ def _check_audit_chain() -> Check: ) +def _check_screen_capture() -> Check: + """Report which grabber reads the screen, and on Wayland which tool. + + A Wayland session with no capture helper installed is the one case where + the framework can still move the mouse but sees nothing, so it is worth + naming before a locator fails with "template not found". + + The Wayland answer also carries ``cursor_may_be_captured``. wlroots + composites a *software* cursor into the output buffer whenever the + backend has no cursor plane, and that buffer is what ``wlr-screencopy`` + hands back, so the pointer can appear in a capture that never asked for + it — unlike BitBlt on Windows or Pillow/mss on X11. A pointer resting on + a target puts a pointer-shaped hole in the middle of it, and the bundle + that explains a failed locator should say so. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import backend_grab_image + if backend_grab_image() is None: + return Check( + name="screen_capture", ok=True, severity=_SEVERITY_INFO, + detail="generic capture (Pillow / mss)", + extra={"grabber": "generic"}, + ) + from je_auto_control.linux_wayland.capture import available_tool + tool = available_tool() + if tool is None: + return Check( + name="screen_capture", ok=False, severity=_SEVERITY_ERROR, + detail="Wayland session with no capture tool and no session " + "bus: install grim (wlroots), gnome-screenshot (GNOME) or " + "spectacle (KDE); the xdg-desktop-portal fallback needs " + "DBUS_SESSION_BUS_ADDRESS set", + extra={"grabber": "wayland", "tool": None}, + ) + return Check( + name="screen_capture", ok=True, severity=_SEVERITY_INFO, + detail=f"Wayland capture via {tool}; the pointer may be composited " + "into the image, so park it away from the target", + extra={"grabber": "wayland", "tool": tool, + "cursor_may_be_captured": True}, + ) + + def _check_screenshot() -> Check: from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot image = pil_screenshot() @@ -218,6 +260,7 @@ def _check_executor() -> Check: _check_optional_deps, _check_executor, _check_audit_chain, + _check_screen_capture, _check_screenshot, _check_mouse, _check_disk_space, diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index b86a15d6..d5a5d39b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -145,8 +145,8 @@ def screenshot(file_path: Optional[str] = None, def list_monitors() -> List[Dict[str, Any]]: """Return every monitor's geometry. Index 0 spans all monitors.""" - import mss - with mss.mss() as sct: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as sct: return [ { "index": index, "left": int(monitor["left"]), @@ -160,10 +160,10 @@ def list_monitors() -> List[Dict[str, Any]]: def _grab_monitor(index: int): - """Capture a single monitor via ``mss`` and return a PIL Image.""" - import mss + """Capture a single monitor through the platform grabber; returns a PIL Image.""" + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber from PIL import Image - with mss.mss() as sct: + with mss_grabber() as sct: if index < 0 or index >= len(sct.monitors): raise ValueError( f"monitor index {index} out of range " diff --git a/je_auto_control/utils/monitor_layout/logical_frame.py b/je_auto_control/utils/monitor_layout/logical_frame.py index 3fd3de7b..04da7a5e 100644 --- a/je_auto_control/utils/monitor_layout/logical_frame.py +++ b/je_auto_control/utils/monitor_layout/logical_frame.py @@ -19,6 +19,14 @@ desktop starts at negative coordinates whenever a monitor sits left of or above the primary one. +Wayland has the same requirement without the DPI half: its capture spans the +compositor's whole output layout, and that layout starts at a negative +coordinate whenever an output sits left of or above the origin. There is no +``GetSystemMetrics`` to ask, so the origin comes from the backend itself +(``screen_grabber.backend_layout_rect``) — without it a hit found in the frame +is reported 1920 px (or whatever the left-hand monitor is wide) to the right of +where it was matched. + The arithmetic (:func:`needs_rescale`, :func:`logical_scale`) is pure and unit-testable; the OS reader and the grabber are both injectable. Imports no ``PySide6``. @@ -72,10 +80,29 @@ def needs_rescale(physical: Tuple[int, int], logical: Tuple[int, int]) -> bool: return bool(logical[0] and logical[1]) and tuple(physical) != tuple(logical) +def _backend_frame_origin() -> Tuple[int, int]: + """Where a self-capturing backend's frame starts, ``(0, 0)`` by default. + + ``logical_virtual_rect`` reads ``GetSystemMetrics``, so off Windows it + has nothing to say — but the Wayland backend captures the compositor's + whole output layout, and that layout starts at a negative coordinate + whenever an output sits left of or above the origin. Treating the frame + as starting at ``(0, 0)`` there offsets every located hit by the origin, + which reads as "the click lands on the wrong monitor" rather than as a + failure to find. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import backend_layout_origin + return backend_layout_origin() + + def _load_image_grab(): - """Import Pillow's ``ImageGrab`` lazily (optional dependency at runtime).""" - from PIL import ImageGrab - return ImageGrab + """Load the platform's ``ImageGrab``-shaped grabber lazily. + + Pillow off Wayland, the compositor's capture tool on it — see + :mod:`je_auto_control.utils.cv2_utils.screen_grabber`. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber + return image_grabber() def _resample(): @@ -105,7 +132,7 @@ def grab_logical(region: Optional[Sequence[int]] = None, *, image = image_grab.grab(all_screens=True) rect = logical_virtual_rect(metrics) - origin_x, origin_y = (rect[0], rect[1]) if rect else (0, 0) + origin_x, origin_y = (rect[0], rect[1]) if rect else _backend_frame_origin() if rect and needs_rescale((image.width, image.height), (rect[2], rect[3])): image = image.resize((rect[2], rect[3]), _resample()) if region is None: diff --git a/je_auto_control/utils/monitor_layout/monitor_layout.py b/je_auto_control/utils/monitor_layout/monitor_layout.py index 2fb46f32..99ba0ed9 100644 --- a/je_auto_control/utils/monitor_layout/monitor_layout.py +++ b/je_auto_control/utils/monitor_layout/monitor_layout.py @@ -130,8 +130,8 @@ def remap_point(src: Monitor, dst: Monitor, local_x: int, def _mss_rows() -> List[Dict[str, Any]]: - import mss - with mss.mss() as screen: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as screen: monitors = screen.monitors return [{"x": int(m["left"]), "y": int(m["top"]), "width": int(m["width"]), "height": int(m["height"])} diff --git a/je_auto_control/utils/ocr/ocr_engine.py b/je_auto_control/utils/ocr/ocr_engine.py index 0d13b3c2..2b15cb06 100644 --- a/je_auto_control/utils/ocr/ocr_engine.py +++ b/je_auto_control/utils/ocr/ocr_engine.py @@ -27,17 +27,24 @@ def _load_image_grab(): + """Cache the platform's ``ImageGrab``-shaped grabber. + + Not Pillow directly: under Wayland the screen has to come from the + compositor's own capture tool, or OCR reads a blank XWayland root. + """ global _image_grab if _image_grab is not None: return _image_grab + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber try: - from PIL import ImageGrab + # image_grabber imports Pillow lazily, so a missing Pillow surfaces + # here rather than on the import above. + _image_grab = image_grabber() except ImportError as error: raise RuntimeError( "OCR requires Pillow for screen capture. Install with: pip install Pillow" ) from error - _image_grab = ImageGrab - return ImageGrab + return _image_grab @dataclass(frozen=True) diff --git a/je_auto_control/utils/remote_desktop/webrtc_transport.py b/je_auto_control/utils/remote_desktop/webrtc_transport.py index 9fd3cb8b..b84d1722 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_transport.py +++ b/je_auto_control/utils/remote_desktop/webrtc_transport.py @@ -30,10 +30,14 @@ ) from exc try: - import mss # type: ignore + import mss # type: ignore # noqa: F401 # reason: annotations + install probe except ImportError as exc: # pragma: no cover - mss is a base dep raise ImportError("mss is required for screen capture") from exc +# Frames come from the platform grabber rather than mss directly: on Wayland +# mss reads the XWayland root, which holds none of the native Wayland windows. +from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + _DEFAULT_STUN = "stun:stun.l.google.com:19302" _DEFAULT_STUN_SERVERS = ( @@ -215,7 +219,7 @@ def _resolve_monitor(sct: "mss.base.MSSBase", index: int) -> dict: def _capture_frame(monitor: dict) -> "np.ndarray": sct = getattr(_capture_local, "sct", None) if sct is None: - sct = mss.mss() + sct = mss_grabber() _capture_local.sct = sct img = sct.grab(monitor) arr = np.frombuffer(img.bgra, dtype=np.uint8).reshape( @@ -279,7 +283,7 @@ def _resolve(self) -> dict: else: sct = getattr(_capture_local, "sct", None) if sct is None: - sct = mss.mss() + sct = mss_grabber() _capture_local.sct = sct self._monitor = _resolve_monitor(sct, self._monitor_index) return self._monitor diff --git a/je_auto_control/utils/smart_waits/waits.py b/je_auto_control/utils/smart_waits/waits.py index 2e08b12f..ce79d247 100644 --- a/je_auto_control/utils/smart_waits/waits.py +++ b/je_auto_control/utils/smart_waits/waits.py @@ -54,15 +54,16 @@ class Frame: def _default_sampler(region: Optional[Sequence[int]]) -> Frame: - """Use PIL ImageGrab to snapshot once. Fails closed on missing dep.""" + """Snapshot once through the platform's grabber. Fails closed on missing dep.""" + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber try: - from PIL import ImageGrab + grabber = image_grabber() except ImportError as error: raise RuntimeError( "Smart waits require Pillow for screen capture.", ) from error bbox = tuple(int(v) for v in region) if region else None - image = ImageGrab.grab(bbox=bbox).convert("RGB") + image = grabber.grab(bbox=bbox).convert("RGB") return Frame(width=image.width, height=image.height, pixels=image.tobytes()) diff --git a/je_auto_control/utils/visual_regression/compare.py b/je_auto_control/utils/visual_regression/compare.py index 9a4d52f0..d26d5f1b 100644 --- a/je_auto_control/utils/visual_regression/compare.py +++ b/je_auto_control/utils/visual_regression/compare.py @@ -89,13 +89,14 @@ def take_golden(path, def _grab(region: Optional[Sequence[int]]) -> Image.Image: - """Screen capture via PIL.ImageGrab; raises if not available.""" - from PIL import ImageGrab + """Screen capture via the platform's grabber; raises if not available.""" + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber + grabber = image_grabber() if region is not None: x, y, width, height = (int(v) for v in region) - return ImageGrab.grab(bbox=(x, y, x + width, y + height), - all_screens=True) - return ImageGrab.grab(all_screens=True) + return grabber.grab(bbox=(x, y, x + width, y + height), + all_screens=True) + return grabber.grab(all_screens=True) def image_difference(actual: Image.Image, expected: Image.Image, diff --git a/test/unit_test/headless/test_diagnostics.py b/test/unit_test/headless/test_diagnostics.py index 2f207e35..4a7d4c64 100644 --- a/test/unit_test/headless/test_diagnostics.py +++ b/test/unit_test/headless/test_diagnostics.py @@ -1,6 +1,7 @@ """Tests for the system diagnostics runner (round 28).""" import subprocess import sys +from unittest.mock import patch from je_auto_control.utils.diagnostics.diagnostics import ( Check, DiagnosticsReport, run_diagnostics, @@ -52,3 +53,38 @@ def test_cli_exits_zero_when_all_green(): # cleanly with one of those codes and emit the summary line. assert completed.returncode in (0, 1), completed.returncode assert "Summary:" in completed.stdout + + +def _wayland_capture_check(tool="grim"): + """Run the capture check with the Wayland grabber and ``tool`` present.""" + from je_auto_control.linux_wayland import capture + from je_auto_control.utils.cv2_utils import screen_grabber + from je_auto_control.utils.diagnostics import diagnostics + with patch.object(screen_grabber, "backend_grab_image", + return_value=lambda *a, **k: None), patch.object(capture, "available_tool", return_value=tool): + return diagnostics._check_screen_capture() + + +def test_wayland_capture_reports_the_tool_it_found(): + check = _wayland_capture_check() + assert check.ok is True + assert check.extra["grabber"] == "wayland" + assert check.extra["tool"] == "grim" + + +def test_wayland_capture_warns_that_the_pointer_may_be_in_the_image(): + """wlroots composites a software cursor into the buffer screencopy hands + back, so a locator can find a pointer-shaped hole in its target. The + bundle that explains that failure has to carry the fact.""" + check = _wayland_capture_check() + assert check.extra["cursor_may_be_captured"] is True + assert "pointer" in check.detail + + +def test_the_generic_grabber_says_nothing_about_the_pointer(): + """BitBlt and Pillow/mss never include it; only Wayland does.""" + from je_auto_control.utils.cv2_utils import screen_grabber + from je_auto_control.utils.diagnostics import diagnostics + with patch.object(screen_grabber, "backend_grab_image", return_value=None): + check = diagnostics._check_screen_capture() + assert check.extra == {"grabber": "generic"} diff --git a/test/unit_test/headless/test_r3_vision_capture.py b/test/unit_test/headless/test_r3_vision_capture.py index 8f157f4a..f8e36cd8 100644 --- a/test/unit_test/headless/test_r3_vision_capture.py +++ b/test/unit_test/headless/test_r3_vision_capture.py @@ -126,7 +126,9 @@ def grab(*_args, **_kwargs): def test_screenshot_save_failure_raises(monkeypatch, tmp_path): pytest.importorskip("PIL") - monkeypatch.setattr(ss, "ImageGrab", _FakeGrab) + # pil_screenshot now asks the platform layer for its grabber rather than + # importing ImageGrab itself, so Wayland gets the compositor's capture. + monkeypatch.setattr(ss, "image_grabber", lambda: _FakeGrab) bad_path = str(tmp_path / "no_such_dir" / "shot.png") # parent missing with pytest.raises(AutoControlScreenException): ss.pil_screenshot(file_path=bad_path) diff --git a/test/unit_test/headless/test_screen_grabber.py b/test/unit_test/headless/test_screen_grabber.py new file mode 100644 index 00000000..f4e7ed4a --- /dev/null +++ b/test/unit_test/headless/test_screen_grabber.py @@ -0,0 +1,278 @@ +"""The capture layer must reach the platform backend, not Pillow directly. + +Every locator, OCR call, screenshot, recording and remote-desktop frame in +the framework used to call ``PIL.ImageGrab`` or ``mss`` itself. Both read the +X11 root window on Linux, which under Wayland belongs to XWayland and holds +none of the native Wayland windows — so all of them captured a blank screen +while the Wayland backend's own capture sat unused and unreachable. + +These tests pin the seam that fixes it: when the active backend publishes +``grab_image`` every capture path goes through it, and when it does not +(Windows, macOS, X11) the real libraries are handed back untouched. No Qt, +and no dependency on the host's display server. +""" +import contextlib +import sys +from unittest.mock import patch + +import pytest +from PIL import Image + +from je_auto_control.utils.cv2_utils import screen_grabber + + +class _FakeBackendScreen: + """Minimal stand-in for a backend screen module that can capture.""" + + def __init__(self, size=(8, 4), colour=(10, 20, 30), origin=(0, 0)): + self._size = size + self._colour = colour + self._origin = origin + self.calls = [] + + def grab_image(self, screen_region=None): + self.calls.append(screen_region) + if screen_region is None: + return Image.new("RGB", self._size, self._colour) + x1, y1, x2, y2 = screen_region + return Image.new("RGB", (x2 - x1, y2 - y1), self._colour) + + def size(self): + return self._size + + def layout_origin(self): + return self._origin + + +@contextlib.contextmanager +def _with_backend(backend): + """Patch the wrapper's ``screen`` so the grabbers see our fake backend. + + Both halves of the seam have to come from the fake. ``mss_grabber`` + reports its layout from ``platform_wrapper.screen.size()``, so patching + only ``grab_image`` would leave ``monitors`` describing whatever display + the test host happens to have -- passing on a 1920x1080 desktop and + failing under a 1280x800 Xvfb for reasons unrelated to the code here. + """ + with patch.object(screen_grabber, "backend_grab_image", + return_value=backend.grab_image): + with patch.object(screen_grabber, "_backend_screen_size", + side_effect=backend.size): + with patch.object(screen_grabber, "backend_layout_origin", + side_effect=backend.layout_origin): + yield + + +# === Backend detection ===================================================== + +def test_backend_grab_image_is_none_without_a_publishing_backend(): + """Windows / macOS / X11 publish no grab_image, and must keep using the + real libraries — this fix must change nothing on those platforms.""" + class _PlainScreen: + def size(self): + return (1920, 1080) + + with patch.dict(sys.modules, + {"je_auto_control.wrapper.platform_wrapper": + type(sys)("stub")}): + sys.modules["je_auto_control.wrapper.platform_wrapper"].screen = \ + _PlainScreen() + assert screen_grabber.backend_grab_image() is None + + +def test_image_grabber_returns_pillow_when_no_backend_publishes_capture(): + with patch.object(screen_grabber, "backend_grab_image", + return_value=None): + from PIL import ImageGrab + assert screen_grabber.image_grabber() is ImageGrab + + +# === ImageGrab-shaped adapter ============================================== + +def test_image_grabber_routes_full_screen_grab_to_the_backend(): + backend = _FakeBackendScreen() + with _with_backend(backend): + image = screen_grabber.image_grabber().grab() + assert backend.calls == [None] + assert image.size == (8, 4) + + +def test_image_grabber_passes_the_bbox_through_as_a_region(): + backend = _FakeBackendScreen() + with _with_backend(backend): + image = screen_grabber.image_grabber().grab(bbox=(2, 3, 6, 9)) + assert backend.calls == [[2, 3, 6, 9]] + assert image.size == (4, 6) + + +def test_image_grabber_tolerates_pillow_only_keywords(): + """Callers pass all_screens / xdisplay; a Wayland capture already spans + the whole layout, so these must be accepted and ignored rather than + raising TypeError deep inside a locator.""" + backend = _FakeBackendScreen() + with _with_backend(backend): + grabber = screen_grabber.image_grabber() + grabber.grab(all_screens=True) + grabber.grab(bbox=(0, 0, 2, 2), all_screens=True, xdisplay=None) + assert backend.calls == [None, [0, 0, 2, 2]] + + +def test_logical_frame_capture_goes_through_the_backend(): + """grab_logical feeds the template, OCR and visual-match locators; it + resolves its grabber lazily, so the seam has to hold there too.""" + from je_auto_control.utils.monitor_layout import logical_frame + backend = _FakeBackendScreen() + # Pin the virtual-desktop metrics so the assertion is about the grabber + # seam, not about whatever monitors this test host happens to have. + flat_desktop = {76: 0, 77: 0, 78: 8, 79: 4} + with _with_backend(backend): + image, origin_x, origin_y = logical_frame.grab_logical( + metrics=flat_desktop.__getitem__) + assert (origin_x, origin_y) == (0, 0) + assert image.size == (8, 4) + assert backend.calls == [None] + + +def test_pil_screenshot_goes_through_the_backend(tmp_path): + from je_auto_control.utils.cv2_utils import screenshot as screenshot_module + backend = _FakeBackendScreen() + target = tmp_path / "shot.png" + with _with_backend(backend): + image = screenshot_module.pil_screenshot(file_path=str(target)) + assert image.size == (8, 4) + assert target.exists() + assert backend.calls == [None] + + +# === mss-shaped adapter ==================================================== + +def test_mss_grabber_reports_the_layout_as_monitor_zero_and_one(): + backend = _FakeBackendScreen(size=(1920, 1080)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + monitors = sct.monitors + assert len(monitors) == 2 + assert monitors[0] == {"left": 0, "top": 0, + "width": 1920, "height": 1080} + # Callers that default to "the first real screen" must still get a frame. + assert monitors[1] == monitors[0] + + +def test_mss_grabber_reports_a_negative_layout_origin(): + """Regression: ``monitors`` was hardcoded to start at (0, 0). + + A Wayland layout starts at a negative coordinate whenever an output + sits left of or above the origin, and mss's own callers already expect + a negative ``left`` there. ``grab`` feeds the rectangle straight back as + a capture region, so claiming (0, 0) grabbed half of one monitor plus + the empty space past the desktop's right edge. + """ + backend = _FakeBackendScreen(size=(2560, 720), origin=(-1280, 0)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + monitors = sct.monitors + sct.grab(monitors[1]) + assert monitors[0] == {"left": -1280, "top": 0, + "width": 2560, "height": 720} + assert backend.calls == [[-1280, 0, 1280, 720]] + + +def test_backend_layout_origin_is_the_origin_without_a_publishing_backend(): + """Windows / macOS / X11 publish no layout_origin, and a guess would be worse.""" + class _PlainScreen: + def size(self): + return (1920, 1080) + + with patch.dict(sys.modules, + {"je_auto_control.wrapper.platform_wrapper": + type(sys)("stub")}): + sys.modules["je_auto_control.wrapper.platform_wrapper"].screen = \ + _PlainScreen() + assert screen_grabber.backend_layout_origin() == (0, 0) + + +def test_backend_layout_origin_reads_a_publishing_backend(): + backend = _FakeBackendScreen(size=(2560, 720), origin=(-1280, -200)) + with patch.dict(sys.modules, + {"je_auto_control.wrapper.platform_wrapper": + type(sys)("stub")}): + sys.modules["je_auto_control.wrapper.platform_wrapper"].screen = backend + assert screen_grabber.backend_layout_origin() == (-1280, -200) + + +def test_mss_shot_exposes_the_bgra_buffer_callers_read(): + """mss consumers read .bgra / .size directly (the MCP monitor grab) and + hand the shot to numpy (the recorder), so both shapes must hold.""" + backend = _FakeBackendScreen(size=(2, 1), colour=(10, 20, 30)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + shot = sct.grab({"left": 0, "top": 0, "width": 2, "height": 1}) + assert shot.size == (2, 1) + assert (shot.width, shot.height) == (2, 1) + # BGRA order, so the blue channel comes first. + assert shot.bgra[:4] == bytes((30, 20, 10, 255)) + assert shot.rgb[:3] == bytes((10, 20, 30)) + rebuilt = Image.frombytes("RGB", shot.size, shot.bgra, "raw", "BGRX") + assert rebuilt.getpixel((0, 0)) == (10, 20, 30) + + +def test_mss_shot_converts_to_a_numpy_bgra_array(): + numpy = pytest.importorskip("numpy") + backend = _FakeBackendScreen(size=(3, 2)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + shot = sct.grab({"left": 0, "top": 0, "width": 3, "height": 2}) + array = numpy.array(shot) + assert array.shape == (2, 3, 4) + + +# === Whole chain, public API down to the compositor's tool ================= + +def _solid_png(size, rgb) -> bytes: + from io import BytesIO + buffer = BytesIO() + Image.new("RGB", size, rgb).save(buffer, format="PNG") + return buffer.getvalue() + + +def test_public_screenshot_returns_the_compositors_pixels_on_wayland(): + """The regression this whole seam exists for. + + ``je_auto_control.screenshot()`` used to call ImageGrab, so on Wayland it + returned the XWayland root — blank — while the backend's grim capture was + unreachable. Drive the real public entry point with a Wayland backend and + a stubbed grim, and the bytes that come back have to be grim's. + """ + import subprocess + + from je_auto_control.linux_wayland import capture as wayland_capture + from je_auto_control.linux_wayland import screen as wayland_screen + from je_auto_control.wrapper.auto_control_screen import screenshot + + png = _solid_png((3, 2), (10, 20, 30)) + + def fake_grim(argv, **_kwargs): + assert argv[0] == "/usr/bin/grim" + return subprocess.CompletedProcess(argv, 0, png, b"") # nosemgrep + + with patch.object(screen_grabber, "backend_grab_image", + return_value=wayland_screen.grab_image), \ + patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_capture.subprocess, "run", + side_effect=fake_grim): + frame = screenshot() + + # The wrapper hands back BGR for OpenCV, so RGB (10, 20, 30) arrives + # reversed — proof these are grim's pixels and not an empty grab. + assert frame.shape == (2, 3, 3) + assert tuple(int(v) for v in frame[0][0]) == (30, 20, 10) + + +def test_mss_grabber_grabs_the_requested_monitor_rectangle(): + backend = _FakeBackendScreen() + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + shot = sct.grab({"left": 5, "top": 6, "width": 3, "height": 4}) + assert backend.calls == [[5, 6, 8, 10]] + assert (shot.left, shot.top) == (5, 6) From c6c57ce074c3bc13255b9a20082d2932f05a34d0 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:15:30 +0800 Subject: [PATCH 09/21] Stop reporting success for Wayland input that went nowhere MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four ways this backend claimed to have moved something it had not. ydotool 0.1.x — what Debian bookworm and every current Ubuntu ship under that name — exits 0 for every argument this backend builds and emits nothing at all. Run with check=True, nothing raised. The CLI generation is now classified once per process and an old one is refused by name; a version the probe does not recognise is let through so this cannot block a future release. libei discards absolute motion outside the regions the compositor advertises, without a return code, an event or an error. Region offsets are uint32, so no region can cover the negative layout space a monitor left of the primary one creates: the pointer went nowhere and never reached the ydotool fallback. The sender now maps the point into region space, retries normalised by the layout origin, and refuses what neither covers so _select_input hands over. An emission a live backend refused raised instead of falling back, as libei's own docstring claimed it did not; a chord refused part-way now releases what it already pressed. LibeiUnavailable derives from AutoControlException so it stops escaping every containment boundary, and a completed session is released rather than leaked. Absolute moves through ydotool now subtract layout_origin(): --absolute counts from the layout's top-left corner, not layout (0, 0). What the compositor then does to that displacement is acceleration nobody can read back, so JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL is how an operator declares it — flat moves silently, strict refuses rather than land somewhere else, unset keeps the warn-once default. Scroll also goes through libei now, with the vertical axis negated: ydotool counts detents in the kernel's REL_WHEEL frame and libei in wl_pointer's. --- je_auto_control/linux_wayland/__init__.py | 18 +- je_auto_control/linux_wayland/_ctypes_bind.py | 75 ++ .../linux_wayland/_select_input.py | 38 +- je_auto_control/linux_wayland/_ydotool_cli.py | 134 +++ je_auto_control/linux_wayland/keyboard.py | 67 +- je_auto_control/linux_wayland/libei.py | 700 ++++++++++--- je_auto_control/linux_wayland/mouse.py | 262 ++++- je_auto_control/linux_wayland/oeffis.py | 196 ++++ test/unit_test/headless/test_wayland_libei.py | 965 +++++++++++++++--- .../unit_test/headless/test_wayland_oeffis.py | 164 +++ .../headless/test_wayland_pointer_accel.py | 170 +++ .../headless/test_wayland_ydotool_cli.py | 206 ++++ .../headless/test_wayland_ydotool_origin.py | 175 ++++ 13 files changed, 2810 insertions(+), 360 deletions(-) create mode 100644 je_auto_control/linux_wayland/_ctypes_bind.py create mode 100644 je_auto_control/linux_wayland/_ydotool_cli.py create mode 100644 je_auto_control/linux_wayland/oeffis.py create mode 100644 test/unit_test/headless/test_wayland_oeffis.py create mode 100644 test/unit_test/headless/test_wayland_pointer_accel.py create mode 100644 test/unit_test/headless/test_wayland_ydotool_cli.py create mode 100644 test/unit_test/headless/test_wayland_ydotool_origin.py diff --git a/je_auto_control/linux_wayland/__init__.py b/je_auto_control/linux_wayland/__init__.py index d49b0f5b..ce073388 100644 --- a/je_auto_control/linux_wayland/__init__.py +++ b/je_auto_control/linux_wayland/__init__.py @@ -7,9 +7,16 @@ * **wtype** — keyboard input via the ``wlr-virtual-keyboard-v1`` protocol (works on wlroots compositors: sway, hyprland, river); * **ydotool** — keyboard + mouse via ``/dev/uinput`` (works on - GNOME / KDE / wlroots, but the daemon needs uinput permission); + GNOME / KDE / wlroots, but the daemon needs uinput permission, and + it must be **1.0 or newer**: 0.1.x answers this backend's argv with + exit code 0 and no events, so ``_ydotool_cli`` refuses it); +* **libei** — the compositor's own input-emulation protocol, reached + through the ``RemoteDesktop`` desktop portal (see ``libei`` and + ``oeffis``). Preferred where it comes up, since it emits without + spawning a process per keystroke; falls back to ydotool otherwise; * **grim** — screenshot via the ``wlr-screencopy`` protocol (wlroots); - ``gnome-screenshot`` is used as a fallback on GNOME / KDE. + ``gnome-screenshot``, ``spectacle`` and the desktop portal back it up + (see ``capture`` and ``portal``). Each helper module probes for the matching binary lazily, so importing this package on a non-Linux host (e.g. CI on Windows / macOS) does not @@ -24,17 +31,18 @@ select_display_server, ) from je_auto_control.linux_wayland._select_input import ( - select_input_backend, + active_backend, select_input_backend, ) from je_auto_control.linux_wayland.libei import ( - LibeiBackend, LibeiUnavailable, get_default_backend, + LibeiBackend, LibeiUnavailable, connected_backend, get_default_backend, ) __all__ = [ "LibeiBackend", "LibeiUnavailable", "WAYLAND_GRIM", "WAYLAND_WTYPE", "WAYLAND_YDOTOOL", - "binary_path", "get_default_backend", "is_wayland_session", + "active_backend", "binary_path", "connected_backend", + "get_default_backend", "is_wayland_session", "missing_dependencies", "select_display_server", "select_input_backend", ] diff --git a/je_auto_control/linux_wayland/_ctypes_bind.py b/je_auto_control/linux_wayland/_ctypes_bind.py new file mode 100644 index 00000000..573bf434 --- /dev/null +++ b/je_auto_control/linux_wayland/_ctypes_bind.py @@ -0,0 +1,75 @@ +"""Shared ctypes plumbing for the Wayland native bindings. + +``libei`` and ``liboeffis`` are reached the same way: find the shared object +on the loader path, resolve a table of entry points with explicit prototypes, +and expose them by name so a test can substitute any object carrying the same +attributes. Keeping one copy matters more than the few lines it saves — a +second copy is a second place for a prototype to drift away from the header +it was written from. +""" +from __future__ import annotations + +import ctypes +import ctypes.util +from typing import Any, Iterable, Optional, Sequence, Tuple + + +#: ``(name, restype, argtypes)`` as ctypes wants them. +Prototype = Tuple[str, Any, tuple] + + +class BoundSymbols: + """Entry points resolved out of one shared object, addressed by name.""" + + def __init__(self, lib: ctypes.CDLL, prototypes: Iterable[Prototype], *, + unchecked: Sequence[str] = ()) -> None: + self._fn = {} + self.lib = lib + for name, restype, argtypes in prototypes: + entry = getattr(lib, name) + entry.restype = restype + entry.argtypes = argtypes + self._fn[name] = entry + for name in unchecked: + # Variadic entry points: ctypes has to infer each argument's type + # at the call site, so an argtypes tuple would be wrong here. + self._fn[name] = getattr(lib, name) + + def __getattr__(self, name: str): + try: + return self._fn[name] + except KeyError: + raise AttributeError(name) from None + + +def load_library(candidates: Iterable[str]) -> Optional[ctypes.CDLL]: + """Return the first of ``candidates`` that loads, or None if none do.""" + for name in candidates: + resolved = ctypes.util.find_library(name) + if resolved is None: + continue + try: + return ctypes.CDLL(resolved, use_errno=True) + except (OSError, RuntimeError): + continue + return None + + +def bind(candidates: Iterable[str], prototypes: Iterable[Prototype], *, + unchecked: Sequence[str] = ()) -> Optional[BoundSymbols]: + """Load a library and resolve its prototypes, or None if either fails. + + A missing entry point means the installed library is not the one these + prototypes were written for, which is a "not available" rather than an + error the caller should have to handle separately. + """ + lib = load_library(candidates) + if lib is None: + return None + try: + return BoundSymbols(lib, prototypes, unchecked=unchecked) + except AttributeError: + return None + + +__all__ = ["BoundSymbols", "Prototype", "bind", "load_library"] diff --git a/je_auto_control/linux_wayland/_select_input.py b/je_auto_control/linux_wayland/_select_input.py index a37b66c7..60bff5ea 100644 --- a/je_auto_control/linux_wayland/_select_input.py +++ b/je_auto_control/linux_wayland/_select_input.py @@ -46,4 +46,40 @@ def _libei_loadable() -> bool: return False -__all__ = ["select_input_backend"] +def active_backend(): + """Return a connected libei backend, or None to use the ydotool CLI. + + The single entry point ``keyboard`` and ``mouse`` use. It lives here + rather than in :mod:`libei` so the dependency runs one way — this module + decides which input path is wanted, :mod:`libei` only knows how to bring + one up. None is the answer on any host where libei is absent, the + portal declines, or the handshake does not complete. + """ + try: + if select_input_backend() != "libei": + return None + from je_auto_control.linux_wayland.libei import connected_backend + return connected_backend() + except (ImportError, OSError, RuntimeError): + return None + + +def emitted(backend, send) -> bool: + """Run one emission on ``backend``; False means the CLI has to take it. + + A backend that finished its handshake can still refuse a single + emission: the compositor pauses a device, or the session ends between + two calls. :mod:`libei` is documented as the fast path and never the + only one, so a refusal falls through to the ``ydotool`` / ``wtype`` + shims here rather than reaching the caller — and if those are missing + too, *they* raise. Nothing is swallowed; the failure just changes hands. + """ + from je_auto_control.linux_wayland.libei import LibeiUnavailable + try: + send(backend) + except LibeiUnavailable: + return False + return True + + +__all__ = ["active_backend", "emitted", "select_input_backend"] diff --git a/je_auto_control/linux_wayland/_ydotool_cli.py b/je_auto_control/linux_wayland/_ydotool_cli.py new file mode 100644 index 00000000..1bdb1ab5 --- /dev/null +++ b/je_auto_control/linux_wayland/_ydotool_cli.py @@ -0,0 +1,134 @@ +"""Which ydotool command line is installed — and refusing the one that lies. + +ydotool 1.0 replaced its command line wholesale. Everything this backend +builds arrived in that release: ``mousemove --absolute``, ``mousemove +--wheel``, hex button bitmasks for ``click`` (so a press and a release can be +sent separately, which is what makes a drag possible), and ``key CODE:STATE`` +taking numeric evdev codes instead of key names. + +The 0.1.x series is still what ``apt install ydotool`` — the hint this +backend used to print — installs on Debian bookworm and on every current +Ubuntu. It does not merely reject the newer argv. Measured against a real +uinput device (``docker/ydotool_verify.py`` runs the same comparison in CI): + +=========================== ==== ======================================= +AutoControl's call rc what reached the kernel +=========================== ==== ======================================= +``click 0xc0`` (left) 0 BTN_LEFT down+up — right by coincidence +``click 0x40`` (press only) 0 *nothing* +``click 0xc1`` (right) 0 *nothing* +``mousemove --absolute ...`` 0 *nothing* — "unrecognised option" +``mousemove --wheel ...`` 0 *nothing* — "unrecognised option" +``key 30:1`` 0 *nothing* +=========================== ==== ======================================= + +Every one exits **0**, including the two that print ``unrecognised option``. +The backends run ydotool with ``check=True``, so a non-zero status is the +only thing that would raise — which means on those distributions a script +clicked nothing, typed nothing and moved nothing while every call reported +success. A silent no-op is the one failure mode a GUI automation library +cannot ship, so the legacy CLI is refused up front instead. + +Neither series implements ``--version``, and 1.x needs its daemon running +before it will answer ``--help``, so the probe uses the one thing both print +without a daemon and without side effects: the no-argument command list. +""" +from __future__ import annotations + +import subprocess # nosec B404 # reason: argv-list, path from shutil.which, no shell +import threading +from typing import Dict, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: ydotool 0.1.x listed a ``recorder`` command; 1.0 dropped it. +_LEGACY_COMMAND = "recorder" + +#: 1.x is daemon-only and advertises the socket override in its usage banner. +#: 0.1.x, which talks to ``/dev/uinput`` directly, has no such line. +_MODERN_MARKER = "ydotool_socket" + +MODERN = "modern" +LEGACY = "legacy" +UNKNOWN = "unknown" + +#: Probing costs a subprocess, and mouse / key dispatch must not pay that per +#: event, so the answer is cached per resolved binary path. +_cache: Dict[str, str] = {} +_cache_lock = threading.Lock() + +_PROBE_TIMEOUT = 5.0 + +_LEGACY_HINT = ( + "ydotool 0.1.x is installed at {path}, and its command line cannot " + "express what this backend needs: absolute cursor moves, separate " + "button press and release edges, wheel events, and numeric evdev key " + "codes all arrived in ydotool 1.0. Worse, 0.1.x exits 0 while emitting " + "nothing for those arguments, so every call would report success and do " + "nothing. Install ydotool 1.0 or newer (Debian: from unstable, which " + "packages 1.0.4; Arch: `pacman -S ydotool`; Fedora: `dnf install " + "ydotool`; or build from https://github.com/ReimuNotMoe/ydotool), or " + "set JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 to drive XWayland instead." +) + + +def _probe(path: str) -> str: + """Classify the installed ydotool by its no-argument usage banner.""" + try: + completed = subprocess.run( # nosec B603 # nosemgrep + [path], capture_output=True, timeout=_PROBE_TIMEOUT, check=False, + ) + except (OSError, subprocess.SubprocessError): + # An unreadable probe is not evidence of the broken version; let the + # real call fail with its own message rather than blaming the CLI. + return UNKNOWN + banner = (completed.stdout + completed.stderr).decode( + "utf-8", errors="replace").lower() + if _MODERN_MARKER in banner: + return MODERN + if _LEGACY_COMMAND in banner.split(): + return LEGACY + return UNKNOWN + + +def cli_generation(path: str) -> str: + """Return ``MODERN`` / ``LEGACY`` / ``UNKNOWN`` for the ydotool at ``path``. + + The result is cached per path for the life of the process. + """ + with _cache_lock: + cached = _cache.get(path) + if cached is not None: + return cached + generation = _probe(path) + with _cache_lock: + _cache[path] = generation + return generation + + +def reject_legacy_cli(path: str) -> str: + """Return ``path``, or raise when it is the 0.1.x CLI that silently no-ops. + + ``UNKNOWN`` is allowed through on purpose: only the version measured to + fail silently is refused, so a future release that drops the markers this + probe reads is not blocked by a stale detector. + """ + if cli_generation(path) == LEGACY: + raise AutoControlException(_LEGACY_HINT.format(path=path)) + return path + + +def reset_cache(path: Optional[str] = None) -> None: + """Forget probe results — for tests, and after installing a new ydotool.""" + with _cache_lock: + if path is None: + _cache.clear() + else: + _cache.pop(path, None) + + +__all__ = [ + "LEGACY", "MODERN", "UNKNOWN", "cli_generation", "reject_legacy_cli", + "reset_cache", +] diff --git a/je_auto_control/linux_wayland/keyboard.py b/je_auto_control/linux_wayland/keyboard.py index 2ce527cd..9766e876 100644 --- a/je_auto_control/linux_wayland/keyboard.py +++ b/je_auto_control/linux_wayland/keyboard.py @@ -20,6 +20,8 @@ from je_auto_control.linux_wayland._detect import ( WAYLAND_WTYPE, WAYLAND_YDOTOOL, binary_path, ) +from je_auto_control.linux_wayland._select_input import emitted +from je_auto_control.linux_wayland._ydotool_cli import reject_legacy_cli from je_auto_control.utils.exception.exceptions import AutoControlException @@ -28,9 +30,11 @@ "Install with your package manager (e.g. `apt install wtype`)." ) _INSTALL_HINT_YDOTOOL = ( - "ydotool is required for Wayland key events. " - "Install with your package manager (e.g. `apt install ydotool`) " - "and ensure ydotoold is running with /dev/uinput permission." + "ydotool 1.0 or newer is required for Wayland key events. " + "Install it with your package manager (Arch: `pacman -S ydotool`; " + "Fedora: `dnf install ydotool`; Debian: from unstable — trixie ships no " + "package and bookworm's 0.1.8 is too old) and ensure ydotoold is running " + "with /dev/uinput permission." ) @@ -41,6 +45,11 @@ def _require(name: str, hint: str) -> str: return path +def _require_ydotool() -> str: + """Resolve ydotool, refusing the 0.1.x CLI that fails silently.""" + return reject_legacy_cli(_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL)) + + def _run(argv: list, *, timeout: float = 5.0) -> None: # argv comes from a private allow-list (wtype / ydotool absolute # paths via shutil.which), never user input; no shell=True. @@ -64,38 +73,29 @@ def press_key(keycode: int) -> None: """Press one evdev key code. Uses libei when available, ydotool otherwise.""" _validate_keycode(keycode) libei = _try_libei() - if libei is not None: - libei.press_key(int(keycode)) + if libei is not None and emitted( + libei, lambda device: device.press_key(int(keycode))): return time.sleep(0.01) - _run([_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL), - "key", f"{int(keycode)}:1"]) + _run([_require_ydotool(), "key", f"{int(keycode)}:1"]) def release_key(keycode: int) -> None: """Release one evdev key code. Uses libei when available, ydotool otherwise.""" _validate_keycode(keycode) libei = _try_libei() - if libei is not None: - libei.release_key(int(keycode)) + if libei is not None and emitted( + libei, lambda device: device.release_key(int(keycode))): return time.sleep(0.01) - _run([_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL), - "key", f"{int(keycode)}:0"]) + _run([_require_ydotool(), "key", f"{int(keycode)}:0"]) def _try_libei(): """Return a connected :class:`LibeiBackend`, or None when CLI should win.""" try: - from je_auto_control.linux_wayland import select_input_backend - if select_input_backend() != "libei": - return None - from je_auto_control.linux_wayland.libei import get_default_backend - backend = get_default_backend() - if backend is None: - return None - backend.connect() - return backend + from je_auto_control.linux_wayland._select_input import active_backend + return active_backend() except (ImportError, RuntimeError, OSError): return None @@ -111,12 +111,37 @@ def hotkey(keycodes: Iterable[int]) -> None: codes = [int(code) for code in keycodes] if not codes: raise ValueError("hotkey requires at least one keycode") - args = [_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL), "key"] + libei = _try_libei() + if libei is not None and _libei_chord(libei, codes): + return + args = [_require_ydotool(), "key"] args.extend(f"{code}:1" for code in codes) args.extend(f"{code}:0" for code in reversed(codes)) _run(args) +def _libei_chord(libei, codes: list) -> bool: + """Send a whole chord through libei; False means the CLI has to redo it. + + Whatever went down is released in reverse before giving up, so a refusal + part-way through cannot leave a modifier held. The CLI then replays the + chord from a clean state — and if it was a *release* that was refused, + replaying is also what unsticks the key. + """ + pressed = [] + taken = True + for code in codes: + if not emitted(libei, lambda device, key=code: device.press_key(key)): + taken = False + break + pressed.append(code) + for code in reversed(pressed): + if not emitted(libei, + lambda device, key=code: device.release_key(key)): + taken = False + return taken + + def write(text: str) -> None: """Type a UTF-8 string via wtype (Unicode-aware, no key-code conversion).""" if not isinstance(text, str): diff --git a/je_auto_control/linux_wayland/libei.py b/je_auto_control/linux_wayland/libei.py index 3fd87223..d4a1b2cb 100644 --- a/je_auto_control/linux_wayland/libei.py +++ b/je_auto_control/linux_wayland/libei.py @@ -1,242 +1,610 @@ -"""ctypes binding for libei — Wayland's HID-layer input emulation library. - -libei runs at the kernel-input-event layer, which is *much* faster -than spawning ``wtype`` / ``ydotool`` per keystroke (a few µs vs. a -few ms). Compositors that expose the -``zwp_input_method_unstable_v2`` portal or the libei sender protocol -let an unprivileged client emit pointer / key events without the -sandbox veto. - -The binding is opt-in: callers go through :class:`LibeiBackend`, and -:func:`LibeiBackend.is_available` probes for ``libei.so.1`` on the -loader path. When the library is missing, callers should fall through -to the CLI shims in :mod:`keyboard` / :mod:`mouse` — that's what the -backend selector in :func:`select_input_backend` does. - -This module deliberately stays pure-Python so the test suite can run -on hosts without libei: the ctypes bindings are introduced lazily -through :class:`_LibeiSymbols`, and every external call is wrapped in -a method that's easy to mock. +"""ctypes binding for libei — Wayland's input-emulation protocol. + +libei is not a "call a function and a key is pressed" library. A sender has +to complete a handshake before it may emit anything: + +1. open a backend — an EIS **file descriptor** from the desktop portal + (:mod:`oeffis`), or a socket path where a compositor exposes one; +2. pump ``ei_dispatch`` / ``ei_get_event`` and answer what arrives; +3. on ``SEAT_ADDED``, bind the capabilities this sender wants; +4. on ``DEVICE_ADDED``, keep the device that carries each capability — + **devices only ever come from an event**, never from the context; +5. on ``DEVICE_RESUMED``, call ``ei_device_start_emulating``; +6. per emission, send the event *and* ``ei_device_frame``, or nothing is + delivered. + +The previous binding did none of that. It held only the ``struct ei *`` +context and passed it to entry points that take a ``struct ei_device *`` — +pointer type confusion in a C library — and without ``frame()`` no event +would have arrived even had the pointers been right. + +**Fail-closed by construction.** Every failure below raises +:class:`LibeiUnavailable`, which ``keyboard`` / ``mouse`` already treat as +"use the ydotool CLI". The handshake is bounded by a deadline and the result +is cached per process (:func:`connected_backend`), so a host where libei is +installed but unusable pays the probe once, not once per keystroke. + +The enum values are libei's, from ``libei.h``. They are the one part of this +module that a wrong guess would silently change — see ``Progress.md``. A +mismatch degrades safely rather than misfiring: capabilities that do not +match mean no device ever reports them, the handshake times out, and the CLI +takes over. """ from __future__ import annotations import ctypes -import ctypes.util +import os +import select import threading -from dataclasses import dataclass -from typing import Optional +import time +from typing import Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.linux_wayland import oeffis +from je_auto_control.linux_wayland._ctypes_bind import BoundSymbols, bind +from je_auto_control.linux_wayland._layout import layout_origin +from je_auto_control.utils.exception.exceptions import AutoControlException _LIBRARY_CANDIDATES = ("ei", "libei", "libei.so.1", "libei.so.0") +# enum ei_device_capability — a **bitmask**, not a sequence. Verified against +# libei.h (Debian 1.5.0-3 and upstream main, which agree on these six). +EI_DEVICE_CAP_POINTER = 1 << 0 +EI_DEVICE_CAP_POINTER_ABSOLUTE = 1 << 1 +EI_DEVICE_CAP_KEYBOARD = 1 << 2 +EI_DEVICE_CAP_TOUCH = 1 << 3 +EI_DEVICE_CAP_SCROLL = 1 << 4 +EI_DEVICE_CAP_BUTTON = 1 << 5 + +# enum ei_event_type — only the ones this sender has to answer. The block +# starts at 1 and increments implicitly through DEVICE_RESUMED before jumping +# to 90; the values below are that first run, verified against libei.h. +EI_EVENT_CONNECT = 1 +EI_EVENT_DISCONNECT = 2 +EI_EVENT_SEAT_ADDED = 3 +EI_EVENT_SEAT_REMOVED = 4 +EI_EVENT_DEVICE_ADDED = 5 +EI_EVENT_DEVICE_REMOVED = 6 +EI_EVENT_DEVICE_PAUSED = 7 +EI_EVENT_DEVICE_RESUMED = 8 + +#: Capabilities bound on every seat. Buttons and scroll ride along with the +#: pointer device wherever the compositor grants them. +_WANTED_CAPS = (EI_DEVICE_CAP_KEYBOARD, EI_DEVICE_CAP_POINTER_ABSOLUTE, + EI_DEVICE_CAP_BUTTON, EI_DEVICE_CAP_SCROLL) + +#: Without these two, half the public API would silently do nothing, so a +#: partial grant is treated as no grant at all. +_REQUIRED_CAPS = (EI_DEVICE_CAP_KEYBOARD, EI_DEVICE_CAP_POINTER_ABSOLUTE) + +#: Handshake budget once the EIS fd is in hand. The portal's own consent +#: wait already happened in :mod:`oeffis`; this is just protocol round trips. +HANDSHAKE_TIMEOUT = 3.0 + +#: One region of an absolute pointer's coordinate space: ``(x, y, w, h)``. +Region = Tuple[int, int, int, int] + +#: Ceiling on ``ei_device_get_region`` enumeration. libei ends the list with +#: NULL and a desktop has a handful of outputs; this only stops a library +#: whose contract differs from hanging the caller's mouse move forever. +_MAX_REGIONS = 64 + +#: One wheel click, in the units ``ei_device_scroll_discrete`` takes. From +#: libei.h: "A discrete scroll event is based logical scroll units (equivalent +#: to one mouse wheel click). The value for one scroll unit is 120." +SCROLL_UNIT = 120 + +_VOID = ctypes.c_void_p +_PROTOTYPES = ( + ("ei_new_sender", _VOID, (_VOID,)), + ("ei_unref", _VOID, (_VOID,)), + ("ei_setup_backend_fd", ctypes.c_int, (_VOID, ctypes.c_int)), + ("ei_setup_backend_socket", ctypes.c_int, (_VOID, ctypes.c_char_p)), + ("ei_get_fd", ctypes.c_int, (_VOID,)), + ("ei_dispatch", None, (_VOID,)), + ("ei_get_event", _VOID, (_VOID,)), + ("ei_event_unref", _VOID, (_VOID,)), + ("ei_event_get_type", ctypes.c_int, (_VOID,)), + ("ei_event_get_seat", _VOID, (_VOID,)), + ("ei_event_get_device", _VOID, (_VOID,)), + ("ei_device_ref", _VOID, (_VOID,)), + ("ei_device_unref", _VOID, (_VOID,)), + ("ei_device_has_capability", ctypes.c_bool, (_VOID, ctypes.c_int)), + ("ei_device_start_emulating", None, (_VOID, ctypes.c_uint32)), + ("ei_device_stop_emulating", None, (_VOID,)), + ("ei_device_frame", None, (_VOID, ctypes.c_uint64)), + ("ei_device_keyboard_key", None, (_VOID, ctypes.c_uint32, ctypes.c_bool)), + ("ei_device_pointer_motion_absolute", None, + (_VOID, ctypes.c_double, ctypes.c_double)), + ("ei_device_get_region", _VOID, (_VOID, ctypes.c_size_t)), + ("ei_region_get_x", ctypes.c_uint32, (_VOID,)), + ("ei_region_get_y", ctypes.c_uint32, (_VOID,)), + ("ei_region_get_width", ctypes.c_uint32, (_VOID,)), + ("ei_region_get_height", ctypes.c_uint32, (_VOID,)), + ("ei_device_button_button", None, (_VOID, ctypes.c_uint32, ctypes.c_bool)), + ("ei_device_scroll_discrete", None, + (_VOID, ctypes.c_int32, ctypes.c_int32)), + ("ei_now", ctypes.c_uint64, (_VOID,)), +) + + +class LibeiUnavailable(AutoControlException, RuntimeError): + """libei is missing, or a sender cannot be brought up on this session. + + Inherits both bases on purpose. ``AutoControlException`` is what every + containment boundary in the framework catches, and a sibling of it + escapes all of them — see that class's docstring. ``RuntimeError`` is + kept because the backend probes here and in ``_select_input`` were + written to catch it, and because "libei will not come up" is a runtime + environment fact rather than a caller mistake. + """ -class LibeiUnavailable(RuntimeError): - """Raised when libei isn't installed or the sender can't connect.""" +def _load_symbols() -> Optional[BoundSymbols]: + """Resolve every libei entry point, or None if one is missing. -@dataclass(frozen=True) -class _LibeiSymbols: - """Function pointers we resolve out of ``libei.so.*``.""" + ``ei_seat_bind_capabilities`` is bound unchecked because it is variadic: + ctypes has to infer each argument's type at the call site. + """ + return bind(_LIBRARY_CANDIDATES, _PROTOTYPES, + unchecked=("ei_seat_bind_capabilities",)) - lib: ctypes.CDLL - ei_new_sender: ctypes.CFUNCTYPE - ei_setup_backend_socket: ctypes.CFUNCTYPE - ei_device_keyboard_key: ctypes.CFUNCTYPE - ei_device_pointer_motion_absolute: ctypes.CFUNCTYPE - ei_device_button_button: ctypes.CFUNCTYPE - ei_device_scroll: ctypes.CFUNCTYPE - ei_unref: ctypes.CFUNCTYPE +def _default_socket_path() -> bytes: + """Where a compositor that exposes EIS on disk would put the socket.""" + runtime = os.environ.get("XDG_RUNTIME_DIR") + if not runtime: + return b"/run/user/1000/eis-0" + return f"{runtime}/eis-0".encode("utf-8") -def _try_load_library() -> Optional[ctypes.CDLL]: - """Probe ``libei.so.*`` on the loader path; return None if absent.""" - for name in _LIBRARY_CANDIDATES: - resolved = ctypes.util.find_library(name) - if resolved is None: - continue - try: - return ctypes.CDLL(resolved, use_errno=True) - except (OSError, RuntimeError): - continue - return None +def _in_any_region(regions: Sequence[Region], x: int, y: int) -> bool: + """Whether ``(x, y)`` falls inside one of ``regions``. -def _bind_symbols(lib: ctypes.CDLL) -> Optional[_LibeiSymbols]: - """Pull every libei function we use out of the shared object.""" - try: - new_sender = lib.ei_new_sender - setup_socket = lib.ei_setup_backend_socket - device_key = lib.ei_device_keyboard_key - device_motion = lib.ei_device_pointer_motion_absolute - device_button = lib.ei_device_button_button - device_scroll = lib.ei_device_scroll - unref = lib.ei_unref - except AttributeError: - return None - # Argument / return types match the upstream API. libei uses opaque - # handles + ints + uint32 booleans, so the binding stays compact. - new_sender.restype = ctypes.c_void_p - new_sender.argtypes = (ctypes.c_char_p,) - setup_socket.restype = ctypes.c_int - setup_socket.argtypes = (ctypes.c_void_p, ctypes.c_char_p) - device_key.restype = ctypes.c_int - device_key.argtypes = (ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32) - device_motion.restype = ctypes.c_int - device_motion.argtypes = (ctypes.c_void_p, ctypes.c_double, ctypes.c_double) - device_button.restype = ctypes.c_int - device_button.argtypes = (ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32) - device_scroll.restype = ctypes.c_int - device_scroll.argtypes = (ctypes.c_void_p, ctypes.c_double, ctypes.c_double) - unref.restype = None - unref.argtypes = (ctypes.c_void_p,) - return _LibeiSymbols( - lib=lib, ei_new_sender=new_sender, - ei_setup_backend_socket=setup_socket, - ei_device_keyboard_key=device_key, - ei_device_pointer_motion_absolute=device_motion, - ei_device_button_button=device_button, - ei_device_scroll=device_scroll, ei_unref=unref, - ) + The right and bottom edges are outside, which is measured rather than + assumed: against a real EIS peer offering one ``(0, 0, 1920, 1080)`` + region, ``(1919, 1079)`` arrives and ``(1920, 1080)`` does not. + """ + return any(left <= x < left + width and top <= y < top + height + for left, top, width, height in regions) class LibeiBackend: - """Native libei sender — thread-safe, lazy initialisation. + """A libei sender that has completed the handshake and can emit input. - ``LibeiBackend()`` cheaply probes for libei but does NOT connect; - call :meth:`connect` to open the socket. Tests inject a fake - :class:`_LibeiSymbols` via the ``symbols=`` keyword so the - behaviour can be exercised without ctypes. + ``LibeiBackend()`` only probes for the library; :meth:`connect` performs + the portal request and the protocol handshake. Tests inject a fake + ``symbols=`` object exposing the same entry-point names. """ - def __init__(self, *, sender_name: bytes = b"je_auto_control", - symbols: Optional[_LibeiSymbols] = None) -> None: - self._sender_name = sender_name + def __init__(self, *, symbols: Optional[BoundSymbols] = None, + portal_connect: Optional[Callable[..., tuple]] = None) -> None: self._symbols = symbols if symbols is not None else _load_symbols() - self._handle: Optional[int] = None - self._lock = threading.Lock() + self._portal_connect = portal_connect or oeffis.connect_eis_fd + self._ei: Optional[int] = None + self._backend_open = False + self._handshake_complete = False + self._session = None + self._devices: Dict[int, int] = {} + self._emulating: Dict[int, bool] = {} + self._sequence = 0 + self._lock = threading.RLock() @property def is_available(self) -> bool: + """Whether libei itself resolved. Says nothing about the session.""" return self._symbols is not None - def connect(self, *, socket_path: Optional[bytes] = None) -> None: - """Open the libei sender socket (default: $XDG_RUNTIME_DIR/eis-0).""" + @property + def is_connected(self) -> bool: + """Whether the handshake finished and a device is emulating.""" + return self._ei is not None and self._has_required_devices() + + def connect(self, *, timeout: float = HANDSHAKE_TIMEOUT, + socket_path: Optional[bytes] = None) -> None: + """Open a backend and run the handshake through to a live device. + + :param timeout: seconds for the protocol handshake, once connected. + :param socket_path: bypass the portal and use this EIS socket. + """ if not self.is_available: - raise LibeiUnavailable( - "libei.so.* not found on the loader path", - ) + raise LibeiUnavailable("libei.so.* not found on the loader path") with self._lock: - if self._handle is not None: + if self.is_connected: return - sender = self._symbols.ei_new_sender(self._sender_name) + sender = self._symbols.ei_new_sender(None) if not sender: raise LibeiUnavailable("ei_new_sender returned NULL") - chosen_socket = socket_path or _default_socket_path() - rc = self._symbols.ei_setup_backend_socket(sender, chosen_socket) - if rc != 0: - self._symbols.ei_unref(sender) - raise LibeiUnavailable( - f"ei_setup_backend_socket returned {rc}", - ) - self._handle = sender + self._ei = sender + try: + self._open_backend(socket_path) + self._handshake(time.monotonic() + max(0.0, timeout)) + # Reaching a live device is what makes ei_unref safe on this + # libei, so _teardown reads this rather than guessing. + self._handshake_complete = True + except BaseException: + self._teardown() + raise def disconnect(self) -> None: + """Release the devices, the sender, and the portal session.""" with self._lock: - if self._handle is None: - return - if self._symbols is not None: - self._symbols.ei_unref(self._handle) - self._handle = None + self._teardown() + + # --- emission --------------------------------------------------------- def press_key(self, keycode: int) -> None: - """Send a keydown for one evdev key code via libei.""" - self._require_connected() - self._symbols.ei_device_keyboard_key(self._handle, int(keycode), 1) + """Send a keydown for one evdev key code.""" + self._emit(EI_DEVICE_CAP_KEYBOARD, lambda device: + self._symbols.ei_device_keyboard_key( + device, int(keycode), True)) def release_key(self, keycode: int) -> None: - self._require_connected() - self._symbols.ei_device_keyboard_key(self._handle, int(keycode), 0) + """Send a keyup for one evdev key code.""" + self._emit(EI_DEVICE_CAP_KEYBOARD, lambda device: + self._symbols.ei_device_keyboard_key( + device, int(keycode), False)) def set_position(self, x: int, y: int) -> None: - self._require_connected() - self._symbols.ei_device_pointer_motion_absolute( - self._handle, float(x), float(y), - ) + """Move the pointer to an absolute screen position. - def click_button(self, button_code: int) -> None: - """Press + release one BTN_* code (e.g. 272 for left click).""" - self._require_connected() - self._symbols.ei_device_button_button(self._handle, int(button_code), 1) - self._symbols.ei_device_button_button(self._handle, int(button_code), 0) + The coordinate is mapped into the device's own space first — see + :meth:`_region_point`, which is also where a point no region covers + is turned into a refusal instead of a silent no-op. + """ + self._emit(EI_DEVICE_CAP_POINTER_ABSOLUTE, lambda device: + self._symbols.ei_device_pointer_motion_absolute( + device, *self._region_point(device, x, y))) def press_button(self, button_code: int) -> None: - self._require_connected() - self._symbols.ei_device_button_button(self._handle, int(button_code), 1) + """Press one BTN_* code (272 is BTN_LEFT).""" + self._emit(EI_DEVICE_CAP_BUTTON, lambda device: + self._symbols.ei_device_button_button( + device, int(button_code), True)) def release_button(self, button_code: int) -> None: - self._require_connected() - self._symbols.ei_device_button_button(self._handle, int(button_code), 0) - - def scroll(self, dx: int, dy: int) -> None: - self._require_connected() - self._symbols.ei_device_scroll(self._handle, float(dx), float(dy)) + """Release one BTN_* code.""" + self._emit(EI_DEVICE_CAP_BUTTON, lambda device: + self._symbols.ei_device_button_button( + device, int(button_code), False)) - def _require_connected(self) -> None: - if not self.is_available: - raise LibeiUnavailable("libei not loaded") - if self._handle is None: - raise LibeiUnavailable( - "libei sender not connected; call connect() first", - ) + def click_button(self, button_code: int) -> None: + """Press then release one BTN_* code.""" + self.press_button(button_code) + self.release_button(button_code) + def scroll(self, dx: int, dy: int) -> None: + """Scroll by whole wheel clicks on either axis. + + libei measures discrete scroll in 120ths of a click — the same + convention as Windows' ``WHEEL_DELTA`` — so a raw detent count is a + 120th of the scroll the caller asked for. Measured against a real EIS + peer, which is also where libei's own "suspicious discrete event value + 1, did you mean 120?" bug warning showed up. + """ + self._emit(EI_DEVICE_CAP_SCROLL, lambda device: + self._symbols.ei_device_scroll_discrete( + device, int(dx) * SCROLL_UNIT, int(dy) * SCROLL_UNIT)) + + # --- handshake -------------------------------------------------------- + + def _open_backend(self, socket_path: Optional[bytes]) -> None: + """Attach the sender to an EIS fd from the portal, or to a socket. + + The portal is the route that works on GNOME and KDE. Where + liboeffis is not installed there is still the well-known socket + some compositors publish, so that is tried rather than giving up. + """ + if socket_path is None and not oeffis.is_available(): + socket_path = _default_socket_path() + if socket_path is not None: + code = self._symbols.ei_setup_backend_socket(self._ei, socket_path) + if code != 0: + raise LibeiUnavailable( + f"ei_setup_backend_socket returned {code}", + ) + self._backend_open = True + return + try: + eis_fd, session = self._portal_connect() + except oeffis.OeffisUnavailable as error: + raise LibeiUnavailable(str(error)) from error + self._session = session + code = self._symbols.ei_setup_backend_fd(self._ei, int(eis_fd)) + if code != 0: + raise LibeiUnavailable(f"ei_setup_backend_fd returned {code}") + self._backend_open = True + + def _handshake(self, deadline: float) -> None: + """Pump events until the devices we need are emulating.""" + while not self._has_required_devices(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise LibeiUnavailable( + "libei connected but no seat offered both a keyboard and " + "an absolute pointer within the handshake timeout", + ) + self._pump(min(remaining, 0.25)) + + def _pump(self, timeout: float) -> None: + """Dispatch libei and answer every event that is waiting.""" + poll_fd = int(self._symbols.ei_get_fd(self._ei)) + if poll_fd < 0: + raise LibeiUnavailable("ei_get_fd returned no pollable fd") + if timeout > 0: + ready, _, _ = select.select([poll_fd], [], [], timeout) + if not ready: + return + self._symbols.ei_dispatch(self._ei) + while True: + event = self._symbols.ei_get_event(self._ei) + if not event: + return + try: + self._on_event(event) + finally: + self._symbols.ei_event_unref(event) + + def _on_event(self, event: int) -> None: + """Answer one libei event.""" + event_type = int(self._symbols.ei_event_get_type(event)) + if event_type == EI_EVENT_SEAT_ADDED: + self._bind_seat(self._symbols.ei_event_get_seat(event)) + elif event_type == EI_EVENT_DEVICE_ADDED: + self._remember_device(self._symbols.ei_event_get_device(event)) + elif event_type == EI_EVENT_DEVICE_RESUMED: + self._start_emulating(self._symbols.ei_event_get_device(event)) + elif event_type in (EI_EVENT_DEVICE_PAUSED, EI_EVENT_DEVICE_REMOVED): + self._forget_device(self._symbols.ei_event_get_device(event)) + elif event_type == EI_EVENT_DISCONNECT: + raise LibeiUnavailable("the compositor disconnected the sender") + + def _bind_seat(self, seat: int) -> None: + """Ask a seat for the capabilities this sender emits. + + ``ei_seat_bind_capabilities`` is variadic and NULL-terminated, so the + arguments are passed as explicit ctypes values — an ``argtypes`` + tuple cannot describe a variadic call. + """ + if not seat: + return + args = [ctypes.c_void_p(seat)] + args.extend(ctypes.c_int(cap) for cap in _WANTED_CAPS) + args.append(ctypes.c_void_p(None)) + self._symbols.ei_seat_bind_capabilities(*args) + + def _remember_device(self, device: int) -> None: + """Keep a reference to a device for each capability it carries.""" + if not device: + return + kept = False + for cap in _WANTED_CAPS: + if not self._symbols.ei_device_has_capability(device, cap): + continue + if not kept: + self._symbols.ei_device_ref(device) + kept = True + self._devices[cap] = device + + def _start_emulating(self, device: int) -> None: + """Mark a resumed device ready and open its emulation sequence.""" + if not device or device not in self._devices.values(): + return + self._sequence += 1 + self._symbols.ei_device_start_emulating(device, self._sequence) + self._emulating[device] = True + + def _forget_device(self, device: int) -> None: + """Drop a paused or removed device; emissions then fail closed.""" + if not device: + return + self._emulating.pop(device, None) + for cap, known in list(self._devices.items()): + if known == device: + del self._devices[cap] + self._symbols.ei_device_unref(device) + + def _has_required_devices(self) -> bool: + return all(self._emulating.get(self._devices.get(cap, 0), False) + for cap in _REQUIRED_CAPS) + + # --- coordinate space ------------------------------------------------- + + def _device_regions(self, device: int) -> List[Region]: + """The regions this device accepts absolute motion in. + + libei reports them in its own space, offsets included: a device whose + region sits at ``x=1280`` takes ``1380`` for a point 100 pixels into + it, not ``100``. An empty list means the device declared none, and a + device with no regions accepts any coordinate — both measured against + a real EIS peer in ``docker/eis_verify.py``. + """ + regions: List[Region] = [] + for index in range(_MAX_REGIONS): + region = self._symbols.ei_device_get_region(device, index) + if not region: + break + regions.append(( + int(self._symbols.ei_region_get_x(region)), + int(self._symbols.ei_region_get_y(region)), + int(self._symbols.ei_region_get_width(region)), + int(self._symbols.ei_region_get_height(region)), + )) + return regions + + def _region_point(self, device: int, x: int, y: int) -> Tuple[float, float]: + """Map a layout coordinate into the space ``device`` accepts. + + **libei discards an absolute motion that lands in no region, and says + nothing about it** — no return code, no event, no log the caller can + see. ``set_position`` would return as though the pointer had moved. + That is the failure this method exists to prevent, and it is measured, + not inferred: against a real EIS peer, a point outside every region + produced no server-side event at all. + + The two spaces can genuinely differ. Region offsets are ``uint32``, so + no compositor *can* advertise a region left of or above the origin, + while this project's layout space starts at + :func:`je_auto_control.linux_wayland._layout.layout_origin` and + goes negative the moment a monitor sits left of the primary one — the + exact layout ``docker/wayland_verify.py`` exercises on the capture + side. Where the raw point misses, the origin-normalised one is tried, + which is the translation that keeps input and capture addressing the + same pixel. If that misses too, the caller gets a refusal and + ``_select_input.emitted`` hands the move to the ydotool path. + """ + regions = self._device_regions(device) + if not regions or _in_any_region(regions, x, y): + return (float(x), float(y)) + origin_x, origin_y = layout_origin() + moved_x, moved_y = x - origin_x, y - origin_y + if (origin_x or origin_y) and _in_any_region(regions, moved_x, moved_y): + return (float(moved_x), float(moved_y)) + raise LibeiUnavailable( + f"({x}, {y}) lies outside every region this pointer accepts " + f"{regions}; libei drops such a motion without reporting it, so " + "the move is refused here for the CLI path to take", + ) -def _load_symbols() -> Optional[_LibeiSymbols]: - lib = _try_load_library() - if lib is None: - return None - return _bind_symbols(lib) + # --- plumbing --------------------------------------------------------- + def _emit(self, capability: int, send: Callable[[int], None]) -> None: + """Send one event on the device carrying ``capability``, then frame. -def _default_socket_path() -> bytes: - import os - runtime = os.environ.get("XDG_RUNTIME_DIR") - if not runtime: - return b"/run/user/1000/eis-0" - return f"{runtime}/eis-0".encode("utf-8") + Without the trailing ``ei_device_frame`` libei buffers the event and + the compositor never sees it, so the two belong in one place. + """ + with self._lock: + if self._ei is None: + raise LibeiUnavailable("libei sender is not connected") + # Pause / resume arrive asynchronously; read them before deciding + # the device is still usable. + self._pump(0.0) + device = self._devices.get(capability) + if not device or not self._emulating.get(device, False): + raise LibeiUnavailable( + f"no libei device is emulating capability {capability}", + ) + send(device) + self._symbols.ei_device_frame(device, self._symbols.ei_now(self._ei)) + + def _teardown(self) -> None: + """Release what is safe to release; abandon what is not. + + ``ei_unref`` **segfaults** on libei 1.3.901 on a context whose backend + was opened but whose handshake never progressed. Measured, not + inferred — ``docker/libei_verify.py`` walks the states one call at a + time against the real library, and ``docker/eis_verify.py`` adds the + live one by running a real EIS peer: + + ========================================= ========== + state ``ei_unref`` + ========================================= ========== + context created, no backend set up safe + ``ei_setup_backend_socket`` failed (-2) safe + backend open, handshake never progressed **SIGSEGV** + handshake completed, devices emulating safe + ========================================= ========== + + ``ei_disconnect`` crashes in the same state, so it is not a + refcounting mistake on our side — it is tearing down a connection + whose EI handshake never progressed. The header documents ``unref`` + as the correct release for every outcome, so this reads as an + upstream bug rather than misuse. + + So the release is chosen by state. A session that reached a live + device is unreffed normally, which is the common path and no longer + leaks. Only the crashing state is abandoned — its context and fd are + dropped without unref, a few hundred bytes and one descriptor per + process, since :func:`connected_backend` probes only once. A segfault + in a library that drives someone's desktop is far worse than that. + """ + if self._ei is not None and self._safe_to_unref(): + for device in set(self._devices.values()): + _quietly(lambda handle=device: + self._symbols.ei_device_unref(handle)) + _quietly(lambda: self._symbols.ei_unref(self._ei)) + self._devices.clear() + self._emulating.clear() + self._ei = None + self._backend_open = False + self._handshake_complete = False + if self._session is not None: + # liboeffis is a different library and closing the portal session + # is what actually revokes the grant, so this always runs. + _quietly(self._session.close) + self._session = None + + def _safe_to_unref(self) -> bool: + """Whether this context is in a state libei survives being unreffed in. + + The two safe states are "no backend was ever opened" and "the + handshake completed". Everything in between is the measured crash. + """ + return not self._backend_open or self._handshake_complete + + +def _quietly(action: Callable[[], object]) -> None: + """Run a cleanup step; a failing one must not mask the real error.""" + try: + action() + except (AttributeError, OSError, ValueError, RuntimeError): + pass -# Cached default sender — created on first use, kept for the process -# lifetime so input latency isn't dominated by sender setup. _DEFAULT_BACKEND: Optional[LibeiBackend] = None +_PROBE_FAILED = False _DEFAULT_LOCK = threading.Lock() -def get_default_backend() -> Optional[LibeiBackend]: - """Return the cached :class:`LibeiBackend`, or None when libei is absent.""" - global _DEFAULT_BACKEND +def connected_backend() -> Optional[LibeiBackend]: + """Return a connected backend, or None — probing at most once. + + The probe involves a portal round trip and a consent dialog, so a host + where libei cannot be used must pay for that discovery once rather than + on every keystroke. Callers treat None as "use the ydotool CLI". + """ + global _DEFAULT_BACKEND, _PROBE_FAILED with _DEFAULT_LOCK: if _DEFAULT_BACKEND is not None: return _DEFAULT_BACKEND + if _PROBE_FAILED: + return None backend = LibeiBackend() if not backend.is_available: + _PROBE_FAILED = True + return None + try: + backend.connect() + except (LibeiUnavailable, OSError, ValueError, AttributeError): + _PROBE_FAILED = True return None _DEFAULT_BACKEND = backend - return _DEFAULT_BACKEND + return _DEFAULT_BACKEND + + +def get_default_backend() -> Optional[LibeiBackend]: + """Return the cached backend if libei resolved, without connecting.""" + with _DEFAULT_LOCK: + if _DEFAULT_BACKEND is not None: + return _DEFAULT_BACKEND + backend = LibeiBackend() + return backend if backend.is_available else None def reset_default_backend() -> None: - """Test hook — drop the cached default so probe runs fresh.""" - global _DEFAULT_BACKEND + """Test hook — drop the cached backend so the probe runs fresh.""" + global _DEFAULT_BACKEND, _PROBE_FAILED with _DEFAULT_LOCK: if _DEFAULT_BACKEND is not None: - try: - _DEFAULT_BACKEND.disconnect() - except LibeiUnavailable: - pass + _quietly(_DEFAULT_BACKEND.disconnect) _DEFAULT_BACKEND = None + _PROBE_FAILED = False __all__ = [ - "LibeiBackend", "LibeiUnavailable", - "get_default_backend", "reset_default_backend", + "EI_DEVICE_CAP_BUTTON", "EI_DEVICE_CAP_KEYBOARD", + "EI_DEVICE_CAP_POINTER_ABSOLUTE", "EI_DEVICE_CAP_SCROLL", + "HANDSHAKE_TIMEOUT", "LibeiBackend", "LibeiUnavailable", + "connected_backend", "get_default_backend", "reset_default_backend", ] diff --git a/je_auto_control/linux_wayland/mouse.py b/je_auto_control/linux_wayland/mouse.py index 0d94e617..82308146 100644 --- a/je_auto_control/linux_wayland/mouse.py +++ b/je_auto_control/linux_wayland/mouse.py @@ -8,12 +8,18 @@ """ from __future__ import annotations +import os import subprocess # nosec B404 # reason: argv-list, no shell interpolation import time -from typing import Optional, Tuple +from functools import lru_cache +from typing import Mapping, Optional, Tuple from je_auto_control.linux_wayland._detect import WAYLAND_YDOTOOL, binary_path +from je_auto_control.linux_wayland._layout import layout_origin +from je_auto_control.linux_wayland._select_input import emitted +from je_auto_control.linux_wayland._ydotool_cli import reject_legacy_cli from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger # ydotool ``click`` accepts hex bitmasks: the low nibble selects the @@ -33,9 +39,11 @@ wayland_scroll_direction_right = 2 _INSTALL_HINT = ( - "ydotool is required for Wayland mouse input. " - "Install with your package manager (e.g. `apt install ydotool`) " - "and ensure ydotoold runs with /dev/uinput permission." + "ydotool 1.0 or newer is required for Wayland mouse input. " + "Install it with your package manager (Arch: `pacman -S ydotool`; " + "Fedora: `dnf install ydotool`; Debian: from unstable — trixie ships no " + "package and bookworm's 0.1.8 is too old) and ensure ydotoold runs with " + "/dev/uinput permission." ) @@ -43,7 +51,7 @@ def _require_ydotool() -> str: path = binary_path(WAYLAND_YDOTOOL) if path is None: raise AutoControlException(_INSTALL_HINT) - return path + return reject_legacy_cli(path) def _run(argv: list, *, timeout: float = 5.0) -> None: @@ -73,32 +81,173 @@ def position() -> Tuple[int, int]: ) +POINTER_ACCEL_ENV = "JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL" +POINTER_ACCEL_MODES = ("warn", "flat", "strict") + +_ACCELERATION_ADVICE = ( + "Disable pointer acceleration for the ydotoold device (sway: `input " + "type:pointer accel_profile flat` plus `pointer_accel 0`) or install " + "liboeffis so the libei path can be used." +) + +_ACCELERATION_WARNING = ( + "Wayland absolute move fell back to ydotool. `mousemove --absolute` is " + "relative motion under the hood, so the compositor's pointer " + "acceleration scales it: measured against a real wlroots session, " + "libinput's default adaptive profile lands the cursor twice as far from " + f"the corner as asked. {_ACCELERATION_ADVICE} With acceleration off, set " + f"{POINTER_ACCEL_ENV}=flat to silence this; set it to strict to have the " + "move refused instead of landing somewhere else." +) + +_ACCELERATION_REFUSAL = ( + f"{POINTER_ACCEL_ENV}=strict, and this absolute move fell back to " + "ydotool, whose `mousemove --absolute` is relative motion the compositor " + f"scales — so the cursor would not land where asked. {_ACCELERATION_ADVICE} " + f"Then set {POINTER_ACCEL_ENV}=flat, or unset it to warn and move anyway." +) + + +def pointer_accel_mode(environ: Optional[Mapping[str, str]] = None) -> str: + """Return the operator's declared pointer-acceleration policy. + + The compositor's acceleration factor cannot be read back by the client, + so this backend cannot compensate for it — only the operator knows + whether it is switched off. This is how they say so: + + * ``warn`` — the default, and what an unset or unrecognised value means: + log the caveat once per process and send the move regardless. + * ``flat`` — acceleration is off for the ydotoold device, the move is + pixel-accurate, and nothing needs saying. + * ``strict`` — refuse to send an absolute move the compositor would + scale, rather than let a click land silently in the wrong place. + """ + env = environ if environ is not None else os.environ + declared = (env.get(POINTER_ACCEL_ENV) or "").strip().lower() + if not declared: + return "warn" + if declared in POINTER_ACCEL_MODES: + return declared + _warn_once( + f"{POINTER_ACCEL_ENV}={declared} is not one of " + f"{', '.join(POINTER_ACCEL_MODES)}; treating it as warn.", + ) + return "warn" + + +def _apply_accel_policy() -> None: + """Gate a ydotool absolute move on :func:`pointer_accel_mode`.""" + mode = pointer_accel_mode() + if mode == "flat": + return + if mode == "strict": + raise AutoControlException(_ACCELERATION_REFUSAL) + _warn_about_acceleration() + + +def _ydotool_point(x: int, y: int) -> Tuple[int, int]: + """Translate a layout coordinate into ydotool's absolute space. + + ``mousemove --absolute`` emits no absolute event. It sends ``INT32_MIN`` + on both axes to drive the cursor into the corner the compositor clamps + to, then sends the target as a relative displacement — so its origin is + the top-left of the *output layout*, not layout ``(0, 0)``. The two are + the same point only while every output sits at a non-negative position; + put a monitor left of the primary one and they differ by + :func:`~je_auto_control.linux_wayland._layout.layout_origin`, which is + exactly the correction the capture path already applies. + + Measured rather than reasoned: on a real wlroots session whose left-hand + output sits at ``x=-1280`` (``docker/Dockerfile.seat``), with the + pointer's acceleration disabled, ``--absolute -x 10 -y 10`` puts the + cursor at layout ``(-1270, 10)``. Without this subtraction a caller + asking for layout ``(-1270, 10)`` would land 1,280 pixels away, on the + other monitor. + """ + origin_x, origin_y = layout_origin() + return x - origin_x, y - origin_y + + def set_position(x: int, y: int) -> None: - """Move the cursor to absolute (x, y). Uses libei when available.""" + """Move the cursor to absolute (x, y). Uses libei when available. + + The libei path is absolute at the protocol level and lands exactly. The + ydotool fallback cannot: see :func:`_ydotool_point` for the origin it + counts from, and note that the compositor's pointer acceleration is + applied to the relative motion it really sends, so the move is only + pixel-accurate where that acceleration is switched off. ydotool's own + ``--help`` says as much; by default this backend logs it once per + process rather than letting a click land silently in the wrong place. + :func:`pointer_accel_mode` documents the environment variable an + operator sets to silence that (acceleration is off) or to have the move + refused outright. + + :raises AutoControlException: on the ydotool path when the operator has + set ``JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=strict``. + """ libei = _try_libei() - if libei is not None: - libei.set_position(int(x), int(y)) + if libei is not None and emitted( + libei, lambda device: device.set_position(int(x), int(y))): return + _apply_accel_policy() time.sleep(0.01) + target_x, target_y = _ydotool_point(int(x), int(y)) _run([_require_ydotool(), "mousemove", "--absolute", - "-x", str(int(x)), "-y", str(int(y))]) + "-x", str(target_x), "-y", str(target_y)]) + + +@lru_cache(maxsize=32) +def _warn_once(message: str) -> None: + """Log ``message`` at warning level, once per process per distinct text. + + Cached rather than latched on a module global: these caveats are worth + saying and not worth repeating on every move of a script that makes + thousands. ``_warn_once.cache_clear()`` re-arms them. The bound is there + because the key includes an operator-supplied env value; evicting only + costs a repeated line. + """ + autocontrol_logger.warning(message) + + +def _warn_about_acceleration() -> None: + """Log the ydotool absolute-move caveat, once per process.""" + _warn_once(_ACCELERATION_WARNING) def _try_libei(): + """Return a connected :class:`LibeiBackend`, or None when CLI should win.""" try: - from je_auto_control.linux_wayland import select_input_backend - if select_input_backend() != "libei": - return None - from je_auto_control.linux_wayland.libei import get_default_backend - backend = get_default_backend() - if backend is None: - return None - backend.connect() - return backend + from je_auto_control.linux_wayland._select_input import active_backend + return active_backend() except (ImportError, RuntimeError, OSError): return None +# ydotool's ``click`` verb and libei speak different button numbers: the +# constants above are ydotool's bitmask nibbles, libei takes raw evdev +# BTN_* codes. Scroll needs a conversion of its own — see +# ``_LIBEI_VERTICAL_SIGN``. +_BTN_LEFT = 272 +_BTN_RIGHT = 273 +_BTN_MIDDLE = 274 + +_EVDEV_BUTTONS = { + wayland_mouse_left: _BTN_LEFT, + wayland_mouse_right: _BTN_RIGHT, + wayland_mouse_middle: _BTN_MIDDLE, +} + + +def _evdev_button(mouse_keycode: int) -> Optional[int]: + """Map this module's public button code onto an evdev BTN_* code. + + A press- or release-only code has one edge bit cleared; restoring both + gives back the canonical constant the table is keyed on. + """ + canonical = int(mouse_keycode) | _YDOTOOL_DOWN_BIT | _YDOTOOL_UP_BIT + return _EVDEV_BUTTONS.get(canonical) + + def _press_code(mouse_keycode: int) -> int: """Button-down only: set the down-bit, clear the up-bit.""" return (int(mouse_keycode) | _YDOTOOL_DOWN_BIT) & ~_YDOTOOL_UP_BIT @@ -111,25 +260,78 @@ def _release_code(mouse_keycode: int) -> int: def press_mouse(mouse_keycode: int) -> None: """Press a mouse button and hold it (down edge only).""" + if _emit_button(mouse_keycode, press=True): + return time.sleep(0.01) _run([_require_ydotool(), "click", f"{_press_code(mouse_keycode):#x}"]) def release_mouse(mouse_keycode: int) -> None: """Release a held mouse button (up edge only).""" + if _emit_button(mouse_keycode, press=False): + return time.sleep(0.01) _run([_require_ydotool(), "click", f"{_release_code(mouse_keycode):#x}"]) +def _emit_button(mouse_keycode: int, *, press: bool) -> bool: + """Send one button edge through libei; False means fall back to the CLI.""" + button = _evdev_button(mouse_keycode) + if button is None: + return False + libei = _try_libei() + if libei is None: + return False + if press: + return emitted(libei, lambda device: device.press_button(button)) + return emitted(libei, lambda device: device.release_button(button)) + + def click_mouse(mouse_keycode: int, x: Optional[int] = None, y: Optional[int] = None) -> None: """Press + release a mouse button, optionally moving first.""" if x is not None and y is not None: set_position(int(x), int(y)) + if _emit_button(mouse_keycode, press=True): + # Not _emit_button: a refused release has to reach the CLI, or the + # button stays down for the rest of the session. + release_mouse(mouse_keycode) + return time.sleep(0.01) _run([_require_ydotool(), "click", f"{int(mouse_keycode):#x}"]) +#: Sign applied to the vertical axis on the way to libei. +#: +#: The two sides count wheels in opposite directions. This module's +#: ``wayland_scroll_direction_*`` constants are in the kernel's ``REL_WHEEL`` +#: frame, because that is what ydotool writes into ``/dev/uinput``: positive +#: is up. libei is in the ``wl_pointer`` / libinput frame, where positive is +#: down — libinput's own evdev reader negates ``REL_WHEEL`` to get there, and +#: the other libei sender that documents its sign (enigo) passes a +#: "positive scrolls down" value straight through to ``scroll_discrete``. +#: +#: Horizontal needs no flip: ``REL_HWHEEL`` and libinput both count right as +#: positive, and libinput passes that axis through unnegated. +#: +#: What ``docker/eis_verify.py`` measured is the half either frame agrees on: +#: ``scroll(0, 1)`` reaches a real EIS server as ``(0, 120)`` — one whole +#: click, on the y axis, sign preserved end to end, no axis swap. The frame +#: the compositor reads that in is the half above. +_LIBEI_VERTICAL_SIGN = -1 + + +def _wheel_deltas(scroll_value: int, scroll_direction: int) -> Tuple[int, int]: + """Split ``(value, direction)`` into the ``(x, y)`` detents to send. + + ``scroll_direction`` carries axis and sign, per this module's + ``wayland_scroll_direction_*`` constants: +-1 vertical, +-2 horizontal. + """ + direction = int(scroll_direction) + amount = abs(int(scroll_value)) * (1 if direction > 0 else -1) + return (0, amount) if abs(direction) == 1 else (amount, 0) + + def scroll(scroll_value: int, scroll_direction: int = wayland_scroll_direction_down) -> None: """Scroll ``scroll_value`` notches in ``scroll_direction``. @@ -140,13 +342,22 @@ def scroll(scroll_value: int, previous ``(direction, x, y)`` shape silently bound the wrapper's direction to ``x`` and then dropped it, so every scroll went the same way. - ``scroll_direction`` carries axis and sign, per this module's - ``wayland_scroll_direction_*`` constants: +-1 vertical, +-2 horizontal. + Prefers libei, which delivers whole wheel clicks without a uinput daemon; + the vertical axis is negated on the way out (``_LIBEI_VERTICAL_SIGN``). + + On the ydotool fallback both axes are always passed, as ydotool's own + documented example does (``ydotool mousemove -w -x 0 -y -1``) — leaving + one out would rely on it defaulting to zero, which the tool does not + promise. Positive is up / right there, matching the kernel's + ``REL_WHEEL`` convention that ydotool writes. """ - direction = int(scroll_direction) - axis = "-y" if abs(direction) == 1 else "-x" - amount = abs(int(scroll_value)) * (1 if direction > 0 else -1) - _run([_require_ydotool(), "mousemove", "--wheel", axis, str(amount)]) + delta_x, delta_y = _wheel_deltas(scroll_value, scroll_direction) + libei = _try_libei() + if libei is not None and emitted(libei, lambda device: device.scroll( + delta_x, _LIBEI_VERTICAL_SIGN * delta_y)): + return + _run([_require_ydotool(), "mousemove", "--wheel", + "-x", str(delta_x), "-y", str(delta_y)]) def send_mouse_event_to_window(*_args, **_kwargs) -> None: @@ -159,7 +370,8 @@ def send_mouse_event_to_window(*_args, **_kwargs) -> None: __all__ = [ - "click_mouse", "position", "press_mouse", "release_mouse", + "POINTER_ACCEL_ENV", "POINTER_ACCEL_MODES", "click_mouse", + "pointer_accel_mode", "position", "press_mouse", "release_mouse", "scroll", "send_mouse_event_to_window", "set_position", "wayland_mouse_left", "wayland_mouse_middle", "wayland_mouse_right", "wayland_scroll_direction_down", "wayland_scroll_direction_left", diff --git a/je_auto_control/linux_wayland/oeffis.py b/je_auto_control/linux_wayland/oeffis.py new file mode 100644 index 00000000..174dfa0a --- /dev/null +++ b/je_auto_control/linux_wayland/oeffis.py @@ -0,0 +1,196 @@ +"""ctypes binding for liboeffis — the portal half of the libei handshake. + +libei needs an EIS socket, and on GNOME / KDE that socket is not a path on +disk: it is a file descriptor handed over D-Bus by +``org.freedesktop.portal.RemoteDesktop.ConnectToEIS``, at the end of a +three-call asynchronous session dance (``CreateSession`` → ``SelectDevices`` +→ ``Start``). A file descriptor cannot be received through the ``gdbus`` +command line, so the CLI trick that works for the Screenshot portal (see +:mod:`portal`) cannot work here — it would need a D-Bus client that speaks +SCM_RIGHTS. + +``liboeffis`` ships with libei precisely so clients do not have to write +that. It performs the whole portal dance and hands back the EIS fd, which is +what :mod:`libei` then passes to ``ei_setup_backend_fd``. + +Everything here is fail-closed: if the library is missing, the portal denies +the request, or the user dismisses the consent dialog, the caller gets None +or an exception and falls back to the ydotool CLI. +""" +from __future__ import annotations + +import ctypes +import select +import time +from typing import Optional, Tuple + +from je_auto_control.linux_wayland._ctypes_bind import BoundSymbols, bind + + +_LIBRARY_CANDIDATES = ("oeffis", "liboeffis", "liboeffis.so.1", "liboeffis.so.0") + +# enum oeffis_device — a bitmask of what the session may do. Verified against +# liboeffis.h (Debian 1.5.0-3 and upstream main agree). +OEFFIS_DEVICE_ALL_DEVICES = 0 # the header's "everything" sentinel +OEFFIS_DEVICE_KEYBOARD = 1 << 0 +OEFFIS_DEVICE_POINTER = 1 << 1 +OEFFIS_DEVICE_TOUCHSCREEN = 1 << 2 + +#: What this backend actually emits. Asking for the touchscreen we never use +#: would widen the grant the user is consenting to for no benefit, and an +#: explicit mask cannot be misread as the ``= 0`` sentinel either way. +OEFFIS_DEVICE_DEFAULT = OEFFIS_DEVICE_KEYBOARD | OEFFIS_DEVICE_POINTER + +# enum oeffis_event_type — note CLOSED precedes DISCONNECTED, which is the +# opposite of the order the names suggest. +OEFFIS_EVENT_NONE = 0 +OEFFIS_EVENT_CONNECTED_TO_EIS = 1 +OEFFIS_EVENT_CLOSED = 2 +OEFFIS_EVENT_DISCONNECTED = 3 + +#: The consent dialog is a human in the loop, so this is a human-scale wait. +DEFAULT_TIMEOUT = 30.0 + +_PROTOTYPES = ( + ("oeffis_new", ctypes.c_void_p, (ctypes.c_void_p,)), + ("oeffis_unref", ctypes.c_void_p, (ctypes.c_void_p,)), + ("oeffis_create_session", None, (ctypes.c_void_p, ctypes.c_uint32)), + ("oeffis_get_fd", ctypes.c_int, (ctypes.c_void_p,)), + ("oeffis_dispatch", None, (ctypes.c_void_p,)), + ("oeffis_get_event", ctypes.c_int, (ctypes.c_void_p,)), + ("oeffis_get_eis_fd", ctypes.c_int, (ctypes.c_void_p,)), + ("oeffis_get_error_message", ctypes.c_char_p, (ctypes.c_void_p,)), +) + + +class OeffisUnavailable(RuntimeError): + """liboeffis is missing, or the portal refused to hand over an EIS fd.""" + + +def load_symbols() -> Optional[BoundSymbols]: + """Resolve liboeffis, or None when it is not installed.""" + return bind(_LIBRARY_CANDIDATES, _PROTOTYPES) + + +def is_available() -> bool: + """Whether the portal route to an EIS fd can be attempted at all.""" + return load_symbols() is not None + + +def _describe(symbols: BoundSymbols, handle: int) -> str: + """The library's own error text, when it has one.""" + try: + message = symbols.oeffis_get_error_message(handle) + except (AttributeError, OSError, ValueError): + return "" + if not message: + return "" + if isinstance(message, bytes): + return message.decode("utf-8", errors="replace") + return str(message) + + +def _pump(symbols: BoundSymbols, handle: int, deadline: float) -> int: + """Wait for one oeffis event, or 0 when the deadline passes.""" + poll_fd = symbols.oeffis_get_fd(handle) + if poll_fd < 0: + raise OeffisUnavailable("oeffis_get_fd returned no pollable fd") + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return OEFFIS_EVENT_NONE + ready, _, _ = select.select([poll_fd], [], [], remaining) + if not ready: + return OEFFIS_EVENT_NONE + symbols.oeffis_dispatch(handle) + event = int(symbols.oeffis_get_event(handle)) + if event != OEFFIS_EVENT_NONE: + return event + + +def connect_eis_fd(devices: int = OEFFIS_DEVICE_DEFAULT, + timeout: float = DEFAULT_TIMEOUT, + symbols: Optional[BoundSymbols] = None) -> Tuple[int, object]: + """Run the portal session and return ``(eis_fd, session_handle)``. + + The session handle must be kept alive for as long as the EIS fd is used: + dropping it tears the portal session down and the compositor stops + accepting input from it. + + :param devices: bitmask of ``OEFFIS_DEVICE_*`` to request. + :param timeout: seconds to wait, including any consent dialog. + :param symbols: injected entry points, for tests. + :return: the EIS file descriptor and the handle owning the session. + """ + resolved = symbols if symbols is not None else load_symbols() + if resolved is None: + raise OeffisUnavailable("liboeffis.so.* not found on the loader path") + handle = resolved.oeffis_new(None) + if not handle: + raise OeffisUnavailable("oeffis_new returned NULL") + try: + resolved.oeffis_create_session(handle, int(devices)) + event = _pump(resolved, handle, time.monotonic() + max(0.0, timeout)) + _require_connected(resolved, handle, event, timeout) + eis_fd = int(resolved.oeffis_get_eis_fd(handle)) + if eis_fd < 0: + raise OeffisUnavailable( + "the portal reported success but handed over no EIS fd", + ) + except BaseException: + _release(resolved, handle) + raise + return eis_fd, _Session(resolved, handle) + + +def _require_connected(symbols: BoundSymbols, handle: int, event: int, + timeout: float) -> None: + """Turn anything but a successful connection into a clear failure.""" + if event == OEFFIS_EVENT_CONNECTED_TO_EIS: + return + if event == OEFFIS_EVENT_NONE: + raise OeffisUnavailable( + f"the desktop portal did not answer within {timeout:g}s " + f"(a consent dialog may be waiting)", + ) + detail = _describe(symbols, handle) + reason = ("the desktop portal closed the remote-desktop session" + if event == OEFFIS_EVENT_CLOSED + else "the desktop portal disconnected the remote-desktop session") + raise OeffisUnavailable(f"{reason}{': ' + detail if detail else ''}") + + +def _release(symbols: BoundSymbols, handle: int) -> None: + """Drop the oeffis context; never raise from cleanup.""" + try: + symbols.oeffis_unref(handle) + except (AttributeError, OSError, ValueError): + pass + + +class _Session: + """Keeps the portal session alive; releasing it ends the grant.""" + + def __init__(self, symbols: BoundSymbols, handle: int) -> None: + self._symbols = symbols + self._handle: Optional[int] = handle + + def close(self) -> None: + """End the portal session.""" + if self._handle is None: + return + _release(self._symbols, self._handle) + self._handle = None + + def __del__(self) -> None: + self.close() + + +__all__ = [ + "DEFAULT_TIMEOUT", "OEFFIS_DEVICE_ALL_DEVICES", "OEFFIS_DEVICE_DEFAULT", + "OEFFIS_DEVICE_KEYBOARD", "OEFFIS_DEVICE_POINTER", + "OEFFIS_DEVICE_TOUCHSCREEN", "OEFFIS_EVENT_CLOSED", + "OEFFIS_EVENT_CONNECTED_TO_EIS", "OEFFIS_EVENT_DISCONNECTED", + "OEFFIS_EVENT_NONE", "OeffisUnavailable", "connect_eis_fd", + "is_available", "load_symbols", +] diff --git a/test/unit_test/headless/test_wayland_libei.py b/test/unit_test/headless/test_wayland_libei.py index 00999e33..6756cfd6 100644 --- a/test/unit_test/headless/test_wayland_libei.py +++ b/test/unit_test/headless/test_wayland_libei.py @@ -1,29 +1,232 @@ -"""Tests for the libei backend selector + ctypes binding.""" -from unittest.mock import MagicMock +"""Tests for the libei backend selector, the portal hop, and the handshake. + +libei will not emit anything until a sender has bound a seat's capabilities, +taken a device *out of an event*, and started emulating on it — and every +emission has to be followed by a frame. These tests drive that state machine +against a fake libei whose entry points record what they were called with, so +the protocol order is pinned on any host. + +The one thing they cannot check is the ABI: the enum values and function +signatures come from ``libei.h`` and only a real Wayland session can confirm +them. What the tests *can* guarantee is the fail-closed property — every path +that does not reach a live device raises ``LibeiUnavailable``, which the +keyboard and mouse modules turn into "use the ydotool CLI". +""" +from unittest.mock import MagicMock, patch import pytest from je_auto_control.linux_wayland import ( - LibeiBackend, LibeiUnavailable, get_default_backend, - select_input_backend, + LibeiBackend, LibeiUnavailable, select_input_backend, ) from je_auto_control.linux_wayland import libei as libei_mod +from je_auto_control.linux_wayland import oeffis as oeffis_mod from je_auto_control.linux_wayland import ( _select_input as select_mod, ) +from je_auto_control.linux_wayland import _ydotool_cli + + +SEAT = 0x5EA7 +KEYBOARD_DEVICE = 0xD0001 +POINTER_DEVICE = 0xD0002 +SENDER = 0xEEEE + + +@pytest.fixture(autouse=True) +def _select_always_ready(): + """Report libei's fd readable without touching the real ``select``. + + On Linux ``ei_get_fd`` hands back a genuine epoll descriptor; the fake + returns a plain integer, and Windows' ``select`` accepts only sockets. + Stubbing the readiness check keeps these tests about the handshake + rather than about the host's poll implementation. + """ + with patch.object(libei_mod.select, "select", + side_effect=lambda r, _w, _x, _t: (list(r), [], [])): + yield + + +@pytest.fixture(autouse=True) +def _ydotool_generation_already_known(): + """Keep the ydotool 0.1.x probe out of the captured argv of these tests. + + The CLI fallback paths here assert on the exact argv the backend builds. + Before building any, both backends probe the installed ydotool once — + 0.1.x answers this argv with exit code 0 and no events — and that probe is + a ``subprocess.run`` call, so it would be captured alongside the argv + under test. ``test_wayland_ydotool_cli.py`` covers the probe itself. + """ + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + try: + yield + finally: + _ydotool_cli.reset_cache() + + +class FakeLibei: + """A libei that plays back a scripted event stream and records calls. + + Devices carry capabilities the way real ones do, so the backend has to + ask ``ei_device_has_capability`` rather than assume. + """ + + def __init__(self, events=None, capabilities=None, regions=None): + self.calls = [] + self.pending = list(events if events is not None else _full_handshake()) + self.capabilities = capabilities if capabilities is not None else { + KEYBOARD_DEVICE: {libei_mod.EI_DEVICE_CAP_KEYBOARD}, + POINTER_DEVICE: {libei_mod.EI_DEVICE_CAP_POINTER_ABSOLUTE, + libei_mod.EI_DEVICE_CAP_BUTTON, + libei_mod.EI_DEVICE_CAP_SCROLL}, + } + #: ``device -> [(x, y, width, height)]``. Empty by default, which is + #: the real "device declared no region" case: it accepts anything. + self.regions = dict(regions or {}) + self._queue = [] + self.lib = MagicMock() + # -- context --------------------------------------------------------- + def ei_new_sender(self, _user_data): + self.calls.append(("new_sender",)) + return SENDER -def _fake_symbols() -> object: - """Build a stand-in for :class:`_LibeiSymbols` we can introspect.""" - symbols = MagicMock() - symbols.ei_new_sender = MagicMock(return_value=0xDEADBEEF) - symbols.ei_setup_backend_socket = MagicMock(return_value=0) - symbols.ei_device_keyboard_key = MagicMock() - symbols.ei_device_pointer_motion_absolute = MagicMock() - symbols.ei_device_button_button = MagicMock() - symbols.ei_device_scroll = MagicMock() - symbols.ei_unref = MagicMock() - return symbols + def ei_setup_backend_fd(self, _ei, fd): + self.calls.append(("setup_backend_fd", fd)) + return 0 + + def ei_setup_backend_socket(self, _ei, path): + self.calls.append(("setup_backend_socket", path)) + return 0 + + def ei_get_fd(self, _ei): + return 0 # a real fd number is never polled: timeout is 0 + + def ei_dispatch(self, _ei): + # One dispatch moves the whole scripted batch into the event queue. + self._queue.extend(self.pending) + self.pending = [] + + def ei_get_event(self, _ei): + return self._queue.pop(0) if self._queue else None + + def ei_event_unref(self, _event): + return None + + def ei_now(self, _ei): + return 123456789 + + def ei_unref(self, _ei): + self.calls.append(("unref",)) + return None + + # -- events ---------------------------------------------------------- + def ei_event_get_type(self, event): + return event[0] + + def ei_event_get_seat(self, event): + return event[1] + + def ei_event_get_device(self, event): + return event[1] + + # -- seat / device --------------------------------------------------- + def ei_seat_bind_capabilities(self, *args): + self.calls.append(("bind", tuple( + getattr(a, "value", a) for a in args))) + + def ei_device_has_capability(self, device, cap): + return cap in self.capabilities.get(device, set()) + + def ei_device_ref(self, device): + self.calls.append(("device_ref", device)) + return device + + def ei_device_unref(self, device): + self.calls.append(("device_unref", device)) + return None + + def ei_device_start_emulating(self, device, sequence): + self.calls.append(("start_emulating", device, sequence)) + + def ei_device_stop_emulating(self, device): + self.calls.append(("stop_emulating", device)) + + def ei_device_frame(self, device, when): + self.calls.append(("frame", device, when)) + + def ei_device_keyboard_key(self, device, keycode, is_press): + self.calls.append(("key", device, keycode, is_press)) + + def ei_device_pointer_motion_absolute(self, device, x, y): + self.calls.append(("motion", device, x, y)) + + # -- regions --------------------------------------------------------- + # A region handle is ``(device, index)``: truthy, and enough to answer + # the four getters. libei ends the list by returning NULL. + def ei_device_get_region(self, device, index): + shapes = self.regions.get(device, ()) + return (device, index) if index < len(shapes) else None + + def _region(self, handle): + device, index = handle + return self.regions[device][index] + + def ei_region_get_x(self, handle): + return self._region(handle)[0] + + def ei_region_get_y(self, handle): + return self._region(handle)[1] + + def ei_region_get_width(self, handle): + return self._region(handle)[2] + + def ei_region_get_height(self, handle): + return self._region(handle)[3] + + def ei_device_button_button(self, device, button, is_press): + self.calls.append(("button", device, button, is_press)) + + def ei_device_scroll_discrete(self, device, x, y): + self.calls.append(("scroll", device, x, y)) + + +def _full_handshake(): + """The event stream a cooperating compositor produces.""" + return [ + (libei_mod.EI_EVENT_CONNECT, None), + (libei_mod.EI_EVENT_SEAT_ADDED, SEAT), + (libei_mod.EI_EVENT_DEVICE_ADDED, KEYBOARD_DEVICE), + (libei_mod.EI_EVENT_DEVICE_ADDED, POINTER_DEVICE), + (libei_mod.EI_EVENT_DEVICE_RESUMED, KEYBOARD_DEVICE), + (libei_mod.EI_EVENT_DEVICE_RESUMED, POINTER_DEVICE), + ] + + +def _portal_present(): + """Pretend liboeffis is installed, so the portal route is taken. + + Which route ``_open_backend`` picks is decided by that probe, not by the + injected connector — a host without liboeffis would otherwise fall + through to the well-known socket and these tests would silently stop + exercising the portal. + """ + return patch.object(libei_mod.oeffis, "is_available", return_value=True) + + +def _connected(fake=None, session=None): + """Return a backend that has completed the handshake over a fake fd.""" + fake = fake or FakeLibei() + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, session or MagicMock())) + with _portal_present(): + backend.connect(timeout=1.0) + return backend, fake + + +def _kinds(fake): + return [call[0] for call in fake.calls] # === Selector ============================================================= @@ -60,7 +263,390 @@ def test_select_input_backend_invalid_override_treated_as_auto(monkeypatch): }) == "cli" -# === LibeiBackend (mocked) ================================================ +# === ABI constants ======================================================== + +def test_capability_constants_match_libei_h(): + """``enum ei_device_capability`` is a bitmask, not a sequence. + + Checked against libei.h (Debian 1.5.0-3 and upstream main, which agree on + these six). This was wrong here — KEYBOARD had been guessed as 3 rather + than ``1 << 2`` — and the consequence was invisible: no device would ever + have reported the capability, so the handshake would have timed out and + every session silently fallen back to the ydotool CLI. + """ + assert libei_mod.EI_DEVICE_CAP_POINTER == 1 + assert libei_mod.EI_DEVICE_CAP_POINTER_ABSOLUTE == 2 + assert libei_mod.EI_DEVICE_CAP_KEYBOARD == 4 + assert libei_mod.EI_DEVICE_CAP_TOUCH == 8 + assert libei_mod.EI_DEVICE_CAP_SCROLL == 16 + assert libei_mod.EI_DEVICE_CAP_BUTTON == 32 + + +def test_event_type_constants_match_libei_h(): + """``enum ei_event_type`` starts at 1 and increments implicitly through + DEVICE_RESUMED before jumping to 90.""" + assert libei_mod.EI_EVENT_CONNECT == 1 + assert libei_mod.EI_EVENT_DISCONNECT == 2 + assert libei_mod.EI_EVENT_SEAT_ADDED == 3 + assert libei_mod.EI_EVENT_SEAT_REMOVED == 4 + assert libei_mod.EI_EVENT_DEVICE_ADDED == 5 + assert libei_mod.EI_EVENT_DEVICE_REMOVED == 6 + assert libei_mod.EI_EVENT_DEVICE_PAUSED == 7 + assert libei_mod.EI_EVENT_DEVICE_RESUMED == 8 + + +# === Handshake ============================================================ + +def test_connect_attaches_the_portal_fd_not_a_socket_path(): + """GNOME / KDE hand the EIS socket over as a file descriptor; a path + only exists on compositors that publish one.""" + _, fake = _connected() + assert ("setup_backend_fd", 7) in fake.calls + assert not any(call[0] == "setup_backend_socket" for call in fake.calls) + + +def test_connect_falls_back_to_the_well_known_socket_without_liboeffis(): + """Some compositors publish EIS at ``$XDG_RUNTIME_DIR/eis-0``; without + liboeffis that is the only route left, and it beats giving up.""" + fake = FakeLibei() + backend = LibeiBackend(symbols=fake) + with patch.object(libei_mod.oeffis, "is_available", return_value=False): + backend.connect(timeout=1.0) + setup = next(c for c in fake.calls if c[0] == "setup_backend_socket") + assert setup[1].endswith(b"eis-0") + + +def test_connect_binds_the_seat_capabilities_it_will_emit(): + _, fake = _connected() + bind = next(call for call in fake.calls if call[0] == "bind") + # seat first, then each capability, then the NULL terminator. + assert bind[1][0] == SEAT + assert set(bind[1][1:-1]) == set(libei_mod._WANTED_CAPS) + assert bind[1][-1] is None + + +def test_connect_starts_emulating_on_every_resumed_device(): + _, fake = _connected() + started = [call for call in fake.calls if call[0] == "start_emulating"] + assert {call[1] for call in started} == {KEYBOARD_DEVICE, POINTER_DEVICE} + # The sequence number increases per device, as libei requires. + assert [call[2] for call in started] == [1, 2] + + +def test_devices_are_taken_from_events_never_from_the_context(): + """The defect this rewrite exists for: the old binding passed the ``ei`` + context to entry points that take an ``ei_device``.""" + backend, fake = _connected() + backend.press_key(30) + for call in fake.calls: + if call[0] in ("key", "motion", "button", "scroll", "frame", + "start_emulating"): + assert call[1] in (KEYBOARD_DEVICE, POINTER_DEVICE) + assert call[1] != SENDER + + +def test_handshake_times_out_when_no_device_ever_resumes(): + """A seat that never resumes a device must not hang the caller.""" + fake = FakeLibei(events=[(libei_mod.EI_EVENT_SEAT_ADDED, SEAT)]) + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, MagicMock())) + with _portal_present(), pytest.raises(LibeiUnavailable, + match="handshake timeout"): + backend.connect(timeout=0.05) + + +def test_handshake_rejects_a_partial_grant(): + """Only a pointer and no keyboard would leave every key press silently + doing nothing, so a partial grant counts as no grant.""" + fake = FakeLibei( + events=[(libei_mod.EI_EVENT_SEAT_ADDED, SEAT), + (libei_mod.EI_EVENT_DEVICE_ADDED, POINTER_DEVICE), + (libei_mod.EI_EVENT_DEVICE_RESUMED, POINTER_DEVICE)], + ) + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, MagicMock())) + with _portal_present(), pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + + +def test_a_denied_portal_request_reads_as_libei_unavailable(): + """A dismissed consent dialog is an ordinary fall-back-to-CLI, not a + crash and not a hang.""" + def refuse(): + raise oeffis_mod.OeffisUnavailable("the user dismissed the dialog") + + fake = FakeLibei() + backend = LibeiBackend(symbols=fake, portal_connect=refuse) + with _portal_present(), pytest.raises(LibeiUnavailable, match="dismissed"): + backend.connect(timeout=1.0) + # The sender is released rather than leaked on the way out. + assert ("unref",) in fake.calls + + +def test_connect_abandons_an_opened_context_rather_than_unref_it(): + """``ei_unref`` segfaults on libei 1.3.901 once the backend is open. + + Measured against the real library in ``docker/libei_verify.py``: safe + before setup and after a *failed* setup, SIGSEGV after a successful one. + A crash in a library driving someone's desktop is worse than leaking one + context per process, so an opened backend is dropped without unref. + """ + fake = FakeLibei(events=[]) + session = MagicMock() + backend = LibeiBackend(symbols=fake, portal_connect=lambda: (7, session)) + with _portal_present(), pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + assert ("unref",) not in fake.calls + # The portal session is liboeffis, a different library, and closing it is + # what actually revokes the grant — so that still has to happen. + session.close.assert_called_once() + + +def test_a_context_that_never_opened_a_backend_is_unreffed(): + """The other half of the rule: before any backend is set up, ``unref`` is + the documented release and is safe, so it must still run.""" + fake = FakeLibei() + + def refuse(): + raise oeffis_mod.OeffisUnavailable("no portal") + + backend = LibeiBackend(symbols=fake, portal_connect=refuse) + with _portal_present(), pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + assert ("unref",) in fake.calls + + +def test_a_failed_backend_setup_is_unreffed(): + """``ei_setup_backend_socket`` returning non-zero leaves the context in + the state the header documents as releasable, and the probe confirms is + safe.""" + class _SetupFails(FakeLibei): + def ei_setup_backend_socket(self, _ei, path): + self.calls.append(("setup_backend_socket", path)) + return -2 + + fake = _SetupFails() + backend = LibeiBackend(symbols=fake) + with pytest.raises(LibeiUnavailable, match="returned -2"): + backend.connect(timeout=0.05, socket_path=b"/nope") + assert ("unref",) in fake.calls + + +# === Emission ============================================================= + +def test_every_emission_is_followed_by_a_frame(): + """libei buffers until ``ei_device_frame``; without it the compositor + sees nothing, which is how the old binding could look correct and emit + nothing at all.""" + backend, fake = _connected() + backend.press_key(30) + backend.set_position(4, 9) + backend.press_button(272) + backend.scroll(0, 3) + kinds = _kinds(fake) + for emission in ("key", "motion", "button", "scroll"): + index = kinds.index(emission) + assert kinds[index + 1] == "frame", f"{emission} was not framed" + + +def test_scroll_sends_whole_wheel_clicks_not_raw_detents(): + """libei measures discrete scroll in 120ths of a click, like WHEEL_DELTA. + + Passing a raw detent count is a 120th of the scroll asked for, and libei + says so at runtime — "suspicious discrete event value 1, did you mean + 120?" — which is how ``docker/eis_verify.py`` found it against a real EIS + peer. It reads the value back off the wire there; this pins the unit + without needing one.""" + backend, fake = _connected() + + def _last_scroll(): + return [call for call in fake.calls if call[0] == "scroll"][-1][2:] + + backend.scroll(0, 3) + assert _last_scroll() == (0, 3 * libei_mod.SCROLL_UNIT) + backend.scroll(-2, 0) + assert _last_scroll() == (-2 * libei_mod.SCROLL_UNIT, 0), \ + "the sign has to survive the conversion too" + + +def test_key_press_and_release_use_the_keyboard_device(): + backend, fake = _connected() + backend.press_key(30) + backend.release_key(30) + keys = [call for call in fake.calls if call[0] == "key"] + assert keys == [("key", KEYBOARD_DEVICE, 30, True), + ("key", KEYBOARD_DEVICE, 30, False)] + + +def test_pointer_motion_uses_the_absolute_pointer_device(): + backend, fake = _connected() + backend.set_position(120, 340) + assert ("motion", POINTER_DEVICE, 120.0, 340.0) in fake.calls + + +# === Absolute pointer regions ============================================= +# +# libei drops an absolute motion that lands in no region and reports nothing +# about it: no return code, no event, no error the caller can see. Measured +# against a real EIS peer, which is also where the rest of this section's +# expectations come from — see ``docker/eis_verify.py``. + +def _with_regions(regions): + """A connected backend whose absolute pointer advertises ``regions``.""" + return _connected(FakeLibei(regions={POINTER_DEVICE: regions})) + + +def test_a_device_with_no_region_takes_the_coordinate_unchanged(): + """The common case, and the one every other test here relies on: a + device that declared no region accepts anything, so nothing is mapped.""" + backend, fake = _connected() + backend.set_position(4000, 3000) + assert ("motion", POINTER_DEVICE, 4000.0, 3000.0) in fake.calls + + +def test_a_point_inside_a_region_is_sent_as_it_stands(): + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + backend.set_position(640, 400) + assert ("motion", POINTER_DEVICE, 640.0, 400.0) in fake.calls + + +def test_region_offsets_are_part_of_the_coordinate_not_stripped(): + """A region at ``x=1280`` takes 1380 for a point 100 pixels into it.""" + backend, fake = _with_regions([(1280, 0, 1920, 1080)]) + backend.set_position(1380, 100) + assert ("motion", POINTER_DEVICE, 1380.0, 100.0) in fake.calls + + +def test_the_far_edges_of_a_region_are_outside_it(monkeypatch): + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + backend.set_position(1919, 1079) + assert ("motion", POINTER_DEVICE, 1919.0, 1079.0) in fake.calls + with pytest.raises(LibeiUnavailable, match="outside every region"): + backend.set_position(1920, 1080) + + +def test_a_point_no_region_covers_is_refused_not_silently_dropped(monkeypatch): + """The bug this section exists for. Without the check libei swallows the + motion and ``set_position`` returns as though the pointer had moved.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + with pytest.raises(LibeiUnavailable, match=r"\(4000, 30\) lies outside"): + backend.set_position(4000, 30) + assert not [call for call in fake.calls if call[0] == "motion"] + + +def test_a_refused_motion_is_not_committed_with_a_frame(monkeypatch): + """Nothing was buffered, so nothing may be flushed — a frame here would + commit whatever the previous emission left on the device.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + with pytest.raises(LibeiUnavailable): + backend.set_position(4000, 30) + assert not [call for call in fake.calls if call[0] == "frame"] + + +def test_a_negative_layout_origin_is_normalised_into_region_space(monkeypatch): + """Region offsets are ``uint32``, so a compositor cannot advertise one + left of the origin — but a monitor placed left of the primary gives this + project's layout a negative origin. Input has to make the same shift + capture already makes, or ``get_pixel`` and ``set_position`` name + different pixels.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (-1280, 0)) + backend, fake = _with_regions([(0, 0, 1280, 1024), (1280, 0, 1920, 1080)]) + backend.set_position(-1280, 10) + assert ("motion", POINTER_DEVICE, 0.0, 10.0) in fake.calls + + +def test_normalising_is_not_attempted_on_a_layout_that_starts_at_zero( + monkeypatch): + """A zero origin makes the shift a no-op, so an out-of-region point must + still be refused rather than sent twice.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + with pytest.raises(LibeiUnavailable): + backend.set_position(-5, -5) + assert not [call for call in fake.calls if call[0] == "motion"] + + +def test_a_point_outside_even_after_normalising_is_still_refused(monkeypatch): + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (-1280, 0)) + backend, fake = _with_regions([(0, 0, 1280, 1024)]) + with pytest.raises(LibeiUnavailable, match="outside every region"): + backend.set_position(9000, 9000) + assert not [call for call in fake.calls if call[0] == "motion"] + + +def test_the_layout_origin_is_only_consulted_when_a_point_misses(): + """It costs a ``wlr-randr`` subprocess, so it must stay off the path + every ordinary mouse move takes.""" + asked = [] + with patch.object(libei_mod, "layout_origin", + side_effect=lambda: asked.append(1) or (0, 0)): + backend, _fake = _with_regions([(0, 0, 1920, 1080)]) + backend.set_position(640, 400) + assert asked == [] + + +def test_layout_origin_is_zero_when_the_layout_cannot_be_read(): + """GNOME and KDE have no ``wlr-randr``; they also normalise the layout + themselves, so zero is the right answer rather than a fallback.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + side_effect=OSError("no wlr-randr")): + assert libei_mod.layout_origin() == (0, 0) + + +def test_region_enumeration_stops_at_the_ceiling(): + """A libei whose contract differs must not hang the caller's mouse.""" + fake = FakeLibei() + fake.ei_device_get_region = lambda _device, index: ("endless", index) + fake.ei_region_get_x = lambda _handle: 0 + fake.ei_region_get_y = lambda _handle: 0 + fake.ei_region_get_width = lambda _handle: 1 + fake.ei_region_get_height = lambda _handle: 1 + backend, _fake = _connected(fake) + assert len(backend._device_regions(POINTER_DEVICE)) == libei_mod._MAX_REGIONS + + +def test_a_motion_refused_for_its_region_reaches_the_cli(monkeypatch): + """End to end: the refusal is the *point*, because ``emitted`` turns it + into the ydotool move that libei would have swallowed.""" + from je_auto_control.linux_wayland import mouse as wayland_mouse + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + captured = [] + + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.set_position(4000, 30) + assert not [call for call in fake.calls if call[0] == "motion"] + assert captured[0][1:] == ["mousemove", "--absolute", + "-x", "4000", "-y", "30"] + + +def test_click_button_presses_then_releases(): + backend, fake = _connected() + backend.click_button(272) + buttons = [call for call in fake.calls if call[0] == "button"] + assert buttons == [("button", POINTER_DEVICE, 272, True), + ("button", POINTER_DEVICE, 272, False)] + + +def test_a_paused_device_stops_accepting_emissions(): + """Pause arrives asynchronously; emitting into a paused device would be + silently dropped, so it has to raise and let the CLI take over.""" + backend, fake = _connected() + fake.pending = [(libei_mod.EI_EVENT_DEVICE_PAUSED, KEYBOARD_DEVICE)] + with pytest.raises(LibeiUnavailable, match="no libei device"): + backend.press_key(30) + + +def test_emitting_before_connect_raises(): + backend = LibeiBackend(symbols=FakeLibei()) + with pytest.raises(LibeiUnavailable, match="not connected"): + backend.press_key(30) + def test_backend_reports_unavailable_when_symbols_missing(): backend = LibeiBackend(symbols=None) @@ -69,168 +655,263 @@ def test_backend_reports_unavailable_when_symbols_missing(): backend.connect() -def test_backend_connect_calls_new_sender_and_setup_socket(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"./eis-test") - symbols.ei_new_sender.assert_called_once_with(b"je_auto_control") - symbols.ei_setup_backend_socket.assert_called_once_with( - 0xDEADBEEF, b"./eis-test", - ) +def test_disconnect_ends_the_portal_session_and_releases_a_live_context(): + """A completed handshake is released properly — devices, then the context. + ``ei_unref`` is only unsafe on a context whose backend opened but whose + handshake never progressed; on a live one it is safe, measured against a + real EIS peer in ``docker/eis_verify.py``. The portal session is a + different library and always closes, or the compositor keeps the + remote-desktop grant open.""" + session = MagicMock() + backend, fake = _connected(session=session) + backend.disconnect() + session.close.assert_called_once() + assert backend.is_connected is False + assert ("unref",) in fake.calls + devices = [call for call in fake.calls if call[0] == "device_unref"] + assert devices, "the devices were dropped without being unreffed" + assert _kinds(fake).index("unref") > _kinds(fake).index("device_unref"), \ + "the context was unreffed before its devices, so those are use-after-free" + + +def test_a_handshake_that_never_completed_abandons_the_context(): + """The one state where ``ei_unref`` segfaults must still be abandoned. + + ``docker/libei_verify.py`` measures it: a backend that opened but never + handshook takes the process down on unref, and on ``ei_disconnect`` too, + so it is not a refcount mistake here.""" + fake = FakeLibei(events=[]) # nothing arrives, so no device goes live + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, MagicMock())) + with _portal_present(): + with pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + assert ("unref",) not in fake.calls + + +# === Probe caching ======================================================== + +def test_a_failed_probe_is_not_retried_on_every_keystroke(): + """The probe costs a portal round trip and possibly a consent dialog. + Paying that per key press would be worse than not having libei at all.""" + libei_mod.reset_default_backend() + attempts = [] -def test_backend_connect_is_idempotent_within_one_instance(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.connect(socket_path=b"/x") - assert symbols.ei_new_sender.call_count == 1 + class _Failing(LibeiBackend): + def __init__(self, **_kwargs): + super().__init__(symbols=FakeLibei()) + def connect(self, **_kwargs): + attempts.append(1) + raise LibeiUnavailable("no portal here") -def test_backend_connect_unrefs_on_setup_failure(): - symbols = _fake_symbols() - symbols.ei_setup_backend_socket.return_value = -1 - backend = LibeiBackend(symbols=symbols) - with pytest.raises(LibeiUnavailable, match="setup_backend_socket"): - backend.connect(socket_path=b"/x") - symbols.ei_unref.assert_called_once_with(0xDEADBEEF) + try: + with patch.object(libei_mod, "LibeiBackend", _Failing): + assert libei_mod.connected_backend() is None + assert libei_mod.connected_backend() is None + assert libei_mod.connected_backend() is None + finally: + libei_mod.reset_default_backend() + assert len(attempts) == 1 -def test_backend_press_key_calls_keyboard_key_with_state_one(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.press_key(28) # KEY_ENTER - symbols.ei_device_keyboard_key.assert_called_once_with( - 0xDEADBEEF, 28, 1, - ) +def test_active_backend_is_none_when_the_selector_says_cli(monkeypatch): + monkeypatch.setenv("JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", "cli") + assert select_mod.active_backend() is None -def test_backend_release_key_uses_state_zero(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.release_key(28) - symbols.ei_device_keyboard_key.assert_called_once_with( - 0xDEADBEEF, 28, 0, - ) +# === Keyboard / mouse fall back rather than fail ========================== +def test_keyboard_falls_back_to_ydotool_when_libei_is_unavailable(): + import subprocess -def test_backend_set_position_uses_absolute_motion(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.set_position(120, 240) - symbols.ei_device_pointer_motion_absolute.assert_called_once_with( - 0xDEADBEEF, 120.0, 240.0, - ) + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + captured = [] + with patch.object(wayland_keyboard, "_try_libei", return_value=None), \ + patch.object(wayland_keyboard, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_keyboard.subprocess, "run", + side_effect=lambda argv, **kw: ( + captured.append(list(argv)) + or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep + wayland_keyboard.press_key(28) + assert captured == [["/usr/bin/ydotool", "key", "28:1"]] -def test_backend_click_button_sends_press_then_release(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.click_button(272) - calls = symbols.ei_device_button_button.call_args_list - assert len(calls) == 2 - assert calls[0].args == (0xDEADBEEF, 272, 1) - assert calls[1].args == (0xDEADBEEF, 272, 0) +def test_keyboard_uses_libei_when_it_is_connected(): + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + backend = MagicMock() + with patch.object(wayland_keyboard, "_try_libei", return_value=backend): + wayland_keyboard.press_key(28) + backend.press_key.assert_called_once_with(28) -def test_backend_scroll_forwards_float_deltas(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.scroll(0, 3) - symbols.ei_device_scroll.assert_called_once_with(0xDEADBEEF, 0.0, 3.0) +def test_mouse_button_maps_onto_an_evdev_code_for_libei(): + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() + with patch.object(wayland_mouse, "_try_libei", return_value=backend): + wayland_mouse.press_mouse(wayland_mouse.wayland_mouse_left) + # BTN_LEFT, not ydotool's 0xC0 bitmask. + backend.press_button.assert_called_once_with(272) -def test_backend_disconnect_releases_handle(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.disconnect() - symbols.ei_unref.assert_called_once_with(0xDEADBEEF) +def _scroll_calls(backend): + return [tuple(call.args) for call in backend.scroll.call_args_list] -def test_backend_method_before_connect_raises(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - with pytest.raises(LibeiUnavailable, match="not connected"): - backend.press_key(28) +def test_mouse_scroll_flips_the_vertical_axis_for_libei(): + """The two paths count wheel detents in opposite directions. -def test_get_default_backend_returns_none_when_unavailable(monkeypatch): - libei_mod.reset_default_backend() - monkeypatch.setattr(libei_mod, "_load_symbols", lambda: None) - assert get_default_backend() is None - libei_mod.reset_default_backend() + This module's ``wayland_scroll_direction_*`` constants are in the + kernel's ``REL_WHEEL`` frame — what ydotool writes into ``/dev/uinput``, + where positive is up. libei is in the ``wl_pointer`` / libinput frame, + where positive is down: libinput's own evdev reader negates ``REL_WHEEL`` + to get there. Handing the constant over unchanged would scroll the wrong + way on every libei host, and scrolling the wrong way fails silently. + """ + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + patch.object(wayland_mouse, "binary_path", return_value=None): + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_up) + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_down) + # binary_path is None, so any fall-through to the CLI would have raised. + assert _scroll_calls(backend) == [(0, -5), (0, 5)] -def test_get_default_backend_caches_single_instance(monkeypatch): - libei_mod.reset_default_backend() - symbols = _fake_symbols() - monkeypatch.setattr(libei_mod, "_load_symbols", lambda: symbols) - a = get_default_backend() - b = get_default_backend() - assert a is b - libei_mod.reset_default_backend() +def test_mouse_scroll_does_not_flip_the_horizontal_axis_for_libei(): + """``REL_HWHEEL`` and libinput agree that right is positive, and + libinput passes that axis through unnegated — so flipping both axes + would be as wrong as flipping neither.""" + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() -# === Library probe ======================================================== + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + patch.object(wayland_mouse, "binary_path", return_value=None): + wayland_mouse.scroll(3, wayland_mouse.wayland_scroll_direction_right) + wayland_mouse.scroll(3, wayland_mouse.wayland_scroll_direction_left) + assert _scroll_calls(backend) == [(3, 0), (-3, 0)] -def test_try_load_library_returns_none_when_find_library_fails(monkeypatch): - monkeypatch.setattr(libei_mod.ctypes.util, "find_library", - lambda _name: None) - assert libei_mod._try_load_library() is None +def _cli_capture(module, captured): + """Patch ``module``'s ydotool lookup and subprocess.run, recording argv.""" + import subprocess -# === Keyboard integration ================================================ + return ( + patch.object(module, "binary_path", return_value="/usr/bin/ydotool"), + patch.object(module.subprocess, "run", + side_effect=lambda argv, **kw: ( + captured.append(list(argv)) + or subprocess.CompletedProcess(argv, 0, b"", b""))), # nosemgrep + ) -def test_keyboard_press_key_uses_libei_when_selected(monkeypatch): - from je_auto_control.linux_wayland import keyboard as wayland_kb - libei_mock = MagicMock() - monkeypatch.setattr(wayland_kb, "_try_libei", lambda: libei_mock) - wayland_kb.press_key(28) - libei_mock.press_key.assert_called_once_with(28) +def test_libei_unavailable_is_catchable_as_an_autocontrol_error(): + """It used to inherit ``RuntimeError`` alone, so every ``except + AutoControlException`` boundary — the executor, the poll loops, the + request handlers, the GUI slots — let it straight through.""" + from je_auto_control.utils.exception.exceptions import AutoControlException -def test_keyboard_falls_back_to_ydotool_when_libei_unavailable(monkeypatch): - from je_auto_control.linux_wayland import keyboard as wayland_kb - monkeypatch.setattr(wayland_kb, "_try_libei", lambda: None) - monkeypatch.setattr(wayland_kb, "binary_path", - lambda _name: "/usr/bin/ydotool") - monkeypatch.setattr(wayland_kb.subprocess, "run", - lambda *_args, **_kw: None) - # Should not raise. - wayland_kb.press_key(28) + assert issubclass(LibeiUnavailable, AutoControlException) + # The probes in _select_input and the two input modules catch + # RuntimeError; that has to keep working. + assert issubclass(LibeiUnavailable, RuntimeError) -def test_mouse_set_position_uses_libei_when_selected(monkeypatch): +def test_a_refused_emission_falls_back_to_the_cli(): + """A backend that finished its handshake can still refuse one emission — + a paused device, a session that ended between calls. libei is documented + as the fast path and never the only one, but the refusal used to escape + the module instead of reaching ydotool.""" from je_auto_control.linux_wayland import mouse as wayland_mouse - libei_mock = MagicMock() - monkeypatch.setattr(wayland_mouse, "_try_libei", lambda: libei_mock) - wayland_mouse.set_position(100, 200) - libei_mock.set_position.assert_called_once_with(100, 200) + backend = MagicMock() + backend.set_position.side_effect = LibeiUnavailable("device paused") + captured = [] + + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.set_position(120, 240) + assert captured[0][1:] == ["mousemove", "--absolute", + "-x", "120", "-y", "240"] -def test_mouse_falls_back_to_ydotool_when_libei_unavailable(monkeypatch): +def test_a_refused_scroll_falls_back_to_the_cli_in_ydotools_own_frame(): + """And it falls back *unflipped*: the vertical flip belongs to libei.""" from je_auto_control.linux_wayland import mouse as wayland_mouse - monkeypatch.setattr(wayland_mouse, "_try_libei", lambda: None) - monkeypatch.setattr(wayland_mouse, "binary_path", - lambda _name: "/usr/bin/ydotool") - monkeypatch.setattr(wayland_mouse.subprocess, "run", - lambda *_args, **_kw: None) - wayland_mouse.set_position(0, 0) + backend = MagicMock() + backend.scroll.side_effect = LibeiUnavailable("device paused") + captured = [] + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_up) + assert captured[0][1:] == ["mousemove", "--wheel", "-x", "0", "-y", "5"] -# === Facade =============================================================== -def test_facade_exports_libei_helpers(): - from je_auto_control.linux_wayland import ( - LibeiBackend as LB, LibeiUnavailable as LU, - get_default_backend as gdb, select_input_backend as sib, - ) - assert LB is LibeiBackend - assert LU is LibeiUnavailable - assert callable(gdb) and callable(sib) +def test_a_refused_button_release_still_reaches_the_cli(): + """The worst refusal to drop: the press landed, so giving up on the + release leaves the button held for the rest of the session.""" + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() + backend.release_button.side_effect = LibeiUnavailable("device paused") + captured = [] + + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.click_mouse(wayland_mouse.wayland_mouse_left) + backend.press_button.assert_called_once_with(272) + # 0xC0 with the down-bit cleared: a release-only click. + assert captured[0][1:] == ["click", "0x80"] + + +def test_a_refused_key_press_falls_back_to_the_cli(): + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + backend = MagicMock() + backend.press_key.side_effect = LibeiUnavailable("device paused") + captured = [] + + binary, run = _cli_capture(wayland_keyboard, captured) + with patch.object(wayland_keyboard, "_try_libei", return_value=backend), \ + binary, run: + wayland_keyboard.press_key(28) + assert captured[0][1:] == ["key", "28:1"] + + +def test_a_chord_refused_part_way_releases_what_it_already_pressed(): + """Otherwise the fallback presses Ctrl again on top of a Ctrl that libei + never released, and the modifier is stuck once the CLI chord ends.""" + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + backend = MagicMock() + backend.press_key.side_effect = [None, LibeiUnavailable("device paused")] + captured = [] + + binary, run = _cli_capture(wayland_keyboard, captured) + with patch.object(wayland_keyboard, "_try_libei", return_value=backend), \ + binary, run: + wayland_keyboard.hotkey([29, 42]) + backend.release_key.assert_called_once_with(29) + assert captured[0][1:] == ["key", "29:1", "42:1", "42:0", "29:0"] + + +def test_mouse_scroll_falls_back_to_ydotool_without_libei(): + """No libei means the ydotool argv, in ydotool's own frame — unflipped.""" + import subprocess + + from je_auto_control.linux_wayland import mouse as wayland_mouse + captured = [] + + with patch.object(wayland_mouse, "_try_libei", return_value=None), \ + patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=lambda argv, **kw: ( + captured.append(list(argv)) + or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_down) + assert captured[0][1:] == ["mousemove", "--wheel", "-x", "0", "-y", "-5"] diff --git a/test/unit_test/headless/test_wayland_oeffis.py b/test/unit_test/headless/test_wayland_oeffis.py new file mode 100644 index 00000000..98febaf6 --- /dev/null +++ b/test/unit_test/headless/test_wayland_oeffis.py @@ -0,0 +1,164 @@ +"""The portal hop that hands libei an EIS file descriptor. + +``ConnectToEIS`` returns a file descriptor over D-Bus, which no command-line +tool can pass into this process — so unlike the Screenshot portal there is no +``gdbus`` shortcut, and liboeffis does the session dance instead. These tests +drive its event loop against a fake library: the success path, the user +declining, the portal going away, and the timeout. All four have to end in a +released context, because a leaked oeffis handle keeps a remote-desktop grant +open on the user's session. +""" +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import oeffis as oeffis_mod + + +HANDLE = 0x0EFF15 + + +class FakeOeffis: + """A liboeffis whose event stream the test scripts.""" + + def __init__(self, events=(), eis_fd=11, error=b""): + self.calls = [] + self._events = list(events) + self._eis_fd = eis_fd + self._error = error + + def oeffis_new(self, _user_data): + self.calls.append(("new",)) + return HANDLE + + def oeffis_unref(self, handle): + self.calls.append(("unref", handle)) + return None + + def oeffis_create_session(self, handle, devices): + self.calls.append(("create_session", handle, devices)) + + def oeffis_get_fd(self, _handle): + return 0 + + def oeffis_dispatch(self, _handle): + self.calls.append(("dispatch",)) + + def oeffis_get_event(self, _handle): + return self._events.pop(0) if self._events else oeffis_mod.OEFFIS_EVENT_NONE + + def oeffis_get_eis_fd(self, _handle): + return self._eis_fd + + def oeffis_get_error_message(self, _handle): + return self._error + + +@pytest.fixture(autouse=True) +def _select_always_ready(): + """Report the oeffis fd readable; the fake's fd is not a real socket.""" + with patch.object(oeffis_mod.select, "select", + side_effect=lambda r, _w, _x, _t: (list(r), [], [])): + yield + + +def test_connect_returns_the_eis_fd_and_a_live_session(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS]) + eis_fd, session = oeffis_mod.connect_eis_fd(symbols=fake) + assert eis_fd == 11 + # The session must NOT be released yet: dropping it ends the grant. + assert ("unref", HANDLE) not in fake.calls + session.close() + assert ("unref", HANDLE) in fake.calls + + +def test_connect_requests_only_the_devices_this_backend_emits(): + """The consent dialog shows the user what is being granted, so asking for + a touchscreen this backend never drives would widen the grant for no + benefit.""" + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS]) + oeffis_mod.connect_eis_fd(symbols=fake) + created = next(c for c in fake.calls if c[0] == "create_session") + assert created[2] == (oeffis_mod.OEFFIS_DEVICE_KEYBOARD + | oeffis_mod.OEFFIS_DEVICE_POINTER) + assert not created[2] & oeffis_mod.OEFFIS_DEVICE_TOUCHSCREEN + + +def test_event_constants_match_liboeffis_h(): + """Checked against liboeffis.h (Debian 1.5.0-3 and upstream main, which + agree). CLOSED precedes DISCONNECTED — the opposite of what the names + suggest, and it was wrong here until the header was read. + """ + assert oeffis_mod.OEFFIS_EVENT_NONE == 0 + assert oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS == 1 + assert oeffis_mod.OEFFIS_EVENT_CLOSED == 2 + assert oeffis_mod.OEFFIS_EVENT_DISCONNECTED == 3 + + +def test_device_constants_match_liboeffis_h(): + assert oeffis_mod.OEFFIS_DEVICE_ALL_DEVICES == 0 + assert oeffis_mod.OEFFIS_DEVICE_KEYBOARD == 1 + assert oeffis_mod.OEFFIS_DEVICE_POINTER == 2 + assert oeffis_mod.OEFFIS_DEVICE_TOUCHSCREEN == 4 + + +def test_a_closed_session_reads_as_the_user_declining(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CLOSED], + error=b"permission denied") + with pytest.raises(oeffis_mod.OeffisUnavailable, match="closed"): + oeffis_mod.connect_eis_fd(symbols=fake) + assert ("unref", HANDLE) in fake.calls + + +def test_the_librarys_own_error_text_reaches_the_message(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CLOSED], + error=b"permission denied") + with pytest.raises(oeffis_mod.OeffisUnavailable, match="permission denied"): + oeffis_mod.connect_eis_fd(symbols=fake) + + +def test_a_disconnect_is_reported_and_the_handle_released(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_DISCONNECTED]) + with pytest.raises(oeffis_mod.OeffisUnavailable, match="disconnected"): + oeffis_mod.connect_eis_fd(symbols=fake) + assert ("unref", HANDLE) in fake.calls + + +def test_an_unanswered_consent_dialog_times_out(): + """Nobody clicking the dialog must not wedge the caller's script.""" + fake = FakeOeffis(events=[]) + with pytest.raises(oeffis_mod.OeffisUnavailable, match="did not answer"): + oeffis_mod.connect_eis_fd(symbols=fake, timeout=0.05) + assert ("unref", HANDLE) in fake.calls + + +def test_a_success_without_an_fd_is_still_a_failure(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS], + eis_fd=-1) + with pytest.raises(oeffis_mod.OeffisUnavailable, match="no EIS fd"): + oeffis_mod.connect_eis_fd(symbols=fake) + assert ("unref", HANDLE) in fake.calls + + +def test_a_null_context_is_reported(): + class _NoContext(FakeOeffis): + def oeffis_new(self, _user_data): + return 0 + + with pytest.raises(oeffis_mod.OeffisUnavailable, match="NULL"): + oeffis_mod.connect_eis_fd(symbols=_NoContext()) + + +def test_missing_library_reads_as_unavailable(): + with patch.object(oeffis_mod, "load_symbols", return_value=None): + assert oeffis_mod.is_available() is False + with pytest.raises(oeffis_mod.OeffisUnavailable, match="not found"): + oeffis_mod.connect_eis_fd() + + +def test_closing_a_session_twice_is_harmless(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS]) + _, session = oeffis_mod.connect_eis_fd(symbols=fake) + session.close() + session.close() + assert [c for c in fake.calls if c[0] == "unref"] == [("unref", HANDLE)] diff --git a/test/unit_test/headless/test_wayland_pointer_accel.py b/test/unit_test/headless/test_wayland_pointer_accel.py new file mode 100644 index 00000000..49b88b13 --- /dev/null +++ b/test/unit_test/headless/test_wayland_pointer_accel.py @@ -0,0 +1,170 @@ +"""The operator declares whether the compositor's pointer acceleration is off. + +``mousemove --absolute`` is relative motion under the hood, so the compositor +scales it: measured against a real wlroots session, libinput's default +adaptive profile lands the cursor twice as far from the corner as asked. The +factor is compositor configuration and no client can read it back, so the +library cannot compensate — only the operator knows. ``POINTER_ACCEL_ENV`` is +how they say so, and these tests pin what each answer does. + +The libei path is absolute at the protocol level, so none of this applies to +it; the last test holds that line. +""" +import subprocess +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import ( + _layout, _ydotool_cli, mouse as wayland_mouse, +) +from je_auto_control.utils.exception.exceptions import AutoControlException + + +@pytest.fixture(autouse=True) +def _pin_the_cli_input_path(monkeypatch): + """Force the ydotool argv and re-arm the once-per-process warnings.""" + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + monkeypatch.delenv(wayland_mouse.POINTER_ACCEL_ENV, raising=False) + try: + with patch.object(wayland_mouse, "_try_libei", return_value=None): + yield + finally: + _ydotool_cli.reset_cache() + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + + +def _move(x=10, y=20): + """Run ``set_position`` on the CLI path; return (argv list, warnings).""" + captured: list = [] + warned: list = [] + + def runner(argv, **_kwargs): + captured.append(list(argv)) + # CompletedProcess is a *constructor* (not a process spawn). + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse, "layout_origin", return_value=(0, 0)), \ + patch.object(wayland_mouse.autocontrol_logger, "warning", + side_effect=lambda message, *a, **k: warned.append( + str(message))), \ + patch.object(wayland_mouse.subprocess, "run", side_effect=runner): + wayland_mouse.set_position(x, y) + return captured, warned + + +def test_unset_means_warn_once_and_move_anyway(): + """The default keeps every existing caller working, loudly.""" + argv, warned = _move() + assert argv == [["/usr/bin/ydotool", "mousemove", "--absolute", + "-x", "10", "-y", "20"]] + assert len(warned) == 1 + assert "acceleration" in warned[0] + + +def test_the_warning_names_the_way_out(): + """A warning nobody can act on is noise: it must name the variable.""" + _, warned = _move() + assert wayland_mouse.POINTER_ACCEL_ENV in warned[0] + assert "flat" in warned[0] and "strict" in warned[0] + + +def test_warn_does_not_repeat_on_every_move(): + """A script making thousands of moves gets one line, not thousands.""" + _move() + _, warned = _move() + assert warned == [] + + +def test_flat_moves_silently(monkeypatch): + """The operator has switched acceleration off; there is nothing to say.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "flat") + argv, warned = _move() + assert len(argv) == 1 + assert warned == [] + + +def test_strict_refuses_rather_than_landing_somewhere_else(monkeypatch): + """Fail fast: no argv is sent at all, so nothing lands anywhere.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "strict") + with pytest.raises(AutoControlException) as raised: + _move() + assert wayland_mouse.POINTER_ACCEL_ENV in str(raised.value) + + +def test_strict_sends_no_command(monkeypatch): + """The refusal happens before the subprocess, not after.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "strict") + captured: list = [] + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=lambda argv, **k: captured.append(argv)), \ + pytest.raises(AutoControlException): + wayland_mouse.set_position(10, 20) + assert captured == [] + + +@pytest.mark.parametrize("declared, expected", [ + ("flat", "flat"), + ("FLAT", "flat"), + (" strict ", "strict"), + ("warn", "warn"), +]) +def test_the_value_is_read_case_and_whitespace_insensitively(declared, + expected): + """Operators type these into shell profiles and CI YAML by hand.""" + assert wayland_mouse.pointer_accel_mode( + {wayland_mouse.POINTER_ACCEL_ENV: declared}) == expected + + +@pytest.mark.parametrize("declared", ["", " ", "off", "flatt", "0"]) +def test_an_unrecognised_value_falls_back_to_the_safe_answer(declared): + """A typo must not silently promote a move to trusted-exact.""" + assert wayland_mouse.pointer_accel_mode( + {wayland_mouse.POINTER_ACCEL_ENV: declared}) == "warn" + + +def test_a_typo_says_so_rather_than_being_swallowed(): + """Falling back silently is how an operator believes a lie.""" + warned: list = [] + with patch.object(wayland_mouse.autocontrol_logger, "warning", + side_effect=lambda message, *a, **k: warned.append( + str(message))): + wayland_mouse.pointer_accel_mode( + {wayland_mouse.POINTER_ACCEL_ENV: "flatt"}) + assert len(warned) == 1 + assert "flatt" in warned[0] + assert "warn" in warned[0] + + +def test_a_missing_variable_is_not_a_typo(): + """An absent variable is the default, and says nothing.""" + warned: list = [] + with patch.object(wayland_mouse.autocontrol_logger, "warning", + side_effect=lambda message, *a, **k: warned.append( + str(message))): + assert wayland_mouse.pointer_accel_mode({}) == "warn" + assert warned == [] + + +def test_strict_does_not_touch_the_libei_path(monkeypatch): + """libei is absolute at the protocol level; the caveat is ydotool's.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "strict") + moved: list = [] + + class _Device: + def set_position(self, x, y): + moved.append((x, y)) + + with patch.object(wayland_mouse, "_try_libei", return_value=_Device()), \ + patch.object(wayland_mouse, "emitted", + side_effect=lambda backend, call: (call(backend), True)[1]): + wayland_mouse.set_position(7, 9) + assert moved == [(7, 9)] diff --git a/test/unit_test/headless/test_wayland_ydotool_cli.py b/test/unit_test/headless/test_wayland_ydotool_cli.py new file mode 100644 index 00000000..bd6fa40c --- /dev/null +++ b/test/unit_test/headless/test_wayland_ydotool_cli.py @@ -0,0 +1,206 @@ +"""The ydotool 0.1.x guard: what it classifies, refuses and lets through. + +The bug it exists for is measured in ``docker/ydotool_verify.py`` against a +real uinput device — 0.1.x answers this backend's argv with exit code 0 and +no events, so ``check=True`` sees success while nothing happened. These tests +cover the classification and the two backends' use of it without needing +ydotool installed, which is what makes them CI-gating on every platform. +""" +import subprocess +from unittest import mock + +import pytest + +from je_auto_control.linux_wayland import _ydotool_cli +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: The real no-argument banners, copied from ydotool 0.1.8 (Debian bookworm) +#: and 1.0.4 (Debian unstable / Arch). If a future release changes these the +#: probe answers "unknown" and stops refusing anything, which is the safe +#: direction — but docker/ydotool_verify.py fails loudly when it happens. +LEGACY_BANNER = b"""Usage: ydotool +Available commands: + type + recorder + mousemove + key + click +""" + +MODERN_BANNER = b"""Usage: ydotool +Available commands: + click + mousemove + type + key + debug + bakers +Use environment variable YDOTOOL_SOCKET to specify daemon socket. +""" + + +@pytest.fixture(autouse=True) +def _clear_probe_cache(): + """The probe caches per path for the process; tests must not share it.""" + _ydotool_cli.reset_cache() + yield + _ydotool_cli.reset_cache() + + +def _fake_run(stdout=b"", stderr=b"", returncode=0): + completed = subprocess.CompletedProcess( + args=["ydotool"], returncode=returncode, stdout=stdout, stderr=stderr) + return mock.Mock(return_value=completed) + + +def test_legacy_banner_is_classified_legacy(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=LEGACY_BANNER)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.LEGACY + + +def test_modern_banner_is_classified_modern(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=MODERN_BANNER)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.MODERN + + +def test_banner_on_stderr_is_read_too(): + """0.1.8 prints its usage to stderr on some paths; both streams count.""" + with mock.patch.object(subprocess, "run", _fake_run(stderr=LEGACY_BANNER)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.LEGACY + + +def test_unrecognised_banner_is_unknown_not_legacy(): + """Only the version measured to fail silently is refused.""" + with mock.patch.object(subprocess, "run", + _fake_run(stdout=b"ydotool 3.0\nsomething new\n")): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_a_word_containing_recorder_does_not_trigger_the_legacy_verdict(): + """The marker is a whole command name, not a substring of prose.""" + banner = b"Usage: ydotool \nAvailable commands:\n screenrecorder\n" + with mock.patch.object(subprocess, "run", _fake_run(stdout=banner)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_probe_failure_is_unknown_rather_than_an_accusation(): + with mock.patch.object(subprocess, "run", + side_effect=OSError("no such binary")): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_probe_timeout_is_unknown(): + with mock.patch.object( + subprocess, "run", + side_effect=subprocess.TimeoutExpired(cmd="ydotool", timeout=5.0)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_the_probe_runs_once_per_path(): + """Mouse and key dispatch must not pay a subprocess per event.""" + runner = _fake_run(stdout=MODERN_BANNER) + with mock.patch.object(subprocess, "run", runner): + for _ in range(5): + _ydotool_cli.cli_generation("/usr/bin/ydotool") + assert runner.call_count == 1 + + +def test_the_cache_is_keyed_on_the_path(): + runner = _fake_run(stdout=MODERN_BANNER) + with mock.patch.object(subprocess, "run", runner): + _ydotool_cli.cli_generation("/usr/bin/ydotool") + _ydotool_cli.cli_generation("/opt/other/ydotool") + assert runner.call_count == 2 + + +def test_reject_legacy_cli_raises_with_an_actionable_message(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException) as error: + _ydotool_cli.reject_legacy_cli("/usr/bin/ydotool") + message = str(error.value) + assert "0.1.x" in message + assert "/usr/bin/ydotool" in message + # The three routes out: a newer ydotool, or the X11 backend. + assert "1.0" in message + assert "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11" in message + + +def test_reject_legacy_cli_returns_the_path_for_a_modern_tool(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=MODERN_BANNER)): + assert _ydotool_cli.reject_legacy_cli("/usr/bin/ydotool") == \ + "/usr/bin/ydotool" + + +def test_reject_legacy_cli_lets_an_unknown_version_through(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=b"???")): + assert _ydotool_cli.reject_legacy_cli("/usr/bin/ydotool") == \ + "/usr/bin/ydotool" + + +def test_reset_cache_can_forget_one_path_only(): + runner = _fake_run(stdout=MODERN_BANNER) + with mock.patch.object(subprocess, "run", runner): + _ydotool_cli.cli_generation("/a/ydotool") + _ydotool_cli.cli_generation("/b/ydotool") + _ydotool_cli.reset_cache("/a/ydotool") + _ydotool_cli.cli_generation("/a/ydotool") + _ydotool_cli.cli_generation("/b/ydotool") + assert runner.call_count == 3 + + +# -------------------------------------------------------------------------- +# The two backends have to consult the guard, or it protects nobody. +# -------------------------------------------------------------------------- + +def test_mouse_refuses_to_emit_through_the_legacy_cli(): + from je_auto_control.linux_wayland import mouse + + with mock.patch("je_auto_control.linux_wayland.mouse.binary_path", + return_value="/usr/bin/ydotool"), \ + mock.patch.object(mouse, "_try_libei", return_value=None), \ + mock.patch.object(subprocess, "run", _fake_run( + stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException, match="0.1.x"): + mouse.click_mouse(mouse.wayland_mouse_left) + + +def test_keyboard_refuses_to_emit_through_the_legacy_cli(): + from je_auto_control.linux_wayland import keyboard + + with mock.patch("je_auto_control.linux_wayland.keyboard.binary_path", + return_value="/usr/bin/ydotool"), \ + mock.patch.object(keyboard, "_try_libei", return_value=None), \ + mock.patch.object(subprocess, "run", _fake_run( + stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException, match="0.1.x"): + keyboard.press_key(30) + + +def test_hotkey_refuses_to_emit_through_the_legacy_cli(): + """The chord path builds its argv separately, so it needs its own guard.""" + from je_auto_control.linux_wayland import keyboard + + with mock.patch("je_auto_control.linux_wayland.keyboard.binary_path", + return_value="/usr/bin/ydotool"), \ + mock.patch.object(keyboard, "_try_libei", return_value=None), \ + mock.patch.object(subprocess, "run", _fake_run( + stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException, match="0.1.x"): + keyboard.hotkey([29, 30]) + + +def test_the_install_hints_no_longer_recommend_a_broken_package(): + """`apt install ydotool` gives 0.1.8 on bookworm and nothing on trixie.""" + from je_auto_control.linux_wayland import keyboard, mouse + + for hint in (mouse._INSTALL_HINT, keyboard._INSTALL_HINT_YDOTOOL): + assert "apt install ydotool" not in hint + assert "1.0" in hint diff --git a/test/unit_test/headless/test_wayland_ydotool_origin.py b/test/unit_test/headless/test_wayland_ydotool_origin.py new file mode 100644 index 00000000..19712bfd --- /dev/null +++ b/test/unit_test/headless/test_wayland_ydotool_origin.py @@ -0,0 +1,175 @@ +"""The ydotool fallback counts absolute moves from the layout's corner. + +``mousemove --absolute`` sends no absolute event: it drives the cursor into +the corner the compositor clamps to and then moves relative to it. That corner +is the top-left of the output layout, which is layout ``(0, 0)`` only while +every output sits at a non-negative position — so on a desktop with a monitor +left of the primary one, a raw layout coordinate lands a monitor's width away. + +Measured against a real wlroots session in ``docker/Dockerfile.seat``; these +tests pin the translation that measurement produced, on any host, without a +compositor. +""" +import subprocess +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import ( + _layout, _ydotool_cli, libei as wayland_libei, mouse as wayland_mouse, +) + + +@pytest.fixture(autouse=True) +def _pin_the_cli_input_path(): + """Keep every test here on the ydotool argv, as the sibling file does.""" + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + try: + with patch.object(wayland_mouse, "_try_libei", return_value=None): + yield + finally: + _ydotool_cli.reset_cache() + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + + +def _fake_run(captured): + def runner(argv, **_kwargs): + captured.append(list(argv)) + # CompletedProcess is a *constructor* (not a process spawn). + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + return runner + + +def _argv_for(point, origin): + """The argv ``set_position(*point)`` builds on a layout at ``origin``.""" + captured: list = [] + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse, "layout_origin", return_value=origin), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=_fake_run(captured)): + wayland_mouse.set_position(*point) + assert len(captured) == 1 + return captured[0] + + +def test_a_layout_that_starts_at_the_origin_needs_no_translation(): + assert _argv_for((120, 240), (0, 0)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "120", "-y", "240", + ] + + +def test_a_negative_origin_is_subtracted_from_the_request(): + """A point on the primary monitor is 1,280 px into the layout.""" + assert _argv_for((120, 90), (-1280, 0)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "1400", "-y", "90", + ] + + +def test_a_point_on_the_left_hand_monitor_reaches_the_corner(): + """And the monitor left of the primary starts at the corner itself.""" + assert _argv_for((-1280, 0), (-1280, 0)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "0", "-y", "0", + ] + + +def test_a_negative_vertical_origin_is_subtracted_too(): + assert _argv_for((40, -300), (0, -400)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "40", "-y", "100", + ] + + +def test_the_translation_is_the_difference_the_capture_path_applies(): + """``_ydotool_point`` is exactly ``point - layout_origin()``.""" + with patch.object(wayland_mouse, "layout_origin", return_value=(-1280, -20)): + assert wayland_mouse._ydotool_point(120, 90) == (1400, 110) + + +# === The shared origin lookup ============================================== + +def test_both_input_paths_share_one_origin_lookup(): + """libei and ydotool must not drift apart on what the origin is.""" + assert wayland_libei.layout_origin is _layout.layout_origin + + +def test_the_origin_is_zero_when_the_screen_module_cannot_answer(): + """A host without Pillow or wlr-randr still moves, just untranslated.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + side_effect=OSError("no capture tool")): + assert _layout.layout_origin() == (0, 0) + + +def test_the_origin_is_whatever_the_screen_module_reports(): + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)): + assert _layout.layout_origin() == (-1280, 0) + + +def test_the_origin_is_read_once_for_a_burst_of_moves(): + """A drag emits many moves; each must not spawn its own ``wlr-randr``.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)) as reader: + for _ in range(20): + assert _layout.layout_origin() == (-1280, 0) + assert reader.call_count == 1 + + +def test_a_reset_makes_the_next_call_measure_again(): + """The window is short so a rearranged desktop is picked up, not pinned.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)): + assert _layout.layout_origin() == (-1280, 0) + _layout.reset_cache() + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(0, 0)): + assert _layout.layout_origin() == (0, 0) + + +def test_a_reading_older_than_the_window_is_measured_again(): + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)): + assert _layout.layout_origin() == (-1280, 0) + stamp, value = _layout._CACHE["origin"] + _layout._CACHE["origin"] = (stamp - _layout._CACHE_SECONDS - 1, value) + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(0, -400)) as reader: + assert _layout.layout_origin() == (0, -400) + assert reader.call_count == 1 + + +# === The acceleration caveat =============================================== + +def test_the_acceleration_caveat_is_logged_once_per_process(): + """Silently landing in the wrong place is the failure being prevented.""" + captured: list = [] + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse, "layout_origin", return_value=(0, 0)), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=_fake_run(captured)), \ + patch.object(wayland_mouse.autocontrol_logger, "warning") as warned: + wayland_mouse.set_position(10, 10) + wayland_mouse.set_position(20, 20) + assert warned.call_count == 1 + assert "acceleration" in warned.call_args[0][0] + + +def test_the_libei_path_says_nothing_about_acceleration(): + """It is absolute at the protocol level, so the caveat does not apply.""" + sent: list = [] + + class _Device: + def set_position(self, x, y): + sent.append((x, y)) + + with patch.object(wayland_mouse, "_try_libei", return_value=_Device()), \ + patch.object(wayland_mouse, "emitted", + side_effect=lambda backend, send: (send(backend), True)[1]), \ + patch.object(wayland_mouse.autocontrol_logger, "warning") as warned: + wayland_mouse.set_position(120, 90) + assert sent == [(120, 90)] + assert warned.call_count == 0 From 46f4cd55967019692c2e76fe620b7284fdb6adf4 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:15:55 +0800 Subject: [PATCH 10/21] Split six files back under the line limit Each was one module holding several, and each split along a seam that already existed rather than by line count: * main_widget.py keeps registration and layout; the screenshot, image detection, recording and script tabs move to their own modules. * webrtc_dialogs.py sheds known-host storage and verification. * remote_desktop/host.py sheds viewer access control, host capture and the per-client handler. * webrtc_host.py sheds authentication and media negotiation. * mcp_server/server.py sheds the wire protocol and client-initiated requests; http_transport takes _notification_message from _protocol. * flow_control.py sheds the data-producing commands (ocr / pdf / shell / sql / otp to var, assert_duration). No behaviour change: the moves are re-exported where callers expect them, and the tests follow the symbols to their new homes. --- je_auto_control/gui/_image_detect_tab.py | 106 +++ je_auto_control/gui/_record_tab.py | 101 +++ je_auto_control/gui/_screenshot_tab.py | 127 +++ je_auto_control/gui/_script_tab.py | 105 +++ je_auto_control/gui/main_widget.py | 376 +-------- .../gui/remote_desktop/_helpers.py | 41 + .../gui/remote_desktop/webrtc_dialogs.py | 373 +-------- .../gui/remote_desktop/webrtc_known_hosts.py | 340 ++++++++ .../utils/executor/flow_control.py | 239 +----- .../utils/executor/flow_data_commands.py | 248 ++++++ .../utils/mcp_server/_client_requests.py | 217 ++++++ je_auto_control/utils/mcp_server/_protocol.py | 165 ++++ .../utils/mcp_server/http_transport.py | 5 +- je_auto_control/utils/mcp_server/server.py | 343 +------- .../utils/remote_desktop/__init__.py | 7 +- je_auto_control/utils/remote_desktop/host.py | 731 +----------------- .../utils/remote_desktop/host_access.py | 105 +++ .../utils/remote_desktop/host_capture.py | 280 +++++++ .../utils/remote_desktop/host_client.py | 406 ++++++++++ .../utils/remote_desktop/webrtc_host.py | 326 +------- .../utils/remote_desktop/webrtc_host_auth.py | 195 +++++ .../utils/remote_desktop/webrtc_host_media.py | 172 +++++ .../headless/test_flow_extensions.py | 5 +- .../headless/test_flow_var_commands.py | 6 +- test/unit_test/headless/test_ocr_to_var.py | 2 +- test/unit_test/headless/test_pdf.py | 2 +- .../headless/test_r3_gui_slot_exceptions.py | 10 +- test/unit_test/headless/test_r3_rdusb_host.py | 2 +- .../test_remote_desktop_ip_allowlist.py | 5 +- .../test_remote_desktop_pending_viewer.py | 5 +- .../headless/test_remote_desktop_resume.py | 5 +- test/unit_test/headless/test_shell_to_var.py | 2 +- test/unit_test/headless/test_sql_steps.py | 2 +- .../headless/test_unattended_reliability.py | 2 +- 34 files changed, 2704 insertions(+), 2352 deletions(-) create mode 100644 je_auto_control/gui/_image_detect_tab.py create mode 100644 je_auto_control/gui/_record_tab.py create mode 100644 je_auto_control/gui/_screenshot_tab.py create mode 100644 je_auto_control/gui/_script_tab.py create mode 100644 je_auto_control/gui/remote_desktop/webrtc_known_hosts.py create mode 100644 je_auto_control/utils/executor/flow_data_commands.py create mode 100644 je_auto_control/utils/mcp_server/_client_requests.py create mode 100644 je_auto_control/utils/mcp_server/_protocol.py create mode 100644 je_auto_control/utils/remote_desktop/host_access.py create mode 100644 je_auto_control/utils/remote_desktop/host_capture.py create mode 100644 je_auto_control/utils/remote_desktop/host_client.py create mode 100644 je_auto_control/utils/remote_desktop/webrtc_host_auth.py create mode 100644 je_auto_control/utils/remote_desktop/webrtc_host_media.py diff --git a/je_auto_control/gui/_image_detect_tab.py b/je_auto_control/gui/_image_detect_tab.py new file mode 100644 index 00000000..983ae4e3 --- /dev/null +++ b/je_auto_control/gui/_image_detect_tab.py @@ -0,0 +1,106 @@ +"""Image-detection tab builder (extracted mixin).""" +from PySide6.QtGui import QDoubleValidator +from PySide6.QtWidgets import ( + QCheckBox, QFileDialog, QGridLayout, QLabel, QLineEdit, QMessageBox, + QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.selector import crop_template_to_file +from je_auto_control.wrapper.auto_control_image import ( + locate_all_image, locate_image_center, locate_and_click, +) + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class ImageDetectTabMixin: + """Provides the image-detection tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``). + ``_locate_click`` reuses the auto-click tab's ``mouse_button_combo`` when + the host also mixes in :class:`AutoClickTabMixin`, falling back to the + left button otherwise. + """ + + def _build_image_detect_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Detection inputs; locate/crop commands run from the Actions menu. + grid = QGridLayout() + grid.addWidget(self._tr(QLabel(), "template_image"), 0, 0) + self.img_path_input = QLineEdit() + grid.addWidget(self.img_path_input, 0, 1) + + grid.addWidget(self._tr(QLabel(), "threshold_label"), 1, 0) + self.threshold_input = QLineEdit("0.8") + self.threshold_input.setValidator(QDoubleValidator(0.0, 1.0, 2)) + grid.addWidget(self.threshold_input, 1, 1) + self.draw_check = self._tr(QCheckBox(), "draw_image_check") + grid.addWidget(self.draw_check, 1, 2) + + layout.addLayout(grid) + + layout.addWidget(self._tr(QLabel(), "detection_result")) + self.detect_result_text = QTextEdit() + self.detect_result_text.setReadOnly(True) + layout.addWidget(self.detect_result_text) + tab.setLayout(layout) + return tab + + def _browse_img(self): + path, _ = QFileDialog.getOpenFileName(self, _t("template_image"), "", "Images (*.png *.jpg *.bmp);;All (*)") + if path: + self.img_path_input.setText(path) + + def _crop_template(self): + save_path, _ = QFileDialog.getSaveFileName( + self, _t("crop_template"), "", "PNG (*.png)" + ) + if not save_path: + return + try: + region = crop_template_to_file(save_path, self) + if region is None: + return + self.img_path_input.setText(save_path) + self.detect_result_text.setText(f"Template saved: {save_path} region={region}") + except (OSError, ValueError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _get_detect_params(self): + path = self.img_path_input.text() + if not path: + raise ValueError("Template image path is empty") + threshold = float(self.threshold_input.text() or "0.8") + draw = self.draw_check.isChecked() + return path, threshold, draw + + def _locate_image(self): + try: + path, th, draw = self._get_detect_params() + result = locate_image_center(path, th, draw) + self.detect_result_text.setText(f"Center: {result}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.detect_result_text.setText(f"Error: {error}") + + def _locate_all(self): + try: + path, th, draw = self._get_detect_params() + result = locate_all_image(path, th, draw) + self.detect_result_text.setText(f"Found {len(result)} matches:\n{result}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.detect_result_text.setText(f"Error: {error}") + + def _locate_click(self): + try: + path, th, draw = self._get_detect_params() + btn = self.mouse_button_combo.currentText() if hasattr(self, "mouse_button_combo") else "mouse_left" + result = locate_and_click(path, btn, th, draw) + self.detect_result_text.setText(f"Clicked at: {result}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.detect_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/_record_tab.py b/je_auto_control/gui/_record_tab.py new file mode 100644 index 00000000..1e468c9e --- /dev/null +++ b/je_auto_control/gui/_record_tab.py @@ -0,0 +1,101 @@ +"""Record / playback tab builder (extracted mixin).""" +import json + +from PySide6.QtWidgets import ( + QFileDialog, QLabel, QMessageBox, QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.executor.action_executor import execute_action +from je_auto_control.utils.json.json_file import read_action_json, write_action_json +from je_auto_control.wrapper.auto_control_record import record, stop_record + +_JSON_FILE_FILTER = "JSON (*.json)" + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class RecordTabMixin: + """Provides the record/playback tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``, + ``self._translate(...)``) and a ``self._record_data`` list holding the + last recording. + """ + + def _build_record_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Record/playback/save/load all run from the Actions menu. + self._record_status_key = "record_idle" + self.record_status_label = QLabel() + self._apply_record_status_label() + layout.addWidget(self.record_status_label) + + layout.addWidget(self._tr(QLabel(), "record_list_label")) + self.record_list_text = QTextEdit() + self.record_list_text.setReadOnly(True) + layout.addWidget(self.record_list_text) + tab.setLayout(layout) + return tab + + def _apply_record_status_label(self) -> None: + if hasattr(self, "record_status_label"): + self.record_status_label.setText( + self._translate("record_status") + " " + + self._translate(self._record_status_key), + ) + + def _record_retranslate(self) -> None: + self._apply_record_status_label() + + def _start_record(self): + try: + record() + self._record_status_key = "record_recording" + self._apply_record_status_label() + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _stop_record(self): + try: + self._record_data = stop_record() or [] + self._record_status_key = "record_idle" + self._apply_record_status_label() + self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _playback_record(self): + try: + if not self._record_data: + QMessageBox.warning(self, "Warning", "No recorded data") + return + execute_action(self._record_data) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _save_record(self): + try: + if not self._record_data: + QMessageBox.warning(self, "Warning", "No recorded data") + return + path, _ = QFileDialog.getSaveFileName(self, _t("save_record"), "", _JSON_FILE_FILTER) + if path: + write_action_json(path, self._record_data) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _load_record(self): + try: + path, _ = QFileDialog.getOpenFileName(self, _t("load_record"), "", _JSON_FILE_FILTER) + if path: + self._record_data = read_action_json(path) + self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) diff --git a/je_auto_control/gui/_screenshot_tab.py b/je_auto_control/gui/_screenshot_tab.py new file mode 100644 index 00000000..a86c73d0 --- /dev/null +++ b/je_auto_control/gui/_screenshot_tab.py @@ -0,0 +1,127 @@ +"""Screenshot / pixel-probe tab builder (extracted mixin).""" +from PySide6.QtGui import QIntValidator +from PySide6.QtWidgets import ( + QFileDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QLineEdit, + QMessageBox, QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.selector import open_region_selector +from je_auto_control.wrapper.auto_control_screen import screen_size, screenshot, get_pixel + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class ScreenshotTabMixin: + """Provides the screenshot tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``, + ``self._translate(...)``) so every label registers for live language + switching. + """ + + def _build_screenshot_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Screen size (read via Actions menu -> Get Screen Size) + size_group = self._tr(QGroupBox(), "screen_size_label") + sg = QHBoxLayout() + self.screen_size_label = QLabel("--") + sg.addWidget(self.screen_size_label) + size_group.setLayout(sg) + layout.addWidget(size_group) + + # Screenshot inputs; capture runs from the Actions menu. + ss_group = self._tr(QGroupBox(), "take_screenshot") + ss_grid = QGridLayout() + ss_grid.addWidget(self._tr(QLabel(), "file_path_label"), 0, 0) + self.ss_path_input = QLineEdit() + ss_grid.addWidget(self.ss_path_input, 0, 1) + + ss_grid.addWidget(self._tr(QLabel(), "region_label"), 1, 0) + self.ss_region_input = QLineEdit() + self.ss_region_input.setPlaceholderText("0, 0, 800, 600") + ss_grid.addWidget(self.ss_region_input, 1, 1) + ss_group.setLayout(ss_grid) + layout.addWidget(ss_group) + + # Pixel probe inputs; lookup runs from the Actions menu. + px_group = self._tr(QGroupBox(), "get_pixel_label") + px_grid = QGridLayout() + px_grid.addWidget(self._tr(QLabel(), "pixel_x"), 0, 0) + self.pixel_x_input = QLineEdit("0") + self.pixel_x_input.setValidator(QIntValidator()) + px_grid.addWidget(self.pixel_x_input, 0, 1) + px_grid.addWidget(self._tr(QLabel(), "pixel_y"), 0, 2) + self.pixel_y_input = QLineEdit("0") + self.pixel_y_input.setValidator(QIntValidator()) + px_grid.addWidget(self.pixel_y_input, 0, 3) + self.pixel_result_label = QLabel() + self._pixel_result_suffix = " --" + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) + px_grid.addWidget(self.pixel_result_label, 1, 0, 1, 4) + px_group.setLayout(px_grid) + layout.addWidget(px_group) + + self.ss_result_text = QTextEdit() + self.ss_result_text.setReadOnly(True) + self.ss_result_text.setMaximumHeight(100) + layout.addWidget(self.ss_result_text) + layout.addStretch() + tab.setLayout(layout) + return tab + + def _get_screen_size(self): + try: + w, h = screen_size() + self.screen_size_label.setText(f"{w} x {h}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _browse_ss_path(self): + path, _ = QFileDialog.getSaveFileName(self, _t("save_screenshot"), "", "PNG (*.png);;All (*)") + if path: + self.ss_path_input.setText(path) + + def _pick_ss_region(self): + region = open_region_selector(self) + if region is None: + return + x, y, w, h = region + self.ss_region_input.setText(f"{x}, {y}, {x + w}, {y + h}") + + def _take_screenshot(self): + try: + path = self.ss_path_input.text() or None + region_text = self.ss_region_input.text().strip() + region = None + if region_text: + region = [int(x.strip()) for x in region_text.split(",")] + screenshot(file_path=path, screen_region=region) + self.ss_result_text.setText(f"Screenshot saved: {path or '(not saved)'}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.ss_result_text.setText(f"Error: {error}") + + def _get_pixel_color(self): + try: + x = int(self.pixel_x_input.text()) + y = int(self.pixel_y_input.text()) + color = get_pixel(x, y) + self._pixel_result_suffix = f" {color}" + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.pixel_result_label.setText(f"Error: {error}") + + def _screenshot_retranslate(self) -> None: + if hasattr(self, "pixel_result_label"): + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) diff --git a/je_auto_control/gui/_script_tab.py b/je_auto_control/gui/_script_tab.py new file mode 100644 index 00000000..d9d7fa10 --- /dev/null +++ b/je_auto_control/gui/_script_tab.py @@ -0,0 +1,105 @@ +"""Script-executor tab builder (extracted mixin).""" +import json + +from PySide6.QtWidgets import ( + QFileDialog, QHBoxLayout, QLabel, QLineEdit, QTextEdit, QVBoxLayout, + QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.executor.action_executor import execute_action, execute_files +from je_auto_control.utils.file_process.get_dir_file_list import get_dir_files_as_list +from je_auto_control.utils.json.json_file import read_action_json + +_JSON_FILE_FILTER = "JSON (*.json)" + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class ScriptTabMixin: + """Provides the script-executor tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``). + """ + + def _build_script_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Load/execute commands run from the Actions menu; the tab keeps + # only the path inputs, the editor, and the result view. + file_h = QHBoxLayout() + file_h.addWidget(self._tr(QLabel(), "file_path_label")) + self.script_path_input = QLineEdit() + file_h.addWidget(self.script_path_input) + layout.addLayout(file_h) + + dir_h = QHBoxLayout() + dir_h.addWidget(self._tr(QLabel(), "execute_dir_label")) + self.script_dir_input = QLineEdit() + dir_h.addWidget(self.script_dir_input) + layout.addLayout(dir_h) + + layout.addWidget(self._tr(QLabel(), "script_content")) + self.script_editor = QTextEdit() + self.script_editor.setPlaceholderText('[["AC_type_keyboard", {"keycode": "a"}]]') + layout.addWidget(self.script_editor) + + layout.addWidget(self._tr(QLabel(), "execution_result")) + self.script_result_text = QTextEdit() + self.script_result_text.setReadOnly(True) + layout.addWidget(self.script_result_text) + tab.setLayout(layout) + return tab + + def _browse_script(self): + path, _ = QFileDialog.getOpenFileName(self, _t("load_script"), "", _JSON_FILE_FILTER) + if path: + self.script_path_input.setText(path) + try: + data = read_action_json(path) + self.script_editor.setText(json.dumps(data, indent=2, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error loading: {error}") + + def _execute_script(self): + try: + path = self.script_path_input.text() + if not path: + return + data = read_action_json(path) + result = execute_action(data) + self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error: {error}") + + def _browse_script_dir(self): + path = QFileDialog.getExistingDirectory(self, _t("execute_dir_label")) + if path: + self.script_dir_input.setText(path) + + def _execute_dir(self): + try: + path = self.script_dir_input.text() + if not path: + return + files = get_dir_files_as_list(path) + result = execute_files(files) + self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error: {error}") + + def _execute_manual_script(self): + try: + text = self.script_editor.toPlainText().strip() + if not text: + return + data = json.loads(text) + result = execute_action(data) + self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/main_widget.py b/je_auto_control/gui/main_widget.py index 6483303c..b84a7e25 100644 --- a/je_auto_control/gui/main_widget.py +++ b/je_auto_control/gui/main_widget.py @@ -3,15 +3,17 @@ from typing import Optional from PySide6.QtCore import QTimer, Signal, QObject -from PySide6.QtGui import QIntValidator, QDoubleValidator, QKeyEvent, Qt +from PySide6.QtGui import QKeyEvent, Qt from PySide6.QtWidgets import ( - QWidget, QLineEdit, QVBoxLayout, QLabel, - QGridLayout, QHBoxLayout, QMessageBox, - QTabWidget, QTextEdit, QFileDialog, QCheckBox, QGroupBox + QWidget, QVBoxLayout, QLabel, QTabWidget, ) from je_auto_control.gui._auto_click_tab import AutoClickTabMixin from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui._image_detect_tab import ImageDetectTabMixin +from je_auto_control.gui._record_tab import RecordTabMixin +from je_auto_control.gui._screenshot_tab import ScreenshotTabMixin +from je_auto_control.gui._script_tab import ScriptTabMixin from je_auto_control.gui.accessibility_tab import AccessibilityTab from je_auto_control.gui.assertions_tab import AssertionsTab from je_auto_control.gui.data_source_tab import DataSourceTab @@ -58,7 +60,6 @@ from je_auto_control.gui.flow_editor import FlowEditorTab from je_auto_control.gui.script_builder import ScriptBuilderTab from je_auto_control.gui.self_healing_tab import SelfHealingTab -from je_auto_control.gui.selector import crop_template_to_file, open_region_selector from je_auto_control.gui.triggers_tab import TriggersTab from je_auto_control.gui.webhooks_tab import WebhooksTab from je_auto_control.gui.email_triggers_tab import EmailTriggersTab @@ -66,21 +67,7 @@ from je_auto_control.gui.vlm_tab import VLMTab from je_auto_control.gui.webrunner_tab import WebRunnerTab from je_auto_control.gui.window_tab import WindowManagerTab -from je_auto_control.wrapper.auto_control_screen import screen_size, screenshot, get_pixel -from je_auto_control.wrapper.auto_control_image import locate_all_image, locate_image_center, locate_and_click -from je_auto_control.wrapper.auto_control_record import record, stop_record -from je_auto_control.utils.exception.exceptions import AutoControlException -from je_auto_control.utils.executor.action_executor import execute_action, execute_files -from je_auto_control.utils.json.json_file import read_action_json, write_action_json -from je_auto_control.utils.file_process.get_dir_file_list import get_dir_files_as_list - - -_JSON_FILE_FILTER = "JSON (*.json)" - - -def _t(key: str) -> str: - """language_wrapper shorthand""" - return language_wrapper.translate(key, key) +from je_auto_control.utils.json.json_file import read_action_json class _WorkerSignals(QObject): @@ -102,7 +89,9 @@ class _TabEntry: # Main Widget # ============================================================================= class AutoControlGUIWidget( - TranslatableMixin, AutoClickTabMixin, ReportTabMixin, QWidget, + TranslatableMixin, AutoClickTabMixin, ScreenshotTabMixin, + ImageDetectTabMixin, RecordTabMixin, ScriptTabMixin, ReportTabMixin, + QWidget, ): """Owns the QTabWidget and exposes show/hide/list APIs for the menu bar.""" @@ -424,351 +413,6 @@ def open_script_file(self, path: str) -> None: if entry is not None: self.tabs.setCurrentWidget(entry.widget) - # ========================================================================= - # Tab 2: Screenshot - # ========================================================================= - def _build_screenshot_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Screen size (read via Actions menu -> Get Screen Size) - size_group = self._tr(QGroupBox(), "screen_size_label") - sg = QHBoxLayout() - self.screen_size_label = QLabel("--") - sg.addWidget(self.screen_size_label) - size_group.setLayout(sg) - layout.addWidget(size_group) - - # Screenshot inputs; capture runs from the Actions menu. - ss_group = self._tr(QGroupBox(), "take_screenshot") - ss_grid = QGridLayout() - ss_grid.addWidget(self._tr(QLabel(), "file_path_label"), 0, 0) - self.ss_path_input = QLineEdit() - ss_grid.addWidget(self.ss_path_input, 0, 1) - - ss_grid.addWidget(self._tr(QLabel(), "region_label"), 1, 0) - self.ss_region_input = QLineEdit() - self.ss_region_input.setPlaceholderText("0, 0, 800, 600") - ss_grid.addWidget(self.ss_region_input, 1, 1) - ss_group.setLayout(ss_grid) - layout.addWidget(ss_group) - - # Pixel probe inputs; lookup runs from the Actions menu. - px_group = self._tr(QGroupBox(), "get_pixel_label") - px_grid = QGridLayout() - px_grid.addWidget(self._tr(QLabel(), "pixel_x"), 0, 0) - self.pixel_x_input = QLineEdit("0") - self.pixel_x_input.setValidator(QIntValidator()) - px_grid.addWidget(self.pixel_x_input, 0, 1) - px_grid.addWidget(self._tr(QLabel(), "pixel_y"), 0, 2) - self.pixel_y_input = QLineEdit("0") - self.pixel_y_input.setValidator(QIntValidator()) - px_grid.addWidget(self.pixel_y_input, 0, 3) - self.pixel_result_label = QLabel() - self._pixel_result_suffix = " --" - self.pixel_result_label.setText( - self._translate("pixel_result") + self._pixel_result_suffix, - ) - px_grid.addWidget(self.pixel_result_label, 1, 0, 1, 4) - px_group.setLayout(px_grid) - layout.addWidget(px_group) - - self.ss_result_text = QTextEdit() - self.ss_result_text.setReadOnly(True) - self.ss_result_text.setMaximumHeight(100) - layout.addWidget(self.ss_result_text) - layout.addStretch() - tab.setLayout(layout) - return tab - - def _get_screen_size(self): - try: - w, h = screen_size() - self.screen_size_label.setText(f"{w} x {h}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _browse_ss_path(self): - path, _ = QFileDialog.getSaveFileName(self, _t("save_screenshot"), "", "PNG (*.png);;All (*)") - if path: - self.ss_path_input.setText(path) - - def _pick_ss_region(self): - region = open_region_selector(self) - if region is None: - return - x, y, w, h = region - self.ss_region_input.setText(f"{x}, {y}, {x + w}, {y + h}") - - def _take_screenshot(self): - try: - path = self.ss_path_input.text() or None - region_text = self.ss_region_input.text().strip() - region = None - if region_text: - region = [int(x.strip()) for x in region_text.split(",")] - screenshot(file_path=path, screen_region=region) - self.ss_result_text.setText(f"Screenshot saved: {path or '(not saved)'}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.ss_result_text.setText(f"Error: {error}") - - def _get_pixel_color(self): - try: - x = int(self.pixel_x_input.text()) - y = int(self.pixel_y_input.text()) - color = get_pixel(x, y) - self._pixel_result_suffix = f" {color}" - self.pixel_result_label.setText( - self._translate("pixel_result") + self._pixel_result_suffix, - ) - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.pixel_result_label.setText(f"Error: {error}") - - def _screenshot_retranslate(self) -> None: - if hasattr(self, "pixel_result_label"): - self.pixel_result_label.setText( - self._translate("pixel_result") + self._pixel_result_suffix, - ) - - # ========================================================================= - # Tab 3: Image Detection - # ========================================================================= - def _build_image_detect_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Detection inputs; locate/crop commands run from the Actions menu. - grid = QGridLayout() - grid.addWidget(self._tr(QLabel(), "template_image"), 0, 0) - self.img_path_input = QLineEdit() - grid.addWidget(self.img_path_input, 0, 1) - - grid.addWidget(self._tr(QLabel(), "threshold_label"), 1, 0) - self.threshold_input = QLineEdit("0.8") - self.threshold_input.setValidator(QDoubleValidator(0.0, 1.0, 2)) - grid.addWidget(self.threshold_input, 1, 1) - self.draw_check = self._tr(QCheckBox(), "draw_image_check") - grid.addWidget(self.draw_check, 1, 2) - - layout.addLayout(grid) - - layout.addWidget(self._tr(QLabel(), "detection_result")) - self.detect_result_text = QTextEdit() - self.detect_result_text.setReadOnly(True) - layout.addWidget(self.detect_result_text) - tab.setLayout(layout) - return tab - - def _browse_img(self): - path, _ = QFileDialog.getOpenFileName(self, _t("template_image"), "", "Images (*.png *.jpg *.bmp);;All (*)") - if path: - self.img_path_input.setText(path) - - def _crop_template(self): - save_path, _ = QFileDialog.getSaveFileName( - self, _t("crop_template"), "", "PNG (*.png)" - ) - if not save_path: - return - try: - region = crop_template_to_file(save_path, self) - if region is None: - return - self.img_path_input.setText(save_path) - self.detect_result_text.setText(f"Template saved: {save_path} region={region}") - except (OSError, ValueError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _get_detect_params(self): - path = self.img_path_input.text() - if not path: - raise ValueError("Template image path is empty") - threshold = float(self.threshold_input.text() or "0.8") - draw = self.draw_check.isChecked() - return path, threshold, draw - - def _locate_image(self): - try: - path, th, draw = self._get_detect_params() - result = locate_image_center(path, th, draw) - self.detect_result_text.setText(f"Center: {result}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.detect_result_text.setText(f"Error: {error}") - - def _locate_all(self): - try: - path, th, draw = self._get_detect_params() - result = locate_all_image(path, th, draw) - self.detect_result_text.setText(f"Found {len(result)} matches:\n{result}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.detect_result_text.setText(f"Error: {error}") - - def _locate_click(self): - try: - path, th, draw = self._get_detect_params() - btn = self.mouse_button_combo.currentText() if hasattr(self, "mouse_button_combo") else "mouse_left" - result = locate_and_click(path, btn, th, draw) - self.detect_result_text.setText(f"Clicked at: {result}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.detect_result_text.setText(f"Error: {error}") - - # ========================================================================= - # Tab 4: Record / Playback - # ========================================================================= - def _build_record_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Record/playback/save/load all run from the Actions menu. - self._record_status_key = "record_idle" - self.record_status_label = QLabel() - self._apply_record_status_label() - layout.addWidget(self.record_status_label) - - layout.addWidget(self._tr(QLabel(), "record_list_label")) - self.record_list_text = QTextEdit() - self.record_list_text.setReadOnly(True) - layout.addWidget(self.record_list_text) - tab.setLayout(layout) - return tab - - def _apply_record_status_label(self) -> None: - if hasattr(self, "record_status_label"): - self.record_status_label.setText( - self._translate("record_status") + " " - + self._translate(self._record_status_key), - ) - - def _record_retranslate(self) -> None: - self._apply_record_status_label() - - def _start_record(self): - try: - record() - self._record_status_key = "record_recording" - self._apply_record_status_label() - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _stop_record(self): - try: - self._record_data = stop_record() or [] - self._record_status_key = "record_idle" - self._apply_record_status_label() - self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _playback_record(self): - try: - if not self._record_data: - QMessageBox.warning(self, "Warning", "No recorded data") - return - execute_action(self._record_data) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _save_record(self): - try: - if not self._record_data: - QMessageBox.warning(self, "Warning", "No recorded data") - return - path, _ = QFileDialog.getSaveFileName(self, _t("save_record"), "", _JSON_FILE_FILTER) - if path: - write_action_json(path, self._record_data) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _load_record(self): - try: - path, _ = QFileDialog.getOpenFileName(self, _t("load_record"), "", _JSON_FILE_FILTER) - if path: - self._record_data = read_action_json(path) - self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - # ========================================================================= - # Tab 5: Script Executor - # ========================================================================= - def _build_script_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Load/execute commands run from the Actions menu; the tab keeps - # only the path inputs, the editor, and the result view. - file_h = QHBoxLayout() - file_h.addWidget(self._tr(QLabel(), "file_path_label")) - self.script_path_input = QLineEdit() - file_h.addWidget(self.script_path_input) - layout.addLayout(file_h) - - dir_h = QHBoxLayout() - dir_h.addWidget(self._tr(QLabel(), "execute_dir_label")) - self.script_dir_input = QLineEdit() - dir_h.addWidget(self.script_dir_input) - layout.addLayout(dir_h) - - layout.addWidget(self._tr(QLabel(), "script_content")) - self.script_editor = QTextEdit() - self.script_editor.setPlaceholderText('[["AC_type_keyboard", {"keycode": "a"}]]') - layout.addWidget(self.script_editor) - - layout.addWidget(self._tr(QLabel(), "execution_result")) - self.script_result_text = QTextEdit() - self.script_result_text.setReadOnly(True) - layout.addWidget(self.script_result_text) - tab.setLayout(layout) - return tab - - def _browse_script(self): - path, _ = QFileDialog.getOpenFileName(self, _t("load_script"), "", _JSON_FILE_FILTER) - if path: - self.script_path_input.setText(path) - try: - data = read_action_json(path) - self.script_editor.setText(json.dumps(data, indent=2, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error loading: {error}") - - def _execute_script(self): - try: - path = self.script_path_input.text() - if not path: - return - data = read_action_json(path) - result = execute_action(data) - self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error: {error}") - - def _browse_script_dir(self): - path = QFileDialog.getExistingDirectory(self, _t("execute_dir_label")) - if path: - self.script_dir_input.setText(path) - - def _execute_dir(self): - try: - path = self.script_dir_input.text() - if not path: - return - files = get_dir_files_as_list(path) - result = execute_files(files) - self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error: {error}") - - def _execute_manual_script(self): - try: - text = self.script_editor.toPlainText().strip() - if not text: - return - data = json.loads(text) - result = execute_action(data) - self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error: {error}") - # ========================================================================= # Global keyboard shortcut: Ctrl+4 to stop # ========================================================================= diff --git a/je_auto_control/gui/remote_desktop/_helpers.py b/je_auto_control/gui/remote_desktop/_helpers.py index bb16c2d8..08670268 100644 --- a/je_auto_control/gui/remote_desktop/_helpers.py +++ b/je_auto_control/gui/remote_desktop/_helpers.py @@ -146,3 +146,44 @@ def body(self) -> QWidget: def set_body_layout(self, layout) -> None: self._body.setLayout(layout) + + +def _short_fp(fp: Optional[str]) -> str: + if not fp: + return "" + return fp[:16] + ("..." if len(fp) > 16 else "") + + +def _iso_to_epoch(value: Optional[str]) -> float: + """Parse ISO; return Unix epoch (or 0 if invalid).""" + if not value: + return 0.0 + from datetime import datetime + try: + return datetime.fromisoformat(value).timestamp() + except (TypeError, ValueError): + return 0.0 + + +def _format_short_time(value: Optional[str]) -> str: + if not value: + return "" + from datetime import datetime + try: + dt = datetime.fromisoformat(value) + except (TypeError, ValueError): + return "" + return dt.astimezone().strftime("%m-%d %H:%M") + + +def _format_last_seen(value: Optional[str]) -> str: + if not value: + return "" + # Stored as ISO 8601 (UTC); render as local-readable "YYYY-MM-DD HH:MM" + from datetime import datetime + try: + dt = datetime.fromisoformat(value) + except (TypeError, ValueError): + return value + return dt.astimezone().strftime("%Y-%m-%d %H:%M") + diff --git a/je_auto_control/gui/remote_desktop/webrtc_dialogs.py b/je_auto_control/gui/remote_desktop/webrtc_dialogs.py index 68d3ff68..9905a843 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_dialogs.py +++ b/je_auto_control/gui/remote_desktop/webrtc_dialogs.py @@ -1,22 +1,28 @@ """Custom dialogs / list widgets used by the WebRTC GUI panels. Kept out of ``webrtc_panel.py`` so that file stays focused on layout -construction and signal wiring. +construction and signal wiring. The known-hosts browser lives in +``webrtc_known_hosts`` and is re-exported here, so callers keep importing +every dialog from one place. """ from __future__ import annotations from typing import Optional from PySide6.QtCore import Qt, Signal -from PySide6.QtGui import QColor from PySide6.QtWidgets import ( - QAbstractItemView, QDialog, QFileDialog, QFormLayout, QHBoxLayout, + QAbstractItemView, QDialog, QHBoxLayout, QHeaderView, QLabel, QLineEdit, QListWidget, QListWidgetItem, QMenu, - QMessageBox, QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, + QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) -from je_auto_control.gui.remote_desktop._helpers import _t +from je_auto_control.gui.remote_desktop._helpers import ( + _format_short_time, _iso_to_epoch, _t, +) +from je_auto_control.gui.remote_desktop.webrtc_known_hosts import ( + KnownHostsDialog, +) class PendingViewerDialog(QDialog): @@ -298,363 +304,6 @@ def _show_context_menu(self, position) -> None: self.copy_name_requested.emit(names[0]) -class KnownHostsDialog(QDialog): - """Browse + forget the persistent KnownHosts (TOFU app + DTLS fingerprints).""" - - def __init__(self, known_hosts, parent: Optional[QWidget] = None) -> None: - super().__init__(parent) - self._known = known_hosts - self.setWindowTitle(_t("rd_webrtc_known_hosts_title")) - self.setMinimumSize(720, 360) - layout = QVBoxLayout(self) - self._table = QTableWidget(0, 4) - self._table.setHorizontalHeaderLabels([ - _t("rd_webrtc_kh_col_host"), - _t("rd_webrtc_kh_col_app_fp"), - _t("rd_webrtc_kh_col_dtls_fp"), - _t("rd_webrtc_kh_col_last_seen"), - ]) - self._table.horizontalHeader().setSectionResizeMode( - 1, QHeaderView.ResizeMode.Stretch, - ) - self._table.horizontalHeader().setSectionResizeMode( - 2, QHeaderView.ResizeMode.Stretch, - ) - self._table.setSelectionBehavior( - QAbstractItemView.SelectionBehavior.SelectRows, - ) - self._table.setSelectionMode( - QAbstractItemView.SelectionMode.ExtendedSelection, - ) - self._table.setEditTriggers( - QAbstractItemView.EditTrigger.NoEditTriggers, - ) - layout.addWidget(self._table) - button_row = QHBoxLayout() - add_btn = QPushButton(_t("rd_webrtc_kh_add")) - add_btn.clicked.connect(self._on_add_manual) - button_row.addWidget(add_btn) - import_btn = QPushButton(_t("rd_webrtc_kh_import")) - import_btn.clicked.connect(self._on_import) - button_row.addWidget(import_btn) - export_btn = QPushButton(_t("rd_webrtc_kh_export")) - export_btn.clicked.connect(self._on_export) - button_row.addWidget(export_btn) - copy_app_btn = QPushButton(_t("rd_webrtc_kh_copy_app")) - copy_app_btn.clicked.connect(self._on_copy_app) - button_row.addWidget(copy_app_btn) - copy_dtls_btn = QPushButton(_t("rd_webrtc_kh_copy_dtls")) - copy_dtls_btn.clicked.connect(self._on_copy_dtls) - button_row.addWidget(copy_dtls_btn) - forget_btn = QPushButton(_t("rd_webrtc_kh_forget")) - forget_btn.clicked.connect(self._on_forget) - button_row.addWidget(forget_btn) - forget_stale_btn = QPushButton(_t("rd_webrtc_kh_forget_stale")) - forget_stale_btn.clicked.connect(self._on_forget_stale) - button_row.addWidget(forget_stale_btn) - clear_btn = QPushButton(_t("rd_webrtc_kh_clear_all")) - clear_btn.clicked.connect(self._on_clear_all) - button_row.addWidget(clear_btn) - button_row.addStretch() - close_btn = QPushButton(_t("rd_webrtc_kh_close")) - close_btn.clicked.connect(self.accept) - button_row.addWidget(close_btn) - layout.addLayout(button_row) - self._refresh() - - def _refresh(self) -> None: - from datetime import datetime, timedelta, timezone - stale_after = timedelta(days=90) - now = datetime.now(timezone.utc) - stale_color = QColor("#888") - entries = self._known.list_entries() - self._table.setRowCount(len(entries)) - for row, (host_id, fps) in enumerate(sorted(entries.items())): - self._populate_row(row, host_id, fps, - now=now, stale_after=stale_after, - stale_color=stale_color) - - def _populate_row(self, row: int, host_id: str, fps: dict, *, - now, stale_after, stale_color) -> None: - items = [ - QTableWidgetItem(host_id), - QTableWidgetItem(_short_fp(fps.get("app_fp"))), - QTableWidgetItem(_short_fp(fps.get("dtls_fp"))), - QTableWidgetItem(_format_last_seen(fps.get("last_seen"))), - ] - if self._is_stale(fps.get("last_seen"), now=now, - stale_after=stale_after): - tip = _t("rd_webrtc_kh_stale_tip") - for it in items: - it.setForeground(stale_color) - it.setToolTip(tip) - for col, item in enumerate(items): - self._table.setItem(row, col, item) - - @staticmethod - def _is_stale(last_seen, *, now, stale_after) -> bool: - if not last_seen: - return False - from datetime import datetime - try: - dt = datetime.fromisoformat(last_seen) - except (TypeError, ValueError): - return False - return now - dt > stale_after - - def _on_forget(self) -> None: - rows = sorted( - {i.row() for i in self._table.selectedIndexes()}, reverse=True, - ) - if not rows: - return - for row in rows: - item = self._table.item(row, 0) - if item is None: - continue - self._known.forget(item.text()) - self._refresh() - - def _on_add_manual(self) -> None: - dialog = _ManualKnownHostDialog(parent=self) - if dialog.exec() != QDialog.DialogCode.Accepted: - return - host_id, app_fp, dtls_fp = dialog.values() - if not host_id: - return - if app_fp: - self._known.remember(host_id, app_fp) - if dtls_fp: - self._known.remember_dtls_fingerprint(host_id, dtls_fp) - self._refresh() - - def _on_copy_app(self) -> None: - self._copy_selected_fingerprint("app_fp") - - def _on_copy_dtls(self) -> None: - self._copy_selected_fingerprint("dtls_fp") - - def _copy_selected_fingerprint(self, key: str) -> None: - from PySide6.QtWidgets import QApplication as _QApp - row = self._table.currentRow() - if row < 0: - return - host_item = self._table.item(row, 0) - if host_item is None: - return - entries = self._known.list_entries() - fps = entries.get(host_item.text()) - if not fps: - return - value = fps.get(key) or "" - clipboard = _QApp.clipboard() - if clipboard is not None: - clipboard.setText(value) - - def _on_export(self) -> None: - import json - path, _filter = QFileDialog.getSaveFileName( - self, _t("rd_webrtc_kh_export"), "known_hosts.json", - "JSON (*.json);;All (*)", - ) - if not path: - return - try: - with open(path, "w", encoding="utf-8") as fh: - json.dump(self._known.list_entries(), fh, - indent=2, ensure_ascii=False) - except OSError as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_import(self) -> None: - data = self._prompt_import_data() - if data is None: - return - existing = self._known.list_entries() - added = 0 - skipped = 0 - for host_id, value in data.items(): - outcome = self._import_one(host_id, value, existing) - if outcome == "added": - added += 1 - elif outcome == "skipped": - skipped += 1 - QMessageBox.information( - self, "WebRTC", - _t("rd_webrtc_kh_import_done").format(added=added, skipped=skipped), - ) - self._refresh() - - def _prompt_import_data(self): - import json - path, _filter = QFileDialog.getOpenFileName( - self, _t("rd_webrtc_kh_import"), "", "JSON (*.json);;All (*)", - ) - if not path: - return None - try: - with open(path, "r", encoding="utf-8") as fh: - data = json.load(fh) - except (OSError, json.JSONDecodeError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - return None - if not isinstance(data, dict): - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_kh_import_bad"), - ) - return None - return data - - def _import_one(self, host_id, value, existing) -> str: - """Return ``"added"``, ``"skipped"``, or ``"ignored"`` per entry.""" - if not isinstance(host_id, str): - return "ignored" - app_fp, dtls_fp = self._extract_fingerprints(value) - if app_fp is None and dtls_fp is None: - return "ignored" - if host_id in existing and not self._confirm_overwrite(host_id): - return "skipped" - if isinstance(app_fp, str) and app_fp: - self._known.remember(host_id, app_fp) - if isinstance(dtls_fp, str) and dtls_fp: - self._known.remember_dtls_fingerprint(host_id, dtls_fp) - return "added" - - @staticmethod - def _extract_fingerprints(value): - if isinstance(value, str): - return value, None - if isinstance(value, dict): - return value.get("app_fp"), value.get("dtls_fp") - return None, None - - def _confirm_overwrite(self, host_id: str) -> bool: - result = QMessageBox.question( - self, "WebRTC", - _t("rd_webrtc_kh_import_overwrite").format(host=host_id), - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - return result == QMessageBox.StandardButton.Yes - - def _on_forget_stale(self) -> None: - from datetime import datetime, timedelta, timezone - cutoff = datetime.now(timezone.utc) - timedelta(days=90) - stale_ids = [] - for host_id, fps in self._known.list_entries().items(): - last_seen = fps.get("last_seen") - if not last_seen: - continue - try: - if datetime.fromisoformat(last_seen) < cutoff: - stale_ids.append(host_id) - except (TypeError, ValueError): - continue - if not stale_ids: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_kh_no_stale"), - ) - return - result = QMessageBox.question( - self, "WebRTC", - _t("rd_webrtc_kh_forget_stale_confirm").format(n=len(stale_ids)), - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - if result != QMessageBox.StandardButton.Yes: - return - for host_id in stale_ids: - self._known.forget(host_id) - self._refresh() - - def _on_clear_all(self) -> None: - from PySide6.QtWidgets import QMessageBox as _QMB - result = _QMB.question( - self, "WebRTC", _t("rd_webrtc_kh_clear_confirm"), - _QMB.StandardButton.Yes | _QMB.StandardButton.No, - ) - if result != _QMB.StandardButton.Yes: - return - for host_id in list(self._known.list_entries().keys()): # NOSONAR python:S7504 # forget() mutates the underlying mapping — list() is required to avoid RuntimeError - self._known.forget(host_id) - self._refresh() - - -def _short_fp(fp: Optional[str]) -> str: - if not fp: - return "" - return fp[:16] + ("..." if len(fp) > 16 else "") - - -def _iso_to_epoch(value: Optional[str]) -> float: - """Parse ISO; return Unix epoch (or 0 if invalid).""" - if not value: - return 0.0 - from datetime import datetime - try: - return datetime.fromisoformat(value).timestamp() - except (TypeError, ValueError): - return 0.0 - - -def _format_short_time(value: Optional[str]) -> str: - if not value: - return "" - from datetime import datetime - try: - dt = datetime.fromisoformat(value) - except (TypeError, ValueError): - return "" - return dt.astimezone().strftime("%m-%d %H:%M") - - -def _format_last_seen(value: Optional[str]) -> str: - if not value: - return "" - # Stored as ISO 8601 (UTC); render as local-readable "YYYY-MM-DD HH:MM" - from datetime import datetime - try: - dt = datetime.fromisoformat(value) - except (TypeError, ValueError): - return value - return dt.astimezone().strftime("%Y-%m-%d %H:%M") - - -class _ManualKnownHostDialog(QDialog): - """Tiny form dialog for pinning a host fingerprint out-of-band.""" - - def __init__(self, parent: Optional[QWidget] = None) -> None: - super().__init__(parent) - self.setWindowTitle(_t("rd_webrtc_kh_add")) - self.setMinimumWidth(420) - layout = QVBoxLayout(self) - form = QFormLayout() - self._host_edit = QLineEdit() - self._host_edit.setPlaceholderText(_t("rd_webrtc_kh_add_host_ph")) - self._app_edit = QLineEdit() - self._app_edit.setPlaceholderText(_t("rd_webrtc_kh_add_app_ph")) - self._dtls_edit = QLineEdit() - self._dtls_edit.setPlaceholderText(_t("rd_webrtc_kh_add_dtls_ph")) - form.addRow(_t("rd_webrtc_kh_col_host"), self._host_edit) - form.addRow(_t("rd_webrtc_kh_col_app_fp"), self._app_edit) - form.addRow(_t("rd_webrtc_kh_col_dtls_fp"), self._dtls_edit) - layout.addLayout(form) - button_row = QHBoxLayout() - button_row.addStretch() - ok = QPushButton(_t("rd_webrtc_kh_add")) - ok.clicked.connect(self.accept) - cancel = QPushButton(_t("rd_webrtc_kh_close")) - cancel.clicked.connect(self.reject) - button_row.addWidget(cancel) - button_row.addWidget(ok) - layout.addLayout(button_row) - - def values(self) -> tuple: - return ( - self._host_edit.text().strip(), - self._app_edit.text().strip(), - self._dtls_edit.text().strip(), - ) - - class AuditLogDialog(QDialog): """Browse the SQLite audit log with filter on event_type / host_id.""" diff --git a/je_auto_control/gui/remote_desktop/webrtc_known_hosts.py b/je_auto_control/gui/remote_desktop/webrtc_known_hosts.py new file mode 100644 index 00000000..d3eb3f72 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_known_hosts.py @@ -0,0 +1,340 @@ +"""Known-hosts browser for the WebRTC GUI panels. + +Split out of ``webrtc_dialogs.py``: the TOFU pin store gets its own table +dialog plus a small out-of-band pinning form, and neither shares state with +the viewer / address-book / audit dialogs next door. The shared fingerprint +and timestamp formatters live in ``_helpers``. +""" +from __future__ import annotations + +from typing import Optional + +from PySide6.QtGui import QColor +from PySide6.QtWidgets import ( + QAbstractItemView, QDialog, QFileDialog, QFormLayout, QHBoxLayout, + QHeaderView, QLineEdit, QMessageBox, QPushButton, QTableWidget, + QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _format_last_seen, _short_fp, _t, +) + + +class KnownHostsDialog(QDialog): + """Browse + forget the persistent KnownHosts (TOFU app + DTLS fingerprints).""" + + def __init__(self, known_hosts, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._known = known_hosts + self.setWindowTitle(_t("rd_webrtc_known_hosts_title")) + self.setMinimumSize(720, 360) + layout = QVBoxLayout(self) + self._table = QTableWidget(0, 4) + self._table.setHorizontalHeaderLabels([ + _t("rd_webrtc_kh_col_host"), + _t("rd_webrtc_kh_col_app_fp"), + _t("rd_webrtc_kh_col_dtls_fp"), + _t("rd_webrtc_kh_col_last_seen"), + ]) + self._table.horizontalHeader().setSectionResizeMode( + 1, QHeaderView.ResizeMode.Stretch, + ) + self._table.horizontalHeader().setSectionResizeMode( + 2, QHeaderView.ResizeMode.Stretch, + ) + self._table.setSelectionBehavior( + QAbstractItemView.SelectionBehavior.SelectRows, + ) + self._table.setSelectionMode( + QAbstractItemView.SelectionMode.ExtendedSelection, + ) + self._table.setEditTriggers( + QAbstractItemView.EditTrigger.NoEditTriggers, + ) + layout.addWidget(self._table) + button_row = QHBoxLayout() + add_btn = QPushButton(_t("rd_webrtc_kh_add")) + add_btn.clicked.connect(self._on_add_manual) + button_row.addWidget(add_btn) + import_btn = QPushButton(_t("rd_webrtc_kh_import")) + import_btn.clicked.connect(self._on_import) + button_row.addWidget(import_btn) + export_btn = QPushButton(_t("rd_webrtc_kh_export")) + export_btn.clicked.connect(self._on_export) + button_row.addWidget(export_btn) + copy_app_btn = QPushButton(_t("rd_webrtc_kh_copy_app")) + copy_app_btn.clicked.connect(self._on_copy_app) + button_row.addWidget(copy_app_btn) + copy_dtls_btn = QPushButton(_t("rd_webrtc_kh_copy_dtls")) + copy_dtls_btn.clicked.connect(self._on_copy_dtls) + button_row.addWidget(copy_dtls_btn) + forget_btn = QPushButton(_t("rd_webrtc_kh_forget")) + forget_btn.clicked.connect(self._on_forget) + button_row.addWidget(forget_btn) + forget_stale_btn = QPushButton(_t("rd_webrtc_kh_forget_stale")) + forget_stale_btn.clicked.connect(self._on_forget_stale) + button_row.addWidget(forget_stale_btn) + clear_btn = QPushButton(_t("rd_webrtc_kh_clear_all")) + clear_btn.clicked.connect(self._on_clear_all) + button_row.addWidget(clear_btn) + button_row.addStretch() + close_btn = QPushButton(_t("rd_webrtc_kh_close")) + close_btn.clicked.connect(self.accept) + button_row.addWidget(close_btn) + layout.addLayout(button_row) + self._refresh() + + def _refresh(self) -> None: + from datetime import datetime, timedelta, timezone + stale_after = timedelta(days=90) + now = datetime.now(timezone.utc) + stale_color = QColor("#888") + entries = self._known.list_entries() + self._table.setRowCount(len(entries)) + for row, (host_id, fps) in enumerate(sorted(entries.items())): + self._populate_row(row, host_id, fps, + now=now, stale_after=stale_after, + stale_color=stale_color) + + def _populate_row(self, row: int, host_id: str, fps: dict, *, + now, stale_after, stale_color) -> None: + items = [ + QTableWidgetItem(host_id), + QTableWidgetItem(_short_fp(fps.get("app_fp"))), + QTableWidgetItem(_short_fp(fps.get("dtls_fp"))), + QTableWidgetItem(_format_last_seen(fps.get("last_seen"))), + ] + if self._is_stale(fps.get("last_seen"), now=now, + stale_after=stale_after): + tip = _t("rd_webrtc_kh_stale_tip") + for it in items: + it.setForeground(stale_color) + it.setToolTip(tip) + for col, item in enumerate(items): + self._table.setItem(row, col, item) + + @staticmethod + def _is_stale(last_seen, *, now, stale_after) -> bool: + if not last_seen: + return False + from datetime import datetime + try: + dt = datetime.fromisoformat(last_seen) + except (TypeError, ValueError): + return False + return now - dt > stale_after + + def _on_forget(self) -> None: + rows = sorted( + {i.row() for i in self._table.selectedIndexes()}, reverse=True, + ) + if not rows: + return + for row in rows: + item = self._table.item(row, 0) + if item is None: + continue + self._known.forget(item.text()) + self._refresh() + + def _on_add_manual(self) -> None: + dialog = _ManualKnownHostDialog(parent=self) + if dialog.exec() != QDialog.DialogCode.Accepted: + return + host_id, app_fp, dtls_fp = dialog.values() + if not host_id: + return + if app_fp: + self._known.remember(host_id, app_fp) + if dtls_fp: + self._known.remember_dtls_fingerprint(host_id, dtls_fp) + self._refresh() + + def _on_copy_app(self) -> None: + self._copy_selected_fingerprint("app_fp") + + def _on_copy_dtls(self) -> None: + self._copy_selected_fingerprint("dtls_fp") + + def _copy_selected_fingerprint(self, key: str) -> None: + from PySide6.QtWidgets import QApplication as _QApp + row = self._table.currentRow() + if row < 0: + return + host_item = self._table.item(row, 0) + if host_item is None: + return + entries = self._known.list_entries() + fps = entries.get(host_item.text()) + if not fps: + return + value = fps.get(key) or "" + clipboard = _QApp.clipboard() + if clipboard is not None: + clipboard.setText(value) + + def _on_export(self) -> None: + import json + path, _filter = QFileDialog.getSaveFileName( + self, _t("rd_webrtc_kh_export"), "known_hosts.json", + "JSON (*.json);;All (*)", + ) + if not path: + return + try: + with open(path, "w", encoding="utf-8") as fh: + json.dump(self._known.list_entries(), fh, + indent=2, ensure_ascii=False) + except OSError as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_import(self) -> None: + data = self._prompt_import_data() + if data is None: + return + existing = self._known.list_entries() + added = 0 + skipped = 0 + for host_id, value in data.items(): + outcome = self._import_one(host_id, value, existing) + if outcome == "added": + added += 1 + elif outcome == "skipped": + skipped += 1 + QMessageBox.information( + self, "WebRTC", + _t("rd_webrtc_kh_import_done").format(added=added, skipped=skipped), + ) + self._refresh() + + def _prompt_import_data(self): + import json + path, _filter = QFileDialog.getOpenFileName( + self, _t("rd_webrtc_kh_import"), "", "JSON (*.json);;All (*)", + ) + if not path: + return None + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, json.JSONDecodeError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + return None + if not isinstance(data, dict): + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_kh_import_bad"), + ) + return None + return data + + def _import_one(self, host_id, value, existing) -> str: + """Return ``"added"``, ``"skipped"``, or ``"ignored"`` per entry.""" + if not isinstance(host_id, str): + return "ignored" + app_fp, dtls_fp = self._extract_fingerprints(value) + if app_fp is None and dtls_fp is None: + return "ignored" + if host_id in existing and not self._confirm_overwrite(host_id): + return "skipped" + if isinstance(app_fp, str) and app_fp: + self._known.remember(host_id, app_fp) + if isinstance(dtls_fp, str) and dtls_fp: + self._known.remember_dtls_fingerprint(host_id, dtls_fp) + return "added" + + @staticmethod + def _extract_fingerprints(value): + if isinstance(value, str): + return value, None + if isinstance(value, dict): + return value.get("app_fp"), value.get("dtls_fp") + return None, None + + def _confirm_overwrite(self, host_id: str) -> bool: + result = QMessageBox.question( + self, "WebRTC", + _t("rd_webrtc_kh_import_overwrite").format(host=host_id), + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + return result == QMessageBox.StandardButton.Yes + + def _on_forget_stale(self) -> None: + from datetime import datetime, timedelta, timezone + cutoff = datetime.now(timezone.utc) - timedelta(days=90) + stale_ids = [] + for host_id, fps in self._known.list_entries().items(): + last_seen = fps.get("last_seen") + if not last_seen: + continue + try: + if datetime.fromisoformat(last_seen) < cutoff: + stale_ids.append(host_id) + except (TypeError, ValueError): + continue + if not stale_ids: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_kh_no_stale"), + ) + return + result = QMessageBox.question( + self, "WebRTC", + _t("rd_webrtc_kh_forget_stale_confirm").format(n=len(stale_ids)), + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + if result != QMessageBox.StandardButton.Yes: + return + for host_id in stale_ids: + self._known.forget(host_id) + self._refresh() + + def _on_clear_all(self) -> None: + from PySide6.QtWidgets import QMessageBox as _QMB + result = _QMB.question( + self, "WebRTC", _t("rd_webrtc_kh_clear_confirm"), + _QMB.StandardButton.Yes | _QMB.StandardButton.No, + ) + if result != _QMB.StandardButton.Yes: + return + for host_id in list(self._known.list_entries().keys()): # NOSONAR python:S7504 # forget() mutates the underlying mapping — list() is required to avoid RuntimeError + self._known.forget(host_id) + self._refresh() + + + + +class _ManualKnownHostDialog(QDialog): + """Tiny form dialog for pinning a host fingerprint out-of-band.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self.setWindowTitle(_t("rd_webrtc_kh_add")) + self.setMinimumWidth(420) + layout = QVBoxLayout(self) + form = QFormLayout() + self._host_edit = QLineEdit() + self._host_edit.setPlaceholderText(_t("rd_webrtc_kh_add_host_ph")) + self._app_edit = QLineEdit() + self._app_edit.setPlaceholderText(_t("rd_webrtc_kh_add_app_ph")) + self._dtls_edit = QLineEdit() + self._dtls_edit.setPlaceholderText(_t("rd_webrtc_kh_add_dtls_ph")) + form.addRow(_t("rd_webrtc_kh_col_host"), self._host_edit) + form.addRow(_t("rd_webrtc_kh_col_app_fp"), self._app_edit) + form.addRow(_t("rd_webrtc_kh_col_dtls_fp"), self._dtls_edit) + layout.addLayout(form) + button_row = QHBoxLayout() + button_row.addStretch() + ok = QPushButton(_t("rd_webrtc_kh_add")) + ok.clicked.connect(self.accept) + cancel = QPushButton(_t("rd_webrtc_kh_close")) + cancel.clicked.connect(self.reject) + button_row.addWidget(cancel) + button_row.addWidget(ok) + layout.addLayout(button_row) + + def values(self) -> tuple: + return ( + self._host_edit.text().strip(), + self._app_edit.text().strip(), + self._dtls_edit.text().strip(), + ) diff --git a/je_auto_control/utils/executor/flow_control.py b/je_auto_control/utils/executor/flow_control.py index 87a816ec..aeb1b462 100644 --- a/je_auto_control/utils/executor/flow_control.py +++ b/je_auto_control/utils/executor/flow_control.py @@ -13,6 +13,12 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlActionException, AutoControlException, ImageNotFoundException, ) +from je_auto_control.utils.executor.flow_data_commands import ( + exec_assert_db, exec_assert_duration, exec_assert_var, exec_http_to_var, + exec_now_to_var, exec_ocr_to_var, exec_otp_to_var, exec_pdf_to_var, + exec_random_to_var, exec_read_file_to_var, exec_shell_to_var, + exec_sql_to_var, exec_transform_var, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.wrapper.auto_control_image import locate_image_center from je_auto_control.wrapper.auto_control_screen import get_pixel @@ -389,239 +395,6 @@ def exec_for_each_row(executor: Any, args: Mapping[str, Any]) -> int: return iterations -def exec_shell_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Run a shell command and store its stdout in a flow variable. - - The command is split into an argv list (never ``shell=True``) and its - captured stdout is bound under ``var`` (default ``shell_output``) for - later ``${var}`` use — the shell counterpart of ``AC_ocr_to_var``. - """ - import os - import shlex - import subprocess # nosec B404 — argv list only, no shell - command = args.get("command", args.get("shell_command")) - argv = ([str(part) for part in command] if isinstance(command, list) - else shlex.split(str(command), posix=(os.name != "nt"))) - timeout_s = float(args.get("timeout", 30.0)) - try: - completed = subprocess.run( # nosec B603 — argv list, no shell # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit - argv, capture_output=True, check=False, timeout=timeout_s, - ) - except subprocess.TimeoutExpired as error: - # TimeoutExpired subclasses SubprocessError (not AutoControlException - # nor OSError), so without this it escapes every executor containment - # boundary and aborts the whole script. - raise AutoControlActionException( - f"AC_shell_to_var: command timed out after {timeout_s}s" - ) from error - output = completed.stdout.decode("utf-8", errors="replace").strip() - var_name = args.get("var", "shell_output") - executor.variables.set(var_name, output) - return {"var": var_name, "output": output, - "returncode": completed.returncode} - - -def _now(): - import datetime as _dt - return _dt.datetime.now() - - -def exec_now_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Store the current local time (strftime format) in a flow variable.""" - value = _now().strftime(str(args.get("format", "%Y-%m-%d %H:%M:%S"))) - var_name = args.get("var", "now") - executor.variables.set(var_name, value) - return {"var": var_name, "value": value} - - -def exec_random_to_var(executor: Any, - args: Mapping[str, Any]) -> Dict[str, Any]: - """Store a random value (int / float / choice) in a flow variable.""" - import random - rng = random.Random(args.get("seed")) # nosec B311 # reason: non-crypto test data - kind = str(args.get("kind", "int")) - if kind == "choice": - value: Any = rng.choice(list(args.get("choices") or [None])) # NOSONAR S2245 non-crypto seeded - elif kind == "float": - value = rng.uniform(float(args.get("min", 0.0)), - float(args.get("max", 1.0))) - else: - value = rng.randint(int(args.get("min", 0)), int(args.get("max", 100))) # NOSONAR S2245 non-crypto - var_name = args.get("var", "random") - executor.variables.set(var_name, value) - return {"var": var_name, "value": value} - - -def exec_assert_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert a flow variable satisfies a condition (assertion DSL).""" - from je_auto_control.utils.assertion import assert_variable - name = args["name"] - return assert_variable( - executor.variables.get_value(name), op=str(args.get("op", "eq")), - expected=args.get("value"), name=name, - raise_on_fail=bool(args.get("raise_on_fail", True)), - ).to_dict() - - -def exec_pdf_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Extract a PDF's text (all pages or one page) into a flow variable.""" - from je_auto_control.utils.pdf.pdf_reader import extract_pdf_text - text = extract_pdf_text(args["path"], pages=args.get("page")) - var_name = args.get("var", "pdf_text") - executor.variables.set(var_name, text) - return {"var": var_name, "length": len(text)} - - -def exec_otp_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Generate a TOTP code from a base32 secret into a flow variable (2FA).""" - from je_auto_control.utils.otp import generate_totp - code = generate_totp(args["secret"], step=int(args.get("step", 30)), - digits=int(args.get("digits", 6))) - var_name = args.get("var", "otp") - executor.variables.set(var_name, code) - return {"var": var_name} - - -def exec_sql_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Run a read-only SQLite query and store its result in a flow variable.""" - from je_auto_control.utils.sql.sql_query import query_sqlite - fetch = str(args.get("fetch", "all")) - result = query_sqlite(args["database"], args["query"], - params=args.get("params"), fetch=fetch) - var_name = args.get("var", "sql_result") - executor.variables.set(var_name, result) - return {"var": var_name, "fetch": fetch} - - -def exec_assert_db(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert a scalar SQLite query result satisfies a condition.""" - from je_auto_control.utils.assertion import assert_variable - from je_auto_control.utils.sql.sql_query import query_sqlite - value = query_sqlite(args["database"], args["query"], - params=args.get("params"), fetch="scalar") - return assert_variable( - value, op=str(args.get("op", "eq")), expected=args.get("expected"), - name="AC_assert_db", - raise_on_fail=bool(args.get("raise_on_fail", True)), - ).to_dict() - - -def exec_read_file_to_var(executor: Any, - args: Mapping[str, Any]) -> Dict[str, Any]: - """Read a file's text content into a flow variable.""" - from pathlib import Path - text = Path(args["path"]).read_text(encoding=args.get("encoding", "utf-8")) - var_name = args.get("var", "file_content") - executor.variables.set(var_name, text) - return {"var": var_name, "length": len(text)} - - -def _dig_json(body: str, path: str) -> Any: - """Navigate a dotted JSON path, e.g. ``data.0.name``.""" - data = json.loads(body) - for part in str(path).split("."): - data = data[int(part)] if isinstance(data, list) else data[part] - return data - - -def exec_http_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Request a URL and store the body (or a JSON field) in a flow variable. - - Supports method/headers/json_body/data/auth via the shared HTTP client, - so the same command drives plain GET reads and POST/PUT API calls. - """ - from je_auto_control.utils.http_client.http_client import http_request - response = http_request( - args["url"], method=str(args.get("method", "GET")), - headers=args.get("headers"), json_body=args.get("json_body"), - data=args.get("data"), auth=args.get("auth"), - timeout=float(args.get("timeout", 30.0)), - ) - json_path = args.get("json_path") - body = response["text"] - value = _dig_json(body, json_path) if json_path else body - var_name = args.get("var", "http_response") - executor.variables.set(var_name, value) - return {"var": var_name, "status": response["status"]} - - -_SIMPLE_TRANSFORMS: Dict[str, Callable[[str], str]] = { - "upper": str.upper, "lower": str.lower, "strip": str.strip, - "title": str.title, "lstrip": str.lstrip, "rstrip": str.rstrip, -} - - -def _regex_extract(text: str, args: Mapping[str, Any]) -> str: - import re - match = re.search(str(args.get("pattern", "")), text) - return match.group(int(args.get("group", 0))) if match else "" - - -def _slice_text(text: str, args: Mapping[str, Any]) -> str: - start = args.get("start") - end = args.get("end") - return text[(int(start) if start is not None else None): - (int(end) if end is not None else None)] - - -def _transform_string(text: str, op: str, args: Mapping[str, Any]) -> str: - simple = _SIMPLE_TRANSFORMS.get(op) - if simple is not None: - return simple(text) - if op == "replace": - return text.replace(str(args.get("find", "")), - str(args.get("replace_with", ""))) - if op == "regex": - return _regex_extract(text, args) - if op == "slice": - return _slice_text(text, args) - raise AutoControlActionException(f"AC_transform_var: unknown op {op!r}") - - -def exec_transform_var(executor: Any, - args: Mapping[str, Any]) -> Dict[str, Any]: - """Apply a string transform to a variable (in place or into ``into``).""" - name = args["name"] - value = str(executor.variables.get_value(name, "")) - result = _transform_string(value, str(args.get("op", "strip")), args) - target = args.get("into", name) - executor.variables.set(target, result) - return {"var": target, "value": result} - - -def exec_ocr_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Read OCR text from a screen region into a flow variable. - - Binds the recognised text under ``var`` (default ``ocr_text``) so later - steps can read it as ``${var}`` — the bridge between OCR and the - variable scope for data-driven flows. - """ - from je_auto_control.utils.ocr.ocr_engine import read_text_in_region - region = args.get("region") - if isinstance(region, str): - region = json.loads(region) if region.strip() else None - matches = read_text_in_region( - region=region, lang=args.get("lang", "eng"), - min_confidence=float(args.get("min_confidence", 60.0)), - ) - text = " ".join(match.text for match in matches).strip() - var_name = args.get("var", "ocr_text") - executor.variables.set(var_name, text) - return {"var": var_name, "text": text} - - -def exec_assert_duration(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert ``body`` completes within ``max_ms`` (a performance budget).""" - from je_auto_control.utils.assertion import assert_duration - body = args.get("body") or [] - return assert_duration( - lambda: executor.execute_action(body, _validated=True), - max_ms=float(args.get("max_ms", 1000.0)), - min_ms=float(args.get("min_ms", 0.0)), - raise_on_fail=bool(args.get("raise_on_fail", True)), - ).to_dict() - - def _as_list(value: Any) -> list: """Accept a native list or a JSON-string list (for the visual builder).""" if isinstance(value, str): diff --git a/je_auto_control/utils/executor/flow_data_commands.py b/je_auto_control/utils/executor/flow_data_commands.py new file mode 100644 index 00000000..03cbb142 --- /dev/null +++ b/je_auto_control/utils/executor/flow_data_commands.py @@ -0,0 +1,248 @@ +"""Data-source and transform block commands for the action executor. + +These are the ``*_to_var`` steps plus their variable assertions: they read +from a shell command, a clock, a PRNG, a PDF, a TOTP secret, a database, +a file, an HTTP endpoint or OCR, and store the result in the executor's +variable scope. Unlike :mod:`flow_control` they never execute a nested +action list, so they carry no loop or branch semantics. + +They are registered in ``flow_control.BLOCK_COMMANDS`` alongside the real +flow-control commands, which is the only table the executor reads. +""" +import json +from typing import Any, Callable, Dict, Mapping + +from je_auto_control.utils.exception.exceptions import AutoControlActionException + + +def exec_shell_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Run a shell command and store its stdout in a flow variable. + + The command is split into an argv list (never ``shell=True``) and its + captured stdout is bound under ``var`` (default ``shell_output``) for + later ``${var}`` use — the shell counterpart of ``AC_ocr_to_var``. + """ + import os + import shlex + import subprocess # nosec B404 — argv list only, no shell + command = args.get("command", args.get("shell_command")) + argv = ([str(part) for part in command] if isinstance(command, list) + else shlex.split(str(command), posix=(os.name != "nt"))) + timeout_s = float(args.get("timeout", 30.0)) + try: + completed = subprocess.run( # nosec B603 — argv list, no shell # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit + argv, capture_output=True, check=False, timeout=timeout_s, + ) + except subprocess.TimeoutExpired as error: + # TimeoutExpired subclasses SubprocessError (not AutoControlException + # nor OSError), so without this it escapes every executor containment + # boundary and aborts the whole script. + raise AutoControlActionException( + f"AC_shell_to_var: command timed out after {timeout_s}s" + ) from error + output = completed.stdout.decode("utf-8", errors="replace").strip() + var_name = args.get("var", "shell_output") + executor.variables.set(var_name, output) + return {"var": var_name, "output": output, + "returncode": completed.returncode} + + +def _now(): + import datetime as _dt + return _dt.datetime.now() + + +def exec_now_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Store the current local time (strftime format) in a flow variable.""" + value = _now().strftime(str(args.get("format", "%Y-%m-%d %H:%M:%S"))) + var_name = args.get("var", "now") + executor.variables.set(var_name, value) + return {"var": var_name, "value": value} + + +def exec_random_to_var(executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Store a random value (int / float / choice) in a flow variable.""" + import random + rng = random.Random(args.get("seed")) # nosec B311 # reason: non-crypto test data + kind = str(args.get("kind", "int")) + if kind == "choice": + value: Any = rng.choice(list(args.get("choices") or [None])) # NOSONAR S2245 non-crypto seeded + elif kind == "float": + value = rng.uniform(float(args.get("min", 0.0)), + float(args.get("max", 1.0))) + else: + value = rng.randint(int(args.get("min", 0)), int(args.get("max", 100))) # NOSONAR S2245 non-crypto + var_name = args.get("var", "random") + executor.variables.set(var_name, value) + return {"var": var_name, "value": value} + + +def exec_assert_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert a flow variable satisfies a condition (assertion DSL).""" + from je_auto_control.utils.assertion import assert_variable + name = args["name"] + return assert_variable( + executor.variables.get_value(name), op=str(args.get("op", "eq")), + expected=args.get("value"), name=name, + raise_on_fail=bool(args.get("raise_on_fail", True)), + ).to_dict() + + +def exec_pdf_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Extract a PDF's text (all pages or one page) into a flow variable.""" + from je_auto_control.utils.pdf.pdf_reader import extract_pdf_text + text = extract_pdf_text(args["path"], pages=args.get("page")) + var_name = args.get("var", "pdf_text") + executor.variables.set(var_name, text) + return {"var": var_name, "length": len(text)} + + +def exec_otp_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Generate a TOTP code from a base32 secret into a flow variable (2FA).""" + from je_auto_control.utils.otp import generate_totp + code = generate_totp(args["secret"], step=int(args.get("step", 30)), + digits=int(args.get("digits", 6))) + var_name = args.get("var", "otp") + executor.variables.set(var_name, code) + return {"var": var_name} + + +def exec_sql_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Run a read-only SQLite query and store its result in a flow variable.""" + from je_auto_control.utils.sql.sql_query import query_sqlite + fetch = str(args.get("fetch", "all")) + result = query_sqlite(args["database"], args["query"], + params=args.get("params"), fetch=fetch) + var_name = args.get("var", "sql_result") + executor.variables.set(var_name, result) + return {"var": var_name, "fetch": fetch} + + +def exec_assert_db(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert a scalar SQLite query result satisfies a condition.""" + from je_auto_control.utils.assertion import assert_variable + from je_auto_control.utils.sql.sql_query import query_sqlite + value = query_sqlite(args["database"], args["query"], + params=args.get("params"), fetch="scalar") + return assert_variable( + value, op=str(args.get("op", "eq")), expected=args.get("expected"), + name="AC_assert_db", + raise_on_fail=bool(args.get("raise_on_fail", True)), + ).to_dict() + + +def exec_read_file_to_var(executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Read a file's text content into a flow variable.""" + from pathlib import Path + text = Path(args["path"]).read_text(encoding=args.get("encoding", "utf-8")) + var_name = args.get("var", "file_content") + executor.variables.set(var_name, text) + return {"var": var_name, "length": len(text)} + + +def _dig_json(body: str, path: str) -> Any: + """Navigate a dotted JSON path, e.g. ``data.0.name``.""" + data = json.loads(body) + for part in str(path).split("."): + data = data[int(part)] if isinstance(data, list) else data[part] + return data + + +def exec_http_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Request a URL and store the body (or a JSON field) in a flow variable. + + Supports method/headers/json_body/data/auth via the shared HTTP client, + so the same command drives plain GET reads and POST/PUT API calls. + """ + from je_auto_control.utils.http_client.http_client import http_request + response = http_request( + args["url"], method=str(args.get("method", "GET")), + headers=args.get("headers"), json_body=args.get("json_body"), + data=args.get("data"), auth=args.get("auth"), + timeout=float(args.get("timeout", 30.0)), + ) + json_path = args.get("json_path") + body = response["text"] + value = _dig_json(body, json_path) if json_path else body + var_name = args.get("var", "http_response") + executor.variables.set(var_name, value) + return {"var": var_name, "status": response["status"]} + + +_SIMPLE_TRANSFORMS: Dict[str, Callable[[str], str]] = { + "upper": str.upper, "lower": str.lower, "strip": str.strip, + "title": str.title, "lstrip": str.lstrip, "rstrip": str.rstrip, +} + + +def _regex_extract(text: str, args: Mapping[str, Any]) -> str: + import re + match = re.search(str(args.get("pattern", "")), text) + return match.group(int(args.get("group", 0))) if match else "" + + +def _slice_text(text: str, args: Mapping[str, Any]) -> str: + start = args.get("start") + end = args.get("end") + return text[(int(start) if start is not None else None): + (int(end) if end is not None else None)] + + +def _transform_string(text: str, op: str, args: Mapping[str, Any]) -> str: + simple = _SIMPLE_TRANSFORMS.get(op) + if simple is not None: + return simple(text) + if op == "replace": + return text.replace(str(args.get("find", "")), + str(args.get("replace_with", ""))) + if op == "regex": + return _regex_extract(text, args) + if op == "slice": + return _slice_text(text, args) + raise AutoControlActionException(f"AC_transform_var: unknown op {op!r}") + + +def exec_transform_var(executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Apply a string transform to a variable (in place or into ``into``).""" + name = args["name"] + value = str(executor.variables.get_value(name, "")) + result = _transform_string(value, str(args.get("op", "strip")), args) + target = args.get("into", name) + executor.variables.set(target, result) + return {"var": target, "value": result} + + +def exec_ocr_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Read OCR text from a screen region into a flow variable. + + Binds the recognised text under ``var`` (default ``ocr_text``) so later + steps can read it as ``${var}`` — the bridge between OCR and the + variable scope for data-driven flows. + """ + from je_auto_control.utils.ocr.ocr_engine import read_text_in_region + region = args.get("region") + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = read_text_in_region( + region=region, lang=args.get("lang", "eng"), + min_confidence=float(args.get("min_confidence", 60.0)), + ) + text = " ".join(match.text for match in matches).strip() + var_name = args.get("var", "ocr_text") + executor.variables.set(var_name, text) + return {"var": var_name, "text": text} + + +def exec_assert_duration(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert ``body`` completes within ``max_ms`` (a performance budget).""" + from je_auto_control.utils.assertion import assert_duration + body = args.get("body") or [] + return assert_duration( + lambda: executor.execute_action(body, _validated=True), + max_ms=float(args.get("max_ms", 1000.0)), + min_ms=float(args.get("min_ms", 0.0)), + raise_on_fail=bool(args.get("raise_on_fail", True)), + ).to_dict() diff --git a/je_auto_control/utils/mcp_server/_client_requests.py b/je_auto_control/utils/mcp_server/_client_requests.py new file mode 100644 index 00000000..5d2e0119 --- /dev/null +++ b/je_auto_control/utils/mcp_server/_client_requests.py @@ -0,0 +1,217 @@ +"""Server-initiated requests to the MCP client. + +MCP is bidirectional: besides answering the client, the server asks *it* +for things — ``roots/list`` to learn the workspace, ``elicitation/create`` +to put a question to the user, ``sampling/createMessage`` to borrow the +model. Each one writes a request out and blocks on a slot until the reply +arrives on the inbound side, so the correlation table and the response +router belong together with the senders that populate it. + +Destructive-tool confirmation lives here too: it is an elicitation +round-trip, not a tool-execution step. +""" +import json +import threading +from typing import Any, Dict, List, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server._protocol import ( + _confirm_destructive_enabled, _file_uri_to_path, _MCPError, +) +from je_auto_control.utils.mcp_server.tools import MCPTool + + +class ClientRequestMixin: + """Outbound half of the MCP session, mixed into :class:`MCPServer`. + + Requires the host to provide ``_writer``, ``_client_capabilities``, + ``_resources``, ``_outbound_lock``, ``_pending_outbound``, + ``_outbound_id_counter`` and ``_sampling_id_counter``. + """ + + @staticmethod + def _is_outbound_response(method: Optional[str], msg_id: Any, + message: Dict[str, Any]) -> bool: + """True when ``message`` is a reply to a server-initiated request.""" + return ( + method is None + and msg_id is not None + and ("result" in message or "error" in message) + ) + + def _dispatch_outbound_response(self, msg_id: Any, + message: Dict[str, Any]) -> None: + """Route a JSON-RPC response to the matching pending request.""" + with self._outbound_lock: + slot = self._pending_outbound.get(msg_id) + if slot is None: + autocontrol_logger.debug( + "MCP outbound response for unknown id %r", msg_id, + ) + return + if "error" in message: + slot["error"] = message["error"] + else: + slot["result"] = message.get("result") + slot["event"].set() + + def _maybe_request_roots_async(self) -> None: + """Fire a roots/list request when the client supports it.""" + if "roots" not in self._client_capabilities: + return + if self._writer is None: + return + threading.Thread( + target=self._refresh_roots_safely, daemon=True, + name="MCPRootsRefresh", + ).start() + + def _refresh_roots_safely(self) -> None: + try: + self.refresh_roots(timeout=5.0) + except (RuntimeError, TimeoutError) as error: + autocontrol_logger.info("MCP roots refresh skipped: %r", error) + + def refresh_roots(self, timeout: float = 10.0) -> List[Dict[str, Any]]: + """Send ``roots/list`` to the client and apply the first root.""" + result = self._send_outbound_request( + "roots/list", params={}, timeout=timeout, + ) + roots_list = (result or {}).get("roots") or [] + if not isinstance(roots_list, list) or not roots_list: + return [] + first_uri = roots_list[0].get("uri") if isinstance(roots_list[0], + dict) else None + if isinstance(first_uri, str): + local_path = _file_uri_to_path(first_uri) + if local_path: + self._resources.set_workspace_root(local_path) + autocontrol_logger.info("MCP workspace root → %s", local_path) + return roots_list + + def _send_outbound_request(self, method: str, + params: Dict[str, Any], + timeout: float = 10.0) -> Dict[str, Any]: + """Send a server-initiated request and wait for the response.""" + writer = self._writer + if writer is None: + raise RuntimeError(f"{method} requires an outbound writer") + request_id = f"srv-{next(self._outbound_id_counter)}" + slot = {"event": threading.Event()} + with self._outbound_lock: + self._pending_outbound[request_id] = slot + envelope = json.dumps({ + "jsonrpc": "2.0", "id": request_id, + "method": method, "params": params, + }, ensure_ascii=False, default=str) + try: + writer(envelope) + if not slot["event"].wait(timeout=timeout): + raise TimeoutError(f"{method} timed out after {timeout}s") + finally: + with self._outbound_lock: + self._pending_outbound.pop(request_id, None) + if "error" in slot: + raise RuntimeError(f"{method} failed: {slot['error']}") + return slot.get("result") or {} + + def request_elicitation(self, message: str, + requested_schema: Optional[Dict[str, Any]] = None, + timeout: float = 60.0) -> Dict[str, Any]: + """Ask the connected client to elicit a response from the user. + + Returns the raw payload (typically ``{"action": "accept" | "decline" | "cancel", ...}``). + Requires the client to advertise the ``elicitation`` capability. + """ + params: Dict[str, Any] = {"message": str(message)} + if requested_schema is not None: + params["requestedSchema"] = requested_schema + return self._send_outbound_request( + "elicitation/create", params=params, timeout=timeout, + ) + + def request_sampling(self, messages: List[Dict[str, Any]], + system_prompt: Optional[str] = None, + max_tokens: int = 1024, + model_preferences: Optional[Dict[str, Any]] = None, + timeout: float = 120.0) -> Dict[str, Any]: + """Ask the connected client to run an LLM sampling request. + + Tools that need the model's help (e.g. an OCR fallback that + wants the model to identify a UI element from a screenshot) + can call this and receive the assistant's reply. Requires the + server to be running in concurrent mode with an outbound + writer set — typically meaning ``serve_stdio`` or the HTTP + SSE transport. + """ + writer = self._writer + if writer is None: + raise RuntimeError( + "request_sampling requires an outbound writer; " + "start serve_stdio or call set_writer() first", + ) + request_id = f"sampling-{next(self._sampling_id_counter)}" + params: Dict[str, Any] = { + "messages": list(messages), + "maxTokens": int(max_tokens), + } + if system_prompt is not None: + params["systemPrompt"] = str(system_prompt) + if model_preferences is not None: + params["modelPreferences"] = dict(model_preferences) + slot = {"event": threading.Event()} + with self._outbound_lock: + self._pending_outbound[request_id] = slot + envelope = json.dumps({ + "jsonrpc": "2.0", "id": request_id, + "method": "sampling/createMessage", "params": params, + }, ensure_ascii=False, default=str) + try: + writer(envelope) + if not slot["event"].wait(timeout=timeout): + raise TimeoutError( + f"sampling request {request_id} timed out after {timeout}s" + ) + finally: + with self._outbound_lock: + self._pending_outbound.pop(request_id, None) + if "error" in slot: + raise RuntimeError(f"sampling failed: {slot['error']}") + return slot.get("result") or {} + + def _maybe_confirm_destructive(self, name: str, tool: MCPTool, + arguments: Dict[str, Any]) -> None: + """Ask the client to confirm before running a destructive tool.""" + if not _confirm_destructive_enabled(): + return + annotations = tool.annotations + if annotations.read_only or not annotations.destructive: + return + if "elicitation" not in self._client_capabilities: + autocontrol_logger.info( + "MCP confirmation requested for %s but client lacks " + "elicitation capability — proceeding without prompt", name, + ) + return + if self._writer is None: + return + prompt = (f"AutoControl is about to run a destructive tool " + f"'{name}'. Continue?") + try: + response = self.request_elicitation( + message=prompt, requested_schema={"type": "object", + "properties": {}}, + timeout=60.0, + ) + except (RuntimeError, TimeoutError) as error: + autocontrol_logger.info( + "MCP elicitation for %s failed (%r) — refusing call", + name, error, + ) + raise _MCPError( + -32000, f"User confirmation unavailable for {name}", + ) from error + action = response.get("action") if isinstance(response, dict) else None + if action != "accept": + raise _MCPError(-32000, f"User declined to run {name}: action={action!r}") + del arguments # available for future per-arg confirmation policies diff --git a/je_auto_control/utils/mcp_server/_protocol.py b/je_auto_control/utils/mcp_server/_protocol.py new file mode 100644 index 00000000..f5e1502e --- /dev/null +++ b/je_auto_control/utils/mcp_server/_protocol.py @@ -0,0 +1,165 @@ +"""JSON-RPC 2.0 wire format for the MCP server. + +Everything here is about the protocol rather than the server: the version +and identity constants, the error classes and the error tuples that decide +what a failing tool is allowed to do, the envelope builders, and the pure +functions that normalise a tool's return value into MCP ``content`` blocks. + +None of it touches server state, so it is importable and testable without +starting a server. +""" +import json +import os +import sqlite3 +import subprocess # nosec B404 # reason: only its TimeoutExpired type is referenced +import sys +import time +from typing import Any, Dict, List, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server.tools import MCPContent + + +PROTOCOL_VERSION = "2025-06-18" +SERVER_NAME = "je_auto_control" +SERVER_VERSION = "0.1.0" +_TOOLS_CALL_METHOD = "tools/call" + +# Framework and external-library errors a tool handler may raise. They all +# subclass ``Exception`` directly (not OSError/RuntimeError/…), so without +# listing them here a failing tool would escape both containment layers — the +# stdio worker thread dies and the client waits forever, or the HTTP +# connection aborts with no JSON-RPC reply. ``AutoControlException`` is the +# family base every ``AutoControl*Exception``/``ImageNotFoundException`` now +# derives from. +_FRAMEWORK_TOOL_ERRORS = ( + AutoControlException, subprocess.TimeoutExpired, sqlite3.Error, +) +_BUILTIN_DISPATCH_ERRORS = ( + OSError, RuntimeError, ValueError, TypeError, KeyError, +) +_DISPATCH_ERRORS = _BUILTIN_DISPATCH_ERRORS + _FRAMEWORK_TOOL_ERRORS +_TOOL_INVOKE_ERRORS = ( + _BUILTIN_DISPATCH_ERRORS + (AttributeError,) + _FRAMEWORK_TOOL_ERRORS +) + + +class _MCPError(Exception): + """Raised inside the dispatcher to surface a JSON-RPC error response.""" + + def __init__(self, code: int, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _to_content_blocks(result: Any) -> List[Dict[str, Any]]: + """Normalise a tool's return value into MCP ``content`` blocks.""" + if isinstance(result, MCPContent): + return [result.to_dict()] + if isinstance(result, list) and result and \ + all(isinstance(item, MCPContent) for item in result): + return [item.to_dict() for item in result] + return [{"type": "text", "text": _stringify_result(result)}] + + +def _stringify_result(value: Any) -> str: + """Convert a tool return value into a model-readable string.""" + if isinstance(value, str): + return value + try: + return json.dumps(value, ensure_ascii=False, default=str) + except (TypeError, ValueError): + return repr(value) + + +def _confirm_destructive_enabled() -> bool: + """Return True when the operator wants destructive tools gated on user OK.""" + raw = os.environ.get("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "") + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _capture_error_screenshot(tool_name: str) -> Optional[str]: + """Save a debug screenshot when JE_AUTOCONTROL_MCP_ERROR_SHOTS is set.""" + debug_dir = os.environ.get("JE_AUTOCONTROL_MCP_ERROR_SHOTS") + if not debug_dir: + return None + target_dir = os.path.realpath(os.fspath(debug_dir)) + try: + os.makedirs(target_dir, exist_ok=True) + except OSError as error: + autocontrol_logger.info( + "MCP error-screenshot dir unavailable: %r", error, + ) + return None + filename = f"{tool_name}_{int(time.time() * 1000)}.png" + path = os.path.join(target_dir, filename) + try: + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + pil_screenshot(file_path=path) + except (OSError, RuntimeError, ValueError, AttributeError, + ImportError) as error: + autocontrol_logger.info( + "MCP failed to capture error screenshot: %r", error, + ) + return None + return path + + +def _file_uri_to_path(uri: str) -> Optional[str]: + """Convert a ``file://`` URI to a local filesystem path; ``None`` otherwise.""" + if not isinstance(uri, str) or not uri.startswith("file://"): + return None + from urllib.parse import unquote, urlparse + parsed = urlparse(uri) + raw_path = unquote(parsed.path) + # Windows: file:///C:/foo strips the leading slash before the drive letter. + if sys.platform.startswith("win") and raw_path.startswith("/") and \ + len(raw_path) > 2 and raw_path[2] == ":": + raw_path = raw_path[1:] + return raw_path or None + + +def _is_hashable(value: Any) -> bool: + """Return True when ``value`` can be used as a dict key.""" + try: + hash(value) + except TypeError: + return False + return True + + +def _coerce_params(raw: Any, msg_id: Any) -> tuple: + """Normalise JSON-RPC ``params`` to a dict. + + Returns ``(params, error_line)``. Every handler here expects an object; + a non-object ``params`` yields a ``-32602`` error line for a request and + an empty dict for a notification (which cannot carry an error reply). + """ + if raw is None: + return {}, None + if isinstance(raw, dict): + return raw, None + if msg_id is None: + return {}, None + return {}, _error_response(msg_id, -32602, "Invalid params: expected an object") + + +def _notification_message(method: str, params: Dict[str, Any]) -> str: + return json.dumps({"jsonrpc": "2.0", "method": method, "params": params}, + ensure_ascii=False, default=str) + + +def _result_response(msg_id: Any, result: Any) -> str: + return json.dumps( + {"jsonrpc": "2.0", "id": msg_id, "result": result}, + ensure_ascii=False, default=str, + ) + + +def _error_response(msg_id: Any, code: int, message: str) -> str: + return json.dumps({ + "jsonrpc": "2.0", "id": msg_id, + "error": {"code": code, "message": message}, + }, ensure_ascii=False) diff --git a/je_auto_control/utils/mcp_server/http_transport.py b/je_auto_control/utils/mcp_server/http_transport.py index 8db697b3..46c667fd 100644 --- a/je_auto_control/utils/mcp_server/http_transport.py +++ b/je_auto_control/utils/mcp_server/http_transport.py @@ -20,9 +20,10 @@ from je_auto_control.utils.http_headers import parse_content_length from je_auto_control.utils.logging.logging_instance import autocontrol_logger -from je_auto_control.utils.mcp_server.server import ( - MCPServer, _notification_message, +from je_auto_control.utils.mcp_server._protocol import ( + _notification_message, ) +from je_auto_control.utils.mcp_server.server import MCPServer DEFAULT_PATH = "/mcp" _MAX_BODY = 1_000_000 diff --git a/je_auto_control/utils/mcp_server/server.py b/je_auto_control/utils/mcp_server/server.py index d3021eb1..f85c4122 100644 --- a/je_auto_control/utils/mcp_server/server.py +++ b/je_auto_control/utils/mcp_server/server.py @@ -9,15 +9,11 @@ import contextlib import itertools import json -import os -import sqlite3 -import subprocess # nosec B404 # reason: only its TimeoutExpired type is referenced import sys import threading import time from typing import Any, Callable, Dict, List, Optional, TextIO -from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.mcp_server.audit import AuditLogger from je_auto_control.utils.mcp_server.context import ( @@ -34,46 +30,23 @@ ResourceProvider, default_resource_provider, ) from je_auto_control.utils.mcp_server.tools import ( - MCPContent, MCPTool, build_default_tool_registry, + MCPTool, build_default_tool_registry, ) from je_auto_control.utils.mcp_server.tools._validation import ( validate_arguments, ) - -PROTOCOL_VERSION = "2025-06-18" -SERVER_NAME = "je_auto_control" -SERVER_VERSION = "0.1.0" -_TOOLS_CALL_METHOD = "tools/call" - -# Framework and external-library errors a tool handler may raise. They all -# subclass ``Exception`` directly (not OSError/RuntimeError/…), so without -# listing them here a failing tool would escape both containment layers — the -# stdio worker thread dies and the client waits forever, or the HTTP -# connection aborts with no JSON-RPC reply. ``AutoControlException`` is the -# family base every ``AutoControl*Exception``/``ImageNotFoundException`` now -# derives from. -_FRAMEWORK_TOOL_ERRORS = ( - AutoControlException, subprocess.TimeoutExpired, sqlite3.Error, -) -_BUILTIN_DISPATCH_ERRORS = ( - OSError, RuntimeError, ValueError, TypeError, KeyError, +from je_auto_control.utils.mcp_server._client_requests import ( + ClientRequestMixin, ) -_DISPATCH_ERRORS = _BUILTIN_DISPATCH_ERRORS + _FRAMEWORK_TOOL_ERRORS -_TOOL_INVOKE_ERRORS = ( - _BUILTIN_DISPATCH_ERRORS + (AttributeError,) + _FRAMEWORK_TOOL_ERRORS +from je_auto_control.utils.mcp_server._protocol import ( + PROTOCOL_VERSION, SERVER_NAME, SERVER_VERSION, _capture_error_screenshot, + _coerce_params, _DISPATCH_ERRORS, _error_response, _is_hashable, + _MCPError, _notification_message, _result_response, _to_content_blocks, + _TOOL_INVOKE_ERRORS, _TOOLS_CALL_METHOD, ) -class _MCPError(Exception): - """Raised inside the dispatcher to surface a JSON-RPC error response.""" - - def __init__(self, code: int, message: str) -> None: - super().__init__(message) - self.code = code - self.message = message - - -class MCPServer: +class MCPServer(ClientRequestMixin): """JSON-RPC 2.0 MCP server with a configurable tool registry.""" def __init__(self, tools: Optional[List[MCPTool]] = None, @@ -390,32 +363,6 @@ def _handle_line_safely(self, line: str) -> Optional[str]: autocontrol_logger.exception("MCP handle_line failed; line skipped") return None - @staticmethod - def _is_outbound_response(method: Optional[str], msg_id: Any, - message: Dict[str, Any]) -> bool: - """True when ``message`` is a reply to a server-initiated request.""" - return ( - method is None - and msg_id is not None - and ("result" in message or "error" in message) - ) - - def _dispatch_outbound_response(self, msg_id: Any, - message: Dict[str, Any]) -> None: - """Route a JSON-RPC response to the matching pending request.""" - with self._outbound_lock: - slot = self._pending_outbound.get(msg_id) - if slot is None: - autocontrol_logger.debug( - "MCP outbound response for unknown id %r", msg_id, - ) - return - if "error" in message: - slot["error"] = message["error"] - else: - slot["result"] = message.get("result") - slot["event"].set() - def _dispatch_tools_call_async(self, msg_id: Any, params: Dict[str, Any]) -> None: """Run a tools/call on a worker thread; the worker writes the reply.""" @@ -463,66 +410,6 @@ def _handle_notification(self, method: Optional[str], return autocontrol_logger.debug("MCP notification ignored: %s", method) - def _maybe_request_roots_async(self) -> None: - """Fire a roots/list request when the client supports it.""" - if "roots" not in self._client_capabilities: - return - if self._writer is None: - return - threading.Thread( - target=self._refresh_roots_safely, daemon=True, - name="MCPRootsRefresh", - ).start() - - def _refresh_roots_safely(self) -> None: - try: - self.refresh_roots(timeout=5.0) - except (RuntimeError, TimeoutError) as error: - autocontrol_logger.info("MCP roots refresh skipped: %r", error) - - def refresh_roots(self, timeout: float = 10.0) -> List[Dict[str, Any]]: - """Send ``roots/list`` to the client and apply the first root.""" - result = self._send_outbound_request( - "roots/list", params={}, timeout=timeout, - ) - roots_list = (result or {}).get("roots") or [] - if not isinstance(roots_list, list) or not roots_list: - return [] - first_uri = roots_list[0].get("uri") if isinstance(roots_list[0], - dict) else None - if isinstance(first_uri, str): - local_path = _file_uri_to_path(first_uri) - if local_path: - self._resources.set_workspace_root(local_path) - autocontrol_logger.info("MCP workspace root → %s", local_path) - return roots_list - - def _send_outbound_request(self, method: str, - params: Dict[str, Any], - timeout: float = 10.0) -> Dict[str, Any]: - """Send a server-initiated request and wait for the response.""" - writer = self._writer - if writer is None: - raise RuntimeError(f"{method} requires an outbound writer") - request_id = f"srv-{next(self._outbound_id_counter)}" - slot = {"event": threading.Event()} - with self._outbound_lock: - self._pending_outbound[request_id] = slot - envelope = json.dumps({ - "jsonrpc": "2.0", "id": request_id, - "method": method, "params": params, - }, ensure_ascii=False, default=str) - try: - writer(envelope) - if not slot["event"].wait(timeout=timeout): - raise TimeoutError(f"{method} timed out after {timeout}s") - finally: - with self._outbound_lock: - self._pending_outbound.pop(request_id, None) - if "error" in slot: - raise RuntimeError(f"{method} failed: {slot['error']}") - return slot.get("result") or {} - def _cancel_active_call(self, params: Dict[str, Any]) -> None: """Mark the matching active tool call as cancelled, if any.""" request_id = params.get("requestId") @@ -764,107 +651,6 @@ def _handle_tools_call(self, msg_id: Any, response["structuredContent"] = result return response - def request_elicitation(self, message: str, - requested_schema: Optional[Dict[str, Any]] = None, - timeout: float = 60.0) -> Dict[str, Any]: - """Ask the connected client to elicit a response from the user. - - Returns the raw payload (typically ``{"action": "accept" | "decline" | "cancel", ...}``). - Requires the client to advertise the ``elicitation`` capability. - """ - params: Dict[str, Any] = {"message": str(message)} - if requested_schema is not None: - params["requestedSchema"] = requested_schema - return self._send_outbound_request( - "elicitation/create", params=params, timeout=timeout, - ) - - def request_sampling(self, messages: List[Dict[str, Any]], - system_prompt: Optional[str] = None, - max_tokens: int = 1024, - model_preferences: Optional[Dict[str, Any]] = None, - timeout: float = 120.0) -> Dict[str, Any]: - """Ask the connected client to run an LLM sampling request. - - Tools that need the model's help (e.g. an OCR fallback that - wants the model to identify a UI element from a screenshot) - can call this and receive the assistant's reply. Requires the - server to be running in concurrent mode with an outbound - writer set — typically meaning ``serve_stdio`` or the HTTP - SSE transport. - """ - writer = self._writer - if writer is None: - raise RuntimeError( - "request_sampling requires an outbound writer; " - "start serve_stdio or call set_writer() first", - ) - request_id = f"sampling-{next(self._sampling_id_counter)}" - params: Dict[str, Any] = { - "messages": list(messages), - "maxTokens": int(max_tokens), - } - if system_prompt is not None: - params["systemPrompt"] = str(system_prompt) - if model_preferences is not None: - params["modelPreferences"] = dict(model_preferences) - slot = {"event": threading.Event()} - with self._outbound_lock: - self._pending_outbound[request_id] = slot - envelope = json.dumps({ - "jsonrpc": "2.0", "id": request_id, - "method": "sampling/createMessage", "params": params, - }, ensure_ascii=False, default=str) - try: - writer(envelope) - if not slot["event"].wait(timeout=timeout): - raise TimeoutError( - f"sampling request {request_id} timed out after {timeout}s" - ) - finally: - with self._outbound_lock: - self._pending_outbound.pop(request_id, None) - if "error" in slot: - raise RuntimeError(f"sampling failed: {slot['error']}") - return slot.get("result") or {} - - def _maybe_confirm_destructive(self, name: str, tool: MCPTool, - arguments: Dict[str, Any]) -> None: - """Ask the client to confirm before running a destructive tool.""" - if not _confirm_destructive_enabled(): - return - annotations = tool.annotations - if annotations.read_only or not annotations.destructive: - return - if "elicitation" not in self._client_capabilities: - autocontrol_logger.info( - "MCP confirmation requested for %s but client lacks " - "elicitation capability — proceeding without prompt", name, - ) - return - if self._writer is None: - return - prompt = (f"AutoControl is about to run a destructive tool " - f"'{name}'. Continue?") - try: - response = self.request_elicitation( - message=prompt, requested_schema={"type": "object", - "properties": {}}, - timeout=60.0, - ) - except (RuntimeError, TimeoutError) as error: - autocontrol_logger.info( - "MCP elicitation for %s failed (%r) — refusing call", - name, error, - ) - raise _MCPError( - -32000, f"User confirmation unavailable for {name}", - ) from error - action = response.get("action") if isinstance(response, dict) else None - if action != "accept": - raise _MCPError(-32000, f"User declined to run {name}: action={action!r}") - del arguments # available for future per-arg confirmation policies - def _build_call_context(self, msg_id: Any, params: Dict[str, Any]) -> ToolCallContext: meta = params.get("_meta") if isinstance(params.get("_meta"), @@ -876,117 +662,6 @@ def _build_call_context(self, msg_id: Any, ) -def _to_content_blocks(result: Any) -> List[Dict[str, Any]]: - """Normalise a tool's return value into MCP ``content`` blocks.""" - if isinstance(result, MCPContent): - return [result.to_dict()] - if isinstance(result, list) and result and \ - all(isinstance(item, MCPContent) for item in result): - return [item.to_dict() for item in result] - return [{"type": "text", "text": _stringify_result(result)}] - - -def _stringify_result(value: Any) -> str: - """Convert a tool return value into a model-readable string.""" - if isinstance(value, str): - return value - try: - return json.dumps(value, ensure_ascii=False, default=str) - except (TypeError, ValueError): - return repr(value) - - -def _confirm_destructive_enabled() -> bool: - """Return True when the operator wants destructive tools gated on user OK.""" - raw = os.environ.get("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "") - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -def _capture_error_screenshot(tool_name: str) -> Optional[str]: - """Save a debug screenshot when JE_AUTOCONTROL_MCP_ERROR_SHOTS is set.""" - debug_dir = os.environ.get("JE_AUTOCONTROL_MCP_ERROR_SHOTS") - if not debug_dir: - return None - target_dir = os.path.realpath(os.fspath(debug_dir)) - try: - os.makedirs(target_dir, exist_ok=True) - except OSError as error: - autocontrol_logger.info( - "MCP error-screenshot dir unavailable: %r", error, - ) - return None - filename = f"{tool_name}_{int(time.time() * 1000)}.png" - path = os.path.join(target_dir, filename) - try: - from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot - pil_screenshot(file_path=path) - except (OSError, RuntimeError, ValueError, AttributeError, - ImportError) as error: - autocontrol_logger.info( - "MCP failed to capture error screenshot: %r", error, - ) - return None - return path - - -def _file_uri_to_path(uri: str) -> Optional[str]: - """Convert a ``file://`` URI to a local filesystem path; ``None`` otherwise.""" - if not isinstance(uri, str) or not uri.startswith("file://"): - return None - from urllib.parse import unquote, urlparse - parsed = urlparse(uri) - raw_path = unquote(parsed.path) - # Windows: file:///C:/foo strips the leading slash before the drive letter. - if sys.platform.startswith("win") and raw_path.startswith("/") and \ - len(raw_path) > 2 and raw_path[2] == ":": - raw_path = raw_path[1:] - return raw_path or None - - -def _is_hashable(value: Any) -> bool: - """Return True when ``value`` can be used as a dict key.""" - try: - hash(value) - except TypeError: - return False - return True - - -def _coerce_params(raw: Any, msg_id: Any) -> tuple: - """Normalise JSON-RPC ``params`` to a dict. - - Returns ``(params, error_line)``. Every handler here expects an object; - a non-object ``params`` yields a ``-32602`` error line for a request and - an empty dict for a notification (which cannot carry an error reply). - """ - if raw is None: - return {}, None - if isinstance(raw, dict): - return raw, None - if msg_id is None: - return {}, None - return {}, _error_response(msg_id, -32602, "Invalid params: expected an object") - - -def _notification_message(method: str, params: Dict[str, Any]) -> str: - return json.dumps({"jsonrpc": "2.0", "method": method, "params": params}, - ensure_ascii=False, default=str) - - -def _result_response(msg_id: Any, result: Any) -> str: - return json.dumps( - {"jsonrpc": "2.0", "id": msg_id, "result": result}, - ensure_ascii=False, default=str, - ) - - -def _error_response(msg_id: Any, code: int, message: str) -> str: - return json.dumps({ - "jsonrpc": "2.0", "id": msg_id, - "error": {"code": code, "message": message}, - }, ensure_ascii=False) - - def start_mcp_stdio_server() -> MCPServer: """Start a stdio MCP server in the foreground; blocks until EOF.""" server = MCPServer() diff --git a/je_auto_control/utils/remote_desktop/__init__.py b/je_auto_control/utils/remote_desktop/__init__.py index c6c45839..f700ef90 100644 --- a/je_auto_control/utils/remote_desktop/__init__.py +++ b/je_auto_control/utils/remote_desktop/__init__.py @@ -19,8 +19,11 @@ from je_auto_control.utils.remote_desktop.file_transfer import ( FileReceiver, FileSendResult, FileTransferError, send_file, ) -from je_auto_control.utils.remote_desktop.host import ( - PendingViewer, PendingViewerCallback, RemoteDesktopHost, +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import ( + PendingViewer, PendingViewerCallback, +) +from je_auto_control.utils.remote_desktop.host_capture import ( list_host_monitors, ) from je_auto_control.utils.remote_desktop.host_id import ( diff --git a/je_auto_control/utils/remote_desktop/host.py b/je_auto_control/utils/remote_desktop/host.py index e1ba0863..c9c055f2 100644 --- a/je_auto_control/utils/remote_desktop/host.py +++ b/je_auto_control/utils/remote_desktop/host.py @@ -1,23 +1,20 @@ """TCP host that streams JPEG frames and applies viewer input.""" -import collections import json import socket import ssl import threading import time -from dataclasses import dataclass -from io import BytesIO -from typing import Any, Callable, Deque, Dict, List, Mapping, Optional, Sequence +from typing import Any, Callable, List, Mapping, Optional, Sequence from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop.audio import ( AudioCapture, AudioCaptureConfig, ) from je_auto_control.utils.remote_desktop.auth import ( - make_nonce, verify_response, + verify_response, ) from je_auto_control.utils.remote_desktop.clipboard_sync import ( - ClipboardSyncError, decode as decode_clipboard, encode_image, encode_text, + encode_image, encode_text, ) from je_auto_control.utils.remote_desktop.file_transfer import ( FileReceiver, FileTransferError, send_file, @@ -26,87 +23,42 @@ load_or_create_host_id, validate_host_id, ) from je_auto_control.utils.remote_desktop.input_dispatch import ( - InputDispatchError, dispatch_input, + dispatch_input, ) from je_auto_control.utils.remote_desktop.protocol import ( - AuthenticationError, MessageType, ProtocolError, + MessageType, ) from je_auto_control.utils.remote_desktop.resume_tokens import ( ResumeTokenStore, ) from je_auto_control.utils.remote_desktop.video_codec import ( - CODEC_JPEG, CodecProvider, JpegPassthrough, codec_tag, + CodecProvider, JpegPassthrough, ) from je_auto_control.utils.remote_desktop.transport import ( MessageChannel, TcpMessageChannel, ) +from je_auto_control.utils.remote_desktop.host_access import ( + PendingViewerCallback, _AUTH_TIMEOUT_S, _candidate_totp_codes, + _compile_ip_allowlist, _ip_in_allowlist, +) +from je_auto_control.utils.remote_desktop.host_capture import ( + CursorProvider, FrameProductionMixin, FrameProvider, + _DEFAULT_QUALITY, _default_frame_provider, + _resolve_cursor_provider, _resolve_monitor_region, +) +from je_auto_control.utils.remote_desktop.host_client import ( + _ClientHandler, +) -FrameProvider = Callable[[], bytes] InputDispatcher = Callable[[Mapping[str, Any]], Any] -CursorProvider = Callable[[], Optional[Sequence[int]]] -"""Return ``(x, y)`` in host screen coordinates, or ``None`` to skip a tick.""" - -_CURSOR_POLL_INTERVAL_S = 1.0 / 30.0 # 30 Hz: smooth, low CPU on idle. - - -@dataclass(frozen=True) -class PendingViewer: - """Snapshot of an authenticated viewer awaiting host approval. - - Passed to the ``on_pending_viewer`` callback after the HMAC handshake - succeeds but before the host starts streaming frames. The callback's - return value is interpreted as: - - * ``True`` / ``"full"`` → admit with full control - * ``"view_only"`` → admit, but drop incoming INPUT messages - * ``False`` / ``None`` / etc. → reject - """ - address: tuple - host_id: str - transport: str = "tcp" - - -PendingViewerCallback = Callable[[PendingViewer], Any] -"""Callback signature: see :class:`PendingViewer` for return value semantics.""" - - -PERMISSION_FULL = "full" -PERMISSION_VIEW_ONLY = "view_only" -PERMISSION_DENIED = "denied" - - -def _interpret_approval(result: Any) -> str: - """Map an approval-callback return value to a permission string. - - Backward compatibility: any truthy value other than the literal - ``"view_only"`` / ``"denied"`` strings is treated as full-control - admit. Falsy values are denied. - """ - if result == PERMISSION_VIEW_ONLY: - return PERMISSION_VIEW_ONLY - if result == PERMISSION_DENIED or not result: - return PERMISSION_DENIED - return PERMISSION_FULL - - -_AUTH_TIMEOUT_S = 60.0 # accept() 的輪詢間隔，用來定期檢查 _shutdown 旗標。 # How long accept() blocks before re-checking the _shutdown flag. _ACCEPT_POLL_TIMEOUT_S = 0.5 -_DEFAULT_QUALITY = 70 _FILE_MSG_TYPES = frozenset({ MessageType.FILE_BEGIN, MessageType.FILE_CHUNK, MessageType.FILE_END, }) -def _candidate_totp_codes(secret: str): - """Yield TOTP codes within ±1 step of the current 30-second window.""" - from je_auto_control.utils.remote_desktop.totp import generate_code - now = time.time() - for delta in (-1, 0, 1): - yield generate_code(secret, at=now + (delta * 30.0)) - - def _validate_host_args(token: str, fps: float, quality: int) -> None: """Throw early on bad constructor args so the host never starts broken.""" if not isinstance(token, str) or not token: @@ -117,522 +69,7 @@ def _validate_host_args(token: str, fps: float, quality: int) -> None: raise ValueError("quality must be in [1, 95]") -def list_host_monitors() -> List[Dict[str, Any]]: - """Headless helper: return every monitor's geometry. - - Index 0 spans all monitors (the ``mss`` convention). Returns an - empty list if ``mss`` is not installed, so GUI callers can show a - disabled control instead of crashing. - """ - try: - import mss - except ImportError: - return [] - with mss.mss() as sct: - return [ - { - "index": index, "left": int(monitor["left"]), - "top": int(monitor["top"]), - "width": int(monitor["width"]), - "height": int(monitor["height"]), - "is_combined": index == 0, - } - for index, monitor in enumerate(sct.monitors) - ] - - -def _resolve_monitor_region( - monitor_index: int) -> Optional[Sequence[int]]: - """Map an ``mss`` monitor index to ``(x, y, width, height)``. - - Returns ``None`` (full-screen capture fallback) when ``mss`` is - not available so a stock install still works. - """ - try: - import mss - except ImportError: - autocontrol_logger.warning( - "remote_desktop monitor_index=%d ignored: mss not installed", - monitor_index, - ) - return None - with mss.mss() as sct: - if monitor_index < 0 or monitor_index >= len(sct.monitors): - raise ValueError( - f"monitor_index {monitor_index} out of range " - f"(0..{len(sct.monitors) - 1})" - ) - mon = sct.monitors[monitor_index] - return ( - int(mon["left"]), int(mon["top"]), - int(mon["width"]), int(mon["height"]), - ) - - -def _compile_ip_allowlist( - entries: Optional[Sequence[str]]) -> Optional[List[Any]]: - """Pre-parse ``entries`` into ``ip_address`` / ``ip_network`` objects. - - ``None`` or an empty list → no filtering (allow all). Entries are - plain IPs (``"192.168.1.10"``) or CIDR ranges (``"10.0.0.0/8"``); - unparseable entries are dropped with a warning so a typo doesn't - silently broaden access. - """ - if not entries: - return None - import ipaddress - compiled: List[Any] = [] - for entry in entries: - text = str(entry).strip() - if not text: - continue - try: - if "/" in text: - compiled.append(ipaddress.ip_network(text, strict=False)) - else: - compiled.append(ipaddress.ip_address(text)) - except ValueError: - autocontrol_logger.warning( - "remote_desktop ip_allowlist entry rejected: %r", text, - ) - return compiled or None - - -def _ip_in_allowlist(allowlist: Optional[List[Any]], peer_ip: str) -> bool: - """Return True when ``peer_ip`` matches any allowlist entry (or no list).""" - if not allowlist: - return True - import ipaddress - try: - addr = ipaddress.ip_address(peer_ip) - except ValueError: - return False - for entry in allowlist: - if isinstance(entry, (ipaddress.IPv4Network, ipaddress.IPv6Network)): - if addr in entry: - return True - elif entry == addr: - return True - return False - - -def _resolve_cursor_provider( - explicit: Optional[CursorProvider], - enabled: bool) -> Optional[CursorProvider]: - """Pick the cursor provider — explicit > default > disabled.""" - if explicit is not None: - return explicit - return _default_cursor_provider() if enabled else None - - -def _default_cursor_provider() -> CursorProvider: - """Build a cursor-position poller using the project's mouse wrapper. - - The wrapper is imported lazily inside the closure so importing this - module on platforms where mouse capture is unavailable does not - blow up. Returns ``None`` on read failures so the broadcast loop - silently skips the tick instead of crashing the host. - """ - def provide() -> Optional[Sequence[int]]: - try: - from je_auto_control.wrapper.auto_control_mouse import ( - get_mouse_position, - ) - except ImportError: - return None - try: - return get_mouse_position() - except (OSError, RuntimeError, AttributeError): - return None - return provide - - -def _default_frame_provider(region: Optional[Sequence[int]] = None, - quality: int = _DEFAULT_QUALITY) -> FrameProvider: - """Build a JPEG frame producer using PIL.ImageGrab.""" - def provide() -> bytes: - from PIL import ImageGrab # local import: not needed for unit tests - if region is not None: - x, y, width, height = (int(v) for v in region) - bbox = (x, y, x + width, y + height) - image = ImageGrab.grab(bbox=bbox, all_screens=True) - else: - image = ImageGrab.grab(all_screens=True) - if image.mode != "RGB": - image = image.convert("RGB") - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=int(quality)) - return buffer.getvalue() - return provide - - -class _ClientHandler: - """Per-connection auth + input-receive + frame-send state.""" - - _AUDIO_QUEUE_MAXLEN = 50 # ~2.5 s of buffered chunks at 50 ms each - - def __init__(self, host: "RemoteDesktopHost", - channel: MessageChannel, address) -> None: - self._host = host - self._channel = channel - self._address = address - self._shutdown = threading.Event() - self._sender_thread: Optional[threading.Thread] = None - self._receiver_thread: Optional[threading.Thread] = None - self._audio_queue: Deque[bytes] = collections.deque( - maxlen=self._AUDIO_QUEUE_MAXLEN, - ) - self._audio_lock = threading.Lock() - self._audio_event = threading.Event() - self._audio_sender_thread: Optional[threading.Thread] = None - self.authenticated = False - # Phase 5.3: per-client permission set by the approval callback. - # Default is full control so legacy callers (no callback) keep - # the prior behaviour. - self.permission = PERMISSION_FULL - - @property - def address(self): - return self._address - - def start(self) -> None: - """Run auth (with optional host approval), then start the loops.""" - try: - self._authenticate() - except (AuthenticationError, ProtocolError, OSError) as error: - autocontrol_logger.info( - "remote_desktop client %s rejected: %r", self._address, error, - ) - self._close() - return - self.authenticated = True - # The initial cursor + frame are seeded from _send_loop (the - # per-client sender thread), not here. start() runs on the shared - # accept thread with the socket timeout already cleared, so sending a - # full-screen JPEG to a viewer that authenticates then stops reading - # would block every new accept until its send buffer drains. - self._sender_thread = threading.Thread( - target=self._send_loop, name="rd-sender", daemon=True, - ) - self._receiver_thread = threading.Thread( - target=self._recv_loop, name="rd-recv", daemon=True, - ) - self._sender_thread.start() - self._receiver_thread.start() - if self._host._audio_config.enabled: - self._audio_sender_thread = threading.Thread( - target=self._audio_send_loop, name="rd-audio", daemon=True, - ) - self._audio_sender_thread.start() - - def _send_initial_frame(self) -> None: - """Forward the most recent encoded frame so new clients aren't blank. - - Motion-dedup in :meth:`_capture_loop` means a static desktop - only bumps ``_latest_seq`` once; replaying that frame to the - new client keeps them from sitting on a black popup until the - host moves something. - """ - with self._host._frame_cond: - frame = self._host._latest_frame - if frame is None: - return - try: - self._channel.send_typed(MessageType.FRAME, frame) - except OSError: - pass - - def push_audio(self, chunk: bytes) -> None: - """Enqueue a PCM chunk for delivery; oldest dropped if queue is full.""" - if self._shutdown.is_set() or not self.authenticated: - return - with self._audio_lock: - self._audio_queue.append(chunk) - self._audio_event.set() - - def stop(self) -> None: - """Signal threads and close the socket.""" - self._shutdown.set() - with self._host._frame_cond: - self._host._frame_cond.notify_all() - self._audio_event.set() - self._close() - - def _resolve_permission(self) -> str: - """Run the host's optional approval callback after token auth. - - Returns one of :data:`PERMISSION_FULL` / :data:`PERMISSION_VIEW_ONLY` - (admit) or :data:`PERMISSION_DENIED` (reject). The caller is - expected to send ``AUTH_FAIL`` and raise on denial — keeping - that wire-level handling inside :meth:`_authenticate` so the - viewer sees the rejection before it has a chance to flip into - the post-handshake state where ``AUTH_FAIL`` is ignored. - """ - callback = self._host._on_pending_viewer - if callback is None: - return PERMISSION_FULL - pending = PendingViewer( - address=tuple(self._address) if self._address else (), - host_id=self._host.host_id, - transport=self._host._transport_name(), - ) - try: - return _interpret_approval(callback(pending)) - except (RuntimeError, ValueError, TypeError) as error: - autocontrol_logger.info( - "remote_desktop approval callback raised for %s: %r", - self._address, error, - ) - return PERMISSION_DENIED - - def _authenticate(self) -> None: - nonce = make_nonce() - self._channel.settimeout(_AUTH_TIMEOUT_S) - self._channel.send_typed(MessageType.AUTH_CHALLENGE, nonce) - msg_type, payload = self._channel.read_typed() - if msg_type is not MessageType.AUTH_RESPONSE: - self._channel.send_typed(MessageType.AUTH_FAIL, - b"expected AUTH_RESPONSE") - raise AuthenticationError( - f"expected AUTH_RESPONSE, got {msg_type.name}" - ) - # Phase 6.6: a viewer reconnecting with a valid resume token - # signs with that token directly — host short-circuits the - # approval popup and reuses the saved permission. - resumed = self._host._try_consume_resume(nonce, payload) - if resumed is not None: - self.permission = resumed - else: - if not self._host._verify_token(nonce, payload): - self._channel.send_typed(MessageType.AUTH_FAIL, b"bad token") - raise AuthenticationError("bad token") - # Host operator gates the session *before* AUTH_OK so the - # viewer surfaces the rejection as an AuthenticationError - # instead of connecting and then mysteriously disconnecting. - permission = self._resolve_permission() - if permission == PERMISSION_DENIED: - self._channel.send_typed( - MessageType.AUTH_FAIL, b"rejected by host", - ) - raise AuthenticationError("rejected by host") - self.permission = permission - # Issue a fresh resume token so the viewer can reconnect - # within the store's TTL without the approval popup. - resume_token = self._host._resume_store.issue(self.permission) - ok_payload = json.dumps( - {"host_id": self._host.host_id, - "resume_token": resume_token, - "resume_ttl": self._host._resume_store.ttl, - "codec": self._host._codec_provider.name}, - ensure_ascii=False, - ).encode("utf-8") - self._channel.send_typed(MessageType.AUTH_OK, ok_payload) - self._channel.settimeout(None) - - def _send_loop(self) -> None: - # Seed the viewer with the latest cursor + frame on this per-client - # sender thread (Phase 2.3: motion-aware capture only bumps the seq on - # change, so a viewer joining a static desktop would otherwise sit - # blank until the host moves). Doing it here rather than in start() - # keeps a slow-reading viewer from stalling the accept thread. - self._host._send_initial_cursor(self) - self._send_initial_frame() - last_sent = 0 - while not self._shutdown.is_set(): - with self._host._frame_cond: - while (not self._shutdown.is_set() - and self._host._latest_seq <= last_sent): - self._host._frame_cond.wait(timeout=0.5) - if self._shutdown.is_set(): - return - frame = self._host._latest_frame - seq = self._host._latest_seq - if frame is None: - continue - try: - self._channel.send_typed(MessageType.FRAME, frame) - except OSError as error: - autocontrol_logger.info( - "remote_desktop send to %s failed: %r", - self._address, error, - ) - self.stop() - return - last_sent = seq - - def _audio_send_loop(self) -> None: - while not self._shutdown.is_set(): - self._audio_event.wait(timeout=0.5) - if self._shutdown.is_set(): - return - while True: - with self._audio_lock: - if not self._audio_queue: - self._audio_event.clear() - break - chunk = self._audio_queue.popleft() - try: - self._channel.send_typed(MessageType.AUDIO, chunk) - except OSError as error: - autocontrol_logger.info( - "remote_desktop audio send to %s failed: %r", - self._address, error, - ) - self.stop() - return - - def _recv_loop(self) -> None: - while not self._shutdown.is_set(): - try: - msg_type, payload = self._channel.read_typed() - except (OSError, ProtocolError) as error: - if not self._shutdown.is_set(): - autocontrol_logger.info( - "remote_desktop recv from %s ended: %r", - self._address, error, - ) - self.stop() - return - self._route_incoming(msg_type, payload) - - def _route_incoming(self, msg_type: MessageType, payload: bytes) -> None: - """Dispatch one received message to the matching handler.""" - if msg_type is MessageType.PING: - return - if msg_type is MessageType.INPUT: - # Phase 5.3: drop input from view-only viewers so they can - # watch but cannot drive the mouse / keyboard. - if self.permission != PERMISSION_VIEW_ONLY: - self._handle_input_payload(payload) - return - if msg_type is MessageType.CLIPBOARD: - self._handle_clipboard_payload(payload) - return - if msg_type is MessageType.CHAT: - self._handle_chat_payload(payload) - return - if msg_type is MessageType.USB_LIST_REQUEST: - self._handle_usb_list_request() - return - if msg_type in _FILE_MSG_TYPES: - self._handle_file_payload(msg_type, payload) - return - autocontrol_logger.info( - "remote_desktop unexpected msg %s from %s", - msg_type.name, self._address, - ) - - def _handle_file_payload(self, msg_type: MessageType, - payload: bytes) -> None: - receiver = self._host._ensure_file_receiver() - try: - if msg_type is MessageType.FILE_BEGIN: - receiver.handle_begin(payload) - elif msg_type is MessageType.FILE_CHUNK: - receiver.handle_chunk(payload) - elif msg_type is MessageType.FILE_END: - receiver.handle_end(payload) - except FileTransferError as error: - autocontrol_logger.info( - "remote_desktop bad file message from %s: %r", - self._address, error, - ) - - def _handle_usb_list_request(self) -> None: - """Phase 6.9: enumerate the host's USB devices and ship the list back. - - Uses the existing :func:`list_usb_devices` helper so we get the - same cross-platform behaviour as the standalone USB module. - Errors fall back to an empty payload — the viewer should - treat that as "host has no usable USB backend" rather than - crashing. - """ - try: - from je_auto_control.utils.usb import list_usb_devices - result = list_usb_devices() - body = { - "backend": result.backend, - "devices": [d.to_dict() for d in result.devices], - } - except (ImportError, OSError, RuntimeError) as error: - autocontrol_logger.info( - "usb_list from %s failed: %r", self._address, error, - ) - body = {"backend": "unavailable", "devices": []} - try: - self._channel.send_typed( - MessageType.USB_LIST_RESPONSE, - json.dumps(body, ensure_ascii=False).encode("utf-8"), - ) - except OSError: - pass - - def _handle_chat_payload(self, payload: bytes) -> None: - """Forward viewer-originated chat to the host's optional callback.""" - callback = self._host._on_chat - if callback is None: - return - try: - body = json.loads(payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - return - if not isinstance(body, dict): - return - text = body.get("text") - sender = body.get("sender", "viewer") - if not isinstance(text, str) or not text: - return - try: - callback(str(sender), text) - except Exception: # noqa: BLE001 callback isolation - autocontrol_logger.exception( - "remote_desktop on_chat callback raised" - ) - - def _handle_clipboard_payload(self, payload: bytes) -> None: - try: - kind, data = decode_clipboard(payload) - except ClipboardSyncError as error: - autocontrol_logger.info( - "remote_desktop bad CLIPBOARD from %s: %r", - self._address, error, - ) - return - try: - self._host._apply_clipboard(kind, data) - except (OSError, RuntimeError, TypeError, ValueError) as error: - autocontrol_logger.warning( - "remote_desktop clipboard apply failed for %s: %r", - self._address, error, - ) - - def _handle_input_payload(self, payload: bytes) -> None: - try: - message = json.loads(payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - autocontrol_logger.info( - "remote_desktop bad INPUT from %s: %r", - self._address, error, - ) - return - try: - self._host._dispatch(message) - except InputDispatchError as error: - autocontrol_logger.info( - "remote_desktop rejected INPUT from %s: %r", - self._address, error, - ) - except (OSError, RuntimeError, ValueError, TypeError) as error: - autocontrol_logger.warning( - "remote_desktop input apply failed for %s: %r", - self._address, error, - ) - - def _close(self) -> None: - self._channel.close() - - -class RemoteDesktopHost: +class RemoteDesktopHost(FrameProductionMixin): """Stream the screen to authenticated viewers and apply their input. The instance owns three kinds of threads: one accept loop, one @@ -1030,67 +467,6 @@ def _apply_clipboard(self, kind: str, data: Any) -> None: # internals ----------------------------------------------------------- - def _cursor_loop(self) -> None: - """Poll cursor position at ~30 Hz and push it to viewers as JSON.""" - provider = self._cursor_provider - if provider is None: - return - while not self._shutdown.is_set(): - position = provider() - if position is not None and len(position) >= 2: - payload = json.dumps( - {"x": int(position[0]), "y": int(position[1]), - "visible": True}, - ).encode("utf-8") - with self._cursor_lock: - is_new = payload != self._latest_cursor_payload - self._latest_cursor_payload = payload - if is_new: - self._broadcast_cursor(payload) - if self._shutdown.wait(timeout=_CURSOR_POLL_INTERVAL_S): - return - - def _broadcast_cursor(self, payload: bytes) -> None: - """Send a CURSOR message to every authenticated client. - - Errors per-client are swallowed — a flaky viewer should not - kill the cursor stream to healthy peers. - """ - with self._clients_lock: - clients = [c for c in self._clients - if c.authenticated and not c._shutdown.is_set()] - for client in clients: - try: - client._channel.send_typed(MessageType.CURSOR, payload) - except OSError: - continue - - def broadcast_viewer_cursor(self, viewer_id: str, - x: int, y: int) -> int: - """Phase 5.1: relay one viewer's cursor position to every other viewer. - - Typically called by :class:`MultiViewerHost` when several - viewers share a session so each viewer's overlay can show the - other operators' pointers (Figma / Google Docs style). The - viewer_id is opaque to the host — viewers use it to colour-key - their overlay. - """ - payload = json.dumps( - {"x": int(x), "y": int(y), "visible": True, - "viewer_id": str(viewer_id)}, - ).encode("utf-8") - with self._clients_lock: - clients = [c for c in self._clients - if c.authenticated and not c._shutdown.is_set()] - sent = 0 - for client in clients: - try: - client._channel.send_typed(MessageType.CURSOR, payload) - sent += 1 - except OSError: - continue - return sent - def broadcast_chat(self, text: str, sender: str = "host") -> int: """Phase 5.2: send a chat message to every connected viewer. @@ -1117,23 +493,6 @@ def broadcast_chat(self, text: str, sender: str = "host") -> int: ) return sent - def _send_initial_cursor(self, client: "_ClientHandler") -> None: - """Push the latest known cursor position to a fresh client. - - Sending unconditionally on auth means the viewer sees a cursor - immediately instead of waiting up to ~1 s for the next position - change. Safe to call when the cursor loop is disabled — we - only send if there's a payload to send. - """ - with self._cursor_lock: - payload = self._latest_cursor_payload - if payload is None: - return - try: - client._channel.send_typed(MessageType.CURSOR, payload) - except OSError: - pass - def _ip_allowed(self, address) -> bool: """Apply the Phase 4.3 allowlist; log + reject silently otherwise.""" peer_ip = address[0] if address else "" @@ -1251,58 +610,6 @@ def _maybe_wrap_tls(self, client_sock: socket.socket, pass return None - def _capture_loop(self) -> None: - next_tick = time.monotonic() - last_frame_hash: Optional[int] = None - while not self._shutdown.is_set(): - try: - frame = self._frame_provider() - except (OSError, RuntimeError, ValueError) as error: - autocontrol_logger.warning( - "remote_desktop frame capture failed: %r", error, - ) - self._shutdown.wait(self._period) - continue - # Phase 2.3: drop frames that are byte-identical to the - # previous capture. A static desktop produces the same JPEG - # every tick (JPEG is deterministic for identical input), - # so this skip costs nothing extra on motion-heavy - # workloads and saves a full FPS-worth of TCP / encoder - # bandwidth at idle. - frame_hash = hash(frame) - if frame_hash != last_frame_hash: - # Phase 6.8: hand the JPEG to the configured codec. - # JpegPassthrough yields the bytes unchanged so the - # wire format stays identical for stock clients. - for encoded in self._encode_for_wire(frame): - with self._frame_cond: - self._latest_frame = encoded - self._latest_seq += 1 - self._frame_cond.notify_all() - last_frame_hash = frame_hash - next_tick += self._period - sleep_for = max(0.0, next_tick - time.monotonic()) - if sleep_for <= 0.0: - next_tick = time.monotonic() - self._shutdown.wait(sleep_for) - - def _encode_for_wire(self, jpeg_bytes: bytes): - """Wrap codec output with a 1-byte tag (skipped for JPEG).""" - provider = self._codec_provider - if provider.name == CODEC_JPEG: - yield jpeg_bytes # legacy wire format: no tag, raw JPEG - return - tag = bytes([codec_tag(provider.name)]) - try: - packets = provider.encode_jpeg(jpeg_bytes) - except (OSError, RuntimeError, ValueError) as error: - autocontrol_logger.warning( - "remote_desktop codec %s failed: %r", provider.name, error, - ) - return - for packet in packets: - yield tag + bytes(packet) - def _reap_dead_clients(self) -> None: with self._clients_lock: self._clients = [c for c in self._clients diff --git a/je_auto_control/utils/remote_desktop/host_access.py b/je_auto_control/utils/remote_desktop/host_access.py new file mode 100644 index 00000000..2dce4d79 --- /dev/null +++ b/je_auto_control/utils/remote_desktop/host_access.py @@ -0,0 +1,105 @@ +"""Viewer approval and access control for the TCP remote-desktop host. + +The gate a viewer passes through after the HMAC handshake and before any +frame is sent: the snapshot handed to the approval callback, how that +callback's return value maps to a permission, the TOTP codes accepted for +a share code, and the IP allowlist. Shared by :mod:`host` and +:mod:`host_client`, which is why it is its own module rather than living +in either. +""" +import time +from dataclasses import dataclass +from typing import Any, Callable, List, Optional, Sequence + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +@dataclass(frozen=True) +class PendingViewer: + """Snapshot of an authenticated viewer awaiting host approval. + + Passed to the ``on_pending_viewer`` callback after the HMAC handshake + succeeds but before the host starts streaming frames. The callback's + return value is interpreted as: + + * ``True`` / ``"full"`` → admit with full control + * ``"view_only"`` → admit, but drop incoming INPUT messages + * ``False`` / ``None`` / etc. → reject + """ + address: tuple + host_id: str + transport: str = "tcp" + +PendingViewerCallback = Callable[[PendingViewer], Any] +"""Callback signature: see :class:`PendingViewer` for return value semantics.""" + +PERMISSION_FULL = "full" +PERMISSION_VIEW_ONLY = "view_only" +PERMISSION_DENIED = "denied" + +def _interpret_approval(result: Any) -> str: + """Map an approval-callback return value to a permission string. + + Backward compatibility: any truthy value other than the literal + ``"view_only"`` / ``"denied"`` strings is treated as full-control + admit. Falsy values are denied. + """ + if result == PERMISSION_VIEW_ONLY: + return PERMISSION_VIEW_ONLY + if result == PERMISSION_DENIED or not result: + return PERMISSION_DENIED + return PERMISSION_FULL + +_AUTH_TIMEOUT_S = 60.0 + +def _candidate_totp_codes(secret: str): + """Yield TOTP codes within ±1 step of the current 30-second window.""" + from je_auto_control.utils.remote_desktop.totp import generate_code + now = time.time() + for delta in (-1, 0, 1): + yield generate_code(secret, at=now + (delta * 30.0)) + +def _compile_ip_allowlist( + entries: Optional[Sequence[str]]) -> Optional[List[Any]]: + """Pre-parse ``entries`` into ``ip_address`` / ``ip_network`` objects. + + ``None`` or an empty list → no filtering (allow all). Entries are + plain IPs (``"192.168.1.10"``) or CIDR ranges (``"10.0.0.0/8"``); + unparseable entries are dropped with a warning so a typo doesn't + silently broaden access. + """ + if not entries: + return None + import ipaddress + compiled: List[Any] = [] + for entry in entries: + text = str(entry).strip() + if not text: + continue + try: + if "/" in text: + compiled.append(ipaddress.ip_network(text, strict=False)) + else: + compiled.append(ipaddress.ip_address(text)) + except ValueError: + autocontrol_logger.warning( + "remote_desktop ip_allowlist entry rejected: %r", text, + ) + return compiled or None + +def _ip_in_allowlist(allowlist: Optional[List[Any]], peer_ip: str) -> bool: + """Return True when ``peer_ip`` matches any allowlist entry (or no list).""" + if not allowlist: + return True + import ipaddress + try: + addr = ipaddress.ip_address(peer_ip) + except ValueError: + return False + for entry in allowlist: + if isinstance(entry, (ipaddress.IPv4Network, ipaddress.IPv6Network)): + if addr in entry: + return True + elif entry == addr: + return True + return False diff --git a/je_auto_control/utils/remote_desktop/host_capture.py b/je_auto_control/utils/remote_desktop/host_capture.py new file mode 100644 index 00000000..01ae4f1c --- /dev/null +++ b/je_auto_control/utils/remote_desktop/host_capture.py @@ -0,0 +1,280 @@ +"""Frame and cursor production for the TCP remote-desktop host. + +What the host streams, as opposed to how it streams it: monitor +enumeration, mapping a monitor index to a capture region, and the default +providers that turn a screen grab into JPEG bytes and read the cursor +position. Every one of them degrades to a working fallback when the +optional dependency (``mss``, Pillow) is missing, so a stock install +still hosts. +""" +import json +import time +from io import BytesIO +from typing import ( + TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, +) + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.protocol import MessageType +from je_auto_control.utils.remote_desktop.video_codec import ( + CODEC_JPEG, codec_tag, +) + +if TYPE_CHECKING: # avoids a runtime cycle: host_client is a sibling + from je_auto_control.utils.remote_desktop.host_client import ( + _ClientHandler, + ) + + +FrameProvider = Callable[[], bytes] + +CursorProvider = Callable[[], Optional[Sequence[int]]] +"""Return ``(x, y)`` in host screen coordinates, or ``None`` to skip a tick.""" + +_DEFAULT_QUALITY = 70 + +_CURSOR_POLL_INTERVAL_S = 1.0 / 30.0 # 30 Hz: smooth, low CPU on idle. + +def list_host_monitors() -> List[Dict[str, Any]]: + """Headless helper: return every monitor's geometry. + + Index 0 spans all monitors (the ``mss`` convention). Returns an + empty list if ``mss`` is not installed, so GUI callers can show a + disabled control instead of crashing. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + try: + grabber = mss_grabber() + except ImportError: + return [] + with grabber as sct: + return [ + { + "index": index, "left": int(monitor["left"]), + "top": int(monitor["top"]), + "width": int(monitor["width"]), + "height": int(monitor["height"]), + "is_combined": index == 0, + } + for index, monitor in enumerate(sct.monitors) + ] + +def _resolve_monitor_region( + monitor_index: int) -> Optional[Sequence[int]]: + """Map an ``mss`` monitor index to ``(x, y, width, height)``. + + Returns ``None`` (full-screen capture fallback) when ``mss`` is + not available so a stock install still works. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + try: + grabber = mss_grabber() + except ImportError: + autocontrol_logger.warning( + "remote_desktop monitor_index=%d ignored: mss not installed", + monitor_index, + ) + return None + with grabber as sct: + if monitor_index < 0 or monitor_index >= len(sct.monitors): + raise ValueError( + f"monitor_index {monitor_index} out of range " + f"(0..{len(sct.monitors) - 1})" + ) + mon = sct.monitors[monitor_index] + return ( + int(mon["left"]), int(mon["top"]), + int(mon["width"]), int(mon["height"]), + ) + +def _resolve_cursor_provider( + explicit: Optional[CursorProvider], + enabled: bool) -> Optional[CursorProvider]: + """Pick the cursor provider — explicit > default > disabled.""" + if explicit is not None: + return explicit + return _default_cursor_provider() if enabled else None + +def _default_cursor_provider() -> CursorProvider: + """Build a cursor-position poller using the project's mouse wrapper. + + The wrapper is imported lazily inside the closure so importing this + module on platforms where mouse capture is unavailable does not + blow up. Returns ``None`` on read failures so the broadcast loop + silently skips the tick instead of crashing the host. + """ + def provide() -> Optional[Sequence[int]]: + try: + from je_auto_control.wrapper.auto_control_mouse import ( + get_mouse_position, + ) + except ImportError: + return None + try: + return get_mouse_position() + except (OSError, RuntimeError, AttributeError): + return None + return provide + +def _default_frame_provider(region: Optional[Sequence[int]] = None, + quality: int = _DEFAULT_QUALITY) -> FrameProvider: + """Build a JPEG frame producer using the platform's screen grabber.""" + def provide() -> bytes: + # local import: not needed for unit tests + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber + grabber = image_grabber() + if region is not None: + x, y, width, height = (int(v) for v in region) + bbox = (x, y, x + width, y + height) + image = grabber.grab(bbox=bbox, all_screens=True) + else: + image = grabber.grab(all_screens=True) + if image.mode != "RGB": + image = image.convert("RGB") + buffer = BytesIO() + image.save(buffer, format="JPEG", quality=int(quality)) + return buffer.getvalue() + return provide + + +class FrameProductionMixin: + """Frame and cursor production half of :class:`RemoteDesktopHost`. + + The loops that turn the host screen into what viewers receive: the + ~30 Hz cursor poll, the capture loop that paces frames at the + configured fps, and the codec step between a JPEG frame and the bytes + that go on the wire. Requires the host to provide ``_shutdown``, + ``_clients``/``_clients_lock``, ``_frame_provider``, + ``_cursor_provider``, ``_codec``, ``_fps``, ``_latest_frame`` and + ``_frame_lock``. + """ + + def _cursor_loop(self) -> None: + """Poll cursor position at ~30 Hz and push it to viewers as JSON.""" + provider = self._cursor_provider + if provider is None: + return + while not self._shutdown.is_set(): + position = provider() + if position is not None and len(position) >= 2: + payload = json.dumps( + {"x": int(position[0]), "y": int(position[1]), + "visible": True}, + ).encode("utf-8") + with self._cursor_lock: + is_new = payload != self._latest_cursor_payload + self._latest_cursor_payload = payload + if is_new: + self._broadcast_cursor(payload) + if self._shutdown.wait(timeout=_CURSOR_POLL_INTERVAL_S): + return + + def _broadcast_cursor(self, payload: bytes) -> None: + """Send a CURSOR message to every authenticated client. + + Errors per-client are swallowed — a flaky viewer should not + kill the cursor stream to healthy peers. + """ + with self._clients_lock: + clients = [c for c in self._clients + if c.authenticated and not c._shutdown.is_set()] + for client in clients: + try: + client._channel.send_typed(MessageType.CURSOR, payload) + except OSError: + continue + + def broadcast_viewer_cursor(self, viewer_id: str, + x: int, y: int) -> int: + """Phase 5.1: relay one viewer's cursor position to every other viewer. + + Typically called by :class:`MultiViewerHost` when several + viewers share a session so each viewer's overlay can show the + other operators' pointers (Figma / Google Docs style). The + viewer_id is opaque to the host — viewers use it to colour-key + their overlay. + """ + payload = json.dumps( + {"x": int(x), "y": int(y), "visible": True, + "viewer_id": str(viewer_id)}, + ).encode("utf-8") + with self._clients_lock: + clients = [c for c in self._clients + if c.authenticated and not c._shutdown.is_set()] + sent = 0 + for client in clients: + try: + client._channel.send_typed(MessageType.CURSOR, payload) + sent += 1 + except OSError: + continue + return sent + + def _send_initial_cursor(self, client: "_ClientHandler") -> None: + """Push the latest known cursor position to a fresh client. + + Sending unconditionally on auth means the viewer sees a cursor + immediately instead of waiting up to ~1 s for the next position + change. Safe to call when the cursor loop is disabled — we + only send if there's a payload to send. + """ + with self._cursor_lock: + payload = self._latest_cursor_payload + if payload is None: + return + try: + client._channel.send_typed(MessageType.CURSOR, payload) + except OSError: + pass + + def _capture_loop(self) -> None: + next_tick = time.monotonic() + last_frame_hash: Optional[int] = None + while not self._shutdown.is_set(): + try: + frame = self._frame_provider() + except (OSError, RuntimeError, ValueError) as error: + autocontrol_logger.warning( + "remote_desktop frame capture failed: %r", error, + ) + self._shutdown.wait(self._period) + continue + # Phase 2.3: drop frames that are byte-identical to the + # previous capture. A static desktop produces the same JPEG + # every tick (JPEG is deterministic for identical input), + # so this skip costs nothing extra on motion-heavy + # workloads and saves a full FPS-worth of TCP / encoder + # bandwidth at idle. + frame_hash = hash(frame) + if frame_hash != last_frame_hash: + # Phase 6.8: hand the JPEG to the configured codec. + # JpegPassthrough yields the bytes unchanged so the + # wire format stays identical for stock clients. + for encoded in self._encode_for_wire(frame): + with self._frame_cond: + self._latest_frame = encoded + self._latest_seq += 1 + self._frame_cond.notify_all() + last_frame_hash = frame_hash + next_tick += self._period + sleep_for = max(0.0, next_tick - time.monotonic()) + if sleep_for <= 0.0: + next_tick = time.monotonic() + self._shutdown.wait(sleep_for) + + def _encode_for_wire(self, jpeg_bytes: bytes): + """Wrap codec output with a 1-byte tag (skipped for JPEG).""" + provider = self._codec_provider + if provider.name == CODEC_JPEG: + yield jpeg_bytes # legacy wire format: no tag, raw JPEG + return + tag = bytes([codec_tag(provider.name)]) + try: + packets = provider.encode_jpeg(jpeg_bytes) + except (OSError, RuntimeError, ValueError) as error: + autocontrol_logger.warning( + "remote_desktop codec %s failed: %r", provider.name, error, + ) + return + for packet in packets: + yield tag + bytes(packet) diff --git a/je_auto_control/utils/remote_desktop/host_client.py b/je_auto_control/utils/remote_desktop/host_client.py new file mode 100644 index 00000000..f1ddc6db --- /dev/null +++ b/je_auto_control/utils/remote_desktop/host_client.py @@ -0,0 +1,406 @@ +"""Per-connection handler for the TCP remote-desktop host. + +One instance per connected viewer, owning that viewer's auth exchange, +sender thread, audio sender thread and receiver thread, plus the routing +table that turns an inbound message type into the right handler. The +owning :class:`~je_auto_control.utils.remote_desktop.host.RemoteDesktopHost` +is referenced only through the instance passed to ``__init__``, so this +module does not import it. +""" +import collections +import json +import threading +from typing import TYPE_CHECKING, Deque, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.auth import make_nonce +from je_auto_control.utils.remote_desktop.clipboard_sync import ( + ClipboardSyncError, decode as decode_clipboard, +) +from je_auto_control.utils.remote_desktop.file_transfer import ( + FileTransferError, +) +from je_auto_control.utils.remote_desktop.host_access import ( + PERMISSION_DENIED, PERMISSION_FULL, PERMISSION_VIEW_ONLY, + PendingViewer, _AUTH_TIMEOUT_S, _interpret_approval, +) +from je_auto_control.utils.remote_desktop.input_dispatch import ( + InputDispatchError, +) +from je_auto_control.utils.remote_desktop.protocol import ( + AuthenticationError, MessageType, ProtocolError, +) +from je_auto_control.utils.remote_desktop.transport import MessageChannel + +if TYPE_CHECKING: # avoids a runtime cycle: host imports this module + from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost + +_FILE_MSG_TYPES = frozenset({ + MessageType.FILE_BEGIN, MessageType.FILE_CHUNK, MessageType.FILE_END, +}) + + +class _ClientHandler: + """Per-connection auth + input-receive + frame-send state.""" + + _AUDIO_QUEUE_MAXLEN = 50 # ~2.5 s of buffered chunks at 50 ms each + + def __init__(self, host: "RemoteDesktopHost", + channel: MessageChannel, address) -> None: + self._host = host + self._channel = channel + self._address = address + self._shutdown = threading.Event() + self._sender_thread: Optional[threading.Thread] = None + self._receiver_thread: Optional[threading.Thread] = None + self._audio_queue: Deque[bytes] = collections.deque( + maxlen=self._AUDIO_QUEUE_MAXLEN, + ) + self._audio_lock = threading.Lock() + self._audio_event = threading.Event() + self._audio_sender_thread: Optional[threading.Thread] = None + self.authenticated = False + # Phase 5.3: per-client permission set by the approval callback. + # Default is full control so legacy callers (no callback) keep + # the prior behaviour. + self.permission = PERMISSION_FULL + + @property + def address(self): + return self._address + + def start(self) -> None: + """Run auth (with optional host approval), then start the loops.""" + try: + self._authenticate() + except (AuthenticationError, ProtocolError, OSError) as error: + autocontrol_logger.info( + "remote_desktop client %s rejected: %r", self._address, error, + ) + self._close() + return + self.authenticated = True + # The initial cursor + frame are seeded from _send_loop (the + # per-client sender thread), not here. start() runs on the shared + # accept thread with the socket timeout already cleared, so sending a + # full-screen JPEG to a viewer that authenticates then stops reading + # would block every new accept until its send buffer drains. + self._sender_thread = threading.Thread( + target=self._send_loop, name="rd-sender", daemon=True, + ) + self._receiver_thread = threading.Thread( + target=self._recv_loop, name="rd-recv", daemon=True, + ) + self._sender_thread.start() + self._receiver_thread.start() + if self._host._audio_config.enabled: + self._audio_sender_thread = threading.Thread( + target=self._audio_send_loop, name="rd-audio", daemon=True, + ) + self._audio_sender_thread.start() + + def _send_initial_frame(self) -> None: + """Forward the most recent encoded frame so new clients aren't blank. + + Motion-dedup in :meth:`_capture_loop` means a static desktop + only bumps ``_latest_seq`` once; replaying that frame to the + new client keeps them from sitting on a black popup until the + host moves something. + """ + with self._host._frame_cond: + frame = self._host._latest_frame + if frame is None: + return + try: + self._channel.send_typed(MessageType.FRAME, frame) + except OSError: + pass + + def push_audio(self, chunk: bytes) -> None: + """Enqueue a PCM chunk for delivery; oldest dropped if queue is full.""" + if self._shutdown.is_set() or not self.authenticated: + return + with self._audio_lock: + self._audio_queue.append(chunk) + self._audio_event.set() + + def stop(self) -> None: + """Signal threads and close the socket.""" + self._shutdown.set() + with self._host._frame_cond: + self._host._frame_cond.notify_all() + self._audio_event.set() + self._close() + + def _resolve_permission(self) -> str: + """Run the host's optional approval callback after token auth. + + Returns one of :data:`PERMISSION_FULL` / :data:`PERMISSION_VIEW_ONLY` + (admit) or :data:`PERMISSION_DENIED` (reject). The caller is + expected to send ``AUTH_FAIL`` and raise on denial — keeping + that wire-level handling inside :meth:`_authenticate` so the + viewer sees the rejection before it has a chance to flip into + the post-handshake state where ``AUTH_FAIL`` is ignored. + """ + callback = self._host._on_pending_viewer + if callback is None: + return PERMISSION_FULL + pending = PendingViewer( + address=tuple(self._address) if self._address else (), + host_id=self._host.host_id, + transport=self._host._transport_name(), + ) + try: + return _interpret_approval(callback(pending)) + except (RuntimeError, ValueError, TypeError) as error: + autocontrol_logger.info( + "remote_desktop approval callback raised for %s: %r", + self._address, error, + ) + return PERMISSION_DENIED + + def _authenticate(self) -> None: + nonce = make_nonce() + self._channel.settimeout(_AUTH_TIMEOUT_S) + self._channel.send_typed(MessageType.AUTH_CHALLENGE, nonce) + msg_type, payload = self._channel.read_typed() + if msg_type is not MessageType.AUTH_RESPONSE: + self._channel.send_typed(MessageType.AUTH_FAIL, + b"expected AUTH_RESPONSE") + raise AuthenticationError( + f"expected AUTH_RESPONSE, got {msg_type.name}" + ) + # Phase 6.6: a viewer reconnecting with a valid resume token + # signs with that token directly — host short-circuits the + # approval popup and reuses the saved permission. + resumed = self._host._try_consume_resume(nonce, payload) + if resumed is not None: + self.permission = resumed + else: + if not self._host._verify_token(nonce, payload): + self._channel.send_typed(MessageType.AUTH_FAIL, b"bad token") + raise AuthenticationError("bad token") + # Host operator gates the session *before* AUTH_OK so the + # viewer surfaces the rejection as an AuthenticationError + # instead of connecting and then mysteriously disconnecting. + permission = self._resolve_permission() + if permission == PERMISSION_DENIED: + self._channel.send_typed( + MessageType.AUTH_FAIL, b"rejected by host", + ) + raise AuthenticationError("rejected by host") + self.permission = permission + # Issue a fresh resume token so the viewer can reconnect + # within the store's TTL without the approval popup. + resume_token = self._host._resume_store.issue(self.permission) + ok_payload = json.dumps( + {"host_id": self._host.host_id, + "resume_token": resume_token, + "resume_ttl": self._host._resume_store.ttl, + "codec": self._host._codec_provider.name}, + ensure_ascii=False, + ).encode("utf-8") + self._channel.send_typed(MessageType.AUTH_OK, ok_payload) + self._channel.settimeout(None) + + def _send_loop(self) -> None: + # Seed the viewer with the latest cursor + frame on this per-client + # sender thread (Phase 2.3: motion-aware capture only bumps the seq on + # change, so a viewer joining a static desktop would otherwise sit + # blank until the host moves). Doing it here rather than in start() + # keeps a slow-reading viewer from stalling the accept thread. + self._host._send_initial_cursor(self) + self._send_initial_frame() + last_sent = 0 + while not self._shutdown.is_set(): + with self._host._frame_cond: + while (not self._shutdown.is_set() + and self._host._latest_seq <= last_sent): + self._host._frame_cond.wait(timeout=0.5) + if self._shutdown.is_set(): + return + frame = self._host._latest_frame + seq = self._host._latest_seq + if frame is None: + continue + try: + self._channel.send_typed(MessageType.FRAME, frame) + except OSError as error: + autocontrol_logger.info( + "remote_desktop send to %s failed: %r", + self._address, error, + ) + self.stop() + return + last_sent = seq + + def _audio_send_loop(self) -> None: + while not self._shutdown.is_set(): + self._audio_event.wait(timeout=0.5) + if self._shutdown.is_set(): + return + while True: + with self._audio_lock: + if not self._audio_queue: + self._audio_event.clear() + break + chunk = self._audio_queue.popleft() + try: + self._channel.send_typed(MessageType.AUDIO, chunk) + except OSError as error: + autocontrol_logger.info( + "remote_desktop audio send to %s failed: %r", + self._address, error, + ) + self.stop() + return + + def _recv_loop(self) -> None: + while not self._shutdown.is_set(): + try: + msg_type, payload = self._channel.read_typed() + except (OSError, ProtocolError) as error: + if not self._shutdown.is_set(): + autocontrol_logger.info( + "remote_desktop recv from %s ended: %r", + self._address, error, + ) + self.stop() + return + self._route_incoming(msg_type, payload) + + def _route_incoming(self, msg_type: MessageType, payload: bytes) -> None: + """Dispatch one received message to the matching handler.""" + if msg_type is MessageType.PING: + return + if msg_type is MessageType.INPUT: + # Phase 5.3: drop input from view-only viewers so they can + # watch but cannot drive the mouse / keyboard. + if self.permission != PERMISSION_VIEW_ONLY: + self._handle_input_payload(payload) + return + if msg_type is MessageType.CLIPBOARD: + self._handle_clipboard_payload(payload) + return + if msg_type is MessageType.CHAT: + self._handle_chat_payload(payload) + return + if msg_type is MessageType.USB_LIST_REQUEST: + self._handle_usb_list_request() + return + if msg_type in _FILE_MSG_TYPES: + self._handle_file_payload(msg_type, payload) + return + autocontrol_logger.info( + "remote_desktop unexpected msg %s from %s", + msg_type.name, self._address, + ) + + def _handle_file_payload(self, msg_type: MessageType, + payload: bytes) -> None: + receiver = self._host._ensure_file_receiver() + try: + if msg_type is MessageType.FILE_BEGIN: + receiver.handle_begin(payload) + elif msg_type is MessageType.FILE_CHUNK: + receiver.handle_chunk(payload) + elif msg_type is MessageType.FILE_END: + receiver.handle_end(payload) + except FileTransferError as error: + autocontrol_logger.info( + "remote_desktop bad file message from %s: %r", + self._address, error, + ) + + def _handle_usb_list_request(self) -> None: + """Phase 6.9: enumerate the host's USB devices and ship the list back. + + Uses the existing :func:`list_usb_devices` helper so we get the + same cross-platform behaviour as the standalone USB module. + Errors fall back to an empty payload — the viewer should + treat that as "host has no usable USB backend" rather than + crashing. + """ + try: + from je_auto_control.utils.usb import list_usb_devices + result = list_usb_devices() + body = { + "backend": result.backend, + "devices": [d.to_dict() for d in result.devices], + } + except (ImportError, OSError, RuntimeError) as error: + autocontrol_logger.info( + "usb_list from %s failed: %r", self._address, error, + ) + body = {"backend": "unavailable", "devices": []} + try: + self._channel.send_typed( + MessageType.USB_LIST_RESPONSE, + json.dumps(body, ensure_ascii=False).encode("utf-8"), + ) + except OSError: + pass + + def _handle_chat_payload(self, payload: bytes) -> None: + """Forward viewer-originated chat to the host's optional callback.""" + callback = self._host._on_chat + if callback is None: + return + try: + body = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return + if not isinstance(body, dict): + return + text = body.get("text") + sender = body.get("sender", "viewer") + if not isinstance(text, str) or not text: + return + try: + callback(str(sender), text) + except Exception: # noqa: BLE001 callback isolation + autocontrol_logger.exception( + "remote_desktop on_chat callback raised" + ) + + def _handle_clipboard_payload(self, payload: bytes) -> None: + try: + kind, data = decode_clipboard(payload) + except ClipboardSyncError as error: + autocontrol_logger.info( + "remote_desktop bad CLIPBOARD from %s: %r", + self._address, error, + ) + return + try: + self._host._apply_clipboard(kind, data) + except (OSError, RuntimeError, TypeError, ValueError) as error: + autocontrol_logger.warning( + "remote_desktop clipboard apply failed for %s: %r", + self._address, error, + ) + + def _handle_input_payload(self, payload: bytes) -> None: + try: + message = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + autocontrol_logger.info( + "remote_desktop bad INPUT from %s: %r", + self._address, error, + ) + return + try: + self._host._dispatch(message) + except InputDispatchError as error: + autocontrol_logger.info( + "remote_desktop rejected INPUT from %s: %r", + self._address, error, + ) + except (OSError, RuntimeError, ValueError, TypeError) as error: + autocontrol_logger.warning( + "remote_desktop input apply failed for %s: %r", + self._address, error, + ) + + def _close(self) -> None: + self._channel.close() diff --git a/je_auto_control/utils/remote_desktop/webrtc_host.py b/je_auto_control/utils/remote_desktop/webrtc_host.py index e8f9d374..fa8670d9 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_host.py +++ b/je_auto_control/utils/remote_desktop/webrtc_host.py @@ -21,9 +21,6 @@ from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop.audit_log import default_audit_log -from je_auto_control.utils.remote_desktop.fingerprint import ( - load_or_create_host_fingerprint, -) from je_auto_control.utils.remote_desktop.input_dispatch import dispatch_input from je_auto_control.utils.remote_desktop.permissions import SessionPermissions from je_auto_control.utils.remote_desktop.rate_limit import ( @@ -34,6 +31,12 @@ RTCPeerConnection, RTCSessionDescription, ScreenVideoTrack, WebRTCConfig, get_bridge, wait_for_ice_gathering, ) +from je_auto_control.utils.remote_desktop.webrtc_host_auth import ( + ViewerAuthMixin, +) +from je_auto_control.utils.remote_desktop.webrtc_host_media import ( + MediaNegotiationMixin, +) _AUTH_GRACE_S = 5.0 @@ -45,7 +48,7 @@ ConsentCallback = Callable[[str], bool] -class WebRTCDesktopHost: +class WebRTCDesktopHost(ViewerAuthMixin, MediaNegotiationMixin): """Single-viewer WebRTC host with manual SDP signaling. Multiple simultaneous viewers would require one ``RTCPeerConnection`` @@ -559,155 +562,6 @@ async def _async_apply_renegotiate_answer(self, sdp: str) -> None: self._maybe_resubscribe_viewer_video() self._maybe_resubscribe_viewer_audio() - def _maybe_resubscribe_viewer_video(self) -> None: - if not (self._config.accept_viewer_video - and self._viewer_video_task is None - and self._pc is not None): - return - video_ts = [ - t for t in self._pc.getTransceivers() if t.kind == "video" - ] - for transceiver in video_ts[1:]: # skip our outbound slot - track = self._receiver_track(transceiver) - if track is None: - continue - self._viewer_video_task = self._spawn_bg( - self._consume_viewer_video(track), - ) - autocontrol_logger.info( - "webrtc host: re-spawned viewer video consume task", - ) - return - - def _maybe_resubscribe_viewer_audio(self) -> None: - if not (self._config.accept_viewer_audio_opus - and self._opus_audio_receiver is None - and self._pc is not None): - return - for transceiver in self._pc.getTransceivers(): - if transceiver.kind != "audio": - continue - track = self._receiver_track(transceiver) - if track is None: - continue - self._start_opus_audio_receive(track) - return - - @staticmethod - def _receiver_track(transceiver): - receiver = transceiver.receiver - return receiver.track if receiver is not None else None - - async def _async_renegotiate(self) -> None: - """Host-initiated renegotiation: new offer → viewer over ctrl channel.""" - if self._pc is None: - return - try: - offer = await self._pc.createOffer() - await self._pc.setLocalDescription(offer) - await wait_for_ice_gathering(self._pc) - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("renegotiate offer: %r", error) - return - self._send_ctrl({ - "type": "renegotiate_offer", - "sdp": self._pc.localDescription.sdp, - }) - autocontrol_logger.info("webrtc host: sent renegotiate offer") - - def request_renegotiation(self) -> None: - """Public sync entry: kick off a fresh SDP exchange over ctrl channel.""" - if self._pc is None: - return - get_bridge().call_soon( - lambda: self._spawn_bg(self._async_renegotiate()), - ) - - def enable_accept_viewer_video(self) -> None: - """Live-add a recvonly video transceiver and renegotiate. - - ``enable_*`` only adds capacity — aiortc has no ``removeTransceiver``, - so disabling needs a reconnect (or set the transceiver to inactive). - """ - if self._pc is None: - return - self._config.accept_viewer_video = True - get_bridge().call_soon(self._add_recvonly_video_and_renegotiate) - - def enable_accept_viewer_audio_opus(self) -> None: - """Live-add a recvonly audio transceiver and renegotiate.""" - if self._pc is None: - return - self._config.accept_viewer_audio_opus = True - get_bridge().call_soon(self._add_recvonly_audio_and_renegotiate) - - def _add_recvonly_video_and_renegotiate(self) -> None: - if self._pc is None: - return - already = sum( - 1 for t in self._pc.getTransceivers() if t.kind == "video" - ) - if already < 2: - self._pc.addTransceiver("video", direction="recvonly") - self._spawn_bg(self._async_renegotiate()) - - def _add_recvonly_audio_and_renegotiate(self) -> None: - if self._pc is None: - return - already = sum( - 1 for t in self._pc.getTransceivers() if t.kind == "audio" - ) - if already < 1: - self._pc.addTransceiver("audio", direction="recvonly") - self._spawn_bg(self._async_renegotiate()) - - def disable_accept_viewer_video(self) -> None: - """Mark the recvonly video slot inactive + stop the consume task.""" - if self._pc is None: - return - self._config.accept_viewer_video = False - get_bridge().call_soon(self._deactivate_recvonly_video) - - def disable_accept_viewer_audio_opus(self) -> None: - """Mark the recvonly audio slot inactive + stop the Opus receiver.""" - if self._pc is None: - return - self._config.accept_viewer_audio_opus = False - get_bridge().call_soon(self._deactivate_recvonly_audio) - - def _deactivate_recvonly_video(self) -> None: - if self._pc is None: - return - # Find the second video transceiver (the recvonly one); first is our - # outbound screen track. - video_ts = [t for t in self._pc.getTransceivers() if t.kind == "video"] - if len(video_ts) >= 2: - try: - video_ts[1].direction = "inactive" - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("inactivate video: %r", error) - if self._viewer_video_task is not None: - self._viewer_video_task.cancel() - self._viewer_video_task = None - self._spawn_bg(self._async_renegotiate()) - - def _deactivate_recvonly_audio(self) -> None: - if self._pc is None: - return - audio_ts = [t for t in self._pc.getTransceivers() if t.kind == "audio"] - if audio_ts: - try: - audio_ts[0].direction = "inactive" - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("inactivate audio: %r", error) - if self._opus_audio_receiver is not None: - try: - self._opus_audio_receiver.stop() - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("opus receiver stop: %r", error) - self._opus_audio_receiver = None - self._spawn_bg(self._async_renegotiate()) - def _ensure_files_receiver(self): from je_auto_control.utils.remote_desktop.webrtc_files import ( FileTransferReceiver, @@ -803,172 +657,6 @@ def read_only(self) -> bool: def permissions(self) -> SessionPermissions: return self._permissions - def _handle_send_sas(self) -> None: - try: - from je_auto_control.utils.remote_desktop.session_actions import ( - send_secure_attention_sequence, - ) - send_secure_attention_sequence() - self._send_ctrl({"type": "sas_ok"}) - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("SendSAS: %r", error) - self._send_ctrl({"type": "sas_fail", "error": str(error)}) - - def _handle_auth(self, data: Mapping[str, Any]) -> None: - token = data.get("token") - if not isinstance(token, str) or token != self._token: - self._reject_auth(data) - return - viewer_id = data.get("viewer_id") - self._pending_viewer_id = ( - viewer_id if isinstance(viewer_id, str) else None - ) - if self._auto_approve_via_trust(): - return - if self._auto_approve_via_whitelist(): - return - if self._on_pending_viewer is None: - self._approve_pending_viewer() - return - self._has_pending_viewer = True - try: - self._on_pending_viewer() - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("pending viewer cb: %r", error) - - def _reject_auth(self, data: Mapping[str, Any]) -> None: - self._send_ctrl({"type": "auth_fail"}) - try: - default_audit_log().log( - "auth_fail", - viewer_id=str(data.get("viewer_id", "")) or None, - detail=f"remote_ip={self._remote_ip}", - ) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("audit log auth_fail: %r", error) - get_bridge().call_soon(self._schedule_close_after_fail) - - def _auto_approve_via_trust(self) -> bool: - if not self._is_trusted_viewer(self._pending_viewer_id): - return False - autocontrol_logger.info( - "webrtc host: viewer_id %s is trusted; auto-approving", - self._pending_viewer_id, - ) - if self._trust_list is not None: - try: - self._trust_list.touch(self._pending_viewer_id) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("trust touch: %r", error) - self._approve_pending_viewer() - return True - - def _auto_approve_via_whitelist(self) -> bool: - if not self._is_ip_whitelisted(self._remote_ip): - return False - autocontrol_logger.info( - "webrtc host: remote ip %s matches whitelist; auto-approving", - self._remote_ip, - ) - self._approve_pending_viewer() - return True - - def _is_ip_whitelisted(self, ip: Optional[str]) -> bool: - if not ip or not self._ip_whitelist: - return False - import ipaddress - try: - addr = ipaddress.ip_address(ip) - except ValueError: - return False - for cidr in self._ip_whitelist: - try: - if addr in ipaddress.ip_network(cidr.strip(), strict=False): - return True - except ValueError: - continue - return False - - def _is_trusted_viewer(self, viewer_id: Optional[str]) -> bool: - if self._trust_list is None or not viewer_id: - return False - try: - return self._trust_list.is_trusted(viewer_id) - except (OSError, RuntimeError) as error: - autocontrol_logger.warning("trust list check: %r", error) - return False - - def trust_pending_viewer(self, label: str = "") -> None: - """Add the current pending viewer to the trust list, then approve.""" - viewer_id = self._pending_viewer_id - if self._trust_list is not None and viewer_id: - try: - self._trust_list.add(viewer_id, label=label) - except (OSError, ValueError, RuntimeError) as error: - autocontrol_logger.warning("trust list add: %r", error) - self.approve_pending_viewer() - - @property - def pending_viewer_id(self) -> Optional[str]: - return self._pending_viewer_id - - def approve_pending_viewer(self) -> None: - """Thread-safe accept; call from GUI when user clicks Accept.""" - get_bridge().call_soon(self._approve_pending_viewer) - - def reject_pending_viewer(self) -> None: - """Thread-safe reject; call from GUI when user clicks Reject.""" - get_bridge().call_soon(self._reject_pending_viewer) - - def _approve_pending_viewer(self) -> None: - if not self._has_pending_viewer and self._authenticated: - return - self._has_pending_viewer = False - self._authenticated = True - self._send_ctrl({ - "type": "auth_ok", - "read_only": not self._permissions.allow_input, - "permissions": self._permissions.to_dict(), - "fingerprint": load_or_create_host_fingerprint(), - }) - try: - default_audit_log().log( - "auth_ok", - viewer_id=self._pending_viewer_id, - detail=f"remote_ip={self._remote_ip}", - ) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("audit log auth_ok: %r", error) - if self._auth_deadline_handle is not None: - self._auth_deadline_handle.cancel() - self._auth_deadline_handle = None - if self._on_authenticated is not None: - try: - self._on_authenticated() - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("auth cb: %r", error) - - def _reject_pending_viewer(self) -> None: - self._has_pending_viewer = False - self._send_ctrl({"type": "auth_fail"}) - get_bridge().call_soon(self._schedule_close_after_fail) - - @property - def has_pending_viewer(self) -> bool: - return self._has_pending_viewer - - def _schedule_close_after_fail(self) -> None: - loop = asyncio.get_event_loop() - loop.call_later(0.5, lambda: self._spawn_bg(self._async_stop())) - - def _enforce_auth_deadline(self) -> None: - if self._authenticated: - return - autocontrol_logger.warning( - "webrtc host: viewer failed to authenticate within grace period", - ) - self._spawn_bg(self._async_stop()) - def _dispatch_input_safely(self, payload: Any) -> None: if not isinstance(payload, dict): return diff --git a/je_auto_control/utils/remote_desktop/webrtc_host_auth.py b/je_auto_control/utils/remote_desktop/webrtc_host_auth.py new file mode 100644 index 00000000..434501e9 --- /dev/null +++ b/je_auto_control/utils/remote_desktop/webrtc_host_auth.py @@ -0,0 +1,195 @@ +"""Viewer authentication and approval for the WebRTC host. + +Everything between a viewer's first ``auth`` message and the moment it is +allowed to send input: token check, the auto-approve paths (trust list, +IP whitelist), the manual accept/reject prompt the GUI drives, the SAS +exchange, and the grace-period deadline that closes an unauthenticated +peer. Kept apart from ``webrtc_host`` so the media session lifecycle is +readable on its own. +""" +from __future__ import annotations + +import asyncio +from typing import Any, Mapping, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.audit_log import default_audit_log +from je_auto_control.utils.remote_desktop.fingerprint import ( + load_or_create_host_fingerprint, +) +from je_auto_control.utils.remote_desktop.webrtc_transport import get_bridge + + +class ViewerAuthMixin: + """Auth half of :class:`WebRTCDesktopHost`. + + Requires the host to provide ``_token``, ``_trust_list``, + ``_ip_whitelist``, ``_authenticated``, ``_pending_viewer_id``, + ``_send_ctrl``, ``_spawn_bg`` and ``_async_stop``. + """ + + def _handle_send_sas(self) -> None: + try: + from je_auto_control.utils.remote_desktop.session_actions import ( + send_secure_attention_sequence, + ) + send_secure_attention_sequence() + self._send_ctrl({"type": "sas_ok"}) + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("SendSAS: %r", error) + self._send_ctrl({"type": "sas_fail", "error": str(error)}) + + def _handle_auth(self, data: Mapping[str, Any]) -> None: + token = data.get("token") + if not isinstance(token, str) or token != self._token: + self._reject_auth(data) + return + viewer_id = data.get("viewer_id") + self._pending_viewer_id = ( + viewer_id if isinstance(viewer_id, str) else None + ) + if self._auto_approve_via_trust(): + return + if self._auto_approve_via_whitelist(): + return + if self._on_pending_viewer is None: + self._approve_pending_viewer() + return + self._has_pending_viewer = True + try: + self._on_pending_viewer() + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("pending viewer cb: %r", error) + + def _reject_auth(self, data: Mapping[str, Any]) -> None: + self._send_ctrl({"type": "auth_fail"}) + try: + default_audit_log().log( + "auth_fail", + viewer_id=str(data.get("viewer_id", "")) or None, + detail=f"remote_ip={self._remote_ip}", + ) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("audit log auth_fail: %r", error) + get_bridge().call_soon(self._schedule_close_after_fail) + + def _auto_approve_via_trust(self) -> bool: + if not self._is_trusted_viewer(self._pending_viewer_id): + return False + autocontrol_logger.info( + "webrtc host: viewer_id %s is trusted; auto-approving", + self._pending_viewer_id, + ) + if self._trust_list is not None: + try: + self._trust_list.touch(self._pending_viewer_id) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("trust touch: %r", error) + self._approve_pending_viewer() + return True + + def _auto_approve_via_whitelist(self) -> bool: + if not self._is_ip_whitelisted(self._remote_ip): + return False + autocontrol_logger.info( + "webrtc host: remote ip %s matches whitelist; auto-approving", + self._remote_ip, + ) + self._approve_pending_viewer() + return True + + def _is_ip_whitelisted(self, ip: Optional[str]) -> bool: + if not ip or not self._ip_whitelist: + return False + import ipaddress + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return False + for cidr in self._ip_whitelist: + try: + if addr in ipaddress.ip_network(cidr.strip(), strict=False): + return True + except ValueError: + continue + return False + + def _is_trusted_viewer(self, viewer_id: Optional[str]) -> bool: + if self._trust_list is None or not viewer_id: + return False + try: + return self._trust_list.is_trusted(viewer_id) + except (OSError, RuntimeError) as error: + autocontrol_logger.warning("trust list check: %r", error) + return False + + def trust_pending_viewer(self, label: str = "") -> None: + """Add the current pending viewer to the trust list, then approve.""" + viewer_id = self._pending_viewer_id + if self._trust_list is not None and viewer_id: + try: + self._trust_list.add(viewer_id, label=label) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.warning("trust list add: %r", error) + self.approve_pending_viewer() + + @property + def pending_viewer_id(self) -> Optional[str]: + return self._pending_viewer_id + + def approve_pending_viewer(self) -> None: + """Thread-safe accept; call from GUI when user clicks Accept.""" + get_bridge().call_soon(self._approve_pending_viewer) + + def reject_pending_viewer(self) -> None: + """Thread-safe reject; call from GUI when user clicks Reject.""" + get_bridge().call_soon(self._reject_pending_viewer) + + def _approve_pending_viewer(self) -> None: + if not self._has_pending_viewer and self._authenticated: + return + self._has_pending_viewer = False + self._authenticated = True + self._send_ctrl({ + "type": "auth_ok", + "read_only": not self._permissions.allow_input, + "permissions": self._permissions.to_dict(), + "fingerprint": load_or_create_host_fingerprint(), + }) + try: + default_audit_log().log( + "auth_ok", + viewer_id=self._pending_viewer_id, + detail=f"remote_ip={self._remote_ip}", + ) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("audit log auth_ok: %r", error) + if self._auth_deadline_handle is not None: + self._auth_deadline_handle.cancel() + self._auth_deadline_handle = None + if self._on_authenticated is not None: + try: + self._on_authenticated() + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("auth cb: %r", error) + + def _reject_pending_viewer(self) -> None: + self._has_pending_viewer = False + self._send_ctrl({"type": "auth_fail"}) + get_bridge().call_soon(self._schedule_close_after_fail) + + @property + def has_pending_viewer(self) -> bool: + return self._has_pending_viewer + + def _schedule_close_after_fail(self) -> None: + loop = asyncio.get_event_loop() + loop.call_later(0.5, lambda: self._spawn_bg(self._async_stop())) + + def _enforce_auth_deadline(self) -> None: + if self._authenticated: + return + autocontrol_logger.warning( + "webrtc host: viewer failed to authenticate within grace period", + ) + self._spawn_bg(self._async_stop()) diff --git a/je_auto_control/utils/remote_desktop/webrtc_host_media.py b/je_auto_control/utils/remote_desktop/webrtc_host_media.py new file mode 100644 index 00000000..2921f3b3 --- /dev/null +++ b/je_auto_control/utils/remote_desktop/webrtc_host_media.py @@ -0,0 +1,172 @@ +"""Renegotiation and recvonly-track management for the WebRTC host. + +aiortc has no ``removeTransceiver``, so turning viewer video or Opus audio +on and off is not symmetric: enabling adds a recvonly transceiver and +re-offers, disabling can only set the existing one inactive and stop the +receiver. That asymmetry, the host-initiated offer that carries it, and the +re-subscription that runs after each answer are the whole of this mixin. +""" +from __future__ import annotations + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.webrtc_transport import ( + get_bridge, wait_for_ice_gathering, +) + + +class MediaNegotiationMixin: + """Media-track half of :class:`WebRTCDesktopHost`. + + Requires the host to provide ``_pc``, ``_config``, ``_viewer_video_task``, + ``_opus_audio_receiver``, ``_send_ctrl``, ``_spawn_bg``, + ``_consume_viewer_video`` and ``_start_opus_audio_receive``. + """ + + def _maybe_resubscribe_viewer_video(self) -> None: + if not (self._config.accept_viewer_video + and self._viewer_video_task is None + and self._pc is not None): + return + video_ts = [ + t for t in self._pc.getTransceivers() if t.kind == "video" + ] + for transceiver in video_ts[1:]: # skip our outbound slot + track = self._receiver_track(transceiver) + if track is None: + continue + self._viewer_video_task = self._spawn_bg( + self._consume_viewer_video(track), + ) + autocontrol_logger.info( + "webrtc host: re-spawned viewer video consume task", + ) + return + + def _maybe_resubscribe_viewer_audio(self) -> None: + if not (self._config.accept_viewer_audio_opus + and self._opus_audio_receiver is None + and self._pc is not None): + return + for transceiver in self._pc.getTransceivers(): + if transceiver.kind != "audio": + continue + track = self._receiver_track(transceiver) + if track is None: + continue + self._start_opus_audio_receive(track) + return + + @staticmethod + def _receiver_track(transceiver): + receiver = transceiver.receiver + return receiver.track if receiver is not None else None + + async def _async_renegotiate(self) -> None: + """Host-initiated renegotiation: new offer → viewer over ctrl channel.""" + if self._pc is None: + return + try: + offer = await self._pc.createOffer() + await self._pc.setLocalDescription(offer) + await wait_for_ice_gathering(self._pc) + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("renegotiate offer: %r", error) + return + self._send_ctrl({ + "type": "renegotiate_offer", + "sdp": self._pc.localDescription.sdp, + }) + autocontrol_logger.info("webrtc host: sent renegotiate offer") + + def request_renegotiation(self) -> None: + """Public sync entry: kick off a fresh SDP exchange over ctrl channel.""" + if self._pc is None: + return + get_bridge().call_soon( + lambda: self._spawn_bg(self._async_renegotiate()), + ) + + def enable_accept_viewer_video(self) -> None: + """Live-add a recvonly video transceiver and renegotiate. + + ``enable_*`` only adds capacity — aiortc has no ``removeTransceiver``, + so disabling needs a reconnect (or set the transceiver to inactive). + """ + if self._pc is None: + return + self._config.accept_viewer_video = True + get_bridge().call_soon(self._add_recvonly_video_and_renegotiate) + + def enable_accept_viewer_audio_opus(self) -> None: + """Live-add a recvonly audio transceiver and renegotiate.""" + if self._pc is None: + return + self._config.accept_viewer_audio_opus = True + get_bridge().call_soon(self._add_recvonly_audio_and_renegotiate) + + def _add_recvonly_video_and_renegotiate(self) -> None: + if self._pc is None: + return + already = sum( + 1 for t in self._pc.getTransceivers() if t.kind == "video" + ) + if already < 2: + self._pc.addTransceiver("video", direction="recvonly") + self._spawn_bg(self._async_renegotiate()) + + def _add_recvonly_audio_and_renegotiate(self) -> None: + if self._pc is None: + return + already = sum( + 1 for t in self._pc.getTransceivers() if t.kind == "audio" + ) + if already < 1: + self._pc.addTransceiver("audio", direction="recvonly") + self._spawn_bg(self._async_renegotiate()) + + def disable_accept_viewer_video(self) -> None: + """Mark the recvonly video slot inactive + stop the consume task.""" + if self._pc is None: + return + self._config.accept_viewer_video = False + get_bridge().call_soon(self._deactivate_recvonly_video) + + def disable_accept_viewer_audio_opus(self) -> None: + """Mark the recvonly audio slot inactive + stop the Opus receiver.""" + if self._pc is None: + return + self._config.accept_viewer_audio_opus = False + get_bridge().call_soon(self._deactivate_recvonly_audio) + + def _deactivate_recvonly_video(self) -> None: + if self._pc is None: + return + # Find the second video transceiver (the recvonly one); first is our + # outbound screen track. + video_ts = [t for t in self._pc.getTransceivers() if t.kind == "video"] + if len(video_ts) >= 2: + try: + video_ts[1].direction = "inactive" + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("inactivate video: %r", error) + if self._viewer_video_task is not None: + self._viewer_video_task.cancel() + self._viewer_video_task = None + self._spawn_bg(self._async_renegotiate()) + + def _deactivate_recvonly_audio(self) -> None: + if self._pc is None: + return + audio_ts = [t for t in self._pc.getTransceivers() if t.kind == "audio"] + if audio_ts: + try: + audio_ts[0].direction = "inactive" + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("inactivate audio: %r", error) + if self._opus_audio_receiver is not None: + try: + self._opus_audio_receiver.stop() + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("opus receiver stop: %r", error) + self._opus_audio_receiver = None + self._spawn_bg(self._async_renegotiate()) diff --git a/test/unit_test/headless/test_flow_extensions.py b/test/unit_test/headless/test_flow_extensions.py index 2cc75b74..07dab00c 100644 --- a/test/unit_test/headless/test_flow_extensions.py +++ b/test/unit_test/headless/test_flow_extensions.py @@ -9,7 +9,10 @@ ) from je_auto_control.utils.executor.action_executor import Executor from je_auto_control.utils.executor.flow_control import ( - exec_assert_duration, exec_call_macro, exec_parallel, + exec_call_macro, exec_parallel, +) +from je_auto_control.utils.executor.flow_data_commands import ( + exec_assert_duration, ) diff --git a/test/unit_test/headless/test_flow_var_commands.py b/test/unit_test/headless/test_flow_var_commands.py index cc524238..2cff682d 100644 --- a/test/unit_test/headless/test_flow_var_commands.py +++ b/test/unit_test/headless/test_flow_var_commands.py @@ -7,9 +7,9 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlAssertionException, ) -from je_auto_control.utils.executor import flow_control +from je_auto_control.utils.executor import flow_data_commands from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import ( +from je_auto_control.utils.executor.flow_data_commands import ( exec_assert_var, exec_http_to_var, exec_now_to_var, exec_random_to_var, exec_read_file_to_var, exec_transform_var, ) @@ -76,7 +76,7 @@ def test_transform_var_replace_and_slice(): def test_now_to_var_formats_injected_clock(monkeypatch): - monkeypatch.setattr(flow_control, "_now", + monkeypatch.setattr(flow_data_commands, "_now", lambda: datetime.datetime(2026, 1, 2, 3, 4, 5)) executor = Executor() result = exec_now_to_var(executor, {"format": "%Y-%m-%d", "var": "d"}) diff --git a/test/unit_test/headless/test_ocr_to_var.py b/test/unit_test/headless/test_ocr_to_var.py index 7796a637..83d1aafb 100644 --- a/test/unit_test/headless/test_ocr_to_var.py +++ b/test/unit_test/headless/test_ocr_to_var.py @@ -1,6 +1,6 @@ """Tests for AC_ocr_to_var (OCR a region into a flow variable).""" from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import exec_ocr_to_var +from je_auto_control.utils.executor.flow_data_commands import exec_ocr_to_var class _Match: diff --git a/test/unit_test/headless/test_pdf.py b/test/unit_test/headless/test_pdf.py index 05c93048..7700ad74 100644 --- a/test/unit_test/headless/test_pdf.py +++ b/test/unit_test/headless/test_pdf.py @@ -8,7 +8,7 @@ import je_auto_control as ac from je_auto_control.utils.exception.exceptions import AutoControlAssertionException from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import exec_pdf_to_var +from je_auto_control.utils.executor.flow_data_commands import exec_pdf_to_var from je_auto_control.utils.pdf import pdf_reader diff --git a/test/unit_test/headless/test_r3_gui_slot_exceptions.py b/test/unit_test/headless/test_r3_gui_slot_exceptions.py index 65b9feb1..d0d82268 100644 --- a/test/unit_test/headless/test_r3_gui_slot_exceptions.py +++ b/test/unit_test/headless/test_r3_gui_slot_exceptions.py @@ -55,11 +55,12 @@ def test_builder_run_slot_surfaces_autocontrol_exception(monkeypatch): def test_playback_record_slot_surfaces_autocontrol_exception(monkeypatch): import je_auto_control.gui.main_widget as mw + import je_auto_control.gui._record_tab as rt warned = {} - monkeypatch.setattr(mw.QMessageBox, "warning", + monkeypatch.setattr(rt.QMessageBox, "warning", lambda *a, **k: warned.setdefault("hit", True)) monkeypatch.setattr( - mw, "execute_action", + rt, "execute_action", _raiser(AutoControlExecuteActionException("boom")), ) stub = types.SimpleNamespace(_record_data=[["AC_ok"]]) @@ -70,10 +71,11 @@ def test_playback_record_slot_surfaces_autocontrol_exception(monkeypatch): def test_execute_script_slot_surfaces_autocontrol_exception(monkeypatch): import je_auto_control.gui.main_widget as mw + import je_auto_control.gui._script_tab as st captured = {} - monkeypatch.setattr(mw, "read_action_json", lambda _p: [["AC_ok"]]) + monkeypatch.setattr(st, "read_action_json", lambda _p: [["AC_ok"]]) monkeypatch.setattr( - mw, "execute_action", + st, "execute_action", _raiser(AutoControlExecuteActionException("boom")), ) editor = types.SimpleNamespace(text=lambda: "some.json") diff --git a/test/unit_test/headless/test_r3_rdusb_host.py b/test/unit_test/headless/test_r3_rdusb_host.py index 8dace307..7932f996 100644 --- a/test/unit_test/headless/test_r3_rdusb_host.py +++ b/test/unit_test/headless/test_r3_rdusb_host.py @@ -14,7 +14,7 @@ from je_auto_control.utils.remote_desktop import ( RemoteDesktopHost, RemoteDesktopViewer, ) -from je_auto_control.utils.remote_desktop.host import _ClientHandler +from je_auto_control.utils.remote_desktop.host_client import _ClientHandler def _wait_until(predicate, timeout: float = 5.0, diff --git a/test/unit_test/headless/test_remote_desktop_ip_allowlist.py b/test/unit_test/headless/test_remote_desktop_ip_allowlist.py index 23e877dd..f3c5912f 100644 --- a/test/unit_test/headless/test_remote_desktop_ip_allowlist.py +++ b/test/unit_test/headless/test_remote_desktop_ip_allowlist.py @@ -5,8 +5,9 @@ import pytest -from je_auto_control.utils.remote_desktop.host import ( - RemoteDesktopHost, _compile_ip_allowlist, _ip_in_allowlist, +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import ( + _compile_ip_allowlist, _ip_in_allowlist, ) from je_auto_control.utils.remote_desktop.viewer import RemoteDesktopViewer diff --git a/test/unit_test/headless/test_remote_desktop_pending_viewer.py b/test/unit_test/headless/test_remote_desktop_pending_viewer.py index bbba16fb..538ed7c0 100644 --- a/test/unit_test/headless/test_remote_desktop_pending_viewer.py +++ b/test/unit_test/headless/test_remote_desktop_pending_viewer.py @@ -4,9 +4,8 @@ import pytest -from je_auto_control.utils.remote_desktop.host import ( - PendingViewer, RemoteDesktopHost, -) +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import PendingViewer from je_auto_control.utils.remote_desktop.protocol import ( AuthenticationError, ) diff --git a/test/unit_test/headless/test_remote_desktop_resume.py b/test/unit_test/headless/test_remote_desktop_resume.py index 9574cd62..4c0d5b85 100644 --- a/test/unit_test/headless/test_remote_desktop_resume.py +++ b/test/unit_test/headless/test_remote_desktop_resume.py @@ -4,9 +4,8 @@ import pytest -from je_auto_control.utils.remote_desktop.host import ( - PendingViewer, RemoteDesktopHost, -) +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import PendingViewer from je_auto_control.utils.remote_desktop.resume_tokens import ( ResumeTokenStore, ) diff --git a/test/unit_test/headless/test_shell_to_var.py b/test/unit_test/headless/test_shell_to_var.py index 6e7c85ba..ef56f7dc 100644 --- a/test/unit_test/headless/test_shell_to_var.py +++ b/test/unit_test/headless/test_shell_to_var.py @@ -2,7 +2,7 @@ import sys from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import exec_shell_to_var +from je_auto_control.utils.executor.flow_data_commands import exec_shell_to_var def test_shell_to_var_captures_stdout(): diff --git a/test/unit_test/headless/test_sql_steps.py b/test/unit_test/headless/test_sql_steps.py index 29aeffe4..9fe6aafd 100644 --- a/test/unit_test/headless/test_sql_steps.py +++ b/test/unit_test/headless/test_sql_steps.py @@ -9,7 +9,7 @@ import je_auto_control as ac from je_auto_control.utils.exception.exceptions import AutoControlAssertionException from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import ( +from je_auto_control.utils.executor.flow_data_commands import ( exec_assert_db, exec_sql_to_var, ) from je_auto_control.utils.sql.sql_query import query_sqlite diff --git a/test/unit_test/headless/test_unattended_reliability.py b/test/unit_test/headless/test_unattended_reliability.py index d7677ef5..83960805 100644 --- a/test/unit_test/headless/test_unattended_reliability.py +++ b/test/unit_test/headless/test_unattended_reliability.py @@ -23,7 +23,7 @@ def test_generate_totp_deterministic_and_verifies(): def test_otp_to_var_command(): from je_auto_control.utils.executor.action_executor import Executor - from je_auto_control.utils.executor.flow_control import exec_otp_to_var + from je_auto_control.utils.executor.flow_data_commands import exec_otp_to_var executor = Executor() result = exec_otp_to_var( executor, {"secret": ac.generate_secret(), "var": "code"}) From e8be8ee3e35287ff70b97fba0751effb27e1d94f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:16:22 +0800 Subject: [PATCH 11/21] Answer 400 for a command the executor does not know POST /execute and /execute_file funnelled every executor failure into 500 {"error": "execute_action failed"}, so a client could not tell a typo in its own request from a broken server. Every name in the list is now checked before anything runs, nested flow-control bodies included, and an unrecognised one comes back as 400 naming all of them with nothing executed. /execute_file answers the same way for a path that is unreadable or holds something that is not an action list. --- .../utils/executor/action_executor.py | 15 ++- .../utils/executor/action_schema.py | 107 ++++++++++++------ .../utils/rest_api/rest_handlers.py | 36 ++++++ .../utils/rest_api/rest_openapi.py | 17 +++ test/unit_test/headless/test_rest_server.py | 76 ++++++++++++- 5 files changed, 211 insertions(+), 40 deletions(-) diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index d292c317..ee3cc775 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -24,7 +24,9 @@ from je_auto_control.utils.clipboard.clipboard import ( get_clipboard, set_clipboard, ) -from je_auto_control.utils.executor.action_schema import validate_actions +from je_auto_control.utils.executor.action_schema import ( + unknown_command_names, validate_actions, +) from je_auto_control.utils.executor.flow_control import ( BLOCK_COMMANDS, LoopBreak, LoopContinue, ) @@ -7909,6 +7911,17 @@ def known_commands(self) -> set: """Return the set of all command names the executor recognises.""" return set(self.event_dict.keys()) | set(self._block_commands.keys()) + def unknown_commands_in(self, action_list: Union[list, dict]) -> List[str]: + """Return the unrecognised command names in ``action_list``, in order. + + Nothing is executed. Structural problems raise exactly as + :meth:`execute_action` would, so a caller-facing boundary can tell + "you named a command that does not exist" apart from "the run + failed" *before* the first action moves anything. + """ + return unknown_command_names(self._unwrap_action_list(action_list), + self.known_commands()) + def _resolve_runtime_args(self, args: Any) -> Any: """Interpolate ``${var}`` placeholders against the current scope. diff --git a/je_auto_control/utils/executor/action_schema.py b/je_auto_control/utils/executor/action_schema.py index 1bb9ae79..f719b888 100644 --- a/je_auto_control/utils/executor/action_schema.py +++ b/je_auto_control/utils/executor/action_schema.py @@ -4,7 +4,7 @@ Validates the outer shape (``[name]`` / ``[name, params]``), that names are in the executor allowlist, and that flow-control nested bodies are themselves valid lists. """ -from typing import Any, Iterable, Set +from typing import Any, Iterable, Iterator, List, Tuple from je_auto_control.utils.exception.exceptions import AutoControlActionException @@ -42,52 +42,87 @@ def validate_actions(actions: Any, known_commands: Iterable[str]) -> None: """Validate an action list recursively; raise on the first problem.""" known = set(known_commands) - _validate_list(actions, known, trail="root") + for trail, name in _iter_actions(actions, "root"): + if not isinstance(name, str) or name not in known: + raise AutoControlActionException(f"{trail}: unknown command {name!r}") -def _validate_list(actions: Any, known: Set[str], trail: str) -> None: +def unknown_command_names(actions: Any, + known_commands: Iterable[str]) -> List[str]: + """Return every unrecognised command name in ``actions``, in order. + + Structural problems still raise, exactly as :func:`validate_actions` does; + the only difference is that an unknown *name* is collected instead of + ending the walk. A boundary that has to answer a caller — the REST API — + reports the whole list, so a client fixes every typo in one round trip + rather than one per request. + """ + known = set(known_commands) + unknown: List[str] = [] + for _trail, name in _iter_actions(actions, "root"): + if isinstance(name, str) and name in known: + continue + label = name if isinstance(name, str) else repr(name) + if label not in unknown: + unknown.append(label) + return unknown + + +def _iter_actions(actions: Any, trail: str) -> Iterator[Tuple[str, Any]]: + """Yield ``(trail, command_name)`` for every action in the tree, in order. + + Structural problems raise as they are met, but whether the name is in the + allowlist is left to the caller. That is what lets "reject the first + unknown name" and "collect every unknown name" share one traversal — and, + more importantly, one definition of where a nested action list may hide. + + Laziness is load-bearing: the caller inspects the name at the ``yield`` + before this walk goes on to check the params, so the order in which the + two complaints surface is unchanged from when it was all one function. + """ if not isinstance(actions, list): raise AutoControlActionException( f"{trail}: action list must be a list, got {type(actions).__name__}" ) for idx, action in enumerate(actions): - _validate_single(action, known, f"{trail}[{idx}]") - - -def _validate_single(action: Any, known: Set[str], trail: str) -> None: - if not isinstance(action, list) or not 1 <= len(action) <= 2: - raise AutoControlActionException( - f"{trail}: must be [name] or [name, params]" - ) - name = action[0] - if not isinstance(name, str) or name not in known: - raise AutoControlActionException(f"{trail}: unknown command {name!r}") - if len(action) == 2 and not isinstance(action[1], (dict, list)): - raise AutoControlActionException( - f"{trail}: params must be dict or list" - ) - _validate_nested_bodies(name, action, known, trail) - - -def _validate_nested_bodies(name: str, action: list, known: Set[str], trail: str) -> None: + node = f"{trail}[{idx}]" + if not isinstance(action, list) or not 1 <= len(action) <= 2: + raise AutoControlActionException( + f"{node}: must be [name] or [name, params]" + ) + yield node, action[0] + if len(action) == 2 and not isinstance(action[1], (dict, list)): + raise AutoControlActionException( + f"{node}: params must be dict or list" + ) + yield from _iter_nested_actions(action[0], action, node) + + +def _iter_nested_actions(name: Any, action: list, + trail: str) -> Iterator[Tuple[str, Any]]: + """Yield the actions held in a flow-control command's nested body keys.""" + # ``name`` arrives unvalidated — the collector does not stop on a bad one — + # and an unhashable name would blow up the two lookups below. + if not isinstance(name, str): + return if len(action) < 2 or not isinstance(action[1], dict): return params = action[1] for body_key in FLOW_BODY_KEYS.get(name, ()): body = params.get(body_key) if body is not None: - _validate_list(body, known, f"{trail}.{body_key}") + yield from _iter_actions(body, f"{trail}.{body_key}") for list_key in FLOW_BRANCH_LIST_KEYS.get(name, ()): - _validate_branch_list(params.get(list_key), known, f"{trail}.{list_key}") - - -def _validate_branch_list(branches: Any, known: Set[str], trail: str) -> None: - """Validate a list of action lists (e.g. ``AC_parallel``'s ``branches``).""" - if branches is None: - return - # The visual builder may pass a JSON string, which the runtime parses via - # _as_list. Leave that shape to the runtime rather than reject it here. - if not isinstance(branches, list): - return - for idx, branch in enumerate(branches): - _validate_list(branch, known, f"{trail}[{idx}]") + branches = params.get(list_key) + # The visual builder may pass a JSON string, which the runtime parses + # via _as_list. Leave that shape to the runtime rather than reject it. + if not isinstance(branches, list): + continue + for idx, branch in enumerate(branches): + yield from _iter_actions(branch, f"{trail}.{list_key}[{idx}]") + + +__all__ = [ + "FLOW_BODY_KEYS", "FLOW_BRANCH_LIST_KEYS", + "validate_actions", "unknown_command_names", +] diff --git a/je_auto_control/utils/rest_api/rest_handlers.py b/je_auto_control/utils/rest_api/rest_handlers.py index 708f2b7b..7fd8c933 100644 --- a/je_auto_control/utils/rest_api/rest_handlers.py +++ b/je_auto_control/utils/rest_api/rest_handlers.py @@ -13,6 +13,10 @@ from typing import Any, Dict, List, Optional, Tuple from urllib.parse import parse_qs +from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, AutoControlActionNullException, + AutoControlException, AutoControlJsonActionException, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -138,12 +142,37 @@ def handle_commands(_ctx: RouteContext) -> HandlerResult: return 200, {"commands": names, "count": len(names)} +def _reject_bad_action_list(actions: Any) -> Optional[HandlerResult]: + """Return a 4xx result if ``actions`` is not dispatchable, else ``None``. + + A misspelt ``AC_*`` name is the caller's mistake, so it has to come back + as a 400 naming the command rather than the blanket 500 that any other + executor failure produces — otherwise a client cannot tell a typo from a + broken server. The whole list is checked before anything runs, which is + also what keeps a half-executed action list from being possible here. + """ + from je_auto_control.utils.executor.action_executor import executor + try: + unknown = executor.unknown_commands_in(actions) + except AutoControlException as error: + return 400, {"error": str(error)} + if unknown: + return 400, { + "error": "unknown command name(s): " + ", ".join(unknown), + "unknown_commands": unknown, + } + return None + + def handle_execute(ctx: RouteContext) -> HandlerResult: if not isinstance(ctx.body, dict): return 400, {"error": "body must be JSON object"} actions = ctx.body.get("actions") if actions is None: return 400, {"error": "missing 'actions' field"} + rejection = _reject_bad_action_list(actions) + if rejection is not None: + return rejection try: from je_auto_control.utils.executor.action_executor import execute_action result = execute_action(actions) @@ -162,6 +191,13 @@ def handle_execute_file(ctx: RouteContext) -> HandlerResult: try: from je_auto_control.utils.executor.action_executor import execute_files result = execute_files([path]) + except (AutoControlActionException, AutoControlActionNullException, + AutoControlJsonActionException) as error: + # The caller chose the path, so an unreadable file or an action list + # naming a command that does not exist is their input being wrong — + # same reasoning as _reject_bad_action_list, reported the same way. + autocontrol_logger.info("rest execute_file rejected: %r", error) + return 400, {"error": str(error)} except Exception as error: # noqa: BLE001 # pylint: disable=broad-except # reason: REST boundary must always return JSON, never drop the HTTP response autocontrol_logger.error("rest execute_file failed: %r", error) return 500, {"error": "execute_files failed"} diff --git a/je_auto_control/utils/rest_api/rest_openapi.py b/je_auto_control/utils/rest_api/rest_openapi.py index f6747507..11cf6221 100644 --- a/je_auto_control/utils/rest_api/rest_openapi.py +++ b/je_auto_control/utils/rest_api/rest_openapi.py @@ -218,6 +218,13 @@ }, }, }, + "errors": { + "400": ("Malformed body, or a command name the executor does " + "not know. The whole list is checked before anything " + "runs, so nothing was executed; 'unknown_commands' " + "lists every unrecognised name."), + "500": "The action list was valid but the run failed.", + }, }, ("POST", "/execute_file"): { "summary": "Run a JSON action file by absolute path.", @@ -229,6 +236,11 @@ "path": {"type": "string"}, }, }, + "errors": { + "400": ("Missing 'path', or the file is unreadable, is not a " + "valid action list, or names an unknown command."), + "500": "The action file was valid but the run failed.", + }, }, ("POST", "/config/export"): { "summary": "Export AutoControl user config as a JSON bundle.", @@ -367,6 +379,11 @@ def _build_responses(meta: Dict[str, Any]) -> Dict[str, Any]: "description": "Bad request body.", "content": {_JSON_MEDIA_TYPE: {"schema": _error_schema()}}, } + for code, description in meta.get("errors", {}).items(): + responses[code] = { + "description": description, + "content": {_JSON_MEDIA_TYPE: {"schema": _error_schema()}}, + } return responses diff --git a/test/unit_test/headless/test_rest_server.py b/test/unit_test/headless/test_rest_server.py index b2f02085..93abf31c 100644 --- a/test/unit_test/headless/test_rest_server.py +++ b/test/unit_test/headless/test_rest_server.py @@ -68,17 +68,87 @@ def test_execute_rejects_missing_actions(rest_server): assert "actions" in payload.get("error", "") +def test_execute_rejects_unknown_command_with_400(rest_server): + """A misspelt AC_* name is the caller's error, so 4xx and not 500.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute", method="POST", + body={"actions": [["AC_bogus_command"]]}, + token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert payload.get("unknown_commands") == ["AC_bogus_command"] + assert "AC_bogus_command" in payload.get("error", "") + + +def test_execute_lists_every_unknown_command(rest_server): + """All bad names come back at once, including ones nested in a body.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute", method="POST", + body={"actions": [ + ["AC_typo_one"], + ["AC_loop", {"times": 1, "body": [["AC_typo_two"]]}], + ]}, + token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert payload.get("unknown_commands") == ["AC_typo_one", "AC_typo_two"] + + +def test_execute_rejects_empty_action_list_with_400(rest_server): + """An empty list is bad input, not a server fault.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute", method="POST", + body={"actions": []}, token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert "error" in payload + + +def test_execute_file_rejects_missing_file_with_400(rest_server, tmp_path): + """The caller named the path, so an unreadable file is their error.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute_file", method="POST", + body={"path": str(tmp_path / "does_not_exist.json")}, + token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert "error" in payload + + +def test_execute_file_rejects_unknown_command_with_400(rest_server, tmp_path): + action_file = tmp_path / "actions.json" + action_file.write_text(json.dumps([["AC_bogus_command"]]), encoding="utf-8") + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute_file", method="POST", + body={"path": str(action_file)}, token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert "AC_bogus_command" in payload.get("error", "") + + def test_unknown_path_returns_404(rest_server): with pytest.raises(urllib.error.HTTPError) as exc_info: _request(rest_server, "/nope", token=rest_server.token) assert exc_info.value.code == 404 -def test_handler_crash_returns_500_not_dropped(rest_server): - """Sending an action list that raises must produce JSON, not RST.""" +def test_handler_crash_returns_500_not_dropped(rest_server, monkeypatch): + """A genuine server-side failure must produce JSON, not RST. + + The action list is valid, so the 400 gate lets it through and the crash + comes from the run itself — which is exactly the 500 that misspelt + command names used to be lumped in with. + """ + from je_auto_control.utils.executor import action_executor + + def _boom(_action_list): + raise MemoryError("simulated executor crash") + + monkeypatch.setattr(action_executor, "execute_action", _boom) with pytest.raises(urllib.error.HTTPError) as exc_info: _request(rest_server, "/execute", method="POST", - body={"actions": []}, token=rest_server.token) + body={"actions": [["AC_sleep", {"seconds": 0}]]}, + token=rest_server.token) assert exc_info.value.code == 500 payload = json.loads(exc_info.value.read().decode("utf-8")) assert "error" in payload From 92b08f0106aa70738876d745004929d39fa36b07 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:16:22 +0800 Subject: [PATCH 12/21] Wait out a clipboard another process is holding open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only one process may have the Windows clipboard open at a time, so RuntimeError: OpenClipboard failed escaped whenever anything else was mid-copy — roughly one call in a thousand on a live desktop. win32_clipboard_api.open_clipboard() is now the single place that opens it, retrying for about 200 ms before raising as before. --- je_auto_control/utils/clipboard/clipboard.py | 28 +++---- .../utils/clipboard/win32_clipboard_api.py | 49 ++++++++--- .../clipboard_formats/clipboard_formats.py | 10 +-- .../test_clipboard_win32_prototypes.py | 84 ++++++++++++++++--- 4 files changed, 125 insertions(+), 46 deletions(-) diff --git a/je_auto_control/utils/clipboard/clipboard.py b/je_auto_control/utils/clipboard/clipboard.py index 18ecce03..3759f5e5 100644 --- a/je_auto_control/utils/clipboard/clipboard.py +++ b/je_auto_control/utils/clipboard/clipboard.py @@ -17,8 +17,6 @@ from io import BytesIO from typing import Optional, Union -_OPEN_CLIPBOARD_FAILED = "OpenClipboard failed" - def get_clipboard() -> str: """Return the current clipboard text (empty string if empty).""" @@ -101,6 +99,8 @@ def _win_get() -> str: import ctypes from ctypes import wintypes + from je_auto_control.utils.clipboard.win32_clipboard_api import open_clipboard + user32 = ctypes.WinDLL("user32", use_last_error=True) kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) cf_unicodetext = 13 @@ -114,9 +114,7 @@ def _win_get() -> str: kernel32.GlobalLock.restype = ctypes.c_void_p kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_CLIPBOARD_FAILED) - try: + with open_clipboard(user32): handle = user32.GetClipboardData(cf_unicodetext) if not handle: return "" @@ -127,14 +125,14 @@ def _win_get() -> str: return ctypes.wstring_at(pointer) finally: kernel32.GlobalUnlock(handle) - finally: - user32.CloseClipboard() def _win_set(text: str) -> None: import ctypes from ctypes import wintypes + from je_auto_control.utils.clipboard.win32_clipboard_api import open_clipboard + user32 = ctypes.WinDLL("user32", use_last_error=True) kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) cf_unicodetext = 13 @@ -162,14 +160,10 @@ def _win_set(text: str) -> None: raise RuntimeError("GlobalLock failed") ctypes.memmove(pointer, ctypes.addressof(data), size) # NOSONAR S5655 false positive — Array is accepted by addressof kernel32.GlobalUnlock(handle) - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_CLIPBOARD_FAILED) - try: + with open_clipboard(user32): user32.EmptyClipboard() if not user32.SetClipboardData(cf_unicodetext, handle): raise RuntimeError("SetClipboardData failed") - finally: - user32.CloseClipboard() # === macOS backend =========================================================== @@ -262,6 +256,10 @@ def _win_set_image(png_bytes: bytes) -> None: import ctypes # noqa: PLC0415 from ctypes import wintypes # noqa: PLC0415 + from je_auto_control.utils.clipboard.win32_clipboard_api import ( # noqa: PLC0415 + open_clipboard, + ) + user32 = ctypes.WinDLL("user32", use_last_error=True) kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) cf_dib = 8 @@ -287,14 +285,10 @@ def _win_set_image(png_bytes: bytes) -> None: raise RuntimeError("GlobalLock failed") ctypes.memmove(pointer, dib, len(dib)) kernel32.GlobalUnlock(handle) - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_CLIPBOARD_FAILED) - try: + with open_clipboard(user32): user32.EmptyClipboard() if not user32.SetClipboardData(cf_dib, handle): raise RuntimeError("SetClipboardData(CF_DIB) failed") - finally: - user32.CloseClipboard() def _mac_get_image() -> Optional[bytes]: diff --git a/je_auto_control/utils/clipboard/win32_clipboard_api.py b/je_auto_control/utils/clipboard/win32_clipboard_api.py index 1e7f201a..028def63 100644 --- a/je_auto_control/utils/clipboard/win32_clipboard_api.py +++ b/je_auto_control/utils/clipboard/win32_clipboard_api.py @@ -15,12 +15,25 @@ """ import ctypes import sys +import time +from contextlib import contextmanager from ctypes import wintypes -from typing import Optional, Tuple +from typing import Iterator, Optional, Tuple GMEM_MOVEABLE = 0x0002 _OPEN_FAILED = "OpenClipboard failed" +# Only one process may hold the clipboard open at a time, so OpenClipboard +# fails outright whenever another application is mid-copy — Explorer, Office +# and every browser own it for a few milliseconds at a time. Reporting that as +# the caller's failure is wrong twice over: Win32 documents it as the condition +# to retry, and a library whose job is driving machines that are busy by +# definition cannot treat "somebody else was copying" as an error. Roughly +# 200 ms of waiting covers the transient owners without hanging a script +# behind one that keeps the clipboard for real. +_OPEN_ATTEMPTS = 10 +_OPEN_RETRY_SECONDS = 0.02 + def _require_windows() -> None: if not sys.platform.startswith("win"): @@ -61,6 +74,28 @@ def clipboard_api() -> Tuple[object, object]: return user32, kernel32 +@contextmanager +def open_clipboard(user32: Optional[object] = None) -> Iterator[object]: + """Own the clipboard for the block, waiting out a transiently busy one. + + Pass the ``user32`` handle you already prototyped to keep the declarations + private to your module; omit it and one is built here. The clipboard is + closed on the way out however the block ends — leaving it open locks every + other process on the desktop out of it. + """ + api = clipboard_api()[0] if user32 is None else user32 + for attempt in range(_OPEN_ATTEMPTS): + if api.OpenClipboard(None): + break + if attempt == _OPEN_ATTEMPTS - 1: + raise RuntimeError(_OPEN_FAILED) + time.sleep(_OPEN_RETRY_SECONDS) + try: + yield api + finally: + api.CloseClipboard() + + def register_format(name: str) -> int: """Register (or look up) a named clipboard format id.""" user32, _kernel32 = clipboard_api() @@ -84,23 +119,17 @@ def set_clipboard_format(format_id: int, payload: bytes, *, raise RuntimeError("GlobalLock failed") ctypes.memmove(pointer, payload, len(payload)) kernel32.GlobalUnlock(handle) - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_FAILED) - try: + with open_clipboard(user32): if empty_first: user32.EmptyClipboard() if not user32.SetClipboardData(int(format_id), handle): raise RuntimeError(f"SetClipboardData({format_id}) failed") - finally: - user32.CloseClipboard() def get_clipboard_format(format_id: int) -> Optional[bytes]: """Read the clipboard's ``format_id`` payload, or ``None`` when absent.""" user32, kernel32 = clipboard_api() - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_FAILED) - try: + with open_clipboard(user32): handle = user32.GetClipboardData(int(format_id)) if not handle: return None @@ -111,5 +140,3 @@ def get_clipboard_format(format_id: int) -> Optional[bytes]: return ctypes.string_at(pointer, kernel32.GlobalSize(handle)) finally: kernel32.GlobalUnlock(handle) - finally: - user32.CloseClipboard() diff --git a/je_auto_control/utils/clipboard_formats/clipboard_formats.py b/je_auto_control/utils/clipboard_formats/clipboard_formats.py index 9c7bc738..b25e3148 100644 --- a/je_auto_control/utils/clipboard_formats/clipboard_formats.py +++ b/je_auto_control/utils/clipboard_formats/clipboard_formats.py @@ -118,11 +118,11 @@ def list_clipboard_formats() -> List[Dict[str, Any]]: # Prototypes come from the shared module, on a *private* handle: declaring # them on the process-wide ``ctypes.windll.user32`` would leak into every # other caller in this process. - from je_auto_control.utils.clipboard.win32_clipboard_api import clipboard_api + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + clipboard_api, open_clipboard, + ) user32, _kernel32 = clipboard_api() - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: + with open_clipboard(user32): formats: List[Dict[str, Any]] = [] format_id = user32.EnumClipboardFormats(0) while format_id: @@ -130,8 +130,6 @@ def list_clipboard_formats() -> List[Dict[str, Any]]: "name": _format_name(user32, format_id)}) format_id = user32.EnumClipboardFormats(format_id) return formats - finally: - user32.CloseClipboard() def clipboard_formats() -> Dict[str, Any]: diff --git a/test/unit_test/headless/test_clipboard_win32_prototypes.py b/test/unit_test/headless/test_clipboard_win32_prototypes.py index 518bb8e9..c0d281ca 100644 --- a/test/unit_test/headless/test_clipboard_win32_prototypes.py +++ b/test/unit_test/headless/test_clipboard_win32_prototypes.py @@ -12,6 +12,13 @@ when the clipboard cannot be opened at all — a locked workstation, a session without a window station, or a non-Windows CI runner — rather than reporting a failure the environment made inevitable. + +That real clipboard is machine-global, which used to make these tests depend on +what every *other* process on the box was doing: one write from anything else +between our write and our read failed the run, and during a parallel Docker +build one did. :func:`_round_trip` closes that hole without giving up the real +Win32 calls — see its docstring for why faking the backend instead would delete +the only coverage this file has. """ import sys @@ -20,6 +27,9 @@ _WINDOWS = sys.platform.startswith("win") pytestmark = pytest.mark.skipif(not _WINDOWS, reason="Windows clipboard only") +# How many times a stolen round-trip is worth re-running before giving up. +_ATTEMPTS = 4 + def _clipboard_available() -> bool: from je_auto_control.utils.clipboard.clipboard import get_clipboard @@ -30,6 +40,50 @@ def _clipboard_available() -> bool: return False +def _sequence_number() -> int: + """Win32's clipboard change counter for this window station. + + It moves on every *modification*, by any process, and never on a read — + which is precisely the signal needed to tell "somebody stole my clipboard" + from "my writer is broken". + """ + import ctypes + from ctypes import wintypes + + user32 = ctypes.WinDLL("user32", use_last_error=True) + user32.GetClipboardSequenceNumber.argtypes = [] + user32.GetClipboardSequenceNumber.restype = wintypes.DWORD + return int(user32.GetClipboardSequenceNumber()) + + +def _round_trip(write, read): + """Run ``write`` then ``read`` as one uninterrupted pair; return the value. + + Faking the clipboard backend would make this file deterministic by + deleting the only thing it tests — the Win32 half, which is where all four + historical bugs were and which no pure-function test could reach. So the + calls stay real and the *interference* is detected instead: if the + sequence number has not moved between the end of our write and the end of + our read, nothing else wrote in that window, so the bytes we read are the + bytes we wrote and the assertion that follows is about our code alone. + + Nothing that looks like a bug is retried away: ``write`` raising is the + exact shape of the regression this file exists to catch, so it propagates + on the first attempt. The other half of the race — a clipboard another + process is holding *open* — is not handled here at all, because + ``win32_clipboard_api.open_clipboard`` waits that out for every caller of + the library, not just for this test. + """ + for _attempt in range(_ATTEMPTS): + write() + stamp = _sequence_number() + value = read() + if _sequence_number() == stamp: + return value + pytest.skip("another process kept overwriting the clipboard") + return None # unreachable; keeps the return type honest for linters + + @pytest.fixture() def clipboard(): """Skip when unusable, and put the user's clipboard back afterwards.""" @@ -52,32 +106,38 @@ def test_text_round_trip(clipboard): from je_auto_control.utils.clipboard.clipboard import ( get_clipboard, set_clipboard, ) - set_clipboard("round-trip probe") - assert get_clipboard() == "round-trip probe" + read_back = _round_trip(lambda: set_clipboard("round-trip probe"), + get_clipboard) + assert read_back == "round-trip probe" def test_html_round_trip(clipboard): from je_auto_control.utils.rich_clipboard.rich_clipboard import ( get_clipboard_html, set_clipboard_html, ) - set_clipboard_html("hi") - assert "hi" in (get_clipboard_html() or "") + read_back = _round_trip(lambda: set_clipboard_html("hi"), + get_clipboard_html) + assert "hi" in (read_back or "") def test_rtf_round_trip(clipboard): from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( build_rtf, get_clipboard_rtf, set_clipboard_rtf, ) - set_clipboard_rtf(build_rtf("hello")) - assert "hello" in (get_clipboard_rtf() or "") + read_back = _round_trip(lambda: set_clipboard_rtf(build_rtf("hello")), + get_clipboard_rtf) + assert "hello" in (read_back or "") def test_csv_round_trip(clipboard): from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( get_clipboard_csv, set_clipboard_csv, ) - set_clipboard_csv([["a", "b"], ["c", "d"]]) - assert get_clipboard_csv() == [["a", "b"], ["c", "d"]] + read_back = _round_trip( + lambda: set_clipboard_csv([["a", "b"], ["c", "d"]]), + get_clipboard_csv, + ) + assert read_back == [["a", "b"], ["c", "d"]] def test_file_list_round_trip(clipboard, tmp_path): @@ -86,8 +146,9 @@ def test_file_list_round_trip(clipboard, tmp_path): ) one = tmp_path / "one.png" one.write_bytes(b"x") - set_clipboard_files([str(one)]) - assert get_clipboard_files() == [str(one)] + read_back = _round_trip(lambda: set_clipboard_files([str(one)]), + get_clipboard_files) + assert read_back == [str(one)] def test_format_enumeration_sees_what_was_written(clipboard): @@ -95,8 +156,7 @@ def test_format_enumeration_sees_what_was_written(clipboard): from je_auto_control.utils.clipboard_formats.clipboard_formats import ( clipboard_formats, ) - set_clipboard("text only") - summary = clipboard_formats() + summary = _round_trip(lambda: set_clipboard("text only"), clipboard_formats) assert summary["has_text"] is True assert summary["has_files"] is False From 3928d9a99cb20f008484a6bca01faa44faa15da7 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:16:57 +0800 Subject: [PATCH 13/21] Record what shipped, what was decided, and how to re-measure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CLAUDE.md now says which limits a CI job actually rejects — line length, and nothing else — so the section stops being read as a gate. The file-length limit gets its scope written down: new files always, rewrites and growth yes, flat data tables no, everything else grandfathered in Progress.md and nowhere else. test_doc_line_counts.py measures every line count the architecture map quotes and rewrites all of them with --fix. Hand-editing is how the map ended up quoting one subsystem at two sizes at once: most tables had been counting a phantom trailing line per file while the totals counted correctly. Progress.md loses both DECIDE items. Pointer acceleration became an operator declaration; the software cursor in a Wayland capture becomes documentation, because both ways around it need a cursor position Wayland will not give a client, and masking the wrong place is worse than a visible pointer. What is left there is external: two libeis observations with no peer to drive them, the consent dialog as a dialog, and whether a GitHub runner can modprobe uinput and evdev. --- CHANGELOG.md | 158 +++ CLAUDE.md | 29 +- Progress.md | 123 ++- README.md | 55 +- README/README_zh-CN.md | 44 +- README/README_zh-TW.md | 44 +- README/WHATS_NEW_zh-CN.md | 485 +++++++++ README/WHATS_NEW_zh-TW.md | 485 +++++++++ WHATS_NEW.md | 676 +++++++++++++ architecture_explore.md | 951 +++++++++--------- docs/CAPABILITY_MATRIX.md | 122 ++- examples/22_wayland_backend.py | 15 + .../headless/test_doc_line_counts.py | 350 +++++++ 13 files changed, 3057 insertions(+), 480 deletions(-) create mode 100644 test/unit_test/headless/test_doc_line_counts.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b9ebd26a..5aeea62a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -82,9 +82,20 @@ only when documented here with a migration path. from the facade, with `AC_canonicalize_url` / `AC_normalize_url` / `AC_urls_equal`, the matching `ac_*` MCP tools, and three Script Builder specs. The module and its tests already existed; only the wiring is new. +- `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` — how an operator declares what the + library cannot read back. `flat` says pointer acceleration is off for the + ydotoold device, so an absolute move through the ydotool fallback is exact + and needs no warning; `strict` refuses that move instead of letting a click + land somewhere else; unset (or any unrecognised value, which says so and + falls back) keeps the existing warn-once-and-move behaviour. The libei path + is absolute at the protocol level and is not affected either way. ### Removed +- `je_auto_control.linux_wayland._detect.WAYLAND_GDBUS` is gone, along with the + `gdbus` probe it named: the desktop-portal capture tier no longer shells out + to any binary. `_detect` is a private module and nothing else referenced the + constant. - **Breaking — `je_auto_control.windows.listener` is gone**, with its `Win32KeyboardListener` and `Win32MouseListener` classes. Recording moved to `windows/record/win32_input_hook.py`, after which nothing in the package or @@ -99,6 +110,60 @@ only when documented here with a migration path. ### Changed +- **The `xdg-desktop-portal` capture tier no longer needs `gdbus` installed.** + It speaks D-Bus directly, so `linux_wayland.portal.is_available()` now + reports whether a session bus address is set rather than whether the `gdbus` + binary is on `PATH`. This widens where the last-resort tier runs; the install + hint in the "no capture tool found" error and the `screen_capture` + diagnostics check were reworded to match. +- **`LibeiBackend.scroll()` sends whole wheel clicks, not raw detent counts.** + libei measures discrete scroll in 120ths of a click, so the previous call + asked for 1/120th of the scroll requested and libei logged it as a client + bug. Measured against a real EIS server (`docker/eis_verify.py`). +- **Wayland `mouse.scroll()` goes through libei where libei is up, instead of + always shelling out to ydotool.** Motion, buttons and keys already did; + scroll was held back because its sign was a guess. The two paths count + wheel detents in opposite directions — this repository's + `wayland_scroll_direction_*` constants are in the kernel's `REL_WHEEL` + frame, which is what ydotool writes (positive is up), and libei is in the + `wl_pointer` frame (positive is down) — so the vertical axis is negated on + the way to libei and the horizontal one is not. No API change: scrolling on + a libei host no longer needs ydotool or a uinput daemon at all. +- **A libei emission that a live backend refuses now falls back to the CLI, + as `libei`'s own docstring already claimed it did.** Only the *connection* + degraded; a compositor that paused a device, or a session that ended + between two calls, raised out of `set_position` / `press_key` / `hotkey` + instead of reaching ydotool. A chord refused part-way releases the keys it + already pressed before handing over, so no modifier is left held. +- **`LibeiUnavailable` derives from `AutoControlException`** (as well as + `RuntimeError`, which existing probes catch). It was a bare `RuntimeError`, + so it escaped every `except AutoControlException` containment boundary — + the executor, the poll loops, the request handlers and the GUI slots. +- **A libei session that completed its handshake is released instead of + abandoned.** `ei_unref` segfaults on libei 1.3.901 only for a context whose + backend opened and whose handshake never progressed; with an EIS peer to + test against, the live case is measurably safe. Teardown no longer leaks a + context and a file descriptor per process. + +- **`POST /execute` and `POST /execute_file` answer `400`, not `500`, for a + command name the executor does not know.** Both used to funnel every + executor failure into `500 {"error": "execute_action failed"}`, so a client + could not tell a typo in its own request from a broken server. Every name in + the list — nested flow-control bodies included — is now checked before + anything runs; an unrecognised one comes back as + `400 {"error": ..., "unknown_commands": [...]}` naming all of them, and + nothing was executed. `/execute_file` answers the same way for a path that + is unreadable or holds something that is not an action list. A client that + keyed off `500` to detect a bad request must key off `400` instead. +- **Windows clipboard calls wait out a clipboard another process is holding + open** instead of failing immediately. Only one process may have it open at + a time, so `RuntimeError: OpenClipboard failed` used to escape whenever + anything else was mid-copy — roughly one call in a thousand on a live + desktop. `win32_clipboard_api.open_clipboard()` is the one place that opens + it now, retrying for about 200 ms; the failure is still raised after that. + Callers that relied on an immediate failure will see a call take up to + 200 ms longer in the contended case. + - **`send_key_event_to_window` / `send_mouse_event_to_window` now actually reach the target.** They posted to the top-level frame, but keyboard messages go to the control that *has focus* and a click belongs to the child under the @@ -182,6 +247,99 @@ only when documented here with a migration path. ### Fixed +- **Wayland: an absolute mouse move through the ydotool fallback counted from + the wrong origin.** `ydotool mousemove --absolute` emits no absolute event — + it drives the cursor into the corner the compositor clamps to and then moves + relative to it, and that corner is the top-left of the output layout rather + than layout `(0, 0)`. On a layout with a monitor left of the primary one the + two differ by the layout origin, so `set_position(x, y)` landed a monitor's + width away from the coordinate the capture path had located. It now + subtracts `layout_origin()`, the same correction `grab_image` applies. + Measured against a real wlroots session consuming the real ydotool device + (`docker/Dockerfile.seat`, the new `seat-verification` job). Layouts whose + outputs all sit at non-negative positions are unaffected. +- **Wayland: the same call is only pixel-accurate where pointer acceleration + is off.** The displacement ydotool sends is relative motion, so the + compositor accelerates it — libinput's default adaptive profile moves the + cursor exactly twice as far as asked. This cannot be corrected from inside + the library, because the factor is the compositor's setting; the backend now + logs the caveat once per process rather than mispositioning in silence. + Disable acceleration for the ydotoold device (sway: `input type:pointer + accel_profile flat` and `pointer_accel 0`), or install `liboeffis` so the + libei path — absolute at the protocol level — is used instead. Once it is + off, `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` silences the warning, and + `=strict` refuses the move rather than warn about it. + +- **The Wayland `xdg-desktop-portal` screen-capture tier could never have + succeeded.** `org.freedesktop.portal.Screenshot` returns a request handle and + delivers the image later as a `Response` signal **directed at the connection + that made the call**; the bus routes a directed message to its destination + and nowhere else. The implementation listened on a `gdbus monitor` + subprocess and called from a separate `gdbus` invocation — two connections, + so the listener was never the addressee. Against a real `dbus-daemon` the + capture ran out its full 30-second timeout every time. The tier now speaks + D-Bus itself on a single connection, subscribing to the request path it + predicts before it calls. No API changed; a path that always failed now + works. +- **On Wayland, a monitor placed left of or above the primary one made every + capture path read the wrong pixels.** The compositor lays its outputs out on + one plane, and that plane starts at a negative coordinate as soon as an + output sits left of (or above) the origin — a `-1280,0` + `0,0` pair is one + 2560x720 layout whose top-left pixel is at x=-1280. Three places assumed the + layout began at `(0, 0)`: `screen.size()` returned `max(x + width)`, the + layout's *right edge* (1280) rather than its width (2560), so everything + that composes size with a capture — the mss-shaped shim's monitor list, + `enumerate_monitors`, the recorder, the WebRTC host, the MCP monitor grab — + asked for half the desktop and called it the whole screen; the region crop + taken when the capture tier cannot apply one itself (gnome-screenshot, + spectacle, the portal, an operator's own command) cropped in layout + coordinates on a layout-origin image, which returns black padding instead of + the left-hand monitor; and `grab_logical` reported an origin of `(0, 0)`, so + a template or OCR match found on that monitor was reported 1280 px to the + right of where it was seen and the click landed on the wrong screen. The + Wayland backend now publishes `layout_origin()`, `size()` returns the + bounding box's size, the crop subtracts the origin, and the generic capture + layer exposes `screen_grabber.backend_layout_origin()` for the paths that + map a pixel back to a screen coordinate. Verified against a real headless + sway session laid out that way — the `wayland-verification` job now runs its + 27 checks over both layouts. + +- **On Wayland, `set_position` could move nothing at all and report success.** + libei accepts absolute motion only inside the regions the compositor + advertises for the pointer, and it discards a point outside every one of + them without a return code, an event or an error — so the move was lost in + silence and never reached the ydotool fallback that could have made it. + Region offsets are `uint32`, so no compositor can advertise a region left of + or above the origin, while the layout space this project addresses starts at + `layout_origin()` and goes negative on the same "monitor left of the primary" + desktop fixed above: on such a layout the input and capture halves named + different pixels, and the pointer went nowhere rather than to the wrong + screen. The libei sender now reads the device's regions, sends a covered + coordinate unchanged, retries an uncovered one normalised by the layout + origin, and refuses what neither covers so `_select_input` hands the move to + ydotool. A device that advertises no region is unaffected. Verified against a + real EIS peer — the `eis-verification` job now runs 20 checks, five of them + on this coordinate space. The ydotool path's own origin remains unverified + and unchanged; see `Progress.md`. + +- **The Wayland ydotool fallback reported success while sending nothing on + Debian and Ubuntu.** ydotool 1.0 replaced its entire command line, and every + argument this backend builds arrived in that release (`mousemove + --absolute`, `mousemove --wheel`, hex `click` bitmasks, `key CODE:STATE`). + Debian bookworm, Ubuntu 22.04 and Ubuntu 24.04 all ship 0.1.8 under the name + `ydotool`, and 0.1.8 exits **0** for those arguments while emitting no + events at all — including for the ones it rejects with `unrecognised + option`. Since the backend runs ydotool with `check=True`, nothing raised: + clicks, keystrokes and cursor moves silently did nothing and every call + reported success. AutoControl now classifies the installed ydotool once per + process and raises `AutoControlException` naming the fix instead of + emitting. **Migration**: install ydotool 1.0+ (Arch, Fedora and Debian + unstable package it; Debian trixie packages none at all), or set + `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` to drive XWayland. A version the + probe does not recognise is allowed through, so this cannot block a future + release. The two `ydotool` install hints no longer suggest `apt install + ydotool`, which is what produced the broken version. + - **Four clipboard writers never worked on 64-bit Windows**: `set_clipboard_files`, `set_clipboard_html`, `set_clipboard_rtf` and `set_clipboard_csv` all raised `OverflowError: int too long to convert` on diff --git a/CLAUDE.md b/CLAUDE.md index 23638098..dc90059c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,13 +60,20 @@ The map is only useful while it matches the tree, so **update it in the same cha python -c "from je_auto_control.utils.mcp_server.tools import build_default_tool_registry as b; print(len(b()))" # MCP tools ``` - Module and line counts come from walking the tree with `ast`; recompute §1, the affected §5.4 theme totals, and the §8 size appendix together so they stay consistent. + Every line count in the map — §1's totals, the §5.4 theme tables, the §5.4.17 file tables, the `####` headings and the §8 appendix — is rewritten in one pass by: + + ```bash + python test/unit_test/headless/test_doc_line_counts.py --fix # then review the diff + ``` + + Never adjust one by hand: the counts are `len(text.splitlines())` (what `wc -l` reports), and hand-editing is how the document ended up quoting the same subsystem at two sizes at once — most tables had been counting a phantom trailing line per file while §1 and §8 counted correctly. - A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 308 subpackages; appearing twice or not at all is a defect. - A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. - Keep the header's scan date, version, and branch current. - `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. - **`test/unit_test/headless/test_doc_counts.py` enforces this and fails CI on a mismatch.** It re-measures the command, MCP-tool, `utils/` subpackage and `examples/` counts and compares them against every place the four documents quote them, so code and docs have to move in the same commit. If you reword a sentence that holds one of those numbers, update the test's pattern — it fails loudly when a citation disappears rather than passing on a document it can no longer read. The GUI tab count is guarded the same way but from `test_actions_menu_gui.py`, whose subprocess probe already builds the widget that count needs. +- **`test_doc_line_counts.py` does the same for every line count**, and is also the `--fix` tool above — so any change in module size reddens CI until the map is re-measured, and re-measuring is one command. ### Outstanding work goes in `Progress.md` @@ -94,10 +101,28 @@ Anything agreed but not done — deferred follow-ups, known gaps, half-delivered - **Pin dependency versions**, including transitive ones that can change return shapes (`opencv-python` is bounded `<6` for exactly this reason). Review new dependencies for known vulnerabilities. - Common logic belongs in `wrapper/` or `utils/`, never duplicated across platform backends. -### Limits enforced by CI +### Size and complexity limits Cyclomatic complexity ≤ 10 · cognitive complexity ≤ 15 · function ≤ 75 lines · parameters ≤ 7 · nesting ≤ 4 · file ≤ 750 lines · line ≤ 120 chars · no duplicated block ≥ 10 lines. +**What actually enforces these.** `quality.yml` runs ruff and bandit only, so line length is the only limit a CI job rejects. Complexity is measured by `radon cc -nc` in the pre-commit list below and read by a human. The file-length limit is enforced by nobody — treat this section as a review standard, not a gate, and do not describe it as CI-enforced. + +**Scope of the file-length limit.** It applies to: + +- every **new** file, without exception — if it does not fit in 750 lines it is more than one module; +- any existing file being **substantially rewritten** — reorganising is the moment to split; +- any existing file that would **grow**: an over-limit file may be edited and may shrink, but a change that pushes it further over needs the split to come first, or a `Progress.md` entry saying why not. + +It does not apply to **flat data tables**: a file that is one mapping or list of entries, with no branching logic, where splitting would fragment a single lookup structure. Today that means the `AC_*` dispatch table, the MCP tool registry, the Script Builder command schema, the façade's re-exports, and the per-language string catalogues. The test is structural, not a label — if a file has to be *read* as a table, it is one. Logic that grew alongside a table does not inherit the exemption. + +Everything else currently over the limit is **grandfathered and listed in `Progress.md`**, which is the only place a standing exception may live. An over-limit file with no entry there is a defect, and the list is measured, not remembered: + +```bash +find je_auto_control -name '*.py' -not -path '*__pycache__*' -exec awk 'END{if(NR>750)print NR"\t"FILENAME}' {} \; | sort -rn +``` + +Adding a file to that list is a last resort, not a routine escape. + Docstrings on every public module, class, and function (one-line summary minimum; type hints replace parameter-type prose). Type hints on all public signatures. Import order stdlib → third-party → first-party; no wildcard imports outside the `__init__.py` façade. ### Automated verification diff --git a/Progress.md b/Progress.md index 7088a115..f0777f8c 100644 --- a/Progress.md +++ b/Progress.md @@ -14,16 +14,123 @@ --- -## [TODO] `windows_backend.py` 915 行，超過 750 行上限 +## 750 行上限的既有豁免清單 -`je_auto_control/utils/accessibility/backends/windows_backend.py` 目前 915 行，超出 -`CLAUDE.md` §Limits 的 750 行。拆出 `windows_query.py`（170）與 `windows_state.py`（98）之後 -仍然超標——這個檔在拆之前就已經是 772 行，後續補視窗限定搜尋與控制項模式又長回來。 +`CLAUDE.md` §Size and complexity limits 規定:超標檔案只能列在這裡,列不進來的就是缺陷。 +清單上的檔案**可以改、可以變短,但不得再變長**——要再長就得先拆。 +行數為 2026-08-19 實測（`len(text.splitlines())`）。 -- **注意**：目前**沒有任何 CI job 在檢查行數與複雜度**（`quality.yml` 只有 ruff 與 bandit）， - 所以這條上限實際上靠自律；`action_executor.py` 8,021 行、`_factories.py` 8,866 行同樣超標。 -- **待決**：是要真的拆這個檔、把上限改成符合現況的數字，還是把這條規則的適用範圍寫清楚。 +| 檔案 | 行數 | 為何還沒拆 | +| --- | ---: | --- | +| `utils/mcp_server/tools/_handlers.py` | 4,789 | 676 個 MCP 工具的處理函式本體。與 `_factories.py`（表）不同,這裡是邏輯,應該依主題拆成 `_handlers/` 套件（input／screen／window／file／agent…）。拆點清楚,純粹是量大。 | +| `gui/remote_desktop/webrtc_panel.py` | 2,555 | 單一 Qt 面板,但已含連線、監視器選擇、頻寬自適應、麥克風、錄影五組互動狀態。應拆成 panel + 各控制器。 | +| `utils/accessibility/backends/windows_backend.py` | 915 | 已拆出 `windows_query.py`（170）與 `windows_state.py`（98）。剩下的是同一套 UIA COM 生命週期管理,再拆會把 `CoInitialize`／介面釋放的配對邏輯切散。 | + +**本質豁免（依 `CLAUDE.md` 的「flat data tables」條款,不算既有豁免）**: +`utils/mcp_server/tools/_factories.py`（8,968,MCP 工具註冊表）、 +`utils/executor/action_executor.py`（8,125,`AC_*` 分派表）、 +`gui/script_builder/command_schema.py`（5,051,每個 `AC_*` 的參數 schema）、 +`je_auto_control/__init__.py`（1,970,門面 re-export）、 +`gui/language_wrapper/{english,japanese,traditional_chinese,simplified_chinese}.py` +（1,316／1,203／1,189／1,188,語系字串表）。 + +### 2026-08-19 決議:上表的實測行數就是新的上限 + +2026-08-18 重新實測時,表上原有的七列**全部**變長,而 `CLAUDE.md` 明寫 +「列上的檔案不得再變長,要再長就得先拆」,所以這裡曾標成 `[DECIDE]`。 +**維護者已於 2026-08-19 拍板:接受實測數字當新基準**——不為了回到舊數字而去拆 +`_handlers.py`（4,789）與 `webrtc_panel.py`（2,555）。上表的行數即是各自的新上限, +規則不變:只准變短,再變長就得先拆。 + +同一批裡有六個檔案在 2026-08-19 已經拆回線內、從表上移除,做法寫在 +[WHATS_NEW.md](WHATS_NEW.md)。 + +行數沒有任何 CI 在把關（`quality.yml` 只跑 ruff 與 bandit,而 ruff 只管行寬), +所以這張表只會在有人手動實測時才會被發現對不上——上次就是。 --- -（目前沒有其他待辦。） +## Wayland:剩下的都不是「缺一台機器」 + +這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, +中間還有負原點的擷取。三次都不是,三次都是同一個誤判:**把「合成器／桌面做不到的事」 +當成了「容器做不到的事」**。portal 是 D-Bus 介面,誰佔住那個名字誰就是 portal; +「會吃 libinput 裝置的 seat」是 wlroots 的 `WLR_BACKENDS=headless,libinput` 加 +`LIBSEAT_BACKEND=builtin` 加 `SEATD_VTBOUND=0`（第四個條件是 udev 要比 ydotoold 早起 +來)。都已經是 CI job 了,見下面「已經有答案的」與 [WHATS_NEW.md](WHATS_NEW.md)。 + +**下次要往這裡加「需要一台 VM／真桌面」之前,先問這件事到底是誰做不到。** + +### 還沒有答案的 + +- **`eis_device_pause()` 在 libeis 1.3.901 對 sender client 沒有送出任何東西。** + 對兩個 live device 呼叫 pause 再 dispatch,client 端的 ei fd 4 秒內完全沒有可讀資料。 + 所以 `LibeiBackend._on_event` 的 `DEVICE_PAUSED`／`DEVICE_REMOVED` 分支**仍然沒有 + peer 可以驅動**。`eis_verify.py` 把這件事寫成「要嘛有反應,要嘛根本沒被通知」, + 若哪天 libeis 開始送了,client 忽略它就會當場失敗。真的合成器上會不會不一樣,未知。 +- **`ei_device_start_emulating()` 的 sequence number 沒有被送到對面。** + 刻意送 4242 過去,server 讀回來是 0。我們這邊的計數本身符合標頭檔的約定 + （每次呼叫至少 +1）,所以不影響正確性,只是從對面驗不到。 +- **同意對話框「長什麼樣子、真人要按多久」。** portal 這一層現在驗到的是對話框 + *產生的東西*:准（Response 0）、拒（Response 1）、以及一直不回答。三種我們都在真的 + bus 上跑過,三種都得在自己的時限內收斂。至於真的 mutter 對話框長什麼樣、真人猶豫 + 三十秒會不會撞到別的東西,那是 mutter 的事,CI 裡沒有人可以去按它。 +- **`ydotool-verification` 與 `seat-verification` 兩個 job 在 GitHub runner 上能不能 + `modprobe uinput evdev`。** 本機（Docker Desktop 的 WSL2 kernel, + `CONFIG_INPUT_EVDEV=m`）兩個都確認跑得完,runner 上還沒跑過;job 寫成模組載不起來 + 就明講失敗,不會靜默跳過。`seat-verification` 還多一個前提:它要 `systemd-udevd` + 在容器裡收得到 kernel uevent。本機收得到（非網路裝置的 uevent 會廣播到所有 + network namespace),runner 上同理但未驗;收不到的話 entrypoint 會在 + `libinput list-devices` 那一步就明講失敗。 + +### 已經有答案的（都在 CI 裡,做法見 WHATS_NEW） + +| 面向 | 怎麼驗的 | job | +| --- | --- | --- | +| 擷取路徑 | 真的 wlroots 合成器（sway headless,兩個上不同純色的 output）,27 項 × 2 種版面 | `wayland-verification` | +| libei 協定層 | 真的 `libeis.so.1` server 在 Unix socket 上,20 項 | `eis-verification` | +| RemoteDesktop portal 交握 | 真的 `dbus-daemon` + 真的 `liboeffis`,對面是自己實作的 portal,`ConnectToEIS` 交出通往真 libeis 的活 fd,20 項 | `portal-verification` | +| ydotool CLI | 真的 uinput 裝置,直接讀回 `/dev/input/eventN`,12 項 | `ydotool-verification` | +| ydotool 的絕對移動落在哪 | 真的 wlroots session 吃真的 ydotool 裝置（`headless,libinput` + builtin seat）,游標位置從 `grim -c` 的像素讀回,14 項 × 2 種版面 | `seat-verification` | + +擷取那一列的第二種版面是**負原點**:`output HEADLESS-1 position -1280 0`, +也就是「第二台螢幕在主螢幕左邊」的桌面。sway headless 收這個座標,grim 也收負的 +`-g`,所以這件事根本不必等 GNOME VM——原本記在這裡說測不到,是把「合成器做得到的事」 +當成了「容器做不到的事」。跑起來當場抓到三個真的錯:`size()` 回的是版面右緣不是寬度、 +非 grim 層級的裁切用版面座標去裁一張以版面原點為 (0,0) 的圖、`grab_logical()` 一律回 +原點 (0,0) 所以比對到的座標整個偏掉。修法見 [WHATS_NEW.md](WHATS_NEW.md)。 + +portal 那一列是同一個錯誤犯第二次的結果,而它抓到的東西比前一次更嚴重: +`portal.py` 那條「先開 `gdbus monitor`、再用 `gdbus call` 發請求」的路 +**在任何真的 bus 上都不可能成功**——portal 的 `Response` 是**指名送給發出呼叫的那條 +連線**,兩個 gdbus 行程是兩條連線,監聽的那條永遠不是收件人。在真的 `dbus-daemon` 上 +量到的就是這樣:呼叫看得到,回答永遠等不到,每次都走到 30 秒逾時。修法見 +[WHATS_NEW.md](WHATS_NEW.md)。 + +五者都不需要合成器以外的東西,更不需要 GNOME VM。libei 這一層驗掉的包含 +capability enum 值與 variadic `ei_seat_bind_capabilities`、event-type enum 值、 +`start_emulating` → 事件 → `frame` 的實際上線內容、live context 的 teardown +安全性（原本每個行程漏一個 context + 一個 fd,已修）、以及絕對指標的座標空間 +（region offset 讀得回來且含在座標裡、region 外的移動被靜靜丟掉、負原點的版面要 +正規化）。portal 這一層驗掉的是四個呼叫的順序與 client 自己預測的 request path、 +`SelectDevices` 收到的裝置遮罩（也就是使用者被要求同意的範圍）、交回來的 fd 真的 +承載得起一個 EI session,以及六種拒絕路徑各自都要 fail closed。ydotool 這一層驗掉的是 +`click` 位元遮罩、拆邊的 press／release、`mousemove --absolute` 的實際上線內容、 +捲動正負號與軸向,以及 `mouse`／`keyboard` 自己組出來的 argv。seat 這一層驗掉的是 +`--absolute` 到底相對於哪裡（版面左上角,不是版面座標的 `(0, 0)`)、關掉加速度後 +一像素對一像素、沒轉換的 `(0, 0)` 會打到隔壁螢幕、`set_position` 減掉的正好是原點、 +以及預設 profile 下的 2 倍加速。 + +### 一件關於發行版的事實,會影響使用者拿到什麼 + +- **`liboeffis` 是獨立的二進位套件,`libei1` 不會把它帶進來。** Debian trixie + **有** `liboeffis1`（1.3.901-1,`liboeffis.so.1`,連 libsystemd 的 sd-bus）—— + 這裡原本寫「Debian trixie 沒有」,是錯的,已實測更正。Arch（1.6.0）與 Fedora 也有。 + 但因為它不是 `libei1` 的相依,只裝 libei 的機器上 portal 快速路徑仍然是關閉的, + `connect()` 會退到 `$XDG_RUNTIME_DIR/eis-0` socket,GNOME／KDE 不開那個 socket + → 退回 ydotool。**所以要用 libei 快速路徑,`liboeffis` 得自己裝。** +- 而那條退路本身,在同一批發行版上原本是壞的——0.1.x 對本專案送的 argv 回傳 0 + 卻不送任何事件。已於 2026-08-19 擋掉,見 CHANGELOG 與 WHATS_NEW;此處無待辦。 + +**緩解**:驗不到的擷取部分有逃生門——`JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` 讓操作者 +直接指定自己的擷取指令（`{output}` 會被換成暫存 PNG 路徑）,優先於所有偵測。 diff --git a/README.md b/README.md index 41039c33..18066393 100644 --- a/README.md +++ b/README.md @@ -241,10 +241,63 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Windows 10 / 11 | Win32 ctypes (+ optional Interception driver) | ✅ | ✅ | ✅ | ✅ | | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | | Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ❌ | -| Linux Wayland | libei, or ydotool / wtype / grim | ✅ | ✅ | ❌ | ❌ | +| Linux Wayland | libei via the desktop portal, or ydotool / wtype + a capture tool | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +Wayland input falls back to the `ydotool` CLI wherever libei is not +reachable, and that fallback needs **ydotool 1.0 or newer**. Every argument +AutoControl builds arrived in that release; 0.1.x — which is what Debian +bookworm and every current Ubuntu still ship under that name, and Debian +trixie ships not at all — answers the same arguments with exit code 0 and no +events. AutoControl detects it and refuses rather than reporting success for +input it never sent. Arch, Fedora and Debian unstable package 1.0. + +That fallback also positions the pointer accurately **only where the +compositor's pointer acceleration is off**. `ydotool mousemove --absolute` +sends no absolute event: it drives the cursor into the corner the compositor +clamps to and then moves relative to it, so the compositor accelerates the +move — measured against a real wlroots session, libinput's default profile +travels exactly twice the distance asked for. ydotool's own `--help` says the +same; AutoControl logs it once per process rather than mispositioning in +silence. Turn acceleration off for the ydotoold device (sway: `input +type:pointer accel_profile flat` and `pointer_accel 0`), or install +`liboeffis` so the libei path — absolute at the protocol level — is used. + +The factor is the compositor's own setting and no client can read it back, so +only you know whether it is off. `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` +says it is, and moves silently; `=strict` refuses the move rather than let a +click land somewhere else; leaving it unset keeps the warn-and-move default. + +Wayland screen capture needs the tool your compositor supports, because no +single one covers them all: `grim` on wlroots compositors (sway, Hyprland, +river), `gnome-screenshot` on GNOME, `spectacle` on KDE. Install one and every +capture path — screenshots, image and anchor locators, OCR, screen recording, +remote desktop — goes through it. With none of them installed, `gdbus` is +enough: `xdg-desktop-portal` is tried last, though it may ask for consent the +first time. Failing that, capture fails loudly with an install hint rather than +returning the blank XWayland root. The `screen_capture` check in +`je_auto_control.api.run_diagnostics()` (and the GUI Diagnostics tab) reports +which tier is in use. + +One Wayland-only difference to plan around: **a capture may contain the mouse +cursor.** Nothing here asks for it, but wlroots composites a *software* cursor +into the output buffer whenever the backend has no cursor plane — which +includes any session run with `WLR_NO_HARDWARE_CURSORS=1` — and that buffer is +what screen capture hands back. Windows and X11 never include the pointer, so a +locator, a template match or an OCR read can find a pointer-shaped hole in the +middle of its target here and nowhere else. Wayland does not let a client read +the cursor position, so there is nothing to reliably mask or move around it: +park the pointer away from what you are about to capture. The `screen_capture` +check reports this as `cursor_may_be_captured`. + +For a setup none of that fits, name your own command — it wins over every +detected tool, and `{output}` is replaced with a temporary PNG path: + +```bash +export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" +``` + Wayland forbids global input recording for unprivileged clients — set `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` to record on an X11 session. Window management is currently Windows-only and raises a clear `NotImplementedError` diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 30e193e5..48447f7b 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -230,10 +230,52 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Windows 10 / 11 | Win32 ctypes（可选 Interception 驱动） | ✅ | ✅ | ✅ | ✅ | | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | | Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ❌ | -| Linux Wayland | libei，或 ydotool／wtype／grim | ✅ | ✅ | ❌ | ❌ | +| Linux Wayland | 经桌面 portal 的 libei，或 ydotool／wtype ＋ 截图工具 | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +Wayland 的输入在 libei 走不通时会退回 `ydotool` CLI，而这条退路需要 +**ydotool 1.0 以上**。AutoControl 送的每一个参数都是那一版才有的；0.1.x +（Debian bookworm 与目前所有 Ubuntu 仍以这个名字提供，Debian trixie 则根本没有） +对同一批参数返回 0 却不送出任何事件。AutoControl 会检测并直接拒绝， +而不是为根本没送出的输入回报成功。Arch、Fedora 与 Debian unstable 提供的是 1.0。 + +这条退路要能**准确定位**，还有一个前提:合成器的指针加速度必须是关的。 +`ydotool mousemove --absolute` 并不发任何绝对事件——它先把光标推到合成器夹取的 +那个角落，再发相对位移，所以这段位移会被合成器加速。对真的 wlroots session 量到的是: +libinput 的默认 profile 让光标走的距离正好是请求的两倍。ydotool 自己的 `--help` +也是这样写的；AutoControl 每个进程会记一次警告，而不是安静地把点击放到错的地方。 +请对 ydotoold 的设备关掉加速度（sway:`input type:pointer accel_profile flat` +加上 `pointer_accel 0`），或者装上 `liboeffis`，改走协议层本来就是绝对坐标的 libei。 + +倍率是合成器自己的设置，客户端读不回来，所以只有你知道它关了没有： +`JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示已经关掉，移动就不再出声； +`=strict` 则宁可拒绝这次移动，也不让点击落在别的地方；不设置就维持 +“警告一次后照样移动”的默认。 + +Wayland 的屏幕截取需要合成器对应的工具，因为没有单一工具能覆盖全部：wlroots 系 +（sway、Hyprland、river）用 `grim`，GNOME 用 `gnome-screenshot`，KDE 用 `spectacle`。 +装好其中一个之后，所有截取路径——截图、图像与锚点定位、OCR、屏幕录制、远程桌面——都会 +经由它。三个都没装也还有 `gdbus`：最后会尝试 `xdg-desktop-portal`，只是第一次可能会弹 +同意对话框。再不行，截取会带着安装提示明确失败，而不是返回空白的 XWayland root； +`je_auto_control.api.run_diagnostics()`（以及 GUI 的 Diagnostics 标签页）的 `screen_capture` +检查会报告当前使用的是哪一层。 + +有一件只在 Wayland 出现、需要事先规划的事：**截取回来的图里可能有鼠标光标。** +这里没有任何一条截取请求光标，但只要 backend 没有光标平面（包含任何以 +`WLR_NO_HARDWARE_CURSORS=1` 运行的 session），wlroots 就会画**软件光标**并把它 +合成进输出缓冲区，而截取交回来的正是那一份。Windows 与 X11 都不含光标，所以 +“定位器、模板匹配或 OCR 在目标中间看到一个光标形状的洞”只会在这里发生。 +Wayland 不让客户端读光标位置，所以没有东西可以可靠地遮或避开：请在截取之前 +把指针移离要拍的区域。`screen_capture` 检查会以 `cursor_may_be_captured` 报告这件事。 + +如果以上都不适用你的环境，可以直接指定自己的命令——它优先于所有检测，`{output}` 会被 +换成临时 PNG 路径： + +```bash +export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" +``` + Wayland 禁止非特权客户端进行全局输入录制——若要录制，请设置 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 并在 X11 会话下运行。窗口管理目前仅 Windows 有实现，其他平台会抛出明确的 `NotImplementedError`。对于会忽略合成输入的应用， diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 3699b1bd..b9f04444 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -231,10 +231,52 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Windows 10 / 11 | Win32 ctypes（可選 Interception 驅動） | ✅ | ✅ | ✅ | ✅ | | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | | Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ❌ | -| Linux Wayland | libei，或 ydotool／wtype／grim | ✅ | ✅ | ❌ | ❌ | +| Linux Wayland | 經桌面 portal 的 libei，或 ydotool／wtype ＋ 擷取工具 | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +Wayland 的輸入在 libei 走不通時會退回 `ydotool` CLI，而這條退路需要 +**ydotool 1.0 以上**。AutoControl 送的每一個參數都是那一版才有的；0.1.x +（Debian bookworm 與目前所有 Ubuntu 仍以這個名字提供，Debian trixie 則根本沒有） +對同一批參數回傳 0 卻不送出任何事件。AutoControl 會偵測並直接拒絕， +而不是為根本沒送出的輸入回報成功。Arch、Fedora 與 Debian unstable 提供的是 1.0。 + +這條退路要能**準確定位**，還有一個前提:合成器的指標加速度必須是關的。 +`ydotool mousemove --absolute` 並不送任何絕對事件——它先把游標推到合成器夾取的 +那個角落，再送相對位移，所以這段位移會被合成器加速。對真的 wlroots session 量到的是: +libinput 的預設 profile 讓游標走的距離正好是要求的兩倍。ydotool 自己的 `--help` +也是這樣寫的；AutoControl 每個行程會記一次警告，而不是安靜地把點擊放到錯的地方。 +請對 ydotoold 的裝置關掉加速度（sway:`input type:pointer accel_profile flat` +加上 `pointer_accel 0`），或是裝上 `liboeffis`，改走協定層本來就是絕對座標的 libei。 + +倍率是合成器自己的設定，用戶端讀不回來，所以只有你知道它關了沒有： +`JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示已經關掉，移動就不再出聲； +`=strict` 則寧可拒絕這次移動，也不讓點擊落在別的地方；不設定就維持 +「警告一次後照樣移動」的預設。 + +Wayland 的螢幕擷取需要合成器對應的工具，因為沒有單一工具能涵蓋全部：wlroots 系 +（sway、Hyprland、river）用 `grim`，GNOME 用 `gnome-screenshot`，KDE 用 `spectacle`。 +裝好其中一個之後，所有擷取路徑——截圖、影像與錨點定位、OCR、螢幕錄影、遠端桌面——都會 +經由它。三個都沒裝也還有 `gdbus`：最後會嘗試 `xdg-desktop-portal`，只是第一次可能會跳 +同意對話框。再不行，擷取會帶著安裝提示明確失敗，而不是回傳空白的 XWayland root； +`je_auto_control.api.run_diagnostics()`（以及 GUI 的 Diagnostics 分頁）的 `screen_capture` +檢查會回報目前使用哪一層。 + +有一件只在 Wayland 出現、需要事先規劃的事：**擷取回來的圖裡可能有滑鼠游標。** +這裡沒有任何一條擷取要求游標，但只要 backend 沒有游標平面（包含任何以 +`WLR_NO_HARDWARE_CURSORS=1` 執行的 session），wlroots 就會畫**軟體游標**並把它 +合成進輸出緩衝區，而擷取交回來的正是那一份。Windows 與 X11 都不含游標，所以 +「定位器、樣板比對或 OCR 在目標中間看到一個游標形狀的洞」只會在這裡發生。 +Wayland 不讓用戶端讀游標位置，所以沒有東西可以可靠地遮或閃避：請在擷取之前 +把指標移離要拍的區域。`screen_capture` 檢查會以 `cursor_may_be_captured` 回報這件事。 + +如果以上都不適用你的環境，可以直接指定自己的指令——它優先於所有偵測，`{output}` 會被 +換成暫存 PNG 路徑： + +```bash +export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" +``` + Wayland 禁止非特權用戶端進行全域輸入錄製——若要錄製，請設定 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 並在 X11 session 下執行。視窗管理目前僅 Windows 有實作，其他平台會拋出明確的 `NotImplementedError`。對於會忽略合成輸入的應用程式， diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index ba7e4e4b..b42b6033 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,490 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-19) — Wayland 两个等人拍板的取舍,拍板了 + +`Progress.md` 上挂着两个 `DECIDE`:缺的不是活，是决定。两件事其实是同一个问题犯两次 +——一个合成器的设置,库**量得到**却**读不回来**,所以它必须停止假装自己知道。 + +- **指针加速度改成由操作者声明,库照信。** 量到的事实不变: + `ydotool mousemove --absolute` 发的是相对移动,libinput 默认的 adaptive profile 会 + 把它加成两倍,而倍率没有任何客户端读得回来。没拍板的是「那要怎么办」——继续警告后 + 照发、直接拒绝、还是让操作者自己讲。直接拒绝会让每一台没有 `liboeffis` 的 Wayland + 机器整条 `set_position` 不能用,所以答案是声明: + `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示 ydotoold 的设备已经关掉加速度, + 移动就静静地、准确地发出去;`=strict` 表示宁可拒绝这次移动,也不让点击落在别的地方; + 不设置就维持现在的「警告一次后照样移动」,所以今天能跑的东西明天照样能跑。 + 值打错会退回警告模式,**而且会讲出来**——shell profile 里的一个错字,不可以静静地 + 把一次移动升级成「可信赖的精确」。整个判断只挂在 ydotool 这条路上;libei 在协议层 + 本来就是绝对坐标。 +- **Wayland 截取里的软件光标,写进文档,不绕过。** 这是 `seat-verification` 顺手量到 + 的:项目里没有任何一条截取带 `-c`,也就是没有人请求光标,而光标还是在图里。原因不在 + 我们这边——只要 backend 没有光标平面,wlroots 就会画**软件光标**,而软件光标是合成进 + 输出缓冲区的,`wlr-screencopy` 交出来的正是那一份。headless 永远处在这个状态; + 真桌面上只要跑 `WLR_NO_HARDWARE_CURSORS=1` 也一样。Windows 的 BitBlt 与 X11 那条路 + 都不含光标,所以这是**只在 Wayland 出现的不一致**;指针压在目标上的时候,定位器、 + 模板匹配与 OCR 看到的就是目标中间有一个光标形状的洞。两条绕法——截取前把指针挪开再 + 挪回来、或匹配时把指针周围遮掉——都得先知道指针在哪,而 Wayland 不让客户端读光标 + 位置;靠进程内的记录去猜,用户一动实体鼠标就过期,而遮错位置比看得见光标更糟。 + 所以改成写下来:写进 capability matrix、三份 README,以及诊断包——`screen_capture` + 检查现在会带 `cursor_may_be_captured`,让那份「解释定位器为何失败」的报告自己说出 + 原因。检查本身照量到的样子断言,哪天 wlroots 开始尊重 `overlay_cursor`,CI 会当场 + 红掉来通知我们。 + +## 本次更新 (2026-08-19) — 「会吃 libinput 设备的合成器」只差三个环境变量 + +`ydotool mousemove --absolute` 不是绝对移动。它不发任何 ABS 事件:先在两轴各发一次 +`INT32_MIN` 的相对移动,把光标推到合成器夹取的那个角落,再把目标当成相对位移发出去。 +**那个角落到底在哪**、以及**位移在路上被合成器做了什么**,是 Wayland 输入路径最后 +两个没有答案的问题——而且两个都被记成「需要一台跑真桌面、会吃 libinput 设备的 VM」。 + +- **两个都不需要 VM。** wlroots 吃 `WLR_BACKENDS=headless,libinput`:输出保持虚拟, + 输入那一半是真的 libinput backend。libseat 的 builtin backend 不靠 logind 就能打开 + 设备,而 `SEATD_VTBOUND=0` 让它不要去抢一个容器根本没有的 VT。第四个条件最容易漏: + libinput 是通过 udev 枚举设备,不是通过 `/dev`,所以 `systemd-udevd` 必须在 ydotoold + 创建设备**之前**就已经在跑。四个都到位之后,ydotoold 的 uinput 设备就是一个普通的 + seat 设备,`grim -c` 会把光标合成进截图里,合成器就能用布局坐标回答问题。 +- **那个角落是布局的左上角,不是布局坐标的 `(0, 0)`。** 只有在所有输出都在非负位置时 + 这两点才是同一点。任何「主屏左边还有一台显示器」的桌面,两者就差一个布局原点——所以 + 在原点为 `-1280` 的布局上,没有转换的「布局 `(0, 0)`」会把光标送到**隔壁那台屏幕**, + 差 1,280 像素。`mouse.set_position` 现在会先减掉 `layout_origin()` 再交给 ydotool, + 这正是抓图路径早就在做的同一个修正;两条输入路径共用的那个查询搬进了 + `linux_wayland/_layout.py`,libei 与 ydotool 不会再对「原点是什么」各说各话。 +- **剩下的距离则被指针加速度放大。** 发出去的位移是相对移动,所以 libinput 会加速它: + 对着真的 wlroots session 量到的是,默认的 adaptive profile 让光标离那个角落的距离 + 正好是请求的**两倍**——因为 `--absolute` 的两个事件发在同一个 frame 里,速度直接把 + profile 打到上限。把 `accel_profile flat` 加上 `pointer_accel 0` 之后,同一个调用 + 就精确到像素。ydotool 自己的 `--help` 一直都写着「You need to disable mouse speed + acceleration for correct absolute movement」;后端现在会每个进程记一次这个警告,而不是 + 让点击安静地落在错的地方。库这一侧再多做不了什么——那个倍率是合成器的设置, + 调用端读不回来。 +- **新的 `seat-verification` job 把这些全部钉住。** `docker/Dockerfile.seat` 跑的是 + 抓图镜像的同两种布局,每种 14 项:sway 真的握着 ydotool 设备、`--absolute (0, 0)` + 会把光标画在布局的第一个像素上、关掉加速度后移动是一像素对一像素、没转换的 + `(0, 0)` 会打不中它指名的那台屏幕、`set_position` 减掉的正好是原点而且在两台屏幕上 + 都落在指定的像素、以及加速倍率就是量到的 2 倍。里面没有任何一项依赖光标主题:每个 + 主张都是两张截图之间的差,光标图片相对于热点的偏移会自己抵消掉。 +- **顺手抓到的一件事。** 在这台合成器上,`grim` 明明没有要求叠上光标,截回来的图里 + 还是有光标——因为只要 backend 没有光标平面,wlroots 就会画**软件光标**(headless + 永远如此,任何驱动不给平面、或用户设了 `WLR_NO_HARDWARE_CURSORS=1` 的 session + 也是)。所有定位器、模板匹配与 OCR 都走同一条抓图,所以指针会在它压着的东西上挖出 + 一个指针形状的洞。该检查照量到的样子记下这个行为,要怎么处理则列进 `Progress.md`。 + +## 本次更新 (2026-08-19) — 抓图用的 portal 本来就不可能成功,真的 bus 讲出来了 + +- **`xdg-desktop-portal` 的回答是「指名发给发出调用的那条连接」的信号。** + `Screenshot` 返回的是一个 *request handle*,不是图;图稍后才以 + `org.freedesktop.portal.Request::Response` 送达,收件人是调用者的 unique bus name。 + bus 对指名消息只发给收件人,所以别的连接再怎么加 match rule 也收不到。 +- **旧的做法是两条连接。** 先起一个 `gdbus monitor` 子进程,再用第二次 `gdbus` + 调用发请求,然后拿两条正则去读 monitor 的 stdout。每次 `gdbus` 都会用自己的 + unique name 开一条自己的连接,所以在听的那个进程从来就不是收件的那个。在真的 + `dbus-daemon` 上量到的:monitor 看得到调用经过,之后什么都不再打印,抓图每次都走完 + 整整 30 秒超时。唯一看得到指名信号的是完整的 bus monitor（`dbus-monitor`,它会向 + bus 要 `BecomeMonitor`）——为了一张兜底截图去换「观察用户 session bus 上每一条 + 消息」的权限,不划算。 +- **这一层现在自己讲 D-Bus,而且只用一条连接。** `linux_wayland/_dbus_client.py` + 是只用标准库写的 session bus 客户端:连接、SASL EXTERNAL 认证、`Hello`、 + `AddMatch`、一次方法调用,然后读到对得上的信号为止。它刻意不是通用绑定——没有 + property、没有 introspection、不导出对象、不传文件描述符（需要那一项的那一个调用 + 仍然交给 liboeffis）。`portal.py` 会**先**用自己的 unique name 推算出 request path + 并订阅,再发调用;portal 若不理会 `handle_token`,返回的 handle 也会一起跟。 +- **而且这是拿掉一个依赖,不是加一个。** 这一层原本需要装 `gdbus`（glib2),现在只要 + 有 session bus 就行,所以这条最后兜底的抓图路径能用的桌面比以前**更多**。安装提示 + 与诊断检查都改成这么说了。 +- **在真的 bus 上整条验过。** 新的 `portal-verification` job 跑真的 `dbus-daemon`、 + 真的 portal 实现与真的 client:抓回来的 PNG 字节解得开,而且就是 portal 画的 + 那几个像素,路径是含空格、percent-escape 过的,读完之后 portal 那个文件不见了。 + portal 没能交出图的每一种收场也都跑过——对话框被关掉、对话框一直开着、成功但没有 + URI、URI 不是本机文件——每一种都必须在 AutoControl 自己的时限内 fail closed。 + +## 本次更新 (2026-08-19) — RemoteDesktop portal 握手也从来不需要 GNOME VM + +- **portal 是 D-Bus 接口,不是合成器功能。** 在 GNOME 与 KDE 上要接到 libei,得跑 + `CreateSession` → `SelectDevices` → `Start` → `ConnectToEIS`,最后由 bus 交出一个 + EIS 文件描述符。这件事本来被记成「没有 GNOME VM 就验不到」,理由是 + `xdg-desktop-portal-wlr` 只做 ScreenCast 与 Screenshot、没有 RemoteDesktop——那是把 + 「没有容器附一个」当成了「没有容器当得了一个」。对 `liboeffis` 而言,谁占住 + `org.freedesktop.portal.Desktop`、谁回答那四个调用,谁就**是** portal。 +- **所以验证自己去占那个名字。** `docker/portal_server.py` 是一个跑在私有 session bus + 上的真 D-Bus 服务,它的 `ConnectToEIS` 交回去的是连到 `eis` 那个镜像用的同一台真 + `libeis` server 的活连接。真的 `liboeffis` 跑完真的握手;出来的描述符承载得起一个 + 真的 EI session;通过它发出的按键、绝对移动与按钮边沿,由对面一个独立实现记录下来。 +- **它确定了什么。** 四个调用按规范的顺序、发到 client 自己推算出来的 request path; + `SelectDevices` 要到的是键盘与指针、不多要——所以用户被要求同意的范围就是这个后端 + 真正需要的范围,`OEFFIS_DEVICE_DEFAULT` 也不是那个 `= 0` 的 all-devices 哨兵值; + 交回来的描述符是一个活的、由调用端持有并负责关闭的 socket——这正是把它交给 + `ei_setup_backend_fd`（一个会接管所有权的函数）之所以正确、而不是双重关闭的原因。 +- **以及每一种拒绝。** 同意对话框被关掉、对话框一直开着、描述符被扣住、portal 把 + session 关掉、portal 旧到根本没有 `ConnectToEIS`、bus 上根本没有 portal:每一种都 + 必须在本项目自己的时限内变成拒绝,而不是卡住或悄悄降级。`OEFFIS_EVENT_CLOSED` 这条 + 分支从来没有 peer 驱动得了,现在有了。 +- **仍然没有宣称的是什么。** 同意对话框作为「对话框」本身。CI 里没有人去按它,所以 + 真的 mutter 对话框长什么样、真人会让它开着多久,那仍然是 mutter 的事。对话框 + *产生的东西*——准、拒、沉默——三种都跑过了。 + +## 本次更新 (2026-08-19) — 一件记错了的打包事实 + +- **Debian trixie 有 `liboeffis`。** `Progress.md` 原本写没有,并据此推论 libei 快速 + 路径在 Debian／Ubuntu 上等于是关的。实测:`liboeffis1` 1.3.901-1 就在 trixie/main, + 提供 `liboeffis.so.1`。真正成立、而且对用户真正有影响的是另一件事:它是**独立的 + 二进制包**,`libei1` 并不依赖它——所以只装 libei 的机器上 portal 这条路仍然是关 + 的,`connect()` 会退回 GNOME 与 KDE 都不会开的 `eis-0` socket。要走快速路径, + `liboeffis` 得自己装。 + +## 本次更新 (2026-08-19) — 主屏幕左边多一块屏幕,Wayland 的抓取整条都读错 + +- **Wayland 没有「单块屏幕」这回事,而唯一那个平面也不一定从 `(0, 0)` 开始。** + 合成器把所有 output 排在同一个平面上,只要有一块在原点的左边或上面,这个平面的 + 原点就是负的——那正是「我的第二块屏幕在左边」对合成器的意思。sway 的 headless + backend 收 `output HEADLESS-1 position -1280 0`,所以这是 CI 立得起来的布局: + 两个 1280x720 的 output、一张 2560x720 的抓取、左上角那个像素在 x=-1280。 +- **`size()` 返回的是布局的右缘,不是宽度。** 它算的是各 output 的 `max(x + width)`, + 在上面那个布局是 1280,而 `grab_image()` 回来的画面宽 2560。凡是把这两者兜在一起 + 的人都信了小的那个:mss shim 的 monitor 列表（连带 `enumerate_monitors`）、屏幕 + 录制、WebRTC host、MCP 的 monitor 抓取,全都去要了一块只有桌面一半大的矩形, + 然后当成整个画面回报。 +- **不能自己套区域的层级,裁切裁在错的坐标空间。** 只有 grim 吃几何参数; + gnome-screenshot、spectacle、portal 与 `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` + 都是把整个布局交回来、由 AutoControl 事后裁切。那个裁切用的是布局坐标,而图的 + (0,0) 是布局原点,所以要 `[-1275, 5, -1175, 55]`（左边那块屏幕上的一块）等于向 + Pillow 要一个在画面左外侧 1275 px 的方框,拿回来的是黑色填充。 +- **而且比对到的东西会被回报在错的屏幕上。** `grab_logical()`——模板搜索、OCR 与 + visual match 背后的那个抓取——的原点是读 `GetSystemMetrics` 的,在 Windows 以外 + 无话可说,于是一律返回 `(0, 0)`,每个命中都被回报在实际位置的右边 1280 px 处。 + 这种错的表现是「点到别块屏幕」,比「找不到」更难查。 +- **修在接缝上,不是修在每个调用端。** Wayland backend 发布 `layout_origin()`; + `size()` 返回 bounding box 的**大小**;`grab_image` 裁切前先扣掉原点; + `screen_grabber.backend_layout_origin()` 则是 `grab_logical` 与 mss shim 去问的 + 那一个,让「自己抓取屏幕的后端」有办法说出这张画面从哪里开始。通用库本来就 + 看得见的后端（Windows／macOS／X11）什么都不用发布,行为完全不变——原点只会被 + 问到,不会被猜。 +- **对真的合成器两种布局都验过。** `wayland-verification` job 现在把那 27 项检查 + 跑两遍:一遍是两个 output 从原点并排,一遍是左边那个在 x=-1280。第二遍才是把 + 负原点整条钉死的那一遍——grim 的负 `-g`、回报的尺寸、`layout_origin()`、 + mss shim 的 monitor 矩形、`grab_logical()` 的原点,以及把操作者自定义指令指向 grim + 之后、让一张**真的**整布局 PNG 走过那条必须位移的裁切路径。 + +## 本次更新 (2026-08-19) — 同一个布局问题的输入侧:libei 把移动悄悄丢掉了 + +- **落在所有 region 之外的绝对移动,libei 会直接丢掉,而且什么都不说。** + 没有返回码、没有事件、调用方看不到任何错误——`ei_device_pointer_motion_absolute` + 就是不把这个事件送上线。`set_position` 于是像光标真的移动过一样正常返回。 + 这是对真的 EIS 对端量出来的,不是推论:设备只提供一个 `(0, 0, 1920, 1080)` + 的 region 时,`(1919, 1079)` 会到,`(1920, 1080)` 在 server 端连一个事件都没有。 +- **而那些 region 所在的坐标空间,不一定就是布局的坐标空间。** region 的 offset + 是 `uint32`,所以没有任何合成器**能**声明一个在原点左边或上面的 region——但本项目 + 的布局空间是从 `layout_origin()` 开始的,只要有一块屏幕在主屏幕左边就会变成负的。 + 那正是抓取那一半刚修好的同一种桌面。于是两半差了整整一个原点,也就是 + `get_pixel(x, y)` 与 `set_position(x, y)` 指向不同像素的那个情况——而本来会跑到 + 隔壁屏幕的光标,实际上是安安静静地哪里都没去。 +- **`LibeiBackend` 现在会读设备的 region,再把坐标映射进去。** 绑上了 + `ei_device_get_region` 与四个 `ei_region_get_*`;`_region_point` 对有覆盖的坐标 + 原样送出,对没覆盖的改用减掉布局原点后的坐标再试一次,两者都不覆盖就拒绝。 + 没有声明任何 region 的设备接受任何坐标,原样通过——这一点同样是量出来的,而那 + 就是单屏幕的常见情况,一分钱都不多花。 +- **拒绝是有用的结果,不是失败。** 它是 `LibeiUnavailable`,所以 + `_select_input.emitted` 会像处理「设备被暂停」那样,把这次移动交给 ydotool 那条路。 + libei 一直被写成快速路径而非唯一路径;这个 bug 的真正问题是:被丢掉的移动从来 + 到不了后备路径,因为没有人知道它被丢掉了。拒绝时也不会送 frame——什么都没有被 + 缓冲,那里送一个 frame 只会把上一次发送留在设备上的东西提交出去。 +- **布局原点只在坐标没中的时候才会去问。** 它要花一个 `wlr-randr` 子进程,所以不会 + 出现在每一次普通鼠标移动的路径上;在 GNOME 与 KDE 上它返回 `(0, 0)`——那在那里是 + 正确答案而不是退路,因为那些合成器自己就会把布局归一化。 +- **`eis-verification` job 多了五项对真协议的检查。** client 读回来的 offset 就是 + 合成器声明的那个;位于 `x=1280` 的 region,往内 100 px 的点要送 `1380` 而不是 + `100`;libei 到今天仍然一声不吭地丢掉 region 外的移动——这是整个防护所依据的测量, + 所以哪天 libei 改成夹取,这项会当场说出来;AutoControl 会拒绝这种移动而不是弄丢它; + 以及在原点为 `-1280` 的布局上,`(-1280, 10)` 到达 server 时是 `(0, 10)`。 + 这个 job 现在跑 20 项。 +- **这件事没有解决的部分。** ydotool 的 `mousemove --absolute` 有它自己的原点—— + 它夹到合成器的左上角,再把目标当成相对位移送出——那个角落是不是布局原点,仍然需要 + 一台真的会吃 libinput 设备的合成器才验得到。它继续留在 `Progress.md`,不会靠猜测 + 去改。 + +## 本次更新 (2026-08-19) — ydotool 一直回报成功,其实什么都没做 + +- **`apt install ydotool`——这个 backend 自己打印的安装提示——装到的版本跑不动它送的 + 命令行,而且是用「返回 0」来表达的。** ydotool 1.0 把整套 CLI 换掉了,而 Wayland + backend 送的每一个参数都是那一版才有的:`mousemove --absolute`、`mousemove + --wheel`、`click` 的十六进制位掩码（拆得开 press 与 release,拖拽整个建立在 + 这上面）、以及吃数字 evdev 码的 `key CODE:STATE`。Debian bookworm、Ubuntu 22.04 + 与 24.04 到今天都还是把 0.1.8 叫做 `ydotool`。对真的 uinput 设备实测:0.1.8 收到 + `click 0x40` **什么事件都不送,返回 0**;收到 `mousemove --absolute` 打印 + `unrecognised option`,**一样返回 0**。而 backend 是用 `check=True` 判断成败的, + 只有非零才会抛。所以在这些发行版上,脚本没点到、没打到、没移动,而每一次调用都回报成功。 +- **旧版 CLI 现在在送出任何东西之前就被挡下。** `linux_wayland/_ydotool_cli.py` + 每个进程只判定一次（鼠标与按键派送付不起每个事件一次 subprocess）,并在错误信息里 + 给出三条路:装 1.0+ 的包、自己编、或 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`。 + 两个版本都没有 `--version`,而 1.x 不启 daemon 连 `--help` 都不回答,所以探测读的是 + 两边都会打印、不需要 daemon、也没有副作用的那一样东西:无参数时的命令清单。认不出来的 + 版本一律放行而不是挡掉,免得将来改了字样的新版被过期的检测器锁在门外。 +- **两处安装提示还错在第二件事上**:Debian trixie 根本没有 `ydotool` 包。 + 提示已改成点名真的有的发行版。 + +## 本次更新 (2026-08-19) — 验 ydotool 从来不需要桌面,只需要一个读取端 + +- **两个验证镜像记下来的那个缺口,记错了。** `Dockerfile.wayland` 与 `Dockerfile.eis` + 结尾都写 ydotool「需要 /dev/uinput 以及一个会消费它的 seat」,而 `Progress.md` 把它 + 排在「先建一台 GNOME VM」后面。seat 决定的是注入的事件**会不会送达某处**,不是它 + **能不能被观察**:ydotoold 建的是普通的 uinput 设备,kernel 会挂成 + `/dev/input/eventN`,读那个节点就拿得回 ydotool 写进去的 `input_event`。 + 不需要合成器,不需要桌面 session,不需要 VM。 +- **`docker/Dockerfile.ydotool` 与 `docker/ydotool_verify.py` 就是在 CI 里做这件事, + 12 项检查。** 过去只对着 mock 断言的东西现在有答案了:`0xc0`／`0xc1`／`0xc2` 真的是 + BTN_LEFT／BTN_RIGHT／BTN_MIDDLE;拆边的 `0x40` 与 `0x80` 真的只送 press 或只送 + release(`press_mouse` 与拖拽整个建立在这上面);`key 30:1 30:0` 真的带的是数字 + evdev 码;而**滚动正负号是量出来的,不再是假设的**——`-y 1` 到 kernel 是 + `REL_WHEEL +1`、`-y -1` 是 `-1`、`-x 2` 是 `REL_HWHEEL +2`,而且轴没有互换。 + 最后这一项正是滚动那次改动之后,`Progress.md` 一直标着「未实测」的假设。 +- **第 12 项检查驱动的是 backend 自己的函数,不是手写 argv**,所以「ydotool 拿到这串 + 命令行会做什么」与「AutoControl 送的是什么」是接起来的,不只是并排。 +- **`mousemove --absolute` 送的并不是绝对事件**,这件事在信任它之前值得知道。 + ydotool 1.x 的设备上没有 ABS 轴:它先在两条相对轴上送 `INT32_MIN`,靠合成器把它夹到 + 左上角,再把目标当成相对位移送出去。所以 `set_position` 会落在你要的像素,**是因为 + 那个夹取**。kernel 这一侧现在钉住了;夹取本身是合成器的行为,仍然是开放项。 +- **容器拿到的是 `/dev/uinput` 加字符主号 13,不是 `--privileged`。** ydotoold 是在 + 容器起来**之后**才建输入节点,`--device` 覆盖不到,所以那个 job 只授予 + `--device-cgroup-rule 'c 13:* rmw'`,别的都没有。 + +## 本次更新 (2026-08-19) — Wayland 的滚动不再需要 uinput 常驻程序 + +光标移动、按键、按钮早就走 libei 了,只剩滚动每一格都还要 fork 一次 `ydotool`, +而 `Progress.md` 也写了原因:正负号是猜的,而滚错方向不会报错,只会安静地错。 +现在接上了,猜测也换成了两份独立佐证加一次实测。 + +- **两条路径对「一格」的正负号定义相反。** 本项目的 `wayland_scroll_direction_*` + 常数是 kernel `REL_WHEEL` 那一套,因为 ydotool 写进 `/dev/uinput` 的就是它:正值为上。 + libei 是 `wl_pointer`／libinput 那一套,正值为下——libinput 自己的 evdev 读取端 + 就是把 `REL_WHEEL` 取负号换过去的,而另一个有写明正负号的 libei sender（enigo） + 则是把「正值往下滚」的值原封不动送进 `scroll_discrete`。水平轴不用翻: + `REL_HWHEEL` 与 libinput 都以右为正。所以送往 libei 时垂直轴取负、水平轴不动—— + 这就是 `Progress.md` 在等的那个决定。 +- **这个翻转有对着真的 EIS server 验证,连负值一起。** `docker/eis_verify.py` 多了第 15 项 + 检查,驱动的是**公开的** `mouse.scroll()`（不是上一项检查驱动的 backend 方法）, + 从 server 端读回来:上是 `(0, -120)`、下是 `(0, 120)`、右是 `(120, 0)`。这也是唯一 + 一次把**负的**离散值送上线;先前那项检查只送过正值,正负号的 marshalling 若有毛病 + 根本没有地方会现形。 +- **被拒绝的发送现在会退回 CLI,也就是代码一直宣称的行为。** `libei` 的模块 + docstring 写着每个失败都抛 `LibeiUnavailable`,「`keyboard`／`mouse` 已经把它当成 + *改用 ydotool CLI*」。实际上只有**连接**是这样处理的。backend 交出去之后,合成器 + 暂停了设备、或 session 在两次调用之间结束,都会直接从 `set_position`、`press_key`、 + `hotkey` 抛出去。把滚动也接上 libei 等于再多一条「明明旁边就有可用后备却让脚本死掉」 + 的路,所以这个后备被补成真的:和弦中途被拒会先把已经按下的键反序放开,不会留下卡住的 + 修饰键;按钮的**放开**被拒则改由 ydotool 放开,不会整个 session 都按着。 +- **`LibeiUnavailable` 原本会穿过所有的拦截边界。** 它只继承 `RuntimeError`,而 + `CLAUDE.md` 写得很明白:不是 `AutoControlException` 的框架错误「会安静地逃出每一个 + 边界」——executor、后台轮询循环、请求处理器、GUI slot。现在两个都继承,原本接 + `RuntimeError` 的探测照旧能用,边界也终于看得到它。 + +## 本次更新 (2026-08-18) — libei 输入路径终于有对手可以说话了 + +Wayland 的**撷取**路径已经对着真的合成器验过;**输入**路径没有,而且被记成「需要一台 +GNOME VM」。其实不需要。libeis 就是 libei 自己那套协议的 server 端,Debian 有打包, +两个库可以直接在一条 Unix socket 上对话——所以 `docker/eis_server.py` 起一个真的 EIS +实作,`docker/eis_verify.py` 把 AutoControl 真正的 sender 对着它跑。不需要合成器, +不需要桌面 session,14 项检查,已接进 CI。 + +- **离散捲动差了 120 倍。** libei 的离散捲动以「一格的 120 分之一」为单位——跟 Windows + `WHEEL_DELTA` 同一套惯例——而 `scroll()` 直接送格数,等于一格只送了 1/120 的捲动量。 + libei 自己在执行期就会讲（`suspicious discrete event value 1, did you mean 120?`）, + 而这句话 mock 永远不会印出来。现在 `scroll(0, 1)` 到对面是 `(0, 120)`:一格、正确的轴、 + 正确的正负号。这条路径原本因为「正负号是猜的」而刻意没接线,结果正负号是这题里比较小的一半。 +- **拆除不再每个行程漏一个 context。** `ei_unref` 在 libei 1.3.901 会 segfault——但只在 + 「backend 开了、握手从未推进」那个状态。有了可以完成握手的对手,live 的情况终于测得到, + 而它是安全的。现在拆除会正常释放 device 与 context,只有真的会炸的那个状态才放弃。 +- **mock 检查不到的值都检查了。** server 端 offer 六个 capability,再把 client 真正绑定的 + 读回来:正好是 AutoControl 要的那四个,所以 `EI_DEVICE_CAP_*` 位元遮罩与 variadic + `ei_seat_bind_capabilities` 的编组都是对的。keycode、绝对座标、按键码都从线上读回来比对。 + 每次发送都确认有 frame,每个 device 都确认有先开 emulation transaction——没开的话 + libei 会把事件丢掉。 +- **两件量到但不是我们能修的事**,如实记下而不是含混带过:libeis 1.3.901 的 + `eis_device_pause()` 对 sender client 没有送出任何东西,所以 client 的 `DEVICE_PAUSED` + 处理仍然没有对手可以驱动;`start_emulating` 的 sequence number 也没有被送到对面 + （刻意送 4242,读回来是 0）。检查写成「要嘛有反应,要嘛根本没被通知」, + 哪天 libeis 开始送了而 client 忽略它,就会当场失败。 + +## 本次更新 (2026-08-18) — 架构地图的行数重新变成实测值 + +- **同一个子系统在地图里被写成两个不同的大小。** `CLAUDE.md` 规定 + `architecture_explore.md` 的每个数字都是实测的,但没有任何东西在检查,于是长出了 + 两套并存的计数惯例:§5.4 主题表与 §5.4.17 档案表数了一行不存在的结尾行—— + `len(text.split("\n"))` 会比以换行结尾的档案实际行数多一行,套件则是每个档案多一行 + ——而 §1 的总计与 §8 附录数的是对的。结果 `utils/executor/` 在一节里是 8,811 行、 + 在另一节里是 9,001 行,而 §8 那一栏加起来也不等于它自己的总计。除此之外还有大约 + 五十列根本是旧的,好几个 `####` 标题差了几百行（`linux_wayland/` 还写着 + 10 档／1,093 行,实际是 14／2,235）,而且有一张主题表多了两个子套件,它的摘要行 + 完全不知情。 +- **413 个数字用同一套惯例重新实测**——`len(text.splitlines())`,也就是 `wc -l` 的 + 结果,以及 `CLAUDE.md` 自己那段「超过 750 行」判断所用的算法。§5.4、§5.4.17 与 §8 + 现在对每个子系统都一致,§8 各列加起来也等于它写的总计。 +- **`test_doc_line_counts.py` 既是闸门也是修复工具。** 任何被引用的行数跟树上对不上 + 就让 CI 失败,并指出是哪几行;加 `--fix` 就一次全部就地改写。行数是地图里唯一没有 + 闸门的部分——指令、MCP 工具、子套件与范例数早就有 `test_doc_counts.py` 在管—— + 这正是它会漂掉的原因。 + +## 本次更新 (2026-08-18) — 剪贴板不再因为别的程序正在复制而失败 + +- **Windows 剪贴板同一时间只允许一个进程打开,而 AutoControl 的每一个剪贴板调用 + 在别人打开时都立刻放弃。** 资源管理器、Office 和每个浏览器复制时都会占住剪贴板 + 几毫秒,这段时间里 `OpenClipboard` 一律返回 false,而六个调用点——文字、图片、 + HTML、RTF、CSV、文件拖放、格式枚举——都把它直接翻成 + `RuntimeError: OpenClipboard failed`。在真实桌面上开一个进程循环复制实测:大约 + 千分之一的打开会失败,也就是脚本会因为操作者既看不见也复现不了的原因直接死掉。 + Win32 文档明写这种情况应该重试,而一个专门用来驱动「别的程序本来就很忙」的机器 + 的库,不能把「别人正在复制」当成错误。 +- **`win32_clipboard_api.open_clipboard()` 现在是唯一打开剪贴板的地方**,忙碌时会 + 等约 200 毫秒才报错,而且不论区块怎么结束都会关闭。三个自己手写 open/close 的 + 模块——包括两个比共享模块更早写的——都改走它,所以新的调用点不可能漏掉重试。 +- **剪贴板 round-trip 测试不再依赖机器上其他进程在做什么。** 这些测试跑的是真实的 + Win32 调用,而那正是四个历史 writer bug 唯一能被看见的地方,所以换成假的后端等于 + 删掉覆盖率,而不是让它稳定。改成读 Win32 剪贴板序号:写入与读回之间序号没变,就 + 代表这段时间没有别人写过,断言测的就只有 AutoControl 自己的代码。已在另一个进程 + 于测试期间写入 4,112 次剪贴板的情况下验证通过。 + +## 本次更新 (2026-08-18) — 打错指令名字回 400,不是 500 + +- **`POST /execute` 对不存在的 `AC_*` 名字回 `500 {"error": "execute_action failed"}`**, + 跟伺服器自己坏掉时的回应一模一样。调用端无法区分「我自己打错字」和「服务挂了」, + 错误讯息也没说是哪个名字不认得。 +- **所有指令名字现在都在执行前先校验**,不认得的回 `400`,并在 `unknown_commands` + 里列出**全部**——包含嵌在流程控制区块里的——让调用端一次改完所有拼错的名字。 + `POST /execute_file` 对读不到的档案、不是动作清单的内容、以及不认得的指令名, + 用同样的方式回应。OpenAPI 规格两者都写明了,并说明被拒绝的请求没有执行任何动作。 +- 校验与收集共用同一次走访,所以「嵌套动作清单可能藏在哪里」仍然只有一份定义。 + +## 本次更新 (2026-08-18) — libei 拆除时的 segfault,以及对真实库的核对 + +- **`ei_unref` 在 libei 1.3.901 上,只要 backend 已打开就会让进程崩溃,而我们的 + fallback 路径正好一头撞进去。** libei 握手的每一种失败都会走到 `_teardown()`, + 所以在任何装了 libei 而握手没完成的机器上,AutoControl 会直接 SIGSEGV,而不是 + 安静地改用 ydotool——跟 fail-closed 的承诺完全相反。逐个调用实测的结果:没有 + setup backend 时 `ei_unref` 安全、setup **失败**后安全、setup **成功**后必炸。 + `ei_disconnect` 在同样状态也炸,所以不是我们 refcount 用错;而头文件明写两种结果 + 都该用 `ei_unref`,因此这是上游的 bug。现在已打开的 backend 会被**放弃**而不是 + unref——代价是每个进程漏一个 context,换掉一个会驱动用户桌面的库直接崩溃。 + 验证程序里有一个哨兵会每次重新确认上游状态,修好时会提示可以移除 workaround。 +- **绑定里每一个入口点现在都对真的 `libei.so` 解析过。** 拼错的符号或错的 `argtypes` + 会毫无阻碍地通过假的符号表,只在用户的机器上才爆——22 个 prototype 加上 variadic + 的 `ei_seat_bind_capabilities` 现在都是真的验过。 +- **整条 fail-closed 链端到端跑过**:连到一个不说 EI 的 socket → 握手超时 → + `LibeiUnavailable` → `active_backend()` 返回 None → `press_key` 落到 ydotool CLI。 +- **`liboeffis` 不是每个发行版都有。** Arch 与 Fedora 有包,**Debian trixie 没有**。 + 没有它就没有 portal 路径,`connect()` 会退到众所周知的 EIS socket——而 GNOME 与 KDE + 根本不开那个 socket。所以在那些系统上 libei 快速路径是关闭的,而且会讲出来,不会 + 看起来莫名其妙地没作用。 + +## 本次更新 (2026-08-18) — Wayland 截取路径对上真的合成器了 + +- **`screen.size()` 报单一屏幕,`grab_image()` 却返回整个布局。** `wlr-randr` 的 + parser 抓文档里第一个 `WxH`,也就是第一个 output 的当前模式。双屏布局下那只有 + 半个画面——而 mss shim 正是把这两者组起来用的,所以录制、WebRTC 与 MCP 的屏幕路径 + 会去要一个只有半个画面大的区域,而且真的拿到了。`size()` 现在返回布局的 bounding + box,parser 会读每个启用中 output 的模式**与位置**。这个 bug 是跑真机才发现的: + 没有任何 mock 有第二块屏幕。 +- **`docker/Dockerfile.wayland` 让后端在 headless sway 下跑。** wlroots 的 headless + backend 不需要 GPU、seat 或显示器,所以一个真正的 Wayland session 塞得进容器——现在 + 也进了 CI,就是 `wayland-verification` job。两个 output 涂上不同纯色,因为在单一颜色 + 的画面上,区域抓错位置抓不出来,红蓝通道对调更是完全抓不出来。 +- **21 项此前只能对 mock 验的检查,现在对的是合成器真的画出来的像素**:grim 的 argv + 与 `-g` 几何、RGB 通道顺序、`wlr-randr` 没有文档记载的输出格式、 + `size()`／`grab_image()`／`get_pixel()`／`screenshot()`、 + `je_auto_control.screenshot()` 的 BGR 输出、`grab_logical()`(定位器与 OCR 路径)、 + mss shim、以及 `wtype`。 +- **容器答不了的部分直接写明,不含糊带过。** ydotool 需要 `/dev/uinput`,而 headless + sway 不吃 libinput 设备;`xdg-desktop-portal-wlr` 没有实现 RemoteDesktop,所以根本 + 没有 `ConnectToEIS` 可测。两者都留在 `Progress.md`。 + +## 本次更新 (2026-08-18) — libei 输入,整条打通 + +- **完整的 portal 握手做完了。** libei 不是“调用一个函数就按下一个键”的库: + sender 必须打开 EIS backend、绑定 seat 的能力、**从事件里**取得 device、在其上 + `start_emulating`,而且每次发送之后都要 `ei_device_frame`,否则什么都不会送达。 + 这些现在全部有了,所以 `press_key`、`set_position` 与鼠标按键不再需要每个事件 + spawn 一次进程。 +- **EIS socket 来自桌面 portal。** 在 GNOME 与 KDE 上它不是磁盘上的路径,而是由 + `org.freedesktop.portal.RemoteDesktop.ConnectToEIS` 经 D-Bus 交出的 file + descriptor,前面还有三次异步的 session 调用。**没有任何命令行工具能把 fd 交进 + 这个进程**,所以截图那条 `gdbus` 路在这里走不通;改用 `liboeffis`(libei 项目正是 + 为此附上它)。liboeffis 不存在时,仍会尝试众所周知的 `$XDG_RUNTIME_DIR/eis-0`。 +- **device 一律来自事件,不再由 context 冒充。** 先前的绑定把 `struct ei *` context + 传给收 `struct ei_device *` 的入口点——C 库里的指针类型混淆。现在从结构上 + 不可能发生。 +- **探测失败只付一次代价,不是每次按键。** 握手包含一次 portal 往返,可能还有同意 + 对话框。结果会在进程内缓存,所以“装了 libei 但不可用”的机器不会每次按键重试。 +- **所有情况都仍会回退到 ydotool。** 库缺失、用户拒绝、能力只给一半、device 被 + 暂停、握手没完成——每一种都抛 `LibeiUnavailable`,而 keyboard／mouse 本来就把它 + 当成“改用 CLI”。scroll 刻意留在 ydotool:它的方向约定有测试钉住,而 libei 的正负号 + 一旦猜错是**静默**的错误行为,不是明确的失败。 +- **ABI 常量是对过上游头文件的,不是猜的。** 对完发现三个错: + `enum ei_device_capability` 是**位掩码**,所以 `EI_DEVICE_CAP_KEYBOARD` 是 + `1 << 2` 而不是 3;`OEFFIS_EVENT_CLOSED` 排在 `OEFFIS_EVENT_DISCONNECTED` + **之前**;`OEFFIS_DEVICE_ALL_DEVICES` 是 `= 0` 的哨兵值,不是各设备位的 OR。 + 第一个代价最大——没有 device 会报告那个能力,所以每次连接都会超时然后静默改用 CLI。 + 核对过的值现在有测试钉住。另外 session 只申请实际会用到的键盘与指针,同意对话框 + 不会再要一个没人用的触摸屏授权。 + +## 本次更新 (2026-08-18) — Wayland 截取有了兜底 + +- **`xdg-desktop-portal` 为三个 CLI 工具兜底。** `grim`／`gnome-screenshot`／`spectacle` + 没有任何一个保证会装（GNOME 自 42 起不再默认安装 `gnome-screenshot`），所以最后会经由 + `gdbus` 尝试 `org.freedesktop.portal.Screenshot`，而不是直接放弃。它天生麻烦：portal + 返回的是 request handle、结果之后才以 signal 送达，所以监听必须在调用**之前**启动； + 而且同意对话框可能挡在前面，因此等待有 30 秒上限。 +- **操作者可以指定自己的截取命令。** + `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` 优先于所有检测。 + `{output}` 会换成临时 PNG 路径，在 `shlex.split` 之后逐个参数替换、不经过 shell， + 所以含空格的路径仍然是单个参数。这是给内置分层都不适用的环境用的逃生门——包含我们对 + 某个工具的 argv 猜错的情况。 +- **libei 主动拒绝它根本送不出输入的连接。** 这个绑定从头到尾只持有 `ei` context， + 但每个 device 入口点收的都是 `ei_device`，而且完全没有跑 libei 的 seat／device／ + `start_emulating`／`frame` 握手——所以它送不出输入，却**可能**在会开 + `$XDG_RUNTIME_DIR/eis-0` 的机器上把错误的指针传进 C 库。现在它会在 `connect()` + 就停下并说明原因。调用端本来就把这个情况当成“改用 ydotool CLI”，而那正是所有真实 + 桌面环境一直在做的事。 +- **portal 监听器不会在收尾时卡死。** 它的 pipe 只在读取线程放手之后才关闭；在有线程 + 仍阻塞在 `read()` 时关闭流可能卡在 buffer lock 上，那会让“超时”变成它本来要防止的 + “卡住”。 + +## 本次更新 (2026-08-18) — 容器镜像在 Windows 签出下也能构建、能启动 + +- **在 Windows 上 clone 再 build 出来的容器，一启动就死。** `.gitattributes` 只写了 + `* text=auto`，于是 `docker/entrypoint.sh` 与 `docker/entrypoint-xfce.sh` 被签出成 CRLF。 + shebang 就变成 `#!/bin/sh`，kernel 去找一个名字末尾带回车的解释器，镜像 build 得完全正常， + 一跑就抛出 `exec /usr/local/bin/autocontrol-entrypoint: no such file or directory`—— + 而那个文件明明就在。CI 永远看不到：Linux runner 签出来就是 LF。现在以 + `*.sh text eol=lf` 钉住，并有测试确认没有任何 entrypoint 带 CRLF。 +- **`.dockerignore` 放在 `docker/`，Docker 根本不会去那里读。** Docker 只读 build *context* + 根目录的那一份，而文档里每一条 build 指令都以仓库根目录为 context + （`docker build -f docker/Dockerfile .`），所以那些排除规则一条都没生效：`.git`、 + `.venv`、`test/` 跟各种缓存每次都被丢给 daemon。已移到根目录。 +- **`mss` 垫片的测试量的是开发机的屏幕，不是它自己的假件。** + `test_screen_grabber.py` 只换掉 `backend_grab_image`，`_backend_screen_size` 还是读真的 + `platform_wrapper.screen`，所以 `monitors[0]` 报的是开发者手边那台屏幕。在 1920x1080 + 的桌面上会过，在 1280x800 的 Xvfb 下就失败，而这跟被测代码无关。现在假件同时接管接缝的两半。 + +## 本次更新 (2026-08-17) — Wayland 看得见屏幕了 + +- **所有截取路径改走平台后端。** `screenshot()`、图像与锚点定位、OCR、smart waits、 + 视觉回归、屏幕录制、MCP 屏幕工具与远程桌面，此前各自直接调用 `PIL.ImageGrab` 或 + `mss`。这两者在 Linux 都是读 X11 root window——在 Wayland 下那个 root 属于 XWayland， + 不会合成原生 Wayland 窗口。Pillow 确实有回退到 `gnome-screenshot`／`grim`／`spectacle` + 的路径，但它只在 X11 截取**抛出异常**时才走（`except OSError`）——也就是根本没有 X + display 的情况；只要 XWayland 在跑（GNOME、KDE、sway 的默认）就不会触发。`mss` 则是 + 任何情况下都没有兜底。现在 `utils/cv2_utils/screen_grabber.py` 是唯一决定“这台机器怎么 + 读屏幕”的地方：后端若发布 `grab_image`，就把它包成调用端已在用的形状。Windows、 + macOS 与 Linux X11 不发布任何东西，仍然使用真正的库，行为完全不变。 +- **Wayland 截取覆盖三种合成器家族，而不只一种。** `grim` 只会说 `wlr-screencopy`， + GNOME 与 KDE 都没有实现——所以 `linux_wayland/capture.py` 依次尝试 `grim` + （sway、Hyprland、river）、`gnome-screenshot`、`spectacle`，并报告用了哪一个。只有 + `grim` 能自己接受区域参数，其余的截取整个屏幕后再裁剪。 +- **没有任何截取工具时明确失败。** 错误信息会列出各合成器该装什么，而不是返回空白的 + XWayland 截取——后者在下游只会表现成“找不到模板”。新的 `screen_capture` 诊断检查 + 会在任何失败发生前先说清楚当前用的是哪一个工具。 +- **`screen.size()` 与 `get_pixel` 在 GNOME／KDE 也能用。** 分辨率从 `wlr-randr` 回退 + 到以截取结果测量；`get_pixel` 从可用的截取路径裁出 1x1 区域。 + +**仍未完成**：libei 原生输入路径与 Wayland 真机验证，见 [Progress.md](../Progress.md)。 + ## 本次更新 (2026-07-03) — 稳定 API、失败诊断包与发布工程 给新集成用的版本化入口点、便携式失败诊断格式,以及强化后的发布管线。完整参考:[`docs/API_LIFECYCLE.md`](../docs/API_LIFECYCLE.md) 与 [`docs/CAPABILITY_MATRIX.md`](../docs/CAPABILITY_MATRIX.md)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 3ba94c5a..1de15dd2 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,490 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-19) — Wayland 兩個等人拍板的取捨,拍板了 + +`Progress.md` 上掛著兩個 `DECIDE`:缺的不是工，是決定。兩件事其實是同一個問題犯兩次 +——一個合成器的設定,函式庫**量得到**卻**讀不回來**,所以它必須停止假裝自己知道。 + +- **指標加速度改成由操作者宣告,函式庫照信。** 量到的事實不變: + `ydotool mousemove --absolute` 送的是相對移動,libinput 預設的 adaptive profile 會 + 把它加成兩倍,而倍率沒有任何用戶端讀得回來。沒拍板的是「那要怎麼辦」——繼續警告後 + 照送、直接拒絕、還是讓操作者自己講。直接拒絕會讓每一台沒有 `liboeffis` 的 Wayland + 機器整條 `set_position` 不能用,所以答案是宣告: + `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示 ydotoold 的裝置已經關掉加速度, + 移動就靜靜地、準確地送出去;`=strict` 表示寧可拒絕這次移動,也不讓點擊落在別的地方; + 不設定就維持現在的「警告一次後照樣移動」,所以今天能跑的東西明天照樣能跑。 + 值打錯會退回警告模式,**而且會講出來**——shell profile 裡的一個錯字,不可以靜靜地 + 把一次移動升級成「可信賴的精確」。整個判斷只掛在 ydotool 這條路上;libei 在協定層 + 本來就是絕對座標。 +- **Wayland 擷取裡的軟體游標,寫進文件,不繞過。** 這是 `seat-verification` 順手量到 + 的:專案裡沒有任何一條擷取帶 `-c`,也就是沒有人要求游標,而游標還是在圖裡。原因不在 + 我們這邊——只要 backend 沒有游標平面,wlroots 就會畫**軟體游標**,而軟體游標是合成進 + 輸出緩衝區的,`wlr-screencopy` 交出來的正是那一份。headless 永遠處在這個狀態; + 真桌面上只要跑 `WLR_NO_HARDWARE_CURSORS=1` 也一樣。Windows 的 BitBlt 與 X11 那條路 + 都不含游標,所以這是**只在 Wayland 出現的不一致**;指標壓在目標上的時候,定位器、 + 樣板比對與 OCR 看到的就是目標中間有一個游標形狀的洞。兩條繞法——擷取前把指標移開再 + 移回來、或比對時把指標周圍遮掉——都得先知道指標在哪,而 Wayland 不讓用戶端讀游標 + 位置;靠行程內的記錄去猜,使用者一動實體滑鼠就過期,而遮錯位置比看得見游標更糟。 + 所以改成寫下來:寫進 capability matrix、三份 README,以及診斷包——`screen_capture` + 檢查現在會帶 `cursor_may_be_captured`,讓那份「解釋定位器為何失敗」的報告自己說出 + 原因。檢查本身照量到的樣子斷言,哪天 wlroots 開始尊重 `overlay_cursor`,CI 會當場 + 紅掉來通知我們。 + +## 本次更新 (2026-08-19) — 「會吃 libinput 裝置的合成器」只差三個環境變數 + +`ydotool mousemove --absolute` 不是絕對移動。它不送任何 ABS 事件:先在兩軸各送一次 +`INT32_MIN` 的相對移動,把游標推到合成器夾取的那個角落,再把目標當成相對位移送出去。 +**那個角落到底是哪裡**、以及**位移在路上被合成器做了什麼**,是 Wayland 輸入路徑最後 +兩個沒有答案的問題——而且兩個都被記成「需要一台跑真桌面、會吃 libinput 裝置的 VM」。 + +- **兩個都不需要 VM。** wlroots 吃 `WLR_BACKENDS=headless,libinput`:輸出維持虛擬, + 輸入那一半是真的 libinput backend。libseat 的 builtin backend 不靠 logind 就能開 + 裝置,而 `SEATD_VTBOUND=0` 讓它不要去搶一個容器根本沒有的 VT。第四個條件最容易漏: + libinput 是透過 udev 列舉裝置,不是透過 `/dev`,所以 `systemd-udevd` 必須在 ydotoold + 建立裝置**之前**就已經在跑。四個都到位之後,ydotoold 的 uinput 裝置就是一個普通的 + seat 裝置,`grim -c` 會把游標合成進截圖裡,合成器就能用版面座標回答問題。 +- **那個角落是版面的左上角,不是版面座標的 `(0, 0)`。** 只有在所有輸出都在非負位置時 + 這兩點才是同一點。任何「主螢幕左邊還有一台螢幕」的桌面,兩者就差一個版面原點——所以 + 在原點為 `-1280` 的版面上,沒有轉換的「版面 `(0, 0)`」會把游標送到**隔壁那台螢幕**, + 差 1,280 像素。`mouse.set_position` 現在會先減掉 `layout_origin()` 再交給 ydotool, + 這正是擷取路徑早就在做的同一個修正;兩條輸入路徑共用的那個查詢搬進了 + `linux_wayland/_layout.py`,libei 與 ydotool 不會再對「原點是什麼」各說各話。 +- **剩下的距離則被指標加速度放大。** 送出去的位移是相對移動,所以 libinput 會加速它: + 對著真的 wlroots session 量到的是,預設的 adaptive profile 讓游標離那個角落的距離 + 正好是要求的**兩倍**——因為 `--absolute` 的兩個事件送在同一個 frame 裡,速度直接把 + profile 打到上限。把 `accel_profile flat` 加上 `pointer_accel 0` 之後,同一個呼叫 + 就精準到像素。ydotool 自己的 `--help` 一直都寫著「You need to disable mouse speed + acceleration for correct absolute movement」;後端現在會每個行程記一次這個警告,而不是 + 讓點擊安靜地落在錯的地方。函式庫這一側再多做不了什麼——那個倍率是合成器的設定, + 呼叫端讀不回來。 +- **新的 `seat-verification` job 把這些全部釘住。** `docker/Dockerfile.seat` 跑的是 + 擷取映像的同兩種版面,每種 14 項:sway 真的握著 ydotool 裝置、`--absolute (0, 0)` + 會把游標畫在版面的第一個像素上、關掉加速度後移動是一像素對一像素、沒轉換的 + `(0, 0)` 會打不中它指名的那台螢幕、`set_position` 減掉的正好是原點而且在兩台螢幕上 + 都落在指定的像素、以及加速倍率就是量到的 2 倍。裡面沒有任何一項依賴游標主題:每個 + 主張都是兩張截圖之間的差,游標圖片相對於熱點的偏移會自己抵銷掉。 +- **順手抓到的一件事。** 在這台合成器上,`grim` 明明沒有要求疊上游標,截回來的圖裡 + 還是有游標——因為只要 backend 沒有游標平面,wlroots 就會畫**軟體游標**(headless + 永遠如此,任何驅動不給平面、或使用者設了 `WLR_NO_HARDWARE_CURSORS=1` 的 session + 也是)。所有定位器、樣板比對與 OCR 都走同一條擷取,所以指標會在它壓著的東西上挖出 + 一個指標形狀的洞。該檢查照量到的樣子記下這個行為,要怎麼處理則列進 `Progress.md`。 + +## 本次更新 (2026-08-19) — 擷取用的 portal 本來就不可能成功,真的 bus 講出來了 + +- **`xdg-desktop-portal` 的回答是「指名送給發出呼叫的那條連線」的訊號。** + `Screenshot` 回的是一個 *request handle*,不是圖;圖稍後才以 + `org.freedesktop.portal.Request::Response` 送達,收件人是呼叫者的 unique bus name。 + bus 對指名訊息只送給收件人,所以別條連線再怎麼加 match rule 也接不到。 +- **舊的做法是兩條連線。** 先起一個 `gdbus monitor` 子行程,再用第二次 `gdbus` + 呼叫發請求,然後拿兩條正規表示式去讀 monitor 的 stdout。每次 `gdbus` 都會用自己的 + unique name 開一條自己的連線,所以在聽的那個行程從來就不是收件的那個。在真的 + `dbus-daemon` 上量到的:monitor 看得到呼叫經過,之後什麼都不再印,擷取每次都走完 + 整整 30 秒逾時。唯一看得到指名訊號的是完整的 bus monitor（`dbus-monitor`,它會向 + bus 要 `BecomeMonitor`）——為了一張備援截圖去換「觀察使用者 session bus 上每一則 + 訊息」的權限,不划算。 +- **這一層現在自己講 D-Bus,而且只用一條連線。** `linux_wayland/_dbus_client.py` + 是只用標準函式庫寫的 session bus 客戶端:連線、SASL EXTERNAL 認證、`Hello`、 + `AddMatch`、一次方法呼叫,然後讀到對得上的訊號為止。它刻意不是通用綁定——沒有 + property、沒有 introspection、不匯出物件、不傳檔案描述子（需要那一項的那一個呼叫 + 仍然交給 liboeffis）。`portal.py` 會**先**用自己的 unique name 推算出 request path + 並訂閱,再發呼叫;portal 若不理會 `handle_token`,回傳的 handle 也會一起跟。 +- **而且這是拿掉一個相依,不是加一個。** 這一層原本需要裝 `gdbus`（glib2),現在只要 + 有 session bus 就行,所以這條最後備援的擷取路徑能用的桌面比以前**更多**。安裝提示 + 與診斷檢查都改成這麼說了。 +- **在真的 bus 上整條驗過。** 新的 `portal-verification` job 跑真的 `dbus-daemon`、 + 真的 portal 實作與真的 client:擷取回來的 PNG 位元組解得開,而且就是 portal 畫的 + 那幾個像素,路徑是含空白、percent-escape 過的,讀完之後 portal 那個檔案不見了。 + portal 沒能交出圖的每一種收場也都跑過——對話框被關掉、對話框一直開著、成功但沒有 + URI、URI 不是本機檔案——每一種都必須在 AutoControl 自己的時限內 fail closed。 + +## 本次更新 (2026-08-19) — RemoteDesktop portal 交握也從來不需要 GNOME VM + +- **portal 是 D-Bus 介面,不是合成器功能。** 在 GNOME 與 KDE 上要接到 libei,得跑 + `CreateSession` → `SelectDevices` → `Start` → `ConnectToEIS`,最後由 bus 交出一個 + EIS 檔案描述子。這件事本來被記成「沒有 GNOME VM 就驗不到」,理由是 + `xdg-desktop-portal-wlr` 只做 ScreenCast 與 Screenshot、沒有 RemoteDesktop——那是把 + 「沒有容器附一個」當成了「沒有容器當得了一個」。對 `liboeffis` 而言,誰佔住 + `org.freedesktop.portal.Desktop`、誰回答那四個呼叫,誰就**是** portal。 +- **所以驗證自己去佔那個名字。** `docker/portal_server.py` 是一個跑在私有 session bus + 上的真 D-Bus 服務,它的 `ConnectToEIS` 交回去的是連到 `eis` 那個映像用的同一台真 + `libeis` server 的活連線。真的 `liboeffis` 跑完真的交握;出來的描述子承載得起一個 + 真的 EI session;透過它送出的按鍵、絕對移動與按鈕邊緣,由對面一個獨立實作記錄下來。 +- **它確定了什麼。** 四個呼叫依規範的順序、送到 client 自己推算出來的 request path; + `SelectDevices` 要到的是鍵盤與指標、不多要——所以使用者被要求同意的範圍就是這個後端 + 真正需要的範圍,`OEFFIS_DEVICE_DEFAULT` 也不是那個 `= 0` 的 all-devices 哨兵值; + 交回來的描述子是一個活的、由呼叫端持有並負責關閉的 socket——這正是把它交給 + `ei_setup_backend_fd`（一個會接管所有權的函式）之所以正確、而不是雙重關閉的原因。 +- **以及每一種拒絕。** 同意對話框被關掉、對話框一直開著、描述子被扣住、portal 把 + session 關掉、portal 舊到根本沒有 `ConnectToEIS`、bus 上根本沒有 portal:每一種都 + 必須在本專案自己的時限內變成拒絕,而不是卡住或悄悄降級。`OEFFIS_EVENT_CLOSED` 這條 + 分支從來沒有 peer 驅動得了,現在有了。 +- **仍然沒有宣稱的是什麼。** 同意對話框作為「對話框」本身。CI 裡沒有人去按它,所以 + 真的 mutter 對話框長什麼樣、真人會讓它開著多久,那仍然是 mutter 的事。對話框 + *產生的東西*——准、拒、沉默——三種都跑過了。 + +## 本次更新 (2026-08-19) — 一件記錯了的套件事實 + +- **Debian trixie 有 `liboeffis`。** `Progress.md` 原本寫沒有,並據此推論 libei 快速 + 路徑在 Debian／Ubuntu 上等於是關的。實測:`liboeffis1` 1.3.901-1 就在 trixie/main, + 提供 `liboeffis.so.1`。真正成立、而且對使用者真正有影響的是另一件事:它是**獨立的 + 二進位套件**,`libei1` 並不相依於它——所以只裝 libei 的機器上 portal 這條路仍然是關 + 的,`connect()` 會退回 GNOME 與 KDE 都不會開的 `eis-0` socket。要走快速路徑, + `liboeffis` 得自己裝。 + +## 本次更新 (2026-08-19) — 主螢幕左邊多一台螢幕,Wayland 的擷取整條都讀錯 + +- **Wayland 沒有「單一螢幕」這回事,而唯一那個平面也不一定從 `(0, 0)` 開始。** + 合成器把所有 output 排在同一個平面上,只要有一台在原點的左邊或上面,這個平面的 + 原點就是負的——那正是「我的第二台螢幕在左邊」對合成器的意思。sway 的 headless + backend 收 `output HEADLESS-1 position -1280 0`,所以這是 CI 立得起來的版面: + 兩個 1280x720 的 output、一張 2560x720 的擷取、左上角那個像素在 x=-1280。 +- **`size()` 回的是版面的右緣,不是寬度。** 它算的是各 output 的 `max(x + width)`, + 在上面那個版面是 1280,而 `grab_image()` 回來的畫面寬 2560。凡是把這兩者兜在一起 + 的人都信了小的那個:mss shim 的 monitor 清單（連帶 `enumerate_monitors`）、螢幕 + 錄影、WebRTC host、MCP 的 monitor 擷取,全都去要了一塊只有桌面一半大的矩形, + 然後當成整個畫面回報。 +- **不能自己套區域的層級,裁切裁在錯的座標空間。** 只有 grim 吃幾何參數; + gnome-screenshot、spectacle、portal 與 `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` + 都是把整個版面交回來、由 AutoControl 事後裁切。那個裁切用的是版面座標,而圖的 + (0,0) 是版面原點,所以要 `[-1275, 5, -1175, 55]`（左邊那台螢幕上的一塊）等於向 + Pillow 要一個在畫面左外側 1275 px 的方框,拿回來的是黑色填充。 +- **而且比對到的東西會被回報在錯的螢幕上。** `grab_logical()`——樣板搜尋、OCR 與 + visual match 背後的那個擷取——的原點是讀 `GetSystemMetrics` 的,在 Windows 以外 + 無話可說,於是一律回 `(0, 0)`,每個命中都被回報在實際位置的右邊 1280 px 處。 + 這種錯的表現是「點到別台螢幕」,比「找不到」更難查。 +- **修在接縫上,不是修在每個呼叫端。** Wayland backend 發布 `layout_origin()`; + `size()` 回 bounding box 的**大小**;`grab_image` 裁切前先扣掉原點; + `screen_grabber.backend_layout_origin()` 則是 `grab_logical` 與 mss shim 去問的 + 那一個,讓「自己擷取螢幕的後端」有辦法說出這張畫面從哪裡開始。通用函式庫本來就 + 看得見的後端（Windows／macOS／X11）什麼都不用發布,行為完全不變——原點只會被 + 問到,不會被猜。 +- **對真的合成器兩種版面都驗過。** `wayland-verification` job 現在把那 27 項檢查 + 跑兩遍:一遍是兩個 output 從原點並排,一遍是左邊那個在 x=-1280。第二遍才是把 + 負原點整條釘死的那一遍——grim 的負 `-g`、回報的尺寸、`layout_origin()`、 + mss shim 的 monitor 矩形、`grab_logical()` 的原點,以及把操作者自訂指令指向 grim + 之後、讓一張**真的**整版面 PNG 走過那條必須位移的裁切路徑。 + +## 本次更新 (2026-08-19) — 同一個版面問題的輸入側:libei 把移動悄悄丟掉了 + +- **落在所有 region 之外的絕對移動,libei 會直接丟掉,而且什麼都不說。** + 沒有回傳碼、沒有事件、呼叫端看不到任何錯誤——`ei_device_pointer_motion_absolute` + 就是不把這個事件送上線。`set_position` 於是像游標真的移動過一樣正常返回。 + 這是對真的 EIS 對端量出來的,不是推論:裝置只提供一個 `(0, 0, 1920, 1080)` + 的 region 時,`(1919, 1079)` 會到,`(1920, 1080)` 在 server 端連一個事件都沒有。 +- **而那些 region 所在的座標空間,不一定就是版面的座標空間。** region 的 offset + 是 `uint32`,所以沒有任何合成器**能**宣告一個在原點左邊或上面的 region——但本專案 + 的版面空間是從 `layout_origin()` 開始的,只要有一台螢幕在主螢幕左邊就會變成負的。 + 那正是擷取那一半剛修好的同一種桌面。於是兩半差了整整一個原點,也就是 + `get_pixel(x, y)` 與 `set_position(x, y)` 指到不同像素的那個情況——而本來會跑到 + 隔壁螢幕的游標,實際上是安安靜靜地哪裡都沒去。 +- **`LibeiBackend` 現在會讀裝置的 region,再把座標映射進去。** 綁上了 + `ei_device_get_region` 與四個 `ei_region_get_*`;`_region_point` 對有涵蓋的座標 + 原樣送出,對沒涵蓋的改用扣掉版面原點後的座標再試一次,兩者都不涵蓋就拒絕。 + 沒有宣告任何 region 的裝置接受任何座標,原樣通過——這一點同樣是量出來的,而那 + 就是單螢幕的常見情況,一毛錢都不多花。 +- **拒絕是有用的結果,不是失敗。** 它是 `LibeiUnavailable`,所以 + `_select_input.emitted` 會像處理「裝置被暫停」那樣,把這次移動交給 ydotool 那條路。 + libei 一直被寫成快速路徑而非唯一路徑;這個 bug 的真正問題是:被丟掉的移動從來 + 到不了後備路徑,因為沒有人知道它被丟掉了。拒絕時也不會送 frame——什麼都沒有被 + 緩衝,那裡送一個 frame 只會把上一次發送留在裝置上的東西提交出去。 +- **版面原點只在座標沒中的時候才會去問。** 它要花一個 `wlr-randr` 子行程,所以不會 + 出現在每一次普通滑鼠移動的路徑上;在 GNOME 與 KDE 上它回 `(0, 0)`——那在那裡是 + 正確答案而不是退路,因為那些合成器自己就會把版面正規化。 +- **`eis-verification` job 多了五項對真協定的檢查。** client 讀回來的 offset 就是 + 合成器宣告的那個;位在 `x=1280` 的 region,往內 100 px 的點要送 `1380` 而不是 + `100`;libei 到今天仍然一聲不吭地丟掉 region 外的移動——這是整個防護所依據的量測, + 所以哪天 libei 改成夾取,這項會當場說出來;AutoControl 會拒絕這種移動而不是弄丟它; + 以及在原點為 `-1280` 的版面上,`(-1280, 10)` 到達 server 時是 `(0, 10)`。 + 這個 job 現在跑 20 項。 +- **這件事沒有解決的部分。** ydotool 的 `mousemove --absolute` 有它自己的原點—— + 它夾到合成器的左上角,再把目標當成相對位移送出——那個角落是不是版面原點,仍然需要 + 一台真的會吃 libinput 裝置的合成器才驗得到。它繼續留在 `Progress.md`,不會靠猜測 + 去改。 + +## 本次更新 (2026-08-19) — ydotool 一直回報成功,其實什麼都沒做 + +- **`apt install ydotool`——這個 backend 自己印的安裝提示——裝到的版本跑不動它送的 + 命令列,而且是用「回傳 0」來表達的。** ydotool 1.0 把整套 CLI 換掉了,而 Wayland + backend 送的每一個參數都是那一版才有的:`mousemove --absolute`、`mousemove + --wheel`、`click` 的十六進位位元遮罩（拆得開 press 與 release,拖曳整個建立在 + 這上面）、以及吃數字 evdev 碼的 `key CODE:STATE`。Debian bookworm、Ubuntu 22.04 + 與 24.04 到今天都還是把 0.1.8 叫做 `ydotool`。對真的 uinput 裝置實測:0.1.8 收到 + `click 0x40` **什麼事件都不送,回傳 0**;收到 `mousemove --absolute` 印 + `unrecognised option`,**一樣回傳 0**。而 backend 是用 `check=True` 判斷成敗的, + 只有非零才會拋。所以在這些發行版上,腳本沒點到、沒打到、沒移動,而每一次呼叫都回報成功。 +- **舊版 CLI 現在在送出任何東西之前就被擋下。** `linux_wayland/_ydotool_cli.py` + 每個行程只判定一次（滑鼠與按鍵派送付不起每個事件一次 subprocess）,並在錯誤訊息裡 + 給出三條路:裝 1.0+ 的套件、自己編、或 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`。 + 兩個版本都沒有 `--version`,而 1.x 沒起 daemon 連 `--help` 都不回答,所以探測讀的是 + 兩邊都會印、不需要 daemon、也沒有副作用的那一樣東西:無參數時的指令清單。認不出來的 + 版本一律放行而不是擋掉,免得將來改了字樣的新版被過期的偵測器鎖在門外。 +- **兩處安裝提示還錯在第二件事上**:Debian trixie 根本沒有 `ydotool` 套件。 + 提示已改成點名真的有的發行版。 + +## 本次更新 (2026-08-19) — 驗 ydotool 從來不需要桌面,只需要一個讀取端 + +- **兩個驗證映像記下來的那個缺口,記錯了。** `Dockerfile.wayland` 與 `Dockerfile.eis` + 結尾都寫 ydotool「需要 /dev/uinput 以及一個會消費它的 seat」,而 `Progress.md` 把它 + 排在「先建一台 GNOME VM」後面。seat 決定的是注入的事件**會不會送達某處**,不是它 + **能不能被觀察**:ydotoold 建的是普通的 uinput 裝置,kernel 會掛成 + `/dev/input/eventN`,讀那個節點就拿得回 ydotool 寫進去的 `input_event`。 + 不需要合成器,不需要桌面 session,不需要 VM。 +- **`docker/Dockerfile.ydotool` 與 `docker/ydotool_verify.py` 就是在 CI 裡做這件事, + 12 項檢查。** 過去只對著 mock 斷言的東西現在有答案了:`0xc0`／`0xc1`／`0xc2` 真的是 + BTN_LEFT／BTN_RIGHT／BTN_MIDDLE;拆邊的 `0x40` 與 `0x80` 真的只送 press 或只送 + release(`press_mouse` 與拖曳整個建立在這上面);`key 30:1 30:0` 真的帶的是數字 + evdev 碼;而**捲動正負號是量出來的,不再是假設的**——`-y 1` 到 kernel 是 + `REL_WHEEL +1`、`-y -1` 是 `-1`、`-x 2` 是 `REL_HWHEEL +2`,而且軸沒有互換。 + 最後這一項正是捲動那次改動之後,`Progress.md` 一直標著「未實測」的假設。 +- **第 12 項檢查驅動的是 backend 自己的函式,不是手寫 argv**,所以「ydotool 拿到這串 + 命令列會做什麼」與「AutoControl 送的是什麼」是接起來的,不只是並排。 +- **`mousemove --absolute` 送的並不是絕對事件**,這件事在信任它之前值得知道。 + ydotool 1.x 的裝置上沒有 ABS 軸:它先在兩條相對軸上送 `INT32_MIN`,靠合成器把它夾到 + 左上角,再把目標當成相對位移送出去。所以 `set_position` 會落在你要的像素,**是因為 + 那個夾取**。kernel 這一側現在釘住了;夾取本身是合成器的行為,仍然是開放項。 +- **容器拿到的是 `/dev/uinput` 加字元主號 13,不是 `--privileged`。** ydotoold 是在 + 容器起來**之後**才建輸入節點,`--device` 涵蓋不到,所以那個 job 只授予 + `--device-cgroup-rule 'c 13:* rmw'`,別的都沒有。 + +## 本次更新 (2026-08-19) — Wayland 的捲動不再需要 uinput 常駐程式 + +游標移動、按鍵、按鈕早就走 libei 了,只剩捲動每一格都還要 fork 一次 `ydotool`, +而 `Progress.md` 也寫了原因:正負號是猜的,而捲錯方向不會噴錯,只會安靜地錯。 +現在接上了,猜測也換成了兩份獨立佐證加一次實測。 + +- **兩條路徑對「一格」的正負號定義相反。** 本專案的 `wayland_scroll_direction_*` + 常數是 kernel `REL_WHEEL` 那一套,因為 ydotool 寫進 `/dev/uinput` 的就是它:正值為上。 + libei 是 `wl_pointer`／libinput 那一套,正值為下——libinput 自己的 evdev 讀取端 + 就是把 `REL_WHEEL` 取負號換過去的,而另一個有寫明正負號的 libei sender（enigo） + 則是把「正值往下捲」的值原封不動送進 `scroll_discrete`。水平軸不用翻: + `REL_HWHEEL` 與 libinput 都以右為正。所以送往 libei 時垂直軸取負、水平軸不動—— + 這就是 `Progress.md` 在等的那個決定。 +- **這個翻轉有對著真的 EIS server 驗,連負值一起。** `docker/eis_verify.py` 多了第 15 項 + 檢查,驅動的是**公開的** `mouse.scroll()`（不是上一項檢查驅動的 backend 方法）, + 從 server 端讀回來:上是 `(0, -120)`、下是 `(0, 120)`、右是 `(120, 0)`。這也是唯一 + 一次把**負的**離散值送上線;先前那項檢查只送過正值,正負號的 marshalling 若有毛病 + 根本沒有地方會現形。 +- **被拒絕的發送現在會退回 CLI,也就是程式碼一直宣稱的行為。** `libei` 的模組 + docstring 寫著每個失敗都拋 `LibeiUnavailable`,「`keyboard`／`mouse` 已經把它當成 + *改用 ydotool CLI*」。實際上只有**連線**是這樣處理的。backend 交出去之後,合成器 + 暫停了裝置、或 session 在兩次呼叫之間結束,都會直接從 `set_position`、`press_key`、 + `hotkey` 拋出去。把捲動也接上 libei 等於再多一條「明明旁邊就有可用後備卻讓腳本死掉」 + 的路,所以這個後備被補成真的:和弦中途被拒會先把已經按下的鍵反序放開,不會留下卡住的 + 修飾鍵;按鈕的**放開**被拒則改由 ydotool 放開,不會整個 session 都按著。 +- **`LibeiUnavailable` 原本會穿過所有的攔截邊界。** 它只繼承 `RuntimeError`,而 + `CLAUDE.md` 寫得很明白:不是 `AutoControlException` 的框架錯誤「會安靜地逃出每一個 + 邊界」——executor、背景輪詢迴圈、請求處理器、GUI slot。現在兩個都繼承,原本接 + `RuntimeError` 的探測照舊能用,邊界也終於看得到它。 + +## 本次更新 (2026-08-18) — libei 輸入路徑終於有對手可以說話了 + +Wayland 的**擷取**路徑已經對著真的合成器驗過;**輸入**路徑沒有,而且被記成「需要一台 +GNOME VM」。其實不需要。libeis 就是 libei 自己那套協定的 server 端,Debian 有打包, +兩個函式庫可以直接在一條 Unix socket 上對話——所以 `docker/eis_server.py` 起一個真的 EIS +實作,`docker/eis_verify.py` 把 AutoControl 真正的 sender 對著它跑。不需要合成器, +不需要桌面 session,14 項檢查,已接進 CI。 + +- **離散捲動差了 120 倍。** libei 的離散捲動以「一格的 120 分之一」為單位——跟 Windows + `WHEEL_DELTA` 同一套慣例——而 `scroll()` 直接送格數,等於一格只送了 1/120 的捲動量。 + libei 自己在執行期就會講（`suspicious discrete event value 1, did you mean 120?`）, + 而這句話 mock 永遠不會印出來。現在 `scroll(0, 1)` 到對面是 `(0, 120)`:一格、正確的軸、 + 正確的正負號。這條路徑原本因為「正負號是猜的」而刻意沒接線,結果正負號是這題裡比較小的一半。 +- **拆除不再每個行程漏一個 context。** `ei_unref` 在 libei 1.3.901 會 segfault——但只在 + 「backend 開了、握手從未推進」那個狀態。有了可以完成握手的對手,live 的情況終於測得到, + 而它是安全的。現在拆除會正常釋放 device 與 context,只有真的會炸的那個狀態才放棄。 +- **mock 檢查不到的值都檢查了。** server 端 offer 六個 capability,再把 client 真正綁定的 + 讀回來:正好是 AutoControl 要的那四個,所以 `EI_DEVICE_CAP_*` 位元遮罩與 variadic + `ei_seat_bind_capabilities` 的編組都是對的。keycode、絕對座標、按鍵碼都從線上讀回來比對。 + 每次發送都確認有 frame,每個 device 都確認有先開 emulation transaction——沒開的話 + libei 會把事件丟掉。 +- **兩件量到但不是我們能修的事**,如實記下而不是含混帶過:libeis 1.3.901 的 + `eis_device_pause()` 對 sender client 沒有送出任何東西,所以 client 的 `DEVICE_PAUSED` + 處理仍然沒有對手可以驅動;`start_emulating` 的 sequence number 也沒有被送到對面 + （刻意送 4242,讀回來是 0）。檢查寫成「要嘛有反應,要嘛根本沒被通知」, + 哪天 libeis 開始送了而 client 忽略它,就會當場失敗。 + +## 本次更新 (2026-08-18) — 架構地圖的行數重新變成實測值 + +- **同一個子系統在地圖裡被寫成兩個不同的大小。** `CLAUDE.md` 規定 + `architecture_explore.md` 的每個數字都是實測的,但沒有任何東西在檢查,於是長出了 + 兩套並存的計數慣例:§5.4 主題表與 §5.4.17 檔案表數了一行不存在的結尾行—— + `len(text.split("\n"))` 會比以換行結尾的檔案實際行數多一行,套件則是每個檔案多一行 + ——而 §1 的總計與 §8 附錄數的是對的。結果 `utils/executor/` 在一節裡是 8,811 行、 + 在另一節裡是 9,001 行,而 §8 那一欄加起來也不等於它自己的總計。除此之外還有大約 + 五十列根本是舊的,好幾個 `####` 標題差了幾百行（`linux_wayland/` 還寫著 + 10 檔／1,093 行,實際是 14／2,235）,而且有一張主題表多了兩個子套件,它的摘要行 + 完全不知情。 +- **413 個數字用同一套慣例重新實測**——`len(text.splitlines())`,也就是 `wc -l` 的 + 結果,以及 `CLAUDE.md` 自己那段「超過 750 行」判斷所用的算法。§5.4、§5.4.17 與 §8 + 現在對每個子系統都一致,§8 各列加起來也等於它寫的總計。 +- **`test_doc_line_counts.py` 既是閘門也是修復工具。** 任何被引用的行數跟樹上對不上 + 就讓 CI 失敗,並指出是哪幾行;加 `--fix` 就一次全部就地改寫。行數是地圖裡唯一沒有 + 閘門的部分——指令、MCP 工具、子套件與範例數早就有 `test_doc_counts.py` 在管—— + 這正是它會漂掉的原因。 + +## 本次更新 (2026-08-18) — 剪貼簿不再因為別的程式正在複製而失敗 + +- **Windows 剪貼簿同一時間只允許一個行程開啟,而 AutoControl 的每一個剪貼簿呼叫 + 在別人開著時都立刻放棄。** 檔案總管、Office 和每個瀏覽器複製時都會佔住剪貼簿 + 幾毫秒,這段時間裡 `OpenClipboard` 一律回傳 false,而六個呼叫點——文字、影像、 + HTML、RTF、CSV、檔案拖放、格式列舉——都把它直接翻成 + `RuntimeError: OpenClipboard failed`。在真實桌面上開一個行程迴圈複製實測:大約 + 千分之一的開啟會失敗,也就是腳本會因為操作者既看不見也重現不了的原因直接死掉。 + Win32 文件明寫這種情況應該重試,而一個專門用來驅動「別的程式本來就很忙」的機器 + 的函式庫,不能把「別人正在複製」當成錯誤。 +- **`win32_clipboard_api.open_clipboard()` 現在是唯一開啟剪貼簿的地方**,忙碌時會 + 等約 200 毫秒才報錯,而且不論區塊怎麼結束都會關閉。三個自己手寫 open/close 的 + 模組——包括兩個比共用模組更早寫的——都改走它,所以新的呼叫點不可能漏掉重試。 +- **剪貼簿 round-trip 測試不再依賴機器上其他行程在做什麼。** 這些測試跑的是真實的 + Win32 呼叫,而那正是四個歷史 writer bug 唯一能被看見的地方,所以換成假的後端等於 + 刪掉覆蓋率,而不是讓它穩定。改成讀 Win32 剪貼簿序號:寫入與讀回之間序號沒變,就 + 代表這段時間沒有別人寫過,斷言測的就只有 AutoControl 自己的程式碼。已在另一個行程 + 於測試期間寫入 4,112 次剪貼簿的情況下驗證通過。 + +## 本次更新 (2026-08-18) — 打錯指令名字回 400,不是 500 + +- **`POST /execute` 對不存在的 `AC_*` 名字回 `500 {"error": "execute_action failed"}`**, + 跟伺服器自己壞掉時的回應一模一樣。呼叫端無法區分「我自己打錯字」和「服務掛了」, + 錯誤訊息也沒說是哪個名字不認得。 +- **所有指令名字現在都在執行前先校驗**,不認得的回 `400`,並在 `unknown_commands` + 裡列出**全部**——包含嵌在流程控制區塊裡的——讓呼叫端一次改完所有拼錯的名字。 + `POST /execute_file` 對讀不到的檔案、不是動作清單的內容、以及不認得的指令名, + 用同樣的方式回應。OpenAPI 規格兩者都寫明了,並說明被拒絕的請求沒有執行任何動作。 +- 校驗與收集共用同一次走訪,所以「巢狀動作清單可能藏在哪裡」仍然只有一份定義。 + +## 本次更新 (2026-08-18) — libei 拆除時的 segfault,以及對真實函式庫的核對 + +- **`ei_unref` 在 libei 1.3.901 上,只要 backend 已開啟就會讓行程崩潰,而我們的 + fallback 路徑正好一頭撞進去。** libei 握手的每一種失敗都會走到 `_teardown()`, + 所以在任何裝了 libei 而握手沒完成的機器上,AutoControl 會直接 SIGSEGV,而不是 + 安靜地改用 ydotool——跟 fail-closed 的承諾完全相反。逐個呼叫實測的結果:沒有 + setup backend 時 `ei_unref` 安全、setup **失敗**後安全、setup **成功**後必炸。 + `ei_disconnect` 在同樣狀態也炸,所以不是我們 refcount 用錯;而標頭檔明寫兩種結果 + 都該用 `ei_unref`,因此這是上游的 bug。現在已開啟的 backend 會被**放棄**而不是 + unref——代價是每個行程漏一個 context,換掉一個會驅動使用者桌面的函式庫直接崩潰。 + 驗證程式裡有一個哨兵會每次重新確認上游狀態,修好時會提示可以移除 workaround。 +- **綁定裡每一個進入點現在都對真的 `libei.so` 解析過。** 拼錯的符號或錯的 `argtypes` + 會毫無阻礙地通過假的符號表,只在使用者的機器上才爆——22 個 prototype 加上 variadic + 的 `ei_seat_bind_capabilities` 現在都是真的驗過。 +- **整條 fail-closed 鏈端到端跑過**:連到一個不說 EI 的 socket → 握手逾時 → + `LibeiUnavailable` → `active_backend()` 回 None → `press_key` 落到 ydotool CLI。 +- **`liboeffis` 不是每個發行版都有。** Arch 與 Fedora 有包,**Debian trixie 沒有**。 + 沒有它就沒有 portal 路徑,`connect()` 會退到眾所周知的 EIS socket——而 GNOME 與 KDE + 根本不開那個 socket。所以在那些系統上 libei 快速路徑是關閉的,而且會講出來,不會 + 看起來莫名其妙地沒作用。 + +## 本次更新 (2026-08-18) — Wayland 擷取路徑對上真的合成器了 + +- **`screen.size()` 報單一螢幕,`grab_image()` 卻回傳整個佈局。** `wlr-randr` 的 + parser 抓文件裡第一個 `WxH`,也就是第一個 output 的當前模式。雙螢幕佈局下那只有 + 半個畫面——而 mss shim 正是把這兩者組起來用的,所以錄影、WebRTC 與 MCP 的螢幕路徑 + 會去要一個只有半個畫面大的區域,而且真的拿到了。`size()` 現在回傳佈局的 bounding + box,parser 會讀每個啟用中 output 的模式**與位置**。這個 bug 是跑真機才發現的: + 沒有任何 mock 有第二個螢幕。 +- **`docker/Dockerfile.wayland` 讓後端在 headless sway 底下跑。** wlroots 的 headless + backend 不需要 GPU、seat 或顯示器,所以一個真正的 Wayland session 塞得進容器——現在 + 也進了 CI,就是 `wayland-verification` job。兩個 output 塗上不同純色,因為在單一顏色 + 的畫面上,區域抓錯位置抓不出來,紅藍通道對調更是完全抓不出來。 +- **21 項先前只能對 mock 驗的檢查,現在對的是合成器真的畫出來的像素**:grim 的 argv + 與 `-g` 幾何、RGB 通道順序、`wlr-randr` 沒有文件記載的輸出格式、 + `size()`／`grab_image()`／`get_pixel()`／`screenshot()`、 + `je_auto_control.screenshot()` 的 BGR 輸出、`grab_logical()`(定位器與 OCR 路徑)、 + mss shim、以及 `wtype`。 +- **容器答不了的部分直接寫明,不含糊帶過。** ydotool 需要 `/dev/uinput`,而 headless + sway 不吃 libinput 裝置;`xdg-desktop-portal-wlr` 沒有實作 RemoteDesktop,所以根本 + 沒有 `ConnectToEIS` 可測。兩者都留在 `Progress.md`。 + +## 本次更新 (2026-08-18) — libei 輸入,整條打通 + +- **完整的 portal 握手做完了。** libei 不是「呼叫一個函式就按下一個鍵」的函式庫: + sender 必須開啟 EIS backend、綁定 seat 的能力、**從事件裡**取得 device、在其上 + `start_emulating`,而且每次發送之後都要 `ei_device_frame`,否則什麼都不會送達。 + 這些現在全部有了,所以 `press_key`、`set_position` 與滑鼠按鍵不再需要每個事件 + spawn 一次行程。 +- **EIS socket 來自桌面 portal。** 在 GNOME 與 KDE 上它不是磁碟上的路徑,而是由 + `org.freedesktop.portal.RemoteDesktop.ConnectToEIS` 經 D-Bus 交出的 file + descriptor,前面還有三次非同步的 session 呼叫。**沒有任何命令列工具能把 fd 交進 + 這個行程**,所以截圖那條 `gdbus` 路在這裡走不通;改用 `liboeffis`(libei 專案正是 + 為此附上它)。liboeffis 不存在時,仍會嘗試眾所周知的 `$XDG_RUNTIME_DIR/eis-0`。 +- **device 一律來自事件,不再由 context 冒充。** 先前的綁定把 `struct ei *` context + 傳給收 `struct ei_device *` 的進入點——C 函式庫裡的指標型別混淆。現在從結構上 + 不可能發生。 +- **探測失敗只付一次代價,不是每次按鍵。** 握手包含一次 portal 往返,可能還有同意 + 對話框。結果會在行程內快取,所以「裝了 libei 但不可用」的機器不會每次按鍵重試。 +- **所有情況都仍會退回 ydotool。** 函式庫缺失、使用者拒絕、能力只給一半、device 被 + 暫停、握手沒完成——每一種都丟 `LibeiUnavailable`,而 keyboard／mouse 本來就把它 + 當成「改用 CLI」。scroll 刻意留在 ydotool:它的方向約定有測試釘住,而 libei 的正負號 + 一旦猜錯是**靜默**的錯誤行為,不是明確的失敗。 +- **ABI 常數是對過上游標頭檔的,不是猜的。** 對完發現三個錯: + `enum ei_device_capability` 是**位元遮罩**,所以 `EI_DEVICE_CAP_KEYBOARD` 是 + `1 << 2` 而不是 3;`OEFFIS_EVENT_CLOSED` 排在 `OEFFIS_EVENT_DISCONNECTED` + **之前**;`OEFFIS_DEVICE_ALL_DEVICES` 是 `= 0` 的哨兵值,不是各裝置位元的 OR。 + 第一個代價最大——沒有 device 會回報那個能力,所以每次連線都會逾時然後靜默改用 CLI。 + 核對過的值現在有測試釘住。另外 session 只申請實際會用到的鍵盤與指標,同意對話框 + 不會再要一個沒人用的觸控螢幕授權。 + +## 本次更新 (2026-08-18) — Wayland 擷取有了保底 + +- **`xdg-desktop-portal` 為三個 CLI 工具兜底。** `grim`／`gnome-screenshot`／`spectacle` + 沒有任何一個保證會裝（GNOME 自 42 起不再預設安裝 `gnome-screenshot`），所以最後會經由 + `gdbus` 嘗試 `org.freedesktop.portal.Screenshot`，而不是直接放棄。它天生麻煩：portal + 回傳的是 request handle、結果之後才以 signal 送達，所以監聽必須在呼叫**之前**啟動； + 而且同意對話框可能擋在前面，因此等待有 30 秒上限。 +- **操作者可以指定自己的擷取指令。** + `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` 優先於所有偵測。 + `{output}` 會換成暫存 PNG 路徑，在 `shlex.split` 之後逐一參數替換、不經過 shell， + 所以含空白的路徑仍然是單一參數。這是給內建分層都不適用的環境用的逃生門——包含我們對 + 某個工具的 argv 猜錯的情況。 +- **libei 主動拒絕它根本送不出輸入的連線。** 這個綁定從頭到尾只握有 `ei` context， + 但每個 device 進入點收的都是 `ei_device`，而且完全沒有跑 libei 的 seat／device／ + `start_emulating`／`frame` 握手——所以它送不出輸入，卻**可能**在會開 + `$XDG_RUNTIME_DIR/eis-0` 的機器上把錯誤的指標傳進 C 函式庫。現在它會在 `connect()` + 就停下並說明原因。呼叫端本來就把這個情況當成「改用 ydotool CLI」，而那正是所有真實 + 桌面環境一直在做的事。 +- **portal 監聽器不會在收尾時卡死。** 它的 pipe 只在讀取執行緒放手之後才關閉；在有執行緒 + 仍阻塞在 `read()` 時關閉串流可能卡在 buffer lock 上，那會讓「逾時」變成它本來要防止的 + 「卡住」。 + +## 本次更新 (2026-08-18) — 容器映像在 Windows 簽出下也能建、能啟 + +- **在 Windows 上 clone 再 build 出來的容器，一啟動就死。** `.gitattributes` 只寫了 + `* text=auto`，於是 `docker/entrypoint.sh` 與 `docker/entrypoint-xfce.sh` 被簽出成 CRLF。 + shebang 就變成 `#!/bin/sh`，kernel 去找一個名字末尾帶回車的直譯器，映像 build 得完全正常， + 一跑就丟出 `exec /usr/local/bin/autocontrol-entrypoint: no such file or directory`—— + 而那個檔明明就在。CI 永遠看不到：Linux runner 簽出來就是 LF。現在以 + `*.sh text eol=lf` 釘住，並有測試確認沒有任何 entrypoint 帶 CRLF。 +- **`.dockerignore` 放在 `docker/`，Docker 根本不會去那裡讀。** Docker 只讀 build *context* + 根目錄的那一份，而文件裡每一道 build 指令都以倉庫根目錄為 context + （`docker build -f docker/Dockerfile .`），所以那些排除規則一條都沒生效：`.git`、 + `.venv`、`test/` 跟各種快取每次都被丟給 daemon。已移到根目錄。 +- **`mss` 墊片的測試量的是開發機的螢幕，不是它自己的假件。** + `test_screen_grabber.py` 只換掉 `backend_grab_image`，`_backend_screen_size` 還是讀真的 + `platform_wrapper.screen`，所以 `monitors[0]` 報的是開發者手邊那台螢幕。在 1920x1080 + 的桌面上會過，在 1280x800 的 Xvfb 下就失敗，而這跟被測程式碼無關。現在假件同時接管接縫的兩半。 + +## 本次更新 (2026-08-17) — Wayland 看得見螢幕了 + +- **所有擷取路徑改走平台後端。** `screenshot()`、影像與錨點定位、OCR、smart waits、 + 視覺回歸、螢幕錄影、MCP 螢幕工具與遠端桌面，先前各自直接呼叫 `PIL.ImageGrab` 或 + `mss`。這兩者在 Linux 都是讀 X11 root window——在 Wayland 下那個 root 屬於 XWayland， + 不會合成原生 Wayland 視窗。Pillow 確實有退回 `gnome-screenshot`／`grim`／`spectacle` + 的路徑，但它只在 X11 擷取**拋出例外**時才走（`except OSError`）——也就是根本沒有 X + display 的情況；只要 XWayland 在跑（GNOME、KDE、sway 的預設）就不會觸發。`mss` 則是 + 任何情況下都沒有後備。現在 `utils/cv2_utils/screen_grabber.py` 是唯一決定「這台機器怎麼 + 讀螢幕」的地方：後端若發布 `grab_image`，就把它包成呼叫端已在用的形狀。Windows、 + macOS 與 Linux X11 不發布任何東西，仍然使用真正的函式庫，行為完全不變。 +- **Wayland 擷取涵蓋三種合成器家族，而不只一種。** `grim` 只會說 `wlr-screencopy`， + GNOME 與 KDE 都沒有實作——所以 `linux_wayland/capture.py` 依序嘗試 `grim` + （sway、Hyprland、river）、`gnome-screenshot`、`spectacle`，並回報用了哪一個。只有 + `grim` 能自己接受區域參數，其餘的擷取整個螢幕後再裁切。 +- **沒有任何擷取工具時明確失敗。** 錯誤訊息會列出各合成器該裝什麼，而不是回傳空白的 + XWayland 擷取——後者在下游只會表現成「找不到樣板」。新的 `screen_capture` 診斷檢查 + 會在任何失敗發生前先講清楚目前用的是哪一個工具。 +- **`screen.size()` 與 `get_pixel` 在 GNOME／KDE 也能用。** 解析度從 `wlr-randr` 退回 + 以擷取結果量測；`get_pixel` 從可用的擷取路徑裁出 1x1 區域。 + +**仍未完成**：libei 原生輸入路徑與 Wayland 真機驗證，見 [Progress.md](../Progress.md)。 + ## 本次更新 (2026-07-03) — 穩定 API、失敗診斷包與發佈工程 給新整合用的版本化進入點、可攜式失敗診斷格式,以及強化後的發佈管線。完整參考:[`docs/API_LIFECYCLE.md`](../docs/API_LIFECYCLE.md) 與 [`docs/CAPABILITY_MATRIX.md`](../docs/CAPABILITY_MATRIX.md)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 1c084aa5..fac41b3f 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,7 +1,683 @@ # What's New — AutoControl +## What's new (2026-08-19) + +### Two Wayland Judgement Calls, Settled + +Two items had been sitting in `Progress.md` marked `DECIDE`: not missing work, +missing decisions. They turned out to be the same problem twice — a compositor +setting the library can *measure* but cannot *read*, and therefore must stop +pretending to know. + +- **Pointer acceleration is now something the operator declares, and the + library believes.** The measurement stands: `ydotool mousemove --absolute` + sends relative motion, libinput's default adaptive profile doubles it, and no + client can read the factor back. What was undecided was what to do about it — + keep warning and move anyway, refuse outright, or let the operator say. + Refusing outright would have taken `set_position` away from every Wayland + machine without `liboeffis`, so the answer is a declaration: + `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` means acceleration is off for the + ydotoold device, and the move goes out silently and exactly; `=strict` means + refuse the move rather than let a click land somewhere else; unset keeps + today's warn-once-then-move, so nothing that works now stops working. An + unrecognised value falls back to the warning *and says that it did* — a typo + in a shell profile must not quietly promote a move to trusted-exact. The + whole gate is on the ydotool path; libei is absolute at the protocol level. +- **The software cursor in a Wayland capture is documented, not worked + around.** `seat-verification` measured it in passing: no capture in this + project passes `grim -c`, so none asks for the pointer, and the pointer is in + the image anyway. The reason is not ours — wlroots draws a *software* cursor + whenever the backend has no cursor plane, and a software cursor is composited + into the output buffer, which is exactly the buffer `wlr-screencopy` hands + back. Headless is permanently in that state; so is any real desktop running + `WLR_NO_HARDWARE_CURSORS=1`. Windows' BitBlt and the X11 path never include + the pointer, so this is a Wayland-only inconsistency, and with the pointer + resting on its target a locator, a template match or an OCR read sees a + pointer-shaped hole in the middle of it. Both ways out — move the pointer + away and back, or mask around it — need to know where the pointer is, and + Wayland does not let a client read that; an in-process guess goes stale the + moment the user touches their own mouse, and masking the wrong place is worse + than a visible cursor. So it is written down instead: in the capability + matrix, in all three READMEs, and in the diagnostics bundle, where the + `screen_capture` check now carries `cursor_may_be_captured` so the report + that explains a failed locator names the reason. `seat-verification` asserts + the behaviour as measured, so if wlroots ever honours `overlay_cursor` for + software cursors, CI goes red and tells us. + +### A Compositor That Consumes Input Was Three Environment Variables Away + +`ydotool mousemove --absolute` is not absolute. It emits no absolute event: +it sends `INT32_MIN` on both axes to drive the cursor into whatever corner the +compositor clamps to, then sends the target as a relative displacement. What +that corner *is*, and what the compositor does to the displacement on the way, +were the last two open questions on the Wayland input path — and both were +recorded as needing a VM running a desktop that consumes libinput devices. + +- **They needed no VM.** wlroots takes `WLR_BACKENDS=headless,libinput`: the + outputs stay virtual while the input half is the real libinput backend. + libseat's builtin backend opens the devices without logind, and + `SEATD_VTBOUND=0` stops it reaching for a VT no container owns. The fourth + requirement is the one that is easy to miss — libinput enumerates through + udev rather than through `/dev`, so `systemd-udevd` has to be running before + ydotoold creates its device. With those four in place, ydotoold's uinput + device is an ordinary seat device, and `grim -c` composites the cursor into + a screenshot, so the compositor answers in layout coordinates. +- **The corner is the layout's top-left, not layout `(0, 0)`.** Those are the + same point only while every output sits at a non-negative position. On the + layout every desktop with a monitor left of the primary one has, they differ + by the layout origin — so on a `-1280` layout an untranslated request for + layout `(0, 0)` put the cursor on the *other monitor*, 1,280 pixels away. + `mouse.set_position` now subtracts `layout_origin()` before handing the + coordinate to ydotool, which is the same correction the capture path already + applies; the lookup both input paths share moved into + `linux_wayland/_layout.py` so libei and ydotool cannot drift apart on it. +- **And pointer acceleration scales the rest.** The displacement is relative + motion, so libinput accelerates it: measured against a real wlroots session, + the default adaptive profile lands the cursor exactly twice as far from that + corner as asked, because `--absolute` sends both of its events in one frame + and the velocity saturates the profile. With `accel_profile flat` and + `pointer_accel 0` the same call is pixel-exact. ydotool's own `--help` has + said "You need to disable mouse speed acceleration for correct absolute + movement" all along; the backend now logs that caveat once per process + instead of letting a click land silently in the wrong place. Nothing else + can be done from inside the library — the factor is the compositor's + setting, not something a caller can read back. +- **A new `seat-verification` job holds all of it.** `docker/Dockerfile.seat` + runs the two layouts the capture image runs, and 14 checks each: that sway + really is holding the ydotool device, that `--absolute (0, 0)` draws the + cursor flush into the layout's first pixel, that with acceleration off the + move is one pixel per pixel, that an untranslated `(0, 0)` misses the + monitor it names, that `set_position` subtracts exactly the origin and lands + on the pixel it was given on both monitors, and that the acceleration factor + is the 2x this was measured at. Nothing in it depends on the cursor theme: + every claim is a difference between two captures, which the image's offset + from its hotspot cancels out of. +- **One thing it found on the way.** On this compositor a `grim` capture that + asks for no cursor overlay contains one anyway, because wlroots draws a + *software* cursor whenever the backend has no cursor plane — always on + headless, and on any session where the driver refuses one or the user set + `WLR_NO_HARDWARE_CURSORS=1`. Every locator, template match and OCR read goes + through that capture, so the pointer punches a pointer-shaped hole in + whatever it is sitting on. The check records the behaviour as measured, and + what to do about it is an open item in `Progress.md`. + +### The Screen-Capture Portal Could Never Have Worked, and a Real Bus Said So + +- **`xdg-desktop-portal` answers with a signal directed at the connection that + called it.** `Screenshot` returns a *request handle*, not an image; the image + arrives later as `org.freedesktop.portal.Request::Response`, addressed to the + caller's unique bus name. The bus routes a directed message to its + destination and nowhere else, so no match rule on any other connection can + make it arrive somewhere else. +- **The old implementation was two connections.** It started `gdbus monitor` in + one subprocess, made the call from a second `gdbus` invocation, and read the + monitor's stdout with a pair of regular expressions. Each `gdbus` invocation + opens its own connection under its own unique name, so the process listening + was never the process addressed. Measured against a real `dbus-daemon`: the + monitor sees the call go past and prints nothing else, and the capture runs + out its full 30-second timeout, every time. The only listener that can see a + directed signal is a full bus monitor — `dbus-monitor`, which asks the bus + for `BecomeMonitor` — and needing permission to observe every message on the + user's session bus is a poor price for a fallback screenshot. +- **The tier now speaks D-Bus itself, on one connection.** + `linux_wayland/_dbus_client.py` is a session-bus client in the standard + library alone: connect, SASL EXTERNAL authentication, `Hello`, `AddMatch`, + one method call, then read until the matching signal arrives. It is + deliberately not a general binding — no properties, no introspection, no + object export, no descriptor passing (liboeffis still does the one call that + needs that). `portal.py` subscribes to the request path it predicts from its + own unique name *before* it calls, and follows the returned handle as well + when a portal ignores `handle_token`. +- **Which also removes a dependency rather than adding one.** The tier used to + need `gdbus` (glib2) installed; it now needs nothing but a session bus, so + the last-resort capture path is available on strictly more desktops than + before. The install hint and the diagnostics check say so. +- **Verified end to end on a real bus.** A new `portal-verification` job runs a + real `dbus-daemon`, a real portal implementation and the real client: the + capture comes back as PNG bytes that decode to the pixels the portal painted, + at a percent-escaped path with a space in it, and the portal's file is gone + afterwards. Every way a portal ends without an image is driven too — a + dismissed dialog, a dialog left open, a success carrying no URI, a URI that + is not a local file — and each has to fail closed on AutoControl's own clock. + +### The RemoteDesktop Portal Handshake Never Needed a GNOME VM Either + +- **The portal is a D-Bus interface, not a compositor feature.** Reaching libei + on GNOME and KDE means `CreateSession` → `SelectDevices` → `Start` → + `ConnectToEIS`, ending in an EIS file descriptor passed over the bus. That + was recorded as unverifiable without a GNOME VM because + `xdg-desktop-portal-wlr` implements ScreenCast and Screenshot but not + RemoteDesktop — which confuses "no container ships one" with "no container + can host one". Whatever owns `org.freedesktop.portal.Desktop` and answers + those four calls *is* the portal, as far as `liboeffis` is concerned. +- **So the verification owns the name itself.** `docker/portal_server.py` is a + real D-Bus service on a private session bus, and its `ConnectToEIS` hands + back a live connection to the same real `libeis` server the `eis` image uses. + The real `liboeffis` runs the real handshake; the descriptor that comes out + carries a real EI session; and the key presses, absolute motion and button + edges emitted through it are recorded by an independent implementation at the + far end. +- **What it settles.** That the four calls arrive in the prescribed order at the + request paths the client predicted; that `SelectDevices` is asked for keyboard + and pointer and nothing wider, so the grant a user consents to is the one this + backend needs and `OEFFIS_DEVICE_DEFAULT` is not the `= 0` all-devices + sentinel; that the descriptor is a live socket the caller owns and must close, + which is what makes handing it to `ei_setup_backend_fd` — a function that + takes ownership — correct rather than a double close. +- **And every refusal.** A dismissed consent dialog, a dialog left open, a + withheld descriptor, a session the portal closes, a portal too old to have + `ConnectToEIS` at all, and no portal on the bus: each has to come back as a + refusal on this project's own clock rather than a hang or a silent downgrade. + The `OEFFIS_EVENT_CLOSED` branch had never had a peer able to drive it; it + does now. +- **What is still not claimed.** The consent dialog as a dialog. Nobody + dismisses anything in CI, so what a real mutter dialog looks like, and how + long a real person leaves it open, stays mutter's business. What a dialog + *produces* — a grant, a refusal, silence — is all exercised. + +### One Packaging Fact That Was Recorded Wrong + +- **Debian trixie does ship `liboeffis`.** `Progress.md` said it did not, and + concluded that the libei fast path was effectively off across Debian and + Ubuntu. Measured: `liboeffis1` 1.3.901-1 is in trixie/main, providing + `liboeffis.so.1`. What is true, and what actually matters to a user, is that + it is a *separate binary package* which `libei1` does not depend on — so + installing libei alone still leaves the portal route off and `connect()` + falls back to the `eis-0` socket that GNOME and KDE do not open. Install + `liboeffis` to get the fast path. + +### A Monitor Left of the Primary Broke Every Wayland Capture Path + +- **Wayland has no per-monitor screen, and the one it does have need not start + at `(0, 0)`.** The compositor lays every output out on a single plane, and + that plane starts at a negative coordinate the moment an output sits left of + or above the origin — which is what "my second monitor is on the left" means + to a compositor. sway's headless backend accepts `output HEADLESS-1 position + -1280 0`, so this is now a layout CI can stand up: two 1280x720 outputs, one + 2560x720 capture, top-left pixel at x=-1280. +- **`screen.size()` was returning the layout's right edge, not its width.** It + computed `max(x + width)` over the outputs, which is 1280 on that layout + while `grab_image()` returns a frame 2560 wide. Everything that composes the + two believed the smaller number: the mss-shaped shim's monitor list (and so + `enumerate_monitors`), the screen recorder, the WebRTC host and the MCP + monitor grab all asked for a rectangle half the size of the desktop and + reported it as the whole screen. +- **The crop for the tiers that cannot take a region cropped in the wrong + space.** Only grim accepts a geometry; gnome-screenshot, spectacle, the + portal and `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` all hand back the whole + layout and AutoControl crops afterwards. That crop used layout coordinates + against an image whose origin is the layout origin, so asking for + `[-1275, 5, -1175, 55]` — a rectangle on the left-hand monitor — asked + Pillow for a box 1275 px left of the frame and got black padding. +- **And a match found on that monitor was reported on the wrong one.** + `grab_logical`, the capture behind template search, OCR and visual match, + reads its origin from `GetSystemMetrics`, which says nothing off Windows — + so it returned `(0, 0)` and every hit came back 1280 px to the right of + where it was seen. That reads as "the click lands on the wrong screen" + rather than as a failure to find, which is the worse of the two. +- **Fixed at the seam, not at the call sites.** The Wayland backend publishes + `layout_origin()`; `size()` returns the bounding box's *size*; `grab_image` + subtracts the origin before cropping; and + `screen_grabber.backend_layout_origin()` is what `grab_logical` and the mss + shim ask, so a backend that captures its own screen can say where that + capture starts. Backends the generic libraries can already see (Windows, + macOS, X11) publish nothing and are unchanged — the origin is only ever + asked for, never guessed. +- **Verified against a real compositor, both ways round.** The + `wayland-verification` job now runs its 27 checks twice: once with the + outputs side by side from the origin, once with the left-hand one at + x=-1280. The second run is what pins the negative case end to end — grim's + negative `-g`, the reported size, `layout_origin()`, the mss shim's monitor + rectangle, `grab_logical`'s origin, and the fallback crop driven through the + operator override pointed at grim so a *real* whole-layout PNG goes through + the code path that has to shift it. + +### The Same Layout Problem, on the Input Side — a Move libei Was Dropping in Silence + +- **libei discards an absolute motion that lands in no region, and reports + nothing about it.** No return code, no event, no error the caller can see — + `ei_device_pointer_motion_absolute` simply does not put the event on the + wire. `set_position` then returned as though the pointer had moved. Measured + against a real EIS peer, not inferred: with the device offering one + `(0, 0, 1920, 1080)` region, `(1919, 1079)` arrives and `(1920, 1080)` + produces no server-side event at all. +- **And the space those regions live in need not be the layout's.** A region's + offset is a `uint32`, so no compositor *can* advertise one left of or above + the origin — while this project's layout space starts at `layout_origin()` + and goes negative the moment a monitor sits left of the primary. That is the + exact desktop the capture half was just fixed for. The two halves therefore + disagreed by the origin, which is the case where `get_pixel(x, y)` and + `set_position(x, y)` name different pixels — and the pointer that would have + gone to the wrong monitor instead went nowhere, quietly. +- **`LibeiBackend` now reads the device's regions and maps the point into + them.** `ei_device_get_region` and the four `ei_region_get_*` getters are + bound; `_region_point` sends a covered coordinate unchanged, retries an + uncovered one normalised by the layout origin, and refuses what neither + covers. A device that declared no region accepts anything and is passed + through untouched, which is measured too — that is the common single-monitor + case, and it costs nothing. +- **A refusal is the useful outcome, not a failure.** It is a + `LibeiUnavailable`, so `_select_input.emitted` hands the move to the ydotool + path exactly as it already does for a paused device. libei is documented as + the fast path and never the only one; the bug was that a dropped move never + reached the fallback because nothing knew it had been dropped. The frame is + not sent on a refusal either — nothing was buffered, and a frame there would + commit whatever the previous emission left on the device. +- **The layout origin is only consulted when a point misses.** It costs a + `wlr-randr` subprocess, so it stays off the path every ordinary mouse move + takes, and it answers `(0, 0)` on GNOME and KDE — which is the right answer + there rather than a fallback, because those compositors normalise the layout + themselves. +- **Five new checks in the `eis-verification` job, against the real + protocol.** That the client reads back the offsets the compositor + advertised; that a region at `x=1280` takes `1380` for a point 100 px into + it rather than `100`; that libei still drops an out-of-region motion without + a word — the measurement the whole guard rests on, so a future libei that + clamps instead says so; that AutoControl refuses such a move rather than + losing it; and that `(-1280, 10)` on a layout starting at `-1280` reaches + the server as `(0, 10)`. The job now runs 20 checks. +- **What this does not settle.** ydotool's `mousemove --absolute` has an + origin of its own — it clamps to the compositor's top-left corner and sends + the target as a relative delta — and whether that corner is the layout + origin still needs a compositor that consumes libinput devices. It stays + open in `Progress.md` rather than being changed on a guess. + +### The ydotool Path Was Reporting Success While Doing Nothing + +- **`apt install ydotool` — the hint this backend printed — installs a + version whose command line cannot run it, and which says so by exiting + zero.** ydotool 1.0 replaced the whole CLI, and every argument the Wayland + backend builds arrived in that release: `mousemove --absolute`, `mousemove + --wheel`, hex `click` bitmasks (which is what lets a press and a release be + sent separately, and therefore what makes drag possible), and `key + CODE:STATE` taking numeric evdev codes. Debian bookworm, Ubuntu 22.04 and + Ubuntu 24.04 all still ship 0.1.8 under that name. Measured against a real + uinput device, 0.1.8 answers `click 0x40` with **no events and exit code + 0**, and answers `mousemove --absolute` with `unrecognised option` — also + **exit code 0**. The backend runs ydotool with `check=True`, so a non-zero + status was the only thing that would have raised. On those distributions a + script clicked nothing, typed nothing and moved nothing, and every call + reported success. +- **The legacy CLI is now refused before anything is sent.** + `linux_wayland/_ydotool_cli.py` classifies the installed ydotool once per + process — mouse and key dispatch cannot afford a subprocess per event — and + raises with the three routes out: a 1.0+ package, a source build, or + `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`. Neither series implements + `--version` and 1.x will not answer `--help` without its daemon running, so + the probe reads the one thing both print with no daemon and no side + effects: the no-argument command list. A version it does not recognise is + allowed through rather than blocked, so a future release that changes that + banner cannot be locked out by a stale detector. +- **Both install hints were wrong in a second way.** Debian trixie ships no + `ydotool` package at all. The hints now name the distributions that do. + +### ydotool Never Needed a Desktop to Verify — Only a Reader + +- **The gap both verification images recorded turned out to be the wrong + gap.** `Dockerfile.wayland` and `Dockerfile.eis` each closed by saying + ydotool "needs /dev/uinput and a seat that consumes it", and `Progress.md` + filed that behind building a GNOME VM. A seat is what makes an injected + event *arrive somewhere*. It is not what makes one *observable*: ydotoold + creates an ordinary uinput device, the kernel publishes it as + `/dev/input/eventN`, and reading that node returns the exact `input_event` + structs ydotool wrote. No compositor, no session, no VM. +- **`docker/Dockerfile.ydotool` and `docker/ydotool_verify.py` do that, in + CI, in twelve checks.** They settle what had only ever been asserted + against mocks: `0xc0` / `0xc1` / `0xc2` really are BTN_LEFT / BTN_RIGHT / + BTN_MIDDLE; the split edges `0x40` and `0x80` really do send a press with + no release and a release with no press, which is the entire basis of + `press_mouse` and drag; `key 30:1 30:0` really does carry numeric evdev + codes; and **the wheel signs are measured rather than assumed** — `-y 1` + reaches the kernel as `REL_WHEEL +1`, `-y -1` as `-1`, and `-x 2` as + `REL_HWHEEL +2` with the axes not swapped. That last one is the assumption + `Progress.md` had flagged as untested since the scroll work landed. +- **The twelfth check drives the backend's own functions rather than a + hand-written argv**, so "what ydotool does with this command line" and + "what AutoControl sends" are joined rather than merely adjacent. +- **`mousemove --absolute` does not emit absolute events**, which is worth + knowing before trusting it. ydotool 1.x has no ABS axes on its device: it + sends `INT32_MIN` on both relative axes first, relies on the compositor + clamping that to the top-left corner, and then sends the target as a + relative delta. So `set_position` lands on the requested pixel *because of + that clamp*. The kernel side is now pinned; the clamp is the compositor's + behaviour and stays open. +- **The container gets `/dev/uinput` and character major 13, not + `--privileged`.** ydotoold creates its input node *after* the container + starts, which `--device` cannot cover, so the job grants + `--device-cgroup-rule 'c 13:* rmw'` and nothing else. + +### Scrolling on Wayland Stops Needing a uinput Daemon + +Motion, buttons and keys had already moved onto libei. Scroll was the one +input left shelling out to `ydotool` for every notch, and `Progress.md` said +why: the sign was a guess, and a scroll that goes the wrong way fails +silently. It is wired now, and the guess has been replaced with two +independent readings plus a measurement. + +- **The two paths count wheel detents in opposite directions.** This + repository's `wayland_scroll_direction_*` constants are in the kernel's + `REL_WHEEL` frame, because that is what ydotool writes into `/dev/uinput`: + positive is up. libei is in the `wl_pointer` / libinput frame, where + positive is down — libinput's own evdev reader negates `REL_WHEEL` to get + there, and the other libei sender that documents its sign (enigo) passes a + "positive scrolls down" value straight through to `scroll_discrete`. + Horizontal needs no flip: `REL_HWHEEL` and libinput both count right as + positive. So the vertical axis is negated on the way to libei and the + horizontal one is not, which is the decision `Progress.md` was waiting on. +- **The flip is checked against the real EIS server, negative value and + all.** A fifteenth check in `docker/eis_verify.py` drives the *public* + `mouse.scroll()` — not the backend method the earlier check drives — and + reads back `(0, -120)` for up, `(0, 120)` for down and `(120, 0)` for + right. It is also the only place a negative discrete value reaches the + wire; the earlier check only ever sent a positive one, so a marshalling + fault on the sign had nowhere to show up. +- **A refused emission now falls back to the CLI, which is what the code + always claimed.** `libei`'s module docstring says every failure raises + `LibeiUnavailable`, "which `keyboard` / `mouse` already treat as *use the + ydotool CLI*". Only the *connection* was treated that way. Once a backend + was handed over, a compositor that paused a device — or a session that + ended between two calls — raised straight out of `set_position`, + `press_key` or `hotkey`. Routing scroll through libei would have added a + fourth way for a script to die on a path that has a working fallback + sitting next to it, so the fallback was made real: a chord refused + part-way releases what it already pressed before handing over, so no + modifier is left held, and a button whose *release* is refused is released + by ydotool rather than staying down for the rest of the session. +- **`LibeiUnavailable` was escaping every containment boundary.** It + inherited `RuntimeError` alone, and `CLAUDE.md` is explicit that a + framework error which is not an `AutoControlException` "silently escapes + every boundary" — the executor, the background poll loops, the request + handlers, the GUI slots. It now inherits both, so the probes that catch + `RuntimeError` keep working and the boundaries finally see it. + +## What's new (2026-08-18) + +### The libei Input Path Now Has Something to Talk To + +The Wayland capture path was verified against a real compositor; the *input* +path was not, and was recorded as needing a GNOME VM. It does not. libeis is +the server side of libei's own protocol, Debian packages it, and the two +libraries will talk to each other over a plain Unix socket — so +`docker/eis_server.py` runs a real EIS implementation and +`docker/eis_verify.py` drives AutoControl's real sender against it. No +compositor, no desktop session, 14 checks, wired into CI. + +- **Discrete scroll was off by a factor of 120.** libei measures discrete + scroll in 120ths of a wheel click — the same convention as Windows' + `WHEEL_DELTA` — and `scroll()` was passing raw detent counts, so one click + asked for 1/120th of a scroll. libei says so at runtime ("suspicious + discrete event value 1, did you mean 120?"), which no mock was ever going to + print. `scroll(0, 1)` now arrives at the server as `(0, 120)`: one click, + right axis, right sign. This was the path `Progress.md` left deliberately + unwired because the *sign* was a guess; the sign turned out to be the + smaller half of the question. +- **The teardown no longer leaks a context per process.** `ei_unref` + segfaults on libei 1.3.901 — but only on a context whose backend opened and + whose handshake never progressed. With a peer to complete a handshake + against, the live case is finally testable, and it is safe. Teardown now + releases the devices and the context normally and abandons only the state + that actually crashes, instead of abandoning every opened backend on the + suspicion that it might. +- **The values a mock cannot check are checked.** The server offers six + capabilities and reads back what the client actually bound: exactly the four + AutoControl asks for, so both the `EI_DEVICE_CAP_*` bitmask and the variadic + `ei_seat_bind_capabilities` marshalling are right. Key codes, absolute + coordinates and button codes are read off the wire and compared. Every + emission is confirmed to carry a frame, and every device to open an + emulation transaction first — libei drops events from one that has not. +- **Two things measured but not ours to fix**, recorded rather than papered + over: `eis_device_pause()` puts nothing on the wire for a sender client on + libeis 1.3.901, so the client's `DEVICE_PAUSED` handling still has no peer + to exercise it; and the `start_emulating` sequence number does not survive + the trip (an explicit 4242 reads back as 0), so AutoControl's counter cannot + be checked from the far side. The check is written so that a libeis which + starts sending pauses will fail loudly if the client ignores them. + +### The Architecture Map's Line Counts Are Measured Again + +- **The map quoted the same subsystem at two different sizes.** `CLAUDE.md` + says every figure in `architecture_explore.md` is measured, but nothing + checked it, and two counting conventions had grown up side by side: the §5.4 + theme tables and the §5.4.17 file tables counted a phantom trailing line — + `len(text.split("\n"))` reports one line more than a file that ends in a + newline actually has, and one more *per file* for a package — while §1's + totals and the §8 appendix counted correctly. So `utils/executor/` was 8,811 + lines in one section and 9,001 in another, and the §8 column did not add up + to its own total. On top of that about fifty rows were simply stale, several + `####` headings were hundreds of lines out (`linux_wayland/` was still + quoted at 10 files / 1,093 lines against a real 14 / 2,235), and one theme + table had gained two subpackages its summary line never heard about. +- **413 figures re-measured** on one convention — `len(text.splitlines())`, + what `wc -l` reports and what `CLAUDE.md`'s own over-750-lines snippet + counts. §5.4, §5.4.17 and §8 now agree with each other for every subsystem, + and §8's rows sum to its stated total. +- **`test_doc_line_counts.py` is both the gate and the fix.** It fails CI when + any quoted line count stops matching the tree, naming the offending lines, + and rewrites all of them in place with `--fix`. The line counts were the one + part of the map with no gate — the command, MCP-tool, subpackage and example + counts already had `test_doc_counts.py` — which is exactly why they were the + part that drifted. + +### The Clipboard No Longer Fails Because Another Application Was Copying + +- **One process at a time may hold the Windows clipboard open, and every + clipboard call in AutoControl gave up the instant one did.** Explorer, + Office and every browser own the clipboard for a few milliseconds at a time + while they copy; `OpenClipboard` returns false for that whole window and all + six call sites — text, image, HTML, RTF, CSV, file drops, format + enumeration — turned it straight into `RuntimeError: OpenClipboard failed`. + Measured on a live desktop with a second process copying in a loop: about + one open in a thousand failed, which is a script that dies for no reason the + operator can see or reproduce. Win32 documents this as the condition to + retry, and a library whose job is driving a machine that other applications + are busy on cannot treat "somebody else was copying" as an error. +- **`win32_clipboard_api.open_clipboard()` is now the single place that opens + it**, waiting out a busy clipboard for roughly 200 ms before reporting + failure, and closing it however the block ends. The three modules that + hand-rolled the open/close pair — including the two that predated the shared + module — go through it, so the retry cannot be forgotten at a new call site. +- **The clipboard round-trip tests no longer depend on what the rest of the + machine is doing.** They exercise the real Win32 calls, which is the only + place the four historical writer bugs could ever be seen, so faking the + backend would have deleted the coverage instead of stabilising it. They now + read the Win32 clipboard sequence number instead: unchanged between the + write and the read means nothing else wrote in that window, so the assertion + is about AutoControl's code and nothing else. Verified against a process + making 4,112 competing clipboard writes during the run. + +### A Misspelled Command Name Is a 400, Not a 500 + +- **`POST /execute` answered `500 {"error": "execute_action failed"}` for an + `AC_*` name that does not exist**, which is the same answer it gives when + the server itself breaks. A client could not tell a typo in its own request + from an outage, and the message did not say which name was unrecognised. +- **Every command name is now checked before anything runs**, and an + unrecognised one comes back as `400` listing *all* of them in + `unknown_commands` — nested flow-control bodies included — so a client fixes + every typo in one round trip. `POST /execute_file` answers the same way for + a file that is unreadable, is not an action list, or names an unknown + command. The OpenAPI spec documents both, and states that a rejected request + executed nothing. +- Validation and collection share one traversal in `action_schema`, so there + is still exactly one definition of where a nested action list may hide. + +### A Segfault in the libei Teardown, and the Binding Checked Against the Real Library + +- **`ei_unref` crashes the process on libei 1.3.901 once a backend is open, + and the fallback path ran straight into it.** Every failure mode of the + libei handshake ends in `_teardown()`, so on any host where libei is + installed and the handshake does not complete, AutoControl died with + SIGSEGV instead of quietly using ydotool — the exact opposite of the + fail-closed promise. Measured one call at a time against the real library: + `ei_unref` is safe with no backend set up and safe after a *failed* setup, + and segfaults after a successful one. `ei_disconnect` crashes in the same + state, so it is not a refcounting mistake here; the header documents + `ei_unref` as correct for both outcomes, which makes this an upstream bug. + An opened backend is now abandoned rather than unreffed — a bounded leak of + one context per process, against a crash in a library that drives a desktop. + A sentinel in the verification re-checks the upstream state on every run and + says so when the workaround can go. +- **Every entry point the binding names is now resolved against the real + `libei.so`.** A misspelled symbol or a wrong `argtypes` sails past a mocked + symbol table and only surfaces on a user's machine; all 22 prototypes plus + the variadic `ei_seat_bind_capabilities` are checked for real. +- **The whole fail-closed chain runs end to end**: connect to a socket that + speaks no EI → handshake times out → `LibeiUnavailable` → `active_backend()` + returns None → `press_key` falls through to the ydotool CLI. +- **`liboeffis` is not packaged everywhere.** Arch and Fedora ship it; Debian + trixie does not. Without it there is no portal route, so `connect()` falls + back to the well-known EIS socket — which GNOME and KDE do not create. The + libei fast path is therefore unavailable on those systems, and says so + rather than looking mysteriously idle. + +### The Wayland Capture Path Now Meets a Real Compositor + +- **`screen.size()` reported one monitor while `grab_image()` returned the + whole layout.** The `wlr-randr` parser took the first `WxH` anywhere in the + document, which is the first output's current mode. On a two-monitor layout + that is half the screen — and the two are composed by the mss-shaped shim, + so the recorder, WebRTC and MCP monitor paths asked for a region half the + size of the screen and got it. `size()` is now the layout bounding box, from + a parser that reads every enabled output's mode *and* position. Found by + running against a real compositor; no mock had a second monitor. +- **`docker/Dockerfile.wayland` runs the backend under headless sway.** The + wlroots headless backend needs no GPU, no seat and no display, so a genuine + Wayland session fits in a container — and now in CI, as the + `wayland-verification` job. Two outputs are painted different solid colours, + because on a uniform screen a region grab cannot be caught reading the wrong + rectangle, and a red/blue swap cannot be caught at all. +- **21 checks that were previously mock-only now run against pixels the + compositor painted**: grim's argv and `-g` geometry, RGB channel order, + `wlr-randr`'s undocumented output format, `size()` / `grab_image()` / + `get_pixel()` / `screenshot()`, `je_auto_control.screenshot()`'s BGR output, + `grab_logical()` (the locator and OCR path), the mss shim, and `wtype`. +- **What the container cannot answer is stated rather than glossed over.** + ydotool needs `/dev/uinput` and headless sway consumes no libinput devices; + `xdg-desktop-portal-wlr` implements no RemoteDesktop, so there is no + `ConnectToEIS` to test. Both remain open in `Progress.md`. + +### libei Input, End to End + +- **The full portal handshake is implemented.** libei is not a + call-a-function-and-a-key-is-pressed library: a sender has to open an EIS + backend, bind a seat's capabilities, take a device *out of an event*, start + emulating on it, and follow every emission with `ei_device_frame` or nothing + is delivered. All of that now happens, so `press_key`, `set_position` and the + mouse buttons can emit without spawning a process per event. +- **The EIS socket comes from the desktop portal.** On GNOME and KDE it is not + a path on disk — it is a file descriptor handed over D-Bus by + `org.freedesktop.portal.RemoteDesktop.ConnectToEIS`, after a three-call + asynchronous session dance. No command-line tool can pass a file descriptor + into this process, so the `gdbus` route used for screenshots cannot work + here; `liboeffis` (which ships with libei for exactly this) does the dance. + Where liboeffis is absent, the well-known `$XDG_RUNTIME_DIR/eis-0` socket is + still tried. +- **Devices come from events, never from the context.** The previous binding + passed the `struct ei *` context to entry points that take a + `struct ei_device *` — pointer type confusion in a C library. That is now + impossible by construction. +- **A failed probe is paid once, not per keystroke.** The handshake involves a + portal round trip and possibly a consent dialog. The result is cached for the + process, so a host where libei is installed but unusable does not re-attempt + it on every key press. +- **Everything still falls back to ydotool.** Missing library, declined + consent, a partial capability grant, a paused device, a handshake that does + not complete — each raises `LibeiUnavailable`, which the keyboard and mouse + modules already treat as "use the CLI". Scroll deliberately stays on ydotool: + its direction convention is pinned by tests, and a wrong sign would fail + silently rather than loudly. +- **The ABI constants were checked against the upstream headers**, not + guessed. Three were wrong. `enum ei_device_capability` is a *bitmask*, so + `EI_DEVICE_CAP_KEYBOARD` is `1 << 2`, not 3; `OEFFIS_EVENT_CLOSED` comes + *before* `OEFFIS_EVENT_DISCONNECTED`; and `OEFFIS_DEVICE_ALL_DEVICES` is a + `= 0` sentinel rather than the OR of the device bits. The capability error + was the expensive one — no device would ever have reported the capability, + so every session would have timed out and silently used the CLI. The + verified values are now pinned by tests. The session also asks only for the + keyboard and pointer it actually drives, so the consent dialog does not + request a touchscreen grant nothing uses. + +### Wayland Capture Has a Floor Under It + +- **`xdg-desktop-portal` backs up the three CLI helpers.** None of `grim`, + `gnome-screenshot` or `spectacle` is guaranteed to be installed — GNOME has + not shipped `gnome-screenshot` by default since 42 — so + `org.freedesktop.portal.Screenshot` is tried last through `gdbus` rather than + giving up. It is awkward by nature: the portal returns a request handle and + answers later with a signal, so the listener starts before the call is made, + and the wait is bounded (30s) because a consent dialog can sit in front of it. +- **An operator can name their own capture command.** + `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` wins over + every detected tool. `{output}` becomes a temporary PNG path, substituted per + argument after `shlex.split` and run without a shell, so a path with spaces + stays one argument. This is the escape hatch for a setup none of the built-in + tiers fit — including one where our argv guess for a helper turns out wrong. +- **libei refuses a connection it cannot emit through.** The binding only ever + holds an `ei` context, while every device entry point takes an `ei_device`, + and it runs none of libei's seat / device / `start_emulating` / `frame` + handshake — so it could not deliver input, but *could* pass the wrong pointer + into a C library on a host that opens `$XDG_RUNTIME_DIR/eis-0`. It now stops + at `connect()` with an explanation. Callers already treated that as "use the + ydotool CLI", which is what every real desktop was doing anyway. +- **The portal listener cannot wedge on shutdown.** Its pipe is closed only + after the reader thread lets go of it; closing a stream out from under a + blocked `read()` can hang on the buffer lock, which would have turned a + timed-out capture into the hang the timeout exists to prevent. + +### The Container Image Builds and Starts From a Windows Checkout + +- **Every container built on a Windows clone died on startup.** `.gitattributes` + said `* text=auto`, so `docker/entrypoint.sh` and `docker/entrypoint-xfce.sh` + were checked out with CRLF. The shebang then reads `#!/bin/sh`, the kernel + looks for an interpreter whose name ends in a carriage return, and the image + builds perfectly and then exits with `exec /usr/local/bin/autocontrol-entrypoint: + no such file or directory` — a message that names the file it just failed to + find. CI never saw it: a Linux runner checks the same file out with LF. + `*.sh text eol=lf` now pins it, and a test asserts no entrypoint carries CRLF. +- **`.dockerignore` was in `docker/`, where Docker does not look.** Docker reads + it from the build *context* root, and every documented build passes the + repository root (`docker build -f docker/Dockerfile .`), so the exclusions did + nothing: `.git`, `.venv`, `test/` and the caches were all being shipped to the + daemon on every build. Moved to the root, where it takes effect. +- **The `mss` shim test measured the host's monitor, not its own fake.** + `test_screen_grabber.py` patched `backend_grab_image` but left + `_backend_screen_size` reading the real `platform_wrapper.screen`, so + `monitors[0]` reported whatever display the developer had. It passed on a + 1920x1080 desktop and failed under a 1280x800 Xvfb, for reasons unrelated to + the code under test. The fake now owns both halves of the seam. + ## What's new (2026-08-17) +### Wayland Sees the Screen + +- **Every capture path now goes through the platform backend.** `screenshot()`, + the image and anchor locators, OCR, smart waits, visual regression, screen + recording, the MCP monitor tools and remote desktop each reached for + `PIL.ImageGrab` or `mss` directly. Both read the X11 root window on Linux, + which under Wayland belongs to XWayland and does not composite native Wayland + windows. Pillow does fall back to `gnome-screenshot` / `grim` / `spectacle`, + but only inside `except OSError` around its X11 grab — so it fires when there + is no X display at all, and *not* while XWayland is up, which is the default + on GNOME, KDE and sway. `mss` has no fallback in any configuration. + `utils/cv2_utils/screen_grabber.py` is now the one place that decides how + pixels are read: a backend that publishes `grab_image` gets wrapped in + whichever library shape the caller already uses. Windows, macOS and Linux X11 + publish nothing and keep the real libraries, so their behaviour is + byte-for-byte unchanged. +- **Wayland capture covers three compositor families, not one.** `grim` only + speaks `wlr-screencopy`, which GNOME and KDE do not implement — so + `linux_wayland/capture.py` tries `grim` (sway, Hyprland, river), then + `gnome-screenshot`, then `spectacle`, and reports which one it used. Only + `grim` can take a region itself; the others capture the screen and the region + is cropped from it. +- **A missing capture tool fails loudly.** It raises with the install command + for each compositor rather than handing back an empty XWayland grab that + reads downstream as "template not found". The new `screen_capture` + diagnostics check names the tool in use before anything has to fail. +- **`screen.size()` and `get_pixel` work off GNOME/KDE too.** Resolution falls + back from `wlr-randr` to measuring a capture, and `get_pixel` crops a 1x1 + region from whichever capture path is available. + ### Which Program Owns That Window - **`foreground_window_process_id` / `window_process_id`** (`AC_foreground_window_pid`, diff --git a/architecture_explore.md b/architecture_explore.md index 6df61ba8..d2283838 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-08-16　**版本**：`pyproject.toml` version `0.0.195`　**分支**：`feat/desktop-automation-gaps` +> **掃描時間**：2026-08-19　**版本**：`pyproject.toml` version `0.0.218`　**分支**：`fix/clipboard-handles-and-window-input` --- @@ -19,14 +19,14 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 998 | -| 程式碼總行數 | 133,534 | +| Python 模組總數（含周邊子專案） | 1,016 | +| 程式碼總行數 | 137,497 | | `je_auto_control/utils/` 子套件數 | 308 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | -| 套件門面 `__all__` 公開名稱數 | 1,221 | +| 套件門面 `__all__` 公開名稱數 | 1,238 | | GUI 分頁數（`main_widget` 註冊） | 48 | | MCP 工具數（`build_default_tool_registry()` 實測） | 676 | -| `test_*.py` 測試檔／測試函式 | 458 / 4,319 | +| `test_*.py` 測試檔／測試函式 | 466 / 4,443 | | 範例腳本 | 27 | **技術基線**：Python ≥ 3.10、MIT 授權、必要相依只有 `je_open_cv`／`opencv-python`／`pillow`／`mss`／ @@ -71,7 +71,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 │ windows/ (ctypes Win32 + Interception 驅動) │ │ osx/ (pyobjc Quartz) │ │ linux_with_x11/ (python-Xlib + 選用 uinput) │ -│ linux_wayland/ (libei ctypes + ydotool/wtype/grim CLI) │ +│ linux_wayland/ (libei via portal/EIS + ydotool/wtype + 擷取工具分層) │ │ android/ (ADB + uiautomator2) │ ios/ (WebDriverAgent) │ └──────────────────────────────────────────────────────────────────────────┘ ``` @@ -97,6 +97,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 | **Observer** | `utils/callback/`、`utils/observer/`、`utils/triggers/` | 動作完成後觸發回呼；畫面出現／消失／變化與外部事件（webhook／IMAP／檔案）驅動腳本。 | | **Template Method** | `utils/generate_report/` | HTML／JSON／XML 三個產生器共用「收集紀錄 → 格式化 → 寫檔」骨架，各自實作渲染。 | | **Adapter / Backend seam** | `accessibility/backends/`、`ocr/backends/`、`vision/backends/`、`llm/backends/`、`agent/backends/`、`hotkey/backends/`、`usb/passthrough/*_backend.py`、`usbip/backend.py` | 每個外部能力都有抽象基底 + 具體實作 + null fallback，讓無相依環境仍可載入與測試。 | +| **Adapter（螢幕擷取）** | `utils/cv2_utils/screen_grabber.py` | 全框架唯一決定「這台機器怎麼讀螢幕」的地方。平台後端若發布 `grab_image`（目前只有 Wayland 需要），就把它包成呼叫端已在用的形狀（`ImageGrab` 或 `mss`）；否則原樣交還真正的函式庫，Windows／macOS／X11 行為完全不變。後端另可發布 `layout_origin`，由 `backend_layout_origin()` 轉給需要把畫面上的點換回螢幕座標的路徑（`grab_logical`、mss shim 的 monitor 矩形）。 | | **Registry / Singleton** | `remote_desktop/registry.py`、`rest_api/rest_registry.py`、`profiler`、`run_history`、`secrets` | 行程級單例，讓 `AC_*` 指令能操作長生命週期的伺服器與狀態。 | --- @@ -151,13 +152,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `je_auto_control/__init__.py` | 1,927 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | -| `je_auto_control/__main__.py` | 71 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | -| `je_auto_control/cli.py` | 327 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | -| `je_auto_control/api/__init__.py` | 23 | 版本化整合進入點。 | -| `je_auto_control/api/core.py` | 20 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | -| `je_auto_control/utils/deprecation.py` | 36 | 公開 API 的一致性棄用警告。 | -| `je_auto_control/utils/http_headers.py` | 33 | 入站 HTTP 標頭的共用防禦式解析。 | +| `je_auto_control/__init__.py` | 1,970 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | +| `je_auto_control/__main__.py` | 70 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | +| `je_auto_control/cli.py` | 326 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | +| `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | +| `je_auto_control/utils/deprecation.py` | 35 | 公開 API 的一致性棄用警告。 | +| `je_auto_control/utils/http_headers.py` | 32 | 入站 HTTP 標頭的共用防禦式解析。 | ### 5.2 wrapper 抽象層 @@ -165,21 +166,21 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `wrapper/platform_wrapper.py` | 60 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | -| `wrapper/_platform_windows.py` | 326 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 150 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | -| `wrapper/_platform_linux.py` | 268 | X11 後端組裝（python-Xlib + 選用 uinput）。 | -| `wrapper/_platform_wayland.py` | 58 | Wayland 後端組裝（libei／ydotool／grim）。 | +| `wrapper/platform_wrapper.py` | 59 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | +| `wrapper/_platform_windows.py` | 325 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | +| `wrapper/_platform_osx.py` | 149 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | +| `wrapper/_platform_wayland.py` | 57 | Wayland 後端組裝（libei／ydotool／grim）。 | | `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | | `wrapper/auto_control_keyboard.py` | 273 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | -| `wrapper/auto_control_screen.py` | 98 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | +| `wrapper/auto_control_screen.py` | 97 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | -| `wrapper/auto_control_record.py` | 76 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | +| `wrapper/auto_control_record.py` | 106 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | | `wrapper/auto_control_window.py` | 293 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | ### 5.3 平台後端 -#### Windows（`windows/`，26 檔／1,939 行） +#### Windows（`windows/`，23 檔／1,995 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -197,7 +198,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | | `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | -#### macOS（`osx/`，17 檔／773 行） +#### macOS（`osx/`，17 檔／761 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -210,7 +211,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | -#### Linux X11（`linux_with_x11/`，19 檔／1,189 行） +#### Linux X11（`linux_with_x11/`，19 檔／1,175 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -225,30 +226,37 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，10 檔／1,093 行） +#### Linux Wayland（`linux_wayland/`，17 檔／3,416 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `_detect.py` | 66 | Wayland session 偵測與 CLI 工具探測。 | -| `_select_input.py` | 50 | 決定使用原生 libei 或 CLI shim。 | -| `libei.py` | 243 | libei（Wayland HID 層輸入模擬）的 ctypes 綁定。 | -| `mouse.py` | 168 | 經 ydotool 的滑鼠後端。 | -| `keyboard.py` | 149 | 經 ydotool + wtype 的鍵盤後端。 | -| `keymap.py` | 156 | 友善鍵名 → evdev key code。 | -| `screen.py` | 136 | 經 grim + wlr-randr 的螢幕後端。 | -| `listener.py` / `record.py` | 49 / 35 | 監聽與錄製 stub（Wayland 限制）。 | +| `_detect.py` | 77 | Wayland session 偵測與 CLI 工具探測。 | +| `_ydotool_cli.py` | 134 | 判定安裝的是哪一代 ydotool 命令列，擋掉會靜默失效的 0.1.x（對本專案送的 argv 回傳 0 卻不送任何事件）。 | +| `_ctypes_bind.py` | 75 | libei／liboeffis 共用的 ctypes 載入與 prototype 綁定。 | +| `_dbus_client.py` | 624 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | +| `_select_input.py` | 85 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | +| `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | +| `oeffis.py` | 196 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | +| `libei.py` | 610 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | +| `mouse.py` | 379 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | +| `keyboard.py` | 173 | 鍵盤後端：libei 優先，退回 ydotool／wtype。 | +| `keymap.py` | 155 | 友善鍵名 → evdev key code。 | +| `capture.py` | 236 | 擷取分層：操作者自訂指令 → grim → gnome-screenshot → spectacle → portal。 | +| `portal.py` | 207 | `org.freedesktop.portal.Screenshot` 最後備援,經 `_dbus_client` 直接講 D-Bus（不再需要安裝 `gdbus`,只要有 session bus）。 | +| `screen.py` | 252 | 螢幕後端；發布 `grab_image` 與 `layout_origin`（擷取畫面左上角的版面座標，有螢幕在主螢幕左側／上方時為負），全框架的擷取都經由它。 | +| `listener.py` / `record.py` | 48 / 34 | 監聽與錄製 stub（Wayland 限制）。 | #### 行動裝置 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `android/adb_client.py` | 184 | `adb` CLI 的薄封裝。 | -| `android/client.py` | 92 | `uiautomator2.Device` 的延遲封裝。 | -| `android/find.py` | 105 | uiautomator2 widget 樹的元素查詢。 | -| `ios/client.py` | 95 | `facebook-wda`（WebDriverAgent）封裝。 | -| `ios/find.py` | 87 | XCUITest 無障礙查詢。 | -| `ios/input.py` | 47 | iOS 觸控與按鍵原語。 | -| `ios/screen.py` | 33 | iOS 裝置螢幕擷取與尺寸。 | +| `android/adb_client.py` | 183 | `adb` CLI 的薄封裝。 | +| `android/client.py` | 91 | `uiautomator2.Device` 的延遲封裝。 | +| `android/find.py` | 104 | uiautomator2 widget 樹的元素查詢。 | +| `ios/client.py` | 94 | `facebook-wda`（WebDriverAgent）封裝。 | +| `ios/find.py` | 86 | XCUITest 無障礙查詢。 | +| `ios/input.py` | 46 | iOS 觸控與按鍵原語。 | +| `ios/screen.py` | 32 | iOS 裝置螢幕擷取與尺寸。 | ### 5.4 能力層 `utils/`（308 個子套件） @@ -257,505 +265,513 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,443 行。 +> 24 個套件、約 12,650 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/action_lint/` | 332 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | -| `utils/action_signing/` | 232 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | -| `utils/checkpoint/` | 117 | 流程檢查點與續跑，讓長 action list 具持久性 | -| `utils/codegen/` | 159 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | -| `utils/dag/` | 478 | 跨主機 DAG 編排器（圖模型 + runner） | -| `utils/decision_table/` | 105 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | -| `utils/deterministic/` | 98 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 8,931 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | -| `utils/flow_debugger/` | 138 | action list 的單步除錯器與追蹤器 | -| `utils/input_macro/` | 129 | 定時輸入事件重播與宣告式輸入序列 DSL | -| `utils/json/` | 75 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | -| `utils/json_store/` | 63 | JSON 字典檔持久化的共用小工具（內部管線） | -| `utils/loop_guard/` | 142 | 機械式卡死迴圈偵測（agent loop 用） | -| `utils/plugin_loader/` | 87 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | -| `utils/plugin_sdk/` | 70 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | -| `utils/project/` | 186 | 專案腳手架：建立目錄結構與範本 action 檔 | -| `utils/recording_edit/` | 152 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | -| `utils/saga/` | 95 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | -| `utils/script_vars/` | 193 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | -| `utils/skill_library/` | 118 | 具名可重用 action 序列（skill）的持久化倉庫 | -| `utils/state_machine/` | 183 | 宣告式有限狀態機驅動 action JSON | -| `utils/stubs/` | 239 | 為 `AC_*` 指令面產生型別 stub | -| `utils/test_record/` | 65 | 全域測試紀錄單例，記錄每個動作的參數與例外 | -| `utils/work_queue/` | 176 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | +| `utils/action_lint/` | 328 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | +| `utils/action_signing/` | 229 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/checkpoint/` | 115 | 流程檢查點與續跑，讓長 action list 具持久性 | +| `utils/codegen/` | 157 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | +| `utils/dag/` | 475 | 跨主機 DAG 編排器（圖模型 + runner） | +| `utils/decision_table/` | 103 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | +| `utils/deterministic/` | 96 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | +| `utils/executor/` | 9,070 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/flow_debugger/` | 136 | action list 的單步除錯器與追蹤器 | +| `utils/input_macro/` | 127 | 定時輸入事件重播與宣告式輸入序列 DSL | +| `utils/json/` | 74 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | +| `utils/json_store/` | 61 | JSON 字典檔持久化的共用小工具（內部管線） | +| `utils/loop_guard/` | 140 | 機械式卡死迴圈偵測（agent loop 用） | +| `utils/plugin_loader/` | 85 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | +| `utils/plugin_sdk/` | 68 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | +| `utils/project/` | 182 | 專案腳手架：建立目錄結構與範本 action 檔 | +| `utils/recording_edit/` | 150 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | +| `utils/saga/` | 93 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | +| `utils/script_vars/` | 190 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | +| `utils/skill_library/` | 116 | 具名可重用 action 序列（skill）的持久化倉庫 | +| `utils/state_machine/` | 181 | 宣告式有限狀態機驅動 action JSON | +| `utils/stubs/` | 236 | 為 `AC_*` 指令面產生型別 stub | +| `utils/test_record/` | 64 | 全域測試紀錄單例，記錄每個動作的參數與例外 | +| `utils/work_queue/` | 174 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 12 個套件、約 1,848 行。 +> 12 個套件、約 1,881 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/callback/` | 201 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | -| `utils/config_bundle/` | 402 | 使用者設定的單檔匯出／匯入 | -| `utils/critical_exit/` | 98 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | -| `utils/diagnostics/` | 272 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | -| `utils/exception/` | 186 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | -| `utils/failure_bundle/` | 189 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | -| `utils/file_process/` | 27 | 目錄檔案列舉（`execute_dir` 的後端） | -| `utils/logging/` | 72 | `autocontrol_logger` 單例 + 輪替檔案 handler | -| `utils/package_manager/` | 99 | 動態載入套件並把 executor 注入其中 | -| `utils/path_guard/` | 101 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | -| `utils/shell_process/` | 161 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | -| `utils/start_exe/` | 40 | 啟動另一個執行檔行程 | +| `utils/callback/` | 200 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | +| `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | +| `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | +| `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | +| `utils/exception/` | 194 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | +| `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | +| `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | +| `utils/package_manager/` | 98 | 動態載入套件並把 executor 注入其中 | +| `utils/path_guard/` | 99 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | +| `utils/shell_process/` | 159 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | +| `utils/start_exe/` | 39 | 啟動另一個執行檔行程 | ### 5.4.3 排程、觸發與背景監看 -> 11 個套件、約 3,574 行。 +> 11 個套件、約 3,544 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/hotkey/` | 734 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | -| `utils/idle_keepawake/` | 214 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | -| `utils/lock_session/` | 165 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | -| `utils/observer/` | 222 | 反應式畫面觀察者，在出現／消失／變化時觸發 | -| `utils/recurrence/` | 326 | RFC 5545 重複規則解析與發生時間展開 | -| `utils/scheduler/` | 355 | 間隔式與 cron 式的 action JSON 排程器 | -| `utils/session_guard/` | 64 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | -| `utils/triggers/` | 1,150 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | -| `utils/voice/` | 89 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | -| `utils/watchdog/` | 175 | 背景彈窗／中斷看門狗，供無人值守自動化 | -| `utils/watcher/` | 80 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | +| `utils/hotkey/` | 727 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | +| `utils/idle_keepawake/` | 212 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | +| `utils/lock_session/` | 163 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | +| `utils/observer/` | 220 | 反應式畫面觀察者，在出現／消失／變化時觸發 | +| `utils/recurrence/` | 324 | RFC 5545 重複規則解析與發生時間展開 | +| `utils/scheduler/` | 352 | 間隔式與 cron 式的 action JSON 排程器 | +| `utils/session_guard/` | 62 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | +| `utils/triggers/` | 1,146 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | +| `utils/voice/` | 87 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | +| `utils/watchdog/` | 173 | 背景彈窗／中斷看門狗，供無人值守自動化 | +| `utils/watcher/` | 78 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | ### 5.4.4 輸入模擬與動作品質 -> 20 個套件、約 2,325 行。 +> 22 個套件、約 2,610 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/act_in_view/` | 78 | 先把目標捲進視野，待其可操作後再動作 | -| `utils/act_modes/` | 69 | actionability 閘門之上的 trial／force 動作模式 | -| `utils/action_effect/` | 112 | 判定一個動作是否真的產生效果，並歸因到目標區域 | -| `utils/action_grounding/` | 82 | 動作前的接地守衛（邊界檢查 + 吸附到元素） | -| `utils/actionability/` | 165 | 動作前就緒閘門（可見 + 穩定 + 啟用 + 未被遮擋） | -| `utils/ensure_state/` | 74 | 冪等地把控制項／設定帶到期望狀態 | -| `utils/field_entry/` | 78 | 清空再輸入的欄位填寫慣用法（Playwright `fill`） | -| `utils/gamepad/` | 313 | 虛擬遊戲手把後端（Windows ViGEmBus 驅動） | -| `utils/humanize/` | 186 | 擬人輸入：貝茲曲線滑鼠路徑 + 抖動打字節奏 | -| `utils/ime_state/` | 146 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | -| `utils/key_hold/` | 109 | 按住按鍵一段時間，或以固定頻率自動重複 | -| `utils/modifier_state/` | 78 | 跨一組動作按住修飾鍵，並保證安全釋放 | -| `utils/mouse_path/` | 94 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | -| `utils/mouse_relative/` | 61 | 相對位移滑鼠移動 | -| `utils/postcondition/` | 140 | 宣告式的動作預期結果規格，對照畫面驗證 | -| `utils/step_repair/` | 116 | 失敗／無效動作的修復策略（自我修正迴圈） | -| `utils/table_grid_fill/` | 143 | 以 OCR 文字填滿格線表格，取得可定址的表格 | +| `utils/act_in_view/` | 76 | 先把目標捲進視野，待其可操作後再動作 | +| `utils/act_modes/` | 67 | actionability 閘門之上的 trial／force 動作模式 | +| `utils/action_effect/` | 110 | 判定一個動作是否真的產生效果，並歸因到目標區域 | +| `utils/action_grounding/` | 80 | 動作前的接地守衛（邊界檢查 + 吸附到元素） | +| `utils/actionability/` | 163 | 動作前就緒閘門（可見 + 穩定 + 啟用 + 未被遮擋） | +| `utils/ensure_state/` | 72 | 冪等地把控制項／設定帶到期望狀態 | +| `utils/field_entry/` | 76 | 清空再輸入的欄位填寫慣用法（Playwright `fill`） | +| `utils/gamepad/` | 311 | 虛擬遊戲手把後端（Windows ViGEmBus 驅動） | +| `utils/humanize/` | 183 | 擬人輸入：貝茲曲線滑鼠路徑 + 抖動打字節奏 | +| `utils/ime_state/` | 144 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | +| `utils/key_hold/` | 107 | 按住按鍵一段時間，或以固定頻率自動重複 | +| `utils/modifier_state/` | 76 | 跨一組動作按住修飾鍵，並保證安全釋放 | +| `utils/mouse_path/` | 92 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | +| `utils/mouse_relative/` | 59 | 相對位移滑鼠移動 | +| `utils/postcondition/` | 138 | 宣告式的動作預期結果規格，對照畫面驗證 | +| `utils/step_repair/` | 114 | 失敗／無效動作的修復策略（自我修正迴圈） | +| `utils/table_grid_fill/` | 141 | 以 OCR 文字填滿格線表格，取得可定址的表格 | | `utils/input_reach/` | 111 | 送出去的輸入到不到得了：桌面鎖定查詢（免費）＋ 實際送一個 F13 確認沒有被過濾（有副作用，只給診斷用） | | `utils/keyboard_layout/` | 148 | 向系統問「這個鍵盤配置下每個鍵印出什麼字」（`ToUnicodeEx`），問不到退回 US 對照表 | | `utils/text_unicode/` | 135 | 輸入任意 Unicode（emoji／CJK／重音字）：優先送字元按鍵事件，不支援時退回剪貼簿貼上 | -| `utils/tween_drag/` | 97 | 沿曲線的緩動插值拖曳 | -| `utils/verify_field/` | 114 | 打字後讀回欄位，確認內容確實落地 | +| `utils/tween_drag/` | 95 | 沿曲線的緩動插值拖曳 | +| `utils/verify_field/` | 112 | 打字後讀回欄位，確認內容確實落地 | ### 5.4.5 影像辨識與畫面分析 -> 37 個套件、約 4,572 行。 +> 37 個套件、約 4,999 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/annotate/` | 109 | 截圖標註：畫框、highlight、箭頭、標籤 | -| `utils/barcode/` | 55 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | -| `utils/color_match/` | 105 | 在 HSV 通道上做顏色感知的樣板比對 | -| `utils/color_region/` | 81 | 以顏色定位畫面區域（遮罩 + 連通元件） | -| `utils/color_stats/` | 91 | 區域顏色統計：平均色與主色 | -| `utils/coordinate_space/` | 86 | 模型網格座標與實體像素之間的座標空間對映 | -| `utils/cv2_utils/` | 355 | OpenCV 基礎層：截圖（mss/Pillow）、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | -| `utils/edge_lines/` | 122 | 以 Hough 轉換偵測線條／格線／分隔線 | -| `utils/edge_match/` | 114 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | -| `utils/feature_match/` | 131 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | -| `utils/hsv_segment/` | 93 | HSV 色彩空間分割（抗光照的顏色遮罩 + blob 框） | -| `utils/icon_classify/` | 115 | 從像素形狀判斷一個框是哪一類元件 | -| `utils/image_dedup/` | 85 | 感知雜湊影像去重（Pillow aHash/dHash） | -| `utils/image_quality/` | 79 | 在 OCR／比對前評分影像品質（銳利度／對比／亮度） | -| `utils/img_histogram/` | 101 | 顏色直方圖指紋與變化偵測（抗光照） | -| `utils/marks_layout/` | 126 | Set-of-Marks 標籤的不重疊排版與可讀配色 | -| `utils/match_autothresh/` | 107 | Otsu 自動門檻，免去手動調 `min_score` | -| `utils/match_ensemble/` | 65 | 多樣板共識比對（多張參考圖投票到同一位置） | -| `utils/match_stability/` | 70 | 比對前的靜止閘門與跨影格的比對持續性 | -| `utils/match_trust/` | 138 | 樣板比對可信度評分（次峰比 + peak-to-sidelobe） | -| `utils/monitor_layout/` | 290 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | -| `utils/motion_regions/` | 75 | 兩影格間的局部變化／活動偵測（absdiff） | -| `utils/perceptual_diff/` | 102 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | -| `utils/preprocess/` | 187 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | -| `utils/qr/` | 55 | 從影像或螢幕區域解碼 QR code（OpenCV） | -| `utils/rotated_match/` | 147 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | -| `utils/saliency/` | 109 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | -| `utils/scale_detect/` | 86 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | -| `utils/screen_grid/` | 145 | 供 VLM 接地用的粗粒度標號網格（點 ↔ 格對映） | -| `utils/set_of_marks/` | 152 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | -| `utils/shape_locator/` | 107 | 以邊緣／輪廓偵測定位元件（矩形／形狀，免樣板） | -| `utils/ssim/` | 142 | 結構相似度比較：感知分數 + 變化區域 | -| `utils/subpixel_match/` | 103 | 以二次曲面擬合做次像素級比對精修 | -| `utils/theme_normalize/` | 94 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | -| `utils/video_report/` | 135 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | +| `utils/annotate/` | 107 | 截圖標註：畫框、highlight、箭頭、標籤 | +| `utils/barcode/` | 53 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | +| `utils/color_match/` | 103 | 在 HSV 通道上做顏色感知的樣板比對 | +| `utils/color_region/` | 79 | 以顏色定位畫面區域（遮罩 + 連通元件） | +| `utils/color_stats/` | 89 | 區域顏色統計：平均色與主色 | +| `utils/coordinate_space/` | 84 | 模型網格座標與實體像素之間的座標空間對映 | +| `utils/cv2_utils/` | 592 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | +| `utils/edge_lines/` | 120 | 以 Hough 轉換偵測線條／格線／分隔線 | +| `utils/edge_match/` | 112 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | +| `utils/feature_match/` | 129 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | +| `utils/hsv_segment/` | 91 | HSV 色彩空間分割（抗光照的顏色遮罩 + blob 框） | +| `utils/icon_classify/` | 113 | 從像素形狀判斷一個框是哪一類元件 | +| `utils/image_dedup/` | 83 | 感知雜湊影像去重（Pillow aHash/dHash） | +| `utils/image_quality/` | 77 | 在 OCR／比對前評分影像品質（銳利度／對比／亮度） | +| `utils/img_histogram/` | 99 | 顏色直方圖指紋與變化偵測（抗光照） | +| `utils/marks_layout/` | 124 | Set-of-Marks 標籤的不重疊排版與可讀配色 | +| `utils/match_autothresh/` | 105 | Otsu 自動門檻，免去手動調 `min_score` | +| `utils/match_ensemble/` | 63 | 多樣板共識比對（多張參考圖投票到同一位置） | +| `utils/match_stability/` | 68 | 比對前的靜止閘門與跨影格的比對持續性 | +| `utils/match_trust/` | 136 | 樣板比對可信度評分（次峰比 + peak-to-sidelobe） | +| `utils/monitor_layout/` | 317 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | +| `utils/motion_regions/` | 73 | 兩影格間的局部變化／活動偵測（absdiff） | +| `utils/perceptual_diff/` | 100 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | +| `utils/preprocess/` | 185 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | +| `utils/qr/` | 53 | 從影像或螢幕區域解碼 QR code（OpenCV） | +| `utils/rotated_match/` | 145 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | +| `utils/saliency/` | 107 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | +| `utils/scale_detect/` | 84 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | +| `utils/screen_grid/` | 143 | 供 VLM 接地用的粗粒度標號網格（點 ↔ 格對映） | +| `utils/set_of_marks/` | 150 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | +| `utils/shape_locator/` | 105 | 以邊緣／輪廓偵測定位元件（矩形／形狀，免樣板） | +| `utils/ssim/` | 140 | 結構相似度比較：感知分數 + 變化區域 | +| `utils/subpixel_match/` | 101 | 以二次曲面擬合做次像素級比對精修 | +| `utils/theme_normalize/` | 92 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | +| `utils/video_report/` | 133 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | | `utils/visual_match/` | 427 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | -| `utils/visual_regression/` | 218 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | +| `utils/visual_regression/` | 217 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | ### 5.4.6 OCR 與文字理解 -> 19 個套件、約 3,098 行。 +> 19 個套件、約 3,180 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/bidi_check/` | 118 | 雙向文字 QA（bidi 控制碼、巢狀平衡、Trojan-source 掃描） | -| `utils/column_layout/` | 152 | 從垂直空白推斷欄位，處理無框線表格 | -| `utils/confusables/` | 114 | 易混淆／同形字偵測（Unicode 欺騙骨架） | -| `utils/form_fields/` | 130 | 多方向關聯表單標籤與值，並讀取核取方塊狀態 | -| `utils/fuzzy/` | 96 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | -| `utils/grid_locator/` | 73 | 以 (row, column) 從邊界框定址表格／網格儲存格 | -| `utils/guardrail/` | 110 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | -| `utils/heading_segment/` | 71 | 判定 OCR 行是標題或內文，建出文件大綱 | -| `utils/near_dup/` | 107 | 近似重複文字偵測（SimHash／MinHash） | -| `utils/ocr/` | 1,105 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | -| `utils/pii_text/` | 100 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | -| `utils/readability/` | 139 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | -| `utils/reading_flow/` | 121 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | -| `utils/search_index/` | 142 | 記憶體內 BM25／TF-IDF 全文檢索 | -| `utils/text_blocks/` | 90 | 把 OCR 行組成段落與項目符號／編號清單 | -| `utils/text_diff/` | 150 | unified diff 產生、套用與三方合併 | -| `utils/text_normalize/` | 65 | Unicode 正規化與 slug 產生 | -| `utils/text_regions/` | 159 | 免模型的畫面文字區域偵測（MSER）：區域與行 | -| `utils/text_similarity/` | 167 | 字串距離度量（文字比對用） | +| `utils/bidi_check/` | 116 | 雙向文字 QA（bidi 控制碼、巢狀平衡、Trojan-source 掃描） | +| `utils/column_layout/` | 150 | 從垂直空白推斷欄位，處理無框線表格 | +| `utils/confusables/` | 112 | 易混淆／同形字偵測（Unicode 欺騙骨架） | +| `utils/form_fields/` | 128 | 多方向關聯表單標籤與值，並讀取核取方塊狀態 | +| `utils/fuzzy/` | 94 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | +| `utils/grid_locator/` | 71 | 以 (row, column) 從邊界框定址表格／網格儲存格 | +| `utils/guardrail/` | 108 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | +| `utils/heading_segment/` | 69 | 判定 OCR 行是標題或內文，建出文件大綱 | +| `utils/near_dup/` | 105 | 近似重複文字偵測（SimHash／MinHash） | +| `utils/ocr/` | 1,112 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | +| `utils/pii_text/` | 98 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | +| `utils/readability/` | 137 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | +| `utils/reading_flow/` | 119 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | +| `utils/search_index/` | 140 | 記憶體內 BM25／TF-IDF 全文檢索 | +| `utils/text_blocks/` | 88 | 把 OCR 行組成段落與項目符號／編號清單 | +| `utils/text_diff/` | 148 | unified diff 產生、套用與三方合併 | +| `utils/text_normalize/` | 63 | Unicode 正規化與 slug 產生 | +| `utils/text_regions/` | 157 | 免模型的畫面文字區域偵測（MSER）：區域與行 | +| `utils/text_similarity/` | 165 | 字串距離度量（文字比對用） | ### 5.4.7 無障礙樹與原生控制項 -> 16 個套件、約 3,287 行。 +> 16 個套件、約 3,851 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/a11y_audit/` | 358 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | +| `utils/a11y_audit/` | 355 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | | `utils/accessibility/` | 2,390 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | -| `utils/ax_events/` | 31 | 反應式 UIA 事件等待（focus-changed） | -| `utils/ax_props/` | 46 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | -| `utils/ax_text/` | 104 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | -| `utils/ax_tree_walk/` | 120 | 可讀、可定址的無障礙樹後處理（角色名 + 節點路徑） | -| `utils/contrast_map/` | 122 | 取樣實際顏色以評定畫面文字的可讀性（WCAG） | -| `utils/control_patterns/` | 90 | 延伸 UIA 控制項模式動作（Expand／Select／Range／Scroll） | -| `utils/cvd_simulate/` | 127 | 模擬色覺缺陷並標示在該狀況下會撞色的顏色 | -| `utils/element_repository/` | 107 | 原生 UI 元素的具名定位器倉庫（object repository） | -| `utils/focus_order/` | 97 | 鍵盤焦點順序：預期 Tab 序列、WCAG 稽核與設定焦點 | -| `utils/legacy_accessible/` | 47 | MSAA 橋接，處理 UIA 無法建模的舊控制項 | -| `utils/selection_view/` | 59 | 容器選取狀態與檢視切換（Selection／MultipleView 模式） | -| `utils/table_pattern/` | 67 | 原生表格的表頭與儲存格定址（UIA TablePattern／GridItem） | -| `utils/transform_window/` | 72 | 以 UIA Transform／Window 模式移動、調整大小與視窗狀態 | -| `utils/virtualized/` | 45 | 實體化虛擬化清單／網格中的離屏項目 | +| `utils/ax_events/` | 29 | 反應式 UIA 事件等待（focus-changed） | +| `utils/ax_props/` | 44 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | +| `utils/ax_text/` | 102 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | +| `utils/ax_tree_walk/` | 118 | 可讀、可定址的無障礙樹後處理（角色名 + 節點路徑） | +| `utils/contrast_map/` | 120 | 取樣實際顏色以評定畫面文字的可讀性（WCAG） | +| `utils/control_patterns/` | 88 | 延伸 UIA 控制項模式動作（Expand／Select／Range／Scroll） | +| `utils/cvd_simulate/` | 125 | 模擬色覺缺陷並標示在該狀況下會撞色的顏色 | +| `utils/element_repository/` | 105 | 原生 UI 元素的具名定位器倉庫（object repository） | +| `utils/focus_order/` | 95 | 鍵盤焦點順序：預期 Tab 序列、WCAG 稽核與設定焦點 | +| `utils/legacy_accessible/` | 45 | MSAA 橋接，處理 UIA 無法建模的舊控制項 | +| `utils/selection_view/` | 57 | 容器選取狀態與檢視切換（Selection／MultipleView 模式） | +| `utils/table_pattern/` | 65 | 原生表格的表頭與儲存格定址（UIA TablePattern／GridItem） | +| `utils/transform_window/` | 70 | 以 UIA Transform／Window 模式移動、調整大小與視窗狀態 | +| `utils/virtualized/` | 43 | 實體化虛擬化清單／網格中的離屏項目 | ### 5.4.8 元素定位、自我修復與智慧等待 -> 23 個套件、約 4,044 行。 +> 23 個套件、約 3,995 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/ab_locator/` | 339 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | -| `utils/adaptive_timeout/` | 86 | 由觀測到的步驟耗時推導等待逾時，而非硬猜 | -| `utils/anchor_locator/` | 440 | 錨點定位器：以空間關係組合 影像／OCR／VLM／a11y 四種來源 | -| `utils/app_idle/` | 110 | 等應用程式不再忙碌，再驅動下一步 | -| `utils/change_localize/` | 82 | 把畫面變化歸因到實際改變的元素框 | -| `utils/critic_features/` | 87 | 每步的 critic 特徵集合與規則式步驟評分 | -| `utils/element_diff/` | 90 | 跨影格的幾何感知元素比對（穩定 ID、移動追蹤） | -| `utils/element_parse/` | 108 | 融合並排序畫面元素框（IoU、合併、多來源融合、閱讀順序） | -| `utils/element_proposal/` | 88 | 免樣板、免模型地從原始像素提出乾淨元素清單 | -| `utils/element_scoring/` | 107 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | -| `utils/expect_poll/` | 139 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | -| `utils/grounding_consensus/` | 129 | 對同一目標的多個接地提案做自我一致性投票 | -| `utils/heal_analytics/` | 79 | 自癒事件記錄的分析（治癒率、脆弱定位器） | -| `utils/locator_chain/` | 114 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | -| `utils/locator_repair/` | 119 | 自癒回寫：把修正後的定位器持久化 | -| `utils/observation/` | 94 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | -| `utils/observation_delta/` | 105 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | -| `utils/screen_state/` | 145 | 語義畫面狀態：快照／差異與結構化畫面描述 | -| `utils/scroll_find/` | 86 | 捲動直到目標影像／文字可見 | -| `utils/self_healing/` | 345 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | -| `utils/semantic_recording/` | 427 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | -| `utils/settle_detector/` | 78 | 以純函式介面判定 UI 是否已靜止 | -| `utils/smart_waits/` | 647 | 智慧等待：以影格差異取代 `time.sleep` | +| `utils/ab_locator/` | 336 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | +| `utils/adaptive_timeout/` | 84 | 由觀測到的步驟耗時推導等待逾時，而非硬猜 | +| `utils/anchor_locator/` | 438 | 錨點定位器：以空間關係組合 影像／OCR／VLM／a11y 四種來源 | +| `utils/app_idle/` | 108 | 等應用程式不再忙碌，再驅動下一步 | +| `utils/change_localize/` | 80 | 把畫面變化歸因到實際改變的元素框 | +| `utils/critic_features/` | 85 | 每步的 critic 特徵集合與規則式步驟評分 | +| `utils/element_diff/` | 88 | 跨影格的幾何感知元素比對（穩定 ID、移動追蹤） | +| `utils/element_parse/` | 106 | 融合並排序畫面元素框（IoU、合併、多來源融合、閱讀順序） | +| `utils/element_proposal/` | 86 | 免樣板、免模型地從原始像素提出乾淨元素清單 | +| `utils/element_scoring/` | 105 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | +| `utils/expect_poll/` | 137 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | +| `utils/grounding_consensus/` | 127 | 對同一目標的多個接地提案做自我一致性投票 | +| `utils/heal_analytics/` | 77 | 自癒事件記錄的分析（治癒率、脆弱定位器） | +| `utils/locator_chain/` | 112 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | +| `utils/locator_repair/` | 117 | 自癒回寫：把修正後的定位器持久化 | +| `utils/observation/` | 92 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | +| `utils/observation_delta/` | 103 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | +| `utils/screen_state/` | 143 | 語義畫面狀態：快照／差異與結構化畫面描述 | +| `utils/scroll_find/` | 84 | 捲動直到目標影像／文字可見 | +| `utils/self_healing/` | 342 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | +| `utils/semantic_recording/` | 423 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | +| `utils/settle_detector/` | 76 | 以純函式介面判定 UI 是否已靜止 | +| `utils/smart_waits/` | 646 | 智慧等待：以影格差異取代 `time.sleep` | ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 19,764 行。 +> 13 個套件、約 20,132 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/a2a/` | 94 | A2A（agent-to-agent）agent card 產生 | -| `utils/agent/` | 1,258 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | -| `utils/agent_memory/` | 148 | agent 的持久化情節記憶（goal → trajectory → outcome） | -| `utils/agent_replay/` | 65 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | -| `utils/agent_trace/` | 131 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | -| `utils/cost_telemetry/` | 295 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | -| `utils/cua_action/` | 129 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | -| `utils/llm/` | 363 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | -| `utils/mcp_registry/` | 94 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,441 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | -| `utils/tool_use_schema/` | 182 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | -| `utils/trajectory_eval/` | 108 | agent 軌跡評估：依評分規準為一次執行打分 | -| `utils/vision/` | 456 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | +| `utils/a2a/` | 92 | A2A（agent-to-agent）agent card 產生 | +| `utils/agent/` | 1,250 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | +| `utils/agent_memory/` | 146 | agent 的持久化情節記憶（goal → trajectory → outcome） | +| `utils/agent_replay/` | 63 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | +| `utils/agent_trace/` | 129 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | +| `utils/cost_telemetry/` | 292 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | +| `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | +| `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | +| `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | +| `utils/mcp_server/` | 16,850 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | +| `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | +| `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,622 行。 +> 6 個套件、約 17,703 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/admin/` | 329 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | -| `utils/config_sync/` | 247 | 透過訊令伺服器做跨機器設定同步 | -| `utils/device_matrix/` | 140 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 11,726 | **遠端桌面子系統**（51 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,255 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | -| `utils/usbip/` | 925 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | +| `utils/admin/` | 327 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | +| `utils/config_sync/` | 245 | 透過訊令伺服器做跨機器設定同步 | +| `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | +| `utils/remote_desktop/` | 11,835 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/usb/` | 4,238 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 -> 24 個套件、約 5,889 行。 +> 24 個套件、約 5,881 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/acme_v2/` | 589 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | -| `utils/chatops/` | 632 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | -| `utils/cookie_jar/` | 105 | RFC 6265 cookie jar | -| `utils/email_send/` | 118 | SMTP 寄信（email 觸發器的發送端搭檔） | -| `utils/events/` | 84 | 對外 CloudEvents 發送（執行生命週期事件） | -| `utils/http_cassette/` | 112 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | -| `utils/http_client/` | 134 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | -| `utils/http_conditional/` | 89 | 條件式 HTTP 請求與快取驗證器 | -| `utils/http_content/` | 105 | HTTP 內容協商與回應解壓縮 | -| `utils/http_problem/` | 118 | RFC 9457 problem+json 解析 | -| `utils/jwt/` | 174 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | -| `utils/link_header/` | 114 | RFC 8288 Link header 解析與分頁 | -| `utils/multipart/` | 141 | multipart/form-data 建構與解析 | -| `utils/notify/` | 97 | 跨平台桌面通知 | -| `utils/notify_channels/` | 102 | 對外聊天／webhook 通知（Slack／Discord／Teams／raw） | -| `utils/otp/` | 39 | TOTP 一次性密碼產生（自動化 2FA 登入） | -| `utils/outbox/` | 94 | 交易式 outbox，保證至少一次的事件投遞 | -| `utils/pytest_plugin/` | 377 | pytest 外掛 + BDD step library（`pytest11` entry point） | -| `utils/rest_api/` | 1,693 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | -| `utils/socket_server/` | 133 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | -| `utils/sse_client/` | 114 | Server-Sent Events 用戶端解析 | -| `utils/tls_acme/` | 445 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | -| `utils/url_canon/` | 117 | RFC 3986 URL 正規化與查詢字串工具 | -| `utils/webrunner_bridge/` | 163 | 把 action JSON 橋接到 WebRunner（`je_web_runner`） | +| `utils/acme_v2/` | 586 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | +| `utils/chatops/` | 628 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | +| `utils/cookie_jar/` | 103 | RFC 6265 cookie jar | +| `utils/email_send/` | 116 | SMTP 寄信（email 觸發器的發送端搭檔） | +| `utils/events/` | 82 | 對外 CloudEvents 發送（執行生命週期事件） | +| `utils/http_cassette/` | 110 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | +| `utils/http_client/` | 132 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | +| `utils/http_conditional/` | 87 | 條件式 HTTP 請求與快取驗證器 | +| `utils/http_content/` | 103 | HTTP 內容協商與回應解壓縮 | +| `utils/http_problem/` | 116 | RFC 9457 problem+json 解析 | +| `utils/jwt/` | 172 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | +| `utils/link_header/` | 112 | RFC 8288 Link header 解析與分頁 | +| `utils/multipart/` | 139 | multipart/form-data 建構與解析 | +| `utils/notify/` | 95 | 跨平台桌面通知 | +| `utils/notify_channels/` | 100 | 對外聊天／webhook 通知（Slack／Discord／Teams／raw） | +| `utils/otp/` | 37 | TOTP 一次性密碼產生（自動化 2FA 登入） | +| `utils/outbox/` | 92 | 交易式 outbox，保證至少一次的事件投遞 | +| `utils/pytest_plugin/` | 373 | pytest 外掛 + BDD step library（`pytest11` entry point） | +| `utils/rest_api/` | 1,738 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/socket_server/` | 131 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | +| `utils/sse_client/` | 112 | Server-Sent Events 用戶端解析 | +| `utils/tls_acme/` | 441 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | +| `utils/url_canon/` | 115 | RFC 3986 URL 正規化與查詢字串工具 | +| `utils/webrunner_bridge/` | 161 | 把 action JSON 橋接到 WebRunner（`je_web_runner`） | ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 6,942 行。 +> 34 個套件、約 6,865 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/anomaly/` | 109 | 單一序列異常偵測 | -| `utils/approval/` | 104 | Approval testing：以核可基準線驗證產出物 | -| `utils/assertion/` | 866 | 斷言 DSL：畫面狀態驗證 + 組合子 | -| `utils/baggage/` | 113 | W3C Baggage 傳遞 | -| `utils/canonical_log/` | 92 | canonical log line 與結構化 JSON 日誌 | -| `utils/ci_annotations/` | 64 | 由執行結果輸出 CI 工作流程註記（GitHub Actions） | -| `utils/compliance/` | 138 | 合規：把治理證據對應到 SOC2／ISO 27001 控制項 | -| `utils/failure_hooks/` | 400 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | -| `utils/failure_signature/` | 76 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | -| `utils/flake_cluster/` | 105 | 以共同失敗 Jaccard 相似度為易碎測試分群 | -| `utils/flakiness/` | 152 | 以執行歷史分析不穩定測試 | -| `utils/generate_report/` | 311 | HTML／JSON／XML 三種報表產生器（Template Method） | -| `utils/media_assert/` | 235 | 媒體斷言：音訊活動與影片動態檢查 | -| `utils/observability/` | 665 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | -| `utils/otlp_export/` | 83 | OTLP/JSON span 匯出 | -| `utils/percentiles/` | 105 | 可合併的串流延遲摘要與精確百分位數 | -| `utils/process_doc/` | 87 | 由錄製的 action list 產生逐步 SOP 文件 | -| `utils/process_mining/` | 112 | 流程探勘：從動作日誌挖掘可自動化的候選 | -| `utils/profiler/` | 425 | 逐動作效能剖析器 + 資源剖析器 | -| `utils/quarantine/` | 192 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | -| `utils/run_diff/` | 125 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | -| `utils/run_history/` | 349 | 執行歷史儲存與產出物管理 | -| `utils/sarif/` | 136 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | -| `utils/slo/` | 114 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | -| `utils/smoothing/` | 69 | 數列移動平均平滑 | -| `utils/soft_assert/` | 64 | 軟斷言：累積檢查並在區塊結束時一次拋出 | -| `utils/stats/` | 215 | 描述統計與 A/B 顯著性檢定（純標準庫） | -| `utils/step_timeline/` | 83 | 每次執行的步驟瀑布圖與瓶頸（關鍵路徑）步驟排名 | -| `utils/test_select/` | 125 | 以執行歷史做風險導向的測試選取 | -| `utils/test_shard/` | 89 | 以耗時為權重的套件切分與分片結果合併 | -| `utils/test_suite/` | 446 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | -| `utils/time_travel/` | 384 | 錄製 session 的時光回溯除錯（控制器 + 播放器） | -| `utils/timeseries/` | 145 | 時間序列轉換（rate／降採樣／重採樣） | -| `utils/trace_context/` | 164 | W3C Trace Context 傳遞 | +| `utils/anomaly/` | 107 | 單一序列異常偵測 | +| `utils/approval/` | 102 | Approval testing：以核可基準線驗證產出物 | +| `utils/assertion/` | 863 | 斷言 DSL：畫面狀態驗證 + 組合子 | +| `utils/baggage/` | 111 | W3C Baggage 傳遞 | +| `utils/canonical_log/` | 90 | canonical log line 與結構化 JSON 日誌 | +| `utils/ci_annotations/` | 62 | 由執行結果輸出 CI 工作流程註記（GitHub Actions） | +| `utils/compliance/` | 136 | 合規：把治理證據對應到 SOC2／ISO 27001 控制項 | +| `utils/failure_hooks/` | 396 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | +| `utils/failure_signature/` | 74 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | +| `utils/flake_cluster/` | 103 | 以共同失敗 Jaccard 相似度為易碎測試分群 | +| `utils/flakiness/` | 150 | 以執行歷史分析不穩定測試 | +| `utils/generate_report/` | 310 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/media_assert/` | 233 | 媒體斷言：音訊活動與影片動態檢查 | +| `utils/observability/` | 661 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | +| `utils/otlp_export/` | 81 | OTLP/JSON span 匯出 | +| `utils/percentiles/` | 103 | 可合併的串流延遲摘要與精確百分位數 | +| `utils/process_doc/` | 85 | 由錄製的 action list 產生逐步 SOP 文件 | +| `utils/process_mining/` | 110 | 流程探勘：從動作日誌挖掘可自動化的候選 | +| `utils/profiler/` | 422 | 逐動作效能剖析器 + 資源剖析器 | +| `utils/quarantine/` | 190 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | +| `utils/run_diff/` | 123 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | +| `utils/run_history/` | 346 | 執行歷史儲存與產出物管理 | +| `utils/sarif/` | 134 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | +| `utils/slo/` | 112 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | +| `utils/smoothing/` | 67 | 數列移動平均平滑 | +| `utils/soft_assert/` | 62 | 軟斷言：累積檢查並在區塊結束時一次拋出 | +| `utils/stats/` | 213 | 描述統計與 A/B 顯著性檢定（純標準庫） | +| `utils/step_timeline/` | 81 | 每次執行的步驟瀑布圖與瓶頸（關鍵路徑）步驟排名 | +| `utils/test_select/` | 123 | 以執行歷史做風險導向的測試選取 | +| `utils/test_shard/` | 87 | 以耗時為權重的套件切分與分片結果合併 | +| `utils/test_suite/` | 442 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | +| `utils/time_travel/` | 381 | 錄製 session 的時光回溯除錯（控制器 + 播放器） | +| `utils/timeseries/` | 143 | 時間序列轉換（rate／降採樣／重採樣） | +| `utils/trace_context/` | 162 | W3C Trace Context 傳遞 | ### 5.4.13 資料來源、結構驗證與 i18n -> 24 個套件、約 3,937 行。 +> 24 個套件、約 3,886 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/checksum/` | 134 | 檢查碼演算法：Luhn、Verhoeff、Damm、ISO 7064 MOD 97-10 | -| `utils/config_schema/` | 111 | 型別化設定結構驗證 | -| `utils/data_drift/` | 127 | 分布漂移偵測 | -| `utils/data_profile/` | 123 | 資料剖析與結構推斷 | -| `utils/data_quality/` | 187 | 資料品質：列結構驗證、欄位擷取、遮蔽 | -| `utils/data_source/` | 182 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | -| `utils/dataset_diff/` | 91 | 表格資料列差異比對（CDC 風格） | -| `utils/gettext_catalog/` | 298 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | -| `utils/i18n_test/` | 132 | 國際化／在地化測試輔助 | -| `utils/json_contract/` | 137 | JSON 契約／快照比對：`match_json`、`diff_json`、`snapshot_json` | -| `utils/json_patch/` | 314 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | -| `utils/json_schema/` | 376 | JSON Schema（Draft 2020-12 子集）驗證 | -| `utils/jsonpath/` | 181 | 精簡 JSONPath 查詢 | -| `utils/list_format/` | 74 | 地區感知清單格式化（CLDR 風格的「A、B 和 C」） | -| `utils/locale_collation/` | 130 | 地區感知字串排序（決定性多層排序鍵） | -| `utils/locale_parse/` | 70 | 地區感知數字／貨幣／日期解析與格式化（選用 babel） | -| `utils/message_format/` | 238 | ICU-lite MessageFormat（plural／select／selectordinal） | -| `utils/office/` | 164 | Office 文件無頭讀寫（Excel／Word／PowerPoint） | -| `utils/pdf/` | 89 | PDF 讀取與斷言（選用 pypdf 後端） | -| `utils/referential/` | 77 | 跨資料集的參照完整性檢查 | -| `utils/schema_compat/` | 164 | JSON Schema 相容性分級 | -| `utils/sql/` | 76 | 對 SQLite 的臨時唯讀 SQL 查詢 | -| `utils/test_data/` | 207 | 帶種子的合成測試資料產生（純標準庫） | -| `utils/xml/` | 255 | XML 檔讀寫與結構變更（`defusedxml`） | +| `utils/checksum/` | 132 | 檢查碼演算法：Luhn、Verhoeff、Damm、ISO 7064 MOD 97-10 | +| `utils/config_schema/` | 109 | 型別化設定結構驗證 | +| `utils/data_drift/` | 125 | 分布漂移偵測 | +| `utils/data_profile/` | 121 | 資料剖析與結構推斷 | +| `utils/data_quality/` | 185 | 資料品質：列結構驗證、欄位擷取、遮蔽 | +| `utils/data_source/` | 180 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | +| `utils/dataset_diff/` | 89 | 表格資料列差異比對（CDC 風格） | +| `utils/gettext_catalog/` | 296 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | +| `utils/i18n_test/` | 130 | 國際化／在地化測試輔助 | +| `utils/json_contract/` | 135 | JSON 契約／快照比對：`match_json`、`diff_json`、`snapshot_json` | +| `utils/json_patch/` | 312 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | +| `utils/json_schema/` | 374 | JSON Schema（Draft 2020-12 子集）驗證 | +| `utils/jsonpath/` | 179 | 精簡 JSONPath 查詢 | +| `utils/list_format/` | 72 | 地區感知清單格式化（CLDR 風格的「A、B 和 C」） | +| `utils/locale_collation/` | 128 | 地區感知字串排序（決定性多層排序鍵） | +| `utils/locale_parse/` | 68 | 地區感知數字／貨幣／日期解析與格式化（選用 babel） | +| `utils/message_format/` | 236 | ICU-lite MessageFormat（plural／select／selectordinal） | +| `utils/office/` | 162 | Office 文件無頭讀寫（Excel／Word／PowerPoint） | +| `utils/pdf/` | 87 | PDF 讀取與斷言（選用 pypdf 後端） | +| `utils/referential/` | 75 | 跨資料集的參照完整性檢查 | +| `utils/schema_compat/` | 162 | JSON Schema 相容性分級 | +| `utils/sql/` | 74 | 對 SQLite 的臨時唯讀 SQL 查詢 | +| `utils/test_data/` | 205 | 帶種子的合成測試資料產生（純標準庫） | +| `utils/xml/` | 250 | XML 檔讀寫與結構變更（`defusedxml`） | ### 5.4.14 安全、機密與合規 -> 13 個套件、約 2,290 行。 +> 13 個套件、約 2,261 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/config_redaction/` | 77 | 設定結構與 log 字串的機密遮蔽 | -| `utils/egress/` | 116 | 無頭 HTTP 用戶端的網路外連允許清單守衛 | -| `utils/governance/` | 202 | 治理：maker-checker 核准閘門與即時憑證租約 | -| `utils/license_policy/` | 141 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | -| `utils/provenance/` | 106 | SLSA 建置來源證明（in-toto v1） | -| `utils/rbac/` | 274 | 角色型存取控制與逐使用者稽核歸因 | -| `utils/redaction/` | 461 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | -| `utils/sbom/` | 110 | SBOM（CycloneDX）產生 | -| `utils/secret_ref/` | 128 | URI scheme 形式的值參照解析 | -| `utils/secrets/` | 253 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | -| `utils/secrets_scan/` | 100 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | -| `utils/vex/` | 132 | OpenVEX 陳述撰寫與漏洞分類處置 | -| `utils/vuln_scan/` | 190 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | +| `utils/config_redaction/` | 75 | 設定結構與 log 字串的機密遮蔽 | +| `utils/egress/` | 114 | 無頭 HTTP 用戶端的網路外連允許清單守衛 | +| `utils/governance/` | 199 | 治理：maker-checker 核准閘門與即時憑證租約 | +| `utils/license_policy/` | 139 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | +| `utils/provenance/` | 104 | SLSA 建置來源證明（in-toto v1） | +| `utils/rbac/` | 272 | 角色型存取控制與逐使用者稽核歸因 | +| `utils/redaction/` | 457 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | +| `utils/sbom/` | 108 | SBOM（CycloneDX）產生 | +| `utils/secret_ref/` | 126 | URI scheme 形式的值參照解析 | +| `utils/secrets/` | 251 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | +| `utils/secrets_scan/` | 98 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | +| `utils/vex/` | 130 | OpenVEX 陳述撰寫與漏洞分類處置 | +| `utils/vuln_scan/` | 188 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | ### 5.4.15 韌性、流量控制與設定 -> 14 個套件、約 1,734 行。 +> 14 個套件、約 1,706 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/artifact_store/` | 116 | S3 相容產出物儲存（報表／截圖／錄影） | -| `utils/assets/` | 157 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | -| `utils/bulkhead/` | 136 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | -| `utils/chaos/` | 155 | 決定性混沌實驗（穩態假說 + 故障注入） | -| `utils/dedup_window/` | 65 | 時間視窗內的訊息去重 | -| `utils/dotenv/` | 103 | `.env` 檔解析與序列化 | -| `utils/feature_flags/` | 175 | 功能旗標評估，含目標規則與決定性灰度 | -| `utils/idempotency/` | 116 | 冪等鍵儲存與已存回應重放 | -| `utils/layered_config/` | 112 | 分層設定解析 | -| `utils/optimistic/` | 107 | 樂觀併發的版本化儲存 | -| `utils/rate_limit/` | 164 | 用戶端限流：token bucket、滑動視窗、throttle | -| `utils/resilience/` | 112 | 韌性原語：退避重試與斷路器 | -| `utils/retry_budget/` | 149 | 重試預算：以牆鐘期限與 full jitter 約束重試 | -| `utils/sequence_gap/` | 67 | 逐串流的序號缺口偵測 | +| `utils/artifact_store/` | 114 | S3 相容產出物儲存（報表／截圖／錄影） | +| `utils/assets/` | 155 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | +| `utils/bulkhead/` | 134 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | +| `utils/chaos/` | 153 | 決定性混沌實驗（穩態假說 + 故障注入） | +| `utils/dedup_window/` | 63 | 時間視窗內的訊息去重 | +| `utils/dotenv/` | 101 | `.env` 檔解析與序列化 | +| `utils/feature_flags/` | 173 | 功能旗標評估，含目標規則與決定性灰度 | +| `utils/idempotency/` | 114 | 冪等鍵儲存與已存回應重放 | +| `utils/layered_config/` | 110 | 分層設定解析 | +| `utils/optimistic/` | 105 | 樂觀併發的版本化儲存 | +| `utils/rate_limit/` | 162 | 用戶端限流：token bucket、滑動視窗、throttle | +| `utils/resilience/` | 110 | 韌性原語：退避重試與斷路器 | +| `utils/retry_budget/` | 147 | 重試預算：以牆鐘期限與 full jitter 約束重試 | +| `utils/sequence_gap/` | 65 | 逐串流的序號缺口偵測 | ### 5.4.16 系統、視窗與剪貼簿 -> 16 個套件、約 2,411 行。 +> 16 個套件、約 2,403 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/clipboard/` | 468 | 跨平台無頭剪貼簿存取（文字 + 影像）＋ `win32_clipboard_api.py`：**所有剪貼簿格式共用的 Win32 原型與 open/alloc/lock 流程**（`argtypes` 只宣告一半曾讓四支 writer 在 64 位元上必然丟 `OverflowError`，見 CHANGELOG）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | -| `utils/clipboard_files/` | 119 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | -| `utils/clipboard_formats/` | 152 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | -| `utils/clipboard_history/` | 111 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | -| `utils/clipboard_rich_formats/` | 279 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | -| `utils/file_assoc/` | 94 | 解析哪個應用程式被註冊來開啟某副檔名 | -| `utils/file_dialog/` | 62 | 驅動原生檔案 開啟／儲存／資料夾選擇 對話框 | -| `utils/file_drop/` | 98 | 以 WM_DROPFILES 把檔案拖放到視窗 | -| `utils/rich_clipboard/` | 151 | 豐富剪貼簿格式 — HTML（CF_HTML）建構／解析／存取 | -| `utils/shell_open/` | 99 | 以預設應用開啟檔案，或以預設瀏覽器開啟 URL | -| `utils/system_volume/` | 196 | 讀取與控制系統主音量與靜音狀態 | -| `utils/trash/` | 90 | 把檔案移到系統資源回收筒（可復原刪除） | -| `utils/window_capture/` | 249 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | -| `utils/window_geometry/` | 83 | 視窗客戶區幾何（外框內縮、client→screen 對映） | -| `utils/window_layout/` | 136 | 視窗拼貼／版面規劃器（左右半、四象限、網格、層疊） | -| `utils/window_zorder/` | 78 | 視窗 z 序控制（最上層／移到最前／送到最後） | +| `utils/clipboard/` | 489 | 跨平台無頭剪貼簿存取（文字 + 影像）＋ `win32_clipboard_api.py`：**所有剪貼簿格式共用的 Win32 原型與 open/alloc/lock 流程**（`open_clipboard()` 會等過短暫被別的行程佔住的剪貼簿——Win32 一次只允許一個行程開啟，別人正在複製就必然失敗）（`argtypes` 只宣告一半曾讓四支 writer 在 64 位元上必然丟 `OverflowError`，見 CHANGELOG）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | +| `utils/clipboard_files/` | 96 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | +| `utils/clipboard_formats/` | 151 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | +| `utils/clipboard_history/` | 109 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | +| `utils/clipboard_rich_formats/` | 254 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | +| `utils/file_assoc/` | 92 | 解析哪個應用程式被註冊來開啟某副檔名 | +| `utils/file_dialog/` | 60 | 驅動原生檔案 開啟／儲存／資料夾選擇 對話框 | +| `utils/file_drop/` | 96 | 以 WM_DROPFILES 把檔案拖放到視窗 | +| `utils/rich_clipboard/` | 131 | 豐富剪貼簿格式 — HTML（CF_HTML）建構／解析／存取 | +| `utils/shell_open/` | 97 | 以預設應用開啟檔案，或以預設瀏覽器開啟 URL | +| `utils/system_volume/` | 194 | 讀取與控制系統主音量與靜音狀態 | +| `utils/trash/` | 88 | 把檔案移到系統資源回收筒（可復原刪除） | +| `utils/window_capture/` | 255 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | +| `utils/window_geometry/` | 81 | 視窗客戶區幾何（外框內縮、client→screen 對映） | +| `utils/window_layout/` | 134 | 視窗拼貼／版面規劃器（左右半、四象限、網格、層疊） | +| `utils/window_zorder/` | 76 | 視窗 z 序控制（最上層／移到最前／送到最後） | ### 5.4.17 大型子系統的檔案級剖析 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（8,811 行）— 執行核心 +#### `utils/executor/`（9,070 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `action_executor.py` | 8,042 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | -| `flow_control.py` | 757 | 34 個區塊指令：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`、`AC_*_to_var`）／`AC_assert_var`。`LoopBreak`／`LoopContinue` 以例外實作。 | -| `action_schema.py` | 94 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。 | -| `mouse_aliases.py` | 40 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | +| `action_executor.py` | 8,125 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `flow_control.py` | 530 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | +| `flow_data_commands.py` | 248 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | +| `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | +| `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,441 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（16,850 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | | `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | -| `server.py` | 995 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器。 | +| `server.py` | 669 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | | `http_transport.py` | 323 | MCP 的 HTTP 傳輸。 | -| `resources.py` | 304 | MCP resource 提供者。 | -| `prompts.py` | 221 | MCP prompt 目錄。 | -| `fake_backend.py` | 185 | CI／無頭測試用的記憶體內假後端。 | -| `plugin_watcher.py` | 150 | 檔案變更時熱重載外掛工具的背景 watcher。 | +| `_client_requests.py` | 217 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | +| `_protocol.py` | 165 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | +| `resources.py` | 303 | MCP resource 提供者。 | +| `prompts.py` | 220 | MCP prompt 目錄。 | +| `fake_backend.py` | 184 | CI／無頭測試用的記憶體內假後端。 | +| `plugin_watcher.py` | 149 | 檔案變更時熱重載外掛工具的背景 watcher。 | | `tools/_base.py` | 147 | 工具註冊表的共用型別與輔助。 | | `tools/_validation.py` | 107 | MCP 工具用到的 JSON Schema 子集驗證器。 | | `tools/plugin_tools.py` | 90 | 把外掛載入的 `AC_*` callable 包成 `MCPTool`。 | -| `log_bridge.py` | 91 | 把 Python logging 記錄橋接成 MCP `notifications/message`。 | -| `audit.py` | 79 | MCP 工具呼叫稽核記錄。 | -| `context.py` | 72 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | -| `rate_limit.py` | 49 | 工具呼叫的 token bucket 限流。 | -| `__main__.py` | 88 | `je_auto_control_mcp` console script 進入點。 | +| `log_bridge.py` | 90 | 把 Python logging 記錄橋接成 MCP `notifications/message`。 | +| `audit.py` | 78 | MCP 工具呼叫稽核記錄。 | +| `context.py` | 71 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | +| `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | +| `__main__.py` | 87 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（11,726 行／51 檔） +#### `utils/remote_desktop/`（11,835 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `host.py` | 1,319 | TCP 主機：串流 JPEG 影格並套用檢視端輸入。 | -| `webrtc_host.py` | 996 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入。 | -| `webrtc_viewer.py` | 639 | WebRTC 檢視端：接收視訊並送出輸入。 | -| `viewer.py` | 624 | TCP 檢視端。 | -| `host_service.py` | 543 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | -| `registry.py` | 371 | `AC_remote_*` 指令使用的行程級單例。 | -| `webrtc_transport.py` | 357 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | -| `multi_viewer.py` | 315 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | -| `signaling_server.py` | 298 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | -| `audit_log.py` | 284 | SQLite 雜湊鏈稽核記錄。 | -| `ws_protocol.py` | 278 | 最小 RFC 6455 WebSocket 框架與握手。 | -| `file_transfer.py` | 274 | 分塊檔案傳輸。 | -| `relay.py` | 271 | NAT 穿透失敗時的 TCP 中繼。 | -| `fingerprint.py` | 251 | TOFU 主機指紋驗證。 | -| `turn_config.py` | 235 | coturn 設定產生器。 | -| `presence.py` | 222 | 多檢視者的執行緒安全在場註冊表。 | -| `jpeg_recorder_encrypted.py` | 218 | AES-GCM 加密版 session 錄影。 | -| `address_book.py` | 210 | 檢視端的主機通訊錄。 | +| `webrtc_host.py` | 683 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | +| `webrtc_viewer.py` | 638 | WebRTC 檢視端：接收視訊並送出輸入。 | +| `host.py` | 625 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | +| `viewer.py` | 623 | TCP 檢視端。 | +| `host_service.py` | 542 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | +| `host_client.py` | 406 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | +| `registry.py` | 370 | `AC_remote_*` 指令使用的行程級單例。 | +| `webrtc_transport.py` | 360 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | +| `multi_viewer.py` | 314 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | +| `signaling_server.py` | 297 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | +| `audit_log.py` | 283 | SQLite 雜湊鏈稽核記錄。 | +| `host_capture.py` | 280 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | +| `ws_protocol.py` | 277 | 最小 RFC 6455 WebSocket 框架與握手。 | +| `file_transfer.py` | 273 | 分塊檔案傳輸。 | +| `relay.py` | 270 | NAT 穿透失敗時的 TCP 中繼。 | +| `fingerprint.py` | 250 | TOFU 主機指紋驗證。 | +| `turn_config.py` | 234 | coturn 設定產生器。 | +| `presence.py` | 221 | 多檢視者的執行緒安全在場註冊表。 | +| `jpeg_recorder_encrypted.py` | 217 | AES-GCM 加密版 session 錄影。 | +| `address_book.py` | 209 | 檢視端的主機通訊錄。 | | `audio.py` / `webrtc_audio.py` / `webrtc_mic.py` | 206 / 190 / 152 | 音訊擷取播放、音訊軌、麥克風上行。 | -| `webrtc_files.py` | 206 | 專屬 DataChannel 的分塊檔案傳輸。 | -| `lan_discovery.py` | 190 | mDNS／Zeroconf 區網探索。 | -| `video_codec.py` | 183 | TCP／WS 路徑的可插拔視訊編解碼。 | -| `hw_codec.py` | 170 | 硬體 H.264 編碼偵測與啟用。 | -| `webrtc_stats.py` | 164 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | -| `webrtc_inspector.py` | 139 | 行程級的 `StatsSnapshot` 滾動視窗。 | -| `adaptive_bitrate.py` | 149 | 依統計調整主機擷取 FPS。 | -| `connect_coordinator.py` | 150 | 由使用者輸入的目標決定該用哪條傳輸。 | -| `signaling_client.py` | 146 | 純標準庫的訊令用戶端。 | -| `trust_list.py` | 145 | 自動接受的檢視端信任清單。 | -| `input_dispatch.py` | 134 | 在主機端套用輸入訊息。 | -| `session_recorder.py` | 130 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | -| `totp.py` | 130 | RFC 6238 TOTP（零外部相依）。 | -| `file_sync.py` | 127 | 輪詢式資料夾鏡像。 | -| `transport.py` | 124 | 可插拔的型別化訊息傳輸。 | -| `protocol.py` | 97 | 長度前綴的 TCP 框架。 | +| `webrtc_files.py` | 205 | 專屬 DataChannel 的分塊檔案傳輸。 | +| `webrtc_host_auth.py` | 195 | 檢視端認證與核准：token 檢查、信任清單／IP 白名單自動放行、手動接受／拒絕、SAS、逾時關閉。 | +| `lan_discovery.py` | 189 | mDNS／Zeroconf 區網探索。 | +| `video_codec.py` | 182 | TCP／WS 路徑的可插拔視訊編解碼。 | +| `webrtc_host_media.py` | 172 | 重新協商與 recvonly 軌管理。aiortc 沒有 `removeTransceiver`,所以開／關不對稱——開是加軌重新 offer,關只能設 inactive 並停掉 receiver。 | +| `hw_codec.py` | 169 | 硬體 H.264 編碼偵測與啟用。 | +| `webrtc_stats.py` | 163 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | +| `connect_coordinator.py` | 149 | 由使用者輸入的目標決定該用哪條傳輸。 | +| `adaptive_bitrate.py` | 148 | 依統計調整主機擷取 FPS。 | +| `signaling_client.py` | 145 | 純標準庫的訊令用戶端。 | +| `trust_list.py` | 144 | 自動接受的檢視端信任清單。 | +| `webrtc_inspector.py` | 138 | 行程級的 `StatsSnapshot` 滾動視窗。 | +| `input_dispatch.py` | 133 | 在主機端套用輸入訊息。 | +| `session_recorder.py` | 129 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | +| `totp.py` | 129 | RFC 6238 TOTP（零外部相依）。 | +| `file_sync.py` | 126 | 輪詢式資料夾鏡像。 | +| `transport.py` | 123 | 可插拔的型別化訊息傳輸。 | +| `host_access.py` | 105 | TCP 主機的檢視端核准與存取控制：`PendingViewer`、權限字串、分享碼的 TOTP 候選值、IP 白名單。`host` 與 `host_client` 共用,所以獨立成模組。 | +| `protocol.py` | 96 | 長度前綴的 TCP 框架。 | | `resume_tokens.py` / `session_quality_cache.py` / `rate_limit.py` | 95 / 86 / 85 | 快速重連 token、每 session 品質快取、檢視端限流。 | | `host_id.py` / `viewer_id.py` | 82 / 78 | 主機與檢視端的持久身分。 | | `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,255 行）與 `utils/usbip/`（925 行） +#### `utils/usb/`（4,238 行）與 `utils/usbip/`（920 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -778,17 +794,17 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `usbip/libusb_backend.py` | 209 | 以 PyUSB／libusb 執行 URB 的正式後端。 | | `usbip/backend.py` | 88 | 可插拔 URB 執行後端。 | -#### `utils/rest_api/`（1,693 行） +#### `utils/rest_api/`（1,738 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `rest_server.py` | 468 | HTTP 前端主體。 | -| `rest_handlers.py` | 451 | 端點實作。 | -| `rest_openapi.py` | 406 | 走訪路由表產生 OpenAPI 3.1 規格。 | -| `rest_auth.py` | 144 | Bearer token 驗證 + 逐 client 限流閘門。 | -| `rest_metrics.py` | 76 | Prometheus 曝露端點。 | -| `rest_registry.py` | 76 | 保存執行中 REST 伺服器的行程級單例。 | -| `__main__.py` | 57 | `python -m je_auto_control.utils.rest_api` 進入點。 | +| `rest_server.py` | 467 | HTTP 前端主體。 | +| `rest_handlers.py` | 486 | 端點實作。 | +| `rest_openapi.py` | 422 | 走訪路由表產生 OpenAPI 3.1 規格。 | +| `rest_auth.py` | 143 | Bearer token 驗證 + 逐 client 限流閘門。 | +| `rest_metrics.py` | 75 | Prometheus 曝露端點。 | +| `rest_registry.py` | 75 | 保存執行中 REST 伺服器的行程級單例。 | +| `__main__.py` | 56 | `python -m je_auto_control.utils.rest_api` 進入點。 | #### 其他多檔子套件 @@ -809,7 +825,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `semantic_recording/` | `enrich.py`（加錨點）、`replay.py`（換機重播）、`self_healing.py`（自癒重播） | | `tls_acme/` | `challenge.py`、`keys.py`、`renewal.py` | | `pytest_plugin/` | `plugin.py`（pytest11 進入點）、`keywords.py`、`bdd_steps.py`（Gherkin） | -| `cv2_utils/` | `screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`blobs.py` | +| `cv2_utils/` | `screen_grabber.py`、`screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`blobs.py` | | `action_lint/` | `linter.py`、`schema.py`、`__main__.py`（CI 使用） | | `time_travel/` | `controller.py`、`player.py` | | `dag/` | `graph.py`、`runner.py` | @@ -838,10 +854,14 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `gui/__init__.py` | 24 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | +| `gui/__init__.py` | 23 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | | `main_window.py` | 290 | `QMainWindow`：選單列（File／Actions／View／…）、可關閉分頁、即時語言切換、字級預設、qt-material 主題。分頁分為 core／editing／detection／automation／system 五類。 | -| `main_widget.py` | 780 | 擁有 `QTabWidget`，註冊 48 個分頁，並暴露 show/hide/list API 給選單列。核心分頁在註冊時直接宣告 `(label_key, handler)` 動作對。 | +| `main_widget.py` | 423 | 擁有 `QTabWidget`，註冊 48 個分頁，並暴露 show/hide/list API 給選單列。核心分頁在註冊時直接宣告 `(label_key, handler)` 動作對；分頁本體都在下列 mixin。 | | `_auto_click_tab.py` | 270 | 自動點擊分頁的 mixin 建構器。 | +| `_screenshot_tab.py` | 127 | 截圖／取像素分頁 mixin。 | +| `_image_detect_tab.py` | 106 | 影像偵測分頁 mixin。 | +| `_script_tab.py` | 105 | 腳本執行分頁 mixin。 | +| `_record_tab.py` | 101 | 錄製／回放分頁 mixin。 | | `_report_tab.py` | 81 | 報表分頁 mixin。 | | `_i18n_helpers.py` | 67 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | | `language_wrapper/` | 4,977 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | @@ -856,12 +876,12 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 分頁 | 模組 | 行數 | 職責 | | --- | --- | ---: | --- | | auto_click | `_auto_click_tab.py` | 270 | 自動點擊：座標、間隔、熱鍵、`write`、捲動。 | -| screenshot | `main_widget.py` 內建 | — | 截圖、選區、螢幕尺寸、取像素色。 | -| image_detect | `main_widget.py` 內建 | — | 樣板裁切、定位、定位全部、定位並點擊。 | -| record | `main_widget.py` 內建 | — | 錄製／停止／回放／存檔／載入。 | +| screenshot | `_screenshot_tab.py` | 127 | 截圖、選區、螢幕尺寸、取像素色。 | +| image_detect | `_image_detect_tab.py` | 106 | 樣板裁切、定位、定位全部、定位並點擊。 | +| record | `_record_tab.py` | 101 | 錄製／停止／回放／存檔／載入。 | | script_builder | `script_builder/` | 5,708 | **視覺化腳本編輯器**：`command_schema.py`（4,924 行 `AC_*` 參數綱要）、`step_model.py`（步驟模型與 AC JSON 序列化）、`step_list_view.py`（含巢狀 body 的樹狀檢視）、`step_form_view.py`（綱要驅動表單）、`builder_tab.py`。 | | flow_editor | `flow_editor/` | 490 | 節點式流程圖檢視：`layout.py`（純 Python 佈局演算法，可單測）、`scene.py`（Qt 場景繪製）、`tab.py`。 | -| script | `main_widget.py` 內建 | — | 載入／執行單檔或整個目錄、內建編輯器執行。 | +| script | `_script_tab.py` | 105 | 載入／執行單檔或整個目錄、內建編輯器執行。 | | recording_editor | `recording_editor_tab.py` | 244 | 裁切、過濾、重新縮放錄製內容。 | | variables | `variables_tab.py` | 166 | 檢視、灌入、清除 executor 執行期作用域。 | | secrets | `secrets_tab.py` | 188 | 解鎖保險庫並管理 `${secrets.NAME}`。 | @@ -892,7 +912,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | dag_runner | `dag_tab.py` | 188 | 編輯、驗證、執行跨主機 DAG。 | | chatops | `chatops_tab.py` | 108 | 在接上 Slack 前先本機測試 slash 指令。 | | trace_replay | `trace_replay_tab.py` | 187 | 拖曳捲動時光回溯錄製內容。 | -| remote_desktop | `remote_desktop/`（16 檔） | 6,240 | 見下。 | +| remote_desktop | `remote_desktop/`（17 檔） | 6,240 | 見下。 | | presence | `presence_tab.py` | 152 | 多檢視者在場名單。 | | rest_api | `rest_api_tab.py` | 198 | 啟停 HTTP 前端並顯示 URL 與 token。 | | admin_console | `admin_console_tab.py` | 313 | 管理多個遠端 AutoControl REST 端點。 | @@ -904,25 +924,26 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | -#### 遠端桌面 GUI（`gui/remote_desktop/`，16 檔／6,240 行） +#### 遠端桌面 GUI（`gui/remote_desktop/`，17 檔／6,254 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_panel.py` | 2,556 | WebRTC 子分頁主體。 | -| `webrtc_dialogs.py` | 845 | WebRTC GUI 用的自訂對話框與清單元件。 | -| `connection_screen.py` | 673 | Quick Connect —— AnyDesk 風格單畫面入口。 | -| `viewer_panel.py` | 543 | 「控制另一台機器」子分頁。 | -| `host_panel.py` | 335 | 「分享這台機器」子分頁。 | -| `frame_display.py` | 229 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | -| `webrtc_workers.py` | 196 | 訊令流程的背景 `QThread` worker。 | -| `tab.py` | 166 | 外層容器分頁。 | -| `_helpers.py` | 149 | 面板共用輔助。 | -| `remote_screen_window.py` | 141 | 檢視端的彈出視窗。 | -| `tray_icon.py` | 99 | WebRTC 主機的系統匣圖示。 | -| `annotation_overlay.py` | 89 | 主機端標註的透明最上層覆蓋。 | -| `sparkline.py` | 78 | WebRTC 統計面板的迷你走勢圖。 | -| `blanking_overlay.py` | 72 | 遠端連線期間的隱私遮蔽全螢幕覆蓋。 | -| `viewer_screen_window.py` | 47 | 顯示連入檢視端分享畫面的彈出視窗。 | +| `webrtc_panel.py` | 2,555 | WebRTC 子分頁主體。 | +| `webrtc_dialogs.py` | 493 | WebRTC GUI 用的自訂對話框與清單元件（待審檢視者、信任清單、通訊錄、遠端檔案表、稽核記錄、LAN 瀏覽）。 | +| `connection_screen.py` | 672 | Quick Connect —— AnyDesk 風格單畫面入口。 | +| `viewer_panel.py` | 542 | 「控制另一台機器」子分頁。 | +| `webrtc_known_hosts.py` | 340 | TOFU 釘選庫瀏覽器：`KnownHostsDialog` 與帶外釘選用的小表單。由 `webrtc_dialogs` 再匯出。 | +| `host_panel.py` | 334 | 「分享這台機器」子分頁。 | +| `frame_display.py` | 228 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | +| `webrtc_workers.py` | 195 | 訊令流程的背景 `QThread` worker。 | +| `tab.py` | 165 | 外層容器分頁。 | +| `_helpers.py` | 189 | 面板共用輔助：翻譯、Qt→AC 鍵滑鼠對應、TLS context、狀態徽章、指紋與時間格式化。 | +| `remote_screen_window.py` | 140 | 檢視端的彈出視窗。 | +| `tray_icon.py` | 98 | WebRTC 主機的系統匣圖示。 | +| `annotation_overlay.py` | 88 | 主機端標註的透明最上層覆蓋。 | +| `sparkline.py` | 77 | WebRTC 統計面板的迷你走勢圖。 | +| `blanking_overlay.py` | 71 | 遠端連線期間的隱私遮蔽全螢幕覆蓋。 | +| `viewer_screen_window.py` | 46 | 顯示連入檢視端分享畫面的彈出視窗。 | ### 5.6 周邊子專案與資產 @@ -935,12 +956,12 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `benchmarks/core_latency.py` | 32 行 | 對穩定無頭進入點的可重複煙霧基準測試。 | | `examples/` | 27 個腳本 | 從截圖點擊、OCR、排程、遠端桌面、agent loop、可觀測性，一路到 computer-use、Wayland、跨主機 DAG、chatops、pytest/BDD、anchor locator。 | | `browser-extension/` | manifest v3 擴充 | 瀏覽器端配合元件（background／content script／popup）。 | -| `docker/` | Dockerfile ×2 + compose | 無頭容器與帶 XFCE 桌面的容器。 | +| `docker/` | Dockerfile ×6 + compose + 8 支驗證／伺服器腳本 | 無頭容器（`Dockerfile`）、帶 XFCE 桌面的容器（`Dockerfile.xfce`),以及四個**驗證用**映像:`Dockerfile.wayland`（sway headless,擷取路徑 + `libei_verify.py` 對真的 libei.so 解析符號）、`Dockerfile.eis`（`eis_server.py` 用 ctypes 綁 libeis 起一個真的 EIS server,`eis_verify.py` 把 libei sender 對著它跑完整握手與發送）、`Dockerfile.portal`（`portal_server.py` 自己佔住 `org.freedesktop.portal.Desktop`,真的 `dbus-daemon` + 真的 liboeffis 跑完 RemoteDesktop 交握）、`Dockerfile.ydotool`（真的 uinput 裝置,`ydotool_verify.py` 直接讀回 `/dev/input/eventN`）、`Dockerfile.seat`（`headless,libinput` + builtin seat,合成器真的吃下 ydotool 裝置,`seat_verify.py` 從 `grim -c` 的像素讀回游標落點）。全部接在 `.github/workflows/docker.yml`。 | | `k8s/helm/` | Helm chart | Kubernetes 部署。 | | `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | | `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | -| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 460 個 `test_*.py`／4,333 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 則守住文件引用的數字與實測值一致。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 466 個 `test_*.py`／4,443 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 | --- @@ -994,26 +1015,26 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 84 | 26,367 | -| `utils/mcp_server/` | 18 | 16,441 | -| `utils/remote_desktop/` | 51 | 11,726 | -| `utils/executor/` | 5 | 8,811 | -| `utils/usb/` | 17 | 4,255 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,325 | +| `gui/` | 89 | 26,542 | +| `utils/mcp_server/` | 20 | 16,850 | +| `utils/remote_desktop/` | 56 | 11,835 | +| `utils/executor/` | 6 | 9,070 | +| `utils/usb/` | 17 | 4,238 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | | `utils/accessibility/` | 12 | 2,390 | -| `wrapper/` | 12 | 1,747 | -| `utils/rest_api/` | 8 | 1,693 | -| `windows/` | 26 | 1,939 | -| `utils/agent/` | 8 | 1,258 | -| `linux_with_x11/` | 19 | 1,189 | -| `utils/triggers/` | 4 | 1,150 | -| `linux_wayland/` | 10 | 1,093 | -| `utils/ocr/` | 9 | 1,105 | -| `utils/usbip/` | 5 | 925 | -| `utils/assertion/` | 3 | 866 | -| `osx/` | 17 | 773 | -| `autocontrol-lsp/` | 8 | 752 | -| `utils/hotkey/` | 7 | 734 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,047 | -| **總計** | **989** | **132,598** | +| `wrapper/` | 12 | 2,056 | +| `windows/` | 23 | 1,995 | +| `utils/rest_api/` | 8 | 1,738 | +| `utils/agent/` | 8 | 1,250 | +| `linux_with_x11/` | 19 | 1,175 | +| `linux_wayland/` | 17 | 3,416 | +| `utils/triggers/` | 4 | 1,146 | +| `utils/ocr/` | 9 | 1,112 | +| `utils/usbip/` | 5 | 920 | +| `utils/assertion/` | 3 | 863 | +| `osx/` | 17 | 761 | +| `autocontrol-lsp/` | 8 | 744 | +| `utils/hotkey/` | 7 | 727 | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,238 | +| **總計** | **1,010** | **137,432** | diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index e4c7e063..7a18a802 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -6,9 +6,9 @@ without a compatibility window. | Capability | Status | Windows | Linux X11 | Linux Wayland | macOS | |---|---|---:|---:|---:|---:| -| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb | partial | implementation | +| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb | CI/sway + libeis | implementation | | JSON executor and variables | stable | CI | CI | CI | platform-neutral | -| Image and anchor locators | beta | CI | CI | screenshot-only | implementation | +| Image and anchor locators | beta | CI | CI | implementation | implementation | | Accessibility locator | beta | CI | backend tests | unavailable | backend tests | | Recorder | beta | CI | implementation | unavailable | unavailable | | Reports, trace, failure bundle | stable | CI | CI | CI | platform-neutral | @@ -21,3 +21,121 @@ without a compatibility window. “Implementation” means code exists but the repository does not currently run a real OS runner for it. It must not be interpreted as a production guarantee. Hardware-backed results and known limitations should be attached to releases. + +Linux Wayland is split: **capture is exercised by CI against a real +compositor; input is exercised by CI against a real EI peer and a real portal.** + +Screen capture runs through the compositor's own tool (`grim` on wlroots, +`gnome-screenshot` on GNOME, `spectacle` on KDE), falling back to +`xdg-desktop-portal` over the session bus, instead of the X11-only Pillow/mss +path — +and `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` covers a setup none of those fit. +The `wayland-verification` job in `docker/` runs the whole capture path inside +a headless sway session and checks it against pixels the compositor painted, +which is why this row says CI rather than “implementation”. It runs twice, over +two output layouts: side by side from the origin, and with the left-hand +output at x=-1280 — the layout of any desktop with a monitor left of the +primary one, where the compositor's plane starts at a negative coordinate and +a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of +that monitor. + +One cross-platform difference falls out of the same job, and it is not one this +project can fix: **a Wayland capture may contain the mouse cursor.** No capture +here passes `grim -c`, so none of them asks for the pointer — but wlroots draws +a *software* cursor whenever the backend has no cursor plane, and a software +cursor is composited into the output buffer, which is the buffer +`wlr-screencopy` hands back. Headless is permanently in that state, and so is a +real desktop whose driver offers no cursor plane or whose user set +`WLR_NO_HARDWARE_CURSORS=1`, a common workaround. Windows' BitBlt and the X11 +Pillow/mss path never include the pointer, so this is a Wayland-only +inconsistency rather than something callers already expect: with the pointer +resting on its target, a locator, a template match or an OCR read sees a +pointer-shaped hole in the middle of it. Both ways out need to know where the +pointer is — move it away and back, or mask around it — and Wayland does not let +a client read the cursor position, so the only source would be an in-process +record that goes stale the moment the user touches their own mouse; masking the +wrong place is worse than a visible cursor. So this is documented rather than +worked around: park the pointer away from the region of interest before +capturing. The `seat-verification` job asserts the behaviour as measured, so if +wlroots ever honours `overlay_cursor` for software cursors, CI goes red and says +so. + +Input is verified in four parts, all of which are CI jobs. + +The `eis-verification` job in `docker/` runs AutoControl's real `libei` sender +against a real EIS server — libeis, over a Unix socket, with no compositor +involved — and reads back off the wire what arrived: the capability and +event-type enum values, the variadic seat bind, the key codes, the absolute +coordinates, the button codes, the scroll unit and sign, and a frame per +emission. It also settles the absolute pointer's coordinate space, which is +where the negative-origin layout above reaches the input half: a region's +offset is part of the coordinate rather than something to subtract, and a +motion landing outside every region is dropped by libei without a return code, +an event or an error — so the sender maps the point into region space and +refuses what no region covers, which is what lets the `ydotool` path take it. + +The `ydotool-verification` job covers the CLI fallback, which the `libei` path +drops to at every failure point. A seat is what makes an injected event arrive +somewhere; it is not what makes one observable, so no compositor is needed: +`ydotoold` creates an ordinary uinput device, the kernel publishes it as +`/dev/input/eventN`, and the job reads the `input_event` structs back off it. +That covers the `click` bitmasks, the split press / release edges drag depends +on, what `mousemove --absolute` really puts on the wire, the wheel signs and +axes, numeric key codes, and — in the last check — the argv the mouse and +keyboard backends build for themselves. + +The `seat-verification` job is where an injected event finally reaches a +compositor, and it settles what `mousemove --absolute` is absolute *to*. That +had been recorded as needing a VM running a desktop that consumes libinput +devices; it needs three environment variables instead. wlroots takes +`WLR_BACKENDS=headless,libinput`, so the outputs stay virtual while the input +half is the real libinput backend; libseat's builtin backend opens the device +without logind; and `SEATD_VTBOUND=0` stops it reaching for a VT no container +owns. `grim -c` then draws the cursor into a screenshot, so the compositor +answers in layout coordinates. Two findings come out of it, over the same two +layouts the capture job uses. The origin `--absolute` counts from is the +top-left of the *output layout*, not layout `(0, 0)` — the same distinction +the capture path already makes, and the reason `set_position` now subtracts +`layout_origin()` before calling ydotool. And the displacement is relative +motion, so the compositor's pointer acceleration scales it: libinput's default +adaptive profile moves the cursor twice as far as asked, which is what +ydotool's own `--help` means by "You need to disable mouse speed acceleration +for correct absolute movement". **The ydotool fallback is therefore only +pixel-accurate on a session whose pointer acceleration is off**; the libei +path is absolute at the protocol level and is unaffected. + +The factor is compositor configuration and no client can read it back, so the +library cannot compensate for it — only the operator knows whether it is off. +`JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` is how they say so, and it applies to +the ydotool path alone: unset (or set to anything unrecognised, which says so +and falls back) warns once per process and sends the move anyway, `flat` +declares acceleration off and moves silently, and `strict` refuses the move +rather than let a click land somewhere else. + +The `portal-verification` job covers how a client reaches libei on GNOME and +KDE, which is not a socket path but a file descriptor handed over D-Bus at the +end of the `org.freedesktop.portal.RemoteDesktop` session dance. That had been +recorded as needing a real desktop, on the grounds that no container ships a +RemoteDesktop portal — but the portal is a D-Bus interface, so the job owns the +well-known name itself and runs the real `liboeffis` through the real +handshake, ending in a live connection to the same `libeis` server the +`eis-verification` job uses. It settles the call order and the predicted +request paths, the device mask a user would be consenting to, that the +descriptor carries a real EI session, and that input emitted through it is +recorded at the far end. Every refusal is covered too — a dismissed dialog, a +dialog left open, a withheld descriptor, a closed session, a portal too old for +`ConnectToEIS`, no portal at all — each of which has to fail closed on +AutoControl's own clock. + +What is still not covered is the consent dialog as a *dialog*: no user +dismisses anything in CI, so what a real dialog looks like and how long a real +one blocks stay mutter's business. The compositor also refuses global input +recording, key hooks, cursor-position reads and per-window injection outright; +those are Wayland design decisions, not gaps. See `Progress.md`. + +One packaging note that affects users more than any of the above: ydotool 1.0 +replaced its entire command line, and everything this backend builds arrived +in that release. Debian trixie ships no `ydotool` package; bookworm and every +current Ubuntu ship 0.1.8, which answers this argv with exit code 0 and no +events. AutoControl refuses that version up front rather than reporting +success for input it never sent. diff --git a/examples/22_wayland_backend.py b/examples/22_wayland_backend.py index 2b516cd7..ec9f6c12 100644 --- a/examples/22_wayland_backend.py +++ b/examples/22_wayland_backend.py @@ -21,6 +21,21 @@ Recording, global key listening, and per-window event injection are *not* available on Wayland by design — those calls raise ``NotImplementedError`` with a hint pointing at the X11 fallback. + +Two Wayland-only caveats are worth knowing before a script leans on them: + +* ``ydotool mousemove --absolute`` is relative motion the compositor + accelerates, so an absolute move through that fallback is pixel-exact only + where pointer acceleration is off. The factor cannot be read back, so turn + acceleration off for the ydotoold device and say so with + ``JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat``; ``=strict`` refuses such a + move rather than mispositioning, and leaving it unset warns once per + process and moves anyway. The libei path is absolute and unaffected. +* A capture may contain the mouse cursor. Nothing here asks for it, but + wlroots composites a *software* cursor into the buffer screen capture hands + back whenever the backend has no cursor plane. Windows and X11 never + include the pointer, so park it away from whatever a locator, a template + match or an OCR read is about to look at. """ from je_auto_control.linux_wayland import ( is_wayland_session, missing_dependencies, select_display_server, diff --git a/test/unit_test/headless/test_doc_line_counts.py b/test/unit_test/headless/test_doc_line_counts.py new file mode 100644 index 00000000..aa44bdb1 --- /dev/null +++ b/test/unit_test/headless/test_doc_line_counts.py @@ -0,0 +1,350 @@ +"""Every line count in `architecture_explore.md` is measured, not remembered. + +`CLAUDE.md` requires the map's figures to be re-measured rather than adjusted +by hand. The command / MCP-tool / subpackage / example counts have had a gate +in `test_doc_counts.py`; the *line* counts had none, and drifted two ways at +once. Most table rows counted a phantom trailing line — `len(text.split("\\n"))` +reports one more line than a file that ends in a newline actually has, and one +more *per file* for a package — while the §1 totals and the §8 appendix used +the real count. The same subsystem was therefore quoted at two different sizes +in one document, and roughly fifty rows were additionally just stale. + +This module measures the tree and compares. `len(text.splitlines())` is the +convention it enforces everywhere: it is what `wc -l` reports, what the §8 +appendix already used, and what `CLAUDE.md`'s own over-750-lines snippet +counts. + +Run it directly to rewrite every figure in place rather than hand-editing: + + python test/unit_test/headless/test_doc_line_counts.py --fix +""" +from __future__ import annotations + +import pathlib +import re +import sys +from typing import Dict, List, Optional, Tuple + +ROOT = pathlib.Path(__file__).resolve().parents[3] +PACKAGE = ROOT / "je_auto_control" +DOC = ROOT / "architecture_explore.md" + +FIX_COMMAND = "python test/unit_test/headless/test_doc_line_counts.py --fix" + +# The scan scope quoted in the document header, for the §1 totals. +SCAN_SCOPE = ("je_auto_control", "autocontrol-lsp", "autocontrol_driver", + "AutoControl", "exe", "benchmarks") + +# Only tables whose header names 行數 hold line counts; §1's 指標／數值 table +# holds subpackage and command counts that must not be touched. +_LINE_TABLE_HEADERS = ("| 模組 | 行數 | 職責 |", "| 檔案 | 行數 | 職責 |") +_SIZE_TABLE_HEADER = "| 層／子系統 | 檔案數 | 行數 |" + +_ROW = re.compile(r"^\|\s*`([^`]+)`") +_ONE_NUMBER = re.compile(r"^(\|[^|]*\|\s*)([\d,]+)(\s*\|)") +_TWO_NUMBERS = re.compile(r"^(\|[^|]*\|\s*)([\d,]+)(\s*\|\s*)([\d,]+)(\s*\|)") +_SUBSECTION = re.compile(r"^####\s") +_PATH_TICK = re.compile(r"`([^`]+/)`") +_PAREN = re.compile(r"（[^（）]*）") +_COUNT_OF = re.compile(r"(\d[\d,]*)(\s*(?:檔|行))") +_BLOCKQUOTE = re.compile(r"^> (\d[\d,]*) 個套件、約 (\d[\d,]*) 行。") +_METRIC_TOTAL_LINES = re.compile(r"^(\| 程式碼總行數 \| )([\d,]+)( \|)") +_METRIC_TOTAL_FILES = re.compile(r"^(\| Python 模組總數（含周邊子專案） \| )([\d,]+)( \|)") + +# §8 rows that are not a plain path measurement. +_TOP_LEVEL_ROW = "je_auto_control/" # 頂層 3 檔 — not recursive +_REMAINDER = "其餘模組" +_GRAND_TOTAL = "**總計**" + + +def _lines(path: pathlib.Path) -> int: + return len(path.read_text(encoding="utf-8", errors="ignore").splitlines()) + + +def _py_files(directory: pathlib.Path) -> List[pathlib.Path]: + return [p for p in sorted(directory.rglob("*.py")) + if "__pycache__" not in p.parts] + + +def _measure(target: pathlib.Path, *, recursive: bool = True) -> Tuple[int, int]: + """``(files, lines)`` for a directory tree or a single file.""" + if target.is_file(): + return 1, _lines(target) + files = _py_files(target) if recursive else sorted(target.glob("*.py")) + return len(files), sum(_lines(p) for p in files) + + +def _resolve(name: str, context: Optional[str]) -> Optional[pathlib.Path]: + """Find what a backticked name in the document refers to, or ``None``. + + Names are written relative to ``je_auto_control/`` in most tables, relative + to the repo root in a few (``autocontrol-lsp/``), and as a bare file name + in the §5.4.17 file tables — where the enclosing ``#### `utils/x/``` header + supplies the package. + """ + candidates = [PACKAGE / name, ROOT / name] + if context and "/" not in name: + candidates.insert(0, PACKAGE / context / name) + for candidate in candidates: + if candidate.is_file() or candidate.is_dir(): + return candidate + return None + + +def _replace_numbers(line: str, values: Tuple[int, ...]) -> str: + """Rewrite the one or two numeric cells of a table row.""" + if len(values) == 2: + return _TWO_NUMBERS.sub( + lambda m: f"{m.group(1)}{values[0]:,}{m.group(3)}{values[1]:,}{m.group(5)}", + line, count=1) + return _ONE_NUMBER.sub( + lambda m: f"{m.group(1)}{values[0]:,}{m.group(3)}", line, count=1) + + +def _header_group_package(line: str, group: str) -> Optional[str]: + """The package a ``（…）`` group on a heading describes, if any. + + The path is written either inside the group (``（`windows/`，23 檔…）``) or + immediately before it (`` `utils/usb/`（4,238 行） ``). + """ + inside = _PATH_TICK.search(group) + if inside: + return inside.group(1) + outside = _PATH_TICK.findall(line.split(group)[0]) + return outside[-1] if outside else None + + +def _rewrite_header(line: str) -> str: + """Rewrite the ``（… 檔／… 行）`` figures carried by a `####` heading.""" + result = line + for group in _PAREN.findall(line): + name = _header_group_package(result, group) + target = _resolve(name, None) if name else None + if target is None or not target.is_dir(): + continue + files, total = _measure(target) + result = result.replace(group, _fill_counts(group, files, total), 1) + return result + + +def _fill_counts(text: str, files: int, lines: int) -> str: + """Replace every ``N 檔`` with ``files`` and every ``N 行`` with ``lines``.""" + def _sub(match: "re.Match[str]") -> str: + unit = match.group(2) + return f"{files if '檔' in unit else lines:,}{unit}" + return _COUNT_OF.sub(_sub, text) + + +def _row_name(line: str) -> Optional[str]: + """The first cell of a table row, if it names something measurable.""" + match = _ROW.match(line) + if match: + return match.group(1) + if _REMAINDER in line or _GRAND_TOTAL in line: + return line.split("|")[1].strip() + return None + + +def _table_rows(doc_lines: List[str]) -> Dict[int, Tuple[str, Optional[str]]]: + """Map each measurable table row's index to ``(name, package context)``. + + Only tables headed 行數 are collected. §1's 指標／數值 table holds + subpackage and command counts, which belong to ``test_doc_counts.py`` and + must not be overwritten with a line count. + """ + rows: Dict[int, Tuple[str, Optional[str]]] = {} + context: Optional[str] = None + header: Optional[str] = None + for index, line in enumerate(doc_lines): + if _SUBSECTION.match(line): + paths = _PATH_TICK.findall(line) + context = paths[0].rstrip("/") if paths else None + if not line.startswith("|"): + header = None + continue + stripped = line.rstrip() + if stripped in _LINE_TABLE_HEADERS or stripped == _SIZE_TABLE_HEADER: + header = stripped + continue + name = _row_name(line) if header is not None else None + if name is not None: + rows[index] = (name, context) + return rows + + +def _section_totals(doc_lines: List[str], start: int) -> Optional[Tuple[int, int]]: + """``(packages, lines)`` of the theme table that follows a blockquote.""" + index = start + while index < len(doc_lines) and doc_lines[index].rstrip() not in _LINE_TABLE_HEADERS: + if doc_lines[index].startswith("### "): + return None + index += 1 + if index >= len(doc_lines): + return None + packages = total = 0 + for line in doc_lines[index + 2:]: + if not line.startswith("|"): + break + match = _ROW.match(line) + if not match: + continue + target = _resolve(match.group(1), None) + if target is None: + continue + packages += 1 + total += _measure(target)[1] + return packages, total + + +def _scope_totals() -> Tuple[int, int]: + files = total = 0 + for name in SCAN_SCOPE: + directory = ROOT / name + if not directory.is_dir(): + continue + count, lines = _measure(directory) + files += count + total += lines + return files, total + + +def _rewrite_table_rows(doc_lines: List[str]) -> Dict[str, object]: + """Rewrite every measurable row; report what §8's derived rows need.""" + named_files = named_lines = 0 + derived: Dict[str, object] = {"remainder": None, "total": None} + for index, (name, context) in _table_rows(doc_lines).items(): + line = doc_lines[index] + if name.startswith(_REMAINDER): + derived["remainder"] = index + continue + if name == _GRAND_TOTAL: + derived["total"] = index + continue + two_columns = _TWO_NUMBERS.match(line) is not None + target = _resolve(name, context) + if target is None: + continue + files, total = _measure( + target, recursive=not (two_columns and name == _TOP_LEVEL_ROW)) + if two_columns: + named_files += files + named_lines += total + doc_lines[index] = _replace_numbers( + line, (files, total) if two_columns else (total,)) + derived["named"] = (named_files, named_lines) + return derived + + +def _rewrite_appendix_totals(doc_lines: List[str], + derived: Dict[str, object]) -> None: + """Fill in §8's remainder and grand-total rows. + + Neither is measured on its own: the remainder is whatever the named rows + did not account for, and the total is the appendix's whole scope. Deriving + them is what makes the column actually add up, which it did not before. + """ + files, lines = _measure(PACKAGE) + lsp_files, lsp_lines = _measure(ROOT / "autocontrol-lsp") + files, lines = files + lsp_files, lines + lsp_lines + named_files, named_lines = derived["named"] # type: ignore[misc] + remainder = derived["remainder"] + if remainder is not None: + doc_lines[remainder] = _replace_numbers( + doc_lines[remainder], (files - named_files, lines - named_lines)) + total = derived["total"] + if total is not None: + doc_lines[total] = re.sub( + r"\*\*([\d,]+)\*\* \| \*\*([\d,]+)\*\*", + f"**{files:,}** | **{lines:,}**", doc_lines[total], count=1) + + +def _rewrite_headings(doc_lines: List[str]) -> None: + """Rewrite `####` headings and the ``> N 個套件、約 L 行。`` summaries.""" + for index, line in enumerate(doc_lines): + if _SUBSECTION.match(line): + doc_lines[index] = _rewrite_header(line) + elif _BLOCKQUOTE.match(line): + totals = _section_totals(doc_lines, index) + if totals is not None: + doc_lines[index] = f"> {totals[0]:,} 個套件、約 {totals[1]:,} 行。" + + +def _rewrite_scope_metrics(doc_lines: List[str]) -> None: + """Rewrite §1's module and line totals over the documented scan scope.""" + files, lines = _scope_totals() + for index, line in enumerate(doc_lines): + line = _METRIC_TOTAL_LINES.sub( + lambda m: f"{m.group(1)}{lines:,}{m.group(3)}", line) + doc_lines[index] = _METRIC_TOTAL_FILES.sub( + lambda m: f"{m.group(1)}{files:,}{m.group(3)}", line) + + +def rewrite(text: str) -> str: + """Return ``text`` with every measurable figure replaced by a measurement.""" + doc_lines = text.split("\n") + _rewrite_appendix_totals(doc_lines, _rewrite_table_rows(doc_lines)) + _rewrite_headings(doc_lines) + _rewrite_scope_metrics(doc_lines) + return "\n".join(doc_lines) + + +def mismatches() -> List[str]: + """One line per figure that does not match the tree. + + Only the figures are reported, never the surrounding prose: the document is + Chinese and a console on a cp950 code page cannot print it. + """ + original = DOC.read_text(encoding="utf-8") + fixed = rewrite(original) + if original == fixed: + return [] + found = [] + for number, (before, after) in enumerate( + zip(original.split("\n"), fixed.split("\n")), 1): + if before == after: + continue + was = " / ".join(re.findall(r"[\d,]{2,}", before)) or "?" + now = " / ".join(re.findall(r"[\d,]{2,}", after)) or "?" + found.append(f"line {number}: {was} -> {now}") + return found + + +def test_every_quoted_line_count_matches_the_tree(): + """`architecture_explore.md` quotes measurements, so they have to measure.""" + found = mismatches() + preview = "\n".join(found[:20]) + more = f"\n… and {len(found) - 20} more" if len(found) > 20 else "" + assert not found, ( + f"{len(found)} line-count figures in architecture_explore.md no longer " + f"match the tree. CLAUDE.md says to re-measure rather than adjust by " + f"hand:\n\n {FIX_COMMAND}\n\n{preview}{more}" + ) + + +def test_the_rewriter_is_idempotent(): + """A second pass must be a no-op, or --fix would churn the document.""" + once = rewrite(DOC.read_text(encoding="utf-8")) + assert rewrite(once) == once + + +def _main(argv: List[str]) -> int: + if "--fix" not in argv: + found = mismatches() + for entry in found: + print(entry) + print(f"{len(found)} figures out of date; rerun with --fix" + if found else "every figure matches the tree") + return 1 if found else 0 + original = DOC.read_text(encoding="utf-8") + fixed = rewrite(original) + if original == fixed: + print("every figure already matches the tree") + return 0 + DOC.write_text(fixed, encoding="utf-8") + changed = sum(1 for a, b in zip(original.split("\n"), fixed.split("\n")) if a != b) + print(f"rewrote {changed} lines in {DOC.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(_main(sys.argv[1:])) From 75fcd708dec67c34374c12f5dc399ac4c3b3fbeb Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:48:06 +0800 Subject: [PATCH 14/21] Patch the name libei actually calls for the layout origin The negative-origin check swapped `libei._layout_origin`, which has not existed since the lookup moved to `_layout.py`; `_region_point` calls `layout_origin`, the name libei binds on import. Caught by the eis-verification job on its first run against a real EIS server, which is what that job is for. The swap raises AttributeError on a name that is not there, so the check failed loudly rather than passing while measuring nothing. --- docker/eis_verify.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/eis_verify.py b/docker/eis_verify.py index b4114a50..a2986b9b 100644 --- a/docker/eis_verify.py +++ b/docker/eis_verify.py @@ -447,7 +447,7 @@ def _check_regions_are_read_back(backend, libei) -> str: def _check_offset_region_takes_absolute_coordinates(backend, server, - libei) -> str: + _libei) -> str: """A region at x=1280 takes 1380 for a point 100 pixels into it.""" before = len(server.recording.absolute_motions) backend.set_position(1380, 100) @@ -511,7 +511,9 @@ def _check_negative_origin_is_normalised(backend, server, libei, which no region covers — so it has to arrive as ``(0, 0)``. """ before = len(server.recording.absolute_motions) - with monkey(libei, "_layout_origin", lambda: (-1280, 0)): + # The name bound inside ``libei`` by its ``from _layout import + # layout_origin``, which is what ``_region_point`` actually calls. + with monkey(libei, "layout_origin", lambda: (-1280, 0)): backend.set_position(-1280, 10) _wait_for(lambda: len(server.recording.absolute_motions) > before, 2.0, "the normalised motion") From a46d6138e05af6a9e9ac819e2788c524036d514f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:48:07 +0800 Subject: [PATCH 15/21] Record that the runner can load uinput and evdev All five verification jobs ran on a GitHub runner. modprobe uinput evdev works there and systemd-udevd receives kernel uevents inside the container, which had only been measured locally on a WSL2 kernel. The jobs still fail loudly rather than skipping, so a runner kernel that stops providing either will say so. --- Progress.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/Progress.md b/Progress.md index f0777f8c..42c030d8 100644 --- a/Progress.md +++ b/Progress.md @@ -75,16 +75,15 @@ *產生的東西*:准（Response 0）、拒（Response 1）、以及一直不回答。三種我們都在真的 bus 上跑過,三種都得在自己的時限內收斂。至於真的 mutter 對話框長什麼樣、真人猶豫 三十秒會不會撞到別的東西,那是 mutter 的事,CI 裡沒有人可以去按它。 -- **`ydotool-verification` 與 `seat-verification` 兩個 job 在 GitHub runner 上能不能 - `modprobe uinput evdev`。** 本機（Docker Desktop 的 WSL2 kernel, - `CONFIG_INPUT_EVDEV=m`）兩個都確認跑得完,runner 上還沒跑過;job 寫成模組載不起來 - 就明講失敗,不會靜默跳過。`seat-verification` 還多一個前提:它要 `systemd-udevd` - 在容器裡收得到 kernel uevent。本機收得到（非網路裝置的 uevent 會廣播到所有 - network namespace),runner 上同理但未驗;收不到的話 entrypoint 會在 - `libinput list-devices` 那一步就明講失敗。 ### 已經有答案的（都在 CI 裡,做法見 WHATS_NEW） +五個 job 都在 GitHub runner 上跑過了（2026-08-19,PR #481）。`modprobe uinput evdev` +在 runner 上載得起來,`systemd-udevd` 在容器裡也收得到 kernel uevent——這兩件事原本 +只在本機（Docker Desktop 的 WSL2 kernel）驗過,曾經記在上面當待辦,現在有答案了。 +job 一律寫成模組載不起來就明講失敗,不會靜默跳過,所以哪天 runner 的 kernel 變了會 +當場紅掉。 + | 面向 | 怎麼驗的 | job | | --- | --- | --- | | 擷取路徑 | 真的 wlroots 合成器（sway headless,兩個上不同純色的 output）,27 項 × 2 種版面 | `wayland-verification` | From b117c8d64bcd7a0484d968260c54964be3d12be0 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 21:48:37 +0800 Subject: [PATCH 16/21] Clear the static analysis findings this branch introduced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The one that mattered was the gate: portal_verify built a filesystem path out of a string that had crossed the bus, which reads as user-controlled however the mock produced it. It now builds the path from its own two inputs — the --shot-dir it passed and the file name it expects — and compares the recorded string to that, which is both untainted and a stricter check: a portal writing somewhere else now fails instead of being followed. The rest are quality: * parse_wlr_randr, libei_verify.main and the uinput drain loop were each over the cognitive complexity limit. The first two hold a block of state per iteration, so the per-line fold and the six check closures move out; the third merges two except branches that were always the same branch, since BlockingIOError is an OSError. * _forget_device dropped the copy-then-mutate over the device map for a list of the keys to remove. * eis_server's fixture thread catches Exception rather than BaseException, so an interpreter shutdown is still an exit. * Two dict constructions in _dbus_client, an unused uniform-signature parameter in two places, and a handful of test-only ones: fixtures built before they are installed rather than mutated in place, and constructors hoisted out of pytest.raises blocks so each block names the one call expected to throw. * apt package lists sorted inside the groups they were already in. --- architecture_explore.md | 22 +- docker/Dockerfile.eis | 2 +- docker/Dockerfile.portal | 6 +- docker/Dockerfile.seat | 4 +- docker/Dockerfile.wayland | 2 +- docker/Dockerfile.ydotool | 2 +- docker/eis_server.py | 2 +- docker/libei_verify.py | 301 ++++++++++-------- docker/portal_verify.py | 20 +- docker/ydotool_verify.py | 36 ++- je_auto_control/linux_wayland/_dbus_client.py | 9 +- je_auto_control/linux_wayland/libei.py | 6 +- je_auto_control/linux_wayland/screen.py | 62 ++-- .../utils/executor/flow_data_commands.py | 9 +- .../test_clipboard_win32_prototypes.py | 2 +- .../unit_test/headless/test_screen_grabber.py | 20 +- .../headless/test_wayland_dbus_client.py | 25 +- .../unit_test/headless/test_wayland_oeffis.py | 3 +- .../headless/test_wayland_pointer_accel.py | 3 +- 19 files changed, 304 insertions(+), 232 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index d2283838..c90b5cfd 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,016 | -| 程式碼總行數 | 137,497 | +| 程式碼總行數 | 137,517 | | `je_auto_control/utils/` 子套件數 | 308 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -226,14 +226,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，17 檔／3,416 行） +#### Linux Wayland（`linux_wayland/`，17 檔／3,431 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | | `_detect.py` | 77 | Wayland session 偵測與 CLI 工具探測。 | | `_ydotool_cli.py` | 134 | 判定安裝的是哪一代 ydotool 命令列，擋掉會靜默失效的 0.1.x（對本專案送的 argv 回傳 0 卻不送任何事件）。 | | `_ctypes_bind.py` | 75 | libei／liboeffis 共用的 ctypes 載入與 prototype 綁定。 | -| `_dbus_client.py` | 624 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | +| `_dbus_client.py` | 625 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | | `_select_input.py` | 85 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | | `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | | `oeffis.py` | 196 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | @@ -243,7 +243,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `keymap.py` | 155 | 友善鍵名 → evdev key code。 | | `capture.py` | 236 | 擷取分層：操作者自訂指令 → grim → gnome-screenshot → spectacle → portal。 | | `portal.py` | 207 | `org.freedesktop.portal.Screenshot` 最後備援,經 `_dbus_client` 直接講 D-Bus（不再需要安裝 `gdbus`,只要有 session bus）。 | -| `screen.py` | 252 | 螢幕後端；發布 `grab_image` 與 `layout_origin`（擷取畫面左上角的版面座標，有螢幕在主螢幕左側／上方時為負），全框架的擷取都經由它。 | +| `screen.py` | 266 | 螢幕後端；發布 `grab_image` 與 `layout_origin`（擷取畫面左上角的版面座標，有螢幕在主螢幕左側／上方時為負），全框架的擷取都經由它。 | | `listener.py` / `record.py` | 48 / 34 | 監聽與錄製 stub（Wayland 限制）。 | #### 行動裝置 @@ -265,7 +265,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,650 行。 +> 24 個套件、約 12,655 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -276,7 +276,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/dag/` | 475 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 103 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 96 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 9,070 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/executor/` | 9,075 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 136 | action list 的單步除錯器與追蹤器 | | `utils/input_macro/` | 127 | 定時輸入事件重播與宣告式輸入序列 DSL | | `utils/json/` | 74 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | @@ -687,13 +687,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（9,070 行）— 執行核心 +#### `utils/executor/`（9,075 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `action_executor.py` | 8,125 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | | `flow_control.py` | 530 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | -| `flow_data_commands.py` | 248 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | +| `flow_data_commands.py` | 253 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | @@ -1018,7 +1018,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `gui/` | 89 | 26,542 | | `utils/mcp_server/` | 20 | 16,850 | | `utils/remote_desktop/` | 56 | 11,835 | -| `utils/executor/` | 6 | 9,070 | +| `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,238 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | | `utils/accessibility/` | 12 | 2,390 | @@ -1027,7 +1027,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | | `linux_with_x11/` | 19 | 1,175 | -| `linux_wayland/` | 17 | 3,416 | +| `linux_wayland/` | 17 | 3,431 | | `utils/triggers/` | 4 | 1,146 | | `utils/ocr/` | 9 | 1,112 | | `utils/usbip/` | 5 | 920 | @@ -1036,5 +1036,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,238 | -| **總計** | **1,010** | **137,432** | +| **總計** | **1,010** | **137,452** | diff --git a/docker/Dockerfile.eis b/docker/Dockerfile.eis index 8a4aabae..72c8369b 100644 --- a/docker/Dockerfile.eis +++ b/docker/Dockerfile.eis @@ -57,7 +57,7 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update \ && apt-get install -y --no-install-recommends \ libei1 libeis1 \ - grim wtype wlr-randr \ + grim wlr-randr wtype \ libgl1 libglib2.0-0 \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.portal b/docker/Dockerfile.portal index 664309f3..e23c7f22 100644 --- a/docker/Dockerfile.portal +++ b/docker/Dockerfile.portal @@ -69,9 +69,9 @@ ARG DEBIAN_FRONTEND=noninteractive # libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. RUN apt-get update \ && apt-get install -y --no-install-recommends \ - liboeffis1 libei1 libeis1 \ - dbus python3-gi gir1.2-glib-2.0 \ - grim wtype wlr-randr \ + libei1 libeis1 liboeffis1 \ + dbus gir1.2-glib-2.0 python3-gi \ + grim wlr-randr wtype \ libgl1 libglib2.0-0 \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.seat b/docker/Dockerfile.seat index bb9c8179..75663d2e 100644 --- a/docker/Dockerfile.seat +++ b/docker/Dockerfile.seat @@ -78,8 +78,8 @@ RUN printf 'deb http://deb.debian.org/debian sid main\n' \ && apt-get update \ && apt-get install -y --no-install-recommends -t sid ydotool \ && apt-get install -y --no-install-recommends \ - sway grim wtype wlr-randr \ - udev libinput-tools dmz-cursor-theme \ + grim sway wlr-randr wtype \ + dmz-cursor-theme libinput-tools udev \ libgl1 libglib2.0-0 \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/Dockerfile.wayland b/docker/Dockerfile.wayland index 2750622d..c36524af 100644 --- a/docker/Dockerfile.wayland +++ b/docker/Dockerfile.wayland @@ -54,7 +54,7 @@ ARG DEBIAN_FRONTEND=noninteractive # libgthread-2.0.so.0 at import, so the package cannot even load without them. RUN apt-get update \ && apt-get install -y --no-install-recommends \ - sway grim wtype wlr-randr \ + grim sway wlr-randr wtype \ libei1 \ libgl1 libglib2.0-0 \ ca-certificates \ diff --git a/docker/Dockerfile.ydotool b/docker/Dockerfile.ydotool index fab3098a..b3cebd2b 100644 --- a/docker/Dockerfile.ydotool +++ b/docker/Dockerfile.ydotool @@ -71,7 +71,7 @@ RUN printf 'deb http://deb.debian.org/debian sid main\n' \ && apt-get update \ && apt-get install -y --no-install-recommends -t sid ydotool \ && apt-get install -y --no-install-recommends \ - grim wtype wlr-randr \ + grim wlr-randr wtype \ libgl1 libglib2.0-0 \ ca-certificates \ && rm -rf /var/lib/apt/lists/* diff --git a/docker/eis_server.py b/docker/eis_server.py index 9a7bd6ec..3a7ae7be 100644 --- a/docker/eis_server.py +++ b/docker/eis_server.py @@ -269,7 +269,7 @@ def _serve(self) -> None: continue self._symbols.eis_dispatch(self._eis) self._drain() - except BaseException as error: # noqa: BLE001 # reason: a fixture thread must report, not vanish + except Exception as error: # noqa: BLE001 # reason: a fixture thread must report, not vanish self._error = error def _run_pending(self) -> None: diff --git a/docker/libei_verify.py b/docker/libei_verify.py index 00a7d027..ff6c1e34 100644 --- a/docker/libei_verify.py +++ b/docker/libei_verify.py @@ -78,6 +78,153 @@ def accept_forever() -> None: return server +def _check_every_entry_point_resolves() -> str: + """The check a mock structurally cannot make: does the .so have these?""" + from je_auto_control.linux_wayland import libei + symbols = libei._load_symbols() + if symbols is None: + raise AssertionError( + "not one prototype resolved — either libei.so is absent or a " + "name in _PROTOTYPES does not exist in it") + missing = [name for name, _, _ in libei._PROTOTYPES + if not hasattr(symbols, name)] + if missing: + raise AssertionError(f"unresolved entry points: {missing}") + # The variadic one is bound separately, without argtypes. + if not hasattr(symbols, "ei_seat_bind_capabilities"): + raise AssertionError("ei_seat_bind_capabilities did not resolve") + return f"{len(libei._PROTOTYPES)} prototypes + 1 variadic, all resolved" + + +def _check_each_call_in_isolation(socket_path: str) -> str: + """Walk connect()'s library calls by hand, printing as it goes. + + connect() is half a dozen calls deep; walking them with flushed output + means a crash names the call that caused it rather than the function + that contained it. + """ + from je_auto_control.linux_wayland import libei + symbols = libei._load_symbols() + + def step(message: str) -> None: + print(f" · {message}", flush=True) + + step("ei_new_sender(None) ...") + handle = symbols.ei_new_sender(None) + step(f" -> {handle!r}") + if not handle: + raise AssertionError("ei_new_sender returned NULL") + + step(f"ei_setup_backend_socket(handle, {socket_path!r}) ...") + code = symbols.ei_setup_backend_socket( + handle, socket_path.encode("utf-8")) + step(f" -> {code}") + + step("ei_get_fd(handle) ...") + poll_fd = symbols.ei_get_fd(handle) + step(f" -> {poll_fd}") + + step("ei_dispatch(handle) ...") + symbols.ei_dispatch(handle) + step(" -> returned") + + step("ei_get_event(handle) ...") + event = symbols.ei_get_event(handle) + step(f" -> {event!r}") + while event: + kind = symbols.ei_event_get_type(event) + step(f" event type {kind}") + symbols.ei_event_unref(event) + event = symbols.ei_get_event(handle) + step(f" next -> {event!r}") + + # ei_unref is NOT called here: on this libei it segfaults once the + # backend is open. The sentinel below establishes that separately, + # in a subprocess, so it cannot take this run down with it. + step("(context abandoned — see the ei_unref sentinel)") + return "every call up to teardown behaves" + + +def _check_unref_sentinel(socket_path: str) -> str: + """Is the upstream ei_unref crash this binding works around still there?""" + import subprocess # nosec B404 # reason: argv list, no shell + program = ( + "import ctypes, ctypes.util, os, socket, threading;" + "lib = ctypes.CDLL(ctypes.util.find_library('ei'));" + "lib.ei_new_sender.restype = ctypes.c_void_p;" + "lib.ei_new_sender.argtypes = (ctypes.c_void_p,);" + "lib.ei_setup_backend_socket.restype = ctypes.c_int;" + "lib.ei_setup_backend_socket.argtypes = " + "(ctypes.c_void_p, ctypes.c_char_p);" + "lib.ei_unref.restype = ctypes.c_void_p;" + "lib.ei_unref.argtypes = (ctypes.c_void_p,);" + f"p = {socket_path!r};" + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM);" + "s.connect(p);" + "h = lib.ei_new_sender(None);" + "rc = lib.ei_setup_backend_socket(h, p.encode());" + "assert rc == 0, rc;" + "lib.ei_unref(h)" + ) + finished = subprocess.run([sys.executable, "-c", program], # nosec B603 + capture_output=True) + if finished.returncode == -11: + return ("still segfaults (rc=-11), so the abandon-on-teardown " + "workaround in libei.py::_teardown is still required") + print() + print(" *** REVISIT *** ei_unref no longer crashes on this") + print(" libei (rc=%s). The workaround in LibeiBackend._teardown" + % finished.returncode) + print(" can probably go; see Progress.md.") + print() + return f"no longer crashes (rc={finished.returncode}) — see above" + + +def _check_connect_fails_closed(socket_path: str) -> str: + """A peer that sends nothing must not be reported as a live session.""" + from je_auto_control.linux_wayland import libei + backend = libei.LibeiBackend() + try: + backend.connect(timeout=1.0, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable as error: + return f"LibeiUnavailable: {str(error)[:90]}" + raise AssertionError( + "connect() reported success against a peer that sent nothing, so " + "the handshake is not actually gating on a live device") + + +def _check_teardown_survives(socket_path: str) -> str: + """disconnect() has to be safe after a failed connect, and idempotent.""" + from je_auto_control.linux_wayland import libei + backend = libei.LibeiBackend() + try: + backend.connect(timeout=0.5, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable: + pass + backend.disconnect() # must be safe after a failed connect + backend.disconnect() # and idempotent + return "teardown survived a failed connect, twice" + + +def _check_keyboard_falls_back() -> str: + """With no libei and no ydotool, the CLI path must surface its hint. + + ydotool is deliberately not installed in this image, so what comes back + must be the install hint — not a libei error and not a silent no-op. + """ + from je_auto_control.linux_wayland import keyboard as wl_keyboard + try: + wl_keyboard.press_key(30) + except Exception as error: # noqa: BLE001 # reason: any type is informative + if "ydotool" in str(error): + return f"{type(error).__name__}: {str(error)[:60]}" + raise + raise AssertionError("press_key claimed success with no libei and no " + "ydotool") + + def main() -> int: print("=" * 72) print("AutoControl libei binding — against the real libei.so") @@ -89,24 +236,15 @@ def main() -> int: print("-" * 72) from je_auto_control.linux_wayland import _select_input, libei, oeffis - from je_auto_control.linux_wayland import keyboard as wl_keyboard - # --- the check the mocks structurally cannot make -------------------- - def _symbols(): - symbols = libei._load_symbols() - if symbols is None: - raise AssertionError( - "not one prototype resolved — either libei.so is absent or a " - "name in _PROTOTYPES does not exist in it") - missing = [name for name, _, _ in libei._PROTOTYPES - if not hasattr(symbols, name)] - if missing: - raise AssertionError(f"unresolved entry points: {missing}") - # The variadic one is bound separately, without argtypes. - if not hasattr(symbols, "ei_seat_bind_capabilities"): - raise AssertionError("ei_seat_bind_capabilities did not resolve") - return f"{len(libei._PROTOTYPES)} prototypes + 1 variadic, all resolved" - check("every libei entry point this binding names exists", _symbols) + # --- a real sender against a socket that speaks no EI ---------------- + runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") + socket_path = os.path.join(runtime, "eis-0") + server = serve_silent_socket(socket_path) + print(f" silent EIS stand-in listening at {socket_path}") + + check("every libei entry point this binding names exists", + _check_every_entry_point_resolves) check("LibeiBackend reports the library as available", lambda: _assert_true(libei.LibeiBackend().is_available, @@ -125,116 +263,14 @@ def _symbols(): print(" which is exactly the path exercised below. The portal") print(" route itself is covered by docker/portal_verify.py.)") - # --- a real sender against a socket that speaks no EI ---------------- - runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") - socket_path = os.path.join(runtime, "eis-0") - server = serve_silent_socket(socket_path) - print(f" silent EIS stand-in listening at {socket_path}") - - # --- raw, one call at a time ----------------------------------------- - # connect() is half a dozen library calls deep. Walking them by hand with - # flushed output means a crash names the call that caused it instead of - # just the function that contained it. - def _raw_walk(): - symbols = libei._load_symbols() - step = lambda msg: print(f" · {msg}", flush=True) # noqa: E731 - - step("ei_new_sender(None) ...") - handle = symbols.ei_new_sender(None) - step(f" -> {handle!r}") - if not handle: - raise AssertionError("ei_new_sender returned NULL") - - step(f"ei_setup_backend_socket(handle, {socket_path!r}) ...") - code = symbols.ei_setup_backend_socket( - handle, socket_path.encode("utf-8")) - step(f" -> {code}") - - step("ei_get_fd(handle) ...") - poll_fd = symbols.ei_get_fd(handle) - step(f" -> {poll_fd}") - - step("ei_dispatch(handle) ...") - symbols.ei_dispatch(handle) - step(" -> returned") - - step("ei_get_event(handle) ...") - event = symbols.ei_get_event(handle) - step(f" -> {event!r}") - while event: - kind = symbols.ei_event_get_type(event) - step(f" event type {kind}") - symbols.ei_event_unref(event) - event = symbols.ei_get_event(handle) - step(f" next -> {event!r}") - - # ei_unref is NOT called here: on this libei it segfaults once the - # backend is open. The sentinel below establishes that separately, - # in a subprocess, so it cannot take this run down with it. - step("(context abandoned — see the ei_unref sentinel)") - return "every call up to teardown behaves" - check("each libei call in isolation", _raw_walk) - - # --- the upstream defect this binding works around ------------------- - def _unref_sentinel(): - import subprocess - program = ( - "import ctypes, ctypes.util, os, socket, threading;" - "lib = ctypes.CDLL(ctypes.util.find_library('ei'));" - "lib.ei_new_sender.restype = ctypes.c_void_p;" - "lib.ei_new_sender.argtypes = (ctypes.c_void_p,);" - "lib.ei_setup_backend_socket.restype = ctypes.c_int;" - "lib.ei_setup_backend_socket.argtypes = " - "(ctypes.c_void_p, ctypes.c_char_p);" - "lib.ei_unref.restype = ctypes.c_void_p;" - "lib.ei_unref.argtypes = (ctypes.c_void_p,);" - f"p = {socket_path!r};" - "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM);" - "s.connect(p);" - "h = lib.ei_new_sender(None);" - "rc = lib.ei_setup_backend_socket(h, p.encode());" - "assert rc == 0, rc;" - "lib.ei_unref(h)" - ) - finished = subprocess.run([sys.executable, "-c", program], - capture_output=True) - if finished.returncode == -11: - return ("still segfaults (rc=-11), so the abandon-on-teardown " - "workaround in libei.py::_teardown is still required") - print() - print(" *** REVISIT *** ei_unref no longer crashes on this") - print(" libei (rc=%s). The workaround in LibeiBackend._teardown" - % finished.returncode) - print(" can probably go; see Progress.md.") - print() - return f"no longer crashes (rc={finished.returncode}) — see above" - check("ei_unref after a successful setup — upstream state", _unref_sentinel) - - def _connect_fails_closed(): - backend = libei.LibeiBackend() - try: - backend.connect(timeout=1.0, - socket_path=socket_path.encode("utf-8")) - except libei.LibeiUnavailable as error: - return f"LibeiUnavailable: {str(error)[:90]}" - raise AssertionError( - "connect() reported success against a peer that sent nothing, so " - "the handshake is not actually gating on a live device") + check("each libei call in isolation", + lambda: _check_each_call_in_isolation(socket_path)) + check("ei_unref after a successful setup — upstream state", + lambda: _check_unref_sentinel(socket_path)) check("connect() against a silent peer fails closed, not open", - _connect_fails_closed) - - def _no_crash_on_teardown(): - backend = libei.LibeiBackend() - try: - backend.connect(timeout=0.5, - socket_path=socket_path.encode("utf-8")) - except libei.LibeiUnavailable: - pass - backend.disconnect() # must be safe after a failed connect - backend.disconnect() # and idempotent - return "teardown survived a failed connect, twice" + lambda: _check_connect_fails_closed(socket_path)) check("teardown after a failed handshake does not crash the process", - _no_crash_on_teardown) + lambda: _check_teardown_survives(socket_path)) # --- the fallback the whole design rests on -------------------------- libei.reset_default_backend() @@ -242,21 +278,8 @@ def _no_crash_on_teardown(): lambda: _assert_true(_select_input.active_backend() is None, "active_backend() returned a backend that " "cannot emit")) - - def _keyboard_falls_back(): - # ydotool is deliberately not installed in this image, so the CLI - # path must surface its install hint — not a libei error and not a - # silent no-op. - try: - wl_keyboard.press_key(30) - except Exception as error: # noqa: BLE001 # reason: any type is informative - if "ydotool" in str(error): - return f"{type(error).__name__}: {str(error)[:60]}" - raise - raise AssertionError("press_key claimed success with no libei and no " - "ydotool") check("press_key falls through to the ydotool CLI path", - _keyboard_falls_back) + _check_keyboard_falls_back) server.close() diff --git a/docker/portal_verify.py b/docker/portal_verify.py index 6684c579..bd0c0469 100644 --- a/docker/portal_verify.py +++ b/docker/portal_verify.py @@ -71,6 +71,13 @@ SHOT_SIZE = (4, 3) SHOT_RGB = (0, 128, 255) +#: The file name the mock writes into the ``--shot-dir`` it is given. Both +#: halves of the capture path are therefore this script's own, so the +#: cleanup check below never has to build a path out of what came back over +#: the bus; the equality assertion there is what fails loudly if +#: ``portal_server.SCREENSHOT_NAME`` and this ever drift apart. +SHOT_NAME = "autocontrol portal shot.png" + #: Long enough for a portal that is going to answer; short enough that the #: ones written not to answer do not hold the image up. GRANT_TIMEOUT = 10.0 @@ -211,7 +218,9 @@ def _await_name(self, timeout: float = 15.0) -> None: """Wait until the portal owns the name, not merely until it started.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: - if any("owns " in line for line in list(self.lines)): + # Snapshot: the reader thread appends to this list as we read. + seen = list(self.lines) + if any("owns " in line for line in seen): return if self._process is not None and self._process.poll() is not None: raise RuntimeError( @@ -432,10 +441,13 @@ def _check_capture_returns_the_portal_bytes(wayland_portal, portal: Portal) -> s f"the first pixel is {decoded[0][0]}, not {SHOT_RGB[::-1]}") shot = portal.recorded().get("shot_path", "") _require(bool(shot), "the portal never recorded where it wrote the capture") - _require(not os.path.exists(shot), - f"the portal's file at {shot!r} was left behind") + expected = os.path.join(_runtime_dir(), SHOT_NAME) + _require(shot == expected, + f"the portal wrote {shot!r}, not the {expected!r} it was told to") + _require(not os.path.exists(expected), + f"the portal's file at {expected!r} was left behind") return (f"{len(payload)} bytes, decoded {width}x{height}, " - f"and {os.path.basename(shot)!r} was cleaned up") + f"and {SHOT_NAME!r} was cleaned up") def _check_capture_refused(wayland_portal, needle: str, diff --git a/docker/ydotool_verify.py b/docker/ydotool_verify.py index a530a773..bbfef52d 100644 --- a/docker/ydotool_verify.py +++ b/docker/ydotool_verify.py @@ -129,20 +129,30 @@ def open_all(self) -> List[str]: def drain(self) -> List[Tuple[int, int, int]]: """Return every pending ``(type, code, value)``, SYN frames dropped.""" - events = [] + events: List[Tuple[int, int, int]] = [] for fd in self._fds.values(): - while True: - try: - data = os.read(fd, _EVENT_SIZE) - except BlockingIOError: - break - except OSError: - break - if not data or len(data) < _EVENT_SIZE: - break - _, _, etype, code, value = struct.unpack(_EVENT_FORMAT, data) - if etype != EV_SYN: - events.append((etype, code, value)) + events.extend(self._drain_one(fd)) + return events + + @staticmethod + def _drain_one(fd: int) -> List[Tuple[int, int, int]]: + """Read one input node dry. + + One OSError branch covers both endings: BlockingIOError is a subclass + of it and means nothing more is queued, and anything else means the + node went away — either way this descriptor is done for now. + """ + events: List[Tuple[int, int, int]] = [] + while True: + try: + data = os.read(fd, _EVENT_SIZE) + except OSError: + break + if not data or len(data) < _EVENT_SIZE: + break + _, _, etype, code, value = struct.unpack(_EVENT_FORMAT, data) + if etype != EV_SYN: + events.append((etype, code, value)) return events def close(self) -> None: diff --git a/je_auto_control/linux_wayland/_dbus_client.py b/je_auto_control/linux_wayland/_dbus_client.py index b146eede..6a9d91a5 100644 --- a/je_auto_control/linux_wayland/_dbus_client.py +++ b/je_auto_control/linux_wayland/_dbus_client.py @@ -307,7 +307,7 @@ def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: reader.align(_ALIGNMENT.get(element[0], 1)) items.append(_read_value(reader, _SignatureReader(element))) if element.startswith("{"): - return {key: value for key, value in items} + return dict(items) return items @@ -377,9 +377,10 @@ def _socket_target(address: str) -> Tuple[str, bool]: for candidate in address.split(";"): if not candidate.startswith("unix:"): continue - options = dict( - part.split("=", 1) for part in candidate[len("unix:"):].split(",") - if "=" in part) + fields = [part.split("=", 1) + for part in candidate[len("unix:"):].split(",") + if "=" in part] + options = dict(fields) if "path" in options: return options["path"], False if "abstract" in options: diff --git a/je_auto_control/linux_wayland/libei.py b/je_auto_control/linux_wayland/libei.py index d4a1b2cb..b37491ce 100644 --- a/je_auto_control/linux_wayland/libei.py +++ b/je_auto_control/linux_wayland/libei.py @@ -399,9 +399,9 @@ def _forget_device(self, device: int) -> None: if not device: return self._emulating.pop(device, None) - for cap, known in list(self._devices.items()): - if known == device: - del self._devices[cap] + stale = [cap for cap, known in self._devices.items() if known == device] + for cap in stale: + del self._devices[cap] self._symbols.ei_device_unref(device) def _has_required_devices(self) -> bool: diff --git a/je_auto_control/linux_wayland/screen.py b/je_auto_control/linux_wayland/screen.py index fef24f04..9879514b 100644 --- a/je_auto_control/linux_wayland/screen.py +++ b/je_auto_control/linux_wayland/screen.py @@ -19,7 +19,7 @@ import re from io import BytesIO -from typing import List, Optional, Sequence, Tuple +from typing import List, NamedTuple, Optional, Sequence, Tuple from PIL import Image @@ -168,6 +168,34 @@ def screenshot(file_path: Optional[str] = None, return file_path +class _OutputBlock(NamedTuple): + """One ``wlr-randr`` output block while its fields are being read.""" + + mode: Optional[Tuple[int, int]] = None + position: Optional[Tuple[int, int]] = None + enabled: bool = True + + +def _read_field(block: _OutputBlock, line: str) -> _OutputBlock: + """Fold one indented ``wlr-randr`` field line into the block it belongs to. + + Lines that name none of the three fields leave the block untouched, which + is most of them — modes other than the current one, refresh rates, scale. + """ + enabled_match = _ENABLED_RE.match(line) + if enabled_match: + return block._replace(enabled=enabled_match.group(1).lower() == "yes") + position_match = _POSITION_RE.match(line) + if position_match: + return block._replace(position=(int(position_match.group(1)), + int(position_match.group(2)))) + mode_match = _MODE_RE.search(line) if "current" in line else None + if mode_match: + return block._replace(mode=(int(mode_match.group(1)), + int(mode_match.group(2)))) + return block + + def parse_wlr_randr(text: str) -> List[Tuple[int, int, int, int]]: """Parse ``wlr-randr`` into ``(x, y, width, height)`` per enabled output. @@ -186,34 +214,20 @@ def parse_wlr_randr(text: str) -> List[Tuple[int, int, int, int]]: the whole layout, and the two are composed by the mss-shaped shim. """ rects: List[Tuple[int, int, int, int]] = [] - mode: Optional[Tuple[int, int]] = None - position: Optional[Tuple[int, int]] = None - enabled = True + block = _OutputBlock() - def flush() -> None: - if enabled and mode is not None: - x, y = position if position is not None else (0, 0) - rects.append((x, y, mode[0], mode[1])) + def flush(finished: _OutputBlock) -> None: + if finished.enabled and finished.mode is not None: + x, y = finished.position or (0, 0) + rects.append((x, y, finished.mode[0], finished.mode[1])) for line in text.splitlines(): if line and not line[0].isspace(): - flush() - mode, position, enabled = None, None, True - continue - enabled_match = _ENABLED_RE.match(line) - if enabled_match: - enabled = enabled_match.group(1).lower() == "yes" - continue - position_match = _POSITION_RE.match(line) - if position_match: - position = (int(position_match.group(1)), - int(position_match.group(2))) + flush(block) + block = _OutputBlock() continue - if "current" in line: - mode_match = _MODE_RE.search(line) - if mode_match: - mode = (int(mode_match.group(1)), int(mode_match.group(2))) - flush() + block = _read_field(block, line) + flush(block) return rects diff --git a/je_auto_control/utils/executor/flow_data_commands.py b/je_auto_control/utils/executor/flow_data_commands.py index 03cbb142..e574b410 100644 --- a/je_auto_control/utils/executor/flow_data_commands.py +++ b/je_auto_control/utils/executor/flow_data_commands.py @@ -119,8 +119,13 @@ def exec_sql_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: return {"var": var_name, "fetch": fetch} -def exec_assert_db(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert a scalar SQLite query result satisfies a condition.""" +def exec_assert_db(_executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert a scalar SQLite query result satisfies a condition. + + Takes the executor every ``BLOCK_COMMANDS`` entry is called with and + does not need it: this command asserts on the query, storing nothing. + """ from je_auto_control.utils.assertion import assert_variable from je_auto_control.utils.sql.sql_query import query_sqlite value = query_sqlite(args["database"], args["query"], diff --git a/test/unit_test/headless/test_clipboard_win32_prototypes.py b/test/unit_test/headless/test_clipboard_win32_prototypes.py index c0d281ca..f7060a9c 100644 --- a/test/unit_test/headless/test_clipboard_win32_prototypes.py +++ b/test/unit_test/headless/test_clipboard_win32_prototypes.py @@ -84,7 +84,7 @@ def _round_trip(write, read): return None # unreachable; keeps the return type honest for linters -@pytest.fixture() +@pytest.fixture def clipboard(): """Skip when unusable, and put the user's clipboard back afterwards.""" if not _clipboard_available(): diff --git a/test/unit_test/headless/test_screen_grabber.py b/test/unit_test/headless/test_screen_grabber.py index f4e7ed4a..352e156e 100644 --- a/test/unit_test/headless/test_screen_grabber.py +++ b/test/unit_test/headless/test_screen_grabber.py @@ -72,11 +72,10 @@ class _PlainScreen: def size(self): return (1920, 1080) + stub = type(sys)("stub") + stub.screen = _PlainScreen() with patch.dict(sys.modules, - {"je_auto_control.wrapper.platform_wrapper": - type(sys)("stub")}): - sys.modules["je_auto_control.wrapper.platform_wrapper"].screen = \ - _PlainScreen() + {"je_auto_control.wrapper.platform_wrapper": stub}): assert screen_grabber.backend_grab_image() is None @@ -183,20 +182,19 @@ class _PlainScreen: def size(self): return (1920, 1080) + stub = type(sys)("stub") + stub.screen = _PlainScreen() with patch.dict(sys.modules, - {"je_auto_control.wrapper.platform_wrapper": - type(sys)("stub")}): - sys.modules["je_auto_control.wrapper.platform_wrapper"].screen = \ - _PlainScreen() + {"je_auto_control.wrapper.platform_wrapper": stub}): assert screen_grabber.backend_layout_origin() == (0, 0) def test_backend_layout_origin_reads_a_publishing_backend(): backend = _FakeBackendScreen(size=(2560, 720), origin=(-1280, -200)) + stub = type(sys)("stub") + stub.screen = backend with patch.dict(sys.modules, - {"je_auto_control.wrapper.platform_wrapper": - type(sys)("stub")}): - sys.modules["je_auto_control.wrapper.platform_wrapper"].screen = backend + {"je_auto_control.wrapper.platform_wrapper": stub}): assert screen_grabber.backend_layout_origin() == (-1280, -200) diff --git a/test/unit_test/headless/test_wayland_dbus_client.py b/test/unit_test/headless/test_wayland_dbus_client.py index 77532340..2be6a515 100644 --- a/test/unit_test/headless/test_wayland_dbus_client.py +++ b/test/unit_test/headless/test_wayland_dbus_client.py @@ -153,13 +153,15 @@ def test_an_array_of_structs_survives_the_alignment_it_forces(): def test_a_signature_that_wants_more_members_than_it_was_given_is_rejected(): + writer = _Writer() with pytest.raises(DBusError): - _Writer().value("(us)", (7,)) + writer.value("(us)", (7,)) def test_a_type_this_does_not_marshal_says_so(): + writer = _Writer() with pytest.raises(DBusError, match="cannot marshal"): - _Writer().value("d", 1.5) + writer.value("d", 1.5) # === Whole messages ======================================================== @@ -218,15 +220,17 @@ def test_serials_rise_so_a_reply_can_be_matched_to_its_call(): def test_more_arguments_than_the_signature_declares_is_rejected(): + bus = _bus() with pytest.raises(DBusError, match="more arguments"): - _bus().send(_dbus_client.SIGNAL, - {_dbus_client.FIELD_MEMBER: ("s", "X")}, "s", ["a", "b"]) + bus.send(_dbus_client.SIGNAL, + {_dbus_client.FIELD_MEMBER: ("s", "X")}, "s", ["a", "b"]) def test_fewer_arguments_than_the_signature_declares_is_rejected(): + bus = _bus() with pytest.raises(DBusError, match="fewer arguments"): - _bus().send(_dbus_client.SIGNAL, - {_dbus_client.FIELD_MEMBER: ("s", "X")}, "ss", ["a"]) + bus.send(_dbus_client.SIGNAL, + {_dbus_client.FIELD_MEMBER: ("s", "X")}, "ss", ["a"]) def test_an_implausibly_large_message_is_refused_before_it_is_buffered(): @@ -234,16 +238,18 @@ def test_an_implausibly_large_message_is_refused_before_it_is_buffered(): bus = _bus() bus._socket.readable = struct.pack( "BBBBIII", ord("B"), _dbus_client.SIGNAL, 0, 1, 0, 1, 0) + deadline = _never() with pytest.raises(DBusError, match="big-endian"): - bus.read_message(deadline=_never()) + bus.read_message(deadline=deadline) # === Addresses ============================================================= @@ -273,8 +279,9 @@ def test_availability_follows_the_environment(monkeypatch): def test_connecting_without_an_address_says_which_variable_is_missing(monkeypatch): monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + bus = SessionBus() with pytest.raises(DBusError, match="DBUS_SESSION_BUS_ADDRESS"): - SessionBus().connect() + bus.connect() def test_the_sender_token_is_the_unique_name_as_a_path_element(): diff --git a/test/unit_test/headless/test_wayland_oeffis.py b/test/unit_test/headless/test_wayland_oeffis.py index 98febaf6..7b50939f 100644 --- a/test/unit_test/headless/test_wayland_oeffis.py +++ b/test/unit_test/headless/test_wayland_oeffis.py @@ -145,8 +145,9 @@ class _NoContext(FakeOeffis): def oeffis_new(self, _user_data): return 0 + symbols = _NoContext() with pytest.raises(oeffis_mod.OeffisUnavailable, match="NULL"): - oeffis_mod.connect_eis_fd(symbols=_NoContext()) + oeffis_mod.connect_eis_fd(symbols=symbols) def test_missing_library_reads_as_unavailable(): diff --git a/test/unit_test/headless/test_wayland_pointer_accel.py b/test/unit_test/headless/test_wayland_pointer_accel.py index 49b88b13..bfbf5d2c 100644 --- a/test/unit_test/headless/test_wayland_pointer_accel.py +++ b/test/unit_test/headless/test_wayland_pointer_accel.py @@ -72,7 +72,8 @@ def test_the_warning_names_the_way_out(): """A warning nobody can act on is noise: it must name the variable.""" _, warned = _move() assert wayland_mouse.POINTER_ACCEL_ENV in warned[0] - assert "flat" in warned[0] and "strict" in warned[0] + assert "flat" in warned[0] + assert "strict" in warned[0] def test_warn_does_not_repeat_on_every_move(): From d6661dfdb895587dab2e44aedfc6c53ae78e71ef Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 22:44:44 +0800 Subject: [PATCH 17/21] Give the verification scripts a private directory to write into MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Seven places either defaulted XDG_RUNTIME_DIR to "/tmp" or named a file directly under it — a socket path, a capture, a mock portal's shot directory. Inside the images the fallback never ran, because every Dockerfile sets XDG_RUNTIME_DIR; run one of these by hand and it wrote a predictable name into a directory any user on the host can create entries in, which is what the sockets in particular must not do. They now take XDG_RUNTIME_DIR when it is set, creating it 0700, and a fresh mkdtemp when it is not. The mock portal's --shot-dir defaults to the same rather than to /tmp; the driver has always passed one. --- docker/eis_verify.py | 18 +++++++++++++++++- docker/libei_verify.py | 18 +++++++++++++++++- docker/portal_server.py | 10 +++++++--- docker/portal_verify.py | 16 +++++++++++++--- docker/seat_verify.py | 18 +++++++++++++++++- docker/wayland_verify.py | 6 +++++- 6 files changed, 76 insertions(+), 10 deletions(-) diff --git a/docker/eis_verify.py b/docker/eis_verify.py index a2986b9b..ed1bbae8 100644 --- a/docker/eis_verify.py +++ b/docker/eis_verify.py @@ -38,6 +38,7 @@ import os import subprocess # nosec B404 # reason: runs this interpreter to isolate a known segfault import sys +import tempfile import time import traceback from typing import Any, Callable, List, Tuple @@ -53,6 +54,21 @@ TARGET_POSITION = (640, 400) +def _scratch_dir() -> str: + """A private directory to write into, created 0700 if it is not there. + + The images set ``XDG_RUNTIME_DIR``, so that is what this returns inside + one. Run this script by hand without it and the answer is a fresh + ``mkdtemp`` rather than the ``/tmp`` root itself, which any user on the + host can create entries in. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-verify-") + + def check(name: str, fn: Callable[[], Any]) -> Any: try: detail = fn() @@ -85,7 +101,7 @@ def _wait_for(predicate: Callable[[], bool], timeout: float, def _socket_path() -> str: - runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") # nosec B108 # reason: container fallback only + runtime = _scratch_dir() os.makedirs(runtime, exist_ok=True) path = os.path.join(runtime, "eis-verify") if os.path.exists(path): diff --git a/docker/libei_verify.py b/docker/libei_verify.py index ff6c1e34..ef94bd64 100644 --- a/docker/libei_verify.py +++ b/docker/libei_verify.py @@ -27,6 +27,7 @@ import os import socket import sys +import tempfile import threading import traceback from typing import Any, Callable, List, Tuple @@ -39,6 +40,21 @@ _results: List[Tuple[str, bool]] = [] +def _scratch_dir() -> str: + """A private directory to write into, created 0700 if it is not there. + + The images set ``XDG_RUNTIME_DIR``, so that is what this returns inside + one. Run this script by hand without it and the answer is a fresh + ``mkdtemp`` rather than the ``/tmp`` root itself, which any user on the + host can create entries in. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-verify-") + + def check(name: str, fn: Callable[[], Any]) -> Any: try: detail = fn() @@ -238,7 +254,7 @@ def main() -> int: from je_auto_control.linux_wayland import _select_input, libei, oeffis # --- a real sender against a socket that speaks no EI ---------------- - runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") + runtime = _scratch_dir() socket_path = os.path.join(runtime, "eis-0") server = serve_silent_socket(socket_path) print(f" silent EIS stand-in listening at {socket_path}") diff --git a/docker/portal_server.py b/docker/portal_server.py index 5e1c0575..b9f3c3cd 100644 --- a/docker/portal_server.py +++ b/docker/portal_server.py @@ -39,6 +39,7 @@ import socket import struct import sys +import tempfile import zlib from typing import Any, Dict, Optional @@ -190,13 +191,16 @@ class MockPortal: def __init__(self, eis_socket: str, behaviour: str, record_path: str, version: int = 2, screenshot: str = "grant", - shot_dir: str = "/tmp") -> None: # nosec B108 # reason: container default, overridden by the driver + shot_dir: Optional[str] = None) -> None: self.eis_socket = eis_socket self.behaviour = behaviour self.record_path = record_path self.version = version self.screenshot = screenshot - self.shot_dir = shot_dir + # The driver always passes one; without it, a private + # directory rather than the world-writable /tmp root. + self.shot_dir = shot_dir or tempfile.mkdtemp( + prefix="autocontrol-portal-") self.record: Dict[str, Any] = { "calls": [], "device_types": None, "properties": [], "session_closed_by_client": False, "shot_path": "", @@ -440,7 +444,7 @@ def main() -> int: help="RemoteDesktop interface version to advertise") parser.add_argument("--screenshot", default="grant", choices=SCREENSHOT_BEHAVIOURS) - parser.add_argument("--shot-dir", default="/tmp", # nosec B108 # reason: container default + parser.add_argument("--shot-dir", default=None, help="directory the mock writes its capture into") arguments = parser.parse_args() os.umask(0o077) diff --git a/docker/portal_verify.py b/docker/portal_verify.py index bd0c0469..a6f4c377 100644 --- a/docker/portal_verify.py +++ b/docker/portal_verify.py @@ -45,6 +45,7 @@ import stat import subprocess # nosec B404 # reason: launches dbus-daemon and the mock portal, argv lists, no shell import sys +import tempfile import threading import time import traceback @@ -105,9 +106,18 @@ def _require(condition: bool, message: str) -> None: def _runtime_dir() -> str: - runtime = os.environ.get("XDG_RUNTIME_DIR", "/tmp") # nosec B108 # reason: container fallback only - os.makedirs(runtime, exist_ok=True) - return runtime + """A private directory for the sockets and the capture, created 0700. + + The image sets ``XDG_RUNTIME_DIR``, so that is what this returns inside + one. Run this by hand without it and the answer is a fresh ``mkdtemp`` + rather than the ``/tmp`` root itself, which any user can create entries + in — including the socket names this script predicts. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-portal-verify-") def _fresh_socket_path(name: str) -> str: diff --git a/docker/seat_verify.py b/docker/seat_verify.py index bf7df439..37e7d1cb 100644 --- a/docker/seat_verify.py +++ b/docker/seat_verify.py @@ -50,6 +50,7 @@ import os import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell import sys +import tempfile import time import traceback from typing import Any, Callable, List, Optional, Tuple @@ -84,7 +85,22 @@ #: frame, so the velocity saturates the profile and the factor lands here. _ADAPTIVE_MAX_FACTOR = 2 -CAPTURE = "/tmp/seat-capture.png" # nosec B108 # reason: container-only scratch + +def _scratch_dir() -> str: + """A private directory for the capture, created 0700 if it is not there. + + The image sets ``XDG_RUNTIME_DIR``; without it — running this by hand — + a fresh ``mkdtemp``, rather than a predictable name under the ``/tmp`` + root that any user on the host can create entries in. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-seat-verify-") + + +CAPTURE = os.path.join(_scratch_dir(), "seat-capture.png") def check(name: str, fn: Callable[[], Any]) -> Any: diff --git a/docker/wayland_verify.py b/docker/wayland_verify.py index 7f4ddbf6..6919e3b4 100644 --- a/docker/wayland_verify.py +++ b/docker/wayland_verify.py @@ -32,6 +32,7 @@ import os import subprocess import sys +import tempfile import traceback from dataclasses import dataclass from typing import Any, Callable, Dict, List, Tuple @@ -222,7 +223,10 @@ def check_public_paths(modules: Dict[str, Any], layout: Layout) -> None: def _screenshot_file(): from PIL import Image - path = "/tmp/shot.png" + path = os.path.join( + os.environ.get("XDG_RUNTIME_DIR") + or tempfile.mkdtemp(prefix="autocontrol-wayland-verify-"), + "shot.png") returned = screen.screenshot(path) _assert_eq(returned, path) with Image.open(path) as saved: From bc4c2388e1e42dab83063abc09b5b026e9836273 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 22:49:00 +0800 Subject: [PATCH 18/21] Fetch the unstable ydotool over a verified transport MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The sid source both images add was plain http. apt verifies package signatures either way, so this was not a way in — but the base image already carries the CA certificates pip uses, so there is nothing to trade: `apt-get update` and the `-t sid ydotool` install both complete over https, measured in the same python:3.12-slim these build from. --- docker/Dockerfile.seat | 2 +- docker/Dockerfile.ydotool | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile.seat b/docker/Dockerfile.seat index 75663d2e..9a813f41 100644 --- a/docker/Dockerfile.seat +++ b/docker/Dockerfile.seat @@ -71,7 +71,7 @@ ARG DEBIAN_FRONTEND=noninteractive # - wtype: unused here, but the Wayland backend refuses to import without it # and `import je_auto_control` then falls back to X11 and dies on DISPLAY. # - libgl1 + libglib2.0-0: opencv-python's hard import-time requirements. -RUN printf 'deb http://deb.debian.org/debian sid main\n' \ +RUN printf 'deb https://deb.debian.org/debian sid main\n' \ > /etc/apt/sources.list.d/sid.list \ && printf 'Package: *\nPin: release a=unstable\nPin-Priority: 100\n' \ > /etc/apt/preferences.d/no-sid-by-default \ diff --git a/docker/Dockerfile.ydotool b/docker/Dockerfile.ydotool index b3cebd2b..757dc916 100644 --- a/docker/Dockerfile.ydotool +++ b/docker/Dockerfile.ydotool @@ -64,7 +64,7 @@ ARG DEBIAN_FRONTEND=noninteractive # line of this verification can run. # # libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. -RUN printf 'deb http://deb.debian.org/debian sid main\n' \ +RUN printf 'deb https://deb.debian.org/debian sid main\n' \ > /etc/apt/sources.list.d/sid.list \ && printf 'Package: *\nPin: release a=unstable\nPin-Priority: 100\n' \ > /etc/apt/preferences.d/no-sid-by-default \ From aa38a2e7900bb336b0c5187531574f9d45508337 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Wed, 19 Aug 2026 22:53:38 +0800 Subject: [PATCH 19/21] Justify wayland_verify's subprocess use like its siblings Every other script under docker/ annotates the same argv-list calls with the reason they are safe; this one was the only file where bandit still had something to say, so the harness now reads consistently and a real finding there will stand out. --- docker/wayland_verify.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/wayland_verify.py b/docker/wayland_verify.py index 6919e3b4..c074a33a 100644 --- a/docker/wayland_verify.py +++ b/docker/wayland_verify.py @@ -30,7 +30,7 @@ import json import os -import subprocess +import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell import sys import tempfile import traceback @@ -108,8 +108,9 @@ def note(message: str) -> None: def sway_outputs() -> List[dict]: """The compositor's own description of its outputs.""" - raw = subprocess.run(["swaymsg", "-t", "get_outputs", "-r"], - check=True, stdout=subprocess.PIPE).stdout + raw = subprocess.run( # nosec B603 B607 # nosemgrep + ["swaymsg", "-t", "get_outputs", "-r"], + check=True, stdout=subprocess.PIPE).stdout return json.loads(raw) @@ -157,8 +158,9 @@ def check_detection(modules: Dict[str, Any]) -> None: def check_geometry(screen: Any, layout: Layout) -> None: """wlr-randr's format, the reported size and the origin it implies.""" def _wlr_randr_raw(): - raw = subprocess.run(["wlr-randr"], check=True, - stdout=subprocess.PIPE).stdout.decode() + raw = subprocess.run( # nosec B603 B607 # nosemgrep + ["wlr-randr"], check=True, + stdout=subprocess.PIPE).stdout.decode() print(" wlr-randr prints:") for line in raw.splitlines()[:8]: print(f" | {line}") From 9469627147fb2b92698810ffc734ee05f781ae55 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 00:33:34 +0800 Subject: [PATCH 20/21] Guard the portal's method dispatch on callable, not on None `getattr(self, f"_do_{method}", None)` can return any attribute that happens to carry the name, so checking it against None accepted one that is not callable and crashed a line later. callable() is the check the dispatch actually needs. pylint still reads the call as not-callable, because it infers the type of a dynamic getattr from the default and cannot narrow past a guard; that is what the inline disable says. --- docker/portal_server.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/portal_server.py b/docker/portal_server.py index b9f3c3cd..d98619f4 100644 --- a/docker/portal_server.py +++ b/docker/portal_server.py @@ -267,10 +267,13 @@ def _on_method_call(self, _connection: Gio.DBusConnection, sender: str, invocation: Gio.DBusMethodInvocation) -> None: print(f"portal: {method}{parameters}", flush=True) handler = getattr(self, f"_do_{method}", None) - if handler is None: + if not callable(handler): invocation.return_error_literal( Gio.dbus_error_quark(), Gio.DBusError.UNKNOWN_METHOD, method) return + # pylint: disable=not-callable + # reason: callable() above is the guard; pylint infers the type of a + # dynamic getattr from its default and cannot narrow past the check. handler(sender, parameters, invocation) def _request_path(self, sender: str, options: Dict[str, Any]) -> str: From 6eda3811e71fd30f142efab3ef0aa0f8fbe27248 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 00:58:17 +0800 Subject: [PATCH 21/21] Put the Semgrep waivers on the lines Codacy reports Codacy honours a nosemgrep marker only when it sits on the exact line it reports, and its taint rule reports the tainted argument rather than the call, so a marker on the subprocess.run() line silenced the audit rule and left the taint one standing. Hoist the argument onto the call line so one marker covers both, and mark the two constructors in the ydotool CLI test, which build a result and an exception rather than launching anything. --- docker/eis_verify.py | 5 +++-- docker/libei_verify.py | 3 ++- docker/seat_verify.py | 9 +++++---- docker/ydotool_verify.py | 2 +- je_auto_control/linux_wayland/capture.py | 8 ++++---- test/unit_test/headless/test_wayland_ydotool_cli.py | 5 +++-- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docker/eis_verify.py b/docker/eis_verify.py index ed1bbae8..acd3edb6 100644 --- a/docker/eis_verify.py +++ b/docker/eis_verify.py @@ -408,8 +408,9 @@ def _live_teardown_sentinel(path: str) -> str: "sym.ei_unref(b._ei);" "print('survived')" ) - finished = subprocess.run( # nosec B603 # reason: this interpreter, fixed argv - [sys.executable, "-c", program], capture_output=True, timeout=60) + # This interpreter, running a program built from literals above; no shell. + finished = subprocess.run([sys.executable, "-c", program], # nosec B603 # nosemgrep + capture_output=True, timeout=60) if finished.returncode == 0: return ("safe, which is what _teardown now relies on to release a " "completed session instead of leaking its context") diff --git a/docker/libei_verify.py b/docker/libei_verify.py index ef94bd64..8a8eae76 100644 --- a/docker/libei_verify.py +++ b/docker/libei_verify.py @@ -182,7 +182,8 @@ def _check_unref_sentinel(socket_path: str) -> str: "assert rc == 0, rc;" "lib.ei_unref(h)" ) - finished = subprocess.run([sys.executable, "-c", program], # nosec B603 + # This interpreter, running a program built from literals above; no shell. + finished = subprocess.run([sys.executable, "-c", program], # nosec B603 # nosemgrep capture_output=True) if finished.returncode == -11: return ("still segfaults (rc=-11), so the abandon-on-teardown " diff --git a/docker/seat_verify.py b/docker/seat_verify.py index 37e7d1cb..526aadbb 100644 --- a/docker/seat_verify.py +++ b/docker/seat_verify.py @@ -124,10 +124,11 @@ def _require(condition: bool, message: str) -> None: def _run(argv: List[str], *, timeout: float = 10.0) -> str: """Run a tool from this image and return its stdout.""" - completed = subprocess.run( # nosec B603 # reason: fixed argv, no shell - argv, check=True, timeout=timeout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) + # argv is assembled from literals in this file; no shell, no user input. + completed = subprocess.run(argv, check=True, # nosec B603 # nosemgrep + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) return completed.stdout.decode("utf-8", errors="replace") diff --git a/docker/ydotool_verify.py b/docker/ydotool_verify.py index bbfef52d..f0ffec9d 100644 --- a/docker/ydotool_verify.py +++ b/docker/ydotool_verify.py @@ -320,7 +320,7 @@ def _start_daemon() -> subprocess.Popen: runtime_dir = os.environ.get("XDG_RUNTIME_DIR") if runtime_dir: os.makedirs(runtime_dir, mode=0o700, exist_ok=True) - os.chmod(runtime_dir, 0o700) + os.chmod(runtime_dir, 0o700) # nosemgrep # reason: owner-only is the least ydotoold needs daemon = subprocess.Popen( # nosec B603 B607 # nosemgrep ["ydotoold"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, ) diff --git a/je_auto_control/linux_wayland/capture.py b/je_auto_control/linux_wayland/capture.py index a2672c80..9f454737 100644 --- a/je_auto_control/linux_wayland/capture.py +++ b/je_auto_control/linux_wayland/capture.py @@ -80,10 +80,10 @@ def run_tool(argv: List[str], *, timeout: float = CAPTURE_TIMEOUT) -> bytes: # spectacle / wlr-randr resolved through shutil.which), never user # input; no shell=True. try: - completed = subprocess.run( # nosec B603 # nosemgrep - argv, check=True, timeout=timeout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) + completed = subprocess.run(argv, check=True, # nosec B603 # nosemgrep + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) except subprocess.CalledProcessError as error: message = (error.stderr or b"").decode("utf-8", errors="replace") raise AutoControlScreenException( diff --git a/test/unit_test/headless/test_wayland_ydotool_cli.py b/test/unit_test/headless/test_wayland_ydotool_cli.py index bd6fa40c..18652deb 100644 --- a/test/unit_test/headless/test_wayland_ydotool_cli.py +++ b/test/unit_test/headless/test_wayland_ydotool_cli.py @@ -49,7 +49,7 @@ def _clear_probe_cache(): def _fake_run(stdout=b"", stderr=b"", returncode=0): - completed = subprocess.CompletedProcess( + completed = subprocess.CompletedProcess( # nosemgrep # reason: a result object, not a launch args=["ydotool"], returncode=returncode, stdout=stdout, stderr=stderr) return mock.Mock(return_value=completed) @@ -99,7 +99,8 @@ def test_probe_failure_is_unknown_rather_than_an_accusation(): def test_probe_timeout_is_unknown(): with mock.patch.object( subprocess, "run", - side_effect=subprocess.TimeoutExpired(cmd="ydotool", timeout=5.0)): + side_effect=subprocess.TimeoutExpired( # nosemgrep # reason: an exception object, not a launch + cmd="ydotool", timeout=5.0)): assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ _ydotool_cli.UNKNOWN