From a9865278f6709d1f68fdc72ee3a1c9d4c6d5b61f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 02:04:02 +0800 Subject: [PATCH 01/30] Run the headless suite on the platforms it claims to support The suite ran on windows-2022 alone for its whole life, so every platform assumption inside it went unmeasured on the two operating systems the project also supports. Linux and macOS join the matrix carrying the two ends of the Python range, since what differs between 3.10 and 3.14 is Python and what differs here is the OS. Linux runs under a real Xvfb rather than Qt's offscreen platform: the X11 backend opens a display at import time, and offscreen would hide the breakage this exists to find. --- .github/workflows/quality.yml | 45 ++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index d159b626..b8c5eef1 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -60,11 +60,26 @@ jobs: run: bandit -r je_auto_control/ -c pyproject.toml pytest-headless: - runs-on: windows-2022 + # The suite ran on Windows alone for its whole life, so every + # platform assumption it holds went unmeasured on the two operating + # systems the project also claims to support. Linux and macOS are + # here to measure them; they carry the two ends of the supported + # Python range rather than all five, because what differs between + # 3.10 and 3.14 is Python and what differs here is the OS. + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] + include: + - { os: windows-2022, python-version: "3.10" } + - { os: windows-2022, python-version: "3.11" } + - { os: windows-2022, python-version: "3.12" } + - { os: windows-2022, python-version: "3.13" } + - { os: windows-2022, python-version: "3.14" } + - { os: ubuntu-22.04, python-version: "3.10" } + - { os: ubuntu-22.04, python-version: "3.14" } + - { os: macos-14, python-version: "3.10" } + - { os: macos-14, python-version: "3.14" } steps: - uses: actions/checkout@v4 @@ -74,7 +89,25 @@ jobs: python-version: ${{ matrix.python-version }} cache: "pip" + # Same set the container image installs, and for the same reasons: + # the X11 backend connects to a display at import time, opencv and + # PySide6 hard-require libGL/glib, and Qt's platform plugin needs + # the xcb libraries. Without these the suite fails at collection + # with a linker error rather than a test result. + - name: Install X11 and Qt runtime libraries (Linux) + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + xvfb xauth x11-utils \ + libgl1 libegl1 libglib2.0-0 \ + libxkbcommon-x11-0 libdbus-1-3 \ + libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 \ + libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-sync1 \ + libxcb-xfixes0 libxcb-xinerama0 libxcb-xkb1 + - name: Install dependencies + shell: bash run: | python -m pip install --upgrade pip wheel # Install the editable package FIRST so its source dir is the @@ -91,8 +124,14 @@ jobs: # Paths come from `testpaths` in pyproject.toml. Do NOT pass an explicit # path here: an argument overrides testpaths, which previously meant the # flow_control tests were configured to run but silently never did. + # + # Linux runs under a real Xvfb rather than Qt's offscreen platform: + # the X11 backend opens a display at import time, and offscreen + # would hide exactly the breakage this job exists to find. - name: Run headless pytest suite + shell: bash run: >- + ${{ runner.os == 'Linux' && 'xvfb-run -a -s "-screen 0 1280x800x24"' || '' }} pytest -v --tb=short --timeout=120 --cov=je_auto_control --cov-report=term-missing --cov-report=xml --cov-fail-under=35 @@ -100,7 +139,7 @@ jobs: - name: Upload coverage report uses: actions/upload-artifact@v4 with: - name: coverage-${{ matrix.python-version }} + name: coverage-${{ matrix.os }}-${{ matrix.python-version }} path: coverage.xml typing-stable-api: From fd4fba77c7caf632612bb223076c16c74a7c3872 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 02:35:08 +0800 Subject: [PATCH 02/30] Fix two macOS defects the widened matrix found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit write("\b") had no key route on macOS: the table carries "backspace" but neither "back" nor the raw character, so write() fell through to its space fallback and typed a space where a backspace was asked for. X11 and Wayland both carry the raw character; macOS was the one that did not. system_profiler reports a symbolic vendor id for Apple's own devices — apple_vendor_id, not a number — and that went straight into a field documented as four hex digits. Its leading "a" is a valid hex digit, so a lenient parse turns it into 000a; only a real id can satisfy the contract, so anything else is now None. The device is still listed and its manufacturer still names the vendor. --- je_auto_control/utils/usb/usb_devices.py | 23 ++++++++++----- je_auto_control/wrapper/_platform_osx.py | 6 ++++ test/unit_test/headless/test_usb_devices.py | 31 +++++++++++++++++++++ 3 files changed, 53 insertions(+), 7 deletions(-) diff --git a/je_auto_control/utils/usb/usb_devices.py b/je_auto_control/utils/usb/usb_devices.py index 5357ebc3..e045fa8f 100644 --- a/je_auto_control/utils/usb/usb_devices.py +++ b/je_auto_control/utils/usb/usb_devices.py @@ -242,16 +242,25 @@ def _hex4(value: Any) -> Optional[str]: def _hex4_from_macos(value: Any) -> Optional[str]: + """Parse ``system_profiler``'s vendor/product id into hex, or None. + + Apple's own devices report a *symbolic* id — ``apple_vendor_id`` + rather than a number — and other entries append the vendor name + after it (``0x05ac (Apple Inc.)``). Only a real hex id can satisfy + the 4-hex-digit shape this field documents, so anything else is + None: passing a symbolic name through a field callers parse as hex + is worse than admitting there is no id. The device is still listed, + and its ``manufacturer`` still names the vendor. + """ if value is None: return None text = str(value).strip() - match = re.match(r"0x([0-9A-Fa-f]+)", text) - if match: - try: - return f"{int(match.group(1), 16):04x}" - except ValueError: - return None - return text or None + # The lookahead is what rejects "apple_vendor_id": its leading "a" + # is a valid hex digit, so without it the id would parse as 000a. + match = re.match(r"(?:0x)?([0-9A-Fa-f]{1,4})(?![0-9A-Za-z_])", text) + if match is None: + return None + return f"{int(match.group(1), 16):04x}" def _strip_or_none(value: Any) -> Optional[str]: diff --git a/je_auto_control/wrapper/_platform_osx.py b/je_auto_control/wrapper/_platform_osx.py index 9714a88e..7319d26b 100644 --- a/je_auto_control/wrapper/_platform_osx.py +++ b/je_auto_control/wrapper/_platform_osx.py @@ -97,6 +97,12 @@ "enter": osx_key_enter, "tab": osx_key_tab, "backspace": osx_key_backspace, + # write() routes a control character by name first and by the raw + # character second. Without this entry "\b" matched neither, so + # write() fell through to its space fallback and silently typed a + # space where a backspace was asked for. X11 and Wayland both + # carry the raw character; macOS was the one that did not. + "\b": osx_key_backspace, "esc": osx_key_esc, "command": osx_key_command, "shift": osx_key_shift, diff --git a/test/unit_test/headless/test_usb_devices.py b/test/unit_test/headless/test_usb_devices.py index fa7213a8..487c6b77 100644 --- a/test/unit_test/headless/test_usb_devices.py +++ b/test/unit_test/headless/test_usb_devices.py @@ -46,3 +46,34 @@ def test_vendor_and_product_ids_are_4_hex_chars_when_present(): def test_result_to_dict_count_matches_devices(): result = list_usb_devices() assert result.to_dict()["count"] == len(result.devices) + + +# --- system_profiler id parsing (macOS shapes, checked on every platform) --- + + +def test_symbolic_vendor_id_is_not_smuggled_through_as_hex(): + """Apple's own devices report ``apple_vendor_id`` instead of a number. + + Its leading "a" is a valid hex digit, so a lenient parse turns it into + ``000a`` and a pass-through returns the word itself — both of which break + the 4-hex-digit shape ``UsbDevice`` documents. Only real macOS hardware + produces this, which is why it went unseen while the suite ran on + Windows alone. + """ + from je_auto_control.utils.usb.usb_devices import _hex4_from_macos + + assert _hex4_from_macos("apple_vendor_id") is None + + +def test_hex_ids_from_system_profiler_are_normalised(): + from je_auto_control.utils.usb.usb_devices import _hex4_from_macos + + assert _hex4_from_macos("0x05ac") == "05ac" + assert _hex4_from_macos("0x8600") == "8600" + # system_profiler appends the vendor name to some entries. + assert _hex4_from_macos("0x05AC (Apple Inc.)") == "05ac" + # A bare id, and one that needs padding out to four digits. + assert _hex4_from_macos("05ac") == "05ac" + assert _hex4_from_macos("0x1") == "0001" + assert _hex4_from_macos(None) is None + assert _hex4_from_macos("") is None From b4055cc47fd57803f803a1a0f5f4bcfda5153614 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 02:35:20 +0800 Subject: [PATCH 03/30] Read X11 input back out of a real client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wayland ended up with five jobs that check what reaches a real peer. X11 — the older and more widely deployed of the two Linux paths — had none: every X11 assertion in the suite is made against a mock of python-Xlib, so nothing had confirmed that an injected event reaches a client at all, that it arrives as real input rather than a sent event, or that a captured pixel is the pixel on screen. Ground truth comes from other codebases than the subject: xev is a real X client that prints every event delivered to its window, ImageMagick's import is an independent grabber against a root painted two asymmetric colours, and xdotool and xdpyinfo are the server answering for itself. It runs over one monitor and then two. The assertion worth naming is `synthetic NO`: XSendEvent traffic arrives with YES and is discarded by most toolkits, so a backend that quietly stopped driving real input would still pass any check that only counted events. There is no negative-origin pass because X11 cannot have one — the root window is the union of every monitor and always begins at (0, 0). It also pins a cross-platform split it found: on Linux the scroll direction comes from scroll_direction and the sign of the count is discarded, while Windows and macOS read the direction off that sign. Portable code written against the Windows convention scrolls the wrong way here. Recorded in Progress.md as a decision rather than changed under the maintainer. --- .github/workflows/docker.yml | 48 ++ Progress.md | 20 + architecture_explore.md | 18 +- docker/Dockerfile.x11 | 95 +++ docker/entrypoint-x11.sh | 117 +++ docker/x11_verify.py | 679 ++++++++++++++++++ docs/CAPABILITY_MATRIX.md | 27 +- .../headless/test_docker_artifacts.py | 49 +- 8 files changed, 1042 insertions(+), 11 deletions(-) create mode 100644 docker/Dockerfile.x11 create mode 100644 docker/entrypoint-x11.sh create mode 100644 docker/x11_verify.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d76d7236..d9a58a40 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -241,6 +241,54 @@ jobs: - name: Verify the portal handshake against liboeffis run: docker run --rm autocontrol-portal:ci + x11-verification: + name: X11 backend against a real X server + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the X11 verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.x11 + tags: autocontrol-x11:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # Wayland ended up with five jobs that read back what reached a real + # peer. X11 — the older and more widely deployed of the two Linux + # paths — had none: every X11 assertion in the suite is made against a + # mock of python-Xlib, so nothing had confirmed that an injected event + # reaches a client at all. + # + # Ground truth deliberately comes from other codebases than the one + # under test. xev is a real X client that prints every event delivered + # to its window, so a click is read back the way the ydotool job reads + # its events off /dev/input/eventN — including `synthetic NO`, which is + # what separates real server input from XSendEvent traffic that + # toolkits discard. ImageMagick's `import` is an independent grabber, + # in the role grim plays for Wayland, against a root window painted two + # asymmetric colours so a wrong rectangle cannot look right. + # + # It runs twice: one monitor over the whole screen, then two RANDR + # monitors side by side. There is deliberately no negative-origin pass + # — on X11 the root window is the union of every monitor and always + # starts at (0, 0), so the Wayland job's second layout has no analogue + # here. That is a protocol difference, not an untested case. + # + # The container exits with the number of failed checks. + - name: Verify the X11 backend against a real X server + run: docker run --rm autocontrol-x11:ci + seat-verification: name: ydotool absolute move against a seat that consumes it needs: build-image diff --git a/Progress.md b/Progress.md index 42c030d8..ee67f2c9 100644 --- a/Progress.md +++ b/Progress.md @@ -50,6 +50,26 @@ --- +## `mouse_scroll` 的方向在三個平台上不是同一回事 + +`DECIDE` — `wrapper/auto_control_mouse.py::mouse_scroll` + +Windows 與 macOS 用 `scroll_value` 的**正負號**決定捲動方向；Linux 不看正負號, +方向來自 `scroll_direction` 參數,值只取 `abs()`。後者是刻意的:負值以前會讓 +`range()` 變空,結果是靜靜地什麼都不捲。 + +**問題在於:照 Windows 寫法寫出來的可攜程式碼,在 Linux 上不會往上捲,而是往下捲 +同樣的次數。**沒有例外、沒有警告,方向就是反的——和 macOS 那個 `write("\b")` +變成打空白是同一類缺陷:靜靜地做了跟要求相反的事。 + +`x11-verification` job 現在把**實測到的**行為釘住了（四個方向各一項,外加 +「負號不決定方向」一項）,所以哪天行為變了 CI 會當場說。要不要讓三個平台一致, +以及一致成哪一種,是相容性決定,需要維護者拍板: + +- 讓 Linux 也認正負號 → 修好可攜性,但會改掉 `scroll_direction` 已文件化的語意; +- 維持現狀 → 就得在 `mouse_scroll` 的 docstring 與 README 明講這個平台差異; +- 折衷:正負號在三個平台都認,`scroll_direction` 只在 Linux 當預設方向。 + ## Wayland:剩下的都不是「缺一台機器」 這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, diff --git a/architecture_explore.md b/architecture_explore.md index c90b5cfd..6d4feffd 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,016 | -| 程式碼總行數 | 137,517 | +| 程式碼總行數 | 137,532 | | `je_auto_control/utils/` 子套件數 | 308 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -168,7 +168,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | ---: | --- | | `wrapper/platform_wrapper.py` | 59 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/_platform_windows.py` | 325 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 149 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_osx.py` | 155 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | | `wrapper/_platform_wayland.py` | 57 | Wayland 後端組裝（libei／ydotool／grim）。 | | `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | @@ -505,7 +505,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,703 行。 +> 6 個套件、約 17,712 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -513,7 +513,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/config_sync/` | 245 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | | `utils/remote_desktop/` | 11,835 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,238 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usb/` | 4,247 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 @@ -771,7 +771,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,238 行）與 `utils/usbip/`（920 行） +#### `utils/usb/`（4,247 行）與 `utils/usbip/`（920 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -956,7 +956,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `benchmarks/core_latency.py` | 32 行 | 對穩定無頭進入點的可重複煙霧基準測試。 | | `examples/` | 27 個腳本 | 從截圖點擊、OCR、排程、遠端桌面、agent loop、可觀測性，一路到 computer-use、Wayland、跨主機 DAG、chatops、pytest/BDD、anchor locator。 | | `browser-extension/` | manifest v3 擴充 | 瀏覽器端配合元件（background／content script／popup）。 | -| `docker/` | Dockerfile ×6 + compose + 8 支驗證／伺服器腳本 | 無頭容器（`Dockerfile`）、帶 XFCE 桌面的容器（`Dockerfile.xfce`),以及四個**驗證用**映像:`Dockerfile.wayland`（sway headless,擷取路徑 + `libei_verify.py` 對真的 libei.so 解析符號）、`Dockerfile.eis`（`eis_server.py` 用 ctypes 綁 libeis 起一個真的 EIS server,`eis_verify.py` 把 libei sender 對著它跑完整握手與發送）、`Dockerfile.portal`（`portal_server.py` 自己佔住 `org.freedesktop.portal.Desktop`,真的 `dbus-daemon` + 真的 liboeffis 跑完 RemoteDesktop 交握）、`Dockerfile.ydotool`（真的 uinput 裝置,`ydotool_verify.py` 直接讀回 `/dev/input/eventN`）、`Dockerfile.seat`（`headless,libinput` + builtin seat,合成器真的吃下 ydotool 裝置,`seat_verify.py` 從 `grim -c` 的像素讀回游標落點）。全部接在 `.github/workflows/docker.yml`。 | +| `docker/` | Dockerfile ×8 + compose + 9 支驗證／伺服器腳本 | 無頭容器（`Dockerfile`）、帶 XFCE 桌面的容器（`Dockerfile.xfce`),以及四個**驗證用**映像:`Dockerfile.wayland`（sway headless,擷取路徑 + `libei_verify.py` 對真的 libei.so 解析符號）、`Dockerfile.eis`（`eis_server.py` 用 ctypes 綁 libeis 起一個真的 EIS server,`eis_verify.py` 把 libei sender 對著它跑完整握手與發送）、`Dockerfile.portal`（`portal_server.py` 自己佔住 `org.freedesktop.portal.Desktop`,真的 `dbus-daemon` + 真的 liboeffis 跑完 RemoteDesktop 交握）、`Dockerfile.ydotool`（真的 uinput 裝置,`ydotool_verify.py` 直接讀回 `/dev/input/eventN`）、`Dockerfile.seat`（`headless,libinput` + builtin seat,合成器真的吃下 ydotool 裝置,`seat_verify.py` 從 `grim -c` 的像素讀回游標落點）、`Dockerfile.x11`（真的 Xvfb + openbox,`x11_verify.py` 用 `xev` 把注入的事件從真的客戶端讀回（含 `synthetic NO`,這是 XTest 跟 `XSendEvent` 的差別）,另用 ImageMagick `import` 做獨立擷取對照,跑兩種螢幕版面）。全部接在 `.github/workflows/docker.yml`。 | | `k8s/helm/` | Helm chart | Kubernetes 部署。 | | `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | @@ -1019,10 +1019,10 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/mcp_server/` | 20 | 16,850 | | `utils/remote_desktop/` | 56 | 11,835 | | `utils/executor/` | 6 | 9,075 | -| `utils/usb/` | 17 | 4,238 | +| `utils/usb/` | 17 | 4,247 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | | `utils/accessibility/` | 12 | 2,390 | -| `wrapper/` | 12 | 2,056 | +| `wrapper/` | 12 | 2,062 | | `windows/` | 23 | 1,995 | | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | @@ -1036,5 +1036,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,238 | -| **總計** | **1,010** | **137,452** | +| **總計** | **1,010** | **137,467** | diff --git a/docker/Dockerfile.x11 b/docker/Dockerfile.x11 new file mode 100644 index 00000000..ef3c737f --- /dev/null +++ b/docker/Dockerfile.x11 @@ -0,0 +1,95 @@ +# AutoControl X11 verification image — a real X server, a real WM, real clients. +# +# Wayland got five jobs that check what actually arrives at a real peer. +# X11 — the older, more widely deployed of the two Linux paths — got none: +# every X11 assertion in the suite is made against a mock of python-Xlib, so +# nothing had ever confirmed that an injected event reaches a client, that it +# reaches it as *real* input rather than a sent event, or that a captured +# pixel is the pixel on screen. +# +# Xvfb needs no GPU and no seat, so a plain runner can host a genuine X +# server. Ground truth deliberately comes from other codebases than the one +# under test: +# +# * ``xev`` — a real X client whose window prints every event that reaches +# it. An XTest-injected event has to travel through the server and be +# delivered like any other, so this reads back what arrived, the same way +# the ydotool job reads back ``/dev/input/eventN``. +# * ``import`` (ImageMagick) — an independent screen grabber, standing in +# for the role ``grim`` plays in the Wayland job. +# * ``xdotool`` / ``xdpyinfo`` — the server's own answers for pointer +# position and screen geometry. +# +# openbox rather than a full desktop on purpose: it is EWMH-compliant, which +# is what the window-management checks need, and it does not paint over the +# root window, which is where the pixel checks put their known colours. +# +# What this image CANNOT answer, and why: +# * A negative layout origin. On X11 the root window is the union of every +# monitor and always begins at (0, 0); a monitor placed to the left +# shifts the others right rather than making the origin negative. The +# Wayland job's second layout has no X11 analogue — this is a protocol +# difference, not an untested case. +# * The uinput input path. It needs /dev/uinput, which is the host's to +# grant; docker/Dockerfile.ydotool already reads those events back off +# the kernel device. +# +# Build: docker build -f docker/Dockerfile.x11 -t autocontrol-x11:latest . +# Run: docker run --rm autocontrol-x11:latest + +# Build every wheel in a throwaway stage, as docker/Dockerfile does, so the +# runtime layer installs binaries only and the dependency set is fixed here +# rather than re-resolved against PyPI at run time. +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# - xvfb + xauth: the X server itself, with RANDR so a monitor layout exists. +# - x11-utils: xev (the event reader) and xdpyinfo (the server's geometry). +# - x11-xserver-utils: xrandr, which declares the monitor layout and reads +# it back. Debian ships it separately from x11-utils. +# - xdotool: the server's own answer for where the pointer is. +# - imagemagick: `import -window root`, an independent grabber. +# - openbox: a real EWMH window manager, without a desktop over the root. +# - xterm: a real client to own a real window. +# - libgl1 + libglib2.0-0: opencv-python hard-requires libGL.so.1 and +# libgthread-2.0.so.0 at import, so the package cannot even load without them. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + xvfb xauth x11-utils x11-xserver-utils xdotool \ + imagemagick openbox xterm \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/x11_verify.py /opt/verify/x11_verify.py +COPY docker/entrypoint-x11.sh /usr/local/bin/autocontrol-x11-verify +RUN chmod +x /usr/local/bin/autocontrol-x11-verify + +# Runs as root on purpose: this image exists to run one verification and +# exit. It is not a deployable service image — docker/Dockerfile is. +ENV PYTHONUNBUFFERED=1 \ + DISPLAY=:99 \ + XDG_SESSION_TYPE=x11 \ + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 \ + SCREEN_GEOMETRY=1280x800x24 + +ENTRYPOINT ["/usr/local/bin/autocontrol-x11-verify"] diff --git a/docker/entrypoint-x11.sh b/docker/entrypoint-x11.sh new file mode 100644 index 00000000..5a9909be --- /dev/null +++ b/docker/entrypoint-x11.sh @@ -0,0 +1,117 @@ +#!/bin/sh +# Bring up a real X server with a real window manager, then run the X11 +# verification inside it. +# +# The verification runs twice, over two monitor layouts: +# +# * one monitor covering the whole screen — the single-display desktop; +# * two monitors side by side, declared with `xrandr --setmonitor` — what +# a dual-display desktop reports to its clients. X11 exposes that through +# RANDR monitors rather than through separate screens, so this is the +# shape a client actually sees. +# +# There is deliberately no negative-origin layout: on X11 the root window is +# the union of every monitor and always starts at (0, 0), so a monitor to the +# left shifts the others right instead of moving the origin. The Wayland +# job's second layout has no analogue here. +# +# Nothing below is allowed to skip quietly. If the server, the window manager +# or the monitor layout cannot be brought up, this says so and fails. +set -eu + +GEOMETRY="${SCREEN_GEOMETRY:-1280x800x24}" +DISPLAY_NUM="${DISPLAY:-:99}" + +echo "starting Xvfb on ${DISPLAY_NUM} at ${GEOMETRY}" +Xvfb "${DISPLAY_NUM}" -screen 0 "${GEOMETRY}" +extension RANDR \ + -nolisten tcp >/tmp/xvfb.log 2>&1 & +XVFB_PID=$! + +# xdpyinfo is the server answering for itself, so it is also the readiest +# proof that the server is up — a fixed sleep would be a guess. +ready=0 +for _ in $(seq 1 100); do + if xdpyinfo >/dev/null 2>&1; then + ready=1 + break + fi + if ! kill -0 "$XVFB_PID" 2>/dev/null; then + echo "Xvfb exited before accepting a connection; its log follows:" + cat /tmp/xvfb.log + exit 1 + fi + sleep 0.1 +done +if [ "$ready" -ne 1 ]; then + echo "Xvfb never accepted a connection; its log follows:" + cat /tmp/xvfb.log + exit 1 +fi +echo "Xvfb is up" + +# openbox owns _NET_* — the window-management checks read the properties it +# maintains, so a session without a window manager would test nothing. +openbox >/tmp/openbox.log 2>&1 & +OPENBOX_PID=$! +wm=0 +for _ in $(seq 1 100); do + if xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | grep -q window; then + wm=1 + break + fi + if ! kill -0 "$OPENBOX_PID" 2>/dev/null; then + echo "openbox exited before claiming the screen; its log follows:" + cat /tmp/openbox.log + exit 1 + fi + sleep 0.1 +done +if [ "$wm" -ne 1 ]; then + echo "no window manager claimed the screen; openbox log follows:" + cat /tmp/openbox.log + exit 1 +fi +echo "openbox is managing ${DISPLAY_NUM}" + +# Runs one pass over one monitor layout; echoes the number of failed checks. +run_pass() { + label="$1" + echo "========================================================================" + echo "layout: ${label}" + echo "========================================================================" + xrandr --listmonitors || true + rc=0 + python3 /opt/verify/x11_verify.py || rc=$? + return "$rc" +} + +total=0 + +run_pass "one monitor over the whole screen" || total=$((total + $?)) + +# Split the same screen into two RANDR monitors. Physical size is required by +# the syntax and irrelevant here; `none` means the monitor is backed by no +# physical output, which is exactly what a virtual layout is. +echo +half_width=$(xdpyinfo | awk '/dimensions:/ {split($2, d, "x"); print int(d[1] / 2)}') +height=$(xdpyinfo | awk '/dimensions:/ {split($2, d, "x"); print d[2]}') +if ! xrandr --setmonitor AC-left "${half_width}/169x${height}/211+0+0" none \ + || ! xrandr --setmonitor AC-right \ + "${half_width}/169x${height}/211+${half_width}+0" none; then + echo "FAILED to declare a two-monitor layout with xrandr --setmonitor." + echo "That is a real failure, not a reason to skip: the dual-display" + echo "geometry below is exactly what has never been checked." + exit 1 +fi + +run_pass "two monitors side by side" || total=$((total + $?)) + +echo +echo "========================================================================" +echo "total failed checks across both layouts: ${total}" +echo "========================================================================" + +kill "$OPENBOX_PID" 2>/dev/null || true +kill "$XVFB_PID" 2>/dev/null || true + +exit "$total" diff --git a/docker/x11_verify.py b/docker/x11_verify.py new file mode 100644 index 00000000..803fff84 --- /dev/null +++ b/docker/x11_verify.py @@ -0,0 +1,679 @@ +"""Verify AutoControl's X11 backend against a real X server and real clients. + +Runs inside the session ``entrypoint-x11.sh`` brings up (see +``Dockerfile.x11``). Every X11 assertion in the unit suite is made against a +mock of ``python-Xlib``, so none of it could answer the questions that matter: +does an injected event actually reach a client, does it arrive as *real* input +rather than a sent event most applications ignore, and is a captured pixel the +pixel that is on screen. + +Ground truth deliberately comes from outside the code under test: + +* ``xev`` is a real X client. Its window prints every event delivered to it, + so an XTest-injected click is read back the way the ydotool job reads its + events back off ``/dev/input/eventN`` — including ``synthetic NO``, which is + what separates server-level input from ``XSendEvent`` traffic that toolkits + routinely discard. +* ``import`` (ImageMagick) is an independent grabber, in the role ``grim`` + plays for Wayland. The root window is painted two asymmetric colours first, + because on a uniform screen any wrong rectangle looks right. +* ``xdotool`` and ``xdpyinfo`` are the server answering for itself about + pointer position and geometry. + +The entrypoint runs this twice: once with a single monitor covering the +screen, once with two RANDR monitors side by side. Nothing below is written +for one layout or the other — every coordinate is derived from what the +server reports. + +There is deliberately no negative-origin pass. On X11 the root window is the +union of every monitor and always starts at ``(0, 0)``, so a monitor placed to +the left shifts the others right rather than moving the origin. The Wayland +job's second layout has no analogue here; that is a protocol difference, not +an untested case. + +Exit status is the number of failed checks, so the container's exit code says +whether this passed. +""" +from __future__ import annotations + +import os +import re +import subprocess # nosec B404 # reason: argv lists of fixed tool names, no shell +import sys +import tempfile +import time +import traceback +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple + +#: Painted onto the two halves of the root window. Asymmetric in every +#: channel so a red/blue swap cannot pass, and different from each other so a +#: region grab has something to get wrong. +LEFT_COLOUR = (0x12, 0x34, 0x56) +RIGHT_COLOUR = (0xAB, 0xCD, 0xEF) + +#: How long an injected event is given to come back out of a real client. +EVENT_TIMEOUT = 5.0 + +#: Everything this run writes goes here, as the sibling verification +#: scripts do, rather than into shared /tmp paths another process could +#: already own. +SCRATCH = tempfile.mkdtemp(prefix="autocontrol-x11-verify-") + +_results: List[Tuple[str, bool, str]] = [] + + +def check(name: str, fn: Callable[[], Any]) -> Any: + """Run one check, record pass/fail, and keep going either way.""" + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a failed check must not stop the rest + _results.append((name, False, traceback.format_exc(limit=3).strip())) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True, str(detail))) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def note(message: str) -> None: + """Print an indented remark that is not a check.""" + print(f" {message}") + + +def _run(argv: List[str], *, timeout: float = 15.0) -> str: + """Run one tool from this image and return its stdout.""" + # argv is assembled from literals in this file; no shell, no user input. + completed = subprocess.run(argv, check=True, # nosec B603 B607 # nosemgrep + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return completed.stdout.decode("utf-8", errors="replace") + + +def _assert_eq(actual: Any, expected: Any) -> str: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + return repr(actual) + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +# --- the server's own description of itself -------------------------------- + + +@dataclass(frozen=True) +class Monitor: + """One RANDR monitor, as ``xrandr --listmonitors`` reports it.""" + + name: str + x: int + y: int + width: int + height: int + + def inside(self, dx: int = 40, dy: int = 40) -> Tuple[int, int]: + """A screen coordinate ``(dx, dy)`` into this monitor.""" + return (self.x + dx, self.y + dy) + + +@dataclass(frozen=True) +class Layout: + """The screen and the monitors carved out of it.""" + + width: int + height: int + monitors: List[Monitor] + + @property + def multi(self) -> bool: + """Whether there is more than one monitor to tell apart.""" + return len(self.monitors) > 1 + + +_MONITOR_RE = re.compile( + r"^\s*\d+:\s+\+?\*?(?P\S+)\s+" + r"(?P\d+)/\d+x(?P\d+)/\d+\+(?P\d+)\+(?P\d+)", +) + + +def read_layout() -> Layout: + """Describe the live screen from the server's own answers.""" + dimensions = re.search(r"dimensions:\s+(\d+)x(\d+)", _run(["xdpyinfo"])) + if dimensions is None: + raise RuntimeError("xdpyinfo reported no dimensions") + width, height = int(dimensions.group(1)), int(dimensions.group(2)) + + monitors: List[Monitor] = [] + for line in _run(["xrandr", "--listmonitors"]).splitlines(): + found = _MONITOR_RE.match(line) + if found is None: + continue + monitors.append(Monitor( + name=found.group("name"), + x=int(found.group("x")), y=int(found.group("y")), + width=int(found.group("w")), height=int(found.group("h")), + )) + if not monitors: + # A screen with no RANDR monitor is still a screen; treat the whole + # root window as one rather than inventing geometry. + monitors.append(Monitor("screen", 0, 0, width, height)) + monitors.sort(key=lambda monitor: monitor.x) + return Layout(width=width, height=height, monitors=monitors) + + +def paint_root(layout: Layout) -> None: + """Paint the root window's two halves in the known colours. + + Uses python-Xlib directly. Painting is setup, not subject: what is under + test is whether the *readers* agree with each other and with an + independent grabber about what ended up there. + """ + from Xlib import display as xdisplay + + connection = xdisplay.Display() + screen = connection.screen() + root = screen.root + pixmap = root.create_pixmap(layout.width, layout.height, screen.root_depth) + context = pixmap.create_gc() + half = layout.width // 2 + for colour, left, span in ( + (LEFT_COLOUR, 0, half), + (RIGHT_COLOUR, half, layout.width - half), + ): + context.change(foreground=(colour[0] << 16) | (colour[1] << 8) | colour[2]) + pixmap.fill_rectangle(context, left, 0, span, layout.height) + root.change_attributes(background_pixmap=pixmap) + root.clear_area(x=0, y=0, width=layout.width, height=layout.height) + connection.sync() + # The pixmap must outlive this function: the server keeps a reference for + # the background, but freeing the client-side resource here would take it + # with us. Parking it on the module keeps the background painted. + globals()["_ROOT_PIXMAP"] = (connection, pixmap) + + +def expected_colour(layout: Layout, x: int) -> Tuple[int, int, int]: + """The colour :func:`paint_root` put at screen column ``x``.""" + return LEFT_COLOUR if x < layout.width // 2 else RIGHT_COLOUR + + +def truth_capture(path: Optional[str] = None): + """Grab the root window with ImageMagick and return it as a Pillow image.""" + from PIL import Image + + path = path or os.path.join(SCRATCH, "truth.png") + _run(["import", "-window", "root", "-silent", path]) + with Image.open(path) as opened: + return opened.convert("RGB").copy() + + +# --- a real client that reports what reached it ---------------------------- + + +_EVENT_HEAD_RE = re.compile(r"^(\w+) event,") +_BUTTON_RE = re.compile(r"\bbutton (\d+)\b") +_KEYCODE_RE = re.compile(r"\bkeycode (\d+)\b") +_KEYSYM_RE = re.compile(r"\(keysym 0x[0-9a-f]+, (\S+)\)") +_ROOT_XY_RE = re.compile(r"\broot:\((-?\d+),(-?\d+)\)") +_SYNTHETIC_RE = re.compile(r"\bsynthetic (\w+)\b") + + +class EventTester: + """``xev``: a real X client whose window prints what is delivered to it. + + An event injected through XTest travels through the server and is + dispatched like any other, so what this reads back is the arrival, not the + call. ``xev`` is line-buffered through ``stdbuf`` because its output is a + pipe here rather than a terminal, and a block-buffered log would report + nothing until it filled. + """ + + LOG_PATH = os.path.join(SCRATCH, "xev.log") + WINDOW_NAME = "Event Tester" + + def __init__(self) -> None: + self._process: Optional[subprocess.Popen] = None + self._handle = None + self._offset = 0 + # Text read but not yet forming a whole record. xev is writing while + # this reads, so the tail of any read is routinely half an event: + # parsing it would report a KeyPress whose keycode had not been + # written yet, which looks exactly like a backend that sent no keycode. + self._pending = "" + # Consecutive reads that returned nothing. xev separates records with + # a blank line but writes no terminator after the last one, so the + # most recent event stays unterminated until the *next* one arrives — + # and a press with nothing after it would never be reported at all. + # Two silent reads is what says the record is finished rather than + # half-written. + self._idle_reads = 0 + # Parsed events not yet consumed by a check. Draining must not + # discard what the caller did not ask for — a click writes press and + # release into one chunk, and a reader that returns the press and + # drops the release makes a working click look like a stuck button. + self._events: List[Dict[str, Any]] = [] + self.window_id = 0 + self.rect: Tuple[int, int, int, int] = (0, 0, 0, 0) + + def __enter__(self) -> "EventTester": + self._handle = open(self.LOG_PATH, "wb") # noqa: SIM115 # reason: closed in __exit__ + # stdbuf + xev, both from this image; no shell, no user input. + self._process = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["stdbuf", "-oL", "xev", "-geometry", "400x300+80+80"], + stdout=self._handle, stderr=subprocess.STDOUT) + self.window_id = self._await_window() + self.rect = self._read_geometry() + # The window manager decides where the window lands and what has + # focus, so ask for focus rather than assume the map gave it. + _run(["xdotool", "windowactivate", "--sync", str(self.window_id)]) + _run(["xdotool", "windowfocus", "--sync", str(self.window_id)]) + self.flush() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is not None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + if self._handle is not None: + self._handle.close() + + def _await_window(self) -> int: + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + found = _run(["xdotool", "search", "--name", self.WINDOW_NAME]) + except subprocess.CalledProcessError: + found = "" + ids = [line for line in found.split() if line.isdigit()] + if ids: + return int(ids[-1]) + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + f"xev exited {self._process.returncode} before mapping a window") + time.sleep(0.1) + raise RuntimeError("xev never mapped a window") + + def _read_geometry(self) -> Tuple[int, int, int, int]: + shell = _run(["xdotool", "getwindowgeometry", "--shell", + str(self.window_id)]) + values: Dict[str, int] = {} + for line in shell.splitlines(): + if "=" in line: + key, _, value = line.partition("=") + if value.strip().lstrip("-").isdigit(): + values[key.strip()] = int(value) + return (values.get("X", 0), values.get("Y", 0), + values.get("WIDTH", 0), values.get("HEIGHT", 0)) + + def centre(self) -> Tuple[int, int]: + """The screen coordinate at the middle of the tester's window.""" + x, y, width, height = self.rect + return (x + width // 2, y + height // 2) + + def drain(self) -> List[Dict[str, Any]]: + """Buffer every *complete* record written since the last read.""" + with open(self.LOG_PATH, "rb") as log: + log.seek(self._offset) + raw = log.read() + self._offset += len(raw) + self._idle_reads = 0 if raw else self._idle_reads + 1 + self._pending += raw.decode("utf-8", errors="replace") + # xev separates records with a blank line, so everything after the + # last one may still be being written. Keep it for the next read. + chunks = re.split(r"\n\s*\n", self._pending) + self._pending = chunks.pop() + # ...unless nothing has been written for two reads running and what is + # held ends in a newline. Then it is not half-written, it is the last + # record, and holding it would lose every event that happens to be + # final — which is every press that is waiting for its release. + if (self._idle_reads >= 2 and self._pending.endswith("\n") + and _EVENT_HEAD_RE.match(self._pending.strip())): + chunks.append(self._pending) + self._pending = "" + fresh: List[Dict[str, Any]] = [] + for record in chunks: + head = _EVENT_HEAD_RE.match(record.strip()) + if head is None: + continue + fresh.append(_parse_record(head.group(1), record)) + self._events.extend(fresh) + return fresh + + def flush(self) -> None: + """Forget everything so far, so a check starts from a clean slate.""" + self.drain() + self._events.clear() + + def buffered(self, kind: str, settle: float = 0.5) -> List[Dict[str, Any]]: + """Events of ``kind`` that have arrived and nothing has claimed. + + Used to assert an *absence*, so it drains until the reader has gone + quiet: returning early would report "nothing arrived" for an event + still sitting unterminated in the buffer. + """ + deadline = time.monotonic() + settle + while time.monotonic() < deadline: + self.drain() + time.sleep(0.05) + return [event for event in self._events if event["type"] == kind] + + def collect(self, kind: str, count: int = 1, + timeout: float = EVENT_TIMEOUT) -> List[Dict[str, Any]]: + """Wait for ``count`` events of ``kind``, or raise saying what came. + + What is taken is removed from the buffer and what is not stays there, + so consecutive collects for different kinds see the same arrival. + """ + deadline = time.monotonic() + timeout + while True: + self.drain() + wanted = [event for event in self._events if event["type"] == kind] + if len(wanted) >= count: + taken = wanted[:count] + for event in taken: + self._events.remove(event) + return taken + if time.monotonic() >= deadline: + raise AssertionError( + f"waited {timeout}s for {count}x {kind}; the buffer holds " + f"{[event['type'] for event in self._events]}") + time.sleep(0.05) + + +def _parse_record(kind: str, record: str) -> Dict[str, Any]: + """Turn one xev record into the fields the checks care about.""" + button = _BUTTON_RE.search(record) + keycode = _KEYCODE_RE.search(record) + keysym = _KEYSYM_RE.search(record) + root_xy = _ROOT_XY_RE.search(record) + synthetic = _SYNTHETIC_RE.search(record) + return { + "type": kind, + "button": int(button.group(1)) if button else None, + "keycode": int(keycode.group(1)) if keycode else None, + "keysym": keysym.group(1) if keysym else None, + "root": ((int(root_xy.group(1)), int(root_xy.group(2))) + if root_xy else None), + "synthetic": synthetic.group(1) if synthetic else None, + } + + +# --- checks ---------------------------------------------------------------- + + +def report_environment(layout: Layout) -> None: + """Print what this run is actually working against.""" + print("-" * 72) + note(f"DISPLAY={os.environ.get('DISPLAY')!r} " + f"XDG_SESSION_TYPE={os.environ.get('XDG_SESSION_TYPE')!r}") + note(f"screen: {layout.width}x{layout.height}") + for monitor in layout.monitors: + note(f"monitor {monitor.name}: {monitor.width}x{monitor.height} " + f"at ({monitor.x}, {monitor.y})") + print("-" * 72) + + +def check_backend_selection() -> None: + """The wrapper must land on the X11 backend, not fall through to Wayland.""" + def _selected() -> str: + from je_auto_control.wrapper import platform_wrapper + + module = platform_wrapper.mouse.__name__ + _assert_true("linux_with_x11" in module, + f"expected the X11 backend, got {module}") + return module + check("the platform wrapper selects the X11 backend", _selected) + + +def check_geometry(layout: Layout) -> None: + """``screen_size()`` must be the screen the server reports.""" + def _size() -> str: + from je_auto_control import screen_size + + return _assert_eq(tuple(screen_size()), (layout.width, layout.height)) + check("screen_size() equals the server's dimensions", _size) + + +def check_pixels(layout: Layout) -> None: + """Every reader must agree with an independent grabber and each other.""" + from je_auto_control import get_pixel, screenshot + + truth = truth_capture() + left = layout.monitors[0].inside() + right = layout.monitors[-1].inside() if layout.multi else ( + layout.width - 40, 40) + + def _truth_is_painted() -> str: + for point in (left, right): + _assert_eq(truth.getpixel(point), expected_colour(layout, point[0])) + return f"{left} and {right} carry the painted colours" + check("an independent grabber sees the painted colours", _truth_is_painted) + + def _get_pixel() -> str: + for point in (left, right): + _assert_eq(tuple(get_pixel(*point)), expected_colour(layout, point[0])) + return f"get_pixel agrees at {left} and {right}" + check("get_pixel() returns the colour that is on screen", _get_pixel) + + def _full_screenshot() -> str: + frame = screenshot() + _assert_eq((frame.shape[1], frame.shape[0]), (layout.width, layout.height)) + for point in (left, right): + blue, green, red = frame[point[1], point[0]] + _assert_eq((int(red), int(green), int(blue)), + expected_colour(layout, point[0])) + return f"{frame.shape[1]}x{frame.shape[0]}, BGR order confirmed" + check("screenshot() is the whole screen, in BGR", _full_screenshot) + + def _region_screenshot() -> str: + # A rectangle wholly inside the right-hand colour: if the region were + # ignored, or applied from the wrong origin, this would pick up the + # left-hand colour instead of failing silently on a uniform screen. + x1, y1 = right + x2, y2 = x1 + 60, y1 + 40 + frame = screenshot(screen_region=[x1, y1, x2, y2]) + _assert_eq((frame.shape[1], frame.shape[0]), (x2 - x1, y2 - y1)) + blue, green, red = frame[2, 2] + _assert_eq((int(red), int(green), int(blue)), expected_colour(layout, x1)) + return f"region [{x1},{y1},{x2},{y2}] cropped and coloured correctly" + check("screenshot(screen_region=...) crops from the right origin", + _region_screenshot) + + +def check_pointer(layout: Layout) -> None: + """Where the pointer is put must be where the server says it is.""" + from je_auto_control import get_mouse_position, set_mouse_position + + def _pointer_at(point: Tuple[int, int]) -> str: + set_mouse_position(*point) + shell = _run(["xdotool", "getmouselocation", "--shell"]) + values = dict( + line.split("=", 1) for line in shell.splitlines() if "=" in line) + server = (int(values["X"]), int(values["Y"])) + _assert_eq(server, point) + _assert_eq(tuple(get_mouse_position()), point) + return f"{point} read back from the server and from get_mouse_position" + + first = layout.monitors[0].inside(120, 120) + check("set_mouse_position lands where the server says", + lambda: _pointer_at(first)) + + if layout.multi: + second = layout.monitors[-1].inside(120, 120) + check("a point on the second monitor lands on the second monitor", + lambda: _pointer_at(second)) + else: + note("one monitor in this layout, so there is no second one to reach; " + "the two-monitor pass covers that.") + + +def check_button_events(tester: EventTester) -> None: + """A click must arrive at a real client, as real input, where aimed.""" + from je_auto_control import click_mouse, press_mouse, release_mouse, set_mouse_position + + target = tester.centre() + + def _click() -> str: + set_mouse_position(*target) + tester.flush() + click_mouse("mouse_left") + press = tester.collect("ButtonPress")[0] + release = tester.collect("ButtonRelease")[0] + _assert_eq(press["button"], 1) + _assert_eq(release["button"], 1) + _assert_eq(press["root"], target) + return f"button 1 press+release at {target}" + check("click_mouse reaches a real client at the aimed point", _click) + + def _not_synthetic() -> str: + set_mouse_position(*target) + tester.flush() + click_mouse("mouse_left") + press = tester.collect("ButtonPress")[0] + # XSendEvent traffic arrives with synthetic YES and is discarded by + # most toolkits. XTest goes through the server, so anything that + # started reporting YES here would mean the backend had quietly + # stopped driving real input. + return _assert_eq(press["synthetic"], "NO") + check("injected input is real server input, not a sent event", + _not_synthetic) + + def _split() -> str: + set_mouse_position(*target) + tester.flush() + press_mouse("mouse_right") + press = tester.collect("ButtonPress")[0] + _assert_eq(press["button"], 3) + _assert_true(not tester.buffered("ButtonRelease"), + "a held button must not release itself") + release_mouse("mouse_right") + _assert_eq(tester.collect("ButtonRelease")[0]["button"], 3) + return "button 3 held, then released, as two events" + check("press_mouse holds the button until release_mouse", _split) + + +def check_scroll_events(tester: EventTester) -> None: + """X11 encodes scrolling as buttons 4-7; the direction must pick correctly. + + On Linux the direction comes from the ``scroll_direction`` argument and + the *sign of the value is discarded* — ``mouse_scroll`` takes ``abs()`` + on purpose, because a negative count used to make ``range()`` empty and + scroll nothing at all. Windows and macOS read the direction off that same + sign instead. The checks below pin the behaviour as measured rather than + as one platform spells it; see Progress.md, where the difference is + recorded as a decision for the maintainer. + """ + from je_auto_control import mouse_scroll, set_mouse_position + + target = tester.centre() + + def _scroll(value: int, direction: str, expected_button: int) -> str: + set_mouse_position(*target) + tester.flush() + mouse_scroll(value, scroll_direction=direction) + presses = tester.collect("ButtonPress", abs(value)) + _assert_eq({event["button"] for event in presses}, {expected_button}) + return (f"{direction} x{abs(value)} arrived as button " + f"{expected_button}") + + check("scroll_direction='scroll_up' arrives as button 4", + lambda: _scroll(2, "scroll_up", 4)) + check("scroll_direction='scroll_down' arrives as button 5", + lambda: _scroll(2, "scroll_down", 5)) + check("scroll_direction='scroll_left' arrives as button 6", + lambda: _scroll(1, "scroll_left", 6)) + check("scroll_direction='scroll_right' arrives as button 7", + lambda: _scroll(1, "scroll_right", 7)) + + def _sign_is_ignored() -> str: + set_mouse_position(*target) + tester.flush() + # Portable code written against the Windows sign convention lands + # here. It does not scroll up; it scrolls down. That is the measured + # contract, and this is what would go red if it ever changed. + mouse_scroll(-2, scroll_direction="scroll_down") + presses = tester.collect("ButtonPress", 2) + _assert_eq({event["button"] for event in presses}, {5}) + return "a negative count scrolls the named direction, not the opposite" + check("the sign of the count does not pick the direction on Linux", + _sign_is_ignored) + + +def check_key_events(tester: EventTester) -> None: + """A keystroke must arrive with the keycode the table promised.""" + from je_auto_control import ( + keyboard_keys_table, press_keyboard_key, release_keyboard_key, write, + ) + + def _single() -> str: + tester.flush() + press_keyboard_key("a") + press = tester.collect("KeyPress")[0] + release_keyboard_key("a") + release = tester.collect("KeyRelease")[0] + # The table holds real X keycodes (keysym_to_keycode at import), so + # this checks the whole name -> keysym -> keycode -> wire path. + _assert_eq(press["keycode"], keyboard_keys_table["a"]) + _assert_eq(release["keycode"], keyboard_keys_table["a"]) + _assert_eq(press["keysym"], "a") + return f"'a' arrived as keycode {press['keycode']}" + check("press/release_keyboard_key deliver the mapped keycode", _single) + + def _string() -> str: + tester.flush() + write("xyz") + presses = tester.collect("KeyPress", 3) + _assert_eq([event["keysym"] for event in presses], ["x", "y", "z"]) + return "write('xyz') arrived as x, y, z in order" + check("write() delivers each character in order", _string) + + +def main() -> int: + print("=" * 72) + print("AutoControl X11 verification — real X server, real clients") + print("=" * 72) + + layout = read_layout() + report_environment(layout) + paint_root(layout) + + check_backend_selection() + check_geometry(layout) + check_pixels(layout) + check_pointer(layout) + + with EventTester() as tester: + note(f"event tester window {tester.window_id} at {tester.rect}") + check_button_events(tester) + check_scroll_events(tester) + check_key_events(tester) + + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("A negative layout origin — X11's root window is the union of every") + note(" monitor and always starts at (0, 0). The Wayland job's second") + note(" layout has no analogue here; this is a protocol difference.") + note("The uinput input path — it needs /dev/uinput, which is the host's") + note(" to grant. docker/ydotool_verify.py reads those events back off") + note(" the kernel device instead.") + + failed = [name for name, ok, _ in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index 7a18a802..9f9dc4e4 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -6,7 +6,7 @@ without a compatibility window. | Capability | Status | Windows | Linux X11 | Linux Wayland | macOS | |---|---|---:|---:|---:|---:| -| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb | CI/sway + libeis | implementation | +| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb + xev | CI/sway + libeis | implementation | | JSON executor and variables | stable | CI | CI | CI | platform-neutral | | Image and anchor locators | beta | CI | CI | implementation | implementation | | Accessibility locator | beta | CI | backend tests | unavailable | backend tests | @@ -25,6 +25,31 @@ Hardware-backed results and known limitations should be attached to releases. Linux Wayland is split: **capture is exercised by CI against a real compositor; input is exercised by CI against a real EI peer and a real portal.** +Linux X11 said `CI/Xvfb` for a long time on the strength of a job that +imported the package under `xvfb-run` and generated two lines of code. Nothing +moved a pointer, and every X11 assertion in the suite is made against a mock +of `python-Xlib`, so the questions that matter went unanswered: does an +injected event reach a client at all, does it arrive as *real* input, and is a +captured pixel the pixel on screen. The `x11-verification` job answers them +against a real Xvfb server with a real window manager, and it takes its ground +truth from other codebases than the one under test — `xev`, a real X client +that prints every event delivered to its window; ImageMagick's `import`, an +independent grabber, against a root window painted two asymmetric colours so a +wrong rectangle cannot look right; and `xdotool` / `xdpyinfo`, the server +answering for itself. It runs twice, over one monitor and then two. + +One assertion there is worth naming, because losing it would be silent: +XTest-injected events must arrive with `synthetic NO`. `XSendEvent` traffic +arrives with `synthetic YES` and is discarded by most toolkits, so a backend +that quietly stopped driving real input would still pass every check that only +counted events. + +There is deliberately no negative-origin X11 pass. On X11 the root window is +the union of every monitor and always begins at `(0, 0)`: a monitor placed to +the left shifts the others right rather than moving the origin. The Wayland +job's second layout has no analogue here — a protocol difference, not an +untested case. + Screen capture runs through the compositor's own tool (`grim` on wlroots, `gnome-screenshot` on GNOME, `spectacle` on KDE), falling back to `xdg-desktop-portal` over the session bus, instead of the X11-only Pillow/mss diff --git a/test/unit_test/headless/test_docker_artifacts.py b/test/unit_test/headless/test_docker_artifacts.py index 5abf85c0..168c28e0 100644 --- a/test/unit_test/headless/test_docker_artifacts.py +++ b/test/unit_test/headless/test_docker_artifacts.py @@ -70,7 +70,7 @@ def test_dockerignore_keeps_build_context_lean(): @pytest.mark.parametrize("script", [ "entrypoint.sh", "entrypoint-xfce.sh", "entrypoint-wayland.sh", - "entrypoint-seat.sh", + "entrypoint-seat.sh", "entrypoint-x11.sh", ]) def test_entrypoints_keep_unix_line_endings(script): """A CRLF shebang makes an image that builds and then cannot start. @@ -239,6 +239,53 @@ def test_portal_verification_job_runs_the_image(): assert "autocontrol-portal:ci" in raw +def test_x11_verification_image_and_script_exist(): + """The X11 path is only verified while these three files are wired up.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.x11").read_text(encoding="utf-8") + # Each tool is ground truth from a different codebase than the subject: + # xev reads events back out of a real client, `import` is an independent + # grabber, xdotool/xdpyinfo are the server answering for itself. + for tool in ("xvfb", "x11-utils", "xdotool", "imagemagick", "openbox"): + assert tool in dockerfile, f"Dockerfile.x11 missing {tool}" + assert "x11_verify.py" in dockerfile + assert "entrypoint-x11.sh" in dockerfile + + verify = (_DOCKER_DIR / "x11_verify.py").read_text(encoding="utf-8") + # XSendEvent traffic arrives with `synthetic YES` and is discarded by most + # toolkits. Losing this assertion would let the backend quietly stop + # driving real input while every other check still passed. + assert "synthetic" in verify + # The three readers have to be checked against each other, not just run. + for reader in ("get_pixel", "screenshot", "truth_capture"): + assert reader in verify, f"x11_verify.py never exercises {reader}" + + entrypoint = (_DOCKER_DIR / "entrypoint-x11.sh").read_text(encoding="utf-8") + # A pass with no window manager tests nothing that reads _NET_*, and a + # skipped second layout would silently drop the dual-monitor geometry. + assert "openbox" in entrypoint + assert "--setmonitor" in entrypoint + + +def test_x11_verification_refuses_to_skip_a_missing_layout(): + """A layout that cannot be declared is a failure, not a reason to skip. + + Every other verification job in docker/ fails loudly when its + precondition is absent; a quiet skip reads as coverage that is not there. + """ + entrypoint = (_DOCKER_DIR / "entrypoint-x11.sh").read_text(encoding="utf-8") + assert "exit 1" in entrypoint + assert "not a reason to skip" in entrypoint + + +def test_x11_verification_job_runs_the_image(): + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "x11-verification" in raw + assert "docker/Dockerfile.x11" in raw + assert "autocontrol-x11:ci" in raw + + def test_gitlab_template_covers_build_test_smoke_stages(): raw = (_REPO_ROOT / "ci_templates" / ".gitlab-ci.yml").read_text( encoding="utf-8", From e2c43de5d67c31ab1957127c9b3cc89d1257b1b1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 02:47:35 +0800 Subject: [PATCH 04/30] Measure what a real macOS runner permits, rather than guess MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit macOS is the one supported platform with no container to put it in, and every macOS row in the capability matrix said "implementation": the code was there and nothing had ever run it on a Mac. Two of these capabilities are gated by TCC, which asks a user to grant Screen Recording and Accessibility, and a CI runner has no user to ask. Which of them a runner grants is not something to guess at — guessing is how the Wayland work twice recorded a desktop's refusal as a container's limitation. So the probe runs in --measure mode: it reports what the runner permits and asserts nothing. Its EXPECTED table starts empty and refuses to pass in assert mode while it is, because a gate that asserts nothing reads as coverage that does not exist. Also clears four Sonar findings this branch introduced. Two are the pip lines the matrix change moved: a `run: |` block scalar swallows NOSONAR entirely, so each command becomes its own single-line step that can carry its own justification. The third is the X11 image running as root, which it has no reason to do — unlike the Wayland image, whose XDG_RUNTIME_DIR ownership is the reason that one is privileged. --- .github/workflows/platform-smoke.yml | 30 ++++ .github/workflows/quality.yml | 35 ++-- architecture_explore.md | 2 +- docker/Dockerfile.x11 | 17 +- test/verify/macos_verify.py | 255 +++++++++++++++++++++++++++ 5 files changed, 322 insertions(+), 17 deletions(-) create mode 100644 test/verify/macos_verify.py diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 5f72b736..353a82f3 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -49,3 +49,33 @@ jobs: name: platform-smoke-${{ matrix.os }}-${{ matrix.python-version }} path: platform-smoke.zip if-no-files-found: warn + + macos-capabilities: + name: What a real macOS runner permits + runs-on: macos-14 + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install -e . # NOSONAR githubactions:S8544 # reason: installs the checked-out project itself, there is no upstream version to lock + + # macOS is the one supported platform with no container to put it in, + # and every macOS row in docs/CAPABILITY_MATRIX.md said + # "implementation": the code was there and nothing had run it on a Mac. + # + # Two of these capabilities are gated by TCC — macOS asks a *user* to + # grant Screen Recording and Accessibility, and a CI runner has no user + # to ask. Which of them a runner grants is not something to guess at, + # and guessing is how the Wayland work twice recorded a desktop's + # refusal as a container's limitation. + # + # So this runs in --measure mode: it reports what the runner permits + # and asserts nothing. Once the measurement is in, EXPECTED in the + # script is filled from it and the flag comes off, at which point a + # capability appearing or disappearing turns this red and says which. + - name: Measure the macOS backend against a real window server + run: python test/verify/macos_verify.py --measure diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index b8c5eef1..a7ed579c 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -106,20 +106,29 @@ jobs: libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-sync1 \ libxcb-xfixes0 libxcb-xinerama0 libxcb-xkb1 - - name: Install dependencies + # One command per step, each on a single line: a `run: |` block scalar + # swallows NOSONAR markers, so a justification inside one is a comment + # that reads as handled and suppresses nothing. + - name: Upgrade the installer shell: bash - run: | - python -m pip install --upgrade pip wheel - # Install the editable package FIRST so its source dir is the - # one Python sees on subsequent imports. We deliberately - # avoid `pip install -r dev_requirements.txt` here because - # that file pulls in `je_auto_control_dev` (a separate PyPI - # package), which ships its own snapshot of `je_auto_control/` - # straight into site-packages and masks the editable install - # for any sub-package the snapshot doesn't include - # (admin, usb, remote_desktop, vision, …). - pip install -e . - pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 pytest-cov==7.0.0 PySide6==6.11.1 + run: python -m pip install --upgrade pip wheel # NOSONAR githubactions:S8544 # reason: pip and wheel are the installer; pinning them here would pin the tool that applies the pins below + + # Install the editable package FIRST so its source dir is the one + # Python sees on subsequent imports. We deliberately avoid + # `pip install -r dev_requirements.txt` here because that file pulls in + # `je_auto_control_dev` (a separate PyPI package), which ships its own + # snapshot of `je_auto_control/` straight into site-packages and masks + # the editable install for any sub-package the snapshot doesn't include + # (admin, usb, remote_desktop, vision, …). + - name: Install the project itself + shell: bash + run: pip install -e . # NOSONAR githubactions:S8544 githubactions:S8541 # reason: installs the checked-out project itself, so there is no upstream version to lock and no third-party setup script to run + + - name: Install the test tooling + shell: bash + # Quoted: `--only-binary :all:` puts a colon-space inside the + # scalar, which YAML reads as a mapping and refuses. + run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 pytest-cov==7.0.0 PySide6==6.11.1" # Paths come from `testpaths` in pyproject.toml. Do NOT pass an explicit # path here: an argument overrides testpaths, which previously meant the diff --git a/architecture_explore.md b/architecture_explore.md index 6d4feffd..87dadc95 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -961,7 +961,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | | `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | -| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 466 個 `test_*.py`／4,443 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`verify`、`test_source` | 466 個 `test_*.py`／4,443 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 `verify/macos_verify.py` 是在真的 `macos-14` runner 上量測 TCC 到底允許什麼的探針（macOS 是唯一沒有容器可用的支援平台），不被 pytest 收集。 | --- diff --git a/docker/Dockerfile.x11 b/docker/Dockerfile.x11 index ef3c737f..7f63e07c 100644 --- a/docker/Dockerfile.x11 +++ b/docker/Dockerfile.x11 @@ -84,9 +84,20 @@ COPY docker/x11_verify.py /opt/verify/x11_verify.py COPY docker/entrypoint-x11.sh /usr/local/bin/autocontrol-x11-verify RUN chmod +x /usr/local/bin/autocontrol-x11-verify -# Runs as root on purpose: this image exists to run one verification and -# exit. It is not a deployable service image — docker/Dockerfile is. -ENV PYTHONUNBUFFERED=1 \ +# Unprivileged, unlike the Wayland verification image. That one runs as root +# because XDG_RUNTIME_DIR ownership would otherwise be the only thing +# standing between it and an answer; X11 has no such requirement — the +# server's socket lives in /tmp and its auth file in $HOME, both of which an +# ordinary user owns. Running as a normal user is also one less difference +# between this container and the desktop it stands in for. +# /app is the working directory, and the library opens its log file in the +# cwd at import, so the verifying user has to own it. +RUN useradd --create-home --shell /bin/sh verify \ + && chown -R verify:verify /app +USER verify + +ENV HOME=/home/verify \ + PYTHONUNBUFFERED=1 \ DISPLAY=:99 \ XDG_SESSION_TYPE=x11 \ JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 \ diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py new file mode 100644 index 00000000..348d0526 --- /dev/null +++ b/test/verify/macos_verify.py @@ -0,0 +1,255 @@ +"""Measure what a real macOS runner lets AutoControl's backend actually do. + +macOS is the one supported platform with no container to put it in, so this +runs directly on a ``macos-14`` GitHub runner (see +``.github/workflows/platform-smoke.yml``). It exists because the capability +matrix said "implementation" for every macOS row: the code was there and +nothing had ever run it on a Mac. + +Two of the capabilities here are gated by TCC — macOS asks the *user* to +grant Screen Recording and Accessibility, and a headless CI runner has no +user to ask. Which of them a runner grants by default is not something to +guess at, and guessing is how the Wayland work lost time twice by recording a +desktop's refusal as a container's limitation. So this has two modes: + +``--measure`` + Run every probe, print what happened, and exit 0. Nothing is asserted; + the output is the measurement. + +default + Assert :data:`EXPECTED` — what the measurement showed the runner permits. + Exit status is the number of checks that did not match, so a capability + that appears or disappears turns CI red and says which one. + +The second mode is only honest once the first has run, which is why +:data:`EXPECTED` starts empty and is filled in from a real run rather than +from what the APIs are documented to do. +""" +from __future__ import annotations + +import argparse +import platform +import sys +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +#: What a ``macos-14`` runner was measured to permit. Keys are probe names; +#: values are ``True`` (works), ``False`` (silently does nothing or is +#: refused), or a string naming the exception type it raises. +#: +#: Empty until the first ``--measure`` run fills it in. An empty table makes +#: the assert mode fail loudly rather than pass vacuously. +EXPECTED: Dict[str, Any] = {} + +_results: List[Tuple[str, bool, str]] = [] + + +def note(message: str) -> None: + """Print an indented remark that is not a probe.""" + print(f" {message}") + + +class Outcome: + """What one probe did, in a form both modes can use.""" + + def __init__(self, worked: bool, detail: str, + error: Optional[str] = None) -> None: + self.worked = worked + self.detail = detail + self.error = error + + @property + def key(self) -> Any: + """The value :data:`EXPECTED` records for this outcome.""" + return self.error if self.error else self.worked + + def __str__(self) -> str: + if self.error: + return f"raised {self.error}: {self.detail}" + return ("works — " if self.worked else "no effect — ") + self.detail + + +def probe(name: str, fn: Callable[[], Outcome]) -> Outcome: + """Run one probe and record what it did, without judging it yet.""" + try: + outcome = fn() + except Exception as error: # noqa: BLE001 # reason: measuring, not asserting + outcome = Outcome(False, traceback.format_exc(limit=2).strip().replace( + "\n", " | "), type(error).__name__) + _results.append((name, outcome.worked, str(outcome))) + print(f" {name}: {outcome}") + return outcome + + +# --- probes ---------------------------------------------------------------- + + +def probe_backend() -> Outcome: + """The wrapper must land on the macOS backend at all.""" + from je_auto_control.wrapper import platform_wrapper + + module = platform_wrapper.mouse.__name__ + return Outcome("osx" in module, module) + + +def probe_screen_size() -> Outcome: + """Quartz reports the display size without any TCC grant.""" + from je_auto_control import screen_size + + width, height = screen_size() + return Outcome(width > 0 and height > 0, f"{width}x{height}") + + +def probe_screenshot() -> Outcome: + """Capture needs Screen Recording on 10.15+; a refusal is silent.""" + from je_auto_control import screen_size, screenshot + + width, height = screen_size() + frame = screenshot() + if frame is None or getattr(frame, "size", 0) == 0: + return Outcome(False, "returned an empty frame") + shape = f"{frame.shape[1]}x{frame.shape[0]}" + # A refused capture comes back as a correctly-sized black rectangle + # rather than an error, so size alone proves nothing. + blank = bool((frame == 0).all()) + return Outcome(not blank, + f"{shape} against a {width}x{height} display" + + (", but every pixel is black" if blank else "")) + + +def probe_get_pixel() -> Outcome: + """The single-pixel read goes through the same capture permission.""" + from je_auto_control import get_pixel + + pixel = get_pixel(10, 10) + return Outcome(pixel is not None, repr(pixel)) + + +def probe_mouse_position() -> Outcome: + """Reading the cursor needs no grant; only moving it does.""" + from je_auto_control import get_mouse_position + + position = get_mouse_position() + return Outcome(position is not None, repr(position)) + + +def probe_mouse_move() -> Outcome: + """CGEventPost is accepted whether or not it is allowed to take effect. + + So the move is measured by reading the cursor back, not by whether the + call returned — a refused post raises nothing at all. + """ + from je_auto_control import get_mouse_position, screen_size, set_mouse_position + + width, height = screen_size() + target = (min(width - 5, 137), min(height - 5, 211)) + set_mouse_position(*target) + landed = tuple(get_mouse_position() or (-1, -1)) + return Outcome(landed == target, f"asked {target}, cursor at {landed}") + + +def probe_keyboard() -> Outcome: + """Key posting is the most restricted of all; measure, do not assume. + + Nothing here has focus, so what is being measured is whether the post is + accepted and the modifier state changes — not where the character went. + """ + from je_auto_control import check_key_is_press, press_keyboard_key, release_keyboard_key + + press_keyboard_key("shift") + try: + held = check_key_is_press("shift") + finally: + release_keyboard_key("shift") + return Outcome(bool(held), f"check_key_is_press('shift') returned {held!r}") + + +def probe_accessibility() -> Outcome: + """The AX tree needs Accessibility; without it the walk returns nothing.""" + from je_auto_control.utils.accessibility.backends import get_backend + + backend = get_backend() + if not backend.available: + return Outcome(False, f"backend {backend.name!r} reports unavailable") + elements = backend.list_elements(max_results=5) + return Outcome(bool(elements), + f"backend {backend.name!r} returned {len(elements)} elements") + + +def probe_recorder() -> Outcome: + """macOS ships no recorder, and must say so rather than look broken. + + ``osx/record/osx_record.py`` exists, but wiring it up would put an + ``NSApplication`` and a blocking run loop into import of the platform + wrapper, so ``recorder`` is None on purpose. This pins that it is a + deliberate absence and not something that quietly stopped working. + """ + from je_auto_control.wrapper import platform_wrapper + + return Outcome(platform_wrapper.recorder is None, + f"recorder is {platform_wrapper.recorder!r}") + + +PROBES: List[Tuple[str, Callable[[], Outcome]]] = [ + ("backend-selection", probe_backend), + ("screen-size", probe_screen_size), + ("screenshot", probe_screenshot), + ("get-pixel", probe_get_pixel), + ("mouse-position", probe_mouse_position), + ("mouse-move", probe_mouse_move), + ("keyboard-post", probe_keyboard), + ("accessibility-tree", probe_accessibility), + ("recorder-absent", probe_recorder), +] + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--measure", action="store_true", + help="report what the runner permits and exit 0, asserting nothing") + options = parser.parse_args(argv) + + print("=" * 72) + print("AutoControl macOS verification — real macOS, real window server") + print("=" * 72) + note(f"{platform.platform()} python {platform.python_version()}") + print("-" * 72) + + outcomes = {name: probe(name, fn) for name, fn in PROBES} + + print("-" * 72) + if options.measure: + print("measurement only — nothing asserted. EXPECTED would be:") + print() + for name, outcome in outcomes.items(): + print(f' "{name}": {outcome.key!r},') + print() + print("Paste that into EXPECTED and drop --measure to make it a gate.") + print("=" * 72) + return 0 + + if not EXPECTED: + print("EXPECTED is empty, so there is nothing to assert. Run with") + print("--measure first and fill it in; passing here would mean") + print("nothing and would read as coverage that does not exist.") + print("=" * 72) + return 1 + + failed = [] + for name, outcome in outcomes.items(): + if name not in EXPECTED: + failed.append(f"{name}: not in EXPECTED (a new probe?)") + elif outcome.key != EXPECTED[name]: + failed.append( + f"{name}: expected {EXPECTED[name]!r}, measured {outcome.key!r}") + print(f"{len(outcomes) - len(failed)}/{len(outcomes)} probes match " + f"what this runner was measured to permit") + for line in failed: + print(f" CHANGED: {line}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) From cb1fe40a2b245699ca86d837e4d4c8cd9b81fc59 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 02:59:47 +0800 Subject: [PATCH 05/30] Turn the macOS probe into a gate, now that it has been measured A macos-14 runner grants both Screen Recording and Accessibility, so every capability the probe covers works on one: capture returns real pixels rather than the black rectangle a refusal produces, CGEventPost moves the cursor and the move reads back exactly, and the AX walk returns real elements. The usual assumption that CI cannot exercise a TCC-gated macOS API is wrong for this runner, which is the whole reason it was measured instead of reasoned about. EXPECTED now holds that measurement and the --measure flag comes off, so a capability appearing or disappearing turns the job red and names it. --- .github/workflows/platform-smoke.yml | 19 ++++++++---- test/verify/macos_verify.py | 46 ++++++++++++++++++++-------- 2 files changed, 47 insertions(+), 18 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 353a82f3..bfdcd9fb 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -73,9 +73,16 @@ jobs: # and guessing is how the Wayland work twice recorded a desktop's # refusal as a container's limitation. # - # So this runs in --measure mode: it reports what the runner permits - # and asserts nothing. Once the measurement is in, EXPECTED in the - # script is filled from it and the flag comes off, at which point a - # capability appearing or disappearing turns this red and says which. - - name: Measure the macOS backend against a real window server - run: python test/verify/macos_verify.py --measure + # Measured first, in --measure mode, and the answer was a surprise: a + # macos-14 runner grants BOTH Screen Recording and Accessibility, so + # every capability works — capture returns real pixels rather than the + # black rectangle a refusal produces, CGEventPost moves the cursor and + # the move reads back exactly, and the AX walk returns real elements. + # The usual assumption that CI cannot exercise a TCC-gated macOS API is + # simply wrong for this runner. + # + # So the flag is off and this is a gate now: EXPECTED in the script + # holds what was measured, and a capability appearing or disappearing + # turns this red and names which one. + - name: Verify the macOS backend against a real window server + run: python test/verify/macos_verify.py diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py index 348d0526..368be557 100644 --- a/test/verify/macos_verify.py +++ b/test/verify/macos_verify.py @@ -10,20 +10,23 @@ grant Screen Recording and Accessibility, and a headless CI runner has no user to ask. Which of them a runner grants by default is not something to guess at, and guessing is how the Wayland work lost time twice by recording a -desktop's refusal as a container's limitation. So this has two modes: +desktop's refusal as a container's limitation. So this was measured first, +and the measurement was a surprise: **a macos-14 runner grants both**, and +every probe below passes on one. See :data:`EXPECTED`. ``--measure`` Run every probe, print what happened, and exit 0. Nothing is asserted; - the output is the measurement. + the output is the measurement. This is how :data:`EXPECTED` is + (re)populated when a runner image changes. default Assert :data:`EXPECTED` — what the measurement showed the runner permits. Exit status is the number of checks that did not match, so a capability - that appears or disappears turns CI red and says which one. + appearing or disappearing turns CI red and names it. This is what the + workflow runs. -The second mode is only honest once the first has run, which is why -:data:`EXPECTED` starts empty and is filled in from a real run rather than -from what the APIs are documented to do. +Assert mode refuses to pass while :data:`EXPECTED` is empty, because a gate +that asserts nothing reads as coverage that does not exist. """ from __future__ import annotations @@ -33,13 +36,32 @@ import traceback from typing import Any, Callable, Dict, List, Optional, Tuple -#: What a ``macos-14`` runner was measured to permit. Keys are probe names; -#: values are ``True`` (works), ``False`` (silently does nothing or is -#: refused), or a string naming the exception type it raises. +#: What a ``macos-14`` runner was measured to permit, on 2026-08-19. Keys are +#: probe names; values are ``True`` (works), ``False`` (silently does nothing +#: or is refused), or a string naming the exception type it raises. #: -#: Empty until the first ``--measure`` run fills it in. An empty table makes -#: the assert mode fail loudly rather than pass vacuously. -EXPECTED: Dict[str, Any] = {} +#: The measurement was a surprise worth writing down: **a GitHub macOS runner +#: grants both Screen Recording and Accessibility to the interpreter**, so +#: every capability here works. Capture returns real pixels rather than the +#: black rectangle a refusal produces, ``CGEventPost`` moves the cursor and +#: the move reads back exactly, and the AX walk returns real elements. The +#: usual assumption — that CI cannot exercise a TCC-gated macOS API — is +#: wrong for this runner, which is why this is measured and not reasoned +#: about. +#: +#: If a future runner image tightens any of that, the mismatch turns this job +#: red and names the capability that changed, which is the point. +EXPECTED: Dict[str, Any] = { + "backend-selection": True, + "screen-size": True, + "screenshot": True, + "get-pixel": True, + "mouse-position": True, + "mouse-move": True, + "keyboard-post": True, + "accessibility-tree": True, + "recorder-absent": True, +} _results: List[Tuple[str, bool, str]] = [] From c0ac50a23c313f2a5940cae47f405b42f926369e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 03:18:54 +0800 Subject: [PATCH 06/30] Give window management a backend, so it works off Windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Window management was Windows-only for the project's whole life: the facade branched on sys.platform and raised NotImplementedError everywhere else, which left 23 AC_* commands and their MCP tools dead on macOS and Linux. It now goes through the same backend seam the accessibility, OCR and vision subsystems already use — Win32, EWMH over python-Xlib on X11, Quartz plus the accessibility API on macOS, and a null fallback that lists nothing and refuses actions with a reason. The seam sits under wrapper/ rather than utils/ because it has to import windows/, linux_with_x11/ and osx/, and utils/ is above those in the layering. Refusals now raise a class that is both an AutoControlException and a NotImplementedError. The GUI tabs and the REST handler already catch the latter to say "not on this platform", and that keeps working; the executor and the background loops catch the former, and a bare NotImplementedError slipped past all of them — aborting a whole script where one action should have been reported as failed. The x11-verification job grew a second script that drives the public facade against a real openbox session, with ground truth from xwininfo and xprop. Two things only a real window manager could show up were wrong first time: * The rectangle is the frame, not the client. Win32's GetWindowRect returns the frame and every caller here is written against that, so reporting the client area was off by the decorations on X11 alone. * A move has to go through _NET_MOVERESIZE_WINDOW. Under a reparenting window manager a client's own x/y are relative to its frame, so a direct ConfigureWindow asks in the wrong coordinate space: asking openbox for (300, 220) that way landed the window at (302, 260). post_key_to_window and post_click_to_window are asserted to arrive flagged synthetic, because that is what XSendEvent traffic is and GTK and Qt discard it by design — the same best-effort caveat Win32's PostMessage carries. macOS has no equivalent at all, so the backend refuses rather than reporting a success that went to whatever had focus. The macOS probe goes back to --measure for one round: its window probe is new and asserting a value nobody has measured is the guess this all avoids. --- .github/workflows/platform-smoke.yml | 7 +- architecture_explore.md | 16 +- docker/Dockerfile.x11 | 1 + docker/entrypoint-x11.sh | 14 +- docker/x11_verify.py | 9 + docker/x11_window_verify.py | 372 ++++++++++++++++ docs/CAPABILITY_MATRIX.md | 35 ++ je_auto_control/utils/exception/exceptions.py | 14 + .../wrapper/auto_control_window.py | 105 ++--- .../wrapper/window_backends/__init__.py | 70 +++ .../wrapper/window_backends/base.py | 120 ++++++ .../wrapper/window_backends/macos_backend.py | 301 +++++++++++++ .../wrapper/window_backends/null_backend.py | 23 + .../window_backends/windows_backend.py | 67 +++ .../wrapper/window_backends/x11_backend.py | 398 ++++++++++++++++++ .../headless/test_window_backends.py | 228 ++++++++++ test/verify/macos_verify.py | 25 ++ 17 files changed, 1734 insertions(+), 71 deletions(-) create mode 100644 docker/x11_window_verify.py create mode 100644 je_auto_control/wrapper/window_backends/__init__.py create mode 100644 je_auto_control/wrapper/window_backends/base.py create mode 100644 je_auto_control/wrapper/window_backends/macos_backend.py create mode 100644 je_auto_control/wrapper/window_backends/null_backend.py create mode 100644 je_auto_control/wrapper/window_backends/windows_backend.py create mode 100644 je_auto_control/wrapper/window_backends/x11_backend.py create mode 100644 test/unit_test/headless/test_window_backends.py diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index bfdcd9fb..1d5b3830 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -84,5 +84,8 @@ jobs: # So the flag is off and this is a gate now: EXPECTED in the script # holds what was measured, and a capability appearing or disappearing # turns this red and names which one. - - name: Verify the macOS backend against a real window server - run: python test/verify/macos_verify.py + # Back in --measure for one round: the window-management probe is new + # and its value is not in EXPECTED yet. Asserting a value nobody has + # measured would be the guess this whole script exists to avoid. + - name: Measure the macOS backend against a real window server + run: python test/verify/macos_verify.py --measure diff --git a/architecture_explore.md b/architecture_explore.md index 87dadc95..92044e96 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,8 +19,8 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,016 | -| 程式碼總行數 | 137,532 | +| Python 模組總數（含周邊子專案） | 1,022 | +| 程式碼總行數 | 138,510 | | `je_auto_control/utils/` 子套件數 | 308 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -176,7 +176,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `wrapper/auto_control_screen.py` | 97 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | | `wrapper/auto_control_record.py` | 106 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | -| `wrapper/auto_control_window.py` | 293 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | +| `wrapper/auto_control_window.py` | 278 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | ### 5.3 平台後端 @@ -296,7 +296,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.2 框架基礎設施 -> 12 個套件、約 1,881 行。 +> 12 個套件、約 1,895 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -304,7 +304,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | -| `utils/exception/` | 194 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/exception/` | 208 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | | `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | | `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | @@ -1022,7 +1022,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/usb/` | 17 | 4,247 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | | `utils/accessibility/` | 12 | 2,390 | -| `wrapper/` | 12 | 2,062 | +| `wrapper/` | 3,026 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | | `windows/` | 23 | 1,995 | | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | @@ -1035,6 +1035,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 761 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,238 | -| **總計** | **1,010** | **137,467** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 685 | 49,278 | +| **總計** | **1,016** | **138,445** | diff --git a/docker/Dockerfile.x11 b/docker/Dockerfile.x11 index 7f63e07c..affafba1 100644 --- a/docker/Dockerfile.x11 +++ b/docker/Dockerfile.x11 @@ -81,6 +81,7 @@ RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ && rm -rf /wheels COPY docker/x11_verify.py /opt/verify/x11_verify.py +COPY docker/x11_window_verify.py /opt/verify/x11_window_verify.py COPY docker/entrypoint-x11.sh /usr/local/bin/autocontrol-x11-verify RUN chmod +x /usr/local/bin/autocontrol-x11-verify diff --git a/docker/entrypoint-x11.sh b/docker/entrypoint-x11.sh index 5a9909be..8fb4524b 100644 --- a/docker/entrypoint-x11.sh +++ b/docker/entrypoint-x11.sh @@ -73,6 +73,10 @@ if [ "$wm" -ne 1 ]; then fi echo "openbox is managing ${DISPLAY_NUM}" +# x11_window_verify imports the harness from x11_verify, so the directory +# holding both has to be on the import path. +export PYTHONPATH=/opt/verify + # Runs one pass over one monitor layout; echoes the number of failed checks. run_pass() { label="$1" @@ -106,9 +110,17 @@ fi run_pass "two monitors side by side" || total=$((total + $?)) +# Window management does not depend on the monitor layout — it is about what +# the window manager agrees to — so it runs once, after both layout passes. +echo +echo "========================================================================" +echo "window management against a real window manager" +echo "========================================================================" +python3 /opt/verify/x11_window_verify.py || total=$((total + $?)) + echo echo "========================================================================" -echo "total failed checks across both layouts: ${total}" +echo "total failed checks: ${total}" echo "========================================================================" kill "$OPENBOX_PID" 2>/dev/null || true diff --git a/docker/x11_verify.py b/docker/x11_verify.py index 803fff84..d8e0ff1f 100644 --- a/docker/x11_verify.py +++ b/docker/x11_verify.py @@ -666,6 +666,15 @@ def main() -> int: note(" to grant. docker/ydotool_verify.py reads those events back off") note(" the kernel device instead.") + return summarise() + + +def summarise() -> int: + """Print the tally and return the number of failed checks. + + Shared with ``x11_window_verify.py``, which runs in the same session and + reports through the same harness. + """ failed = [name for name, ok, _ in _results if not ok] print("=" * 72) print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") diff --git a/docker/x11_window_verify.py b/docker/x11_window_verify.py new file mode 100644 index 00000000..c3d565cd --- /dev/null +++ b/docker/x11_window_verify.py @@ -0,0 +1,372 @@ +"""Verify cross-platform window management against a real window manager. + +Window management was Windows-only for the project's whole life: the facade +branched on ``sys.platform`` and raised everywhere else, so 23 ``AC_*`` +commands and their MCP tools were dead on Linux and macOS. The X11 backend +that replaces that branch talks EWMH to the window manager, and EWMH is +exactly the kind of contract a mock cannot check — what matters is whether +*openbox* agrees, not whether python-Xlib was called with the right atoms. + +So this runs against the real session ``entrypoint-x11.sh`` brings up, drives +the public facade (``je_auto_control.list_windows`` and friends, not the +backend), and takes ground truth from ``xdotool`` and ``xprop`` — the window +manager answering for itself. + +The subject is a real ``xterm``, launched with a title nothing else uses. + +It shares the harness in ``x11_verify`` rather than duplicating it: same +session, same tally, one exit status. +""" +from __future__ import annotations + +import os +import subprocess # nosec B404 # reason: argv lists of fixed tool names, no shell +import sys +import time +from typing import Any, Optional, Tuple + +from x11_verify import ( + EventTester, _assert_eq, _assert_true, _run, check, note, summarise, +) + +#: Unique enough that `xdotool search` cannot match anything else in the image. +WINDOW_TITLE = "autocontrol-window-verify" + +#: How long the window manager is given to act on an EWMH request. These are +#: asynchronous by design — the client message goes to the root window and the +#: window manager gets to it when it gets to it. +WM_TIMEOUT = 5.0 + + +class Xterm: + """A real X client owning a real, managed, titled window.""" + + def __init__(self, title: str = WINDOW_TITLE) -> None: + self.title = title + self._process: Optional[subprocess.Popen] = None + self.window_id = 0 + + def __enter__(self) -> "Xterm": + # -hold keeps the window up after the shell exits, so the subject + # cannot vanish mid-check for a reason unrelated to what is under test. + self._process = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["xterm", "-title", self.title, "-geometry", "40x10+150+150", + "-hold", "-e", "true"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.window_id = self._await_window() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is not None and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + + def _await_window(self) -> int: + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + found = _run(["xdotool", "search", "--name", self.title]) + except subprocess.CalledProcessError: + found = "" + ids = [line for line in found.split() if line.isdigit()] + if ids: + # Wait for the window manager to finish managing it: an EWMH + # request against an unmanaged window is simply dropped. + self._await_managed(int(ids[-1])) + return int(ids[-1]) + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + f"xterm exited {self._process.returncode} without a window") + time.sleep(0.1) + raise RuntimeError("xterm never mapped a window") + + @staticmethod + def _await_managed(window_id: int) -> None: + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline: + listed = _run(["xprop", "-root", "_NET_CLIENT_LIST"]) + if f"{window_id:#x}" in listed.lower().replace("0x", "0x"): + return + time.sleep(0.1) + raise RuntimeError(f"the window manager never took window {window_id}") + + @property + def pid(self) -> int: + return self._process.pid if self._process is not None else 0 + + +def _active_window() -> int: + """The window the server says is active, via xdotool.""" + return int(_run(["xdotool", "getactivewindow"]).strip() or 0) + + +#: How far a size may differ from what was asked before it counts as wrong. +#: xterm resizes in whole character cells, so it rounds a pixel size down to +#: the nearest cell — a property of the subject, not of the backend. A real +#: failure to resize is out by hundreds of pixels, not by one cell. +SIZE_TOLERANCE = 24 + + +def _client_geometry(window_id: int) -> Tuple[int, int, int, int]: + """The *client* ``(x, y, width, height)``, as xwininfo reports it.""" + values: dict = {} + for line in _run(["xwininfo", "-id", str(window_id)]).splitlines(): + stripped = line.strip() + for label, key in (("Absolute upper-left X:", "X"), + ("Absolute upper-left Y:", "Y"), + ("Width:", "WIDTH"), ("Height:", "HEIGHT")): + if stripped.startswith(label): + values[key] = int(stripped[len(label):].strip()) + return (values.get("X", 0), values.get("Y", 0), + values.get("WIDTH", 0), values.get("HEIGHT", 0)) + + +def _frame_extents(window_id: int) -> Tuple[int, int, int, int]: + """``(left, right, top, bottom)`` decoration thickness, or zeroes.""" + try: + raw = _run(["xprop", "-id", str(window_id), "_NET_FRAME_EXTENTS"]) + except subprocess.CalledProcessError: + return (0, 0, 0, 0) + if "not found" in raw or "=" not in raw: + return (0, 0, 0, 0) + parts = [piece.strip() for piece in raw.split("=", 1)[1].split(",")] + if len(parts) < 4 or not all(piece.isdigit() for piece in parts[:4]): + return (0, 0, 0, 0) + return tuple(int(piece) for piece in parts[:4]) # type: ignore[return-value] + + +def _geometry(window_id: int) -> Tuple[int, int, int, int]: + """The *frame* ``(x, y, width, height)``: the client plus its decorations. + + Win32's ``GetWindowRect`` returns the frame, every caller in this project + is written against that, and the X11 backend reports the frame to match — + so the frame is what has to be checked. + + Neither tool reports it directly. ``xwininfo -id`` gives the client + rectangle (``-frame`` changes how a window is *picked*, not what is + reported, so it makes no difference with ``-id``), and ``xprop``'s + ``_NET_FRAME_EXTENTS`` gives the thickness the window manager added + around it. Adding them is both the definition of the frame and a check + that the backend's own walk up to it found the same window. + """ + x, y, width, height = _client_geometry(window_id) + left, right, top, bottom = _frame_extents(window_id) + return (x - left, y - top, width + left + right, height + top + bottom) + + +def _wait_until(predicate, timeout: float = WM_TIMEOUT) -> bool: + """Poll a predicate — every EWMH request is asynchronous.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.1) + return False + + +# --- checks ---------------------------------------------------------------- + + +def check_backend_selection() -> None: + def _selected() -> str: + from je_auto_control.wrapper.window_backends import get_backend + + backend = get_backend() + _assert_true(backend.available, + f"backend {backend.name!r} reports unavailable") + return _assert_eq(backend.name, "x11-ewmh") + check("the window seam selects the X11 backend", _selected) + + +def check_listing(xterm: Xterm) -> None: + import je_auto_control as ac + + def _lists() -> str: + titles = dict(ac.list_windows()) + _assert_true(xterm.window_id in titles, + f"window {xterm.window_id} missing from {sorted(titles)}") + return _assert_eq(titles[xterm.window_id], xterm.title) + check("list_windows sees a real managed window with its title", _lists) + + def _finds() -> str: + hit = ac.find_window(xterm.title) + _assert_true(hit is not None, "find_window returned nothing") + return _assert_eq(hit[0], xterm.window_id) + check("find_window resolves the title substring", _finds) + + def _rect() -> str: + rect = ac.window_rect(xterm.title) + _assert_true(rect is not None, "window_rect returned None") + left, top, right, bottom = rect + x, y, width, height = _geometry(xterm.window_id) + # Both sides are the frame: the rectangle a user sees and drags, and + # the one Win32 reports. See _geometry for why the client rectangle + # is the wrong comparison. + _assert_eq((left, top), (x, y)) + _assert_eq((right - left, bottom - top), (width, height)) + return f"({left}, {top}) {right - left}x{bottom - top}" + check("window_rect matches what the server reports", _rect) + + def _pid() -> str: + # _NET_WM_PID is what the client advertises; xterm advertises its own. + return _assert_eq(ac.window_process_id(xterm.title), xterm.pid) + check("window_process_id is the owning process", _pid) + + def _by_pid() -> str: + owned = dict(ac.windows_for_process_id(xterm.pid)) + _assert_true(xterm.window_id in owned, + f"window missing from the pid's windows: {sorted(owned)}") + return f"{len(owned)} window(s) for pid {xterm.pid}" + check("windows_for_process_id addresses windows by owner", _by_pid) + + +def check_focus(xterm: Xterm) -> None: + import je_auto_control as ac + + def _focus() -> str: + ac.focus_window(xterm.title) + _assert_true(_wait_until(lambda: _active_window() == xterm.window_id), + f"the window manager never made {xterm.window_id} active") + return _assert_eq(ac.foreground_window()[0], xterm.window_id) + check("focus_window makes the window active, and foreground_window agrees", + _focus) + + def _fg_pid() -> str: + return _assert_eq(ac.foreground_window_process_id(), xterm.pid) + check("foreground_window_process_id names the owning process", _fg_pid) + + +def check_move(xterm: Xterm) -> None: + import je_auto_control as ac + + def _move() -> str: + _assert_true(ac.move_window_by_title(xterm.title, 300, 220, 500, 300), + "move_window_by_title reported failure") + _assert_true( + _wait_until(lambda: _geometry(xterm.window_id)[:2] == (300, 220)), + f"window stayed at {_geometry(xterm.window_id)}") + x, y, width, height = _geometry(xterm.window_id) + _assert_eq((x, y), (300, 220)) + _assert_true(abs(width - 500) <= SIZE_TOLERANCE + and abs(height - 300) <= SIZE_TOLERANCE, + f"asked 500x300, frame is {width}x{height}") + return f"frame at ({x}, {y}), {width}x{height}" + check("move_window_by_title moves and resizes for real", _move) + + def _move_keeps_size() -> str: + # Omitting width/height must keep the current size rather than + # collapsing the window, which is what makes a plain reposition usable + # without looking the dimensions up first. + before = _geometry(xterm.window_id) + _assert_true(ac.move_window_by_title(xterm.title, 120, 90), + "move_window_by_title reported failure") + _assert_true( + _wait_until(lambda: _geometry(xterm.window_id)[:2] == (120, 90)), + f"window stayed at {_geometry(xterm.window_id)}") + return _assert_eq(_geometry(xterm.window_id)[2:], before[2:]) + check("moving without a size keeps the size", _move_keeps_size) + + +def check_minimize(xterm: Xterm) -> None: + import je_auto_control as ac + from je_auto_control.wrapper.window_backends import get_backend + + backend = get_backend() + + def _minimize() -> str: + _assert_true(ac.minimize_window_by_title(xterm.title), + "minimize_window_by_title reported failure") + _assert_true( + _wait_until(lambda: backend.is_minimized(xterm.window_id)), + "the window never reported itself hidden") + return "iconified, and _NET_WM_STATE_HIDDEN says so" + check("minimize_window_by_title iconifies the window", _minimize) + + def _restore() -> str: + # focus_window restores before raising, which is the path a caller + # takes to get back to a minimised window without knowing it was one. + ac.focus_window(xterm.title) + _assert_true( + _wait_until(lambda: not backend.is_minimized(xterm.window_id)), + "the window stayed hidden") + return "restored by focus_window" + check("focus_window restores a minimised window", _restore) + + +def check_post_is_synthetic(tester: EventTester) -> None: + """Posting to an unfocused window works, and is honest about what it is.""" + import je_auto_control as ac + + def _post_click() -> str: + tester.flush() + _assert_true( + ac.post_click_to_window("Event Tester", "left", 20, 20), + "post_click_to_window reported failure") + press = tester.collect("ButtonPress")[0] + _assert_eq(press["button"], 1) + # This is the point of the check. XSendEvent traffic arrives flagged + # synthetic and GTK and Qt discard it by design, so post_* is + # best-effort by nature — the same caveat Win32's PostMessage carries. + # Asserting it here stops anyone reading post_* as real input. + return _assert_eq(press["synthetic"], "YES") + check("post_click_to_window arrives, flagged synthetic", _post_click) + + def _post_key() -> str: + from je_auto_control import keyboard_keys_table + + tester.flush() + _assert_true(ac.post_key_to_window("Event Tester", "b"), + "post_key_to_window reported failure") + press = tester.collect("KeyPress")[0] + _assert_eq(press["keycode"], keyboard_keys_table["b"]) + return _assert_eq(press["synthetic"], "YES") + check("post_key_to_window arrives, flagged synthetic", _post_key) + + +def check_close(xterm: Xterm) -> None: + import je_auto_control as ac + + def _close() -> str: + _assert_true(ac.close_window_by_title(xterm.title), + "close_window_by_title reported failure") + _assert_true( + _wait_until(lambda: ac.find_window(xterm.title) is None, 10.0), + "the window is still listed") + return "gone from the client list" + check("close_window_by_title really closes the window", _close) + + +def main() -> int: + print("=" * 72) + print("AutoControl window management — real X11 window manager") + print("=" * 72) + note(f"DISPLAY={os.environ.get('DISPLAY')!r}") + + check_backend_selection() + + with Xterm() as xterm: + note(f"subject: xterm window {xterm.window_id} pid {xterm.pid}") + check_listing(xterm) + check_focus(xterm) + check_move(xterm) + check_minimize(xterm) + check_close(xterm) + + with EventTester() as tester: + check_post_is_synthetic(tester) + + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("Wayland window management — the protocol does not let a client") + note(" enumerate or move another application's windows at all, so there") + note(" is nothing to implement, let alone verify. The backend selector") + note(" says so rather than looking broken.") + + return summarise() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index 9f9dc4e4..78a44aa0 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -10,6 +10,7 @@ without a compatibility window. | JSON executor and variables | stable | CI | CI | CI | platform-neutral | | Image and anchor locators | beta | CI | CI | implementation | implementation | | Accessibility locator | beta | CI | backend tests | unavailable | backend tests | +| Window management | beta | CI | CI/openbox | unavailable | CI (listing) | | Recorder | beta | CI | implementation | unavailable | unavailable | | Reports, trace, failure bundle | stable | CI | CI | CI | platform-neutral | | REST, MCP, scheduler | beta | CI | CI | CI | platform-neutral | @@ -64,6 +65,40 @@ primary one, where the compositor's plane starts at a negative coordinate and a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of that monitor. +Window management had no row here at all until it had more than one platform. +It was Windows-only for the project's whole life — the facade branched on +`sys.platform` and raised everywhere else — which left 23 `AC_*` commands and +their MCP tools dead on macOS and Linux. It now goes through a backend seam: +Win32, EWMH over python-Xlib on X11, and Quartz plus the accessibility API on +macOS. + +The X11 half is exercised by the `x11-verification` job against a real +`openbox` session, driving the public facade and taking ground truth from +`xwininfo` and `xprop`. Two things only a real window manager could have +shown up came out of it, and both were wrong in the first implementation: + +* **The rectangle is the frame, not the client.** Win32's `GetWindowRect` + returns the frame — border and title bar included — and every caller here is + written against that. Reporting the client area was off by the decorations + on X11 alone, silently, and by a different amount per window manager. +* **A move must go through `_NET_MOVERESIZE_WINDOW`.** Under a reparenting + window manager a client's own x/y are relative to its frame, so a direct + `ConfigureWindow` asks for a position in the wrong coordinate space. + Measured against openbox, asking for (300, 220) that way landed the window + at (302, 260). + +`post_key_to_window` and `post_click_to_window` work on X11 and are asserted +to arrive *flagged synthetic*, because that is what they are: `XSendEvent` +traffic, which GTK and Qt discard by design. They are the X11 counterpart of +Win32's `PostMessage`, which carries the same best-effort caveat. macOS has no +equivalent at all — an event goes to whatever has focus — so the backend +refuses rather than reporting a success that went somewhere else. + +Wayland is `unavailable` and will stay that way: the protocol does not let a +client enumerate or move another application's windows. That is a design +decision upstream, not a gap here, and the backend selector says so instead of +looking broken. + One cross-platform difference falls out of the same job, and it is not one this project can fix: **a Wayland capture may contain the mouse cursor.** No capture here passes `grim -c`, so none of them asks for the pointer — but wlroots draws diff --git a/je_auto_control/utils/exception/exceptions.py b/je_auto_control/utils/exception/exceptions.py index ff700e48..a5fc7cf1 100644 --- a/je_auto_control/utils/exception/exceptions.py +++ b/je_auto_control/utils/exception/exceptions.py @@ -114,3 +114,17 @@ class XMLTypeException(AutoControlException): # Execute callback class CallbackExecutorException(AutoControlException): pass + + +# Platform capability +class AutoControlUnsupportedOperationException( + AutoControlException, NotImplementedError): + """An operation the current platform's backend cannot perform. + + Inherits both on purpose. ``NotImplementedError`` is what the GUI tabs and + the REST handler already catch to say "not on this platform", and that + behaviour is kept; ``AutoControlException`` is what the executor and the + other containment boundaries catch, and a bare ``NotImplementedError`` + slipped straight past all of them — aborting a whole script where a single + action should have been reported as failed. + """ diff --git a/je_auto_control/wrapper/auto_control_window.py b/je_auto_control/wrapper/auto_control_window.py index 20b20066..fada874f 100644 --- a/je_auto_control/wrapper/auto_control_window.py +++ b/je_auto_control/wrapper/auto_control_window.py @@ -1,23 +1,23 @@ """Cross-platform window management facade. -On Windows, delegates to ``windows_window_manage`` (Win32 API). -On macOS / Linux, operations raise a clear ``NotImplementedError``. +Delegates to whichever backend :mod:`je_auto_control.wrapper.window_backends` +selects — Win32 on Windows, EWMH over python-Xlib on X11, Quartz plus the +accessibility API on macOS — and keeps here everything that is not +platform-specific: substring matching, waiting, and the compositions like +"move without restating the size". + +This was Windows-only for the project's whole life, which left these +functions, their 23 ``AC_*`` commands and their MCP tools dead on the two +other supported platforms. A platform with no backend still gets a null one +that lists nothing and refuses actions with a reason, so importing never +fails and callers get an answer rather than an ``ImportError``. """ -import sys import time from typing import List, Optional, Tuple, Union from je_auto_control.utils.exception.exceptions import AutoControlActionException from je_auto_control.utils.logging.logging_instance import autocontrol_logger - -_IS_WINDOWS = sys.platform in ("win32", "cygwin", "msys") - - -def _require_windows() -> None: - if not _IS_WINDOWS: - raise NotImplementedError( - f"Window management is only implemented on Windows (got {sys.platform})" - ) +from je_auto_control.wrapper.window_backends import get_backend def list_windows(titled_only: bool = False) -> List[Tuple[int, str]]: @@ -28,9 +28,8 @@ def list_windows(titled_only: bool = False) -> List[Tuple[int, str]]: Most visible windows have no title (shell and helper surfaces); pass ``titled_only`` for just the ones a user would recognise. """ - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm - found = wm.get_all_window_hwnd() + backend = get_backend() + found = backend.list_windows() if titled_only: return [(hwnd, title) for hwnd, title in found if title.strip()] return found @@ -49,20 +48,19 @@ def find_window(title_substring: str, def focus_window(title_substring: str, case_sensitive: bool = False) -> int: """Bring the first matching window to the foreground; return its hwnd.""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: raise AutoControlActionException( f"focus_window: no window matches {title_substring!r}" ) hwnd, title = hit - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() # A minimized window stays invisible however often you foreground it, so # restore it first — but only when it really is minimized: SW_RESTORE on a # maximized window un-maximizes it, which is not what "focus" should do. - if wm.is_window_minimized(hwnd): - wm.show_window(hwnd, wm.SW_RESTORE) - wm.set_foreground_window(hwnd) + if backend.is_minimized(hwnd): + backend.restore(hwnd) + backend.set_foreground(hwnd) autocontrol_logger.info("focused window hwnd=%s title=%r", hwnd, title) return hwnd @@ -72,7 +70,6 @@ def wait_for_window(title_substring: str, poll: float = 0.5, case_sensitive: bool = False) -> int: """Poll until a window with the given title appears; return its hwnd.""" - _require_windows() poll = max(0.05, float(poll)) deadline = time.monotonic() + float(timeout) while time.monotonic() < deadline: @@ -92,33 +89,30 @@ def close_window_by_title(title_substring: str, case_sensitive: bool = False) -> call underneath is named ``CloseWindow`` but minimises. Use :func:`minimize_window_by_title` for the old behaviour. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - return wm.close_window(hit[0]) + backend = get_backend() + return backend.close(hit[0]) def minimize_window_by_title(title_substring: str, case_sensitive: bool = False) -> bool: """Minimise the first matching window. ``False`` if nothing matched.""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - return wm.minimize_window(hit[0]) + backend = get_backend() + return backend.minimize(hit[0]) def foreground_window() -> Optional[Tuple[int, str]]: """The window the user is currently working in, or ``None``.""" - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm - hwnd = wm.get_foreground_window() + backend = get_backend() + hwnd = backend.foreground_window() if not hwnd: return None - titles = dict(wm.get_all_window_hwnd()) + titles = dict(backend.list_windows()) return hwnd, titles.get(hwnd, "") @@ -139,13 +133,12 @@ def post_key_to_window(title_substring: str, key: Union[int, str], the foreground all ignore posted messages. Callers must say so rather than reporting success. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() keycode, character = _resolve_key(key) - return wm.post_key(hit[0], keycode, character) + return backend.post_key(hit[0], keycode, character) def post_click_to_window(title_substring: str, button: str = "left", @@ -158,12 +151,11 @@ def post_click_to_window(title_substring: str, button: str = "left", client coordinates — a click posted to the frame lands nowhere. Same best-effort caveat as :func:`post_key_to_window`. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - return wm.post_click(hit[0], _mouse_button_name(button), int(x), int(y)) + backend = get_backend() + return backend.post_click(hit[0], _mouse_button_name(button), int(x), int(y)) def _resolve_key(key: Union[int, str]) -> Tuple[int, str]: @@ -196,23 +188,21 @@ def foreground_window_process_id() -> Optional[int]: activity probes, "is my automation target focused" — have to go through the process id. """ - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm - hwnd = wm.get_foreground_window() + backend = get_backend() + hwnd = backend.foreground_window() if not hwnd: return None - return wm.get_window_process_id(hwnd) or None + return backend.window_process_id(hwnd) or None def window_process_id(title_substring: str, case_sensitive: bool = False) -> Optional[int]: """The PID owning the first window whose title contains the substring.""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return None - from je_auto_control.windows.window import windows_window_manage as wm - return wm.get_window_process_id(hit[0]) or None + backend = get_backend() + return backend.window_process_id(hit[0]) or None def windows_for_process_id(pid: int, @@ -223,20 +213,18 @@ def windows_for_process_id(pid: int, named after whatever page they show, and several of its processes have no window at all. Ownership is the stable key. """ - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() target = int(pid) return [(hwnd, title) for hwnd, title in list_windows(titled_only) - if wm.get_window_process_id(hwnd) == target] + if backend.window_process_id(hwnd) == target] def minimize_windows_for_process(pid: int) -> int: """Minimise every visible top-level window owned by ``pid``; return the count.""" - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() minimized = 0 for hwnd, _title in windows_for_process_id(pid): - if wm.minimize_window(hwnd): + if backend.minimize(hwnd): minimized += 1 return minimized @@ -249,12 +237,11 @@ def window_rect(title_substring: str, Screen coordinates, so on a multi-monitor desktop the values can be negative for a monitor left of or above the primary one. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return None - from je_auto_control.windows.window import windows_window_manage as wm - return wm.get_window_rect(hit[0]) + backend = get_backend() + return backend.window_rect(hit[0]) def move_window_by_title(title_substring: str, x: int, y: int, @@ -266,28 +253,26 @@ def move_window_by_title(title_substring: str, x: int, y: int, Omitting ``width`` / ``height`` keeps the window's current size, so a plain reposition does not have to restate dimensions the caller has to look up. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() if width is None or height is None: - rect = wm.get_window_rect(hit[0]) + rect = backend.window_rect(hit[0]) if rect is None: return False left, top, right, bottom = rect width = right - left if width is None else width height = bottom - top if height is None else height - return wm.move_window(hit[0], int(x), int(y), int(width), int(height)) + return backend.move(hit[0], int(x), int(y), int(width), int(height)) def show_window_by_title(title_substring: str, cmd_show: int = 1, case_sensitive: bool = False) -> bool: """Show or restore a window (``cmd_show`` follows Win32 ShowWindow).""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - wm.show_window(hit[0], int(cmd_show)) + backend = get_backend() + backend.show(hit[0], int(cmd_show)) return True diff --git a/je_auto_control/wrapper/window_backends/__init__.py b/je_auto_control/wrapper/window_backends/__init__.py new file mode 100644 index 00000000..144c987d --- /dev/null +++ b/je_auto_control/wrapper/window_backends/__init__.py @@ -0,0 +1,70 @@ +"""Per-platform window-management backends. + +Window management was Windows-only for the project's whole life: the facade +branched on ``sys.platform`` and raised ``NotImplementedError`` everywhere +else, which left 23 ``AC_*`` commands and their MCP tools dead on macOS and +Linux. This is the seam that replaces that branch, following the same shape +the accessibility, OCR and vision subsystems already use — abstract base, +concrete implementations, null fallback — so a platform without one still +imports. + +Selection is cached because probing opens an X connection or a Quartz query, +and the answer cannot change inside a process. +""" +import sys +from typing import Optional + +from je_auto_control.wrapper.window_backends.base import WindowManageBackend +from je_auto_control.wrapper.window_backends.null_backend import NullWindowBackend + +_cached_backend: Optional[WindowManageBackend] = None + + +def get_backend() -> WindowManageBackend: + """Return (and cache) the best window backend for this platform.""" + global _cached_backend + if _cached_backend is None: + _cached_backend = _build_backend() + return _cached_backend + + +def reset_backend_cache() -> None: + """Force the next :func:`get_backend` call to re-detect.""" + global _cached_backend + _cached_backend = None + + +def _build_backend() -> WindowManageBackend: + if sys.platform in ("win32", "cygwin", "msys"): + from je_auto_control.wrapper.window_backends.windows_backend import ( + WindowsWindowBackend, + ) + return WindowsWindowBackend() + if sys.platform == "darwin": + from je_auto_control.wrapper.window_backends.macos_backend import ( + MacOSWindowBackend, + ) + backend = MacOSWindowBackend() + if backend.available: + return backend + return NullWindowBackend( + "pyobjc (Quartz, AppKit) is required for macOS window management") + if sys.platform in ("linux", "linux2"): + from je_auto_control.wrapper.window_backends.x11_backend import ( + X11WindowBackend, + ) + backend = X11WindowBackend() + if backend.available: + return backend + # Wayland deliberately does not let a client enumerate or move other + # applications' windows; that is a protocol decision, not a gap here. + return NullWindowBackend( + "no X display. Wayland does not expose other windows to a client, " + "so run under X11 or XWayland for window management") + return NullWindowBackend(f"no window backend for platform {sys.platform!r}") + + +__all__ = [ + "NullWindowBackend", "WindowManageBackend", + "get_backend", "reset_backend_cache", +] diff --git a/je_auto_control/wrapper/window_backends/base.py b/je_auto_control/wrapper/window_backends/base.py new file mode 100644 index 00000000..0695cebd --- /dev/null +++ b/je_auto_control/wrapper/window_backends/base.py @@ -0,0 +1,120 @@ +"""Abstract window-management backend.""" +from typing import List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import ( + AutoControlUnsupportedOperationException, +) + + +class WindowManageBackend: + """One platform's answers about its top-level windows. + + Everything that is not platform-specific — substring matching, waiting, + "move without restating the size" — lives in + :mod:`je_auto_control.wrapper.auto_control_window`, so a backend only has + to list windows and answer about one at a time. + + ``window_id`` is whatever the platform calls a window: an ``HWND`` on + Windows, an X11 window id on Linux, a ``CGWindowID`` on macOS. It is + always a plain ``int``, so it composes with any other call on that + platform. + + A backend that cannot perform an operation raises through + :meth:`_unsupported` rather than returning a falsy value: "this platform + has no such concept" and "it did not work this time" are different + answers, and a caller that cannot tell them apart will retry forever. + """ + + name: str = "abstract" + available: bool = False + + # --- listing ----------------------------------------------------------- + + def list_windows(self) -> List[Tuple[int, str]]: + """``(window_id, title)`` for every visible top-level window. + + Front-most first, so the first match of a title substring is the one + the user is most likely to mean. + """ + raise NotImplementedError + + def foreground_window(self) -> int: + """The window the user is working in, or ``0`` when there is none.""" + self._unsupported("foreground_window") + + # --- reading one window ------------------------------------------------ + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + """``(left, top, right, bottom)`` in screen coordinates, or None. + + Screen coordinates, so on a multi-monitor desktop these can be + negative for a monitor left of or above the primary one. + """ + self._unsupported("window_rect") + + def window_process_id(self, window_id: int) -> int: + """The pid owning the window, or ``0`` when it cannot be determined. + + A title is not identity — applications rewrite theirs at will and + unrelated programs share names like ``Settings`` — so ownership is + what addresses a window stably. + """ + self._unsupported("window_process_id") + + def is_minimized(self, window_id: int) -> bool: + """Whether the window is minimised / iconified.""" + self._unsupported("is_minimized") + + # --- acting on one window ---------------------------------------------- + + def set_foreground(self, window_id: int) -> None: + """Raise the window and give it the keyboard focus.""" + self._unsupported("set_foreground") + + def restore(self, window_id: int) -> None: + """Un-minimise the window without changing anything else. + + Deliberately not "show it however": restoring a *maximised* window + would un-maximise it, which is not what focusing something should do. + """ + self._unsupported("restore") + + def show(self, window_id: int, cmd_show: int) -> None: + """Apply a platform show-state code (Win32 ``ShowWindow`` numbering).""" + self._unsupported("show") + + def close(self, window_id: int) -> bool: + """Ask the window to close; ``False`` when the request was refused.""" + self._unsupported("close") + + def minimize(self, window_id: int) -> bool: + """Minimise the window; ``False`` when the request was refused.""" + self._unsupported("minimize") + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + """Move and resize the window; ``False`` when the request was refused.""" + self._unsupported("move") + + # --- acting on a window that does not have focus ----------------------- + + def post_key(self, window_id: int, keycode: int, + character: str = "") -> bool: + """Deliver one key to the window without focusing it.""" + self._unsupported("post_key") + + def post_click(self, window_id: int, button: str, x: int, y: int) -> bool: + """Deliver one click to the window without focusing it. + + ``x`` / ``y`` are relative to the window's own top-left corner. + """ + self._unsupported("post_click") + + # --- refusal ----------------------------------------------------------- + + def _unsupported(self, operation: str): + """Raise a clear error naming what this backend cannot do.""" + raise AutoControlUnsupportedOperationException( + f"{operation} is not supported by the {self.name} window backend", + ) diff --git a/je_auto_control/wrapper/window_backends/macos_backend.py b/je_auto_control/wrapper/window_backends/macos_backend.py new file mode 100644 index 00000000..edaf30e0 --- /dev/null +++ b/je_auto_control/wrapper/window_backends/macos_backend.py @@ -0,0 +1,301 @@ +"""macOS window-management backend, over Quartz and the accessibility API. + +Reading and acting are two different APIs on macOS, with two different +permission stories, and this backend needs both: + +* **Quartz** (``CGWindowListCopyWindowInfo``) enumerates every on-screen + window with its id, title, owning pid and bounds. It needs no grant, so + listing, rectangles and ownership work out of the box. +* **The accessibility API** is the only way to *move*, *close*, *minimise* or + *raise* someone else's window. It is gated by TCC: the user grants + Accessibility to the interpreter, and until they do every action silently + does nothing. :meth:`MacOSWindowBackend.available` reports the Quartz half, + and each action raises through the base class when AX refuses, rather than + returning a false success. + +Quartz has no window handle that the accessibility API accepts, so a +``CGWindowID`` is matched to its ``AXUIElement`` by owner, title and frame. +That is what the two APIs give us to work with; the alternative is a private +symbol (``_AXUIElementGetWindow``) this project will not depend on. +""" +import sys +from typing import Any, List, Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + +#: ``ShowWindow`` codes with a macOS meaning; see the X11 backend for why the +#: project spells show-state in Win32's numbering. +SW_SHOWNORMAL = 1 +SW_SHOWMINIMIZED = 2 +SW_MINIMIZE = 6 +SW_RESTORE = 9 + + +class MacOSWindowBackend(WindowManageBackend): + """Quartz for what a window *is*, accessibility for what it *does*.""" + + name = "macos-quartz-ax" + + def __init__(self) -> None: + self.available = sys.platform == "darwin" and self._probe() + + def _probe(self) -> bool: + try: + import Quartz # noqa: F401 # reason: probe import + import AppKit # noqa: F401 # reason: probe import + return True + except ImportError as error: + autocontrol_logger.info( + "macOS window backend unavailable: %r", error) + return False + + # --- listing, via Quartz ----------------------------------------------- + + def _window_info(self) -> List[dict]: + """Every on-screen window Quartz will admit to, front-most first.""" + import Quartz + + options = (Quartz.kCGWindowListOptionOnScreenOnly + | Quartz.kCGWindowListExcludeDesktopElements) + found = Quartz.CGWindowListCopyWindowInfo(options, Quartz.kCGNullWindowID) + return list(found or []) + + def list_windows(self) -> List[Tuple[int, str]]: + import Quartz + + windows = [] + for info in self._window_info(): + # Layer 0 is the ordinary application layer. Menu bars, the Dock + # and status items live above it and are not windows a caller + # means when they say "the Safari window". + if int(info.get(Quartz.kCGWindowLayer, 0) or 0) != 0: + continue + number = int(info.get(Quartz.kCGWindowNumber, 0) or 0) + if not number: + continue + windows.append((number, str(info.get(Quartz.kCGWindowName, "") or ""))) + return windows + + def _info_for(self, window_id: int) -> Optional[dict]: + import Quartz + + for info in self._window_info(): + if int(info.get(Quartz.kCGWindowNumber, 0) or 0) == int(window_id): + return info + return None + + def foreground_window(self) -> int: + import AppKit + import Quartz + + workspace = AppKit.NSWorkspace.sharedWorkspace() + frontmost = workspace.frontmostApplication() + if frontmost is None: + return 0 + pid = int(frontmost.processIdentifier()) + for info in self._window_info(): + if (int(info.get(Quartz.kCGWindowOwnerPID, 0) or 0) == pid + and int(info.get(Quartz.kCGWindowLayer, 0) or 0) == 0): + # The list is front-to-back, so the owning app's first entry + # is the window that is actually in front. + return int(info.get(Quartz.kCGWindowNumber, 0) or 0) + return 0 + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + import Quartz + + info = self._info_for(window_id) + if info is None: + return None + bounds = info.get(Quartz.kCGWindowBounds) + if not bounds: + return None + left, top = int(bounds["X"]), int(bounds["Y"]) + return (left, top, left + int(bounds["Width"]), + top + int(bounds["Height"])) + + def window_process_id(self, window_id: int) -> int: + import Quartz + + info = self._info_for(window_id) + if info is None: + return 0 + return int(info.get(Quartz.kCGWindowOwnerPID, 0) or 0) + + # --- acting, via the accessibility API --------------------------------- + + def _ax_windows_for(self, pid: int) -> list: + """Every accessibility window belonging to a process.""" + import ApplicationServices as ax + + application = ax.AXUIElementCreateApplication(pid) + error, windows = ax.AXUIElementCopyAttributeValue( + application, "AXWindows", None) + return [] if error or not windows else list(windows) + + def _ax_window(self, window_id: int): + """The ``AXUIElement`` for a ``CGWindowID``, or None. + + Quartz ids and accessibility elements are separate namespaces with no + public bridge, so the window is found again inside its owning + application by frame and title — the two properties both APIs report. + """ + import Quartz + + info = self._info_for(window_id) + if info is None: + return None + pid = int(info.get(Quartz.kCGWindowOwnerPID, 0) or 0) + if not pid: + return None + bounds = info.get(Quartz.kCGWindowBounds) or {} + return _best_match( + self._ax_windows_for(pid), + (int(bounds.get("X", 0)), int(bounds.get("Y", 0))), + str(info.get(Quartz.kCGWindowName, "") or "")) + + def _require_ax_window(self, window_id: int, operation: str): + window = self._ax_window(window_id) + if window is None: + raise _refusal(operation, window_id) + return window + + def is_minimized(self, window_id: int) -> bool: + import ApplicationServices as ax + + window = self._ax_window(window_id) + if window is None: + # A minimised window is not in the on-screen list at all, so + # failing to find it is itself the answer here. + return self._info_for(window_id) is None + _error, value = ax.AXUIElementCopyAttributeValue( + window, "AXMinimized", None) + return bool(value) + + def set_foreground(self, window_id: int) -> None: + import AppKit + import ApplicationServices as ax + + pid = self.window_process_id(window_id) + if not pid: + raise _refusal("set_foreground", window_id) + running = AppKit.NSRunningApplication.runningApplicationWithProcessIdentifier_(pid) + if running is not None: + running.activateWithOptions_( + AppKit.NSApplicationActivateIgnoringOtherApps) + # Activating the application brings its front window forward, which is + # not necessarily the one asked for, so raise that one specifically. + window = self._ax_window(window_id) + if window is not None: + ax.AXUIElementPerformAction(window, "AXRaise") + + def restore(self, window_id: int) -> None: + import ApplicationServices as ax + + window = self._require_ax_window(window_id, "restore") + ax.AXUIElementSetAttributeValue(window, "AXMinimized", False) + + def show(self, window_id: int, cmd_show: int) -> None: + code = int(cmd_show) + if code in (SW_SHOWNORMAL, SW_RESTORE): + self.restore(window_id) + elif code in (SW_MINIMIZE, SW_SHOWMINIMIZED): + self.minimize(window_id) + else: + # macOS has no hide-this-window or maximise-this-window that maps + # onto the remaining Win32 codes; zoom is not maximise and the + # difference matters to the caller. + self._unsupported(f"show(cmd_show={code})") + + def close(self, window_id: int) -> bool: + import ApplicationServices as ax + + window = self._ax_window(window_id) + if window is None: + return False + error, button = ax.AXUIElementCopyAttributeValue( + window, "AXCloseButton", None) + if error or button is None: + return False + return not ax.AXUIElementPerformAction(button, "AXPress") + + def minimize(self, window_id: int) -> bool: + import ApplicationServices as ax + + window = self._ax_window(window_id) + if window is None: + return False + return not ax.AXUIElementSetAttributeValue(window, "AXMinimized", True) + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + import ApplicationServices as ax + import Quartz + + window = self._ax_window(window_id) + if window is None: + return False + position = ax.AXValueCreate( + Quartz.kAXValueCGPointType, Quartz.CGPoint(float(x), float(y))) + size = ax.AXValueCreate( + Quartz.kAXValueCGSizeType, + Quartz.CGSize(float(width), float(height))) + moved = ax.AXUIElementSetAttributeValue(window, "AXPosition", position) + resized = ax.AXUIElementSetAttributeValue(window, "AXSize", size) + return not moved and not resized + + # post_key / post_click are deliberately left to the base class. macOS has + # no equivalent of PostMessage or XSendEvent: an event goes to whatever + # has focus, and there is no way to address one window without raising it. + # Refusing says so; a "success" that focused something else would not. + + +def _best_match(candidates: list, wanted_origin: Tuple[int, int], + wanted_title: str): + """Pick the accessibility window that is the Quartz window described. + + Frame is the stronger signal and is tried first: a title can be empty, or + duplicated across an application's windows, while two windows cannot share + an origin at the same moment. + """ + import ApplicationServices as ax + + fallback = None + for window in candidates: + _error, position = ax.AXUIElementCopyAttributeValue( + window, "AXPosition", None) + if _point(position) == wanted_origin: + return window + if fallback is None and wanted_title: + _error, title = ax.AXUIElementCopyAttributeValue( + window, "AXTitle", None) + if str(title or "") == wanted_title: + fallback = window + return fallback + + +def _point(value: Any) -> Tuple[int, int]: + """Read an ``AXValue`` point as ``(x, y)``, or ``(-1, -1)``.""" + import ApplicationServices as ax + import Quartz + + if value is None: + return (-1, -1) + ok, point = ax.AXValueGetValue(value, Quartz.kAXValueCGPointType, None) + if not ok or point is None: + return (-1, -1) + return (int(point.x), int(point.y)) + + +def _refusal(operation: str, window_id: int): + from je_auto_control.utils.exception.exceptions import ( + AutoControlUnsupportedOperationException, + ) + + return AutoControlUnsupportedOperationException( + f"{operation}: no accessibility element for window {window_id}. " + "macOS gates window actions behind Accessibility — grant it to this " + "interpreter in System Settings > Privacy & Security > Accessibility.", + ) diff --git a/je_auto_control/wrapper/window_backends/null_backend.py b/je_auto_control/wrapper/window_backends/null_backend.py new file mode 100644 index 00000000..0f1ecf1f --- /dev/null +++ b/je_auto_control/wrapper/window_backends/null_backend.py @@ -0,0 +1,23 @@ +"""Fallback window backend for a platform with no implementation.""" +from typing import List, Tuple + +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + + +class NullWindowBackend(WindowManageBackend): + """Answers every operation with a refusal that says why. + + ``list_windows`` returns an empty list rather than raising: "there are no + windows I can see" is a truthful answer that lets a caller iterate, and + every operation that would *act* on a window still refuses loudly. + """ + + name = "null" + + def __init__(self, reason: str = "") -> None: + self.available = False + self.reason = reason or "no window backend for this platform" + self.name = f"null ({self.reason})" + + def list_windows(self) -> List[Tuple[int, str]]: + return [] diff --git a/je_auto_control/wrapper/window_backends/windows_backend.py b/je_auto_control/wrapper/window_backends/windows_backend.py new file mode 100644 index 00000000..39e87c39 --- /dev/null +++ b/je_auto_control/wrapper/window_backends/windows_backend.py @@ -0,0 +1,67 @@ +"""Windows window-management backend, over the existing Win32 module.""" +import sys +from typing import List, Optional, Tuple + +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + + +class WindowsWindowBackend(WindowManageBackend): + """Delegates to :mod:`je_auto_control.windows.window.windows_window_manage`. + + That module is unchanged and stays the single home of the Win32 calls; + this only adapts it to the shape the other platforms also answer in. + """ + + name = "win32" + + def __init__(self) -> None: + self.available = sys.platform in ("win32", "cygwin", "msys") + + @property + def _wm(self): + """The Win32 module, imported lazily so other platforms never load it.""" + from je_auto_control.windows.window import windows_window_manage + + return windows_window_manage + + def list_windows(self) -> List[Tuple[int, str]]: + return self._wm.get_all_window_hwnd() + + def foreground_window(self) -> int: + return self._wm.get_foreground_window() + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + return self._wm.get_window_rect(window_id) + + def window_process_id(self, window_id: int) -> int: + return self._wm.get_window_process_id(window_id) + + def is_minimized(self, window_id: int) -> bool: + return self._wm.is_window_minimized(window_id) + + def set_foreground(self, window_id: int) -> None: + self._wm.set_foreground_window(window_id) + + def restore(self, window_id: int) -> None: + self._wm.show_window(window_id, self._wm.SW_RESTORE) + + def show(self, window_id: int, cmd_show: int) -> None: + self._wm.show_window(window_id, int(cmd_show)) + + def close(self, window_id: int) -> bool: + return self._wm.close_window(window_id) + + def minimize(self, window_id: int) -> bool: + return self._wm.minimize_window(window_id) + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + return self._wm.move_window(window_id, x, y, width, height) + + def post_key(self, window_id: int, keycode: int, + character: str = "") -> bool: + return self._wm.post_key(window_id, keycode, character) + + def post_click(self, window_id: int, button: str, x: int, y: int) -> bool: + return self._wm.post_click(window_id, button, x, y) diff --git a/je_auto_control/wrapper/window_backends/x11_backend.py b/je_auto_control/wrapper/window_backends/x11_backend.py new file mode 100644 index 00000000..50ba1960 --- /dev/null +++ b/je_auto_control/wrapper/window_backends/x11_backend.py @@ -0,0 +1,398 @@ +"""X11 window-management backend, over EWMH and python-Xlib. + +Everything here goes through the EWMH properties and client messages a +window manager maintains (``_NET_CLIENT_LIST_STACKING``, ``_NET_ACTIVE_WINDOW``, +``_NET_WM_PID``, ``_NET_WM_STATE``, …) rather than by poking at windows +directly, because on X11 the window manager owns stacking, focus and +iconification — a client that reparents or raises behind its back gets +overruled or, worse, half-obeyed. + +``python-Xlib`` is already a hard dependency on Linux, so this adds nothing to +install. + +One caveat is written into :meth:`X11WindowBackend.post_key` and +:meth:`X11WindowBackend.post_click` rather than hidden: events delivered with +``XSendEvent`` arrive at the client flagged *synthetic*, and most toolkits +(GTK and Qt among them) ignore synthetic input by design. They are the closest +X11 equivalent of Win32's ``PostMessage`` — which also bypasses focus — and +they are honest about being best-effort. +""" +import sys +import time +from typing import Any, List, Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + +#: ``ShowWindow`` codes this backend can honour, mapped to what X11 calls the +#: same idea. The Win32 numbering is the project's cross-platform spelling of +#: show-state; the codes with no X11 meaning are refused rather than guessed. +SW_HIDE = 0 +SW_SHOWNORMAL = 1 +SW_SHOWMINIMIZED = 2 +SW_MAXIMIZE = 3 +SW_MINIMIZE = 6 +SW_RESTORE = 9 + +#: ICCCM window states, from the specification. +_ICONIC_STATE = 3 + +#: Button names the project uses, in X11's button numbering. +_BUTTONS = {"left": 1, "middle": 2, "right": 3} + + +def _is_linux() -> bool: + return sys.platform in ("linux", "linux2") + + +class X11WindowBackend(WindowManageBackend): + """Window management through the window manager's own EWMH contract.""" + + name = "x11-ewmh" + + def __init__(self) -> None: + self._connection = None + self._atoms: dict = {} + self.available = _is_linux() and self._probe() + + # --- connection -------------------------------------------------------- + + def _probe(self) -> bool: + try: + self._display() + return True + except Exception as error: # noqa: BLE001 # reason: any X failure means unavailable + autocontrol_logger.info( + "X11 window backend unavailable: %r", error) + return False + + def _display(self): + """The X connection, opened once and kept. + + Its own connection rather than the input backend's: this module is + selected by the wrapper independently of which input path is active, + and on Wayland-with-XWayland the input backend may not have opened one + at all. + """ + if self._connection is None: + from Xlib import display as xdisplay + + self._connection = xdisplay.Display() + return self._connection + + def _atom(self, name: str) -> int: + """Intern an atom once per connection.""" + if name not in self._atoms: + self._atoms[name] = self._display().intern_atom(name) + return self._atoms[name] + + def _root(self): + return self._display().screen().root + + def _window(self, window_id: int): + return self._display().create_resource_object("window", int(window_id)) + + def _property(self, window, name: str, kind: Optional[int] = None): + """Read a property's value list, or ``None`` when it is absent.""" + from Xlib import X + + found = window.get_full_property( + self._atom(name), X.AnyPropertyType if kind is None else kind) + return None if found is None else found.value + + def _client_message(self, window, name: str, data: List[int]) -> None: + """Send an EWMH client message to the root window. + + EWMH requests are addressed to the root with + ``SubstructureRedirect``: that is what routes them to the window + manager, which is the only party allowed to act on them. + """ + from Xlib import X, protocol + + padded = (list(data) + [0, 0, 0, 0, 0])[:5] + event = protocol.event.ClientMessage( + window=window, client_type=self._atom(name), data=(32, padded)) + self._root().send_event( + event, + event_mask=X.SubstructureRedirectMask | X.SubstructureNotifyMask) + self._display().flush() + + # --- listing ----------------------------------------------------------- + + def list_windows(self) -> List[Tuple[int, str]]: + from Xlib import Xatom + + # Stacking order is bottom-to-top, so reversing it puts the front-most + # window first — which is what makes "the first title that matches" the + # one the user meant. _NET_CLIENT_LIST carries no order at all, so it + # is only the fallback. + ids = self._property(self._root(), "_NET_CLIENT_LIST_STACKING", + Xatom.WINDOW) + if ids is None: + ids = self._property(self._root(), "_NET_CLIENT_LIST", + Xatom.WINDOW) or [] + ordered = list(ids) + else: + ordered = list(reversed(list(ids))) + return [(int(window_id), self._title(int(window_id))) + for window_id in ordered] + + def _title(self, window_id: int) -> str: + """``_NET_WM_NAME`` if the client sets it, else the legacy ``WM_NAME``.""" + window = self._window(window_id) + try: + value = self._property(window, "_NET_WM_NAME") + if value: + return _as_text(value) + legacy = window.get_wm_name() + return legacy if isinstance(legacy, str) else _as_text(legacy) + except Exception: # noqa: BLE001 # reason: a window can vanish mid-walk + return "" + + def foreground_window(self) -> int: + from Xlib import Xatom + + active = self._property(self._root(), "_NET_ACTIVE_WINDOW", Xatom.WINDOW) + return int(active[0]) if active else 0 + + # --- reading one window ------------------------------------------------ + + def _frame(self, window_id: int): + """The window manager's frame around a client, or the client itself. + + A reparenting window manager makes the client a grandchild of the + root, inside a frame that carries the border and title bar. The frame + is what the user sees and drags, so it is the window every coordinate + here is about. + """ + window = self._window(window_id) + root_id = self._root().id + # Bounded rather than `while True`: a frame is one or two levels, and + # a cycle here would hang the caller instead of returning something. + for _ in range(16): + tree = window.query_tree() + parent = getattr(tree, "parent", None) + if parent is None or parent.id == root_id: + return window + window = parent + return window + + def _frame_extents(self, window_id: int) -> Tuple[int, int, int, int]: + """``(left, right, top, bottom)`` decoration thickness, or zeroes.""" + from Xlib import Xatom + + try: + value = self._property(self._window(window_id), + "_NET_FRAME_EXTENTS", Xatom.CARDINAL) + except Exception: # noqa: BLE001 # reason: the window may be gone + return (0, 0, 0, 0) + if not value or len(value) < 4: + return (0, 0, 0, 0) + return (int(value[0]), int(value[1]), int(value[2]), int(value[3])) + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + """The *frame* rectangle, decorations included. + + Win32's ``GetWindowRect`` returns the frame, and every caller in this + project is written against that, so returning the client area here + would be off by the border and title bar on X11 alone — silently, and + by a different amount per window manager. + """ + try: + frame = self._frame(window_id) + geometry = frame.get_geometry() + # The frame is a direct child of the root, so its own x/y are + # already root coordinates. Translating instead would add the + # window's border width, because XTranslateCoordinates starts + # from inside the border — measured as a one-pixel error against + # xwininfo on an undecorated window. + left, top = int(geometry.x), int(geometry.y) + return (left, top, left + int(geometry.width), + top + int(geometry.height)) + except Exception as error: # noqa: BLE001 # reason: the window may be gone + autocontrol_logger.info("window_rect(%s) failed: %r", window_id, error) + return None + + def window_process_id(self, window_id: int) -> int: + from Xlib import Xatom + + try: + value = self._property(self._window(window_id), "_NET_WM_PID", + Xatom.CARDINAL) + except Exception: # noqa: BLE001 # reason: the window may be gone + return 0 + return int(value[0]) if value else 0 + + def is_minimized(self, window_id: int) -> bool: + try: + states = self._property(self._window(window_id), "_NET_WM_STATE") or [] + except Exception: # noqa: BLE001 # reason: the window may be gone + return False + return self._atom("_NET_WM_STATE_HIDDEN") in list(states) + + # --- acting on one window ---------------------------------------------- + + def set_foreground(self, window_id: int) -> None: + window = self._window(window_id) + # Source 2 ("pager") is what a window manager honours without the + # focus-stealing prevention it applies to source 1 ("application"). + self._client_message(window, "_NET_ACTIVE_WINDOW", + [2, int(time.time()), 0]) + self._display().flush() + + def restore(self, window_id: int) -> None: + window = self._window(window_id) + window.map() + self._client_message(window, "_NET_ACTIVE_WINDOW", + [2, int(time.time()), 0]) + self._display().flush() + + def show(self, window_id: int, cmd_show: int) -> None: + window = self._window(window_id) + code = int(cmd_show) + if code == SW_HIDE: + window.unmap() + elif code in (SW_SHOWNORMAL, SW_RESTORE): + self.restore(window_id) + elif code in (SW_MINIMIZE, SW_SHOWMINIMIZED): + self.minimize(window_id) + elif code == SW_MAXIMIZE: + # _NET_WM_STATE_ADD is 1; both axes have to be named or the window + # only grows one way. + self._client_message( + window, "_NET_WM_STATE", + [1, self._atom("_NET_WM_STATE_MAXIMIZED_HORZ"), + self._atom("_NET_WM_STATE_MAXIMIZED_VERT"), 2]) + else: + # Win32 has show codes with no X11 meaning (SW_SHOWNA, + # SW_FORCEMINIMIZE, …). Guessing at one would move a window in a + # way the caller did not ask for. + self._unsupported(f"show(cmd_show={code})") + self._display().flush() + + def close(self, window_id: int) -> bool: + try: + self._client_message(self._window(window_id), "_NET_CLOSE_WINDOW", + [int(time.time()), 2]) + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("close(%s) failed: %r", window_id, error) + return False + + def minimize(self, window_id: int) -> bool: + try: + # Iconifying is ICCCM, not EWMH: there is no _NET_ message for it, + # and WM_CHANGE_STATE is what every window manager implements. + self._client_message(self._window(window_id), "WM_CHANGE_STATE", + [_ICONIC_STATE]) + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("minimize(%s) failed: %r", window_id, error) + return False + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + """Move and resize the *frame*, matching :meth:`window_rect`. + + Goes through ``_NET_MOVERESIZE_WINDOW`` rather than configuring the + window directly: under a reparenting window manager a client's own + x/y are relative to its frame, so a direct ``ConfigureWindow`` asks + for a position in the wrong coordinate space and the window manager + applies its own arithmetic on top. Measured against openbox, asking + for (300, 220) that way landed the window at (302, 260). + + EWMH sizes the *client*, while Win32's ``MoveWindow`` sizes the + frame, so the decorations come off the requested size — which is what + makes this round-trip with :meth:`window_rect`. + """ + try: + border_left, border_right, border_top, border_bottom = \ + self._frame_extents(window_id) + client_width = max(1, int(width) - border_left - border_right) + client_height = max(1, int(height) - border_top - border_bottom) + # Bits 8-11 say which of x/y/width/height are supplied; bits 12-13 + # are the source indication, and 2 means "pager", which a window + # manager honours without focus-stealing prevention. Gravity 0 + # leaves the window's own gravity in charge, so x/y place the + # frame's top-left corner. + flags = (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (2 << 12) + self._client_message( + self._window(window_id), "_NET_MOVERESIZE_WINDOW", + [flags, int(x), int(y), client_width, client_height]) + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("move(%s) failed: %r", window_id, error) + return False + + # --- acting on a window that does not have focus ----------------------- + + def post_key(self, window_id: int, keycode: int, + character: str = "") -> bool: + """Send one key to the window without focusing it. + + Delivered with ``XSendEvent``, so it arrives flagged synthetic and + GTK and Qt discard it by design. This is the X11 counterpart of + Win32's ``PostMessage`` — best-effort, and useful mainly for the + older toolkits that do accept it. For input that always lands, focus + the window and use the ordinary keyboard API. + """ + from Xlib import X, protocol + + del character # X11 addresses keys by keycode; the character is Win32's + try: + window = self._window(window_id) + for factory, mask in ((protocol.event.KeyPress, X.KeyPressMask), + (protocol.event.KeyRelease, X.KeyReleaseMask)): + window.send_event( + factory(time=X.CurrentTime, root=self._root(), + window=window, same_screen=1, child=X.NONE, + root_x=0, root_y=0, event_x=0, event_y=0, + state=0, detail=int(keycode)), + propagate=True, event_mask=mask) + self._display().flush() + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("post_key(%s) failed: %r", window_id, error) + return False + + def post_click(self, window_id: int, button: str, x: int, y: int) -> bool: + """Send one click to the window without focusing it. + + Carries the same synthetic-event caveat as :meth:`post_key`. + """ + from Xlib import X, protocol + + number = _BUTTONS.get(str(button).lower().removeprefix("mouse_")) + if number is None: + self._unsupported(f"post_click(button={button!r})") + try: + window = self._window(window_id) + for factory, mask in ( + (protocol.event.ButtonPress, X.ButtonPressMask), + (protocol.event.ButtonRelease, X.ButtonReleaseMask)): + window.send_event( + factory(time=X.CurrentTime, root=self._root(), + window=window, same_screen=1, child=X.NONE, + root_x=0, root_y=0, event_x=int(x), event_y=int(y), + state=0, detail=number), + propagate=True, event_mask=mask) + self._display().flush() + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("post_click(%s) failed: %r", window_id, error) + return False + + +def _as_text(value: Any) -> str: + """Decode a property value that may be bytes, str, or an array of either.""" + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, str): + return value + try: + return bytes(value).decode("utf-8", errors="replace").rstrip("\x00") + except (TypeError, ValueError): + return str(value) diff --git a/test/unit_test/headless/test_window_backends.py b/test/unit_test/headless/test_window_backends.py new file mode 100644 index 00000000..94d9aa08 --- /dev/null +++ b/test/unit_test/headless/test_window_backends.py @@ -0,0 +1,228 @@ +"""Headless tests for the window-management backend seam. No Qt. + +The old window tests are Windows-only, because the facade was: it branched on +``sys.platform`` and raised everywhere else, so there was nothing to test on +another platform. Now that the platform detail sits behind a backend, the +facade's own logic — substring matching, ordering, "move without restating the +size", waiting — is platform-neutral and can be checked anywhere, which is +what these do by driving a fake backend. +""" +import sys + +import pytest + +from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, AutoControlException, + AutoControlUnsupportedOperationException, +) +from je_auto_control.wrapper import auto_control_window as w +from je_auto_control.wrapper.window_backends import ( + NullWindowBackend, WindowManageBackend, get_backend, reset_backend_cache, +) + + +class FakeBackend(WindowManageBackend): + """A backend that records what it was asked and answers predictably.""" + + name = "fake" + + def __init__(self): + self.available = True + self.calls = [] + self.windows = [(11, "Editor"), (12, " "), (13, "Browser")] + self.rect = (10, 20, 110, 220) + self.minimized = False + + def list_windows(self): + return list(self.windows) + + def foreground_window(self): + return 13 + + def window_rect(self, window_id): + self.calls.append(("window_rect", window_id)) + return self.rect + + def window_process_id(self, window_id): + return {11: 111, 12: 112, 13: 113}.get(window_id, 0) + + def is_minimized(self, window_id): + return self.minimized + + def set_foreground(self, window_id): + self.calls.append(("set_foreground", window_id)) + + def restore(self, window_id): + self.calls.append(("restore", window_id)) + + def show(self, window_id, cmd_show): + self.calls.append(("show", window_id, cmd_show)) + + def close(self, window_id): + self.calls.append(("close", window_id)) + return True + + def minimize(self, window_id): + self.calls.append(("minimize", window_id)) + return True + + def move(self, window_id, x, y, width, height): + self.calls.append(("move", window_id, x, y, width, height)) + return True + + def post_key(self, window_id, keycode, character=""): + self.calls.append(("post_key", window_id, keycode, character)) + return True + + def post_click(self, window_id, button, x, y): + self.calls.append(("post_click", window_id, button, x, y)) + return True + + +@pytest.fixture() +def backend(monkeypatch): + """Point the facade at a fake backend, on every platform.""" + fake = FakeBackend() + monkeypatch.setattr(w, "get_backend", lambda: fake) + return fake + + +# --- the refusal contract -------------------------------------------------- + + +def test_unsupported_is_catchable_as_both_families(): + """A bare NotImplementedError escaped every containment boundary. + + The GUI tabs and the REST handler catch ``NotImplementedError`` to say + "not on this platform", so that has to keep working. The executor and the + background loops catch ``AutoControlException``, and a bare + ``NotImplementedError`` slipped straight past them — aborting a whole + script where one action should have been reported as failed. + """ + assert issubclass(AutoControlUnsupportedOperationException, + NotImplementedError) + assert issubclass(AutoControlUnsupportedOperationException, + AutoControlException) + + +def test_base_backend_refuses_by_name(): + base = WindowManageBackend() + with pytest.raises(AutoControlUnsupportedOperationException) as caught: + base.close(1) + assert "close" in str(caught.value) + assert "abstract" in str(caught.value) + + +def test_null_backend_lists_nothing_but_refuses_actions(): + """Listing is answerable without a backend; acting is not. + + "There are no windows I can see" lets a caller iterate and move on. A + silent False from ``close`` would read as "the window refused", which is + a different thing and would have callers retrying forever. + """ + null = NullWindowBackend("no display here") + assert null.list_windows() == [] + assert not null.available + assert "no display here" in null.name + with pytest.raises(AutoControlUnsupportedOperationException): + null.close(1) + + +def test_selection_is_cached_and_resettable(): + reset_backend_cache() + first = get_backend() + assert get_backend() is first + reset_backend_cache() + assert get_backend() is not first + + +def test_every_platform_gets_a_backend_that_imports(): + """No platform may fail to import, whatever it can or cannot do.""" + reset_backend_cache() + chosen = get_backend() + assert isinstance(chosen, WindowManageBackend) + assert chosen.name + + +@pytest.mark.skipif(sys.platform not in ("win32", "cygwin", "msys"), + reason="the Win32 backend only builds on Windows") +def test_windows_selects_the_win32_backend(): + reset_backend_cache() + assert get_backend().name == "win32" + + +# --- the facade's platform-neutral logic ----------------------------------- + + +def test_list_windows_can_drop_the_untitled(backend): + assert w.list_windows() == backend.windows + assert w.list_windows(titled_only=True) == [(11, "Editor"), (13, "Browser")] + + +def test_find_window_matches_case_insensitively_by_default(backend): + assert w.find_window("edit") == (11, "Editor") + assert w.find_window("EDIT") == (11, "Editor") + assert w.find_window("EDIT", case_sensitive=True) is None + assert w.find_window("nothing") is None + + +def test_focus_window_restores_only_when_minimised(backend): + """SW_RESTORE on a maximised window un-maximises it. + + So focusing must not restore unconditionally: that would shrink a + maximised window as a side effect of being asked to focus it. + """ + w.focus_window("Editor") + assert ("restore", 11) not in backend.calls + assert ("set_foreground", 11) in backend.calls + + backend.calls.clear() + backend.minimized = True + w.focus_window("Editor") + assert ("restore", 11) in backend.calls + + +def test_focus_window_reports_a_missing_window_as_an_action_failure(backend): + with pytest.raises(AutoControlActionException): + w.focus_window("nothing matches this") + + +def test_move_without_a_size_keeps_the_current_size(backend): + """Otherwise a plain reposition has to restate dimensions it must look up.""" + assert w.move_window_by_title("Editor", 5, 6) + assert ("move", 11, 5, 6, 100, 200) in backend.calls + + +def test_move_with_a_size_passes_it_through(backend): + assert w.move_window_by_title("Editor", 5, 6, 70, 80) + assert ("move", 11, 5, 6, 70, 80) in backend.calls + + +def test_close_and_minimize_report_no_match_as_false(backend): + assert w.close_window_by_title("nothing") is False + assert w.minimize_window_by_title("nothing") is False + assert backend.calls == [] + + +def test_windows_for_process_id_filters_by_owner(backend): + assert w.windows_for_process_id(111) == [(11, "Editor")] + assert w.windows_for_process_id(999) == [] + + +def test_minimize_windows_for_process_counts_what_it_minimised(backend): + assert w.minimize_windows_for_process(113) == 1 + assert ("minimize", 13) in backend.calls + + +def test_foreground_window_pairs_the_id_with_its_title(backend): + assert w.foreground_window() == (13, "Browser") + assert w.foreground_window_process_id() == 113 + + +def test_wait_for_window_returns_as_soon_as_it_appears(backend): + assert w.wait_for_window("Browser", timeout=1.0, poll=0.05) == 13 + + +def test_wait_for_window_times_out_as_an_action_failure(backend): + with pytest.raises(AutoControlActionException): + w.wait_for_window("never", timeout=0.2, poll=0.05) diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py index 368be557..e870033c 100644 --- a/test/verify/macos_verify.py +++ b/test/verify/macos_verify.py @@ -212,6 +212,30 @@ def probe_recorder() -> Outcome: f"recorder is {platform_wrapper.recorder!r}") +def probe_window_management() -> Outcome: + """Quartz lists windows without a grant; acting on one needs Accessibility. + + Listing is what this measures, because it is the half that has to work + before any of the rest means anything — and because a runner with no + windows at all would make every window command untestable here. + """ + import je_auto_control as ac + from je_auto_control.wrapper.window_backends import get_backend + + backend = get_backend() + if not backend.available: + return Outcome(False, f"backend {backend.name!r} reports unavailable") + windows = ac.list_windows() + if not windows: + return Outcome(False, f"backend {backend.name!r} listed no windows") + window_id, title = windows[0] + rect = backend.window_rect(window_id) + pid = backend.window_process_id(window_id) + return Outcome(rect is not None and pid > 0, + f"{len(windows)} window(s); first {title!r} " + f"rect={rect} pid={pid}") + + PROBES: List[Tuple[str, Callable[[], Outcome]]] = [ ("backend-selection", probe_backend), ("screen-size", probe_screen_size), @@ -222,6 +246,7 @@ def probe_recorder() -> Outcome: ("keyboard-post", probe_keyboard), ("accessibility-tree", probe_accessibility), ("recorder-absent", probe_recorder), + ("window-management", probe_window_management), ] From 688dd2be301ea532ee2e76053bb8e33a0e0f895b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 03:37:47 +0800 Subject: [PATCH 07/30] Give Linux an accessibility backend, over AT-SPI2 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Linux had none: the selector fell through to the null backend while the capability matrix claimed "backend tests" for Linux X11. AT-SPI2 is a D-Bus protocol rather than a library, which is what makes it reachable without a new dependency — pyatspi and gi.repository.Atspi are distribution packages built against the system introspection data and cannot be installed into a virtual environment, so depending on them would be depending on something most users cannot get. The D-Bus client written for the portal handshake moves from linux_wayland/ to utils/dbus_client/ to make that possible: utils/ sits above the per-OS packages, so an accessibility backend reaching down into a platform backend to borrow its D-Bus code would invert the layering. The old path re-exports it, so the portal code is untouched; the client's own tests follow it, since a shim forwarding private names too would be a second copy of the surface to keep in step. Verifying it against a real bus and a real GTK application immediately found a gap in that client: it could not demarshal signed integers. The portal never needed one, and AT-SPI reports a component's extents as four signed values, because a window on a monitor left of or above the primary one is at a negative coordinate — so the backend could read a tree but not where anything in it was. The whole fixed-width numeric set marshals now, except UNIX_FD, which stays an error on purpose: it is an index into a descriptor array this client does not receive, so returning it would hand a caller a number that addresses nothing. The x11-verification job grew a third script for it, against zenity on a D-Bus-activated accessibility bus. Neither half can be mocked usefully — an application only appears on the bus if its toolkit bridge loaded, and the tree's shape is the toolkit's business. Because AT-SPI is a bus rather than a display protocol, this is the one capability where Wayland is not the restricted case: the same bus serves both Linux sessions. --- CLAUDE.md | 6 +- README.md | 2 +- README/README_zh-CN.md | 2 +- README/README_zh-TW.md | 2 +- architecture_explore.md | 29 +- docker/Dockerfile.x11 | 12 + docker/entrypoint-x11.sh | 14 + docker/x11_atspi_verify.py | 191 +++++ docs/CAPABILITY_MATRIX.md | 32 +- je_auto_control/linux_wayland/_dbus_client.py | 633 +---------------- .../utils/accessibility/backends/__init__.py | 14 + .../accessibility/backends/linux_backend.py | 414 +++++++++++ je_auto_control/utils/dbus_client/__init__.py | 24 + .../utils/dbus_client/session_bus.py | 656 ++++++++++++++++++ .../headless/test_accessibility_linux.py | 233 +++++++ .../headless/test_docker_artifacts.py | 21 + .../headless/test_wayland_dbus_client.py | 39 +- 17 files changed, 1683 insertions(+), 641 deletions(-) create mode 100644 docker/x11_atspi_verify.py create mode 100644 je_auto_control/utils/accessibility/backends/linux_backend.py create mode 100644 je_auto_control/utils/dbus_client/__init__.py create mode 100644 je_auto_control/utils/dbus_client/session_bus.py create mode 100644 test/unit_test/headless/test_accessibility_linux.py diff --git a/CLAUDE.md b/CLAUDE.md index dc90059c..2ceb846e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mouse and keyboard control, image recognition, OCR, accessibility-tree and VLM element location, action scripting, and report generation behind one API. Backends: Windows (Win32 ctypes), macOS (pyobjc/Quartz), Linux X11 (python-Xlib), Linux Wayland (libei / ydotool), Android (adb), iOS (WebDriverAgent). - **Package**: `je_auto_control` · **Python** ≥ 3.10 · **License**: MIT · **Author**: JE-Chen -- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 308 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. +- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 309 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. ## Architecture @@ -18,7 +18,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mo | Template Method | `utils/generate_report/` | HTML / JSON / XML share collect → format → write. | | Backend seam | `backends/` under `accessibility`, `ocr`, `vision`, `llm`, `agent`, `hotkey`, `usb`, `usbip` | Abstract base + concrete impls + null fallback, so dependency-free environments still import. | -Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (308 headless subpackages) → `wrapper/` → per-OS backend. +Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (309 headless subpackages) → `wrapper/` → per-OS backend. ## Development Commands @@ -68,7 +68,7 @@ The map is only useful while it matches the tree, so **update it in the same cha Never adjust one by hand: the counts are `len(text.splitlines())` (what `wc -l` reports), and hand-editing is how the document ended up quoting the same subsystem at two sizes at once — most tables had been counting a phantom trailing line per file while §1 and §8 counted correctly. -- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 308 subpackages; appearing twice or not at all is a defect. +- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 309 subpackages; appearing twice or not at all is a defect. - A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. - Keep the header's scan date, version, and branch current. - `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. diff --git a/README.md b/README.md index 18066393..6d679ea4 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Diagnostics | `run_diagnostics` | `AC_diagnose` | Diagnostics | | Test-code generation | `generate_code` | — | — | -Beyond this table, `utils/` holds 308 headless packages covering assertions, resilience, +Beyond this table, `utils/` holds 309 headless packages covering assertions, resilience, data quality, i18n auditing, redaction, governance, observability, and more. The full per-module map is in **[architecture_explore.md](architecture_explore.md)**. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 48447f7b..047eb5be 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -147,7 +147,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 系统诊断 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 测试代码生成 | `generate_code` | — | — | -除了这张表，`utils/` 下还有 308 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 +除了这张表，`utils/` 下还有 309 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 治理、可观测性等等。完整的逐模块地图在 **[architecture_explore.md](../architecture_explore.md)**。 --- diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index b9f04444..7be0fa71 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -147,7 +147,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 系統診斷 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 測試碼產生 | `generate_code` | — | — | -除了這張表，`utils/` 底下還有 308 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 +除了這張表，`utils/` 底下還有 309 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 治理、可觀測性等等。完整的逐模組地圖在 **[architecture_explore.md](../architecture_explore.md)**。 --- diff --git a/architecture_explore.md b/architecture_explore.md index 92044e96..5cbc0157 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,9 +19,9 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,022 | -| 程式碼總行數 | 138,510 | -| `je_auto_control/utils/` 子套件數 | 308 | +| Python 模組總數（含周邊子專案） | 1,025 | +| 程式碼總行數 | 139,017 | +| `je_auto_control/utils/` 子套件數 | 309 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | | GUI 分頁數（`main_widget` 註冊） | 48 | @@ -55,7 +55,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 └───────────────────────────────┬──────────────────────────────────────────┘ │ ┌───────────────────────────────▼──────────────────────────────────────────┐ -│ 能力層 utils/（308 個子套件，全部無 Qt 相依） │ +│ 能力層 utils/（309 個子套件，全部無 Qt 相依） │ │ 影像辨識 │ OCR │ 無障礙樹 │ 定位自癒 │ AI/Agent │ 遠端桌面 │ USB │ │ 報表觀測 │ 資料 │ 安全 │ 韌性 │ 系統整合 │ 排程觸發 │ 網路協定 │ └───────────────────────────────┬──────────────────────────────────────────┘ @@ -226,14 +226,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，17 檔／3,431 行） +#### Linux Wayland（`linux_wayland/`，17 檔／2,830 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | | `_detect.py` | 77 | Wayland session 偵測與 CLI 工具探測。 | | `_ydotool_cli.py` | 134 | 判定安裝的是哪一代 ydotool 命令列，擋掉會靜默失效的 0.1.x（對本專案送的 argv 回傳 0 卻不送任何事件）。 | | `_ctypes_bind.py` | 75 | libei／liboeffis 共用的 ctypes 載入與 prototype 綁定。 | -| `_dbus_client.py` | 625 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | +| `_dbus_client.py` | 24 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | | `_select_input.py` | 85 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | | `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | | `oeffis.py` | 196 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | @@ -258,7 +258,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `ios/input.py` | 46 | iOS 觸控與按鍵原語。 | | `ios/screen.py` | 32 | iOS 裝置螢幕擷取與尺寸。 | -### 5.4 能力層 `utils/`（308 個子套件） +### 5.4 能力層 `utils/`（309 個子套件） 以下依主題分組。每個子套件都是獨立可匯入的無頭模組，不含任何 Qt 相依。 @@ -296,7 +296,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.2 框架基礎設施 -> 12 個套件、約 1,895 行。 +> 13 個套件、約 2,575 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -304,6 +304,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | +| `utils/dbus_client/` | 680 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | | `utils/exception/` | 208 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | | `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | @@ -432,12 +433,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.7 無障礙樹與原生控制項 -> 16 個套件、約 3,851 行。 +> 16 個套件、約 4,279 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a11y_audit/` | 355 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | -| `utils/accessibility/` | 2,390 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | +| `utils/accessibility/` | 2,818 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | | `utils/ax_events/` | 29 | 反應式 UIA 事件等待（focus-changed） | | `utils/ax_props/` | 44 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | | `utils/ax_text/` | 102 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | @@ -1021,13 +1022,13 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,247 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | -| `utils/accessibility/` | 12 | 2,390 | +| `utils/accessibility/` | 13 | 2,818 | | `wrapper/` | 3,026 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | | `windows/` | 23 | 1,995 | | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | | `linux_with_x11/` | 19 | 1,175 | -| `linux_wayland/` | 17 | 3,431 | +| `linux_wayland/` | 17 | 2,830 | | `utils/triggers/` | 4 | 1,146 | | `utils/ocr/` | 9 | 1,112 | | `utils/usbip/` | 5 | 920 | @@ -1035,6 +1036,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 761 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 685 | 49,278 | -| **總計** | **1,016** | **138,445** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 687 | 49,958 | +| **總計** | **1,019** | **138,952** | diff --git a/docker/Dockerfile.x11 b/docker/Dockerfile.x11 index affafba1..75ff25d8 100644 --- a/docker/Dockerfile.x11 +++ b/docker/Dockerfile.x11 @@ -62,12 +62,18 @@ ARG DEBIAN_FRONTEND=noninteractive # - imagemagick: `import -window root`, an independent grabber. # - openbox: a real EWMH window manager, without a desktop over the root. # - xterm: a real client to own a real window. +# - at-spi2-core + dbus-x11: the accessibility bus. AT-SPI is a D-Bus +# protocol, so the backend needs a session bus and the two services +# D-Bus activates on it (org.a11y.Bus and the registry). +# - zenity: a real GTK application, so there is an actual accessible +# tree to walk. xterm exposes none — it is not a toolkit application. # - libgl1 + libglib2.0-0: opencv-python hard-requires libGL.so.1 and # libgthread-2.0.so.0 at import, so the package cannot even load without them. RUN apt-get update \ && apt-get install -y --no-install-recommends \ xvfb xauth x11-utils x11-xserver-utils xdotool \ imagemagick openbox xterm \ + at-spi2-core dbus-x11 zenity \ libgl1 libglib2.0-0 \ ca-certificates \ && rm -rf /var/lib/apt/lists/* @@ -82,6 +88,7 @@ RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ COPY docker/x11_verify.py /opt/verify/x11_verify.py COPY docker/x11_window_verify.py /opt/verify/x11_window_verify.py +COPY docker/x11_atspi_verify.py /opt/verify/x11_atspi_verify.py COPY docker/entrypoint-x11.sh /usr/local/bin/autocontrol-x11-verify RUN chmod +x /usr/local/bin/autocontrol-x11-verify @@ -97,7 +104,12 @@ RUN useradd --create-home --shell /bin/sh verify \ && chown -R verify:verify /app USER verify +# GTK3 only loads the accessibility bridge when it is asked to, and +# what normally asks is a desktop setting this container has no +# store for. Naming the module directly is what bridges the app. ENV HOME=/home/verify \ + GTK_MODULES=gail:atk-bridge \ + NO_AT_BRIDGE=0 \ PYTHONUNBUFFERED=1 \ DISPLAY=:99 \ XDG_SESSION_TYPE=x11 \ diff --git a/docker/entrypoint-x11.sh b/docker/entrypoint-x11.sh index 8fb4524b..4966c163 100644 --- a/docker/entrypoint-x11.sh +++ b/docker/entrypoint-x11.sh @@ -19,6 +19,14 @@ # or the monitor layout cannot be brought up, this says so and fails. set -eu +# AT-SPI is a D-Bus protocol: without a session bus there is no accessibility +# bus to activate, and the backend would correctly report itself unavailable +# for a reason that is this container's fault rather than the code's. Re-exec +# under one, once. +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ]; then + exec dbus-run-session -- "$0" "$@" +fi + GEOMETRY="${SCREEN_GEOMETRY:-1280x800x24}" DISPLAY_NUM="${DISPLAY:-:99}" @@ -118,6 +126,12 @@ echo "window management against a real window manager" echo "========================================================================" python3 /opt/verify/x11_window_verify.py || total=$((total + $?)) +echo +echo "========================================================================" +echo "accessibility against a real AT-SPI bus" +echo "========================================================================" +python3 /opt/verify/x11_atspi_verify.py || total=$((total + $?)) + echo echo "========================================================================" echo "total failed checks: ${total}" diff --git a/docker/x11_atspi_verify.py b/docker/x11_atspi_verify.py new file mode 100644 index 00000000..5385cf9a --- /dev/null +++ b/docker/x11_atspi_verify.py @@ -0,0 +1,191 @@ +"""Verify the Linux accessibility backend against a real AT-SPI bus. + +Linux had no accessibility backend at all — the selector fell through to the +null one — while the capability matrix claimed "backend tests" for Linux X11. +The backend that closes that speaks AT-SPI2 over D-Bus directly, with no +binding, so what has to be checked is whether a *real* accessibility bus and a +*real* toolkit application agree with the bytes it sends. + +Neither half can be mocked usefully. The bus is activated by D-Bus rather than +started by hand, an application only appears on it if its toolkit bridge +loaded, and the tree's shape is the toolkit's business. So the subject here is +``zenity`` — a GTK application with a label and buttons whose names are known +because this script chose them. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import os +import subprocess # nosec B404 # reason: argv lists of fixed tool names, no shell +import sys +import time +from typing import Any, Optional + +from x11_verify import _assert_eq, _assert_true, check, note, summarise + +#: The text zenity is told to show. Distinctive so a match cannot be an +#: accident of some other accessible carrying the same words. +DIALOG_TITLE = "autocontrol-atspi-dialog" +DIALOG_TEXT = "autocontrol-atspi-label" + +#: How long the toolkit is given to bridge itself onto the bus. This is the +#: slowest part of the whole image: the application has to start, load the +#: bridge module, and register with the registry. +BRIDGE_TIMEOUT = 30.0 + + +class Zenity: + """A real GTK application, bridged onto the accessibility bus.""" + + def __init__(self) -> None: + self._process: Optional[subprocess.Popen] = None + + def __enter__(self) -> "Zenity": + self._process = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["zenity", "--info", "--title", DIALOG_TITLE, + "--text", DIALOG_TEXT], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is not None and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + + def alive(self) -> bool: + return self._process is not None and self._process.poll() is None + + +def _await_bridged(backend, timeout: float = BRIDGE_TIMEOUT) -> list: + """Wait until the application shows up on the bus, then return its tree.""" + deadline = time.monotonic() + timeout + last: list = [] + while time.monotonic() < deadline: + last = backend.list_elements(max_results=400) + if any(DIALOG_TEXT in element.name or DIALOG_TITLE in element.name + for element in last): + return last + time.sleep(0.5) + return last + + +def check_backend_selection() -> None: + def _selected() -> str: + from je_auto_control.utils.accessibility.backends import ( + get_backend, reset_backend_cache, + ) + + reset_backend_cache() + backend = get_backend() + _assert_true( + backend.available, + f"backend {backend.name!r} reports unavailable — if this says " + f"'null', the accessibility bus did not come up in this container") + return _assert_eq(backend.name, "linux-atspi") + check("Linux selects the AT-SPI backend, not the null one", _selected) + + +def check_tree(backend, zenity: Zenity) -> None: + elements = _await_bridged(backend) + + def _application_appears() -> str: + _assert_true(zenity.alive(), "zenity exited before it was inspected") + names = [element.name for element in elements] + _assert_true( + any(DIALOG_TEXT in name or DIALOG_TITLE in name for name in names), + f"the dialog never appeared on the bus; saw {names[:20]}") + return f"{len(elements)} accessibles, including the dialog" + check("a real GTK application is reachable over the bus", + _application_appears) + + def _roles_are_named() -> str: + # GetRoleName is a separate call from the name, and a backend that + # skipped it would still list every element — with no role at all, + # which is what a role= filter searches on. + roles = {element.role for element in elements if element.role} + _assert_true(bool(roles), "no element reported a role") + return f"{len(roles)} distinct roles, e.g. {sorted(roles)[:4]}" + check("every accessible carries the role AT-SPI reports", _roles_are_named) + + def _extents_are_screen_pixels() -> str: + # GetExtents takes a coordinate space, and the wrong one returns + # window-relative numbers that look plausible and click the wrong + # place. A window on this screen has to have a non-zero size. + sized = [element for element in elements + if element.bounds[2] > 0 and element.bounds[3] > 0] + _assert_true(bool(sized), "no accessible reported a rectangle") + widest = max(sized, key=lambda element: element.bounds[2]) + return (f"{len(sized)} with a rectangle; widest {widest.role!r} " + f"{widest.bounds}") + check("Component.GetExtents returns real screen rectangles", + _extents_are_screen_pixels) + + def _application_is_named() -> str: + # The walk is per-application, so every element has to carry the name + # of the application it came from — that is what app_name filters on. + owners = {element.app_name for element in elements if element.app_name} + _assert_true(bool(owners), "no element carried an application name") + return f"applications on the bus: {sorted(owners)}" + check("elements carry the application they belong to", _application_is_named) + + def _scoped_walk_is_narrower() -> str: + scoped = backend.list_elements(max_results=400, + window_title=DIALOG_TITLE) + _assert_true(len(scoped) <= len(elements), + f"scoping widened the walk: {len(scoped)} > {len(elements)}") + return f"{len(scoped)} scoped vs {len(elements)} unscoped" + check("scoping to a window narrows the walk", _scoped_walk_is_narrower) + + +def check_state(backend) -> None: + def _state() -> str: + # The state bitfield arrives as two 32-bit words; reading only the + # first silently drops every state above bit 31. + state = backend.get_state(role="push button") + if state is None: + state = backend.get_state(role="label") + _assert_true(state is not None, + "get_state found neither a button nor a label") + _assert_true("enabled" in state, + f"no enabled flag in {sorted(state)}") + return f"{sorted(state)}" + check("get_state reads the AT-SPI state bitfield", _state) + + +def main() -> int: + print("=" * 72) + print("AutoControl accessibility — real AT-SPI bus, real GTK application") + print("=" * 72) + note(f"DBUS_SESSION_BUS_ADDRESS set: " + f"{bool(os.environ.get('DBUS_SESSION_BUS_ADDRESS'))}") + + check_backend_selection() + + from je_auto_control.utils.accessibility.backends import get_backend + + backend = get_backend() + if not backend.available: + print("-" * 72) + print("The accessibility bus is not up, so nothing below can run.") + print("That is a failure of this container, not a reason to skip:") + print("every other check here depends on it.") + return summarise() or 1 + + with Zenity() as zenity: + check_tree(backend, zenity) + check_state(backend) + + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("Anything needing a screen reader's own state — at-spi2 exposes") + note(" ScreenReaderEnabled, and nothing here is a screen reader.") + + return summarise() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index 78a44aa0..da87997d 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -9,7 +9,7 @@ without a compatibility window. | Mouse, keyboard, screenshot | stable | CI | CI/Xvfb + xev | CI/sway + libeis | implementation | | JSON executor and variables | stable | CI | CI | CI | platform-neutral | | Image and anchor locators | beta | CI | CI | implementation | implementation | -| Accessibility locator | beta | CI | backend tests | unavailable | backend tests | +| Accessibility locator | beta | CI | CI/AT-SPI | CI/AT-SPI | CI (tree read) | | Window management | beta | CI | CI/openbox | unavailable | CI (listing) | | Recorder | beta | CI | implementation | unavailable | unavailable | | Reports, trace, failure bundle | stable | CI | CI | CI | platform-neutral | @@ -65,6 +65,36 @@ primary one, where the compositor's plane starts at a negative coordinate and a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of that monitor. +The accessibility row said `backend tests` for Linux X11 and meant nothing by +it: there was no Linux backend at all, and `_build_backend()` fell straight +through to the null one. There is one now, over **AT-SPI2** — which is a D-Bus +protocol rather than a library, and that is what makes it reachable without a +new dependency. The usual bindings (`pyatspi`, `gi.repository.Atspi`) are +distribution packages built against the system introspection data and cannot +be installed into a virtual environment, so depending on them would be +depending on something most users cannot get. The client written for the XDG +portal handshake already spoke enough D-Bus. + +It is exercised by the `x11-verification` job against a real accessibility bus +and a real GTK application (`zenity`), because neither half can be mocked +usefully: the bus is D-Bus-activated rather than started by hand, an +application only appears on it if its toolkit bridge loaded, and the tree's +shape is the toolkit's business. + +That job immediately found a gap in the shared D-Bus client: **it could not +demarshal signed integers.** The portal handshake never needed one, and +AT-SPI reports a component's extents as four *signed* values — because a +window on a monitor left of or above the primary one is at a negative +coordinate. Without it the backend could read a tree but not where anything +was. The client now handles the whole fixed-width numeric set except `h` +(UNIX_FD), which stays an error on purpose: it is an index into a descriptor +array this client does not receive, so returning it would hand a caller a +number that addresses nothing. + +Because AT-SPI is a bus rather than a display protocol, this row is `CI/AT-SPI` +for **both** Linux entries: a Wayland session runs the same accessibility bus, +so this is the one capability where Wayland is not the restricted case. + Window management had no row here at all until it had more than one platform. It was Windows-only for the project's whole life — the facade branched on `sys.platform` and raised everywhere else — which left 23 `AC_*` commands and diff --git a/je_auto_control/linux_wayland/_dbus_client.py b/je_auto_control/linux_wayland/_dbus_client.py index 6a9d91a5..000f10cc 100644 --- a/je_auto_control/linux_wayland/_dbus_client.py +++ b/je_auto_control/linux_wayland/_dbus_client.py @@ -1,625 +1,24 @@ -"""A minimal D-Bus session-bus client, in the standard library alone. +"""The D-Bus client this package uses, re-exported from its new home. -This exists because of one property of the XDG portal protocol: a portal call -returns only a *request handle*, and the answer arrives later as a ``Response`` -signal — **directed at the unique bus name that made the call**. The bus routes -a directed message to its destination and nowhere else, so a listener on a -second connection never receives it, whatever match rules it adds. +The implementation moved to :mod:`je_auto_control.utils.dbus_client` when the +Linux accessibility backend became its second caller: ``utils/`` sits above +the per-OS packages in this project's layering, so an accessibility backend +reaching down into ``linux_wayland/`` to borrow its D-Bus code would invert +that. -That rules out the obvious shell-out. ``gdbus call`` and ``gdbus monitor`` are -two processes with two unique names, so the monitor is not the caller and the -Response is not addressed to it; measured against a real ``dbus-daemon``, it -sees the call go past and the answer never arrive. The only listener that -does see it is a full bus monitor (``dbus-monitor``, which asks the bus for -``BecomeMonitor``), and making a screenshot require permission to observe every -message on the user's session bus is a poor trade for a fallback tier. - -So the subscription and the call happen on one connection here, which is what -every portal client library does and what the protocol is designed for. The -cost is marshalling D-Bus by hand; the scope is kept to exactly what a portal -conversation needs — connect, authenticate, ``Hello``, ``AddMatch``, one method -call, and read signals until the matching one arrives. It is deliberately not a -general D-Bus binding: no properties, no introspection, no object export, no -file-descriptor passing (:mod:`je_auto_control.linux_wayland.oeffis` uses -liboeffis for the one call that needs that). - -Every failure is an :class:`AutoControlException` subclass, so the containment -boundaries that catch the family keep working. +This module stays because the portal code and its tests import it by this +name, and moving a working protocol implementation is not a reason to churn +either. """ -from __future__ import annotations - -import contextlib -import os -import socket -import struct -import time -from typing import Any, Dict, List, Optional, Tuple - -from je_auto_control.utils.exception.exceptions import AutoControlException - - -#: Message types, from the D-Bus specification. -METHOD_CALL = 1 -METHOD_RETURN = 2 -ERROR = 3 -SIGNAL = 4 - -#: Header field codes. -FIELD_PATH = 1 -FIELD_INTERFACE = 2 -FIELD_MEMBER = 3 -FIELD_ERROR_NAME = 4 -FIELD_REPLY_SERIAL = 5 -FIELD_DESTINATION = 6 -FIELD_SENDER = 7 -FIELD_SIGNATURE = 8 - -#: ``NO_REPLY_EXPECTED``. Declared for completeness; nothing here uses it, -#: because every call this makes is one whose answer is worth waiting for. -FLAG_NO_REPLY_EXPECTED = 1 - -BUS_NAME = "org.freedesktop.DBus" -BUS_PATH = "/org/freedesktop/DBus" -BUS_INTERFACE = "org.freedesktop.DBus" - -_ADDRESS_ENV = "DBUS_SESSION_BUS_ADDRESS" -_MAX_MESSAGE = 128 * 1024 * 1024 # the specification's own ceiling -_ALIGNMENT = {"y": 1, "b": 4, "n": 2, "q": 2, "i": 4, "u": 4, - "x": 8, "t": 8, "d": 8, "s": 4, "o": 4, "g": 1, - "a": 4, "(": 8, "{": 8, "v": 1, "h": 4} - - -class DBusError(AutoControlException): - """The session bus is unreachable, or answered with an error.""" - - -class Variant: - """A value with an explicit D-Bus type, for the ``a{sv}`` option maps.""" - - __slots__ = ("signature", "value") - - def __init__(self, signature: str, value: Any) -> None: - self.signature = signature - self.value = value - - def __repr__(self) -> str: - return f"Variant({self.signature!r}, {self.value!r})" - - -# --- marshalling ---------------------------------------------------------- - - -class _Writer: - """Little-endian marshaller. Alignment is relative to the message start.""" - - def __init__(self, offset: int = 0) -> None: - self._parts: List[bytes] = [] - self._length = offset - - def align(self, boundary: int) -> None: - padding = (-self._length) % boundary - if padding: - self._parts.append(bytes(padding)) - self._length += padding - - def raw(self, data: bytes) -> None: - self._parts.append(data) - self._length += len(data) - - def byte(self, value: int) -> None: - self.raw(struct.pack(" None: - self.align(4) - self.raw(struct.pack(" None: - encoded = value.encode("utf-8") - self.uint32(len(encoded)) - self.raw(encoded + b"\x00") - - def signature(self, value: str) -> None: - encoded = value.encode("ascii") - self.byte(len(encoded)) - self.raw(encoded + b"\x00") - - def value(self, sig: str, value: Any) -> None: - """Write one complete value of the given single signature.""" - _write_value(self, _SignatureReader(sig), value) - - @property - def data(self) -> bytes: - return b"".join(self._parts) - - def __len__(self) -> int: - return self._length - - -class _SignatureReader: - """Walks a signature string one complete type at a time.""" - - def __init__(self, text: str) -> None: - self.text = text - self.index = 0 - - def done(self) -> bool: - return self.index >= len(self.text) - - def peek(self) -> str: - return self.text[self.index] - - def take(self) -> str: - code = self.text[self.index] - self.index += 1 - return code - - def take_complete(self) -> str: - """Take one whole type, following containers to their close.""" - start = self.index - code = self.take() - if code == "a": - self.take_complete() - elif code in "({": - closing = ")" if code == "(" else "}" - while self.peek() != closing: - self.take_complete() - self.take() - return self.text[start:self.index] - - -def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: - code = reader.take() - if code == "y": - writer.byte(int(value)) - elif code == "b": - writer.uint32(1 if value else 0) - elif code == "u": - writer.uint32(int(value)) - elif code in "so": - writer.string(str(value)) - elif code == "g": - writer.signature(str(value)) - elif code == "v": - _write_variant(writer, value) - elif code == "a": - _write_array(writer, reader, value) - elif code in "({": - _write_struct(writer, reader, code, value) - else: - raise DBusError(f"cannot marshal D-Bus type {code!r}") - - -def _write_variant(writer: _Writer, value: Any) -> None: - variant = value if isinstance(value, Variant) else _guess_variant(value) - writer.signature(variant.signature) - writer.value(variant.signature, variant.value) - - -def _guess_variant(value: Any) -> Variant: - if isinstance(value, bool): - return Variant("b", value) - if isinstance(value, int): - return Variant("u", value) - if isinstance(value, str): - return Variant("s", value) - raise DBusError(f"no obvious D-Bus type for {type(value).__name__}") - - -def _write_array(writer: _Writer, reader: _SignatureReader, value: Any) -> None: - element = reader.take_complete() - writer.align(4) - # The length prefix counts the elements only, and it is written before - # the padding that aligns the first one — so the body is built separately - # at the offset it will actually occupy. - body_start = len(writer) + 4 - padding = (-body_start) % _ALIGNMENT.get(element[0], 1) - body = _Writer(body_start + padding) - items = value.items() if isinstance(value, dict) else value - for item in items: - body.align(_ALIGNMENT.get(element[0], 1)) - _write_value(body, _SignatureReader(element), item) - writer.uint32(len(body) - body_start - padding) - writer.raw(bytes(padding)) - writer.raw(body.data) - - -def _write_struct(writer: _Writer, reader: _SignatureReader, code: str, - value: Any) -> None: - closing = ")" if code == "(" else "}" - writer.align(8) - for item in value: - if reader.peek() == closing: - raise DBusError("too many members for this struct signature") - _write_value(writer, reader, item) - if reader.peek() != closing: - raise DBusError("too few members for this struct signature") - reader.take() - - -class _Reader: - """Little-endian demarshaller over one complete message.""" - - def __init__(self, data: bytes, offset: int = 0) -> None: - self.data = data - self.offset = offset - - def align(self, boundary: int) -> None: - self.offset += (-self.offset) % boundary - - def take(self, count: int) -> bytes: - if self.offset + count > len(self.data): - raise DBusError("truncated D-Bus message") - chunk = self.data[self.offset:self.offset + count] - self.offset += count - return chunk - - def byte(self) -> int: - return self.take(1)[0] - - def uint32(self) -> int: - self.align(4) - return struct.unpack(" str: - length = self.uint32() - text = self.take(length).decode("utf-8", errors="replace") - self.take(1) - return text - - def signature(self) -> str: - length = self.byte() - text = self.take(length).decode("ascii", errors="replace") - self.take(1) - return text - - def value(self, sig: str) -> Any: - return _read_value(self, _SignatureReader(sig)) - - -def _read_value(reader: _Reader, signature: _SignatureReader) -> Any: - code = signature.take() - if code == "y": - return reader.byte() - if code == "b": - return bool(reader.uint32()) - if code == "u": - return reader.uint32() - if code in "so": - return reader.string() - if code == "g": - return reader.signature() - if code == "v": - return reader.value(reader.signature()) - if code == "a": - return _read_array(reader, signature) - if code in "({": - return _read_struct(reader, signature, code) - raise DBusError(f"cannot demarshal D-Bus type {code!r}") - - -def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: - element = signature.take_complete() - length = reader.uint32() - reader.align(_ALIGNMENT.get(element[0], 1)) - end = reader.offset + length - items = [] - while reader.offset < end: - reader.align(_ALIGNMENT.get(element[0], 1)) - items.append(_read_value(reader, _SignatureReader(element))) - if element.startswith("{"): - return dict(items) - return items - - -def _read_struct(reader: _Reader, signature: _SignatureReader, - code: str) -> tuple: - closing = ")" if code == "(" else "}" - reader.align(8) - members = [] - while signature.peek() != closing: - members.append(_read_value(reader, signature)) - signature.take() - return tuple(members) - - -class Message: - """One decoded D-Bus message: the header fields that matter, and the body.""" - - __slots__ = ("type", "serial", "fields", "body") - - def __init__(self, message_type: int, serial: int, - fields: Dict[int, Any], body: List[Any]) -> None: - self.type = message_type - self.serial = serial - self.fields = fields - self.body = body - - @property - def path(self) -> str: - return self.fields.get(FIELD_PATH, "") - - @property - def interface(self) -> str: - return self.fields.get(FIELD_INTERFACE, "") - - @property - def member(self) -> str: - return self.fields.get(FIELD_MEMBER, "") - - @property - def reply_serial(self) -> int: - return self.fields.get(FIELD_REPLY_SERIAL, 0) - - @property - def error_name(self) -> str: - return self.fields.get(FIELD_ERROR_NAME, "") - - -# --- the connection ------------------------------------------------------- - - -def session_address() -> Optional[str]: - """The session bus address, or None when this process has no session bus.""" - address = os.environ.get(_ADDRESS_ENV, "").strip() - return address or None - - -def is_available() -> bool: - """Whether a session bus address is set, so connecting is worth trying.""" - return session_address() is not None - - -def _socket_target(address: str) -> Tuple[str, bool]: - """Pick a connectable ``unix:`` transport out of a bus address. - - :return: the socket path and whether it is in the abstract namespace. - """ - for candidate in address.split(";"): - if not candidate.startswith("unix:"): - continue - fields = [part.split("=", 1) - for part in candidate[len("unix:"):].split(",") - if "=" in part] - options = dict(fields) - if "path" in options: - return options["path"], False - if "abstract" in options: - return options["abstract"], True - raise DBusError(f"no usable unix transport in {address!r}") - - -class SessionBus: - """An authenticated connection to the session bus, as a context manager.""" - - def __init__(self, address: Optional[str] = None) -> None: - self.address = address or session_address() - self.unique_name = "" - self._socket: Optional[socket.socket] = None - self._serial = 0 - self._buffer = b"" - #: Messages read while waiting for a method reply. A portal's Response - #: signal and its method return come from two different senders, so - #: the bus gives no ordering between them and the signal can arrive - #: first — dropping it here would be a wait that never ends. - self._queued: List[Message] = [] - - # --- lifecycle -------------------------------------------------------- - - def __enter__(self) -> "SessionBus": - self.connect() - return self - - def __exit__(self, *_exception: Any) -> None: - self.close() - - def connect(self) -> None: - """Open the socket, authenticate, and say ``Hello``.""" - if self.address is None: - raise DBusError( - f"{_ADDRESS_ENV} is not set, so there is no session bus to " - f"reach the desktop portal on", - ) - path, abstract = _socket_target(self.address) - try: - self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._socket.connect(("\0" + path) if abstract else path) - except OSError as error: - self.close() - raise DBusError(f"cannot reach the session bus: {error}") from error - self._authenticate() - self.unique_name = self.call( - BUS_NAME, BUS_PATH, BUS_INTERFACE, "Hello", "", [])[0] - - def close(self) -> None: - """Drop the connection; never raise from teardown.""" - if self._socket is not None: - with contextlib.suppress(OSError): - self._socket.close() - self._socket = None - - @property - def sender_token(self) -> str: - """This connection's name as the portal spells it inside object paths.""" - return self.unique_name.lstrip(":").replace(".", "_") - - # --- authentication --------------------------------------------------- - - def _authenticate(self) -> None: - """The SASL EXTERNAL handshake, which is a uid in hex over a socket.""" - uid = str(os.getuid()).encode("ascii") - self._send_raw(b"\x00AUTH EXTERNAL " + uid.hex().encode("ascii") - + b"\r\n") - reply = self._read_line() - if not reply.startswith("OK"): - raise DBusError(f"the session bus refused authentication: {reply}") - self._send_raw(b"BEGIN\r\n") - - def _send_raw(self, data: bytes) -> None: - if self._socket is None: - raise DBusError("the session bus connection is closed") - try: - self._socket.sendall(data) - except OSError as error: - raise DBusError(f"writing to the session bus failed: {error}") from error - - def _read_line(self, timeout: float = 10.0) -> str: - """Read one CRLF-terminated line of the auth conversation.""" - deadline = time.monotonic() + timeout - while b"\r\n" not in self._buffer: - self._fill(deadline - time.monotonic()) - line, _, self._buffer = self._buffer.partition(b"\r\n") - return line.decode("utf-8", errors="replace") - - # --- reading ---------------------------------------------------------- - - def _fill(self, timeout: float) -> None: - """Read whatever is available, or fail once the deadline has passed.""" - if self._socket is None: - raise DBusError("the session bus connection is closed") - if timeout <= 0: - raise DBusError("the session bus did not answer in time") - self._socket.settimeout(timeout) - try: - chunk = self._socket.recv(65536) - except socket.timeout as error: - raise DBusError("the session bus did not answer in time") from error - except OSError as error: - raise DBusError(f"reading from the session bus failed: {error}") from error - if not chunk: - raise DBusError("the session bus closed the connection") - self._buffer += chunk - - def read_message(self, deadline: float) -> Message: - """Read one whole message, or fail when the deadline passes.""" - while len(self._buffer) < 16: - self._fill(deadline - time.monotonic()) - endian, message_type, _flags, _version = struct.unpack( - " _MAX_MESSAGE: - raise DBusError("the session bus sent an implausibly large message") - while len(self._buffer) < total: - self._fill(deadline - time.monotonic()) - raw, self._buffer = self._buffer[:total], self._buffer[total:] - return _decode(raw, message_type, serial, fields_length, header_end) - - # --- writing ---------------------------------------------------------- - - def _next_serial(self) -> int: - self._serial += 1 - return self._serial - - def send(self, message_type: int, fields: Dict[int, Tuple[str, Any]], - signature: str, body: List[Any], flags: int = 0) -> int: - """Marshal and send one message; return its serial.""" - payload = _Writer() - for code, value in body_pairs(signature, body): - payload.value(code, value) - serial = self._next_serial() - header = _Writer() - header.raw(struct.pack(" List[Any]: - """Make a method call and return the reply body.""" - fields = { - FIELD_PATH: ("o", path), - FIELD_INTERFACE: ("s", interface), - FIELD_MEMBER: ("s", member), - FIELD_DESTINATION: ("s", destination), - } - if signature: - fields[FIELD_SIGNATURE] = ("g", signature) - serial = self.send(METHOD_CALL, fields, signature, body) - deadline = time.monotonic() + timeout - while True: - message = self.read_message(deadline) - if message.reply_serial != serial: - self._queued.append(message) - continue - if message.type == ERROR: - detail = message.body[0] if message.body else "" - raise DBusError(f"{message.error_name}: {detail}".strip(": ")) - if message.type == METHOD_RETURN: - return message.body - - def add_match(self, rule: str) -> None: - """Subscribe to signals, and wait for the bus to confirm the rule. - - Waiting matters twice over: a malformed rule is reported rather than - silently never matching, and the round trip proves the subscription is - in place before the call that provokes the signal is made. - """ - self.call(BUS_NAME, BUS_PATH, BUS_INTERFACE, "AddMatch", "s", [rule]) - - def wait_for_signal(self, paths: List[str], interface: str, member: str, - timeout: float) -> List[Any]: - """Read until the signal we subscribed to arrives, or time out.""" - def matches(message: Message) -> bool: - return (message.type == SIGNAL and message.member == member - and message.interface == interface - and message.path in paths) - - for index, message in enumerate(self._queued): - if matches(message): - del self._queued[index] - return message.body - self._queued.clear() - deadline = time.monotonic() + max(0.0, timeout) - while True: - message = self.read_message(deadline) - if matches(message): - return message.body - - -def body_pairs(signature: str, body: List[Any]): - """Pair each top-level type in a signature with its argument.""" - reader = _SignatureReader(signature) - for value in body: - if reader.done(): - raise DBusError("more arguments than the signature declares") - yield reader.take_complete(), value - if not reader.done(): - raise DBusError("fewer arguments than the signature declares") - - -def _decode(raw: bytes, message_type: int, serial: int, fields_length: int, - header_end: int) -> Message: - """Turn one complete message's bytes into a :class:`Message`.""" - reader = _Reader(raw, 12) - fields: Dict[int, Any] = {} - reader.uint32() # the header array's own length, already read - end = 16 + fields_length - while reader.offset < end: - reader.align(8) - code = reader.byte() - fields[code] = reader.value(reader.signature()) - reader.offset = header_end + ((-header_end) % 8) - body: List[Any] = [] - signature = fields.get(FIELD_SIGNATURE, "") - if signature: - walker = _SignatureReader(signature) - while not walker.done(): - body.append(_read_value(reader, _SignatureReader( - walker.take_complete()))) - return Message(message_type, serial, fields, body) - +from je_auto_control.utils.dbus_client.session_bus import ( # noqa: F401 # reason: re-export + BUS_INTERFACE, BUS_NAME, BUS_PATH, DBusError, ERROR, FIELD_ERROR_NAME, + FIELD_INTERFACE, FIELD_MEMBER, FIELD_PATH, FIELD_REPLY_SERIAL, + FIELD_SENDER, FIELD_SIGNATURE, METHOD_CALL, METHOD_RETURN, Message, + SIGNAL, SessionBus, Variant, body_pairs, is_available, session_address, +) __all__ = [ "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", - "Variant", "is_available", "session_address", + "Variant", "body_pairs", "is_available", "session_address", ] diff --git a/je_auto_control/utils/accessibility/backends/__init__.py b/je_auto_control/utils/accessibility/backends/__init__.py index 4f39ffea..ed1c3553 100644 --- a/je_auto_control/utils/accessibility/backends/__init__.py +++ b/je_auto_control/utils/accessibility/backends/__init__.py @@ -48,6 +48,20 @@ def _build_backend() -> AccessibilityBackend: return NullAccessibilityBackend( "pyobjc (ApplicationServices, AppKit) is required on macOS", ) + if sys.platform.startswith("linux"): + from je_auto_control.utils.accessibility.backends.linux_backend import ( + LinuxAccessibilityBackend, + ) + backend = LinuxAccessibilityBackend() + if backend.available: + return backend + # AT-SPI is a bus, not a library, so "not installed" and "installed + # but nothing is bridged to it" look the same from here. Name both. + return NullAccessibilityBackend( + "no AT-SPI accessibility bus. Install and start at-spi2-core, and " + "enable the toolkit bridge (GTK_MODULES=gail:atk-bridge, " + "QT_ACCESSIBILITY=1)", + ) return NullAccessibilityBackend( f"no accessibility backend for platform {sys.platform!r}", ) diff --git a/je_auto_control/utils/accessibility/backends/linux_backend.py b/je_auto_control/utils/accessibility/backends/linux_backend.py new file mode 100644 index 00000000..58cb1412 --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/linux_backend.py @@ -0,0 +1,414 @@ +"""Linux accessibility backend, over AT-SPI2. + +Linux had no accessibility backend at all: :func:`_build_backend` fell through +to the null one, while ``docs/CAPABILITY_MATRIX.md`` claimed "backend tests" +for Linux X11. This is the backend that makes the claim true. + +AT-SPI2 is a D-Bus protocol, not a library, which is what makes it reachable +here without adding a dependency. The usual bindings (``pyatspi``, +``gi.repository.Atspi``) are distribution packages built against the system +GObject introspection data: they cannot be installed into a virtual +environment, so a project that depends on them is a project most users cannot +run. :mod:`je_auto_control.utils.dbus_client`, written for the XDG portal +handshake, already speaks enough D-Bus for this. + +The shape of the protocol: + +* The accessibility bus is *not* the session bus. Its address comes from + ``org.a11y.Bus.GetAddress`` on the session bus, and everything else happens + on a second connection to that address. +* An accessible object is addressed by a **pair** — the bus name of the + application that owns it and an object path inside it — so references are + ``(sender, path)`` tuples throughout rather than single strings. +* The root's children are applications; theirs are windows; theirs are the + controls. Walking is therefore per-application, which is also what makes + ``app_name`` cheap to honour. + +Every read is defensive. An accessible can disappear between the call that +lists it and the call that describes it — the application closed a dialog, or +exited — and a walk that raised on that would fail more often than it +succeeded. +""" +import os +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.accessibility.backends.base import AccessibilityBackend +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, element_matches, +) +from je_auto_control.utils.dbus_client import DBusError, SessionBus, Variant +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +#: Where the session bus publishes the accessibility bus's address. +_BUS_NAME = "org.a11y.Bus" +_BUS_PATH = "/org/a11y/bus" + +#: The registry that owns the tree's root. +_REGISTRY = "org.a11y.atspi.Registry" +_ROOT_PATH = "/org/a11y/atspi/accessible/root" + +#: AT-SPI2 interfaces, as they appear on the wire. +_ACCESSIBLE = "org.a11y.atspi.Accessible" +_ACTION = "org.a11y.atspi.Action" +_COMPONENT = "org.a11y.atspi.Component" +_EDITABLE = "org.a11y.atspi.EditableText" +_TEXT = "org.a11y.atspi.Text" +_VALUE = "org.a11y.atspi.Value" +_PROPERTIES = "org.freedesktop.DBus.Properties" + +#: ``GetExtents`` coordinate spaces. 0 is the screen, which is the only one +#: whose numbers mean anything to a caller about to click them. +_COORDS_SCREEN = 0 + +#: Bit positions in the ``GetState`` bitfield that this backend reads. +_STATE_ENABLED = 8 +_STATE_FOCUSED = 12 +_STATE_SELECTED = 25 + +#: A reference to one accessible: the owning application's bus name and the +#: object path inside it. +Reference = Tuple[str, str] + + +def _is_available() -> bool: + """Whether an accessibility bus can be reached at all.""" + if os.name != "posix": + return False + try: + with _AtspiConnection() as connection: + connection.children(connection.root) + return True + except (DBusError, OSError) as error: + autocontrol_logger.info("AT-SPI unavailable: %r", error) + return False + + +class _AtspiConnection: + """One connection to the accessibility bus, for the length of one call.""" + + def __init__(self) -> None: + self._bus: Optional[SessionBus] = None + + def __enter__(self) -> "_AtspiConnection": + self._bus = SessionBus(address=self._address()) + self._bus.connect() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._bus is not None: + self._bus.close() + self._bus = None + + @staticmethod + def _address() -> str: + """Ask the session bus where the accessibility bus is.""" + with SessionBus() as session: + reply = session.call(_BUS_NAME, _BUS_PATH, _BUS_NAME, + "GetAddress", "", []) + if not reply or not isinstance(reply[0], str): + raise DBusError("org.a11y.Bus.GetAddress returned no address") + return reply[0] + + @property + def root(self) -> Reference: + return (_REGISTRY, _ROOT_PATH) + + def _call(self, reference: Reference, interface: str, member: str, + signature: str = "", body: Optional[List[Any]] = None, + timeout: float = 10.0) -> List[Any]: + sender, path = reference + return self._bus.call(sender, path, interface, member, + signature, body or [], timeout=timeout) + + # --- reads ------------------------------------------------------------- + + def children(self, reference: Reference) -> List[Reference]: + """The accessible's children, as ``(sender, path)`` pairs.""" + reply = self._call(reference, _ACCESSIBLE, "GetChildren") + if not reply: + return [] + return [(str(item[0]), str(item[1])) for item in reply[0] + if isinstance(item, (list, tuple)) and len(item) >= 2] + + def property(self, reference: Reference, name: str, + interface: str = _ACCESSIBLE) -> Any: + """One D-Bus property, unwrapped from its variant.""" + reply = self._call(reference, _PROPERTIES, "Get", "ss", + [interface, name]) + value = reply[0] if reply else None + return value.value if isinstance(value, Variant) else value + + def role_name(self, reference: Reference) -> str: + reply = self._call(reference, _ACCESSIBLE, "GetRoleName") + return str(reply[0]) if reply else "" + + def state(self, reference: Reference) -> int: + """The state bitfield, as one integer. + + AT-SPI sends it as two 32-bit words rather than one 64-bit value, so + reading only the first would silently drop every state above bit 31. + """ + reply = self._call(reference, _ACCESSIBLE, "GetState") + words = list(reply[0]) if reply and reply[0] else [] + bits = 0 + for index, word in enumerate(words[:2]): + bits |= int(word) << (32 * index) + return bits + + def extents(self, reference: Reference) -> Tuple[int, int, int, int]: + """``(left, top, width, height)`` in screen pixels, or zeroes.""" + try: + reply = self._call(reference, _COMPONENT, "GetExtents", "u", + [_COORDS_SCREEN]) + except DBusError: + # Not every accessible implements Component — a plain text node + # has no rectangle, and that is not an error. + return (0, 0, 0, 0) + if not reply or len(reply[0]) < 4: + return (0, 0, 0, 0) + return tuple(int(value) for value in reply[0][:4]) # type: ignore[return-value] + + # --- writes ------------------------------------------------------------ + + def do_action(self, reference: Reference, index: int = 0) -> bool: + reply = self._call(reference, _ACTION, "DoAction", "i", [int(index)]) + return bool(reply and reply[0]) + + def set_text(self, reference: Reference, value: str) -> bool: + reply = self._call(reference, _EDITABLE, "SetTextContents", "s", + [str(value)]) + return bool(reply and reply[0]) + + def text(self, reference: Reference) -> Optional[str]: + try: + reply = self._call(reference, _TEXT, "GetText", "ii", [0, -1]) + except DBusError: + return None + return str(reply[0]) if reply else None + + def number(self, reference: Reference) -> Optional[float]: + try: + return float(self.property(reference, "CurrentValue", _VALUE)) + except (DBusError, TypeError, ValueError): + return None + + def grab_focus(self, reference: Reference) -> bool: + reply = self._call(reference, _COMPONENT, "GrabFocus") + return bool(reply and reply[0]) + + +class LinuxAccessibilityBackend(AccessibilityBackend): + """The AT-SPI2 tree, walked over D-Bus.""" + + name = "linux-atspi" + + def __init__(self) -> None: + self.available = _is_available() + + def _require(self) -> None: + if not self.available: + raise AccessibilityNotAvailableError( + "no AT-SPI accessibility bus. Install and start at-spi2-core, " + "and make sure the toolkit's accessibility bridge is enabled " + "(GTK_MODULES=gail:atk-bridge, QT_ACCESSIBILITY=1).", + ) + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + window_title: Optional[str] = None, + ) -> List[AccessibilityElement]: + self._require() + results: List[AccessibilityElement] = [] + with _AtspiConnection() as connection: + for application in connection.children(connection.root): + if len(results) >= max_results: + break + name = _safe_name(connection, application) + if app_name is not None and name != app_name: + continue + self._walk(connection, application, name, results, + max_results, window_title) + return results[:max_results] + + def _walk(self, connection: _AtspiConnection, reference: Reference, + app_name: str, results: List[AccessibilityElement], + max_results: int, window_title: Optional[str], + depth: int = 0) -> None: + """Depth-first, and bounded: a broken tree must not hang the caller.""" + if len(results) >= max_results or depth > 32: + return + try: + children = connection.children(reference) + except DBusError: + return + for child in children: + if len(results) >= max_results: + return + converted = _convert(connection, child, app_name) + if converted is None: + continue + # Scoping to a window is not just filtering: below a window the + # tree is orders of magnitude smaller, so this both narrows the + # answer and shortens the walk. + if depth == 0 and window_title is not None: + if window_title.lower() not in converted.name.lower(): + continue + results.append(converted) + self._walk(connection, child, app_name, results, max_results, + window_title, depth + 1) + + # --- control patterns -------------------------------------------------- + + def _find(self, connection: _AtspiConnection, name: Optional[str], + role: Optional[str], app_name: Optional[str], + window_title: Optional[str] = None, + contains: bool = False) -> Optional[Reference]: + """The reference behind the first element that matches.""" + for application in connection.children(connection.root): + owner = _safe_name(connection, application) + if app_name is not None and owner != app_name: + continue + found = self._search(connection, application, owner, name, role, + contains) + if found is not None: + return found + del window_title # accepted for signature parity with the base class + return None + + def _search(self, connection: _AtspiConnection, reference: Reference, + app_name: str, name: Optional[str], role: Optional[str], + contains: bool, depth: int = 0) -> Optional[Reference]: + if depth > 32: + return None + try: + children = connection.children(reference) + except DBusError: + return None + for child in children: + converted = _convert(connection, child, app_name) + if converted is not None and element_matches( + converted, name, role, app_name, contains): + return child + deeper = self._search(connection, child, app_name, name, role, + contains, depth + 1) + if deeper is not None: + return deeper + return None + + def get_value(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[str]: + self._require() + del automation_id # AT-SPI has no equivalent of a UIA automation id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name, + window_title, contains) + if reference is None: + return None + text = connection.text(reference) + if text is not None: + return text + number = connection.number(reference) + return None if number is None else str(number) + + def set_value(self, value: str, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name) + if reference is None: + return False + try: + return connection.set_text(reference, value) + except DBusError as error: + autocontrol_logger.info("set_value failed: %r", error) + return False + + def invoke(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name) + if reference is None: + return False + try: + return connection.do_action(reference, 0) + except DBusError as error: + autocontrol_logger.info("invoke failed: %r", error) + return False + + def set_focus(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name) + if reference is None: + return False + try: + return connection.grab_focus(reference) + except DBusError as error: + autocontrol_logger.info("set_focus failed: %r", error) + return False + + def get_state(self, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[Dict[str, Any]]: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name, + window_title, contains) + if reference is None: + return None + bits = connection.state(reference) + state: Dict[str, Any] = { + "enabled": bool(bits & (1 << _STATE_ENABLED)), + "focused": bool(bits & (1 << _STATE_FOCUSED)), + "selected": bool(bits & (1 << _STATE_SELECTED)), + } + # A key is absent when the control does not have the concept, + # which is a different answer from the value being empty. + text = connection.text(reference) + if text is not None: + state["value"] = text + number = connection.number(reference) + if number is not None: + state["number"] = number + return state + + +def _safe_name(connection: _AtspiConnection, reference: Reference) -> str: + try: + return str(connection.property(reference, "Name") or "") + except DBusError: + return "" + + +def _convert(connection: _AtspiConnection, reference: Reference, + app_name: str) -> Optional[AccessibilityElement]: + """One accessible as an :class:`AccessibilityElement`, or None.""" + try: + name = _safe_name(connection, reference) + role = connection.role_name(reference) + bounds = connection.extents(reference) + bits = connection.state(reference) + except DBusError: + # The accessible went away between being listed and being described. + return None + if not name and not role: + return None + return AccessibilityElement( + name=name, role=role, bounds=bounds, app_name=app_name, + native_id=reference[1], + enabled=bool(bits & (1 << _STATE_ENABLED)), + ) diff --git a/je_auto_control/utils/dbus_client/__init__.py b/je_auto_control/utils/dbus_client/__init__.py new file mode 100644 index 00000000..6dd33b6e --- /dev/null +++ b/je_auto_control/utils/dbus_client/__init__.py @@ -0,0 +1,24 @@ +"""A minimal D-Bus session-bus client, in the standard library alone. + +Written for the XDG portal handshake — see :mod:`.session_bus` for why that +could not be a shell-out to ``gdbus`` — and kept general enough for anything +else that has to speak D-Bus without pulling in a binding. AT-SPI2, the Linux +accessibility bus, is the second caller. + +It lives here rather than under ``linux_wayland/`` because ``utils/`` sits +above the per-OS packages in this project's layering: an accessibility backend +in ``utils/`` reaching down into a platform backend to borrow its D-Bus code +would invert that. ``je_auto_control.linux_wayland._dbus_client`` re-exports +this module, so the Wayland code and its tests are unchanged. +""" +from je_auto_control.utils.dbus_client.session_bus import ( + BUS_INTERFACE, BUS_NAME, BUS_PATH, DBusError, ERROR, METHOD_CALL, + METHOD_RETURN, Message, SIGNAL, SessionBus, Variant, body_pairs, + is_available, session_address, +) + +__all__ = [ + "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", + "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", + "Variant", "body_pairs", "is_available", "session_address", +] diff --git a/je_auto_control/utils/dbus_client/session_bus.py b/je_auto_control/utils/dbus_client/session_bus.py new file mode 100644 index 00000000..d5777292 --- /dev/null +++ b/je_auto_control/utils/dbus_client/session_bus.py @@ -0,0 +1,656 @@ +"""A minimal D-Bus session-bus client, in the standard library alone. + +This exists because of one property of the XDG portal protocol: a portal call +returns only a *request handle*, and the answer arrives later as a ``Response`` +signal — **directed at the unique bus name that made the call**. The bus routes +a directed message to its destination and nowhere else, so a listener on a +second connection never receives it, whatever match rules it adds. + +That rules out the obvious shell-out. ``gdbus call`` and ``gdbus monitor`` are +two processes with two unique names, so the monitor is not the caller and the +Response is not addressed to it; measured against a real ``dbus-daemon``, it +sees the call go past and the answer never arrive. The only listener that +does see it is a full bus monitor (``dbus-monitor``, which asks the bus for +``BecomeMonitor``), and making a screenshot require permission to observe every +message on the user's session bus is a poor trade for a fallback tier. + +So the subscription and the call happen on one connection here, which is what +every portal client library does and what the protocol is designed for. The +cost is marshalling D-Bus by hand; the scope is kept to exactly what a portal +conversation needs — connect, authenticate, ``Hello``, ``AddMatch``, one method +call, and read signals until the matching one arrives. It is deliberately not a +general D-Bus binding: no properties, no introspection, no object export, no +file-descriptor passing (:mod:`je_auto_control.linux_wayland.oeffis` uses +liboeffis for the one call that needs that). + +Every failure is an :class:`AutoControlException` subclass, so the containment +boundaries that catch the family keep working. +""" +from __future__ import annotations + +import contextlib +import os +import socket +import struct +import time +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: Message types, from the D-Bus specification. +METHOD_CALL = 1 +METHOD_RETURN = 2 +ERROR = 3 +SIGNAL = 4 + +#: The fixed-width numeric types, as ``struct`` format and byte width. Each is +#: aligned to its own width on the wire. +#: +#: ``i`` matters more than it looks: AT-SPI reports a component's extents as +#: four *signed* integers, because a window on a monitor left of or above the +#: primary one is at a negative coordinate. Without it the accessibility +#: backend could read a tree but not where anything was. +_FIXED = { + "n": (" None: + self.signature = signature + self.value = value + + def __repr__(self) -> str: + return f"Variant({self.signature!r}, {self.value!r})" + + +# --- marshalling ---------------------------------------------------------- + + +class _Writer: + """Little-endian marshaller. Alignment is relative to the message start.""" + + def __init__(self, offset: int = 0) -> None: + self._parts: List[bytes] = [] + self._length = offset + + def align(self, boundary: int) -> None: + padding = (-self._length) % boundary + if padding: + self._parts.append(bytes(padding)) + self._length += padding + + def raw(self, data: bytes) -> None: + self._parts.append(data) + self._length += len(data) + + def byte(self, value: int) -> None: + self.raw(struct.pack(" None: + self.align(4) + self.raw(struct.pack(" None: + """Write one fixed-width number, aligned to its own width.""" + fmt, width = _FIXED[code] + self.align(width) + self.raw(struct.pack(fmt, float(value) if fmt == " None: + encoded = value.encode("utf-8") + self.uint32(len(encoded)) + self.raw(encoded + b"\x00") + + def signature(self, value: str) -> None: + encoded = value.encode("ascii") + self.byte(len(encoded)) + self.raw(encoded + b"\x00") + + def value(self, sig: str, value: Any) -> None: + """Write one complete value of the given single signature.""" + _write_value(self, _SignatureReader(sig), value) + + @property + def data(self) -> bytes: + return b"".join(self._parts) + + def __len__(self) -> int: + return self._length + + +class _SignatureReader: + """Walks a signature string one complete type at a time.""" + + def __init__(self, text: str) -> None: + self.text = text + self.index = 0 + + def done(self) -> bool: + return self.index >= len(self.text) + + def peek(self) -> str: + return self.text[self.index] + + def take(self) -> str: + code = self.text[self.index] + self.index += 1 + return code + + def take_complete(self) -> str: + """Take one whole type, following containers to their close.""" + start = self.index + code = self.take() + if code == "a": + self.take_complete() + elif code in "({": + closing = ")" if code == "(" else "}" + while self.peek() != closing: + self.take_complete() + self.take() + return self.text[start:self.index] + + +def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + code = reader.take() + if code == "y": + writer.byte(int(value)) + elif code == "b": + writer.uint32(1 if value else 0) + elif code in _FIXED: + writer.fixed(code, value) + elif code in "so": + writer.string(str(value)) + elif code == "g": + writer.signature(str(value)) + elif code == "v": + _write_variant(writer, value) + elif code == "a": + _write_array(writer, reader, value) + elif code in "({": + _write_struct(writer, reader, code, value) + else: + raise DBusError(f"cannot marshal D-Bus type {code!r}") + + +def _write_variant(writer: _Writer, value: Any) -> None: + variant = value if isinstance(value, Variant) else _guess_variant(value) + writer.signature(variant.signature) + writer.value(variant.signature, variant.value) + + +def _guess_variant(value: Any) -> Variant: + if isinstance(value, bool): + return Variant("b", value) + if isinstance(value, int): + return Variant("u", value) + if isinstance(value, str): + return Variant("s", value) + raise DBusError(f"no obvious D-Bus type for {type(value).__name__}") + + +def _write_array(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + element = reader.take_complete() + writer.align(4) + # The length prefix counts the elements only, and it is written before + # the padding that aligns the first one — so the body is built separately + # at the offset it will actually occupy. + body_start = len(writer) + 4 + padding = (-body_start) % _ALIGNMENT.get(element[0], 1) + body = _Writer(body_start + padding) + items = value.items() if isinstance(value, dict) else value + for item in items: + body.align(_ALIGNMENT.get(element[0], 1)) + _write_value(body, _SignatureReader(element), item) + writer.uint32(len(body) - body_start - padding) + writer.raw(bytes(padding)) + writer.raw(body.data) + + +def _write_struct(writer: _Writer, reader: _SignatureReader, code: str, + value: Any) -> None: + closing = ")" if code == "(" else "}" + writer.align(8) + for item in value: + if reader.peek() == closing: + raise DBusError("too many members for this struct signature") + _write_value(writer, reader, item) + if reader.peek() != closing: + raise DBusError("too few members for this struct signature") + reader.take() + + +class _Reader: + """Little-endian demarshaller over one complete message.""" + + def __init__(self, data: bytes, offset: int = 0) -> None: + self.data = data + self.offset = offset + + def align(self, boundary: int) -> None: + self.offset += (-self.offset) % boundary + + def take(self, count: int) -> bytes: + if self.offset + count > len(self.data): + raise DBusError("truncated D-Bus message") + chunk = self.data[self.offset:self.offset + count] + self.offset += count + return chunk + + def byte(self) -> int: + return self.take(1)[0] + + def uint32(self) -> int: + self.align(4) + return struct.unpack(" Any: + """Read one fixed-width number, aligned to its own width.""" + fmt, width = _FIXED[code] + self.align(width) + return struct.unpack(fmt, self.take(width))[0] + + def string(self) -> str: + length = self.uint32() + text = self.take(length).decode("utf-8", errors="replace") + self.take(1) + return text + + def signature(self) -> str: + length = self.byte() + text = self.take(length).decode("ascii", errors="replace") + self.take(1) + return text + + def value(self, sig: str) -> Any: + return _read_value(self, _SignatureReader(sig)) + + +def _read_value(reader: _Reader, signature: _SignatureReader) -> Any: + code = signature.take() + if code == "y": + return reader.byte() + if code == "b": + return bool(reader.uint32()) + if code in _FIXED: + return reader.fixed(code) + if code in "so": + return reader.string() + if code == "g": + return reader.signature() + if code == "v": + return reader.value(reader.signature()) + if code == "a": + return _read_array(reader, signature) + if code in "({": + return _read_struct(reader, signature, code) + raise DBusError(f"cannot demarshal D-Bus type {code!r}") + + +def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: + element = signature.take_complete() + length = reader.uint32() + reader.align(_ALIGNMENT.get(element[0], 1)) + end = reader.offset + length + items = [] + while reader.offset < end: + reader.align(_ALIGNMENT.get(element[0], 1)) + items.append(_read_value(reader, _SignatureReader(element))) + if element.startswith("{"): + return dict(items) + return items + + +def _read_struct(reader: _Reader, signature: _SignatureReader, + code: str) -> tuple: + closing = ")" if code == "(" else "}" + reader.align(8) + members = [] + while signature.peek() != closing: + members.append(_read_value(reader, signature)) + signature.take() + return tuple(members) + + +class Message: + """One decoded D-Bus message: the header fields that matter, and the body.""" + + __slots__ = ("type", "serial", "fields", "body") + + def __init__(self, message_type: int, serial: int, + fields: Dict[int, Any], body: List[Any]) -> None: + self.type = message_type + self.serial = serial + self.fields = fields + self.body = body + + @property + def path(self) -> str: + return self.fields.get(FIELD_PATH, "") + + @property + def interface(self) -> str: + return self.fields.get(FIELD_INTERFACE, "") + + @property + def member(self) -> str: + return self.fields.get(FIELD_MEMBER, "") + + @property + def reply_serial(self) -> int: + return self.fields.get(FIELD_REPLY_SERIAL, 0) + + @property + def error_name(self) -> str: + return self.fields.get(FIELD_ERROR_NAME, "") + + +# --- the connection ------------------------------------------------------- + + +def session_address() -> Optional[str]: + """The session bus address, or None when this process has no session bus.""" + address = os.environ.get(_ADDRESS_ENV, "").strip() + return address or None + + +def is_available() -> bool: + """Whether a session bus address is set, so connecting is worth trying.""" + return session_address() is not None + + +def _socket_target(address: str) -> Tuple[str, bool]: + """Pick a connectable ``unix:`` transport out of a bus address. + + :return: the socket path and whether it is in the abstract namespace. + """ + for candidate in address.split(";"): + if not candidate.startswith("unix:"): + continue + fields = [part.split("=", 1) + for part in candidate[len("unix:"):].split(",") + if "=" in part] + options = dict(fields) + if "path" in options: + return options["path"], False + if "abstract" in options: + return options["abstract"], True + raise DBusError(f"no usable unix transport in {address!r}") + + +class SessionBus: + """An authenticated connection to the session bus, as a context manager.""" + + def __init__(self, address: Optional[str] = None) -> None: + self.address = address or session_address() + self.unique_name = "" + self._socket: Optional[socket.socket] = None + self._serial = 0 + self._buffer = b"" + #: Messages read while waiting for a method reply. A portal's Response + #: signal and its method return come from two different senders, so + #: the bus gives no ordering between them and the signal can arrive + #: first — dropping it here would be a wait that never ends. + self._queued: List[Message] = [] + + # --- lifecycle -------------------------------------------------------- + + def __enter__(self) -> "SessionBus": + self.connect() + return self + + def __exit__(self, *_exception: Any) -> None: + self.close() + + def connect(self) -> None: + """Open the socket, authenticate, and say ``Hello``.""" + if self.address is None: + raise DBusError( + f"{_ADDRESS_ENV} is not set, so there is no session bus to " + f"reach the desktop portal on", + ) + path, abstract = _socket_target(self.address) + try: + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._socket.connect(("\0" + path) if abstract else path) + except OSError as error: + self.close() + raise DBusError(f"cannot reach the session bus: {error}") from error + self._authenticate() + self.unique_name = self.call( + BUS_NAME, BUS_PATH, BUS_INTERFACE, "Hello", "", [])[0] + + def close(self) -> None: + """Drop the connection; never raise from teardown.""" + if self._socket is not None: + with contextlib.suppress(OSError): + self._socket.close() + self._socket = None + + @property + def sender_token(self) -> str: + """This connection's name as the portal spells it inside object paths.""" + return self.unique_name.lstrip(":").replace(".", "_") + + # --- authentication --------------------------------------------------- + + def _authenticate(self) -> None: + """The SASL EXTERNAL handshake, which is a uid in hex over a socket.""" + uid = str(os.getuid()).encode("ascii") + self._send_raw(b"\x00AUTH EXTERNAL " + uid.hex().encode("ascii") + + b"\r\n") + reply = self._read_line() + if not reply.startswith("OK"): + raise DBusError(f"the session bus refused authentication: {reply}") + self._send_raw(b"BEGIN\r\n") + + def _send_raw(self, data: bytes) -> None: + if self._socket is None: + raise DBusError("the session bus connection is closed") + try: + self._socket.sendall(data) + except OSError as error: + raise DBusError(f"writing to the session bus failed: {error}") from error + + def _read_line(self, timeout: float = 10.0) -> str: + """Read one CRLF-terminated line of the auth conversation.""" + deadline = time.monotonic() + timeout + while b"\r\n" not in self._buffer: + self._fill(deadline - time.monotonic()) + line, _, self._buffer = self._buffer.partition(b"\r\n") + return line.decode("utf-8", errors="replace") + + # --- reading ---------------------------------------------------------- + + def _fill(self, timeout: float) -> None: + """Read whatever is available, or fail once the deadline has passed.""" + if self._socket is None: + raise DBusError("the session bus connection is closed") + if timeout <= 0: + raise DBusError("the session bus did not answer in time") + self._socket.settimeout(timeout) + try: + chunk = self._socket.recv(65536) + except socket.timeout as error: + raise DBusError("the session bus did not answer in time") from error + except OSError as error: + raise DBusError(f"reading from the session bus failed: {error}") from error + if not chunk: + raise DBusError("the session bus closed the connection") + self._buffer += chunk + + def read_message(self, deadline: float) -> Message: + """Read one whole message, or fail when the deadline passes.""" + while len(self._buffer) < 16: + self._fill(deadline - time.monotonic()) + endian, message_type, _flags, _version = struct.unpack( + " _MAX_MESSAGE: + raise DBusError("the session bus sent an implausibly large message") + while len(self._buffer) < total: + self._fill(deadline - time.monotonic()) + raw, self._buffer = self._buffer[:total], self._buffer[total:] + return _decode(raw, message_type, serial, fields_length, header_end) + + # --- writing ---------------------------------------------------------- + + def _next_serial(self) -> int: + self._serial += 1 + return self._serial + + def send(self, message_type: int, fields: Dict[int, Tuple[str, Any]], + signature: str, body: List[Any], flags: int = 0) -> int: + """Marshal and send one message; return its serial.""" + payload = _Writer() + for code, value in body_pairs(signature, body): + payload.value(code, value) + serial = self._next_serial() + header = _Writer() + header.raw(struct.pack(" List[Any]: + """Make a method call and return the reply body.""" + fields = { + FIELD_PATH: ("o", path), + FIELD_INTERFACE: ("s", interface), + FIELD_MEMBER: ("s", member), + FIELD_DESTINATION: ("s", destination), + } + if signature: + fields[FIELD_SIGNATURE] = ("g", signature) + serial = self.send(METHOD_CALL, fields, signature, body) + deadline = time.monotonic() + timeout + while True: + message = self.read_message(deadline) + if message.reply_serial != serial: + self._queued.append(message) + continue + if message.type == ERROR: + detail = message.body[0] if message.body else "" + raise DBusError(f"{message.error_name}: {detail}".strip(": ")) + if message.type == METHOD_RETURN: + return message.body + + def add_match(self, rule: str) -> None: + """Subscribe to signals, and wait for the bus to confirm the rule. + + Waiting matters twice over: a malformed rule is reported rather than + silently never matching, and the round trip proves the subscription is + in place before the call that provokes the signal is made. + """ + self.call(BUS_NAME, BUS_PATH, BUS_INTERFACE, "AddMatch", "s", [rule]) + + def wait_for_signal(self, paths: List[str], interface: str, member: str, + timeout: float) -> List[Any]: + """Read until the signal we subscribed to arrives, or time out.""" + def matches(message: Message) -> bool: + return (message.type == SIGNAL and message.member == member + and message.interface == interface + and message.path in paths) + + for index, message in enumerate(self._queued): + if matches(message): + del self._queued[index] + return message.body + self._queued.clear() + deadline = time.monotonic() + max(0.0, timeout) + while True: + message = self.read_message(deadline) + if matches(message): + return message.body + + +def body_pairs(signature: str, body: List[Any]): + """Pair each top-level type in a signature with its argument.""" + reader = _SignatureReader(signature) + for value in body: + if reader.done(): + raise DBusError("more arguments than the signature declares") + yield reader.take_complete(), value + if not reader.done(): + raise DBusError("fewer arguments than the signature declares") + + +def _decode(raw: bytes, message_type: int, serial: int, fields_length: int, + header_end: int) -> Message: + """Turn one complete message's bytes into a :class:`Message`.""" + reader = _Reader(raw, 12) + fields: Dict[int, Any] = {} + reader.uint32() # the header array's own length, already read + end = 16 + fields_length + while reader.offset < end: + reader.align(8) + code = reader.byte() + fields[code] = reader.value(reader.signature()) + reader.offset = header_end + ((-header_end) % 8) + body: List[Any] = [] + signature = fields.get(FIELD_SIGNATURE, "") + if signature: + walker = _SignatureReader(signature) + while not walker.done(): + body.append(_read_value(reader, _SignatureReader( + walker.take_complete()))) + return Message(message_type, serial, fields, body) + + +__all__ = [ + "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", + "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", + "Variant", "is_available", "session_address", +] diff --git a/test/unit_test/headless/test_accessibility_linux.py b/test/unit_test/headless/test_accessibility_linux.py new file mode 100644 index 00000000..c8340500 --- /dev/null +++ b/test/unit_test/headless/test_accessibility_linux.py @@ -0,0 +1,233 @@ +"""Headless tests for the Linux AT-SPI accessibility backend. No Qt, no bus. + +The backend's real behaviour is checked against a live bus and a live GTK +application by ``docker/x11_atspi_verify.py`` — a mock cannot tell you whether +at-spi2 agrees with the bytes you send. What is worth pinning here is the +part that has nothing to do with the bus: how a reply is turned into an +:class:`AccessibilityElement`, how the walk is bounded, and what the backend +does when there is no bus at all, which is the case on every developer +machine that is not Linux. +""" +import sys + +import pytest + +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, +) +from je_auto_control.utils.dbus_client import DBusError + + +atspi = pytest.importorskip( + "je_auto_control.utils.accessibility.backends.linux_backend", + exc_type=ImportError) + + +class FakeConnection: + """A tree of accessibles, answering the calls the backend makes.""" + + def __init__(self, tree=None, names=None, roles=None, extents=None, + states=None): + self.tree = tree or {} + self.names = names or {} + self.roles = roles or {} + self.extents_map = extents or {} + self.states = states or {} + self.entered = 0 + self.exited = 0 + + # context manager, so the backend's `with` blocks work unchanged + def __enter__(self): + self.entered += 1 + return self + + def __exit__(self, *_exception): + self.exited += 1 + + @property + def root(self): + return ("registry", "/root") + + def children(self, reference): + return list(self.tree.get(reference, [])) + + def property(self, reference, name, interface=None): + del interface + return self.names.get(reference, "") if name == "Name" else "" + + def role_name(self, reference): + return self.roles.get(reference, "") + + def state(self, reference): + return self.states.get(reference, 1 << 8) + + def extents(self, reference): + return self.extents_map.get(reference, (0, 0, 0, 0)) + + def text(self, reference): + return None + + def number(self, reference): + return None + + +APP = ("app", "/app") +WINDOW = ("app", "/window") +BUTTON = ("app", "/button") + + +def _tree_connection(): + return FakeConnection( + tree={("registry", "/root"): [APP], APP: [WINDOW], WINDOW: [BUTTON]}, + names={APP: "zenity", WINDOW: "autocontrol-dialog", BUTTON: "OK"}, + roles={WINDOW: "dialog", BUTTON: "push button"}, + extents={WINDOW: (-1280, 0, 310, 233), BUTTON: (-1200, 100, 60, 24)}, + ) + + +@pytest.fixture() +def backend(monkeypatch): + """An available backend whose bus is the fake tree above.""" + connection = _tree_connection() + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + instance.connection = connection + return instance + + +# --- availability ---------------------------------------------------------- + + +def test_unavailable_backend_says_what_to_install(): + """"No bus" and "no package" look the same from here, so name both.""" + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = False + with pytest.raises(AccessibilityNotAvailableError) as caught: + instance.list_elements() + message = str(caught.value) + assert "at-spi2-core" in message + assert "atk-bridge" in message + + +@pytest.mark.skipif(sys.platform.startswith("linux"), + reason="on Linux the real probe decides") +def test_off_linux_the_backend_is_never_available(): + assert atspi._is_available() is False + + +# --- turning a reply into an element --------------------------------------- + + +def test_walk_collects_the_tree_below_each_application(backend): + found = backend.list_elements() + assert [element.name for element in found] == [ + "autocontrol-dialog", "OK"] + assert all(isinstance(element, AccessibilityElement) for element in found) + + +def test_elements_carry_the_application_they_came_from(backend): + """app_name is what a caller filters on, and the walk is per-application.""" + assert {element.app_name for element in backend.list_elements()} == {"zenity"} + + +def test_negative_extents_survive_as_negative(backend): + """A monitor left of the primary one puts a window at a negative x. + + Reading the extents as unsigned would turn -1280 into 4293967296 and send + every click derived from it to the wrong screen. + """ + dialog = backend.list_elements()[0] + assert dialog.bounds == (-1280, 0, 310, 233) + assert dialog.center == (-1280 + 155, 116) + + +def test_app_name_filter_skips_other_applications(backend): + assert backend.list_elements(app_name="zenity") + assert backend.list_elements(app_name="something else") == [] + + +def test_max_results_bounds_the_walk(backend): + assert len(backend.list_elements(max_results=1)) == 1 + + +def test_window_title_scopes_to_one_window(backend): + """Below a window the tree is orders of magnitude smaller. + + So scoping both narrows the answer and shortens the walk, rather than + filtering a full result set afterwards. + """ + assert [element.name + for element in backend.list_elements(window_title="autocontrol")] \ + == ["autocontrol-dialog", "OK"] + assert backend.list_elements(window_title="no such window") == [] + + +def test_an_accessible_that_vanishes_mid_walk_is_skipped(monkeypatch): + """Applications close dialogs while they are being listed.""" + connection = _tree_connection() + + def _explode(reference): + if reference == BUTTON: + raise DBusError("no such object") + return connection.roles.get(reference, "") + + monkeypatch.setattr(connection, "role_name", _explode) + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + assert [element.name for element in instance.list_elements()] == [ + "autocontrol-dialog"] + + +def test_a_cyclic_tree_cannot_hang_the_caller(monkeypatch): + """A malformed tree must bottom out rather than recurse forever.""" + loop = ("app", "/loop") + connection = FakeConnection( + tree={("registry", "/root"): [APP], APP: [loop], loop: [loop]}, + names={APP: "app", loop: "loop"}, roles={loop: "panel"}) + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + found = instance.list_elements(max_results=500) + assert 0 < len(found) < 500 + + +def test_a_nameless_and_roleless_accessible_is_dropped(monkeypatch): + """A node with neither is not an element a caller could ever address.""" + connection = FakeConnection( + tree={("registry", "/root"): [APP], APP: [BUTTON]}, + names={APP: "app"}, roles={}) + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + assert instance.list_elements() == [] + + +# --- state ----------------------------------------------------------------- + + +def test_state_reads_both_halves_of_the_bitfield(): + """AT-SPI sends the state as two 32-bit words, not one 64-bit value. + + Reading only the first silently drops every state above bit 31. + """ + class TwoWordBus: + def __enter__(self): + return self + + def __exit__(self, *_exception): + return None + + def call(self, *_args, **_kwargs): + # low word carries ENABLED, high word carries bit 32 + return [[1 << 8, 1]] + + connection = atspi._AtspiConnection() + connection._bus = TwoWordBus() + assert connection.state(BUTTON) == (1 << 8) | (1 << 32) diff --git a/test/unit_test/headless/test_docker_artifacts.py b/test/unit_test/headless/test_docker_artifacts.py index 168c28e0..88098598 100644 --- a/test/unit_test/headless/test_docker_artifacts.py +++ b/test/unit_test/headless/test_docker_artifacts.py @@ -266,6 +266,27 @@ def test_x11_verification_image_and_script_exist(): assert "--setmonitor" in entrypoint +def test_x11_image_can_reach_an_accessibility_bus(): + """AT-SPI is a bus, so the image needs one and something bridged to it.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.x11").read_text(encoding="utf-8") + # at-spi2-core provides the services D-Bus activates; dbus-x11 provides + # the session bus to activate them on; zenity is a real GTK application, + # which xterm is not — it exposes no accessible tree at all. + for package in ("at-spi2-core", "dbus-x11", "zenity"): + assert package in dockerfile, f"Dockerfile.x11 missing {package}" + # GTK3 only loads its bridge when asked, and what normally asks is a + # desktop setting this container has no store for. + assert "atk-bridge" in dockerfile + + entrypoint = (_DOCKER_DIR / "entrypoint-x11.sh").read_text(encoding="utf-8") + assert "dbus-run-session" in entrypoint + + verify = (_DOCKER_DIR / "x11_atspi_verify.py").read_text(encoding="utf-8") + # Extents are the check that caught the signed-integer gap in the D-Bus + # client; losing it would let that regress silently. + assert "GetExtents" in verify + + def test_x11_verification_refuses_to_skip_a_missing_layout(): """A layout that cannot be declared is a failure, not a reason to skip. diff --git a/test/unit_test/headless/test_wayland_dbus_client.py b/test/unit_test/headless/test_wayland_dbus_client.py index 2be6a515..e5d15d85 100644 --- a/test/unit_test/headless/test_wayland_dbus_client.py +++ b/test/unit_test/headless/test_wayland_dbus_client.py @@ -20,8 +20,12 @@ import pytest -from je_auto_control.linux_wayland import _dbus_client -from je_auto_control.linux_wayland._dbus_client import ( +# These exercise the marshalling internals, so they import the client +# where it lives rather than through the Wayland package's re-export: +# a shim that forwarded private names too would be a second copy of +# the module's surface to keep in step. +from je_auto_control.utils.dbus_client import session_bus as _dbus_client +from je_auto_control.utils.dbus_client.session_bus import ( DBusError, SessionBus, Variant, _Reader, _SignatureReader, _Writer, ) @@ -159,9 +163,38 @@ def test_a_signature_that_wants_more_members_than_it_was_given_is_rejected(): def test_a_type_this_does_not_marshal_says_so(): + """``h`` is the one fixed-width type this refuses, and it refuses on purpose. + + A UNIX_FD is a 32-bit index into a descriptor array carried in the + message's control data, which this client does not receive. Marshalling + the index alone would hand a caller a number that addresses nothing. + """ writer = _Writer() with pytest.raises(DBusError, match="cannot marshal"): - writer.value("d", 1.5) + writer.value("h", 0) + + +def test_signed_and_wide_numbers_round_trip(): + """AT-SPI reports extents as signed integers, because coordinates can be. + + A monitor left of or above the primary one puts a window at a negative + coordinate, so reading these as unsigned would turn -1280 into + 4293967296 and send every click to the wrong screen. + """ + for signature, value in (("i", -1280), ("i", 42), ("n", -5), ("q", 5), + ("x", -(2 ** 40)), ("t", 2 ** 40), ("d", 1.5)): + writer = _Writer() + writer.value(signature, value) + reader = _dbus_client._Reader(writer.data) + assert reader.value(signature) == value, signature + + +def test_a_struct_of_signed_integers_round_trips(): + """The exact shape AT-SPI's Component.GetExtents returns.""" + writer = _Writer() + writer.value("(iiii)", [-1280, 0, 640, 480]) + reader = _dbus_client._Reader(writer.data) + assert reader.value("(iiii)") == (-1280, 0, 640, 480) # === Whole messages ======================================================== From dfbce67449e4cc6dcee4b7f0f0dba5aef794d6f8 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 03:38:56 +0800 Subject: [PATCH 08/30] Gate the macOS window probe on the code, not on the runner's session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A macos-14 runner was measured to have no ordinary application windows at all, so a probe that asserted a count would be asserting a property of the runner image rather than of this project — and would go red the day that image happens to open one. It now asserts that the backend is selected, the Quartz query runs and every window it does return can be described, and reports the counts either side of the layer filter so an empty session and a filter that dropped everything stay told apart. --- .github/workflows/platform-smoke.yml | 7 ++---- test/verify/macos_verify.py | 33 ++++++++++++++++++---------- 2 files changed, 24 insertions(+), 16 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 1d5b3830..bfdcd9fb 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -84,8 +84,5 @@ jobs: # So the flag is off and this is a gate now: EXPECTED in the script # holds what was measured, and a capability appearing or disappearing # turns this red and names which one. - # Back in --measure for one round: the window-management probe is new - # and its value is not in EXPECTED yet. Asserting a value nobody has - # measured would be the guess this whole script exists to avoid. - - name: Measure the macOS backend against a real window server - run: python test/verify/macos_verify.py --measure + - name: Verify the macOS backend against a real window server + run: python test/verify/macos_verify.py diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py index e870033c..2fbf5b6e 100644 --- a/test/verify/macos_verify.py +++ b/test/verify/macos_verify.py @@ -61,6 +61,10 @@ "keyboard-post": True, "accessibility-tree": True, "recorder-absent": True, + # True means "the code answered", not "the runner had windows": a + # macos-14 runner was measured to have none at the application layer, so + # the count is reported and not asserted. + "window-management": True, } _results: List[Tuple[str, bool, str]] = [] @@ -215,9 +219,12 @@ def probe_recorder() -> Outcome: def probe_window_management() -> Outcome: """Quartz lists windows without a grant; acting on one needs Accessibility. - Listing is what this measures, because it is the half that has to work - before any of the rest means anything — and because a runner with no - windows at all would make every window command untestable here. + What is asserted is that the *code* answers — the backend is selected, the + Quartz query runs, and every window it returns can be described. What is + only *reported* is how many there are, because that is a property of the + runner's session rather than of this project: a GitHub macOS runner was + measured to have no ordinary application windows at all. Gating on a + count would go red the day the runner image happens to open one. """ import je_auto_control as ac from je_auto_control.wrapper.window_backends import get_backend @@ -225,15 +232,19 @@ def probe_window_management() -> Outcome: backend = get_backend() if not backend.available: return Outcome(False, f"backend {backend.name!r} reports unavailable") + # Before the layer filter, so "the session is empty" and "the filter + # dropped everything" are told apart rather than guessed at. + raw = len(backend._window_info()) windows = ac.list_windows() - if not windows: - return Outcome(False, f"backend {backend.name!r} listed no windows") - window_id, title = windows[0] - rect = backend.window_rect(window_id) - pid = backend.window_process_id(window_id) - return Outcome(rect is not None and pid > 0, - f"{len(windows)} window(s); first {title!r} " - f"rect={rect} pid={pid}") + described = [] + for window_id, title in windows[:3]: + described.append((title, backend.window_rect(window_id), + backend.window_process_id(window_id))) + complete = all(rect is not None and pid > 0 + for _title, rect, pid in described) + return Outcome(complete, + f"{raw} on-screen window(s) from Quartz, {len(windows)} at " + f"the application layer; described {described}") PROBES: List[Tuple[str, Callable[[], Outcome]]] = [ From 3afc45303dc3d73aa3de4066bdb61710dc3c2614 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 03:46:13 +0800 Subject: [PATCH 09/30] Support the BSDs and arm64, and run something on both MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit platform_wrapper refused to start on anything that was not win32/cygwin/msys, darwin or linux/linux2, and each of the seven X11 backend modules carried its own copy of the same Linux-only guard. A FreeBSD, OpenBSD or NetBSD desktop runs the same X server and the same python-Xlib, so all of that was refusing a system the code already worked on — and python-Xlib was pinned to platform_system=='Linux', so relaxing the guards alone would have left the backend without its one dependency. sys.platform was being compared against literal lists in over a hundred places, and every list named the same three families, so the fix is one place that decides: utils/platform_id. The guards now ask is_x11_unix() — "is this an X11 unix", which is the question they were always trying to ask — rather than whether the kernel is Linux. None of that is worth anything unless something runs it on a BSD, and no hosted runner is one, so the freebsd job boots a real FreeBSD 14 VM inside the runner: it asserts the classification, then imports the X11 modules under a real X server and moves the pointer and reads it back. It covers the platform layer rather than the whole package, because opencv has no FreeBSD wheel and a source build would take an hour or fail — that limit is stated in the job rather than left to be discovered. arm64 joins the smoke matrix as ubuntu-22.04-arm and windows-11-arm (macos-14 was already arm64). The dependency set is where the architecture shows: opencv, pillow and cryptography all ship native wheels, and a missing one is a source build rather than a clean failure. --- .github/workflows/platform-smoke.yml | 78 ++++++++++++++++++- CLAUDE.md | 6 +- README.md | 2 +- README/README_zh-CN.md | 2 +- README/README_zh-TW.md | 2 +- architecture_explore.md | 27 +++---- docs/CAPABILITY_MATRIX.md | 22 ++++++ .../core/utils/x11_linux_display.py | 7 +- .../linux_with_x11/core/utils/x11_linux_vk.py | 7 +- .../keyboard/x11_linux_keyboard_control.py | 7 +- .../listener/x11_linux_listener.py | 7 +- .../mouse/x11_linux_mouse_control.py | 7 +- .../linux_with_x11/record/x11_linux_record.py | 7 +- .../linux_with_x11/screen/x11_linux_screen.py | 7 +- je_auto_control/utils/platform_id/__init__.py | 62 +++++++++++++++ je_auto_control/wrapper/platform_wrapper.py | 22 +++++- pyproject.toml | 5 +- test/unit_test/headless/test_platform_id.py | 73 +++++++++++++++++ 18 files changed, 311 insertions(+), 39 deletions(-) create mode 100644 je_auto_control/utils/platform_id/__init__.py create mode 100644 test/unit_test/headless/test_platform_id.py diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index bfdcd9fb..2e5ab0e8 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -14,8 +14,18 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-2022, ubuntu-22.04, macos-14] + # arm64 is not a rounding error on the desktop any more, and the + # dependency set is where it shows: opencv, pillow and cryptography + # all ship native wheels, and a missing arm64 wheel means a source + # build that either takes an hour or fails. macos-14 is already + # arm64; these add the other two. + os: [windows-2022, ubuntu-22.04, macos-14, ubuntu-22.04-arm, + windows-11-arm] python-version: ["3.10", "3.14"] + exclude: + # windows-11-arm has no 3.10 build available through setup-python. + - os: windows-11-arm + python-version: "3.10" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -50,6 +60,72 @@ jobs: path: platform-smoke.zip if-no-files-found: warn + freebsd: + name: The X11 backend's platform guards on a real FreeBSD + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + # The X11 backend was gated on sys.platform being linux/linux2, so it + # refused to import on a FreeBSD desktop that runs the same X server, + # the same python-Xlib and the same code. Relaxing that guard is only + # worth anything if something actually runs it on a BSD, and no hosted + # runner is one — so this boots a real FreeBSD VM inside the runner. + # + # What it checks is the platform layer, not the whole package: the + # heavy dependencies (opencv in particular) have no FreeBSD wheels, so + # a full install would build from source for an hour or fail. The + # guards and the wrapper's routing are the change; they are what runs. + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + - uses: vmactions/freebsd-vm@v1 # NOSONAR githubactions:S7637 + with: + release: "14.2" + usesh: true + prepare: | + pkg install -y python311 py311-pip py311-xlib xorg-vfbserver xauth + run: | + set -eu + echo "uname: $(uname -a)" + python3.11 - <<'PROBE' + import sys + sys.path.insert(0, ".") + from je_auto_control.utils.platform_id import ( + current_family, is_bsd, is_x11_unix, + ) + print("sys.platform:", sys.platform) + assert sys.platform.startswith("freebsd"), sys.platform + assert is_bsd(), "FreeBSD is not recognised as a BSD" + assert is_x11_unix(), "FreeBSD is not recognised as an X11 unix" + assert current_family() == "bsd", current_family() + print("platform_id: OK") + PROBE + # The guards are what refused to load here. Import the X11 + # modules directly, under a real X server, without dragging in + # the package's heavy dependencies. + Xvfb :99 -screen 0 1280x800x24 & + sleep 3 + DISPLAY=:99 python3.11 - <<'BACKEND' + import sys + sys.path.insert(0, ".") + from je_auto_control.linux_with_x11.core.utils import ( + x11_linux_display, x11_linux_vk, + ) + from je_auto_control.linux_with_x11.mouse import ( + x11_linux_mouse_control as mouse, + ) + from je_auto_control.linux_with_x11.screen import ( + x11_linux_screen as screen, + ) + print("display:", x11_linux_display.display) + print("screen size:", screen.size()) + mouse.set_position(321, 123) + landed = mouse.position() + assert landed == (321, 123), landed + print("pointer round-trip on FreeBSD:", landed) + print("keycode for 'a':", x11_linux_vk.x11_linux_key_a) + BACKEND + macos-capabilities: name: What a real macOS runner permits runs-on: macos-14 diff --git a/CLAUDE.md b/CLAUDE.md index 2ceb846e..4c083688 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mouse and keyboard control, image recognition, OCR, accessibility-tree and VLM element location, action scripting, and report generation behind one API. Backends: Windows (Win32 ctypes), macOS (pyobjc/Quartz), Linux X11 (python-Xlib), Linux Wayland (libei / ydotool), Android (adb), iOS (WebDriverAgent). - **Package**: `je_auto_control` · **Python** ≥ 3.10 · **License**: MIT · **Author**: JE-Chen -- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 309 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. +- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 310 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. ## Architecture @@ -18,7 +18,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mo | Template Method | `utils/generate_report/` | HTML / JSON / XML share collect → format → write. | | Backend seam | `backends/` under `accessibility`, `ocr`, `vision`, `llm`, `agent`, `hotkey`, `usb`, `usbip` | Abstract base + concrete impls + null fallback, so dependency-free environments still import. | -Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (309 headless subpackages) → `wrapper/` → per-OS backend. +Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (310 headless subpackages) → `wrapper/` → per-OS backend. ## Development Commands @@ -68,7 +68,7 @@ The map is only useful while it matches the tree, so **update it in the same cha Never adjust one by hand: the counts are `len(text.splitlines())` (what `wc -l` reports), and hand-editing is how the document ended up quoting the same subsystem at two sizes at once — most tables had been counting a phantom trailing line per file while §1 and §8 counted correctly. -- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 309 subpackages; appearing twice or not at all is a defect. +- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 310 subpackages; appearing twice or not at all is a defect. - A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. - Keep the header's scan date, version, and branch current. - `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. diff --git a/README.md b/README.md index 6d679ea4..162a707d 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Diagnostics | `run_diagnostics` | `AC_diagnose` | Diagnostics | | Test-code generation | `generate_code` | — | — | -Beyond this table, `utils/` holds 309 headless packages covering assertions, resilience, +Beyond this table, `utils/` holds 310 headless packages covering assertions, resilience, data quality, i18n auditing, redaction, governance, observability, and more. The full per-module map is in **[architecture_explore.md](architecture_explore.md)**. diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 047eb5be..b90b3202 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -147,7 +147,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 系统诊断 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 测试代码生成 | `generate_code` | — | — | -除了这张表，`utils/` 下还有 309 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 +除了这张表，`utils/` 下还有 310 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 治理、可观测性等等。完整的逐模块地图在 **[architecture_explore.md](../architecture_explore.md)**。 --- diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 7be0fa71..3cd13826 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -147,7 +147,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 系統診斷 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 測試碼產生 | `generate_code` | — | — | -除了這張表，`utils/` 底下還有 309 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 +除了這張表，`utils/` 底下還有 310 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 治理、可觀測性等等。完整的逐模組地圖在 **[architecture_explore.md](../architecture_explore.md)**。 --- diff --git a/architecture_explore.md b/architecture_explore.md index 5cbc0157..2b3bed22 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,9 +19,9 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,025 | -| 程式碼總行數 | 139,017 | -| `je_auto_control/utils/` 子套件數 | 309 | +| Python 模組總數（含周邊子專案） | 1,026 | +| 程式碼總行數 | 139,114 | +| `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | | GUI 分頁數（`main_widget` 註冊） | 48 | @@ -55,7 +55,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 └───────────────────────────────┬──────────────────────────────────────────┘ │ ┌───────────────────────────────▼──────────────────────────────────────────┐ -│ 能力層 utils/（309 個子套件，全部無 Qt 相依） │ +│ 能力層 utils/（310 個子套件，全部無 Qt 相依） │ │ 影像辨識 │ OCR │ 無障礙樹 │ 定位自癒 │ AI/Agent │ 遠端桌面 │ USB │ │ 報表觀測 │ 資料 │ 安全 │ 韌性 │ 系統整合 │ 排程觸發 │ 網路協定 │ └───────────────────────────────┬──────────────────────────────────────────┘ @@ -166,7 +166,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `wrapper/platform_wrapper.py` | 59 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | +| `wrapper/platform_wrapper.py` | 73 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/_platform_windows.py` | 325 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | | `wrapper/_platform_osx.py` | 155 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | @@ -211,7 +211,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | -#### Linux X11（`linux_with_x11/`，19 檔／1,175 行） +#### Linux X11（`linux_with_x11/`，19 檔／1,196 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -258,7 +258,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `ios/input.py` | 46 | iOS 觸控與按鍵原語。 | | `ios/screen.py` | 32 | iOS 裝置螢幕擷取與尺寸。 | -### 5.4 能力層 `utils/`（309 個子套件） +### 5.4 能力層 `utils/`（310 個子套件） 以下依主題分組。每個子套件都是獨立可匯入的無頭模組，不含任何 Qt 相依。 @@ -296,7 +296,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.2 框架基礎設施 -> 13 個套件、約 2,575 行。 +> 14 個套件、約 2,637 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -311,6 +311,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | | `utils/package_manager/` | 98 | 動態載入套件並把 executor 注入其中 | | `utils/path_guard/` | 99 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | +| `utils/platform_id/` | 62 | 作業系統家族的單一判定點。`sys.platform` 原本在一百多處跟字面清單比對，而那些清單都沒有 BSD；`is_x11_unix()` 問的是「這是不是 X11 unix」，這才是守衛一直想問的問題 | | `utils/shell_process/` | 159 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | | `utils/start_exe/` | 39 | 啟動另一個執行檔行程 | @@ -920,7 +921,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | audit_log | `audit_log_tab.py` | 192 | 瀏覽並驗證防竄改雜湊鏈。 | | inspector | `inspector_tab.py` | 121 | WebRTC 檢測器：即時摘要與近期統計取樣。 | | usb_devices | `usb_devices_tab.py` | 121 | 唯讀列舉 + 熱插拔監看控制。 | -| usb_browser | `usb_browser_tab.py` | 309 | 檢視端 USB 裝置瀏覽器。 | +| usb_browser | `usb_browser_tab.py` | 310 | 檢視端 USB 裝置瀏覽器。 | | usb_share | `usb_passthrough_panel.py` + `usb_passthrough_prompt.py` | 704 | AnyDesk 風格 USB 直通面板與主機端 ACL 授權對話框。 | | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | @@ -1023,11 +1024,11 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/usb/` | 17 | 4,247 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | | `utils/accessibility/` | 13 | 2,818 | -| `wrapper/` | 3,026 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | +| `wrapper/` | 3,040 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | | `windows/` | 23 | 1,995 | | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | -| `linux_with_x11/` | 19 | 1,175 | +| `linux_with_x11/` | 19 | 1,196 | | `linux_wayland/` | 17 | 2,830 | | `utils/triggers/` | 4 | 1,146 | | `utils/ocr/` | 9 | 1,112 | @@ -1036,6 +1037,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 761 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 687 | 49,958 | -| **總計** | **1,019** | **138,952** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 688 | 50,034 | +| **總計** | **1,020** | **139,049** | diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index da87997d..57e57926 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -65,6 +65,28 @@ primary one, where the compositor's plane starts at a negative coordinate and a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of that monitor. +The table has four columns because those are the four desktops with their own +backend, not because they are the only supported systems. Two more axes now +have CI behind them: + +**The BSDs.** `platform_wrapper` refused to start on anything that was not +win32/cygwin/msys, darwin or linux/linux2, and every X11 backend module +carried its own copy of the same Linux-only guard — so a FreeBSD, OpenBSD or +NetBSD desktop, which runs the same X server and the same `python-Xlib` as +Linux, could not import the package at all. `python-Xlib` was pinned to +`platform_system=='Linux'` too, so even relaxing the guards would have left +the backend without its one dependency. The guards now ask +`utils/platform_id.is_x11_unix()` — "is this an X11 unix", which is the +question they were always trying to ask — and the `freebsd` job boots a real +FreeBSD 14 VM to run them, moving the pointer and reading it back. It checks +the platform layer rather than the whole package: opencv has no FreeBSD +wheel, and a source build would take an hour or fail. + +**arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` and +`windows-11-arm` join the smoke matrix. The dependency set is where the +architecture shows — opencv, pillow and cryptography all ship native wheels, +and a missing one means a source build rather than a clean failure. + The accessibility row said `backend tests` for Linux X11 and meant nothing by it: there was no Linux backend at all, and `_build_backend()` fell straight through to the null one. There is one now, over **AT-SPI2** — which is a D-Bus diff --git a/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py b/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py index f5ee1f86..fe93c594 100644 --- a/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py +++ b/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py @@ -1,9 +1,12 @@ -import sys from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) import os diff --git a/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py b/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py index 6c295691..b81fbe1e 100644 --- a/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py +++ b/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py @@ -1,9 +1,12 @@ -import sys from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib import XK diff --git a/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py b/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py index 9fe6febc..f6477877 100644 --- a/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py +++ b/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py @@ -1,12 +1,15 @@ -import sys import time from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from je_auto_control.linux_with_x11.core.utils.x11_linux_display import display diff --git a/je_auto_control/linux_with_x11/listener/x11_linux_listener.py b/je_auto_control/linux_with_x11/listener/x11_linux_listener.py index 52590c59..9bf0b02c 100644 --- a/je_auto_control/linux_with_x11/listener/x11_linux_listener.py +++ b/je_auto_control/linux_with_x11/listener/x11_linux_listener.py @@ -1,13 +1,16 @@ -import sys from queue import Queue from threading import Thread from je_auto_control.utils.exception.exception_tags import linux_import_error_message, listener_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib.display import Display diff --git a/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py b/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py index dff94b70..844a7733 100644 --- a/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py +++ b/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py @@ -1,13 +1,16 @@ -import sys import time from typing import Optional, Tuple from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib import X, protocol diff --git a/je_auto_control/linux_with_x11/record/x11_linux_record.py b/je_auto_control/linux_with_x11/record/x11_linux_record.py index 01744b4a..691bba00 100644 --- a/je_auto_control/linux_with_x11/record/x11_linux_record.py +++ b/je_auto_control/linux_with_x11/record/x11_linux_record.py @@ -1,13 +1,16 @@ -import sys from typing import Any from queue import Queue from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from je_auto_control.linux_with_x11.listener.x11_linux_listener import ( diff --git a/je_auto_control/linux_with_x11/screen/x11_linux_screen.py b/je_auto_control/linux_with_x11/screen/x11_linux_screen.py index d509fbdf..7e888734 100644 --- a/je_auto_control/linux_with_x11/screen/x11_linux_screen.py +++ b/je_auto_control/linux_with_x11/screen/x11_linux_screen.py @@ -1,12 +1,15 @@ -import sys from typing import Tuple from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib import X diff --git a/je_auto_control/utils/platform_id/__init__.py b/je_auto_control/utils/platform_id/__init__.py new file mode 100644 index 00000000..840a4a43 --- /dev/null +++ b/je_auto_control/utils/platform_id/__init__.py @@ -0,0 +1,62 @@ +"""Which operating-system family this is, asked once and spelled one way. + +``sys.platform`` was compared against literal lists in over a hundred places, +and every one of those lists named ``win32``/``cygwin``/``msys``, ``darwin``, +``linux``/``linux2`` and nothing else. That is fine until an operating system +turns up that is none of them and still runs X11 — a FreeBSD, OpenBSD or +NetBSD desktop is an ordinary X11 desktop, and the X11 backend works there — +at which point every one of those lists is a separate place to be wrong. + +The distinction that actually matters to this project is not "which kernel" +but **which input and display stack**, so that is what these answer. +""" +import sys + +__all__ = [ + "BSD_PREFIXES", "current_family", "is_bsd", "is_macos", "is_windows", + "is_x11_unix", +] + +#: ``sys.platform`` on the BSDs carries the major version — ``freebsd14``, +#: ``openbsd7`` — so these are matched by prefix rather than by equality. +BSD_PREFIXES = ("freebsd", "openbsd", "netbsd", "dragonfly") + + +def is_windows(platform: str = "") -> bool: + """Windows, including the POSIX emulation layers that ship the Win32 API.""" + return (platform or sys.platform) in ("win32", "cygwin", "msys") + + +def is_macos(platform: str = "") -> bool: + """macOS, where the backend is Quartz rather than X11.""" + return (platform or sys.platform) == "darwin" + + +def is_bsd(platform: str = "") -> bool: + """One of the BSDs, which run the same X11 stack as Linux.""" + return (platform or sys.platform).startswith(BSD_PREFIXES) + + +def is_x11_unix(platform: str = "") -> bool: + """A Unix whose desktop is X11 (or Wayland with XWayland underneath). + + Linux and the BSDs both qualify. This is the test the X11 backend + modules guard on: what they need is an X server and ``python-Xlib``, + neither of which is Linux-specific. + """ + name = platform or sys.platform + return name.startswith("linux") or is_bsd(name) + + +def current_family(platform: str = "") -> str: + """``windows`` / ``macos`` / ``linux`` / ``bsd`` / the raw name.""" + name = platform or sys.platform + if is_windows(name): + return "windows" + if is_macos(name): + return "macos" + if name.startswith("linux"): + return "linux" + if is_bsd(name): + return "bsd" + return name diff --git a/je_auto_control/wrapper/platform_wrapper.py b/je_auto_control/wrapper/platform_wrapper.py index 6940935a..e844d53e 100644 --- a/je_auto_control/wrapper/platform_wrapper.py +++ b/je_auto_control/wrapper/platform_wrapper.py @@ -1,20 +1,21 @@ import sys from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_bsd, is_macos, is_windows -if sys.platform in ["win32", "cygwin", "msys"]: +if is_windows(): from je_auto_control.wrapper._platform_windows import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) -elif sys.platform == "darwin": +elif is_macos(): from je_auto_control.wrapper._platform_osx import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) -elif sys.platform in ["linux", "linux2"]: +elif sys.platform.startswith("linux"): from je_auto_control.linux_wayland import select_display_server from je_auto_control.utils.logging.logging_instance import ( autocontrol_logger, @@ -45,8 +46,21 @@ mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) +elif is_bsd(): + # A FreeBSD, OpenBSD or NetBSD desktop is an ordinary X11 desktop: the + # same X server, the same python-Xlib, the same backend. There is no + # Wayland branch here because the Wayland backend's fast path is libei, + # whose socket and portal are Linux desktop infrastructure; a BSD running + # Wayland reaches this through XWayland like any other X11 client. + from je_auto_control.wrapper._platform_linux import ( # noqa: F401 # reason: facade re-export + keyboard, keyboard_check, keyboard_keys_table, + mouse, mouse_keys_table, special_mouse_keys_table, + screen, recorder, + ) else: - raise AutoControlException("unknown operating system") + raise AutoControlException( + f"unknown operating system: {sys.platform!r}. Windows, macOS, Linux " + f"and the BSDs are supported.") if None in [keyboard_keys_table, mouse_keys_table, keyboard, mouse, screen]: raise AutoControlException("Can't init auto control") diff --git a/pyproject.toml b/pyproject.toml index b52afa4f..4cdd6065 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,10 @@ dependencies = [ "pillow==12.3.0", "pyobjc-core==12.2.1;platform_system=='Darwin'", "pyobjc==12.2.1;platform_system=='Darwin'", - "python-Xlib==0.33;platform_system=='Linux'", + # The BSDs run the same X11 stack, and platform_system is the kernel + # name there ("FreeBSD", "OpenBSD", "NetBSD"), so a Linux-only marker + # left the X11 backend without its one dependency on every one of them. + "python-Xlib==0.33;platform_system=='Linux' or platform_system=='FreeBSD' or platform_system=='OpenBSD' or platform_system=='NetBSD'", "mss==10.2.0", "defusedxml==0.7.1", "cryptography>=48.0.1" diff --git a/test/unit_test/headless/test_platform_id.py b/test/unit_test/headless/test_platform_id.py new file mode 100644 index 00000000..f91ba948 --- /dev/null +++ b/test/unit_test/headless/test_platform_id.py @@ -0,0 +1,73 @@ +"""Headless tests for platform identification. No Qt. + +``sys.platform`` was compared against literal lists in over a hundred places, +and every one named win32/cygwin/msys, darwin and linux/linux2 — so an +operating system that is none of those and still runs X11 was a hundred +separate places to be wrong. These pin the one place that decides now. + +The real FreeBSD check is the ``freebsd`` job in ``platform-smoke.yml``, which +boots a VM and imports the X11 backend on it; what is worth pinning here is +the classification itself, which is pure and can be checked from anywhere. +""" +import pytest + +from je_auto_control.utils.platform_id import ( + current_family, is_bsd, is_macos, is_windows, is_x11_unix, +) + + +@pytest.mark.parametrize("platform,family", [ + ("win32", "windows"), ("cygwin", "windows"), ("msys", "windows"), + ("darwin", "macos"), + ("linux", "linux"), ("linux2", "linux"), + ("freebsd14", "bsd"), ("freebsd13", "bsd"), + ("openbsd7", "bsd"), ("netbsd10", "bsd"), ("dragonfly6", "bsd"), +]) +def test_families_are_classified(platform, family): + assert current_family(platform) == family + + +def test_an_unknown_platform_keeps_its_own_name(): + """Better a name the reader can look up than a wrong family.""" + assert current_family("haiku1") == "haiku1" + assert not is_x11_unix("haiku1") + + +@pytest.mark.parametrize("platform", [ + "freebsd14", "openbsd7", "netbsd10", "dragonfly6"]) +def test_the_bsds_are_x11_unixes(platform): + """What the X11 backend needs is an X server and python-Xlib. + + Neither is Linux-specific, and a BSD desktop is an ordinary X11 desktop, + so the guards ask this rather than asking whether the kernel is Linux. + """ + assert is_bsd(platform) + assert is_x11_unix(platform) + assert not is_windows(platform) + assert not is_macos(platform) + + +def test_the_bsd_version_suffix_does_not_matter(): + """sys.platform carries the major version there — freebsd14, not freebsd. + + An equality check against "freebsd" matches no real system at all, which + is the trap this prefix match exists to avoid. + """ + assert is_bsd("freebsd") + assert is_bsd("freebsd15") + # Prefix, not equality: matching "freebsd" exactly matches no real + # system. Python's own porting guidance says to compare this way. + assert not is_bsd("linux") + assert not is_bsd("darwin") + + +def test_macos_is_not_an_x11_unix(): + """It is a unix, but its backend is Quartz — the distinction that matters.""" + assert is_macos("darwin") + assert not is_x11_unix("darwin") + + +def test_the_helpers_default_to_this_interpreter(): + """Called with no argument they answer about the running platform.""" + families = {is_windows(), is_macos(), is_x11_unix()} + assert True in families, "this platform matches none of the families" From f550eba1d3ad8bf10855e2ccae72bf571ed7fd88 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 03:50:33 +0800 Subject: [PATCH 10/30] Record what shipped, and the one thing deliberately left MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WHATS_NEW and both translations describe the six phases; CHANGELOG records the compatibility-visible parts — the window backends, the AT-SPI backend, platform_id, the new exception class and the D-Bus client's move. Progress.md gains the macOS recorder: OSXRecorder is a complete implementation that the platform wrapper never selects, and that is not an oversight. osx_listener builds an NSApplication at import time and stopping a recording needs a blocking run loop, so wiring it up as it stands would move both onto the import path of the whole package. The capability matrix already says 'unavailable', which matches. --- CHANGELOG.md | 32 ++++++++++++ Progress.md | 15 ++++++ README/WHATS_NEW_zh-CN.md | 64 +++++++++++++++++++++++ README/WHATS_NEW_zh-TW.md | 64 +++++++++++++++++++++++ WHATS_NEW.md | 107 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 282 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aeea62a..e630b6c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,25 @@ only when documented here with a migration path. ### Added +- Cross-platform window management. The 23 `AC_*` window commands and their + MCP tools now work on macOS and Linux/X11 as well as Windows, through a + backend seam (`je_auto_control.wrapper.window_backends`). Wayland remains + unsupported: the protocol does not let a client enumerate or move another + application's windows. +- Linux accessibility backend over AT-SPI2 + (`je_auto_control.utils.accessibility.backends.linux_backend`), with no new + dependency. Serves both X11 and Wayland sessions. +- `je_auto_control.utils.platform_id` — one place that classifies the + operating system family, and the BSDs are now one of them. FreeBSD, OpenBSD, + NetBSD and DragonFly route to the X11 backend instead of raising "unknown + operating system". +- `AutoControlUnsupportedOperationException`, raised when a platform backend + cannot perform an operation. It subclasses both `AutoControlException` and + `NotImplementedError`, so existing `except NotImplementedError` handlers are + unaffected while the executor's containment boundaries now catch it. +- `je_auto_control.utils.dbus_client` — the D-Bus client, moved out of + `linux_wayland/` so `utils/` can use it. The old path re-exports it. + - Stable, headless `je_auto_control.api` façade. - Portable `autocontrol.failure-bundle/v1` diagnostic archives and CLI command. - Public API lifecycle, capability matrix, security policy, coverage and type @@ -247,6 +266,19 @@ only when documented here with a migration path. ### Fixed +- macOS: `write()` typed a space instead of a backspace, because `"\b"` had + no route in the macOS key table and fell through to the space fallback. +- macOS: USB enumeration returned `apple_vendor_id` in `vendor_id`, a field + documented as a four-hex-digit string. A value that is not a hex id is now + `None`; the device is still listed and `manufacturer` still names the vendor. +- Linux/X11: `window_rect` returned the client area rather than the frame, + disagreeing with Win32's `GetWindowRect` by the window decorations. +- Linux/X11: `move_window_by_title` configured the client window directly, + which under a reparenting window manager positions it in the wrong + coordinate space. It now goes through `_NET_MOVERESIZE_WINDOW`. +- The D-Bus client could not marshal or demarshal signed integers, so any + protocol using them (AT-SPI extents among them) failed to decode. + - **Wayland: an absolute mouse move through the ydotool fallback counted from the wrong origin.** `ydotool mousemove --absolute` emits no absolute event — it drives the cursor into the corner the compositor clamps to and then moves diff --git a/Progress.md b/Progress.md index ee67f2c9..59019607 100644 --- a/Progress.md +++ b/Progress.md @@ -50,6 +50,21 @@ --- +## macOS 的錄製器寫了，但接不上去 + +`TODO` — `osx/record/osx_record.py`、`osx/listener/osx_listener.py` + +`OSXRecorder` 是完整實作，但 `wrapper/_platform_osx.py` 裡寫的是 +`recorder = None`，所以它永遠不會被選到。這不是疏失： +`osx_listener.py` 在 **import 時**就呼叫 `NSApplication.sharedApplication()`， +而停止錄製要靠 `AppHelper.runEventLoop()`，那是一個會卡住呼叫緒的 +事件迴圈。直接接上去會把這兩件事都搬進 `import je_auto_control` +的路徑上，那是回歸而不是修好。 + +要接上去得先把 listener 改成：不在 import 時建立 NSApplication， +且把 run loop 放到自己的執行緒。`docs/CAPABILITY_MATRIX.md` 的 Recorder +macOS 格寫的是 `unavailable`，跟現狀一致。 + ## `mouse_scroll` 的方向在三個平台上不是同一回事 `DECIDE` — `wrapper/auto_control_mouse.py::mouse_scroll` diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index b42b6033..76958668 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,69 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-20) — 声称支援的平台，這回真的量過了 + +整套测试一直只在 `windows-2022` 跑，另加容器裡一次 Linux +執行。macOS 只跑兩行指令。Wayland 有五個 job 對真的對等體 +讀回輸入；X11——兩條 Linux 路徑中更老、部署更廣的那一 +條——一個都沒有，而且套件裡每一條 X11 斷言都是對著 +`python-Xlib` 的 mock 做的。 + +**测试套件現在真的在它宣稱支援的平台上跑。** +`pytest-headless` 改成 OS 矩陣；Linux 跑在真的 Xvfb 上而不是 Qt 的 +offscreen，因為 X11 後端在 import 時就連線，offscreen 會將正好要找的 +毛病蓋掉。第一輪就抓到兩個真的 macOS 缺陷： + +- `write("\b")` 在 macOS 沒有任何按鍵路徑，會落到空白鍵 + fallback——要求退格，打出來的是空白。 +- `system_profiler` 對 Apple 自家裝置回的是 **符號式** vendor + id（`apple_vendor_id`），而那個欄位文件上寫的是四位十六進位。 + +**X11 的輸入現在從真的客戶端讀回。** 新的 `x11-verification` +job 跑在真的 Xvfb + 真的視窗管理員上，對照組來自受測對象以外的 +程式碼：`xev`、ImageMagick 的 `import`、`xdotool` 與 `xdpyinfo`。 +最值得點名的一項是 `synthetic NO`——`XSendEvent` 的事件帶的是 +`YES`，大多數 toolkit 會直接丟掉，所以一個患患停止驅動真實 +輸入的後端，在只數事件的檢查下仍然會全綠。 + +**macOS 在 CI 裡其實完全驗得了，跟一般假設相反。** +`macos-14` runner **兩個 TCC 權限都給**：擷取回來的是真像素而不是 +被拒時的全黑矩形，`CGEventPost` 真的移得動游標且讀回完全相符， +AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期望表 +還是空的時候會拒絕通過。 + +### 視窗管理不再是 Windows 專屬 + +以前是：門面在 `sys.platform` 上分支，其他平台一律丟例外， +23 個 `AC_*` 指令跟對應的 MCP 工具在 macOS 與 Linux 上都是死的。 +現在走平台縫：Win32、X11 的 EWMH、macOS 的 Quartz + 無障礙 API。 + +兩件只有真的視窗管理員才能披露的錯，第一版都錯了： + +- **矩形是外框，不是客戶區。** Win32 的 `GetWindowRect` + 回的是外框，所有呼叫端都是照那個寫的。 +- **移動必須走 `_NET_MOVERESIZE_WINDOW`。** 在 reparenting + 視窗管理員下，客戶端自己的 x/y 是相對於外框的；對 openbox + 要 (300, 220)，直接 `ConfigureWindow` 的結果是落在 (302, 260)。 + +### Linux 有無障礙後端了 + +之前完全沒有。新後端走 **AT-SPI2**——它是 D-Bus 協定而不是 +函式庫，這就是它不用加新相依的原因：`pyatspi` 與 +`gi.repository.Atspi` 是發行版套件，裝不進 venv。 + +拿真的 bus 跟真的 GTK 程式一驗，當場抓到 D-Bus 客戶端的一個缺口： +**它不會解有號整數**。portal 從來不需要，而 AT-SPI 的 extents 是四個 +**有號**值——因為主螢幕左邊（或上方）的螢幕上，視窗坐標是負的。 + +### BSD 與 arm64 + +`platform_wrapper` 對非 win/darwin/linux 一律丟「unknown operating +system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 +新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 +真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 +`ubuntu-22.04-arm` 與 `windows-11-arm` 加進 smoke 矩陣。 + + ## 本次更新 (2026-08-19) — Wayland 两个等人拍板的取舍,拍板了 `Progress.md` 上挂着两个 `DECIDE`:缺的不是活，是决定。两件事其实是同一个问题犯两次 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 1de15dd2..09ba0fe2 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,69 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-20) — 嬣稱支援的平台，這回真的量過了 + +整套測試一直只在 `windows-2022` 跑，另加容器裡一次 Linux +執行。macOS 只跑兩行指令。Wayland 有五個 job 對真的對等體 +讀回輸入；X11——兩條 Linux 路徑中更老、部署更廣的那一 +條——一個都沒有，而且套件裡每一條 X11 斷言都是對著 +`python-Xlib` 的 mock 做的。 + +**測試套件現在真的在它宣稱支援的平台上跑。** +`pytest-headless` 改成 OS 矩陣；Linux 跑在真的 Xvfb 上而不是 Qt 的 +offscreen，因為 X11 後端在 import 時就連線，offscreen 會將正好要找的 +毛病蓋掉。第一輪就抓到兩個真的 macOS 缺陷： + +- `write("\b")` 在 macOS 沒有任何按鍵路徑，會落到空白鍵 + fallback——要求退格，打出來的是空白。 +- `system_profiler` 對 Apple 自家裝置回的是 **符號式** vendor + id（`apple_vendor_id`），而那個欄位文件上寫的是四位十六進位。 + +**X11 的輸入現在從真的客戶端讀回。** 新的 `x11-verification` +job 跑在真的 Xvfb + 真的視窗管理員上，對照組來自受測對象以外的 +程式碼：`xev`、ImageMagick 的 `import`、`xdotool` 與 `xdpyinfo`。 +最值得點名的一項是 `synthetic NO`——`XSendEvent` 的事件帶的是 +`YES`，大多數 toolkit 會直接丟掉，所以一個患患停止驅動真實 +輸入的後端，在只數事件的檢查下仍然會全綠。 + +**macOS 在 CI 裡其實完全驗得了，跟一般假設相反。** +`macos-14` runner **兩個 TCC 權限都給**：擷取回來的是真像素而不是 +被拒時的全黑矩形，`CGEventPost` 真的移得動游標且讀回完全相符， +AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期望表 +還是空的時候會拒絕通過。 + +### 視窗管理不再是 Windows 專屬 + +以前是：門面在 `sys.platform` 上分支，其他平台一律丟例外， +23 個 `AC_*` 指令跟對應的 MCP 工具在 macOS 與 Linux 上都是死的。 +現在走平台縫：Win32、X11 的 EWMH、macOS 的 Quartz + 無障礙 API。 + +兩件只有真的視窗管理員才能披露的錯，第一版都錯了： + +- **矩形是外框，不是客戶區。** Win32 的 `GetWindowRect` + 回的是外框，所有呼叫端都是照那個寫的。 +- **移動必須走 `_NET_MOVERESIZE_WINDOW`。** 在 reparenting + 視窗管理員下，客戶端自己的 x/y 是相對於外框的；對 openbox + 要 (300, 220)，直接 `ConfigureWindow` 的結果是落在 (302, 260)。 + +### Linux 有無障礙後端了 + +之前完全沒有。新後端走 **AT-SPI2**——它是 D-Bus 協定而不是 +函式庫，這就是它不用加新相依的原因：`pyatspi` 與 +`gi.repository.Atspi` 是發行版套件，裝不進 venv。 + +拿真的 bus 跟真的 GTK 程式一驗，當場抓到 D-Bus 客戶端的一個缺口： +**它不會解有號整數**。portal 從來不需要，而 AT-SPI 的 extents 是四個 +**有號**值——因為主螢幕左邊（或上方）的螢幕上，視窗坐標是負的。 + +### BSD 與 arm64 + +`platform_wrapper` 對非 win/darwin/linux 一律丟「unknown operating +system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 +新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 +真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 +`ubuntu-22.04-arm` 與 `windows-11-arm` 加進 smoke 矩陣。 + + ## 本次更新 (2026-08-19) — Wayland 兩個等人拍板的取捨,拍板了 `Progress.md` 上掛著兩個 `DECIDE`:缺的不是工，是決定。兩件事其實是同一個問題犯兩次 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index fac41b3f..05f6b0d4 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,112 @@ # What's New — AutoControl +## What's new (2026-08-20) + +### The Platforms This Project Claims, Now Measured + +The suite ran on `windows-2022` alone for its whole life, plus one Linux +container run. macOS got two commands and nothing else. Wayland had five jobs +reading input back off a real peer; X11 — the older and more widely deployed +of the two Linux paths — had none, and every X11 assertion in the suite was +made against a mock of `python-Xlib`. + +**The suite now runs where the project says it runs.** `pytest-headless` +became an OS matrix: Windows keeps all five Pythons, Linux and macOS carry the +two ends of the range. Linux runs under a real Xvfb rather than Qt's offscreen +platform, because the X11 backend opens a display at import time and offscreen +would hide exactly the breakage this exists to find. It found two real macOS +defects on the first run: + +- `write("\b")` had no key route on macOS, so it fell through to the space + fallback and typed a space where a backspace was asked for. X11 and Wayland + both carry the raw character; macOS was the one that did not. +- `system_profiler` reports a *symbolic* vendor id for Apple's own devices — + `apple_vendor_id`, not a number — and that went straight into a field + documented as four hex digits. Its leading `a` is a valid hex digit, so a + lenient parse turns it into `000a`. + +**X11 input is read back out of a real client.** A new `x11-verification` job +runs against a real Xvfb server with a real window manager, taking ground +truth from other codebases than the subject: `xev`, a real X client that +prints every event delivered to its window; ImageMagick's `import` against a +root painted two asymmetric colours; `xdotool` and `xdpyinfo`. The assertion +worth naming is `synthetic NO` — `XSendEvent` traffic arrives with `YES` and +is discarded by most toolkits, so a backend that quietly stopped driving real +input would still pass any check that only counted events. + +**macOS turned out to be fully testable in CI, contrary to the usual +assumption.** A `macos-14` runner grants *both* Screen Recording and +Accessibility: capture returns real pixels rather than the black rectangle a +refusal produces, `CGEventPost` moves the cursor and the move reads back +exactly, and the AX walk returns real elements. That was measured first and +asserted second, and the probe still refuses to pass while its expectations +table is empty. + +### Window Management Is No Longer Windows-Only + +It was: the facade branched on `sys.platform` and raised everywhere else, +leaving 23 `AC_*` commands and their MCP tools dead on macOS and Linux. It now +goes through a backend seam — Win32, EWMH over `python-Xlib` on X11, Quartz +plus the accessibility API on macOS, and a null fallback that lists nothing +and refuses actions with a reason. + +Two things only a real window manager could show up were wrong first time: + +- **The rectangle is the frame, not the client.** Win32's `GetWindowRect` + returns the frame, and every caller is written against that, so reporting + the client area was off by the decorations on X11 alone — silently, and by + a different amount per window manager. +- **A move has to go through `_NET_MOVERESIZE_WINDOW`.** Under a reparenting + window manager a client's own x/y are relative to its frame, so a direct + `ConfigureWindow` asks in the wrong coordinate space. Asking openbox for + (300, 220) that way landed the window at (302, 260). + +Refusals now raise a class that is both an `AutoControlException` and a +`NotImplementedError`. The GUI tabs and the REST handler already catch the +latter to say "not on this platform"; the executor catches the former, and a +bare `NotImplementedError` slipped past every containment boundary — aborting +a whole script where one action should have been reported as failed. + +### Linux Has an Accessibility Backend + +It had none — the selector fell through to the null one while the capability +matrix claimed "backend tests" for Linux X11. The new backend speaks +**AT-SPI2**, which is a D-Bus protocol rather than a library, and that is what +makes it reachable without a new dependency: `pyatspi` and +`gi.repository.Atspi` are distribution packages built against the system +introspection data and cannot be installed into a virtual environment. + +The D-Bus client written for the portal handshake moved from `linux_wayland/` +to `utils/dbus_client/` to make that possible, and verifying the backend +against a real bus and a real GTK application immediately found a gap in it: +**it could not demarshal signed integers.** The portal never needed one, and +AT-SPI reports a component's extents as four *signed* values, because a window +on a monitor left of or above the primary one is at a negative coordinate — so +the backend could read a tree but not where anything in it was. + +Because AT-SPI is a bus rather than a display protocol, this is the one +capability where Wayland is not the restricted case: the same bus serves both +Linux sessions. + +### The BSDs, and arm64 + +`platform_wrapper` refused to start on anything that was not +win32/cygwin/msys, darwin or linux/linux2, and each of the seven X11 backend +modules carried its own copy of the same Linux-only guard — so a FreeBSD, +OpenBSD or NetBSD desktop, which runs the same X server and the same +`python-Xlib`, could not import the package at all. `sys.platform` was being +compared against literal lists in over a hundred places, so the fix is one +place that decides: `utils/platform_id`, whose `is_x11_unix()` asks the +question those guards were always trying to ask. + +A `freebsd` job boots a real FreeBSD 14 VM inside the runner, imports the X11 +modules under a real X server, and moves the pointer and reads it back. It +covers the platform layer rather than the whole package, because opencv has no +FreeBSD wheel — a limit stated in the job rather than left to be discovered. +`ubuntu-22.04-arm` and `windows-11-arm` join the smoke matrix; `macos-14` was +already arm64. + + ## What's new (2026-08-19) ### Two Wayland Judgement Calls, Settled From fb60702cd076d20ba64e69f6e177efc203fe46cb Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 04:12:17 +0800 Subject: [PATCH 11/30] Poll the macOS modifier state instead of racing it check_key_is_press after a posted shift returned True on one runner round and False on the next: CGEventSourceKeyState reflects the window server's state, and the posted event has to reach it first. Reading once made the probe a coin toss, and a gate that is a coin toss is worse than no gate. Also records what the window probe measured: Quartz reports five on-screen windows on a macos-14 runner and none of them is at the application layer, so the count stays reported rather than asserted. --- test/verify/macos_verify.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py index 2fbf5b6e..e1bf2c44 100644 --- a/test/verify/macos_verify.py +++ b/test/verify/macos_verify.py @@ -33,6 +33,7 @@ import argparse import platform import sys +import time import traceback from typing import Any, Callable, Dict, List, Optional, Tuple @@ -61,12 +62,16 @@ "keyboard-post": True, "accessibility-tree": True, "recorder-absent": True, - # True means "the code answered", not "the runner had windows": a - # macos-14 runner was measured to have none at the application layer, so + # True means "the code answered", not "the runner had windows". Measured: + # Quartz reports 5 on-screen windows on a macos-14 runner and *none* of + # them is at the application layer — they are menu bar and system UI. So # the count is reported and not asserted. "window-management": True, } +#: How long the window server is given to reflect a posted modifier. +KEY_STATE_TIMEOUT = 3.0 + _results: List[Tuple[str, bool, str]] = [] @@ -177,17 +182,31 @@ def probe_mouse_move() -> Outcome: def probe_keyboard() -> Outcome: """Key posting is the most restricted of all; measure, do not assume. - Nothing here has focus, so what is being measured is whether the post is + Nothing here has focus, so what is measured is whether the post is accepted and the modifier state changes — not where the character went. + + The read is polled rather than taken once. Measured on a macos-14 runner, + an immediate ``check_key_is_press`` after the post returned True on one + run and False on the next: ``CGEventSourceKeyState`` reflects the window + server's state, and the posted event has to reach it first. Reading once + makes this probe a coin toss, and a gate that is a coin toss is worse + than no gate. """ from je_auto_control import check_key_is_press, press_keyboard_key, release_keyboard_key press_keyboard_key("shift") try: - held = check_key_is_press("shift") + deadline = time.monotonic() + KEY_STATE_TIMEOUT + held = False + while time.monotonic() < deadline: + held = bool(check_key_is_press("shift")) + if held: + break + time.sleep(0.05) finally: release_keyboard_key("shift") - return Outcome(bool(held), f"check_key_is_press('shift') returned {held!r}") + return Outcome(held, f"check_key_is_press('shift') became {held!r} within " + f"{KEY_STATE_TIMEOUT}s") def probe_accessibility() -> Outcome: From 794220cea5b468912520fdade93ce772f799d6d1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 04:25:20 +0800 Subject: [PATCH 12/30] Get FreeBSD's python-Xlib from pip, not from a flavoured port py311-pip and py311-xlib are not in FreeBSD 14.2's repository: the flavoured port names depend on which Python flavour the release defaults to. Only the interpreter and the X server come from pkg now; python-Xlib is pure Python, so pip installs it without a compiler and at the version the project pins. --- .github/workflows/platform-smoke.yml | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 2e5ab0e8..b45e8dbf 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -82,8 +82,13 @@ jobs: with: release: "14.2" usesh: true + # Only the interpreter and an X server come from pkg. The flavoured + # port names (py311-pip, py311-xlib) depend on which Python + # flavour the release defaults to and were not in 14.2's + # repository at all; python-Xlib is pure Python, so pip is both + # simpler and version-exact. prepare: | - pkg install -y python311 py311-pip py311-xlib xorg-vfbserver xauth + pkg install -y python311 xorg-vfbserver xauth run: | set -eu echo "uname: $(uname -a)" @@ -100,6 +105,13 @@ jobs: assert current_family() == "bsd", current_family() print("platform_id: OK") PROBE + # python-Xlib is the X11 backend's one dependency and is pure + # Python, so pip installs it here without a compiler. Failing + # loudly matters: without it the import below would fail for a + # reason that has nothing to do with the platform guards. + python3.11 -m ensurepip --upgrade + python3.11 -m pip install --quiet "python-Xlib==0.33" + # The guards are what refused to load here. Import the X11 # modules directly, under a real X server, without dragging in # the package's heavy dependencies. From 775e23c902aea8d07c1473535ab2d96b33fbe3bd Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 04:32:06 +0800 Subject: [PATCH 13/30] Stop losing an MCP tool reply when the transport shuts down MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Windows CI caught this as one response where two were expected, on one matrix cell out of nine. It is not flakiness in the test: tools/call runs on a worker thread under stdio, and serve_stdio's `finally` restores the previous writer as soon as the loop reaches EOF. A worker that had not reached its write yet then read `self._writer` and found it already swapped back — so the reply went down the previous connection, or was dropped with "MCP async tool reply with no writer". Two halves. The writer is now captured when the call is dispatched, so a reply always belongs to the transport that accepted the request; and the transport drains its in-flight workers before restoring anything. The drain is bounded, and names the worker that outstayed its welcome rather than hanging shutdown on one bad tool. Both new tests fail without the fix and log the exact production symptom. Widening the matrix is what surfaced this: the race needs a slow enough machine to lose, and windows-2022 on 3.10 was slow enough. --- architecture_explore.md | 14 +-- je_auto_control/utils/mcp_server/server.py | 50 +++++++++- test/unit_test/headless/test_mcp_server.py | 106 +++++++++++++++++++++ 3 files changed, 160 insertions(+), 10 deletions(-) diff --git a/architecture_explore.md b/architecture_explore.md index 2b3bed22..e0adcfd1 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,026 | -| 程式碼總行數 | 139,114 | +| 程式碼總行數 | 139,158 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -487,7 +487,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,132 行。 +> 13 個套件、約 20,176 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -500,7 +500,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,850 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 16,894 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | @@ -699,13 +699,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,850 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（16,894 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | | `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | -| `server.py` | 669 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | +| `server.py` | 713 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | | `http_transport.py` | 323 | MCP 的 HTTP 傳輸。 | | `_client_requests.py` | 217 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | | `_protocol.py` | 165 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | @@ -1018,7 +1018,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | | `gui/` | 89 | 26,542 | -| `utils/mcp_server/` | 20 | 16,850 | +| `utils/mcp_server/` | 20 | 16,894 | | `utils/remote_desktop/` | 56 | 11,835 | | `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,247 | @@ -1038,5 +1038,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 688 | 50,034 | -| **總計** | **1,020** | **139,049** | +| **總計** | **1,020** | **139,093** | diff --git a/je_auto_control/utils/mcp_server/server.py b/je_auto_control/utils/mcp_server/server.py index f85c4122..5c24f943 100644 --- a/je_auto_control/utils/mcp_server/server.py +++ b/je_auto_control/utils/mcp_server/server.py @@ -45,6 +45,10 @@ _TOOL_INVOKE_ERRORS, _TOOLS_CALL_METHOD, ) +#: How long a transport waits for in-flight tool replies before it gives up +#: and shuts down anyway. +WORKER_DRAIN_TIMEOUT = 10.0 + class MCPServer(ClientRequestMixin): """JSON-RPC 2.0 MCP server with a configurable tool registry.""" @@ -71,6 +75,12 @@ def __init__(self, tools: Optional[List[MCPTool]] = None, self._log_bridge = log_bridge self._stop = threading.Event() self._initialized = False + # tools/call runs on a worker thread under the stdio transport, so a + # reply can still be in flight when the loop reaches EOF. Tracking the + # workers is what lets the transport wait for them instead of pulling + # the writer out from under them. + self._workers: List[threading.Thread] = [] + self._workers_lock = threading.Lock() # Connection-scoped state. The stdio transport has exactly one peer, so # a server-wide default is right for it; an HTTP transport has many, and # each request runs on its own thread. Keeping the notifier/writer in @@ -264,6 +274,10 @@ def serve_stdio(self, stdin: Optional[TextIO] = None, if response is not None: self._write_message(out_stream, response) finally: + # Before anything is restored: a tools/call dispatched just before + # EOF is still running, and its reply has nowhere to go once the + # writer is swapped back. + self._join_workers() self._detach_log_bridge_if_configured() self._notifier = prior_notifier self._writer = prior_writer @@ -366,18 +380,48 @@ def _handle_line_safely(self, line: str) -> Optional[str]: def _dispatch_tools_call_async(self, msg_id: Any, params: Dict[str, Any]) -> None: """Run a tools/call on a worker thread; the worker writes the reply.""" + # Captured here rather than read inside the worker. The worker may not + # reach its write until after the transport's `finally` has restored + # the previous writer, and then the reply would go down the previous + # connection or be dropped as "no writer" — a reply belongs to the + # transport that accepted the request. + writer = self._writer + def worker() -> None: payload = self._build_response(msg_id, _TOOLS_CALL_METHOD, params) - writer = self._writer if writer is None: autocontrol_logger.warning( "MCP async tool reply with no writer; dropping %s", msg_id, ) return writer(payload) - threading.Thread( + thread = threading.Thread( target=worker, daemon=True, name=f"MCPCall-{msg_id}", - ).start() + ) + with self._workers_lock: + self._workers = [live for live in self._workers if live.is_alive()] + self._workers.append(thread) + thread.start() + + def _join_workers(self, timeout: float = WORKER_DRAIN_TIMEOUT) -> None: + """Wait for in-flight tool replies before the transport goes away. + + Bounded: a handler that never returns must not keep the process from + shutting down, so this says which worker outstayed its welcome and + gives up rather than hanging. + """ + deadline = time.monotonic() + timeout + with self._workers_lock: + pending = [live for live in self._workers if live.is_alive()] + for thread in pending: + remaining = deadline - time.monotonic() + if remaining <= 0: + autocontrol_logger.warning( + "MCP shutdown: %s is still running; its reply is lost", + thread.name, + ) + continue + thread.join(remaining) def _build_response(self, msg_id: Any, method: Optional[str], params: Dict[str, Any]) -> str: diff --git a/test/unit_test/headless/test_mcp_server.py b/test/unit_test/headless/test_mcp_server.py index a9c8ab58..6ae2f5bf 100644 --- a/test/unit_test/headless/test_mcp_server.py +++ b/test/unit_test/headless/test_mcp_server.py @@ -1751,3 +1751,109 @@ def test_default_registry_lists_core_automation_tools(): "ac_execute_actions", "ac_list_action_commands", } assert expected.issubset(names) + + +# --- shutdown while a tool reply is in flight ------------------------------ + + +def test_a_slow_tool_reply_is_not_lost_at_eof(): + """serve_stdio must not return while a reply is still being written. + + tools/call runs on a worker thread under stdio, and the loop's `finally` + restores the previous writer. A worker that had not reached its write yet + then found ``self._writer`` already swapped back — so the reply went to + the previous transport or was dropped as "no writer". Windows CI caught it + as one response where two were expected; the sleep here makes the window + wide enough to catch every time. + """ + import threading + import time as _time + + started = threading.Event() + + def slow_handler(): + started.set() + _time.sleep(0.5) + return "pong" + + tool = MCPTool(name="slow_tool", description="slow", + input_schema={"type": "object", "properties": {}}, + handler=slow_handler) + server = MCPServer(tools=[tool]) + stdin = io.StringIO( + _request("initialize", msg_id=1, params={"protocolVersion": "x"}) + + "\n" + + _request("tools/call", msg_id=2, + params={"name": "slow_tool", "arguments": {}}) + + "\n") + stdout = io.StringIO() + server.serve_stdio(stdin=stdin, stdout=stdout) + + assert started.is_set(), "the tool never ran" + responses = [_decode(line) for line in stdout.getvalue().splitlines() + if line and '"id":' in line and '"method"' not in line] + assert len(responses) == 2, [response.get("id") for response in responses] + assert responses[-1]["result"]["content"][0]["text"] == "pong" + + +def test_a_reply_goes_to_the_transport_that_accepted_the_request(): + """The writer is captured at dispatch, not read when the worker finishes. + + Reading it later is what let a late reply land on whatever writer the + server had by then — the next connection's, or none at all. + """ + import threading + + release = threading.Event() + delivered = [] + + def blocking_handler(): + release.wait(timeout=5) + return "late" + + tool = MCPTool(name="late_tool", description="late", + input_schema={"type": "object", "properties": {}}, + handler=blocking_handler) + server = MCPServer(tools=[tool]) + server.set_notifier(None) + server._writer = delivered.append + server._dispatch_tools_call_async( + 7, {"name": "late_tool", "arguments": {}}) + + # The transport goes away while the tool is still running. + server._writer = None + release.set() + server._join_workers(timeout=5) + + assert len(delivered) == 1, delivered + assert '"id": 7' in delivered[0] or '"id":7' in delivered[0] + + +def test_shutdown_does_not_wait_forever_for_a_stuck_tool(): + """A handler that never returns must not keep the transport alive. + + The drain is bounded and says which worker outstayed its welcome, rather + than hanging on one bad tool. + """ + import threading + import time as _time + + release = threading.Event() + + def stuck_handler(): + release.wait(timeout=30) + return "eventually" + + tool = MCPTool(name="stuck_tool", description="stuck", + input_schema={"type": "object", "properties": {}}, + handler=stuck_handler) + server = MCPServer(tools=[tool]) + server._writer = lambda payload: None + server._dispatch_tools_call_async( + 1, {"name": "stuck_tool", "arguments": {}}) + try: + started = _time.monotonic() + server._join_workers(timeout=0.2) + assert _time.monotonic() - started < 5 + finally: + release.set() From 48368c636d638b2fa35b2c92c7d17dcb02ff2347 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 04:50:55 +0800 Subject: [PATCH 14/30] Drop the Windows arm64 runner, because it measured a real blocker MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit opencv-python publishes no win_arm64 wheel, so pip falls back to building it from source and CMake cannot configure for ARM64. The job spent twelve minutes failing at that. This is not a CI problem to work around — the package genuinely cannot be installed on Windows arm64 today, so the cell comes out of the matrix and the reason goes into the workflow comment, the capability matrix and Progress.md, with the runner ready to add back when the wheel exists. Linux arm64 is fine: ubuntu-22.04-arm passes on both Python versions, and macos-14 was already arm64. Windows is the only combination blocked. --- .github/workflows/platform-smoke.yml | 21 +++++++++++---------- Progress.md | 19 +++++++++++++++++++ README/WHATS_NEW_zh-CN.md | 2 +- README/WHATS_NEW_zh-TW.md | 2 +- WHATS_NEW.md | 6 ++++-- docs/CAPABILITY_MATRIX.md | 11 +++++++---- 6 files changed, 43 insertions(+), 18 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index b45e8dbf..8d0f00eb 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -15,17 +15,18 @@ jobs: fail-fast: false matrix: # arm64 is not a rounding error on the desktop any more, and the - # dependency set is where it shows: opencv, pillow and cryptography - # all ship native wheels, and a missing arm64 wheel means a source - # build that either takes an hour or fails. macos-14 is already - # arm64; these add the other two. - os: [windows-2022, ubuntu-22.04, macos-14, ubuntu-22.04-arm, - windows-11-arm] + # dependency set is where it shows. macos-14 is already arm64; + # ubuntu-22.04-arm adds Linux, and it passes. + # + # windows-11-arm is deliberately absent, and it was measured rather + # than assumed: opencv-python publishes no win_arm64 wheel, so pip + # falls back to building it from source and CMake cannot configure + # for ARM64. The job spent twelve minutes failing at that, which is + # not a CI problem to work around — the package genuinely cannot be + # installed on Windows arm64 today. Recorded in Progress.md; add the + # runner back when the wheel exists. + os: [windows-2022, ubuntu-22.04, macos-14, ubuntu-22.04-arm] python-version: ["3.10", "3.14"] - exclude: - # windows-11-arm has no 3.10 build available through setup-python. - - os: windows-11-arm - python-version: "3.10" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 diff --git a/Progress.md b/Progress.md index 59019607..f10bde8c 100644 --- a/Progress.md +++ b/Progress.md @@ -50,6 +50,25 @@ --- +## Windows arm64 裝不起來，卡在 opencv-python + +`BLOCKED` — 上游（opencv-python 沒有 win_arm64 wheel） + +`windows-11-arm` 加進 `platform-smoke.yml` 的矩陣跑了一次， +結果是實測而不是推測：**opencv-python 並沒有發 +win_arm64 wheel**，pip 回退到從原碼建，CMake 在 ARM64 上 +configure 不起來，花了十二分鐘失敗。cryptography +也在同一輪裡被拉去建。 + +這不是 CI 設定問題，是**這個套件今天在 +Windows arm64 上裝不起來**。所以那一格已從矩陣 +移除，並把原因寫在 workflow 的註解裡；哪天上游 +發了 wheel，把 runner 加回去就好。 + +**Linux arm64 是好的**——`ubuntu-22.04-arm` 兩個 Python 版本 +都綠，macOS 本來就是 arm64。所以卡住的只有 Windows +這一個組合。 + ## macOS 的錄製器寫了，但接不上去 `TODO` — `osx/record/osx_record.py`、`osx/listener/osx_listener.py` diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index 76958668..ffcfe552 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -61,7 +61,7 @@ AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期 system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 -`ubuntu-22.04-arm` 與 `windows-11-arm` 加進 smoke 矩陣。 +`ubuntu-22.04-arm` 加進 smoke 矩陣且全绿。`windows-11-arm` 试过后拿掉了：opencv-python 根本没发 `win_arm64` wheel，这个包今天在 Windows arm64 上装不起来——是量出来的，已记在 `Progress.md`。 ## 本次更新 (2026-08-19) — Wayland 两个等人拍板的取舍,拍板了 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 09ba0fe2..f95ae303 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -61,7 +61,7 @@ AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期 system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 -`ubuntu-22.04-arm` 與 `windows-11-arm` 加進 smoke 矩陣。 +`ubuntu-22.04-arm` 加進 smoke 矩陣且全綠。`windows-11-arm` 試過後拿掉了：opencv-python 根本沒發 `win_arm64` wheel，這個套件今天在 Windows arm64 上裝不起來——是量出來的，已記在 `Progress.md`。 ## 本次更新 (2026-08-19) — Wayland 兩個等人拍板的取捨,拍板了 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 05f6b0d4..df893fa9 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -103,8 +103,10 @@ A `freebsd` job boots a real FreeBSD 14 VM inside the runner, imports the X11 modules under a real X server, and moves the pointer and reads it back. It covers the platform layer rather than the whole package, because opencv has no FreeBSD wheel — a limit stated in the job rather than left to be discovered. -`ubuntu-22.04-arm` and `windows-11-arm` join the smoke matrix; `macos-14` was -already arm64. +`ubuntu-22.04-arm` joins the smoke matrix and passes; `macos-14` was already +arm64. `windows-11-arm` was tried and removed: opencv-python publishes no +`win_arm64` wheel, so the package cannot be installed there at all today — +measured, not assumed, and recorded in `Progress.md`. ## What's new (2026-08-19) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index 57e57926..c293340e 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -82,10 +82,13 @@ FreeBSD 14 VM to run them, moving the pointer and reading it back. It checks the platform layer rather than the whole package: opencv has no FreeBSD wheel, and a source build would take an hour or fail. -**arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` and -`windows-11-arm` join the smoke matrix. The dependency set is where the -architecture shows — opencv, pillow and cryptography all ship native wheels, -and a missing one means a source build rather than a clean failure. +**arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` joins the +smoke matrix and passes. `windows-11-arm` was tried and removed, on +measurement rather than assumption: **opencv-python publishes no `win_arm64` +wheel**, so pip falls back to building it from source and CMake cannot +configure for ARM64. That is not a CI problem to work around — the package +genuinely cannot be installed on Windows arm64 today, which is recorded in +`Progress.md` with the runner ready to add back when the wheel exists. The accessibility row said `backend tests` for Linux X11 and meant nothing by it: there was no Linux backend at all, and `_build_backend()` fell straight From a42d2185a37fffbcafcf081b391d438175e64a18 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 05:02:54 +0800 Subject: [PATCH 15/30] Keep the FreeBSD probe to the modules that need no Pillow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The screen module imports Pillow at module scope, and Pillow is one of the heavy dependencies this job deliberately does not install — it has no FreeBSD wheel either. Its guard is the same one every other X11 module carries, so dropping it from the probe costs no coverage of the change. What runs instead is the display, keymap, mouse and keyboard modules: a real pointer move on a real X server read back from the server, and an XTest key injection the platform accepts. That is the claim being made. --- .github/workflows/platform-smoke.yml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 8d0f00eb..4b3ffd9b 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -118,25 +118,39 @@ jobs: # the package's heavy dependencies. Xvfb :99 -screen 0 1280x800x24 & sleep 3 + # The display, keymap, mouse and keyboard modules are the ones + # that carried the Linux-only guard, and python-Xlib is all they + # need. The screen module is deliberately not imported here: it + # pulls in Pillow, which is one of the heavy dependencies this + # job does not install, and its guard is the same one. DISPLAY=:99 python3.11 - <<'BACKEND' import sys sys.path.insert(0, ".") from je_auto_control.linux_with_x11.core.utils import ( x11_linux_display, x11_linux_vk, ) + from je_auto_control.linux_with_x11.keyboard import ( + x11_linux_keyboard_control as keyboard, + ) from je_auto_control.linux_with_x11.mouse import ( x11_linux_mouse_control as mouse, ) - from je_auto_control.linux_with_x11.screen import ( - x11_linux_screen as screen, - ) print("display:", x11_linux_display.display) - print("screen size:", screen.size()) + print("keycode for 'a':", x11_linux_vk.x11_linux_key_a) + assert x11_linux_vk.x11_linux_key_a > 0, "no keymap on FreeBSD" + + # A real pointer move on a real X server, read back from the + # server. This is the whole claim: the X11 backend works here. mouse.set_position(321, 123) landed = mouse.position() assert landed == (321, 123), landed print("pointer round-trip on FreeBSD:", landed) - print("keycode for 'a':", x11_linux_vk.x11_linux_key_a) + + # Press and release a key: nothing has focus, so what is checked + # is that XTest accepts the call on this platform at all. + keyboard.press_key(x11_linux_vk.x11_linux_key_a) + keyboard.release_key(x11_linux_vk.x11_linux_key_a) + print("XTest key injection accepted on FreeBSD") BACKEND macos-capabilities: From 949684b2222a9af786f623346ade1f6fe61a73c6 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 05:25:48 +0800 Subject: [PATCH 16/30] Give the FreeBSD VM the dependencies the facade imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importing anything under je_auto_control runs the package facade, which imports Pillow and OpenCV at module scope — so 'just the platform layer' is not something the import system will hand over, and both probes died on PIL before reaching what they were checking. The dependencies come from FreeBSD's own repository rather than from pip: neither publishes a FreeBSD wheel, and building OpenCV from source in a CI VM does not finish. Each package installs on its own line so a name that is not in the repository names itself in the log instead of taking the step down with it — and nothing is skipped quietly, because whatever is genuinely missing still surfaces as an ImportError. The platform_id probe now loads that module by file path instead: the classification is what decides which backend a BSD gets, and it should be checkable without dragging OpenCV in behind it. --- .github/workflows/platform-smoke.yml | 51 ++++++++++++++++++++-------- 1 file changed, 37 insertions(+), 14 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 4b3ffd9b..90616846 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -83,22 +83,43 @@ jobs: with: release: "14.2" usesh: true - # Only the interpreter and an X server come from pkg. The flavoured - # port names (py311-pip, py311-xlib) depend on which Python - # flavour the release defaults to and were not in 14.2's - # repository at all; python-Xlib is pure Python, so pip is both - # simpler and version-exact. + # Importing anything under je_auto_control runs the package + # facade, which imports Pillow and OpenCV at module scope — so + # "just the platform layer" is not a thing the import system will + # give us, and the heavy dependencies have to be present. They + # come from FreeBSD's own repository rather than from pip: neither + # publishes a FreeBSD wheel, and building OpenCV from source in a + # CI VM is not a thing that finishes. + # + # Each package is installed on its own line so a name that is not + # in the repository names itself in the log instead of taking the + # whole step down with it. Nothing is skipped quietly: whatever is + # genuinely missing surfaces as an ImportError below. prepare: | pkg install -y python311 xorg-vfbserver xauth + pkg install -y py311-xlib || echo "MISSING PACKAGE: py311-xlib" + pkg install -y py311-pillow || echo "MISSING PACKAGE: py311-pillow" + pkg install -y py311-numpy || echo "MISSING PACKAGE: py311-numpy" + pkg install -y py311-opencv || echo "MISSING PACKAGE: py311-opencv" + pkg install -y py311-sqlite3 || echo "MISSING PACKAGE: py311-sqlite3" + pkg info | grep -iE "py3.*(xlib|pillow|numpy|opencv)" || true run: | set -eu echo "uname: $(uname -a)" + # Loaded by file path rather than imported: the classification is + # what decides which backend a BSD gets, and it should be + # checkable without dragging the whole facade (and OpenCV) in. python3.11 - <<'PROBE' + import importlib.util import sys - sys.path.insert(0, ".") - from je_auto_control.utils.platform_id import ( - current_family, is_bsd, is_x11_unix, - ) + + spec = importlib.util.spec_from_file_location( + "platform_id", "je_auto_control/utils/platform_id/__init__.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + current_family = module.current_family + is_bsd = module.is_bsd + is_x11_unix = module.is_x11_unix print("sys.platform:", sys.platform) assert sys.platform.startswith("freebsd"), sys.platform assert is_bsd(), "FreeBSD is not recognised as a BSD" @@ -106,12 +127,14 @@ jobs: assert current_family() == "bsd", current_family() print("platform_id: OK") PROBE - # python-Xlib is the X11 backend's one dependency and is pure - # Python, so pip installs it here without a compiler. Failing - # loudly matters: without it the import below would fail for a - # reason that has nothing to do with the platform guards. + # python-Xlib is pure Python, so pip is the reliable route to it + # whatever the port is called. --break-system-packages because + # FreeBSD marks the interpreter's site-packages as externally + # managed, and this VM exists for exactly one run. python3.11 -m ensurepip --upgrade - python3.11 -m pip install --quiet "python-Xlib==0.33" + python3.11 -m pip install --quiet --break-system-packages \ + "python-Xlib==0.33" || \ + python3.11 -m pip install --quiet "python-Xlib==0.33" # The guards are what refused to load here. Import the X11 # modules directly, under a real X server, without dragging in From 47257cd2ca670c00f0d077bc72b579c446ebba49 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 06:29:50 +0800 Subject: [PATCH 17/30] Scope the FreeBSD job to the decision a BSD is needed to answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The job was trying to import the X11 backend, which means importing the package facade, which imports OpenCV and cryptography at module scope. Neither publishes a FreeBSD wheel; installing them from ports pulled a dependency tree that had not finished after fifty minutes, so the run was cancelled. A smoke job cannot cost an hour. What a BSD is uniquely needed to answer is the classification itself: sys.platform really reads freebsd14 there, and is_x11_unix() — the question every relaxed guard now asks — returns True on it. utils/platform_id imports nothing but sys, so it loads by file path with no dependencies at all, and that is what the job runs. Driving real input on a BSD stays uncovered, and Progress.md says so with the five lines that would close it on a machine that has the dependency set. --- .github/workflows/platform-smoke.yml | 123 +++++++++------------------ Progress.md | 27 ++++++ docs/CAPABILITY_MATRIX.md | 12 ++- 3 files changed, 74 insertions(+), 88 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 90616846..f93dd29b 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -62,53 +62,42 @@ jobs: if-no-files-found: warn freebsd: - name: The X11 backend's platform guards on a real FreeBSD + name: The BSD platform decision on a real FreeBSD runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 # The X11 backend was gated on sys.platform being linux/linux2, so it - # refused to import on a FreeBSD desktop that runs the same X server, - # the same python-Xlib and the same code. Relaxing that guard is only - # worth anything if something actually runs it on a BSD, and no hosted - # runner is one — so this boots a real FreeBSD VM inside the runner. + # refused to load on a FreeBSD desktop that runs the same X server, the + # same python-Xlib and the same code. Relaxing that guard is only worth + # something if a BSD actually runs the decision, and no hosted runner + # is one — so this boots a real FreeBSD VM inside the runner. # - # What it checks is the platform layer, not the whole package: the - # heavy dependencies (opencv in particular) have no FreeBSD wheels, so - # a full install would build from source for an hour or fail. The - # guards and the wrapper's routing are the change; they are what runs. + # It checks the *decision*, not the whole backend, and the reason is + # measured rather than assumed: importing anything under + # je_auto_control runs the package facade, which imports OpenCV and + # cryptography at module scope. Neither publishes a FreeBSD wheel, and + # installing them from ports pulled in a dependency tree that had not + # finished after fifty minutes. So utils/platform_id is loaded by file + # path — it imports nothing but sys, which is the point of it being one + # small module — and what a BSD is uniquely needed for is exactly what + # runs here: that sys.platform really looks like this, and that the + # classification every guard now asks answers correctly on it. + # + # What that leaves uncovered is the backend actually driving input on a + # BSD. See Progress.md; it needs a machine with the dependency set on + # it, not a different CI trick. # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: vmactions/freebsd-vm@v1 # NOSONAR githubactions:S7637 with: release: "14.2" usesh: true - # Importing anything under je_auto_control runs the package - # facade, which imports Pillow and OpenCV at module scope — so - # "just the platform layer" is not a thing the import system will - # give us, and the heavy dependencies have to be present. They - # come from FreeBSD's own repository rather than from pip: neither - # publishes a FreeBSD wheel, and building OpenCV from source in a - # CI VM is not a thing that finishes. - # - # Each package is installed on its own line so a name that is not - # in the repository names itself in the log instead of taking the - # whole step down with it. Nothing is skipped quietly: whatever is - # genuinely missing surfaces as an ImportError below. prepare: | - pkg install -y python311 xorg-vfbserver xauth - pkg install -y py311-xlib || echo "MISSING PACKAGE: py311-xlib" - pkg install -y py311-pillow || echo "MISSING PACKAGE: py311-pillow" - pkg install -y py311-numpy || echo "MISSING PACKAGE: py311-numpy" - pkg install -y py311-opencv || echo "MISSING PACKAGE: py311-opencv" - pkg install -y py311-sqlite3 || echo "MISSING PACKAGE: py311-sqlite3" - pkg info | grep -iE "py3.*(xlib|pillow|numpy|opencv)" || true + pkg install -y python311 run: | set -eu echo "uname: $(uname -a)" - # Loaded by file path rather than imported: the classification is - # what decides which backend a BSD gets, and it should be - # checkable without dragging the whole facade (and OpenCV) in. python3.11 - <<'PROBE' import importlib.util import sys @@ -117,64 +106,28 @@ jobs: "platform_id", "je_auto_control/utils/platform_id/__init__.py") module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) - current_family = module.current_family - is_bsd = module.is_bsd - is_x11_unix = module.is_x11_unix + print("sys.platform:", sys.platform) assert sys.platform.startswith("freebsd"), sys.platform - assert is_bsd(), "FreeBSD is not recognised as a BSD" - assert is_x11_unix(), "FreeBSD is not recognised as an X11 unix" - assert current_family() == "bsd", current_family() - print("platform_id: OK") - PROBE - # python-Xlib is pure Python, so pip is the reliable route to it - # whatever the port is called. --break-system-packages because - # FreeBSD marks the interpreter's site-packages as externally - # managed, and this VM exists for exactly one run. - python3.11 -m ensurepip --upgrade - python3.11 -m pip install --quiet --break-system-packages \ - "python-Xlib==0.33" || \ - python3.11 -m pip install --quiet "python-Xlib==0.33" - # The guards are what refused to load here. Import the X11 - # modules directly, under a real X server, without dragging in - # the package's heavy dependencies. - Xvfb :99 -screen 0 1280x800x24 & - sleep 3 - # The display, keymap, mouse and keyboard modules are the ones - # that carried the Linux-only guard, and python-Xlib is all they - # need. The screen module is deliberately not imported here: it - # pulls in Pillow, which is one of the heavy dependencies this - # job does not install, and its guard is the same one. - DISPLAY=:99 python3.11 - <<'BACKEND' - import sys - sys.path.insert(0, ".") - from je_auto_control.linux_with_x11.core.utils import ( - x11_linux_display, x11_linux_vk, - ) - from je_auto_control.linux_with_x11.keyboard import ( - x11_linux_keyboard_control as keyboard, - ) - from je_auto_control.linux_with_x11.mouse import ( - x11_linux_mouse_control as mouse, - ) - print("display:", x11_linux_display.display) - print("keycode for 'a':", x11_linux_vk.x11_linux_key_a) - assert x11_linux_vk.x11_linux_key_a > 0, "no keymap on FreeBSD" - - # A real pointer move on a real X server, read back from the - # server. This is the whole claim: the X11 backend works here. - mouse.set_position(321, 123) - landed = mouse.position() - assert landed == (321, 123), landed - print("pointer round-trip on FreeBSD:", landed) + # The four questions every relaxed guard now asks. Before this + # change the answer to the third was False on exactly this + # platform, and the package refused to import at all. + assert module.is_bsd(), "FreeBSD is not recognised as a BSD" + assert module.is_x11_unix(), "FreeBSD is not an X11 unix" + assert not module.is_windows(), "FreeBSD claimed to be Windows" + assert not module.is_macos(), "FreeBSD claimed to be macOS" + assert module.current_family() == "bsd", module.current_family() - # Press and release a key: nothing has focus, so what is checked - # is that XTest accepts the call on this platform at all. - keyboard.press_key(x11_linux_vk.x11_linux_key_a) - keyboard.release_key(x11_linux_vk.x11_linux_key_a) - print("XTest key injection accepted on FreeBSD") - BACKEND + # The version suffix is the trap: sys.platform is freebsd14 here, + # never a bare "freebsd", so an equality check would match no + # real system at all. + assert sys.platform != "freebsd", ( + "this release stopped carrying a version suffix; the prefix " + "match still works, but the comment explaining why it exists " + "no longer describes reality") + print("platform_id on FreeBSD: OK") + PROBE macos-capabilities: name: What a real macOS runner permits diff --git a/Progress.md b/Progress.md index f10bde8c..266da46c 100644 --- a/Progress.md +++ b/Progress.md @@ -50,6 +50,33 @@ --- +## BSD 上只驗了「判定」，沒驗到「真的驅動輸入」 + +`TODO` — `.github/workflows/platform-smoke.yml` 的 `freebsd` job + +`freebsd` job 在 runner 裡開真的 FreeBSD 14 VM，驗的是 +`utils/platform_id`：`sys.platform` 真的長成 `freebsd14`、 +`is_x11_unix()` 在上面回 True——也就是每個放寬後的守衛 +現在問的那個問題，而這件事只有 BSD 能回答。 + +**沒驗到的是：X11 backend 在 BSD 上真的移滑鼠、真的送 +按鍵。** 原因是量出來的，不是懶：import 任何 +`je_auto_control` 底下的東西都會跑門面，而門面在 +module scope import OpenCV 與 cryptography。這兩個都沒發 FreeBSD +wheel，改用 ports 裝（`py311-opencv`）拉出來的相依樹跑了 +**五十分鐘還沒裝完**，只好取消。一個 smoke job 不能 +花一小時。 + +要補這塊，需要的是一台已經裝好相依套件的真 +FreeBSD（或者一個預先烤好依賴的自訂映像），而不是 +另一個 CI 小技巧。跑的時候把下面這段跑完就算驗到： + +```python +from je_auto_control.linux_with_x11.mouse import x11_linux_mouse_control as m +m.set_position(321, 123) +assert m.position() == (321, 123) +``` + ## Windows arm64 裝不起來，卡在 opencv-python `BLOCKED` — 上游（opencv-python 沒有 win_arm64 wheel） diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index c293340e..e6eadb1f 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -78,9 +78,15 @@ Linux, could not import the package at all. `python-Xlib` was pinned to the backend without its one dependency. The guards now ask `utils/platform_id.is_x11_unix()` — "is this an X11 unix", which is the question they were always trying to ask — and the `freebsd` job boots a real -FreeBSD 14 VM to run them, moving the pointer and reading it back. It checks -the platform layer rather than the whole package: opencv has no FreeBSD -wheel, and a source build would take an hour or fail. +FreeBSD 14 VM to run that decision on a system that is genuinely one. + +It checks the decision and not the backend, for a measured reason: importing +anything under `je_auto_control` runs the package facade, which imports +OpenCV and cryptography at module scope, and neither publishes a FreeBSD +wheel. Installing them from ports pulled a dependency tree that had not +finished after fifty minutes. So `utils/platform_id` is loaded by file path, +and what a BSD is uniquely needed to answer is what runs. Driving real input +on a BSD is still uncovered and is recorded in `Progress.md`. **arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` joins the smoke matrix and passes. `windows-11-arm` was tried and removed, on From 19ea5064fb7e36c1e9478b297e30d39ec484c298 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 11:00:24 +0800 Subject: [PATCH 18/30] Give macOS the recorder it already had the code for OSXRecorder was a complete implementation sitting behind recorder = None, and that was not an oversight: the listener called NSApplication.sharedApplication() at import time, and stopping a recording meant AppHelper.runEventLoop(), a loop that never returns to its caller. Wiring it up would have put both on the path of import je_auto_control. The premise is wrong. A CGEventTap needs a run loop, not an application: create the tap on a dedicated thread, add its source to that thread's run loop, and pump the loop in short CFRunLoopRunInMode slices so a stop flag is honoured between them. Nothing touches AppKit, nothing runs at import, and record() returns immediately. The macOS hotkey backend had been driving a tap exactly this way in the same tree. The tap is listen-only, which is load-bearing rather than a detail: a recorder that consumed events would swallow the input it is recording. Two defects were in that code and only a Mac could show either. Coordinates came from NSEvent.mouseLocation(), a bottom-left origin, while every replay posts into the top-left space osx_mouse uses -- so a click recorded near the top of the screen replayed near the bottom. And modifiers were not recorded at all: macOS sends no key-down for Shift, Control, Option or Command, only a flagsChanged carrying the new flag set, so a recording could not say a modifier was held across what followed. Everything after the capture is platform-neutral and now lives in utils/input_macro/recorder_base.py, which both backends subclass. A second hand-written copy of the queue and timeline shaping would diverge silently, and it would surface as a recording made on one OS replaying wrongly on the other. Verifying that end to end turned up a defect that was never macOS-specific: replay_timeline's dispatch table held the run_sequence DSL's vocabulary and the recorders emit their own, and the two sets were disjoint. Feeding stop_record_timeline() to replay_timeline() -- the pipeline the docstrings and the ac_record_stop_timeline tool both prescribe -- matched no handler, replayed an empty session, and returned every event as played. The macos-capabilities job records a real session on a real window server now, and the README platform table stops claiming gaps the code does not have: recording on macOS, and window management on macOS and Linux/X11, which landed earlier on this branch without the table being updated. --- CHANGELOG.md | 40 +++ Progress.md | 15 - README.md | 9 +- README/README_zh-CN.md | 8 +- README/README_zh-TW.md | 8 +- WHATS_NEW.md | 64 ++++ architecture_explore.md | 50 +-- docs/CAPABILITY_MATRIX.md | 28 +- je_auto_control/cli.py | 3 - je_auto_control/osx/listener/osx_listener.py | 331 +++++++++++++----- je_auto_control/osx/record/osx_record.py | 57 ++- .../utils/exception/exception_tags.py | 4 +- je_auto_control/utils/input_macro/__init__.py | 8 +- .../utils/input_macro/input_macro.py | 44 ++- .../utils/input_macro/recorder_base.py | 169 +++++++++ .../utils/mcp_server/tools/_factories.py | 3 +- .../windows/record/win32_input_hook.py | 28 +- .../windows/record/win32_record.py | 125 ++----- je_auto_control/wrapper/_platform_osx.py | 5 +- .../wrapper/auto_control_record.py | 17 +- test/unit_test/headless/test_osx_input_tap.py | 227 ++++++++++++ test/unit_test/headless/test_recorder_base.py | 290 +++++++++++++++ test/verify/macos_verify.py | 61 +++- 23 files changed, 1271 insertions(+), 323 deletions(-) create mode 100644 je_auto_control/utils/input_macro/recorder_base.py create mode 100644 test/unit_test/headless/test_osx_input_tap.py create mode 100644 test/unit_test/headless/test_recorder_base.py diff --git a/CHANGELOG.md b/CHANGELOG.md index e630b6c6..beee31cb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,22 @@ only when documented here with a migration path. ### Added +- **The macOS recorder works.** `record()`, `stop_record()`, + `stop_record_timeline()`, the `AC_record*` commands, the `ac_record_*` MCP + tools and `je_auto_control record` all run on macOS now; they used to refuse + outright with "Cannot use recorder on macOS". Capture goes through a + listen-only Quartz `CGEventTap` on its own thread + (`je_auto_control.osx.listener.osx_listener.OSXInputTap`), so it records + presses, releases, the wheel and per-event timing, exactly as the Windows + hook does. Requires Accessibility permission; without it the tap raises + `AutoControlRecordException` naming the permission — where the facade's + `record()` logs it, as it does every other backend's start failure — rather + than starting a session that silently records nothing. +- `je_auto_control.utils.input_macro.recorder_base` — the platform-neutral + half of recording: `timeline()`, `legacy_action_queue()` and the + `InputRecorder` base the Windows and macOS recorders now share. + `timeline` keeps working when imported from + `je_auto_control.windows.record.win32_input_hook`, where it used to live. - Cross-platform window management. The 23 `AC_*` window commands and their MCP tools now work on macOS and Linux/X11 as well as Windows, through a backend seam (`je_auto_control.wrapper.window_backends`). Wayland remains @@ -129,6 +145,10 @@ only when documented here with a migration path. ### Changed +- **`macos_record_error_message` now names a permission, not a platform.** It + read "Cannot use recorder on macOS", which described a limitation that no + longer exists; it now names the Accessibility grant that recording actually + needs, and is raised from the event tap rather than from the wrapper. - **The `xdg-desktop-portal` capture tier no longer needs `gdbus` installed.** It speaks D-Bus directly, so `linux_wayland.portal.is_available()` now reports whether a session bus address is set rather than whether the `gdbus` @@ -266,6 +286,26 @@ only when documented here with a migration path. ### Fixed +- **A recorded timeline replayed nothing.** `replay_timeline`'s dispatch table + held the `run_sequence` DSL's vocabulary (`press` / `click` / `key`) and the + recorders emit their own (`key_down` / `mouse_up` / `scroll`), and the two + were disjoint — so `stop_record_timeline()` fed to `replay_timeline()`, the + pipeline both the docstrings and the `ac_record_stop_timeline` tool + prescribe, matched no handler, replayed an empty session, and still returned + every event as played. The recorder ops dispatch now, and the wheel reads + `delta` as well as `value` (reading only `value` fell back to the default of + one notch, so a three-notch scroll down replayed as one notch the other + way). Affects every platform, not only macOS. +- macOS: recorded mouse coordinates were mirrored vertically. The listener + read `NSEvent.mouseLocation()`, whose origin is the bottom-left of the + display, while every replay posts into the top-left space `osx_mouse` uses — + so a click recorded near the top of the screen replayed near the bottom. It + now reads `CGEventGetLocation`, which is already in the space the replay + posts into. +- macOS: modifier keys were not recorded at all. macOS sends no key-down for + Shift, Control, Option or Command, only a `flagsChanged` event carrying the + new flag set, so a recording could not say a modifier was held across the + actions that followed. They are reconstructed from the flags now. - macOS: `write()` typed a space instead of a backspace, because `"\b"` had no route in the macOS key table and fell through to the space fallback. - macOS: USB enumeration returned `apple_vendor_id` in `vendor_id`, a field diff --git a/Progress.md b/Progress.md index 266da46c..e39e6b07 100644 --- a/Progress.md +++ b/Progress.md @@ -96,21 +96,6 @@ Windows arm64 上裝不起來**。所以那一格已從矩陣 都綠，macOS 本來就是 arm64。所以卡住的只有 Windows 這一個組合。 -## macOS 的錄製器寫了，但接不上去 - -`TODO` — `osx/record/osx_record.py`、`osx/listener/osx_listener.py` - -`OSXRecorder` 是完整實作，但 `wrapper/_platform_osx.py` 裡寫的是 -`recorder = None`，所以它永遠不會被選到。這不是疏失： -`osx_listener.py` 在 **import 時**就呼叫 `NSApplication.sharedApplication()`， -而停止錄製要靠 `AppHelper.runEventLoop()`，那是一個會卡住呼叫緒的 -事件迴圈。直接接上去會把這兩件事都搬進 `import je_auto_control` -的路徑上，那是回歸而不是修好。 - -要接上去得先把 listener 改成：不在 import 時建立 NSApplication， -且把 run loop 放到自己的執行緒。`docs/CAPABILITY_MATRIX.md` 的 Recorder -macOS 格寫的是 `unavailable`，跟現狀一致。 - ## `mouse_scroll` 的方向在三個平台上不是同一回事 `DECIDE` — `wrapper/auto_control_mouse.py::mouse_scroll` diff --git a/README.md b/README.md index 162a707d..3b0aed1c 100644 --- a/README.md +++ b/README.md @@ -239,12 +239,17 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Platform | Backend | Input | Screen capture | Recording | Window management | |---|---|:---:|:---:|:---:|:---:| | Windows 10 / 11 | Win32 ctypes (+ optional Interception driver) | ✅ | ✅ | ✅ | ✅ | -| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | -| Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ❌ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | +| Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | libei via the desktop portal, or ydotool / wtype + a capture tool | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +¹ macOS recording captures through a Quartz event tap and needs +**Accessibility** permission (System Settings → Privacy & Security → +Accessibility). Without it recording raises and names the permission +rather than returning an empty session. + Wayland input falls back to the `ydotool` CLI wherever libei is not reachable, and that fallback needs **ydotool 1.0 or newer**. Every argument AutoControl builds arrived in that release; 0.1.x — which is what Debian diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index b90b3202..67028de3 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -228,12 +228,16 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | 平台 | 后端 | 输入 | 屏幕捕获 | 录制 | 窗口管理 | |---|---|:---:|:---:|:---:|:---:| | Windows 10 / 11 | Win32 ctypes（可选 Interception 驱动） | ✅ | ✅ | ✅ | ✅ | -| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | -| Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ❌ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | +| Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | 经桌面 portal 的 libei，或 ydotool／wtype ＋ 截图工具 | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +¹ macOS 的录制走 Quartz event tap，需要**辅助功能**权限 +（系统设置 → 隐私与安全性 → 辅助功能）。没有授权时会直接抛出并指名 +缺的是哪个权限，而不是安静地录到一个空的 session。 + Wayland 的输入在 libei 走不通时会退回 `ydotool` CLI，而这条退路需要 **ydotool 1.0 以上**。AutoControl 送的每一个参数都是那一版才有的；0.1.x （Debian bookworm 与目前所有 Ubuntu 仍以这个名字提供，Debian trixie 则根本没有） diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index 3cd13826..f3a8a1df 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -229,12 +229,16 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | 平台 | 後端 | 輸入 | 螢幕擷取 | 錄製 | 視窗管理 | |---|---|:---:|:---:|:---:|:---:| | Windows 10 / 11 | Win32 ctypes（可選 Interception 驅動） | ✅ | ✅ | ✅ | ✅ | -| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | -| Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ❌ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | +| Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | 經桌面 portal 的 libei，或 ydotool／wtype ＋ 擷取工具 | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +¹ macOS 的錄製走 Quartz event tap，需要**輔助使用**權限 +（系統設定 → 隱私權與安全性 → 輔助使用）。沒有授權時會直接拋出並指名 +缺的是哪個權限，而不是安靜地錄到一個空的 session。 + Wayland 的輸入在 libei 走不通時會退回 `ydotool` CLI，而這條退路需要 **ydotool 1.0 以上**。AutoControl 送的每一個參數都是那一版才有的；0.1.x （Debian bookworm 與目前所有 Ubuntu 仍以這個名字提供，Debian trixie 則根本沒有） diff --git a/WHATS_NEW.md b/WHATS_NEW.md index df893fa9..ac35e234 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -42,6 +42,70 @@ exactly, and the AX walk returns real elements. That was measured first and asserted second, and the probe still refuses to pass while its expectations table is empty. +### The macOS Recorder Was Written, Unreachable, and Wrong + +`OSXRecorder` had been a complete implementation for as long as +`wrapper/_platform_osx.py` had said `recorder = None`, and that was not an +oversight. The listener called `NSApplication.sharedApplication()` **at import +time**, and stopping a recording meant `AppHelper.runEventLoop()` — a loop +that never returns to its caller. Wiring it up would have put both on the path +of `import je_auto_control`, which is a regression, not a fix. So `record()`, +`stop_record()` and `je_auto_control record` all refused on macOS with +"Cannot use recorder on macOS", and the capability matrix said `unavailable`. + +**The premise was wrong: a `CGEventTap` needs a run loop, not an +application.** Create the tap on a dedicated thread, add its source to *that* +thread's run loop, and pump the loop in short `CFRunLoopRunInMode` slices so a +stop flag is honoured between them. Nothing touches AppKit, nothing runs at +import, and `record()` returns immediately. The macOS hotkey backend had been +driving a tap exactly this way in the same tree the whole time. + +The tap is **listen-only**, which is load-bearing rather than a detail: a +recorder that consumed events would swallow the very input it is recording, so +the user's clicks would stop working the moment recording started. + +Two defects were sitting in that code, and only a Mac could show either: + +- **Recorded clicks were mirrored vertically.** Coordinates came from + `NSEvent.mouseLocation()`, whose origin is the **bottom-left** of the + display, while every replay posts into the top-left space `osx_mouse` uses. + A click recorded near the top of the screen replayed near the bottom, with + no error anywhere — the same silent shape as `write("\b")` typing a space. + It reads `CGEventGetLocation` now, which is already the replay's space. +- **Modifiers were not recorded at all.** macOS sends no key-down for Shift, + Control, Option or Command; it sends one `flagsChanged` event carrying the + new flag set. A recording therefore could not say a modifier was held across + the actions that followed — which is one of the two reasons the timeline + exists. They are reconstructed from the flag bits now. + +**The half that is not platform-specific stopped being copied.** Everything +after the capture — the down-events-only queue the executor is handed, the +`delta_ms` timeline a replay consumes, the mouse-only and keyboard-only +filters — is one implementation in +`utils/input_macro/recorder_base.py`, and both backends subclass it. A second +hand-written copy of that shaping would have diverged silently, and the way it +would surface is a recording made on one OS replaying wrongly on the other. + +**And the recording had nowhere to go.** Verifying the new backend end to end +turned up a defect that was never macOS-specific: `replay_timeline`'s dispatch +table held the `run_sequence` DSL's vocabulary — `press`, `click`, `key` — +while every recorder emits its own — `key_down`, `mouse_up`, `scroll`. The two +sets were **disjoint**. So `stop_record_timeline()` handed to +`replay_timeline()`, which is the pipeline the docstrings and the +`ac_record_stop_timeline` tool description both prescribe, matched no handler +at all: it replayed an empty session and returned the full event count as +played. The one op that did match, `scroll`, read a key the recorder does not +write, so it fell back to a single notch in the default direction. Both are +fixed, on every platform. + +**It is verified on a real window server, not against a fake.** The +`macos-capabilities` job posts a move, a click and a keypress through the +public API while recording, and asserts they come back out of the tap with the +release and with the coordinates they were posted at. Decoding is unit-tested +separately against genuine `CGEventCreate*` events, which needs no +Accessibility grant — so the parts that can be tested without a permission +are, and the one part that cannot is where the permission is measured. + ### Window Management Is No Longer Windows-Only It was: the facade branched on `sys.platform` and raised everywhere else, diff --git a/architecture_explore.md b/architecture_explore.md index e0adcfd1..f7c40078 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,8 +19,8 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,026 | -| 程式碼總行數 | 139,158 | +| Python 模組總數（含周邊子專案） | 1,027 | +| 程式碼總行數 | 139,420 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -154,7 +154,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | ---: | --- | | `je_auto_control/__init__.py` | 1,970 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | | `je_auto_control/__main__.py` | 70 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | -| `je_auto_control/cli.py` | 326 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/cli.py` | 323 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | | `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | | `je_auto_control/utils/deprecation.py` | 35 | 公開 API 的一致性棄用警告。 | @@ -168,19 +168,19 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | ---: | --- | | `wrapper/platform_wrapper.py` | 73 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/_platform_windows.py` | 325 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 155 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_osx.py` | 156 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | | `wrapper/_platform_wayland.py` | 57 | Wayland 後端組裝（libei／ydotool／grim）。 | | `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | | `wrapper/auto_control_keyboard.py` | 273 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | | `wrapper/auto_control_screen.py` | 97 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | -| `wrapper/auto_control_record.py` | 106 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | +| `wrapper/auto_control_record.py` | 107 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | | `wrapper/auto_control_window.py` | 278 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | ### 5.3 平台後端 -#### Windows（`windows/`，23 檔／1,995 行） +#### Windows（`windows/`，23 檔／1,894 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -189,8 +189,8 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `core/utils/win32_keypress_check.py` | 21 | `GetAsyncKeyState` 按鍵狀態查詢。 | | `mouse/win32_ctype_mouse_control.py` | 220 | 滑鼠事件產生（含多螢幕絕對座標換算）。 | | `keyboard/win32_ctype_keyboard_control.py` | 55 | 鍵盤事件產生。 | -| `record/win32_input_hook.py` | 223 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | -| `record/win32_record.py` | 126 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）。 | +| `record/win32_input_hook.py` | 207 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | +| `record/win32_record.py` | 41 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）；整形本體與 macOS 共用 `utils/input_macro/recorder_base.py`。 | | `screen/win32_screen.py` | 89 | 螢幕尺寸與像素讀取。**每支 Win32 函式都明寫 argtypes/restype**（HDC 是指標寬度，走預設的 c_int 會截斷，錯誤會沉默地擴散到 GetPixel／ReleaseDC），並持有自己的 user32／gdi32 handle。import 時呼叫 `SetProcessDPIAware()`——**行程層級且不可還原**，實體↔邏輯座標換算請走 `utils/monitor_layout`。 | | `window/windows_window_manage.py` | 366 | 視窗列舉／聚焦／關閉／最小化／幾何／所屬行程 PID／投遞式輸入（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | | `message/window_message.py` | 97 | 直接對視窗送 `WM_*` 訊息（背景輸入）。 | @@ -198,7 +198,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | | `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | -#### macOS（`osx/`，17 檔／761 行） +#### macOS（`osx/`，17 檔／907 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -206,8 +206,8 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `mouse/osx_mouse.py` | 137 | Quartz `CGEvent` 滑鼠事件。 | | `keyboard/osx_keyboard.py` | 129 | Quartz 鍵盤事件。 | | `keyboard/osx_keyboard_check.py` | 24 | 按鍵狀態查詢。 | -| `listener/osx_listener.py` | 96 | `CGEventTap` 監聽。 | -| `record/osx_record.py` | 52 | 錄製（CLI `record` 於 macOS 明確不支援）。 | +| `listener/osx_listener.py` | 253 | 專屬執行緒上的 listen-only `CGEventTap`＋自己的 `CFRunLoopRunInMode` 切片；不在 import 時建 `NSApplication`，也不用會卡住呼叫緒的 `AppHelper.runEventLoop()`。修飾鍵由 `flagsChanged` 的旗標還原成 press／release，座標取 `CGEventGetLocation`（左上原點，與重播送出的座標同一空間）。 | +| `record/osx_record.py` | 41 | 錄製。捕捉後的整形（舊版按下事件 Queue、時間軸、只錄滑鼠／只錄鍵盤）走共用的 `utils/input_macro/recorder_base.py`。 | | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | @@ -265,7 +265,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,655 行。 +> 24 個套件、約 12,870 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -278,7 +278,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/deterministic/` | 96 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | | `utils/executor/` | 9,075 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 136 | action list 的單步除錯器與追蹤器 | -| `utils/input_macro/` | 127 | 定時輸入事件重播與宣告式輸入序列 DSL | +| `utils/input_macro/` | 342 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | | `utils/json/` | 74 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | | `utils/json_store/` | 61 | JSON 字典檔持久化的共用小工具（內部管線） | | `utils/loop_guard/` | 140 | 機械式卡死迴圈偵測（agent loop 用） | @@ -296,7 +296,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.2 框架基礎設施 -> 14 個套件、約 2,637 行。 +> 14 個套件、約 2,639 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -305,7 +305,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | | `utils/dbus_client/` | 680 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | -| `utils/exception/` | 208 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/exception/` | 210 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | | `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | | `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | @@ -487,7 +487,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,176 行。 +> 13 個套件、約 20,177 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -500,7 +500,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,894 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 16,895 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | @@ -699,7 +699,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,894 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（16,895 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -1018,14 +1018,14 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | | `gui/` | 89 | 26,542 | -| `utils/mcp_server/` | 20 | 16,894 | +| `utils/mcp_server/` | 20 | 16,895 | | `utils/remote_desktop/` | 56 | 11,835 | | `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,247 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,363 | | `utils/accessibility/` | 13 | 2,818 | -| `wrapper/` | 3,040 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | -| `windows/` | 23 | 1,995 | +| `wrapper/` | 3,042 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | +| `windows/` | 23 | 1,894 | | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | | `linux_with_x11/` | 19 | 1,196 | @@ -1034,9 +1034,9 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/ocr/` | 9 | 1,112 | | `utils/usbip/` | 5 | 920 | | `utils/assertion/` | 3 | 863 | -| `osx/` | 17 | 761 | +| `osx/` | 17 | 907 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 688 | 50,034 | -| **總計** | **1,020** | **139,093** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 689 | 50,253 | +| **總計** | **1,021** | **139,355** | diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index e6eadb1f..0ac53406 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -11,7 +11,7 @@ without a compatibility window. | Image and anchor locators | beta | CI | CI | implementation | implementation | | Accessibility locator | beta | CI | CI/AT-SPI | CI/AT-SPI | CI (tree read) | | Window management | beta | CI | CI/openbox | unavailable | CI (listing) | -| Recorder | beta | CI | implementation | unavailable | unavailable | +| Recorder | beta | CI | implementation | unavailable | CI | | Reports, trace, failure bundle | stable | CI | CI | CI | platform-neutral | | REST, MCP, scheduler | beta | CI | CI | CI | platform-neutral | | Remote desktop / WebRTC | beta | tests | tests | tests | tests | @@ -65,6 +65,32 @@ primary one, where the compositor's plane starts at a negative coordinate and a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of that monitor. +The recorder row said `unavailable` for macOS while the code for one sat in +the tree unused, and the reason was real rather than an oversight: the old +listener built an `NSApplication` at import time and stopped recording with +`AppHelper.runEventLoop()`, a loop that never returns to its caller. Wiring +that up would have put both on the path of `import je_auto_control`, so +`wrapper/_platform_osx.py` set `recorder = None` instead. + +Neither was necessary. A `CGEventTap` needs a **run loop**, not an +application: the tap is created on a dedicated thread, its source is added to +that thread's run loop, and the loop is pumped in short `CFRunLoopRunInMode` +slices so a stop flag is honoured between them — the same shape the macOS +hotkey backend already used. The tap is listen-only, because a recorder that +consumed events would swallow the input it is recording. This row says `CI` +because the `macos-capabilities` job records a real session on a real window +server: it posts a move, a click and a keypress through the public API and +asserts they come back out of the tap with the release and the coordinates +they were posted at. + +Two defects were in that code and only a Mac could show them. Coordinates came +from `NSEvent.mouseLocation()`, whose origin is the **bottom-left** of the +display, while every replay posts into the top-left space `osx_mouse` uses — +so a click recorded near the top of the screen replayed near the bottom. And +modifiers were not recorded at all: macOS sends no key-down for Shift, +Control, Option or Command, only a `flagsChanged` event carrying the new flag +set, so a recording could not say a modifier was held across what followed. + The table has four columns because those are the four desktops with their own backend, not because they are the only supported systems. Two more axes now have CI behind them: diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py index 6864efff..ff46edca 100644 --- a/je_auto_control/cli.py +++ b/je_auto_control/cli.py @@ -116,9 +116,6 @@ def cmd_fmt(args: argparse.Namespace) -> int: def cmd_record(args: argparse.Namespace) -> int: """Record mouse/keyboard input into an action file.""" - if sys.platform == "darwin": - sys.stderr.write("record is not supported on macOS\n") - return 1 from je_auto_control.wrapper.auto_control_record import record_to_json stop_event = threading.Event() if args.duration is None: diff --git a/je_auto_control/osx/listener/osx_listener.py b/je_auto_control/osx/listener/osx_listener.py index f6fe4e61..9995cd1f 100644 --- a/je_auto_control/osx/listener/osx_listener.py +++ b/je_auto_control/osx/listener/osx_listener.py @@ -1,96 +1,253 @@ +"""macOS keyboard/mouse capture over a Quartz event tap, on its own thread. + +This module used to build an ``NSApplication`` **at import time** and stop +recording by way of ``AppHelper.runEventLoop()``, a loop that never returns to +its caller. Both sat on the path of ``import je_auto_control``, which is why +``wrapper/_platform_osx.py`` set ``recorder = None`` and macOS shipped without +a recorder at all rather than take that regression. + +Neither is necessary. A ``CGEventTap`` needs a **run loop**, not an +application: create the tap on a dedicated thread, add its source to *that* +thread's run loop, and pump the loop in short ``CFRunLoopRunInMode`` slices so +the stop flag is honoured between them. Nothing touches AppKit, nothing runs +at import, and the caller is never blocked. The macOS hotkey backend in +``utils/hotkey/backends/macos_backend.py`` already drives a tap this way; this +is the same shape. + +Two properties are deliberate and both are load-bearing: + +* **The tap is listen-only.** A recorder that consumed events would swallow + the very input it is recording, so the user's clicks would stop working the + moment recording started. +* **Modifiers are reconstructed from ``flagsChanged``.** macOS sends no + key-down/key-up for Shift, Control, Option or Command; it sends one event + carrying the new flag set. Without decoding that, a recording could not say + a modifier was held across the actions that followed. + +Coordinates come from ``CGEventGetLocation``, whose origin is the top-left of +the display, which is the space ``osx_mouse`` posts into. The previous +listener read ``NSEvent.mouseLocation()``, a **bottom-left** origin, so every +recorded click was mirrored vertically against where a replay would put it. + +Requires Accessibility permission (System Settings -> Privacy & Security -> +Accessibility). Without it ``CGEventTapCreate`` returns ``None`` and +:meth:`OSXInputTap.start` raises rather than recording silence. + +**Everything typed while recording is captured, passwords included.** Callers +must treat the result as sensitive. +""" import sys -from queue import Queue +import threading +import time +from typing import Any, Dict, List, Optional, Tuple -from je_auto_control.utils.exception.exception_tags import osx_import_error_message -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.exception.exception_tags import ( + macos_record_error_message, osx_import_error_message, +) +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlRecordException, +) +from je_auto_control.utils.input_macro.recorder_base import MAX_EVENTS +from je_auto_control.utils.logging.logging_instance import autocontrol_logger # === 平台檢查 Platform Check === # 僅允許在 macOS (Darwin) 環境執行，否則拋出例外 if sys.platform not in ["darwin"]: raise AutoControlException(osx_import_error_message) -from Cocoa import ( - NSApplication, - NSEvent, - NSEventMaskKeyDown, - NSEventMaskLeftMouseDown, - NSEventMaskRightMouseDown, - NSObject, +import Quartz + +#: How long one run-loop slice runs before the stop flag is re-checked. +POLL_SECONDS = 0.1 + +#: How long :meth:`OSXInputTap.start` waits for the tap to come up or fail. +START_TIMEOUT = 5.0 + +#: Quartz event type -> ``(op, button)``. ``otherMouse`` is the middle button +#: for every mouse this project addresses. +_MOUSE_EVENTS = { + Quartz.kCGEventLeftMouseDown: ("mouse_down", "left"), + Quartz.kCGEventLeftMouseUp: ("mouse_up", "left"), + Quartz.kCGEventRightMouseDown: ("mouse_down", "right"), + Quartz.kCGEventRightMouseUp: ("mouse_up", "right"), + Quartz.kCGEventOtherMouseDown: ("mouse_down", "middle"), + Quartz.kCGEventOtherMouseUp: ("mouse_up", "middle"), +} + +#: Modifier keycode -> the flag bit that says it is currently held. macOS +#: reports modifiers only as a flag set, so the keycode alone cannot say +#: whether an event was a press or a release; the bit can. +_MODIFIER_FLAG = { + 54: Quartz.kCGEventFlagMaskCommand, # right command + 55: Quartz.kCGEventFlagMaskCommand, # command + 56: Quartz.kCGEventFlagMaskShift, # shift + 57: Quartz.kCGEventFlagMaskAlphaShift, # caps lock + 58: Quartz.kCGEventFlagMaskAlternate, # option + 59: Quartz.kCGEventFlagMaskControl, # control + 60: Quartz.kCGEventFlagMaskShift, # right shift + 61: Quartz.kCGEventFlagMaskAlternate, # right option + 62: Quartz.kCGEventFlagMaskControl, # right control + 63: Quartz.kCGEventFlagMaskSecondaryFn, # fn +} + +#: ``CGEventMaskBit`` is a C macro rather than a symbol, so the shift is +#: written out — the same form the macOS hotkey backend uses. +_TAP_MASK = sum( + 1 << event for event in + (Quartz.kCGEventKeyDown, Quartz.kCGEventKeyUp, + Quartz.kCGEventFlagsChanged, Quartz.kCGEventScrollWheel, + *_MOUSE_EVENTS) ) -from PyObjCTools import AppHelper - -# === 全域事件記錄 Queue Global event record queue === -record_queue = Queue() - -# 建立 NSApplication 實例 Create NSApplication instance -app = NSApplication.sharedApplication() - - -class AppDelegate(NSObject): - """ - AppDelegate - 應用程式委派類別 - - 負責在應用程式啟動後註冊全域事件監聽器 - """ - - def applicationDidFinishLaunching_(self, notification): # noqa: N802 # reason: ObjC selector signature - """ - 註冊全域事件監聽器 - Register global event monitors - """ - NSEvent.addGlobalMonitorForEventsMatchingMask_handler_( - NSEventMaskKeyDown, keyboard_handler - ) - NSEvent.addGlobalMonitorForEventsMatchingMask_handler_( - NSEventMaskLeftMouseDown, mouse_left_handler - ) - NSEvent.addGlobalMonitorForEventsMatchingMask_handler_( - NSEventMaskRightMouseDown, mouse_right_handler - ) - - -def mouse_left_handler(event) -> None: - """ - 滑鼠左鍵事件處理器 - Mouse left button handler - """ - loc = NSEvent.mouseLocation() - record_queue.put(("AC_mouse_left", loc.x, loc.y)) - - -def mouse_right_handler(event) -> None: - """ - 滑鼠右鍵事件處理器 - Mouse right button handler - """ - loc = NSEvent.mouseLocation() - record_queue.put(("AC_mouse_right", loc.x, loc.y)) - - -def keyboard_handler(event) -> None: - """ - 鍵盤事件處理器 - Keyboard event handler - """ - keycode = int(event.keyCode()) - if keycode == 98: # 特殊情況：忽略 keycode 98 - return - record_queue.put(("AC_type_keyboard", keycode)) - - -def osx_record() -> None: - """ - 開始錄製事件 - Start recording events - """ - delegate = AppDelegate.alloc().init() - app.setDelegate_(delegate) - AppHelper.runEventLoop() - - -def osx_stop_record() -> Queue: - """ - 停止錄製並回傳事件 Queue - Stop recording and return event queue - """ - return record_queue \ No newline at end of file + +# A tap that takes too long is disabled by the window server rather than +# allowed to slow input down; re-enabling it is the tap owner's job. +_TAP_DISABLED = (Quartz.kCGEventTapDisabledByTimeout, + Quartz.kCGEventTapDisabledByUserInput) + + +class OSXInputTap: + """Captures keyboard and mouse events, with releases, wheel and timing.""" + + def __init__(self, max_events: int = MAX_EVENTS) -> None: + self.events: List[Dict[str, Any]] = [] + self.started = time.monotonic() + self.error: Optional[str] = None + self.max_events = int(max_events) + self._stop = threading.Event() + self._ready = threading.Event() + self._thread: Optional[threading.Thread] = None + + # -- public ------------------------------------------------------------ + def start(self) -> None: + """Create the tap and record. Raises if the tap cannot be created.""" + self._stop.clear() + self._ready.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + self._ready.wait(timeout=START_TIMEOUT) + if self.error: + raise AutoControlRecordException(self.error) + + def stop(self) -> List[Dict[str, Any]]: + """Stop recording and return the raw events.""" + self._stop.set() + thread, self._thread = self._thread, None + if thread is not None: + # A run-loop slice cannot be interrupted part-way, so allow for + # several: a thread left running keeps a live tap on the session. + thread.join(timeout=POLL_SECONDS * 10) + if thread.is_alive(): + autocontrol_logger.error( + "recorder thread did not stop within %.1fs", + POLL_SECONDS * 10) + return self.events + + # -- tap thread -------------------------------------------------------- + def _run(self) -> None: + from CoreFoundation import CFRunLoopRunInMode, kCFRunLoopDefaultMode + + tap = source = run_loop = None + try: + tap = Quartz.CGEventTapCreate( + Quartz.kCGSessionEventTap, Quartz.kCGHeadInsertEventTap, + # Listen-only: a recorder that consumed events would swallow + # the input it is recording. + Quartz.kCGEventTapOptionListenOnly, _TAP_MASK, + self._callback, None, + ) + if tap is None: + autocontrol_logger.error( + "CGEventTapCreate returned None - Accessibility not granted") + self.error = macos_record_error_message + return + source = Quartz.CFMachPortCreateRunLoopSource(None, tap, 0) + run_loop = Quartz.CFRunLoopGetCurrent() + Quartz.CFRunLoopAddSource(run_loop, source, kCFRunLoopDefaultMode) + Quartz.CGEventTapEnable(tap, True) + except Exception as error: # noqa: BLE001 # reason: see below + # Anything at all here has to reach start(), which is blocked on + # _ready: a thread that dies quietly leaves the caller believing + # it is recording, and the session comes back empty with no + # explanation of why. + autocontrol_logger.error("recorder start failed: %r", error) + self.error = f"could not create the macOS event tap: {error!r}" + return + finally: + self._ready.set() + try: + while not self._stop.is_set(): + CFRunLoopRunInMode(kCFRunLoopDefaultMode, POLL_SECONDS, False) + finally: + # Both, and in this order: an enabled tap whose source is already + # gone is what leaks one live tap per record cycle. + Quartz.CGEventTapEnable(tap, False) + if source is not None and run_loop is not None: + Quartz.CFRunLoopRemoveSource( + run_loop, source, kCFRunLoopDefaultMode) + + def _callback(self, proxy, event_type, event, _refcon): + """Called by the window server on the tap thread, for every event.""" + # Anything escaping here tears down the run loop, and the recording + # then stops without saying so — one unexpected event shape would end + # the session. The catch is broad on purpose, and it is bounded: the + # error is logged and the next event is still decoded. + try: + if event_type in _TAP_DISABLED: + # Re-arm, rather than record silence for the rest of the run. + autocontrol_logger.info( + "event tap disabled (%s), re-enabling", event_type) + Quartz.CGEventTapEnable(proxy, True) + else: + self.decode(int(event_type), event) + except Exception as error: # noqa: BLE001 # reason: an OS callback; see above + autocontrol_logger.error("recorder decode failed: %r", error) + return event + + # -- decoding ---------------------------------------------------------- + def decode(self, event_type: int, event: Any) -> None: + """Record one Quartz event. Split out so it is testable off the tap.""" + if event_type == Quartz.kCGEventKeyDown: + self._put({"op": "key_down", "vk": self._keycode(event)}) + elif event_type == Quartz.kCGEventKeyUp: + self._put({"op": "key_up", "vk": self._keycode(event)}) + elif event_type == Quartz.kCGEventFlagsChanged: + self._modifier(event) + elif event_type == Quartz.kCGEventScrollWheel: + self._scroll(event) + elif event_type in _MOUSE_EVENTS: + operation, button = _MOUSE_EVENTS[event_type] + x, y = self._location(event) + self._put({"op": operation, "button": button, "x": x, "y": y}) + + @staticmethod + def _keycode(event: Any) -> int: + return int(Quartz.CGEventGetIntegerValueField( + event, Quartz.kCGKeyboardEventKeycode)) + + @staticmethod + def _location(event: Any) -> Tuple[int, int]: + point = Quartz.CGEventGetLocation(event) + return int(point.x), int(point.y) + + def _modifier(self, event: Any) -> None: + """Turn a flag set into the press or release of one modifier key.""" + keycode = self._keycode(event) + mask = _MODIFIER_FLAG.get(keycode) + if mask is None: + return + held = bool(int(Quartz.CGEventGetFlags(event)) & mask) + self._put({"op": "key_down" if held else "key_up", "vk": keycode}) + + def _scroll(self, event: Any) -> None: + delta = int(Quartz.CGEventGetIntegerValueField( + event, Quartz.kCGScrollWheelEventDeltaAxis1)) + x, y = self._location(event) + self._put({"op": "scroll", "delta": delta, "x": x, "y": y}) + + def _put(self, event: Dict[str, Any]) -> None: + """Record one event, stopping the tap once the cap is reached.""" + if len(self.events) >= self.max_events: + self._stop.set() + return + event["time"] = time.monotonic() + self.events.append(event) diff --git a/je_auto_control/osx/record/osx_record.py b/je_auto_control/osx/record/osx_record.py index f119508f..6c16441a 100644 --- a/je_auto_control/osx/record/osx_record.py +++ b/je_auto_control/osx/record/osx_record.py @@ -1,52 +1,41 @@ +"""macOS recorder: the Quartz event tap shaped into the recorder surface. + +Everything after the capture — the down-events-only queue the executor has +always been handed, the ``delta_ms`` timeline a replay needs, and the +mouse-only / keyboard-only filters — is platform-neutral and lives in +:mod:`je_auto_control.utils.input_macro.recorder_base`, so this backend and +the Windows one cannot drift in the shape they produce. +""" import sys -from queue import Queue -from je_auto_control.utils.exception.exception_tags import osx_import_error_message -from je_auto_control.utils.exception.exceptions import AutoControlException, AutoControlJsonActionException +from je_auto_control.utils.exception.exception_tags import ( + osx_import_error_message, +) +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.input_macro.recorder_base import InputRecorder # === 平台檢查 Platform Check === # 僅允許在 macOS (Darwin) 環境執行，否則拋出例外 if sys.platform not in ["darwin"]: raise AutoControlException(osx_import_error_message) -from je_auto_control.osx.listener.osx_listener import osx_record, osx_stop_record +from je_auto_control.osx.listener.osx_listener import OSXInputTap -class OSXRecorder: +class OSXRecorder(InputRecorder): """ OSXRecorder macOS 事件錄製控制器 - - 提供開始與停止錄製的介面 - - 將錄製結果存入 Queue - """ - - def __init__(self): - self.record_flag: bool = False - - def record(self) -> None: - """ - Start recording events - 開始錄製事件 - """ - self.record_flag = True - osx_record() - def stop_record(self) -> Queue: - """ - Stop recording and return recorded events - 停止錄製並回傳事件隊列 - - :raises AutoControlJsonActionException: 若沒有錄製到任何事件 - :return: Queue of recorded events 錄製事件的隊列 - """ - record_queue = osx_stop_record() - self.record_flag = False - - if record_queue is None: - raise AutoControlJsonActionException + Capture runs through :class:`OSXInputTap`, a listen-only ``CGEventTap`` on + its own thread, so ``record()`` returns immediately and the user's input + keeps working while it is being recorded. + """ - return record_queue + def new_hook(self) -> OSXInputTap: + """Return a fresh, unstarted event tap.""" + return OSXInputTap() # === 全域 Recorder 實例 Global Recorder Instance === -osx_recorder = OSXRecorder() \ No newline at end of file +osx_recorder = OSXRecorder() diff --git a/je_auto_control/utils/exception/exception_tags.py b/je_auto_control/utils/exception/exception_tags.py index d997f152..62c968c9 100644 --- a/je_auto_control/utils/exception/exception_tags.py +++ b/je_auto_control/utils/exception/exception_tags.py @@ -6,7 +6,9 @@ linux_import_error_message: str = "Should only be loaded on Linux" osx_import_error_message: str = "Should only be loaded on macOS" windows_import_error_message: str = "Should only be loaded on Windows" -macos_record_error_message: str = "Cannot use recorder on macOS" +macos_record_error_message: str = ( + "Cannot record on macOS without Accessibility permission " + "(System Settings -> Privacy & Security -> Accessibility)") # keyboard tags keyboard_error_message: str = "Auto-control keyboard error" diff --git a/je_auto_control/utils/input_macro/__init__.py b/je_auto_control/utils/input_macro/__init__.py index 9e7a3729..03f24968 100644 --- a/je_auto_control/utils/input_macro/__init__.py +++ b/je_auto_control/utils/input_macro/__init__.py @@ -1,6 +1,10 @@ -"""Timed input-event replay and a declarative input-sequence DSL.""" +"""Timed input events: capture shaping, replay, and an input-sequence DSL.""" from je_auto_control.utils.input_macro.input_macro import ( replay_timeline, run_sequence, ) +from je_auto_control.utils.input_macro.recorder_base import ( + InputRecorder, legacy_action_queue, timeline, +) -__all__ = ["replay_timeline", "run_sequence"] +__all__ = ["InputRecorder", "legacy_action_queue", "replay_timeline", + "run_sequence", "timeline"] diff --git a/je_auto_control/utils/input_macro/input_macro.py b/je_auto_control/utils/input_macro/input_macro.py index b17c2492..b5f0b82d 100644 --- a/je_auto_control/utils/input_macro/input_macro.py +++ b/je_auto_control/utils/input_macro/input_macro.py @@ -33,7 +33,12 @@ def _sink_click(event: Dict[str, Any]) -> None: def _sink_scroll(event: Dict[str, Any]) -> None: from je_auto_control.wrapper.auto_control_mouse import mouse_scroll - mouse_scroll(int(event.get("value", 1))) + # ``value`` is the DSL's name for it, ``delta`` the recorder's. The sign + # is kept: it is what decides the direction on Windows and macOS. Linux + # takes its direction from `scroll_direction` instead, which is an open + # cross-platform decision recorded in Progress.md, not something to + # settle here. + mouse_scroll(int(event.get("value", event.get("delta", 1)))) def _sink_press(event: Dict[str, Any]) -> None: @@ -52,9 +57,46 @@ def _sink_key(event: Dict[str, Any]) -> None: type_keyboard(event["key"]) +# The recorder names a button "left"; the input API names it "mouse_left". +_RECORDED_BUTTON = {"left": "mouse_left", "right": "mouse_right", + "middle": "mouse_middle"} + + +def _sink_key_down(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_keyboard import press_keyboard_key + press_keyboard_key(event["vk"]) + + +def _sink_key_up(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_keyboard import ( + release_keyboard_key) + release_keyboard_key(event["vk"]) + + +def _sink_mouse_down(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_mouse import press_mouse + press_mouse(_RECORDED_BUTTON.get(event.get("button", ""), "mouse_left"), + int(event.get("x", 0)), int(event.get("y", 0))) + + +def _sink_mouse_up(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_mouse import release_mouse + release_mouse(_RECORDED_BUTTON.get(event.get("button", ""), "mouse_left"), + int(event.get("x", 0)), int(event.get("y", 0))) + + +#: Two vocabularies reach this table and both have to work. The `run_sequence` +#: DSL writes ``press`` / ``click`` / ``key``; the recorders write +#: ``key_down`` / ``mouse_up`` and so on. They used to be disjoint, so feeding +#: ``stop_record_timeline()`` to :func:`replay_timeline` — the pipeline the +#: docstrings and the ``ac_record_stop_timeline`` tool both prescribe — +#: matched nothing and replayed an empty session while reporting the full +#: event count as played. _SINKS: Dict[str, Callable[[Dict[str, Any]], None]] = { "move": _sink_move, "click": _sink_click, "scroll": _sink_scroll, "press": _sink_press, "release": _sink_release, "key": _sink_key, + "key_down": _sink_key_down, "key_up": _sink_key_up, + "mouse_down": _sink_mouse_down, "mouse_up": _sink_mouse_up, } diff --git a/je_auto_control/utils/input_macro/recorder_base.py b/je_auto_control/utils/input_macro/recorder_base.py new file mode 100644 index 00000000..26140afe --- /dev/null +++ b/je_auto_control/utils/input_macro/recorder_base.py @@ -0,0 +1,169 @@ +"""Platform-neutral shaping of recorded input, and the recorder base class. + +Windows and macOS capture input through entirely different OS machinery — a +low-level ``WH_KEYBOARD_LL`` hook driven by a message pump, and a Quartz +``CGEventTap`` driven by a run loop — but everything *after* the capture is +identical: the same event dictionaries, the same three ways of asking for +them, and the same two output shapes. That part belongs here rather than +being copied into the second backend. + +* :func:`timeline` turns raw timestamped events into the ``delta_ms`` form + :func:`je_auto_control.utils.input_macro.replay_timeline` plays back. +* :func:`legacy_action_queue` produces the historical down-events-only queue + that :func:`je_auto_control.wrapper.auto_control_record.stop_record` and the + executor have always been handed. +* :class:`InputRecorder` is the ``record`` / ``record_mouse`` / + ``record_keyboard`` surface every platform recorder exposes. A backend + supplies only :meth:`InputRecorder.new_hook`. + +The divergence this prevents is not hypothetical and would be silent: a +recording made on one OS has to replay on the other, and two hand-written +copies of the queue shaping drift without anything going red. + +**Everything typed while recording is captured, passwords included.** Callers +must treat the result as sensitive. +""" +from queue import Queue +from typing import Any, Dict, List, Optional, Sequence, Tuple + +#: A hook left installed grows its event list forever. Recording is bounded so +#: a forgotten session cannot consume the process. +MAX_EVENTS = 20000 + +#: Legacy queue entries: the down-event half, shaped as executor commands. +LEGACY_MOUSE_COMMAND = {"left": "AC_mouse_left", "right": "AC_mouse_right", + "middle": "AC_mouse_middle"} + +#: Both capture kinds, which is what a plain ``record()`` asks for. +ALL_KINDS: Tuple[str, ...] = ("keyboard", "mouse") + + +def event_kind(event: Dict[str, Any]) -> str: + """Return ``"keyboard"`` or ``"mouse"`` for one raw event.""" + return "keyboard" if str(event.get("op", "")).startswith("key_") else "mouse" + + +def timeline(events: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert raw events to ``delta_ms`` form for ``replay_timeline``. + + The first event has no gap before it; every later one carries the real + pause that preceded it, which is what makes a replay track the original + pace. + """ + out: List[Dict[str, Any]] = [] + previous: Optional[float] = None + for event in events: + moment = float(event.get("time", 0.0)) + item = {key: value for key, value in event.items() if key != "time"} + item["delta_ms"] = 0 if previous is None else max( + 0, int((moment - previous) * 1000)) + out.append(item) + previous = moment + return out + + +def legacy_action_queue(events: List[Dict[str, Any]]) -> Queue: + """The historical shape: one executor command per press, no releases.""" + queue: Queue = Queue() + for event in events: + operation = event.get("op") + if operation == "key_down": + queue.put(("AC_type_keyboard", int(event.get("vk", 0)))) + elif operation == "mouse_down": + command = LEGACY_MOUSE_COMMAND.get(event.get("button", "")) + if command: + queue.put((command, int(event.get("x", 0)), + int(event.get("y", 0)))) + return queue + + +class InputRecorder: + """``record`` / ``stop_record`` over a platform hook, with timeline output. + + ``stop_record`` keeps returning the down-events-only queue so existing + callers are unaffected; ``stop_record_timeline`` returns everything a + replay needs — releases, wheel movement and per-event gaps. + """ + + def __init__(self) -> None: + self.hook: Any = None + self.record_queue: Optional[Queue] = None + self.result_queue: Optional[Queue] = None + self._kinds: Tuple[str, ...] = ALL_KINDS + + def new_hook(self) -> Any: + """Return a fresh, unstarted platform hook. Backends override this.""" + raise NotImplementedError + + # -- capture ----------------------------------------------------------- + def _start(self, kinds: Sequence[str]) -> None: + self._kinds = tuple(kinds) + self.hook = self.new_hook() + self.record_queue = Queue() + self.hook.start() + + def _stop(self) -> List[Dict[str, Any]]: + if self.hook is None: + return [] + events = [event for event in self.hook.stop() + if event_kind(event) in self._kinds] + self.hook = None + return events + + def _as_queue(self, events: List[Dict[str, Any]]) -> Queue: + queue = legacy_action_queue(events) + self.record_queue = None + self.result_queue = queue + return queue + + # -- public ------------------------------------------------------------ + def record(self) -> None: + """ + 開始錄製滑鼠與鍵盤事件 + Start recording both mouse and keyboard events + """ + self._start(ALL_KINDS) + + def stop_record(self) -> Queue: + """ + 停止錄製並回傳事件 + Stop recording and return recorded events + """ + return self._as_queue(self._stop()) + + def stop_record_timeline(self) -> List[Dict[str, Any]]: + """ + 停止錄製並回傳含放開、滾輪與間隔時間的完整事件 + Stop recording and return press *and* release, wheel and ``delta_ms`` + + Ready for :func:`je_auto_control.utils.input_macro.replay_timeline`. + """ + return timeline(self._stop()) + + def record_mouse(self) -> None: + """ + 開始錄製滑鼠事件 + Start recording mouse events + """ + self._start(("mouse",)) + + def stop_record_mouse(self) -> Queue: + """ + 停止錄製滑鼠事件並回傳結果 + Stop recording mouse events and return results + """ + return self._as_queue(self._stop()) + + def record_keyboard(self) -> None: + """ + 開始錄製鍵盤事件 + Start recording keyboard events + """ + self._start(("keyboard",)) + + def stop_record_keyboard(self) -> Queue: + """ + 停止錄製鍵盤事件並回傳結果 + Stop recording keyboard events and return results + """ + return self._as_queue(self._stop()) diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 17960fb9..8b842ebb 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -560,7 +560,8 @@ def recording_tools() -> List[MCPTool]: name="ac_record_start", description=("Start recording mouse and keyboard events in the " "background. Call ac_record_stop to retrieve the " - "captured action list. Not supported on macOS."), + "captured action list. On macOS this needs " + "Accessibility permission."), input_schema=schema({}), handler=h.record_start, annotations=SIDE_EFFECT_ONLY, diff --git a/je_auto_control/windows/record/win32_input_hook.py b/je_auto_control/windows/record/win32_input_hook.py index 44877253..d2f171f4 100644 --- a/je_auto_control/windows/record/win32_input_hook.py +++ b/je_auto_control/windows/record/win32_input_hook.py @@ -35,6 +35,12 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlException, AutoControlRecordException, ) +# Re-exported: the event cap and the timeline shaping are platform-neutral and +# shared with the macOS tap, but this is where callers have always imported +# them from. +from je_auto_control.utils.input_macro.recorder_base import ( # noqa: F401 + MAX_EVENTS, timeline, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger if sys.platform not in ["win32", "cygwin", "msys"]: @@ -55,10 +61,6 @@ 0x0207: ("mouse_down", "middle"), 0x0208: ("mouse_up", "middle"), } -# A hook that is left installed keeps growing this list forever. Recording is -# bounded so a forgotten session cannot consume the process. -MAX_EVENTS = 20000 - class _KBDLLHOOKSTRUCT(ctypes.Structure): _fields_ = [("vkCode", wintypes.DWORD), ("scanCode", wintypes.DWORD), @@ -203,21 +205,3 @@ def _mouse_event(self, l_param, message: int) -> None: raw = ctypes.c_short((int(data.mouseData) >> 16) & 0xFFFF).value self._put({"op": "scroll", "delta": raw // _WHEEL_NOTCH, "x": int(data.pt.x), "y": int(data.pt.y)}) - - -def timeline(events: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Convert raw events to ``delta_ms`` form for ``replay_timeline``. - - The first event has no gap before it; every later one carries the real pause - that preceded it, which is what makes a replay track the original pace. - """ - out: List[Dict[str, Any]] = [] - previous: Optional[float] = None - for event in events: - moment = float(event.get("time", 0.0)) - item = {key: value for key, value in event.items() if key != "time"} - item["delta_ms"] = 0 if previous is None else max( - 0, int((moment - previous) * 1000)) - out.append(item) - previous = moment - return out diff --git a/je_auto_control/windows/record/win32_record.py b/je_auto_control/windows/record/win32_record.py index 5892e8c6..ceb45e0d 100644 --- a/je_auto_control/windows/record/win32_record.py +++ b/je_auto_control/windows/record/win32_record.py @@ -1,126 +1,41 @@ +"""Windows recorder: the low-level input hook shaped into the recorder surface. + +Everything after the capture — the down-events-only queue the executor has +always been handed, the ``delta_ms`` timeline a replay needs, and the +mouse-only / keyboard-only filters — is platform-neutral and lives in +:mod:`je_auto_control.utils.input_macro.recorder_base`, so this backend and +the macOS one cannot drift in the shape they produce. +""" import sys -from typing import Any, Dict, List, Optional -from queue import Queue from je_auto_control.utils.exception.exception_tags import windows_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.input_macro.recorder_base import InputRecorder if sys.platform not in ["win32", "cygwin", "msys"]: raise AutoControlException(windows_import_error_message) -from je_auto_control.windows.record.win32_input_hook import ( - Win32InputHook, timeline, -) +from je_auto_control.windows.record.win32_input_hook import Win32InputHook -# Legacy queue entries: the down-event half, shaped as executor commands. -_LEGACY_MOUSE_COMMAND = {"left": "AC_mouse_left", "right": "AC_mouse_right", - "middle": "AC_mouse_middle"} - -class Win32Recorder: +class Win32Recorder(InputRecorder): """ Win32Recorder Windows 錄製器 - 可同時錄製滑鼠與鍵盤事件 - 可選擇只錄製滑鼠或鍵盤 - Capture runs through :class:`Win32InputHook`, which records releases, wheel - movement and timing as well as presses. ``stop_record`` still returns the - historical down-events-only queue so existing callers are unaffected; use - ``stop_record_timeline`` for everything needed to reproduce the session. + Capture runs through :class:`Win32InputHook`, which records releases, + wheel movement and timing as well as presses. ``stop_record`` still + returns the historical down-events-only queue so existing callers are + unaffected; use ``stop_record_timeline`` for everything needed to + reproduce the session. """ - def __init__(self): - self.hook: Optional[Win32InputHook] = None - self.record_queue: Optional[Queue] = None - self.result_queue: Optional[Queue] = None - self._kinds: tuple = ("keyboard", "mouse") - - def _start(self, kinds: tuple) -> None: - self._kinds = kinds - self.hook = Win32InputHook() - self.record_queue = Queue() - self.hook.start() - - def _stop(self) -> List[Dict[str, Any]]: - if self.hook is None: - return [] - events = [event for event in self.hook.stop() - if self._wanted(event)] - self.hook = None - return events - - def _wanted(self, event: Dict[str, Any]) -> bool: - keyboard = event.get("op", "").startswith("key_") - return ("keyboard" if keyboard else "mouse") in self._kinds - - def _as_queue(self, events: List[Dict[str, Any]]) -> Queue: - """The historical shape: one executor command per press, no releases.""" - queue: Queue = Queue() - for event in events: - operation = event.get("op") - if operation == "key_down": - queue.put(("AC_type_keyboard", int(event.get("vk", 0)))) - elif operation == "mouse_down": - command = _LEGACY_MOUSE_COMMAND.get(event.get("button", "")) - if command: - queue.put((command, int(event.get("x", 0)), - int(event.get("y", 0)))) - self.record_queue = None - self.result_queue = queue - return queue - - def record(self) -> None: - """ - 開始錄製滑鼠與鍵盤事件 - Start recording both mouse and keyboard events - """ - self._start(("keyboard", "mouse")) - - def stop_record(self) -> Queue: - """ - 停止錄製並回傳事件 - Stop recording and return recorded events - """ - return self._as_queue(self._stop()) - - def stop_record_timeline(self) -> List[Dict[str, Any]]: - """ - 停止錄製並回傳含放開、滾輪與間隔時間的完整事件 - Stop recording and return press *and* release, wheel and ``delta_ms`` - - Ready for :func:`je_auto_control.utils.input_macro.replay_timeline`. - """ - return timeline(self._stop()) - - def record_mouse(self) -> None: - """ - 開始錄製滑鼠事件 - Start recording mouse events - """ - self._start(("mouse",)) - - def stop_record_mouse(self) -> Queue: - """ - 停止錄製滑鼠事件並回傳結果 - Stop recording mouse events and return results - """ - return self._as_queue(self._stop()) - - def record_keyboard(self) -> None: - """ - 開始錄製鍵盤事件 - Start recording keyboard events - """ - self._start(("keyboard",)) - - def stop_record_keyboard(self) -> Queue: - """ - 停止錄製鍵盤事件並回傳結果 - Stop recording keyboard events and return results - """ - return self._as_queue(self._stop()) + def new_hook(self) -> Win32InputHook: + """Return a fresh, uninstalled hook.""" + return Win32InputHook() # 全域錄製器實例 Global recorder instance -win32_recorder = Win32Recorder() \ No newline at end of file +win32_recorder = Win32Recorder() diff --git a/je_auto_control/wrapper/_platform_osx.py b/je_auto_control/wrapper/_platform_osx.py index 7319d26b..18a97fe0 100644 --- a/je_auto_control/wrapper/_platform_osx.py +++ b/je_auto_control/wrapper/_platform_osx.py @@ -37,6 +37,7 @@ ) from je_auto_control.osx.keyboard import osx_keyboard, osx_keyboard_check from je_auto_control.osx.mouse import osx_mouse +from je_auto_control.osx.record.osx_record import osx_recorder from je_auto_control.osx.screen import osx_screen from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -149,7 +150,7 @@ keyboard_check = osx_keyboard_check mouse = osx_mouse screen = osx_screen -recorder = None +recorder = osx_recorder -if None in [keyboard_keys_table, mouse_keys_table, keyboard_check, keyboard, mouse, screen]: +if None in [keyboard_keys_table, mouse_keys_table, keyboard_check, keyboard, mouse, screen, recorder]: raise AutoControlException("Can't init auto control") diff --git a/je_auto_control/wrapper/auto_control_record.py b/je_auto_control/wrapper/auto_control_record.py index d161d3b7..df07af6a 100644 --- a/je_auto_control/wrapper/auto_control_record.py +++ b/je_auto_control/wrapper/auto_control_record.py @@ -1,9 +1,16 @@ +"""Start and stop input recording, and save what was recorded. + +macOS used to be refused here outright: the recorder existed but wiring it up +would have put an ``NSApplication`` and a blocking run loop into the import of +the platform wrapper. It no longer does — capture runs on a Quartz event tap +on its own thread — so the refusal is gone and every platform with a recorder +goes down the same path. A macOS session without Accessibility permission now +fails where that is actually true, in the tap, naming the permission. +""" import os -import sys import threading from typing import Optional -from je_auto_control.utils.exception.exception_tags import macos_record_error_message from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.exception.exceptions import AutoControlJsonActionException from je_auto_control.utils.json.json_file import write_action_json @@ -18,8 +25,6 @@ def record() -> None: """ autocontrol_logger.info("record") try: - if sys.platform == "darwin": - raise AutoControlException(macos_record_error_message) record_action_to_list("record", None) recorder.record() except (OSError, RuntimeError, AttributeError, TypeError, ValueError, AutoControlException, AutoControlJsonActionException) as error: @@ -33,8 +38,6 @@ def stop_record() -> list: """ autocontrol_logger.info("stop_record") try: - if sys.platform == "darwin": - raise AutoControlException(macos_record_error_message) action_queue = recorder.stop_record() if action_queue is None: raise AutoControlJsonActionException @@ -66,8 +69,6 @@ def stop_record_timeline() -> list: """ autocontrol_logger.info("stop_record_timeline") try: - if sys.platform == "darwin": - raise AutoControlException(macos_record_error_message) collect = getattr(recorder, "stop_record_timeline", None) if collect is None: autocontrol_logger.error( diff --git a/test/unit_test/headless/test_osx_input_tap.py b/test/unit_test/headless/test_osx_input_tap.py new file mode 100644 index 00000000..f1ea3f2c --- /dev/null +++ b/test/unit_test/headless/test_osx_input_tap.py @@ -0,0 +1,227 @@ +"""The macOS event tap, driven by real CGEvents. Runs on macOS CI only. + +These build genuine Quartz events with ``CGEventCreate*`` and hand them to the +decoder directly, which needs **no** Accessibility grant and no user at the +keyboard — only creating a live *tap* needs the grant, and that is verified +separately by ``test/verify/macos_verify.py`` on a real runner. + +The regression being guarded is why macOS shipped without a recorder at all: +the old listener built an ``NSApplication`` at import and stopped recording +via ``AppHelper.runEventLoop()``, a loop that never returns. Both would have +landed on the path of ``import je_auto_control``. +""" +import sys +import threading + +import pytest + +if sys.platform != "darwin": # pragma: no cover + pytest.skip("macOS event tap", allow_module_level=True) + +import Quartz # noqa: E402 + +from je_auto_control.osx.listener import osx_listener # noqa: E402 +from je_auto_control.osx.listener.osx_listener import OSXInputTap # noqa: E402 +from je_auto_control.osx.record.osx_record import OSXRecorder # noqa: E402 +from je_auto_control.utils.exception.exceptions import ( # noqa: E402 + AutoControlRecordException, +) + + +def _key_event(keycode, down): + return Quartz.CGEventCreateKeyboardEvent(None, keycode, down) + + +def _mouse_event(event_type, x, y, button=0): + return Quartz.CGEventCreateMouseEvent(None, event_type, (x, y), button) + + +def _decode(events): + """Feed ``(type, event)`` pairs to a fresh tap and return what it kept.""" + tap = OSXInputTap() + for event_type, event in events: + tap.decode(event_type, event) + return tap.events + + +# --- the regression that kept the recorder unwired ------------------------ + +def test_importing_the_listener_builds_no_application(): + # The old module ran NSApplication.sharedApplication() at module scope, so + # every `import je_auto_control` on a Mac created one, and stopping a + # recording meant AppHelper.runEventLoop(). Neither name is here now. + assert not hasattr(osx_listener, "app") + assert not hasattr(osx_listener, "NSApplication") + assert not hasattr(osx_listener, "AppHelper") + + +def test_the_platform_wrapper_now_selects_a_recorder(): + from je_auto_control.wrapper import platform_wrapper + + assert isinstance(platform_wrapper.recorder, OSXRecorder) + + +def test_recording_does_not_block_the_caller(): + # AppHelper.runEventLoop() never returns, so record() used to be a call + # you could not come back from. The tap runs on its own thread. + recorder = OSXRecorder() + finished = threading.Event() + + def _drive(): + try: + recorder.record() + except AutoControlRecordException: + pass # no Accessibility grant here; still returned + finally: + finished.set() + + threading.Thread(target=_drive, daemon=True).start() + assert finished.wait(timeout=osx_listener.START_TIMEOUT + 5.0), ( + "record() did not return; a blocking run loop is back") + recorder.stop_record() + + +# --- decoding ------------------------------------------------------------- + +def test_tap_records_key_press_and_release(): + events = _decode([ + (Quartz.kCGEventKeyDown, _key_event(0, True)), + (Quartz.kCGEventKeyUp, _key_event(0, False)), + ]) + assert [e["op"] for e in events] == ["key_down", "key_up"] + assert events[0]["vk"] == 0 # kVK_ANSI_A + assert all("time" in e for e in events) + + +def test_tap_records_button_release_not_just_press(): + # Without the release a drag is indistinguishable from a click. + events = _decode([ + (Quartz.kCGEventLeftMouseDown, + _mouse_event(Quartz.kCGEventLeftMouseDown, 10, 20)), + (Quartz.kCGEventLeftMouseUp, + _mouse_event(Quartz.kCGEventLeftMouseUp, 90, 60)), + ]) + assert [(e["op"], e["button"], e["x"], e["y"]) for e in events] == [ + ("mouse_down", "left", 10, 20), ("mouse_up", "left", 90, 60)] + + +def test_tap_records_every_button_this_project_addresses(): + events = _decode([ + (Quartz.kCGEventRightMouseDown, + _mouse_event(Quartz.kCGEventRightMouseDown, 1, 2, + Quartz.kCGMouseButtonRight)), + (Quartz.kCGEventOtherMouseDown, + _mouse_event(Quartz.kCGEventOtherMouseDown, 3, 4, + Quartz.kCGMouseButtonCenter)), + ]) + assert [e["button"] for e in events] == ["right", "middle"] + + +def test_recorded_coordinates_are_the_ones_a_replay_posts(): + # CGEventGetLocation has a top-left origin, which is the space osx_mouse + # posts into. The old listener read NSEvent.mouseLocation(), a bottom-left + # origin, so every recorded click replayed vertically mirrored. + height = Quartz.CGDisplayBounds(Quartz.CGMainDisplayID()).size.height + near_top = int(height * 0.1) + events = _decode([ + (Quartz.kCGEventLeftMouseDown, + _mouse_event(Quartz.kCGEventLeftMouseDown, 40, near_top)), + ]) + assert events[0]["y"] == near_top + assert events[0]["y"] < height / 2 + + +def test_tap_decodes_a_scroll_with_its_sign(): + # Reading the wheel unsigned turns a scroll down into a scroll up. + down = Quartz.CGEventCreateScrollWheelEvent( + None, Quartz.kCGScrollEventUnitLine, 1, -3) + up = Quartz.CGEventCreateScrollWheelEvent( + None, Quartz.kCGScrollEventUnitLine, 1, 3) + events = _decode([(Quartz.kCGEventScrollWheel, down), + (Quartz.kCGEventScrollWheel, up)]) + assert [e["op"] for e in events] == ["scroll", "scroll"] + assert [e["delta"] for e in events] == [-3, 3] + + +def test_a_held_modifier_becomes_a_press_and_a_release(): + # macOS sends no key-down for Shift, only a flagsChanged carrying the new + # flag set. Without decoding it a recording cannot say a modifier was held + # across the actions that followed. + pressed = _key_event(56, True) + Quartz.CGEventSetFlags(pressed, Quartz.kCGEventFlagMaskShift) + released = _key_event(56, False) + Quartz.CGEventSetFlags(released, 0) + events = _decode([(Quartz.kCGEventFlagsChanged, pressed), + (Quartz.kCGEventFlagsChanged, released)]) + assert [(e["op"], e["vk"]) for e in events] == [ + ("key_down", 56), ("key_up", 56)] + + +def test_a_flags_change_for_no_known_modifier_is_dropped(): + other = _key_event(0, True) + Quartz.CGEventSetFlags(other, Quartz.kCGEventFlagMaskShift) + assert _decode([(Quartz.kCGEventFlagsChanged, other)]) == [] + + +def test_mouse_movement_is_not_recorded(): + # Every pixel of travel would drown the events that matter, exactly as on + # Windows, where WM_MOUSEMOVE is ignored for the same reason. + assert _decode([ + (Quartz.kCGEventMouseMoved, + _mouse_event(Quartz.kCGEventMouseMoved, 5, 5))]) == [] + + +def test_tap_stops_itself_at_the_event_cap(): + # A forgotten recording must not grow without bound. + tap = OSXInputTap(max_events=3) + for _ in range(10): + tap.decode(Quartz.kCGEventKeyDown, _key_event(0, True)) + assert len(tap.events) == 3 + + +# --- failure and lifecycle ------------------------------------------------ + +def test_a_tap_that_cannot_be_created_fails_loudly(monkeypatch): + # Without Accessibility, CGEventTapCreate returns None. Recording silence + # would look like "the user did nothing" for the rest of the session. + monkeypatch.setattr(Quartz, "CGEventTapCreate", + lambda *args, **kwargs: None) + tap = OSXInputTap() + with pytest.raises(AutoControlRecordException) as caught: + tap.start() + assert "Accessibility" in str(caught.value) + + +def test_stopping_a_tap_that_never_started_returns_nothing(): + assert OSXInputTap().stop() == [] + + +def test_a_decode_failure_does_not_end_the_recording(monkeypatch): + # An exception escaping the callback tears down the run loop, and the + # recording then stops without saying so — so one unusable event must not + # cost the session. The next event still has to be decoded. + tap = OSXInputTap() + calls = [] + + def _explode(*_args, **_kwargs): + calls.append(1) + raise RuntimeError("unexpected event shape") + + monkeypatch.setattr(tap, "decode", _explode) + event = _key_event(0, True) + assert tap._callback(None, Quartz.kCGEventKeyDown, event, None) is event + assert tap._callback(None, Quartz.kCGEventKeyUp, event, None) is event + assert len(calls) == 2 + assert tap.events == [] + + +def test_a_disabled_tap_is_re_armed_rather_than_left_deaf(monkeypatch): + # The window server disables a tap that takes too long. Leaving it + # disabled records silence for the rest of the session. + enabled = [] + monkeypatch.setattr(Quartz, "CGEventTapEnable", + lambda tap, state: enabled.append((tap, state))) + tap = OSXInputTap() + tap._callback("proxy", Quartz.kCGEventTapDisabledByTimeout, None, None) + assert enabled == [("proxy", True)] + assert tap.events == [] diff --git a/test/unit_test/headless/test_recorder_base.py b/test/unit_test/headless/test_recorder_base.py new file mode 100644 index 00000000..bc0579cb --- /dev/null +++ b/test/unit_test/headless/test_recorder_base.py @@ -0,0 +1,290 @@ +"""The recorder shaping every platform shares. No Qt, no real input, no OS. + +Windows captures input with a low-level hook and macOS with a Quartz event +tap, but everything after the capture is one implementation — the queue the +executor is handed, the timeline a replay consumes, and the mouse-only / +keyboard-only filters. This exercises that shared half on any platform, so a +change to it cannot be merged on the strength of a Windows-only run. +""" +from queue import Queue + +import pytest + +from je_auto_control.utils.input_macro.recorder_base import ( + ALL_KINDS, InputRecorder, event_kind, legacy_action_queue, timeline, +) + + +def _events(): + """A press, a release, and a scroll — the three things replay needs.""" + return [ + {"op": "key_down", "vk": 65, "time": 10.0}, + {"op": "key_up", "vk": 65, "time": 10.25}, + {"op": "scroll", "delta": -3, "x": 5, "y": 6, "time": 10.75}, + ] + + +class _FakeHook: + """Stands in for Win32InputHook / OSXInputTap: the same two methods.""" + + def __init__(self, events): + self._events = events + self.started = False + self.stopped = False + + def start(self): + self.started = True + + def stop(self): + self.stopped = True + return list(self._events) + + +class _FakeRecorder(InputRecorder): + def __init__(self, events): + super().__init__() + self._events = events + self.hooks = [] + + def new_hook(self): + hook = _FakeHook(self._events) + self.hooks.append(hook) + return hook + + +# --- timeline ------------------------------------------------------------- + +def test_timeline_turns_timestamps_into_gaps(): + # Without the gaps every step replays at once and no real interface keeps + # up: the window that was supposed to open has not opened yet. + out = timeline(_events()) + assert [event["delta_ms"] for event in out] == [0, 250, 500] + assert "time" not in out[0] + + +def test_timeline_keeps_releases_and_wheel(): + # A press-only recording cannot tell a drag from a click, and loses + # scrolling entirely. + out = timeline(_events()) + assert [event["op"] for event in out] == ["key_down", "key_up", "scroll"] + assert out[2]["delta"] == -3 + + +def test_timeline_of_nothing_is_empty(): + assert timeline([]) == [] + + +def test_timeline_never_reports_a_negative_gap(): + # Monotonic time should not go backwards, but a clamped gap beats a replay + # that tries to sleep a negative amount. + out = timeline([{"op": "key_down", "vk": 1, "time": 5.0}, + {"op": "key_up", "vk": 1, "time": 4.0}]) + assert out[1]["delta_ms"] == 0 + + +def test_timeline_feeds_replay_timeline_unchanged(): + # The two halves are only useful together, so pin that the shape one emits + # is the shape the other consumes. + from je_auto_control.utils.input_macro import replay_timeline + + played = [] + count = replay_timeline(timeline(_events()), sink=played.append, + sleep=lambda _seconds: None) + assert count == 3 + assert [event["op"] for event in played] == [ + "key_down", "key_up", "scroll"] + + +def test_the_default_replay_sink_knows_every_op_a_recorder_emits(): + # These two vocabularies used to be disjoint: the recorder emitted + # key_down / mouse_up / scroll and the sink table held press / click / + # key, so feeding stop_record_timeline() to replay_timeline() — the + # pipeline the docstrings and the MCP tool both prescribe — matched + # nothing, replayed an empty session, and still reported every event as + # played. + from je_auto_control.utils.input_macro.input_macro import _SINKS + + emitted = {"key_down", "key_up", "mouse_down", "mouse_up", "scroll"} + assert emitted <= set(_SINKS), sorted(emitted - set(_SINKS)) + + +def test_replaying_a_recording_drives_the_matching_input_calls(monkeypatch): + # Through the real default sink, with only the input calls themselves + # faked — so the op-to-call routing is what is being exercised. + from je_auto_control.utils.input_macro import replay_timeline + + calls = [] + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_keyboard.press_keyboard_key", + lambda key, *a, **k: calls.append(("press_key", key))) + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_keyboard.release_keyboard_key", + lambda key, *a, **k: calls.append(("release_key", key))) + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_mouse.press_mouse", + lambda button, x, y: calls.append(("press_mouse", button, x, y))) + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_mouse.release_mouse", + lambda button, x, y: calls.append(("release_mouse", button, x, y))) + + replay_timeline(timeline([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "key_up", "vk": 65, "time": 1.1}, + {"op": "mouse_down", "button": "right", "x": 1, "y": 2, "time": 1.2}, + {"op": "mouse_up", "button": "right", "x": 1, "y": 2, "time": 1.3}, + ]), sleep=lambda _seconds: None) + assert calls == [ + ("press_key", 65), ("release_key", 65), + # "right" as the recorder writes it, "mouse_right" as the input API + # names it — a mismatch here presses the wrong button silently. + ("press_mouse", "mouse_right", 1, 2), + ("release_mouse", "mouse_right", 1, 2)] + + +def test_the_recorders_wheel_delta_is_not_dropped_by_the_sink(monkeypatch): + # The DSL calls it `value` and the recorder calls it `delta`; reading only + # `value` fell back to the default of 1, so a three-notch scroll down + # replayed as a single notch in the other direction. + from je_auto_control.utils.input_macro.input_macro import _sink_scroll + + asked = [] + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_mouse.mouse_scroll", + lambda value, *a, **k: asked.append(value)) + _sink_scroll({"op": "scroll", "delta": -3}) + _sink_scroll({"op": "scroll", "value": 7}) + assert asked == [-3, 7] + + +# --- the legacy queue ----------------------------------------------------- + +def test_legacy_queue_is_presses_only_as_executor_commands(): + queue = legacy_action_queue([ + {"op": "key_down", "vk": 65}, + {"op": "key_up", "vk": 65}, + {"op": "mouse_down", "button": "left", "x": 7, "y": 8}, + {"op": "mouse_up", "button": "left", "x": 7, "y": 8}, + {"op": "mouse_down", "button": "right", "x": 1, "y": 2}, + {"op": "mouse_down", "button": "middle", "x": 3, "y": 4}, + ]) + assert list(queue.queue) == [ + ("AC_type_keyboard", 65), ("AC_mouse_left", 7, 8), + ("AC_mouse_right", 1, 2), ("AC_mouse_middle", 3, 4)] + + +def test_legacy_queue_drops_a_button_it_has_no_command_for(): + # An unknown button must not become a malformed action the executor then + # rejects at replay time, far from where it was recorded. + queue = legacy_action_queue([ + {"op": "mouse_down", "button": "x1", "x": 1, "y": 2}]) + assert list(queue.queue) == [] + + +def test_legacy_queue_ignores_the_wheel(): + # stop_record() has never carried scrolling; stop_record_timeline() does. + assert list(legacy_action_queue( + [{"op": "scroll", "delta": 1, "x": 0, "y": 0}]).queue) == [] + + +# --- kinds ---------------------------------------------------------------- + +@pytest.mark.parametrize("op, kind", [ + ("key_down", "keyboard"), ("key_up", "keyboard"), + ("mouse_down", "mouse"), ("mouse_up", "mouse"), ("scroll", "mouse"), +]) +def test_every_op_is_classified(op, kind): + assert event_kind({"op": op}) == kind + + +def test_an_unknown_op_counts_as_mouse_rather_than_vanishing(): + # It is recorded by a backend that knows what it is; dropping it silently + # would be worse than filing it under the broader of the two kinds. + assert event_kind({"op": "gesture"}) == "mouse" + assert ALL_KINDS == ("keyboard", "mouse") + + +# --- the recorder surface ------------------------------------------------- + +def test_record_then_stop_returns_the_legacy_queue(): + recorder = _FakeRecorder(_events()) + recorder.record() + assert recorder.hooks[0].started + queue = recorder.stop_record() + assert isinstance(queue, Queue) + assert list(queue.queue) == [("AC_type_keyboard", 65)] + assert recorder.result_queue is queue + assert recorder.record_queue is None + + +def test_timeline_stop_returns_everything(): + recorder = _FakeRecorder(_events()) + recorder.record() + out = recorder.stop_record_timeline() + assert [event["op"] for event in out] == ["key_down", "key_up", "scroll"] + + +def test_keyboard_only_recording_drops_mouse_events(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_keyboard() + assert [e["op"] for e in recorder.stop_record_timeline()] == ["key_down"] + + +def test_mouse_only_recording_drops_key_events(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "scroll", "delta": 1, "time": 1.1}, + ]) + recorder.record_mouse() + assert [e["op"] for e in recorder.stop_record_timeline()] == ["scroll"] + + +def test_mouse_only_stop_variant_also_filters(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_mouse() + assert list(recorder.stop_record_mouse().queue) == [("AC_mouse_left", 1, 2)] + + +def test_keyboard_only_stop_variant_also_filters(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_keyboard() + assert list(recorder.stop_record_keyboard().queue) == [ + ("AC_type_keyboard", 65)] + + +def test_stopping_without_recording_is_not_an_error(): + assert _FakeRecorder([]).stop_record_timeline() == [] + assert list(_FakeRecorder([]).stop_record().queue) == [] + + +def test_a_second_stop_does_not_reuse_the_finished_hook(): + # The hook is dropped on stop, so a stray second stop returns nothing + # rather than replaying the previous session's events. + recorder = _FakeRecorder(_events()) + recorder.record() + recorder.stop_record() + assert recorder.stop_record_timeline() == [] + + +def test_each_recording_gets_a_fresh_hook(): + # Reusing a stopped hook is how a second recording comes back with the + # first one's events appended to it. + recorder = _FakeRecorder(_events()) + recorder.record() + recorder.stop_record() + recorder.record() + assert len(recorder.hooks) == 2 + assert recorder.hooks[0] is not recorder.hooks[1] + + +def test_a_backend_must_supply_a_hook(): + with pytest.raises(NotImplementedError): + InputRecorder().record() diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py index e1bf2c44..c2879d8e 100644 --- a/test/verify/macos_verify.py +++ b/test/verify/macos_verify.py @@ -61,7 +61,13 @@ "mouse-move": True, "keyboard-post": True, "accessibility-tree": True, - "recorder-absent": True, + # The one row here that a CI run measured rather than a local one: there + # is no Mac in the loop where this is written. It is asserted True from + # the same grant the rows above it were measured to have — an event tap + # needs exactly the Accessibility permission `keyboard-post` already + # proved is granted. If that reasoning is wrong the job says so, names + # this probe, and prints how many events the tap actually saw. + "recorder": True, # True means "the code answered", not "the runner had windows". Measured: # Quartz reports 5 on-screen windows on a macos-14 runner and *none* of # them is at the application layer — they are menu bar and system UI. So @@ -72,6 +78,10 @@ #: How long the window server is given to reflect a posted modifier. KEY_STATE_TIMEOUT = 3.0 +#: How long the recorder's tap thread is given to see the posted events. Its +#: run loop advances in slices, so one slice is not enough to rely on. +RECORDER_SETTLE_SECONDS = 1.0 + _results: List[Tuple[str, bool, str]] = [] @@ -222,17 +232,48 @@ def probe_accessibility() -> Outcome: def probe_recorder() -> Outcome: - """macOS ships no recorder, and must say so rather than look broken. - - ``osx/record/osx_record.py`` exists, but wiring it up would put an - ``NSApplication`` and a blocking run loop into import of the platform - wrapper, so ``recorder`` is None on purpose. This pins that it is a - deliberate absence and not something that quietly stopped working. + """Recording needs Accessibility, a live event tap, and real events in it. + + macOS shipped without a recorder for as long as the code did exist: the + old listener built an ``NSApplication`` at import and stopped recording + with ``AppHelper.runEventLoop()``, so wiring it up would have put both on + the path of ``import je_auto_control``. It now captures through a + listen-only ``CGEventTap`` on its own thread, and this is where that gets + exercised against a real window server rather than a fake. + + Three things have to hold together and only a Mac can answer any of them: + the tap can be created at all (that is the Accessibility grant), events + posted into the session reach it, and what comes back out carries the + coordinates and the release — not just the press. """ + import je_auto_control as ac from je_auto_control.wrapper import platform_wrapper - return Outcome(platform_wrapper.recorder is None, - f"recorder is {platform_wrapper.recorder!r}") + if platform_wrapper.recorder is None: + return Outcome(False, "no recorder is selected on this platform") + + target = (321, 123) + ac.record() + try: + # Posted through the public API, so this exercises the same path a + # user's session does rather than a private Quartz call. + ac.set_mouse_position(*target) + ac.click_mouse("mouse_left", *target) + ac.press_keyboard_key("a") + ac.release_keyboard_key("a") + # The tap thread runs the loop in slices; give it more than one. + time.sleep(RECORDER_SETTLE_SECONDS) + finally: + events = ac.stop_record_timeline() + + operations = [event.get("op") for event in events] + clicks = [event for event in events if event.get("op") == "mouse_down"] + landed = [(event.get("x"), event.get("y")) for event in clicks] + return Outcome( + "mouse_down" in operations and "mouse_up" in operations + and "key_down" in operations and target in landed, + f"{len(events)} event(s): {operations}; clicks landed at {landed}, " + f"posted {target}") def probe_window_management() -> Outcome: @@ -275,7 +316,7 @@ def probe_window_management() -> Outcome: ("mouse-move", probe_mouse_move), ("keyboard-post", probe_keyboard), ("accessibility-tree", probe_accessibility), - ("recorder-absent", probe_recorder), + ("recorder", probe_recorder), ("window-management", probe_window_management), ] From 9cbc2bd1ea1d4a5d736a9a24d72c0ab732fc252c Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 11:11:12 +0800 Subject: [PATCH 19/30] Drop the record-on-macOS assertion the gate no longer backs cmd_record stopped short-circuiting on darwin when macOS got a working recorder, but test_record_subcommand_delegates_to_helper still asserted the old refusal, so both macos-14 pytest-headless jobs failed on rc 0 == 1. The branch was only ever describing the gate; with the gate gone the delegation assertions apply on every platform. --- test/unit_test/headless/test_cli.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/test/unit_test/headless/test_cli.py b/test/unit_test/headless/test_cli.py index 3a097ec2..d0d6f25b 100644 --- a/test/unit_test/headless/test_cli.py +++ b/test/unit_test/headless/test_cli.py @@ -120,12 +120,9 @@ def fake_record_to_json(output_path, *, stop_event, timeout=None): fake_record_to_json) out = str(tmp_path / "rec.json") rc = main(["record", out, "--duration", "0"]) - if sys.platform == "darwin": - assert rc == 1 - else: - assert rc == 0 - assert captured["output"] == out - assert captured["timeout"] == pytest.approx(0.0) + assert rc == 0 + assert captured["output"] == out + assert captured["timeout"] == pytest.approx(0.0) def test_record_to_json_helper_writes_file(tmp_path, monkeypatch): From 2a49f3093d7a71bcbf2d4cfce2be3e404ea9aace Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 11:47:55 +0800 Subject: [PATCH 20/30] Stop requiring OpenCV to move a mouse, and verify a BSD really does Progress.md recorded the missing BSD input coverage as needing "a machine with the dependency set on it, not a different CI trick". That was the mistake its own Wayland section warns about three lines earlier: asking what the environment cannot do instead of asking who actually cannot. It was not FreeBSD that could not run the X11 backend. It was this package, which imported five image and crypto wheels before it would let you move a pointer. Ten modules on the facade's import path pulled in OpenCV, NumPy, Pillow, je_open_cv or cryptography at module scope, across about sixty call sites, while most of utils/ had been importing OpenCV lazily all along with the docstrings to say so. Those ten now do the same; the Pillow annotations moved under TYPE_CHECKING and the two ImageSource aliases keep Pillow in the union as a forward reference. What import je_auto_control needs is defusedxml, plus python-Xlib on X11 -- both pure Python. The five stay hard dependencies and still install by default; what changes is when a missing one is reported. So the FreeBSD VM installs python-Xlib, defusedxml and an X server in seconds where the ports build had not finished in fifty minutes, and freebsd_verify.py runs the backend rather than the guard. Ground truth is the X server answering for itself: query_pointer for the cursor and the button mask, query_keymap for whether an injected key really went down, and a mapped X window that asked for button events for the wheel. That last one was needed for a defect nothing else could see: mouse_scroll matched Windows, then macOS, then a literal ["linux", "linux2"] -- one of the hand-written platform lists platform_id exists to replace -- so on a BSD it fell off the end of the chain with no backend call, no exception and no log line. A wheel event never appears in the pointer mask, because X11 delivers a scroll as a press and release of button 4/5/6/7 too fast to sample. The scroll direction question that had been sitting at DECIDE is settled the way the maintainer chose: the sign of scroll_value reverses the direction on every platform, and scroll_direction names the direction a positive count takes. X11 turned the sign back into the opposite button; Wayland had the same abs() in _wheel_deltas and lost it the same way. Migration for anyone relying on the magnitude alone is in CHANGELOG.md. test_facade_import_is_light.py blocks all five wheels in a subprocess and imports the facade anyway, because one convenience import undoes this silently and every runner with wheels keeps passing. --- .github/workflows/platform-smoke.yml | 88 +++-- CHANGELOG.md | 28 ++ Progress.md | 55 +-- README.md | 16 +- README/README_zh-CN.md | 13 +- README/README_zh-TW.md | 13 +- WHATS_NEW.md | 83 +++- architecture_explore.md | 48 +-- docker/x11_verify.py | 37 +- docs/CAPABILITY_MATRIX.md | 24 +- je_auto_control/linux_wayland/mouse.py | 7 +- .../mouse/x11_linux_mouse_control.py | 31 +- .../utils/action_signing/cipher.py | 5 +- je_auto_control/utils/annotate/annotate.py | 15 +- .../utils/color_stats/color_stats.py | 14 +- .../utils/cv2_utils/screen_record.py | 3 +- je_auto_control/utils/cv2_utils/screenshot.py | 8 +- .../utils/cv2_utils/template_detection.py | 4 +- .../utils/cv2_utils/video_recording.py | 4 +- .../utils/mcp_server/tools/_factories.py | 7 +- je_auto_control/utils/qr/qr.py | 12 +- .../utils/visual_regression/compare.py | 8 +- je_auto_control/wrapper/auto_control_mouse.py | 32 +- .../wrapper/auto_control_screen.py | 11 +- .../headless/test_facade_import_is_light.py | 87 +++++ .../headless/test_r3_vision_capture.py | 6 +- .../headless/test_scroll_sign_is_portable.py | 170 +++++++++ test/verify/freebsd_verify.py | 358 ++++++++++++++++++ 28 files changed, 995 insertions(+), 192 deletions(-) create mode 100644 test/unit_test/headless/test_facade_import_is_light.py create mode 100644 test/unit_test/headless/test_scroll_sign_is_portable.py create mode 100644 test/verify/freebsd_verify.py diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index f93dd29b..50f88b02 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -62,8 +62,9 @@ jobs: if-no-files-found: warn freebsd: - name: The BSD platform decision on a real FreeBSD + name: The X11 backend driving input on a real FreeBSD runs-on: ubuntu-22.04 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 @@ -71,64 +72,61 @@ jobs: # The X11 backend was gated on sys.platform being linux/linux2, so it # refused to load on a FreeBSD desktop that runs the same X server, the # same python-Xlib and the same code. Relaxing that guard is only worth - # something if a BSD actually runs the decision, and no hosted runner - # is one — so this boots a real FreeBSD VM inside the runner. + # something if a BSD actually runs it, and no hosted runner is one — so + # this boots a real FreeBSD VM inside the runner. # - # It checks the *decision*, not the whole backend, and the reason is - # measured rather than assumed: importing anything under - # je_auto_control runs the package facade, which imports OpenCV and - # cryptography at module scope. Neither publishes a FreeBSD wheel, and - # installing them from ports pulled in a dependency tree that had not - # finished after fifty minutes. So utils/platform_id is loaded by file - # path — it imports nothing but sys, which is the point of it being one - # small module — and what a BSD is uniquely needed for is exactly what - # runs here: that sys.platform really looks like this, and that the - # classification every guard now asks answers correctly on it. + # For a while it could only check the *decision*, because importing + # anything under je_auto_control ran the facade and the facade imported + # OpenCV and cryptography at module scope. Neither publishes a FreeBSD + # wheel and building them from ports had not finished after fifty + # minutes, so utils/platform_id was loaded by file path and the backend + # itself went untested. # - # What that leaves uncovered is the backend actually driving input on a - # BSD. See Progress.md; it needs a machine with the dependency set on - # it, not a different CI trick. + # That was the wrong thing to work around. Moving a mouse needs neither + # package, and the facade no longer insists on them — they are imported + # by the functions that use them, which test_facade_import_is_light.py + # keeps true. What is left for this VM is python-Xlib and defusedxml, + # both pure Python, plus an X server. So the whole backend runs here now + # and the reads come back off the server itself: query_pointer for the + # cursor, its button mask for the buttons, query_keymap for the keys. # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha - uses: vmactions/freebsd-vm@v1 # NOSONAR githubactions:S7637 with: release: "14.2" usesh: true prepare: | - pkg install -y python311 + pkg install -y python311 py311-pip xorg-vfbserver run: | set -eu echo "uname: $(uname -a)" - python3.11 - <<'PROBE' - import importlib.util - import sys - spec = importlib.util.spec_from_file_location( - "platform_id", "je_auto_control/utils/platform_id/__init__.py") - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + # The two pure-Python dependencies the facade still needs, at the + # versions pyproject pins, by their PyPI names. FreeBSD does not + # mark its system Python externally-managed today; if that changes + # the retry says so in the log rather than the job failing on a + # pip policy error. + python3.11 -m pip install --no-deps python-xlib==0.33 defusedxml==0.7.1 \ + || { echo "pip refused the system environment; retrying opted-in"; \ + python3.11 -m pip install --break-system-packages --no-deps \ + python-xlib==0.33 defusedxml==0.7.1; } - print("sys.platform:", sys.platform) - assert sys.platform.startswith("freebsd"), sys.platform - - # The four questions every relaxed guard now asks. Before this - # change the answer to the third was False on exactly this - # platform, and the package refused to import at all. - assert module.is_bsd(), "FreeBSD is not recognised as a BSD" - assert module.is_x11_unix(), "FreeBSD is not an X11 unix" - assert not module.is_windows(), "FreeBSD claimed to be Windows" - assert not module.is_macos(), "FreeBSD claimed to be macOS" - assert module.current_family() == "bsd", module.current_family() - - # The version suffix is the trap: sys.platform is freebsd14 here, - # never a bare "freebsd", so an equality check would match no - # real system at all. - assert sys.platform != "freebsd", ( - "this release stopped carrying a version suffix; the prefix " - "match still works, but the comment explaining why it exists " - "no longer describes reality") - print("platform_id on FreeBSD: OK") - PROBE + # The backend connects to a display at import time, so the server + # has to be up first. 1280x1024 because the verification drives the + # cursor to the far corner and reads it back. + Xvfb :99 -screen 0 1280x1024x24 & + xvfb_pid=$! + trap 'kill "$xvfb_pid" 2>/dev/null || true' EXIT + waited=0 + while [ ! -e /tmp/.X11-unix/X99 ]; do + waited=$((waited + 1)) + if [ "$waited" -gt 100 ]; then + echo "Xvfb never created /tmp/.X11-unix/X99" >&2 + exit 1 + fi + sleep 0.1 + done + DISPLAY=:99 PYTHONPATH="$(pwd)" python3.11 test/verify/freebsd_verify.py macos-capabilities: name: What a real macOS runner permits runs-on: macos-14 diff --git a/CHANGELOG.md b/CHANGELOG.md index beee31cb..0a66b828 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,28 @@ only when documented here with a migration path. ### Changed +- **The sign of `scroll_value` picks the scroll direction on every platform.** + Windows and macOS have always read it that way; X11 and Wayland took the + direction from `scroll_direction` alone and used `abs(scroll_value)`, so + `mouse_scroll(-3)` — code written and tested against the Windows convention — + scrolled *down* three notches on Linux instead of up, with no exception and + no warning. `scroll_direction` now names the direction a **positive** count + takes, and a negative count reverses it, on all four backends. + + *Migration.* Code that passed a negative `scroll_value` to Linux or Wayland + and relied on the magnitude alone now scrolls the opposite way. Take + `abs()` at the call site to keep the old behaviour: + `mouse_scroll(abs(value), scroll_direction="scroll_down")`. Code that passed + a positive count is unaffected, as is every Windows and macOS caller. +- **`import je_auto_control` no longer imports OpenCV, NumPy, Pillow, + `je_open_cv` or `cryptography`.** They are imported by the functions that use + them. The facade pulled all five in at module scope, so a platform without + wheels for them — a FreeBSD desktop, for one — could not use the input + automation half of the package at all, though it needs none of them. Nothing + moves in the public API and the packages remain hard dependencies; what + changes is *when* a missing one is reported, which is now at the first image + or encryption call rather than at import. `test_facade_import_is_light.py` + keeps it that way. - **`macos_record_error_message` now names a permission, not a platform.** It read "Cannot use recorder on macOS", which described a limitation that no longer exists; it now names the Accessibility grant that recording actually @@ -286,6 +308,12 @@ only when documented here with a migration path. ### Fixed +- **`mouse_scroll` did nothing at all on the BSDs.** It matched Windows, then + macOS, then a literal `["linux", "linux2"]`, so a FreeBSD, OpenBSD, NetBSD or + DragonFly caller fell off the end of the chain: no backend call, no + exception, no log line. It asks `platform_id.is_x11_unix()` now, and an + unrecognised platform raises `AutoControlMouseException` instead of returning + as though it had scrolled. - **A recorded timeline replayed nothing.** `replay_timeline`'s dispatch table held the `run_sequence` DSL's vocabulary (`press` / `click` / `key`) and the recorders emit their own (`key_down` / `mouse_up` / `scroll`), and the two diff --git a/Progress.md b/Progress.md index e39e6b07..77693cc0 100644 --- a/Progress.md +++ b/Progress.md @@ -50,33 +50,6 @@ --- -## BSD 上只驗了「判定」，沒驗到「真的驅動輸入」 - -`TODO` — `.github/workflows/platform-smoke.yml` 的 `freebsd` job - -`freebsd` job 在 runner 裡開真的 FreeBSD 14 VM，驗的是 -`utils/platform_id`：`sys.platform` 真的長成 `freebsd14`、 -`is_x11_unix()` 在上面回 True——也就是每個放寬後的守衛 -現在問的那個問題，而這件事只有 BSD 能回答。 - -**沒驗到的是：X11 backend 在 BSD 上真的移滑鼠、真的送 -按鍵。** 原因是量出來的，不是懶：import 任何 -`je_auto_control` 底下的東西都會跑門面，而門面在 -module scope import OpenCV 與 cryptography。這兩個都沒發 FreeBSD -wheel，改用 ports 裝（`py311-opencv`）拉出來的相依樹跑了 -**五十分鐘還沒裝完**，只好取消。一個 smoke job 不能 -花一小時。 - -要補這塊，需要的是一台已經裝好相依套件的真 -FreeBSD（或者一個預先烤好依賴的自訂映像），而不是 -另一個 CI 小技巧。跑的時候把下面這段跑完就算驗到： - -```python -from je_auto_control.linux_with_x11.mouse import x11_linux_mouse_control as m -m.set_position(321, 123) -assert m.position() == (321, 123) -``` - ## Windows arm64 裝不起來，卡在 opencv-python `BLOCKED` — 上游（opencv-python 沒有 win_arm64 wheel） @@ -92,30 +65,18 @@ Windows arm64 上裝不起來**。所以那一格已從矩陣 移除，並把原因寫在 workflow 的註解裡；哪天上游 發了 wheel，把 runner 加回去就好。 +門面已經不在 module scope import OpenCV 了（見 +[WHATS_NEW.md](WHATS_NEW.md)），但這裡卡的不是 import +而是 **pip 裝不起來**:`opencv-python` 仍列在 +`pyproject.toml` 的 `dependencies`,`pip install -e .` +第一步就會去建它。要讓 arm64 只裝輸入的部分,得先決定 +把 OpenCV／Pillow 移到 optional extra——那是相容性決定, +沒有人要求之前不做。 + **Linux arm64 是好的**——`ubuntu-22.04-arm` 兩個 Python 版本 都綠，macOS 本來就是 arm64。所以卡住的只有 Windows 這一個組合。 -## `mouse_scroll` 的方向在三個平台上不是同一回事 - -`DECIDE` — `wrapper/auto_control_mouse.py::mouse_scroll` - -Windows 與 macOS 用 `scroll_value` 的**正負號**決定捲動方向；Linux 不看正負號, -方向來自 `scroll_direction` 參數,值只取 `abs()`。後者是刻意的:負值以前會讓 -`range()` 變空,結果是靜靜地什麼都不捲。 - -**問題在於:照 Windows 寫法寫出來的可攜程式碼,在 Linux 上不會往上捲,而是往下捲 -同樣的次數。**沒有例外、沒有警告,方向就是反的——和 macOS 那個 `write("\b")` -變成打空白是同一類缺陷:靜靜地做了跟要求相反的事。 - -`x11-verification` job 現在把**實測到的**行為釘住了（四個方向各一項,外加 -「負號不決定方向」一項）,所以哪天行為變了 CI 會當場說。要不要讓三個平台一致, -以及一致成哪一種,是相容性決定,需要維護者拍板: - -- 讓 Linux 也認正負號 → 修好可攜性,但會改掉 `scroll_direction` 已文件化的語意; -- 維持現狀 → 就得在 `mouse_scroll` 的 docstring 與 README 明講這個平台差異; -- 折衷:正負號在三個平台都認,`scroll_direction` 只在 Linux 當預設方向。 - ## Wayland:剩下的都不是「缺一台機器」 這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, diff --git a/README.md b/README.md index 3b0aed1c..d57eb80d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ **AutoControl** is a cross-platform GUI automation framework for Python. It drives the mouse and keyboard, finds things on screen (template matching, OCR, the OS accessibility tree, or a vision model), records and replays flows, and runs them from JSON action -files — on Windows, macOS, Linux (X11 and Wayland), Android, and iOS. +files — on Windows, macOS, Linux (X11 and Wayland), the BSDs, Android, +and iOS. Every capability ships three ways: a **Python API**, an **`AC_*` action command** usable from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. @@ -19,7 +20,7 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. ## Why AutoControl -- **One API, six platforms.** `wrapper/platform_wrapper.py` picks the backend at import +- **One API, seven platforms.** `wrapper/platform_wrapper.py` picks the backend at import time; your script does not change between Windows, macOS, X11, and Wayland. - **Scriptable without Python.** 773 `AC_*` commands cover the whole feature set, so a JSON file can do anything the library can — including loops, branches, try/catch, @@ -242,6 +243,7 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | | Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | libei via the desktop portal, or ydotool / wtype + a capture tool | ✅ | ✅ | ❌ | ❌ | +| FreeBSD / OpenBSD / NetBSD | python-Xlib, the same X11 backend as Linux | ✅² | ⚠️² | ✅² | ✅² | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | @@ -250,6 +252,16 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) Accessibility). Without it recording raises and names the permission rather than returning an empty session. +² The BSDs run the X11 backend unchanged — the same X server, the same +`python-Xlib`, which is the only dependency input, recording and window +management have. A `freebsd` CI job drives real input on a real FreeBSD 14 and +reads it back off the X server; OpenBSD and NetBSD take the same code path but +have no CI runner. Screen capture is the exception, and the reason is +packaging rather than the platform: it goes through Pillow/mss and OpenCV, and +`opencv-python`, `pillow` and `cryptography` publish no FreeBSD wheels. Build +those from ports and capture, image matching, OCR and action encryption work +too — `import je_auto_control` no longer requires any of them. + Wayland input falls back to the `ydotool` CLI wherever libei is not reachable, and that fallback needs **ydotool 1.0 or newer**. Every argument AutoControl builds arrived in that release; 0.1.x — which is what Debian diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 67028de3..452e56c8 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -7,7 +7,7 @@ **AutoControl** 是一套跨平台的 Python GUI 自动化框架。它能驱动鼠标与键盘、在画面上找到目标 （模板匹配、OCR、操作系统无障碍树，或视觉模型）、录制与回放操作流程，并以 JSON 动作文件执行—— -支持 Windows、macOS、Linux（X11 与 Wayland）、Android 与 iOS。 +支持 Windows、macOS、Linux（X11 与 Wayland）、BSD、Android 与 iOS。 每项能力都以三种形式提供：**Python API**、可在 JSON 文件／CLI／服务器使用的 **`AC_*` 动作命令**， 以及 **GUI 标签页**。没有任何功能只存在于 GUI。 @@ -18,7 +18,7 @@ ## 为什么选择 AutoControl -- **一套 API，六个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 +- **一套 API，七个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 Windows、macOS、X11 与 Wayland 上都不需要改写。 - **不写 Python 也能脚本化。** 773 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 能做的任何事——包含循环、分支、try/catch、宏与变量。 @@ -231,6 +231,7 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | | Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | 经桌面 portal 的 libei，或 ydotool／wtype ＋ 截图工具 | ✅ | ✅ | ❌ | ❌ | +| FreeBSD／OpenBSD／NetBSD | python-Xlib，与 Linux 同一套 X11 后端 | ✅² | ⚠️² | ✅² | ✅² | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | @@ -238,6 +239,14 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) （系统设置 → 隐私与安全性 → 辅助功能）。没有授权时会直接抛出并指名 缺的是哪个权限，而不是安静地录到一个空的 session。 +² BSD 直接跑同一套 X11 后端——同一个 X server、同一个 `python-Xlib`，而输入、 +录制与窗口管理就只依赖这一个包。`freebsd` CI job 在真的 FreeBSD 14 上驱动真的 +输入，再从 X server 读回来；OpenBSD 与 NetBSD 走同一条代码路径，只是没有 CI +runner。唯一的例外是屏幕截取，卡住的是打包而不是平台：它走 Pillow／mss 与 +OpenCV，而 `opencv-python`、`pillow`、`cryptography` 都没有发 FreeBSD wheel。 +从 ports 构建之后，截取、图像匹配、OCR 与动作加密也都能用—— +`import je_auto_control` 本身已经不需要它们任何一个。 + Wayland 的输入在 libei 走不通时会退回 `ydotool` CLI，而这条退路需要 **ydotool 1.0 以上**。AutoControl 送的每一个参数都是那一版才有的；0.1.x （Debian bookworm 与目前所有 Ubuntu 仍以这个名字提供，Debian trixie 则根本没有） diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index f3a8a1df..97e31d4b 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -7,7 +7,7 @@ **AutoControl** 是一套跨平台的 Python GUI 自動化框架。它能驅動滑鼠與鍵盤、在畫面上找到目標 （樣板比對、OCR、作業系統無障礙樹，或視覺模型）、錄製與重播操作流程，並以 JSON 動作檔執行—— -支援 Windows、macOS、Linux（X11 與 Wayland）、Android 與 iOS。 +支援 Windows、macOS、Linux（X11 與 Wayland）、BSD、Android 與 iOS。 每項能力都以三種形式提供：**Python API**、可在 JSON 檔／CLI／伺服器使用的 **`AC_*` 動作指令**， 以及 **GUI 分頁**。沒有任何功能只存在於 GUI。 @@ -18,7 +18,7 @@ ## 為什麼選擇 AutoControl -- **一套 API，六個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 +- **一套 API，七個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 Windows、macOS、X11 與 Wayland 上都不需要改寫。 - **不寫 Python 也能腳本化。** 773 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 @@ -232,6 +232,7 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | | Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | 經桌面 portal 的 libei，或 ydotool／wtype ＋ 擷取工具 | ✅ | ✅ | ❌ | ❌ | +| FreeBSD／OpenBSD／NetBSD | python-Xlib，與 Linux 同一套 X11 後端 | ✅² | ⚠️² | ✅² | ✅² | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | @@ -239,6 +240,14 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) （系統設定 → 隱私權與安全性 → 輔助使用）。沒有授權時會直接拋出並指名 缺的是哪個權限，而不是安靜地錄到一個空的 session。 +² BSD 直接跑同一套 X11 後端——同一個 X server、同一個 `python-Xlib`，而輸入、 +錄製與視窗管理就只相依這一個套件。`freebsd` CI job 在真的 FreeBSD 14 上驅動真的 +輸入，再從 X server 讀回來；OpenBSD 與 NetBSD 走同一條程式路徑，只是沒有 CI +runner。唯一的例外是螢幕擷取，而卡的是打包不是平台：它走 Pillow／mss 與 OpenCV， +而 `opencv-python`、`pillow`、`cryptography` 都沒有發 FreeBSD wheel。從 ports +建起來之後，擷取、影像比對、OCR 與動作加密也都能用——`import je_auto_control` +本身已經不需要它們任何一個。 + Wayland 的輸入在 libei 走不通時會退回 `ydotool` CLI，而這條退路需要 **ydotool 1.0 以上**。AutoControl 送的每一個參數都是那一版才有的；0.1.x （Debian bookworm 與目前所有 Ubuntu 仍以這個名字提供，Debian trixie 則根本沒有） diff --git a/WHATS_NEW.md b/WHATS_NEW.md index ac35e234..341cee25 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -163,15 +163,90 @@ compared against literal lists in over a hundred places, so the fix is one place that decides: `utils/platform_id`, whose `is_x11_unix()` asks the question those guards were always trying to ask. -A `freebsd` job boots a real FreeBSD 14 VM inside the runner, imports the X11 -modules under a real X server, and moves the pointer and reads it back. It -covers the platform layer rather than the whole package, because opencv has no -FreeBSD wheel — a limit stated in the job rather than left to be discovered. +A `freebsd` job boots a real FreeBSD 14 VM inside the runner. It could at +first only check the *decision* — that `sys.platform` really reads `freebsd14`, +and that the classification every relaxed guard asks answers correctly on it — +because importing anything under `je_auto_control` ran the facade, and the +facade imported OpenCV and cryptography at module scope. `utils/platform_id` +had to be loaded by file path to get even that far. See below: that turned out +to be the wrong thing to work around, and the job now drives the whole backend. + `ubuntu-22.04-arm` joins the smoke matrix and passes; `macos-14` was already arm64. `windows-11-arm` was tried and removed: opencv-python publishes no `win_arm64` wheel, so the package cannot be installed there at all today — measured, not assumed, and recorded in `Progress.md`. +### The Facade Insisted on OpenCV to Move a Mouse + +`Progress.md` recorded the missing BSD coverage as needing "a machine with the +dependency set on it, not a different CI trick". That entry was making the +mistake its own Wayland section warns about three lines further up: **asking +what the environment could not do, instead of asking who actually could not do +it.** It was not FreeBSD that could not run the X11 backend. It was this +package, which imported five image and crypto wheels before it would let you +move a pointer. + +The measurement was small. Ten modules on the facade's import path pulled in +OpenCV, NumPy, Pillow, `je_open_cv` or `cryptography` at module scope, across +about sixty call sites — while most of `utils/` had been importing OpenCV +lazily all along, with the docstrings to say so. Those ten now do the same. The +type annotations that referenced Pillow moved under `TYPE_CHECKING`, and the +two public `ImageSource` aliases keep Pillow in the union as a forward +reference, so nothing changes for a caller or a type checker. + +What `import je_auto_control` needs now is `defusedxml`, plus `python-Xlib` on +an X11 platform. Both are pure Python. The heavy five are still hard +dependencies and still install by default; the difference is that a platform +with no wheel for them can now use the half of the package that never needed +them. `test/unit_test/headless/test_facade_import_is_light.py` blocks all five +in a subprocess and imports the facade anyway, because this is a property one +convenience import silently undoes and every runner with wheels keeps passing. + +### The BSD Job Drives Real Input Now, and Found the Defect That Was Waiting + +With the facade light, the FreeBSD VM needs python-Xlib, defusedxml and an X +server — an install measured in seconds, where the ports build for OpenCV had +not finished after fifty minutes. So `test/verify/freebsd_verify.py` runs the +backend rather than the guard, and takes its ground truth from the X server +answering for itself: `query_pointer` for where the cursor is, its button mask +for which buttons the server believes are down, and `query_keymap` — the bitmap +of every physically-held key — for whether an injected key press really landed. +That last one is why no second process is needed here; the Linux +`x11-verification` job already reads events back out of `xev`, and what a BSD is +uniquely needed to answer is whether this code drives the same server on a +different kernel. + +It also maps a real X window that has asked for button events, because one +defect could not be caught any other way: **`mouse_scroll` did nothing at all +on a BSD.** It matched Windows, then macOS, then a literal +`["linux", "linux2"]` — one of the hundred-odd hand-written platform lists +`platform_id` exists to replace, and one that had been missed — so a BSD caller +fell off the end of the chain with no backend call, no exception and no log +line. A wheel event never shows up in the pointer mask, since X11 delivers a +scroll as a press *and* release of button 4/5/6/7 too fast to sample, so only a +client reading the event queue can see it happen or not happen. + +### `mouse_scroll` Means the Same Thing on Every Platform + +This had been sitting in `Progress.md` as a `DECIDE`, and the maintainer +settled it: **the sign of `scroll_value` reverses the direction everywhere, and +`scroll_direction` names the direction a positive count takes.** + +Windows and macOS had always read the sign. X11 encodes direction as a button +rather than a signed delta, so it took the direction from `scroll_direction` +and discarded the sign — deliberately, because a negative count used to make +`range()` empty and scroll nothing at all. The cost was portability with no +symptom to debug: `mouse_scroll(-3)`, written and tested on Windows, scrolled +three notches *down* on Linux instead of up. Wayland had the same `abs()` in +`_wheel_deltas` and the same result. + +Both now turn a negative count back into the opposite direction — a button swap +on X11, a sign on the Wayland delta. `docker/x11_verify.py` pins it against a +real X server through `xev`, `freebsd_verify.py` pins it on a BSD, and +`test_scroll_sign_is_portable.py` pins it on every runner without needing a +display. The migration note for anyone who was relying on the magnitude alone +is in `CHANGELOG.md`. + ## What's new (2026-08-19) diff --git a/architecture_explore.md b/architecture_explore.md index f7c40078..0f185f04 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-08-19　**版本**：`pyproject.toml` version `0.0.218`　**分支**：`fix/clipboard-handles-and-window-input` +> **掃描時間**：2026-08-20　**版本**：`pyproject.toml` version `0.0.219`　**分支**：`feat/cross-platform-verification` --- @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,027 | -| 程式碼總行數 | 139,420 | +| 程式碼總行數 | 139,499 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -171,9 +171,9 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `wrapper/_platform_osx.py` | 156 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | | `wrapper/_platform_wayland.py` | 57 | Wayland 後端組裝（libei／ydotool／grim）。 | -| `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_mouse.py` | 366 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | | `wrapper/auto_control_keyboard.py` | 273 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | -| `wrapper/auto_control_screen.py` | 97 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | +| `wrapper/auto_control_screen.py` | 100 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | | `wrapper/auto_control_record.py` | 107 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | | `wrapper/auto_control_window.py` | 278 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | @@ -211,7 +211,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | -#### Linux X11（`linux_with_x11/`，19 檔／1,196 行） +#### Linux X11（`linux_with_x11/`，19 檔／1,215 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -226,7 +226,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，17 檔／2,830 行） +#### Linux Wayland（`linux_wayland/`，17 檔／2,835 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -238,7 +238,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | | `oeffis.py` | 196 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | | `libei.py` | 610 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | -| `mouse.py` | 379 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | +| `mouse.py` | 384 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | | `keyboard.py` | 173 | 鍵盤後端：libei 優先，退回 ydotool／wtype。 | | `keymap.py` | 155 | 友善鍵名 → evdev key code。 | | `capture.py` | 236 | 擷取分層：操作者自訂指令 → grim → gnome-screenshot → spectacle → portal。 | @@ -265,12 +265,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,870 行。 +> 24 個套件、約 12,871 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/action_lint/` | 328 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | -| `utils/action_signing/` | 229 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/action_signing/` | 230 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | | `utils/checkpoint/` | 115 | 流程檢查點與續跑，讓長 action list 具持久性 | | `utils/codegen/` | 157 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | | `utils/dag/` | 475 | 跨主機 DAG 編排器（圖模型 + runner） | @@ -364,17 +364,17 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.5 影像辨識與畫面分析 -> 37 個套件、約 4,999 行。 +> 37 個套件、約 5,027 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/annotate/` | 107 | 截圖標註：畫框、highlight、箭頭、標籤 | +| `utils/annotate/` | 114 | 截圖標註：畫框、highlight、箭頭、標籤 | | `utils/barcode/` | 53 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | | `utils/color_match/` | 103 | 在 HSV 通道上做顏色感知的樣板比對 | | `utils/color_region/` | 79 | 以顏色定位畫面區域（遮罩 + 連通元件） | -| `utils/color_stats/` | 89 | 區域顏色統計：平均色與主色 | +| `utils/color_stats/` | 95 | 區域顏色統計：平均色與主色 | | `utils/coordinate_space/` | 84 | 模型網格座標與實體像素之間的座標空間對映 | -| `utils/cv2_utils/` | 592 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | +| `utils/cv2_utils/` | 597 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | | `utils/edge_lines/` | 120 | 以 Hough 轉換偵測線條／格線／分隔線 | | `utils/edge_match/` | 112 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | | `utils/feature_match/` | 129 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | @@ -392,7 +392,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/motion_regions/` | 73 | 兩影格間的局部變化／活動偵測（absdiff） | | `utils/perceptual_diff/` | 100 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | | `utils/preprocess/` | 185 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | -| `utils/qr/` | 53 | 從影像或螢幕區域解碼 QR code（OpenCV） | +| `utils/qr/` | 59 | 從影像或螢幕區域解碼 QR code（OpenCV） | | `utils/rotated_match/` | 145 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | | `utils/saliency/` | 107 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | | `utils/scale_detect/` | 84 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | @@ -404,7 +404,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/theme_normalize/` | 92 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | | `utils/video_report/` | 133 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | | `utils/visual_match/` | 427 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | -| `utils/visual_regression/` | 217 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | +| `utils/visual_regression/` | 221 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | ### 5.4.6 OCR 與文字理解 @@ -487,7 +487,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,177 行。 +> 13 個套件、約 20,180 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -500,7 +500,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,895 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 16,898 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | @@ -699,7 +699,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,895 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（16,898 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -1018,18 +1018,18 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | | `gui/` | 89 | 26,542 | -| `utils/mcp_server/` | 20 | 16,895 | +| `utils/mcp_server/` | 20 | 16,898 | | `utils/remote_desktop/` | 56 | 11,835 | | `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,247 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,363 | | `utils/accessibility/` | 13 | 2,818 | -| `wrapper/` | 3,042 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | +| `wrapper/` | 3,065 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | | `windows/` | 23 | 1,894 | | `utils/rest_api/` | 8 | 1,738 | | `utils/agent/` | 8 | 1,250 | -| `linux_with_x11/` | 19 | 1,196 | -| `linux_wayland/` | 17 | 2,830 | +| `linux_with_x11/` | 19 | 1,215 | +| `linux_wayland/` | 17 | 2,835 | | `utils/triggers/` | 4 | 1,146 | | `utils/ocr/` | 9 | 1,112 | | `utils/usbip/` | 5 | 920 | @@ -1037,6 +1037,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 907 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 689 | 50,253 | -| **總計** | **1,021** | **139,355** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 689 | 50,305 | +| **總計** | **1,021** | **139,434** | diff --git a/docker/x11_verify.py b/docker/x11_verify.py index d8e0ff1f..b12f6ffc 100644 --- a/docker/x11_verify.py +++ b/docker/x11_verify.py @@ -564,13 +564,12 @@ def _split() -> str: def check_scroll_events(tester: EventTester) -> None: """X11 encodes scrolling as buttons 4-7; the direction must pick correctly. - On Linux the direction comes from the ``scroll_direction`` argument and - the *sign of the value is discarded* — ``mouse_scroll`` takes ``abs()`` - on purpose, because a negative count used to make ``range()`` empty and - scroll nothing at all. Windows and macOS read the direction off that same - sign instead. The checks below pin the behaviour as measured rather than - as one platform spells it; see Progress.md, where the difference is - recorded as a decision for the maintainer. + ``scroll_direction`` names the direction a *positive* count scrolls in, + and a negative count reverses it — the same rule on all three platforms + since the maintainer settled it. X11 used to discard the sign, so code + written against the Windows convention scrolled the opposite way here, + silently. The last check is that regression: it fails if the sign ever + stops being read. """ from je_auto_control import mouse_scroll, set_mouse_position @@ -594,18 +593,26 @@ def _scroll(value: int, direction: str, expected_button: int) -> str: check("scroll_direction='scroll_right' arrives as button 7", lambda: _scroll(1, "scroll_right", 7)) - def _sign_is_ignored() -> str: + def _negative_reverses(direction: str, expected_button: int) -> str: set_mouse_position(*target) tester.flush() # Portable code written against the Windows sign convention lands - # here. It does not scroll up; it scrolls down. That is the measured - # contract, and this is what would go red if it ever changed. - mouse_scroll(-2, scroll_direction="scroll_down") + # here, and now gets the direction it asked for. + mouse_scroll(-2, scroll_direction=direction) presses = tester.collect("ButtonPress", 2) - _assert_eq({event["button"] for event in presses}, {5}) - return "a negative count scrolls the named direction, not the opposite" - check("the sign of the count does not pick the direction on Linux", - _sign_is_ignored) + _assert_eq({event["button"] for event in presses}, + {expected_button}) + return (f"-2 with {direction} arrived as button " + f"{expected_button}") + + check("a negative count reverses scroll_down into button 4", + lambda: _negative_reverses("scroll_down", 4)) + check("a negative count reverses scroll_up into button 5", + lambda: _negative_reverses("scroll_up", 5)) + check("a negative count reverses scroll_left into button 7", + lambda: _negative_reverses("scroll_left", 7)) + check("a negative count reverses scroll_right into button 6", + lambda: _negative_reverses("scroll_right", 6)) def check_key_events(tester: EventTester) -> None: diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index 0ac53406..aaf65d4e 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -106,13 +106,23 @@ the backend without its one dependency. The guards now ask question they were always trying to ask — and the `freebsd` job boots a real FreeBSD 14 VM to run that decision on a system that is genuinely one. -It checks the decision and not the backend, for a measured reason: importing -anything under `je_auto_control` runs the package facade, which imports -OpenCV and cryptography at module scope, and neither publishes a FreeBSD -wheel. Installing them from ports pulled a dependency tree that had not -finished after fifty minutes. So `utils/platform_id` is loaded by file path, -and what a BSD is uniquely needed to answer is what runs. Driving real input -on a BSD is still uncovered and is recorded in `Progress.md`. +For a while it checked that decision and nothing else, for a measured reason: +importing anything under `je_auto_control` ran the package facade, which +imported OpenCV and cryptography at module scope, and neither publishes a +FreeBSD wheel — installing them from ports pulled a dependency tree that had +not finished after fifty minutes. + +That was the wrong thing to work around. Moving a pointer needs neither +package, so the facade stopped importing them (and NumPy, Pillow and +`je_open_cv`) at module scope; they belong to the functions that use them. +What the VM installs now is `python-Xlib`, `defusedxml` and an X server, all +of which take seconds, and `test/verify/freebsd_verify.py` drives the whole +backend on it. The reads come off the X server rather than out of this +codebase: `query_pointer` for the cursor and the button mask, `query_keymap` +for whether an injected key really went down, and a mapped X window that has +asked for button events for the wheel — which is what caught `mouse_scroll` +matching a literal `["linux", "linux2"]` and therefore doing nothing at all, +silently, on every BSD. **arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` joins the smoke matrix and passes. `windows-11-arm` was tried and removed, on diff --git a/je_auto_control/linux_wayland/mouse.py b/je_auto_control/linux_wayland/mouse.py index 82308146..ae22aae3 100644 --- a/je_auto_control/linux_wayland/mouse.py +++ b/je_auto_control/linux_wayland/mouse.py @@ -326,9 +326,14 @@ def _wheel_deltas(scroll_value: int, scroll_direction: int) -> Tuple[int, int]: ``scroll_direction`` carries axis and sign, per this module's ``wayland_scroll_direction_*`` constants: +-1 vertical, +-2 horizontal. + + A negative ``scroll_value`` reverses that direction, the same rule the + other three backends follow. The magnitude used to be taken with + ``abs()``, which is what let a portable ``mouse_scroll(-3)`` scroll the + named direction here instead of the opposite one. """ direction = int(scroll_direction) - amount = abs(int(scroll_value)) * (1 if direction > 0 else -1) + amount = int(scroll_value) * (1 if direction > 0 else -1) return (0, amount) if abs(direction) == 1 else (amount, 0) diff --git a/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py b/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py index 844a7733..154efe19 100644 --- a/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py +++ b/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py @@ -26,6 +26,17 @@ x11_linux_scroll_direction_left = 6 x11_linux_scroll_direction_right = 7 +#: Each scroll direction and the one a negative count turns it into. +#: Windows and macOS have always read the direction off the sign of +#: the count; X11 encodes direction as a *button*, so the sign has to +#: be turned back into one here rather than handed to the server. +_OPPOSITE_SCROLL_DIRECTION = { + x11_linux_scroll_direction_up: x11_linux_scroll_direction_down, + x11_linux_scroll_direction_down: x11_linux_scroll_direction_up, + x11_linux_scroll_direction_left: x11_linux_scroll_direction_right, + x11_linux_scroll_direction_right: x11_linux_scroll_direction_left, +} + def position() -> Tuple[int, int]: """ @@ -94,17 +105,25 @@ def scroll(scroll_value: int, scroll_direction: int) -> None: 模擬滑鼠滾動 :param scroll_value: number of scroll units 滾動次數 - 方向由 scroll_direction 決定，因此這裡只取絕對值。 - Direction comes from scroll_direction, so only the magnitude is used - here: range() on a negative value is empty, which silently scrolled - nothing at all. - :param scroll_direction: scroll direction 滾動方向 + 負數會反轉方向，與 Windows／macOS 一致；絕對值是滾動次數。 + A negative count reverses ``scroll_direction``, which is what + Windows and macOS have always done with the sign; the magnitude + is the number of notches. (It used to be discarded, so portable + code written against the Windows convention scrolled the *named* + direction instead of the opposite one — silently, and only on + this backend.) + :param scroll_direction: 未帶負號時的預設方向 The direction a + positive count scrolls in 4 = up 上 5 = down 下 6 = left 左 7 = right 右 """ - for _ in range(abs(int(scroll_value))): + scroll_value = int(scroll_value) + if scroll_value < 0: + scroll_direction = _OPPOSITE_SCROLL_DIRECTION.get( + scroll_direction, scroll_direction) + for _ in range(abs(scroll_value)): click_mouse(scroll_direction) diff --git a/je_auto_control/utils/action_signing/cipher.py b/je_auto_control/utils/action_signing/cipher.py index a80dc398..c7593bbb 100644 --- a/je_auto_control/utils/action_signing/cipher.py +++ b/je_auto_control/utils/action_signing/cipher.py @@ -14,8 +14,6 @@ from pathlib import Path from typing import Optional, Union -from cryptography.fernet import Fernet, InvalidToken - from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -27,6 +25,7 @@ def _persistent_key() -> bytes: """Read the per-user Fernet key, creating it (0600) on first use.""" + from cryptography.fernet import Fernet if _DEFAULT_KEY_PATH.exists(): return _DEFAULT_KEY_PATH.read_bytes() _DEFAULT_KEY_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -49,6 +48,7 @@ def _fernet_key(key: KeyType) -> bytes: def encrypt_action_file(path: Union[str, Path], key: KeyType = None) -> str: """Encrypt the file at ``path`` to ``.enc``; return the enc path.""" + from cryptography.fernet import Fernet target = Path(path) token = Fernet(_fernet_key(key)).encrypt(target.read_bytes()) enc_path = target.with_name(target.name + _ENC_SUFFIX) @@ -65,6 +65,7 @@ def decrypt_action_file(enc_path: Union[str, Path], key: KeyType = None, dropped. Raises :class:`AutoControlException` on a wrong key or a tampered file. """ + from cryptography.fernet import Fernet, InvalidToken enc = Path(enc_path) try: plaintext = Fernet(_fernet_key(key)).decrypt(enc.read_bytes()) diff --git a/je_auto_control/utils/annotate/annotate.py b/je_auto_control/utils/annotate/annotate.py index 08cf5376..615c6577 100644 --- a/je_auto_control/utils/annotate/annotate.py +++ b/je_auto_control/utils/annotate/annotate.py @@ -13,18 +13,24 @@ {"type": "arrow", "start": [x1, y1], "end": [x2, y2], "color": [...]} {"type": "text", "position": [x, y], "text": "step 3", "color": [...]} """ +from __future__ import annotations + import io import math from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union, +) -from PIL import Image, ImageDraw +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image, ImageDraw -ImageSource = Union[str, Path, bytes, Image.Image] +ImageSource = Union[str, Path, bytes, "Image.Image"] -def _load_image(source: ImageSource) -> Image.Image: +def _load_image(source: ImageSource) -> "Image.Image": """Load ``source`` (path / bytes / PIL image) as an RGBA image.""" + from PIL import Image if isinstance(source, Image.Image): return source.convert("RGBA") if isinstance(source, bytes): @@ -84,6 +90,7 @@ def annotate_screenshot(source: ImageSource, PIL image; ``annotations`` is a list of box / highlight / arrow / text dicts. Unknown annotation types are ignored. """ + from PIL import Image, ImageDraw base = _load_image(source) highlight_layer = Image.new("RGBA", base.size, (0, 0, 0, 0)) highlight_draw = ImageDraw.Draw(highlight_layer) diff --git a/je_auto_control/utils/color_stats/color_stats.py b/je_auto_control/utils/color_stats/color_stats.py index 5c3469f1..a3839f19 100644 --- a/je_auto_control/utils/color_stats/color_stats.py +++ b/je_auto_control/utils/color_stats/color_stats.py @@ -7,15 +7,20 @@ representative. Pure Pillow — no Qt, no screen capture — so it is fully unit-testable. """ +from __future__ import annotations + import io from collections import Counter from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union, +) -from PIL import Image +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image -ImageSource = Union[str, Path, bytes, Image.Image] +ImageSource = Union[str, Path, bytes, "Image.Image"] RGB = Tuple[int, int, int] @@ -32,7 +37,8 @@ def to_dict(self) -> Dict[str, Any]: return asdict(self) -def _load_rgb(source: ImageSource) -> Image.Image: +def _load_rgb(source: ImageSource) -> "Image.Image": + from PIL import Image if isinstance(source, Image.Image): return source.convert("RGB") if isinstance(source, bytes): diff --git a/je_auto_control/utils/cv2_utils/screen_record.py b/je_auto_control/utils/cv2_utils/screen_record.py index e46bd51a..bc4ddd72 100644 --- a/je_auto_control/utils/cv2_utils/screen_record.py +++ b/je_auto_control/utils/cv2_utils/screen_record.py @@ -1,6 +1,5 @@ import threading from typing import Dict, Tuple -import cv2 from je_auto_control.wrapper.auto_control_screen import screenshot @@ -69,6 +68,7 @@ class ScreenRecordThread(threading.Thread): def __init__(self, path_and_filename, codec, frame_per_sec, resolution: Tuple[int, int]): super().__init__() + import cv2 self.fourcc = cv2.VideoWriter.fourcc(*codec) self.video_writer = cv2.VideoWriter(path_and_filename, self.fourcc, frame_per_sec, resolution) # 用 Event 而非布林旗標:run() 之前呼叫 stop() 也能被遵守,不會被覆寫。 @@ -79,6 +79,7 @@ def __init__(self, path_and_filename, codec, frame_per_sec, resolution: Tuple[in self.resolution = resolution def run(self) -> None: + import cv2 try: while not self._stop_event.is_set(): # 擷取螢幕畫面 Capture screen frame diff --git a/je_auto_control/utils/cv2_utils/screenshot.py b/je_auto_control/utils/cv2_utils/screenshot.py index 3c6c84a4..649269f7 100644 --- a/je_auto_control/utils/cv2_utils/screenshot.py +++ b/je_auto_control/utils/cv2_utils/screenshot.py @@ -1,5 +1,9 @@ -from PIL import Image -from typing import List, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber from je_auto_control.utils.exception.exceptions import AutoControlScreenException diff --git a/je_auto_control/utils/cv2_utils/template_detection.py b/je_auto_control/utils/cv2_utils/template_detection.py index 9c151460..98f27bd5 100644 --- a/je_auto_control/utils/cv2_utils/template_detection.py +++ b/je_auto_control/utils/cv2_utils/template_detection.py @@ -8,8 +8,6 @@ """ from typing import Any, List, Optional, Sequence, Tuple -from je_open_cv import template_detection - from je_auto_control.utils.monitor_layout.logical_frame import grab_logical @@ -54,6 +52,7 @@ def find_image(image: Any, detect_threshold: float = 1.0, :param screen_region: Limit the search to (x, y, width, height) 限定搜尋範圍 :return: [found, [x1, y1, x2, y2]] 座標為螢幕座標 """ + from je_open_cv import template_detection grab_image, origin_x, origin_y = grab_logical( screen_region, all_screens=all_screens) result = template_detection.find_object( @@ -80,6 +79,7 @@ def find_image_multi(image: Any, detect_threshold: float = 1.0, :param screen_region: Limit the search to (x, y, width, height) 限定搜尋範圍 :return: [found, [[x1, y1, x2, y2], ...]] 座標為螢幕座標 """ + from je_open_cv import template_detection grab_image, origin_x, origin_y = grab_logical( screen_region, all_screens=all_screens) result = template_detection.find_multi_object( diff --git a/je_auto_control/utils/cv2_utils/video_recording.py b/je_auto_control/utils/cv2_utils/video_recording.py index 3128b8c2..8df2d28f 100644 --- a/je_auto_control/utils/cv2_utils/video_recording.py +++ b/je_auto_control/utils/cv2_utils/video_recording.py @@ -1,6 +1,4 @@ import threading -import cv2 -import numpy as np from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -44,6 +42,8 @@ def run(self): 執行錄影迴圈 Run recording loop """ + import cv2 + import numpy as np with mss_grabber() as sct: resolution = sct.monitors[0] output_file = self.video_name + ".mp4" diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 8b842ebb..ee2bfc39 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -49,8 +49,11 @@ def mouse_tools() -> List[MCPTool]: ), MCPTool( name="ac_mouse_scroll", - description=("Scroll the mouse wheel by scroll_value units. " - "scroll_direction is Linux-only: scroll_up | scroll_down."), + description=("Scroll the mouse wheel by scroll_value units; " + "a negative scroll_value reverses the direction. " + "scroll_direction names the direction a positive " + "value takes and is read on X11/Wayland only: " + "scroll_up | scroll_down | scroll_left | scroll_right."), input_schema=schema({ "scroll_value": {"type": "integer"}, "x": {"type": "integer"}, diff --git a/je_auto_control/utils/qr/qr.py b/je_auto_control/utils/qr/qr.py index b561eb4c..fb07c822 100644 --- a/je_auto_control/utils/qr/qr.py +++ b/je_auto_control/utils/qr/qr.py @@ -4,18 +4,24 @@ dependency), so no extra package is needed. The decoder is injectable so the wrapper is unit-testable without a real QR image. GUI-free. """ +from __future__ import annotations + import io from pathlib import Path -from typing import Any, Callable, List, Optional, Sequence, Union +from typing import ( + TYPE_CHECKING, Any, Callable, List, Optional, Sequence, Union, +) -from PIL import Image +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image -ImageSource = Union[str, Path, bytes, Image.Image] +ImageSource = Union[str, Path, bytes, "Image.Image"] QRDecoder = Callable[[Any], List[str]] def _load_np(source: ImageSource, region: Optional[Sequence[int]]): import numpy as np + from PIL import Image if isinstance(source, Image.Image): image = source elif isinstance(source, bytes): diff --git a/je_auto_control/utils/visual_regression/compare.py b/je_auto_control/utils/visual_regression/compare.py index d26d5f1b..12482002 100644 --- a/je_auto_control/utils/visual_regression/compare.py +++ b/je_auto_control/utils/visual_regression/compare.py @@ -4,9 +4,10 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Optional, Sequence, Tuple -from PIL import Image, ImageChops, ImageDraw +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image @dataclass(frozen=True) @@ -60,6 +61,7 @@ def _expand_path(path) -> Path: def _apply_masks(image: Image.Image, masks: Sequence[MaskRegion]) -> Image.Image: """Black out the masked regions on a *copy* so the input stays intact.""" + from PIL import ImageDraw if not masks: return image result = image.copy() @@ -110,6 +112,7 @@ def image_difference(actual: Image.Image, expected: Image.Image, the comparison). ``masks`` blacks out those regions on *both* sides before comparing. """ + from PIL import ImageChops, ImageDraw if actual.size != expected.size: raise ValueError( f"image sizes differ: actual={actual.size}, " @@ -151,6 +154,7 @@ def compare_to_golden(golden_path, a tiny rendering wobble doesn't fail every CI run). Defaults to ``0.0`` for strictest comparison. """ + from PIL import Image target = _expand_path(golden_path) if not target.exists(): raise FileNotFoundError(f"golden image not found: {target}") diff --git a/je_auto_control/wrapper/auto_control_mouse.py b/je_auto_control/wrapper/auto_control_mouse.py index ab97cb09..9f736fd6 100644 --- a/je_auto_control/wrapper/auto_control_mouse.py +++ b/je_auto_control/wrapper/auto_control_mouse.py @@ -12,6 +12,9 @@ AutoControlCantFindKeyException, AutoControlMouseException ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.platform_id import ( + is_macos, is_windows, is_x11_unix +) from je_auto_control.utils.test_record.record_test_class import record_action_to_list from je_auto_control.wrapper.auto_control_screen import screen_size from je_auto_control.wrapper.platform_wrapper import mouse, mouse_keys_table, special_mouse_keys_table @@ -242,10 +245,19 @@ def mouse_scroll(scroll_value: int, x: int = None, y: int = None, 模擬滑鼠滾輪操作 Simulate mouse scroll - :param scroll_value: 滾動數值 Scroll value + 每個平台的規則相同：``scroll_value`` 為負就反向，絕對值是滾動格數。 + The sign of ``scroll_value`` reverses the direction on every platform, so a + call written on one works on the others. X11 and Wayland used to discard it + and always scroll ``scroll_direction``, which meant portable code scrolled + the opposite way there with no error and no warning. + + :param scroll_value: 滾動數值，負數代表反向 Scroll value; negative reverses :param x: X 座標，指定時會先將游標移到該處 X position; the cursor moves here first :param y: Y 座標，指定時會先將游標移到該處 Y position; the cursor moves here first - :param scroll_direction: 滾動方向 (Linux only) Scroll direction + :param scroll_direction: 未帶負號時的方向，只有 X11／Wayland 後端會讀。 + The direction a *positive* count scrolls in. Only the X11 and Wayland + backends read it — Windows and macOS have a single wheel axis and take + the direction from the sign alone. :return: (scroll_value, scroll_direction) """ autocontrol_logger.info(f"mouse_scroll, value={scroll_value}, x={x}, y={y}, direction={scroll_direction}") @@ -261,13 +273,21 @@ def mouse_scroll(scroll_value: int, x: int = None, y: int = None, if x is not None or y is not None: _scroll_to(x, y) - if sys.platform in ["win32", "cygwin", "msys"]: - mouse.scroll(scroll_value) - elif sys.platform == "darwin": + # 用 platform_id 問「哪一種輸入堆疊」，而不是再列一次 OS 名單： + # 原本的 ["linux", "linux2"] 把 BSD 漏在所有分支之外，滾動在 + # FreeBSD 上不會報錯，只是什麼都不做。 + # Ask platform_id which input stack this is rather than spelling out + # another list of OS names: the ["linux", "linux2"] one left the BSDs + # outside every branch, so scrolling on FreeBSD raised nothing and + # did nothing. + if is_windows() or is_macos(): mouse.scroll(scroll_value) - elif sys.platform in ["linux", "linux2"]: + elif is_x11_unix(): scroll_direction = special_mouse_keys_table.get(scroll_direction, scroll_direction) mouse.scroll(scroll_value, scroll_direction) + else: + raise AutoControlMouseException( + f"mouse_scroll: no backend for {sys.platform!r}") record_action_to_list("mouse_scroll", param) return scroll_value, scroll_direction diff --git a/je_auto_control/wrapper/auto_control_screen.py b/je_auto_control/wrapper/auto_control_screen.py index 6d2571ce..46711836 100644 --- a/je_auto_control/wrapper/auto_control_screen.py +++ b/je_auto_control/wrapper/auto_control_screen.py @@ -1,9 +1,6 @@ import sys from typing import Tuple, List -import cv2 -import numpy as np - from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot from je_auto_control.utils.exception.exception_tags import screen_get_size_error_message from je_auto_control.utils.exception.exception_tags import screen_screenshot_error_message @@ -40,7 +37,13 @@ def screenshot(file_path: str = None, screen_region: list = None) -> List[int]: :param screen_region: screenshot region 截圖區域 """ autocontrol_logger.info(f"screenshot, file_path: {file_path}, screen_region: {screen_region}") - param = locals() + # 明寫成 dict,不用 locals():下面的 import 也會出現在 locals() 裡, + # 錄下來的參數就會多兩個模組物件。 + # Spelled out rather than locals(): the imports below land in locals() too, + # so the recorded parameters would carry two module objects. + param = {"file_path": file_path, "screen_region": screen_region} + import cv2 # noqa: E402 # reason: kept off the facade's import path + import numpy as np # noqa: E402 # reason: kept off the facade's import path try: record_action_to_list("AC_screenshot", param) return cv2.cvtColor( diff --git a/test/unit_test/headless/test_facade_import_is_light.py b/test/unit_test/headless/test_facade_import_is_light.py new file mode 100644 index 00000000..62dac290 --- /dev/null +++ b/test/unit_test/headless/test_facade_import_is_light.py @@ -0,0 +1,87 @@ +"""``import je_auto_control`` must not need the image/crypto wheels. + +The facade used to import OpenCV, NumPy, Pillow, ``je_open_cv`` and +``cryptography`` at module scope, so *any* import under ``je_auto_control`` +pulled all five in. That is what kept the X11 backend off a FreeBSD desktop: +none of those five publishes a FreeBSD wheel, and moving a mouse needs none of +them. They are now imported by the functions that use them. + +This is a property that regresses silently — one convenience import at the top +of one module puts the whole set back on the path, and every platform that has +wheels keeps passing. So the check is made the way a machine without them would +make it: the modules are blocked outright, in a subprocess, and the facade has +to import anyway. +""" +import os +import pathlib +import subprocess # nosec B404 # reason: fixed argv, sys.executable, no shell +import sys + +import pytest + +#: Blocked wholesale. ``mss`` is in the list because screen capture is as +#: optional to input automation as OpenCV is; it was already lazy, and this +#: keeps it that way. +HEAVY = ("cv2", "numpy", "PIL", "cryptography", "je_open_cv", "mss") + +#: The working tree, so the subprocess tests the checkout rather than whatever +#: version of the package happens to be installed in site-packages. +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + +_PROBE = ''' +import sys + + +class _Blocker: + """Refuse the heavy wheels the way an absent wheel would.""" + + BLOCKED = {blocked!r} + + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".")[0] in self.BLOCKED: + raise ImportError( + "No module named %r (blocked by the probe)" % fullname) + return None + + +sys.meta_path.insert(0, _Blocker()) +import je_auto_control # noqa: E402 + +leaked = sorted(name for name in sys.modules + if name.split(".")[0] in _Blocker.BLOCKED) +if leaked: + raise SystemExit("heavy modules reached sys.modules: %s" % leaked) +print("ok", len(je_auto_control.__all__)) +''' + + +def test_facade_imports_without_the_heavy_wheels(): + """The facade imports with OpenCV / Pillow / cryptography absent.""" + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT)) + result = subprocess.run( # nosec B603 # reason: fixed argv, no shell + [sys.executable, "-c", _PROBE.format(blocked=HEAVY)], + capture_output=True, text=True, timeout=180, check=False, env=env) + assert result.returncode == 0, ( + "import je_auto_control needs one of " + f"{', '.join(HEAVY)}:\n{result.stdout}\n{result.stderr}") + assert result.stdout.startswith("ok "), result.stdout + + +def test_the_lazy_modules_still_work_when_the_wheels_are_there(): + """Deferring an import must not have broken the function that uses it. + + The probe above proves the import path is clean; this proves the call path + still is, so a local import placed in the wrong function cannot pass as a + win. + """ + pytest.importorskip("PIL") + from PIL import Image + + from je_auto_control.utils.color_stats.color_stats import ( + region_color_stats, + ) + + image = Image.new("RGB", (8, 8), (10, 20, 30)) + stats = region_color_stats(image) + assert stats.average_rgb == (10, 20, 30) + assert stats.dominant_rgb == (10, 20, 30) diff --git a/test/unit_test/headless/test_r3_vision_capture.py b/test/unit_test/headless/test_r3_vision_capture.py index f8e36cd8..0fca56cd 100644 --- a/test/unit_test/headless/test_r3_vision_capture.py +++ b/test/unit_test/headless/test_r3_vision_capture.py @@ -12,7 +12,7 @@ import pytest np = pytest.importorskip("numpy") -pytest.importorskip("cv2") +cv2 = pytest.importorskip("cv2") from je_auto_control.utils.cv2_utils import screen_record as sr # noqa: E402 from je_auto_control.utils.cv2_utils import screenshot as ss # noqa: E402 @@ -43,7 +43,7 @@ def release(self): # --- finding 2: recorder stop honoured + writer released ----------------- def test_stop_before_run_is_honored(monkeypatch): - monkeypatch.setattr(sr.cv2, "VideoWriter", _FakeVideoWriter) + monkeypatch.setattr(cv2, "VideoWriter", _FakeVideoWriter) thread = sr.ScreenRecordThread("out.avi", "XVID", 30, (4, 4)) frame = np.zeros((4, 4, 3), dtype=np.uint8) @@ -64,7 +64,7 @@ def screenshot_bounds_the_buggy_path(): def test_normal_run_writes_frames_and_releases(monkeypatch): - monkeypatch.setattr(sr.cv2, "VideoWriter", _FakeVideoWriter) + monkeypatch.setattr(cv2, "VideoWriter", _FakeVideoWriter) thread = sr.ScreenRecordThread("out.avi", "XVID", 30, (4, 4)) frame = np.zeros((4, 4, 3), dtype=np.uint8) calls = {"n": 0} diff --git a/test/unit_test/headless/test_scroll_sign_is_portable.py b/test/unit_test/headless/test_scroll_sign_is_portable.py new file mode 100644 index 00000000..123532d0 --- /dev/null +++ b/test/unit_test/headless/test_scroll_sign_is_portable.py @@ -0,0 +1,170 @@ +"""The sign of ``scroll_value`` picks the direction on every platform. + +Windows and macOS have always read the direction off the sign. X11 encodes +direction as a button (4/5/6/7) rather than a signed delta, so it took the +direction from ``scroll_direction`` and discarded the sign — deliberately, +because a negative count used to make ``range()`` empty and scroll nothing. + +The cost was that ``mouse_scroll(-3)`` written and tested on Windows scrolled +*down* three notches on Linux instead of up, with no exception and no warning. +The maintainer settled it: the sign reverses the direction everywhere, and +``scroll_direction`` names the direction a positive count takes. + +The real X server side of this is pinned by ``docker/x11_verify.py``, which +reads the buttons back out of ``xev``. These tests hold the same contract +without a display, so a change to the mapping fails on every runner rather than +only in the container job. +""" +from __future__ import annotations + +import importlib +import sys +import types + +import pytest + + +@pytest.fixture() +def x11_mouse(monkeypatch): + """Import the X11 mouse backend with python-Xlib and the display faked.""" + monkeypatch.setattr(sys, "platform", "linux") + + fake_xlib = types.ModuleType("Xlib") + fake_xlib.X = types.SimpleNamespace( + ZPixmap=2, ButtonPress=4, ButtonRelease=5, CurrentTime=0) + fake_xlib.protocol = types.SimpleNamespace(event=types.SimpleNamespace()) + monkeypatch.setitem(sys.modules, "Xlib", fake_xlib) + monkeypatch.setitem(sys.modules, "Xlib.protocol", fake_xlib.protocol) + + xtest = types.ModuleType("Xlib.ext.xtest") + xtest.fake_input = lambda *args, **kwargs: None + monkeypatch.setitem(sys.modules, "Xlib.ext", types.ModuleType("Xlib.ext")) + monkeypatch.setitem(sys.modules, "Xlib.ext.xtest", xtest) + + display_mod = types.ModuleType("x11_linux_display") + display_mod.display = types.SimpleNamespace(sync=lambda: None) + monkeypatch.setitem( + sys.modules, + "je_auto_control.linux_with_x11.core.utils.x11_linux_display", + display_mod) + monkeypatch.delitem( + sys.modules, + "je_auto_control.linux_with_x11.mouse.x11_linux_mouse_control", + raising=False) + + module = importlib.import_module( + "je_auto_control.linux_with_x11.mouse.x11_linux_mouse_control") + clicked: list[int] = [] + monkeypatch.setattr(module, "click_mouse", clicked.append) + module.clicked = clicked # for the assertions below + return module + + +UP, DOWN, LEFT, RIGHT = 4, 5, 6, 7 + + +@pytest.mark.parametrize(("direction", "expected"), [ + (UP, UP), (DOWN, DOWN), (LEFT, LEFT), (RIGHT, RIGHT), +]) +def test_a_positive_count_scrolls_the_named_direction(x11_mouse, direction, + expected): + """``scroll_direction`` still means what it always meant.""" + x11_mouse.scroll(3, direction) + assert x11_mouse.clicked == [expected] * 3 + + +@pytest.mark.parametrize(("direction", "expected"), [ + (UP, DOWN), (DOWN, UP), (LEFT, RIGHT), (RIGHT, LEFT), +]) +def test_a_negative_count_reverses_it(x11_mouse, direction, expected): + """This is the behaviour change: the sign wins, as it does elsewhere.""" + x11_mouse.scroll(-3, direction) + assert x11_mouse.clicked == [expected] * 3 + + +def test_the_magnitude_is_still_the_notch_count(x11_mouse): + """A negative count must not go back to scrolling nothing at all.""" + x11_mouse.scroll(-5, DOWN) + assert len(x11_mouse.clicked) == 5 + + +def test_zero_scrolls_nothing(x11_mouse): + """Zero is neither direction and must stay a no-op.""" + x11_mouse.scroll(0, DOWN) + assert x11_mouse.clicked == [] + + +def test_an_unknown_direction_is_passed_through(x11_mouse): + """A button this table does not know is forwarded, not silently remapped.""" + x11_mouse.scroll(-1, 99) + assert x11_mouse.clicked == [99] + + +def test_the_wrapper_routes_the_bsds_to_the_x11_backend(monkeypatch): + """``["linux", "linux2"]`` left FreeBSD outside every branch. + + The wrapper matched Windows, then macOS, then a literal list of Linux + names — so on a BSD ``mouse_scroll`` fell off the end, raised nothing and + scrolled nothing. It asks ``platform_id`` which input stack this is now. + """ + from je_auto_control.wrapper import auto_control_mouse + + calls: list[tuple] = [] + monkeypatch.setattr(auto_control_mouse, "mouse", + types.SimpleNamespace( + scroll=lambda *args: calls.append(args))) + monkeypatch.setattr(auto_control_mouse, "special_mouse_keys_table", + {"scroll_down": DOWN}) + monkeypatch.setattr(sys, "platform", "freebsd14") + + auto_control_mouse.mouse_scroll(2, scroll_direction="scroll_down") + + assert calls == [(2, DOWN)], ( + "a BSD must reach the X11 backend; it used to match no branch at all") + + +# --- the Wayland backend, which had the same abs() ------------------------ + + +def _wayland_mouse(): + """The Wayland mouse module, or a skip when its imports are unavailable.""" + return pytest.importorskip( + "je_auto_control.linux_wayland.mouse", exc_type=ImportError) + + +@pytest.mark.parametrize(("direction_name", "expected"), [ + ("wayland_scroll_direction_up", (0, 3)), + ("wayland_scroll_direction_down", (0, -3)), + ("wayland_scroll_direction_left", (-3, 0)), + ("wayland_scroll_direction_right", (3, 0)), +]) +def test_wayland_positive_count_keeps_the_named_direction(direction_name, + expected): + """Unchanged behaviour, pinned so the sign fix cannot have moved it.""" + mouse = _wayland_mouse() + direction = getattr(mouse, direction_name) + assert mouse._wheel_deltas(3, direction) == expected + + +@pytest.mark.parametrize(("direction_name", "expected"), [ + ("wayland_scroll_direction_up", (0, -3)), + ("wayland_scroll_direction_down", (0, 3)), + ("wayland_scroll_direction_left", (3, 0)), + ("wayland_scroll_direction_right", (-3, 0)), +]) +def test_wayland_negative_count_reverses_it(direction_name, expected): + """``_wheel_deltas`` took ``abs()`` of the count, so the sign was lost. + + Wayland reaches the same wrapper branch as X11, so it had the same defect + and needed the same fix — otherwise "the sign wins everywhere" would have + been true of three backends out of four. + """ + mouse = _wayland_mouse() + direction = getattr(mouse, direction_name) + assert mouse._wheel_deltas(-3, direction) == expected + + +def test_wayland_zero_is_still_a_no_op(): + """Zero has no direction, and must not become a one-notch scroll.""" + mouse = _wayland_mouse() + assert mouse._wheel_deltas(0, mouse.wayland_scroll_direction_down) == (0, 0) diff --git a/test/verify/freebsd_verify.py b/test/verify/freebsd_verify.py new file mode 100644 index 00000000..6b8aef23 --- /dev/null +++ b/test/verify/freebsd_verify.py @@ -0,0 +1,358 @@ +"""Drive real X11 input from a real FreeBSD, and read it back off the server. + +The X11 backend was gated on ``sys.platform`` being ``linux``/``linux2``, so it +refused to load on a FreeBSD desktop running the same X server, the same +python-Xlib and the same code. Relaxing that guard is only worth something if a +BSD actually runs it, and no hosted runner is one — so +``.github/workflows/platform-smoke.yml`` boots a FreeBSD VM inside an Ubuntu +runner and runs this. + +For a while this could only check the *decision* — that ``sys.platform`` really +looks like ``freebsd14``, and that the classification every relaxed guard asks +answers correctly on it — because importing anything under ``je_auto_control`` +ran the facade, and the facade imported OpenCV and cryptography at module scope. +Neither publishes a FreeBSD wheel, and building them from ports had not finished +after fifty minutes. + +That was the wrong thing to fix. Moving a mouse needs neither package, and the +facade had no business insisting on them: they are imported by the functions +that use them now (``test_facade_import_is_light.py`` keeps it that way), which +leaves this VM needing python-Xlib and defusedxml — both pure Python — and an X +server. So the whole backend runs here, not just the guard. + +Ground truth is the X server answering for itself, never this codebase: +``query_pointer`` for where the cursor actually is, its button mask for which +buttons the server believes are down, and ``query_keymap`` — a bitmap of every +physically-pressed key — for whether an injected key press really landed. That +last one is what makes a second X client unnecessary here; the Linux +``x11-verification`` job already reads events back out of ``xev``, and what a +BSD is uniquely needed to answer is whether this code drives the same server on +a different kernel. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import os +import sys +import time +import traceback +from typing import Any, Callable, List, Tuple + +#: Points the cursor is driven to. The corners matter: an off-by-one in the +#: coordinate space shows up at (0, 0) and at the far edge, not in the middle. +PROBE_POINTS: Tuple[Tuple[int, int], ...] = ( + (321, 123), (0, 0), (1, 1), (640, 480), (1279, 1023), +) + +#: How long the server is given to settle after an injected event. +SETTLE = 0.05 + +_results: List[Tuple[str, bool, str]] = [] + + +def check(name: str, fn: Callable[[], Any]) -> Any: + """Run one check, record pass/fail, and keep going either way.""" + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a failed check must not stop the rest + _results.append((name, False, traceback.format_exc(limit=3).strip())) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True, str(detail))) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _assert_eq(actual: Any, expected: Any) -> str: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + return f"{actual!r}" + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +# --- the decision only a BSD can answer --------------------------------- + + +def check_platform_identity() -> None: + """What ``sys.platform`` is here, and what every relaxed guard makes of it.""" + from je_auto_control.utils import platform_id + + def _is_freebsd() -> str: + _assert_true(sys.platform.startswith("freebsd"), + f"not a FreeBSD: sys.platform is {sys.platform!r}") + return sys.platform + check("sys.platform really is a FreeBSD", _is_freebsd) + + # Before the guards were relaxed the answer to the second of these was + # False on exactly this platform, and the package refused to import at all. + check("is_bsd() recognises it", + lambda: _assert_true(platform_id.is_bsd(), "not recognised as a BSD")) + check("is_x11_unix() recognises it", + lambda: _assert_true(platform_id.is_x11_unix(), "not an X11 unix")) + check("is_windows() does not", + lambda: _assert_true(not platform_id.is_windows(), "claimed Windows")) + check("is_macos() does not", + lambda: _assert_true(not platform_id.is_macos(), "claimed macOS")) + check("current_family() is bsd", + lambda: _assert_eq(platform_id.current_family(), "bsd")) + + # The version suffix is the trap: sys.platform is freebsd14 here, never a + # bare "freebsd", so an equality check would match no real system at all. + def _suffixed() -> str: + _assert_true( + sys.platform != "freebsd", + "this release stopped carrying a version suffix; the prefix match " + "still works, but the comment explaining why it exists no longer " + "describes reality") + return sys.platform + check("sys.platform still carries the major version", _suffixed) + + +def check_facade_is_importable() -> None: + """The facade imports here — which is the thing that used to be impossible. + + And it imports *without* the wheels FreeBSD has none of. Asserting they are + genuinely absent is the point: if some future image happens to have OpenCV + installed, this job would quietly stop testing the property it exists for. + """ + def _absent() -> str: + import importlib.util + + present = [name for name in ("cv2", "PIL", "cryptography", "je_open_cv") + if importlib.util.find_spec(name) is not None] + _assert_true(not present, + f"expected these to be absent on the VM, found: {present}") + return "cv2, PIL, cryptography, je_open_cv all absent" + check("the heavy wheels really are not installed here", _absent) + + def _facade() -> str: + import je_auto_control + + return f"{len(je_auto_control.__all__)} public names" + check("import je_auto_control", _facade) + + def _backend() -> str: + from je_auto_control.wrapper import platform_wrapper + + module = platform_wrapper.mouse.__name__ + _assert_true("linux_with_x11" in module, + f"expected the X11 backend, got {module}") + return module + check("the wrapper selects the X11 backend on a BSD", _backend) + + +# --- input, driven and read back off the server ------------------------- + + +def check_mouse_position() -> None: + """Every point the backend is told to move to is where the server puts it.""" + from je_auto_control.linux_with_x11.mouse import x11_linux_mouse_control as m + + for x, y in PROBE_POINTS: + def _round_trip(x=x, y=y) -> str: + m.set_position(x, y) + time.sleep(SETTLE) + return _assert_eq(m.position(), (x, y)) + check(f"set_position({x}, {y}) lands where it was asked", _round_trip) + + +def check_mouse_buttons() -> None: + """A pressed button is a button the *server* reports as down.""" + from je_auto_control.linux_with_x11.core.utils.x11_linux_display import ( + display, + ) + from je_auto_control.linux_with_x11.mouse import x11_linux_mouse_control as m + + def _mask() -> int: + return display.screen().root.query_pointer()._data["mask"] + + #: Button 1 in the pointer mask reported by the server. + button_1 = 1 << 8 + + def _press() -> str: + m.set_position(400, 300) + m.press_mouse(m.x11_linux_mouse_left) + time.sleep(SETTLE) + held = _mask() + m.release_mouse(m.x11_linux_mouse_left) + time.sleep(SETTLE) + _assert_true(bool(held & button_1), + f"the server did not report button 1 down (mask {held:#x})") + _assert_true(not _mask() & button_1, + "button 1 stayed down after release_mouse") + return "pressed and released" + check("press_mouse reaches the server, release_mouse clears it", _press) + + def _click() -> str: + m.click_mouse(m.x11_linux_mouse_left, 500, 260) + time.sleep(SETTLE) + _assert_eq(m.position(), (500, 260)) + _assert_true(not _mask() & button_1, + "click_mouse left button 1 held down") + return "moved and clicked cleanly" + check("click_mouse moves and leaves no button held", _click) + + +def check_keyboard() -> None: + """An injected key press is a key the server reports as physically down. + + ``query_keymap`` is a 32-byte bitmap of every key currently held, straight + out of the server, so this needs no second client to read the event back. + """ + from je_auto_control.linux_with_x11.core.utils.x11_linux_display import ( + display, + ) + from je_auto_control.linux_with_x11.keyboard import ( + x11_linux_keyboard_control as k, + ) + + def _is_down(keycode: int) -> bool: + keymap = display.query_keymap() + return bool(keymap[keycode // 8] & (1 << (keycode % 8))) + + # Keycode 38 is 'a' on the standard PC layout Xvfb comes up with. The + # check does not depend on which character it produces — only on the + # server agreeing that this physical key went down and came back up. + keycode = 38 + + def _press() -> str: + _assert_true(not _is_down(keycode), "the key was already down") + k.press_key(keycode) + time.sleep(SETTLE) + down = _is_down(keycode) + k.release_key(keycode) + time.sleep(SETTLE) + _assert_true(down, "the server never saw the key go down") + _assert_true(not _is_down(keycode), "the key stayed down after release") + return f"keycode {keycode} down then up" + check("press_key/release_key reach the server", _press) + + +def check_scroll() -> None: + """Scrolling is the one BSD-only defect this job exists to catch. + + ``mouse_scroll`` matched Windows, then macOS, then a literal + ``["linux", "linux2"]`` — so on a BSD it fell off the end of the chain, + raised nothing and scrolled nothing. Nothing above would notice: the + pointer mask never shows a wheel button, because X11 delivers a scroll as a + press *and* release of button 4/5/6/7 too fast to sample. + + So this maps a real X window that has asked for button events, puts the + cursor inside it, and reads the buttons back out of the event queue — the + same ground truth the Linux job gets from ``xev``, without needing a second + process. It also holds the sign contract that was settled for all three + platforms: a negative count reverses the direction. + """ + from Xlib import X + + from je_auto_control.linux_with_x11.core.utils.x11_linux_display import ( + display, + ) + + screen = display.screen() + window = screen.root.create_window( + 0, 0, 400, 400, 0, screen.root_depth, + X.InputOutput, X.CopyFromParent, + background_pixel=screen.white_pixel, + event_mask=X.ButtonPressMask | X.ButtonReleaseMask) + window.map() + display.sync() + + def _drain() -> None: + while display.pending_events(): + display.next_event() + + def _collect(count: int) -> List[int]: + """The button of every ButtonPress that arrives, up to ``count``.""" + buttons: List[int] = [] + deadline = time.time() + 5.0 + while len(buttons) < count and time.time() < deadline: + if not display.pending_events(): + time.sleep(0.01) + continue + event = display.next_event() + if event.type == X.ButtonPress: + buttons.append(event.detail) + return buttons + + def _scroll(value: int, direction: str, expected: int) -> str: + import je_auto_control as ac + + ac.set_mouse_position(200, 200) + time.sleep(SETTLE) + _drain() + ac.mouse_scroll(value, scroll_direction=direction) + buttons = _collect(abs(value)) + _assert_eq(buttons, [expected] * abs(value)) + return f"{value:+d} {direction} arrived as button {expected}" + + # A positive count scrolls the direction it names... + check("scroll_down arrives as button 5", + lambda: _scroll(2, "scroll_down", 5)) + check("scroll_up arrives as button 4", + lambda: _scroll(2, "scroll_up", 4)) + # ...and a negative one reverses it, on this platform as on the others. + check("a negative count reverses scroll_down into button 4", + lambda: _scroll(-2, "scroll_down", 4)) + check("a negative count reverses scroll_up into button 5", + lambda: _scroll(-2, "scroll_up", 5)) + + window.destroy() + display.sync() + + +def check_facade_input() -> None: + """The same thing through the public API, so the binding is covered too.""" + def _through_facade() -> str: + import je_auto_control as ac + + ac.set_mouse_position(210, 340) + time.sleep(SETTLE) + return _assert_eq(tuple(ac.get_mouse_position()), (210, 340)) + check("set_mouse_position/get_mouse_position through the facade", + _through_facade) + + +def report_environment() -> None: + """Print what this is running on, so a failure has context in the log.""" + import platform + + print(f"uname : {' '.join(platform.uname())}") + print(f"sys.platform : {sys.platform}") + print(f"python : {sys.version.split()[0]}") + print(f"DISPLAY : {os.environ.get('DISPLAY', '(unset)')}") + print() + + +def summarise() -> int: + """Print the tally; return the number of failed checks.""" + failed = [name for name, ok, _ in _results if not ok] + print() + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + return len(failed) + + +def main() -> int: + report_environment() + check_platform_identity() + check_facade_is_importable() + check_mouse_position() + check_mouse_buttons() + check_scroll() + check_keyboard() + check_facade_input() + return summarise() + + +if __name__ == "__main__": + raise SystemExit(main()) From 2a70ccbd14aef253927bd8b607c1ae406d7a642f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 11:50:50 +0800 Subject: [PATCH 21/30] Bootstrap the FreeBSD VM's pip from ensurepip, not from pkg FreeBSD 14.2's package repository has python311 but not py311-pip, so `pkg install py311-pip` fails outright and the job never reaches the verification. The flavoured port names are the unreliable part here, which is also why the two dependencies come from PyPI by their own names rather than as py311-xlib and py311-defusedxml. Both are pure Python, so pip is enough. The externally-managed retry goes with it: FreeBSD does not mark the system Python that way, and an ensurepip-installed pip owns what it installed. Also re-measures _factories.py in the Progress.md exemption list, which the scroll tool's description moved from 8,968 to 8,972 lines. --- .github/workflows/platform-smoke.yml | 18 ++++++++---------- Progress.md | 2 +- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 50f88b02..2ef568ab 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -95,20 +95,18 @@ jobs: release: "14.2" usesh: true prepare: | - pkg install -y python311 py311-pip xorg-vfbserver + pkg install -y python311 xorg-vfbserver run: | set -eu echo "uname: $(uname -a)" - # The two pure-Python dependencies the facade still needs, at the - # versions pyproject pins, by their PyPI names. FreeBSD does not - # mark its system Python externally-managed today; if that changes - # the retry says so in the log rather than the job failing on a - # pip policy error. - python3.11 -m pip install --no-deps python-xlib==0.33 defusedxml==0.7.1 \ - || { echo "pip refused the system environment; retrying opted-in"; \ - python3.11 -m pip install --break-system-packages --no-deps \ - python-xlib==0.33 defusedxml==0.7.1; } + # pip comes from ensurepip, not from pkg: FreeBSD 14.2's repository + # has no py311-pip (the flavoured port names are not dependable + # here, while python311 itself is). The two dependencies are the + # only ones the facade still needs, both pure Python, at the + # versions pyproject pins and by their PyPI names. + python3.11 -m ensurepip --upgrade + python3.11 -m pip install --no-deps python-xlib==0.33 defusedxml==0.7.1 # The backend connects to a display at import time, so the server # has to be up first. 1280x1024 because the verification drives the diff --git a/Progress.md b/Progress.md index 77693cc0..0f853caa 100644 --- a/Progress.md +++ b/Progress.md @@ -27,7 +27,7 @@ | `utils/accessibility/backends/windows_backend.py` | 915 | 已拆出 `windows_query.py`（170）與 `windows_state.py`（98）。剩下的是同一套 UIA COM 生命週期管理,再拆會把 `CoInitialize`／介面釋放的配對邏輯切散。 | **本質豁免（依 `CLAUDE.md` 的「flat data tables」條款,不算既有豁免）**: -`utils/mcp_server/tools/_factories.py`（8,968,MCP 工具註冊表）、 +`utils/mcp_server/tools/_factories.py`（8,972,MCP 工具註冊表）、 `utils/executor/action_executor.py`（8,125,`AC_*` 分派表）、 `gui/script_builder/command_schema.py`（5,051,每個 `AC_*` 的參數 schema）、 `je_auto_control/__init__.py`（1,970,門面 re-export）、 From f5347eaad3d234cfb0e82ce1ea8e114e2c848b0f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 12:03:32 +0800 Subject: [PATCH 22/30] Name six explicitly, since --no-deps is the point of the FreeBSD job python-Xlib 0.33 imports six from Xlib.display, and --no-deps meant nothing installed it, so the verification died on ModuleNotFoundError before it reached a single check. Dropping --no-deps would have fixed it and thrown away what the flag is there for: it is what proves nothing heavy is being pulled in behind the job that exists to show the backend needs nothing heavy. six is pure Python and now named alongside the other two. Everything before that point worked on the first run: ensurepip bootstrapped pip, Xvfb came up on :99, and sys.platform read freebsd14. --- .github/workflows/platform-smoke.yml | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 2ef568ab..444d1a43 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -102,11 +102,18 @@ jobs: # pip comes from ensurepip, not from pkg: FreeBSD 14.2's repository # has no py311-pip (the flavoured port names are not dependable - # here, while python311 itself is). The two dependencies are the - # only ones the facade still needs, both pure Python, at the - # versions pyproject pins and by their PyPI names. + # here, while python311 itself is). These are the only dependencies + # the facade still needs, all pure Python, at the versions + # pyproject pins and by their PyPI names. + # + # --no-deps is the point of this job rather than a detail: it is + # what proves nothing heavy is being dragged in behind the + # verification. six is therefore named explicitly — python-Xlib + # 0.33 imports it from Xlib.display, and with --no-deps nothing + # else would install it. python3.11 -m ensurepip --upgrade - python3.11 -m pip install --no-deps python-xlib==0.33 defusedxml==0.7.1 + python3.11 -m pip install --no-deps \ + python-xlib==0.33 six defusedxml==0.7.1 # The backend connects to a display at import time, so the server # has to be up first. 1280x1024 because the verification drives the From d46bfecfa03d10e705ebeae648643a76285e41c0 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 12:45:57 +0800 Subject: [PATCH 23/30] Stop needing a database to move a mouse, and let a BSD prove it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The FreeBSD VM was added to test the X11 backend and never got there: FreeBSD's python311 has no sqlite3 — it is the separate databases/py-sqlite3 package — and ten subsystems imported it at module scope, every one of them reachable from the facade. So `import je_auto_control` failed outright on a stock FreeBSD, on a machine where moving a mouse needs no database at all. Same shape as the OpenCV finding one commit earlier, from a direction no reasoning about wheels reaches: the standard library is not the same size on every platform. They go through utils/sqlite_support now, which fails at the first call that opens a database instead of at import, and raises the unsupported-operation type the GUI tabs, the REST handler and the executor already report as "not available here" rather than an ImportError none of them catch. HistoryStore connects on first use for the same reason its singleton is built while the facade is importing — which is also why importing the package no longer creates ~/.je_auto_control as a side effect. The FreeBSD job asserts the module is absent, so an image that later ships py311-sqlite3 goes red instead of quietly retiring the property. --- .github/workflows/platform-smoke.yml | 7 + CHANGELOG.md | 24 +++ WHATS_NEW.md | 42 +++++ architecture_explore.md | 57 ++++--- .../utils/agent_memory/agent_memory.py | 21 ++- je_auto_control/utils/chatops/router.py | 6 +- .../utils/checkpoint/checkpoint.py | 17 +- .../utils/data_source/data_source.py | 8 +- .../utils/diagnostics/diagnostics.py | 10 ++ je_auto_control/utils/mcp_server/_protocol.py | 4 +- .../utils/remote_desktop/audit_log.py | 19 ++- je_auto_control/utils/rest_api/rest_server.py | 11 +- .../utils/run_history/history_store.py | 79 ++++++--- je_auto_control/utils/sql/sql_query.py | 14 +- je_auto_control/utils/sqlite_support.py | 56 ++++++ .../utils/work_queue/work_queue.py | 19 ++- .../headless/test_sqlite_is_optional.py | 159 ++++++++++++++++++ test/verify/freebsd_verify.py | 51 ++++++ 18 files changed, 506 insertions(+), 98 deletions(-) create mode 100644 je_auto_control/utils/sqlite_support.py create mode 100644 test/unit_test/headless/test_sqlite_is_optional.py diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 444d1a43..d964e514 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -111,6 +111,13 @@ jobs: # verification. six is therefore named explicitly — python-Xlib # 0.33 imports it from Xlib.display, and with --no-deps nothing # else would install it. + # + # py311-sqlite3 is deliberately not installed either. FreeBSD + # packages sqlite3 apart from python311, this VM is the only + # machine in CI that does, and it is what caught ten subsystems + # importing it at module scope — which made `import + # je_auto_control` fail outright on a stock FreeBSD. Adding the + # package here would make that regression invisible again. python3.11 -m ensurepip --upgrade python3.11 -m pip install --no-deps \ python-xlib==0.33 six defusedxml==0.7.1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a66b828..49d2e362 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -145,6 +145,14 @@ only when documented here with a migration path. ### Changed +- The default run-history database is created when it is first written to, + not while `je_auto_control` is being imported. `HistoryStore` opens its + connection (and makes its parent directory) on first use, so merely + importing the package no longer creates + `~/.je_auto_control/run_history.sqlite`. Every method behaves as before; + a store that was never used and then closed simply never touched the + disk. + - **The sign of `scroll_value` picks the scroll direction on every platform.** Windows and macOS have always read it that way; X11 and Wayland took the direction from `scroll_direction` alone and used `abs(scroll_value)`, so @@ -308,6 +316,22 @@ only when documented here with a migration path. ### Fixed +- **`import je_auto_control` needed a Python built with `sqlite3`, and + FreeBSD's is not.** `sqlite3` is in the standard library but not in every + build of it: CPython links it against a system library, and FreeBSD ships + the result as the separate `databases/py-sqlite3` package. Ten subsystems + imported it at module scope — run history, checkpoints, the work queue, + agent memory, the remote-desktop audit log, SQL data sources, and the + error tuples in the REST, chat-ops and MCP containment boundaries — and + all ten are reachable from the facade, so the whole package failed to + import on a stock FreeBSD, mouse and keyboard included. They go through + `je_auto_control.utils.sqlite_support` now, which fails at the first call + that opens a database rather than at import, and raises + `AutoControlUnsupportedOperationException` — the type the GUI tabs, the + REST handler and the executor already report as "not available here" — + instead of an `ImportError` none of them catch. `run_diagnostics()` lists + `sqlite3` among the optional dependencies, so the gap is visible without + reading a traceback. - **`mouse_scroll` did nothing at all on the BSDs.** It matched Windows, then macOS, then a literal `["linux", "linux2"]`, so a FreeBSD, OpenBSD, NetBSD or DragonFly caller fell off the end of the chain: no backend call, no diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 341cee25..1a21b246 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -42,6 +42,48 @@ exactly, and the AX walk returns real elements. That was measured first and asserted second, and the probe still refuses to pass while its expectations table is empty. +### A BSD Found the Same Mistake in a Second Place + +The FreeBSD VM was added to prove the X11 backend drives a real BSD. It failed +before it got there, and what it failed on was not a wheel: **FreeBSD's +`python311` has no `sqlite3`.** The module is in the standard library but not +in every build of it — CPython links it against a system library, and FreeBSD +packages the result separately as `databases/py-sqlite3`. + +Ten subsystems imported it at module scope: run history, checkpoints, the work +queue, agent memory, the remote-desktop audit log, SQL data sources, and the +`except` tuples that keep a database error from killing the REST handler +thread, the chat-ops poll loop and the MCP transport. Every one of them is +reachable from the facade, so `import je_auto_control` failed outright — on a +machine where moving a mouse needs no database at all. This is the same shape +as the OpenCV/Pillow finding one commit earlier, from a source that reasoning +about wheels would never reach: the standard library is not the same size on +every platform. + +**The ten now go through `je_auto_control/utils/sqlite_support.py`.** +`require_sqlite3()` returns the module or raises +`AutoControlUnsupportedOperationException` — deliberately the type the platform +backends already raise for something they cannot do, so the GUI tabs, the REST +handler and the executor report "not available here" instead of dying on an +`ImportError` that none of them catch. `SQLITE_ERRORS` and +`SQLITE_OPERATIONAL_ERRORS` are tuples rather than classes, so +`except (ValueError, *SQLITE_ERRORS)` stays a valid handler that catches +exactly the right amount — nothing — where nothing can raise them. + +That left one thing still opening a database during import: `HistoryStore` +connected in its constructor, and `default_history_store` is built while the +facade is importing. It connects on first use now, which is also why +`import je_auto_control` no longer creates `~/.je_auto_control/` as a side +effect of being imported. + +Three things keep it fixed. `test_sqlite_is_optional.py` blocks `_sqlite3` in a +subprocess — exactly what FreeBSD reports — and requires the facade to import +anyway, with the error tuples empty. The FreeBSD job asserts the module is +*absent* on the VM, so a future image that happens to ship `py311-sqlite3` +turns the job red rather than quietly retiring the property it was added to +test. And `run_diagnostics()` lists `sqlite3` among the optional dependencies, +so an operator sees the gap as a line in a report instead of a traceback. + ### The macOS Recorder Was Written, Unreachable, and Wrong `OSXRecorder` had been a complete implementation for as long as diff --git a/architecture_explore.md b/architecture_explore.md index 0f185f04..ce9ad6d1 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,14 +19,14 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,027 | -| 程式碼總行數 | 139,499 | +| Python 模組總數（含周邊子專案） | 1,028 | +| 程式碼總行數 | 139,623 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | | GUI 分頁數（`main_widget` 註冊） | 48 | | MCP 工具數（`build_default_tool_registry()` 實測） | 676 | -| `test_*.py` 測試檔／測試函式 | 466 / 4,443 | +| `test_*.py` 測試檔／測試函式 | 478 / 4,654 | | 範例腳本 | 27 | **技術基線**：Python ≥ 3.10、MIT 授權、必要相依只有 `je_open_cv`／`opencv-python`／`pillow`／`mss`／ @@ -159,6 +159,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | | `je_auto_control/utils/deprecation.py` | 35 | 公開 API 的一致性棄用警告。 | | `je_auto_control/utils/http_headers.py` | 32 | 入站 HTTP 標頭的共用防禦式解析。 | +| `je_auto_control/utils/sqlite_support.py` | 56 | 選用標準函式庫 `sqlite3` 的取用點：`require_sqlite3()`／`sqlite3_available()`／`SQLITE_ERRORS`。十個以 SQLite 存放狀態的子系統都經由這裡，所以 FreeBSD 這種把 `sqlite3` 另外包成 `databases/py-sqlite3` 的 Python 仍然 import 得起門面。 | ### 5.2 wrapper 抽象層 @@ -265,13 +266,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,871 行。 +> 24 個套件、約 12,881 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/action_lint/` | 328 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | | `utils/action_signing/` | 230 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | -| `utils/checkpoint/` | 115 | 流程檢查點與續跑，讓長 action list 具持久性 | +| `utils/checkpoint/` | 120 | 流程檢查點與續跑，讓長 action list 具持久性 | | `utils/codegen/` | 157 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | | `utils/dag/` | 475 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 103 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | @@ -292,18 +293,18 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/state_machine/` | 181 | 宣告式有限狀態機驅動 action JSON | | `utils/stubs/` | 236 | 為 `AC_*` 指令面產生型別 stub | | `utils/test_record/` | 64 | 全域測試紀錄單例，記錄每個動作的參數與例外 | -| `utils/work_queue/` | 174 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | +| `utils/work_queue/` | 179 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 14 個套件、約 2,639 行。 +> 14 個套件、約 2,649 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/callback/` | 200 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | | `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | -| `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | +| `utils/diagnostics/` | 322 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | | `utils/dbus_client/` | 680 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | | `utils/exception/` | 210 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | @@ -487,13 +488,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,180 行。 +> 13 個套件、約 20,185 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a2a/` | 92 | A2A（agent-to-agent）agent card 產生 | | `utils/agent/` | 1,250 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | -| `utils/agent_memory/` | 146 | agent 的持久化情節記憶（goal → trajectory → outcome） | +| `utils/agent_memory/` | 151 | agent 的持久化情節記憶（goal → trajectory → outcome） | | `utils/agent_replay/` | 63 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | | `utils/agent_trace/` | 129 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | | `utils/cost_telemetry/` | 292 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | @@ -507,20 +508,20 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,712 行。 +> 6 個套件、約 17,717 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 327 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | | `utils/config_sync/` | 245 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 11,835 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/remote_desktop/` | 11,840 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | | `utils/usb/` | 4,247 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 -> 24 個套件、約 5,881 行。 +> 24 個套件、約 5,882 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -542,7 +543,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/otp/` | 37 | TOTP 一次性密碼產生（自動化 2FA 登入） | | `utils/outbox/` | 92 | 交易式 outbox，保證至少一次的事件投遞 | | `utils/pytest_plugin/` | 373 | pytest 外掛 + BDD step library（`pytest11` entry point） | -| `utils/rest_api/` | 1,738 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/rest_api/` | 1,739 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | | `utils/socket_server/` | 131 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | | `utils/sse_client/` | 112 | Server-Sent Events 用戶端解析 | | `utils/tls_acme/` | 441 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | @@ -551,7 +552,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 6,865 行。 +> 34 個套件、約 6,896 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -576,7 +577,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/profiler/` | 422 | 逐動作效能剖析器 + 資源剖析器 | | `utils/quarantine/` | 190 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | | `utils/run_diff/` | 123 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | -| `utils/run_history/` | 346 | 執行歷史儲存與產出物管理 | +| `utils/run_history/` | 377 | 執行歷史儲存與產出物管理 | | `utils/sarif/` | 134 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | | `utils/slo/` | 112 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | | `utils/smoothing/` | 67 | 數列移動平均平滑 | @@ -592,7 +593,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.13 資料來源、結構驗證與 i18n -> 24 個套件、約 3,886 行。 +> 24 個套件、約 3,892 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -601,7 +602,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/data_drift/` | 125 | 分布漂移偵測 | | `utils/data_profile/` | 121 | 資料剖析與結構推斷 | | `utils/data_quality/` | 185 | 資料品質：列結構驗證、欄位擷取、遮蔽 | -| `utils/data_source/` | 180 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | +| `utils/data_source/` | 182 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | | `utils/dataset_diff/` | 89 | 表格資料列差異比對（CDC 風格） | | `utils/gettext_catalog/` | 296 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | | `utils/i18n_test/` | 130 | 國際化／在地化測試輔助 | @@ -617,7 +618,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/pdf/` | 87 | PDF 讀取與斷言（選用 pypdf 後端） | | `utils/referential/` | 75 | 跨資料集的參照完整性檢查 | | `utils/schema_compat/` | 162 | JSON Schema 相容性分級 | -| `utils/sql/` | 74 | 對 SQLite 的臨時唯讀 SQL 查詢 | +| `utils/sql/` | 78 | 對 SQLite 的臨時唯讀 SQL 查詢 | | `utils/test_data/` | 205 | 帶種子的合成測試資料產生（純標準庫） | | `utils/xml/` | 250 | XML 檔讀寫與結構變更（`defusedxml`） | @@ -722,7 +723,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | | `__main__.py` | 87 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（11,835 行／56 檔） +#### `utils/remote_desktop/`（11,840 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 @@ -738,7 +739,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `webrtc_transport.py` | 360 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | | `multi_viewer.py` | 314 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | | `signaling_server.py` | 297 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | -| `audit_log.py` | 283 | SQLite 雜湊鏈稽核記錄。 | +| `audit_log.py` | 288 | SQLite 雜湊鏈稽核記錄。 | | `host_capture.py` | 280 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | | `ws_protocol.py` | 277 | 最小 RFC 6455 WebSocket 框架與握手。 | | `file_transfer.py` | 273 | 分塊檔案傳輸。 | @@ -796,11 +797,11 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `usbip/libusb_backend.py` | 209 | 以 PyUSB／libusb 執行 URB 的正式後端。 | | `usbip/backend.py` | 88 | 可插拔 URB 執行後端。 | -#### `utils/rest_api/`（1,738 行） +#### `utils/rest_api/`（1,739 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `rest_server.py` | 467 | HTTP 前端主體。 | +| `rest_server.py` | 468 | HTTP 前端主體。 | | `rest_handlers.py` | 486 | 端點實作。 | | `rest_openapi.py` | 422 | 走訪路由表產生 OpenAPI 3.1 規格。 | | `rest_auth.py` | 143 | Bearer token 驗證 + 逐 client 限流閘門。 | @@ -963,7 +964,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | | `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | -| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`verify`、`test_source` | 466 個 `test_*.py`／4,443 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 `verify/macos_verify.py` 是在真的 `macos-14` runner 上量測 TCC 到底允許什麼的探針（macOS 是唯一沒有容器可用的支援平台），不被 pytest 收集。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`verify`、`test_source` | 478 個 `test_*.py`／4,654 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 `verify/macos_verify.py` 是在真的 `macos-14` runner 上量測 TCC 到底允許什麼的探針（macOS 是唯一沒有容器可用的支援平台），不被 pytest 收集。 | --- @@ -1019,14 +1020,14 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | --- | ---: | ---: | | `gui/` | 89 | 26,542 | | `utils/mcp_server/` | 20 | 16,898 | -| `utils/remote_desktop/` | 56 | 11,835 | +| `utils/remote_desktop/` | 56 | 11,840 | | `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,247 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,363 | | `utils/accessibility/` | 13 | 2,818 | | `wrapper/` | 3,065 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | | `windows/` | 23 | 1,894 | -| `utils/rest_api/` | 8 | 1,738 | +| `utils/rest_api/` | 8 | 1,739 | | `utils/agent/` | 8 | 1,250 | | `linux_with_x11/` | 19 | 1,215 | | `linux_wayland/` | 17 | 2,835 | @@ -1037,6 +1038,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 907 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 689 | 50,305 | -| **總計** | **1,021** | **139,434** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 690 | 50,423 | +| **總計** | **1,022** | **139,558** | diff --git a/je_auto_control/utils/agent_memory/agent_memory.py b/je_auto_control/utils/agent_memory/agent_memory.py index cffbfadf..15cd44c8 100644 --- a/je_auto_control/utils/agent_memory/agent_memory.py +++ b/je_auto_control/utils/agent_memory/agent_memory.py @@ -16,10 +16,14 @@ """ import json import re -import sqlite3 import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 _TOKEN = re.compile(r"[a-z0-9]+") @@ -40,7 +44,7 @@ def _tokens(text: str) -> List[str]: return _TOKEN.findall((text or "").lower()) -def _row_to_episode(row: sqlite3.Row, score: float = 0.0) -> Episode: +def _row_to_episode(row: "sqlite3.Row", score: float = 0.0) -> Episode: return Episode( id=int(row["id"]), goal=row["goal"], steps=json.loads(row["steps"] or "[]"), @@ -55,10 +59,11 @@ def __init__(self, db_path: str) -> None: self._db_path = db_path self._ensure_schema() - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self._db_path, timeout=30.0, - isolation_level=None) - conn.row_factory = sqlite3.Row + def _connect(self) -> "sqlite3.Connection": + driver = require_sqlite3() + conn = driver.connect(self._db_path, timeout=30.0, + isolation_level=None) + conn.row_factory = driver.Row return conn def _ensure_schema(self) -> None: @@ -131,7 +136,7 @@ def stats(self) -> Dict[str, int]: return {"episodes": int(row["c"])} -def _relevance(row: sqlite3.Row, terms: List[str]) -> float: +def _relevance(row: "sqlite3.Row", terms: List[str]) -> float: haystack = " ".join([row["goal"] or "", row["outcome"] or "", " ".join(json.loads(row["tags"] or "[]"))]) counts = _tokens(haystack) diff --git a/je_auto_control/utils/chatops/router.py b/je_auto_control/utils/chatops/router.py index 0521bdc7..ede88c72 100644 --- a/je_auto_control/utils/chatops/router.py +++ b/je_auto_control/utils/chatops/router.py @@ -10,12 +10,12 @@ from __future__ import annotations import shlex -import sqlite3 import threading from dataclasses import asdict, dataclass, field from typing import Any, Callable, Dict, List, Optional from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.sqlite_support import SQLITE_ERRORS # Built-in commands always available even when the operator only @@ -138,14 +138,14 @@ def _dispatch_argv(self, argv: List[str], ) # The router is the containment boundary for handler failures: a bad # script (AutoControlException) or a run-history read error - # (sqlite3.Error) must come back as a chat reply, not escape and kill + # (a sqlite3 error) must come back as a chat reply, not escape and kill # the transport's poll loop. try: return spec.handler(rest, context) except ChatOpsError as error: return CommandResult(text=f"{name}: {error}", succeeded=False) except (RuntimeError, OSError, ValueError, TypeError, LookupError, - AttributeError, AutoControlException, sqlite3.Error) as error: + AttributeError, AutoControlException, *SQLITE_ERRORS) as error: return CommandResult( text=f"{name} failed: {type(error).__name__}: {error}", succeeded=False, diff --git a/je_auto_control/utils/checkpoint/checkpoint.py b/je_auto_control/utils/checkpoint/checkpoint.py index 9bad332e..542c4802 100644 --- a/je_auto_control/utils/checkpoint/checkpoint.py +++ b/je_auto_control/utils/checkpoint/checkpoint.py @@ -11,10 +11,14 @@ without a real crash. """ import json -import sqlite3 import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 @dataclass @@ -33,10 +37,11 @@ def __init__(self, db_path: str) -> None: self._db_path = db_path self._ensure_schema() - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self._db_path, timeout=30.0, - isolation_level=None) - conn.row_factory = sqlite3.Row + def _connect(self) -> "sqlite3.Connection": + driver = require_sqlite3() + conn = driver.connect(self._db_path, timeout=30.0, + isolation_level=None) + conn.row_factory = driver.Row return conn def _ensure_schema(self) -> None: diff --git a/je_auto_control/utils/data_source/data_source.py b/je_auto_control/utils/data_source/data_source.py index 80f24de4..cc7b1f49 100644 --- a/je_auto_control/utils/data_source/data_source.py +++ b/je_auto_control/utils/data_source/data_source.py @@ -23,12 +23,13 @@ import csv import json import os -import sqlite3 from contextlib import closing from pathlib import Path from typing import Any, Callable, Dict, List, Optional from urllib.parse import quote +from je_auto_control.utils.sqlite_support import require_sqlite3 + _READ_ONLY_SQL_PREFIXES = ("select", "with") @@ -96,8 +97,9 @@ def _load_sqlite(source: Dict[str, Any]) -> List[Dict[str, Any]]: uri = f"file:{safe_path}?mode=ro" # closing(): the sqlite3 context manager commits/rolls back but never closes # the connection, so the file handle leaks until GC. Close it explicitly. - with closing(sqlite3.connect(uri, uri=True)) as conn: - conn.row_factory = sqlite3.Row + driver = require_sqlite3() + with closing(driver.connect(uri, uri=True)) as conn: + conn.row_factory = driver.Row rows = conn.execute(query).fetchall() return [dict(row) for row in rows] diff --git a/je_auto_control/utils/diagnostics/diagnostics.py b/je_auto_control/utils/diagnostics/diagnostics.py index 4b214995..71ce1e5c 100644 --- a/je_auto_control/utils/diagnostics/diagnostics.py +++ b/je_auto_control/utils/diagnostics/diagnostics.py @@ -99,6 +99,7 @@ def _check_optional_deps() -> Check: ("pytesseract", "OCR engine"), ("cv2", "image recognition"), ("PySide6", "GUI"), + ("sqlite3", "run history / checkpoints / work queue / SQL sources"), ) available, missing = [], [] for module_name, purpose in optional_modules: @@ -122,6 +123,15 @@ def _check_optional_deps() -> Check: def _check_audit_chain() -> Check: from je_auto_control.utils.remote_desktop.audit_log import default_audit_log + from je_auto_control.utils.sqlite_support import sqlite3_available + + # Without sqlite3 there is no chain to verify. Reporting that as a + # broken chain would be a false alarm about tampering. + if not sqlite3_available(): + return Check( + name="audit_chain", ok=True, severity=_SEVERITY_WARN, + detail="no audit log on this Python: it has no sqlite3", + ) result = default_audit_log().verify_chain() if result.ok: return Check( diff --git a/je_auto_control/utils/mcp_server/_protocol.py b/je_auto_control/utils/mcp_server/_protocol.py index f5e1502e..7c7f1381 100644 --- a/je_auto_control/utils/mcp_server/_protocol.py +++ b/je_auto_control/utils/mcp_server/_protocol.py @@ -10,7 +10,6 @@ """ import json import os -import sqlite3 import subprocess # nosec B404 # reason: only its TimeoutExpired type is referenced import sys import time @@ -19,6 +18,7 @@ from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.mcp_server.tools import MCPContent +from je_auto_control.utils.sqlite_support import SQLITE_ERRORS PROTOCOL_VERSION = "2025-06-18" @@ -34,7 +34,7 @@ # family base every ``AutoControl*Exception``/``ImageNotFoundException`` now # derives from. _FRAMEWORK_TOOL_ERRORS = ( - AutoControlException, subprocess.TimeoutExpired, sqlite3.Error, + AutoControlException, subprocess.TimeoutExpired, *SQLITE_ERRORS, ) _BUILTIN_DISPATCH_ERRORS = ( OSError, RuntimeError, ValueError, TypeError, KeyError, diff --git a/je_auto_control/utils/remote_desktop/audit_log.py b/je_auto_control/utils/remote_desktop/audit_log.py index 686ad9eb..d3b5b94e 100644 --- a/je_auto_control/utils/remote_desktop/audit_log.py +++ b/je_auto_control/utils/remote_desktop/audit_log.py @@ -20,7 +20,6 @@ import hashlib import json import os -import sqlite3 import threading from dataclasses import dataclass from datetime import datetime, timezone @@ -28,6 +27,9 @@ from typing import List, Optional, Tuple from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.sqlite_support import ( + SQLITE_ERRORS, SQLITE_OPERATIONAL_ERRORS, require_sqlite3, +) _DEFAULT_PATH_RELATIVE = ".je_auto_control/audit.db" @@ -55,8 +57,11 @@ class AuditLog: def __init__(self, path: Optional[Path] = None) -> None: self._path = Path(path) if path is not None else default_audit_log_path() self._lock = threading.Lock() + # Asked for before the directory is made, so a Python without + # sqlite3 does not leave an empty ~/.je_auto_control behind. + driver = require_sqlite3() self._path.parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect( + self._conn = driver.connect( str(self._path), check_same_thread=False, isolation_level=None, ) self._init_schema() @@ -87,11 +92,11 @@ def _init_schema(self) -> None: # raw-SQL-construction rules without resorting to suppressions. try: self._conn.execute("ALTER TABLE events ADD COLUMN prev_hash TEXT") - except sqlite3.OperationalError: + except SQLITE_OPERATIONAL_ERRORS: pass # Column already exists — that's fine. try: self._conn.execute("ALTER TABLE events ADD COLUMN row_hash TEXT") - except sqlite3.OperationalError: + except SQLITE_OPERATIONAL_ERRORS: pass # Column already exists — that's fine. self._backfill_chain_locked() @@ -148,7 +153,7 @@ def log(self, event_type: str, *, ) self._last_hash = row_hash self._maybe_prune_locked() - except sqlite3.Error as error: + except SQLITE_ERRORS as error: autocontrol_logger.warning("audit log insert: %r", error) def _maybe_prune_locked(self) -> None: @@ -178,7 +183,7 @@ def query(self, *, try: cur = self._conn.execute(sql, args) rows = cur.fetchall() - except sqlite3.Error as error: + except SQLITE_ERRORS as error: autocontrol_logger.warning("audit log query: %r", error) return [] return [ @@ -227,7 +232,7 @@ def close(self) -> None: with self._lock: try: self._conn.close() - except sqlite3.Error: + except SQLITE_ERRORS: pass diff --git a/je_auto_control/utils/rest_api/rest_server.py b/je_auto_control/utils/rest_api/rest_server.py index e2a5f6b5..1582374e 100644 --- a/je_auto_control/utils/rest_api/rest_server.py +++ b/je_auto_control/utils/rest_api/rest_server.py @@ -11,7 +11,6 @@ import json import re -import sqlite3 import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -38,6 +37,7 @@ handle_usb_remote_open, ) from je_auto_control.utils.rest_api.rest_metrics import RestMetrics +from je_auto_control.utils.sqlite_support import SQLITE_ERRORS HandlerFn = Callable[[RouteContext], HandlerResult] @@ -191,11 +191,12 @@ def _dispatch(self, method: str, routes: Dict[str, HandlerFn], try: status, payload = handler(ctx) # AutoControlException is the family base (every framework error derives - # from it). sqlite3.Error is separate: handlers such as /history read the - # shared run-history DB, and a locked/corrupt DB otherwise escaped the - # handler thread and dropped the connection with no response. + # from it). The sqlite3 errors are separate: handlers such as /history + # read the shared run-history DB, and a locked/corrupt DB otherwise + # escaped the handler thread and dropped the connection with no + # response. The tuple is empty on a Python built without sqlite3. except (OSError, RuntimeError, ValueError, TypeError, - AutoControlException, sqlite3.Error) as error: + AutoControlException, *SQLITE_ERRORS) as error: autocontrol_logger.error( "rest-api %s %s handler raised: %r", method, parsed.path, error, ) diff --git a/je_auto_control/utils/run_history/history_store.py b/je_auto_control/utils/run_history/history_store.py index b02db3a9..78f814eb 100644 --- a/je_auto_control/utils/run_history/history_store.py +++ b/je_auto_control/utils/run_history/history_store.py @@ -6,14 +6,19 @@ pruning via :meth:`clear` or :meth:`prune`. """ import os -import sqlite3 import threading import time from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.sqlite_support import ( + SQLITE_ERRORS, require_sqlite3, +) + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 SOURCE_SCHEDULER = "scheduler" SOURCE_TRIGGER = "trigger" @@ -96,23 +101,41 @@ class HistoryStore: def __init__(self, path: Union[str, Path] = _IN_MEMORY_DB) -> None: self._path = str(path) if path == _IN_MEMORY_DB else str(Path(path)) + self._lock = threading.Lock() + # The database is opened on first use, not here. The module-level + # ``default_history_store`` is built during ``import + # je_auto_control``, and connecting there made importing the package + # create a file on disk and require a Python built with sqlite3 -- + # which FreeBSD ships as a separate package, so importing the + # package failed outright there, mouse and keyboard included. + self._conn: Optional["sqlite3.Connection"] = None + + def _connection(self) -> "sqlite3.Connection": + """Return the open connection, creating the database on first call. + + Every caller already holds ``self._lock``, so the open happens once. + """ + if self._conn is not None: + return self._conn + driver = require_sqlite3() if self._path != _IN_MEMORY_DB: os.makedirs(os.path.dirname(self._path) or ".", exist_ok=True) - self._lock = threading.Lock() - self._conn = sqlite3.connect( + conn = driver.connect( self._path, check_same_thread=False, isolation_level=None, ) - self._conn.row_factory = sqlite3.Row - self._conn.executescript(_SCHEMA) - self._migrate_schema() + conn.row_factory = driver.Row + conn.executescript(_SCHEMA) + self._conn = conn + self._migrate_schema_locked(conn) + return conn - def _migrate_schema(self) -> None: + def _migrate_schema_locked(self, conn: "sqlite3.Connection") -> None: """Add columns that older store files are missing.""" - cols = {row["name"] for row in self._conn.execute( + cols = {row["name"] for row in conn.execute( "PRAGMA table_info(runs)", ).fetchall()} if "artifact_path" not in cols: - self._conn.execute( + conn.execute( "ALTER TABLE runs ADD COLUMN artifact_path TEXT", ) @@ -127,7 +150,7 @@ def start_run(self, source_type: str, source_id: str, _validate_source(source_type) ts = float(started_at) if started_at is not None else time.time() with self._lock: - cursor = self._conn.execute( + cursor = self._connection().execute( "INSERT INTO runs (source_type, source_id, script_path," " started_at, status) VALUES (?, ?, ?, ?, ?)", (source_type, source_id, script_path, ts, STATUS_RUNNING), @@ -144,7 +167,7 @@ def finish_run(self, run_id: int, status: str, raise ValueError("cannot finish a run with status=running") ts = float(finished_at) if finished_at is not None else time.time() with self._lock: - cursor = self._conn.execute( + cursor = self._connection().execute( "UPDATE runs SET finished_at = ?, status = ?, error_text = ?," " artifact_path = ? WHERE id = ?", (ts, status, error_text, artifact_path, int(run_id)), @@ -154,7 +177,7 @@ def finish_run(self, run_id: int, status: str, def attach_artifact(self, run_id: int, artifact_path: str) -> bool: """Attach or replace the artifact path on a finished run.""" with self._lock: - cursor = self._conn.execute( + cursor = self._connection().execute( "UPDATE runs SET artifact_path = ? WHERE id = ?", (artifact_path, int(run_id)), ) @@ -169,7 +192,7 @@ def list_runs(self, limit: int = 100, bound_limit = int(limit) if source_type is None: with self._lock: - rows = self._conn.execute( + rows = self._connection().execute( "SELECT * FROM runs " "ORDER BY started_at DESC LIMIT ?", (bound_limit,), @@ -177,7 +200,7 @@ def list_runs(self, limit: int = 100, else: _validate_source(source_type) with self._lock: - rows = self._conn.execute( + rows = self._connection().execute( "SELECT * FROM runs WHERE source_type = ? " "ORDER BY started_at DESC LIMIT ?", (source_type, bound_limit), @@ -187,7 +210,7 @@ def list_runs(self, limit: int = 100, def get_run(self, run_id: int) -> Optional[RunRecord]: """Return a specific row or ``None`` if absent.""" with self._lock: - row = self._conn.execute( + row = self._connection().execute( "SELECT * FROM runs WHERE id = ?", (int(run_id),), ).fetchone() return _row_to_record(row) if row is not None else None @@ -197,22 +220,22 @@ def count(self, source_type: Optional[str] = None) -> int: if source_type is not None: _validate_source(source_type) with self._lock: - row = self._conn.execute( + row = self._connection().execute( "SELECT COUNT(*) FROM runs WHERE source_type = ?", (source_type,), ).fetchone() else: with self._lock: - row = self._conn.execute("SELECT COUNT(*) FROM runs").fetchone() + row = self._connection().execute("SELECT COUNT(*) FROM runs").fetchone() return int(row[0]) def clear(self) -> int: """Delete every row (and its artifact file); return rows removed.""" with self._lock: - paths = [r[0] for r in self._conn.execute( + paths = [r[0] for r in self._connection().execute( "SELECT artifact_path FROM runs WHERE artifact_path IS NOT NULL", ).fetchall()] - cursor = self._conn.execute("DELETE FROM runs") + cursor = self._connection().execute("DELETE FROM runs") removed = int(cursor.rowcount) _remove_artifact_files(paths) return removed @@ -222,13 +245,13 @@ def prune(self, keep_latest: int) -> int: if keep_latest < 0: raise ValueError("keep_latest must be >= 0") with self._lock: - paths = [r[0] for r in self._conn.execute( + paths = [r[0] for r in self._connection().execute( "SELECT artifact_path FROM runs WHERE artifact_path IS NOT NULL" " AND id NOT IN (" "SELECT id FROM runs ORDER BY started_at DESC LIMIT ?)", (int(keep_latest),), ).fetchall()] - cursor = self._conn.execute( + cursor = self._connection().execute( "DELETE FROM runs WHERE id NOT IN (" "SELECT id FROM runs ORDER BY started_at DESC LIMIT ?" ")", @@ -239,14 +262,22 @@ def prune(self, keep_latest: int) -> int: return removed def close(self) -> None: + """Close the database if it was ever opened. + + The connection object is deliberately kept: a call after ``close()`` + must keep raising sqlite3's "closed database" error rather than + silently reopening and, for an in-memory store, losing every row. + """ with self._lock: + if self._conn is None: + return try: self._conn.close() - except sqlite3.Error as error: + except SQLITE_ERRORS as error: autocontrol_logger.warning("history close failed: %r", error) -def _row_to_record(row: sqlite3.Row) -> RunRecord: +def _row_to_record(row: "sqlite3.Row") -> RunRecord: artifact = row["artifact_path"] if "artifact_path" in row.keys() else None return RunRecord( id=int(row["id"]), diff --git a/je_auto_control/utils/sql/sql_query.py b/je_auto_control/utils/sql/sql_query.py index 11d045d3..c2912cc7 100644 --- a/je_auto_control/utils/sql/sql_query.py +++ b/je_auto_control/utils/sql/sql_query.py @@ -7,15 +7,18 @@ parameters (never string-interpolated) to avoid SQL injection. Imports no ``PySide6`` so it stays fully headless. """ -import sqlite3 from contextlib import closing from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from urllib.parse import quote from je_auto_control.utils.data_source.data_source import ( _resolve_path, _validate_select, ) +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 _FetchResult = Union[List[Dict[str, Any]], Dict[str, Any], Any, None] @@ -33,7 +36,7 @@ def _read_only_uri(path: Path) -> str: return f"file:{safe_path}?mode=ro" -def _shape(cursor: sqlite3.Cursor, fetch: str) -> _FetchResult: +def _shape(cursor: "sqlite3.Cursor", fetch: str) -> _FetchResult: """Reduce a cursor to the requested result shape.""" if fetch in ("all", "rows"): return [dict(row) for row in cursor.fetchall()] @@ -63,8 +66,9 @@ def query_sqlite(database: str, query: str, uri = _read_only_uri(path) # closing(): sqlite3's own context manager only commits/rolls back the # transaction, it does not close the connection (leaking the handle until GC). - with closing(sqlite3.connect(uri, uri=True)) as connection: - connection.row_factory = sqlite3.Row + driver = require_sqlite3() + with closing(driver.connect(uri, uri=True)) as connection: + connection.row_factory = driver.Row cursor = connection.execute( statement, params if params is not None else ()) return _shape(cursor, fetch) diff --git a/je_auto_control/utils/sqlite_support.py b/je_auto_control/utils/sqlite_support.py new file mode 100644 index 00000000..9dec7cdc --- /dev/null +++ b/je_auto_control/utils/sqlite_support.py @@ -0,0 +1,56 @@ +"""Access to the optional standard-library ``sqlite3`` module. + +CPython links ``sqlite3`` against a system library that is not always shipped +with the interpreter: FreeBSD packages it separately as +``databases/py-sqlite3``, and minimal builds leave it out entirely. Ten +AutoControl subsystems keep their state in SQLite and every one of them is +reachable from ``import je_auto_control``, so importing ``sqlite3`` at module +scope made the whole package unimportable on such a Python -- mouse and +keyboard included, neither of which touches a database. Going through here +instead defers the failure to the first call that actually opens one. +""" +from typing import Tuple, Type + +from je_auto_control.utils.exception.exceptions import \ + AutoControlUnsupportedOperationException + +try: + import sqlite3 as _sqlite3 + + #: For ``except`` clauses in the callers that contain database failures. + #: The classes are named here rather than at each site so the tuple can be + #: empty when there is no ``sqlite3`` -- nothing can raise them then, so an + #: empty tuple catches exactly the right amount: nothing. + SQLITE_ERRORS: Tuple[Type[BaseException], ...] = (_sqlite3.Error,) + SQLITE_OPERATIONAL_ERRORS: Tuple[Type[BaseException], ...] = ( + _sqlite3.OperationalError, + ) +except ImportError: # reason: interpreters built without the sqlite3 extension + _sqlite3 = None # type: ignore[assignment] + SQLITE_ERRORS = () + SQLITE_OPERATIONAL_ERRORS = () + +_UNAVAILABLE_MESSAGE = ( + "This Python has no sqlite3 module, so the SQLite-backed features (run " + "history, checkpoints, work queue, agent memory, remote-desktop audit " + "log, SQL data sources) cannot run. Install it for this interpreter -- on " + "FreeBSD it is the separate databases/py-sqlite3 package." +) + + +def sqlite3_available() -> bool: + """Whether this interpreter can open SQLite databases.""" + return _sqlite3 is not None + + +def require_sqlite3(): + """Return the ``sqlite3`` module, raising if this build does not have it. + + Raises ``AutoControlUnsupportedOperationException`` -- the same type the + platform backends raise for an operation they cannot perform, so the GUI + tabs, the REST handler and the executor already report it as "unavailable + here" instead of dying on an ``ImportError`` none of them catch. + """ + if _sqlite3 is None: + raise AutoControlUnsupportedOperationException(_UNAVAILABLE_MESSAGE) + return _sqlite3 diff --git a/je_auto_control/utils/work_queue/work_queue.py b/je_auto_control/utils/work_queue/work_queue.py index 0e6db16b..698399f5 100644 --- a/je_auto_control/utils/work_queue/work_queue.py +++ b/je_auto_control/utils/work_queue/work_queue.py @@ -16,10 +16,14 @@ Pure standard library (``sqlite3``); imports no ``PySide6``. """ import json -import sqlite3 import time from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 STATUS_NEW = "new" STATUS_IN_PROGRESS = "in_progress" @@ -51,10 +55,11 @@ def __init__(self, db_path: str, name: str = "default") -> None: self._name = name self._ensure_schema() - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self._db_path, timeout=30.0, - isolation_level=None) - conn.row_factory = sqlite3.Row + def _connect(self) -> "sqlite3.Connection": + driver = require_sqlite3() + conn = driver.connect(self._db_path, timeout=30.0, + isolation_level=None) + conn.row_factory = driver.Row return conn def _ensure_schema(self) -> None: @@ -79,7 +84,7 @@ def add(self, data: Dict[str, Any], *, reference: Optional[str] = None, time.time())) return int(cur.lastrowid) - def _has_pending(self, conn: sqlite3.Connection, reference: str) -> bool: + def _has_pending(self, conn: "sqlite3.Connection", reference: str) -> bool: row = conn.execute( "SELECT 1 FROM work_items WHERE queue=? AND reference=? AND " "status IN (?, ?) LIMIT 1", diff --git a/test/unit_test/headless/test_sqlite_is_optional.py b/test/unit_test/headless/test_sqlite_is_optional.py new file mode 100644 index 00000000..ddaf5693 --- /dev/null +++ b/test/unit_test/headless/test_sqlite_is_optional.py @@ -0,0 +1,159 @@ +"""``import je_auto_control`` must not need a Python built with sqlite3. + +``sqlite3`` is in the standard library but not in every build of it: CPython +links it against a system library, and FreeBSD ships the result as the separate +``databases/py-sqlite3`` package. Ten AutoControl subsystems keep their state in +SQLite -- run history, checkpoints, the work queue, agent memory, the +remote-desktop audit log, SQL data sources, and the three error tuples that +catch their failures -- and every one of them is reachable from the facade, so +``import sqlite3`` at module scope made the whole package unimportable on such +an interpreter. Moving a mouse needs no database; that is what this file pins. + +Like ``test_facade_import_is_light``, the check is made the way the machine +without it would make it: ``_sqlite3`` is blocked outright in a subprocess -- +which is precisely what FreeBSD's stock ``python311`` reports -- and the facade +has to import anyway. +""" +import os +import pathlib +import subprocess # nosec B404 # reason: fixed argv, sys.executable, no shell +import sys + +import pytest + +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlUnsupportedOperationException, +) +from je_auto_control.utils.run_history.history_store import HistoryStore +from je_auto_control.utils import sqlite_support + +#: The working tree, so the subprocess tests the checkout rather than whatever +#: version of the package happens to be installed in site-packages. +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + +_PROBE = ''' +import pathlib +import sys + + +class _NoSqlite: + """Refuse the sqlite3 extension the way FreeBSD's python311 does.""" + + def find_spec(self, fullname, path=None, target=None): + if fullname == "_sqlite3": + raise ModuleNotFoundError( + "No module named '_sqlite3'", name="_sqlite3") + return None + + +for name in [m for m in sys.modules if m.split(".")[0] in ("sqlite3", "_sqlite3")]: + del sys.modules[name] +sys.meta_path.insert(0, _NoSqlite()) + +import je_auto_control # noqa: E402 + +if "sqlite3" in sys.modules: + raise SystemExit("sqlite3 reached sys.modules") + +# The catch tuples the containment boundaries splice in have to survive the +# absence: empty, so `except (ValueError, *SQLITE_ERRORS)` still compiles and +# still catches everything it did before. +from je_auto_control.utils import sqlite_support # noqa: E402 + +if sqlite_support.SQLITE_ERRORS or sqlite_support.SQLITE_OPERATIONAL_ERRORS: + raise SystemExit("the sqlite3 error tuples are not empty without sqlite3") +if sqlite_support.sqlite3_available(): + raise SystemExit("sqlite3_available() lied") + +# Importing must not open (or create) the default run-history database either. +db = pathlib.Path.home() / ".je_auto_control" / "run_history.sqlite" +if db.exists(): + raise SystemExit("importing the facade created %s" % db) + +print("ok", len(je_auto_control.__all__)) +''' + + +def test_facade_imports_without_sqlite3(tmp_path): + """The facade imports on an interpreter with no ``_sqlite3``.""" + home = tmp_path / "home" + home.mkdir() + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT), + HOME=str(home), USERPROFILE=str(home)) + env.pop("HOMEDRIVE", None) + env.pop("HOMEPATH", None) + result = subprocess.run( # nosec B603 # reason: fixed argv, no shell + [sys.executable, "-c", _PROBE], + capture_output=True, text=True, timeout=180, check=False, env=env) + assert result.returncode == 0, ( + "import je_auto_control needs sqlite3:\n" + f"{result.stdout}\n{result.stderr}") + assert result.stdout.startswith("ok "), result.stdout + + +def test_require_sqlite3_reports_it_the_way_backends_do(monkeypatch): + """Without the module, using a store raises the unsupported-operation type. + + That type is what the GUI tabs, the REST handler and the executor already + translate into "not available here"; a bare ``ImportError`` would escape + every one of those boundaries. + """ + monkeypatch.setattr(sqlite_support, "_sqlite3", None) + assert sqlite_support.sqlite3_available() is False + with pytest.raises(AutoControlUnsupportedOperationException) as caught: + sqlite_support.require_sqlite3() + assert isinstance(caught.value, AutoControlException) + assert "databases/py-sqlite3" in str(caught.value) + + +def test_error_tuples_hold_the_real_classes_when_sqlite3_is_there(): + """With the module present the catch tuples are its exception classes. + + They are tuples rather than the classes themselves so that the same + ``except`` clauses stay valid, and catch nothing, on a build without it -- + which the subprocess probe above asserts on the empty side. + """ + sqlite3 = pytest.importorskip("sqlite3") + assert sqlite_support.sqlite3_available() is True + assert sqlite_support.SQLITE_ERRORS == (sqlite3.Error,) + assert sqlite_support.SQLITE_OPERATIONAL_ERRORS == (sqlite3.OperationalError,) + + +def test_history_store_opens_the_database_on_first_use(tmp_path): + """Constructing the store must not touch the disk; using it must. + + The module-level ``default_history_store`` is built while the facade is + importing, so anything its constructor does happens on every ``import + je_auto_control``. + """ + pytest.importorskip("sqlite3") + db_path = tmp_path / "nested" / "run_history.sqlite" + store = HistoryStore(path=db_path) + assert not db_path.parent.exists(), "constructing the store made directories" + + run_id = store.start_run("scheduler", "job-1", "script.json") + assert db_path.exists() + assert store.get_run(run_id) is not None + store.close() + + +def test_closing_a_store_that_never_opened_is_a_no_op(tmp_path): + """``close()`` before any query neither connects nor raises.""" + db_path = tmp_path / "unused.sqlite" + store = HistoryStore(path=db_path) + store.close() + assert not db_path.exists() + + +def test_a_closed_store_does_not_silently_reopen(tmp_path): + """After ``close()``, a query still fails instead of starting a new file. + + The lazy connection could otherwise resurrect a closed store -- and for an + in-memory store that means quietly returning an empty history. + """ + sqlite3 = pytest.importorskip("sqlite3") + store = HistoryStore(path=tmp_path / "closed.sqlite") + store.start_run("scheduler", "job-1", "script.json") + store.close() + with pytest.raises(sqlite3.ProgrammingError): + store.count() diff --git a/test/verify/freebsd_verify.py b/test/verify/freebsd_verify.py index 6b8aef23..c05bca5c 100644 --- a/test/verify/freebsd_verify.py +++ b/test/verify/freebsd_verify.py @@ -20,6 +20,12 @@ leaves this VM needing python-Xlib and defusedxml — both pure Python — and an X server. So the whole backend runs here, not just the guard. +Running it then found the same mistake in a second place, one no wheel-shaped +reasoning would have caught: FreeBSD's ``python311`` has no ``sqlite3`` (it is +the separate ``databases/py-sqlite3`` package), and ten subsystems imported it +at module scope, so the facade still would not import. That is why the checks +below assert the module is *absent* here and that the package works anyway. + Ground truth is the X server answering for itself, never this codebase: ``query_pointer`` for where the cursor actually is, its button mask for which buttons the server believes are down, and ``query_keymap`` — a bitmap of every @@ -133,6 +139,20 @@ def _absent() -> str: return "cv2, PIL, cryptography, je_open_cv all absent" check("the heavy wheels really are not installed here", _absent) + # Not a wheel — a piece of the standard library the OS packages + # separately. Same trap, and asserting it is absent is what keeps this + # job testing the property: if a later image ships py311-sqlite3, the + # check goes red rather than quietly passing on an easier machine. + def _no_sqlite3() -> str: + import importlib.util + + _assert_true( + importlib.util.find_spec("_sqlite3") is None, + "this image has sqlite3, so importing the facade here no longer " + "proves the package works without it") + return "no _sqlite3, as FreeBSD's python311 ships it" + check("sqlite3 really is not installed here", _no_sqlite3) + def _facade() -> str: import je_auto_control @@ -149,6 +169,36 @@ def _backend() -> str: check("the wrapper selects the X11 backend on a BSD", _backend) +def check_sqlite_backed_features_degrade() -> None: + """Without sqlite3, the stores fail on use — not on import, and not fatally. + + The failure has to be the type every containment boundary already + understands. A bare ``ImportError`` escapes all of them, which is how one + absent database module took the mouse down with it. + """ + from je_auto_control.utils import sqlite_support + + check("sqlite3_available() reports the truth here", + lambda: _assert_eq(sqlite_support.sqlite3_available(), False)) + + def _typed() -> str: + from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlUnsupportedOperationException, + ) + from je_auto_control.utils.run_history.history_store import ( + default_history_store, + ) + + try: + default_history_store.count() + except AutoControlUnsupportedOperationException as error: + _assert_true(isinstance(error, AutoControlException), + "not in the family the executor catches") + return type(error).__name__ + raise AssertionError("reading run history did not raise without sqlite3") + check("using a store raises the unsupported-operation error", _typed) + + # --- input, driven and read back off the server ------------------------- @@ -346,6 +396,7 @@ def main() -> int: report_environment() check_platform_identity() check_facade_is_importable() + check_sqlite_backed_features_degrade() check_mouse_position() check_mouse_buttons() check_scroll() From b07de3c62aeae4d96e03238c0eb352518ef363bb Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 12:47:36 +0800 Subject: [PATCH 24/30] Mark the two import probes the way Codacy's Semgrep honours MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Codacy reports "subprocess function 'run' without a static string" on the call line and suppresses only a marker on that exact line — the comment above does not count, unlike Semgrep itself. Both probes build argv from sys.executable plus a module-level literal, with no shell. --- test/unit_test/headless/test_facade_import_is_light.py | 4 +++- test/unit_test/headless/test_sqlite_is_optional.py | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/test/unit_test/headless/test_facade_import_is_light.py b/test/unit_test/headless/test_facade_import_is_light.py index 62dac290..d6376aed 100644 --- a/test/unit_test/headless/test_facade_import_is_light.py +++ b/test/unit_test/headless/test_facade_import_is_light.py @@ -58,7 +58,9 @@ def find_spec(self, fullname, path=None, target=None): def test_facade_imports_without_the_heavy_wheels(): """The facade imports with OpenCV / Pillow / cryptography absent.""" env = dict(os.environ, PYTHONPATH=str(REPO_ROOT)) - result = subprocess.run( # nosec B603 # reason: fixed argv, no shell + # argv is this interpreter plus a module-level literal probe; the only + # interpolation is the blocked-module tuple, also a literal. No shell. + result = subprocess.run( # nosec B603 # nosemgrep # reason: literal argv, no shell [sys.executable, "-c", _PROBE.format(blocked=HEAVY)], capture_output=True, text=True, timeout=180, check=False, env=env) assert result.returncode == 0, ( diff --git a/test/unit_test/headless/test_sqlite_is_optional.py b/test/unit_test/headless/test_sqlite_is_optional.py index ddaf5693..a5ee8961 100644 --- a/test/unit_test/headless/test_sqlite_is_optional.py +++ b/test/unit_test/headless/test_sqlite_is_optional.py @@ -24,8 +24,8 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlException, AutoControlUnsupportedOperationException, ) -from je_auto_control.utils.run_history.history_store import HistoryStore from je_auto_control.utils import sqlite_support +from je_auto_control.utils.run_history.history_store import HistoryStore #: The working tree, so the subprocess tests the checkout rather than whatever #: version of the package happens to be installed in site-packages. @@ -82,7 +82,8 @@ def test_facade_imports_without_sqlite3(tmp_path): HOME=str(home), USERPROFILE=str(home)) env.pop("HOMEDRIVE", None) env.pop("HOMEPATH", None) - result = subprocess.run( # nosec B603 # reason: fixed argv, no shell + # argv is this interpreter plus a module-level literal probe. No shell. + result = subprocess.run( # nosec B603 # nosemgrep # reason: literal argv, no shell [sys.executable, "-c", _PROBE], capture_output=True, text=True, timeout=180, check=False, env=env) assert result.returncode == 0, ( From ae65cba09d6402c214dc519cc028ac9b4793fc6b Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 12:57:32 +0800 Subject: [PATCH 25/30] Put the last five errors inside the family that gets caught MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round 3 reparented the exception hierarchy so the executor, the poll loops, the request handlers and the GUI slots could each contain the whole family in one except. Five classes were missed, and one was reachable from an action list: AC_config_import on a malformed bundle raised ConfigBundleError, which inherits Exception directly, so the per-action clause did not catch it — the error went past the boundary and took every remaining action with it, under raise_on_error=False, where the contract is that a failed action is recorded. Measured on a two-action list that lost its second action. AC_usb_remote_devices and AC_usb_remote_open had the same path through UsbClientError. The guard is structural rather than a list of the five: the new test walks the package with ast and fails on any class inheriting Exception directly, against an allowlist that must say why — LoopBreak and LoopContinue because a family handler swallowing a break is the mirror-image bug, the MCP error carrier because it never leaves the dispatcher that raised it. --- CHANGELOG.md | 14 +++ WHATS_NEW.md | 25 +++++ architecture_explore.md | 22 ++-- .../utils/config_bundle/config_bundle.py | 3 +- .../utils/usb/passthrough/protocol.py | 3 +- .../utils/usb/passthrough/session.py | 3 +- .../utils/usb/passthrough/viewer_client.py | 3 +- .../utils/work_queue/work_queue.py | 3 +- .../headless/test_exception_family_is_flat.py | 103 ++++++++++++++++++ 9 files changed, 163 insertions(+), 16 deletions(-) create mode 100644 test/unit_test/headless/test_exception_family_is_flat.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 49d2e362..b8fd3e51 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -316,6 +316,20 @@ only when documented here with a migration path. ### Fixed +- **A rejected config bundle aborted the rest of the script.** Five + framework errors still inherited `Exception` directly — + `ConfigBundleError`, the USB passthrough `ProtocolError`, + `SessionError` and `UsbClientError`, and the work queue's + `BusinessError` — and the containment boundaries all catch the + `AutoControlException` family, so none of them caught these. A + malformed bundle passed to `AC_config_import` therefore raised straight + past the executor's per-action boundary and killed every remaining + action, even under `raise_on_error=False`; `AC_usb_remote_devices` and + `AC_usb_remote_open` had the same path through `UsbClientError`. All + five derive from `AutoControlException` now, so they are recorded as a + failed action like every other framework error. `LoopBreak`, + `LoopContinue` and the MCP dispatcher's private error carrier stay + outside the family deliberately — they are control flow, not failure. - **`import je_auto_control` needed a Python built with `sqlite3`, and FreeBSD's is not.** `sqlite3` is in the standard library but not in every build of it: CPython links it against a system library, and FreeBSD ships diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 1a21b246..8464c7e2 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -42,6 +42,31 @@ exactly, and the AX walk returns real elements. That was measured first and asserted second, and the probe still refuses to pass while its expectations table is empty. +### Five Errors That Every Boundary Missed + +The exception hierarchy is flat so that the executor, the poll loops, the +request handlers and the GUI slots can each contain the whole family in one +`except`. Round 3 reparented the family for that reason; five classes were +missed, and one of them was reachable from an action list. + +`AC_config_import` with a malformed bundle raised `ConfigBundleError`, which +inherits `Exception` directly, so the executor's per-action clause — which +lists `AutoControlException` and the builtins — did not catch it. The error +went past the boundary and took every remaining action with it, under +`raise_on_error=False`, where the contract is that a failed action is +*recorded*. Measured, not reasoned: a two-action list lost its second action. +`AC_usb_remote_devices` and `AC_usb_remote_open` had the same path through +`UsbClientError`. + +All five now derive from `AutoControlException`. What keeps the next one out is +not a list of the five: `test_exception_family_is_flat.py` walks the package +with `ast` and fails on *any* class inheriting `Exception` directly, against a +three-entry allowlist that has to state why. `LoopBreak` and `LoopContinue` are +on it because they are control flow — a family handler swallowing a `break` +would be the mirror-image bug — and the MCP dispatcher's private error carrier +because it never leaves the dispatcher that raised it. The allowlist is checked +in both directions, so a stale entry fails too. + ### A BSD Found the Same Mistake in a Second Place The FreeBSD VM was added to prove the X11 backend drives a real BSD. It failed diff --git a/architecture_explore.md b/architecture_explore.md index ce9ad6d1..ef74733a 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -20,7 +20,7 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | | Python 模組總數（含周邊子專案） | 1,028 | -| 程式碼總行數 | 139,623 | +| 程式碼總行數 | 139,628 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -266,7 +266,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,881 行。 +> 24 個套件、約 12,882 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -293,16 +293,16 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/state_machine/` | 181 | 宣告式有限狀態機驅動 action JSON | | `utils/stubs/` | 236 | 為 `AC_*` 指令面產生型別 stub | | `utils/test_record/` | 64 | 全域測試紀錄單例，記錄每個動作的參數與例外 | -| `utils/work_queue/` | 179 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | +| `utils/work_queue/` | 180 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 14 個套件、約 2,649 行。 +> 14 個套件、約 2,650 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/callback/` | 200 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | -| `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | +| `utils/config_bundle/` | 400 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | | `utils/diagnostics/` | 322 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | | `utils/dbus_client/` | 680 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | @@ -508,7 +508,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,717 行。 +> 6 個套件、約 17,720 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -516,7 +516,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/config_sync/` | 245 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | | `utils/remote_desktop/` | 11,840 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,247 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usb/` | 4,250 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 @@ -774,7 +774,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,247 行）與 `utils/usbip/`（920 行） +#### `utils/usb/`（4,250 行）與 `utils/usbip/`（920 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -1022,7 +1022,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `utils/mcp_server/` | 20 | 16,898 | | `utils/remote_desktop/` | 56 | 11,840 | | `utils/executor/` | 6 | 9,075 | -| `utils/usb/` | 17 | 4,247 | +| `utils/usb/` | 17 | 4,250 | | `je_auto_control/`（頂層 3 檔） | 3 | 2,363 | | `utils/accessibility/` | 13 | 2,818 | | `wrapper/` | 3,065 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | @@ -1038,6 +1038,6 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `osx/` | 17 | 907 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 690 | 50,423 | -| **總計** | **1,022** | **139,558** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 690 | 50,425 | +| **總計** | **1,022** | **139,563** | diff --git a/je_auto_control/utils/config_bundle/config_bundle.py b/je_auto_control/utils/config_bundle/config_bundle.py index 126fc73b..53ef33a9 100644 --- a/je_auto_control/utils/config_bundle/config_bundle.py +++ b/je_auto_control/utils/config_bundle/config_bundle.py @@ -38,6 +38,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -60,7 +61,7 @@ } -class ConfigBundleError(Exception): +class ConfigBundleError(AutoControlException): """Raised when bundle parsing or writing fails in a recoverable way.""" diff --git a/je_auto_control/utils/usb/passthrough/protocol.py b/je_auto_control/utils/usb/passthrough/protocol.py index bf240764..0f05192f 100644 --- a/je_auto_control/utils/usb/passthrough/protocol.py +++ b/je_auto_control/utils/usb/passthrough/protocol.py @@ -19,6 +19,7 @@ import enum import struct from dataclasses import dataclass +from je_auto_control.utils.exception.exceptions import AutoControlException _HEADER_FORMAT = "!BBH" @@ -43,7 +44,7 @@ class Opcode(enum.IntEnum): ERROR = 0xFF -class ProtocolError(Exception): +class ProtocolError(AutoControlException): """Raised on malformed frames or invariant violations.""" diff --git a/je_auto_control/utils/usb/passthrough/session.py b/je_auto_control/utils/usb/passthrough/session.py index 597e598b..208a69d8 100644 --- a/je_auto_control/utils/usb/passthrough/session.py +++ b/je_auto_control/utils/usb/passthrough/session.py @@ -65,6 +65,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.usb.passthrough.acl import UsbAcl from je_auto_control.utils.usb.passthrough.backend import UsbBackend, UsbHandle @@ -84,7 +85,7 @@ _ABUSE_LOCKOUT_S = 5.0 -class SessionError(Exception): +class SessionError(AutoControlException): """Raised on session-level invariant violations (not protocol parse errors).""" diff --git a/je_auto_control/utils/usb/passthrough/viewer_client.py b/je_auto_control/utils/usb/passthrough/viewer_client.py index e55e5e58..ff4e005e 100644 --- a/je_auto_control/utils/usb/passthrough/viewer_client.py +++ b/je_auto_control/utils/usb/passthrough/viewer_client.py @@ -38,6 +38,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.usb.passthrough.protocol import ( FLAG_EOF, Frame, Opcode, @@ -50,7 +51,7 @@ _CLIENT_SHUT_DOWN_MSG = "client is shut down" -class UsbClientError(Exception): +class UsbClientError(AutoControlException): """The host reported a transfer or open failure.""" diff --git a/je_auto_control/utils/work_queue/work_queue.py b/je_auto_control/utils/work_queue/work_queue.py index 698399f5..9c13d58d 100644 --- a/je_auto_control/utils/work_queue/work_queue.py +++ b/je_auto_control/utils/work_queue/work_queue.py @@ -20,6 +20,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.sqlite_support import require_sqlite3 if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations @@ -31,7 +32,7 @@ STATUS_FAILED = "failed" -class BusinessError(Exception): +class BusinessError(AutoControlException): """A non-retryable, data-level failure of a work item.""" diff --git a/test/unit_test/headless/test_exception_family_is_flat.py b/test/unit_test/headless/test_exception_family_is_flat.py new file mode 100644 index 00000000..429945c0 --- /dev/null +++ b/test/unit_test/headless/test_exception_family_is_flat.py @@ -0,0 +1,103 @@ +"""Every framework error must subclass ``AutoControlException``. + +The hierarchy is flat on purpose: the containment boundaries — the executor's +per-action `except`, the background poll loops, the REST/socket/MCP request +handlers, the GUI slots — all catch the family in one clause. A class that +inherits ``Exception`` directly is caught by none of them, so one malformed +argument to one command aborts the whole script instead of being recorded as a +failed action. That is not hypothetical: ``ConfigBundleError`` did exactly that +through ``AC_config_import``, and ``UsbClientError`` could through +``AC_usb_remote_devices``. + +The rule is checked structurally rather than class by class, because the way it +gets broken is a *new* subsystem defining its own error — which no list of +existing classes would notice. +""" +import ast +import pathlib + +import pytest + +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: The package tree, read as source: importing every module to inspect its +#: classes would drag in optional backends this test does not need. +PACKAGE_ROOT = pathlib.Path(__file__).resolve().parents[3] / "je_auto_control" + +#: Deliberately outside the family, and each for a reason that would break if +#: it were inside: +#: +#: * ``LoopBreak`` / ``LoopContinue`` are control flow, not failure. The +#: executor re-raises them *before* the family clause; making them relatives +#: would let a plain ``except AutoControlException`` swallow a ``break``. +#: * ``_MCPError`` carries a JSON-RPC error code to the dispatcher that raised +#: it, and is caught there by name. It never crosses a boundary. +#: * ``AutoControlException`` is the root of the family, so it is the one class +#: that has to inherit ``Exception`` itself. +DELIBERATELY_OUTSIDE = { + ("utils/exception/exceptions.py", "AutoControlException"), + ("utils/executor/flow_control.py", "LoopBreak"), + ("utils/executor/flow_control.py", "LoopContinue"), + ("utils/mcp_server/_protocol.py", "_MCPError"), +} + + +def _classes_inheriting_exception_directly(): + """Yield ``(relative path, class name)`` for every ``class X(Exception)``.""" + for path in sorted(PACKAGE_ROOT.rglob("*.py")): + if "__pycache__" in path.parts: + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for base in node.bases: + if isinstance(base, ast.Name) and base.id == "Exception": + rel = path.relative_to(PACKAGE_ROOT).as_posix() + yield rel, node.name + + +def test_no_framework_error_escapes_the_family(): + """Only the three control-flow carriers may inherit ``Exception``.""" + found = set(_classes_inheriting_exception_directly()) + unexpected = found - DELIBERATELY_OUTSIDE + assert not unexpected, ( + "these inherit Exception directly, so every containment boundary " + "misses them — derive them from AutoControlException, or add them to " + f"DELIBERATELY_OUTSIDE with the reason: {sorted(unexpected)}") + + # And the allowlist itself has to stay real: a stale entry would quietly + # licence a class that no longer exists, or has since been reparented. + stale = DELIBERATELY_OUTSIDE - found + assert not stale, f"allowlisted classes that are no longer there: {stale}" + + +@pytest.mark.parametrize("module_path, class_name", [ + ("je_auto_control.utils.config_bundle.config_bundle", "ConfigBundleError"), + ("je_auto_control.utils.usb.passthrough.protocol", "ProtocolError"), + ("je_auto_control.utils.usb.passthrough.session", "SessionError"), + ("je_auto_control.utils.usb.passthrough.viewer_client", "UsbClientError"), + ("je_auto_control.utils.work_queue.work_queue", "BusinessError"), +]) +def test_the_reparented_five_are_in_the_family(module_path, class_name): + """The classes that used to escape, named so the fix is legible.""" + module = pytest.importorskip(module_path) + assert issubclass(getattr(module, class_name), AutoControlException) + + +def test_a_rejected_config_bundle_does_not_abort_the_script(): + """The failure that proved the rule matters, pinned end to end. + + ``AC_config_import`` on a malformed bundle used to raise past the + per-action boundary and take every remaining action with it, even under + ``raise_on_error=False``. + """ + from je_auto_control.utils.executor.action_executor import executor + + record = executor.execute_action( + [["AC_config_import", {"bundle": {"not": "a bundle"}}], + ["AC_sleep", {"sleep_time": 0.01}]], + raise_on_error=False) + + assert len(record) == 2, "the second action never ran" + assert any("ConfigBundleError" in str(value) for value in record.values()) From 501f1465c97574a70fb404c9d01cc2254c326138 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 12:59:24 +0800 Subject: [PATCH 26/30] Pin that importing the package writes nothing to the user's home MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The history store's constructor opened its database, and that constructor runs while the facade is importing — so importing the package created ~/.je_auto_control/run_history.sqlite whether or not the caller ever recorded a run. The check reads the whole home directory rather than that one path, because what puts it back is any module-level singleton that opens a file, not that path in particular. --- .../headless/test_facade_import_is_light.py | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/test/unit_test/headless/test_facade_import_is_light.py b/test/unit_test/headless/test_facade_import_is_light.py index d6376aed..202df616 100644 --- a/test/unit_test/headless/test_facade_import_is_light.py +++ b/test/unit_test/headless/test_facade_import_is_light.py @@ -16,6 +16,7 @@ import pathlib import subprocess # nosec B404 # reason: fixed argv, sys.executable, no shell import sys +import tempfile import pytest @@ -87,3 +88,31 @@ def test_the_lazy_modules_still_work_when_the_wheels_are_there(): stats = region_color_stats(image) assert stats.average_rgb == (10, 20, 30) assert stats.dominant_rgb == (10, 20, 30) + + +def test_importing_the_facade_writes_nothing_to_the_users_home(): + """Importing must not create files, only make the names available. + + `default_history_store` used to open its SQLite database in its + constructor, and that constructor runs while the facade is importing — so + `import je_auto_control` created `~/.je_auto_control/run_history.sqlite` + whether or not the caller ever recorded a run. Any module-level singleton + that opens a file puts that back, which is why this reads the whole home + directory rather than naming that one path. + """ + with tempfile.TemporaryDirectory(prefix="ac-home-") as raw_home: + home = pathlib.Path(raw_home) + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT), + HOME=str(home), USERPROFILE=str(home)) + env.pop("HOMEDRIVE", None) + env.pop("HOMEPATH", None) + # argv is this interpreter plus a literal statement. No shell. + result = subprocess.run( # nosec B603 # nosemgrep # reason: literal argv, no shell + [sys.executable, "-c", "import je_auto_control"], + capture_output=True, text=True, timeout=180, check=False, env=env) + assert result.returncode == 0, result.stderr + + created = sorted(p.relative_to(home).as_posix() for p in home.rglob("*")) + assert not created, ( + "importing je_auto_control created these under the user's home: " + f"{created}") From 7ee8eb9a47529527ce5056f31279803ad17c2b2e Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 13:00:55 +0800 Subject: [PATCH 27/30] Scope the import side-effect check to the directory we own Reading the whole home directory would have gone red for a third-party import writing its own cache there, which is someone else's business and would read as our regression. Every store, key file and cache this package owns is under ~/.je_auto_control, so its absence is the property. --- .../headless/test_facade_import_is_light.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/test/unit_test/headless/test_facade_import_is_light.py b/test/unit_test/headless/test_facade_import_is_light.py index 202df616..5d5054cd 100644 --- a/test/unit_test/headless/test_facade_import_is_light.py +++ b/test/unit_test/headless/test_facade_import_is_light.py @@ -90,15 +90,17 @@ def test_the_lazy_modules_still_work_when_the_wheels_are_there(): assert stats.dominant_rgb == (10, 20, 30) -def test_importing_the_facade_writes_nothing_to_the_users_home(): +def test_importing_the_facade_writes_no_state_of_its_own(): """Importing must not create files, only make the names available. `default_history_store` used to open its SQLite database in its constructor, and that constructor runs while the facade is importing — so `import je_auto_control` created `~/.je_auto_control/run_history.sqlite` - whether or not the caller ever recorded a run. Any module-level singleton - that opens a file puts that back, which is why this reads the whole home - directory rather than naming that one path. + whether or not the caller ever recorded a run. Every store, key file and + cache this package owns lives under that one directory, so its absence is + the whole property; the rest of the home directory is left out because a + third-party import writing its own cache there would be someone else's + business, and a red test for it would be noise. """ with tempfile.TemporaryDirectory(prefix="ac-home-") as raw_home: home = pathlib.Path(raw_home) @@ -112,7 +114,9 @@ def test_importing_the_facade_writes_nothing_to_the_users_home(): capture_output=True, text=True, timeout=180, check=False, env=env) assert result.returncode == 0, result.stderr - created = sorted(p.relative_to(home).as_posix() for p in home.rglob("*")) - assert not created, ( - "importing je_auto_control created these under the user's home: " - f"{created}") + state_dir = home / ".je_auto_control" + created = sorted(path.relative_to(home).as_posix() + for path in state_dir.rglob("*")) + assert not state_dir.exists(), ( + "importing je_auto_control created its state directory, " + f"holding: {created}") From d8b6384a42a29a0f0fd8030c5dfdea8a4ded256f Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 13:44:50 +0800 Subject: [PATCH 28/30] Name the second package that blocks Windows arm64 The entry blamed opencv-python alone and proposed moving OpenCV and Pillow to an optional extra to unblock the install. Re-measuring shows that fix would not have worked, and that half of it was aimed at the wrong package: - cryptography is a second, independent blocker. Wheels stop at 46.0.3; 46.0.4 onwards publish no win_arm64. Our floor is >=48.0.1, set to clear GHSA-537c-gmf6-5ccf, so it cannot be lowered to reach one. - Pillow ships win_arm64 wheels and never blocked anything. - PySide6 and qt-material resolve, so the GUI extra is not implicated; aiortc fails on its own transitive google-crc32c. None of this needs an arm64 runner: pip resolves for a foreign platform with --dry-run --only-binary=:all: --platform win_arm64, which answers in seconds what previously cost twelve minutes of a runner to fail at. The command is now recorded next to the finding, so the next re-check is cheap. --- .github/workflows/platform-smoke.yml | 25 ++++++++++---- Progress.md | 49 ++++++++++++++++++++-------- README/WHATS_NEW_zh-CN.md | 2 +- README/WHATS_NEW_zh-TW.md | 2 +- WHATS_NEW.md | 12 +++++-- docs/CAPABILITY_MATRIX.md | 14 +++++--- 6 files changed, 74 insertions(+), 30 deletions(-) diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index d964e514..f22bb1a4 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -19,12 +19,25 @@ jobs: # ubuntu-22.04-arm adds Linux, and it passes. # # windows-11-arm is deliberately absent, and it was measured rather - # than assumed: opencv-python publishes no win_arm64 wheel, so pip - # falls back to building it from source and CMake cannot configure - # for ARM64. The job spent twelve minutes failing at that, which is - # not a CI problem to work around — the package genuinely cannot be - # installed on Windows arm64 today. Recorded in Progress.md; add the - # runner back when the wheel exists. + # than assumed. Two dependencies have no win_arm64 wheel, and both + # have to go before the runner is worth adding back: + # + # opencv-python — no win_arm64 wheel in any version, so pip falls + # back to building from source and CMake cannot + # configure for ARM64. Twelve minutes, then failure. + # cryptography — wheels stop at 46.0.3; 46.0.4 onwards ship none. + # Our floor is >=48.0.1 and that is a security floor + # (GHSA-537c-gmf6-5ccf), so it cannot be lowered. + # + # Neither is a CI problem to work around — the package genuinely + # cannot be installed on Windows arm64 today. Re-check without a + # runner, in about ten seconds: + # + # pip install --dry-run --only-binary=:all: --platform win_arm64 \ + # --python-version 3.12 --target /tmp/probe \ + # 'opencv-python>=4.8,<6' 'cryptography>=48.0.1' + # + # Recorded in Progress.md; add the runner back when both resolve. os: [windows-2022, ubuntu-22.04, macos-14, ubuntu-22.04-arm] python-version: ["3.10", "3.14"] runs-on: ${{ matrix.os }} diff --git a/Progress.md b/Progress.md index 0f853caa..009c2bae 100644 --- a/Progress.md +++ b/Progress.md @@ -50,28 +50,49 @@ --- -## Windows arm64 裝不起來，卡在 opencv-python +## Windows arm64 裝不起來——是兩個上游，不是一個 -`BLOCKED` — 上游（opencv-python 沒有 win_arm64 wheel） +`BLOCKED` — 上游（`opencv-python` 沒有 win_arm64 wheel；`cryptography` 在安全下限之上也沒有） `windows-11-arm` 加進 `platform-smoke.yml` 的矩陣跑了一次， 結果是實測而不是推測：**opencv-python 並沒有發 win_arm64 wheel**，pip 回退到從原碼建，CMake 在 ARM64 上 -configure 不起來，花了十二分鐘失敗。cryptography -也在同一輪裡被拉去建。 - -這不是 CI 設定問題，是**這個套件今天在 -Windows arm64 上裝不起來**。所以那一格已從矩陣 -移除，並把原因寫在 workflow 的註解裡；哪天上游 -發了 wheel，把 runner 加回去就好。 +configure 不起來，花了十二分鐘失敗。所以那一格已從矩陣 +移除，並把原因寫在 workflow 的註解裡。 門面已經不在 module scope import OpenCV 了（見 [WHATS_NEW.md](WHATS_NEW.md)），但這裡卡的不是 import -而是 **pip 裝不起來**:`opencv-python` 仍列在 -`pyproject.toml` 的 `dependencies`,`pip install -e .` -第一步就會去建它。要讓 arm64 只裝輸入的部分,得先決定 -把 OpenCV／Pillow 移到 optional extra——那是相容性決定, -沒有人要求之前不做。 +而是 **pip 裝不起來**：那幾個套件仍列在 `pyproject.toml` +的 `dependencies`，`pip install -e .` 第一步就會去建它們。 + +### 2026-08-20 重新實測：當初只數到一半 + +不必開 runner——`pip` 可以替別的平台解析，十秒就給出答案： + +| 依賴 | win_arm64 | 實測 | +| --- | --- | --- | +| `opencv-python>=4.8,<6` | **沒有** | 任何版本都沒有，pip 回的是 `from versions: none`。`je_open_cv` 自己是純 Python，但它相依 opencv-python，所以一起卡。 | +| `cryptography>=48.0.1` | **沒有** | wheel 只出到 **46.0.3**，46.0.4 起上游就不再發 win_arm64。而 `>=48.0.1` 是 347ec1e 為了 GHSA-537c-gmf6-5ccf（high）訂的**安全下限**，不能為了 arm64 降回去。 | +| `pillow==12.3.0` | 有 | `pillow-12.3.0-cp3xx-win_arm64.whl` 一直都在。**原本這裡寫「把 OpenCV／Pillow 移到 extra」，Pillow 那半是猜的，它從來不是卡點。** | +| `mss`／`defusedxml`／`je_open_cv` | 有 | 純 Python。 | +| `PySide6==6.11.1`／`qt-material==2.17` | 有 | `[gui]` extra 在 arm64 上裝得起來。 | +| `aiortc` | **沒有** | 卡在傳遞相依 `google-crc32c`，與本專案的選擇無關；`av` 自己有 wheel。 | + +**所以原本那句「把 OpenCV／Pillow 移到 optional extra，arm64 就能只裝輸入的部分」 +是不成立的**——就算 OpenCV 移走，`cryptography` 還是會把 `pip install` 擋在同一個 +地方，而它的下限是安全下限，沒有往下讓的空間。要真的讓 arm64 裝得起來，**兩個都得 +離開必裝集合**；`cryptography` 今天被六個模組用到（`acme_v2`、`tls_acme`、 +`action_signing`、`secrets`、`remote_desktop` 的加密錄影），那是比 OpenCV 更大的 +相容性決定。沒有人要求之前不做。 + +重驗指令（不需要 arm64 機器，也不需要 runner）： + +```bash +pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version 3.12 --target /tmp/probe 'opencv-python>=4.8,<6' 'cryptography>=48.0.1' +``` + +兩行 `ERROR: No matching distribution` 就是現況。哪天其中一行不見了，就是上游發了 +wheel，那時把 `windows-11-arm` 加回 `platform-smoke.yml` 的矩陣。 **Linux arm64 是好的**——`ubuntu-22.04-arm` 兩個 Python 版本 都綠，macOS 本來就是 arm64。所以卡住的只有 Windows diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index ffcfe552..0a07ea53 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -61,7 +61,7 @@ AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期 system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 -`ubuntu-22.04-arm` 加進 smoke 矩陣且全绿。`windows-11-arm` 试过后拿掉了：opencv-python 根本没发 `win_arm64` wheel，这个包今天在 Windows arm64 上装不起来——是量出来的，已记在 `Progress.md`。 +`ubuntu-22.04-arm` 加进 smoke 矩阵且全绿。`windows-11-arm` 试过后拿掉了，而重新实测又挖出当初漏掉的**第二个**卡点：opencv-python 任何版本都没发 `win_arm64` wheel，cryptography 则从 46.0.4 起不再发，而本项目的下限 `>=48.0.1` 是安全下限（GHSA-537c-gmf6-5ccf），不能为了凑 wheel 往下让。原本跟 OpenCV 并列的 Pillow 其实一直都有 arm64 wheel，从来不是卡点。这些都不需要 arm64 机器就验得到——`pip install --dry-run --only-binary=:all: --platform win_arm64` 十秒给答案，指令已连同结论记在 `Progress.md`。 ## 本次更新 (2026-08-19) — Wayland 两个等人拍板的取舍,拍板了 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index f95ae303..ef67d949 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -61,7 +61,7 @@ AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期 system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 -`ubuntu-22.04-arm` 加進 smoke 矩陣且全綠。`windows-11-arm` 試過後拿掉了：opencv-python 根本沒發 `win_arm64` wheel，這個套件今天在 Windows arm64 上裝不起來——是量出來的，已記在 `Progress.md`。 +`ubuntu-22.04-arm` 加進 smoke 矩陣且全綠。`windows-11-arm` 試過後拿掉了，而重新實測又挖出當初漏掉的**第二個**卡點：opencv-python 任何版本都沒發 `win_arm64` wheel，cryptography 則從 46.0.4 起不再發，而本專案的下限 `>=48.0.1` 是安全下限（GHSA-537c-gmf6-5ccf），不能為了湊 wheel 往下讓。原本跟 OpenCV 並列的 Pillow 其實一直都有 arm64 wheel，從來不是卡點。這些都不需要 arm64 機器就驗得到——`pip install --dry-run --only-binary=:all: --platform win_arm64` 十秒給答案，指令已連同結論記在 `Progress.md`。 ## 本次更新 (2026-08-19) — Wayland 兩個等人拍板的取捨,拍板了 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 8464c7e2..69e191f4 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -239,9 +239,15 @@ had to be loaded by file path to get even that far. See below: that turned out to be the wrong thing to work around, and the job now drives the whole backend. `ubuntu-22.04-arm` joins the smoke matrix and passes; `macos-14` was already -arm64. `windows-11-arm` was tried and removed: opencv-python publishes no -`win_arm64` wheel, so the package cannot be installed there at all today — -measured, not assumed, and recorded in `Progress.md`. +arm64. `windows-11-arm` was tried and removed, and re-measuring turned up a +**second** blocker the first pass had missed: opencv-python publishes no +`win_arm64` wheel in any version, and cryptography stopped publishing one after +46.0.3 — while this project's floor is `>=48.0.1`, a security floor +(GHSA-537c-gmf6-5ccf) that cannot be lowered to reach a wheel. Pillow, named +alongside OpenCV in the original entry, ships `win_arm64` wheels and was never +part of the problem. None of that needs an arm64 machine to check: `pip +install --dry-run --only-binary=:all: --platform win_arm64` answers it in +seconds, and `Progress.md` records the command next to the finding. ### The Facade Insisted on OpenCV to Move a Mouse diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index aaf65d4e..59358ebe 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -126,11 +126,15 @@ silently, on every BSD. **arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` joins the smoke matrix and passes. `windows-11-arm` was tried and removed, on -measurement rather than assumption: **opencv-python publishes no `win_arm64` -wheel**, so pip falls back to building it from source and CMake cannot -configure for ARM64. That is not a CI problem to work around — the package -genuinely cannot be installed on Windows arm64 today, which is recorded in -`Progress.md` with the runner ready to add back when the wheel exists. +measurement rather than assumption, and **two** dependencies are why: +**opencv-python publishes no `win_arm64` wheel** in any version, so pip falls +back to building from source and CMake cannot configure for ARM64; and +**cryptography stopped publishing one after 46.0.3**, while this project's +floor is `>=48.0.1` — a security floor (GHSA-537c-gmf6-5ccf) that cannot be +lowered to reach a wheel. Neither is a CI problem to work around: the package +genuinely cannot be installed on Windows arm64 today. `Progress.md` records +both, alongside a `pip --dry-run --platform win_arm64` command that re-checks +them in seconds without an arm64 machine. The accessibility row said `backend tests` for Linux X11 and meant nothing by it: there was no Linux backend at all, and `_build_backend()` fell straight From d7ff13f8218a586550da87ddc9c23725830376a1 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 17:16:58 +0800 Subject: [PATCH 29/30] Say where the destructive-tool prompt actually works The docs promise JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE gates every destructive tool, caveated only on the client advertising elicitation. Measured against a real HttpMCPServer, it fires on stdio and never fires over HTTP: a plain POST has no server-to-client channel to carry the prompt, and an SSE POST closes its connection, so the capability the client advertised at initialize is forgotten before the next call. The operator gets no confirmation and one INFO line. Document the limit in both translations, point at the controls that do work over HTTP, and pin the current behaviour so closing the gap reddens a test instead of leaving a stale warning. Which fix to take is a behaviour change either way, so it goes to Progress.md as a decision. --- Progress.md | 51 ++++++++++++++ WHATS_NEW.md | 28 ++++++++ .../Eng/doc/mcp_server/mcp_server_doc.rst | 14 ++++ .../Zh/doc/mcp_server/mcp_server_doc.rst | 12 ++++ .../headless/test_mcp_http_transport.py | 66 +++++++++++++++++++ 5 files changed, 171 insertions(+) diff --git a/Progress.md b/Progress.md index 009c2bae..6a68a757 100644 --- a/Progress.md +++ b/Progress.md @@ -98,6 +98,57 @@ wheel，那時把 `windows-11-arm` 加回 `platform-smoke.yml` 的矩陣。 都綠，macOS 本來就是 arm64。所以卡住的只有 Windows 這一個組合。 +## MCP 的破壞性動作確認,在 HTTP transport 上一次都不會觸發 + +`DECIDE` — 要嘛補上 session 身分,要嘛改成 fail closed;兩條都會動到行為 + +`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` 的用意是「每個 destructive 工具執行前 +先問過人」。這件事在 stdio 上是好的,**在 HTTP transport 上一次都不會觸發**—— +而且不是文件原本以為的「initialize 跟 tools/call 落在不同連線時才失效」,是**四種 +組合全部失效**。實測(2026-08-20,真的 `HttpMCPServer`,真的 destructive 工具): + +| 情境 | 結果 | +| --- | --- | +| 行程內(等同 stdio),同一個 server 物件 | 送出 `elicitation/create`,拒絕就不執行 ✅ | +| HTTP,plain POST,同一條連線 | **不問就執行** | +| HTTP,plain POST,兩條連線 | **不問就執行** | +| HTTP,SSE,同一條連線 | **不問就執行** | +| HTTP,SSE,兩條連線 | **不問就執行** | + +原因有兩層,兩層都得處理: + +1. `_maybe_confirm_destructive` 在 `self._writer is None` 時直接 return。plain POST + 的 `connection_scope` 沒有 writer(一次 request/response,沒有 server→client 通道), + 所以這條路連問的能力都沒有。 +2. 就算是有 writer 的 SSE,`_dispatch_sse` 會把 `close_connection` 設成 True, + `finish()` 接著呼叫 `forget_connection(id(self))`。而 connection scope 是用 + `id(self)`(TCP 連線)當 key,不是用 MCP session,所以 `initialize` 帶進來的 + `capabilities`(裡面才有 `elicitation`)在下一個 `tools/call` 一定已經被忘掉, + 於是走進「client 沒有 elicitation 能力」那條分支,留一行 INFO log 後**放行**。 + +也就是說,操作者設了這個變數、以為擋住了,實際上什麼都沒擋,而且只有 INFO log。 +文件原本寫「gate every destructive tool」,只註明「舊 client 會 fall through」, +沒說 HTTP 上根本不會啟動——**這一半已經修好了**:兩份 `mcp_server_doc.rst` 都加了 +warning,說明它目前只在 stdio 有效,HTTP 請改靠 bearer token 與綁 `127.0.0.1`。 +`test_mcp_http_transport.py::test_destructive_confirm_gate_never_fires_over_http` +把現況釘住(plain POST 與 SSE 兩種),哪天有人補好了,那個測試會當場紅掉, +提醒他同一筆改動要把文件的 warning 跟本節一起改掉。 + +**還沒決定的是行為要往哪邊走**,兩個選項都不是純加法: + +- **(A) fail closed** — 變數開著又問不到人時,直接拒絕執行 destructive 工具。 + 最貼近操作者的意圖,改動也最小。但會翻掉現有的 + `test_destructive_confirmation_skipped_when_client_lacks_capability`,而且會擋掉 + 今天所有沒有 elicitation 能力的 client(包含 HTTP 上的每一個)。 +- **(B) 補上 `Mcp-Session-Id`** — 讓 scope 跟著 MCP session 走而不是跟著 TCP 連線, + 這樣 SSE 那條路才有機會真的問出來。這是 MCP 自己的答案,但這個 transport 是 + **刻意**做成 sessionless 的(`do_DELETE` 的註解就這麼寫),而 per-connection 隔離 + 正是先前一次重構的重點,有 `test_r3_mcp_connection_isolation` 在守。要動得連那條 + 隔離一起重新想,不能只是把 key 換掉。plain POST 就算有了 session 仍然沒有通道, + 所以 (B) 落地後,plain POST 還是得靠 (A) 收尾。 + +重驗方式:把上表跑一次即可,四種 HTTP 組合都應該是「不問就執行」。 + ## Wayland:剩下的都不是「缺一台機器」 這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 69e191f4..32fd83d3 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -320,6 +320,34 @@ real X server through `xev`, `freebsd_verify.py` pins it on a BSD, and display. The migration note for anyone who was relying on the magnitude alone is in `CHANGELOG.md`. +### The Destructive-Action Prompt Was Documented as Covering a Transport It Never Reached + +`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` is documented as gating *every* +destructive MCP tool behind a confirmation prompt, with one stated caveat: the +client has to advertise the `elicitation` capability. Measured against a real +`HttpMCPServer` with a real destructive tool, the gate fires on stdio and +**never fires over HTTP** — not in any of the four combinations of plain POST +or SSE, one connection or two. + +Two independent reasons, and the second one is why keeping the connection open +does not help. A plain `POST` is one request and one response, so its +connection scope has no server→client channel and the prompt has nothing to +travel down. An SSE `POST` does have one, but `_dispatch_sse` sets +`close_connection`, so the scope keyed on that TCP connection is forgotten +before the next request — and the `elicitation` capability the client +advertised at `initialize` goes with it. The call then takes the "client +cannot be prompted" branch, logs one INFO line, and runs the tool. + +An operator who sets that variable on an HTTP-exposed server is getting no +confirmation at all. Both `mcp_server_doc.rst` translations now say so, and +point at the bearer token and the `127.0.0.1` bind as the controls that do +work there. `test_destructive_confirm_gate_never_fires_over_http` pins the +current behaviour for both content types, so whoever closes the gap gets a red +test telling them to update the warning in the same change. Which of the two +fixes to take — refuse when the prompt is impossible, or give the transport +real session identity — is a behaviour change either way and is written up in +`Progress.md`. + ## What's new (2026-08-19) diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 13192771..75ff0b3f 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -283,6 +283,20 @@ error to the model without running the action. Requires the client to advertise the ``elicitation`` capability — older clients fall through with a logged warning. +.. warning:: + + **This gate is effective on the stdio transport only.** Over the + HTTP transport it never fires: the prompt needs a server→client + channel bound to the same connection scope that received + ``initialize``, and the HTTP transport is deliberately sessionless + — a plain ``POST`` has no such channel at all, and an SSE ``POST`` + closes its connection after the final event, so no later + ``tools/call`` can reach the capabilities the client advertised. + Destructive tools therefore run **unprompted** over HTTP even with + this variable set. Do not rely on it as the only control on an + HTTP-exposed server; use the bearer token and bind to + ``127.0.0.1``. + Audit log ========= diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index d1d1e93a..d5f13027 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -268,6 +268,18 @@ Bearer token 也可從 ``JE_AUTOCONTROL_MCP_TOKEN`` 環境變數讀取。 使用者拒絕時模型會收到乾淨的錯誤,不會執行動作。需要 client 自己 聲明 ``elicitation`` 能力;舊 client 會留下 warning log 後繼續執行。 +.. warning:: + + **這道關卡目前只在 stdio transport 上有效。** 走 HTTP transport 時 + 它一次都不會觸發:送出提示需要一條與收到 ``initialize`` 的同一個 + connection scope 綁在一起的 server→client 通道,而 HTTP transport + 是刻意設計成 sessionless 的——普通 ``POST`` 根本沒有這種通道, + SSE ``POST`` 則在送完最後一個事件後就關掉連線,所以後續的 + ``tools/call`` 拿不到 client 當初聲明的能力。也就是說,即使設了這個 + 環境變數,destructive 工具在 HTTP 上仍然會**不經詢問直接執行**。 + 不要把它當成 HTTP 服務的唯一控制手段;請改用 bearer token 並綁在 + ``127.0.0.1``。 + 稽核 Log ======== diff --git a/test/unit_test/headless/test_mcp_http_transport.py b/test/unit_test/headless/test_mcp_http_transport.py index 10655e22..5deebbb9 100644 --- a/test/unit_test/headless/test_mcp_http_transport.py +++ b/test/unit_test/headless/test_mcp_http_transport.py @@ -277,3 +277,69 @@ def test_malformed_json_returns_parse_error(http_server): assert response.status == 200 payload = json.loads(response.read().decode("utf-8")) assert payload["error"]["code"] == -32700 + + +def _post_with_accept(server, body, accept): + """POST and return the raw body, letting the caller choose ``Accept``.""" + host, port = server.address + url = f"{_TEST_SCHEME}://{host}:{port}{DEFAULT_PATH}" + req = urllib.request.Request( + url, data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": accept}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=5) as response: # nosec B310 + return response.read().decode("utf-8") + + +@pytest.mark.parametrize("accept", ["application/json", "text/event-stream"]) +def test_destructive_confirm_gate_never_fires_over_http(monkeypatch, accept): + """Pin the known gap: JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE is stdio-only. + + The elicitation prompt needs a server->client channel bound to the same + connection scope that received ``initialize``. A plain POST has no such + channel, and an SSE POST closes its connection after the final event, so + the capabilities the client advertised are gone by the next ``tools/call``. + Destructive tools therefore run unprompted over HTTP even with the gate on. + + This is recorded in ``Progress.md`` and warned about in the MCP docs. When + the transport grows real session identity, this test fails -- that is the + point: update the docs warning and the Progress.md entry in the same change. + """ + from je_auto_control.utils.mcp_server.tools import ( + MCPTool, MCPToolAnnotations, + ) + + monkeypatch.setenv("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "1") + ran = [] + tool = MCPTool( + name="zap_probe", description="destructive probe", + input_schema={"type": "object", "properties": {}}, + handler=lambda: ran.append(1) or "RAN", + annotations=MCPToolAnnotations(destructive=True, read_only=False), + ) + server = HttpMCPServer(mcp=MCPServer( + tools=[tool], resource_provider=ChainProvider([]), + prompt_provider=StaticPromptProvider([]), + ), host="127.0.0.1", port=0) + server.start() + try: + init_body = _post_with_accept(server, { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {"elicitation": {}}, + "clientInfo": {"name": "probe", "version": "1"}, + }, + }, accept) + call_body = _post_with_accept(server, { + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": {"name": "zap_probe", "arguments": {}}, + }, accept) + finally: + server.stop(timeout=1.0) + + assert "elicitation/create" not in init_body + assert "elicitation/create" not in call_body + assert ran == [1], "gate did not fire, so the tool should have run" + assert '"isError": false' in call_body From f52662c1907124d668d529f241cde58157ea3698 Mon Sep 17 00:00:00 2001 From: JeffreyChen Date: Thu, 20 Aug 2026 18:02:19 +0800 Subject: [PATCH 30/30] Give the HTTP transport the session identity MCP specifies MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The destructive-action confirmation fired on stdio and never over HTTP. The prompt is a question asked between receiving a call and answering it, so it needs a channel bound to the scope that received initialize — and that scope was keyed on the TCP connection, which a plain POST ends and an SSE POST closes after its last event. The capability advertised at initialize was always gone by the tools/call. Key the scope on Mcp-Session-Id instead. initialize mints one and returns it as a header; GET with an SSE Accept opens the standing server-to-client stream; DELETE terminates. The dispatcher needed no change, since it already scoped capabilities and call slots on an opaque connection_id, so per-connection isolation is preserved as-is — a session is just an identity that outlives a socket. Sessions are swept when idle and capped. A client that echoes the id and gives the server somewhere to send the question is now prompted for real, over either the standing stream or an SSE POST's own response stream. One that does neither still proceeds, as a stdio client without elicitation does; that fallback is now documented and pinned rather than being the only behaviour. Also stop draining a request body that was already read. The new 404 and 409 are decided after parsing, as the "body must be UTF-8" 400 always was, and the drain then blocked on bytes that were gone until the read timeout. --- CHANGELOG.md | 38 ++ Progress.md | 51 -- WHATS_NEW.md | 86 ++- architecture_explore.md | 17 +- .../Eng/doc/mcp_server/mcp_server_doc.rst | 66 ++- .../Zh/doc/mcp_server/mcp_server_doc.rst | 51 +- .../utils/mcp_server/http_sessions.py | 234 ++++++++ .../utils/mcp_server/http_transport.py | 247 ++++++++- .../headless/test_mcp_http_sessions.py | 522 ++++++++++++++++++ .../headless/test_mcp_http_transport.py | 30 +- 10 files changed, 1200 insertions(+), 142 deletions(-) create mode 100644 je_auto_control/utils/mcp_server/http_sessions.py create mode 100644 test/unit_test/headless/test_mcp_http_sessions.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b8fd3e51..7af7c5f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,26 @@ only when documented here with a migration path. land somewhere else; unset (or any unrecognised value, which says so and falls back) keeps the existing warn-once-and-move behaviour. The libei path is absolute at the protocol level and is not affected either way. +- **MCP sessions over HTTP.** `initialize` now mints an `Mcp-Session-Id` and + returns it as a response header. A client that echoes it keeps one + dispatcher scope — the capabilities it advertised, and the slots its + in-flight calls occupy — across every connection it opens, instead of one + scope per TCP connection. `GET /mcp` with `Accept: text/event-stream` and a + valid session id opens the standing server-to-client SSE stream (one per + session; a second gets 409), `DELETE /mcp` with the id terminates the + session, and a server request is answered by `POST`ing an ordinary JSON-RPC + response on any connection. Sessions are swept after ten minutes untouched + and capped at 128. `je_auto_control.utils.mcp_server.http_sessions` holds + the registry. +- **`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` now works over HTTP** — for a + client that echoes `Mcp-Session-Id` and holds the `GET` stream open. It + previously fired only on stdio: the prompt needs a server-to-client channel + bound to the scope that received `initialize`, and a connection-keyed scope + never survived to the `tools/call`. A client that does neither still cannot + be prompted and its destructive calls still proceed, exactly as for a stdio + client that never advertised `elicitation`; that fallback is documented and + is not a substitute for the bearer token, the `127.0.0.1` bind or + `JE_AUTOCONTROL_MCP_READONLY`. ### Removed @@ -145,6 +165,15 @@ only when documented here with a migration path. ### Changed +- The MCP HTTP transport answers `GET /mcp` differently. It used to return + `405` with `{"error": "GET stream not supported"}` for every request; it now + serves the session's SSE stream when the request carries + `Accept: text/event-stream` and a valid `Mcp-Session-Id`, and still returns + `405` when the `Accept` header does not ask for a stream. A request — of any + method — carrying an `Mcp-Session-Id` the server does not know is refused + with `404` rather than served under a fresh scope, which is the signal to + re-run `initialize`. `DELETE /mcp` without a session header is still + accepted as a no-op, so clients that never adopt sessions are unaffected. - The default run-history database is created when it is first written to, not while `je_auto_control` is being imported. `HistoryStore` opens its connection (and makes its parent directory) on first use, so merely @@ -316,6 +345,15 @@ only when documented here with a migration path. ### Fixed +- The MCP HTTP transport no longer tries to drain a request body it has + already read. Any `4xx` decided *after* the body was parsed — the new + unknown-session `404` and duplicate-stream `409`, and the pre-existing + "body must be UTF-8" `400` — called `_drain_body()`, which then blocked + reading bytes that were gone until the 30-second socket timeout, pinning + that worker and logging a `ConnectionAbortedError` traceback when the peer + closed first. The drain is now skipped once the body is consumed, and a + peer that has already vanished ends it quietly instead of raising. + - **A rejected config bundle aborted the rest of the script.** Five framework errors still inherited `Exception` directly — `ConfigBundleError`, the USB passthrough `ProtocolError`, diff --git a/Progress.md b/Progress.md index 6a68a757..009c2bae 100644 --- a/Progress.md +++ b/Progress.md @@ -98,57 +98,6 @@ wheel，那時把 `windows-11-arm` 加回 `platform-smoke.yml` 的矩陣。 都綠，macOS 本來就是 arm64。所以卡住的只有 Windows 這一個組合。 -## MCP 的破壞性動作確認,在 HTTP transport 上一次都不會觸發 - -`DECIDE` — 要嘛補上 session 身分,要嘛改成 fail closed;兩條都會動到行為 - -`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` 的用意是「每個 destructive 工具執行前 -先問過人」。這件事在 stdio 上是好的,**在 HTTP transport 上一次都不會觸發**—— -而且不是文件原本以為的「initialize 跟 tools/call 落在不同連線時才失效」,是**四種 -組合全部失效**。實測(2026-08-20,真的 `HttpMCPServer`,真的 destructive 工具): - -| 情境 | 結果 | -| --- | --- | -| 行程內(等同 stdio),同一個 server 物件 | 送出 `elicitation/create`,拒絕就不執行 ✅ | -| HTTP,plain POST,同一條連線 | **不問就執行** | -| HTTP,plain POST,兩條連線 | **不問就執行** | -| HTTP,SSE,同一條連線 | **不問就執行** | -| HTTP,SSE,兩條連線 | **不問就執行** | - -原因有兩層,兩層都得處理: - -1. `_maybe_confirm_destructive` 在 `self._writer is None` 時直接 return。plain POST - 的 `connection_scope` 沒有 writer(一次 request/response,沒有 server→client 通道), - 所以這條路連問的能力都沒有。 -2. 就算是有 writer 的 SSE,`_dispatch_sse` 會把 `close_connection` 設成 True, - `finish()` 接著呼叫 `forget_connection(id(self))`。而 connection scope 是用 - `id(self)`(TCP 連線)當 key,不是用 MCP session,所以 `initialize` 帶進來的 - `capabilities`(裡面才有 `elicitation`)在下一個 `tools/call` 一定已經被忘掉, - 於是走進「client 沒有 elicitation 能力」那條分支,留一行 INFO log 後**放行**。 - -也就是說,操作者設了這個變數、以為擋住了,實際上什麼都沒擋,而且只有 INFO log。 -文件原本寫「gate every destructive tool」,只註明「舊 client 會 fall through」, -沒說 HTTP 上根本不會啟動——**這一半已經修好了**:兩份 `mcp_server_doc.rst` 都加了 -warning,說明它目前只在 stdio 有效,HTTP 請改靠 bearer token 與綁 `127.0.0.1`。 -`test_mcp_http_transport.py::test_destructive_confirm_gate_never_fires_over_http` -把現況釘住(plain POST 與 SSE 兩種),哪天有人補好了,那個測試會當場紅掉, -提醒他同一筆改動要把文件的 warning 跟本節一起改掉。 - -**還沒決定的是行為要往哪邊走**,兩個選項都不是純加法: - -- **(A) fail closed** — 變數開著又問不到人時,直接拒絕執行 destructive 工具。 - 最貼近操作者的意圖,改動也最小。但會翻掉現有的 - `test_destructive_confirmation_skipped_when_client_lacks_capability`,而且會擋掉 - 今天所有沒有 elicitation 能力的 client(包含 HTTP 上的每一個)。 -- **(B) 補上 `Mcp-Session-Id`** — 讓 scope 跟著 MCP session 走而不是跟著 TCP 連線, - 這樣 SSE 那條路才有機會真的問出來。這是 MCP 自己的答案,但這個 transport 是 - **刻意**做成 sessionless 的(`do_DELETE` 的註解就這麼寫),而 per-connection 隔離 - 正是先前一次重構的重點,有 `test_r3_mcp_connection_isolation` 在守。要動得連那條 - 隔離一起重新想,不能只是把 key 換掉。plain POST 就算有了 session 仍然沒有通道, - 所以 (B) 落地後,plain POST 還是得靠 (A) 收尾。 - -重驗方式:把上表跑一次即可,四種 HTTP 組合都應該是「不問就執行」。 - ## Wayland:剩下的都不是「缺一台機器」 這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, diff --git a/WHATS_NEW.md b/WHATS_NEW.md index 32fd83d3..457a16ed 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -320,34 +320,76 @@ real X server through `xev`, `freebsd_verify.py` pins it on a BSD, and display. The migration note for anyone who was relying on the magnitude alone is in `CHANGELOG.md`. -### The Destructive-Action Prompt Was Documented as Covering a Transport It Never Reached +### The Destructive-Action Prompt Reached Only One of the Two Transports `JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` is documented as gating *every* destructive MCP tool behind a confirmation prompt, with one stated caveat: the client has to advertise the `elicitation` capability. Measured against a real -`HttpMCPServer` with a real destructive tool, the gate fires on stdio and -**never fires over HTTP** — not in any of the four combinations of plain POST +`HttpMCPServer` with a real destructive tool, the gate fired on stdio and +**never fired over HTTP** — not in any of the four combinations of plain POST or SSE, one connection or two. -Two independent reasons, and the second one is why keeping the connection open -does not help. A plain `POST` is one request and one response, so its -connection scope has no server→client channel and the prompt has nothing to -travel down. An SSE `POST` does have one, but `_dispatch_sse` sets -`close_connection`, so the scope keyed on that TCP connection is forgotten -before the next request — and the `elicitation` capability the client -advertised at `initialize` goes with it. The call then takes the "client -cannot be prompted" branch, logs one INFO line, and runs the tool. - -An operator who sets that variable on an HTTP-exposed server is getting no -confirmation at all. Both `mcp_server_doc.rst` translations now say so, and -point at the bearer token and the `127.0.0.1` bind as the controls that do -work there. `test_destructive_confirm_gate_never_fires_over_http` pins the -current behaviour for both content types, so whoever closes the gap gets a red -test telling them to update the warning in the same change. Which of the two -fixes to take — refuse when the prompt is impossible, or give the transport -real session identity — is a behaviour change either way and is written up in -`Progress.md`. - +Two independent reasons, and the second is why keeping the connection open did +not help. A plain `POST` is one request and one response, so its connection +scope had no server→client channel and the prompt had nothing to travel down. +An SSE `POST` did have one, but `_dispatch_sse` sets `close_connection`, so the +scope keyed on that TCP connection was forgotten before the next request — and +the `elicitation` capability advertised at `initialize` went with it. The call +then took the "client cannot be prompted" branch, logged one INFO line, and ran +the tool. An operator who set that variable on an HTTP-exposed server was +getting no confirmation at all. + +**The transport now has the identity MCP actually specifies.** `initialize` +mints an `Mcp-Session-Id` and returns it as a response header; a client that +echoes it keeps one dispatcher scope across every connection it opens. `GET` +with `Accept: text/event-stream` opens the standing server→client stream that +server-initiated traffic belongs on, and answering a server request is an +ordinary `POST` matched back to the waiting call by id. `DELETE` terminates. +The dispatcher itself needed no change — it already scoped capabilities and +active-call slots on an opaque `connection_id`, so a session id simply takes +that slot, and the per-connection isolation guarded by +`test_r3_mcp_connection_isolation` is preserved verbatim: a session is just an +identity that outlives a socket. + +So the confirmation now round-trips over HTTP, and the test that proves it uses +four separate connections — one that initialized, one holding the stream, one +carrying the `tools/call`, one carrying the decline — none of which shared a +socket with the handshake. Declining blocks the tool; accepting runs it. + +Measuring it turned up a second way through that was worth pinning: an SSE +`POST` carrying a session id needs no standing stream at all, because its own +response stream is already a server-to-client channel. The `elicitation/create` +goes out ahead of the result on the same socket the call arrived on. Both paths +now have a test. + +What did *not* change is the fallback: a client that ignores the session header, +or that only ever sends plain JSON `POST`s with no stream open, has given the +server nowhere to ask, and its destructive calls still proceed — exactly as they +do for a stdio client that never advertised `elicitation`. That is now a +documented boundary with a test on each side of it rather than an accident — but +it is still a boundary, so the bearer token, the `127.0.0.1` bind and +`JE_AUTOCONTROL_MCP_READONLY` remain the controls that do not depend on the +client behaving. + +Sessions are bounded in both directions: swept after ten minutes untouched, and +the registry evicts the least recently seen once it holds 128. Dropping a +session — however it goes — releases the dispatcher state held under its id, +which is the same release a closing socket used to perform, moved to the +identity that actually owns that state. Because every `initialize` mints a +session, including for the many clients that ignore the header and never come +back, evicting one that was never used after its handshake is logged as routine; +the warning is saved for evicting a session someone was holding, which is the +one that means the cap is too low. + +The new refusals also exposed an old assumption in the transport. Every `4xx` +runs `_drain_body()` first, so the client can read the response before the +socket closes — but the new unknown-session `404` and duplicate-stream `409` are +decided *after* the body has been parsed, and so was the pre-existing "body must +be UTF-8" `400`. The drain then went looking for bytes that were already gone +and blocked until the thirty-second read timeout, pinning that worker and +printing a `ConnectionAbortedError` traceback whenever the peer closed first. +It now skips the drain once the body is consumed, and treats a peer that has +already vanished as nothing left to be courteous to. ## What's new (2026-08-19) diff --git a/architecture_explore.md b/architecture_explore.md index ef74733a..edf5f599 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -19,8 +19,8 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,028 | -| 程式碼總行數 | 139,628 | +| Python 模組總數（含周邊子專案） | 1,029 | +| 程式碼總行數 | 140,053 | | `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | @@ -488,7 +488,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,185 行。 +> 13 個套件、約 20,610 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -501,7 +501,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,898 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 17,323 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | @@ -700,14 +700,15 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,898 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（17,323 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | | `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | | `server.py` | 713 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | -| `http_transport.py` | 323 | MCP 的 HTTP 傳輸。 | +| `http_transport.py` | 514 | MCP 的 HTTP 傳輸。 | +| `http_sessions.py` | 234 | MCP 的 HTTP 傳輸用的 session 身分:`Mcp-Session-Id` 註冊表,以及每個 session 那條常駐的 server→client SSE 串流。 | | `_client_requests.py` | 217 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | | `_protocol.py` | 165 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | | `resources.py` | 303 | MCP resource 提供者。 | @@ -1019,7 +1020,7 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | | `gui/` | 89 | 26,542 | -| `utils/mcp_server/` | 20 | 16,898 | +| `utils/mcp_server/` | 21 | 17,323 | | `utils/remote_desktop/` | 56 | 11,840 | | `utils/executor/` | 6 | 9,075 | | `utils/usb/` | 17 | 4,250 | @@ -1039,5 +1040,5 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | | 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 690 | 50,425 | -| **總計** | **1,022** | **139,563** | +| **總計** | **1,023** | **139,988** | diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 75ff0b3f..44c2e332 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -253,6 +253,38 @@ box), start the same dispatcher behind HTTP: Bearer token can also come from ``JE_AUTOCONTROL_MCP_TOKEN``. +Sessions +======== + +``initialize`` mints a session and returns it in an +``Mcp-Session-Id`` response header. Echo that header on every later +request and the server keeps one scope for you — the capabilities +you advertised, and the slots your in-flight calls occupy — no +matter how many TCP connections you use. Without it each request is +scoped to its own connection, which is all the transport used to +offer. + +- ``GET /mcp`` with ``Accept: text/event-stream`` and a valid + ``Mcp-Session-Id`` opens the **standing server-to-client stream**. + Server-initiated traffic that is not a reply to a specific request + travels down it: progress notifications, and the + ``elicitation/create`` behind the confirmation gate below. One + stream per session; a second ``GET`` gets 409. The stream carries + an SSE comment as a heartbeat so an abandoned socket surfaces as a + write error rather than a parked thread. +- Answer a server request by ``POST``\ ing an ordinary JSON-RPC + response — with the session header — on any connection. It is + matched to the waiting call by id, and acked with 202. +- ``DELETE /mcp`` with the header terminates the session and + releases everything scoped to it. Without the header it is + accepted as a no-op, so sessionless clients keep working. +- An unknown or expired id is refused with **404**, not served under + a fresh scope: the client holds state the server does not, and + needs to know to re-initialize. +- Sessions are bounded. They are swept after ten minutes untouched + (a standing stream keeps its own session fresh), and the registry + evicts the least recently seen once it holds 128. + Read-only / safe mode ===================== @@ -283,19 +315,31 @@ error to the model without running the action. Requires the client to advertise the ``elicitation`` capability — older clients fall through with a logged warning. +The prompt is a question the server asks *between* receiving a call +and answering it, so it needs a channel the client is listening on +at that moment. What that means per transport: + +- **stdio** — always available; the client is on the other end of + the same pipe. +- **HTTP** — available to a client that echoes ``Mcp-Session-Id`` + (see `Sessions`_) and gives the server somewhere to send the + question. Either channel does: the standing ``GET`` stream, or an + SSE ``POST``, whose own response stream carries the + ``elicitation/create`` before the result. Either way the answer + comes back as a separate ``POST`` — the client is busy reading the + stream it asked on. + .. warning:: - **This gate is effective on the stdio transport only.** Over the - HTTP transport it never fires: the prompt needs a server→client - channel bound to the same connection scope that received - ``initialize``, and the HTTP transport is deliberately sessionless - — a plain ``POST`` has no such channel at all, and an SSE ``POST`` - closes its connection after the final event, so no later - ``tools/call`` can reach the capabilities the client advertised. - Destructive tools therefore run **unprompted** over HTTP even with - this variable set. Do not rely on it as the only control on an - HTTP-exposed server; use the bearer token and bind to - ``127.0.0.1``. + A client that ignores ``Mcp-Session-Id``, or that only ever sends + plain JSON ``POST``\ s with no stream open, has given the server + nowhere to ask — so + destructive tools run **unprompted**, exactly as they do for a + stdio client that never advertised ``elicitation``. That fallback + is logged, but it is a fallback: on an HTTP-exposed server treat + the bearer token, the ``127.0.0.1`` bind and + ``JE_AUTOCONTROL_MCP_READONLY`` as the controls that do not + depend on the client behaving. Audit log ========= diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index d5f13027..a50740a8 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -240,6 +240,31 @@ HTTP 傳輸(含 SSE / Auth / TLS) Bearer token 也可從 ``JE_AUTOCONTROL_MCP_TOKEN`` 環境變數讀取。 +Session +======= + +``initialize`` 會產生一個 session,並用 ``Mcp-Session-Id`` 回應標頭 +交給 client。之後每個請求都帶上這個標頭,伺服器就會把它們視為同一個 +scope——包含你在 ``initialize`` 聲明的能力,以及進行中呼叫佔用的槽位 +——不管你開了幾條 TCP 連線。不帶這個標頭時,每個請求只以自己的連線 +為範圍,也就是這條傳輸從前唯一的行為。 + +- ``GET /mcp`` 帶 ``Accept: text/event-stream`` 與有效的 + ``Mcp-Session-Id``,會開啟**常駐的 server→client 串流**。凡是不 + 屬於某個特定請求之回覆的伺服器主動訊息都走這條:進度通知,以及 + 下面確認關卡要送的 ``elicitation/create``。一個 session 只能有一 + 條串流,第二個 ``GET`` 會收到 409。串流會定期送出 SSE 註解當作 + 心跳,好讓斷掉的 socket 變成一個寫入錯誤,而不是一條卡住的執行緒。 +- 要回答伺服器的請求,就在任何一條連線上 ``POST`` 一個普通的 + JSON-RPC response(同樣帶 session 標頭)。伺服器會依 id 對回正在 + 等待的那個呼叫,並以 202 回覆。 +- ``DELETE /mcp`` 帶標頭會終止 session,並釋放掛在它底下的所有狀態; + 不帶標頭時照舊接受,不影響沒有 session 概念的 client。 +- 未知或已過期的 id 一律回 **404**,不會改用一個新的 scope 服務它: + client 手上有伺服器沒有的狀態,它需要知道自己該重新 initialize。 +- session 有上下界。十分鐘沒被碰過就會被掃掉(常駐串流會讓自己的 + session 保持新鮮),而註冊表滿 128 個時,最久沒動的那個會被淘汰。 + 唯讀 / 安全模式 =============== @@ -268,17 +293,25 @@ Bearer token 也可從 ``JE_AUTOCONTROL_MCP_TOKEN`` 環境變數讀取。 使用者拒絕時模型會收到乾淨的錯誤,不會執行動作。需要 client 自己 聲明 ``elicitation`` 能力;舊 client 會留下 warning log 後繼續執行。 +這個提示是伺服器在「收到呼叫」與「回答呼叫」**之間**問出去的問題, +所以它需要一條 client 當下正在聽的通道。各傳輸的情況: + +- **stdio** — 一定有;client 就在同一條 pipe 的另一端。 +- **HTTP** — 只要 client 回送 ``Mcp-Session-Id``(見 `Session`_), + 並且給伺服器一條送問題的通道就有。兩種通道都算:常駐的 ``GET`` + 串流,或是一個 SSE ``POST``——它自己的回應串流會在結果之前先送出 + ``elicitation/create``。兩種情況下,答案都要用另一個 ``POST`` 送 + 回來,因為 client 正忙著讀它問過去的那條串流。 + .. warning:: - **這道關卡目前只在 stdio transport 上有效。** 走 HTTP transport 時 - 它一次都不會觸發:送出提示需要一條與收到 ``initialize`` 的同一個 - connection scope 綁在一起的 server→client 通道,而 HTTP transport - 是刻意設計成 sessionless 的——普通 ``POST`` 根本沒有這種通道, - SSE ``POST`` 則在送完最後一個事件後就關掉連線,所以後續的 - ``tools/call`` 拿不到 client 當初聲明的能力。也就是說,即使設了這個 - 環境變數,destructive 工具在 HTTP 上仍然會**不經詢問直接執行**。 - 不要把它當成 HTTP 服務的唯一控制手段;請改用 bearer token 並綁在 - ``127.0.0.1``。 + 如果 client 不回送 ``Mcp-Session-Id``,或是自始至終只送普通的 + JSON ``POST``、一條串流都沒開,伺服器就沒有地方可問——這時 + destructive 工具會**不經詢問直接執行**,和一個從未聲明 + ``elicitation`` 的 stdio client 完全一樣。 + 這條退路會留下 log,但它終究是退路:對外開放的 HTTP 服務,請把 + bearer token、綁定 ``127.0.0.1`` 與 ``JE_AUTOCONTROL_MCP_READONLY`` + 當成真正的控制手段,它們不需要 client 配合。 稽核 Log ======== diff --git a/je_auto_control/utils/mcp_server/http_sessions.py b/je_auto_control/utils/mcp_server/http_sessions.py new file mode 100644 index 00000000..4e988410 --- /dev/null +++ b/je_auto_control/utils/mcp_server/http_sessions.py @@ -0,0 +1,234 @@ +"""MCP session identity for the HTTP transport. + +The Streamable HTTP transport used to scope every peer on the identity of +its TCP connection. That is not what MCP means by a session: a client is +free to send ``initialize`` on one connection and ``tools/call`` on the +next, and an SSE response closes its connection after the final event. Any +state established at ``initialize`` — most importantly the ``elicitation`` +capability the destructive-action confirmation gate depends on — was +therefore gone by the time it was needed. + +This module holds the identity that outlives a connection: a registry of +``Mcp-Session-Id`` values, each owning the optional standing server-to-client +SSE stream opened by ``GET``. The dispatcher itself needs no changes — it +already scopes capabilities and active calls on an opaque ``connection_id``, +so a session id simply takes that slot. + +Sessions are bounded in both directions: idle ones are swept, and the +registry never grows past ``max_sessions``. Whenever a session is dropped +the registry reports it through ``on_drop`` so the transport can release the +dispatcher state held under that id. +""" +import secrets +import threading +import time +from typing import Any, Callable, Dict, List, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +# MCP names the header in this casing; HTTP lookups are case-insensitive. +SESSION_HEADER = "Mcp-Session-Id" +# A session costs one dict entry plus at most one streaming thread, but an +# unbounded registry is still a memory target for an unauthenticated peer. +DEFAULT_MAX_SESSIONS = 128 +# How long a session may go untouched before it is swept. Clients that keep a +# standing GET stream open touch it on every heartbeat, so this bounds only +# genuinely abandoned sessions. +DEFAULT_IDLE_TIMEOUT = 600.0 +# Bytes of entropy in a session id. The spec asks for cryptographically +# secure and globally unique; 32 bytes is comfortably both. +_ID_ENTROPY_BYTES = 32 + + +class HttpSession: + """One MCP session: an identity, plus its optional outbound stream. + + The stream is the ``GET`` SSE channel the client may open to receive + server-initiated traffic — progress notifications, and the + ``elicitation/create`` request the confirmation gate sends. Without it + the server has no way to ask the client anything between a request and + its response, which is why a session with no stream still cannot be + prompted. + """ + + def __init__(self, session_id: str, now: float) -> None: + self.id = session_id + self.created_at = now + self.last_seen = now + self.closed = threading.Event() + self._lock = threading.Lock() + self._stream_writer: Optional[Callable[[str], None]] = None + + @property + def has_stream(self) -> bool: + """True while a standing server-to-client SSE stream is attached.""" + with self._lock: + return self._stream_writer is not None + + @property + def stream_writer(self) -> Optional[Callable[[str], None]]: + """The attached stream's emit callable, or ``None``.""" + with self._lock: + return self._stream_writer + + def attach_stream(self, writer: Callable[[str], None]) -> bool: + """Attach the standing stream; False when one is already attached. + + Refusing the second stream is deliberate. Two streams on one session + would make "which socket does this elicitation go down" ambiguous, + and the client cannot tell which one the server picked. + """ + with self._lock: + if self._stream_writer is not None: + return False + self._stream_writer = writer + return True + + def detach_stream(self, writer: Callable[[str], None]) -> None: + """Detach ``writer`` if it is still the attached one.""" + with self._lock: + if self._stream_writer is writer: + self._stream_writer = None + + +def _log_eviction(victim: HttpSession, cap: int) -> None: + """Report an eviction, loudly only when the victim was in use. + + Every ``initialize`` mints a session, including for the many clients that + ignore the header and never come back. Evicting one of those is routine + capacity work; evicting a session a client is actually holding means the + cap is too low for the load, and that is worth a warning. + """ + abandoned = not victim.has_stream and victim.last_seen == victim.created_at + if abandoned: + autocontrol_logger.info( + "MCP session cap %d reached — evicting a session that was never " + "used after initialize", cap, + ) + return + autocontrol_logger.warning( + "MCP session cap %d reached — evicting a live session (%s); raise " + "the cap if clients are being dropped mid-conversation", + cap, victim.id, + ) + + +class SessionRegistry: + """Bounded, sweeping registry of :class:`HttpSession` by id. + + ``on_drop`` is invoked — outside the registry lock — for every session + the registry removes, whichever way it goes: explicit termination, idle + sweep, capacity eviction or shutdown. The transport uses it to release + the dispatcher state scoped to that session id. + """ + + def __init__(self, *, + max_sessions: int = DEFAULT_MAX_SESSIONS, + idle_timeout: float = DEFAULT_IDLE_TIMEOUT, + clock: Callable[[], float] = time.monotonic, + on_drop: Optional[Callable[[HttpSession], None]] = None, + ) -> None: + self._sessions: Dict[str, HttpSession] = {} + self._lock = threading.Lock() + self._max_sessions = max(1, int(max_sessions)) + self._idle_timeout = float(idle_timeout) + self._clock = clock + self._on_drop = on_drop + + def __len__(self) -> int: + with self._lock: + return len(self._sessions) + + def create(self) -> HttpSession: + """Mint a session, sweeping expired ones and honouring the cap.""" + session_id = secrets.token_urlsafe(_ID_ENTROPY_BYTES) + now = self._clock() + dropped: List[HttpSession] = [] + with self._lock: + dropped.extend(self._expired_locked(now)) + while len(self._sessions) >= self._max_sessions: + victim = min(self._sessions.values(), + key=lambda item: item.last_seen) + del self._sessions[victim.id] + dropped.append(victim) + _log_eviction(victim, self._max_sessions) + session = HttpSession(session_id, now) + self._sessions[session_id] = session + self._announce(dropped) + return session + + def get(self, session_id: Optional[str]) -> Optional[HttpSession]: + """Return the live session for ``session_id``, touching it.""" + if not session_id: + return None + now = self._clock() + dropped: List[HttpSession] = [] + with self._lock: + dropped.extend(self._expired_locked(now)) + session = self._sessions.get(session_id) + if session is not None: + session.last_seen = now + self._announce(dropped) + return session + + def terminate(self, session_id: Optional[str]) -> Optional[HttpSession]: + """Drop ``session_id`` and close its stream; None when unknown.""" + if not session_id: + return None + with self._lock: + session = self._sessions.pop(session_id, None) + if session is not None: + self._announce([session]) + return session + + def terminate_all(self) -> List[HttpSession]: + """Drop every session — used when the transport shuts down.""" + with self._lock: + sessions = list(self._sessions.values()) + self._sessions.clear() + self._announce(sessions) + return sessions + + def _expired_locked(self, now: float) -> List[HttpSession]: + """Remove idle sessions; caller holds the lock and announces.""" + if self._idle_timeout <= 0: + return [] + stale = [session for session in self._sessions.values() + if now - session.last_seen > self._idle_timeout] + for session in stale: + del self._sessions[session.id] + return stale + + def _announce(self, dropped: List[HttpSession]) -> None: + """Close each dropped session's stream, then report it.""" + for session in dropped: + session.closed.set() + if self._on_drop is None: + continue + try: + self._on_drop(session) + except (RuntimeError, OSError, ValueError, KeyError, + AttributeError) as error: + # A session can be dropped from a sweep triggered by an + # unrelated request; a failing hook must not surface there. + autocontrol_logger.warning( + "MCP session drop hook failed for %s: %r", + session.id, error, + ) + + +def session_id_from_headers(headers: Any) -> Optional[str]: + """Read ``Mcp-Session-Id`` off a request's headers, or ``None``.""" + if headers is None: + return None + raw = headers.get(SESSION_HEADER) + if not isinstance(raw, str): + return None + value = raw.strip() + return value or None + + +__all__ = [ + "DEFAULT_IDLE_TIMEOUT", "DEFAULT_MAX_SESSIONS", "HttpSession", + "SESSION_HEADER", "SessionRegistry", "session_id_from_headers", +] diff --git a/je_auto_control/utils/mcp_server/http_transport.py b/je_auto_control/utils/mcp_server/http_transport.py index 46c667fd..d2969825 100644 --- a/je_auto_control/utils/mcp_server/http_transport.py +++ b/je_auto_control/utils/mcp_server/http_transport.py @@ -1,14 +1,22 @@ """HTTP transport for the MCP server. -Implements a minimal Streamable HTTP transport (JSON-only, no SSE -streaming) so MCP clients that prefer HTTP — or that need to reach -the server from another process / container — can talk to the same -:class:`MCPServer` dispatcher already used by the stdio transport. +Implements a Streamable HTTP transport so MCP clients that prefer HTTP — or +that need to reach the server from another process / container — can talk to +the same :class:`MCPServer` dispatcher already used by the stdio transport. Notifications are answered with ``202 Accepted`` per the MCP spec; ordinary requests return their JSON-RPC response with ``Content-Type: application/json``. The default bind is ``127.0.0.1`` to honour the project's least-privilege policy. + +**Sessions.** ``initialize`` mints an ``Mcp-Session-Id`` and returns it as a +response header; a client that echoes it back keeps one dispatcher scope +across every connection it makes, and may open a standing server-to-client +SSE stream with ``GET``. That stream is what lets the server ask the client +something mid-call — the ``elicitation/create`` behind the destructive-action +confirmation gate. A client that ignores the header still works exactly as +before, scoped to its TCP connection, but cannot be prompted: there is no +channel to carry the question. See :mod:`.http_sessions`. """ import hmac import json @@ -16,13 +24,16 @@ import ssl import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple from je_auto_control.utils.http_headers import parse_content_length from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.mcp_server._protocol import ( _notification_message, ) +from je_auto_control.utils.mcp_server.http_sessions import ( + HttpSession, SESSION_HEADER, SessionRegistry, session_id_from_headers, +) from je_auto_control.utils.mcp_server.server import MCPServer DEFAULT_PATH = "/mcp" @@ -37,12 +48,37 @@ # Bound the TLS handshake so one silent client can't wedge the single accept # thread waiting for a ClientHello that never arrives. _HANDSHAKE_TIMEOUT = 10.0 +# How often a standing GET stream writes an SSE comment. It keeps the session +# off the idle sweep and turns a client that vanished without a FIN into a +# write error instead of a thread parked forever. +_STREAM_HEARTBEAT = 15.0 + + +def _is_initialize(line: str) -> bool: + """True when ``line`` is an ``initialize`` request; tolerant of junk.""" + try: + message = json.loads(line) + except ValueError: + return False + return isinstance(message, dict) and message.get("method") == "initialize" + + +def _notifier_for(writer: Optional[Callable[[str], None]]): + """Wrap a raw writer as a (method, params) notifier, or ``None``.""" + if writer is None: + return None + return lambda method, params: writer( + _notification_message(method, params), + ) class _MCPHttpHandler(BaseHTTPRequestHandler): """Bridges HTTP requests onto :meth:`MCPServer.handle_line`.""" server_version = "AutoControlMCP/1.0" + # Set once this request's body has been read off the socket, so a later + # error response knows there is nothing left to drain. + _body_consumed = False # socketserver applies this to the connection socket in setup(); it bounds # every read (headers *and* body) so a stalled request cannot pin a worker. timeout = _REQUEST_TIMEOUT @@ -62,23 +98,60 @@ def do_POST(self) -> None: # noqa: N802 # reason: stdlib API if line is None: return bridge: MCPServer = self.server.mcp # type: ignore[attr-defined] - conn_id = id(self) + session, resolved = self._resolve_session(line) + if not resolved: + return + # Prefer the session's identity over the socket's: a client that + # echoes Mcp-Session-Id keeps one scope — and so keeps the + # capabilities it advertised at initialize — across connections. + conn_id = session.id if session is not None else id(self) + extra = {SESSION_HEADER: session.id} if session is not None else None if self._client_accepts_sse(): - self._dispatch_sse(bridge, line, conn_id) + self._dispatch_sse(bridge, line, conn_id, extra) return - # Scope this connection's identity so active-call slots and client - # capabilities don't collide with another peer that reuses the same - # JSON-RPC ids. No notifier/writer: a plain POST has no stream. - with bridge.connection_scope(connection_id=conn_id): + # A plain POST has no stream of its own, but a session may have a + # standing GET stream; server-initiated traffic belongs on it. Absent + # one there is no writer, rather than whichever other peer's socket + # happens to be open. + writer = session.stream_writer if session is not None else None + with bridge.connection_scope(connection_id=conn_id, writer=writer, + notifier=_notifier_for(writer)): response = bridge.handle_line(line) if response is None: # MCP notification — no body, ack with 202. self._send_blank(status=202) return - self._send_raw_json(response) + self._send_raw_json(response, extra_headers=extra) + + def _resolve_session(self, line: str) -> Tuple[Optional[HttpSession], + bool]: + """Resolve this request's session; False means a reply was sent. + + A request carrying an unknown or expired id is refused with 404 + rather than silently served under a fresh scope — the client has + state we do not, and it needs to know to re-initialize. + """ + registry: SessionRegistry = self.server.sessions # type: ignore[attr-defined] + header_id = session_id_from_headers(self.headers) + if header_id is not None: + session = registry.get(header_id) + if session is None: + self._send_json( + {"error": "unknown or expired session"}, status=404, + ) + return None, False + return session, True + if _is_initialize(line): + return registry.create(), True + return None, True def finish(self) -> None: - """Release this connection's per-peer server state, then close.""" + """Release this connection's per-peer server state, then close. + + Only the anonymous, connection-keyed scope is dropped here. State + held under a session id outlives the socket by design and is + released when the session is terminated, swept or evicted. + """ try: super().finish() finally: @@ -106,7 +179,8 @@ def _client_accepts_sse(self) -> bool: return _SSE_MEDIA_TYPE in accept def _dispatch_sse(self, bridge: MCPServer, line: str, - conn_id: int) -> None: + conn_id: Any, + extra_headers: Optional[Dict[str, str]] = None) -> None: """Stream progress notifications + the final response as SSE events.""" # Force connection close so the client gets EOF after the last event. self.close_connection = True @@ -115,6 +189,8 @@ def _dispatch_sse(self, bridge: MCPServer, line: str, f"{_SSE_MEDIA_TYPE}; charset=utf-8") self.send_header("Cache-Control", "no-cache") self.send_header("Connection", "close") + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) self.end_headers() send_lock = threading.Lock() @@ -144,15 +220,97 @@ def emit(payload: str) -> None: emit(response) def do_GET(self) -> None: # noqa: N802 # reason: stdlib API + """Open the session's standing server→client SSE stream.""" if not self._authorize(): return - # MCP optionally allows server→client SSE on GET; not used here. - self._send_json({"error": "GET stream not supported"}, status=405) + if self.path != DEFAULT_PATH: + self._send_json({"error": "unknown path"}, status=404) + return + if not self._client_accepts_sse(): + self._send_json( + {"error": f"GET requires Accept: {_SSE_MEDIA_TYPE}"}, + status=405, + ) + return + registry: SessionRegistry = self.server.sessions # type: ignore[attr-defined] + session = registry.get(session_id_from_headers(self.headers)) + if session is None: + self._send_json( + {"error": "unknown or expired session"}, status=404, + ) + return + self._stream_session(registry, session) + + def _stream_session(self, registry: SessionRegistry, + session: HttpSession) -> None: + """Hold this socket open as the session's outbound channel.""" + self.close_connection = True + send_lock = threading.Lock() + + def emit(payload: str) -> None: + with send_lock: + self.wfile.write(b"data: ") + self.wfile.write(payload.encode("utf-8")) + self.wfile.write(b"\n\n") + self.wfile.flush() + + # Claim the slot before writing headers, and write them under the + # same lock, so an elicitation racing the handshake cannot land in + # front of the status line. + if not session.attach_stream(emit): + self._send_json( + {"error": "session already has an open stream"}, status=409, + ) + return + try: + with send_lock: + self.send_response(200) + self.send_header("Content-Type", + f"{_SSE_MEDIA_TYPE}; charset=utf-8") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.send_header(SESSION_HEADER, session.id) + self.end_headers() + self.wfile.flush() + self._heartbeat_until_closed(registry, session, send_lock) + except OSError as error: + # The client went away, or stopped reading long enough for the + # send to time out. Either way this stream is over. + autocontrol_logger.info( + "MCP session stream %s ended: %r", session.id, error, + ) + finally: + session.detach_stream(emit) + + def _heartbeat_until_closed(self, registry: SessionRegistry, + session: HttpSession, + send_lock: threading.Lock) -> None: + """Write an SSE comment periodically until the session ends.""" + while not session.closed.wait(timeout=_STREAM_HEARTBEAT): + # Touching through the registry both keeps this session off the + # idle sweep and tells us when it has already been dropped. + if registry.get(session.id) is None: + return + with send_lock: + self.wfile.write(b": keep-alive\n\n") + self.wfile.flush() def do_DELETE(self) -> None: # noqa: N802 # reason: stdlib API + """Terminate the session named by the header, if there is one.""" if not self._authorize(): return - # Sessionless server — accept the terminate so clients can cleanup. + registry: SessionRegistry = self.server.sessions # type: ignore[attr-defined] + header_id = session_id_from_headers(self.headers) + if header_id is None: + # No session to drop — accept it so sessionless clients can + # still run their cleanup unchanged. + self._send_json({"status": "session terminated"}) + return + if registry.terminate(header_id) is None: + self._send_json( + {"error": "unknown or expired session"}, status=404, + ) + return self._send_json({"status": "session terminated"}) # --- helpers ------------------------------------------------------------- @@ -163,15 +321,21 @@ def _read_body(self) -> Optional[str]: self._send_json({"error": "invalid Content-Length"}, status=400) return None raw = self.rfile.read(length) + # From here the body is gone from the socket. Any 4xx we send later + # must not try to drain it again: there is nothing left to read, so + # the drain would block on the next request's bytes until the socket + # timeout and pin this worker for thirty seconds. + self._body_consumed = True try: return raw.decode("utf-8").strip() except UnicodeDecodeError: self._send_json({"error": "body must be UTF-8"}, status=400) return None - def _send_json(self, payload: Any, status: int = 200) -> None: + def _send_json(self, payload: Any, status: int = 200, + extra_headers: Optional[Dict[str, str]] = None) -> None: body = json.dumps(payload, ensure_ascii=False).encode("utf-8") - self._write_headers(status, body) + self._write_headers(status, body, extra_headers) self.wfile.write(body) if status >= 400: # Drain any unread request body before the socket closes. @@ -181,20 +345,29 @@ def _send_json(self, payload: Any, status: int = 200) -> None: self._drain_body() def _drain_body(self) -> None: + if self._body_consumed: + return declared = parse_content_length(self.headers) if declared <= 0: return cap = min(declared, _MAX_BODY * _DRAIN_CAP_MULTIPLE) remaining = cap - while remaining > 0: - chunk = self.rfile.read(min(remaining, _DRAIN_CHUNK)) - if not chunk: - break - remaining -= len(chunk) - - def _send_raw_json(self, raw_json: str) -> None: + try: + while remaining > 0: + chunk = self.rfile.read(min(remaining, _DRAIN_CHUNK)) + if not chunk: + break + remaining -= len(chunk) + except OSError as error: + # Draining is a courtesy to the client's read of our 4xx. If the + # peer has already gone, there is nothing left to be courteous + # about — and letting this escape logs a whole traceback for it. + autocontrol_logger.debug("MCP drain aborted: %r", error) + + def _send_raw_json(self, raw_json: str, + extra_headers: Optional[Dict[str, str]] = None) -> None: body = raw_json.encode("utf-8") - self._write_headers(200, body) + self._write_headers(200, body, extra_headers) self.wfile.write(body) def _send_blank(self, status: int) -> None: @@ -202,10 +375,14 @@ def _send_blank(self, status: int) -> None: self.send_header("Content-Length", "0") self.end_headers() - def _write_headers(self, status: int, body: bytes) -> None: + def _write_headers(self, status: int, body: bytes, + extra_headers: Optional[Dict[str, str]] = None, + ) -> None: self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) self.end_headers() @@ -218,6 +395,12 @@ def __init__(self, server_address: Tuple[str, int], super().__init__(server_address, _MCPHttpHandler) self.mcp = mcp self.auth_token = auth_token + # Dropping a session releases the dispatcher state scoped to its id — + # the same release a closing socket used to perform, moved to the + # identity that actually owns that state. + self.sessions = SessionRegistry( + on_drop=lambda session: mcp.forget_connection(session.id), + ) # No sse_lock: SSE requests used to swap server-wide notifier/writer # state and needed serialising. They now bind that state to their own # thread via MCPServer.connection_scope, so concurrent SSE streams no @@ -271,6 +454,11 @@ def address(self) -> Tuple[str, int]: def mcp(self) -> MCPServer: return self._mcp + @property + def sessions(self) -> Optional[SessionRegistry]: + """The live session registry, or ``None`` before :meth:`start`.""" + return self._server.sessions if self._server is not None else None + def start(self) -> None: """Bind the socket and begin serving on a background thread.""" if self._server is not None: @@ -298,6 +486,9 @@ def start(self) -> None: def stop(self, timeout: float = 2.0) -> None: if self._server is None: return + # Close the sessions first: a standing GET stream parks a worker on + # its heartbeat, and terminating releases it without waiting one out. + self._server.sessions.terminate_all() self._server.shutdown() self._server.server_close() if self._thread is not None: diff --git a/test/unit_test/headless/test_mcp_http_sessions.py b/test/unit_test/headless/test_mcp_http_sessions.py new file mode 100644 index 00000000..3ad934c3 --- /dev/null +++ b/test/unit_test/headless/test_mcp_http_sessions.py @@ -0,0 +1,522 @@ +"""Headless tests for MCP session identity over the HTTP transport. + +Two halves. The registry half is pure and drives an injected clock. The +transport half runs a real ``HttpMCPServer`` over real sockets, because the +thing worth proving — that a destructive tool can be confirmed over HTTP — +only exists once a session id, a standing GET stream and three separate +connections are all in play at once. +""" +import http.client +import json +import logging +import threading + +import pytest + +from je_auto_control.utils.mcp_server.http_sessions import ( + SESSION_HEADER, SessionRegistry, session_id_from_headers, +) +from je_auto_control.utils.mcp_server.http_transport import ( + DEFAULT_PATH, HttpMCPServer, _MCPHttpHandler, +) +from je_auto_control.utils.mcp_server.prompts import StaticPromptProvider +from je_auto_control.utils.mcp_server.resources import ChainProvider +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import MCPTool, MCPToolAnnotations + + +class _Clock: + """Manual monotonic clock.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + +# --- registry --------------------------------------------------------------- + + +def test_created_sessions_are_distinct_and_resolvable(): + registry = SessionRegistry(clock=_Clock()) + first, second = registry.create(), registry.create() + assert first.id != second.id + assert registry.get(first.id) is first + assert registry.get(second.id) is second + assert len(registry) == 2 + + +def test_unknown_and_blank_session_ids_resolve_to_none(): + registry = SessionRegistry(clock=_Clock()) + assert registry.get("nope") is None + assert registry.get("") is None + assert registry.get(None) is None + + +def test_idle_sessions_are_swept_and_announced(): + clock = _Clock() + dropped = [] + registry = SessionRegistry(clock=clock, idle_timeout=100.0, + on_drop=dropped.append) + session = registry.create() + clock.now += 50.0 + assert registry.get(session.id) is session # touched, so still live + clock.now += 60.0 # 60 < 100 since the touch + assert registry.get(session.id) is session + clock.now += 101.0 + assert registry.get(session.id) is None + assert [item.id for item in dropped] == [session.id] + assert session.closed.is_set() + + +def test_capacity_evicts_the_least_recently_seen(): + clock = _Clock() + dropped = [] + registry = SessionRegistry(clock=clock, max_sessions=2, + idle_timeout=0, on_drop=dropped.append) + first, second = registry.create(), registry.create() + clock.now += 1.0 + registry.get(second.id) # second is now the fresher of the two + clock.now += 1.0 + third = registry.create() + assert [item.id for item in dropped] == [first.id] + assert registry.get(first.id) is None + assert registry.get(second.id) is second + assert registry.get(third.id) is third + + +def test_terminate_and_terminate_all_announce_once(): + dropped = [] + registry = SessionRegistry(clock=_Clock(), on_drop=dropped.append) + first, second = registry.create(), registry.create() + assert registry.terminate(first.id) is first + assert registry.terminate(first.id) is None + registry.terminate_all() + assert [item.id for item in dropped] == [first.id, second.id] + assert second.closed.is_set() + assert len(registry) == 0 + + +def test_a_failing_drop_hook_does_not_break_the_caller(): + def boom(_session): + raise RuntimeError("hook exploded") + + registry = SessionRegistry(clock=_Clock(), on_drop=boom) + session = registry.create() + assert registry.terminate(session.id) is session # no exception escapes + + +def test_only_one_stream_attaches_per_session(): + registry = SessionRegistry(clock=_Clock()) + session = registry.create() + first, second = (lambda _payload: None), (lambda _payload: None) + assert session.attach_stream(first) is True + assert session.has_stream is True + assert session.attach_stream(second) is False + session.detach_stream(second) # not the owner — no-op + assert session.stream_writer is first + session.detach_stream(first) + assert session.has_stream is False + + +def test_session_id_header_is_read_case_insensitively(): + assert session_id_from_headers({"Mcp-Session-Id": " abc "}) == "abc" + assert session_id_from_headers({"Mcp-Session-Id": " "}) is None + assert session_id_from_headers({}) is None + assert session_id_from_headers(None) is None + + +# --- transport -------------------------------------------------------------- + + +def _destructive_tool(ran): + return MCPTool( + name="zap", description="destructive probe", + input_schema={"type": "object", "properties": {}}, + handler=lambda: ran.append(1) or "RAN", + annotations=MCPToolAnnotations(destructive=True, read_only=False), + ) + + +@pytest.fixture() +def live_server(monkeypatch): + """A real HttpMCPServer carrying one destructive tool, gate armed.""" + monkeypatch.setenv("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "1") + ran = [] + server = HttpMCPServer(mcp=MCPServer( + tools=[_destructive_tool(ran)], resource_provider=ChainProvider([]), + prompt_provider=StaticPromptProvider([]), + ), host="127.0.0.1", port=0) + server.start() + try: + yield server, ran + finally: + server.stop(timeout=2.0) + + +def _connect(server): + host, port = server.address + return http.client.HTTPConnection(host, port, timeout=20) + + +def _post(conn, payload, session_id=None, accept="application/json"): + headers = {"Content-Type": "application/json", "Accept": accept} + if session_id is not None: + headers[SESSION_HEADER] = session_id + conn.request("POST", DEFAULT_PATH, body=json.dumps(payload), + headers=headers) + response = conn.getresponse() + body = response.read().decode("utf-8") + return response, body + + +_INIT = { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {"elicitation": {}}, + "clientInfo": {"name": "probe", "version": "1"}, + }, +} +_CALL = { + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": {"name": "zap", "arguments": {}}, +} + + +def _initialize(server): + """Run initialize on its own connection; return the minted session id.""" + conn = _connect(server) + try: + response, _body = _post(conn, _INIT) + assert response.status == 200 + session_id = response.getheader(SESSION_HEADER) + assert session_id, "initialize must mint a session id" + return session_id + finally: + conn.close() + + +def _read_sse_event(response): + """Read one SSE event's data payload, skipping comments and blanks. + + The connection's socket timeout is what bounds this, so a stream that + never carries the expected event fails the test instead of hanging it. + """ + while True: + raw = response.readline() + if not raw: + return None + line = raw.decode("utf-8").rstrip("\r\n") + if line.startswith("data: "): + return json.loads(line[len("data: "):]) + + +def test_initialize_mints_a_session_id(live_server): + server, _ran = live_server + session_id = _initialize(server) + assert server.sessions.get(session_id) is not None + + +def test_capabilities_survive_a_new_connection_when_the_id_is_echoed( + live_server): + server, _ran = live_server + session_id = _initialize(server) + # A second, entirely separate TCP connection. + conn = _connect(server) + try: + with server.mcp.connection_scope(connection_id=session_id): + assert "elicitation" in server.mcp._client_capabilities + finally: + conn.close() + + +def test_unknown_session_id_is_refused(live_server): + server, _ran = live_server + conn = _connect(server) + try: + response, _body = _post(conn, _CALL, session_id="not-a-session") + assert response.status == 404 + finally: + conn.close() + + +def test_delete_terminates_the_session(live_server): + server, _ran = live_server + session_id = _initialize(server) + conn = _connect(server) + try: + conn.request("DELETE", DEFAULT_PATH, + headers={SESSION_HEADER: session_id}) + assert conn.getresponse().read() is not None + conn.close() + conn = _connect(server) + response, _body = _post(conn, _CALL, session_id=session_id) + assert response.status == 404 + finally: + conn.close() + + +def test_get_without_sse_accept_is_rejected(live_server): + server, _ran = live_server + session_id = _initialize(server) + conn = _connect(server) + try: + conn.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "application/json", + }) + assert conn.getresponse().status == 405 + finally: + conn.close() + + +def test_get_stream_without_a_session_is_rejected(live_server): + server, _ran = live_server + conn = _connect(server) + try: + conn.request("GET", DEFAULT_PATH, + headers={"Accept": "text/event-stream"}) + assert conn.getresponse().status == 404 + finally: + conn.close() + + +def test_destructive_confirmation_round_trips_over_http(live_server): + """The whole point of session identity: a decline blocks the tool. + + Three connections, as a real Streamable HTTP client would use: one that + initialized, one holding the standing GET stream, and one carrying the + tools/call. The elicitation goes out on the stream and its reply comes + back on a fourth — none of which shared a socket with the handshake. + """ + server, ran = live_server + session_id = _initialize(server) + + stream_conn = _connect(server) + stream_conn.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + stream = stream_conn.getresponse() + assert stream.status == 200 + + call_result = {} + + def run_call(): + conn = _connect(server) + try: + _response, body = _post(conn, _CALL, session_id=session_id) + call_result["body"] = body + finally: + conn.close() + + caller = threading.Thread(target=run_call, daemon=True) + caller.start() + try: + prompt = _read_sse_event(stream) + assert prompt is not None, "expected elicitation on the stream" + assert prompt["method"] == "elicitation/create" + + reply_conn = _connect(server) + try: + response, _body = _post(reply_conn, { + "jsonrpc": "2.0", "id": prompt["id"], + "result": {"action": "decline"}, + }, session_id=session_id) + assert response.status == 202 + finally: + reply_conn.close() + + caller.join(timeout=20.0) + assert not caller.is_alive() + finally: + stream_conn.close() + + payload = json.loads(call_result["body"]) + assert payload["error"]["code"] == -32000 + assert "declined" in payload["error"]["message"] + assert ran == [], "a declined destructive tool must not run" + + +def test_accepting_the_confirmation_runs_the_tool(live_server): + server, ran = live_server + session_id = _initialize(server) + + stream_conn = _connect(server) + stream_conn.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + stream = stream_conn.getresponse() + call_result = {} + + def run_call(): + conn = _connect(server) + try: + _response, body = _post(conn, _CALL, session_id=session_id) + call_result["body"] = body + finally: + conn.close() + + caller = threading.Thread(target=run_call, daemon=True) + caller.start() + try: + prompt = _read_sse_event(stream) + reply_conn = _connect(server) + try: + _post(reply_conn, { + "jsonrpc": "2.0", "id": prompt["id"], + "result": {"action": "accept", "content": {}}, + }, session_id=session_id) + finally: + reply_conn.close() + caller.join(timeout=20.0) + finally: + stream_conn.close() + + payload = json.loads(call_result["body"]) + assert payload["result"]["isError"] is False + assert ran == [1] + + +def test_second_stream_on_one_session_is_refused(live_server): + server, _ran = live_server + session_id = _initialize(server) + first = _connect(server) + first.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + assert first.getresponse().status == 200 + second = _connect(server) + try: + second.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + assert second.getresponse().status == 409 + finally: + second.close() + first.close() + + +def test_evicting_an_abandoned_session_is_not_logged_as_a_warning(caplog): + """Every initialize mints a session; churning through them is routine. + + A client that ignores the header leaves a session behind on every + handshake. Warning on each eviction would put one line per request in + the log of any busy sessionless server, which buries the case that does + matter: a session someone was holding. + """ + clock = _Clock() + registry = SessionRegistry(clock=clock, max_sessions=1, idle_timeout=0) + registry.create() + with caplog.at_level(logging.INFO): + registry.create() + assert not [record for record in caplog.records + if record.levelno >= logging.WARNING] + assert any("never used after initialize" in record.getMessage() + for record in caplog.records) + + +def test_evicting_a_session_in_use_warns(caplog): + clock = _Clock() + registry = SessionRegistry(clock=clock, max_sessions=1, idle_timeout=0) + held = registry.create() + clock.now += 1.0 + registry.get(held.id) # touched, so no longer an abandoned shell + with caplog.at_level(logging.INFO): + registry.create() + warnings = [record for record in caplog.records + if record.levelno >= logging.WARNING] + assert len(warnings) == 1 + assert "evicting a live session" in warnings[0].getMessage() + + +def test_an_sse_post_can_carry_its_own_confirmation(live_server): + """A session'd SSE POST is promptable without a standing GET stream. + + The response stream is itself a server-to-client channel, so the + elicitation goes down the same socket the call arrived on. The reply + still comes back as a separate POST, because the client is busy reading + this one. + """ + server, ran = live_server + session_id = _initialize(server) + + conn = _connect(server) + conn.request("POST", DEFAULT_PATH, body=json.dumps(_CALL), headers={ + "Content-Type": "application/json", "Accept": "text/event-stream", + SESSION_HEADER: session_id, + }) + stream = conn.getresponse() + try: + prompt = _read_sse_event(stream) + assert prompt is not None and prompt["method"] == "elicitation/create" + + reply_conn = _connect(server) + try: + _post(reply_conn, { + "jsonrpc": "2.0", "id": prompt["id"], + "result": {"action": "decline"}, + }, session_id=session_id) + finally: + reply_conn.close() + + final = _read_sse_event(stream) + assert final["error"]["code"] == -32000 + assert "declined" in final["error"]["message"] + finally: + conn.close() + assert ran == [] + + +class _RecordingReader: + """Stands in for a handler's ``rfile``, remembering every read.""" + + def __init__(self, chunks=b""): + self.reads = [] + self._buffer = chunks + + def read(self, size): + self.reads.append(size) + chunk, self._buffer = self._buffer[:size], self._buffer[size:] + return chunk + + +class _VanishedReader: + """An ``rfile`` whose peer is already gone.""" + + def read(self, _size): + raise ConnectionAbortedError(10053, "connection aborted") + + +def _bare_handler(*, consumed, reader, length="10"): + """A handler instance with no socket, for the body-drain logic alone.""" + handler = _MCPHttpHandler.__new__(_MCPHttpHandler) + handler._body_consumed = consumed + handler.rfile = reader + handler.headers = {"Content-Length": length} + return handler + + +def test_drain_is_skipped_once_the_body_has_been_read(): + """A 4xx raised after _read_body must not go looking for more body. + + It would find the *next* request's bytes, or nothing at all, and block + on the socket until the 30s read timeout — pinning the worker for every + unknown-session 404 and every duplicate-stream 409. + """ + reader = _RecordingReader(b"leftover!!") + handler = _bare_handler(consumed=True, reader=reader) + handler._drain_body() + assert reader.reads == [] + + +def test_drain_still_runs_when_the_body_was_never_read(): + reader = _RecordingReader(b"0123456789") + handler = _bare_handler(consumed=False, reader=reader) + handler._drain_body() + assert reader.reads, "an unread body is still drained" + + +def test_drain_survives_a_peer_that_vanished(): + handler = _bare_handler(consumed=False, reader=_VanishedReader()) + handler._drain_body() # must not raise: courtesy to a peer that has gone diff --git a/test/unit_test/headless/test_mcp_http_transport.py b/test/unit_test/headless/test_mcp_http_transport.py index 5deebbb9..236b4ca1 100644 --- a/test/unit_test/headless/test_mcp_http_transport.py +++ b/test/unit_test/headless/test_mcp_http_transport.py @@ -293,18 +293,21 @@ def _post_with_accept(server, body, accept): @pytest.mark.parametrize("accept", ["application/json", "text/event-stream"]) -def test_destructive_confirm_gate_never_fires_over_http(monkeypatch, accept): - """Pin the known gap: JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE is stdio-only. - - The elicitation prompt needs a server->client channel bound to the same - connection scope that received ``initialize``. A plain POST has no such - channel, and an SSE POST closes its connection after the final event, so - the capabilities the client advertised are gone by the next ``tools/call``. - Destructive tools therefore run unprompted over HTTP even with the gate on. - - This is recorded in ``Progress.md`` and warned about in the MCP docs. When - the transport grows real session identity, this test fails -- that is the - point: update the docs warning and the Progress.md entry in the same change. +def test_destructive_confirm_gate_needs_a_session_the_client_keeps( + monkeypatch, accept): + """A client that ignores Mcp-Session-Id cannot be prompted, by construction. + + The prompt is an ``elicitation/create`` the server has to send *between* + receiving a call and answering it, so it needs a channel the client is + listening on. A client that neither echoes the session id nor opens the + standing GET stream has given the server nowhere to ask, and the call + proceeds — the same fallback a stdio client without the ``elicitation`` + capability takes. + + A client that does echo the id and open the stream is prompted for real; + that is ``test_mcp_http_sessions.py``. This test pins the other side of + that line so the fallback stays deliberate rather than becoming the only + behaviour again. """ from je_auto_control.utils.mcp_server.tools import ( MCPTool, MCPToolAnnotations, @@ -332,6 +335,7 @@ def test_destructive_confirm_gate_never_fires_over_http(monkeypatch, accept): "clientInfo": {"name": "probe", "version": "1"}, }, }, accept) + # No Mcp-Session-Id echoed back, and no GET stream opened. call_body = _post_with_accept(server, { "jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": "zap_probe", "arguments": {}}, @@ -341,5 +345,5 @@ def test_destructive_confirm_gate_never_fires_over_http(monkeypatch, accept): assert "elicitation/create" not in init_body assert "elicitation/create" not in call_body - assert ran == [1], "gate did not fire, so the tool should have run" + assert ran == [1], "with nowhere to ask, the call proceeds" assert '"isError": false' in call_body