diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d76d7236..d9a58a40 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -241,6 +241,54 @@ jobs: - name: Verify the portal handshake against liboeffis run: docker run --rm autocontrol-portal:ci + x11-verification: + name: X11 backend against a real X server + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the X11 verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.x11 + tags: autocontrol-x11:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # Wayland ended up with five jobs that read back what reached a real + # peer. X11 — the older and more widely deployed of the two Linux + # paths — had none: every X11 assertion in the suite is made against a + # mock of python-Xlib, so nothing had confirmed that an injected event + # reaches a client at all. + # + # Ground truth deliberately comes from other codebases than the one + # under test. xev is a real X client that prints every event delivered + # to its window, so a click is read back the way the ydotool job reads + # its events off /dev/input/eventN — including `synthetic NO`, which is + # what separates real server input from XSendEvent traffic that + # toolkits discard. ImageMagick's `import` is an independent grabber, + # in the role grim plays for Wayland, against a root window painted two + # asymmetric colours so a wrong rectangle cannot look right. + # + # It runs twice: one monitor over the whole screen, then two RANDR + # monitors side by side. There is deliberately no negative-origin pass + # — on X11 the root window is the union of every monitor and always + # starts at (0, 0), so the Wayland job's second layout has no analogue + # here. That is a protocol difference, not an untested case. + # + # The container exits with the number of failed checks. + - name: Verify the X11 backend against a real X server + run: docker run --rm autocontrol-x11:ci + seat-verification: name: ydotool absolute move against a seat that consumes it needs: build-image diff --git a/.github/workflows/platform-smoke.yml b/.github/workflows/platform-smoke.yml index 5f72b736..f22bb1a4 100644 --- a/.github/workflows/platform-smoke.yml +++ b/.github/workflows/platform-smoke.yml @@ -14,7 +14,31 @@ jobs: strategy: fail-fast: false matrix: - os: [windows-2022, ubuntu-22.04, macos-14] + # arm64 is not a rounding error on the desktop any more, and the + # dependency set is where it shows. macos-14 is already arm64; + # ubuntu-22.04-arm adds Linux, and it passes. + # + # windows-11-arm is deliberately absent, and it was measured rather + # than assumed. Two dependencies have no win_arm64 wheel, and both + # have to go before the runner is worth adding back: + # + # opencv-python — no win_arm64 wheel in any version, so pip falls + # back to building from source and CMake cannot + # configure for ARM64. Twelve minutes, then failure. + # cryptography — wheels stop at 46.0.3; 46.0.4 onwards ship none. + # Our floor is >=48.0.1 and that is a security floor + # (GHSA-537c-gmf6-5ccf), so it cannot be lowered. + # + # Neither is a CI problem to work around — the package genuinely + # cannot be installed on Windows arm64 today. Re-check without a + # runner, in about ten seconds: + # + # pip install --dry-run --only-binary=:all: --platform win_arm64 \ + # --python-version 3.12 --target /tmp/probe \ + # 'opencv-python>=4.8,<6' 'cryptography>=48.0.1' + # + # Recorded in Progress.md; add the runner back when both resolve. + os: [windows-2022, ubuntu-22.04, macos-14, ubuntu-22.04-arm] python-version: ["3.10", "3.14"] runs-on: ${{ matrix.os }} steps: @@ -49,3 +73,118 @@ jobs: name: platform-smoke-${{ matrix.os }}-${{ matrix.python-version }} path: platform-smoke.zip if-no-files-found: warn + + freebsd: + name: The X11 backend driving input on a real FreeBSD + runs-on: ubuntu-22.04 + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v4 + + # The X11 backend was gated on sys.platform being linux/linux2, so it + # refused to load on a FreeBSD desktop that runs the same X server, the + # same python-Xlib and the same code. Relaxing that guard is only worth + # something if a BSD actually runs it, and no hosted runner is one — so + # this boots a real FreeBSD VM inside the runner. + # + # For a while it could only check the *decision*, because importing + # anything under je_auto_control ran the facade and the facade imported + # OpenCV and cryptography at module scope. Neither publishes a FreeBSD + # wheel and building them from ports had not finished after fifty + # minutes, so utils/platform_id was loaded by file path and the backend + # itself went untested. + # + # That was the wrong thing to work around. Moving a mouse needs neither + # package, and the facade no longer insists on them — they are imported + # by the functions that use them, which test_facade_import_is_light.py + # keeps true. What is left for this VM is python-Xlib and defusedxml, + # both pure Python, plus an X server. So the whole backend runs here now + # and the reads come back off the server itself: query_pointer for the + # cursor, its button mask for the buttons, query_keymap for the keys. + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + - uses: vmactions/freebsd-vm@v1 # NOSONAR githubactions:S7637 + with: + release: "14.2" + usesh: true + prepare: | + pkg install -y python311 xorg-vfbserver + run: | + set -eu + echo "uname: $(uname -a)" + + # pip comes from ensurepip, not from pkg: FreeBSD 14.2's repository + # has no py311-pip (the flavoured port names are not dependable + # here, while python311 itself is). These are the only dependencies + # the facade still needs, all pure Python, at the versions + # pyproject pins and by their PyPI names. + # + # --no-deps is the point of this job rather than a detail: it is + # what proves nothing heavy is being dragged in behind the + # verification. six is therefore named explicitly — python-Xlib + # 0.33 imports it from Xlib.display, and with --no-deps nothing + # else would install it. + # + # py311-sqlite3 is deliberately not installed either. FreeBSD + # packages sqlite3 apart from python311, this VM is the only + # machine in CI that does, and it is what caught ten subsystems + # importing it at module scope — which made `import + # je_auto_control` fail outright on a stock FreeBSD. Adding the + # package here would make that regression invisible again. + python3.11 -m ensurepip --upgrade + python3.11 -m pip install --no-deps \ + python-xlib==0.33 six defusedxml==0.7.1 + + # The backend connects to a display at import time, so the server + # has to be up first. 1280x1024 because the verification drives the + # cursor to the far corner and reads it back. + Xvfb :99 -screen 0 1280x1024x24 & + xvfb_pid=$! + trap 'kill "$xvfb_pid" 2>/dev/null || true' EXIT + waited=0 + while [ ! -e /tmp/.X11-unix/X99 ]; do + waited=$((waited + 1)) + if [ "$waited" -gt 100 ]; then + echo "Xvfb never created /tmp/.X11-unix/X99" >&2 + exit 1 + fi + sleep 0.1 + done + + DISPLAY=:99 PYTHONPATH="$(pwd)" python3.11 test/verify/freebsd_verify.py + macos-capabilities: + name: What a real macOS runner permits + runs-on: macos-14 + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - run: python -m pip install -e . # NOSONAR githubactions:S8544 # reason: installs the checked-out project itself, there is no upstream version to lock + + # macOS is the one supported platform with no container to put it in, + # and every macOS row in docs/CAPABILITY_MATRIX.md said + # "implementation": the code was there and nothing had run it on a Mac. + # + # Two of these capabilities are gated by TCC — macOS asks a *user* to + # grant Screen Recording and Accessibility, and a CI runner has no user + # to ask. Which of them a runner grants is not something to guess at, + # and guessing is how the Wayland work twice recorded a desktop's + # refusal as a container's limitation. + # + # Measured first, in --measure mode, and the answer was a surprise: a + # macos-14 runner grants BOTH Screen Recording and Accessibility, so + # every capability works — capture returns real pixels rather than the + # black rectangle a refusal produces, CGEventPost moves the cursor and + # the move reads back exactly, and the AX walk returns real elements. + # The usual assumption that CI cannot exercise a TCC-gated macOS API is + # simply wrong for this runner. + # + # So the flag is off and this is a gate now: EXPECTED in the script + # holds what was measured, and a capability appearing or disappearing + # turns this red and names which one. + - name: Verify the macOS backend against a real window server + run: python test/verify/macos_verify.py diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index d159b626..a7ed579c 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -60,11 +60,26 @@ jobs: run: bandit -r je_auto_control/ -c pyproject.toml pytest-headless: - runs-on: windows-2022 + # The suite ran on Windows alone for its whole life, so every + # platform assumption it holds went unmeasured on the two operating + # systems the project also claims to support. Linux and macOS are + # here to measure them; they carry the two ends of the supported + # Python range rather than all five, because what differs between + # 3.10 and 3.14 is Python and what differs here is the OS. + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: [ "3.10", "3.11", "3.12", "3.13", "3.14" ] + include: + - { os: windows-2022, python-version: "3.10" } + - { os: windows-2022, python-version: "3.11" } + - { os: windows-2022, python-version: "3.12" } + - { os: windows-2022, python-version: "3.13" } + - { os: windows-2022, python-version: "3.14" } + - { os: ubuntu-22.04, python-version: "3.10" } + - { os: ubuntu-22.04, python-version: "3.14" } + - { os: macos-14, python-version: "3.10" } + - { os: macos-14, python-version: "3.14" } steps: - uses: actions/checkout@v4 @@ -74,25 +89,58 @@ jobs: python-version: ${{ matrix.python-version }} cache: "pip" - - name: Install dependencies + # Same set the container image installs, and for the same reasons: + # the X11 backend connects to a display at import time, opencv and + # PySide6 hard-require libGL/glib, and Qt's platform plugin needs + # the xcb libraries. Without these the suite fails at collection + # with a linker error rather than a test result. + - name: Install X11 and Qt runtime libraries (Linux) + if: runner.os == 'Linux' run: | - python -m pip install --upgrade pip wheel - # Install the editable package FIRST so its source dir is the - # one Python sees on subsequent imports. We deliberately - # avoid `pip install -r dev_requirements.txt` here because - # that file pulls in `je_auto_control_dev` (a separate PyPI - # package), which ships its own snapshot of `je_auto_control/` - # straight into site-packages and masks the editable install - # for any sub-package the snapshot doesn't include - # (admin, usb, remote_desktop, vision, …). - pip install -e . - pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 pytest-cov==7.0.0 PySide6==6.11.1 + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + xvfb xauth x11-utils \ + libgl1 libegl1 libglib2.0-0 \ + libxkbcommon-x11-0 libdbus-1-3 \ + libxcb-cursor0 libxcb-icccm4 libxcb-image0 libxcb-keysyms1 \ + libxcb-randr0 libxcb-render-util0 libxcb-shape0 libxcb-sync1 \ + libxcb-xfixes0 libxcb-xinerama0 libxcb-xkb1 + + # One command per step, each on a single line: a `run: |` block scalar + # swallows NOSONAR markers, so a justification inside one is a comment + # that reads as handled and suppresses nothing. + - name: Upgrade the installer + shell: bash + run: python -m pip install --upgrade pip wheel # NOSONAR githubactions:S8544 # reason: pip and wheel are the installer; pinning them here would pin the tool that applies the pins below + + # Install the editable package FIRST so its source dir is the one + # Python sees on subsequent imports. We deliberately avoid + # `pip install -r dev_requirements.txt` here because that file pulls in + # `je_auto_control_dev` (a separate PyPI package), which ships its own + # snapshot of `je_auto_control/` straight into site-packages and masks + # the editable install for any sub-package the snapshot doesn't include + # (admin, usb, remote_desktop, vision, …). + - name: Install the project itself + shell: bash + run: pip install -e . # NOSONAR githubactions:S8544 githubactions:S8541 # reason: installs the checked-out project itself, so there is no upstream version to lock and no third-party setup script to run + + - name: Install the test tooling + shell: bash + # Quoted: `--only-binary :all:` puts a colon-space inside the + # scalar, which YAML reads as a mapping and refuses. + run: "pip install --only-binary :all: ruff==0.15.22 bandit==1.9.4 pytest==9.1.1 pytest-timeout==2.4.0 pytest-rerunfailures==15.1 pytest-cov==7.0.0 PySide6==6.11.1" # Paths come from `testpaths` in pyproject.toml. Do NOT pass an explicit # path here: an argument overrides testpaths, which previously meant the # flow_control tests were configured to run but silently never did. + # + # Linux runs under a real Xvfb rather than Qt's offscreen platform: + # the X11 backend opens a display at import time, and offscreen + # would hide exactly the breakage this job exists to find. - name: Run headless pytest suite + shell: bash run: >- + ${{ runner.os == 'Linux' && 'xvfb-run -a -s "-screen 0 1280x800x24"' || '' }} pytest -v --tb=short --timeout=120 --cov=je_auto_control --cov-report=term-missing --cov-report=xml --cov-fail-under=35 @@ -100,7 +148,7 @@ jobs: - name: Upload coverage report uses: actions/upload-artifact@v4 with: - name: coverage-${{ matrix.python-version }} + name: coverage-${{ matrix.os }}-${{ matrix.python-version }} path: coverage.xml typing-stable-api: diff --git a/CHANGELOG.md b/CHANGELOG.md index 5aeea62a..7af7c5f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,41 @@ only when documented here with a migration path. ### Added +- **The macOS recorder works.** `record()`, `stop_record()`, + `stop_record_timeline()`, the `AC_record*` commands, the `ac_record_*` MCP + tools and `je_auto_control record` all run on macOS now; they used to refuse + outright with "Cannot use recorder on macOS". Capture goes through a + listen-only Quartz `CGEventTap` on its own thread + (`je_auto_control.osx.listener.osx_listener.OSXInputTap`), so it records + presses, releases, the wheel and per-event timing, exactly as the Windows + hook does. Requires Accessibility permission; without it the tap raises + `AutoControlRecordException` naming the permission — where the facade's + `record()` logs it, as it does every other backend's start failure — rather + than starting a session that silently records nothing. +- `je_auto_control.utils.input_macro.recorder_base` — the platform-neutral + half of recording: `timeline()`, `legacy_action_queue()` and the + `InputRecorder` base the Windows and macOS recorders now share. + `timeline` keeps working when imported from + `je_auto_control.windows.record.win32_input_hook`, where it used to live. +- Cross-platform window management. The 23 `AC_*` window commands and their + MCP tools now work on macOS and Linux/X11 as well as Windows, through a + backend seam (`je_auto_control.wrapper.window_backends`). Wayland remains + unsupported: the protocol does not let a client enumerate or move another + application's windows. +- Linux accessibility backend over AT-SPI2 + (`je_auto_control.utils.accessibility.backends.linux_backend`), with no new + dependency. Serves both X11 and Wayland sessions. +- `je_auto_control.utils.platform_id` — one place that classifies the + operating system family, and the BSDs are now one of them. FreeBSD, OpenBSD, + NetBSD and DragonFly route to the X11 backend instead of raising "unknown + operating system". +- `AutoControlUnsupportedOperationException`, raised when a platform backend + cannot perform an operation. It subclasses both `AutoControlException` and + `NotImplementedError`, so existing `except NotImplementedError` handlers are + unaffected while the executor's containment boundaries now catch it. +- `je_auto_control.utils.dbus_client` — the D-Bus client, moved out of + `linux_wayland/` so `utils/` can use it. The old path re-exports it. + - Stable, headless `je_auto_control.api` façade. - Portable `autocontrol.failure-bundle/v1` diagnostic archives and CLI command. - Public API lifecycle, capability matrix, security policy, coverage and type @@ -89,6 +124,26 @@ only when documented here with a migration path. land somewhere else; unset (or any unrecognised value, which says so and falls back) keeps the existing warn-once-and-move behaviour. The libei path is absolute at the protocol level and is not affected either way. +- **MCP sessions over HTTP.** `initialize` now mints an `Mcp-Session-Id` and + returns it as a response header. A client that echoes it keeps one + dispatcher scope — the capabilities it advertised, and the slots its + in-flight calls occupy — across every connection it opens, instead of one + scope per TCP connection. `GET /mcp` with `Accept: text/event-stream` and a + valid session id opens the standing server-to-client SSE stream (one per + session; a second gets 409), `DELETE /mcp` with the id terminates the + session, and a server request is answered by `POST`ing an ordinary JSON-RPC + response on any connection. Sessions are swept after ten minutes untouched + and capped at 128. `je_auto_control.utils.mcp_server.http_sessions` holds + the registry. +- **`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` now works over HTTP** — for a + client that echoes `Mcp-Session-Id` and holds the `GET` stream open. It + previously fired only on stdio: the prompt needs a server-to-client channel + bound to the scope that received `initialize`, and a connection-keyed scope + never survived to the `tools/call`. A client that does neither still cannot + be prompted and its destructive calls still proceed, exactly as for a stdio + client that never advertised `elicitation`; that fallback is documented and + is not a substitute for the bearer token, the `127.0.0.1` bind or + `JE_AUTOCONTROL_MCP_READONLY`. ### Removed @@ -110,6 +165,49 @@ only when documented here with a migration path. ### Changed +- The MCP HTTP transport answers `GET /mcp` differently. It used to return + `405` with `{"error": "GET stream not supported"}` for every request; it now + serves the session's SSE stream when the request carries + `Accept: text/event-stream` and a valid `Mcp-Session-Id`, and still returns + `405` when the `Accept` header does not ask for a stream. A request — of any + method — carrying an `Mcp-Session-Id` the server does not know is refused + with `404` rather than served under a fresh scope, which is the signal to + re-run `initialize`. `DELETE /mcp` without a session header is still + accepted as a no-op, so clients that never adopt sessions are unaffected. +- The default run-history database is created when it is first written to, + not while `je_auto_control` is being imported. `HistoryStore` opens its + connection (and makes its parent directory) on first use, so merely + importing the package no longer creates + `~/.je_auto_control/run_history.sqlite`. Every method behaves as before; + a store that was never used and then closed simply never touched the + disk. + +- **The sign of `scroll_value` picks the scroll direction on every platform.** + Windows and macOS have always read it that way; X11 and Wayland took the + direction from `scroll_direction` alone and used `abs(scroll_value)`, so + `mouse_scroll(-3)` — code written and tested against the Windows convention — + scrolled *down* three notches on Linux instead of up, with no exception and + no warning. `scroll_direction` now names the direction a **positive** count + takes, and a negative count reverses it, on all four backends. + + *Migration.* Code that passed a negative `scroll_value` to Linux or Wayland + and relied on the magnitude alone now scrolls the opposite way. Take + `abs()` at the call site to keep the old behaviour: + `mouse_scroll(abs(value), scroll_direction="scroll_down")`. Code that passed + a positive count is unaffected, as is every Windows and macOS caller. +- **`import je_auto_control` no longer imports OpenCV, NumPy, Pillow, + `je_open_cv` or `cryptography`.** They are imported by the functions that use + them. The facade pulled all five in at module scope, so a platform without + wheels for them — a FreeBSD desktop, for one — could not use the input + automation half of the package at all, though it needs none of them. Nothing + moves in the public API and the packages remain hard dependencies; what + changes is *when* a missing one is reported, which is now at the first image + or encryption call rather than at import. `test_facade_import_is_light.py` + keeps it that way. +- **`macos_record_error_message` now names a permission, not a platform.** It + read "Cannot use recorder on macOS", which described a limitation that no + longer exists; it now names the Accessibility grant that recording actually + needs, and is raised from the event tap rather than from the wrapper. - **The `xdg-desktop-portal` capture tier no longer needs `gdbus` installed.** It speaks D-Bus directly, so `linux_wayland.portal.is_available()` now reports whether a session bus address is set rather than whether the `gdbus` @@ -247,6 +345,84 @@ only when documented here with a migration path. ### Fixed +- The MCP HTTP transport no longer tries to drain a request body it has + already read. Any `4xx` decided *after* the body was parsed — the new + unknown-session `404` and duplicate-stream `409`, and the pre-existing + "body must be UTF-8" `400` — called `_drain_body()`, which then blocked + reading bytes that were gone until the 30-second socket timeout, pinning + that worker and logging a `ConnectionAbortedError` traceback when the peer + closed first. The drain is now skipped once the body is consumed, and a + peer that has already vanished ends it quietly instead of raising. + +- **A rejected config bundle aborted the rest of the script.** Five + framework errors still inherited `Exception` directly — + `ConfigBundleError`, the USB passthrough `ProtocolError`, + `SessionError` and `UsbClientError`, and the work queue's + `BusinessError` — and the containment boundaries all catch the + `AutoControlException` family, so none of them caught these. A + malformed bundle passed to `AC_config_import` therefore raised straight + past the executor's per-action boundary and killed every remaining + action, even under `raise_on_error=False`; `AC_usb_remote_devices` and + `AC_usb_remote_open` had the same path through `UsbClientError`. All + five derive from `AutoControlException` now, so they are recorded as a + failed action like every other framework error. `LoopBreak`, + `LoopContinue` and the MCP dispatcher's private error carrier stay + outside the family deliberately — they are control flow, not failure. +- **`import je_auto_control` needed a Python built with `sqlite3`, and + FreeBSD's is not.** `sqlite3` is in the standard library but not in every + build of it: CPython links it against a system library, and FreeBSD ships + the result as the separate `databases/py-sqlite3` package. Ten subsystems + imported it at module scope — run history, checkpoints, the work queue, + agent memory, the remote-desktop audit log, SQL data sources, and the + error tuples in the REST, chat-ops and MCP containment boundaries — and + all ten are reachable from the facade, so the whole package failed to + import on a stock FreeBSD, mouse and keyboard included. They go through + `je_auto_control.utils.sqlite_support` now, which fails at the first call + that opens a database rather than at import, and raises + `AutoControlUnsupportedOperationException` — the type the GUI tabs, the + REST handler and the executor already report as "not available here" — + instead of an `ImportError` none of them catch. `run_diagnostics()` lists + `sqlite3` among the optional dependencies, so the gap is visible without + reading a traceback. +- **`mouse_scroll` did nothing at all on the BSDs.** It matched Windows, then + macOS, then a literal `["linux", "linux2"]`, so a FreeBSD, OpenBSD, NetBSD or + DragonFly caller fell off the end of the chain: no backend call, no + exception, no log line. It asks `platform_id.is_x11_unix()` now, and an + unrecognised platform raises `AutoControlMouseException` instead of returning + as though it had scrolled. +- **A recorded timeline replayed nothing.** `replay_timeline`'s dispatch table + held the `run_sequence` DSL's vocabulary (`press` / `click` / `key`) and the + recorders emit their own (`key_down` / `mouse_up` / `scroll`), and the two + were disjoint — so `stop_record_timeline()` fed to `replay_timeline()`, the + pipeline both the docstrings and the `ac_record_stop_timeline` tool + prescribe, matched no handler, replayed an empty session, and still returned + every event as played. The recorder ops dispatch now, and the wheel reads + `delta` as well as `value` (reading only `value` fell back to the default of + one notch, so a three-notch scroll down replayed as one notch the other + way). Affects every platform, not only macOS. +- macOS: recorded mouse coordinates were mirrored vertically. The listener + read `NSEvent.mouseLocation()`, whose origin is the bottom-left of the + display, while every replay posts into the top-left space `osx_mouse` uses — + so a click recorded near the top of the screen replayed near the bottom. It + now reads `CGEventGetLocation`, which is already in the space the replay + posts into. +- macOS: modifier keys were not recorded at all. macOS sends no key-down for + Shift, Control, Option or Command, only a `flagsChanged` event carrying the + new flag set, so a recording could not say a modifier was held across the + actions that followed. They are reconstructed from the flags now. +- macOS: `write()` typed a space instead of a backspace, because `"\b"` had + no route in the macOS key table and fell through to the space fallback. +- macOS: USB enumeration returned `apple_vendor_id` in `vendor_id`, a field + documented as a four-hex-digit string. A value that is not a hex id is now + `None`; the device is still listed and `manufacturer` still names the vendor. +- Linux/X11: `window_rect` returned the client area rather than the frame, + disagreeing with Win32's `GetWindowRect` by the window decorations. +- Linux/X11: `move_window_by_title` configured the client window directly, + which under a reparenting window manager positions it in the wrong + coordinate space. It now goes through `_NET_MOVERESIZE_WINDOW`. +- The D-Bus client could not marshal or demarshal signed integers, so any + protocol using them (AT-SPI extents among them) failed to decode. + - **Wayland: an absolute mouse move through the ydotool fallback counted from the wrong origin.** `ydotool mousemove --absolute` emits no absolute event — it drives the cursor into the corner the compositor clamps to and then moves diff --git a/CLAUDE.md b/CLAUDE.md index dc90059c..4c083688 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mouse and keyboard control, image recognition, OCR, accessibility-tree and VLM element location, action scripting, and report generation behind one API. Backends: Windows (Win32 ctypes), macOS (pyobjc/Quartz), Linux X11 (python-Xlib), Linux Wayland (libei / ydotool), Android (adb), iOS (WebDriverAgent). - **Package**: `je_auto_control` · **Python** ≥ 3.10 · **License**: MIT · **Author**: JE-Chen -- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 308 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. +- **[architecture_explore.md](architecture_explore.md)** is the per-module map — read it before changing structure; it lists all 310 `utils/` subpackages, every GUI tab, and file-level tables for the large subsystems. ## Architecture @@ -18,7 +18,7 @@ AutoControl (`je_auto_control`) is a cross-platform GUI automation framework: mo | Template Method | `utils/generate_report/` | HTML / JSON / XML share collect → format → write. | | Backend seam | `backends/` under `accessibility`, `ocr`, `vision`, `llm`, `agent`, `hotkey`, `usb`, `usbip` | Abstract base + concrete impls + null fallback, so dependency-free environments still import. | -Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (308 headless subpackages) → `wrapper/` → per-OS backend. +Layering: entry points (`cli.py`, `gui/`, socket / REST / MCP servers) → executor → `utils/` (310 headless subpackages) → `wrapper/` → per-OS backend. ## Development Commands @@ -68,7 +68,7 @@ The map is only useful while it matches the tree, so **update it in the same cha Never adjust one by hand: the counts are `len(text.splitlines())` (what `wc -l` reports), and hand-editing is how the document ended up quoting the same subsystem at two sizes at once — most tables had been counting a phantom trailing line per file while §1 and §8 counted correctly. -- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 308 subpackages; appearing twice or not at all is a defect. +- A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 310 subpackages; appearing twice or not at all is a defect. - A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. - Keep the header's scan date, version, and branch current. - `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. diff --git a/Progress.md b/Progress.md index 42c030d8..009c2bae 100644 --- a/Progress.md +++ b/Progress.md @@ -27,7 +27,7 @@ | `utils/accessibility/backends/windows_backend.py` | 915 | 已拆出 `windows_query.py`（170）與 `windows_state.py`（98）。剩下的是同一套 UIA COM 生命週期管理,再拆會把 `CoInitialize`／介面釋放的配對邏輯切散。 | **本質豁免（依 `CLAUDE.md` 的「flat data tables」條款,不算既有豁免）**: -`utils/mcp_server/tools/_factories.py`（8,968,MCP 工具註冊表）、 +`utils/mcp_server/tools/_factories.py`（8,972,MCP 工具註冊表）、 `utils/executor/action_executor.py`（8,125,`AC_*` 分派表）、 `gui/script_builder/command_schema.py`（5,051,每個 `AC_*` 的參數 schema）、 `je_auto_control/__init__.py`（1,970,門面 re-export）、 @@ -50,6 +50,54 @@ --- +## Windows arm64 裝不起來——是兩個上游，不是一個 + +`BLOCKED` — 上游（`opencv-python` 沒有 win_arm64 wheel；`cryptography` 在安全下限之上也沒有） + +`windows-11-arm` 加進 `platform-smoke.yml` 的矩陣跑了一次， +結果是實測而不是推測：**opencv-python 並沒有發 +win_arm64 wheel**，pip 回退到從原碼建，CMake 在 ARM64 上 +configure 不起來，花了十二分鐘失敗。所以那一格已從矩陣 +移除，並把原因寫在 workflow 的註解裡。 + +門面已經不在 module scope import OpenCV 了（見 +[WHATS_NEW.md](WHATS_NEW.md)），但這裡卡的不是 import +而是 **pip 裝不起來**：那幾個套件仍列在 `pyproject.toml` +的 `dependencies`，`pip install -e .` 第一步就會去建它們。 + +### 2026-08-20 重新實測：當初只數到一半 + +不必開 runner——`pip` 可以替別的平台解析，十秒就給出答案： + +| 依賴 | win_arm64 | 實測 | +| --- | --- | --- | +| `opencv-python>=4.8,<6` | **沒有** | 任何版本都沒有，pip 回的是 `from versions: none`。`je_open_cv` 自己是純 Python，但它相依 opencv-python，所以一起卡。 | +| `cryptography>=48.0.1` | **沒有** | wheel 只出到 **46.0.3**，46.0.4 起上游就不再發 win_arm64。而 `>=48.0.1` 是 347ec1e 為了 GHSA-537c-gmf6-5ccf（high）訂的**安全下限**，不能為了 arm64 降回去。 | +| `pillow==12.3.0` | 有 | `pillow-12.3.0-cp3xx-win_arm64.whl` 一直都在。**原本這裡寫「把 OpenCV／Pillow 移到 extra」，Pillow 那半是猜的，它從來不是卡點。** | +| `mss`／`defusedxml`／`je_open_cv` | 有 | 純 Python。 | +| `PySide6==6.11.1`／`qt-material==2.17` | 有 | `[gui]` extra 在 arm64 上裝得起來。 | +| `aiortc` | **沒有** | 卡在傳遞相依 `google-crc32c`，與本專案的選擇無關；`av` 自己有 wheel。 | + +**所以原本那句「把 OpenCV／Pillow 移到 optional extra，arm64 就能只裝輸入的部分」 +是不成立的**——就算 OpenCV 移走，`cryptography` 還是會把 `pip install` 擋在同一個 +地方，而它的下限是安全下限，沒有往下讓的空間。要真的讓 arm64 裝得起來，**兩個都得 +離開必裝集合**；`cryptography` 今天被六個模組用到（`acme_v2`、`tls_acme`、 +`action_signing`、`secrets`、`remote_desktop` 的加密錄影），那是比 OpenCV 更大的 +相容性決定。沒有人要求之前不做。 + +重驗指令（不需要 arm64 機器，也不需要 runner）： + +```bash +pip install --dry-run --only-binary=:all: --platform win_arm64 --python-version 3.12 --target /tmp/probe 'opencv-python>=4.8,<6' 'cryptography>=48.0.1' +``` + +兩行 `ERROR: No matching distribution` 就是現況。哪天其中一行不見了，就是上游發了 +wheel，那時把 `windows-11-arm` 加回 `platform-smoke.yml` 的矩陣。 + +**Linux arm64 是好的**——`ubuntu-22.04-arm` 兩個 Python 版本 +都綠，macOS 本來就是 arm64。所以卡住的只有 Windows +這一個組合。 + ## Wayland:剩下的都不是「缺一台機器」 這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, diff --git a/README.md b/README.md index 18066393..d57eb80d 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ **AutoControl** is a cross-platform GUI automation framework for Python. It drives the mouse and keyboard, finds things on screen (template matching, OCR, the OS accessibility tree, or a vision model), records and replays flows, and runs them from JSON action -files — on Windows, macOS, Linux (X11 and Wayland), Android, and iOS. +files — on Windows, macOS, Linux (X11 and Wayland), the BSDs, Android, +and iOS. Every capability ships three ways: a **Python API**, an **`AC_*` action command** usable from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. @@ -19,7 +20,7 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. ## Why AutoControl -- **One API, six platforms.** `wrapper/platform_wrapper.py` picks the backend at import +- **One API, seven platforms.** `wrapper/platform_wrapper.py` picks the backend at import time; your script does not change between Windows, macOS, X11, and Wayland. - **Scriptable without Python.** 773 `AC_*` commands cover the whole feature set, so a JSON file can do anything the library can — including loops, branches, try/catch, @@ -153,7 +154,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Diagnostics | `run_diagnostics` | `AC_diagnose` | Diagnostics | | Test-code generation | `generate_code` | — | — | -Beyond this table, `utils/` holds 308 headless packages covering assertions, resilience, +Beyond this table, `utils/` holds 310 headless packages covering assertions, resilience, data quality, i18n auditing, redaction, governance, observability, and more. The full per-module map is in **[architecture_explore.md](architecture_explore.md)**. @@ -239,12 +240,28 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Platform | Backend | Input | Screen capture | Recording | Window management | |---|---|:---:|:---:|:---:|:---:| | Windows 10 / 11 | Win32 ctypes (+ optional Interception driver) | ✅ | ✅ | ✅ | ✅ | -| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | -| Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ❌ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | +| Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | libei via the desktop portal, or ydotool / wtype + a capture tool | ✅ | ✅ | ❌ | ❌ | +| FreeBSD / OpenBSD / NetBSD | python-Xlib, the same X11 backend as Linux | ✅² | ⚠️² | ✅² | ✅² | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +¹ macOS recording captures through a Quartz event tap and needs +**Accessibility** permission (System Settings → Privacy & Security → +Accessibility). Without it recording raises and names the permission +rather than returning an empty session. + +² The BSDs run the X11 backend unchanged — the same X server, the same +`python-Xlib`, which is the only dependency input, recording and window +management have. A `freebsd` CI job drives real input on a real FreeBSD 14 and +reads it back off the X server; OpenBSD and NetBSD take the same code path but +have no CI runner. Screen capture is the exception, and the reason is +packaging rather than the platform: it goes through Pillow/mss and OpenCV, and +`opencv-python`, `pillow` and `cryptography` publish no FreeBSD wheels. Build +those from ports and capture, image matching, OCR and action encryption work +too — `import je_auto_control` no longer requires any of them. + Wayland input falls back to the `ydotool` CLI wherever libei is not reachable, and that fallback needs **ydotool 1.0 or newer**. Every argument AutoControl builds arrived in that release; 0.1.x — which is what Debian diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index 48447f7b..452e56c8 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -7,7 +7,7 @@ **AutoControl** 是一套跨平台的 Python GUI 自动化框架。它能驱动鼠标与键盘、在画面上找到目标 （模板匹配、OCR、操作系统无障碍树，或视觉模型）、录制与回放操作流程，并以 JSON 动作文件执行—— -支持 Windows、macOS、Linux（X11 与 Wayland）、Android 与 iOS。 +支持 Windows、macOS、Linux（X11 与 Wayland）、BSD、Android 与 iOS。 每项能力都以三种形式提供：**Python API**、可在 JSON 文件／CLI／服务器使用的 **`AC_*` 动作命令**， 以及 **GUI 标签页**。没有任何功能只存在于 GUI。 @@ -18,7 +18,7 @@ ## 为什么选择 AutoControl -- **一套 API，六个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 +- **一套 API，七个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 Windows、macOS、X11 与 Wayland 上都不需要改写。 - **不写 Python 也能脚本化。** 773 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 能做的任何事——包含循环、分支、try/catch、宏与变量。 @@ -147,7 +147,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 系统诊断 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 测试代码生成 | `generate_code` | — | — | -除了这张表，`utils/` 下还有 308 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 +除了这张表，`utils/` 下还有 310 个无头包，覆盖断言、韧性、数据质量、i18n 审计、脱敏、 治理、可观测性等等。完整的逐模块地图在 **[architecture_explore.md](../architecture_explore.md)**。 --- @@ -228,12 +228,25 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | 平台 | 后端 | 输入 | 屏幕捕获 | 录制 | 窗口管理 | |---|---|:---:|:---:|:---:|:---:| | Windows 10 / 11 | Win32 ctypes（可选 Interception 驱动） | ✅ | ✅ | ✅ | ✅ | -| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | -| Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ❌ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | +| Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | 经桌面 portal 的 libei，或 ydotool／wtype ＋ 截图工具 | ✅ | ✅ | ❌ | ❌ | +| FreeBSD／OpenBSD／NetBSD | python-Xlib，与 Linux 同一套 X11 后端 | ✅² | ⚠️² | ✅² | ✅² | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +¹ macOS 的录制走 Quartz event tap，需要**辅助功能**权限 +（系统设置 → 隐私与安全性 → 辅助功能）。没有授权时会直接抛出并指名 +缺的是哪个权限，而不是安静地录到一个空的 session。 + +² BSD 直接跑同一套 X11 后端——同一个 X server、同一个 `python-Xlib`，而输入、 +录制与窗口管理就只依赖这一个包。`freebsd` CI job 在真的 FreeBSD 14 上驱动真的 +输入，再从 X server 读回来；OpenBSD 与 NetBSD 走同一条代码路径，只是没有 CI +runner。唯一的例外是屏幕截取，卡住的是打包而不是平台：它走 Pillow／mss 与 +OpenCV，而 `opencv-python`、`pillow`、`cryptography` 都没有发 FreeBSD wheel。 +从 ports 构建之后，截取、图像匹配、OCR 与动作加密也都能用—— +`import je_auto_control` 本身已经不需要它们任何一个。 + Wayland 的输入在 libei 走不通时会退回 `ydotool` CLI，而这条退路需要 **ydotool 1.0 以上**。AutoControl 送的每一个参数都是那一版才有的；0.1.x （Debian bookworm 与目前所有 Ubuntu 仍以这个名字提供，Debian trixie 则根本没有） diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index b9f04444..97e31d4b 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -7,7 +7,7 @@ **AutoControl** 是一套跨平台的 Python GUI 自動化框架。它能驅動滑鼠與鍵盤、在畫面上找到目標 （樣板比對、OCR、作業系統無障礙樹，或視覺模型）、錄製與重播操作流程，並以 JSON 動作檔執行—— -支援 Windows、macOS、Linux（X11 與 Wayland）、Android 與 iOS。 +支援 Windows、macOS、Linux（X11 與 Wayland）、BSD、Android 與 iOS。 每項能力都以三種形式提供：**Python API**、可在 JSON 檔／CLI／伺服器使用的 **`AC_*` 動作指令**， 以及 **GUI 分頁**。沒有任何功能只存在於 GUI。 @@ -18,7 +18,7 @@ ## 為什麼選擇 AutoControl -- **一套 API，六個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 +- **一套 API，七個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 Windows、macOS、X11 與 Wayland 上都不需要改寫。 - **不寫 Python 也能腳本化。** 773 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 @@ -147,7 +147,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 系統診斷 | `run_diagnostics` | `AC_diagnose` | Diagnostics | | 測試碼產生 | `generate_code` | — | — | -除了這張表，`utils/` 底下還有 308 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 +除了這張表，`utils/` 底下還有 310 個無頭套件，涵蓋斷言、韌性、資料品質、i18n 稽核、遮蔽、 治理、可觀測性等等。完整的逐模組地圖在 **[architecture_explore.md](../architecture_explore.md)**。 --- @@ -229,12 +229,25 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | 平台 | 後端 | 輸入 | 螢幕擷取 | 錄製 | 視窗管理 | |---|---|:---:|:---:|:---:|:---:| | Windows 10 / 11 | Win32 ctypes（可選 Interception 驅動） | ✅ | ✅ | ✅ | ✅ | -| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | -| Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ❌ | +| macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ✅¹ | ✅ | +| Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ✅ | | Linux Wayland | 經桌面 portal 的 libei，或 ydotool／wtype ＋ 擷取工具 | ✅ | ✅ | ❌ | ❌ | +| FreeBSD／OpenBSD／NetBSD | python-Xlib，與 Linux 同一套 X11 後端 | ✅² | ⚠️² | ✅² | ✅² | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +¹ macOS 的錄製走 Quartz event tap，需要**輔助使用**權限 +（系統設定 → 隱私權與安全性 → 輔助使用）。沒有授權時會直接拋出並指名 +缺的是哪個權限，而不是安靜地錄到一個空的 session。 + +² BSD 直接跑同一套 X11 後端——同一個 X server、同一個 `python-Xlib`，而輸入、 +錄製與視窗管理就只相依這一個套件。`freebsd` CI job 在真的 FreeBSD 14 上驅動真的 +輸入，再從 X server 讀回來；OpenBSD 與 NetBSD 走同一條程式路徑，只是沒有 CI +runner。唯一的例外是螢幕擷取，而卡的是打包不是平台：它走 Pillow／mss 與 OpenCV， +而 `opencv-python`、`pillow`、`cryptography` 都沒有發 FreeBSD wheel。從 ports +建起來之後，擷取、影像比對、OCR 與動作加密也都能用——`import je_auto_control` +本身已經不需要它們任何一個。 + Wayland 的輸入在 libei 走不通時會退回 `ydotool` CLI，而這條退路需要 **ydotool 1.0 以上**。AutoControl 送的每一個參數都是那一版才有的；0.1.x （Debian bookworm 與目前所有 Ubuntu 仍以這個名字提供，Debian trixie 則根本沒有） diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index b42b6033..0a07ea53 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,69 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-20) — 声称支援的平台，這回真的量過了 + +整套测试一直只在 `windows-2022` 跑，另加容器裡一次 Linux +執行。macOS 只跑兩行指令。Wayland 有五個 job 對真的對等體 +讀回輸入；X11——兩條 Linux 路徑中更老、部署更廣的那一 +條——一個都沒有，而且套件裡每一條 X11 斷言都是對著 +`python-Xlib` 的 mock 做的。 + +**测试套件現在真的在它宣稱支援的平台上跑。** +`pytest-headless` 改成 OS 矩陣；Linux 跑在真的 Xvfb 上而不是 Qt 的 +offscreen，因為 X11 後端在 import 時就連線，offscreen 會將正好要找的 +毛病蓋掉。第一輪就抓到兩個真的 macOS 缺陷： + +- `write("\b")` 在 macOS 沒有任何按鍵路徑，會落到空白鍵 + fallback——要求退格，打出來的是空白。 +- `system_profiler` 對 Apple 自家裝置回的是 **符號式** vendor + id（`apple_vendor_id`），而那個欄位文件上寫的是四位十六進位。 + +**X11 的輸入現在從真的客戶端讀回。** 新的 `x11-verification` +job 跑在真的 Xvfb + 真的視窗管理員上，對照組來自受測對象以外的 +程式碼：`xev`、ImageMagick 的 `import`、`xdotool` 與 `xdpyinfo`。 +最值得點名的一項是 `synthetic NO`——`XSendEvent` 的事件帶的是 +`YES`，大多數 toolkit 會直接丟掉，所以一個患患停止驅動真實 +輸入的後端，在只數事件的檢查下仍然會全綠。 + +**macOS 在 CI 裡其實完全驗得了，跟一般假設相反。** +`macos-14` runner **兩個 TCC 權限都給**：擷取回來的是真像素而不是 +被拒時的全黑矩形，`CGEventPost` 真的移得動游標且讀回完全相符， +AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期望表 +還是空的時候會拒絕通過。 + +### 視窗管理不再是 Windows 專屬 + +以前是：門面在 `sys.platform` 上分支，其他平台一律丟例外， +23 個 `AC_*` 指令跟對應的 MCP 工具在 macOS 與 Linux 上都是死的。 +現在走平台縫：Win32、X11 的 EWMH、macOS 的 Quartz + 無障礙 API。 + +兩件只有真的視窗管理員才能披露的錯，第一版都錯了： + +- **矩形是外框，不是客戶區。** Win32 的 `GetWindowRect` + 回的是外框，所有呼叫端都是照那個寫的。 +- **移動必須走 `_NET_MOVERESIZE_WINDOW`。** 在 reparenting + 視窗管理員下，客戶端自己的 x/y 是相對於外框的；對 openbox + 要 (300, 220)，直接 `ConfigureWindow` 的結果是落在 (302, 260)。 + +### Linux 有無障礙後端了 + +之前完全沒有。新後端走 **AT-SPI2**——它是 D-Bus 協定而不是 +函式庫，這就是它不用加新相依的原因：`pyatspi` 與 +`gi.repository.Atspi` 是發行版套件，裝不進 venv。 + +拿真的 bus 跟真的 GTK 程式一驗，當場抓到 D-Bus 客戶端的一個缺口： +**它不會解有號整數**。portal 從來不需要，而 AT-SPI 的 extents 是四個 +**有號**值——因為主螢幕左邊（或上方）的螢幕上，視窗坐標是負的。 + +### BSD 與 arm64 + +`platform_wrapper` 對非 win/darwin/linux 一律丟「unknown operating +system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 +新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 +真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 +`ubuntu-22.04-arm` 加进 smoke 矩阵且全绿。`windows-11-arm` 试过后拿掉了，而重新实测又挖出当初漏掉的**第二个**卡点：opencv-python 任何版本都没发 `win_arm64` wheel，cryptography 则从 46.0.4 起不再发，而本项目的下限 `>=48.0.1` 是安全下限（GHSA-537c-gmf6-5ccf），不能为了凑 wheel 往下让。原本跟 OpenCV 并列的 Pillow 其实一直都有 arm64 wheel，从来不是卡点。这些都不需要 arm64 机器就验得到——`pip install --dry-run --only-binary=:all: --platform win_arm64` 十秒给答案，指令已连同结论记在 `Progress.md`。 + + ## 本次更新 (2026-08-19) — Wayland 两个等人拍板的取舍,拍板了 `Progress.md` 上挂着两个 `DECIDE`:缺的不是活，是决定。两件事其实是同一个问题犯两次 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 1de15dd2..ef67d949 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,69 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-20) — 嬣稱支援的平台，這回真的量過了 + +整套測試一直只在 `windows-2022` 跑，另加容器裡一次 Linux +執行。macOS 只跑兩行指令。Wayland 有五個 job 對真的對等體 +讀回輸入；X11——兩條 Linux 路徑中更老、部署更廣的那一 +條——一個都沒有，而且套件裡每一條 X11 斷言都是對著 +`python-Xlib` 的 mock 做的。 + +**測試套件現在真的在它宣稱支援的平台上跑。** +`pytest-headless` 改成 OS 矩陣；Linux 跑在真的 Xvfb 上而不是 Qt 的 +offscreen，因為 X11 後端在 import 時就連線，offscreen 會將正好要找的 +毛病蓋掉。第一輪就抓到兩個真的 macOS 缺陷： + +- `write("\b")` 在 macOS 沒有任何按鍵路徑，會落到空白鍵 + fallback——要求退格，打出來的是空白。 +- `system_profiler` 對 Apple 自家裝置回的是 **符號式** vendor + id（`apple_vendor_id`），而那個欄位文件上寫的是四位十六進位。 + +**X11 的輸入現在從真的客戶端讀回。** 新的 `x11-verification` +job 跑在真的 Xvfb + 真的視窗管理員上，對照組來自受測對象以外的 +程式碼：`xev`、ImageMagick 的 `import`、`xdotool` 與 `xdpyinfo`。 +最值得點名的一項是 `synthetic NO`——`XSendEvent` 的事件帶的是 +`YES`，大多數 toolkit 會直接丟掉，所以一個患患停止驅動真實 +輸入的後端，在只數事件的檢查下仍然會全綠。 + +**macOS 在 CI 裡其實完全驗得了，跟一般假設相反。** +`macos-14` runner **兩個 TCC 權限都給**：擷取回來的是真像素而不是 +被拒時的全黑矩形，`CGEventPost` 真的移得動游標且讀回完全相符， +AX 樹也走得出真的元素。這是先量再斷言的，而且探針在期望表 +還是空的時候會拒絕通過。 + +### 視窗管理不再是 Windows 專屬 + +以前是：門面在 `sys.platform` 上分支，其他平台一律丟例外， +23 個 `AC_*` 指令跟對應的 MCP 工具在 macOS 與 Linux 上都是死的。 +現在走平台縫：Win32、X11 的 EWMH、macOS 的 Quartz + 無障礙 API。 + +兩件只有真的視窗管理員才能披露的錯，第一版都錯了： + +- **矩形是外框，不是客戶區。** Win32 的 `GetWindowRect` + 回的是外框，所有呼叫端都是照那個寫的。 +- **移動必須走 `_NET_MOVERESIZE_WINDOW`。** 在 reparenting + 視窗管理員下，客戶端自己的 x/y 是相對於外框的；對 openbox + 要 (300, 220)，直接 `ConfigureWindow` 的結果是落在 (302, 260)。 + +### Linux 有無障礙後端了 + +之前完全沒有。新後端走 **AT-SPI2**——它是 D-Bus 協定而不是 +函式庫，這就是它不用加新相依的原因：`pyatspi` 與 +`gi.repository.Atspi` 是發行版套件，裝不進 venv。 + +拿真的 bus 跟真的 GTK 程式一驗，當場抓到 D-Bus 客戶端的一個缺口： +**它不會解有號整數**。portal 從來不需要，而 AT-SPI 的 extents 是四個 +**有號**值——因為主螢幕左邊（或上方）的螢幕上，視窗坐標是負的。 + +### BSD 與 arm64 + +`platform_wrapper` 對非 win/darwin/linux 一律丟「unknown operating +system」，七個 X11 後端模組又各自帶一份同樣的 Linux 專屬守衛。 +新的 `utils/platform_id` 是唯一的判定點，`freebsd` job 在 runner 裡開 +真的 FreeBSD 14 VM，在真的 X server 上 import X11 模組並把游標移完讀回。 +`ubuntu-22.04-arm` 加進 smoke 矩陣且全綠。`windows-11-arm` 試過後拿掉了，而重新實測又挖出當初漏掉的**第二個**卡點：opencv-python 任何版本都沒發 `win_arm64` wheel，cryptography 則從 46.0.4 起不再發，而本專案的下限 `>=48.0.1` 是安全下限（GHSA-537c-gmf6-5ccf），不能為了湊 wheel 往下讓。原本跟 OpenCV 並列的 Pillow 其實一直都有 arm64 wheel，從來不是卡點。這些都不需要 arm64 機器就驗得到——`pip install --dry-run --only-binary=:all: --platform win_arm64` 十秒給答案，指令已連同結論記在 `Progress.md`。 + + ## 本次更新 (2026-08-19) — Wayland 兩個等人拍板的取捨,拍板了 `Progress.md` 上掛著兩個 `DECIDE`:缺的不是工，是決定。兩件事其實是同一個問題犯兩次 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index fac41b3f..457a16ed 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,396 @@ # What's New — AutoControl +## What's new (2026-08-20) + +### The Platforms This Project Claims, Now Measured + +The suite ran on `windows-2022` alone for its whole life, plus one Linux +container run. macOS got two commands and nothing else. Wayland had five jobs +reading input back off a real peer; X11 — the older and more widely deployed +of the two Linux paths — had none, and every X11 assertion in the suite was +made against a mock of `python-Xlib`. + +**The suite now runs where the project says it runs.** `pytest-headless` +became an OS matrix: Windows keeps all five Pythons, Linux and macOS carry the +two ends of the range. Linux runs under a real Xvfb rather than Qt's offscreen +platform, because the X11 backend opens a display at import time and offscreen +would hide exactly the breakage this exists to find. It found two real macOS +defects on the first run: + +- `write("\b")` had no key route on macOS, so it fell through to the space + fallback and typed a space where a backspace was asked for. X11 and Wayland + both carry the raw character; macOS was the one that did not. +- `system_profiler` reports a *symbolic* vendor id for Apple's own devices — + `apple_vendor_id`, not a number — and that went straight into a field + documented as four hex digits. Its leading `a` is a valid hex digit, so a + lenient parse turns it into `000a`. + +**X11 input is read back out of a real client.** A new `x11-verification` job +runs against a real Xvfb server with a real window manager, taking ground +truth from other codebases than the subject: `xev`, a real X client that +prints every event delivered to its window; ImageMagick's `import` against a +root painted two asymmetric colours; `xdotool` and `xdpyinfo`. The assertion +worth naming is `synthetic NO` — `XSendEvent` traffic arrives with `YES` and +is discarded by most toolkits, so a backend that quietly stopped driving real +input would still pass any check that only counted events. + +**macOS turned out to be fully testable in CI, contrary to the usual +assumption.** A `macos-14` runner grants *both* Screen Recording and +Accessibility: capture returns real pixels rather than the black rectangle a +refusal produces, `CGEventPost` moves the cursor and the move reads back +exactly, and the AX walk returns real elements. That was measured first and +asserted second, and the probe still refuses to pass while its expectations +table is empty. + +### Five Errors That Every Boundary Missed + +The exception hierarchy is flat so that the executor, the poll loops, the +request handlers and the GUI slots can each contain the whole family in one +`except`. Round 3 reparented the family for that reason; five classes were +missed, and one of them was reachable from an action list. + +`AC_config_import` with a malformed bundle raised `ConfigBundleError`, which +inherits `Exception` directly, so the executor's per-action clause — which +lists `AutoControlException` and the builtins — did not catch it. The error +went past the boundary and took every remaining action with it, under +`raise_on_error=False`, where the contract is that a failed action is +*recorded*. Measured, not reasoned: a two-action list lost its second action. +`AC_usb_remote_devices` and `AC_usb_remote_open` had the same path through +`UsbClientError`. + +All five now derive from `AutoControlException`. What keeps the next one out is +not a list of the five: `test_exception_family_is_flat.py` walks the package +with `ast` and fails on *any* class inheriting `Exception` directly, against a +three-entry allowlist that has to state why. `LoopBreak` and `LoopContinue` are +on it because they are control flow — a family handler swallowing a `break` +would be the mirror-image bug — and the MCP dispatcher's private error carrier +because it never leaves the dispatcher that raised it. The allowlist is checked +in both directions, so a stale entry fails too. + +### A BSD Found the Same Mistake in a Second Place + +The FreeBSD VM was added to prove the X11 backend drives a real BSD. It failed +before it got there, and what it failed on was not a wheel: **FreeBSD's +`python311` has no `sqlite3`.** The module is in the standard library but not +in every build of it — CPython links it against a system library, and FreeBSD +packages the result separately as `databases/py-sqlite3`. + +Ten subsystems imported it at module scope: run history, checkpoints, the work +queue, agent memory, the remote-desktop audit log, SQL data sources, and the +`except` tuples that keep a database error from killing the REST handler +thread, the chat-ops poll loop and the MCP transport. Every one of them is +reachable from the facade, so `import je_auto_control` failed outright — on a +machine where moving a mouse needs no database at all. This is the same shape +as the OpenCV/Pillow finding one commit earlier, from a source that reasoning +about wheels would never reach: the standard library is not the same size on +every platform. + +**The ten now go through `je_auto_control/utils/sqlite_support.py`.** +`require_sqlite3()` returns the module or raises +`AutoControlUnsupportedOperationException` — deliberately the type the platform +backends already raise for something they cannot do, so the GUI tabs, the REST +handler and the executor report "not available here" instead of dying on an +`ImportError` that none of them catch. `SQLITE_ERRORS` and +`SQLITE_OPERATIONAL_ERRORS` are tuples rather than classes, so +`except (ValueError, *SQLITE_ERRORS)` stays a valid handler that catches +exactly the right amount — nothing — where nothing can raise them. + +That left one thing still opening a database during import: `HistoryStore` +connected in its constructor, and `default_history_store` is built while the +facade is importing. It connects on first use now, which is also why +`import je_auto_control` no longer creates `~/.je_auto_control/` as a side +effect of being imported. + +Three things keep it fixed. `test_sqlite_is_optional.py` blocks `_sqlite3` in a +subprocess — exactly what FreeBSD reports — and requires the facade to import +anyway, with the error tuples empty. The FreeBSD job asserts the module is +*absent* on the VM, so a future image that happens to ship `py311-sqlite3` +turns the job red rather than quietly retiring the property it was added to +test. And `run_diagnostics()` lists `sqlite3` among the optional dependencies, +so an operator sees the gap as a line in a report instead of a traceback. + +### The macOS Recorder Was Written, Unreachable, and Wrong + +`OSXRecorder` had been a complete implementation for as long as +`wrapper/_platform_osx.py` had said `recorder = None`, and that was not an +oversight. The listener called `NSApplication.sharedApplication()` **at import +time**, and stopping a recording meant `AppHelper.runEventLoop()` — a loop +that never returns to its caller. Wiring it up would have put both on the path +of `import je_auto_control`, which is a regression, not a fix. So `record()`, +`stop_record()` and `je_auto_control record` all refused on macOS with +"Cannot use recorder on macOS", and the capability matrix said `unavailable`. + +**The premise was wrong: a `CGEventTap` needs a run loop, not an +application.** Create the tap on a dedicated thread, add its source to *that* +thread's run loop, and pump the loop in short `CFRunLoopRunInMode` slices so a +stop flag is honoured between them. Nothing touches AppKit, nothing runs at +import, and `record()` returns immediately. The macOS hotkey backend had been +driving a tap exactly this way in the same tree the whole time. + +The tap is **listen-only**, which is load-bearing rather than a detail: a +recorder that consumed events would swallow the very input it is recording, so +the user's clicks would stop working the moment recording started. + +Two defects were sitting in that code, and only a Mac could show either: + +- **Recorded clicks were mirrored vertically.** Coordinates came from + `NSEvent.mouseLocation()`, whose origin is the **bottom-left** of the + display, while every replay posts into the top-left space `osx_mouse` uses. + A click recorded near the top of the screen replayed near the bottom, with + no error anywhere — the same silent shape as `write("\b")` typing a space. + It reads `CGEventGetLocation` now, which is already the replay's space. +- **Modifiers were not recorded at all.** macOS sends no key-down for Shift, + Control, Option or Command; it sends one `flagsChanged` event carrying the + new flag set. A recording therefore could not say a modifier was held across + the actions that followed — which is one of the two reasons the timeline + exists. They are reconstructed from the flag bits now. + +**The half that is not platform-specific stopped being copied.** Everything +after the capture — the down-events-only queue the executor is handed, the +`delta_ms` timeline a replay consumes, the mouse-only and keyboard-only +filters — is one implementation in +`utils/input_macro/recorder_base.py`, and both backends subclass it. A second +hand-written copy of that shaping would have diverged silently, and the way it +would surface is a recording made on one OS replaying wrongly on the other. + +**And the recording had nowhere to go.** Verifying the new backend end to end +turned up a defect that was never macOS-specific: `replay_timeline`'s dispatch +table held the `run_sequence` DSL's vocabulary — `press`, `click`, `key` — +while every recorder emits its own — `key_down`, `mouse_up`, `scroll`. The two +sets were **disjoint**. So `stop_record_timeline()` handed to +`replay_timeline()`, which is the pipeline the docstrings and the +`ac_record_stop_timeline` tool description both prescribe, matched no handler +at all: it replayed an empty session and returned the full event count as +played. The one op that did match, `scroll`, read a key the recorder does not +write, so it fell back to a single notch in the default direction. Both are +fixed, on every platform. + +**It is verified on a real window server, not against a fake.** The +`macos-capabilities` job posts a move, a click and a keypress through the +public API while recording, and asserts they come back out of the tap with the +release and with the coordinates they were posted at. Decoding is unit-tested +separately against genuine `CGEventCreate*` events, which needs no +Accessibility grant — so the parts that can be tested without a permission +are, and the one part that cannot is where the permission is measured. + +### Window Management Is No Longer Windows-Only + +It was: the facade branched on `sys.platform` and raised everywhere else, +leaving 23 `AC_*` commands and their MCP tools dead on macOS and Linux. It now +goes through a backend seam — Win32, EWMH over `python-Xlib` on X11, Quartz +plus the accessibility API on macOS, and a null fallback that lists nothing +and refuses actions with a reason. + +Two things only a real window manager could show up were wrong first time: + +- **The rectangle is the frame, not the client.** Win32's `GetWindowRect` + returns the frame, and every caller is written against that, so reporting + the client area was off by the decorations on X11 alone — silently, and by + a different amount per window manager. +- **A move has to go through `_NET_MOVERESIZE_WINDOW`.** Under a reparenting + window manager a client's own x/y are relative to its frame, so a direct + `ConfigureWindow` asks in the wrong coordinate space. Asking openbox for + (300, 220) that way landed the window at (302, 260). + +Refusals now raise a class that is both an `AutoControlException` and a +`NotImplementedError`. The GUI tabs and the REST handler already catch the +latter to say "not on this platform"; the executor catches the former, and a +bare `NotImplementedError` slipped past every containment boundary — aborting +a whole script where one action should have been reported as failed. + +### Linux Has an Accessibility Backend + +It had none — the selector fell through to the null one while the capability +matrix claimed "backend tests" for Linux X11. The new backend speaks +**AT-SPI2**, which is a D-Bus protocol rather than a library, and that is what +makes it reachable without a new dependency: `pyatspi` and +`gi.repository.Atspi` are distribution packages built against the system +introspection data and cannot be installed into a virtual environment. + +The D-Bus client written for the portal handshake moved from `linux_wayland/` +to `utils/dbus_client/` to make that possible, and verifying the backend +against a real bus and a real GTK application immediately found a gap in it: +**it could not demarshal signed integers.** The portal never needed one, and +AT-SPI reports a component's extents as four *signed* values, because a window +on a monitor left of or above the primary one is at a negative coordinate — so +the backend could read a tree but not where anything in it was. + +Because AT-SPI is a bus rather than a display protocol, this is the one +capability where Wayland is not the restricted case: the same bus serves both +Linux sessions. + +### The BSDs, and arm64 + +`platform_wrapper` refused to start on anything that was not +win32/cygwin/msys, darwin or linux/linux2, and each of the seven X11 backend +modules carried its own copy of the same Linux-only guard — so a FreeBSD, +OpenBSD or NetBSD desktop, which runs the same X server and the same +`python-Xlib`, could not import the package at all. `sys.platform` was being +compared against literal lists in over a hundred places, so the fix is one +place that decides: `utils/platform_id`, whose `is_x11_unix()` asks the +question those guards were always trying to ask. + +A `freebsd` job boots a real FreeBSD 14 VM inside the runner. It could at +first only check the *decision* — that `sys.platform` really reads `freebsd14`, +and that the classification every relaxed guard asks answers correctly on it — +because importing anything under `je_auto_control` ran the facade, and the +facade imported OpenCV and cryptography at module scope. `utils/platform_id` +had to be loaded by file path to get even that far. See below: that turned out +to be the wrong thing to work around, and the job now drives the whole backend. + +`ubuntu-22.04-arm` joins the smoke matrix and passes; `macos-14` was already +arm64. `windows-11-arm` was tried and removed, and re-measuring turned up a +**second** blocker the first pass had missed: opencv-python publishes no +`win_arm64` wheel in any version, and cryptography stopped publishing one after +46.0.3 — while this project's floor is `>=48.0.1`, a security floor +(GHSA-537c-gmf6-5ccf) that cannot be lowered to reach a wheel. Pillow, named +alongside OpenCV in the original entry, ships `win_arm64` wheels and was never +part of the problem. None of that needs an arm64 machine to check: `pip +install --dry-run --only-binary=:all: --platform win_arm64` answers it in +seconds, and `Progress.md` records the command next to the finding. + +### The Facade Insisted on OpenCV to Move a Mouse + +`Progress.md` recorded the missing BSD coverage as needing "a machine with the +dependency set on it, not a different CI trick". That entry was making the +mistake its own Wayland section warns about three lines further up: **asking +what the environment could not do, instead of asking who actually could not do +it.** It was not FreeBSD that could not run the X11 backend. It was this +package, which imported five image and crypto wheels before it would let you +move a pointer. + +The measurement was small. Ten modules on the facade's import path pulled in +OpenCV, NumPy, Pillow, `je_open_cv` or `cryptography` at module scope, across +about sixty call sites — while most of `utils/` had been importing OpenCV +lazily all along, with the docstrings to say so. Those ten now do the same. The +type annotations that referenced Pillow moved under `TYPE_CHECKING`, and the +two public `ImageSource` aliases keep Pillow in the union as a forward +reference, so nothing changes for a caller or a type checker. + +What `import je_auto_control` needs now is `defusedxml`, plus `python-Xlib` on +an X11 platform. Both are pure Python. The heavy five are still hard +dependencies and still install by default; the difference is that a platform +with no wheel for them can now use the half of the package that never needed +them. `test/unit_test/headless/test_facade_import_is_light.py` blocks all five +in a subprocess and imports the facade anyway, because this is a property one +convenience import silently undoes and every runner with wheels keeps passing. + +### The BSD Job Drives Real Input Now, and Found the Defect That Was Waiting + +With the facade light, the FreeBSD VM needs python-Xlib, defusedxml and an X +server — an install measured in seconds, where the ports build for OpenCV had +not finished after fifty minutes. So `test/verify/freebsd_verify.py` runs the +backend rather than the guard, and takes its ground truth from the X server +answering for itself: `query_pointer` for where the cursor is, its button mask +for which buttons the server believes are down, and `query_keymap` — the bitmap +of every physically-held key — for whether an injected key press really landed. +That last one is why no second process is needed here; the Linux +`x11-verification` job already reads events back out of `xev`, and what a BSD is +uniquely needed to answer is whether this code drives the same server on a +different kernel. + +It also maps a real X window that has asked for button events, because one +defect could not be caught any other way: **`mouse_scroll` did nothing at all +on a BSD.** It matched Windows, then macOS, then a literal +`["linux", "linux2"]` — one of the hundred-odd hand-written platform lists +`platform_id` exists to replace, and one that had been missed — so a BSD caller +fell off the end of the chain with no backend call, no exception and no log +line. A wheel event never shows up in the pointer mask, since X11 delivers a +scroll as a press *and* release of button 4/5/6/7 too fast to sample, so only a +client reading the event queue can see it happen or not happen. + +### `mouse_scroll` Means the Same Thing on Every Platform + +This had been sitting in `Progress.md` as a `DECIDE`, and the maintainer +settled it: **the sign of `scroll_value` reverses the direction everywhere, and +`scroll_direction` names the direction a positive count takes.** + +Windows and macOS had always read the sign. X11 encodes direction as a button +rather than a signed delta, so it took the direction from `scroll_direction` +and discarded the sign — deliberately, because a negative count used to make +`range()` empty and scroll nothing at all. The cost was portability with no +symptom to debug: `mouse_scroll(-3)`, written and tested on Windows, scrolled +three notches *down* on Linux instead of up. Wayland had the same `abs()` in +`_wheel_deltas` and the same result. + +Both now turn a negative count back into the opposite direction — a button swap +on X11, a sign on the Wayland delta. `docker/x11_verify.py` pins it against a +real X server through `xev`, `freebsd_verify.py` pins it on a BSD, and +`test_scroll_sign_is_portable.py` pins it on every runner without needing a +display. The migration note for anyone who was relying on the magnitude alone +is in `CHANGELOG.md`. + +### The Destructive-Action Prompt Reached Only One of the Two Transports + +`JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE=1` is documented as gating *every* +destructive MCP tool behind a confirmation prompt, with one stated caveat: the +client has to advertise the `elicitation` capability. Measured against a real +`HttpMCPServer` with a real destructive tool, the gate fired on stdio and +**never fired over HTTP** — not in any of the four combinations of plain POST +or SSE, one connection or two. + +Two independent reasons, and the second is why keeping the connection open did +not help. A plain `POST` is one request and one response, so its connection +scope had no server→client channel and the prompt had nothing to travel down. +An SSE `POST` did have one, but `_dispatch_sse` sets `close_connection`, so the +scope keyed on that TCP connection was forgotten before the next request — and +the `elicitation` capability advertised at `initialize` went with it. The call +then took the "client cannot be prompted" branch, logged one INFO line, and ran +the tool. An operator who set that variable on an HTTP-exposed server was +getting no confirmation at all. + +**The transport now has the identity MCP actually specifies.** `initialize` +mints an `Mcp-Session-Id` and returns it as a response header; a client that +echoes it keeps one dispatcher scope across every connection it opens. `GET` +with `Accept: text/event-stream` opens the standing server→client stream that +server-initiated traffic belongs on, and answering a server request is an +ordinary `POST` matched back to the waiting call by id. `DELETE` terminates. +The dispatcher itself needed no change — it already scoped capabilities and +active-call slots on an opaque `connection_id`, so a session id simply takes +that slot, and the per-connection isolation guarded by +`test_r3_mcp_connection_isolation` is preserved verbatim: a session is just an +identity that outlives a socket. + +So the confirmation now round-trips over HTTP, and the test that proves it uses +four separate connections — one that initialized, one holding the stream, one +carrying the `tools/call`, one carrying the decline — none of which shared a +socket with the handshake. Declining blocks the tool; accepting runs it. + +Measuring it turned up a second way through that was worth pinning: an SSE +`POST` carrying a session id needs no standing stream at all, because its own +response stream is already a server-to-client channel. The `elicitation/create` +goes out ahead of the result on the same socket the call arrived on. Both paths +now have a test. + +What did *not* change is the fallback: a client that ignores the session header, +or that only ever sends plain JSON `POST`s with no stream open, has given the +server nowhere to ask, and its destructive calls still proceed — exactly as they +do for a stdio client that never advertised `elicitation`. That is now a +documented boundary with a test on each side of it rather than an accident — but +it is still a boundary, so the bearer token, the `127.0.0.1` bind and +`JE_AUTOCONTROL_MCP_READONLY` remain the controls that do not depend on the +client behaving. + +Sessions are bounded in both directions: swept after ten minutes untouched, and +the registry evicts the least recently seen once it holds 128. Dropping a +session — however it goes — releases the dispatcher state held under its id, +which is the same release a closing socket used to perform, moved to the +identity that actually owns that state. Because every `initialize` mints a +session, including for the many clients that ignore the header and never come +back, evicting one that was never used after its handshake is logged as routine; +the warning is saved for evicting a session someone was holding, which is the +one that means the cap is too low. + +The new refusals also exposed an old assumption in the transport. Every `4xx` +runs `_drain_body()` first, so the client can read the response before the +socket closes — but the new unknown-session `404` and duplicate-stream `409` are +decided *after* the body has been parsed, and so was the pre-existing "body must +be UTF-8" `400`. The drain then went looking for bytes that were already gone +and blocked until the thirty-second read timeout, pinning that worker and +printing a `ConnectionAbortedError` traceback whenever the peer closed first. +It now skips the drain once the body is consumed, and treats a peer that has +already vanished as nothing left to be courteous to. + ## What's new (2026-08-19) ### Two Wayland Judgement Calls, Settled diff --git a/architecture_explore.md b/architecture_explore.md index c90b5cfd..edf5f599 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-08-19　**版本**：`pyproject.toml` version `0.0.218`　**分支**：`fix/clipboard-handles-and-window-input` +> **掃描時間**：2026-08-20　**版本**：`pyproject.toml` version `0.0.219`　**分支**：`feat/cross-platform-verification` --- @@ -19,14 +19,14 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 1,016 | -| 程式碼總行數 | 137,517 | -| `je_auto_control/utils/` 子套件數 | 308 | +| Python 模組總數（含周邊子專案） | 1,029 | +| 程式碼總行數 | 140,053 | +| `je_auto_control/utils/` 子套件數 | 310 | | `AC_*` 動作指令數（`known_commands()` 實測） | 773 | | 套件門面 `__all__` 公開名稱數 | 1,238 | | GUI 分頁數（`main_widget` 註冊） | 48 | | MCP 工具數（`build_default_tool_registry()` 實測） | 676 | -| `test_*.py` 測試檔／測試函式 | 466 / 4,443 | +| `test_*.py` 測試檔／測試函式 | 478 / 4,654 | | 範例腳本 | 27 | **技術基線**：Python ≥ 3.10、MIT 授權、必要相依只有 `je_open_cv`／`opencv-python`／`pillow`／`mss`／ @@ -55,7 +55,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 └───────────────────────────────┬──────────────────────────────────────────┘ │ ┌───────────────────────────────▼──────────────────────────────────────────┐ -│ 能力層 utils/（308 個子套件，全部無 Qt 相依） │ +│ 能力層 utils/（310 個子套件，全部無 Qt 相依） │ │ 影像辨識 │ OCR │ 無障礙樹 │ 定位自癒 │ AI/Agent │ 遠端桌面 │ USB │ │ 報表觀測 │ 資料 │ 安全 │ 韌性 │ 系統整合 │ 排程觸發 │ 網路協定 │ └───────────────────────────────┬──────────────────────────────────────────┘ @@ -154,11 +154,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | --- | ---: | --- | | `je_auto_control/__init__.py` | 1,970 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | | `je_auto_control/__main__.py` | 70 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | -| `je_auto_control/cli.py` | 326 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/cli.py` | 323 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | | `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | | `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | | `je_auto_control/utils/deprecation.py` | 35 | 公開 API 的一致性棄用警告。 | | `je_auto_control/utils/http_headers.py` | 32 | 入站 HTTP 標頭的共用防禦式解析。 | +| `je_auto_control/utils/sqlite_support.py` | 56 | 選用標準函式庫 `sqlite3` 的取用點：`require_sqlite3()`／`sqlite3_available()`／`SQLITE_ERRORS`。十個以 SQLite 存放狀態的子系統都經由這裡，所以 FreeBSD 這種把 `sqlite3` 另外包成 `databases/py-sqlite3` 的 Python 仍然 import 得起門面。 | ### 5.2 wrapper 抽象層 @@ -166,21 +167,21 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `wrapper/platform_wrapper.py` | 59 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | +| `wrapper/platform_wrapper.py` | 73 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | | `wrapper/_platform_windows.py` | 325 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 149 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_osx.py` | 156 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | | `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | | `wrapper/_platform_wayland.py` | 57 | Wayland 後端組裝（libei／ydotool／grim）。 | -| `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_mouse.py` | 366 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | | `wrapper/auto_control_keyboard.py` | 273 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | -| `wrapper/auto_control_screen.py` | 97 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | +| `wrapper/auto_control_screen.py` | 100 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | -| `wrapper/auto_control_record.py` | 106 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | -| `wrapper/auto_control_window.py` | 293 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | +| `wrapper/auto_control_record.py` | 107 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | +| `wrapper/auto_control_window.py` | 278 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | ### 5.3 平台後端 -#### Windows（`windows/`，23 檔／1,995 行） +#### Windows（`windows/`，23 檔／1,894 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -189,8 +190,8 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `core/utils/win32_keypress_check.py` | 21 | `GetAsyncKeyState` 按鍵狀態查詢。 | | `mouse/win32_ctype_mouse_control.py` | 220 | 滑鼠事件產生（含多螢幕絕對座標換算）。 | | `keyboard/win32_ctype_keyboard_control.py` | 55 | 鍵盤事件產生。 | -| `record/win32_input_hook.py` | 223 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | -| `record/win32_record.py` | 126 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）。 | +| `record/win32_input_hook.py` | 207 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | +| `record/win32_record.py` | 41 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）；整形本體與 macOS 共用 `utils/input_macro/recorder_base.py`。 | | `screen/win32_screen.py` | 89 | 螢幕尺寸與像素讀取。**每支 Win32 函式都明寫 argtypes/restype**（HDC 是指標寬度，走預設的 c_int 會截斷，錯誤會沉默地擴散到 GetPixel／ReleaseDC），並持有自己的 user32／gdi32 handle。import 時呼叫 `SetProcessDPIAware()`——**行程層級且不可還原**，實體↔邏輯座標換算請走 `utils/monitor_layout`。 | | `window/windows_window_manage.py` | 366 | 視窗列舉／聚焦／關閉／最小化／幾何／所屬行程 PID／投遞式輸入（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | | `message/window_message.py` | 97 | 直接對視窗送 `WM_*` 訊息（背景輸入）。 | @@ -198,7 +199,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | | `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | -#### macOS（`osx/`，17 檔／761 行） +#### macOS（`osx/`，17 檔／907 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -206,12 +207,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `mouse/osx_mouse.py` | 137 | Quartz `CGEvent` 滑鼠事件。 | | `keyboard/osx_keyboard.py` | 129 | Quartz 鍵盤事件。 | | `keyboard/osx_keyboard_check.py` | 24 | 按鍵狀態查詢。 | -| `listener/osx_listener.py` | 96 | `CGEventTap` 監聽。 | -| `record/osx_record.py` | 52 | 錄製（CLI `record` 於 macOS 明確不支援）。 | +| `listener/osx_listener.py` | 253 | 專屬執行緒上的 listen-only `CGEventTap`＋自己的 `CFRunLoopRunInMode` 切片；不在 import 時建 `NSApplication`，也不用會卡住呼叫緒的 `AppHelper.runEventLoop()`。修飾鍵由 `flagsChanged` 的旗標還原成 press／release，座標取 `CGEventGetLocation`（左上原點，與重播送出的座標同一空間）。 | +| `record/osx_record.py` | 41 | 錄製。捕捉後的整形（舊版按下事件 Queue、時間軸、只錄滑鼠／只錄鍵盤）走共用的 `utils/input_macro/recorder_base.py`。 | | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | -#### Linux X11（`linux_with_x11/`，19 檔／1,175 行） +#### Linux X11（`linux_with_x11/`，19 檔／1,215 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -226,19 +227,19 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，17 檔／3,431 行） +#### Linux Wayland（`linux_wayland/`，17 檔／2,835 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | | `_detect.py` | 77 | Wayland session 偵測與 CLI 工具探測。 | | `_ydotool_cli.py` | 134 | 判定安裝的是哪一代 ydotool 命令列，擋掉會靜默失效的 0.1.x（對本專案送的 argv 回傳 0 卻不送任何事件）。 | | `_ctypes_bind.py` | 75 | libei／liboeffis 共用的 ctypes 載入與 prototype 綁定。 | -| `_dbus_client.py` | 625 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | +| `_dbus_client.py` | 24 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | | `_select_input.py` | 85 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | | `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | | `oeffis.py` | 196 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | | `libei.py` | 610 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | -| `mouse.py` | 379 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | +| `mouse.py` | 384 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | | `keyboard.py` | 173 | 鍵盤後端：libei 優先，退回 ydotool／wtype。 | | `keymap.py` | 155 | 友善鍵名 → evdev key code。 | | `capture.py` | 236 | 擷取分層：操作者自訂指令 → grim → gnome-screenshot → spectacle → portal。 | @@ -258,27 +259,27 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `ios/input.py` | 46 | iOS 觸控與按鍵原語。 | | `ios/screen.py` | 32 | iOS 裝置螢幕擷取與尺寸。 | -### 5.4 能力層 `utils/`（308 個子套件） +### 5.4 能力層 `utils/`（310 個子套件） 以下依主題分組。每個子套件都是獨立可匯入的無頭模組，不含任何 Qt 相依。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,655 行。 +> 24 個套件、約 12,882 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/action_lint/` | 328 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | -| `utils/action_signing/` | 229 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | -| `utils/checkpoint/` | 115 | 流程檢查點與續跑，讓長 action list 具持久性 | +| `utils/action_signing/` | 230 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/checkpoint/` | 120 | 流程檢查點與續跑，讓長 action list 具持久性 | | `utils/codegen/` | 157 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | | `utils/dag/` | 475 | 跨主機 DAG 編排器（圖模型 + runner） | | `utils/decision_table/` | 103 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | | `utils/deterministic/` | 96 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | | `utils/executor/` | 9,075 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | | `utils/flow_debugger/` | 136 | action list 的單步除錯器與追蹤器 | -| `utils/input_macro/` | 127 | 定時輸入事件重播與宣告式輸入序列 DSL | +| `utils/input_macro/` | 342 | 定時輸入事件：錄製結果的整形（`timeline`／`InputRecorder`，Windows 與 macOS 共用）、重播與宣告式輸入序列 DSL | | `utils/json/` | 74 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | | `utils/json_store/` | 61 | JSON 字典檔持久化的共用小工具（內部管線） | | `utils/loop_guard/` | 140 | 機械式卡死迴圈偵測（agent loop 用） | @@ -292,24 +293,26 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/state_machine/` | 181 | 宣告式有限狀態機驅動 action JSON | | `utils/stubs/` | 236 | 為 `AC_*` 指令面產生型別 stub | | `utils/test_record/` | 64 | 全域測試紀錄單例，記錄每個動作的參數與例外 | -| `utils/work_queue/` | 174 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | +| `utils/work_queue/` | 180 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 12 個套件、約 1,881 行。 +> 14 個套件、約 2,650 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/callback/` | 200 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | -| `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | +| `utils/config_bundle/` | 400 | 使用者設定的單檔匯出／匯入 | | `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | -| `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | -| `utils/exception/` | 194 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/diagnostics/` | 322 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | +| `utils/dbus_client/` | 680 | 只用標準函式庫的 D-Bus session bus 客戶端。原本在 `linux_wayland/` 為 portal 交握而寫，AT-SPI 無障礙後端成為第二個使用者後搬到這裡（`utils/` 在分層上在各 OS 套件之上） | +| `utils/exception/` | 210 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | | `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | | `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | | `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | | `utils/package_manager/` | 98 | 動態載入套件並把 executor 注入其中 | | `utils/path_guard/` | 99 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | +| `utils/platform_id/` | 62 | 作業系統家族的單一判定點。`sys.platform` 原本在一百多處跟字面清單比對，而那些清單都沒有 BSD；`is_x11_unix()` 問的是「這是不是 X11 unix」，這才是守衛一直想問的問題 | | `utils/shell_process/` | 159 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | | `utils/start_exe/` | 39 | 啟動另一個執行檔行程 | @@ -362,17 +365,17 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.5 影像辨識與畫面分析 -> 37 個套件、約 4,999 行。 +> 37 個套件、約 5,027 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/annotate/` | 107 | 截圖標註：畫框、highlight、箭頭、標籤 | +| `utils/annotate/` | 114 | 截圖標註：畫框、highlight、箭頭、標籤 | | `utils/barcode/` | 53 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | | `utils/color_match/` | 103 | 在 HSV 通道上做顏色感知的樣板比對 | | `utils/color_region/` | 79 | 以顏色定位畫面區域（遮罩 + 連通元件） | -| `utils/color_stats/` | 89 | 區域顏色統計：平均色與主色 | +| `utils/color_stats/` | 95 | 區域顏色統計：平均色與主色 | | `utils/coordinate_space/` | 84 | 模型網格座標與實體像素之間的座標空間對映 | -| `utils/cv2_utils/` | 592 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | +| `utils/cv2_utils/` | 597 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | | `utils/edge_lines/` | 120 | 以 Hough 轉換偵測線條／格線／分隔線 | | `utils/edge_match/` | 112 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | | `utils/feature_match/` | 129 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | @@ -390,7 +393,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/motion_regions/` | 73 | 兩影格間的局部變化／活動偵測（absdiff） | | `utils/perceptual_diff/` | 100 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | | `utils/preprocess/` | 185 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | -| `utils/qr/` | 53 | 從影像或螢幕區域解碼 QR code（OpenCV） | +| `utils/qr/` | 59 | 從影像或螢幕區域解碼 QR code（OpenCV） | | `utils/rotated_match/` | 145 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | | `utils/saliency/` | 107 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | | `utils/scale_detect/` | 84 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | @@ -402,7 +405,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/theme_normalize/` | 92 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | | `utils/video_report/` | 133 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | | `utils/visual_match/` | 427 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | -| `utils/visual_regression/` | 217 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | +| `utils/visual_regression/` | 221 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | ### 5.4.6 OCR 與文字理解 @@ -432,12 +435,12 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.7 無障礙樹與原生控制項 -> 16 個套件、約 3,851 行。 +> 16 個套件、約 4,279 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a11y_audit/` | 355 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | -| `utils/accessibility/` | 2,390 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | +| `utils/accessibility/` | 2,818 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | | `utils/ax_events/` | 29 | 反應式 UIA 事件等待（focus-changed） | | `utils/ax_props/` | 44 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | | `utils/ax_text/` | 102 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | @@ -485,40 +488,40 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 20,132 行。 +> 13 個套件、約 20,610 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/a2a/` | 92 | A2A（agent-to-agent）agent card 產生 | | `utils/agent/` | 1,250 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | -| `utils/agent_memory/` | 146 | agent 的持久化情節記憶（goal → trajectory → outcome） | +| `utils/agent_memory/` | 151 | agent 的持久化情節記憶（goal → trajectory → outcome） | | `utils/agent_replay/` | 63 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | | `utils/agent_trace/` | 129 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | | `utils/cost_telemetry/` | 292 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | | `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | | `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | | `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,850 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/mcp_server/` | 17,323 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | | `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | | `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | | `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,703 行。 +> 6 個套件、約 17,720 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | | `utils/admin/` | 327 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | | `utils/config_sync/` | 245 | 透過訊令伺服器做跨機器設定同步 | | `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 11,835 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,238 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/remote_desktop/` | 11,840 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/usb/` | 4,250 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | | `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 -> 24 個套件、約 5,881 行。 +> 24 個套件、約 5,882 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -540,7 +543,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/otp/` | 37 | TOTP 一次性密碼產生（自動化 2FA 登入） | | `utils/outbox/` | 92 | 交易式 outbox，保證至少一次的事件投遞 | | `utils/pytest_plugin/` | 373 | pytest 外掛 + BDD step library（`pytest11` entry point） | -| `utils/rest_api/` | 1,738 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/rest_api/` | 1,739 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | | `utils/socket_server/` | 131 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | | `utils/sse_client/` | 112 | Server-Sent Events 用戶端解析 | | `utils/tls_acme/` | 441 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | @@ -549,7 +552,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 6,865 行。 +> 34 個套件、約 6,896 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -574,7 +577,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/profiler/` | 422 | 逐動作效能剖析器 + 資源剖析器 | | `utils/quarantine/` | 190 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | | `utils/run_diff/` | 123 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | -| `utils/run_history/` | 346 | 執行歷史儲存與產出物管理 | +| `utils/run_history/` | 377 | 執行歷史儲存與產出物管理 | | `utils/sarif/` | 134 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | | `utils/slo/` | 112 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | | `utils/smoothing/` | 67 | 數列移動平均平滑 | @@ -590,7 +593,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.13 資料來源、結構驗證與 i18n -> 24 個套件、約 3,886 行。 +> 24 個套件、約 3,892 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -599,7 +602,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/data_drift/` | 125 | 分布漂移偵測 | | `utils/data_profile/` | 121 | 資料剖析與結構推斷 | | `utils/data_quality/` | 185 | 資料品質：列結構驗證、欄位擷取、遮蔽 | -| `utils/data_source/` | 180 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | +| `utils/data_source/` | 182 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | | `utils/dataset_diff/` | 89 | 表格資料列差異比對（CDC 風格） | | `utils/gettext_catalog/` | 296 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | | `utils/i18n_test/` | 130 | 國際化／在地化測試輔助 | @@ -615,7 +618,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `utils/pdf/` | 87 | PDF 讀取與斷言（選用 pypdf 後端） | | `utils/referential/` | 75 | 跨資料集的參照完整性檢查 | | `utils/schema_compat/` | 162 | JSON Schema 相容性分級 | -| `utils/sql/` | 74 | 對 SQLite 的臨時唯讀 SQL 查詢 | +| `utils/sql/` | 78 | 對 SQLite 的臨時唯讀 SQL 查詢 | | `utils/test_data/` | 205 | 帶種子的合成測試資料產生（純標準庫） | | `utils/xml/` | 250 | XML 檔讀寫與結構變更（`defusedxml`） | @@ -697,14 +700,15 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | | `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,850 行，676 個工具）— 最大子系統 +#### `utils/mcp_server/`（17,323 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | | `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | -| `server.py` | 669 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | -| `http_transport.py` | 323 | MCP 的 HTTP 傳輸。 | +| `server.py` | 713 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | +| `http_transport.py` | 514 | MCP 的 HTTP 傳輸。 | +| `http_sessions.py` | 234 | MCP 的 HTTP 傳輸用的 session 身分:`Mcp-Session-Id` 註冊表,以及每個 session 那條常駐的 server→client SSE 串流。 | | `_client_requests.py` | 217 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | | `_protocol.py` | 165 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | | `resources.py` | 303 | MCP resource 提供者。 | @@ -720,7 +724,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | | `__main__.py` | 87 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（11,835 行／56 檔） +#### `utils/remote_desktop/`（11,840 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 @@ -736,7 +740,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `webrtc_transport.py` | 360 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | | `multi_viewer.py` | 314 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | | `signaling_server.py` | 297 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | -| `audit_log.py` | 283 | SQLite 雜湊鏈稽核記錄。 | +| `audit_log.py` | 288 | SQLite 雜湊鏈稽核記錄。 | | `host_capture.py` | 280 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | | `ws_protocol.py` | 277 | 最小 RFC 6455 WebSocket 框架與握手。 | | `file_transfer.py` | 273 | 分塊檔案傳輸。 | @@ -771,7 +775,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,238 行）與 `utils/usbip/`（920 行） +#### `utils/usb/`（4,250 行）與 `utils/usbip/`（920 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -794,11 +798,11 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `usbip/libusb_backend.py` | 209 | 以 PyUSB／libusb 執行 URB 的正式後端。 | | `usbip/backend.py` | 88 | 可插拔 URB 執行後端。 | -#### `utils/rest_api/`（1,738 行） +#### `utils/rest_api/`（1,739 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `rest_server.py` | 467 | HTTP 前端主體。 | +| `rest_server.py` | 468 | HTTP 前端主體。 | | `rest_handlers.py` | 486 | 端點實作。 | | `rest_openapi.py` | 422 | 走訪路由表產生 OpenAPI 3.1 規格。 | | `rest_auth.py` | 143 | Bearer token 驗證 + 逐 client 限流閘門。 | @@ -919,7 +923,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | audit_log | `audit_log_tab.py` | 192 | 瀏覽並驗證防竄改雜湊鏈。 | | inspector | `inspector_tab.py` | 121 | WebRTC 檢測器：即時摘要與近期統計取樣。 | | usb_devices | `usb_devices_tab.py` | 121 | 唯讀列舉 + 熱插拔監看控制。 | -| usb_browser | `usb_browser_tab.py` | 309 | 檢視端 USB 裝置瀏覽器。 | +| usb_browser | `usb_browser_tab.py` | 310 | 檢視端 USB 裝置瀏覽器。 | | usb_share | `usb_passthrough_panel.py` + `usb_passthrough_prompt.py` | 704 | AnyDesk 風格 USB 直通面板與主機端 ACL 授權對話框。 | | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | @@ -956,12 +960,12 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `benchmarks/core_latency.py` | 32 行 | 對穩定無頭進入點的可重複煙霧基準測試。 | | `examples/` | 27 個腳本 | 從截圖點擊、OCR、排程、遠端桌面、agent loop、可觀測性，一路到 computer-use、Wayland、跨主機 DAG、chatops、pytest/BDD、anchor locator。 | | `browser-extension/` | manifest v3 擴充 | 瀏覽器端配合元件（background／content script／popup）。 | -| `docker/` | Dockerfile ×6 + compose + 8 支驗證／伺服器腳本 | 無頭容器（`Dockerfile`）、帶 XFCE 桌面的容器（`Dockerfile.xfce`),以及四個**驗證用**映像:`Dockerfile.wayland`（sway headless,擷取路徑 + `libei_verify.py` 對真的 libei.so 解析符號）、`Dockerfile.eis`（`eis_server.py` 用 ctypes 綁 libeis 起一個真的 EIS server,`eis_verify.py` 把 libei sender 對著它跑完整握手與發送）、`Dockerfile.portal`（`portal_server.py` 自己佔住 `org.freedesktop.portal.Desktop`,真的 `dbus-daemon` + 真的 liboeffis 跑完 RemoteDesktop 交握）、`Dockerfile.ydotool`（真的 uinput 裝置,`ydotool_verify.py` 直接讀回 `/dev/input/eventN`）、`Dockerfile.seat`（`headless,libinput` + builtin seat,合成器真的吃下 ydotool 裝置,`seat_verify.py` 從 `grim -c` 的像素讀回游標落點）。全部接在 `.github/workflows/docker.yml`。 | +| `docker/` | Dockerfile ×8 + compose + 9 支驗證／伺服器腳本 | 無頭容器（`Dockerfile`）、帶 XFCE 桌面的容器（`Dockerfile.xfce`),以及四個**驗證用**映像:`Dockerfile.wayland`（sway headless,擷取路徑 + `libei_verify.py` 對真的 libei.so 解析符號）、`Dockerfile.eis`（`eis_server.py` 用 ctypes 綁 libeis 起一個真的 EIS server,`eis_verify.py` 把 libei sender 對著它跑完整握手與發送）、`Dockerfile.portal`（`portal_server.py` 自己佔住 `org.freedesktop.portal.Desktop`,真的 `dbus-daemon` + 真的 liboeffis 跑完 RemoteDesktop 交握）、`Dockerfile.ydotool`（真的 uinput 裝置,`ydotool_verify.py` 直接讀回 `/dev/input/eventN`）、`Dockerfile.seat`（`headless,libinput` + builtin seat,合成器真的吃下 ydotool 裝置,`seat_verify.py` 從 `grim -c` 的像素讀回游標落點）、`Dockerfile.x11`（真的 Xvfb + openbox,`x11_verify.py` 用 `xev` 把注入的事件從真的客戶端讀回（含 `synthetic NO`,這是 XTest 跟 `XSendEvent` 的差別）,另用 ImageMagick `import` 做獨立擷取對照,跑兩種螢幕版面）。全部接在 `.github/workflows/docker.yml`。 | | `k8s/helm/` | Helm chart | Kubernetes 部署。 | | `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | | `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | -| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 466 個 `test_*.py`／4,443 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`verify`、`test_source` | 478 個 `test_*.py`／4,654 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 `verify/macos_verify.py` 是在真的 `macos-14` runner 上量測 TCC 到底允許什麼的探針（macOS 是唯一沒有容器可用的支援平台），不被 pytest 收集。 | --- @@ -1016,25 +1020,25 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | | `gui/` | 89 | 26,542 | -| `utils/mcp_server/` | 20 | 16,850 | -| `utils/remote_desktop/` | 56 | 11,835 | +| `utils/mcp_server/` | 21 | 17,323 | +| `utils/remote_desktop/` | 56 | 11,840 | | `utils/executor/` | 6 | 9,075 | -| `utils/usb/` | 17 | 4,238 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | -| `utils/accessibility/` | 12 | 2,390 | -| `wrapper/` | 12 | 2,056 | -| `windows/` | 23 | 1,995 | -| `utils/rest_api/` | 8 | 1,738 | +| `utils/usb/` | 17 | 4,250 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,363 | +| `utils/accessibility/` | 13 | 2,818 | +| `wrapper/` | 3,065 | 3,013 新增 `window_backends/`：視窗管理的平台縫（`base` / `windows_backend` / `x11_backend` / `macos_backend` / `null_backend`）。放在 `wrapper/` 而不是 `utils/`，因為它必須 import `windows/`、`linux_with_x11/`、`osx/`，而 `utils/` 在分層上在那三者之上。 | +| `windows/` | 23 | 1,894 | +| `utils/rest_api/` | 8 | 1,739 | | `utils/agent/` | 8 | 1,250 | -| `linux_with_x11/` | 19 | 1,175 | -| `linux_wayland/` | 17 | 3,431 | +| `linux_with_x11/` | 19 | 1,215 | +| `linux_wayland/` | 17 | 2,835 | | `utils/triggers/` | 4 | 1,146 | | `utils/ocr/` | 9 | 1,112 | | `utils/usbip/` | 5 | 920 | | `utils/assertion/` | 3 | 863 | -| `osx/` | 17 | 761 | +| `osx/` | 17 | 907 | | `autocontrol-lsp/` | 8 | 744 | | `utils/hotkey/` | 7 | 727 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,238 | -| **總計** | **1,010** | **137,452** | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 690 | 50,425 | +| **總計** | **1,023** | **139,988** | diff --git a/docker/Dockerfile.x11 b/docker/Dockerfile.x11 new file mode 100644 index 00000000..75ff25d8 --- /dev/null +++ b/docker/Dockerfile.x11 @@ -0,0 +1,119 @@ +# AutoControl X11 verification image — a real X server, a real WM, real clients. +# +# Wayland got five jobs that check what actually arrives at a real peer. +# X11 — the older, more widely deployed of the two Linux paths — got none: +# every X11 assertion in the suite is made against a mock of python-Xlib, so +# nothing had ever confirmed that an injected event reaches a client, that it +# reaches it as *real* input rather than a sent event, or that a captured +# pixel is the pixel on screen. +# +# Xvfb needs no GPU and no seat, so a plain runner can host a genuine X +# server. Ground truth deliberately comes from other codebases than the one +# under test: +# +# * ``xev`` — a real X client whose window prints every event that reaches +# it. An XTest-injected event has to travel through the server and be +# delivered like any other, so this reads back what arrived, the same way +# the ydotool job reads back ``/dev/input/eventN``. +# * ``import`` (ImageMagick) — an independent screen grabber, standing in +# for the role ``grim`` plays in the Wayland job. +# * ``xdotool`` / ``xdpyinfo`` — the server's own answers for pointer +# position and screen geometry. +# +# openbox rather than a full desktop on purpose: it is EWMH-compliant, which +# is what the window-management checks need, and it does not paint over the +# root window, which is where the pixel checks put their known colours. +# +# What this image CANNOT answer, and why: +# * A negative layout origin. On X11 the root window is the union of every +# monitor and always begins at (0, 0); a monitor placed to the left +# shifts the others right rather than making the origin negative. The +# Wayland job's second layout has no X11 analogue — this is a protocol +# difference, not an untested case. +# * The uinput input path. It needs /dev/uinput, which is the host's to +# grant; docker/Dockerfile.ydotool already reads those events back off +# the kernel device. +# +# Build: docker build -f docker/Dockerfile.x11 -t autocontrol-x11:latest . +# Run: docker run --rm autocontrol-x11:latest + +# Build every wheel in a throwaway stage, as docker/Dockerfile does, so the +# runtime layer installs binaries only and the dependency set is fixed here +# rather than re-resolved against PyPI at run time. +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# - xvfb + xauth: the X server itself, with RANDR so a monitor layout exists. +# - x11-utils: xev (the event reader) and xdpyinfo (the server's geometry). +# - x11-xserver-utils: xrandr, which declares the monitor layout and reads +# it back. Debian ships it separately from x11-utils. +# - xdotool: the server's own answer for where the pointer is. +# - imagemagick: `import -window root`, an independent grabber. +# - openbox: a real EWMH window manager, without a desktop over the root. +# - xterm: a real client to own a real window. +# - at-spi2-core + dbus-x11: the accessibility bus. AT-SPI is a D-Bus +# protocol, so the backend needs a session bus and the two services +# D-Bus activates on it (org.a11y.Bus and the registry). +# - zenity: a real GTK application, so there is an actual accessible +# tree to walk. xterm exposes none — it is not a toolkit application. +# - libgl1 + libglib2.0-0: opencv-python hard-requires libGL.so.1 and +# libgthread-2.0.so.0 at import, so the package cannot even load without them. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + xvfb xauth x11-utils x11-xserver-utils xdotool \ + imagemagick openbox xterm \ + at-spi2-core dbus-x11 zenity \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/x11_verify.py /opt/verify/x11_verify.py +COPY docker/x11_window_verify.py /opt/verify/x11_window_verify.py +COPY docker/x11_atspi_verify.py /opt/verify/x11_atspi_verify.py +COPY docker/entrypoint-x11.sh /usr/local/bin/autocontrol-x11-verify +RUN chmod +x /usr/local/bin/autocontrol-x11-verify + +# Unprivileged, unlike the Wayland verification image. That one runs as root +# because XDG_RUNTIME_DIR ownership would otherwise be the only thing +# standing between it and an answer; X11 has no such requirement — the +# server's socket lives in /tmp and its auth file in $HOME, both of which an +# ordinary user owns. Running as a normal user is also one less difference +# between this container and the desktop it stands in for. +# /app is the working directory, and the library opens its log file in the +# cwd at import, so the verifying user has to own it. +RUN useradd --create-home --shell /bin/sh verify \ + && chown -R verify:verify /app +USER verify + +# GTK3 only loads the accessibility bridge when it is asked to, and +# what normally asks is a desktop setting this container has no +# store for. Naming the module directly is what bridges the app. +ENV HOME=/home/verify \ + GTK_MODULES=gail:atk-bridge \ + NO_AT_BRIDGE=0 \ + PYTHONUNBUFFERED=1 \ + DISPLAY=:99 \ + XDG_SESSION_TYPE=x11 \ + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 \ + SCREEN_GEOMETRY=1280x800x24 + +ENTRYPOINT ["/usr/local/bin/autocontrol-x11-verify"] diff --git a/docker/entrypoint-x11.sh b/docker/entrypoint-x11.sh new file mode 100644 index 00000000..4966c163 --- /dev/null +++ b/docker/entrypoint-x11.sh @@ -0,0 +1,143 @@ +#!/bin/sh +# Bring up a real X server with a real window manager, then run the X11 +# verification inside it. +# +# The verification runs twice, over two monitor layouts: +# +# * one monitor covering the whole screen — the single-display desktop; +# * two monitors side by side, declared with `xrandr --setmonitor` — what +# a dual-display desktop reports to its clients. X11 exposes that through +# RANDR monitors rather than through separate screens, so this is the +# shape a client actually sees. +# +# There is deliberately no negative-origin layout: on X11 the root window is +# the union of every monitor and always starts at (0, 0), so a monitor to the +# left shifts the others right instead of moving the origin. The Wayland +# job's second layout has no analogue here. +# +# Nothing below is allowed to skip quietly. If the server, the window manager +# or the monitor layout cannot be brought up, this says so and fails. +set -eu + +# AT-SPI is a D-Bus protocol: without a session bus there is no accessibility +# bus to activate, and the backend would correctly report itself unavailable +# for a reason that is this container's fault rather than the code's. Re-exec +# under one, once. +if [ -z "${DBUS_SESSION_BUS_ADDRESS:-}" ]; then + exec dbus-run-session -- "$0" "$@" +fi + +GEOMETRY="${SCREEN_GEOMETRY:-1280x800x24}" +DISPLAY_NUM="${DISPLAY:-:99}" + +echo "starting Xvfb on ${DISPLAY_NUM} at ${GEOMETRY}" +Xvfb "${DISPLAY_NUM}" -screen 0 "${GEOMETRY}" +extension RANDR \ + -nolisten tcp >/tmp/xvfb.log 2>&1 & +XVFB_PID=$! + +# xdpyinfo is the server answering for itself, so it is also the readiest +# proof that the server is up — a fixed sleep would be a guess. +ready=0 +for _ in $(seq 1 100); do + if xdpyinfo >/dev/null 2>&1; then + ready=1 + break + fi + if ! kill -0 "$XVFB_PID" 2>/dev/null; then + echo "Xvfb exited before accepting a connection; its log follows:" + cat /tmp/xvfb.log + exit 1 + fi + sleep 0.1 +done +if [ "$ready" -ne 1 ]; then + echo "Xvfb never accepted a connection; its log follows:" + cat /tmp/xvfb.log + exit 1 +fi +echo "Xvfb is up" + +# openbox owns _NET_* — the window-management checks read the properties it +# maintains, so a session without a window manager would test nothing. +openbox >/tmp/openbox.log 2>&1 & +OPENBOX_PID=$! +wm=0 +for _ in $(seq 1 100); do + if xprop -root _NET_SUPPORTING_WM_CHECK 2>/dev/null | grep -q window; then + wm=1 + break + fi + if ! kill -0 "$OPENBOX_PID" 2>/dev/null; then + echo "openbox exited before claiming the screen; its log follows:" + cat /tmp/openbox.log + exit 1 + fi + sleep 0.1 +done +if [ "$wm" -ne 1 ]; then + echo "no window manager claimed the screen; openbox log follows:" + cat /tmp/openbox.log + exit 1 +fi +echo "openbox is managing ${DISPLAY_NUM}" + +# x11_window_verify imports the harness from x11_verify, so the directory +# holding both has to be on the import path. +export PYTHONPATH=/opt/verify + +# Runs one pass over one monitor layout; echoes the number of failed checks. +run_pass() { + label="$1" + echo "========================================================================" + echo "layout: ${label}" + echo "========================================================================" + xrandr --listmonitors || true + rc=0 + python3 /opt/verify/x11_verify.py || rc=$? + return "$rc" +} + +total=0 + +run_pass "one monitor over the whole screen" || total=$((total + $?)) + +# Split the same screen into two RANDR monitors. Physical size is required by +# the syntax and irrelevant here; `none` means the monitor is backed by no +# physical output, which is exactly what a virtual layout is. +echo +half_width=$(xdpyinfo | awk '/dimensions:/ {split($2, d, "x"); print int(d[1] / 2)}') +height=$(xdpyinfo | awk '/dimensions:/ {split($2, d, "x"); print d[2]}') +if ! xrandr --setmonitor AC-left "${half_width}/169x${height}/211+0+0" none \ + || ! xrandr --setmonitor AC-right \ + "${half_width}/169x${height}/211+${half_width}+0" none; then + echo "FAILED to declare a two-monitor layout with xrandr --setmonitor." + echo "That is a real failure, not a reason to skip: the dual-display" + echo "geometry below is exactly what has never been checked." + exit 1 +fi + +run_pass "two monitors side by side" || total=$((total + $?)) + +# Window management does not depend on the monitor layout — it is about what +# the window manager agrees to — so it runs once, after both layout passes. +echo +echo "========================================================================" +echo "window management against a real window manager" +echo "========================================================================" +python3 /opt/verify/x11_window_verify.py || total=$((total + $?)) + +echo +echo "========================================================================" +echo "accessibility against a real AT-SPI bus" +echo "========================================================================" +python3 /opt/verify/x11_atspi_verify.py || total=$((total + $?)) + +echo +echo "========================================================================" +echo "total failed checks: ${total}" +echo "========================================================================" + +kill "$OPENBOX_PID" 2>/dev/null || true +kill "$XVFB_PID" 2>/dev/null || true + +exit "$total" diff --git a/docker/x11_atspi_verify.py b/docker/x11_atspi_verify.py new file mode 100644 index 00000000..5385cf9a --- /dev/null +++ b/docker/x11_atspi_verify.py @@ -0,0 +1,191 @@ +"""Verify the Linux accessibility backend against a real AT-SPI bus. + +Linux had no accessibility backend at all — the selector fell through to the +null one — while the capability matrix claimed "backend tests" for Linux X11. +The backend that closes that speaks AT-SPI2 over D-Bus directly, with no +binding, so what has to be checked is whether a *real* accessibility bus and a +*real* toolkit application agree with the bytes it sends. + +Neither half can be mocked usefully. The bus is activated by D-Bus rather than +started by hand, an application only appears on it if its toolkit bridge +loaded, and the tree's shape is the toolkit's business. So the subject here is +``zenity`` — a GTK application with a label and buttons whose names are known +because this script chose them. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import os +import subprocess # nosec B404 # reason: argv lists of fixed tool names, no shell +import sys +import time +from typing import Any, Optional + +from x11_verify import _assert_eq, _assert_true, check, note, summarise + +#: The text zenity is told to show. Distinctive so a match cannot be an +#: accident of some other accessible carrying the same words. +DIALOG_TITLE = "autocontrol-atspi-dialog" +DIALOG_TEXT = "autocontrol-atspi-label" + +#: How long the toolkit is given to bridge itself onto the bus. This is the +#: slowest part of the whole image: the application has to start, load the +#: bridge module, and register with the registry. +BRIDGE_TIMEOUT = 30.0 + + +class Zenity: + """A real GTK application, bridged onto the accessibility bus.""" + + def __init__(self) -> None: + self._process: Optional[subprocess.Popen] = None + + def __enter__(self) -> "Zenity": + self._process = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["zenity", "--info", "--title", DIALOG_TITLE, + "--text", DIALOG_TEXT], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is not None and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + + def alive(self) -> bool: + return self._process is not None and self._process.poll() is None + + +def _await_bridged(backend, timeout: float = BRIDGE_TIMEOUT) -> list: + """Wait until the application shows up on the bus, then return its tree.""" + deadline = time.monotonic() + timeout + last: list = [] + while time.monotonic() < deadline: + last = backend.list_elements(max_results=400) + if any(DIALOG_TEXT in element.name or DIALOG_TITLE in element.name + for element in last): + return last + time.sleep(0.5) + return last + + +def check_backend_selection() -> None: + def _selected() -> str: + from je_auto_control.utils.accessibility.backends import ( + get_backend, reset_backend_cache, + ) + + reset_backend_cache() + backend = get_backend() + _assert_true( + backend.available, + f"backend {backend.name!r} reports unavailable — if this says " + f"'null', the accessibility bus did not come up in this container") + return _assert_eq(backend.name, "linux-atspi") + check("Linux selects the AT-SPI backend, not the null one", _selected) + + +def check_tree(backend, zenity: Zenity) -> None: + elements = _await_bridged(backend) + + def _application_appears() -> str: + _assert_true(zenity.alive(), "zenity exited before it was inspected") + names = [element.name for element in elements] + _assert_true( + any(DIALOG_TEXT in name or DIALOG_TITLE in name for name in names), + f"the dialog never appeared on the bus; saw {names[:20]}") + return f"{len(elements)} accessibles, including the dialog" + check("a real GTK application is reachable over the bus", + _application_appears) + + def _roles_are_named() -> str: + # GetRoleName is a separate call from the name, and a backend that + # skipped it would still list every element — with no role at all, + # which is what a role= filter searches on. + roles = {element.role for element in elements if element.role} + _assert_true(bool(roles), "no element reported a role") + return f"{len(roles)} distinct roles, e.g. {sorted(roles)[:4]}" + check("every accessible carries the role AT-SPI reports", _roles_are_named) + + def _extents_are_screen_pixels() -> str: + # GetExtents takes a coordinate space, and the wrong one returns + # window-relative numbers that look plausible and click the wrong + # place. A window on this screen has to have a non-zero size. + sized = [element for element in elements + if element.bounds[2] > 0 and element.bounds[3] > 0] + _assert_true(bool(sized), "no accessible reported a rectangle") + widest = max(sized, key=lambda element: element.bounds[2]) + return (f"{len(sized)} with a rectangle; widest {widest.role!r} " + f"{widest.bounds}") + check("Component.GetExtents returns real screen rectangles", + _extents_are_screen_pixels) + + def _application_is_named() -> str: + # The walk is per-application, so every element has to carry the name + # of the application it came from — that is what app_name filters on. + owners = {element.app_name for element in elements if element.app_name} + _assert_true(bool(owners), "no element carried an application name") + return f"applications on the bus: {sorted(owners)}" + check("elements carry the application they belong to", _application_is_named) + + def _scoped_walk_is_narrower() -> str: + scoped = backend.list_elements(max_results=400, + window_title=DIALOG_TITLE) + _assert_true(len(scoped) <= len(elements), + f"scoping widened the walk: {len(scoped)} > {len(elements)}") + return f"{len(scoped)} scoped vs {len(elements)} unscoped" + check("scoping to a window narrows the walk", _scoped_walk_is_narrower) + + +def check_state(backend) -> None: + def _state() -> str: + # The state bitfield arrives as two 32-bit words; reading only the + # first silently drops every state above bit 31. + state = backend.get_state(role="push button") + if state is None: + state = backend.get_state(role="label") + _assert_true(state is not None, + "get_state found neither a button nor a label") + _assert_true("enabled" in state, + f"no enabled flag in {sorted(state)}") + return f"{sorted(state)}" + check("get_state reads the AT-SPI state bitfield", _state) + + +def main() -> int: + print("=" * 72) + print("AutoControl accessibility — real AT-SPI bus, real GTK application") + print("=" * 72) + note(f"DBUS_SESSION_BUS_ADDRESS set: " + f"{bool(os.environ.get('DBUS_SESSION_BUS_ADDRESS'))}") + + check_backend_selection() + + from je_auto_control.utils.accessibility.backends import get_backend + + backend = get_backend() + if not backend.available: + print("-" * 72) + print("The accessibility bus is not up, so nothing below can run.") + print("That is a failure of this container, not a reason to skip:") + print("every other check here depends on it.") + return summarise() or 1 + + with Zenity() as zenity: + check_tree(backend, zenity) + check_state(backend) + + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("Anything needing a screen reader's own state — at-spi2 exposes") + note(" ScreenReaderEnabled, and nothing here is a screen reader.") + + return summarise() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/x11_verify.py b/docker/x11_verify.py new file mode 100644 index 00000000..b12f6ffc --- /dev/null +++ b/docker/x11_verify.py @@ -0,0 +1,695 @@ +"""Verify AutoControl's X11 backend against a real X server and real clients. + +Runs inside the session ``entrypoint-x11.sh`` brings up (see +``Dockerfile.x11``). Every X11 assertion in the unit suite is made against a +mock of ``python-Xlib``, so none of it could answer the questions that matter: +does an injected event actually reach a client, does it arrive as *real* input +rather than a sent event most applications ignore, and is a captured pixel the +pixel that is on screen. + +Ground truth deliberately comes from outside the code under test: + +* ``xev`` is a real X client. Its window prints every event delivered to it, + so an XTest-injected click is read back the way the ydotool job reads its + events back off ``/dev/input/eventN`` — including ``synthetic NO``, which is + what separates server-level input from ``XSendEvent`` traffic that toolkits + routinely discard. +* ``import`` (ImageMagick) is an independent grabber, in the role ``grim`` + plays for Wayland. The root window is painted two asymmetric colours first, + because on a uniform screen any wrong rectangle looks right. +* ``xdotool`` and ``xdpyinfo`` are the server answering for itself about + pointer position and geometry. + +The entrypoint runs this twice: once with a single monitor covering the +screen, once with two RANDR monitors side by side. Nothing below is written +for one layout or the other — every coordinate is derived from what the +server reports. + +There is deliberately no negative-origin pass. On X11 the root window is the +union of every monitor and always starts at ``(0, 0)``, so a monitor placed to +the left shifts the others right rather than moving the origin. The Wayland +job's second layout has no analogue here; that is a protocol difference, not +an untested case. + +Exit status is the number of failed checks, so the container's exit code says +whether this passed. +""" +from __future__ import annotations + +import os +import re +import subprocess # nosec B404 # reason: argv lists of fixed tool names, no shell +import sys +import tempfile +import time +import traceback +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple + +#: Painted onto the two halves of the root window. Asymmetric in every +#: channel so a red/blue swap cannot pass, and different from each other so a +#: region grab has something to get wrong. +LEFT_COLOUR = (0x12, 0x34, 0x56) +RIGHT_COLOUR = (0xAB, 0xCD, 0xEF) + +#: How long an injected event is given to come back out of a real client. +EVENT_TIMEOUT = 5.0 + +#: Everything this run writes goes here, as the sibling verification +#: scripts do, rather than into shared /tmp paths another process could +#: already own. +SCRATCH = tempfile.mkdtemp(prefix="autocontrol-x11-verify-") + +_results: List[Tuple[str, bool, str]] = [] + + +def check(name: str, fn: Callable[[], Any]) -> Any: + """Run one check, record pass/fail, and keep going either way.""" + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a failed check must not stop the rest + _results.append((name, False, traceback.format_exc(limit=3).strip())) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True, str(detail))) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def note(message: str) -> None: + """Print an indented remark that is not a check.""" + print(f" {message}") + + +def _run(argv: List[str], *, timeout: float = 15.0) -> str: + """Run one tool from this image and return its stdout.""" + # argv is assembled from literals in this file; no shell, no user input. + completed = subprocess.run(argv, check=True, # nosec B603 B607 # nosemgrep + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return completed.stdout.decode("utf-8", errors="replace") + + +def _assert_eq(actual: Any, expected: Any) -> str: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + return repr(actual) + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +# --- the server's own description of itself -------------------------------- + + +@dataclass(frozen=True) +class Monitor: + """One RANDR monitor, as ``xrandr --listmonitors`` reports it.""" + + name: str + x: int + y: int + width: int + height: int + + def inside(self, dx: int = 40, dy: int = 40) -> Tuple[int, int]: + """A screen coordinate ``(dx, dy)`` into this monitor.""" + return (self.x + dx, self.y + dy) + + +@dataclass(frozen=True) +class Layout: + """The screen and the monitors carved out of it.""" + + width: int + height: int + monitors: List[Monitor] + + @property + def multi(self) -> bool: + """Whether there is more than one monitor to tell apart.""" + return len(self.monitors) > 1 + + +_MONITOR_RE = re.compile( + r"^\s*\d+:\s+\+?\*?(?P\S+)\s+" + r"(?P\d+)/\d+x(?P\d+)/\d+\+(?P\d+)\+(?P\d+)", +) + + +def read_layout() -> Layout: + """Describe the live screen from the server's own answers.""" + dimensions = re.search(r"dimensions:\s+(\d+)x(\d+)", _run(["xdpyinfo"])) + if dimensions is None: + raise RuntimeError("xdpyinfo reported no dimensions") + width, height = int(dimensions.group(1)), int(dimensions.group(2)) + + monitors: List[Monitor] = [] + for line in _run(["xrandr", "--listmonitors"]).splitlines(): + found = _MONITOR_RE.match(line) + if found is None: + continue + monitors.append(Monitor( + name=found.group("name"), + x=int(found.group("x")), y=int(found.group("y")), + width=int(found.group("w")), height=int(found.group("h")), + )) + if not monitors: + # A screen with no RANDR monitor is still a screen; treat the whole + # root window as one rather than inventing geometry. + monitors.append(Monitor("screen", 0, 0, width, height)) + monitors.sort(key=lambda monitor: monitor.x) + return Layout(width=width, height=height, monitors=monitors) + + +def paint_root(layout: Layout) -> None: + """Paint the root window's two halves in the known colours. + + Uses python-Xlib directly. Painting is setup, not subject: what is under + test is whether the *readers* agree with each other and with an + independent grabber about what ended up there. + """ + from Xlib import display as xdisplay + + connection = xdisplay.Display() + screen = connection.screen() + root = screen.root + pixmap = root.create_pixmap(layout.width, layout.height, screen.root_depth) + context = pixmap.create_gc() + half = layout.width // 2 + for colour, left, span in ( + (LEFT_COLOUR, 0, half), + (RIGHT_COLOUR, half, layout.width - half), + ): + context.change(foreground=(colour[0] << 16) | (colour[1] << 8) | colour[2]) + pixmap.fill_rectangle(context, left, 0, span, layout.height) + root.change_attributes(background_pixmap=pixmap) + root.clear_area(x=0, y=0, width=layout.width, height=layout.height) + connection.sync() + # The pixmap must outlive this function: the server keeps a reference for + # the background, but freeing the client-side resource here would take it + # with us. Parking it on the module keeps the background painted. + globals()["_ROOT_PIXMAP"] = (connection, pixmap) + + +def expected_colour(layout: Layout, x: int) -> Tuple[int, int, int]: + """The colour :func:`paint_root` put at screen column ``x``.""" + return LEFT_COLOUR if x < layout.width // 2 else RIGHT_COLOUR + + +def truth_capture(path: Optional[str] = None): + """Grab the root window with ImageMagick and return it as a Pillow image.""" + from PIL import Image + + path = path or os.path.join(SCRATCH, "truth.png") + _run(["import", "-window", "root", "-silent", path]) + with Image.open(path) as opened: + return opened.convert("RGB").copy() + + +# --- a real client that reports what reached it ---------------------------- + + +_EVENT_HEAD_RE = re.compile(r"^(\w+) event,") +_BUTTON_RE = re.compile(r"\bbutton (\d+)\b") +_KEYCODE_RE = re.compile(r"\bkeycode (\d+)\b") +_KEYSYM_RE = re.compile(r"\(keysym 0x[0-9a-f]+, (\S+)\)") +_ROOT_XY_RE = re.compile(r"\broot:\((-?\d+),(-?\d+)\)") +_SYNTHETIC_RE = re.compile(r"\bsynthetic (\w+)\b") + + +class EventTester: + """``xev``: a real X client whose window prints what is delivered to it. + + An event injected through XTest travels through the server and is + dispatched like any other, so what this reads back is the arrival, not the + call. ``xev`` is line-buffered through ``stdbuf`` because its output is a + pipe here rather than a terminal, and a block-buffered log would report + nothing until it filled. + """ + + LOG_PATH = os.path.join(SCRATCH, "xev.log") + WINDOW_NAME = "Event Tester" + + def __init__(self) -> None: + self._process: Optional[subprocess.Popen] = None + self._handle = None + self._offset = 0 + # Text read but not yet forming a whole record. xev is writing while + # this reads, so the tail of any read is routinely half an event: + # parsing it would report a KeyPress whose keycode had not been + # written yet, which looks exactly like a backend that sent no keycode. + self._pending = "" + # Consecutive reads that returned nothing. xev separates records with + # a blank line but writes no terminator after the last one, so the + # most recent event stays unterminated until the *next* one arrives — + # and a press with nothing after it would never be reported at all. + # Two silent reads is what says the record is finished rather than + # half-written. + self._idle_reads = 0 + # Parsed events not yet consumed by a check. Draining must not + # discard what the caller did not ask for — a click writes press and + # release into one chunk, and a reader that returns the press and + # drops the release makes a working click look like a stuck button. + self._events: List[Dict[str, Any]] = [] + self.window_id = 0 + self.rect: Tuple[int, int, int, int] = (0, 0, 0, 0) + + def __enter__(self) -> "EventTester": + self._handle = open(self.LOG_PATH, "wb") # noqa: SIM115 # reason: closed in __exit__ + # stdbuf + xev, both from this image; no shell, no user input. + self._process = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["stdbuf", "-oL", "xev", "-geometry", "400x300+80+80"], + stdout=self._handle, stderr=subprocess.STDOUT) + self.window_id = self._await_window() + self.rect = self._read_geometry() + # The window manager decides where the window lands and what has + # focus, so ask for focus rather than assume the map gave it. + _run(["xdotool", "windowactivate", "--sync", str(self.window_id)]) + _run(["xdotool", "windowfocus", "--sync", str(self.window_id)]) + self.flush() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is not None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + if self._handle is not None: + self._handle.close() + + def _await_window(self) -> int: + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + found = _run(["xdotool", "search", "--name", self.WINDOW_NAME]) + except subprocess.CalledProcessError: + found = "" + ids = [line for line in found.split() if line.isdigit()] + if ids: + return int(ids[-1]) + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + f"xev exited {self._process.returncode} before mapping a window") + time.sleep(0.1) + raise RuntimeError("xev never mapped a window") + + def _read_geometry(self) -> Tuple[int, int, int, int]: + shell = _run(["xdotool", "getwindowgeometry", "--shell", + str(self.window_id)]) + values: Dict[str, int] = {} + for line in shell.splitlines(): + if "=" in line: + key, _, value = line.partition("=") + if value.strip().lstrip("-").isdigit(): + values[key.strip()] = int(value) + return (values.get("X", 0), values.get("Y", 0), + values.get("WIDTH", 0), values.get("HEIGHT", 0)) + + def centre(self) -> Tuple[int, int]: + """The screen coordinate at the middle of the tester's window.""" + x, y, width, height = self.rect + return (x + width // 2, y + height // 2) + + def drain(self) -> List[Dict[str, Any]]: + """Buffer every *complete* record written since the last read.""" + with open(self.LOG_PATH, "rb") as log: + log.seek(self._offset) + raw = log.read() + self._offset += len(raw) + self._idle_reads = 0 if raw else self._idle_reads + 1 + self._pending += raw.decode("utf-8", errors="replace") + # xev separates records with a blank line, so everything after the + # last one may still be being written. Keep it for the next read. + chunks = re.split(r"\n\s*\n", self._pending) + self._pending = chunks.pop() + # ...unless nothing has been written for two reads running and what is + # held ends in a newline. Then it is not half-written, it is the last + # record, and holding it would lose every event that happens to be + # final — which is every press that is waiting for its release. + if (self._idle_reads >= 2 and self._pending.endswith("\n") + and _EVENT_HEAD_RE.match(self._pending.strip())): + chunks.append(self._pending) + self._pending = "" + fresh: List[Dict[str, Any]] = [] + for record in chunks: + head = _EVENT_HEAD_RE.match(record.strip()) + if head is None: + continue + fresh.append(_parse_record(head.group(1), record)) + self._events.extend(fresh) + return fresh + + def flush(self) -> None: + """Forget everything so far, so a check starts from a clean slate.""" + self.drain() + self._events.clear() + + def buffered(self, kind: str, settle: float = 0.5) -> List[Dict[str, Any]]: + """Events of ``kind`` that have arrived and nothing has claimed. + + Used to assert an *absence*, so it drains until the reader has gone + quiet: returning early would report "nothing arrived" for an event + still sitting unterminated in the buffer. + """ + deadline = time.monotonic() + settle + while time.monotonic() < deadline: + self.drain() + time.sleep(0.05) + return [event for event in self._events if event["type"] == kind] + + def collect(self, kind: str, count: int = 1, + timeout: float = EVENT_TIMEOUT) -> List[Dict[str, Any]]: + """Wait for ``count`` events of ``kind``, or raise saying what came. + + What is taken is removed from the buffer and what is not stays there, + so consecutive collects for different kinds see the same arrival. + """ + deadline = time.monotonic() + timeout + while True: + self.drain() + wanted = [event for event in self._events if event["type"] == kind] + if len(wanted) >= count: + taken = wanted[:count] + for event in taken: + self._events.remove(event) + return taken + if time.monotonic() >= deadline: + raise AssertionError( + f"waited {timeout}s for {count}x {kind}; the buffer holds " + f"{[event['type'] for event in self._events]}") + time.sleep(0.05) + + +def _parse_record(kind: str, record: str) -> Dict[str, Any]: + """Turn one xev record into the fields the checks care about.""" + button = _BUTTON_RE.search(record) + keycode = _KEYCODE_RE.search(record) + keysym = _KEYSYM_RE.search(record) + root_xy = _ROOT_XY_RE.search(record) + synthetic = _SYNTHETIC_RE.search(record) + return { + "type": kind, + "button": int(button.group(1)) if button else None, + "keycode": int(keycode.group(1)) if keycode else None, + "keysym": keysym.group(1) if keysym else None, + "root": ((int(root_xy.group(1)), int(root_xy.group(2))) + if root_xy else None), + "synthetic": synthetic.group(1) if synthetic else None, + } + + +# --- checks ---------------------------------------------------------------- + + +def report_environment(layout: Layout) -> None: + """Print what this run is actually working against.""" + print("-" * 72) + note(f"DISPLAY={os.environ.get('DISPLAY')!r} " + f"XDG_SESSION_TYPE={os.environ.get('XDG_SESSION_TYPE')!r}") + note(f"screen: {layout.width}x{layout.height}") + for monitor in layout.monitors: + note(f"monitor {monitor.name}: {monitor.width}x{monitor.height} " + f"at ({monitor.x}, {monitor.y})") + print("-" * 72) + + +def check_backend_selection() -> None: + """The wrapper must land on the X11 backend, not fall through to Wayland.""" + def _selected() -> str: + from je_auto_control.wrapper import platform_wrapper + + module = platform_wrapper.mouse.__name__ + _assert_true("linux_with_x11" in module, + f"expected the X11 backend, got {module}") + return module + check("the platform wrapper selects the X11 backend", _selected) + + +def check_geometry(layout: Layout) -> None: + """``screen_size()`` must be the screen the server reports.""" + def _size() -> str: + from je_auto_control import screen_size + + return _assert_eq(tuple(screen_size()), (layout.width, layout.height)) + check("screen_size() equals the server's dimensions", _size) + + +def check_pixels(layout: Layout) -> None: + """Every reader must agree with an independent grabber and each other.""" + from je_auto_control import get_pixel, screenshot + + truth = truth_capture() + left = layout.monitors[0].inside() + right = layout.monitors[-1].inside() if layout.multi else ( + layout.width - 40, 40) + + def _truth_is_painted() -> str: + for point in (left, right): + _assert_eq(truth.getpixel(point), expected_colour(layout, point[0])) + return f"{left} and {right} carry the painted colours" + check("an independent grabber sees the painted colours", _truth_is_painted) + + def _get_pixel() -> str: + for point in (left, right): + _assert_eq(tuple(get_pixel(*point)), expected_colour(layout, point[0])) + return f"get_pixel agrees at {left} and {right}" + check("get_pixel() returns the colour that is on screen", _get_pixel) + + def _full_screenshot() -> str: + frame = screenshot() + _assert_eq((frame.shape[1], frame.shape[0]), (layout.width, layout.height)) + for point in (left, right): + blue, green, red = frame[point[1], point[0]] + _assert_eq((int(red), int(green), int(blue)), + expected_colour(layout, point[0])) + return f"{frame.shape[1]}x{frame.shape[0]}, BGR order confirmed" + check("screenshot() is the whole screen, in BGR", _full_screenshot) + + def _region_screenshot() -> str: + # A rectangle wholly inside the right-hand colour: if the region were + # ignored, or applied from the wrong origin, this would pick up the + # left-hand colour instead of failing silently on a uniform screen. + x1, y1 = right + x2, y2 = x1 + 60, y1 + 40 + frame = screenshot(screen_region=[x1, y1, x2, y2]) + _assert_eq((frame.shape[1], frame.shape[0]), (x2 - x1, y2 - y1)) + blue, green, red = frame[2, 2] + _assert_eq((int(red), int(green), int(blue)), expected_colour(layout, x1)) + return f"region [{x1},{y1},{x2},{y2}] cropped and coloured correctly" + check("screenshot(screen_region=...) crops from the right origin", + _region_screenshot) + + +def check_pointer(layout: Layout) -> None: + """Where the pointer is put must be where the server says it is.""" + from je_auto_control import get_mouse_position, set_mouse_position + + def _pointer_at(point: Tuple[int, int]) -> str: + set_mouse_position(*point) + shell = _run(["xdotool", "getmouselocation", "--shell"]) + values = dict( + line.split("=", 1) for line in shell.splitlines() if "=" in line) + server = (int(values["X"]), int(values["Y"])) + _assert_eq(server, point) + _assert_eq(tuple(get_mouse_position()), point) + return f"{point} read back from the server and from get_mouse_position" + + first = layout.monitors[0].inside(120, 120) + check("set_mouse_position lands where the server says", + lambda: _pointer_at(first)) + + if layout.multi: + second = layout.monitors[-1].inside(120, 120) + check("a point on the second monitor lands on the second monitor", + lambda: _pointer_at(second)) + else: + note("one monitor in this layout, so there is no second one to reach; " + "the two-monitor pass covers that.") + + +def check_button_events(tester: EventTester) -> None: + """A click must arrive at a real client, as real input, where aimed.""" + from je_auto_control import click_mouse, press_mouse, release_mouse, set_mouse_position + + target = tester.centre() + + def _click() -> str: + set_mouse_position(*target) + tester.flush() + click_mouse("mouse_left") + press = tester.collect("ButtonPress")[0] + release = tester.collect("ButtonRelease")[0] + _assert_eq(press["button"], 1) + _assert_eq(release["button"], 1) + _assert_eq(press["root"], target) + return f"button 1 press+release at {target}" + check("click_mouse reaches a real client at the aimed point", _click) + + def _not_synthetic() -> str: + set_mouse_position(*target) + tester.flush() + click_mouse("mouse_left") + press = tester.collect("ButtonPress")[0] + # XSendEvent traffic arrives with synthetic YES and is discarded by + # most toolkits. XTest goes through the server, so anything that + # started reporting YES here would mean the backend had quietly + # stopped driving real input. + return _assert_eq(press["synthetic"], "NO") + check("injected input is real server input, not a sent event", + _not_synthetic) + + def _split() -> str: + set_mouse_position(*target) + tester.flush() + press_mouse("mouse_right") + press = tester.collect("ButtonPress")[0] + _assert_eq(press["button"], 3) + _assert_true(not tester.buffered("ButtonRelease"), + "a held button must not release itself") + release_mouse("mouse_right") + _assert_eq(tester.collect("ButtonRelease")[0]["button"], 3) + return "button 3 held, then released, as two events" + check("press_mouse holds the button until release_mouse", _split) + + +def check_scroll_events(tester: EventTester) -> None: + """X11 encodes scrolling as buttons 4-7; the direction must pick correctly. + + ``scroll_direction`` names the direction a *positive* count scrolls in, + and a negative count reverses it — the same rule on all three platforms + since the maintainer settled it. X11 used to discard the sign, so code + written against the Windows convention scrolled the opposite way here, + silently. The last check is that regression: it fails if the sign ever + stops being read. + """ + from je_auto_control import mouse_scroll, set_mouse_position + + target = tester.centre() + + def _scroll(value: int, direction: str, expected_button: int) -> str: + set_mouse_position(*target) + tester.flush() + mouse_scroll(value, scroll_direction=direction) + presses = tester.collect("ButtonPress", abs(value)) + _assert_eq({event["button"] for event in presses}, {expected_button}) + return (f"{direction} x{abs(value)} arrived as button " + f"{expected_button}") + + check("scroll_direction='scroll_up' arrives as button 4", + lambda: _scroll(2, "scroll_up", 4)) + check("scroll_direction='scroll_down' arrives as button 5", + lambda: _scroll(2, "scroll_down", 5)) + check("scroll_direction='scroll_left' arrives as button 6", + lambda: _scroll(1, "scroll_left", 6)) + check("scroll_direction='scroll_right' arrives as button 7", + lambda: _scroll(1, "scroll_right", 7)) + + def _negative_reverses(direction: str, expected_button: int) -> str: + set_mouse_position(*target) + tester.flush() + # Portable code written against the Windows sign convention lands + # here, and now gets the direction it asked for. + mouse_scroll(-2, scroll_direction=direction) + presses = tester.collect("ButtonPress", 2) + _assert_eq({event["button"] for event in presses}, + {expected_button}) + return (f"-2 with {direction} arrived as button " + f"{expected_button}") + + check("a negative count reverses scroll_down into button 4", + lambda: _negative_reverses("scroll_down", 4)) + check("a negative count reverses scroll_up into button 5", + lambda: _negative_reverses("scroll_up", 5)) + check("a negative count reverses scroll_left into button 7", + lambda: _negative_reverses("scroll_left", 7)) + check("a negative count reverses scroll_right into button 6", + lambda: _negative_reverses("scroll_right", 6)) + + +def check_key_events(tester: EventTester) -> None: + """A keystroke must arrive with the keycode the table promised.""" + from je_auto_control import ( + keyboard_keys_table, press_keyboard_key, release_keyboard_key, write, + ) + + def _single() -> str: + tester.flush() + press_keyboard_key("a") + press = tester.collect("KeyPress")[0] + release_keyboard_key("a") + release = tester.collect("KeyRelease")[0] + # The table holds real X keycodes (keysym_to_keycode at import), so + # this checks the whole name -> keysym -> keycode -> wire path. + _assert_eq(press["keycode"], keyboard_keys_table["a"]) + _assert_eq(release["keycode"], keyboard_keys_table["a"]) + _assert_eq(press["keysym"], "a") + return f"'a' arrived as keycode {press['keycode']}" + check("press/release_keyboard_key deliver the mapped keycode", _single) + + def _string() -> str: + tester.flush() + write("xyz") + presses = tester.collect("KeyPress", 3) + _assert_eq([event["keysym"] for event in presses], ["x", "y", "z"]) + return "write('xyz') arrived as x, y, z in order" + check("write() delivers each character in order", _string) + + +def main() -> int: + print("=" * 72) + print("AutoControl X11 verification — real X server, real clients") + print("=" * 72) + + layout = read_layout() + report_environment(layout) + paint_root(layout) + + check_backend_selection() + check_geometry(layout) + check_pixels(layout) + check_pointer(layout) + + with EventTester() as tester: + note(f"event tester window {tester.window_id} at {tester.rect}") + check_button_events(tester) + check_scroll_events(tester) + check_key_events(tester) + + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("A negative layout origin — X11's root window is the union of every") + note(" monitor and always starts at (0, 0). The Wayland job's second") + note(" layout has no analogue here; this is a protocol difference.") + note("The uinput input path — it needs /dev/uinput, which is the host's") + note(" to grant. docker/ydotool_verify.py reads those events back off") + note(" the kernel device instead.") + + return summarise() + + +def summarise() -> int: + """Print the tally and return the number of failed checks. + + Shared with ``x11_window_verify.py``, which runs in the same session and + reports through the same harness. + """ + failed = [name for name, ok, _ in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/x11_window_verify.py b/docker/x11_window_verify.py new file mode 100644 index 00000000..c3d565cd --- /dev/null +++ b/docker/x11_window_verify.py @@ -0,0 +1,372 @@ +"""Verify cross-platform window management against a real window manager. + +Window management was Windows-only for the project's whole life: the facade +branched on ``sys.platform`` and raised everywhere else, so 23 ``AC_*`` +commands and their MCP tools were dead on Linux and macOS. The X11 backend +that replaces that branch talks EWMH to the window manager, and EWMH is +exactly the kind of contract a mock cannot check — what matters is whether +*openbox* agrees, not whether python-Xlib was called with the right atoms. + +So this runs against the real session ``entrypoint-x11.sh`` brings up, drives +the public facade (``je_auto_control.list_windows`` and friends, not the +backend), and takes ground truth from ``xdotool`` and ``xprop`` — the window +manager answering for itself. + +The subject is a real ``xterm``, launched with a title nothing else uses. + +It shares the harness in ``x11_verify`` rather than duplicating it: same +session, same tally, one exit status. +""" +from __future__ import annotations + +import os +import subprocess # nosec B404 # reason: argv lists of fixed tool names, no shell +import sys +import time +from typing import Any, Optional, Tuple + +from x11_verify import ( + EventTester, _assert_eq, _assert_true, _run, check, note, summarise, +) + +#: Unique enough that `xdotool search` cannot match anything else in the image. +WINDOW_TITLE = "autocontrol-window-verify" + +#: How long the window manager is given to act on an EWMH request. These are +#: asynchronous by design — the client message goes to the root window and the +#: window manager gets to it when it gets to it. +WM_TIMEOUT = 5.0 + + +class Xterm: + """A real X client owning a real, managed, titled window.""" + + def __init__(self, title: str = WINDOW_TITLE) -> None: + self.title = title + self._process: Optional[subprocess.Popen] = None + self.window_id = 0 + + def __enter__(self) -> "Xterm": + # -hold keeps the window up after the shell exits, so the subject + # cannot vanish mid-check for a reason unrelated to what is under test. + self._process = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["xterm", "-title", self.title, "-geometry", "40x10+150+150", + "-hold", "-e", "true"], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + self.window_id = self._await_window() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is not None and self._process.poll() is None: + self._process.terminate() + try: + self._process.wait(timeout=5) + except subprocess.TimeoutExpired: + self._process.kill() + + def _await_window(self) -> int: + deadline = time.monotonic() + 20.0 + while time.monotonic() < deadline: + try: + found = _run(["xdotool", "search", "--name", self.title]) + except subprocess.CalledProcessError: + found = "" + ids = [line for line in found.split() if line.isdigit()] + if ids: + # Wait for the window manager to finish managing it: an EWMH + # request against an unmanaged window is simply dropped. + self._await_managed(int(ids[-1])) + return int(ids[-1]) + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + f"xterm exited {self._process.returncode} without a window") + time.sleep(0.1) + raise RuntimeError("xterm never mapped a window") + + @staticmethod + def _await_managed(window_id: int) -> None: + deadline = time.monotonic() + 10.0 + while time.monotonic() < deadline: + listed = _run(["xprop", "-root", "_NET_CLIENT_LIST"]) + if f"{window_id:#x}" in listed.lower().replace("0x", "0x"): + return + time.sleep(0.1) + raise RuntimeError(f"the window manager never took window {window_id}") + + @property + def pid(self) -> int: + return self._process.pid if self._process is not None else 0 + + +def _active_window() -> int: + """The window the server says is active, via xdotool.""" + return int(_run(["xdotool", "getactivewindow"]).strip() or 0) + + +#: How far a size may differ from what was asked before it counts as wrong. +#: xterm resizes in whole character cells, so it rounds a pixel size down to +#: the nearest cell — a property of the subject, not of the backend. A real +#: failure to resize is out by hundreds of pixels, not by one cell. +SIZE_TOLERANCE = 24 + + +def _client_geometry(window_id: int) -> Tuple[int, int, int, int]: + """The *client* ``(x, y, width, height)``, as xwininfo reports it.""" + values: dict = {} + for line in _run(["xwininfo", "-id", str(window_id)]).splitlines(): + stripped = line.strip() + for label, key in (("Absolute upper-left X:", "X"), + ("Absolute upper-left Y:", "Y"), + ("Width:", "WIDTH"), ("Height:", "HEIGHT")): + if stripped.startswith(label): + values[key] = int(stripped[len(label):].strip()) + return (values.get("X", 0), values.get("Y", 0), + values.get("WIDTH", 0), values.get("HEIGHT", 0)) + + +def _frame_extents(window_id: int) -> Tuple[int, int, int, int]: + """``(left, right, top, bottom)`` decoration thickness, or zeroes.""" + try: + raw = _run(["xprop", "-id", str(window_id), "_NET_FRAME_EXTENTS"]) + except subprocess.CalledProcessError: + return (0, 0, 0, 0) + if "not found" in raw or "=" not in raw: + return (0, 0, 0, 0) + parts = [piece.strip() for piece in raw.split("=", 1)[1].split(",")] + if len(parts) < 4 or not all(piece.isdigit() for piece in parts[:4]): + return (0, 0, 0, 0) + return tuple(int(piece) for piece in parts[:4]) # type: ignore[return-value] + + +def _geometry(window_id: int) -> Tuple[int, int, int, int]: + """The *frame* ``(x, y, width, height)``: the client plus its decorations. + + Win32's ``GetWindowRect`` returns the frame, every caller in this project + is written against that, and the X11 backend reports the frame to match — + so the frame is what has to be checked. + + Neither tool reports it directly. ``xwininfo -id`` gives the client + rectangle (``-frame`` changes how a window is *picked*, not what is + reported, so it makes no difference with ``-id``), and ``xprop``'s + ``_NET_FRAME_EXTENTS`` gives the thickness the window manager added + around it. Adding them is both the definition of the frame and a check + that the backend's own walk up to it found the same window. + """ + x, y, width, height = _client_geometry(window_id) + left, right, top, bottom = _frame_extents(window_id) + return (x - left, y - top, width + left + right, height + top + bottom) + + +def _wait_until(predicate, timeout: float = WM_TIMEOUT) -> bool: + """Poll a predicate — every EWMH request is asynchronous.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return True + time.sleep(0.1) + return False + + +# --- checks ---------------------------------------------------------------- + + +def check_backend_selection() -> None: + def _selected() -> str: + from je_auto_control.wrapper.window_backends import get_backend + + backend = get_backend() + _assert_true(backend.available, + f"backend {backend.name!r} reports unavailable") + return _assert_eq(backend.name, "x11-ewmh") + check("the window seam selects the X11 backend", _selected) + + +def check_listing(xterm: Xterm) -> None: + import je_auto_control as ac + + def _lists() -> str: + titles = dict(ac.list_windows()) + _assert_true(xterm.window_id in titles, + f"window {xterm.window_id} missing from {sorted(titles)}") + return _assert_eq(titles[xterm.window_id], xterm.title) + check("list_windows sees a real managed window with its title", _lists) + + def _finds() -> str: + hit = ac.find_window(xterm.title) + _assert_true(hit is not None, "find_window returned nothing") + return _assert_eq(hit[0], xterm.window_id) + check("find_window resolves the title substring", _finds) + + def _rect() -> str: + rect = ac.window_rect(xterm.title) + _assert_true(rect is not None, "window_rect returned None") + left, top, right, bottom = rect + x, y, width, height = _geometry(xterm.window_id) + # Both sides are the frame: the rectangle a user sees and drags, and + # the one Win32 reports. See _geometry for why the client rectangle + # is the wrong comparison. + _assert_eq((left, top), (x, y)) + _assert_eq((right - left, bottom - top), (width, height)) + return f"({left}, {top}) {right - left}x{bottom - top}" + check("window_rect matches what the server reports", _rect) + + def _pid() -> str: + # _NET_WM_PID is what the client advertises; xterm advertises its own. + return _assert_eq(ac.window_process_id(xterm.title), xterm.pid) + check("window_process_id is the owning process", _pid) + + def _by_pid() -> str: + owned = dict(ac.windows_for_process_id(xterm.pid)) + _assert_true(xterm.window_id in owned, + f"window missing from the pid's windows: {sorted(owned)}") + return f"{len(owned)} window(s) for pid {xterm.pid}" + check("windows_for_process_id addresses windows by owner", _by_pid) + + +def check_focus(xterm: Xterm) -> None: + import je_auto_control as ac + + def _focus() -> str: + ac.focus_window(xterm.title) + _assert_true(_wait_until(lambda: _active_window() == xterm.window_id), + f"the window manager never made {xterm.window_id} active") + return _assert_eq(ac.foreground_window()[0], xterm.window_id) + check("focus_window makes the window active, and foreground_window agrees", + _focus) + + def _fg_pid() -> str: + return _assert_eq(ac.foreground_window_process_id(), xterm.pid) + check("foreground_window_process_id names the owning process", _fg_pid) + + +def check_move(xterm: Xterm) -> None: + import je_auto_control as ac + + def _move() -> str: + _assert_true(ac.move_window_by_title(xterm.title, 300, 220, 500, 300), + "move_window_by_title reported failure") + _assert_true( + _wait_until(lambda: _geometry(xterm.window_id)[:2] == (300, 220)), + f"window stayed at {_geometry(xterm.window_id)}") + x, y, width, height = _geometry(xterm.window_id) + _assert_eq((x, y), (300, 220)) + _assert_true(abs(width - 500) <= SIZE_TOLERANCE + and abs(height - 300) <= SIZE_TOLERANCE, + f"asked 500x300, frame is {width}x{height}") + return f"frame at ({x}, {y}), {width}x{height}" + check("move_window_by_title moves and resizes for real", _move) + + def _move_keeps_size() -> str: + # Omitting width/height must keep the current size rather than + # collapsing the window, which is what makes a plain reposition usable + # without looking the dimensions up first. + before = _geometry(xterm.window_id) + _assert_true(ac.move_window_by_title(xterm.title, 120, 90), + "move_window_by_title reported failure") + _assert_true( + _wait_until(lambda: _geometry(xterm.window_id)[:2] == (120, 90)), + f"window stayed at {_geometry(xterm.window_id)}") + return _assert_eq(_geometry(xterm.window_id)[2:], before[2:]) + check("moving without a size keeps the size", _move_keeps_size) + + +def check_minimize(xterm: Xterm) -> None: + import je_auto_control as ac + from je_auto_control.wrapper.window_backends import get_backend + + backend = get_backend() + + def _minimize() -> str: + _assert_true(ac.minimize_window_by_title(xterm.title), + "minimize_window_by_title reported failure") + _assert_true( + _wait_until(lambda: backend.is_minimized(xterm.window_id)), + "the window never reported itself hidden") + return "iconified, and _NET_WM_STATE_HIDDEN says so" + check("minimize_window_by_title iconifies the window", _minimize) + + def _restore() -> str: + # focus_window restores before raising, which is the path a caller + # takes to get back to a minimised window without knowing it was one. + ac.focus_window(xterm.title) + _assert_true( + _wait_until(lambda: not backend.is_minimized(xterm.window_id)), + "the window stayed hidden") + return "restored by focus_window" + check("focus_window restores a minimised window", _restore) + + +def check_post_is_synthetic(tester: EventTester) -> None: + """Posting to an unfocused window works, and is honest about what it is.""" + import je_auto_control as ac + + def _post_click() -> str: + tester.flush() + _assert_true( + ac.post_click_to_window("Event Tester", "left", 20, 20), + "post_click_to_window reported failure") + press = tester.collect("ButtonPress")[0] + _assert_eq(press["button"], 1) + # This is the point of the check. XSendEvent traffic arrives flagged + # synthetic and GTK and Qt discard it by design, so post_* is + # best-effort by nature — the same caveat Win32's PostMessage carries. + # Asserting it here stops anyone reading post_* as real input. + return _assert_eq(press["synthetic"], "YES") + check("post_click_to_window arrives, flagged synthetic", _post_click) + + def _post_key() -> str: + from je_auto_control import keyboard_keys_table + + tester.flush() + _assert_true(ac.post_key_to_window("Event Tester", "b"), + "post_key_to_window reported failure") + press = tester.collect("KeyPress")[0] + _assert_eq(press["keycode"], keyboard_keys_table["b"]) + return _assert_eq(press["synthetic"], "YES") + check("post_key_to_window arrives, flagged synthetic", _post_key) + + +def check_close(xterm: Xterm) -> None: + import je_auto_control as ac + + def _close() -> str: + _assert_true(ac.close_window_by_title(xterm.title), + "close_window_by_title reported failure") + _assert_true( + _wait_until(lambda: ac.find_window(xterm.title) is None, 10.0), + "the window is still listed") + return "gone from the client list" + check("close_window_by_title really closes the window", _close) + + +def main() -> int: + print("=" * 72) + print("AutoControl window management — real X11 window manager") + print("=" * 72) + note(f"DISPLAY={os.environ.get('DISPLAY')!r}") + + check_backend_selection() + + with Xterm() as xterm: + note(f"subject: xterm window {xterm.window_id} pid {xterm.pid}") + check_listing(xterm) + check_focus(xterm) + check_move(xterm) + check_minimize(xterm) + check_close(xterm) + + with EventTester() as tester: + check_post_is_synthetic(tester) + + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("Wayland window management — the protocol does not let a client") + note(" enumerate or move another application's windows at all, so there") + note(" is nothing to implement, let alone verify. The backend selector") + note(" says so rather than looking broken.") + + return summarise() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index 7a18a802..59358ebe 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -6,11 +6,12 @@ without a compatibility window. | Capability | Status | Windows | Linux X11 | Linux Wayland | macOS | |---|---|---:|---:|---:|---:| -| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb | CI/sway + libeis | implementation | +| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb + xev | CI/sway + libeis | implementation | | JSON executor and variables | stable | CI | CI | CI | platform-neutral | | Image and anchor locators | beta | CI | CI | implementation | implementation | -| Accessibility locator | beta | CI | backend tests | unavailable | backend tests | -| Recorder | beta | CI | implementation | unavailable | unavailable | +| Accessibility locator | beta | CI | CI/AT-SPI | CI/AT-SPI | CI (tree read) | +| Window management | beta | CI | CI/openbox | unavailable | CI (listing) | +| Recorder | beta | CI | implementation | unavailable | CI | | Reports, trace, failure bundle | stable | CI | CI | CI | platform-neutral | | REST, MCP, scheduler | beta | CI | CI | CI | platform-neutral | | Remote desktop / WebRTC | beta | tests | tests | tests | tests | @@ -25,6 +26,31 @@ Hardware-backed results and known limitations should be attached to releases. Linux Wayland is split: **capture is exercised by CI against a real compositor; input is exercised by CI against a real EI peer and a real portal.** +Linux X11 said `CI/Xvfb` for a long time on the strength of a job that +imported the package under `xvfb-run` and generated two lines of code. Nothing +moved a pointer, and every X11 assertion in the suite is made against a mock +of `python-Xlib`, so the questions that matter went unanswered: does an +injected event reach a client at all, does it arrive as *real* input, and is a +captured pixel the pixel on screen. The `x11-verification` job answers them +against a real Xvfb server with a real window manager, and it takes its ground +truth from other codebases than the one under test — `xev`, a real X client +that prints every event delivered to its window; ImageMagick's `import`, an +independent grabber, against a root window painted two asymmetric colours so a +wrong rectangle cannot look right; and `xdotool` / `xdpyinfo`, the server +answering for itself. It runs twice, over one monitor and then two. + +One assertion there is worth naming, because losing it would be silent: +XTest-injected events must arrive with `synthetic NO`. `XSendEvent` traffic +arrives with `synthetic YES` and is discarded by most toolkits, so a backend +that quietly stopped driving real input would still pass every check that only +counted events. + +There is deliberately no negative-origin X11 pass. On X11 the root window is +the union of every monitor and always begins at `(0, 0)`: a monitor placed to +the left shifts the others right rather than moving the origin. The Wayland +job's second layout has no analogue here — a protocol difference, not an +untested case. + Screen capture runs through the compositor's own tool (`grim` on wlroots, `gnome-screenshot` on GNOME, `spectacle` on KDE), falling back to `xdg-desktop-portal` over the session bus, instead of the X11-only Pillow/mss @@ -39,6 +65,141 @@ primary one, where the compositor's plane starts at a negative coordinate and a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of that monitor. +The recorder row said `unavailable` for macOS while the code for one sat in +the tree unused, and the reason was real rather than an oversight: the old +listener built an `NSApplication` at import time and stopped recording with +`AppHelper.runEventLoop()`, a loop that never returns to its caller. Wiring +that up would have put both on the path of `import je_auto_control`, so +`wrapper/_platform_osx.py` set `recorder = None` instead. + +Neither was necessary. A `CGEventTap` needs a **run loop**, not an +application: the tap is created on a dedicated thread, its source is added to +that thread's run loop, and the loop is pumped in short `CFRunLoopRunInMode` +slices so a stop flag is honoured between them — the same shape the macOS +hotkey backend already used. The tap is listen-only, because a recorder that +consumed events would swallow the input it is recording. This row says `CI` +because the `macos-capabilities` job records a real session on a real window +server: it posts a move, a click and a keypress through the public API and +asserts they come back out of the tap with the release and the coordinates +they were posted at. + +Two defects were in that code and only a Mac could show them. Coordinates came +from `NSEvent.mouseLocation()`, whose origin is the **bottom-left** of the +display, while every replay posts into the top-left space `osx_mouse` uses — +so a click recorded near the top of the screen replayed near the bottom. And +modifiers were not recorded at all: macOS sends no key-down for Shift, +Control, Option or Command, only a `flagsChanged` event carrying the new flag +set, so a recording could not say a modifier was held across what followed. + +The table has four columns because those are the four desktops with their own +backend, not because they are the only supported systems. Two more axes now +have CI behind them: + +**The BSDs.** `platform_wrapper` refused to start on anything that was not +win32/cygwin/msys, darwin or linux/linux2, and every X11 backend module +carried its own copy of the same Linux-only guard — so a FreeBSD, OpenBSD or +NetBSD desktop, which runs the same X server and the same `python-Xlib` as +Linux, could not import the package at all. `python-Xlib` was pinned to +`platform_system=='Linux'` too, so even relaxing the guards would have left +the backend without its one dependency. The guards now ask +`utils/platform_id.is_x11_unix()` — "is this an X11 unix", which is the +question they were always trying to ask — and the `freebsd` job boots a real +FreeBSD 14 VM to run that decision on a system that is genuinely one. + +For a while it checked that decision and nothing else, for a measured reason: +importing anything under `je_auto_control` ran the package facade, which +imported OpenCV and cryptography at module scope, and neither publishes a +FreeBSD wheel — installing them from ports pulled a dependency tree that had +not finished after fifty minutes. + +That was the wrong thing to work around. Moving a pointer needs neither +package, so the facade stopped importing them (and NumPy, Pillow and +`je_open_cv`) at module scope; they belong to the functions that use them. +What the VM installs now is `python-Xlib`, `defusedxml` and an X server, all +of which take seconds, and `test/verify/freebsd_verify.py` drives the whole +backend on it. The reads come off the X server rather than out of this +codebase: `query_pointer` for the cursor and the button mask, `query_keymap` +for whether an injected key really went down, and a mapped X window that has +asked for button events for the wheel — which is what caught `mouse_scroll` +matching a literal `["linux", "linux2"]` and therefore doing nothing at all, +silently, on every BSD. + +**arm64.** `macos-14` was already arm64; `ubuntu-22.04-arm` joins the +smoke matrix and passes. `windows-11-arm` was tried and removed, on +measurement rather than assumption, and **two** dependencies are why: +**opencv-python publishes no `win_arm64` wheel** in any version, so pip falls +back to building from source and CMake cannot configure for ARM64; and +**cryptography stopped publishing one after 46.0.3**, while this project's +floor is `>=48.0.1` — a security floor (GHSA-537c-gmf6-5ccf) that cannot be +lowered to reach a wheel. Neither is a CI problem to work around: the package +genuinely cannot be installed on Windows arm64 today. `Progress.md` records +both, alongside a `pip --dry-run --platform win_arm64` command that re-checks +them in seconds without an arm64 machine. + +The accessibility row said `backend tests` for Linux X11 and meant nothing by +it: there was no Linux backend at all, and `_build_backend()` fell straight +through to the null one. There is one now, over **AT-SPI2** — which is a D-Bus +protocol rather than a library, and that is what makes it reachable without a +new dependency. The usual bindings (`pyatspi`, `gi.repository.Atspi`) are +distribution packages built against the system introspection data and cannot +be installed into a virtual environment, so depending on them would be +depending on something most users cannot get. The client written for the XDG +portal handshake already spoke enough D-Bus. + +It is exercised by the `x11-verification` job against a real accessibility bus +and a real GTK application (`zenity`), because neither half can be mocked +usefully: the bus is D-Bus-activated rather than started by hand, an +application only appears on it if its toolkit bridge loaded, and the tree's +shape is the toolkit's business. + +That job immediately found a gap in the shared D-Bus client: **it could not +demarshal signed integers.** The portal handshake never needed one, and +AT-SPI reports a component's extents as four *signed* values — because a +window on a monitor left of or above the primary one is at a negative +coordinate. Without it the backend could read a tree but not where anything +was. The client now handles the whole fixed-width numeric set except `h` +(UNIX_FD), which stays an error on purpose: it is an index into a descriptor +array this client does not receive, so returning it would hand a caller a +number that addresses nothing. + +Because AT-SPI is a bus rather than a display protocol, this row is `CI/AT-SPI` +for **both** Linux entries: a Wayland session runs the same accessibility bus, +so this is the one capability where Wayland is not the restricted case. + +Window management had no row here at all until it had more than one platform. +It was Windows-only for the project's whole life — the facade branched on +`sys.platform` and raised everywhere else — which left 23 `AC_*` commands and +their MCP tools dead on macOS and Linux. It now goes through a backend seam: +Win32, EWMH over python-Xlib on X11, and Quartz plus the accessibility API on +macOS. + +The X11 half is exercised by the `x11-verification` job against a real +`openbox` session, driving the public facade and taking ground truth from +`xwininfo` and `xprop`. Two things only a real window manager could have +shown up came out of it, and both were wrong in the first implementation: + +* **The rectangle is the frame, not the client.** Win32's `GetWindowRect` + returns the frame — border and title bar included — and every caller here is + written against that. Reporting the client area was off by the decorations + on X11 alone, silently, and by a different amount per window manager. +* **A move must go through `_NET_MOVERESIZE_WINDOW`.** Under a reparenting + window manager a client's own x/y are relative to its frame, so a direct + `ConfigureWindow` asks for a position in the wrong coordinate space. + Measured against openbox, asking for (300, 220) that way landed the window + at (302, 260). + +`post_key_to_window` and `post_click_to_window` work on X11 and are asserted +to arrive *flagged synthetic*, because that is what they are: `XSendEvent` +traffic, which GTK and Qt discard by design. They are the X11 counterpart of +Win32's `PostMessage`, which carries the same best-effort caveat. macOS has no +equivalent at all — an event goes to whatever has focus — so the backend +refuses rather than reporting a success that went somewhere else. + +Wayland is `unavailable` and will stay that way: the protocol does not let a +client enumerate or move another application's windows. That is a design +decision upstream, not a gap here, and the backend selector says so instead of +looking broken. + One cross-platform difference falls out of the same job, and it is not one this project can fix: **a Wayland capture may contain the mouse cursor.** No capture here passes `grim -c`, so none of them asks for the pointer — but wlroots draws diff --git a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst index 13192771..44c2e332 100644 --- a/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Eng/doc/mcp_server/mcp_server_doc.rst @@ -253,6 +253,38 @@ box), start the same dispatcher behind HTTP: Bearer token can also come from ``JE_AUTOCONTROL_MCP_TOKEN``. +Sessions +======== + +``initialize`` mints a session and returns it in an +``Mcp-Session-Id`` response header. Echo that header on every later +request and the server keeps one scope for you — the capabilities +you advertised, and the slots your in-flight calls occupy — no +matter how many TCP connections you use. Without it each request is +scoped to its own connection, which is all the transport used to +offer. + +- ``GET /mcp`` with ``Accept: text/event-stream`` and a valid + ``Mcp-Session-Id`` opens the **standing server-to-client stream**. + Server-initiated traffic that is not a reply to a specific request + travels down it: progress notifications, and the + ``elicitation/create`` behind the confirmation gate below. One + stream per session; a second ``GET`` gets 409. The stream carries + an SSE comment as a heartbeat so an abandoned socket surfaces as a + write error rather than a parked thread. +- Answer a server request by ``POST``\ ing an ordinary JSON-RPC + response — with the session header — on any connection. It is + matched to the waiting call by id, and acked with 202. +- ``DELETE /mcp`` with the header terminates the session and + releases everything scoped to it. Without the header it is + accepted as a no-op, so sessionless clients keep working. +- An unknown or expired id is refused with **404**, not served under + a fresh scope: the client holds state the server does not, and + needs to know to re-initialize. +- Sessions are bounded. They are swept after ten minutes untouched + (a standing stream keeps its own session fresh), and the registry + evicts the least recently seen once it holds 128. + Read-only / safe mode ===================== @@ -283,6 +315,32 @@ error to the model without running the action. Requires the client to advertise the ``elicitation`` capability — older clients fall through with a logged warning. +The prompt is a question the server asks *between* receiving a call +and answering it, so it needs a channel the client is listening on +at that moment. What that means per transport: + +- **stdio** — always available; the client is on the other end of + the same pipe. +- **HTTP** — available to a client that echoes ``Mcp-Session-Id`` + (see `Sessions`_) and gives the server somewhere to send the + question. Either channel does: the standing ``GET`` stream, or an + SSE ``POST``, whose own response stream carries the + ``elicitation/create`` before the result. Either way the answer + comes back as a separate ``POST`` — the client is busy reading the + stream it asked on. + +.. warning:: + + A client that ignores ``Mcp-Session-Id``, or that only ever sends + plain JSON ``POST``\ s with no stream open, has given the server + nowhere to ask — so + destructive tools run **unprompted**, exactly as they do for a + stdio client that never advertised ``elicitation``. That fallback + is logged, but it is a fallback: on an HTTP-exposed server treat + the bearer token, the ``127.0.0.1`` bind and + ``JE_AUTOCONTROL_MCP_READONLY`` as the controls that do not + depend on the client behaving. + Audit log ========= diff --git a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst index d1d1e93a..a50740a8 100644 --- a/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst +++ b/docs/source/Zh/doc/mcp_server/mcp_server_doc.rst @@ -240,6 +240,31 @@ HTTP 傳輸(含 SSE / Auth / TLS) Bearer token 也可從 ``JE_AUTOCONTROL_MCP_TOKEN`` 環境變數讀取。 +Session +======= + +``initialize`` 會產生一個 session,並用 ``Mcp-Session-Id`` 回應標頭 +交給 client。之後每個請求都帶上這個標頭,伺服器就會把它們視為同一個 +scope——包含你在 ``initialize`` 聲明的能力,以及進行中呼叫佔用的槽位 +——不管你開了幾條 TCP 連線。不帶這個標頭時,每個請求只以自己的連線 +為範圍,也就是這條傳輸從前唯一的行為。 + +- ``GET /mcp`` 帶 ``Accept: text/event-stream`` 與有效的 + ``Mcp-Session-Id``,會開啟**常駐的 server→client 串流**。凡是不 + 屬於某個特定請求之回覆的伺服器主動訊息都走這條:進度通知,以及 + 下面確認關卡要送的 ``elicitation/create``。一個 session 只能有一 + 條串流,第二個 ``GET`` 會收到 409。串流會定期送出 SSE 註解當作 + 心跳,好讓斷掉的 socket 變成一個寫入錯誤,而不是一條卡住的執行緒。 +- 要回答伺服器的請求,就在任何一條連線上 ``POST`` 一個普通的 + JSON-RPC response(同樣帶 session 標頭)。伺服器會依 id 對回正在 + 等待的那個呼叫,並以 202 回覆。 +- ``DELETE /mcp`` 帶標頭會終止 session,並釋放掛在它底下的所有狀態; + 不帶標頭時照舊接受,不影響沒有 session 概念的 client。 +- 未知或已過期的 id 一律回 **404**,不會改用一個新的 scope 服務它: + client 手上有伺服器沒有的狀態,它需要知道自己該重新 initialize。 +- session 有上下界。十分鐘沒被碰過就會被掃掉(常駐串流會讓自己的 + session 保持新鮮),而註冊表滿 128 個時,最久沒動的那個會被淘汰。 + 唯讀 / 安全模式 =============== @@ -268,6 +293,26 @@ Bearer token 也可從 ``JE_AUTOCONTROL_MCP_TOKEN`` 環境變數讀取。 使用者拒絕時模型會收到乾淨的錯誤,不會執行動作。需要 client 自己 聲明 ``elicitation`` 能力;舊 client 會留下 warning log 後繼續執行。 +這個提示是伺服器在「收到呼叫」與「回答呼叫」**之間**問出去的問題, +所以它需要一條 client 當下正在聽的通道。各傳輸的情況: + +- **stdio** — 一定有;client 就在同一條 pipe 的另一端。 +- **HTTP** — 只要 client 回送 ``Mcp-Session-Id``(見 `Session`_), + 並且給伺服器一條送問題的通道就有。兩種通道都算:常駐的 ``GET`` + 串流,或是一個 SSE ``POST``——它自己的回應串流會在結果之前先送出 + ``elicitation/create``。兩種情況下,答案都要用另一個 ``POST`` 送 + 回來,因為 client 正忙著讀它問過去的那條串流。 + +.. warning:: + + 如果 client 不回送 ``Mcp-Session-Id``,或是自始至終只送普通的 + JSON ``POST``、一條串流都沒開,伺服器就沒有地方可問——這時 + destructive 工具會**不經詢問直接執行**,和一個從未聲明 + ``elicitation`` 的 stdio client 完全一樣。 + 這條退路會留下 log,但它終究是退路:對外開放的 HTTP 服務,請把 + bearer token、綁定 ``127.0.0.1`` 與 ``JE_AUTOCONTROL_MCP_READONLY`` + 當成真正的控制手段,它們不需要 client 配合。 + 稽核 Log ======== diff --git a/je_auto_control/cli.py b/je_auto_control/cli.py index 6864efff..ff46edca 100644 --- a/je_auto_control/cli.py +++ b/je_auto_control/cli.py @@ -116,9 +116,6 @@ def cmd_fmt(args: argparse.Namespace) -> int: def cmd_record(args: argparse.Namespace) -> int: """Record mouse/keyboard input into an action file.""" - if sys.platform == "darwin": - sys.stderr.write("record is not supported on macOS\n") - return 1 from je_auto_control.wrapper.auto_control_record import record_to_json stop_event = threading.Event() if args.duration is None: diff --git a/je_auto_control/linux_wayland/_dbus_client.py b/je_auto_control/linux_wayland/_dbus_client.py index 6a9d91a5..000f10cc 100644 --- a/je_auto_control/linux_wayland/_dbus_client.py +++ b/je_auto_control/linux_wayland/_dbus_client.py @@ -1,625 +1,24 @@ -"""A minimal D-Bus session-bus client, in the standard library alone. +"""The D-Bus client this package uses, re-exported from its new home. -This exists because of one property of the XDG portal protocol: a portal call -returns only a *request handle*, and the answer arrives later as a ``Response`` -signal — **directed at the unique bus name that made the call**. The bus routes -a directed message to its destination and nowhere else, so a listener on a -second connection never receives it, whatever match rules it adds. +The implementation moved to :mod:`je_auto_control.utils.dbus_client` when the +Linux accessibility backend became its second caller: ``utils/`` sits above +the per-OS packages in this project's layering, so an accessibility backend +reaching down into ``linux_wayland/`` to borrow its D-Bus code would invert +that. -That rules out the obvious shell-out. ``gdbus call`` and ``gdbus monitor`` are -two processes with two unique names, so the monitor is not the caller and the -Response is not addressed to it; measured against a real ``dbus-daemon``, it -sees the call go past and the answer never arrive. The only listener that -does see it is a full bus monitor (``dbus-monitor``, which asks the bus for -``BecomeMonitor``), and making a screenshot require permission to observe every -message on the user's session bus is a poor trade for a fallback tier. - -So the subscription and the call happen on one connection here, which is what -every portal client library does and what the protocol is designed for. The -cost is marshalling D-Bus by hand; the scope is kept to exactly what a portal -conversation needs — connect, authenticate, ``Hello``, ``AddMatch``, one method -call, and read signals until the matching one arrives. It is deliberately not a -general D-Bus binding: no properties, no introspection, no object export, no -file-descriptor passing (:mod:`je_auto_control.linux_wayland.oeffis` uses -liboeffis for the one call that needs that). - -Every failure is an :class:`AutoControlException` subclass, so the containment -boundaries that catch the family keep working. +This module stays because the portal code and its tests import it by this +name, and moving a working protocol implementation is not a reason to churn +either. """ -from __future__ import annotations - -import contextlib -import os -import socket -import struct -import time -from typing import Any, Dict, List, Optional, Tuple - -from je_auto_control.utils.exception.exceptions import AutoControlException - - -#: Message types, from the D-Bus specification. -METHOD_CALL = 1 -METHOD_RETURN = 2 -ERROR = 3 -SIGNAL = 4 - -#: Header field codes. -FIELD_PATH = 1 -FIELD_INTERFACE = 2 -FIELD_MEMBER = 3 -FIELD_ERROR_NAME = 4 -FIELD_REPLY_SERIAL = 5 -FIELD_DESTINATION = 6 -FIELD_SENDER = 7 -FIELD_SIGNATURE = 8 - -#: ``NO_REPLY_EXPECTED``. Declared for completeness; nothing here uses it, -#: because every call this makes is one whose answer is worth waiting for. -FLAG_NO_REPLY_EXPECTED = 1 - -BUS_NAME = "org.freedesktop.DBus" -BUS_PATH = "/org/freedesktop/DBus" -BUS_INTERFACE = "org.freedesktop.DBus" - -_ADDRESS_ENV = "DBUS_SESSION_BUS_ADDRESS" -_MAX_MESSAGE = 128 * 1024 * 1024 # the specification's own ceiling -_ALIGNMENT = {"y": 1, "b": 4, "n": 2, "q": 2, "i": 4, "u": 4, - "x": 8, "t": 8, "d": 8, "s": 4, "o": 4, "g": 1, - "a": 4, "(": 8, "{": 8, "v": 1, "h": 4} - - -class DBusError(AutoControlException): - """The session bus is unreachable, or answered with an error.""" - - -class Variant: - """A value with an explicit D-Bus type, for the ``a{sv}`` option maps.""" - - __slots__ = ("signature", "value") - - def __init__(self, signature: str, value: Any) -> None: - self.signature = signature - self.value = value - - def __repr__(self) -> str: - return f"Variant({self.signature!r}, {self.value!r})" - - -# --- marshalling ---------------------------------------------------------- - - -class _Writer: - """Little-endian marshaller. Alignment is relative to the message start.""" - - def __init__(self, offset: int = 0) -> None: - self._parts: List[bytes] = [] - self._length = offset - - def align(self, boundary: int) -> None: - padding = (-self._length) % boundary - if padding: - self._parts.append(bytes(padding)) - self._length += padding - - def raw(self, data: bytes) -> None: - self._parts.append(data) - self._length += len(data) - - def byte(self, value: int) -> None: - self.raw(struct.pack(" None: - self.align(4) - self.raw(struct.pack(" None: - encoded = value.encode("utf-8") - self.uint32(len(encoded)) - self.raw(encoded + b"\x00") - - def signature(self, value: str) -> None: - encoded = value.encode("ascii") - self.byte(len(encoded)) - self.raw(encoded + b"\x00") - - def value(self, sig: str, value: Any) -> None: - """Write one complete value of the given single signature.""" - _write_value(self, _SignatureReader(sig), value) - - @property - def data(self) -> bytes: - return b"".join(self._parts) - - def __len__(self) -> int: - return self._length - - -class _SignatureReader: - """Walks a signature string one complete type at a time.""" - - def __init__(self, text: str) -> None: - self.text = text - self.index = 0 - - def done(self) -> bool: - return self.index >= len(self.text) - - def peek(self) -> str: - return self.text[self.index] - - def take(self) -> str: - code = self.text[self.index] - self.index += 1 - return code - - def take_complete(self) -> str: - """Take one whole type, following containers to their close.""" - start = self.index - code = self.take() - if code == "a": - self.take_complete() - elif code in "({": - closing = ")" if code == "(" else "}" - while self.peek() != closing: - self.take_complete() - self.take() - return self.text[start:self.index] - - -def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: - code = reader.take() - if code == "y": - writer.byte(int(value)) - elif code == "b": - writer.uint32(1 if value else 0) - elif code == "u": - writer.uint32(int(value)) - elif code in "so": - writer.string(str(value)) - elif code == "g": - writer.signature(str(value)) - elif code == "v": - _write_variant(writer, value) - elif code == "a": - _write_array(writer, reader, value) - elif code in "({": - _write_struct(writer, reader, code, value) - else: - raise DBusError(f"cannot marshal D-Bus type {code!r}") - - -def _write_variant(writer: _Writer, value: Any) -> None: - variant = value if isinstance(value, Variant) else _guess_variant(value) - writer.signature(variant.signature) - writer.value(variant.signature, variant.value) - - -def _guess_variant(value: Any) -> Variant: - if isinstance(value, bool): - return Variant("b", value) - if isinstance(value, int): - return Variant("u", value) - if isinstance(value, str): - return Variant("s", value) - raise DBusError(f"no obvious D-Bus type for {type(value).__name__}") - - -def _write_array(writer: _Writer, reader: _SignatureReader, value: Any) -> None: - element = reader.take_complete() - writer.align(4) - # The length prefix counts the elements only, and it is written before - # the padding that aligns the first one — so the body is built separately - # at the offset it will actually occupy. - body_start = len(writer) + 4 - padding = (-body_start) % _ALIGNMENT.get(element[0], 1) - body = _Writer(body_start + padding) - items = value.items() if isinstance(value, dict) else value - for item in items: - body.align(_ALIGNMENT.get(element[0], 1)) - _write_value(body, _SignatureReader(element), item) - writer.uint32(len(body) - body_start - padding) - writer.raw(bytes(padding)) - writer.raw(body.data) - - -def _write_struct(writer: _Writer, reader: _SignatureReader, code: str, - value: Any) -> None: - closing = ")" if code == "(" else "}" - writer.align(8) - for item in value: - if reader.peek() == closing: - raise DBusError("too many members for this struct signature") - _write_value(writer, reader, item) - if reader.peek() != closing: - raise DBusError("too few members for this struct signature") - reader.take() - - -class _Reader: - """Little-endian demarshaller over one complete message.""" - - def __init__(self, data: bytes, offset: int = 0) -> None: - self.data = data - self.offset = offset - - def align(self, boundary: int) -> None: - self.offset += (-self.offset) % boundary - - def take(self, count: int) -> bytes: - if self.offset + count > len(self.data): - raise DBusError("truncated D-Bus message") - chunk = self.data[self.offset:self.offset + count] - self.offset += count - return chunk - - def byte(self) -> int: - return self.take(1)[0] - - def uint32(self) -> int: - self.align(4) - return struct.unpack(" str: - length = self.uint32() - text = self.take(length).decode("utf-8", errors="replace") - self.take(1) - return text - - def signature(self) -> str: - length = self.byte() - text = self.take(length).decode("ascii", errors="replace") - self.take(1) - return text - - def value(self, sig: str) -> Any: - return _read_value(self, _SignatureReader(sig)) - - -def _read_value(reader: _Reader, signature: _SignatureReader) -> Any: - code = signature.take() - if code == "y": - return reader.byte() - if code == "b": - return bool(reader.uint32()) - if code == "u": - return reader.uint32() - if code in "so": - return reader.string() - if code == "g": - return reader.signature() - if code == "v": - return reader.value(reader.signature()) - if code == "a": - return _read_array(reader, signature) - if code in "({": - return _read_struct(reader, signature, code) - raise DBusError(f"cannot demarshal D-Bus type {code!r}") - - -def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: - element = signature.take_complete() - length = reader.uint32() - reader.align(_ALIGNMENT.get(element[0], 1)) - end = reader.offset + length - items = [] - while reader.offset < end: - reader.align(_ALIGNMENT.get(element[0], 1)) - items.append(_read_value(reader, _SignatureReader(element))) - if element.startswith("{"): - return dict(items) - return items - - -def _read_struct(reader: _Reader, signature: _SignatureReader, - code: str) -> tuple: - closing = ")" if code == "(" else "}" - reader.align(8) - members = [] - while signature.peek() != closing: - members.append(_read_value(reader, signature)) - signature.take() - return tuple(members) - - -class Message: - """One decoded D-Bus message: the header fields that matter, and the body.""" - - __slots__ = ("type", "serial", "fields", "body") - - def __init__(self, message_type: int, serial: int, - fields: Dict[int, Any], body: List[Any]) -> None: - self.type = message_type - self.serial = serial - self.fields = fields - self.body = body - - @property - def path(self) -> str: - return self.fields.get(FIELD_PATH, "") - - @property - def interface(self) -> str: - return self.fields.get(FIELD_INTERFACE, "") - - @property - def member(self) -> str: - return self.fields.get(FIELD_MEMBER, "") - - @property - def reply_serial(self) -> int: - return self.fields.get(FIELD_REPLY_SERIAL, 0) - - @property - def error_name(self) -> str: - return self.fields.get(FIELD_ERROR_NAME, "") - - -# --- the connection ------------------------------------------------------- - - -def session_address() -> Optional[str]: - """The session bus address, or None when this process has no session bus.""" - address = os.environ.get(_ADDRESS_ENV, "").strip() - return address or None - - -def is_available() -> bool: - """Whether a session bus address is set, so connecting is worth trying.""" - return session_address() is not None - - -def _socket_target(address: str) -> Tuple[str, bool]: - """Pick a connectable ``unix:`` transport out of a bus address. - - :return: the socket path and whether it is in the abstract namespace. - """ - for candidate in address.split(";"): - if not candidate.startswith("unix:"): - continue - fields = [part.split("=", 1) - for part in candidate[len("unix:"):].split(",") - if "=" in part] - options = dict(fields) - if "path" in options: - return options["path"], False - if "abstract" in options: - return options["abstract"], True - raise DBusError(f"no usable unix transport in {address!r}") - - -class SessionBus: - """An authenticated connection to the session bus, as a context manager.""" - - def __init__(self, address: Optional[str] = None) -> None: - self.address = address or session_address() - self.unique_name = "" - self._socket: Optional[socket.socket] = None - self._serial = 0 - self._buffer = b"" - #: Messages read while waiting for a method reply. A portal's Response - #: signal and its method return come from two different senders, so - #: the bus gives no ordering between them and the signal can arrive - #: first — dropping it here would be a wait that never ends. - self._queued: List[Message] = [] - - # --- lifecycle -------------------------------------------------------- - - def __enter__(self) -> "SessionBus": - self.connect() - return self - - def __exit__(self, *_exception: Any) -> None: - self.close() - - def connect(self) -> None: - """Open the socket, authenticate, and say ``Hello``.""" - if self.address is None: - raise DBusError( - f"{_ADDRESS_ENV} is not set, so there is no session bus to " - f"reach the desktop portal on", - ) - path, abstract = _socket_target(self.address) - try: - self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - self._socket.connect(("\0" + path) if abstract else path) - except OSError as error: - self.close() - raise DBusError(f"cannot reach the session bus: {error}") from error - self._authenticate() - self.unique_name = self.call( - BUS_NAME, BUS_PATH, BUS_INTERFACE, "Hello", "", [])[0] - - def close(self) -> None: - """Drop the connection; never raise from teardown.""" - if self._socket is not None: - with contextlib.suppress(OSError): - self._socket.close() - self._socket = None - - @property - def sender_token(self) -> str: - """This connection's name as the portal spells it inside object paths.""" - return self.unique_name.lstrip(":").replace(".", "_") - - # --- authentication --------------------------------------------------- - - def _authenticate(self) -> None: - """The SASL EXTERNAL handshake, which is a uid in hex over a socket.""" - uid = str(os.getuid()).encode("ascii") - self._send_raw(b"\x00AUTH EXTERNAL " + uid.hex().encode("ascii") - + b"\r\n") - reply = self._read_line() - if not reply.startswith("OK"): - raise DBusError(f"the session bus refused authentication: {reply}") - self._send_raw(b"BEGIN\r\n") - - def _send_raw(self, data: bytes) -> None: - if self._socket is None: - raise DBusError("the session bus connection is closed") - try: - self._socket.sendall(data) - except OSError as error: - raise DBusError(f"writing to the session bus failed: {error}") from error - - def _read_line(self, timeout: float = 10.0) -> str: - """Read one CRLF-terminated line of the auth conversation.""" - deadline = time.monotonic() + timeout - while b"\r\n" not in self._buffer: - self._fill(deadline - time.monotonic()) - line, _, self._buffer = self._buffer.partition(b"\r\n") - return line.decode("utf-8", errors="replace") - - # --- reading ---------------------------------------------------------- - - def _fill(self, timeout: float) -> None: - """Read whatever is available, or fail once the deadline has passed.""" - if self._socket is None: - raise DBusError("the session bus connection is closed") - if timeout <= 0: - raise DBusError("the session bus did not answer in time") - self._socket.settimeout(timeout) - try: - chunk = self._socket.recv(65536) - except socket.timeout as error: - raise DBusError("the session bus did not answer in time") from error - except OSError as error: - raise DBusError(f"reading from the session bus failed: {error}") from error - if not chunk: - raise DBusError("the session bus closed the connection") - self._buffer += chunk - - def read_message(self, deadline: float) -> Message: - """Read one whole message, or fail when the deadline passes.""" - while len(self._buffer) < 16: - self._fill(deadline - time.monotonic()) - endian, message_type, _flags, _version = struct.unpack( - " _MAX_MESSAGE: - raise DBusError("the session bus sent an implausibly large message") - while len(self._buffer) < total: - self._fill(deadline - time.monotonic()) - raw, self._buffer = self._buffer[:total], self._buffer[total:] - return _decode(raw, message_type, serial, fields_length, header_end) - - # --- writing ---------------------------------------------------------- - - def _next_serial(self) -> int: - self._serial += 1 - return self._serial - - def send(self, message_type: int, fields: Dict[int, Tuple[str, Any]], - signature: str, body: List[Any], flags: int = 0) -> int: - """Marshal and send one message; return its serial.""" - payload = _Writer() - for code, value in body_pairs(signature, body): - payload.value(code, value) - serial = self._next_serial() - header = _Writer() - header.raw(struct.pack(" List[Any]: - """Make a method call and return the reply body.""" - fields = { - FIELD_PATH: ("o", path), - FIELD_INTERFACE: ("s", interface), - FIELD_MEMBER: ("s", member), - FIELD_DESTINATION: ("s", destination), - } - if signature: - fields[FIELD_SIGNATURE] = ("g", signature) - serial = self.send(METHOD_CALL, fields, signature, body) - deadline = time.monotonic() + timeout - while True: - message = self.read_message(deadline) - if message.reply_serial != serial: - self._queued.append(message) - continue - if message.type == ERROR: - detail = message.body[0] if message.body else "" - raise DBusError(f"{message.error_name}: {detail}".strip(": ")) - if message.type == METHOD_RETURN: - return message.body - - def add_match(self, rule: str) -> None: - """Subscribe to signals, and wait for the bus to confirm the rule. - - Waiting matters twice over: a malformed rule is reported rather than - silently never matching, and the round trip proves the subscription is - in place before the call that provokes the signal is made. - """ - self.call(BUS_NAME, BUS_PATH, BUS_INTERFACE, "AddMatch", "s", [rule]) - - def wait_for_signal(self, paths: List[str], interface: str, member: str, - timeout: float) -> List[Any]: - """Read until the signal we subscribed to arrives, or time out.""" - def matches(message: Message) -> bool: - return (message.type == SIGNAL and message.member == member - and message.interface == interface - and message.path in paths) - - for index, message in enumerate(self._queued): - if matches(message): - del self._queued[index] - return message.body - self._queued.clear() - deadline = time.monotonic() + max(0.0, timeout) - while True: - message = self.read_message(deadline) - if matches(message): - return message.body - - -def body_pairs(signature: str, body: List[Any]): - """Pair each top-level type in a signature with its argument.""" - reader = _SignatureReader(signature) - for value in body: - if reader.done(): - raise DBusError("more arguments than the signature declares") - yield reader.take_complete(), value - if not reader.done(): - raise DBusError("fewer arguments than the signature declares") - - -def _decode(raw: bytes, message_type: int, serial: int, fields_length: int, - header_end: int) -> Message: - """Turn one complete message's bytes into a :class:`Message`.""" - reader = _Reader(raw, 12) - fields: Dict[int, Any] = {} - reader.uint32() # the header array's own length, already read - end = 16 + fields_length - while reader.offset < end: - reader.align(8) - code = reader.byte() - fields[code] = reader.value(reader.signature()) - reader.offset = header_end + ((-header_end) % 8) - body: List[Any] = [] - signature = fields.get(FIELD_SIGNATURE, "") - if signature: - walker = _SignatureReader(signature) - while not walker.done(): - body.append(_read_value(reader, _SignatureReader( - walker.take_complete()))) - return Message(message_type, serial, fields, body) - +from je_auto_control.utils.dbus_client.session_bus import ( # noqa: F401 # reason: re-export + BUS_INTERFACE, BUS_NAME, BUS_PATH, DBusError, ERROR, FIELD_ERROR_NAME, + FIELD_INTERFACE, FIELD_MEMBER, FIELD_PATH, FIELD_REPLY_SERIAL, + FIELD_SENDER, FIELD_SIGNATURE, METHOD_CALL, METHOD_RETURN, Message, + SIGNAL, SessionBus, Variant, body_pairs, is_available, session_address, +) __all__ = [ "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", - "Variant", "is_available", "session_address", + "Variant", "body_pairs", "is_available", "session_address", ] diff --git a/je_auto_control/linux_wayland/mouse.py b/je_auto_control/linux_wayland/mouse.py index 82308146..ae22aae3 100644 --- a/je_auto_control/linux_wayland/mouse.py +++ b/je_auto_control/linux_wayland/mouse.py @@ -326,9 +326,14 @@ def _wheel_deltas(scroll_value: int, scroll_direction: int) -> Tuple[int, int]: ``scroll_direction`` carries axis and sign, per this module's ``wayland_scroll_direction_*`` constants: +-1 vertical, +-2 horizontal. + + A negative ``scroll_value`` reverses that direction, the same rule the + other three backends follow. The magnitude used to be taken with + ``abs()``, which is what let a portable ``mouse_scroll(-3)`` scroll the + named direction here instead of the opposite one. """ direction = int(scroll_direction) - amount = abs(int(scroll_value)) * (1 if direction > 0 else -1) + amount = int(scroll_value) * (1 if direction > 0 else -1) return (0, amount) if abs(direction) == 1 else (amount, 0) diff --git a/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py b/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py index f5ee1f86..fe93c594 100644 --- a/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py +++ b/je_auto_control/linux_with_x11/core/utils/x11_linux_display.py @@ -1,9 +1,12 @@ -import sys from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) import os diff --git a/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py b/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py index 6c295691..b81fbe1e 100644 --- a/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py +++ b/je_auto_control/linux_with_x11/core/utils/x11_linux_vk.py @@ -1,9 +1,12 @@ -import sys from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib import XK diff --git a/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py b/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py index 9fe6febc..f6477877 100644 --- a/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py +++ b/je_auto_control/linux_with_x11/keyboard/x11_linux_keyboard_control.py @@ -1,12 +1,15 @@ -import sys import time from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from je_auto_control.linux_with_x11.core.utils.x11_linux_display import display diff --git a/je_auto_control/linux_with_x11/listener/x11_linux_listener.py b/je_auto_control/linux_with_x11/listener/x11_linux_listener.py index 52590c59..9bf0b02c 100644 --- a/je_auto_control/linux_with_x11/listener/x11_linux_listener.py +++ b/je_auto_control/linux_with_x11/listener/x11_linux_listener.py @@ -1,13 +1,16 @@ -import sys from queue import Queue from threading import Thread from je_auto_control.utils.exception.exception_tags import linux_import_error_message, listener_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib.display import Display diff --git a/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py b/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py index dff94b70..154efe19 100644 --- a/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py +++ b/je_auto_control/linux_with_x11/mouse/x11_linux_mouse_control.py @@ -1,13 +1,16 @@ -import sys import time from typing import Optional, Tuple from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib import X, protocol @@ -23,6 +26,17 @@ x11_linux_scroll_direction_left = 6 x11_linux_scroll_direction_right = 7 +#: Each scroll direction and the one a negative count turns it into. +#: Windows and macOS have always read the direction off the sign of +#: the count; X11 encodes direction as a *button*, so the sign has to +#: be turned back into one here rather than handed to the server. +_OPPOSITE_SCROLL_DIRECTION = { + x11_linux_scroll_direction_up: x11_linux_scroll_direction_down, + x11_linux_scroll_direction_down: x11_linux_scroll_direction_up, + x11_linux_scroll_direction_left: x11_linux_scroll_direction_right, + x11_linux_scroll_direction_right: x11_linux_scroll_direction_left, +} + def position() -> Tuple[int, int]: """ @@ -91,17 +105,25 @@ def scroll(scroll_value: int, scroll_direction: int) -> None: 模擬滑鼠滾動 :param scroll_value: number of scroll units 滾動次數 - 方向由 scroll_direction 決定，因此這裡只取絕對值。 - Direction comes from scroll_direction, so only the magnitude is used - here: range() on a negative value is empty, which silently scrolled - nothing at all. - :param scroll_direction: scroll direction 滾動方向 + 負數會反轉方向，與 Windows／macOS 一致；絕對值是滾動次數。 + A negative count reverses ``scroll_direction``, which is what + Windows and macOS have always done with the sign; the magnitude + is the number of notches. (It used to be discarded, so portable + code written against the Windows convention scrolled the *named* + direction instead of the opposite one — silently, and only on + this backend.) + :param scroll_direction: 未帶負號時的預設方向 The direction a + positive count scrolls in 4 = up 上 5 = down 下 6 = left 左 7 = right 右 """ - for _ in range(abs(int(scroll_value))): + scroll_value = int(scroll_value) + if scroll_value < 0: + scroll_direction = _OPPOSITE_SCROLL_DIRECTION.get( + scroll_direction, scroll_direction) + for _ in range(abs(scroll_value)): click_mouse(scroll_direction) diff --git a/je_auto_control/linux_with_x11/record/x11_linux_record.py b/je_auto_control/linux_with_x11/record/x11_linux_record.py index 01744b4a..691bba00 100644 --- a/je_auto_control/linux_with_x11/record/x11_linux_record.py +++ b/je_auto_control/linux_with_x11/record/x11_linux_record.py @@ -1,13 +1,16 @@ -import sys from typing import Any from queue import Queue from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from je_auto_control.linux_with_x11.listener.x11_linux_listener import ( diff --git a/je_auto_control/linux_with_x11/screen/x11_linux_screen.py b/je_auto_control/linux_with_x11/screen/x11_linux_screen.py index d509fbdf..7e888734 100644 --- a/je_auto_control/linux_with_x11/screen/x11_linux_screen.py +++ b/je_auto_control/linux_with_x11/screen/x11_linux_screen.py @@ -1,12 +1,15 @@ -import sys from typing import Tuple from je_auto_control.utils.exception.exception_tags import linux_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_x11_unix # === 平台檢查 Platform Check === # 僅允許在 Linux 環境執行，否則拋出例外 -if sys.platform not in ["linux", "linux2"]: +# Not "is this Linux" but "is this an X11 unix": a FreeBSD, OpenBSD or +# NetBSD desktop runs the same X server and the same python-Xlib, and +# nothing below is Linux-specific. +if not is_x11_unix(): raise AutoControlException(linux_import_error_message) from Xlib import X diff --git a/je_auto_control/osx/listener/osx_listener.py b/je_auto_control/osx/listener/osx_listener.py index f6fe4e61..9995cd1f 100644 --- a/je_auto_control/osx/listener/osx_listener.py +++ b/je_auto_control/osx/listener/osx_listener.py @@ -1,96 +1,253 @@ +"""macOS keyboard/mouse capture over a Quartz event tap, on its own thread. + +This module used to build an ``NSApplication`` **at import time** and stop +recording by way of ``AppHelper.runEventLoop()``, a loop that never returns to +its caller. Both sat on the path of ``import je_auto_control``, which is why +``wrapper/_platform_osx.py`` set ``recorder = None`` and macOS shipped without +a recorder at all rather than take that regression. + +Neither is necessary. A ``CGEventTap`` needs a **run loop**, not an +application: create the tap on a dedicated thread, add its source to *that* +thread's run loop, and pump the loop in short ``CFRunLoopRunInMode`` slices so +the stop flag is honoured between them. Nothing touches AppKit, nothing runs +at import, and the caller is never blocked. The macOS hotkey backend in +``utils/hotkey/backends/macos_backend.py`` already drives a tap this way; this +is the same shape. + +Two properties are deliberate and both are load-bearing: + +* **The tap is listen-only.** A recorder that consumed events would swallow + the very input it is recording, so the user's clicks would stop working the + moment recording started. +* **Modifiers are reconstructed from ``flagsChanged``.** macOS sends no + key-down/key-up for Shift, Control, Option or Command; it sends one event + carrying the new flag set. Without decoding that, a recording could not say + a modifier was held across the actions that followed. + +Coordinates come from ``CGEventGetLocation``, whose origin is the top-left of +the display, which is the space ``osx_mouse`` posts into. The previous +listener read ``NSEvent.mouseLocation()``, a **bottom-left** origin, so every +recorded click was mirrored vertically against where a replay would put it. + +Requires Accessibility permission (System Settings -> Privacy & Security -> +Accessibility). Without it ``CGEventTapCreate`` returns ``None`` and +:meth:`OSXInputTap.start` raises rather than recording silence. + +**Everything typed while recording is captured, passwords included.** Callers +must treat the result as sensitive. +""" import sys -from queue import Queue +import threading +import time +from typing import Any, Dict, List, Optional, Tuple -from je_auto_control.utils.exception.exception_tags import osx_import_error_message -from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.exception.exception_tags import ( + macos_record_error_message, osx_import_error_message, +) +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlRecordException, +) +from je_auto_control.utils.input_macro.recorder_base import MAX_EVENTS +from je_auto_control.utils.logging.logging_instance import autocontrol_logger # === 平台檢查 Platform Check === # 僅允許在 macOS (Darwin) 環境執行，否則拋出例外 if sys.platform not in ["darwin"]: raise AutoControlException(osx_import_error_message) -from Cocoa import ( - NSApplication, - NSEvent, - NSEventMaskKeyDown, - NSEventMaskLeftMouseDown, - NSEventMaskRightMouseDown, - NSObject, +import Quartz + +#: How long one run-loop slice runs before the stop flag is re-checked. +POLL_SECONDS = 0.1 + +#: How long :meth:`OSXInputTap.start` waits for the tap to come up or fail. +START_TIMEOUT = 5.0 + +#: Quartz event type -> ``(op, button)``. ``otherMouse`` is the middle button +#: for every mouse this project addresses. +_MOUSE_EVENTS = { + Quartz.kCGEventLeftMouseDown: ("mouse_down", "left"), + Quartz.kCGEventLeftMouseUp: ("mouse_up", "left"), + Quartz.kCGEventRightMouseDown: ("mouse_down", "right"), + Quartz.kCGEventRightMouseUp: ("mouse_up", "right"), + Quartz.kCGEventOtherMouseDown: ("mouse_down", "middle"), + Quartz.kCGEventOtherMouseUp: ("mouse_up", "middle"), +} + +#: Modifier keycode -> the flag bit that says it is currently held. macOS +#: reports modifiers only as a flag set, so the keycode alone cannot say +#: whether an event was a press or a release; the bit can. +_MODIFIER_FLAG = { + 54: Quartz.kCGEventFlagMaskCommand, # right command + 55: Quartz.kCGEventFlagMaskCommand, # command + 56: Quartz.kCGEventFlagMaskShift, # shift + 57: Quartz.kCGEventFlagMaskAlphaShift, # caps lock + 58: Quartz.kCGEventFlagMaskAlternate, # option + 59: Quartz.kCGEventFlagMaskControl, # control + 60: Quartz.kCGEventFlagMaskShift, # right shift + 61: Quartz.kCGEventFlagMaskAlternate, # right option + 62: Quartz.kCGEventFlagMaskControl, # right control + 63: Quartz.kCGEventFlagMaskSecondaryFn, # fn +} + +#: ``CGEventMaskBit`` is a C macro rather than a symbol, so the shift is +#: written out — the same form the macOS hotkey backend uses. +_TAP_MASK = sum( + 1 << event for event in + (Quartz.kCGEventKeyDown, Quartz.kCGEventKeyUp, + Quartz.kCGEventFlagsChanged, Quartz.kCGEventScrollWheel, + *_MOUSE_EVENTS) ) -from PyObjCTools import AppHelper - -# === 全域事件記錄 Queue Global event record queue === -record_queue = Queue() - -# 建立 NSApplication 實例 Create NSApplication instance -app = NSApplication.sharedApplication() - - -class AppDelegate(NSObject): - """ - AppDelegate - 應用程式委派類別 - - 負責在應用程式啟動後註冊全域事件監聽器 - """ - - def applicationDidFinishLaunching_(self, notification): # noqa: N802 # reason: ObjC selector signature - """ - 註冊全域事件監聽器 - Register global event monitors - """ - NSEvent.addGlobalMonitorForEventsMatchingMask_handler_( - NSEventMaskKeyDown, keyboard_handler - ) - NSEvent.addGlobalMonitorForEventsMatchingMask_handler_( - NSEventMaskLeftMouseDown, mouse_left_handler - ) - NSEvent.addGlobalMonitorForEventsMatchingMask_handler_( - NSEventMaskRightMouseDown, mouse_right_handler - ) - - -def mouse_left_handler(event) -> None: - """ - 滑鼠左鍵事件處理器 - Mouse left button handler - """ - loc = NSEvent.mouseLocation() - record_queue.put(("AC_mouse_left", loc.x, loc.y)) - - -def mouse_right_handler(event) -> None: - """ - 滑鼠右鍵事件處理器 - Mouse right button handler - """ - loc = NSEvent.mouseLocation() - record_queue.put(("AC_mouse_right", loc.x, loc.y)) - - -def keyboard_handler(event) -> None: - """ - 鍵盤事件處理器 - Keyboard event handler - """ - keycode = int(event.keyCode()) - if keycode == 98: # 特殊情況：忽略 keycode 98 - return - record_queue.put(("AC_type_keyboard", keycode)) - - -def osx_record() -> None: - """ - 開始錄製事件 - Start recording events - """ - delegate = AppDelegate.alloc().init() - app.setDelegate_(delegate) - AppHelper.runEventLoop() - - -def osx_stop_record() -> Queue: - """ - 停止錄製並回傳事件 Queue - Stop recording and return event queue - """ - return record_queue \ No newline at end of file + +# A tap that takes too long is disabled by the window server rather than +# allowed to slow input down; re-enabling it is the tap owner's job. +_TAP_DISABLED = (Quartz.kCGEventTapDisabledByTimeout, + Quartz.kCGEventTapDisabledByUserInput) + + +class OSXInputTap: + """Captures keyboard and mouse events, with releases, wheel and timing.""" + + def __init__(self, max_events: int = MAX_EVENTS) -> None: + self.events: List[Dict[str, Any]] = [] + self.started = time.monotonic() + self.error: Optional[str] = None + self.max_events = int(max_events) + self._stop = threading.Event() + self._ready = threading.Event() + self._thread: Optional[threading.Thread] = None + + # -- public ------------------------------------------------------------ + def start(self) -> None: + """Create the tap and record. Raises if the tap cannot be created.""" + self._stop.clear() + self._ready.clear() + self._thread = threading.Thread(target=self._run, daemon=True) + self._thread.start() + self._ready.wait(timeout=START_TIMEOUT) + if self.error: + raise AutoControlRecordException(self.error) + + def stop(self) -> List[Dict[str, Any]]: + """Stop recording and return the raw events.""" + self._stop.set() + thread, self._thread = self._thread, None + if thread is not None: + # A run-loop slice cannot be interrupted part-way, so allow for + # several: a thread left running keeps a live tap on the session. + thread.join(timeout=POLL_SECONDS * 10) + if thread.is_alive(): + autocontrol_logger.error( + "recorder thread did not stop within %.1fs", + POLL_SECONDS * 10) + return self.events + + # -- tap thread -------------------------------------------------------- + def _run(self) -> None: + from CoreFoundation import CFRunLoopRunInMode, kCFRunLoopDefaultMode + + tap = source = run_loop = None + try: + tap = Quartz.CGEventTapCreate( + Quartz.kCGSessionEventTap, Quartz.kCGHeadInsertEventTap, + # Listen-only: a recorder that consumed events would swallow + # the input it is recording. + Quartz.kCGEventTapOptionListenOnly, _TAP_MASK, + self._callback, None, + ) + if tap is None: + autocontrol_logger.error( + "CGEventTapCreate returned None - Accessibility not granted") + self.error = macos_record_error_message + return + source = Quartz.CFMachPortCreateRunLoopSource(None, tap, 0) + run_loop = Quartz.CFRunLoopGetCurrent() + Quartz.CFRunLoopAddSource(run_loop, source, kCFRunLoopDefaultMode) + Quartz.CGEventTapEnable(tap, True) + except Exception as error: # noqa: BLE001 # reason: see below + # Anything at all here has to reach start(), which is blocked on + # _ready: a thread that dies quietly leaves the caller believing + # it is recording, and the session comes back empty with no + # explanation of why. + autocontrol_logger.error("recorder start failed: %r", error) + self.error = f"could not create the macOS event tap: {error!r}" + return + finally: + self._ready.set() + try: + while not self._stop.is_set(): + CFRunLoopRunInMode(kCFRunLoopDefaultMode, POLL_SECONDS, False) + finally: + # Both, and in this order: an enabled tap whose source is already + # gone is what leaks one live tap per record cycle. + Quartz.CGEventTapEnable(tap, False) + if source is not None and run_loop is not None: + Quartz.CFRunLoopRemoveSource( + run_loop, source, kCFRunLoopDefaultMode) + + def _callback(self, proxy, event_type, event, _refcon): + """Called by the window server on the tap thread, for every event.""" + # Anything escaping here tears down the run loop, and the recording + # then stops without saying so — one unexpected event shape would end + # the session. The catch is broad on purpose, and it is bounded: the + # error is logged and the next event is still decoded. + try: + if event_type in _TAP_DISABLED: + # Re-arm, rather than record silence for the rest of the run. + autocontrol_logger.info( + "event tap disabled (%s), re-enabling", event_type) + Quartz.CGEventTapEnable(proxy, True) + else: + self.decode(int(event_type), event) + except Exception as error: # noqa: BLE001 # reason: an OS callback; see above + autocontrol_logger.error("recorder decode failed: %r", error) + return event + + # -- decoding ---------------------------------------------------------- + def decode(self, event_type: int, event: Any) -> None: + """Record one Quartz event. Split out so it is testable off the tap.""" + if event_type == Quartz.kCGEventKeyDown: + self._put({"op": "key_down", "vk": self._keycode(event)}) + elif event_type == Quartz.kCGEventKeyUp: + self._put({"op": "key_up", "vk": self._keycode(event)}) + elif event_type == Quartz.kCGEventFlagsChanged: + self._modifier(event) + elif event_type == Quartz.kCGEventScrollWheel: + self._scroll(event) + elif event_type in _MOUSE_EVENTS: + operation, button = _MOUSE_EVENTS[event_type] + x, y = self._location(event) + self._put({"op": operation, "button": button, "x": x, "y": y}) + + @staticmethod + def _keycode(event: Any) -> int: + return int(Quartz.CGEventGetIntegerValueField( + event, Quartz.kCGKeyboardEventKeycode)) + + @staticmethod + def _location(event: Any) -> Tuple[int, int]: + point = Quartz.CGEventGetLocation(event) + return int(point.x), int(point.y) + + def _modifier(self, event: Any) -> None: + """Turn a flag set into the press or release of one modifier key.""" + keycode = self._keycode(event) + mask = _MODIFIER_FLAG.get(keycode) + if mask is None: + return + held = bool(int(Quartz.CGEventGetFlags(event)) & mask) + self._put({"op": "key_down" if held else "key_up", "vk": keycode}) + + def _scroll(self, event: Any) -> None: + delta = int(Quartz.CGEventGetIntegerValueField( + event, Quartz.kCGScrollWheelEventDeltaAxis1)) + x, y = self._location(event) + self._put({"op": "scroll", "delta": delta, "x": x, "y": y}) + + def _put(self, event: Dict[str, Any]) -> None: + """Record one event, stopping the tap once the cap is reached.""" + if len(self.events) >= self.max_events: + self._stop.set() + return + event["time"] = time.monotonic() + self.events.append(event) diff --git a/je_auto_control/osx/record/osx_record.py b/je_auto_control/osx/record/osx_record.py index f119508f..6c16441a 100644 --- a/je_auto_control/osx/record/osx_record.py +++ b/je_auto_control/osx/record/osx_record.py @@ -1,52 +1,41 @@ +"""macOS recorder: the Quartz event tap shaped into the recorder surface. + +Everything after the capture — the down-events-only queue the executor has +always been handed, the ``delta_ms`` timeline a replay needs, and the +mouse-only / keyboard-only filters — is platform-neutral and lives in +:mod:`je_auto_control.utils.input_macro.recorder_base`, so this backend and +the Windows one cannot drift in the shape they produce. +""" import sys -from queue import Queue -from je_auto_control.utils.exception.exception_tags import osx_import_error_message -from je_auto_control.utils.exception.exceptions import AutoControlException, AutoControlJsonActionException +from je_auto_control.utils.exception.exception_tags import ( + osx_import_error_message, +) +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.input_macro.recorder_base import InputRecorder # === 平台檢查 Platform Check === # 僅允許在 macOS (Darwin) 環境執行，否則拋出例外 if sys.platform not in ["darwin"]: raise AutoControlException(osx_import_error_message) -from je_auto_control.osx.listener.osx_listener import osx_record, osx_stop_record +from je_auto_control.osx.listener.osx_listener import OSXInputTap -class OSXRecorder: +class OSXRecorder(InputRecorder): """ OSXRecorder macOS 事件錄製控制器 - - 提供開始與停止錄製的介面 - - 將錄製結果存入 Queue - """ - - def __init__(self): - self.record_flag: bool = False - - def record(self) -> None: - """ - Start recording events - 開始錄製事件 - """ - self.record_flag = True - osx_record() - def stop_record(self) -> Queue: - """ - Stop recording and return recorded events - 停止錄製並回傳事件隊列 - - :raises AutoControlJsonActionException: 若沒有錄製到任何事件 - :return: Queue of recorded events 錄製事件的隊列 - """ - record_queue = osx_stop_record() - self.record_flag = False - - if record_queue is None: - raise AutoControlJsonActionException + Capture runs through :class:`OSXInputTap`, a listen-only ``CGEventTap`` on + its own thread, so ``record()`` returns immediately and the user's input + keeps working while it is being recorded. + """ - return record_queue + def new_hook(self) -> OSXInputTap: + """Return a fresh, unstarted event tap.""" + return OSXInputTap() # === 全域 Recorder 實例 Global Recorder Instance === -osx_recorder = OSXRecorder() \ No newline at end of file +osx_recorder = OSXRecorder() diff --git a/je_auto_control/utils/accessibility/backends/__init__.py b/je_auto_control/utils/accessibility/backends/__init__.py index 4f39ffea..ed1c3553 100644 --- a/je_auto_control/utils/accessibility/backends/__init__.py +++ b/je_auto_control/utils/accessibility/backends/__init__.py @@ -48,6 +48,20 @@ def _build_backend() -> AccessibilityBackend: return NullAccessibilityBackend( "pyobjc (ApplicationServices, AppKit) is required on macOS", ) + if sys.platform.startswith("linux"): + from je_auto_control.utils.accessibility.backends.linux_backend import ( + LinuxAccessibilityBackend, + ) + backend = LinuxAccessibilityBackend() + if backend.available: + return backend + # AT-SPI is a bus, not a library, so "not installed" and "installed + # but nothing is bridged to it" look the same from here. Name both. + return NullAccessibilityBackend( + "no AT-SPI accessibility bus. Install and start at-spi2-core, and " + "enable the toolkit bridge (GTK_MODULES=gail:atk-bridge, " + "QT_ACCESSIBILITY=1)", + ) return NullAccessibilityBackend( f"no accessibility backend for platform {sys.platform!r}", ) diff --git a/je_auto_control/utils/accessibility/backends/linux_backend.py b/je_auto_control/utils/accessibility/backends/linux_backend.py new file mode 100644 index 00000000..58cb1412 --- /dev/null +++ b/je_auto_control/utils/accessibility/backends/linux_backend.py @@ -0,0 +1,414 @@ +"""Linux accessibility backend, over AT-SPI2. + +Linux had no accessibility backend at all: :func:`_build_backend` fell through +to the null one, while ``docs/CAPABILITY_MATRIX.md`` claimed "backend tests" +for Linux X11. This is the backend that makes the claim true. + +AT-SPI2 is a D-Bus protocol, not a library, which is what makes it reachable +here without adding a dependency. The usual bindings (``pyatspi``, +``gi.repository.Atspi``) are distribution packages built against the system +GObject introspection data: they cannot be installed into a virtual +environment, so a project that depends on them is a project most users cannot +run. :mod:`je_auto_control.utils.dbus_client`, written for the XDG portal +handshake, already speaks enough D-Bus for this. + +The shape of the protocol: + +* The accessibility bus is *not* the session bus. Its address comes from + ``org.a11y.Bus.GetAddress`` on the session bus, and everything else happens + on a second connection to that address. +* An accessible object is addressed by a **pair** — the bus name of the + application that owns it and an object path inside it — so references are + ``(sender, path)`` tuples throughout rather than single strings. +* The root's children are applications; theirs are windows; theirs are the + controls. Walking is therefore per-application, which is also what makes + ``app_name`` cheap to honour. + +Every read is defensive. An accessible can disappear between the call that +lists it and the call that describes it — the application closed a dialog, or +exited — and a walk that raised on that would fail more often than it +succeeded. +""" +import os +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.accessibility.backends.base import AccessibilityBackend +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, element_matches, +) +from je_auto_control.utils.dbus_client import DBusError, SessionBus, Variant +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +#: Where the session bus publishes the accessibility bus's address. +_BUS_NAME = "org.a11y.Bus" +_BUS_PATH = "/org/a11y/bus" + +#: The registry that owns the tree's root. +_REGISTRY = "org.a11y.atspi.Registry" +_ROOT_PATH = "/org/a11y/atspi/accessible/root" + +#: AT-SPI2 interfaces, as they appear on the wire. +_ACCESSIBLE = "org.a11y.atspi.Accessible" +_ACTION = "org.a11y.atspi.Action" +_COMPONENT = "org.a11y.atspi.Component" +_EDITABLE = "org.a11y.atspi.EditableText" +_TEXT = "org.a11y.atspi.Text" +_VALUE = "org.a11y.atspi.Value" +_PROPERTIES = "org.freedesktop.DBus.Properties" + +#: ``GetExtents`` coordinate spaces. 0 is the screen, which is the only one +#: whose numbers mean anything to a caller about to click them. +_COORDS_SCREEN = 0 + +#: Bit positions in the ``GetState`` bitfield that this backend reads. +_STATE_ENABLED = 8 +_STATE_FOCUSED = 12 +_STATE_SELECTED = 25 + +#: A reference to one accessible: the owning application's bus name and the +#: object path inside it. +Reference = Tuple[str, str] + + +def _is_available() -> bool: + """Whether an accessibility bus can be reached at all.""" + if os.name != "posix": + return False + try: + with _AtspiConnection() as connection: + connection.children(connection.root) + return True + except (DBusError, OSError) as error: + autocontrol_logger.info("AT-SPI unavailable: %r", error) + return False + + +class _AtspiConnection: + """One connection to the accessibility bus, for the length of one call.""" + + def __init__(self) -> None: + self._bus: Optional[SessionBus] = None + + def __enter__(self) -> "_AtspiConnection": + self._bus = SessionBus(address=self._address()) + self._bus.connect() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._bus is not None: + self._bus.close() + self._bus = None + + @staticmethod + def _address() -> str: + """Ask the session bus where the accessibility bus is.""" + with SessionBus() as session: + reply = session.call(_BUS_NAME, _BUS_PATH, _BUS_NAME, + "GetAddress", "", []) + if not reply or not isinstance(reply[0], str): + raise DBusError("org.a11y.Bus.GetAddress returned no address") + return reply[0] + + @property + def root(self) -> Reference: + return (_REGISTRY, _ROOT_PATH) + + def _call(self, reference: Reference, interface: str, member: str, + signature: str = "", body: Optional[List[Any]] = None, + timeout: float = 10.0) -> List[Any]: + sender, path = reference + return self._bus.call(sender, path, interface, member, + signature, body or [], timeout=timeout) + + # --- reads ------------------------------------------------------------- + + def children(self, reference: Reference) -> List[Reference]: + """The accessible's children, as ``(sender, path)`` pairs.""" + reply = self._call(reference, _ACCESSIBLE, "GetChildren") + if not reply: + return [] + return [(str(item[0]), str(item[1])) for item in reply[0] + if isinstance(item, (list, tuple)) and len(item) >= 2] + + def property(self, reference: Reference, name: str, + interface: str = _ACCESSIBLE) -> Any: + """One D-Bus property, unwrapped from its variant.""" + reply = self._call(reference, _PROPERTIES, "Get", "ss", + [interface, name]) + value = reply[0] if reply else None + return value.value if isinstance(value, Variant) else value + + def role_name(self, reference: Reference) -> str: + reply = self._call(reference, _ACCESSIBLE, "GetRoleName") + return str(reply[0]) if reply else "" + + def state(self, reference: Reference) -> int: + """The state bitfield, as one integer. + + AT-SPI sends it as two 32-bit words rather than one 64-bit value, so + reading only the first would silently drop every state above bit 31. + """ + reply = self._call(reference, _ACCESSIBLE, "GetState") + words = list(reply[0]) if reply and reply[0] else [] + bits = 0 + for index, word in enumerate(words[:2]): + bits |= int(word) << (32 * index) + return bits + + def extents(self, reference: Reference) -> Tuple[int, int, int, int]: + """``(left, top, width, height)`` in screen pixels, or zeroes.""" + try: + reply = self._call(reference, _COMPONENT, "GetExtents", "u", + [_COORDS_SCREEN]) + except DBusError: + # Not every accessible implements Component — a plain text node + # has no rectangle, and that is not an error. + return (0, 0, 0, 0) + if not reply or len(reply[0]) < 4: + return (0, 0, 0, 0) + return tuple(int(value) for value in reply[0][:4]) # type: ignore[return-value] + + # --- writes ------------------------------------------------------------ + + def do_action(self, reference: Reference, index: int = 0) -> bool: + reply = self._call(reference, _ACTION, "DoAction", "i", [int(index)]) + return bool(reply and reply[0]) + + def set_text(self, reference: Reference, value: str) -> bool: + reply = self._call(reference, _EDITABLE, "SetTextContents", "s", + [str(value)]) + return bool(reply and reply[0]) + + def text(self, reference: Reference) -> Optional[str]: + try: + reply = self._call(reference, _TEXT, "GetText", "ii", [0, -1]) + except DBusError: + return None + return str(reply[0]) if reply else None + + def number(self, reference: Reference) -> Optional[float]: + try: + return float(self.property(reference, "CurrentValue", _VALUE)) + except (DBusError, TypeError, ValueError): + return None + + def grab_focus(self, reference: Reference) -> bool: + reply = self._call(reference, _COMPONENT, "GrabFocus") + return bool(reply and reply[0]) + + +class LinuxAccessibilityBackend(AccessibilityBackend): + """The AT-SPI2 tree, walked over D-Bus.""" + + name = "linux-atspi" + + def __init__(self) -> None: + self.available = _is_available() + + def _require(self) -> None: + if not self.available: + raise AccessibilityNotAvailableError( + "no AT-SPI accessibility bus. Install and start at-spi2-core, " + "and make sure the toolkit's accessibility bridge is enabled " + "(GTK_MODULES=gail:atk-bridge, QT_ACCESSIBILITY=1).", + ) + + def list_elements(self, app_name: Optional[str] = None, + max_results: int = 200, + window_title: Optional[str] = None, + ) -> List[AccessibilityElement]: + self._require() + results: List[AccessibilityElement] = [] + with _AtspiConnection() as connection: + for application in connection.children(connection.root): + if len(results) >= max_results: + break + name = _safe_name(connection, application) + if app_name is not None and name != app_name: + continue + self._walk(connection, application, name, results, + max_results, window_title) + return results[:max_results] + + def _walk(self, connection: _AtspiConnection, reference: Reference, + app_name: str, results: List[AccessibilityElement], + max_results: int, window_title: Optional[str], + depth: int = 0) -> None: + """Depth-first, and bounded: a broken tree must not hang the caller.""" + if len(results) >= max_results or depth > 32: + return + try: + children = connection.children(reference) + except DBusError: + return + for child in children: + if len(results) >= max_results: + return + converted = _convert(connection, child, app_name) + if converted is None: + continue + # Scoping to a window is not just filtering: below a window the + # tree is orders of magnitude smaller, so this both narrows the + # answer and shortens the walk. + if depth == 0 and window_title is not None: + if window_title.lower() not in converted.name.lower(): + continue + results.append(converted) + self._walk(connection, child, app_name, results, max_results, + window_title, depth + 1) + + # --- control patterns -------------------------------------------------- + + def _find(self, connection: _AtspiConnection, name: Optional[str], + role: Optional[str], app_name: Optional[str], + window_title: Optional[str] = None, + contains: bool = False) -> Optional[Reference]: + """The reference behind the first element that matches.""" + for application in connection.children(connection.root): + owner = _safe_name(connection, application) + if app_name is not None and owner != app_name: + continue + found = self._search(connection, application, owner, name, role, + contains) + if found is not None: + return found + del window_title # accepted for signature parity with the base class + return None + + def _search(self, connection: _AtspiConnection, reference: Reference, + app_name: str, name: Optional[str], role: Optional[str], + contains: bool, depth: int = 0) -> Optional[Reference]: + if depth > 32: + return None + try: + children = connection.children(reference) + except DBusError: + return None + for child in children: + converted = _convert(connection, child, app_name) + if converted is not None and element_matches( + converted, name, role, app_name, contains): + return child + deeper = self._search(connection, child, app_name, name, role, + contains, depth + 1) + if deeper is not None: + return deeper + return None + + def get_value(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[str]: + self._require() + del automation_id # AT-SPI has no equivalent of a UIA automation id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name, + window_title, contains) + if reference is None: + return None + text = connection.text(reference) + if text is not None: + return text + number = connection.number(reference) + return None if number is None else str(number) + + def set_value(self, value: str, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name) + if reference is None: + return False + try: + return connection.set_text(reference, value) + except DBusError as error: + autocontrol_logger.info("set_value failed: %r", error) + return False + + def invoke(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name) + if reference is None: + return False + try: + return connection.do_action(reference, 0) + except DBusError as error: + autocontrol_logger.info("invoke failed: %r", error) + return False + + def set_focus(self, name: Optional[str] = None, role: Optional[str] = None, + app_name: Optional[str] = None, + automation_id: Optional[str] = None) -> bool: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name) + if reference is None: + return False + try: + return connection.grab_focus(reference) + except DBusError as error: + autocontrol_logger.info("set_focus failed: %r", error) + return False + + def get_state(self, name: Optional[str] = None, + role: Optional[str] = None, app_name: Optional[str] = None, + automation_id: Optional[str] = None, + window_title: Optional[str] = None, + contains: bool = False) -> Optional[Dict[str, Any]]: + self._require() + del automation_id + with _AtspiConnection() as connection: + reference = self._find(connection, name, role, app_name, + window_title, contains) + if reference is None: + return None + bits = connection.state(reference) + state: Dict[str, Any] = { + "enabled": bool(bits & (1 << _STATE_ENABLED)), + "focused": bool(bits & (1 << _STATE_FOCUSED)), + "selected": bool(bits & (1 << _STATE_SELECTED)), + } + # A key is absent when the control does not have the concept, + # which is a different answer from the value being empty. + text = connection.text(reference) + if text is not None: + state["value"] = text + number = connection.number(reference) + if number is not None: + state["number"] = number + return state + + +def _safe_name(connection: _AtspiConnection, reference: Reference) -> str: + try: + return str(connection.property(reference, "Name") or "") + except DBusError: + return "" + + +def _convert(connection: _AtspiConnection, reference: Reference, + app_name: str) -> Optional[AccessibilityElement]: + """One accessible as an :class:`AccessibilityElement`, or None.""" + try: + name = _safe_name(connection, reference) + role = connection.role_name(reference) + bounds = connection.extents(reference) + bits = connection.state(reference) + except DBusError: + # The accessible went away between being listed and being described. + return None + if not name and not role: + return None + return AccessibilityElement( + name=name, role=role, bounds=bounds, app_name=app_name, + native_id=reference[1], + enabled=bool(bits & (1 << _STATE_ENABLED)), + ) diff --git a/je_auto_control/utils/action_signing/cipher.py b/je_auto_control/utils/action_signing/cipher.py index a80dc398..c7593bbb 100644 --- a/je_auto_control/utils/action_signing/cipher.py +++ b/je_auto_control/utils/action_signing/cipher.py @@ -14,8 +14,6 @@ from pathlib import Path from typing import Optional, Union -from cryptography.fernet import Fernet, InvalidToken - from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -27,6 +25,7 @@ def _persistent_key() -> bytes: """Read the per-user Fernet key, creating it (0600) on first use.""" + from cryptography.fernet import Fernet if _DEFAULT_KEY_PATH.exists(): return _DEFAULT_KEY_PATH.read_bytes() _DEFAULT_KEY_PATH.parent.mkdir(parents=True, exist_ok=True) @@ -49,6 +48,7 @@ def _fernet_key(key: KeyType) -> bytes: def encrypt_action_file(path: Union[str, Path], key: KeyType = None) -> str: """Encrypt the file at ``path`` to ``.enc``; return the enc path.""" + from cryptography.fernet import Fernet target = Path(path) token = Fernet(_fernet_key(key)).encrypt(target.read_bytes()) enc_path = target.with_name(target.name + _ENC_SUFFIX) @@ -65,6 +65,7 @@ def decrypt_action_file(enc_path: Union[str, Path], key: KeyType = None, dropped. Raises :class:`AutoControlException` on a wrong key or a tampered file. """ + from cryptography.fernet import Fernet, InvalidToken enc = Path(enc_path) try: plaintext = Fernet(_fernet_key(key)).decrypt(enc.read_bytes()) diff --git a/je_auto_control/utils/agent_memory/agent_memory.py b/je_auto_control/utils/agent_memory/agent_memory.py index cffbfadf..15cd44c8 100644 --- a/je_auto_control/utils/agent_memory/agent_memory.py +++ b/je_auto_control/utils/agent_memory/agent_memory.py @@ -16,10 +16,14 @@ """ import json import re -import sqlite3 import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 _TOKEN = re.compile(r"[a-z0-9]+") @@ -40,7 +44,7 @@ def _tokens(text: str) -> List[str]: return _TOKEN.findall((text or "").lower()) -def _row_to_episode(row: sqlite3.Row, score: float = 0.0) -> Episode: +def _row_to_episode(row: "sqlite3.Row", score: float = 0.0) -> Episode: return Episode( id=int(row["id"]), goal=row["goal"], steps=json.loads(row["steps"] or "[]"), @@ -55,10 +59,11 @@ def __init__(self, db_path: str) -> None: self._db_path = db_path self._ensure_schema() - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self._db_path, timeout=30.0, - isolation_level=None) - conn.row_factory = sqlite3.Row + def _connect(self) -> "sqlite3.Connection": + driver = require_sqlite3() + conn = driver.connect(self._db_path, timeout=30.0, + isolation_level=None) + conn.row_factory = driver.Row return conn def _ensure_schema(self) -> None: @@ -131,7 +136,7 @@ def stats(self) -> Dict[str, int]: return {"episodes": int(row["c"])} -def _relevance(row: sqlite3.Row, terms: List[str]) -> float: +def _relevance(row: "sqlite3.Row", terms: List[str]) -> float: haystack = " ".join([row["goal"] or "", row["outcome"] or "", " ".join(json.loads(row["tags"] or "[]"))]) counts = _tokens(haystack) diff --git a/je_auto_control/utils/annotate/annotate.py b/je_auto_control/utils/annotate/annotate.py index 08cf5376..615c6577 100644 --- a/je_auto_control/utils/annotate/annotate.py +++ b/je_auto_control/utils/annotate/annotate.py @@ -13,18 +13,24 @@ {"type": "arrow", "start": [x1, y1], "end": [x2, y2], "color": [...]} {"type": "text", "position": [x, y], "text": "step 3", "color": [...]} """ +from __future__ import annotations + import io import math from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union, +) -from PIL import Image, ImageDraw +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image, ImageDraw -ImageSource = Union[str, Path, bytes, Image.Image] +ImageSource = Union[str, Path, bytes, "Image.Image"] -def _load_image(source: ImageSource) -> Image.Image: +def _load_image(source: ImageSource) -> "Image.Image": """Load ``source`` (path / bytes / PIL image) as an RGBA image.""" + from PIL import Image if isinstance(source, Image.Image): return source.convert("RGBA") if isinstance(source, bytes): @@ -84,6 +90,7 @@ def annotate_screenshot(source: ImageSource, PIL image; ``annotations`` is a list of box / highlight / arrow / text dicts. Unknown annotation types are ignored. """ + from PIL import Image, ImageDraw base = _load_image(source) highlight_layer = Image.new("RGBA", base.size, (0, 0, 0, 0)) highlight_draw = ImageDraw.Draw(highlight_layer) diff --git a/je_auto_control/utils/chatops/router.py b/je_auto_control/utils/chatops/router.py index 0521bdc7..ede88c72 100644 --- a/je_auto_control/utils/chatops/router.py +++ b/je_auto_control/utils/chatops/router.py @@ -10,12 +10,12 @@ from __future__ import annotations import shlex -import sqlite3 import threading from dataclasses import asdict, dataclass, field from typing import Any, Callable, Dict, List, Optional from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.sqlite_support import SQLITE_ERRORS # Built-in commands always available even when the operator only @@ -138,14 +138,14 @@ def _dispatch_argv(self, argv: List[str], ) # The router is the containment boundary for handler failures: a bad # script (AutoControlException) or a run-history read error - # (sqlite3.Error) must come back as a chat reply, not escape and kill + # (a sqlite3 error) must come back as a chat reply, not escape and kill # the transport's poll loop. try: return spec.handler(rest, context) except ChatOpsError as error: return CommandResult(text=f"{name}: {error}", succeeded=False) except (RuntimeError, OSError, ValueError, TypeError, LookupError, - AttributeError, AutoControlException, sqlite3.Error) as error: + AttributeError, AutoControlException, *SQLITE_ERRORS) as error: return CommandResult( text=f"{name} failed: {type(error).__name__}: {error}", succeeded=False, diff --git a/je_auto_control/utils/checkpoint/checkpoint.py b/je_auto_control/utils/checkpoint/checkpoint.py index 9bad332e..542c4802 100644 --- a/je_auto_control/utils/checkpoint/checkpoint.py +++ b/je_auto_control/utils/checkpoint/checkpoint.py @@ -11,10 +11,14 @@ without a real crash. """ import json -import sqlite3 import time from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 @dataclass @@ -33,10 +37,11 @@ def __init__(self, db_path: str) -> None: self._db_path = db_path self._ensure_schema() - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self._db_path, timeout=30.0, - isolation_level=None) - conn.row_factory = sqlite3.Row + def _connect(self) -> "sqlite3.Connection": + driver = require_sqlite3() + conn = driver.connect(self._db_path, timeout=30.0, + isolation_level=None) + conn.row_factory = driver.Row return conn def _ensure_schema(self) -> None: diff --git a/je_auto_control/utils/color_stats/color_stats.py b/je_auto_control/utils/color_stats/color_stats.py index 5c3469f1..a3839f19 100644 --- a/je_auto_control/utils/color_stats/color_stats.py +++ b/je_auto_control/utils/color_stats/color_stats.py @@ -7,15 +7,20 @@ representative. Pure Pillow — no Qt, no screen capture — so it is fully unit-testable. """ +from __future__ import annotations + import io from collections import Counter from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import ( + TYPE_CHECKING, Any, Dict, List, Optional, Sequence, Tuple, Union, +) -from PIL import Image +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image -ImageSource = Union[str, Path, bytes, Image.Image] +ImageSource = Union[str, Path, bytes, "Image.Image"] RGB = Tuple[int, int, int] @@ -32,7 +37,8 @@ def to_dict(self) -> Dict[str, Any]: return asdict(self) -def _load_rgb(source: ImageSource) -> Image.Image: +def _load_rgb(source: ImageSource) -> "Image.Image": + from PIL import Image if isinstance(source, Image.Image): return source.convert("RGB") if isinstance(source, bytes): diff --git a/je_auto_control/utils/config_bundle/config_bundle.py b/je_auto_control/utils/config_bundle/config_bundle.py index 126fc73b..53ef33a9 100644 --- a/je_auto_control/utils/config_bundle/config_bundle.py +++ b/je_auto_control/utils/config_bundle/config_bundle.py @@ -38,6 +38,7 @@ from pathlib import Path from typing import Any, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -60,7 +61,7 @@ } -class ConfigBundleError(Exception): +class ConfigBundleError(AutoControlException): """Raised when bundle parsing or writing fails in a recoverable way.""" diff --git a/je_auto_control/utils/cv2_utils/screen_record.py b/je_auto_control/utils/cv2_utils/screen_record.py index e46bd51a..bc4ddd72 100644 --- a/je_auto_control/utils/cv2_utils/screen_record.py +++ b/je_auto_control/utils/cv2_utils/screen_record.py @@ -1,6 +1,5 @@ import threading from typing import Dict, Tuple -import cv2 from je_auto_control.wrapper.auto_control_screen import screenshot @@ -69,6 +68,7 @@ class ScreenRecordThread(threading.Thread): def __init__(self, path_and_filename, codec, frame_per_sec, resolution: Tuple[int, int]): super().__init__() + import cv2 self.fourcc = cv2.VideoWriter.fourcc(*codec) self.video_writer = cv2.VideoWriter(path_and_filename, self.fourcc, frame_per_sec, resolution) # 用 Event 而非布林旗標:run() 之前呼叫 stop() 也能被遵守,不會被覆寫。 @@ -79,6 +79,7 @@ def __init__(self, path_and_filename, codec, frame_per_sec, resolution: Tuple[in self.resolution = resolution def run(self) -> None: + import cv2 try: while not self._stop_event.is_set(): # 擷取螢幕畫面 Capture screen frame diff --git a/je_auto_control/utils/cv2_utils/screenshot.py b/je_auto_control/utils/cv2_utils/screenshot.py index 3c6c84a4..649269f7 100644 --- a/je_auto_control/utils/cv2_utils/screenshot.py +++ b/je_auto_control/utils/cv2_utils/screenshot.py @@ -1,5 +1,9 @@ -from PIL import Image -from typing import List, Optional +from __future__ import annotations + +from typing import TYPE_CHECKING, List, Optional + +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber from je_auto_control.utils.exception.exceptions import AutoControlScreenException diff --git a/je_auto_control/utils/cv2_utils/template_detection.py b/je_auto_control/utils/cv2_utils/template_detection.py index 9c151460..98f27bd5 100644 --- a/je_auto_control/utils/cv2_utils/template_detection.py +++ b/je_auto_control/utils/cv2_utils/template_detection.py @@ -8,8 +8,6 @@ """ from typing import Any, List, Optional, Sequence, Tuple -from je_open_cv import template_detection - from je_auto_control.utils.monitor_layout.logical_frame import grab_logical @@ -54,6 +52,7 @@ def find_image(image: Any, detect_threshold: float = 1.0, :param screen_region: Limit the search to (x, y, width, height) 限定搜尋範圍 :return: [found, [x1, y1, x2, y2]] 座標為螢幕座標 """ + from je_open_cv import template_detection grab_image, origin_x, origin_y = grab_logical( screen_region, all_screens=all_screens) result = template_detection.find_object( @@ -80,6 +79,7 @@ def find_image_multi(image: Any, detect_threshold: float = 1.0, :param screen_region: Limit the search to (x, y, width, height) 限定搜尋範圍 :return: [found, [[x1, y1, x2, y2], ...]] 座標為螢幕座標 """ + from je_open_cv import template_detection grab_image, origin_x, origin_y = grab_logical( screen_region, all_screens=all_screens) result = template_detection.find_multi_object( diff --git a/je_auto_control/utils/cv2_utils/video_recording.py b/je_auto_control/utils/cv2_utils/video_recording.py index 3128b8c2..8df2d28f 100644 --- a/je_auto_control/utils/cv2_utils/video_recording.py +++ b/je_auto_control/utils/cv2_utils/video_recording.py @@ -1,6 +1,4 @@ import threading -import cv2 -import numpy as np from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -44,6 +42,8 @@ def run(self): 執行錄影迴圈 Run recording loop """ + import cv2 + import numpy as np with mss_grabber() as sct: resolution = sct.monitors[0] output_file = self.video_name + ".mp4" diff --git a/je_auto_control/utils/data_source/data_source.py b/je_auto_control/utils/data_source/data_source.py index 80f24de4..cc7b1f49 100644 --- a/je_auto_control/utils/data_source/data_source.py +++ b/je_auto_control/utils/data_source/data_source.py @@ -23,12 +23,13 @@ import csv import json import os -import sqlite3 from contextlib import closing from pathlib import Path from typing import Any, Callable, Dict, List, Optional from urllib.parse import quote +from je_auto_control.utils.sqlite_support import require_sqlite3 + _READ_ONLY_SQL_PREFIXES = ("select", "with") @@ -96,8 +97,9 @@ def _load_sqlite(source: Dict[str, Any]) -> List[Dict[str, Any]]: uri = f"file:{safe_path}?mode=ro" # closing(): the sqlite3 context manager commits/rolls back but never closes # the connection, so the file handle leaks until GC. Close it explicitly. - with closing(sqlite3.connect(uri, uri=True)) as conn: - conn.row_factory = sqlite3.Row + driver = require_sqlite3() + with closing(driver.connect(uri, uri=True)) as conn: + conn.row_factory = driver.Row rows = conn.execute(query).fetchall() return [dict(row) for row in rows] diff --git a/je_auto_control/utils/dbus_client/__init__.py b/je_auto_control/utils/dbus_client/__init__.py new file mode 100644 index 00000000..6dd33b6e --- /dev/null +++ b/je_auto_control/utils/dbus_client/__init__.py @@ -0,0 +1,24 @@ +"""A minimal D-Bus session-bus client, in the standard library alone. + +Written for the XDG portal handshake — see :mod:`.session_bus` for why that +could not be a shell-out to ``gdbus`` — and kept general enough for anything +else that has to speak D-Bus without pulling in a binding. AT-SPI2, the Linux +accessibility bus, is the second caller. + +It lives here rather than under ``linux_wayland/`` because ``utils/`` sits +above the per-OS packages in this project's layering: an accessibility backend +in ``utils/`` reaching down into a platform backend to borrow its D-Bus code +would invert that. ``je_auto_control.linux_wayland._dbus_client`` re-exports +this module, so the Wayland code and its tests are unchanged. +""" +from je_auto_control.utils.dbus_client.session_bus import ( + BUS_INTERFACE, BUS_NAME, BUS_PATH, DBusError, ERROR, METHOD_CALL, + METHOD_RETURN, Message, SIGNAL, SessionBus, Variant, body_pairs, + is_available, session_address, +) + +__all__ = [ + "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", + "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", + "Variant", "body_pairs", "is_available", "session_address", +] diff --git a/je_auto_control/utils/dbus_client/session_bus.py b/je_auto_control/utils/dbus_client/session_bus.py new file mode 100644 index 00000000..d5777292 --- /dev/null +++ b/je_auto_control/utils/dbus_client/session_bus.py @@ -0,0 +1,656 @@ +"""A minimal D-Bus session-bus client, in the standard library alone. + +This exists because of one property of the XDG portal protocol: a portal call +returns only a *request handle*, and the answer arrives later as a ``Response`` +signal — **directed at the unique bus name that made the call**. The bus routes +a directed message to its destination and nowhere else, so a listener on a +second connection never receives it, whatever match rules it adds. + +That rules out the obvious shell-out. ``gdbus call`` and ``gdbus monitor`` are +two processes with two unique names, so the monitor is not the caller and the +Response is not addressed to it; measured against a real ``dbus-daemon``, it +sees the call go past and the answer never arrive. The only listener that +does see it is a full bus monitor (``dbus-monitor``, which asks the bus for +``BecomeMonitor``), and making a screenshot require permission to observe every +message on the user's session bus is a poor trade for a fallback tier. + +So the subscription and the call happen on one connection here, which is what +every portal client library does and what the protocol is designed for. The +cost is marshalling D-Bus by hand; the scope is kept to exactly what a portal +conversation needs — connect, authenticate, ``Hello``, ``AddMatch``, one method +call, and read signals until the matching one arrives. It is deliberately not a +general D-Bus binding: no properties, no introspection, no object export, no +file-descriptor passing (:mod:`je_auto_control.linux_wayland.oeffis` uses +liboeffis for the one call that needs that). + +Every failure is an :class:`AutoControlException` subclass, so the containment +boundaries that catch the family keep working. +""" +from __future__ import annotations + +import contextlib +import os +import socket +import struct +import time +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: Message types, from the D-Bus specification. +METHOD_CALL = 1 +METHOD_RETURN = 2 +ERROR = 3 +SIGNAL = 4 + +#: The fixed-width numeric types, as ``struct`` format and byte width. Each is +#: aligned to its own width on the wire. +#: +#: ``i`` matters more than it looks: AT-SPI reports a component's extents as +#: four *signed* integers, because a window on a monitor left of or above the +#: primary one is at a negative coordinate. Without it the accessibility +#: backend could read a tree but not where anything was. +_FIXED = { + "n": (" None: + self.signature = signature + self.value = value + + def __repr__(self) -> str: + return f"Variant({self.signature!r}, {self.value!r})" + + +# --- marshalling ---------------------------------------------------------- + + +class _Writer: + """Little-endian marshaller. Alignment is relative to the message start.""" + + def __init__(self, offset: int = 0) -> None: + self._parts: List[bytes] = [] + self._length = offset + + def align(self, boundary: int) -> None: + padding = (-self._length) % boundary + if padding: + self._parts.append(bytes(padding)) + self._length += padding + + def raw(self, data: bytes) -> None: + self._parts.append(data) + self._length += len(data) + + def byte(self, value: int) -> None: + self.raw(struct.pack(" None: + self.align(4) + self.raw(struct.pack(" None: + """Write one fixed-width number, aligned to its own width.""" + fmt, width = _FIXED[code] + self.align(width) + self.raw(struct.pack(fmt, float(value) if fmt == " None: + encoded = value.encode("utf-8") + self.uint32(len(encoded)) + self.raw(encoded + b"\x00") + + def signature(self, value: str) -> None: + encoded = value.encode("ascii") + self.byte(len(encoded)) + self.raw(encoded + b"\x00") + + def value(self, sig: str, value: Any) -> None: + """Write one complete value of the given single signature.""" + _write_value(self, _SignatureReader(sig), value) + + @property + def data(self) -> bytes: + return b"".join(self._parts) + + def __len__(self) -> int: + return self._length + + +class _SignatureReader: + """Walks a signature string one complete type at a time.""" + + def __init__(self, text: str) -> None: + self.text = text + self.index = 0 + + def done(self) -> bool: + return self.index >= len(self.text) + + def peek(self) -> str: + return self.text[self.index] + + def take(self) -> str: + code = self.text[self.index] + self.index += 1 + return code + + def take_complete(self) -> str: + """Take one whole type, following containers to their close.""" + start = self.index + code = self.take() + if code == "a": + self.take_complete() + elif code in "({": + closing = ")" if code == "(" else "}" + while self.peek() != closing: + self.take_complete() + self.take() + return self.text[start:self.index] + + +def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + code = reader.take() + if code == "y": + writer.byte(int(value)) + elif code == "b": + writer.uint32(1 if value else 0) + elif code in _FIXED: + writer.fixed(code, value) + elif code in "so": + writer.string(str(value)) + elif code == "g": + writer.signature(str(value)) + elif code == "v": + _write_variant(writer, value) + elif code == "a": + _write_array(writer, reader, value) + elif code in "({": + _write_struct(writer, reader, code, value) + else: + raise DBusError(f"cannot marshal D-Bus type {code!r}") + + +def _write_variant(writer: _Writer, value: Any) -> None: + variant = value if isinstance(value, Variant) else _guess_variant(value) + writer.signature(variant.signature) + writer.value(variant.signature, variant.value) + + +def _guess_variant(value: Any) -> Variant: + if isinstance(value, bool): + return Variant("b", value) + if isinstance(value, int): + return Variant("u", value) + if isinstance(value, str): + return Variant("s", value) + raise DBusError(f"no obvious D-Bus type for {type(value).__name__}") + + +def _write_array(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + element = reader.take_complete() + writer.align(4) + # The length prefix counts the elements only, and it is written before + # the padding that aligns the first one — so the body is built separately + # at the offset it will actually occupy. + body_start = len(writer) + 4 + padding = (-body_start) % _ALIGNMENT.get(element[0], 1) + body = _Writer(body_start + padding) + items = value.items() if isinstance(value, dict) else value + for item in items: + body.align(_ALIGNMENT.get(element[0], 1)) + _write_value(body, _SignatureReader(element), item) + writer.uint32(len(body) - body_start - padding) + writer.raw(bytes(padding)) + writer.raw(body.data) + + +def _write_struct(writer: _Writer, reader: _SignatureReader, code: str, + value: Any) -> None: + closing = ")" if code == "(" else "}" + writer.align(8) + for item in value: + if reader.peek() == closing: + raise DBusError("too many members for this struct signature") + _write_value(writer, reader, item) + if reader.peek() != closing: + raise DBusError("too few members for this struct signature") + reader.take() + + +class _Reader: + """Little-endian demarshaller over one complete message.""" + + def __init__(self, data: bytes, offset: int = 0) -> None: + self.data = data + self.offset = offset + + def align(self, boundary: int) -> None: + self.offset += (-self.offset) % boundary + + def take(self, count: int) -> bytes: + if self.offset + count > len(self.data): + raise DBusError("truncated D-Bus message") + chunk = self.data[self.offset:self.offset + count] + self.offset += count + return chunk + + def byte(self) -> int: + return self.take(1)[0] + + def uint32(self) -> int: + self.align(4) + return struct.unpack(" Any: + """Read one fixed-width number, aligned to its own width.""" + fmt, width = _FIXED[code] + self.align(width) + return struct.unpack(fmt, self.take(width))[0] + + def string(self) -> str: + length = self.uint32() + text = self.take(length).decode("utf-8", errors="replace") + self.take(1) + return text + + def signature(self) -> str: + length = self.byte() + text = self.take(length).decode("ascii", errors="replace") + self.take(1) + return text + + def value(self, sig: str) -> Any: + return _read_value(self, _SignatureReader(sig)) + + +def _read_value(reader: _Reader, signature: _SignatureReader) -> Any: + code = signature.take() + if code == "y": + return reader.byte() + if code == "b": + return bool(reader.uint32()) + if code in _FIXED: + return reader.fixed(code) + if code in "so": + return reader.string() + if code == "g": + return reader.signature() + if code == "v": + return reader.value(reader.signature()) + if code == "a": + return _read_array(reader, signature) + if code in "({": + return _read_struct(reader, signature, code) + raise DBusError(f"cannot demarshal D-Bus type {code!r}") + + +def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: + element = signature.take_complete() + length = reader.uint32() + reader.align(_ALIGNMENT.get(element[0], 1)) + end = reader.offset + length + items = [] + while reader.offset < end: + reader.align(_ALIGNMENT.get(element[0], 1)) + items.append(_read_value(reader, _SignatureReader(element))) + if element.startswith("{"): + return dict(items) + return items + + +def _read_struct(reader: _Reader, signature: _SignatureReader, + code: str) -> tuple: + closing = ")" if code == "(" else "}" + reader.align(8) + members = [] + while signature.peek() != closing: + members.append(_read_value(reader, signature)) + signature.take() + return tuple(members) + + +class Message: + """One decoded D-Bus message: the header fields that matter, and the body.""" + + __slots__ = ("type", "serial", "fields", "body") + + def __init__(self, message_type: int, serial: int, + fields: Dict[int, Any], body: List[Any]) -> None: + self.type = message_type + self.serial = serial + self.fields = fields + self.body = body + + @property + def path(self) -> str: + return self.fields.get(FIELD_PATH, "") + + @property + def interface(self) -> str: + return self.fields.get(FIELD_INTERFACE, "") + + @property + def member(self) -> str: + return self.fields.get(FIELD_MEMBER, "") + + @property + def reply_serial(self) -> int: + return self.fields.get(FIELD_REPLY_SERIAL, 0) + + @property + def error_name(self) -> str: + return self.fields.get(FIELD_ERROR_NAME, "") + + +# --- the connection ------------------------------------------------------- + + +def session_address() -> Optional[str]: + """The session bus address, or None when this process has no session bus.""" + address = os.environ.get(_ADDRESS_ENV, "").strip() + return address or None + + +def is_available() -> bool: + """Whether a session bus address is set, so connecting is worth trying.""" + return session_address() is not None + + +def _socket_target(address: str) -> Tuple[str, bool]: + """Pick a connectable ``unix:`` transport out of a bus address. + + :return: the socket path and whether it is in the abstract namespace. + """ + for candidate in address.split(";"): + if not candidate.startswith("unix:"): + continue + fields = [part.split("=", 1) + for part in candidate[len("unix:"):].split(",") + if "=" in part] + options = dict(fields) + if "path" in options: + return options["path"], False + if "abstract" in options: + return options["abstract"], True + raise DBusError(f"no usable unix transport in {address!r}") + + +class SessionBus: + """An authenticated connection to the session bus, as a context manager.""" + + def __init__(self, address: Optional[str] = None) -> None: + self.address = address or session_address() + self.unique_name = "" + self._socket: Optional[socket.socket] = None + self._serial = 0 + self._buffer = b"" + #: Messages read while waiting for a method reply. A portal's Response + #: signal and its method return come from two different senders, so + #: the bus gives no ordering between them and the signal can arrive + #: first — dropping it here would be a wait that never ends. + self._queued: List[Message] = [] + + # --- lifecycle -------------------------------------------------------- + + def __enter__(self) -> "SessionBus": + self.connect() + return self + + def __exit__(self, *_exception: Any) -> None: + self.close() + + def connect(self) -> None: + """Open the socket, authenticate, and say ``Hello``.""" + if self.address is None: + raise DBusError( + f"{_ADDRESS_ENV} is not set, so there is no session bus to " + f"reach the desktop portal on", + ) + path, abstract = _socket_target(self.address) + try: + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._socket.connect(("\0" + path) if abstract else path) + except OSError as error: + self.close() + raise DBusError(f"cannot reach the session bus: {error}") from error + self._authenticate() + self.unique_name = self.call( + BUS_NAME, BUS_PATH, BUS_INTERFACE, "Hello", "", [])[0] + + def close(self) -> None: + """Drop the connection; never raise from teardown.""" + if self._socket is not None: + with contextlib.suppress(OSError): + self._socket.close() + self._socket = None + + @property + def sender_token(self) -> str: + """This connection's name as the portal spells it inside object paths.""" + return self.unique_name.lstrip(":").replace(".", "_") + + # --- authentication --------------------------------------------------- + + def _authenticate(self) -> None: + """The SASL EXTERNAL handshake, which is a uid in hex over a socket.""" + uid = str(os.getuid()).encode("ascii") + self._send_raw(b"\x00AUTH EXTERNAL " + uid.hex().encode("ascii") + + b"\r\n") + reply = self._read_line() + if not reply.startswith("OK"): + raise DBusError(f"the session bus refused authentication: {reply}") + self._send_raw(b"BEGIN\r\n") + + def _send_raw(self, data: bytes) -> None: + if self._socket is None: + raise DBusError("the session bus connection is closed") + try: + self._socket.sendall(data) + except OSError as error: + raise DBusError(f"writing to the session bus failed: {error}") from error + + def _read_line(self, timeout: float = 10.0) -> str: + """Read one CRLF-terminated line of the auth conversation.""" + deadline = time.monotonic() + timeout + while b"\r\n" not in self._buffer: + self._fill(deadline - time.monotonic()) + line, _, self._buffer = self._buffer.partition(b"\r\n") + return line.decode("utf-8", errors="replace") + + # --- reading ---------------------------------------------------------- + + def _fill(self, timeout: float) -> None: + """Read whatever is available, or fail once the deadline has passed.""" + if self._socket is None: + raise DBusError("the session bus connection is closed") + if timeout <= 0: + raise DBusError("the session bus did not answer in time") + self._socket.settimeout(timeout) + try: + chunk = self._socket.recv(65536) + except socket.timeout as error: + raise DBusError("the session bus did not answer in time") from error + except OSError as error: + raise DBusError(f"reading from the session bus failed: {error}") from error + if not chunk: + raise DBusError("the session bus closed the connection") + self._buffer += chunk + + def read_message(self, deadline: float) -> Message: + """Read one whole message, or fail when the deadline passes.""" + while len(self._buffer) < 16: + self._fill(deadline - time.monotonic()) + endian, message_type, _flags, _version = struct.unpack( + " _MAX_MESSAGE: + raise DBusError("the session bus sent an implausibly large message") + while len(self._buffer) < total: + self._fill(deadline - time.monotonic()) + raw, self._buffer = self._buffer[:total], self._buffer[total:] + return _decode(raw, message_type, serial, fields_length, header_end) + + # --- writing ---------------------------------------------------------- + + def _next_serial(self) -> int: + self._serial += 1 + return self._serial + + def send(self, message_type: int, fields: Dict[int, Tuple[str, Any]], + signature: str, body: List[Any], flags: int = 0) -> int: + """Marshal and send one message; return its serial.""" + payload = _Writer() + for code, value in body_pairs(signature, body): + payload.value(code, value) + serial = self._next_serial() + header = _Writer() + header.raw(struct.pack(" List[Any]: + """Make a method call and return the reply body.""" + fields = { + FIELD_PATH: ("o", path), + FIELD_INTERFACE: ("s", interface), + FIELD_MEMBER: ("s", member), + FIELD_DESTINATION: ("s", destination), + } + if signature: + fields[FIELD_SIGNATURE] = ("g", signature) + serial = self.send(METHOD_CALL, fields, signature, body) + deadline = time.monotonic() + timeout + while True: + message = self.read_message(deadline) + if message.reply_serial != serial: + self._queued.append(message) + continue + if message.type == ERROR: + detail = message.body[0] if message.body else "" + raise DBusError(f"{message.error_name}: {detail}".strip(": ")) + if message.type == METHOD_RETURN: + return message.body + + def add_match(self, rule: str) -> None: + """Subscribe to signals, and wait for the bus to confirm the rule. + + Waiting matters twice over: a malformed rule is reported rather than + silently never matching, and the round trip proves the subscription is + in place before the call that provokes the signal is made. + """ + self.call(BUS_NAME, BUS_PATH, BUS_INTERFACE, "AddMatch", "s", [rule]) + + def wait_for_signal(self, paths: List[str], interface: str, member: str, + timeout: float) -> List[Any]: + """Read until the signal we subscribed to arrives, or time out.""" + def matches(message: Message) -> bool: + return (message.type == SIGNAL and message.member == member + and message.interface == interface + and message.path in paths) + + for index, message in enumerate(self._queued): + if matches(message): + del self._queued[index] + return message.body + self._queued.clear() + deadline = time.monotonic() + max(0.0, timeout) + while True: + message = self.read_message(deadline) + if matches(message): + return message.body + + +def body_pairs(signature: str, body: List[Any]): + """Pair each top-level type in a signature with its argument.""" + reader = _SignatureReader(signature) + for value in body: + if reader.done(): + raise DBusError("more arguments than the signature declares") + yield reader.take_complete(), value + if not reader.done(): + raise DBusError("fewer arguments than the signature declares") + + +def _decode(raw: bytes, message_type: int, serial: int, fields_length: int, + header_end: int) -> Message: + """Turn one complete message's bytes into a :class:`Message`.""" + reader = _Reader(raw, 12) + fields: Dict[int, Any] = {} + reader.uint32() # the header array's own length, already read + end = 16 + fields_length + while reader.offset < end: + reader.align(8) + code = reader.byte() + fields[code] = reader.value(reader.signature()) + reader.offset = header_end + ((-header_end) % 8) + body: List[Any] = [] + signature = fields.get(FIELD_SIGNATURE, "") + if signature: + walker = _SignatureReader(signature) + while not walker.done(): + body.append(_read_value(reader, _SignatureReader( + walker.take_complete()))) + return Message(message_type, serial, fields, body) + + +__all__ = [ + "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", + "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", + "Variant", "is_available", "session_address", +] diff --git a/je_auto_control/utils/diagnostics/diagnostics.py b/je_auto_control/utils/diagnostics/diagnostics.py index 4b214995..71ce1e5c 100644 --- a/je_auto_control/utils/diagnostics/diagnostics.py +++ b/je_auto_control/utils/diagnostics/diagnostics.py @@ -99,6 +99,7 @@ def _check_optional_deps() -> Check: ("pytesseract", "OCR engine"), ("cv2", "image recognition"), ("PySide6", "GUI"), + ("sqlite3", "run history / checkpoints / work queue / SQL sources"), ) available, missing = [], [] for module_name, purpose in optional_modules: @@ -122,6 +123,15 @@ def _check_optional_deps() -> Check: def _check_audit_chain() -> Check: from je_auto_control.utils.remote_desktop.audit_log import default_audit_log + from je_auto_control.utils.sqlite_support import sqlite3_available + + # Without sqlite3 there is no chain to verify. Reporting that as a + # broken chain would be a false alarm about tampering. + if not sqlite3_available(): + return Check( + name="audit_chain", ok=True, severity=_SEVERITY_WARN, + detail="no audit log on this Python: it has no sqlite3", + ) result = default_audit_log().verify_chain() if result.ok: return Check( diff --git a/je_auto_control/utils/exception/exception_tags.py b/je_auto_control/utils/exception/exception_tags.py index d997f152..62c968c9 100644 --- a/je_auto_control/utils/exception/exception_tags.py +++ b/je_auto_control/utils/exception/exception_tags.py @@ -6,7 +6,9 @@ linux_import_error_message: str = "Should only be loaded on Linux" osx_import_error_message: str = "Should only be loaded on macOS" windows_import_error_message: str = "Should only be loaded on Windows" -macos_record_error_message: str = "Cannot use recorder on macOS" +macos_record_error_message: str = ( + "Cannot record on macOS without Accessibility permission " + "(System Settings -> Privacy & Security -> Accessibility)") # keyboard tags keyboard_error_message: str = "Auto-control keyboard error" diff --git a/je_auto_control/utils/exception/exceptions.py b/je_auto_control/utils/exception/exceptions.py index ff700e48..a5fc7cf1 100644 --- a/je_auto_control/utils/exception/exceptions.py +++ b/je_auto_control/utils/exception/exceptions.py @@ -114,3 +114,17 @@ class XMLTypeException(AutoControlException): # Execute callback class CallbackExecutorException(AutoControlException): pass + + +# Platform capability +class AutoControlUnsupportedOperationException( + AutoControlException, NotImplementedError): + """An operation the current platform's backend cannot perform. + + Inherits both on purpose. ``NotImplementedError`` is what the GUI tabs and + the REST handler already catch to say "not on this platform", and that + behaviour is kept; ``AutoControlException`` is what the executor and the + other containment boundaries catch, and a bare ``NotImplementedError`` + slipped straight past all of them — aborting a whole script where a single + action should have been reported as failed. + """ diff --git a/je_auto_control/utils/input_macro/__init__.py b/je_auto_control/utils/input_macro/__init__.py index 9e7a3729..03f24968 100644 --- a/je_auto_control/utils/input_macro/__init__.py +++ b/je_auto_control/utils/input_macro/__init__.py @@ -1,6 +1,10 @@ -"""Timed input-event replay and a declarative input-sequence DSL.""" +"""Timed input events: capture shaping, replay, and an input-sequence DSL.""" from je_auto_control.utils.input_macro.input_macro import ( replay_timeline, run_sequence, ) +from je_auto_control.utils.input_macro.recorder_base import ( + InputRecorder, legacy_action_queue, timeline, +) -__all__ = ["replay_timeline", "run_sequence"] +__all__ = ["InputRecorder", "legacy_action_queue", "replay_timeline", + "run_sequence", "timeline"] diff --git a/je_auto_control/utils/input_macro/input_macro.py b/je_auto_control/utils/input_macro/input_macro.py index b17c2492..b5f0b82d 100644 --- a/je_auto_control/utils/input_macro/input_macro.py +++ b/je_auto_control/utils/input_macro/input_macro.py @@ -33,7 +33,12 @@ def _sink_click(event: Dict[str, Any]) -> None: def _sink_scroll(event: Dict[str, Any]) -> None: from je_auto_control.wrapper.auto_control_mouse import mouse_scroll - mouse_scroll(int(event.get("value", 1))) + # ``value`` is the DSL's name for it, ``delta`` the recorder's. The sign + # is kept: it is what decides the direction on Windows and macOS. Linux + # takes its direction from `scroll_direction` instead, which is an open + # cross-platform decision recorded in Progress.md, not something to + # settle here. + mouse_scroll(int(event.get("value", event.get("delta", 1)))) def _sink_press(event: Dict[str, Any]) -> None: @@ -52,9 +57,46 @@ def _sink_key(event: Dict[str, Any]) -> None: type_keyboard(event["key"]) +# The recorder names a button "left"; the input API names it "mouse_left". +_RECORDED_BUTTON = {"left": "mouse_left", "right": "mouse_right", + "middle": "mouse_middle"} + + +def _sink_key_down(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_keyboard import press_keyboard_key + press_keyboard_key(event["vk"]) + + +def _sink_key_up(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_keyboard import ( + release_keyboard_key) + release_keyboard_key(event["vk"]) + + +def _sink_mouse_down(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_mouse import press_mouse + press_mouse(_RECORDED_BUTTON.get(event.get("button", ""), "mouse_left"), + int(event.get("x", 0)), int(event.get("y", 0))) + + +def _sink_mouse_up(event: Dict[str, Any]) -> None: + from je_auto_control.wrapper.auto_control_mouse import release_mouse + release_mouse(_RECORDED_BUTTON.get(event.get("button", ""), "mouse_left"), + int(event.get("x", 0)), int(event.get("y", 0))) + + +#: Two vocabularies reach this table and both have to work. The `run_sequence` +#: DSL writes ``press`` / ``click`` / ``key``; the recorders write +#: ``key_down`` / ``mouse_up`` and so on. They used to be disjoint, so feeding +#: ``stop_record_timeline()`` to :func:`replay_timeline` — the pipeline the +#: docstrings and the ``ac_record_stop_timeline`` tool both prescribe — +#: matched nothing and replayed an empty session while reporting the full +#: event count as played. _SINKS: Dict[str, Callable[[Dict[str, Any]], None]] = { "move": _sink_move, "click": _sink_click, "scroll": _sink_scroll, "press": _sink_press, "release": _sink_release, "key": _sink_key, + "key_down": _sink_key_down, "key_up": _sink_key_up, + "mouse_down": _sink_mouse_down, "mouse_up": _sink_mouse_up, } diff --git a/je_auto_control/utils/input_macro/recorder_base.py b/je_auto_control/utils/input_macro/recorder_base.py new file mode 100644 index 00000000..26140afe --- /dev/null +++ b/je_auto_control/utils/input_macro/recorder_base.py @@ -0,0 +1,169 @@ +"""Platform-neutral shaping of recorded input, and the recorder base class. + +Windows and macOS capture input through entirely different OS machinery — a +low-level ``WH_KEYBOARD_LL`` hook driven by a message pump, and a Quartz +``CGEventTap`` driven by a run loop — but everything *after* the capture is +identical: the same event dictionaries, the same three ways of asking for +them, and the same two output shapes. That part belongs here rather than +being copied into the second backend. + +* :func:`timeline` turns raw timestamped events into the ``delta_ms`` form + :func:`je_auto_control.utils.input_macro.replay_timeline` plays back. +* :func:`legacy_action_queue` produces the historical down-events-only queue + that :func:`je_auto_control.wrapper.auto_control_record.stop_record` and the + executor have always been handed. +* :class:`InputRecorder` is the ``record`` / ``record_mouse`` / + ``record_keyboard`` surface every platform recorder exposes. A backend + supplies only :meth:`InputRecorder.new_hook`. + +The divergence this prevents is not hypothetical and would be silent: a +recording made on one OS has to replay on the other, and two hand-written +copies of the queue shaping drift without anything going red. + +**Everything typed while recording is captured, passwords included.** Callers +must treat the result as sensitive. +""" +from queue import Queue +from typing import Any, Dict, List, Optional, Sequence, Tuple + +#: A hook left installed grows its event list forever. Recording is bounded so +#: a forgotten session cannot consume the process. +MAX_EVENTS = 20000 + +#: Legacy queue entries: the down-event half, shaped as executor commands. +LEGACY_MOUSE_COMMAND = {"left": "AC_mouse_left", "right": "AC_mouse_right", + "middle": "AC_mouse_middle"} + +#: Both capture kinds, which is what a plain ``record()`` asks for. +ALL_KINDS: Tuple[str, ...] = ("keyboard", "mouse") + + +def event_kind(event: Dict[str, Any]) -> str: + """Return ``"keyboard"`` or ``"mouse"`` for one raw event.""" + return "keyboard" if str(event.get("op", "")).startswith("key_") else "mouse" + + +def timeline(events: List[Dict[str, Any]]) -> List[Dict[str, Any]]: + """Convert raw events to ``delta_ms`` form for ``replay_timeline``. + + The first event has no gap before it; every later one carries the real + pause that preceded it, which is what makes a replay track the original + pace. + """ + out: List[Dict[str, Any]] = [] + previous: Optional[float] = None + for event in events: + moment = float(event.get("time", 0.0)) + item = {key: value for key, value in event.items() if key != "time"} + item["delta_ms"] = 0 if previous is None else max( + 0, int((moment - previous) * 1000)) + out.append(item) + previous = moment + return out + + +def legacy_action_queue(events: List[Dict[str, Any]]) -> Queue: + """The historical shape: one executor command per press, no releases.""" + queue: Queue = Queue() + for event in events: + operation = event.get("op") + if operation == "key_down": + queue.put(("AC_type_keyboard", int(event.get("vk", 0)))) + elif operation == "mouse_down": + command = LEGACY_MOUSE_COMMAND.get(event.get("button", "")) + if command: + queue.put((command, int(event.get("x", 0)), + int(event.get("y", 0)))) + return queue + + +class InputRecorder: + """``record`` / ``stop_record`` over a platform hook, with timeline output. + + ``stop_record`` keeps returning the down-events-only queue so existing + callers are unaffected; ``stop_record_timeline`` returns everything a + replay needs — releases, wheel movement and per-event gaps. + """ + + def __init__(self) -> None: + self.hook: Any = None + self.record_queue: Optional[Queue] = None + self.result_queue: Optional[Queue] = None + self._kinds: Tuple[str, ...] = ALL_KINDS + + def new_hook(self) -> Any: + """Return a fresh, unstarted platform hook. Backends override this.""" + raise NotImplementedError + + # -- capture ----------------------------------------------------------- + def _start(self, kinds: Sequence[str]) -> None: + self._kinds = tuple(kinds) + self.hook = self.new_hook() + self.record_queue = Queue() + self.hook.start() + + def _stop(self) -> List[Dict[str, Any]]: + if self.hook is None: + return [] + events = [event for event in self.hook.stop() + if event_kind(event) in self._kinds] + self.hook = None + return events + + def _as_queue(self, events: List[Dict[str, Any]]) -> Queue: + queue = legacy_action_queue(events) + self.record_queue = None + self.result_queue = queue + return queue + + # -- public ------------------------------------------------------------ + def record(self) -> None: + """ + 開始錄製滑鼠與鍵盤事件 + Start recording both mouse and keyboard events + """ + self._start(ALL_KINDS) + + def stop_record(self) -> Queue: + """ + 停止錄製並回傳事件 + Stop recording and return recorded events + """ + return self._as_queue(self._stop()) + + def stop_record_timeline(self) -> List[Dict[str, Any]]: + """ + 停止錄製並回傳含放開、滾輪與間隔時間的完整事件 + Stop recording and return press *and* release, wheel and ``delta_ms`` + + Ready for :func:`je_auto_control.utils.input_macro.replay_timeline`. + """ + return timeline(self._stop()) + + def record_mouse(self) -> None: + """ + 開始錄製滑鼠事件 + Start recording mouse events + """ + self._start(("mouse",)) + + def stop_record_mouse(self) -> Queue: + """ + 停止錄製滑鼠事件並回傳結果 + Stop recording mouse events and return results + """ + return self._as_queue(self._stop()) + + def record_keyboard(self) -> None: + """ + 開始錄製鍵盤事件 + Start recording keyboard events + """ + self._start(("keyboard",)) + + def stop_record_keyboard(self) -> Queue: + """ + 停止錄製鍵盤事件並回傳結果 + Stop recording keyboard events and return results + """ + return self._as_queue(self._stop()) diff --git a/je_auto_control/utils/mcp_server/_protocol.py b/je_auto_control/utils/mcp_server/_protocol.py index f5e1502e..7c7f1381 100644 --- a/je_auto_control/utils/mcp_server/_protocol.py +++ b/je_auto_control/utils/mcp_server/_protocol.py @@ -10,7 +10,6 @@ """ import json import os -import sqlite3 import subprocess # nosec B404 # reason: only its TimeoutExpired type is referenced import sys import time @@ -19,6 +18,7 @@ from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.mcp_server.tools import MCPContent +from je_auto_control.utils.sqlite_support import SQLITE_ERRORS PROTOCOL_VERSION = "2025-06-18" @@ -34,7 +34,7 @@ # family base every ``AutoControl*Exception``/``ImageNotFoundException`` now # derives from. _FRAMEWORK_TOOL_ERRORS = ( - AutoControlException, subprocess.TimeoutExpired, sqlite3.Error, + AutoControlException, subprocess.TimeoutExpired, *SQLITE_ERRORS, ) _BUILTIN_DISPATCH_ERRORS = ( OSError, RuntimeError, ValueError, TypeError, KeyError, diff --git a/je_auto_control/utils/mcp_server/http_sessions.py b/je_auto_control/utils/mcp_server/http_sessions.py new file mode 100644 index 00000000..4e988410 --- /dev/null +++ b/je_auto_control/utils/mcp_server/http_sessions.py @@ -0,0 +1,234 @@ +"""MCP session identity for the HTTP transport. + +The Streamable HTTP transport used to scope every peer on the identity of +its TCP connection. That is not what MCP means by a session: a client is +free to send ``initialize`` on one connection and ``tools/call`` on the +next, and an SSE response closes its connection after the final event. Any +state established at ``initialize`` — most importantly the ``elicitation`` +capability the destructive-action confirmation gate depends on — was +therefore gone by the time it was needed. + +This module holds the identity that outlives a connection: a registry of +``Mcp-Session-Id`` values, each owning the optional standing server-to-client +SSE stream opened by ``GET``. The dispatcher itself needs no changes — it +already scopes capabilities and active calls on an opaque ``connection_id``, +so a session id simply takes that slot. + +Sessions are bounded in both directions: idle ones are swept, and the +registry never grows past ``max_sessions``. Whenever a session is dropped +the registry reports it through ``on_drop`` so the transport can release the +dispatcher state held under that id. +""" +import secrets +import threading +import time +from typing import Any, Callable, Dict, List, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + +# MCP names the header in this casing; HTTP lookups are case-insensitive. +SESSION_HEADER = "Mcp-Session-Id" +# A session costs one dict entry plus at most one streaming thread, but an +# unbounded registry is still a memory target for an unauthenticated peer. +DEFAULT_MAX_SESSIONS = 128 +# How long a session may go untouched before it is swept. Clients that keep a +# standing GET stream open touch it on every heartbeat, so this bounds only +# genuinely abandoned sessions. +DEFAULT_IDLE_TIMEOUT = 600.0 +# Bytes of entropy in a session id. The spec asks for cryptographically +# secure and globally unique; 32 bytes is comfortably both. +_ID_ENTROPY_BYTES = 32 + + +class HttpSession: + """One MCP session: an identity, plus its optional outbound stream. + + The stream is the ``GET`` SSE channel the client may open to receive + server-initiated traffic — progress notifications, and the + ``elicitation/create`` request the confirmation gate sends. Without it + the server has no way to ask the client anything between a request and + its response, which is why a session with no stream still cannot be + prompted. + """ + + def __init__(self, session_id: str, now: float) -> None: + self.id = session_id + self.created_at = now + self.last_seen = now + self.closed = threading.Event() + self._lock = threading.Lock() + self._stream_writer: Optional[Callable[[str], None]] = None + + @property + def has_stream(self) -> bool: + """True while a standing server-to-client SSE stream is attached.""" + with self._lock: + return self._stream_writer is not None + + @property + def stream_writer(self) -> Optional[Callable[[str], None]]: + """The attached stream's emit callable, or ``None``.""" + with self._lock: + return self._stream_writer + + def attach_stream(self, writer: Callable[[str], None]) -> bool: + """Attach the standing stream; False when one is already attached. + + Refusing the second stream is deliberate. Two streams on one session + would make "which socket does this elicitation go down" ambiguous, + and the client cannot tell which one the server picked. + """ + with self._lock: + if self._stream_writer is not None: + return False + self._stream_writer = writer + return True + + def detach_stream(self, writer: Callable[[str], None]) -> None: + """Detach ``writer`` if it is still the attached one.""" + with self._lock: + if self._stream_writer is writer: + self._stream_writer = None + + +def _log_eviction(victim: HttpSession, cap: int) -> None: + """Report an eviction, loudly only when the victim was in use. + + Every ``initialize`` mints a session, including for the many clients that + ignore the header and never come back. Evicting one of those is routine + capacity work; evicting a session a client is actually holding means the + cap is too low for the load, and that is worth a warning. + """ + abandoned = not victim.has_stream and victim.last_seen == victim.created_at + if abandoned: + autocontrol_logger.info( + "MCP session cap %d reached — evicting a session that was never " + "used after initialize", cap, + ) + return + autocontrol_logger.warning( + "MCP session cap %d reached — evicting a live session (%s); raise " + "the cap if clients are being dropped mid-conversation", + cap, victim.id, + ) + + +class SessionRegistry: + """Bounded, sweeping registry of :class:`HttpSession` by id. + + ``on_drop`` is invoked — outside the registry lock — for every session + the registry removes, whichever way it goes: explicit termination, idle + sweep, capacity eviction or shutdown. The transport uses it to release + the dispatcher state scoped to that session id. + """ + + def __init__(self, *, + max_sessions: int = DEFAULT_MAX_SESSIONS, + idle_timeout: float = DEFAULT_IDLE_TIMEOUT, + clock: Callable[[], float] = time.monotonic, + on_drop: Optional[Callable[[HttpSession], None]] = None, + ) -> None: + self._sessions: Dict[str, HttpSession] = {} + self._lock = threading.Lock() + self._max_sessions = max(1, int(max_sessions)) + self._idle_timeout = float(idle_timeout) + self._clock = clock + self._on_drop = on_drop + + def __len__(self) -> int: + with self._lock: + return len(self._sessions) + + def create(self) -> HttpSession: + """Mint a session, sweeping expired ones and honouring the cap.""" + session_id = secrets.token_urlsafe(_ID_ENTROPY_BYTES) + now = self._clock() + dropped: List[HttpSession] = [] + with self._lock: + dropped.extend(self._expired_locked(now)) + while len(self._sessions) >= self._max_sessions: + victim = min(self._sessions.values(), + key=lambda item: item.last_seen) + del self._sessions[victim.id] + dropped.append(victim) + _log_eviction(victim, self._max_sessions) + session = HttpSession(session_id, now) + self._sessions[session_id] = session + self._announce(dropped) + return session + + def get(self, session_id: Optional[str]) -> Optional[HttpSession]: + """Return the live session for ``session_id``, touching it.""" + if not session_id: + return None + now = self._clock() + dropped: List[HttpSession] = [] + with self._lock: + dropped.extend(self._expired_locked(now)) + session = self._sessions.get(session_id) + if session is not None: + session.last_seen = now + self._announce(dropped) + return session + + def terminate(self, session_id: Optional[str]) -> Optional[HttpSession]: + """Drop ``session_id`` and close its stream; None when unknown.""" + if not session_id: + return None + with self._lock: + session = self._sessions.pop(session_id, None) + if session is not None: + self._announce([session]) + return session + + def terminate_all(self) -> List[HttpSession]: + """Drop every session — used when the transport shuts down.""" + with self._lock: + sessions = list(self._sessions.values()) + self._sessions.clear() + self._announce(sessions) + return sessions + + def _expired_locked(self, now: float) -> List[HttpSession]: + """Remove idle sessions; caller holds the lock and announces.""" + if self._idle_timeout <= 0: + return [] + stale = [session for session in self._sessions.values() + if now - session.last_seen > self._idle_timeout] + for session in stale: + del self._sessions[session.id] + return stale + + def _announce(self, dropped: List[HttpSession]) -> None: + """Close each dropped session's stream, then report it.""" + for session in dropped: + session.closed.set() + if self._on_drop is None: + continue + try: + self._on_drop(session) + except (RuntimeError, OSError, ValueError, KeyError, + AttributeError) as error: + # A session can be dropped from a sweep triggered by an + # unrelated request; a failing hook must not surface there. + autocontrol_logger.warning( + "MCP session drop hook failed for %s: %r", + session.id, error, + ) + + +def session_id_from_headers(headers: Any) -> Optional[str]: + """Read ``Mcp-Session-Id`` off a request's headers, or ``None``.""" + if headers is None: + return None + raw = headers.get(SESSION_HEADER) + if not isinstance(raw, str): + return None + value = raw.strip() + return value or None + + +__all__ = [ + "DEFAULT_IDLE_TIMEOUT", "DEFAULT_MAX_SESSIONS", "HttpSession", + "SESSION_HEADER", "SessionRegistry", "session_id_from_headers", +] diff --git a/je_auto_control/utils/mcp_server/http_transport.py b/je_auto_control/utils/mcp_server/http_transport.py index 46c667fd..d2969825 100644 --- a/je_auto_control/utils/mcp_server/http_transport.py +++ b/je_auto_control/utils/mcp_server/http_transport.py @@ -1,14 +1,22 @@ """HTTP transport for the MCP server. -Implements a minimal Streamable HTTP transport (JSON-only, no SSE -streaming) so MCP clients that prefer HTTP — or that need to reach -the server from another process / container — can talk to the same -:class:`MCPServer` dispatcher already used by the stdio transport. +Implements a Streamable HTTP transport so MCP clients that prefer HTTP — or +that need to reach the server from another process / container — can talk to +the same :class:`MCPServer` dispatcher already used by the stdio transport. Notifications are answered with ``202 Accepted`` per the MCP spec; ordinary requests return their JSON-RPC response with ``Content-Type: application/json``. The default bind is ``127.0.0.1`` to honour the project's least-privilege policy. + +**Sessions.** ``initialize`` mints an ``Mcp-Session-Id`` and returns it as a +response header; a client that echoes it back keeps one dispatcher scope +across every connection it makes, and may open a standing server-to-client +SSE stream with ``GET``. That stream is what lets the server ask the client +something mid-call — the ``elicitation/create`` behind the destructive-action +confirmation gate. A client that ignores the header still works exactly as +before, scoped to its TCP connection, but cannot be prompted: there is no +channel to carry the question. See :mod:`.http_sessions`. """ import hmac import json @@ -16,13 +24,16 @@ import ssl import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer -from typing import Any, Optional, Tuple +from typing import Any, Callable, Dict, Optional, Tuple from je_auto_control.utils.http_headers import parse_content_length from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.mcp_server._protocol import ( _notification_message, ) +from je_auto_control.utils.mcp_server.http_sessions import ( + HttpSession, SESSION_HEADER, SessionRegistry, session_id_from_headers, +) from je_auto_control.utils.mcp_server.server import MCPServer DEFAULT_PATH = "/mcp" @@ -37,12 +48,37 @@ # Bound the TLS handshake so one silent client can't wedge the single accept # thread waiting for a ClientHello that never arrives. _HANDSHAKE_TIMEOUT = 10.0 +# How often a standing GET stream writes an SSE comment. It keeps the session +# off the idle sweep and turns a client that vanished without a FIN into a +# write error instead of a thread parked forever. +_STREAM_HEARTBEAT = 15.0 + + +def _is_initialize(line: str) -> bool: + """True when ``line`` is an ``initialize`` request; tolerant of junk.""" + try: + message = json.loads(line) + except ValueError: + return False + return isinstance(message, dict) and message.get("method") == "initialize" + + +def _notifier_for(writer: Optional[Callable[[str], None]]): + """Wrap a raw writer as a (method, params) notifier, or ``None``.""" + if writer is None: + return None + return lambda method, params: writer( + _notification_message(method, params), + ) class _MCPHttpHandler(BaseHTTPRequestHandler): """Bridges HTTP requests onto :meth:`MCPServer.handle_line`.""" server_version = "AutoControlMCP/1.0" + # Set once this request's body has been read off the socket, so a later + # error response knows there is nothing left to drain. + _body_consumed = False # socketserver applies this to the connection socket in setup(); it bounds # every read (headers *and* body) so a stalled request cannot pin a worker. timeout = _REQUEST_TIMEOUT @@ -62,23 +98,60 @@ def do_POST(self) -> None: # noqa: N802 # reason: stdlib API if line is None: return bridge: MCPServer = self.server.mcp # type: ignore[attr-defined] - conn_id = id(self) + session, resolved = self._resolve_session(line) + if not resolved: + return + # Prefer the session's identity over the socket's: a client that + # echoes Mcp-Session-Id keeps one scope — and so keeps the + # capabilities it advertised at initialize — across connections. + conn_id = session.id if session is not None else id(self) + extra = {SESSION_HEADER: session.id} if session is not None else None if self._client_accepts_sse(): - self._dispatch_sse(bridge, line, conn_id) + self._dispatch_sse(bridge, line, conn_id, extra) return - # Scope this connection's identity so active-call slots and client - # capabilities don't collide with another peer that reuses the same - # JSON-RPC ids. No notifier/writer: a plain POST has no stream. - with bridge.connection_scope(connection_id=conn_id): + # A plain POST has no stream of its own, but a session may have a + # standing GET stream; server-initiated traffic belongs on it. Absent + # one there is no writer, rather than whichever other peer's socket + # happens to be open. + writer = session.stream_writer if session is not None else None + with bridge.connection_scope(connection_id=conn_id, writer=writer, + notifier=_notifier_for(writer)): response = bridge.handle_line(line) if response is None: # MCP notification — no body, ack with 202. self._send_blank(status=202) return - self._send_raw_json(response) + self._send_raw_json(response, extra_headers=extra) + + def _resolve_session(self, line: str) -> Tuple[Optional[HttpSession], + bool]: + """Resolve this request's session; False means a reply was sent. + + A request carrying an unknown or expired id is refused with 404 + rather than silently served under a fresh scope — the client has + state we do not, and it needs to know to re-initialize. + """ + registry: SessionRegistry = self.server.sessions # type: ignore[attr-defined] + header_id = session_id_from_headers(self.headers) + if header_id is not None: + session = registry.get(header_id) + if session is None: + self._send_json( + {"error": "unknown or expired session"}, status=404, + ) + return None, False + return session, True + if _is_initialize(line): + return registry.create(), True + return None, True def finish(self) -> None: - """Release this connection's per-peer server state, then close.""" + """Release this connection's per-peer server state, then close. + + Only the anonymous, connection-keyed scope is dropped here. State + held under a session id outlives the socket by design and is + released when the session is terminated, swept or evicted. + """ try: super().finish() finally: @@ -106,7 +179,8 @@ def _client_accepts_sse(self) -> bool: return _SSE_MEDIA_TYPE in accept def _dispatch_sse(self, bridge: MCPServer, line: str, - conn_id: int) -> None: + conn_id: Any, + extra_headers: Optional[Dict[str, str]] = None) -> None: """Stream progress notifications + the final response as SSE events.""" # Force connection close so the client gets EOF after the last event. self.close_connection = True @@ -115,6 +189,8 @@ def _dispatch_sse(self, bridge: MCPServer, line: str, f"{_SSE_MEDIA_TYPE}; charset=utf-8") self.send_header("Cache-Control", "no-cache") self.send_header("Connection", "close") + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) self.end_headers() send_lock = threading.Lock() @@ -144,15 +220,97 @@ def emit(payload: str) -> None: emit(response) def do_GET(self) -> None: # noqa: N802 # reason: stdlib API + """Open the session's standing server→client SSE stream.""" if not self._authorize(): return - # MCP optionally allows server→client SSE on GET; not used here. - self._send_json({"error": "GET stream not supported"}, status=405) + if self.path != DEFAULT_PATH: + self._send_json({"error": "unknown path"}, status=404) + return + if not self._client_accepts_sse(): + self._send_json( + {"error": f"GET requires Accept: {_SSE_MEDIA_TYPE}"}, + status=405, + ) + return + registry: SessionRegistry = self.server.sessions # type: ignore[attr-defined] + session = registry.get(session_id_from_headers(self.headers)) + if session is None: + self._send_json( + {"error": "unknown or expired session"}, status=404, + ) + return + self._stream_session(registry, session) + + def _stream_session(self, registry: SessionRegistry, + session: HttpSession) -> None: + """Hold this socket open as the session's outbound channel.""" + self.close_connection = True + send_lock = threading.Lock() + + def emit(payload: str) -> None: + with send_lock: + self.wfile.write(b"data: ") + self.wfile.write(payload.encode("utf-8")) + self.wfile.write(b"\n\n") + self.wfile.flush() + + # Claim the slot before writing headers, and write them under the + # same lock, so an elicitation racing the handshake cannot land in + # front of the status line. + if not session.attach_stream(emit): + self._send_json( + {"error": "session already has an open stream"}, status=409, + ) + return + try: + with send_lock: + self.send_response(200) + self.send_header("Content-Type", + f"{_SSE_MEDIA_TYPE}; charset=utf-8") + self.send_header("Cache-Control", "no-cache") + self.send_header("Connection", "close") + self.send_header(SESSION_HEADER, session.id) + self.end_headers() + self.wfile.flush() + self._heartbeat_until_closed(registry, session, send_lock) + except OSError as error: + # The client went away, or stopped reading long enough for the + # send to time out. Either way this stream is over. + autocontrol_logger.info( + "MCP session stream %s ended: %r", session.id, error, + ) + finally: + session.detach_stream(emit) + + def _heartbeat_until_closed(self, registry: SessionRegistry, + session: HttpSession, + send_lock: threading.Lock) -> None: + """Write an SSE comment periodically until the session ends.""" + while not session.closed.wait(timeout=_STREAM_HEARTBEAT): + # Touching through the registry both keeps this session off the + # idle sweep and tells us when it has already been dropped. + if registry.get(session.id) is None: + return + with send_lock: + self.wfile.write(b": keep-alive\n\n") + self.wfile.flush() def do_DELETE(self) -> None: # noqa: N802 # reason: stdlib API + """Terminate the session named by the header, if there is one.""" if not self._authorize(): return - # Sessionless server — accept the terminate so clients can cleanup. + registry: SessionRegistry = self.server.sessions # type: ignore[attr-defined] + header_id = session_id_from_headers(self.headers) + if header_id is None: + # No session to drop — accept it so sessionless clients can + # still run their cleanup unchanged. + self._send_json({"status": "session terminated"}) + return + if registry.terminate(header_id) is None: + self._send_json( + {"error": "unknown or expired session"}, status=404, + ) + return self._send_json({"status": "session terminated"}) # --- helpers ------------------------------------------------------------- @@ -163,15 +321,21 @@ def _read_body(self) -> Optional[str]: self._send_json({"error": "invalid Content-Length"}, status=400) return None raw = self.rfile.read(length) + # From here the body is gone from the socket. Any 4xx we send later + # must not try to drain it again: there is nothing left to read, so + # the drain would block on the next request's bytes until the socket + # timeout and pin this worker for thirty seconds. + self._body_consumed = True try: return raw.decode("utf-8").strip() except UnicodeDecodeError: self._send_json({"error": "body must be UTF-8"}, status=400) return None - def _send_json(self, payload: Any, status: int = 200) -> None: + def _send_json(self, payload: Any, status: int = 200, + extra_headers: Optional[Dict[str, str]] = None) -> None: body = json.dumps(payload, ensure_ascii=False).encode("utf-8") - self._write_headers(status, body) + self._write_headers(status, body, extra_headers) self.wfile.write(body) if status >= 400: # Drain any unread request body before the socket closes. @@ -181,20 +345,29 @@ def _send_json(self, payload: Any, status: int = 200) -> None: self._drain_body() def _drain_body(self) -> None: + if self._body_consumed: + return declared = parse_content_length(self.headers) if declared <= 0: return cap = min(declared, _MAX_BODY * _DRAIN_CAP_MULTIPLE) remaining = cap - while remaining > 0: - chunk = self.rfile.read(min(remaining, _DRAIN_CHUNK)) - if not chunk: - break - remaining -= len(chunk) - - def _send_raw_json(self, raw_json: str) -> None: + try: + while remaining > 0: + chunk = self.rfile.read(min(remaining, _DRAIN_CHUNK)) + if not chunk: + break + remaining -= len(chunk) + except OSError as error: + # Draining is a courtesy to the client's read of our 4xx. If the + # peer has already gone, there is nothing left to be courteous + # about — and letting this escape logs a whole traceback for it. + autocontrol_logger.debug("MCP drain aborted: %r", error) + + def _send_raw_json(self, raw_json: str, + extra_headers: Optional[Dict[str, str]] = None) -> None: body = raw_json.encode("utf-8") - self._write_headers(200, body) + self._write_headers(200, body, extra_headers) self.wfile.write(body) def _send_blank(self, status: int) -> None: @@ -202,10 +375,14 @@ def _send_blank(self, status: int) -> None: self.send_header("Content-Length", "0") self.end_headers() - def _write_headers(self, status: int, body: bytes) -> None: + def _write_headers(self, status: int, body: bytes, + extra_headers: Optional[Dict[str, str]] = None, + ) -> None: self.send_response(status) self.send_header("Content-Type", "application/json; charset=utf-8") self.send_header("Content-Length", str(len(body))) + for name, value in (extra_headers or {}).items(): + self.send_header(name, value) self.end_headers() @@ -218,6 +395,12 @@ def __init__(self, server_address: Tuple[str, int], super().__init__(server_address, _MCPHttpHandler) self.mcp = mcp self.auth_token = auth_token + # Dropping a session releases the dispatcher state scoped to its id — + # the same release a closing socket used to perform, moved to the + # identity that actually owns that state. + self.sessions = SessionRegistry( + on_drop=lambda session: mcp.forget_connection(session.id), + ) # No sse_lock: SSE requests used to swap server-wide notifier/writer # state and needed serialising. They now bind that state to their own # thread via MCPServer.connection_scope, so concurrent SSE streams no @@ -271,6 +454,11 @@ def address(self) -> Tuple[str, int]: def mcp(self) -> MCPServer: return self._mcp + @property + def sessions(self) -> Optional[SessionRegistry]: + """The live session registry, or ``None`` before :meth:`start`.""" + return self._server.sessions if self._server is not None else None + def start(self) -> None: """Bind the socket and begin serving on a background thread.""" if self._server is not None: @@ -298,6 +486,9 @@ def start(self) -> None: def stop(self, timeout: float = 2.0) -> None: if self._server is None: return + # Close the sessions first: a standing GET stream parks a worker on + # its heartbeat, and terminating releases it without waiting one out. + self._server.sessions.terminate_all() self._server.shutdown() self._server.server_close() if self._thread is not None: diff --git a/je_auto_control/utils/mcp_server/server.py b/je_auto_control/utils/mcp_server/server.py index f85c4122..5c24f943 100644 --- a/je_auto_control/utils/mcp_server/server.py +++ b/je_auto_control/utils/mcp_server/server.py @@ -45,6 +45,10 @@ _TOOL_INVOKE_ERRORS, _TOOLS_CALL_METHOD, ) +#: How long a transport waits for in-flight tool replies before it gives up +#: and shuts down anyway. +WORKER_DRAIN_TIMEOUT = 10.0 + class MCPServer(ClientRequestMixin): """JSON-RPC 2.0 MCP server with a configurable tool registry.""" @@ -71,6 +75,12 @@ def __init__(self, tools: Optional[List[MCPTool]] = None, self._log_bridge = log_bridge self._stop = threading.Event() self._initialized = False + # tools/call runs on a worker thread under the stdio transport, so a + # reply can still be in flight when the loop reaches EOF. Tracking the + # workers is what lets the transport wait for them instead of pulling + # the writer out from under them. + self._workers: List[threading.Thread] = [] + self._workers_lock = threading.Lock() # Connection-scoped state. The stdio transport has exactly one peer, so # a server-wide default is right for it; an HTTP transport has many, and # each request runs on its own thread. Keeping the notifier/writer in @@ -264,6 +274,10 @@ def serve_stdio(self, stdin: Optional[TextIO] = None, if response is not None: self._write_message(out_stream, response) finally: + # Before anything is restored: a tools/call dispatched just before + # EOF is still running, and its reply has nowhere to go once the + # writer is swapped back. + self._join_workers() self._detach_log_bridge_if_configured() self._notifier = prior_notifier self._writer = prior_writer @@ -366,18 +380,48 @@ def _handle_line_safely(self, line: str) -> Optional[str]: def _dispatch_tools_call_async(self, msg_id: Any, params: Dict[str, Any]) -> None: """Run a tools/call on a worker thread; the worker writes the reply.""" + # Captured here rather than read inside the worker. The worker may not + # reach its write until after the transport's `finally` has restored + # the previous writer, and then the reply would go down the previous + # connection or be dropped as "no writer" — a reply belongs to the + # transport that accepted the request. + writer = self._writer + def worker() -> None: payload = self._build_response(msg_id, _TOOLS_CALL_METHOD, params) - writer = self._writer if writer is None: autocontrol_logger.warning( "MCP async tool reply with no writer; dropping %s", msg_id, ) return writer(payload) - threading.Thread( + thread = threading.Thread( target=worker, daemon=True, name=f"MCPCall-{msg_id}", - ).start() + ) + with self._workers_lock: + self._workers = [live for live in self._workers if live.is_alive()] + self._workers.append(thread) + thread.start() + + def _join_workers(self, timeout: float = WORKER_DRAIN_TIMEOUT) -> None: + """Wait for in-flight tool replies before the transport goes away. + + Bounded: a handler that never returns must not keep the process from + shutting down, so this says which worker outstayed its welcome and + gives up rather than hanging. + """ + deadline = time.monotonic() + timeout + with self._workers_lock: + pending = [live for live in self._workers if live.is_alive()] + for thread in pending: + remaining = deadline - time.monotonic() + if remaining <= 0: + autocontrol_logger.warning( + "MCP shutdown: %s is still running; its reply is lost", + thread.name, + ) + continue + thread.join(remaining) def _build_response(self, msg_id: Any, method: Optional[str], params: Dict[str, Any]) -> str: diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 17960fb9..ee2bfc39 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -49,8 +49,11 @@ def mouse_tools() -> List[MCPTool]: ), MCPTool( name="ac_mouse_scroll", - description=("Scroll the mouse wheel by scroll_value units. " - "scroll_direction is Linux-only: scroll_up | scroll_down."), + description=("Scroll the mouse wheel by scroll_value units; " + "a negative scroll_value reverses the direction. " + "scroll_direction names the direction a positive " + "value takes and is read on X11/Wayland only: " + "scroll_up | scroll_down | scroll_left | scroll_right."), input_schema=schema({ "scroll_value": {"type": "integer"}, "x": {"type": "integer"}, @@ -560,7 +563,8 @@ def recording_tools() -> List[MCPTool]: name="ac_record_start", description=("Start recording mouse and keyboard events in the " "background. Call ac_record_stop to retrieve the " - "captured action list. Not supported on macOS."), + "captured action list. On macOS this needs " + "Accessibility permission."), input_schema=schema({}), handler=h.record_start, annotations=SIDE_EFFECT_ONLY, diff --git a/je_auto_control/utils/platform_id/__init__.py b/je_auto_control/utils/platform_id/__init__.py new file mode 100644 index 00000000..840a4a43 --- /dev/null +++ b/je_auto_control/utils/platform_id/__init__.py @@ -0,0 +1,62 @@ +"""Which operating-system family this is, asked once and spelled one way. + +``sys.platform`` was compared against literal lists in over a hundred places, +and every one of those lists named ``win32``/``cygwin``/``msys``, ``darwin``, +``linux``/``linux2`` and nothing else. That is fine until an operating system +turns up that is none of them and still runs X11 — a FreeBSD, OpenBSD or +NetBSD desktop is an ordinary X11 desktop, and the X11 backend works there — +at which point every one of those lists is a separate place to be wrong. + +The distinction that actually matters to this project is not "which kernel" +but **which input and display stack**, so that is what these answer. +""" +import sys + +__all__ = [ + "BSD_PREFIXES", "current_family", "is_bsd", "is_macos", "is_windows", + "is_x11_unix", +] + +#: ``sys.platform`` on the BSDs carries the major version — ``freebsd14``, +#: ``openbsd7`` — so these are matched by prefix rather than by equality. +BSD_PREFIXES = ("freebsd", "openbsd", "netbsd", "dragonfly") + + +def is_windows(platform: str = "") -> bool: + """Windows, including the POSIX emulation layers that ship the Win32 API.""" + return (platform or sys.platform) in ("win32", "cygwin", "msys") + + +def is_macos(platform: str = "") -> bool: + """macOS, where the backend is Quartz rather than X11.""" + return (platform or sys.platform) == "darwin" + + +def is_bsd(platform: str = "") -> bool: + """One of the BSDs, which run the same X11 stack as Linux.""" + return (platform or sys.platform).startswith(BSD_PREFIXES) + + +def is_x11_unix(platform: str = "") -> bool: + """A Unix whose desktop is X11 (or Wayland with XWayland underneath). + + Linux and the BSDs both qualify. This is the test the X11 backend + modules guard on: what they need is an X server and ``python-Xlib``, + neither of which is Linux-specific. + """ + name = platform or sys.platform + return name.startswith("linux") or is_bsd(name) + + +def current_family(platform: str = "") -> str: + """``windows`` / ``macos`` / ``linux`` / ``bsd`` / the raw name.""" + name = platform or sys.platform + if is_windows(name): + return "windows" + if is_macos(name): + return "macos" + if name.startswith("linux"): + return "linux" + if is_bsd(name): + return "bsd" + return name diff --git a/je_auto_control/utils/qr/qr.py b/je_auto_control/utils/qr/qr.py index b561eb4c..fb07c822 100644 --- a/je_auto_control/utils/qr/qr.py +++ b/je_auto_control/utils/qr/qr.py @@ -4,18 +4,24 @@ dependency), so no extra package is needed. The decoder is injectable so the wrapper is unit-testable without a real QR image. GUI-free. """ +from __future__ import annotations + import io from pathlib import Path -from typing import Any, Callable, List, Optional, Sequence, Union +from typing import ( + TYPE_CHECKING, Any, Callable, List, Optional, Sequence, Union, +) -from PIL import Image +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image -ImageSource = Union[str, Path, bytes, Image.Image] +ImageSource = Union[str, Path, bytes, "Image.Image"] QRDecoder = Callable[[Any], List[str]] def _load_np(source: ImageSource, region: Optional[Sequence[int]]): import numpy as np + from PIL import Image if isinstance(source, Image.Image): image = source elif isinstance(source, bytes): diff --git a/je_auto_control/utils/remote_desktop/audit_log.py b/je_auto_control/utils/remote_desktop/audit_log.py index 686ad9eb..d3b5b94e 100644 --- a/je_auto_control/utils/remote_desktop/audit_log.py +++ b/je_auto_control/utils/remote_desktop/audit_log.py @@ -20,7 +20,6 @@ import hashlib import json import os -import sqlite3 import threading from dataclasses import dataclass from datetime import datetime, timezone @@ -28,6 +27,9 @@ from typing import List, Optional, Tuple from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.sqlite_support import ( + SQLITE_ERRORS, SQLITE_OPERATIONAL_ERRORS, require_sqlite3, +) _DEFAULT_PATH_RELATIVE = ".je_auto_control/audit.db" @@ -55,8 +57,11 @@ class AuditLog: def __init__(self, path: Optional[Path] = None) -> None: self._path = Path(path) if path is not None else default_audit_log_path() self._lock = threading.Lock() + # Asked for before the directory is made, so a Python without + # sqlite3 does not leave an empty ~/.je_auto_control behind. + driver = require_sqlite3() self._path.parent.mkdir(parents=True, exist_ok=True) - self._conn = sqlite3.connect( + self._conn = driver.connect( str(self._path), check_same_thread=False, isolation_level=None, ) self._init_schema() @@ -87,11 +92,11 @@ def _init_schema(self) -> None: # raw-SQL-construction rules without resorting to suppressions. try: self._conn.execute("ALTER TABLE events ADD COLUMN prev_hash TEXT") - except sqlite3.OperationalError: + except SQLITE_OPERATIONAL_ERRORS: pass # Column already exists — that's fine. try: self._conn.execute("ALTER TABLE events ADD COLUMN row_hash TEXT") - except sqlite3.OperationalError: + except SQLITE_OPERATIONAL_ERRORS: pass # Column already exists — that's fine. self._backfill_chain_locked() @@ -148,7 +153,7 @@ def log(self, event_type: str, *, ) self._last_hash = row_hash self._maybe_prune_locked() - except sqlite3.Error as error: + except SQLITE_ERRORS as error: autocontrol_logger.warning("audit log insert: %r", error) def _maybe_prune_locked(self) -> None: @@ -178,7 +183,7 @@ def query(self, *, try: cur = self._conn.execute(sql, args) rows = cur.fetchall() - except sqlite3.Error as error: + except SQLITE_ERRORS as error: autocontrol_logger.warning("audit log query: %r", error) return [] return [ @@ -227,7 +232,7 @@ def close(self) -> None: with self._lock: try: self._conn.close() - except sqlite3.Error: + except SQLITE_ERRORS: pass diff --git a/je_auto_control/utils/rest_api/rest_server.py b/je_auto_control/utils/rest_api/rest_server.py index e2a5f6b5..1582374e 100644 --- a/je_auto_control/utils/rest_api/rest_server.py +++ b/je_auto_control/utils/rest_api/rest_server.py @@ -11,7 +11,6 @@ import json import re -import sqlite3 import threading from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path @@ -38,6 +37,7 @@ handle_usb_remote_open, ) from je_auto_control.utils.rest_api.rest_metrics import RestMetrics +from je_auto_control.utils.sqlite_support import SQLITE_ERRORS HandlerFn = Callable[[RouteContext], HandlerResult] @@ -191,11 +191,12 @@ def _dispatch(self, method: str, routes: Dict[str, HandlerFn], try: status, payload = handler(ctx) # AutoControlException is the family base (every framework error derives - # from it). sqlite3.Error is separate: handlers such as /history read the - # shared run-history DB, and a locked/corrupt DB otherwise escaped the - # handler thread and dropped the connection with no response. + # from it). The sqlite3 errors are separate: handlers such as /history + # read the shared run-history DB, and a locked/corrupt DB otherwise + # escaped the handler thread and dropped the connection with no + # response. The tuple is empty on a Python built without sqlite3. except (OSError, RuntimeError, ValueError, TypeError, - AutoControlException, sqlite3.Error) as error: + AutoControlException, *SQLITE_ERRORS) as error: autocontrol_logger.error( "rest-api %s %s handler raised: %r", method, parsed.path, error, ) diff --git a/je_auto_control/utils/run_history/history_store.py b/je_auto_control/utils/run_history/history_store.py index b02db3a9..78f814eb 100644 --- a/je_auto_control/utils/run_history/history_store.py +++ b/je_auto_control/utils/run_history/history_store.py @@ -6,14 +6,19 @@ pruning via :meth:`clear` or :meth:`prune`. """ import os -import sqlite3 import threading import time from dataclasses import dataclass from pathlib import Path -from typing import List, Optional, Union +from typing import TYPE_CHECKING, List, Optional, Union from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.sqlite_support import ( + SQLITE_ERRORS, require_sqlite3, +) + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 SOURCE_SCHEDULER = "scheduler" SOURCE_TRIGGER = "trigger" @@ -96,23 +101,41 @@ class HistoryStore: def __init__(self, path: Union[str, Path] = _IN_MEMORY_DB) -> None: self._path = str(path) if path == _IN_MEMORY_DB else str(Path(path)) + self._lock = threading.Lock() + # The database is opened on first use, not here. The module-level + # ``default_history_store`` is built during ``import + # je_auto_control``, and connecting there made importing the package + # create a file on disk and require a Python built with sqlite3 -- + # which FreeBSD ships as a separate package, so importing the + # package failed outright there, mouse and keyboard included. + self._conn: Optional["sqlite3.Connection"] = None + + def _connection(self) -> "sqlite3.Connection": + """Return the open connection, creating the database on first call. + + Every caller already holds ``self._lock``, so the open happens once. + """ + if self._conn is not None: + return self._conn + driver = require_sqlite3() if self._path != _IN_MEMORY_DB: os.makedirs(os.path.dirname(self._path) or ".", exist_ok=True) - self._lock = threading.Lock() - self._conn = sqlite3.connect( + conn = driver.connect( self._path, check_same_thread=False, isolation_level=None, ) - self._conn.row_factory = sqlite3.Row - self._conn.executescript(_SCHEMA) - self._migrate_schema() + conn.row_factory = driver.Row + conn.executescript(_SCHEMA) + self._conn = conn + self._migrate_schema_locked(conn) + return conn - def _migrate_schema(self) -> None: + def _migrate_schema_locked(self, conn: "sqlite3.Connection") -> None: """Add columns that older store files are missing.""" - cols = {row["name"] for row in self._conn.execute( + cols = {row["name"] for row in conn.execute( "PRAGMA table_info(runs)", ).fetchall()} if "artifact_path" not in cols: - self._conn.execute( + conn.execute( "ALTER TABLE runs ADD COLUMN artifact_path TEXT", ) @@ -127,7 +150,7 @@ def start_run(self, source_type: str, source_id: str, _validate_source(source_type) ts = float(started_at) if started_at is not None else time.time() with self._lock: - cursor = self._conn.execute( + cursor = self._connection().execute( "INSERT INTO runs (source_type, source_id, script_path," " started_at, status) VALUES (?, ?, ?, ?, ?)", (source_type, source_id, script_path, ts, STATUS_RUNNING), @@ -144,7 +167,7 @@ def finish_run(self, run_id: int, status: str, raise ValueError("cannot finish a run with status=running") ts = float(finished_at) if finished_at is not None else time.time() with self._lock: - cursor = self._conn.execute( + cursor = self._connection().execute( "UPDATE runs SET finished_at = ?, status = ?, error_text = ?," " artifact_path = ? WHERE id = ?", (ts, status, error_text, artifact_path, int(run_id)), @@ -154,7 +177,7 @@ def finish_run(self, run_id: int, status: str, def attach_artifact(self, run_id: int, artifact_path: str) -> bool: """Attach or replace the artifact path on a finished run.""" with self._lock: - cursor = self._conn.execute( + cursor = self._connection().execute( "UPDATE runs SET artifact_path = ? WHERE id = ?", (artifact_path, int(run_id)), ) @@ -169,7 +192,7 @@ def list_runs(self, limit: int = 100, bound_limit = int(limit) if source_type is None: with self._lock: - rows = self._conn.execute( + rows = self._connection().execute( "SELECT * FROM runs " "ORDER BY started_at DESC LIMIT ?", (bound_limit,), @@ -177,7 +200,7 @@ def list_runs(self, limit: int = 100, else: _validate_source(source_type) with self._lock: - rows = self._conn.execute( + rows = self._connection().execute( "SELECT * FROM runs WHERE source_type = ? " "ORDER BY started_at DESC LIMIT ?", (source_type, bound_limit), @@ -187,7 +210,7 @@ def list_runs(self, limit: int = 100, def get_run(self, run_id: int) -> Optional[RunRecord]: """Return a specific row or ``None`` if absent.""" with self._lock: - row = self._conn.execute( + row = self._connection().execute( "SELECT * FROM runs WHERE id = ?", (int(run_id),), ).fetchone() return _row_to_record(row) if row is not None else None @@ -197,22 +220,22 @@ def count(self, source_type: Optional[str] = None) -> int: if source_type is not None: _validate_source(source_type) with self._lock: - row = self._conn.execute( + row = self._connection().execute( "SELECT COUNT(*) FROM runs WHERE source_type = ?", (source_type,), ).fetchone() else: with self._lock: - row = self._conn.execute("SELECT COUNT(*) FROM runs").fetchone() + row = self._connection().execute("SELECT COUNT(*) FROM runs").fetchone() return int(row[0]) def clear(self) -> int: """Delete every row (and its artifact file); return rows removed.""" with self._lock: - paths = [r[0] for r in self._conn.execute( + paths = [r[0] for r in self._connection().execute( "SELECT artifact_path FROM runs WHERE artifact_path IS NOT NULL", ).fetchall()] - cursor = self._conn.execute("DELETE FROM runs") + cursor = self._connection().execute("DELETE FROM runs") removed = int(cursor.rowcount) _remove_artifact_files(paths) return removed @@ -222,13 +245,13 @@ def prune(self, keep_latest: int) -> int: if keep_latest < 0: raise ValueError("keep_latest must be >= 0") with self._lock: - paths = [r[0] for r in self._conn.execute( + paths = [r[0] for r in self._connection().execute( "SELECT artifact_path FROM runs WHERE artifact_path IS NOT NULL" " AND id NOT IN (" "SELECT id FROM runs ORDER BY started_at DESC LIMIT ?)", (int(keep_latest),), ).fetchall()] - cursor = self._conn.execute( + cursor = self._connection().execute( "DELETE FROM runs WHERE id NOT IN (" "SELECT id FROM runs ORDER BY started_at DESC LIMIT ?" ")", @@ -239,14 +262,22 @@ def prune(self, keep_latest: int) -> int: return removed def close(self) -> None: + """Close the database if it was ever opened. + + The connection object is deliberately kept: a call after ``close()`` + must keep raising sqlite3's "closed database" error rather than + silently reopening and, for an in-memory store, losing every row. + """ with self._lock: + if self._conn is None: + return try: self._conn.close() - except sqlite3.Error as error: + except SQLITE_ERRORS as error: autocontrol_logger.warning("history close failed: %r", error) -def _row_to_record(row: sqlite3.Row) -> RunRecord: +def _row_to_record(row: "sqlite3.Row") -> RunRecord: artifact = row["artifact_path"] if "artifact_path" in row.keys() else None return RunRecord( id=int(row["id"]), diff --git a/je_auto_control/utils/sql/sql_query.py b/je_auto_control/utils/sql/sql_query.py index 11d045d3..c2912cc7 100644 --- a/je_auto_control/utils/sql/sql_query.py +++ b/je_auto_control/utils/sql/sql_query.py @@ -7,15 +7,18 @@ parameters (never string-interpolated) to avoid SQL injection. Imports no ``PySide6`` so it stays fully headless. """ -import sqlite3 from contextlib import closing from pathlib import Path -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union from urllib.parse import quote from je_auto_control.utils.data_source.data_source import ( _resolve_path, _validate_select, ) +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 _FetchResult = Union[List[Dict[str, Any]], Dict[str, Any], Any, None] @@ -33,7 +36,7 @@ def _read_only_uri(path: Path) -> str: return f"file:{safe_path}?mode=ro" -def _shape(cursor: sqlite3.Cursor, fetch: str) -> _FetchResult: +def _shape(cursor: "sqlite3.Cursor", fetch: str) -> _FetchResult: """Reduce a cursor to the requested result shape.""" if fetch in ("all", "rows"): return [dict(row) for row in cursor.fetchall()] @@ -63,8 +66,9 @@ def query_sqlite(database: str, query: str, uri = _read_only_uri(path) # closing(): sqlite3's own context manager only commits/rolls back the # transaction, it does not close the connection (leaking the handle until GC). - with closing(sqlite3.connect(uri, uri=True)) as connection: - connection.row_factory = sqlite3.Row + driver = require_sqlite3() + with closing(driver.connect(uri, uri=True)) as connection: + connection.row_factory = driver.Row cursor = connection.execute( statement, params if params is not None else ()) return _shape(cursor, fetch) diff --git a/je_auto_control/utils/sqlite_support.py b/je_auto_control/utils/sqlite_support.py new file mode 100644 index 00000000..9dec7cdc --- /dev/null +++ b/je_auto_control/utils/sqlite_support.py @@ -0,0 +1,56 @@ +"""Access to the optional standard-library ``sqlite3`` module. + +CPython links ``sqlite3`` against a system library that is not always shipped +with the interpreter: FreeBSD packages it separately as +``databases/py-sqlite3``, and minimal builds leave it out entirely. Ten +AutoControl subsystems keep their state in SQLite and every one of them is +reachable from ``import je_auto_control``, so importing ``sqlite3`` at module +scope made the whole package unimportable on such a Python -- mouse and +keyboard included, neither of which touches a database. Going through here +instead defers the failure to the first call that actually opens one. +""" +from typing import Tuple, Type + +from je_auto_control.utils.exception.exceptions import \ + AutoControlUnsupportedOperationException + +try: + import sqlite3 as _sqlite3 + + #: For ``except`` clauses in the callers that contain database failures. + #: The classes are named here rather than at each site so the tuple can be + #: empty when there is no ``sqlite3`` -- nothing can raise them then, so an + #: empty tuple catches exactly the right amount: nothing. + SQLITE_ERRORS: Tuple[Type[BaseException], ...] = (_sqlite3.Error,) + SQLITE_OPERATIONAL_ERRORS: Tuple[Type[BaseException], ...] = ( + _sqlite3.OperationalError, + ) +except ImportError: # reason: interpreters built without the sqlite3 extension + _sqlite3 = None # type: ignore[assignment] + SQLITE_ERRORS = () + SQLITE_OPERATIONAL_ERRORS = () + +_UNAVAILABLE_MESSAGE = ( + "This Python has no sqlite3 module, so the SQLite-backed features (run " + "history, checkpoints, work queue, agent memory, remote-desktop audit " + "log, SQL data sources) cannot run. Install it for this interpreter -- on " + "FreeBSD it is the separate databases/py-sqlite3 package." +) + + +def sqlite3_available() -> bool: + """Whether this interpreter can open SQLite databases.""" + return _sqlite3 is not None + + +def require_sqlite3(): + """Return the ``sqlite3`` module, raising if this build does not have it. + + Raises ``AutoControlUnsupportedOperationException`` -- the same type the + platform backends raise for an operation they cannot perform, so the GUI + tabs, the REST handler and the executor already report it as "unavailable + here" instead of dying on an ``ImportError`` none of them catch. + """ + if _sqlite3 is None: + raise AutoControlUnsupportedOperationException(_UNAVAILABLE_MESSAGE) + return _sqlite3 diff --git a/je_auto_control/utils/usb/passthrough/protocol.py b/je_auto_control/utils/usb/passthrough/protocol.py index bf240764..0f05192f 100644 --- a/je_auto_control/utils/usb/passthrough/protocol.py +++ b/je_auto_control/utils/usb/passthrough/protocol.py @@ -19,6 +19,7 @@ import enum import struct from dataclasses import dataclass +from je_auto_control.utils.exception.exceptions import AutoControlException _HEADER_FORMAT = "!BBH" @@ -43,7 +44,7 @@ class Opcode(enum.IntEnum): ERROR = 0xFF -class ProtocolError(Exception): +class ProtocolError(AutoControlException): """Raised on malformed frames or invariant violations.""" diff --git a/je_auto_control/utils/usb/passthrough/session.py b/je_auto_control/utils/usb/passthrough/session.py index 597e598b..208a69d8 100644 --- a/je_auto_control/utils/usb/passthrough/session.py +++ b/je_auto_control/utils/usb/passthrough/session.py @@ -65,6 +65,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.usb.passthrough.acl import UsbAcl from je_auto_control.utils.usb.passthrough.backend import UsbBackend, UsbHandle @@ -84,7 +85,7 @@ _ABUSE_LOCKOUT_S = 5.0 -class SessionError(Exception): +class SessionError(AutoControlException): """Raised on session-level invariant violations (not protocol parse errors).""" diff --git a/je_auto_control/utils/usb/passthrough/viewer_client.py b/je_auto_control/utils/usb/passthrough/viewer_client.py index e55e5e58..ff4e005e 100644 --- a/je_auto_control/utils/usb/passthrough/viewer_client.py +++ b/je_auto_control/utils/usb/passthrough/viewer_client.py @@ -38,6 +38,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional +from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.usb.passthrough.protocol import ( FLAG_EOF, Frame, Opcode, @@ -50,7 +51,7 @@ _CLIENT_SHUT_DOWN_MSG = "client is shut down" -class UsbClientError(Exception): +class UsbClientError(AutoControlException): """The host reported a transfer or open failure.""" diff --git a/je_auto_control/utils/usb/usb_devices.py b/je_auto_control/utils/usb/usb_devices.py index 5357ebc3..e045fa8f 100644 --- a/je_auto_control/utils/usb/usb_devices.py +++ b/je_auto_control/utils/usb/usb_devices.py @@ -242,16 +242,25 @@ def _hex4(value: Any) -> Optional[str]: def _hex4_from_macos(value: Any) -> Optional[str]: + """Parse ``system_profiler``'s vendor/product id into hex, or None. + + Apple's own devices report a *symbolic* id — ``apple_vendor_id`` + rather than a number — and other entries append the vendor name + after it (``0x05ac (Apple Inc.)``). Only a real hex id can satisfy + the 4-hex-digit shape this field documents, so anything else is + None: passing a symbolic name through a field callers parse as hex + is worse than admitting there is no id. The device is still listed, + and its ``manufacturer`` still names the vendor. + """ if value is None: return None text = str(value).strip() - match = re.match(r"0x([0-9A-Fa-f]+)", text) - if match: - try: - return f"{int(match.group(1), 16):04x}" - except ValueError: - return None - return text or None + # The lookahead is what rejects "apple_vendor_id": its leading "a" + # is a valid hex digit, so without it the id would parse as 000a. + match = re.match(r"(?:0x)?([0-9A-Fa-f]{1,4})(?![0-9A-Za-z_])", text) + if match is None: + return None + return f"{int(match.group(1), 16):04x}" def _strip_or_none(value: Any) -> Optional[str]: diff --git a/je_auto_control/utils/visual_regression/compare.py b/je_auto_control/utils/visual_regression/compare.py index d26d5f1b..12482002 100644 --- a/je_auto_control/utils/visual_regression/compare.py +++ b/je_auto_control/utils/visual_regression/compare.py @@ -4,9 +4,10 @@ import os from dataclasses import dataclass, field from pathlib import Path -from typing import Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Optional, Sequence, Tuple -from PIL import Image, ImageChops, ImageDraw +if TYPE_CHECKING: # pragma: no cover - annotations only + from PIL import Image @dataclass(frozen=True) @@ -60,6 +61,7 @@ def _expand_path(path) -> Path: def _apply_masks(image: Image.Image, masks: Sequence[MaskRegion]) -> Image.Image: """Black out the masked regions on a *copy* so the input stays intact.""" + from PIL import ImageDraw if not masks: return image result = image.copy() @@ -110,6 +112,7 @@ def image_difference(actual: Image.Image, expected: Image.Image, the comparison). ``masks`` blacks out those regions on *both* sides before comparing. """ + from PIL import ImageChops, ImageDraw if actual.size != expected.size: raise ValueError( f"image sizes differ: actual={actual.size}, " @@ -151,6 +154,7 @@ def compare_to_golden(golden_path, a tiny rendering wobble doesn't fail every CI run). Defaults to ``0.0`` for strictest comparison. """ + from PIL import Image target = _expand_path(golden_path) if not target.exists(): raise FileNotFoundError(f"golden image not found: {target}") diff --git a/je_auto_control/utils/work_queue/work_queue.py b/je_auto_control/utils/work_queue/work_queue.py index 0e6db16b..9c13d58d 100644 --- a/je_auto_control/utils/work_queue/work_queue.py +++ b/je_auto_control/utils/work_queue/work_queue.py @@ -16,10 +16,15 @@ Pure standard library (``sqlite3``); imports no ``PySide6``. """ import json -import sqlite3 import time from dataclasses import dataclass -from typing import Any, Dict, List, Optional +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.sqlite_support import require_sqlite3 + +if TYPE_CHECKING: # reason: sqlite3 types are named only in annotations + import sqlite3 STATUS_NEW = "new" STATUS_IN_PROGRESS = "in_progress" @@ -27,7 +32,7 @@ STATUS_FAILED = "failed" -class BusinessError(Exception): +class BusinessError(AutoControlException): """A non-retryable, data-level failure of a work item.""" @@ -51,10 +56,11 @@ def __init__(self, db_path: str, name: str = "default") -> None: self._name = name self._ensure_schema() - def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self._db_path, timeout=30.0, - isolation_level=None) - conn.row_factory = sqlite3.Row + def _connect(self) -> "sqlite3.Connection": + driver = require_sqlite3() + conn = driver.connect(self._db_path, timeout=30.0, + isolation_level=None) + conn.row_factory = driver.Row return conn def _ensure_schema(self) -> None: @@ -79,7 +85,7 @@ def add(self, data: Dict[str, Any], *, reference: Optional[str] = None, time.time())) return int(cur.lastrowid) - def _has_pending(self, conn: sqlite3.Connection, reference: str) -> bool: + def _has_pending(self, conn: "sqlite3.Connection", reference: str) -> bool: row = conn.execute( "SELECT 1 FROM work_items WHERE queue=? AND reference=? AND " "status IN (?, ?) LIMIT 1", diff --git a/je_auto_control/windows/record/win32_input_hook.py b/je_auto_control/windows/record/win32_input_hook.py index 44877253..d2f171f4 100644 --- a/je_auto_control/windows/record/win32_input_hook.py +++ b/je_auto_control/windows/record/win32_input_hook.py @@ -35,6 +35,12 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlException, AutoControlRecordException, ) +# Re-exported: the event cap and the timeline shaping are platform-neutral and +# shared with the macOS tap, but this is where callers have always imported +# them from. +from je_auto_control.utils.input_macro.recorder_base import ( # noqa: F401 + MAX_EVENTS, timeline, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger if sys.platform not in ["win32", "cygwin", "msys"]: @@ -55,10 +61,6 @@ 0x0207: ("mouse_down", "middle"), 0x0208: ("mouse_up", "middle"), } -# A hook that is left installed keeps growing this list forever. Recording is -# bounded so a forgotten session cannot consume the process. -MAX_EVENTS = 20000 - class _KBDLLHOOKSTRUCT(ctypes.Structure): _fields_ = [("vkCode", wintypes.DWORD), ("scanCode", wintypes.DWORD), @@ -203,21 +205,3 @@ def _mouse_event(self, l_param, message: int) -> None: raw = ctypes.c_short((int(data.mouseData) >> 16) & 0xFFFF).value self._put({"op": "scroll", "delta": raw // _WHEEL_NOTCH, "x": int(data.pt.x), "y": int(data.pt.y)}) - - -def timeline(events: List[Dict[str, Any]]) -> List[Dict[str, Any]]: - """Convert raw events to ``delta_ms`` form for ``replay_timeline``. - - The first event has no gap before it; every later one carries the real pause - that preceded it, which is what makes a replay track the original pace. - """ - out: List[Dict[str, Any]] = [] - previous: Optional[float] = None - for event in events: - moment = float(event.get("time", 0.0)) - item = {key: value for key, value in event.items() if key != "time"} - item["delta_ms"] = 0 if previous is None else max( - 0, int((moment - previous) * 1000)) - out.append(item) - previous = moment - return out diff --git a/je_auto_control/windows/record/win32_record.py b/je_auto_control/windows/record/win32_record.py index 5892e8c6..ceb45e0d 100644 --- a/je_auto_control/windows/record/win32_record.py +++ b/je_auto_control/windows/record/win32_record.py @@ -1,126 +1,41 @@ +"""Windows recorder: the low-level input hook shaped into the recorder surface. + +Everything after the capture — the down-events-only queue the executor has +always been handed, the ``delta_ms`` timeline a replay needs, and the +mouse-only / keyboard-only filters — is platform-neutral and lives in +:mod:`je_auto_control.utils.input_macro.recorder_base`, so this backend and +the macOS one cannot drift in the shape they produce. +""" import sys -from typing import Any, Dict, List, Optional -from queue import Queue from je_auto_control.utils.exception.exception_tags import windows_import_error_message from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.input_macro.recorder_base import InputRecorder if sys.platform not in ["win32", "cygwin", "msys"]: raise AutoControlException(windows_import_error_message) -from je_auto_control.windows.record.win32_input_hook import ( - Win32InputHook, timeline, -) +from je_auto_control.windows.record.win32_input_hook import Win32InputHook -# Legacy queue entries: the down-event half, shaped as executor commands. -_LEGACY_MOUSE_COMMAND = {"left": "AC_mouse_left", "right": "AC_mouse_right", - "middle": "AC_mouse_middle"} - -class Win32Recorder: +class Win32Recorder(InputRecorder): """ Win32Recorder Windows 錄製器 - 可同時錄製滑鼠與鍵盤事件 - 可選擇只錄製滑鼠或鍵盤 - Capture runs through :class:`Win32InputHook`, which records releases, wheel - movement and timing as well as presses. ``stop_record`` still returns the - historical down-events-only queue so existing callers are unaffected; use - ``stop_record_timeline`` for everything needed to reproduce the session. + Capture runs through :class:`Win32InputHook`, which records releases, + wheel movement and timing as well as presses. ``stop_record`` still + returns the historical down-events-only queue so existing callers are + unaffected; use ``stop_record_timeline`` for everything needed to + reproduce the session. """ - def __init__(self): - self.hook: Optional[Win32InputHook] = None - self.record_queue: Optional[Queue] = None - self.result_queue: Optional[Queue] = None - self._kinds: tuple = ("keyboard", "mouse") - - def _start(self, kinds: tuple) -> None: - self._kinds = kinds - self.hook = Win32InputHook() - self.record_queue = Queue() - self.hook.start() - - def _stop(self) -> List[Dict[str, Any]]: - if self.hook is None: - return [] - events = [event for event in self.hook.stop() - if self._wanted(event)] - self.hook = None - return events - - def _wanted(self, event: Dict[str, Any]) -> bool: - keyboard = event.get("op", "").startswith("key_") - return ("keyboard" if keyboard else "mouse") in self._kinds - - def _as_queue(self, events: List[Dict[str, Any]]) -> Queue: - """The historical shape: one executor command per press, no releases.""" - queue: Queue = Queue() - for event in events: - operation = event.get("op") - if operation == "key_down": - queue.put(("AC_type_keyboard", int(event.get("vk", 0)))) - elif operation == "mouse_down": - command = _LEGACY_MOUSE_COMMAND.get(event.get("button", "")) - if command: - queue.put((command, int(event.get("x", 0)), - int(event.get("y", 0)))) - self.record_queue = None - self.result_queue = queue - return queue - - def record(self) -> None: - """ - 開始錄製滑鼠與鍵盤事件 - Start recording both mouse and keyboard events - """ - self._start(("keyboard", "mouse")) - - def stop_record(self) -> Queue: - """ - 停止錄製並回傳事件 - Stop recording and return recorded events - """ - return self._as_queue(self._stop()) - - def stop_record_timeline(self) -> List[Dict[str, Any]]: - """ - 停止錄製並回傳含放開、滾輪與間隔時間的完整事件 - Stop recording and return press *and* release, wheel and ``delta_ms`` - - Ready for :func:`je_auto_control.utils.input_macro.replay_timeline`. - """ - return timeline(self._stop()) - - def record_mouse(self) -> None: - """ - 開始錄製滑鼠事件 - Start recording mouse events - """ - self._start(("mouse",)) - - def stop_record_mouse(self) -> Queue: - """ - 停止錄製滑鼠事件並回傳結果 - Stop recording mouse events and return results - """ - return self._as_queue(self._stop()) - - def record_keyboard(self) -> None: - """ - 開始錄製鍵盤事件 - Start recording keyboard events - """ - self._start(("keyboard",)) - - def stop_record_keyboard(self) -> Queue: - """ - 停止錄製鍵盤事件並回傳結果 - Stop recording keyboard events and return results - """ - return self._as_queue(self._stop()) + def new_hook(self) -> Win32InputHook: + """Return a fresh, uninstalled hook.""" + return Win32InputHook() # 全域錄製器實例 Global recorder instance -win32_recorder = Win32Recorder() \ No newline at end of file +win32_recorder = Win32Recorder() diff --git a/je_auto_control/wrapper/_platform_osx.py b/je_auto_control/wrapper/_platform_osx.py index 9714a88e..18a97fe0 100644 --- a/je_auto_control/wrapper/_platform_osx.py +++ b/je_auto_control/wrapper/_platform_osx.py @@ -37,6 +37,7 @@ ) from je_auto_control.osx.keyboard import osx_keyboard, osx_keyboard_check from je_auto_control.osx.mouse import osx_mouse +from je_auto_control.osx.record.osx_record import osx_recorder from je_auto_control.osx.screen import osx_screen from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -97,6 +98,12 @@ "enter": osx_key_enter, "tab": osx_key_tab, "backspace": osx_key_backspace, + # write() routes a control character by name first and by the raw + # character second. Without this entry "\b" matched neither, so + # write() fell through to its space fallback and silently typed a + # space where a backspace was asked for. X11 and Wayland both + # carry the raw character; macOS was the one that did not. + "\b": osx_key_backspace, "esc": osx_key_esc, "command": osx_key_command, "shift": osx_key_shift, @@ -143,7 +150,7 @@ keyboard_check = osx_keyboard_check mouse = osx_mouse screen = osx_screen -recorder = None +recorder = osx_recorder -if None in [keyboard_keys_table, mouse_keys_table, keyboard_check, keyboard, mouse, screen]: +if None in [keyboard_keys_table, mouse_keys_table, keyboard_check, keyboard, mouse, screen, recorder]: raise AutoControlException("Can't init auto control") diff --git a/je_auto_control/wrapper/auto_control_mouse.py b/je_auto_control/wrapper/auto_control_mouse.py index ab97cb09..9f736fd6 100644 --- a/je_auto_control/wrapper/auto_control_mouse.py +++ b/je_auto_control/wrapper/auto_control_mouse.py @@ -12,6 +12,9 @@ AutoControlCantFindKeyException, AutoControlMouseException ) from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.platform_id import ( + is_macos, is_windows, is_x11_unix +) from je_auto_control.utils.test_record.record_test_class import record_action_to_list from je_auto_control.wrapper.auto_control_screen import screen_size from je_auto_control.wrapper.platform_wrapper import mouse, mouse_keys_table, special_mouse_keys_table @@ -242,10 +245,19 @@ def mouse_scroll(scroll_value: int, x: int = None, y: int = None, 模擬滑鼠滾輪操作 Simulate mouse scroll - :param scroll_value: 滾動數值 Scroll value + 每個平台的規則相同：``scroll_value`` 為負就反向，絕對值是滾動格數。 + The sign of ``scroll_value`` reverses the direction on every platform, so a + call written on one works on the others. X11 and Wayland used to discard it + and always scroll ``scroll_direction``, which meant portable code scrolled + the opposite way there with no error and no warning. + + :param scroll_value: 滾動數值，負數代表反向 Scroll value; negative reverses :param x: X 座標，指定時會先將游標移到該處 X position; the cursor moves here first :param y: Y 座標，指定時會先將游標移到該處 Y position; the cursor moves here first - :param scroll_direction: 滾動方向 (Linux only) Scroll direction + :param scroll_direction: 未帶負號時的方向，只有 X11／Wayland 後端會讀。 + The direction a *positive* count scrolls in. Only the X11 and Wayland + backends read it — Windows and macOS have a single wheel axis and take + the direction from the sign alone. :return: (scroll_value, scroll_direction) """ autocontrol_logger.info(f"mouse_scroll, value={scroll_value}, x={x}, y={y}, direction={scroll_direction}") @@ -261,13 +273,21 @@ def mouse_scroll(scroll_value: int, x: int = None, y: int = None, if x is not None or y is not None: _scroll_to(x, y) - if sys.platform in ["win32", "cygwin", "msys"]: - mouse.scroll(scroll_value) - elif sys.platform == "darwin": + # 用 platform_id 問「哪一種輸入堆疊」，而不是再列一次 OS 名單： + # 原本的 ["linux", "linux2"] 把 BSD 漏在所有分支之外，滾動在 + # FreeBSD 上不會報錯，只是什麼都不做。 + # Ask platform_id which input stack this is rather than spelling out + # another list of OS names: the ["linux", "linux2"] one left the BSDs + # outside every branch, so scrolling on FreeBSD raised nothing and + # did nothing. + if is_windows() or is_macos(): mouse.scroll(scroll_value) - elif sys.platform in ["linux", "linux2"]: + elif is_x11_unix(): scroll_direction = special_mouse_keys_table.get(scroll_direction, scroll_direction) mouse.scroll(scroll_value, scroll_direction) + else: + raise AutoControlMouseException( + f"mouse_scroll: no backend for {sys.platform!r}") record_action_to_list("mouse_scroll", param) return scroll_value, scroll_direction diff --git a/je_auto_control/wrapper/auto_control_record.py b/je_auto_control/wrapper/auto_control_record.py index d161d3b7..df07af6a 100644 --- a/je_auto_control/wrapper/auto_control_record.py +++ b/je_auto_control/wrapper/auto_control_record.py @@ -1,9 +1,16 @@ +"""Start and stop input recording, and save what was recorded. + +macOS used to be refused here outright: the recorder existed but wiring it up +would have put an ``NSApplication`` and a blocking run loop into the import of +the platform wrapper. It no longer does — capture runs on a Quartz event tap +on its own thread — so the refusal is gone and every platform with a recorder +goes down the same path. A macOS session without Accessibility permission now +fails where that is actually true, in the tap, naming the permission. +""" import os -import sys import threading from typing import Optional -from je_auto_control.utils.exception.exception_tags import macos_record_error_message from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.exception.exceptions import AutoControlJsonActionException from je_auto_control.utils.json.json_file import write_action_json @@ -18,8 +25,6 @@ def record() -> None: """ autocontrol_logger.info("record") try: - if sys.platform == "darwin": - raise AutoControlException(macos_record_error_message) record_action_to_list("record", None) recorder.record() except (OSError, RuntimeError, AttributeError, TypeError, ValueError, AutoControlException, AutoControlJsonActionException) as error: @@ -33,8 +38,6 @@ def stop_record() -> list: """ autocontrol_logger.info("stop_record") try: - if sys.platform == "darwin": - raise AutoControlException(macos_record_error_message) action_queue = recorder.stop_record() if action_queue is None: raise AutoControlJsonActionException @@ -66,8 +69,6 @@ def stop_record_timeline() -> list: """ autocontrol_logger.info("stop_record_timeline") try: - if sys.platform == "darwin": - raise AutoControlException(macos_record_error_message) collect = getattr(recorder, "stop_record_timeline", None) if collect is None: autocontrol_logger.error( diff --git a/je_auto_control/wrapper/auto_control_screen.py b/je_auto_control/wrapper/auto_control_screen.py index 6d2571ce..46711836 100644 --- a/je_auto_control/wrapper/auto_control_screen.py +++ b/je_auto_control/wrapper/auto_control_screen.py @@ -1,9 +1,6 @@ import sys from typing import Tuple, List -import cv2 -import numpy as np - from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot from je_auto_control.utils.exception.exception_tags import screen_get_size_error_message from je_auto_control.utils.exception.exception_tags import screen_screenshot_error_message @@ -40,7 +37,13 @@ def screenshot(file_path: str = None, screen_region: list = None) -> List[int]: :param screen_region: screenshot region 截圖區域 """ autocontrol_logger.info(f"screenshot, file_path: {file_path}, screen_region: {screen_region}") - param = locals() + # 明寫成 dict,不用 locals():下面的 import 也會出現在 locals() 裡, + # 錄下來的參數就會多兩個模組物件。 + # Spelled out rather than locals(): the imports below land in locals() too, + # so the recorded parameters would carry two module objects. + param = {"file_path": file_path, "screen_region": screen_region} + import cv2 # noqa: E402 # reason: kept off the facade's import path + import numpy as np # noqa: E402 # reason: kept off the facade's import path try: record_action_to_list("AC_screenshot", param) return cv2.cvtColor( diff --git a/je_auto_control/wrapper/auto_control_window.py b/je_auto_control/wrapper/auto_control_window.py index 20b20066..fada874f 100644 --- a/je_auto_control/wrapper/auto_control_window.py +++ b/je_auto_control/wrapper/auto_control_window.py @@ -1,23 +1,23 @@ """Cross-platform window management facade. -On Windows, delegates to ``windows_window_manage`` (Win32 API). -On macOS / Linux, operations raise a clear ``NotImplementedError``. +Delegates to whichever backend :mod:`je_auto_control.wrapper.window_backends` +selects — Win32 on Windows, EWMH over python-Xlib on X11, Quartz plus the +accessibility API on macOS — and keeps here everything that is not +platform-specific: substring matching, waiting, and the compositions like +"move without restating the size". + +This was Windows-only for the project's whole life, which left these +functions, their 23 ``AC_*`` commands and their MCP tools dead on the two +other supported platforms. A platform with no backend still gets a null one +that lists nothing and refuses actions with a reason, so importing never +fails and callers get an answer rather than an ``ImportError``. """ -import sys import time from typing import List, Optional, Tuple, Union from je_auto_control.utils.exception.exceptions import AutoControlActionException from je_auto_control.utils.logging.logging_instance import autocontrol_logger - -_IS_WINDOWS = sys.platform in ("win32", "cygwin", "msys") - - -def _require_windows() -> None: - if not _IS_WINDOWS: - raise NotImplementedError( - f"Window management is only implemented on Windows (got {sys.platform})" - ) +from je_auto_control.wrapper.window_backends import get_backend def list_windows(titled_only: bool = False) -> List[Tuple[int, str]]: @@ -28,9 +28,8 @@ def list_windows(titled_only: bool = False) -> List[Tuple[int, str]]: Most visible windows have no title (shell and helper surfaces); pass ``titled_only`` for just the ones a user would recognise. """ - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm - found = wm.get_all_window_hwnd() + backend = get_backend() + found = backend.list_windows() if titled_only: return [(hwnd, title) for hwnd, title in found if title.strip()] return found @@ -49,20 +48,19 @@ def find_window(title_substring: str, def focus_window(title_substring: str, case_sensitive: bool = False) -> int: """Bring the first matching window to the foreground; return its hwnd.""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: raise AutoControlActionException( f"focus_window: no window matches {title_substring!r}" ) hwnd, title = hit - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() # A minimized window stays invisible however often you foreground it, so # restore it first — but only when it really is minimized: SW_RESTORE on a # maximized window un-maximizes it, which is not what "focus" should do. - if wm.is_window_minimized(hwnd): - wm.show_window(hwnd, wm.SW_RESTORE) - wm.set_foreground_window(hwnd) + if backend.is_minimized(hwnd): + backend.restore(hwnd) + backend.set_foreground(hwnd) autocontrol_logger.info("focused window hwnd=%s title=%r", hwnd, title) return hwnd @@ -72,7 +70,6 @@ def wait_for_window(title_substring: str, poll: float = 0.5, case_sensitive: bool = False) -> int: """Poll until a window with the given title appears; return its hwnd.""" - _require_windows() poll = max(0.05, float(poll)) deadline = time.monotonic() + float(timeout) while time.monotonic() < deadline: @@ -92,33 +89,30 @@ def close_window_by_title(title_substring: str, case_sensitive: bool = False) -> call underneath is named ``CloseWindow`` but minimises. Use :func:`minimize_window_by_title` for the old behaviour. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - return wm.close_window(hit[0]) + backend = get_backend() + return backend.close(hit[0]) def minimize_window_by_title(title_substring: str, case_sensitive: bool = False) -> bool: """Minimise the first matching window. ``False`` if nothing matched.""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - return wm.minimize_window(hit[0]) + backend = get_backend() + return backend.minimize(hit[0]) def foreground_window() -> Optional[Tuple[int, str]]: """The window the user is currently working in, or ``None``.""" - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm - hwnd = wm.get_foreground_window() + backend = get_backend() + hwnd = backend.foreground_window() if not hwnd: return None - titles = dict(wm.get_all_window_hwnd()) + titles = dict(backend.list_windows()) return hwnd, titles.get(hwnd, "") @@ -139,13 +133,12 @@ def post_key_to_window(title_substring: str, key: Union[int, str], the foreground all ignore posted messages. Callers must say so rather than reporting success. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() keycode, character = _resolve_key(key) - return wm.post_key(hit[0], keycode, character) + return backend.post_key(hit[0], keycode, character) def post_click_to_window(title_substring: str, button: str = "left", @@ -158,12 +151,11 @@ def post_click_to_window(title_substring: str, button: str = "left", client coordinates — a click posted to the frame lands nowhere. Same best-effort caveat as :func:`post_key_to_window`. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - return wm.post_click(hit[0], _mouse_button_name(button), int(x), int(y)) + backend = get_backend() + return backend.post_click(hit[0], _mouse_button_name(button), int(x), int(y)) def _resolve_key(key: Union[int, str]) -> Tuple[int, str]: @@ -196,23 +188,21 @@ def foreground_window_process_id() -> Optional[int]: activity probes, "is my automation target focused" — have to go through the process id. """ - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm - hwnd = wm.get_foreground_window() + backend = get_backend() + hwnd = backend.foreground_window() if not hwnd: return None - return wm.get_window_process_id(hwnd) or None + return backend.window_process_id(hwnd) or None def window_process_id(title_substring: str, case_sensitive: bool = False) -> Optional[int]: """The PID owning the first window whose title contains the substring.""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return None - from je_auto_control.windows.window import windows_window_manage as wm - return wm.get_window_process_id(hit[0]) or None + backend = get_backend() + return backend.window_process_id(hit[0]) or None def windows_for_process_id(pid: int, @@ -223,20 +213,18 @@ def windows_for_process_id(pid: int, named after whatever page they show, and several of its processes have no window at all. Ownership is the stable key. """ - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() target = int(pid) return [(hwnd, title) for hwnd, title in list_windows(titled_only) - if wm.get_window_process_id(hwnd) == target] + if backend.window_process_id(hwnd) == target] def minimize_windows_for_process(pid: int) -> int: """Minimise every visible top-level window owned by ``pid``; return the count.""" - _require_windows() - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() minimized = 0 for hwnd, _title in windows_for_process_id(pid): - if wm.minimize_window(hwnd): + if backend.minimize(hwnd): minimized += 1 return minimized @@ -249,12 +237,11 @@ def window_rect(title_substring: str, Screen coordinates, so on a multi-monitor desktop the values can be negative for a monitor left of or above the primary one. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return None - from je_auto_control.windows.window import windows_window_manage as wm - return wm.get_window_rect(hit[0]) + backend = get_backend() + return backend.window_rect(hit[0]) def move_window_by_title(title_substring: str, x: int, y: int, @@ -266,28 +253,26 @@ def move_window_by_title(title_substring: str, x: int, y: int, Omitting ``width`` / ``height`` keeps the window's current size, so a plain reposition does not have to restate dimensions the caller has to look up. """ - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm + backend = get_backend() if width is None or height is None: - rect = wm.get_window_rect(hit[0]) + rect = backend.window_rect(hit[0]) if rect is None: return False left, top, right, bottom = rect width = right - left if width is None else width height = bottom - top if height is None else height - return wm.move_window(hit[0], int(x), int(y), int(width), int(height)) + return backend.move(hit[0], int(x), int(y), int(width), int(height)) def show_window_by_title(title_substring: str, cmd_show: int = 1, case_sensitive: bool = False) -> bool: """Show or restore a window (``cmd_show`` follows Win32 ShowWindow).""" - _require_windows() hit = find_window(title_substring, case_sensitive) if hit is None: return False - from je_auto_control.windows.window import windows_window_manage as wm - wm.show_window(hit[0], int(cmd_show)) + backend = get_backend() + backend.show(hit[0], int(cmd_show)) return True diff --git a/je_auto_control/wrapper/platform_wrapper.py b/je_auto_control/wrapper/platform_wrapper.py index 6940935a..e844d53e 100644 --- a/je_auto_control/wrapper/platform_wrapper.py +++ b/je_auto_control/wrapper/platform_wrapper.py @@ -1,20 +1,21 @@ import sys from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.platform_id import is_bsd, is_macos, is_windows -if sys.platform in ["win32", "cygwin", "msys"]: +if is_windows(): from je_auto_control.wrapper._platform_windows import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) -elif sys.platform == "darwin": +elif is_macos(): from je_auto_control.wrapper._platform_osx import ( # noqa: F401 # reason: facade re-export keyboard, keyboard_check, keyboard_keys_table, mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) -elif sys.platform in ["linux", "linux2"]: +elif sys.platform.startswith("linux"): from je_auto_control.linux_wayland import select_display_server from je_auto_control.utils.logging.logging_instance import ( autocontrol_logger, @@ -45,8 +46,21 @@ mouse, mouse_keys_table, special_mouse_keys_table, screen, recorder, ) +elif is_bsd(): + # A FreeBSD, OpenBSD or NetBSD desktop is an ordinary X11 desktop: the + # same X server, the same python-Xlib, the same backend. There is no + # Wayland branch here because the Wayland backend's fast path is libei, + # whose socket and portal are Linux desktop infrastructure; a BSD running + # Wayland reaches this through XWayland like any other X11 client. + from je_auto_control.wrapper._platform_linux import ( # noqa: F401 # reason: facade re-export + keyboard, keyboard_check, keyboard_keys_table, + mouse, mouse_keys_table, special_mouse_keys_table, + screen, recorder, + ) else: - raise AutoControlException("unknown operating system") + raise AutoControlException( + f"unknown operating system: {sys.platform!r}. Windows, macOS, Linux " + f"and the BSDs are supported.") if None in [keyboard_keys_table, mouse_keys_table, keyboard, mouse, screen]: raise AutoControlException("Can't init auto control") diff --git a/je_auto_control/wrapper/window_backends/__init__.py b/je_auto_control/wrapper/window_backends/__init__.py new file mode 100644 index 00000000..144c987d --- /dev/null +++ b/je_auto_control/wrapper/window_backends/__init__.py @@ -0,0 +1,70 @@ +"""Per-platform window-management backends. + +Window management was Windows-only for the project's whole life: the facade +branched on ``sys.platform`` and raised ``NotImplementedError`` everywhere +else, which left 23 ``AC_*`` commands and their MCP tools dead on macOS and +Linux. This is the seam that replaces that branch, following the same shape +the accessibility, OCR and vision subsystems already use — abstract base, +concrete implementations, null fallback — so a platform without one still +imports. + +Selection is cached because probing opens an X connection or a Quartz query, +and the answer cannot change inside a process. +""" +import sys +from typing import Optional + +from je_auto_control.wrapper.window_backends.base import WindowManageBackend +from je_auto_control.wrapper.window_backends.null_backend import NullWindowBackend + +_cached_backend: Optional[WindowManageBackend] = None + + +def get_backend() -> WindowManageBackend: + """Return (and cache) the best window backend for this platform.""" + global _cached_backend + if _cached_backend is None: + _cached_backend = _build_backend() + return _cached_backend + + +def reset_backend_cache() -> None: + """Force the next :func:`get_backend` call to re-detect.""" + global _cached_backend + _cached_backend = None + + +def _build_backend() -> WindowManageBackend: + if sys.platform in ("win32", "cygwin", "msys"): + from je_auto_control.wrapper.window_backends.windows_backend import ( + WindowsWindowBackend, + ) + return WindowsWindowBackend() + if sys.platform == "darwin": + from je_auto_control.wrapper.window_backends.macos_backend import ( + MacOSWindowBackend, + ) + backend = MacOSWindowBackend() + if backend.available: + return backend + return NullWindowBackend( + "pyobjc (Quartz, AppKit) is required for macOS window management") + if sys.platform in ("linux", "linux2"): + from je_auto_control.wrapper.window_backends.x11_backend import ( + X11WindowBackend, + ) + backend = X11WindowBackend() + if backend.available: + return backend + # Wayland deliberately does not let a client enumerate or move other + # applications' windows; that is a protocol decision, not a gap here. + return NullWindowBackend( + "no X display. Wayland does not expose other windows to a client, " + "so run under X11 or XWayland for window management") + return NullWindowBackend(f"no window backend for platform {sys.platform!r}") + + +__all__ = [ + "NullWindowBackend", "WindowManageBackend", + "get_backend", "reset_backend_cache", +] diff --git a/je_auto_control/wrapper/window_backends/base.py b/je_auto_control/wrapper/window_backends/base.py new file mode 100644 index 00000000..0695cebd --- /dev/null +++ b/je_auto_control/wrapper/window_backends/base.py @@ -0,0 +1,120 @@ +"""Abstract window-management backend.""" +from typing import List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import ( + AutoControlUnsupportedOperationException, +) + + +class WindowManageBackend: + """One platform's answers about its top-level windows. + + Everything that is not platform-specific — substring matching, waiting, + "move without restating the size" — lives in + :mod:`je_auto_control.wrapper.auto_control_window`, so a backend only has + to list windows and answer about one at a time. + + ``window_id`` is whatever the platform calls a window: an ``HWND`` on + Windows, an X11 window id on Linux, a ``CGWindowID`` on macOS. It is + always a plain ``int``, so it composes with any other call on that + platform. + + A backend that cannot perform an operation raises through + :meth:`_unsupported` rather than returning a falsy value: "this platform + has no such concept" and "it did not work this time" are different + answers, and a caller that cannot tell them apart will retry forever. + """ + + name: str = "abstract" + available: bool = False + + # --- listing ----------------------------------------------------------- + + def list_windows(self) -> List[Tuple[int, str]]: + """``(window_id, title)`` for every visible top-level window. + + Front-most first, so the first match of a title substring is the one + the user is most likely to mean. + """ + raise NotImplementedError + + def foreground_window(self) -> int: + """The window the user is working in, or ``0`` when there is none.""" + self._unsupported("foreground_window") + + # --- reading one window ------------------------------------------------ + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + """``(left, top, right, bottom)`` in screen coordinates, or None. + + Screen coordinates, so on a multi-monitor desktop these can be + negative for a monitor left of or above the primary one. + """ + self._unsupported("window_rect") + + def window_process_id(self, window_id: int) -> int: + """The pid owning the window, or ``0`` when it cannot be determined. + + A title is not identity — applications rewrite theirs at will and + unrelated programs share names like ``Settings`` — so ownership is + what addresses a window stably. + """ + self._unsupported("window_process_id") + + def is_minimized(self, window_id: int) -> bool: + """Whether the window is minimised / iconified.""" + self._unsupported("is_minimized") + + # --- acting on one window ---------------------------------------------- + + def set_foreground(self, window_id: int) -> None: + """Raise the window and give it the keyboard focus.""" + self._unsupported("set_foreground") + + def restore(self, window_id: int) -> None: + """Un-minimise the window without changing anything else. + + Deliberately not "show it however": restoring a *maximised* window + would un-maximise it, which is not what focusing something should do. + """ + self._unsupported("restore") + + def show(self, window_id: int, cmd_show: int) -> None: + """Apply a platform show-state code (Win32 ``ShowWindow`` numbering).""" + self._unsupported("show") + + def close(self, window_id: int) -> bool: + """Ask the window to close; ``False`` when the request was refused.""" + self._unsupported("close") + + def minimize(self, window_id: int) -> bool: + """Minimise the window; ``False`` when the request was refused.""" + self._unsupported("minimize") + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + """Move and resize the window; ``False`` when the request was refused.""" + self._unsupported("move") + + # --- acting on a window that does not have focus ----------------------- + + def post_key(self, window_id: int, keycode: int, + character: str = "") -> bool: + """Deliver one key to the window without focusing it.""" + self._unsupported("post_key") + + def post_click(self, window_id: int, button: str, x: int, y: int) -> bool: + """Deliver one click to the window without focusing it. + + ``x`` / ``y`` are relative to the window's own top-left corner. + """ + self._unsupported("post_click") + + # --- refusal ----------------------------------------------------------- + + def _unsupported(self, operation: str): + """Raise a clear error naming what this backend cannot do.""" + raise AutoControlUnsupportedOperationException( + f"{operation} is not supported by the {self.name} window backend", + ) diff --git a/je_auto_control/wrapper/window_backends/macos_backend.py b/je_auto_control/wrapper/window_backends/macos_backend.py new file mode 100644 index 00000000..edaf30e0 --- /dev/null +++ b/je_auto_control/wrapper/window_backends/macos_backend.py @@ -0,0 +1,301 @@ +"""macOS window-management backend, over Quartz and the accessibility API. + +Reading and acting are two different APIs on macOS, with two different +permission stories, and this backend needs both: + +* **Quartz** (``CGWindowListCopyWindowInfo``) enumerates every on-screen + window with its id, title, owning pid and bounds. It needs no grant, so + listing, rectangles and ownership work out of the box. +* **The accessibility API** is the only way to *move*, *close*, *minimise* or + *raise* someone else's window. It is gated by TCC: the user grants + Accessibility to the interpreter, and until they do every action silently + does nothing. :meth:`MacOSWindowBackend.available` reports the Quartz half, + and each action raises through the base class when AX refuses, rather than + returning a false success. + +Quartz has no window handle that the accessibility API accepts, so a +``CGWindowID`` is matched to its ``AXUIElement`` by owner, title and frame. +That is what the two APIs give us to work with; the alternative is a private +symbol (``_AXUIElementGetWindow``) this project will not depend on. +""" +import sys +from typing import Any, List, Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + +#: ``ShowWindow`` codes with a macOS meaning; see the X11 backend for why the +#: project spells show-state in Win32's numbering. +SW_SHOWNORMAL = 1 +SW_SHOWMINIMIZED = 2 +SW_MINIMIZE = 6 +SW_RESTORE = 9 + + +class MacOSWindowBackend(WindowManageBackend): + """Quartz for what a window *is*, accessibility for what it *does*.""" + + name = "macos-quartz-ax" + + def __init__(self) -> None: + self.available = sys.platform == "darwin" and self._probe() + + def _probe(self) -> bool: + try: + import Quartz # noqa: F401 # reason: probe import + import AppKit # noqa: F401 # reason: probe import + return True + except ImportError as error: + autocontrol_logger.info( + "macOS window backend unavailable: %r", error) + return False + + # --- listing, via Quartz ----------------------------------------------- + + def _window_info(self) -> List[dict]: + """Every on-screen window Quartz will admit to, front-most first.""" + import Quartz + + options = (Quartz.kCGWindowListOptionOnScreenOnly + | Quartz.kCGWindowListExcludeDesktopElements) + found = Quartz.CGWindowListCopyWindowInfo(options, Quartz.kCGNullWindowID) + return list(found or []) + + def list_windows(self) -> List[Tuple[int, str]]: + import Quartz + + windows = [] + for info in self._window_info(): + # Layer 0 is the ordinary application layer. Menu bars, the Dock + # and status items live above it and are not windows a caller + # means when they say "the Safari window". + if int(info.get(Quartz.kCGWindowLayer, 0) or 0) != 0: + continue + number = int(info.get(Quartz.kCGWindowNumber, 0) or 0) + if not number: + continue + windows.append((number, str(info.get(Quartz.kCGWindowName, "") or ""))) + return windows + + def _info_for(self, window_id: int) -> Optional[dict]: + import Quartz + + for info in self._window_info(): + if int(info.get(Quartz.kCGWindowNumber, 0) or 0) == int(window_id): + return info + return None + + def foreground_window(self) -> int: + import AppKit + import Quartz + + workspace = AppKit.NSWorkspace.sharedWorkspace() + frontmost = workspace.frontmostApplication() + if frontmost is None: + return 0 + pid = int(frontmost.processIdentifier()) + for info in self._window_info(): + if (int(info.get(Quartz.kCGWindowOwnerPID, 0) or 0) == pid + and int(info.get(Quartz.kCGWindowLayer, 0) or 0) == 0): + # The list is front-to-back, so the owning app's first entry + # is the window that is actually in front. + return int(info.get(Quartz.kCGWindowNumber, 0) or 0) + return 0 + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + import Quartz + + info = self._info_for(window_id) + if info is None: + return None + bounds = info.get(Quartz.kCGWindowBounds) + if not bounds: + return None + left, top = int(bounds["X"]), int(bounds["Y"]) + return (left, top, left + int(bounds["Width"]), + top + int(bounds["Height"])) + + def window_process_id(self, window_id: int) -> int: + import Quartz + + info = self._info_for(window_id) + if info is None: + return 0 + return int(info.get(Quartz.kCGWindowOwnerPID, 0) or 0) + + # --- acting, via the accessibility API --------------------------------- + + def _ax_windows_for(self, pid: int) -> list: + """Every accessibility window belonging to a process.""" + import ApplicationServices as ax + + application = ax.AXUIElementCreateApplication(pid) + error, windows = ax.AXUIElementCopyAttributeValue( + application, "AXWindows", None) + return [] if error or not windows else list(windows) + + def _ax_window(self, window_id: int): + """The ``AXUIElement`` for a ``CGWindowID``, or None. + + Quartz ids and accessibility elements are separate namespaces with no + public bridge, so the window is found again inside its owning + application by frame and title — the two properties both APIs report. + """ + import Quartz + + info = self._info_for(window_id) + if info is None: + return None + pid = int(info.get(Quartz.kCGWindowOwnerPID, 0) or 0) + if not pid: + return None + bounds = info.get(Quartz.kCGWindowBounds) or {} + return _best_match( + self._ax_windows_for(pid), + (int(bounds.get("X", 0)), int(bounds.get("Y", 0))), + str(info.get(Quartz.kCGWindowName, "") or "")) + + def _require_ax_window(self, window_id: int, operation: str): + window = self._ax_window(window_id) + if window is None: + raise _refusal(operation, window_id) + return window + + def is_minimized(self, window_id: int) -> bool: + import ApplicationServices as ax + + window = self._ax_window(window_id) + if window is None: + # A minimised window is not in the on-screen list at all, so + # failing to find it is itself the answer here. + return self._info_for(window_id) is None + _error, value = ax.AXUIElementCopyAttributeValue( + window, "AXMinimized", None) + return bool(value) + + def set_foreground(self, window_id: int) -> None: + import AppKit + import ApplicationServices as ax + + pid = self.window_process_id(window_id) + if not pid: + raise _refusal("set_foreground", window_id) + running = AppKit.NSRunningApplication.runningApplicationWithProcessIdentifier_(pid) + if running is not None: + running.activateWithOptions_( + AppKit.NSApplicationActivateIgnoringOtherApps) + # Activating the application brings its front window forward, which is + # not necessarily the one asked for, so raise that one specifically. + window = self._ax_window(window_id) + if window is not None: + ax.AXUIElementPerformAction(window, "AXRaise") + + def restore(self, window_id: int) -> None: + import ApplicationServices as ax + + window = self._require_ax_window(window_id, "restore") + ax.AXUIElementSetAttributeValue(window, "AXMinimized", False) + + def show(self, window_id: int, cmd_show: int) -> None: + code = int(cmd_show) + if code in (SW_SHOWNORMAL, SW_RESTORE): + self.restore(window_id) + elif code in (SW_MINIMIZE, SW_SHOWMINIMIZED): + self.minimize(window_id) + else: + # macOS has no hide-this-window or maximise-this-window that maps + # onto the remaining Win32 codes; zoom is not maximise and the + # difference matters to the caller. + self._unsupported(f"show(cmd_show={code})") + + def close(self, window_id: int) -> bool: + import ApplicationServices as ax + + window = self._ax_window(window_id) + if window is None: + return False + error, button = ax.AXUIElementCopyAttributeValue( + window, "AXCloseButton", None) + if error or button is None: + return False + return not ax.AXUIElementPerformAction(button, "AXPress") + + def minimize(self, window_id: int) -> bool: + import ApplicationServices as ax + + window = self._ax_window(window_id) + if window is None: + return False + return not ax.AXUIElementSetAttributeValue(window, "AXMinimized", True) + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + import ApplicationServices as ax + import Quartz + + window = self._ax_window(window_id) + if window is None: + return False + position = ax.AXValueCreate( + Quartz.kAXValueCGPointType, Quartz.CGPoint(float(x), float(y))) + size = ax.AXValueCreate( + Quartz.kAXValueCGSizeType, + Quartz.CGSize(float(width), float(height))) + moved = ax.AXUIElementSetAttributeValue(window, "AXPosition", position) + resized = ax.AXUIElementSetAttributeValue(window, "AXSize", size) + return not moved and not resized + + # post_key / post_click are deliberately left to the base class. macOS has + # no equivalent of PostMessage or XSendEvent: an event goes to whatever + # has focus, and there is no way to address one window without raising it. + # Refusing says so; a "success" that focused something else would not. + + +def _best_match(candidates: list, wanted_origin: Tuple[int, int], + wanted_title: str): + """Pick the accessibility window that is the Quartz window described. + + Frame is the stronger signal and is tried first: a title can be empty, or + duplicated across an application's windows, while two windows cannot share + an origin at the same moment. + """ + import ApplicationServices as ax + + fallback = None + for window in candidates: + _error, position = ax.AXUIElementCopyAttributeValue( + window, "AXPosition", None) + if _point(position) == wanted_origin: + return window + if fallback is None and wanted_title: + _error, title = ax.AXUIElementCopyAttributeValue( + window, "AXTitle", None) + if str(title or "") == wanted_title: + fallback = window + return fallback + + +def _point(value: Any) -> Tuple[int, int]: + """Read an ``AXValue`` point as ``(x, y)``, or ``(-1, -1)``.""" + import ApplicationServices as ax + import Quartz + + if value is None: + return (-1, -1) + ok, point = ax.AXValueGetValue(value, Quartz.kAXValueCGPointType, None) + if not ok or point is None: + return (-1, -1) + return (int(point.x), int(point.y)) + + +def _refusal(operation: str, window_id: int): + from je_auto_control.utils.exception.exceptions import ( + AutoControlUnsupportedOperationException, + ) + + return AutoControlUnsupportedOperationException( + f"{operation}: no accessibility element for window {window_id}. " + "macOS gates window actions behind Accessibility — grant it to this " + "interpreter in System Settings > Privacy & Security > Accessibility.", + ) diff --git a/je_auto_control/wrapper/window_backends/null_backend.py b/je_auto_control/wrapper/window_backends/null_backend.py new file mode 100644 index 00000000..0f1ecf1f --- /dev/null +++ b/je_auto_control/wrapper/window_backends/null_backend.py @@ -0,0 +1,23 @@ +"""Fallback window backend for a platform with no implementation.""" +from typing import List, Tuple + +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + + +class NullWindowBackend(WindowManageBackend): + """Answers every operation with a refusal that says why. + + ``list_windows`` returns an empty list rather than raising: "there are no + windows I can see" is a truthful answer that lets a caller iterate, and + every operation that would *act* on a window still refuses loudly. + """ + + name = "null" + + def __init__(self, reason: str = "") -> None: + self.available = False + self.reason = reason or "no window backend for this platform" + self.name = f"null ({self.reason})" + + def list_windows(self) -> List[Tuple[int, str]]: + return [] diff --git a/je_auto_control/wrapper/window_backends/windows_backend.py b/je_auto_control/wrapper/window_backends/windows_backend.py new file mode 100644 index 00000000..39e87c39 --- /dev/null +++ b/je_auto_control/wrapper/window_backends/windows_backend.py @@ -0,0 +1,67 @@ +"""Windows window-management backend, over the existing Win32 module.""" +import sys +from typing import List, Optional, Tuple + +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + + +class WindowsWindowBackend(WindowManageBackend): + """Delegates to :mod:`je_auto_control.windows.window.windows_window_manage`. + + That module is unchanged and stays the single home of the Win32 calls; + this only adapts it to the shape the other platforms also answer in. + """ + + name = "win32" + + def __init__(self) -> None: + self.available = sys.platform in ("win32", "cygwin", "msys") + + @property + def _wm(self): + """The Win32 module, imported lazily so other platforms never load it.""" + from je_auto_control.windows.window import windows_window_manage + + return windows_window_manage + + def list_windows(self) -> List[Tuple[int, str]]: + return self._wm.get_all_window_hwnd() + + def foreground_window(self) -> int: + return self._wm.get_foreground_window() + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + return self._wm.get_window_rect(window_id) + + def window_process_id(self, window_id: int) -> int: + return self._wm.get_window_process_id(window_id) + + def is_minimized(self, window_id: int) -> bool: + return self._wm.is_window_minimized(window_id) + + def set_foreground(self, window_id: int) -> None: + self._wm.set_foreground_window(window_id) + + def restore(self, window_id: int) -> None: + self._wm.show_window(window_id, self._wm.SW_RESTORE) + + def show(self, window_id: int, cmd_show: int) -> None: + self._wm.show_window(window_id, int(cmd_show)) + + def close(self, window_id: int) -> bool: + return self._wm.close_window(window_id) + + def minimize(self, window_id: int) -> bool: + return self._wm.minimize_window(window_id) + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + return self._wm.move_window(window_id, x, y, width, height) + + def post_key(self, window_id: int, keycode: int, + character: str = "") -> bool: + return self._wm.post_key(window_id, keycode, character) + + def post_click(self, window_id: int, button: str, x: int, y: int) -> bool: + return self._wm.post_click(window_id, button, x, y) diff --git a/je_auto_control/wrapper/window_backends/x11_backend.py b/je_auto_control/wrapper/window_backends/x11_backend.py new file mode 100644 index 00000000..50ba1960 --- /dev/null +++ b/je_auto_control/wrapper/window_backends/x11_backend.py @@ -0,0 +1,398 @@ +"""X11 window-management backend, over EWMH and python-Xlib. + +Everything here goes through the EWMH properties and client messages a +window manager maintains (``_NET_CLIENT_LIST_STACKING``, ``_NET_ACTIVE_WINDOW``, +``_NET_WM_PID``, ``_NET_WM_STATE``, …) rather than by poking at windows +directly, because on X11 the window manager owns stacking, focus and +iconification — a client that reparents or raises behind its back gets +overruled or, worse, half-obeyed. + +``python-Xlib`` is already a hard dependency on Linux, so this adds nothing to +install. + +One caveat is written into :meth:`X11WindowBackend.post_key` and +:meth:`X11WindowBackend.post_click` rather than hidden: events delivered with +``XSendEvent`` arrive at the client flagged *synthetic*, and most toolkits +(GTK and Qt among them) ignore synthetic input by design. They are the closest +X11 equivalent of Win32's ``PostMessage`` — which also bypasses focus — and +they are honest about being best-effort. +""" +import sys +import time +from typing import Any, List, Optional, Tuple + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.wrapper.window_backends.base import WindowManageBackend + +#: ``ShowWindow`` codes this backend can honour, mapped to what X11 calls the +#: same idea. The Win32 numbering is the project's cross-platform spelling of +#: show-state; the codes with no X11 meaning are refused rather than guessed. +SW_HIDE = 0 +SW_SHOWNORMAL = 1 +SW_SHOWMINIMIZED = 2 +SW_MAXIMIZE = 3 +SW_MINIMIZE = 6 +SW_RESTORE = 9 + +#: ICCCM window states, from the specification. +_ICONIC_STATE = 3 + +#: Button names the project uses, in X11's button numbering. +_BUTTONS = {"left": 1, "middle": 2, "right": 3} + + +def _is_linux() -> bool: + return sys.platform in ("linux", "linux2") + + +class X11WindowBackend(WindowManageBackend): + """Window management through the window manager's own EWMH contract.""" + + name = "x11-ewmh" + + def __init__(self) -> None: + self._connection = None + self._atoms: dict = {} + self.available = _is_linux() and self._probe() + + # --- connection -------------------------------------------------------- + + def _probe(self) -> bool: + try: + self._display() + return True + except Exception as error: # noqa: BLE001 # reason: any X failure means unavailable + autocontrol_logger.info( + "X11 window backend unavailable: %r", error) + return False + + def _display(self): + """The X connection, opened once and kept. + + Its own connection rather than the input backend's: this module is + selected by the wrapper independently of which input path is active, + and on Wayland-with-XWayland the input backend may not have opened one + at all. + """ + if self._connection is None: + from Xlib import display as xdisplay + + self._connection = xdisplay.Display() + return self._connection + + def _atom(self, name: str) -> int: + """Intern an atom once per connection.""" + if name not in self._atoms: + self._atoms[name] = self._display().intern_atom(name) + return self._atoms[name] + + def _root(self): + return self._display().screen().root + + def _window(self, window_id: int): + return self._display().create_resource_object("window", int(window_id)) + + def _property(self, window, name: str, kind: Optional[int] = None): + """Read a property's value list, or ``None`` when it is absent.""" + from Xlib import X + + found = window.get_full_property( + self._atom(name), X.AnyPropertyType if kind is None else kind) + return None if found is None else found.value + + def _client_message(self, window, name: str, data: List[int]) -> None: + """Send an EWMH client message to the root window. + + EWMH requests are addressed to the root with + ``SubstructureRedirect``: that is what routes them to the window + manager, which is the only party allowed to act on them. + """ + from Xlib import X, protocol + + padded = (list(data) + [0, 0, 0, 0, 0])[:5] + event = protocol.event.ClientMessage( + window=window, client_type=self._atom(name), data=(32, padded)) + self._root().send_event( + event, + event_mask=X.SubstructureRedirectMask | X.SubstructureNotifyMask) + self._display().flush() + + # --- listing ----------------------------------------------------------- + + def list_windows(self) -> List[Tuple[int, str]]: + from Xlib import Xatom + + # Stacking order is bottom-to-top, so reversing it puts the front-most + # window first — which is what makes "the first title that matches" the + # one the user meant. _NET_CLIENT_LIST carries no order at all, so it + # is only the fallback. + ids = self._property(self._root(), "_NET_CLIENT_LIST_STACKING", + Xatom.WINDOW) + if ids is None: + ids = self._property(self._root(), "_NET_CLIENT_LIST", + Xatom.WINDOW) or [] + ordered = list(ids) + else: + ordered = list(reversed(list(ids))) + return [(int(window_id), self._title(int(window_id))) + for window_id in ordered] + + def _title(self, window_id: int) -> str: + """``_NET_WM_NAME`` if the client sets it, else the legacy ``WM_NAME``.""" + window = self._window(window_id) + try: + value = self._property(window, "_NET_WM_NAME") + if value: + return _as_text(value) + legacy = window.get_wm_name() + return legacy if isinstance(legacy, str) else _as_text(legacy) + except Exception: # noqa: BLE001 # reason: a window can vanish mid-walk + return "" + + def foreground_window(self) -> int: + from Xlib import Xatom + + active = self._property(self._root(), "_NET_ACTIVE_WINDOW", Xatom.WINDOW) + return int(active[0]) if active else 0 + + # --- reading one window ------------------------------------------------ + + def _frame(self, window_id: int): + """The window manager's frame around a client, or the client itself. + + A reparenting window manager makes the client a grandchild of the + root, inside a frame that carries the border and title bar. The frame + is what the user sees and drags, so it is the window every coordinate + here is about. + """ + window = self._window(window_id) + root_id = self._root().id + # Bounded rather than `while True`: a frame is one or two levels, and + # a cycle here would hang the caller instead of returning something. + for _ in range(16): + tree = window.query_tree() + parent = getattr(tree, "parent", None) + if parent is None or parent.id == root_id: + return window + window = parent + return window + + def _frame_extents(self, window_id: int) -> Tuple[int, int, int, int]: + """``(left, right, top, bottom)`` decoration thickness, or zeroes.""" + from Xlib import Xatom + + try: + value = self._property(self._window(window_id), + "_NET_FRAME_EXTENTS", Xatom.CARDINAL) + except Exception: # noqa: BLE001 # reason: the window may be gone + return (0, 0, 0, 0) + if not value or len(value) < 4: + return (0, 0, 0, 0) + return (int(value[0]), int(value[1]), int(value[2]), int(value[3])) + + def window_rect(self, window_id: int, + ) -> Optional[Tuple[int, int, int, int]]: + """The *frame* rectangle, decorations included. + + Win32's ``GetWindowRect`` returns the frame, and every caller in this + project is written against that, so returning the client area here + would be off by the border and title bar on X11 alone — silently, and + by a different amount per window manager. + """ + try: + frame = self._frame(window_id) + geometry = frame.get_geometry() + # The frame is a direct child of the root, so its own x/y are + # already root coordinates. Translating instead would add the + # window's border width, because XTranslateCoordinates starts + # from inside the border — measured as a one-pixel error against + # xwininfo on an undecorated window. + left, top = int(geometry.x), int(geometry.y) + return (left, top, left + int(geometry.width), + top + int(geometry.height)) + except Exception as error: # noqa: BLE001 # reason: the window may be gone + autocontrol_logger.info("window_rect(%s) failed: %r", window_id, error) + return None + + def window_process_id(self, window_id: int) -> int: + from Xlib import Xatom + + try: + value = self._property(self._window(window_id), "_NET_WM_PID", + Xatom.CARDINAL) + except Exception: # noqa: BLE001 # reason: the window may be gone + return 0 + return int(value[0]) if value else 0 + + def is_minimized(self, window_id: int) -> bool: + try: + states = self._property(self._window(window_id), "_NET_WM_STATE") or [] + except Exception: # noqa: BLE001 # reason: the window may be gone + return False + return self._atom("_NET_WM_STATE_HIDDEN") in list(states) + + # --- acting on one window ---------------------------------------------- + + def set_foreground(self, window_id: int) -> None: + window = self._window(window_id) + # Source 2 ("pager") is what a window manager honours without the + # focus-stealing prevention it applies to source 1 ("application"). + self._client_message(window, "_NET_ACTIVE_WINDOW", + [2, int(time.time()), 0]) + self._display().flush() + + def restore(self, window_id: int) -> None: + window = self._window(window_id) + window.map() + self._client_message(window, "_NET_ACTIVE_WINDOW", + [2, int(time.time()), 0]) + self._display().flush() + + def show(self, window_id: int, cmd_show: int) -> None: + window = self._window(window_id) + code = int(cmd_show) + if code == SW_HIDE: + window.unmap() + elif code in (SW_SHOWNORMAL, SW_RESTORE): + self.restore(window_id) + elif code in (SW_MINIMIZE, SW_SHOWMINIMIZED): + self.minimize(window_id) + elif code == SW_MAXIMIZE: + # _NET_WM_STATE_ADD is 1; both axes have to be named or the window + # only grows one way. + self._client_message( + window, "_NET_WM_STATE", + [1, self._atom("_NET_WM_STATE_MAXIMIZED_HORZ"), + self._atom("_NET_WM_STATE_MAXIMIZED_VERT"), 2]) + else: + # Win32 has show codes with no X11 meaning (SW_SHOWNA, + # SW_FORCEMINIMIZE, …). Guessing at one would move a window in a + # way the caller did not ask for. + self._unsupported(f"show(cmd_show={code})") + self._display().flush() + + def close(self, window_id: int) -> bool: + try: + self._client_message(self._window(window_id), "_NET_CLOSE_WINDOW", + [int(time.time()), 2]) + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("close(%s) failed: %r", window_id, error) + return False + + def minimize(self, window_id: int) -> bool: + try: + # Iconifying is ICCCM, not EWMH: there is no _NET_ message for it, + # and WM_CHANGE_STATE is what every window manager implements. + self._client_message(self._window(window_id), "WM_CHANGE_STATE", + [_ICONIC_STATE]) + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("minimize(%s) failed: %r", window_id, error) + return False + + def move(self, window_id: int, x: int, y: int, + width: int, height: int) -> bool: + """Move and resize the *frame*, matching :meth:`window_rect`. + + Goes through ``_NET_MOVERESIZE_WINDOW`` rather than configuring the + window directly: under a reparenting window manager a client's own + x/y are relative to its frame, so a direct ``ConfigureWindow`` asks + for a position in the wrong coordinate space and the window manager + applies its own arithmetic on top. Measured against openbox, asking + for (300, 220) that way landed the window at (302, 260). + + EWMH sizes the *client*, while Win32's ``MoveWindow`` sizes the + frame, so the decorations come off the requested size — which is what + makes this round-trip with :meth:`window_rect`. + """ + try: + border_left, border_right, border_top, border_bottom = \ + self._frame_extents(window_id) + client_width = max(1, int(width) - border_left - border_right) + client_height = max(1, int(height) - border_top - border_bottom) + # Bits 8-11 say which of x/y/width/height are supplied; bits 12-13 + # are the source indication, and 2 means "pager", which a window + # manager honours without focus-stealing prevention. Gravity 0 + # leaves the window's own gravity in charge, so x/y place the + # frame's top-left corner. + flags = (1 << 8) | (1 << 9) | (1 << 10) | (1 << 11) | (2 << 12) + self._client_message( + self._window(window_id), "_NET_MOVERESIZE_WINDOW", + [flags, int(x), int(y), client_width, client_height]) + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("move(%s) failed: %r", window_id, error) + return False + + # --- acting on a window that does not have focus ----------------------- + + def post_key(self, window_id: int, keycode: int, + character: str = "") -> bool: + """Send one key to the window without focusing it. + + Delivered with ``XSendEvent``, so it arrives flagged synthetic and + GTK and Qt discard it by design. This is the X11 counterpart of + Win32's ``PostMessage`` — best-effort, and useful mainly for the + older toolkits that do accept it. For input that always lands, focus + the window and use the ordinary keyboard API. + """ + from Xlib import X, protocol + + del character # X11 addresses keys by keycode; the character is Win32's + try: + window = self._window(window_id) + for factory, mask in ((protocol.event.KeyPress, X.KeyPressMask), + (protocol.event.KeyRelease, X.KeyReleaseMask)): + window.send_event( + factory(time=X.CurrentTime, root=self._root(), + window=window, same_screen=1, child=X.NONE, + root_x=0, root_y=0, event_x=0, event_y=0, + state=0, detail=int(keycode)), + propagate=True, event_mask=mask) + self._display().flush() + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("post_key(%s) failed: %r", window_id, error) + return False + + def post_click(self, window_id: int, button: str, x: int, y: int) -> bool: + """Send one click to the window without focusing it. + + Carries the same synthetic-event caveat as :meth:`post_key`. + """ + from Xlib import X, protocol + + number = _BUTTONS.get(str(button).lower().removeprefix("mouse_")) + if number is None: + self._unsupported(f"post_click(button={button!r})") + try: + window = self._window(window_id) + for factory, mask in ( + (protocol.event.ButtonPress, X.ButtonPressMask), + (protocol.event.ButtonRelease, X.ButtonReleaseMask)): + window.send_event( + factory(time=X.CurrentTime, root=self._root(), + window=window, same_screen=1, child=X.NONE, + root_x=0, root_y=0, event_x=int(x), event_y=int(y), + state=0, detail=number), + propagate=True, event_mask=mask) + self._display().flush() + return True + except Exception as error: # noqa: BLE001 # reason: report, do not abort + autocontrol_logger.info("post_click(%s) failed: %r", window_id, error) + return False + + +def _as_text(value: Any) -> str: + """Decode a property value that may be bytes, str, or an array of either.""" + if value is None: + return "" + if isinstance(value, bytes): + return value.decode("utf-8", errors="replace") + if isinstance(value, str): + return value + try: + return bytes(value).decode("utf-8", errors="replace").rstrip("\x00") + except (TypeError, ValueError): + return str(value) diff --git a/pyproject.toml b/pyproject.toml index b52afa4f..4cdd6065 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,10 @@ dependencies = [ "pillow==12.3.0", "pyobjc-core==12.2.1;platform_system=='Darwin'", "pyobjc==12.2.1;platform_system=='Darwin'", - "python-Xlib==0.33;platform_system=='Linux'", + # The BSDs run the same X11 stack, and platform_system is the kernel + # name there ("FreeBSD", "OpenBSD", "NetBSD"), so a Linux-only marker + # left the X11 backend without its one dependency on every one of them. + "python-Xlib==0.33;platform_system=='Linux' or platform_system=='FreeBSD' or platform_system=='OpenBSD' or platform_system=='NetBSD'", "mss==10.2.0", "defusedxml==0.7.1", "cryptography>=48.0.1" diff --git a/test/unit_test/headless/test_accessibility_linux.py b/test/unit_test/headless/test_accessibility_linux.py new file mode 100644 index 00000000..c8340500 --- /dev/null +++ b/test/unit_test/headless/test_accessibility_linux.py @@ -0,0 +1,233 @@ +"""Headless tests for the Linux AT-SPI accessibility backend. No Qt, no bus. + +The backend's real behaviour is checked against a live bus and a live GTK +application by ``docker/x11_atspi_verify.py`` — a mock cannot tell you whether +at-spi2 agrees with the bytes you send. What is worth pinning here is the +part that has nothing to do with the bus: how a reply is turned into an +:class:`AccessibilityElement`, how the walk is bounded, and what the backend +does when there is no bus at all, which is the case on every developer +machine that is not Linux. +""" +import sys + +import pytest + +from je_auto_control.utils.accessibility.element import ( + AccessibilityElement, AccessibilityNotAvailableError, +) +from je_auto_control.utils.dbus_client import DBusError + + +atspi = pytest.importorskip( + "je_auto_control.utils.accessibility.backends.linux_backend", + exc_type=ImportError) + + +class FakeConnection: + """A tree of accessibles, answering the calls the backend makes.""" + + def __init__(self, tree=None, names=None, roles=None, extents=None, + states=None): + self.tree = tree or {} + self.names = names or {} + self.roles = roles or {} + self.extents_map = extents or {} + self.states = states or {} + self.entered = 0 + self.exited = 0 + + # context manager, so the backend's `with` blocks work unchanged + def __enter__(self): + self.entered += 1 + return self + + def __exit__(self, *_exception): + self.exited += 1 + + @property + def root(self): + return ("registry", "/root") + + def children(self, reference): + return list(self.tree.get(reference, [])) + + def property(self, reference, name, interface=None): + del interface + return self.names.get(reference, "") if name == "Name" else "" + + def role_name(self, reference): + return self.roles.get(reference, "") + + def state(self, reference): + return self.states.get(reference, 1 << 8) + + def extents(self, reference): + return self.extents_map.get(reference, (0, 0, 0, 0)) + + def text(self, reference): + return None + + def number(self, reference): + return None + + +APP = ("app", "/app") +WINDOW = ("app", "/window") +BUTTON = ("app", "/button") + + +def _tree_connection(): + return FakeConnection( + tree={("registry", "/root"): [APP], APP: [WINDOW], WINDOW: [BUTTON]}, + names={APP: "zenity", WINDOW: "autocontrol-dialog", BUTTON: "OK"}, + roles={WINDOW: "dialog", BUTTON: "push button"}, + extents={WINDOW: (-1280, 0, 310, 233), BUTTON: (-1200, 100, 60, 24)}, + ) + + +@pytest.fixture() +def backend(monkeypatch): + """An available backend whose bus is the fake tree above.""" + connection = _tree_connection() + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + instance.connection = connection + return instance + + +# --- availability ---------------------------------------------------------- + + +def test_unavailable_backend_says_what_to_install(): + """"No bus" and "no package" look the same from here, so name both.""" + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = False + with pytest.raises(AccessibilityNotAvailableError) as caught: + instance.list_elements() + message = str(caught.value) + assert "at-spi2-core" in message + assert "atk-bridge" in message + + +@pytest.mark.skipif(sys.platform.startswith("linux"), + reason="on Linux the real probe decides") +def test_off_linux_the_backend_is_never_available(): + assert atspi._is_available() is False + + +# --- turning a reply into an element --------------------------------------- + + +def test_walk_collects_the_tree_below_each_application(backend): + found = backend.list_elements() + assert [element.name for element in found] == [ + "autocontrol-dialog", "OK"] + assert all(isinstance(element, AccessibilityElement) for element in found) + + +def test_elements_carry_the_application_they_came_from(backend): + """app_name is what a caller filters on, and the walk is per-application.""" + assert {element.app_name for element in backend.list_elements()} == {"zenity"} + + +def test_negative_extents_survive_as_negative(backend): + """A monitor left of the primary one puts a window at a negative x. + + Reading the extents as unsigned would turn -1280 into 4293967296 and send + every click derived from it to the wrong screen. + """ + dialog = backend.list_elements()[0] + assert dialog.bounds == (-1280, 0, 310, 233) + assert dialog.center == (-1280 + 155, 116) + + +def test_app_name_filter_skips_other_applications(backend): + assert backend.list_elements(app_name="zenity") + assert backend.list_elements(app_name="something else") == [] + + +def test_max_results_bounds_the_walk(backend): + assert len(backend.list_elements(max_results=1)) == 1 + + +def test_window_title_scopes_to_one_window(backend): + """Below a window the tree is orders of magnitude smaller. + + So scoping both narrows the answer and shortens the walk, rather than + filtering a full result set afterwards. + """ + assert [element.name + for element in backend.list_elements(window_title="autocontrol")] \ + == ["autocontrol-dialog", "OK"] + assert backend.list_elements(window_title="no such window") == [] + + +def test_an_accessible_that_vanishes_mid_walk_is_skipped(monkeypatch): + """Applications close dialogs while they are being listed.""" + connection = _tree_connection() + + def _explode(reference): + if reference == BUTTON: + raise DBusError("no such object") + return connection.roles.get(reference, "") + + monkeypatch.setattr(connection, "role_name", _explode) + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + assert [element.name for element in instance.list_elements()] == [ + "autocontrol-dialog"] + + +def test_a_cyclic_tree_cannot_hang_the_caller(monkeypatch): + """A malformed tree must bottom out rather than recurse forever.""" + loop = ("app", "/loop") + connection = FakeConnection( + tree={("registry", "/root"): [APP], APP: [loop], loop: [loop]}, + names={APP: "app", loop: "loop"}, roles={loop: "panel"}) + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + found = instance.list_elements(max_results=500) + assert 0 < len(found) < 500 + + +def test_a_nameless_and_roleless_accessible_is_dropped(monkeypatch): + """A node with neither is not an element a caller could ever address.""" + connection = FakeConnection( + tree={("registry", "/root"): [APP], APP: [BUTTON]}, + names={APP: "app"}, roles={}) + monkeypatch.setattr(atspi, "_AtspiConnection", lambda: connection) + instance = atspi.LinuxAccessibilityBackend.__new__( + atspi.LinuxAccessibilityBackend) + instance.available = True + assert instance.list_elements() == [] + + +# --- state ----------------------------------------------------------------- + + +def test_state_reads_both_halves_of_the_bitfield(): + """AT-SPI sends the state as two 32-bit words, not one 64-bit value. + + Reading only the first silently drops every state above bit 31. + """ + class TwoWordBus: + def __enter__(self): + return self + + def __exit__(self, *_exception): + return None + + def call(self, *_args, **_kwargs): + # low word carries ENABLED, high word carries bit 32 + return [[1 << 8, 1]] + + connection = atspi._AtspiConnection() + connection._bus = TwoWordBus() + assert connection.state(BUTTON) == (1 << 8) | (1 << 32) diff --git a/test/unit_test/headless/test_cli.py b/test/unit_test/headless/test_cli.py index 3a097ec2..d0d6f25b 100644 --- a/test/unit_test/headless/test_cli.py +++ b/test/unit_test/headless/test_cli.py @@ -120,12 +120,9 @@ def fake_record_to_json(output_path, *, stop_event, timeout=None): fake_record_to_json) out = str(tmp_path / "rec.json") rc = main(["record", out, "--duration", "0"]) - if sys.platform == "darwin": - assert rc == 1 - else: - assert rc == 0 - assert captured["output"] == out - assert captured["timeout"] == pytest.approx(0.0) + assert rc == 0 + assert captured["output"] == out + assert captured["timeout"] == pytest.approx(0.0) def test_record_to_json_helper_writes_file(tmp_path, monkeypatch): diff --git a/test/unit_test/headless/test_docker_artifacts.py b/test/unit_test/headless/test_docker_artifacts.py index 5abf85c0..88098598 100644 --- a/test/unit_test/headless/test_docker_artifacts.py +++ b/test/unit_test/headless/test_docker_artifacts.py @@ -70,7 +70,7 @@ def test_dockerignore_keeps_build_context_lean(): @pytest.mark.parametrize("script", [ "entrypoint.sh", "entrypoint-xfce.sh", "entrypoint-wayland.sh", - "entrypoint-seat.sh", + "entrypoint-seat.sh", "entrypoint-x11.sh", ]) def test_entrypoints_keep_unix_line_endings(script): """A CRLF shebang makes an image that builds and then cannot start. @@ -239,6 +239,74 @@ def test_portal_verification_job_runs_the_image(): assert "autocontrol-portal:ci" in raw +def test_x11_verification_image_and_script_exist(): + """The X11 path is only verified while these three files are wired up.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.x11").read_text(encoding="utf-8") + # Each tool is ground truth from a different codebase than the subject: + # xev reads events back out of a real client, `import` is an independent + # grabber, xdotool/xdpyinfo are the server answering for itself. + for tool in ("xvfb", "x11-utils", "xdotool", "imagemagick", "openbox"): + assert tool in dockerfile, f"Dockerfile.x11 missing {tool}" + assert "x11_verify.py" in dockerfile + assert "entrypoint-x11.sh" in dockerfile + + verify = (_DOCKER_DIR / "x11_verify.py").read_text(encoding="utf-8") + # XSendEvent traffic arrives with `synthetic YES` and is discarded by most + # toolkits. Losing this assertion would let the backend quietly stop + # driving real input while every other check still passed. + assert "synthetic" in verify + # The three readers have to be checked against each other, not just run. + for reader in ("get_pixel", "screenshot", "truth_capture"): + assert reader in verify, f"x11_verify.py never exercises {reader}" + + entrypoint = (_DOCKER_DIR / "entrypoint-x11.sh").read_text(encoding="utf-8") + # A pass with no window manager tests nothing that reads _NET_*, and a + # skipped second layout would silently drop the dual-monitor geometry. + assert "openbox" in entrypoint + assert "--setmonitor" in entrypoint + + +def test_x11_image_can_reach_an_accessibility_bus(): + """AT-SPI is a bus, so the image needs one and something bridged to it.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.x11").read_text(encoding="utf-8") + # at-spi2-core provides the services D-Bus activates; dbus-x11 provides + # the session bus to activate them on; zenity is a real GTK application, + # which xterm is not — it exposes no accessible tree at all. + for package in ("at-spi2-core", "dbus-x11", "zenity"): + assert package in dockerfile, f"Dockerfile.x11 missing {package}" + # GTK3 only loads its bridge when asked, and what normally asks is a + # desktop setting this container has no store for. + assert "atk-bridge" in dockerfile + + entrypoint = (_DOCKER_DIR / "entrypoint-x11.sh").read_text(encoding="utf-8") + assert "dbus-run-session" in entrypoint + + verify = (_DOCKER_DIR / "x11_atspi_verify.py").read_text(encoding="utf-8") + # Extents are the check that caught the signed-integer gap in the D-Bus + # client; losing it would let that regress silently. + assert "GetExtents" in verify + + +def test_x11_verification_refuses_to_skip_a_missing_layout(): + """A layout that cannot be declared is a failure, not a reason to skip. + + Every other verification job in docker/ fails loudly when its + precondition is absent; a quiet skip reads as coverage that is not there. + """ + entrypoint = (_DOCKER_DIR / "entrypoint-x11.sh").read_text(encoding="utf-8") + assert "exit 1" in entrypoint + assert "not a reason to skip" in entrypoint + + +def test_x11_verification_job_runs_the_image(): + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "x11-verification" in raw + assert "docker/Dockerfile.x11" in raw + assert "autocontrol-x11:ci" in raw + + def test_gitlab_template_covers_build_test_smoke_stages(): raw = (_REPO_ROOT / "ci_templates" / ".gitlab-ci.yml").read_text( encoding="utf-8", diff --git a/test/unit_test/headless/test_exception_family_is_flat.py b/test/unit_test/headless/test_exception_family_is_flat.py new file mode 100644 index 00000000..429945c0 --- /dev/null +++ b/test/unit_test/headless/test_exception_family_is_flat.py @@ -0,0 +1,103 @@ +"""Every framework error must subclass ``AutoControlException``. + +The hierarchy is flat on purpose: the containment boundaries — the executor's +per-action `except`, the background poll loops, the REST/socket/MCP request +handlers, the GUI slots — all catch the family in one clause. A class that +inherits ``Exception`` directly is caught by none of them, so one malformed +argument to one command aborts the whole script instead of being recorded as a +failed action. That is not hypothetical: ``ConfigBundleError`` did exactly that +through ``AC_config_import``, and ``UsbClientError`` could through +``AC_usb_remote_devices``. + +The rule is checked structurally rather than class by class, because the way it +gets broken is a *new* subsystem defining its own error — which no list of +existing classes would notice. +""" +import ast +import pathlib + +import pytest + +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: The package tree, read as source: importing every module to inspect its +#: classes would drag in optional backends this test does not need. +PACKAGE_ROOT = pathlib.Path(__file__).resolve().parents[3] / "je_auto_control" + +#: Deliberately outside the family, and each for a reason that would break if +#: it were inside: +#: +#: * ``LoopBreak`` / ``LoopContinue`` are control flow, not failure. The +#: executor re-raises them *before* the family clause; making them relatives +#: would let a plain ``except AutoControlException`` swallow a ``break``. +#: * ``_MCPError`` carries a JSON-RPC error code to the dispatcher that raised +#: it, and is caught there by name. It never crosses a boundary. +#: * ``AutoControlException`` is the root of the family, so it is the one class +#: that has to inherit ``Exception`` itself. +DELIBERATELY_OUTSIDE = { + ("utils/exception/exceptions.py", "AutoControlException"), + ("utils/executor/flow_control.py", "LoopBreak"), + ("utils/executor/flow_control.py", "LoopContinue"), + ("utils/mcp_server/_protocol.py", "_MCPError"), +} + + +def _classes_inheriting_exception_directly(): + """Yield ``(relative path, class name)`` for every ``class X(Exception)``.""" + for path in sorted(PACKAGE_ROOT.rglob("*.py")): + if "__pycache__" in path.parts: + continue + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + for base in node.bases: + if isinstance(base, ast.Name) and base.id == "Exception": + rel = path.relative_to(PACKAGE_ROOT).as_posix() + yield rel, node.name + + +def test_no_framework_error_escapes_the_family(): + """Only the three control-flow carriers may inherit ``Exception``.""" + found = set(_classes_inheriting_exception_directly()) + unexpected = found - DELIBERATELY_OUTSIDE + assert not unexpected, ( + "these inherit Exception directly, so every containment boundary " + "misses them — derive them from AutoControlException, or add them to " + f"DELIBERATELY_OUTSIDE with the reason: {sorted(unexpected)}") + + # And the allowlist itself has to stay real: a stale entry would quietly + # licence a class that no longer exists, or has since been reparented. + stale = DELIBERATELY_OUTSIDE - found + assert not stale, f"allowlisted classes that are no longer there: {stale}" + + +@pytest.mark.parametrize("module_path, class_name", [ + ("je_auto_control.utils.config_bundle.config_bundle", "ConfigBundleError"), + ("je_auto_control.utils.usb.passthrough.protocol", "ProtocolError"), + ("je_auto_control.utils.usb.passthrough.session", "SessionError"), + ("je_auto_control.utils.usb.passthrough.viewer_client", "UsbClientError"), + ("je_auto_control.utils.work_queue.work_queue", "BusinessError"), +]) +def test_the_reparented_five_are_in_the_family(module_path, class_name): + """The classes that used to escape, named so the fix is legible.""" + module = pytest.importorskip(module_path) + assert issubclass(getattr(module, class_name), AutoControlException) + + +def test_a_rejected_config_bundle_does_not_abort_the_script(): + """The failure that proved the rule matters, pinned end to end. + + ``AC_config_import`` on a malformed bundle used to raise past the + per-action boundary and take every remaining action with it, even under + ``raise_on_error=False``. + """ + from je_auto_control.utils.executor.action_executor import executor + + record = executor.execute_action( + [["AC_config_import", {"bundle": {"not": "a bundle"}}], + ["AC_sleep", {"sleep_time": 0.01}]], + raise_on_error=False) + + assert len(record) == 2, "the second action never ran" + assert any("ConfigBundleError" in str(value) for value in record.values()) diff --git a/test/unit_test/headless/test_facade_import_is_light.py b/test/unit_test/headless/test_facade_import_is_light.py new file mode 100644 index 00000000..5d5054cd --- /dev/null +++ b/test/unit_test/headless/test_facade_import_is_light.py @@ -0,0 +1,122 @@ +"""``import je_auto_control`` must not need the image/crypto wheels. + +The facade used to import OpenCV, NumPy, Pillow, ``je_open_cv`` and +``cryptography`` at module scope, so *any* import under ``je_auto_control`` +pulled all five in. That is what kept the X11 backend off a FreeBSD desktop: +none of those five publishes a FreeBSD wheel, and moving a mouse needs none of +them. They are now imported by the functions that use them. + +This is a property that regresses silently — one convenience import at the top +of one module puts the whole set back on the path, and every platform that has +wheels keeps passing. So the check is made the way a machine without them would +make it: the modules are blocked outright, in a subprocess, and the facade has +to import anyway. +""" +import os +import pathlib +import subprocess # nosec B404 # reason: fixed argv, sys.executable, no shell +import sys +import tempfile + +import pytest + +#: Blocked wholesale. ``mss`` is in the list because screen capture is as +#: optional to input automation as OpenCV is; it was already lazy, and this +#: keeps it that way. +HEAVY = ("cv2", "numpy", "PIL", "cryptography", "je_open_cv", "mss") + +#: The working tree, so the subprocess tests the checkout rather than whatever +#: version of the package happens to be installed in site-packages. +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + +_PROBE = ''' +import sys + + +class _Blocker: + """Refuse the heavy wheels the way an absent wheel would.""" + + BLOCKED = {blocked!r} + + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".")[0] in self.BLOCKED: + raise ImportError( + "No module named %r (blocked by the probe)" % fullname) + return None + + +sys.meta_path.insert(0, _Blocker()) +import je_auto_control # noqa: E402 + +leaked = sorted(name for name in sys.modules + if name.split(".")[0] in _Blocker.BLOCKED) +if leaked: + raise SystemExit("heavy modules reached sys.modules: %s" % leaked) +print("ok", len(je_auto_control.__all__)) +''' + + +def test_facade_imports_without_the_heavy_wheels(): + """The facade imports with OpenCV / Pillow / cryptography absent.""" + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT)) + # argv is this interpreter plus a module-level literal probe; the only + # interpolation is the blocked-module tuple, also a literal. No shell. + result = subprocess.run( # nosec B603 # nosemgrep # reason: literal argv, no shell + [sys.executable, "-c", _PROBE.format(blocked=HEAVY)], + capture_output=True, text=True, timeout=180, check=False, env=env) + assert result.returncode == 0, ( + "import je_auto_control needs one of " + f"{', '.join(HEAVY)}:\n{result.stdout}\n{result.stderr}") + assert result.stdout.startswith("ok "), result.stdout + + +def test_the_lazy_modules_still_work_when_the_wheels_are_there(): + """Deferring an import must not have broken the function that uses it. + + The probe above proves the import path is clean; this proves the call path + still is, so a local import placed in the wrong function cannot pass as a + win. + """ + pytest.importorskip("PIL") + from PIL import Image + + from je_auto_control.utils.color_stats.color_stats import ( + region_color_stats, + ) + + image = Image.new("RGB", (8, 8), (10, 20, 30)) + stats = region_color_stats(image) + assert stats.average_rgb == (10, 20, 30) + assert stats.dominant_rgb == (10, 20, 30) + + +def test_importing_the_facade_writes_no_state_of_its_own(): + """Importing must not create files, only make the names available. + + `default_history_store` used to open its SQLite database in its + constructor, and that constructor runs while the facade is importing — so + `import je_auto_control` created `~/.je_auto_control/run_history.sqlite` + whether or not the caller ever recorded a run. Every store, key file and + cache this package owns lives under that one directory, so its absence is + the whole property; the rest of the home directory is left out because a + third-party import writing its own cache there would be someone else's + business, and a red test for it would be noise. + """ + with tempfile.TemporaryDirectory(prefix="ac-home-") as raw_home: + home = pathlib.Path(raw_home) + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT), + HOME=str(home), USERPROFILE=str(home)) + env.pop("HOMEDRIVE", None) + env.pop("HOMEPATH", None) + # argv is this interpreter plus a literal statement. No shell. + result = subprocess.run( # nosec B603 # nosemgrep # reason: literal argv, no shell + [sys.executable, "-c", "import je_auto_control"], + capture_output=True, text=True, timeout=180, check=False, env=env) + assert result.returncode == 0, result.stderr + + state_dir = home / ".je_auto_control" + created = sorted(path.relative_to(home).as_posix() + for path in state_dir.rglob("*")) + assert not state_dir.exists(), ( + "importing je_auto_control created its state directory, " + f"holding: {created}") diff --git a/test/unit_test/headless/test_mcp_http_sessions.py b/test/unit_test/headless/test_mcp_http_sessions.py new file mode 100644 index 00000000..3ad934c3 --- /dev/null +++ b/test/unit_test/headless/test_mcp_http_sessions.py @@ -0,0 +1,522 @@ +"""Headless tests for MCP session identity over the HTTP transport. + +Two halves. The registry half is pure and drives an injected clock. The +transport half runs a real ``HttpMCPServer`` over real sockets, because the +thing worth proving — that a destructive tool can be confirmed over HTTP — +only exists once a session id, a standing GET stream and three separate +connections are all in play at once. +""" +import http.client +import json +import logging +import threading + +import pytest + +from je_auto_control.utils.mcp_server.http_sessions import ( + SESSION_HEADER, SessionRegistry, session_id_from_headers, +) +from je_auto_control.utils.mcp_server.http_transport import ( + DEFAULT_PATH, HttpMCPServer, _MCPHttpHandler, +) +from je_auto_control.utils.mcp_server.prompts import StaticPromptProvider +from je_auto_control.utils.mcp_server.resources import ChainProvider +from je_auto_control.utils.mcp_server.server import MCPServer +from je_auto_control.utils.mcp_server.tools import MCPTool, MCPToolAnnotations + + +class _Clock: + """Manual monotonic clock.""" + + def __init__(self) -> None: + self.now = 1000.0 + + def __call__(self) -> float: + return self.now + + +# --- registry --------------------------------------------------------------- + + +def test_created_sessions_are_distinct_and_resolvable(): + registry = SessionRegistry(clock=_Clock()) + first, second = registry.create(), registry.create() + assert first.id != second.id + assert registry.get(first.id) is first + assert registry.get(second.id) is second + assert len(registry) == 2 + + +def test_unknown_and_blank_session_ids_resolve_to_none(): + registry = SessionRegistry(clock=_Clock()) + assert registry.get("nope") is None + assert registry.get("") is None + assert registry.get(None) is None + + +def test_idle_sessions_are_swept_and_announced(): + clock = _Clock() + dropped = [] + registry = SessionRegistry(clock=clock, idle_timeout=100.0, + on_drop=dropped.append) + session = registry.create() + clock.now += 50.0 + assert registry.get(session.id) is session # touched, so still live + clock.now += 60.0 # 60 < 100 since the touch + assert registry.get(session.id) is session + clock.now += 101.0 + assert registry.get(session.id) is None + assert [item.id for item in dropped] == [session.id] + assert session.closed.is_set() + + +def test_capacity_evicts_the_least_recently_seen(): + clock = _Clock() + dropped = [] + registry = SessionRegistry(clock=clock, max_sessions=2, + idle_timeout=0, on_drop=dropped.append) + first, second = registry.create(), registry.create() + clock.now += 1.0 + registry.get(second.id) # second is now the fresher of the two + clock.now += 1.0 + third = registry.create() + assert [item.id for item in dropped] == [first.id] + assert registry.get(first.id) is None + assert registry.get(second.id) is second + assert registry.get(third.id) is third + + +def test_terminate_and_terminate_all_announce_once(): + dropped = [] + registry = SessionRegistry(clock=_Clock(), on_drop=dropped.append) + first, second = registry.create(), registry.create() + assert registry.terminate(first.id) is first + assert registry.terminate(first.id) is None + registry.terminate_all() + assert [item.id for item in dropped] == [first.id, second.id] + assert second.closed.is_set() + assert len(registry) == 0 + + +def test_a_failing_drop_hook_does_not_break_the_caller(): + def boom(_session): + raise RuntimeError("hook exploded") + + registry = SessionRegistry(clock=_Clock(), on_drop=boom) + session = registry.create() + assert registry.terminate(session.id) is session # no exception escapes + + +def test_only_one_stream_attaches_per_session(): + registry = SessionRegistry(clock=_Clock()) + session = registry.create() + first, second = (lambda _payload: None), (lambda _payload: None) + assert session.attach_stream(first) is True + assert session.has_stream is True + assert session.attach_stream(second) is False + session.detach_stream(second) # not the owner — no-op + assert session.stream_writer is first + session.detach_stream(first) + assert session.has_stream is False + + +def test_session_id_header_is_read_case_insensitively(): + assert session_id_from_headers({"Mcp-Session-Id": " abc "}) == "abc" + assert session_id_from_headers({"Mcp-Session-Id": " "}) is None + assert session_id_from_headers({}) is None + assert session_id_from_headers(None) is None + + +# --- transport -------------------------------------------------------------- + + +def _destructive_tool(ran): + return MCPTool( + name="zap", description="destructive probe", + input_schema={"type": "object", "properties": {}}, + handler=lambda: ran.append(1) or "RAN", + annotations=MCPToolAnnotations(destructive=True, read_only=False), + ) + + +@pytest.fixture() +def live_server(monkeypatch): + """A real HttpMCPServer carrying one destructive tool, gate armed.""" + monkeypatch.setenv("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "1") + ran = [] + server = HttpMCPServer(mcp=MCPServer( + tools=[_destructive_tool(ran)], resource_provider=ChainProvider([]), + prompt_provider=StaticPromptProvider([]), + ), host="127.0.0.1", port=0) + server.start() + try: + yield server, ran + finally: + server.stop(timeout=2.0) + + +def _connect(server): + host, port = server.address + return http.client.HTTPConnection(host, port, timeout=20) + + +def _post(conn, payload, session_id=None, accept="application/json"): + headers = {"Content-Type": "application/json", "Accept": accept} + if session_id is not None: + headers[SESSION_HEADER] = session_id + conn.request("POST", DEFAULT_PATH, body=json.dumps(payload), + headers=headers) + response = conn.getresponse() + body = response.read().decode("utf-8") + return response, body + + +_INIT = { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {"elicitation": {}}, + "clientInfo": {"name": "probe", "version": "1"}, + }, +} +_CALL = { + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": {"name": "zap", "arguments": {}}, +} + + +def _initialize(server): + """Run initialize on its own connection; return the minted session id.""" + conn = _connect(server) + try: + response, _body = _post(conn, _INIT) + assert response.status == 200 + session_id = response.getheader(SESSION_HEADER) + assert session_id, "initialize must mint a session id" + return session_id + finally: + conn.close() + + +def _read_sse_event(response): + """Read one SSE event's data payload, skipping comments and blanks. + + The connection's socket timeout is what bounds this, so a stream that + never carries the expected event fails the test instead of hanging it. + """ + while True: + raw = response.readline() + if not raw: + return None + line = raw.decode("utf-8").rstrip("\r\n") + if line.startswith("data: "): + return json.loads(line[len("data: "):]) + + +def test_initialize_mints_a_session_id(live_server): + server, _ran = live_server + session_id = _initialize(server) + assert server.sessions.get(session_id) is not None + + +def test_capabilities_survive_a_new_connection_when_the_id_is_echoed( + live_server): + server, _ran = live_server + session_id = _initialize(server) + # A second, entirely separate TCP connection. + conn = _connect(server) + try: + with server.mcp.connection_scope(connection_id=session_id): + assert "elicitation" in server.mcp._client_capabilities + finally: + conn.close() + + +def test_unknown_session_id_is_refused(live_server): + server, _ran = live_server + conn = _connect(server) + try: + response, _body = _post(conn, _CALL, session_id="not-a-session") + assert response.status == 404 + finally: + conn.close() + + +def test_delete_terminates_the_session(live_server): + server, _ran = live_server + session_id = _initialize(server) + conn = _connect(server) + try: + conn.request("DELETE", DEFAULT_PATH, + headers={SESSION_HEADER: session_id}) + assert conn.getresponse().read() is not None + conn.close() + conn = _connect(server) + response, _body = _post(conn, _CALL, session_id=session_id) + assert response.status == 404 + finally: + conn.close() + + +def test_get_without_sse_accept_is_rejected(live_server): + server, _ran = live_server + session_id = _initialize(server) + conn = _connect(server) + try: + conn.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "application/json", + }) + assert conn.getresponse().status == 405 + finally: + conn.close() + + +def test_get_stream_without_a_session_is_rejected(live_server): + server, _ran = live_server + conn = _connect(server) + try: + conn.request("GET", DEFAULT_PATH, + headers={"Accept": "text/event-stream"}) + assert conn.getresponse().status == 404 + finally: + conn.close() + + +def test_destructive_confirmation_round_trips_over_http(live_server): + """The whole point of session identity: a decline blocks the tool. + + Three connections, as a real Streamable HTTP client would use: one that + initialized, one holding the standing GET stream, and one carrying the + tools/call. The elicitation goes out on the stream and its reply comes + back on a fourth — none of which shared a socket with the handshake. + """ + server, ran = live_server + session_id = _initialize(server) + + stream_conn = _connect(server) + stream_conn.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + stream = stream_conn.getresponse() + assert stream.status == 200 + + call_result = {} + + def run_call(): + conn = _connect(server) + try: + _response, body = _post(conn, _CALL, session_id=session_id) + call_result["body"] = body + finally: + conn.close() + + caller = threading.Thread(target=run_call, daemon=True) + caller.start() + try: + prompt = _read_sse_event(stream) + assert prompt is not None, "expected elicitation on the stream" + assert prompt["method"] == "elicitation/create" + + reply_conn = _connect(server) + try: + response, _body = _post(reply_conn, { + "jsonrpc": "2.0", "id": prompt["id"], + "result": {"action": "decline"}, + }, session_id=session_id) + assert response.status == 202 + finally: + reply_conn.close() + + caller.join(timeout=20.0) + assert not caller.is_alive() + finally: + stream_conn.close() + + payload = json.loads(call_result["body"]) + assert payload["error"]["code"] == -32000 + assert "declined" in payload["error"]["message"] + assert ran == [], "a declined destructive tool must not run" + + +def test_accepting_the_confirmation_runs_the_tool(live_server): + server, ran = live_server + session_id = _initialize(server) + + stream_conn = _connect(server) + stream_conn.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + stream = stream_conn.getresponse() + call_result = {} + + def run_call(): + conn = _connect(server) + try: + _response, body = _post(conn, _CALL, session_id=session_id) + call_result["body"] = body + finally: + conn.close() + + caller = threading.Thread(target=run_call, daemon=True) + caller.start() + try: + prompt = _read_sse_event(stream) + reply_conn = _connect(server) + try: + _post(reply_conn, { + "jsonrpc": "2.0", "id": prompt["id"], + "result": {"action": "accept", "content": {}}, + }, session_id=session_id) + finally: + reply_conn.close() + caller.join(timeout=20.0) + finally: + stream_conn.close() + + payload = json.loads(call_result["body"]) + assert payload["result"]["isError"] is False + assert ran == [1] + + +def test_second_stream_on_one_session_is_refused(live_server): + server, _ran = live_server + session_id = _initialize(server) + first = _connect(server) + first.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + assert first.getresponse().status == 200 + second = _connect(server) + try: + second.request("GET", DEFAULT_PATH, headers={ + SESSION_HEADER: session_id, "Accept": "text/event-stream", + }) + assert second.getresponse().status == 409 + finally: + second.close() + first.close() + + +def test_evicting_an_abandoned_session_is_not_logged_as_a_warning(caplog): + """Every initialize mints a session; churning through them is routine. + + A client that ignores the header leaves a session behind on every + handshake. Warning on each eviction would put one line per request in + the log of any busy sessionless server, which buries the case that does + matter: a session someone was holding. + """ + clock = _Clock() + registry = SessionRegistry(clock=clock, max_sessions=1, idle_timeout=0) + registry.create() + with caplog.at_level(logging.INFO): + registry.create() + assert not [record for record in caplog.records + if record.levelno >= logging.WARNING] + assert any("never used after initialize" in record.getMessage() + for record in caplog.records) + + +def test_evicting_a_session_in_use_warns(caplog): + clock = _Clock() + registry = SessionRegistry(clock=clock, max_sessions=1, idle_timeout=0) + held = registry.create() + clock.now += 1.0 + registry.get(held.id) # touched, so no longer an abandoned shell + with caplog.at_level(logging.INFO): + registry.create() + warnings = [record for record in caplog.records + if record.levelno >= logging.WARNING] + assert len(warnings) == 1 + assert "evicting a live session" in warnings[0].getMessage() + + +def test_an_sse_post_can_carry_its_own_confirmation(live_server): + """A session'd SSE POST is promptable without a standing GET stream. + + The response stream is itself a server-to-client channel, so the + elicitation goes down the same socket the call arrived on. The reply + still comes back as a separate POST, because the client is busy reading + this one. + """ + server, ran = live_server + session_id = _initialize(server) + + conn = _connect(server) + conn.request("POST", DEFAULT_PATH, body=json.dumps(_CALL), headers={ + "Content-Type": "application/json", "Accept": "text/event-stream", + SESSION_HEADER: session_id, + }) + stream = conn.getresponse() + try: + prompt = _read_sse_event(stream) + assert prompt is not None and prompt["method"] == "elicitation/create" + + reply_conn = _connect(server) + try: + _post(reply_conn, { + "jsonrpc": "2.0", "id": prompt["id"], + "result": {"action": "decline"}, + }, session_id=session_id) + finally: + reply_conn.close() + + final = _read_sse_event(stream) + assert final["error"]["code"] == -32000 + assert "declined" in final["error"]["message"] + finally: + conn.close() + assert ran == [] + + +class _RecordingReader: + """Stands in for a handler's ``rfile``, remembering every read.""" + + def __init__(self, chunks=b""): + self.reads = [] + self._buffer = chunks + + def read(self, size): + self.reads.append(size) + chunk, self._buffer = self._buffer[:size], self._buffer[size:] + return chunk + + +class _VanishedReader: + """An ``rfile`` whose peer is already gone.""" + + def read(self, _size): + raise ConnectionAbortedError(10053, "connection aborted") + + +def _bare_handler(*, consumed, reader, length="10"): + """A handler instance with no socket, for the body-drain logic alone.""" + handler = _MCPHttpHandler.__new__(_MCPHttpHandler) + handler._body_consumed = consumed + handler.rfile = reader + handler.headers = {"Content-Length": length} + return handler + + +def test_drain_is_skipped_once_the_body_has_been_read(): + """A 4xx raised after _read_body must not go looking for more body. + + It would find the *next* request's bytes, or nothing at all, and block + on the socket until the 30s read timeout — pinning the worker for every + unknown-session 404 and every duplicate-stream 409. + """ + reader = _RecordingReader(b"leftover!!") + handler = _bare_handler(consumed=True, reader=reader) + handler._drain_body() + assert reader.reads == [] + + +def test_drain_still_runs_when_the_body_was_never_read(): + reader = _RecordingReader(b"0123456789") + handler = _bare_handler(consumed=False, reader=reader) + handler._drain_body() + assert reader.reads, "an unread body is still drained" + + +def test_drain_survives_a_peer_that_vanished(): + handler = _bare_handler(consumed=False, reader=_VanishedReader()) + handler._drain_body() # must not raise: courtesy to a peer that has gone diff --git a/test/unit_test/headless/test_mcp_http_transport.py b/test/unit_test/headless/test_mcp_http_transport.py index 10655e22..236b4ca1 100644 --- a/test/unit_test/headless/test_mcp_http_transport.py +++ b/test/unit_test/headless/test_mcp_http_transport.py @@ -277,3 +277,73 @@ def test_malformed_json_returns_parse_error(http_server): assert response.status == 200 payload = json.loads(response.read().decode("utf-8")) assert payload["error"]["code"] == -32700 + + +def _post_with_accept(server, body, accept): + """POST and return the raw body, letting the caller choose ``Accept``.""" + host, port = server.address + url = f"{_TEST_SCHEME}://{host}:{port}{DEFAULT_PATH}" + req = urllib.request.Request( + url, data=json.dumps(body).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": accept}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=5) as response: # nosec B310 + return response.read().decode("utf-8") + + +@pytest.mark.parametrize("accept", ["application/json", "text/event-stream"]) +def test_destructive_confirm_gate_needs_a_session_the_client_keeps( + monkeypatch, accept): + """A client that ignores Mcp-Session-Id cannot be prompted, by construction. + + The prompt is an ``elicitation/create`` the server has to send *between* + receiving a call and answering it, so it needs a channel the client is + listening on. A client that neither echoes the session id nor opens the + standing GET stream has given the server nowhere to ask, and the call + proceeds — the same fallback a stdio client without the ``elicitation`` + capability takes. + + A client that does echo the id and open the stream is prompted for real; + that is ``test_mcp_http_sessions.py``. This test pins the other side of + that line so the fallback stays deliberate rather than becoming the only + behaviour again. + """ + from je_auto_control.utils.mcp_server.tools import ( + MCPTool, MCPToolAnnotations, + ) + + monkeypatch.setenv("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "1") + ran = [] + tool = MCPTool( + name="zap_probe", description="destructive probe", + input_schema={"type": "object", "properties": {}}, + handler=lambda: ran.append(1) or "RAN", + annotations=MCPToolAnnotations(destructive=True, read_only=False), + ) + server = HttpMCPServer(mcp=MCPServer( + tools=[tool], resource_provider=ChainProvider([]), + prompt_provider=StaticPromptProvider([]), + ), host="127.0.0.1", port=0) + server.start() + try: + init_body = _post_with_accept(server, { + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {"elicitation": {}}, + "clientInfo": {"name": "probe", "version": "1"}, + }, + }, accept) + # No Mcp-Session-Id echoed back, and no GET stream opened. + call_body = _post_with_accept(server, { + "jsonrpc": "2.0", "id": 2, "method": "tools/call", + "params": {"name": "zap_probe", "arguments": {}}, + }, accept) + finally: + server.stop(timeout=1.0) + + assert "elicitation/create" not in init_body + assert "elicitation/create" not in call_body + assert ran == [1], "with nowhere to ask, the call proceeds" + assert '"isError": false' in call_body diff --git a/test/unit_test/headless/test_mcp_server.py b/test/unit_test/headless/test_mcp_server.py index a9c8ab58..6ae2f5bf 100644 --- a/test/unit_test/headless/test_mcp_server.py +++ b/test/unit_test/headless/test_mcp_server.py @@ -1751,3 +1751,109 @@ def test_default_registry_lists_core_automation_tools(): "ac_execute_actions", "ac_list_action_commands", } assert expected.issubset(names) + + +# --- shutdown while a tool reply is in flight ------------------------------ + + +def test_a_slow_tool_reply_is_not_lost_at_eof(): + """serve_stdio must not return while a reply is still being written. + + tools/call runs on a worker thread under stdio, and the loop's `finally` + restores the previous writer. A worker that had not reached its write yet + then found ``self._writer`` already swapped back — so the reply went to + the previous transport or was dropped as "no writer". Windows CI caught it + as one response where two were expected; the sleep here makes the window + wide enough to catch every time. + """ + import threading + import time as _time + + started = threading.Event() + + def slow_handler(): + started.set() + _time.sleep(0.5) + return "pong" + + tool = MCPTool(name="slow_tool", description="slow", + input_schema={"type": "object", "properties": {}}, + handler=slow_handler) + server = MCPServer(tools=[tool]) + stdin = io.StringIO( + _request("initialize", msg_id=1, params={"protocolVersion": "x"}) + + "\n" + + _request("tools/call", msg_id=2, + params={"name": "slow_tool", "arguments": {}}) + + "\n") + stdout = io.StringIO() + server.serve_stdio(stdin=stdin, stdout=stdout) + + assert started.is_set(), "the tool never ran" + responses = [_decode(line) for line in stdout.getvalue().splitlines() + if line and '"id":' in line and '"method"' not in line] + assert len(responses) == 2, [response.get("id") for response in responses] + assert responses[-1]["result"]["content"][0]["text"] == "pong" + + +def test_a_reply_goes_to_the_transport_that_accepted_the_request(): + """The writer is captured at dispatch, not read when the worker finishes. + + Reading it later is what let a late reply land on whatever writer the + server had by then — the next connection's, or none at all. + """ + import threading + + release = threading.Event() + delivered = [] + + def blocking_handler(): + release.wait(timeout=5) + return "late" + + tool = MCPTool(name="late_tool", description="late", + input_schema={"type": "object", "properties": {}}, + handler=blocking_handler) + server = MCPServer(tools=[tool]) + server.set_notifier(None) + server._writer = delivered.append + server._dispatch_tools_call_async( + 7, {"name": "late_tool", "arguments": {}}) + + # The transport goes away while the tool is still running. + server._writer = None + release.set() + server._join_workers(timeout=5) + + assert len(delivered) == 1, delivered + assert '"id": 7' in delivered[0] or '"id":7' in delivered[0] + + +def test_shutdown_does_not_wait_forever_for_a_stuck_tool(): + """A handler that never returns must not keep the transport alive. + + The drain is bounded and says which worker outstayed its welcome, rather + than hanging on one bad tool. + """ + import threading + import time as _time + + release = threading.Event() + + def stuck_handler(): + release.wait(timeout=30) + return "eventually" + + tool = MCPTool(name="stuck_tool", description="stuck", + input_schema={"type": "object", "properties": {}}, + handler=stuck_handler) + server = MCPServer(tools=[tool]) + server._writer = lambda payload: None + server._dispatch_tools_call_async( + 1, {"name": "stuck_tool", "arguments": {}}) + try: + started = _time.monotonic() + server._join_workers(timeout=0.2) + assert _time.monotonic() - started < 5 + finally: + release.set() diff --git a/test/unit_test/headless/test_osx_input_tap.py b/test/unit_test/headless/test_osx_input_tap.py new file mode 100644 index 00000000..f1ea3f2c --- /dev/null +++ b/test/unit_test/headless/test_osx_input_tap.py @@ -0,0 +1,227 @@ +"""The macOS event tap, driven by real CGEvents. Runs on macOS CI only. + +These build genuine Quartz events with ``CGEventCreate*`` and hand them to the +decoder directly, which needs **no** Accessibility grant and no user at the +keyboard — only creating a live *tap* needs the grant, and that is verified +separately by ``test/verify/macos_verify.py`` on a real runner. + +The regression being guarded is why macOS shipped without a recorder at all: +the old listener built an ``NSApplication`` at import and stopped recording +via ``AppHelper.runEventLoop()``, a loop that never returns. Both would have +landed on the path of ``import je_auto_control``. +""" +import sys +import threading + +import pytest + +if sys.platform != "darwin": # pragma: no cover + pytest.skip("macOS event tap", allow_module_level=True) + +import Quartz # noqa: E402 + +from je_auto_control.osx.listener import osx_listener # noqa: E402 +from je_auto_control.osx.listener.osx_listener import OSXInputTap # noqa: E402 +from je_auto_control.osx.record.osx_record import OSXRecorder # noqa: E402 +from je_auto_control.utils.exception.exceptions import ( # noqa: E402 + AutoControlRecordException, +) + + +def _key_event(keycode, down): + return Quartz.CGEventCreateKeyboardEvent(None, keycode, down) + + +def _mouse_event(event_type, x, y, button=0): + return Quartz.CGEventCreateMouseEvent(None, event_type, (x, y), button) + + +def _decode(events): + """Feed ``(type, event)`` pairs to a fresh tap and return what it kept.""" + tap = OSXInputTap() + for event_type, event in events: + tap.decode(event_type, event) + return tap.events + + +# --- the regression that kept the recorder unwired ------------------------ + +def test_importing_the_listener_builds_no_application(): + # The old module ran NSApplication.sharedApplication() at module scope, so + # every `import je_auto_control` on a Mac created one, and stopping a + # recording meant AppHelper.runEventLoop(). Neither name is here now. + assert not hasattr(osx_listener, "app") + assert not hasattr(osx_listener, "NSApplication") + assert not hasattr(osx_listener, "AppHelper") + + +def test_the_platform_wrapper_now_selects_a_recorder(): + from je_auto_control.wrapper import platform_wrapper + + assert isinstance(platform_wrapper.recorder, OSXRecorder) + + +def test_recording_does_not_block_the_caller(): + # AppHelper.runEventLoop() never returns, so record() used to be a call + # you could not come back from. The tap runs on its own thread. + recorder = OSXRecorder() + finished = threading.Event() + + def _drive(): + try: + recorder.record() + except AutoControlRecordException: + pass # no Accessibility grant here; still returned + finally: + finished.set() + + threading.Thread(target=_drive, daemon=True).start() + assert finished.wait(timeout=osx_listener.START_TIMEOUT + 5.0), ( + "record() did not return; a blocking run loop is back") + recorder.stop_record() + + +# --- decoding ------------------------------------------------------------- + +def test_tap_records_key_press_and_release(): + events = _decode([ + (Quartz.kCGEventKeyDown, _key_event(0, True)), + (Quartz.kCGEventKeyUp, _key_event(0, False)), + ]) + assert [e["op"] for e in events] == ["key_down", "key_up"] + assert events[0]["vk"] == 0 # kVK_ANSI_A + assert all("time" in e for e in events) + + +def test_tap_records_button_release_not_just_press(): + # Without the release a drag is indistinguishable from a click. + events = _decode([ + (Quartz.kCGEventLeftMouseDown, + _mouse_event(Quartz.kCGEventLeftMouseDown, 10, 20)), + (Quartz.kCGEventLeftMouseUp, + _mouse_event(Quartz.kCGEventLeftMouseUp, 90, 60)), + ]) + assert [(e["op"], e["button"], e["x"], e["y"]) for e in events] == [ + ("mouse_down", "left", 10, 20), ("mouse_up", "left", 90, 60)] + + +def test_tap_records_every_button_this_project_addresses(): + events = _decode([ + (Quartz.kCGEventRightMouseDown, + _mouse_event(Quartz.kCGEventRightMouseDown, 1, 2, + Quartz.kCGMouseButtonRight)), + (Quartz.kCGEventOtherMouseDown, + _mouse_event(Quartz.kCGEventOtherMouseDown, 3, 4, + Quartz.kCGMouseButtonCenter)), + ]) + assert [e["button"] for e in events] == ["right", "middle"] + + +def test_recorded_coordinates_are_the_ones_a_replay_posts(): + # CGEventGetLocation has a top-left origin, which is the space osx_mouse + # posts into. The old listener read NSEvent.mouseLocation(), a bottom-left + # origin, so every recorded click replayed vertically mirrored. + height = Quartz.CGDisplayBounds(Quartz.CGMainDisplayID()).size.height + near_top = int(height * 0.1) + events = _decode([ + (Quartz.kCGEventLeftMouseDown, + _mouse_event(Quartz.kCGEventLeftMouseDown, 40, near_top)), + ]) + assert events[0]["y"] == near_top + assert events[0]["y"] < height / 2 + + +def test_tap_decodes_a_scroll_with_its_sign(): + # Reading the wheel unsigned turns a scroll down into a scroll up. + down = Quartz.CGEventCreateScrollWheelEvent( + None, Quartz.kCGScrollEventUnitLine, 1, -3) + up = Quartz.CGEventCreateScrollWheelEvent( + None, Quartz.kCGScrollEventUnitLine, 1, 3) + events = _decode([(Quartz.kCGEventScrollWheel, down), + (Quartz.kCGEventScrollWheel, up)]) + assert [e["op"] for e in events] == ["scroll", "scroll"] + assert [e["delta"] for e in events] == [-3, 3] + + +def test_a_held_modifier_becomes_a_press_and_a_release(): + # macOS sends no key-down for Shift, only a flagsChanged carrying the new + # flag set. Without decoding it a recording cannot say a modifier was held + # across the actions that followed. + pressed = _key_event(56, True) + Quartz.CGEventSetFlags(pressed, Quartz.kCGEventFlagMaskShift) + released = _key_event(56, False) + Quartz.CGEventSetFlags(released, 0) + events = _decode([(Quartz.kCGEventFlagsChanged, pressed), + (Quartz.kCGEventFlagsChanged, released)]) + assert [(e["op"], e["vk"]) for e in events] == [ + ("key_down", 56), ("key_up", 56)] + + +def test_a_flags_change_for_no_known_modifier_is_dropped(): + other = _key_event(0, True) + Quartz.CGEventSetFlags(other, Quartz.kCGEventFlagMaskShift) + assert _decode([(Quartz.kCGEventFlagsChanged, other)]) == [] + + +def test_mouse_movement_is_not_recorded(): + # Every pixel of travel would drown the events that matter, exactly as on + # Windows, where WM_MOUSEMOVE is ignored for the same reason. + assert _decode([ + (Quartz.kCGEventMouseMoved, + _mouse_event(Quartz.kCGEventMouseMoved, 5, 5))]) == [] + + +def test_tap_stops_itself_at_the_event_cap(): + # A forgotten recording must not grow without bound. + tap = OSXInputTap(max_events=3) + for _ in range(10): + tap.decode(Quartz.kCGEventKeyDown, _key_event(0, True)) + assert len(tap.events) == 3 + + +# --- failure and lifecycle ------------------------------------------------ + +def test_a_tap_that_cannot_be_created_fails_loudly(monkeypatch): + # Without Accessibility, CGEventTapCreate returns None. Recording silence + # would look like "the user did nothing" for the rest of the session. + monkeypatch.setattr(Quartz, "CGEventTapCreate", + lambda *args, **kwargs: None) + tap = OSXInputTap() + with pytest.raises(AutoControlRecordException) as caught: + tap.start() + assert "Accessibility" in str(caught.value) + + +def test_stopping_a_tap_that_never_started_returns_nothing(): + assert OSXInputTap().stop() == [] + + +def test_a_decode_failure_does_not_end_the_recording(monkeypatch): + # An exception escaping the callback tears down the run loop, and the + # recording then stops without saying so — so one unusable event must not + # cost the session. The next event still has to be decoded. + tap = OSXInputTap() + calls = [] + + def _explode(*_args, **_kwargs): + calls.append(1) + raise RuntimeError("unexpected event shape") + + monkeypatch.setattr(tap, "decode", _explode) + event = _key_event(0, True) + assert tap._callback(None, Quartz.kCGEventKeyDown, event, None) is event + assert tap._callback(None, Quartz.kCGEventKeyUp, event, None) is event + assert len(calls) == 2 + assert tap.events == [] + + +def test_a_disabled_tap_is_re_armed_rather_than_left_deaf(monkeypatch): + # The window server disables a tap that takes too long. Leaving it + # disabled records silence for the rest of the session. + enabled = [] + monkeypatch.setattr(Quartz, "CGEventTapEnable", + lambda tap, state: enabled.append((tap, state))) + tap = OSXInputTap() + tap._callback("proxy", Quartz.kCGEventTapDisabledByTimeout, None, None) + assert enabled == [("proxy", True)] + assert tap.events == [] diff --git a/test/unit_test/headless/test_platform_id.py b/test/unit_test/headless/test_platform_id.py new file mode 100644 index 00000000..f91ba948 --- /dev/null +++ b/test/unit_test/headless/test_platform_id.py @@ -0,0 +1,73 @@ +"""Headless tests for platform identification. No Qt. + +``sys.platform`` was compared against literal lists in over a hundred places, +and every one named win32/cygwin/msys, darwin and linux/linux2 — so an +operating system that is none of those and still runs X11 was a hundred +separate places to be wrong. These pin the one place that decides now. + +The real FreeBSD check is the ``freebsd`` job in ``platform-smoke.yml``, which +boots a VM and imports the X11 backend on it; what is worth pinning here is +the classification itself, which is pure and can be checked from anywhere. +""" +import pytest + +from je_auto_control.utils.platform_id import ( + current_family, is_bsd, is_macos, is_windows, is_x11_unix, +) + + +@pytest.mark.parametrize("platform,family", [ + ("win32", "windows"), ("cygwin", "windows"), ("msys", "windows"), + ("darwin", "macos"), + ("linux", "linux"), ("linux2", "linux"), + ("freebsd14", "bsd"), ("freebsd13", "bsd"), + ("openbsd7", "bsd"), ("netbsd10", "bsd"), ("dragonfly6", "bsd"), +]) +def test_families_are_classified(platform, family): + assert current_family(platform) == family + + +def test_an_unknown_platform_keeps_its_own_name(): + """Better a name the reader can look up than a wrong family.""" + assert current_family("haiku1") == "haiku1" + assert not is_x11_unix("haiku1") + + +@pytest.mark.parametrize("platform", [ + "freebsd14", "openbsd7", "netbsd10", "dragonfly6"]) +def test_the_bsds_are_x11_unixes(platform): + """What the X11 backend needs is an X server and python-Xlib. + + Neither is Linux-specific, and a BSD desktop is an ordinary X11 desktop, + so the guards ask this rather than asking whether the kernel is Linux. + """ + assert is_bsd(platform) + assert is_x11_unix(platform) + assert not is_windows(platform) + assert not is_macos(platform) + + +def test_the_bsd_version_suffix_does_not_matter(): + """sys.platform carries the major version there — freebsd14, not freebsd. + + An equality check against "freebsd" matches no real system at all, which + is the trap this prefix match exists to avoid. + """ + assert is_bsd("freebsd") + assert is_bsd("freebsd15") + # Prefix, not equality: matching "freebsd" exactly matches no real + # system. Python's own porting guidance says to compare this way. + assert not is_bsd("linux") + assert not is_bsd("darwin") + + +def test_macos_is_not_an_x11_unix(): + """It is a unix, but its backend is Quartz — the distinction that matters.""" + assert is_macos("darwin") + assert not is_x11_unix("darwin") + + +def test_the_helpers_default_to_this_interpreter(): + """Called with no argument they answer about the running platform.""" + families = {is_windows(), is_macos(), is_x11_unix()} + assert True in families, "this platform matches none of the families" diff --git a/test/unit_test/headless/test_r3_vision_capture.py b/test/unit_test/headless/test_r3_vision_capture.py index f8e36cd8..0fca56cd 100644 --- a/test/unit_test/headless/test_r3_vision_capture.py +++ b/test/unit_test/headless/test_r3_vision_capture.py @@ -12,7 +12,7 @@ import pytest np = pytest.importorskip("numpy") -pytest.importorskip("cv2") +cv2 = pytest.importorskip("cv2") from je_auto_control.utils.cv2_utils import screen_record as sr # noqa: E402 from je_auto_control.utils.cv2_utils import screenshot as ss # noqa: E402 @@ -43,7 +43,7 @@ def release(self): # --- finding 2: recorder stop honoured + writer released ----------------- def test_stop_before_run_is_honored(monkeypatch): - monkeypatch.setattr(sr.cv2, "VideoWriter", _FakeVideoWriter) + monkeypatch.setattr(cv2, "VideoWriter", _FakeVideoWriter) thread = sr.ScreenRecordThread("out.avi", "XVID", 30, (4, 4)) frame = np.zeros((4, 4, 3), dtype=np.uint8) @@ -64,7 +64,7 @@ def screenshot_bounds_the_buggy_path(): def test_normal_run_writes_frames_and_releases(monkeypatch): - monkeypatch.setattr(sr.cv2, "VideoWriter", _FakeVideoWriter) + monkeypatch.setattr(cv2, "VideoWriter", _FakeVideoWriter) thread = sr.ScreenRecordThread("out.avi", "XVID", 30, (4, 4)) frame = np.zeros((4, 4, 3), dtype=np.uint8) calls = {"n": 0} diff --git a/test/unit_test/headless/test_recorder_base.py b/test/unit_test/headless/test_recorder_base.py new file mode 100644 index 00000000..bc0579cb --- /dev/null +++ b/test/unit_test/headless/test_recorder_base.py @@ -0,0 +1,290 @@ +"""The recorder shaping every platform shares. No Qt, no real input, no OS. + +Windows captures input with a low-level hook and macOS with a Quartz event +tap, but everything after the capture is one implementation — the queue the +executor is handed, the timeline a replay consumes, and the mouse-only / +keyboard-only filters. This exercises that shared half on any platform, so a +change to it cannot be merged on the strength of a Windows-only run. +""" +from queue import Queue + +import pytest + +from je_auto_control.utils.input_macro.recorder_base import ( + ALL_KINDS, InputRecorder, event_kind, legacy_action_queue, timeline, +) + + +def _events(): + """A press, a release, and a scroll — the three things replay needs.""" + return [ + {"op": "key_down", "vk": 65, "time": 10.0}, + {"op": "key_up", "vk": 65, "time": 10.25}, + {"op": "scroll", "delta": -3, "x": 5, "y": 6, "time": 10.75}, + ] + + +class _FakeHook: + """Stands in for Win32InputHook / OSXInputTap: the same two methods.""" + + def __init__(self, events): + self._events = events + self.started = False + self.stopped = False + + def start(self): + self.started = True + + def stop(self): + self.stopped = True + return list(self._events) + + +class _FakeRecorder(InputRecorder): + def __init__(self, events): + super().__init__() + self._events = events + self.hooks = [] + + def new_hook(self): + hook = _FakeHook(self._events) + self.hooks.append(hook) + return hook + + +# --- timeline ------------------------------------------------------------- + +def test_timeline_turns_timestamps_into_gaps(): + # Without the gaps every step replays at once and no real interface keeps + # up: the window that was supposed to open has not opened yet. + out = timeline(_events()) + assert [event["delta_ms"] for event in out] == [0, 250, 500] + assert "time" not in out[0] + + +def test_timeline_keeps_releases_and_wheel(): + # A press-only recording cannot tell a drag from a click, and loses + # scrolling entirely. + out = timeline(_events()) + assert [event["op"] for event in out] == ["key_down", "key_up", "scroll"] + assert out[2]["delta"] == -3 + + +def test_timeline_of_nothing_is_empty(): + assert timeline([]) == [] + + +def test_timeline_never_reports_a_negative_gap(): + # Monotonic time should not go backwards, but a clamped gap beats a replay + # that tries to sleep a negative amount. + out = timeline([{"op": "key_down", "vk": 1, "time": 5.0}, + {"op": "key_up", "vk": 1, "time": 4.0}]) + assert out[1]["delta_ms"] == 0 + + +def test_timeline_feeds_replay_timeline_unchanged(): + # The two halves are only useful together, so pin that the shape one emits + # is the shape the other consumes. + from je_auto_control.utils.input_macro import replay_timeline + + played = [] + count = replay_timeline(timeline(_events()), sink=played.append, + sleep=lambda _seconds: None) + assert count == 3 + assert [event["op"] for event in played] == [ + "key_down", "key_up", "scroll"] + + +def test_the_default_replay_sink_knows_every_op_a_recorder_emits(): + # These two vocabularies used to be disjoint: the recorder emitted + # key_down / mouse_up / scroll and the sink table held press / click / + # key, so feeding stop_record_timeline() to replay_timeline() — the + # pipeline the docstrings and the MCP tool both prescribe — matched + # nothing, replayed an empty session, and still reported every event as + # played. + from je_auto_control.utils.input_macro.input_macro import _SINKS + + emitted = {"key_down", "key_up", "mouse_down", "mouse_up", "scroll"} + assert emitted <= set(_SINKS), sorted(emitted - set(_SINKS)) + + +def test_replaying_a_recording_drives_the_matching_input_calls(monkeypatch): + # Through the real default sink, with only the input calls themselves + # faked — so the op-to-call routing is what is being exercised. + from je_auto_control.utils.input_macro import replay_timeline + + calls = [] + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_keyboard.press_keyboard_key", + lambda key, *a, **k: calls.append(("press_key", key))) + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_keyboard.release_keyboard_key", + lambda key, *a, **k: calls.append(("release_key", key))) + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_mouse.press_mouse", + lambda button, x, y: calls.append(("press_mouse", button, x, y))) + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_mouse.release_mouse", + lambda button, x, y: calls.append(("release_mouse", button, x, y))) + + replay_timeline(timeline([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "key_up", "vk": 65, "time": 1.1}, + {"op": "mouse_down", "button": "right", "x": 1, "y": 2, "time": 1.2}, + {"op": "mouse_up", "button": "right", "x": 1, "y": 2, "time": 1.3}, + ]), sleep=lambda _seconds: None) + assert calls == [ + ("press_key", 65), ("release_key", 65), + # "right" as the recorder writes it, "mouse_right" as the input API + # names it — a mismatch here presses the wrong button silently. + ("press_mouse", "mouse_right", 1, 2), + ("release_mouse", "mouse_right", 1, 2)] + + +def test_the_recorders_wheel_delta_is_not_dropped_by_the_sink(monkeypatch): + # The DSL calls it `value` and the recorder calls it `delta`; reading only + # `value` fell back to the default of 1, so a three-notch scroll down + # replayed as a single notch in the other direction. + from je_auto_control.utils.input_macro.input_macro import _sink_scroll + + asked = [] + monkeypatch.setattr( + "je_auto_control.wrapper.auto_control_mouse.mouse_scroll", + lambda value, *a, **k: asked.append(value)) + _sink_scroll({"op": "scroll", "delta": -3}) + _sink_scroll({"op": "scroll", "value": 7}) + assert asked == [-3, 7] + + +# --- the legacy queue ----------------------------------------------------- + +def test_legacy_queue_is_presses_only_as_executor_commands(): + queue = legacy_action_queue([ + {"op": "key_down", "vk": 65}, + {"op": "key_up", "vk": 65}, + {"op": "mouse_down", "button": "left", "x": 7, "y": 8}, + {"op": "mouse_up", "button": "left", "x": 7, "y": 8}, + {"op": "mouse_down", "button": "right", "x": 1, "y": 2}, + {"op": "mouse_down", "button": "middle", "x": 3, "y": 4}, + ]) + assert list(queue.queue) == [ + ("AC_type_keyboard", 65), ("AC_mouse_left", 7, 8), + ("AC_mouse_right", 1, 2), ("AC_mouse_middle", 3, 4)] + + +def test_legacy_queue_drops_a_button_it_has_no_command_for(): + # An unknown button must not become a malformed action the executor then + # rejects at replay time, far from where it was recorded. + queue = legacy_action_queue([ + {"op": "mouse_down", "button": "x1", "x": 1, "y": 2}]) + assert list(queue.queue) == [] + + +def test_legacy_queue_ignores_the_wheel(): + # stop_record() has never carried scrolling; stop_record_timeline() does. + assert list(legacy_action_queue( + [{"op": "scroll", "delta": 1, "x": 0, "y": 0}]).queue) == [] + + +# --- kinds ---------------------------------------------------------------- + +@pytest.mark.parametrize("op, kind", [ + ("key_down", "keyboard"), ("key_up", "keyboard"), + ("mouse_down", "mouse"), ("mouse_up", "mouse"), ("scroll", "mouse"), +]) +def test_every_op_is_classified(op, kind): + assert event_kind({"op": op}) == kind + + +def test_an_unknown_op_counts_as_mouse_rather_than_vanishing(): + # It is recorded by a backend that knows what it is; dropping it silently + # would be worse than filing it under the broader of the two kinds. + assert event_kind({"op": "gesture"}) == "mouse" + assert ALL_KINDS == ("keyboard", "mouse") + + +# --- the recorder surface ------------------------------------------------- + +def test_record_then_stop_returns_the_legacy_queue(): + recorder = _FakeRecorder(_events()) + recorder.record() + assert recorder.hooks[0].started + queue = recorder.stop_record() + assert isinstance(queue, Queue) + assert list(queue.queue) == [("AC_type_keyboard", 65)] + assert recorder.result_queue is queue + assert recorder.record_queue is None + + +def test_timeline_stop_returns_everything(): + recorder = _FakeRecorder(_events()) + recorder.record() + out = recorder.stop_record_timeline() + assert [event["op"] for event in out] == ["key_down", "key_up", "scroll"] + + +def test_keyboard_only_recording_drops_mouse_events(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_keyboard() + assert [e["op"] for e in recorder.stop_record_timeline()] == ["key_down"] + + +def test_mouse_only_recording_drops_key_events(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "scroll", "delta": 1, "time": 1.1}, + ]) + recorder.record_mouse() + assert [e["op"] for e in recorder.stop_record_timeline()] == ["scroll"] + + +def test_mouse_only_stop_variant_also_filters(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_mouse() + assert list(recorder.stop_record_mouse().queue) == [("AC_mouse_left", 1, 2)] + + +def test_keyboard_only_stop_variant_also_filters(): + recorder = _FakeRecorder([ + {"op": "key_down", "vk": 65, "time": 1.0}, + {"op": "mouse_down", "button": "left", "x": 1, "y": 2, "time": 1.1}, + ]) + recorder.record_keyboard() + assert list(recorder.stop_record_keyboard().queue) == [ + ("AC_type_keyboard", 65)] + + +def test_stopping_without_recording_is_not_an_error(): + assert _FakeRecorder([]).stop_record_timeline() == [] + assert list(_FakeRecorder([]).stop_record().queue) == [] + + +def test_a_second_stop_does_not_reuse_the_finished_hook(): + # The hook is dropped on stop, so a stray second stop returns nothing + # rather than replaying the previous session's events. + recorder = _FakeRecorder(_events()) + recorder.record() + recorder.stop_record() + assert recorder.stop_record_timeline() == [] + + +def test_each_recording_gets_a_fresh_hook(): + # Reusing a stopped hook is how a second recording comes back with the + # first one's events appended to it. + recorder = _FakeRecorder(_events()) + recorder.record() + recorder.stop_record() + recorder.record() + assert len(recorder.hooks) == 2 + assert recorder.hooks[0] is not recorder.hooks[1] + + +def test_a_backend_must_supply_a_hook(): + with pytest.raises(NotImplementedError): + InputRecorder().record() diff --git a/test/unit_test/headless/test_scroll_sign_is_portable.py b/test/unit_test/headless/test_scroll_sign_is_portable.py new file mode 100644 index 00000000..123532d0 --- /dev/null +++ b/test/unit_test/headless/test_scroll_sign_is_portable.py @@ -0,0 +1,170 @@ +"""The sign of ``scroll_value`` picks the direction on every platform. + +Windows and macOS have always read the direction off the sign. X11 encodes +direction as a button (4/5/6/7) rather than a signed delta, so it took the +direction from ``scroll_direction`` and discarded the sign — deliberately, +because a negative count used to make ``range()`` empty and scroll nothing. + +The cost was that ``mouse_scroll(-3)`` written and tested on Windows scrolled +*down* three notches on Linux instead of up, with no exception and no warning. +The maintainer settled it: the sign reverses the direction everywhere, and +``scroll_direction`` names the direction a positive count takes. + +The real X server side of this is pinned by ``docker/x11_verify.py``, which +reads the buttons back out of ``xev``. These tests hold the same contract +without a display, so a change to the mapping fails on every runner rather than +only in the container job. +""" +from __future__ import annotations + +import importlib +import sys +import types + +import pytest + + +@pytest.fixture() +def x11_mouse(monkeypatch): + """Import the X11 mouse backend with python-Xlib and the display faked.""" + monkeypatch.setattr(sys, "platform", "linux") + + fake_xlib = types.ModuleType("Xlib") + fake_xlib.X = types.SimpleNamespace( + ZPixmap=2, ButtonPress=4, ButtonRelease=5, CurrentTime=0) + fake_xlib.protocol = types.SimpleNamespace(event=types.SimpleNamespace()) + monkeypatch.setitem(sys.modules, "Xlib", fake_xlib) + monkeypatch.setitem(sys.modules, "Xlib.protocol", fake_xlib.protocol) + + xtest = types.ModuleType("Xlib.ext.xtest") + xtest.fake_input = lambda *args, **kwargs: None + monkeypatch.setitem(sys.modules, "Xlib.ext", types.ModuleType("Xlib.ext")) + monkeypatch.setitem(sys.modules, "Xlib.ext.xtest", xtest) + + display_mod = types.ModuleType("x11_linux_display") + display_mod.display = types.SimpleNamespace(sync=lambda: None) + monkeypatch.setitem( + sys.modules, + "je_auto_control.linux_with_x11.core.utils.x11_linux_display", + display_mod) + monkeypatch.delitem( + sys.modules, + "je_auto_control.linux_with_x11.mouse.x11_linux_mouse_control", + raising=False) + + module = importlib.import_module( + "je_auto_control.linux_with_x11.mouse.x11_linux_mouse_control") + clicked: list[int] = [] + monkeypatch.setattr(module, "click_mouse", clicked.append) + module.clicked = clicked # for the assertions below + return module + + +UP, DOWN, LEFT, RIGHT = 4, 5, 6, 7 + + +@pytest.mark.parametrize(("direction", "expected"), [ + (UP, UP), (DOWN, DOWN), (LEFT, LEFT), (RIGHT, RIGHT), +]) +def test_a_positive_count_scrolls_the_named_direction(x11_mouse, direction, + expected): + """``scroll_direction`` still means what it always meant.""" + x11_mouse.scroll(3, direction) + assert x11_mouse.clicked == [expected] * 3 + + +@pytest.mark.parametrize(("direction", "expected"), [ + (UP, DOWN), (DOWN, UP), (LEFT, RIGHT), (RIGHT, LEFT), +]) +def test_a_negative_count_reverses_it(x11_mouse, direction, expected): + """This is the behaviour change: the sign wins, as it does elsewhere.""" + x11_mouse.scroll(-3, direction) + assert x11_mouse.clicked == [expected] * 3 + + +def test_the_magnitude_is_still_the_notch_count(x11_mouse): + """A negative count must not go back to scrolling nothing at all.""" + x11_mouse.scroll(-5, DOWN) + assert len(x11_mouse.clicked) == 5 + + +def test_zero_scrolls_nothing(x11_mouse): + """Zero is neither direction and must stay a no-op.""" + x11_mouse.scroll(0, DOWN) + assert x11_mouse.clicked == [] + + +def test_an_unknown_direction_is_passed_through(x11_mouse): + """A button this table does not know is forwarded, not silently remapped.""" + x11_mouse.scroll(-1, 99) + assert x11_mouse.clicked == [99] + + +def test_the_wrapper_routes_the_bsds_to_the_x11_backend(monkeypatch): + """``["linux", "linux2"]`` left FreeBSD outside every branch. + + The wrapper matched Windows, then macOS, then a literal list of Linux + names — so on a BSD ``mouse_scroll`` fell off the end, raised nothing and + scrolled nothing. It asks ``platform_id`` which input stack this is now. + """ + from je_auto_control.wrapper import auto_control_mouse + + calls: list[tuple] = [] + monkeypatch.setattr(auto_control_mouse, "mouse", + types.SimpleNamespace( + scroll=lambda *args: calls.append(args))) + monkeypatch.setattr(auto_control_mouse, "special_mouse_keys_table", + {"scroll_down": DOWN}) + monkeypatch.setattr(sys, "platform", "freebsd14") + + auto_control_mouse.mouse_scroll(2, scroll_direction="scroll_down") + + assert calls == [(2, DOWN)], ( + "a BSD must reach the X11 backend; it used to match no branch at all") + + +# --- the Wayland backend, which had the same abs() ------------------------ + + +def _wayland_mouse(): + """The Wayland mouse module, or a skip when its imports are unavailable.""" + return pytest.importorskip( + "je_auto_control.linux_wayland.mouse", exc_type=ImportError) + + +@pytest.mark.parametrize(("direction_name", "expected"), [ + ("wayland_scroll_direction_up", (0, 3)), + ("wayland_scroll_direction_down", (0, -3)), + ("wayland_scroll_direction_left", (-3, 0)), + ("wayland_scroll_direction_right", (3, 0)), +]) +def test_wayland_positive_count_keeps_the_named_direction(direction_name, + expected): + """Unchanged behaviour, pinned so the sign fix cannot have moved it.""" + mouse = _wayland_mouse() + direction = getattr(mouse, direction_name) + assert mouse._wheel_deltas(3, direction) == expected + + +@pytest.mark.parametrize(("direction_name", "expected"), [ + ("wayland_scroll_direction_up", (0, -3)), + ("wayland_scroll_direction_down", (0, 3)), + ("wayland_scroll_direction_left", (3, 0)), + ("wayland_scroll_direction_right", (-3, 0)), +]) +def test_wayland_negative_count_reverses_it(direction_name, expected): + """``_wheel_deltas`` took ``abs()`` of the count, so the sign was lost. + + Wayland reaches the same wrapper branch as X11, so it had the same defect + and needed the same fix — otherwise "the sign wins everywhere" would have + been true of three backends out of four. + """ + mouse = _wayland_mouse() + direction = getattr(mouse, direction_name) + assert mouse._wheel_deltas(-3, direction) == expected + + +def test_wayland_zero_is_still_a_no_op(): + """Zero has no direction, and must not become a one-notch scroll.""" + mouse = _wayland_mouse() + assert mouse._wheel_deltas(0, mouse.wayland_scroll_direction_down) == (0, 0) diff --git a/test/unit_test/headless/test_sqlite_is_optional.py b/test/unit_test/headless/test_sqlite_is_optional.py new file mode 100644 index 00000000..a5ee8961 --- /dev/null +++ b/test/unit_test/headless/test_sqlite_is_optional.py @@ -0,0 +1,160 @@ +"""``import je_auto_control`` must not need a Python built with sqlite3. + +``sqlite3`` is in the standard library but not in every build of it: CPython +links it against a system library, and FreeBSD ships the result as the separate +``databases/py-sqlite3`` package. Ten AutoControl subsystems keep their state in +SQLite -- run history, checkpoints, the work queue, agent memory, the +remote-desktop audit log, SQL data sources, and the three error tuples that +catch their failures -- and every one of them is reachable from the facade, so +``import sqlite3`` at module scope made the whole package unimportable on such +an interpreter. Moving a mouse needs no database; that is what this file pins. + +Like ``test_facade_import_is_light``, the check is made the way the machine +without it would make it: ``_sqlite3`` is blocked outright in a subprocess -- +which is precisely what FreeBSD's stock ``python311`` reports -- and the facade +has to import anyway. +""" +import os +import pathlib +import subprocess # nosec B404 # reason: fixed argv, sys.executable, no shell +import sys + +import pytest + +from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlUnsupportedOperationException, +) +from je_auto_control.utils import sqlite_support +from je_auto_control.utils.run_history.history_store import HistoryStore + +#: The working tree, so the subprocess tests the checkout rather than whatever +#: version of the package happens to be installed in site-packages. +REPO_ROOT = pathlib.Path(__file__).resolve().parents[3] + +_PROBE = ''' +import pathlib +import sys + + +class _NoSqlite: + """Refuse the sqlite3 extension the way FreeBSD's python311 does.""" + + def find_spec(self, fullname, path=None, target=None): + if fullname == "_sqlite3": + raise ModuleNotFoundError( + "No module named '_sqlite3'", name="_sqlite3") + return None + + +for name in [m for m in sys.modules if m.split(".")[0] in ("sqlite3", "_sqlite3")]: + del sys.modules[name] +sys.meta_path.insert(0, _NoSqlite()) + +import je_auto_control # noqa: E402 + +if "sqlite3" in sys.modules: + raise SystemExit("sqlite3 reached sys.modules") + +# The catch tuples the containment boundaries splice in have to survive the +# absence: empty, so `except (ValueError, *SQLITE_ERRORS)` still compiles and +# still catches everything it did before. +from je_auto_control.utils import sqlite_support # noqa: E402 + +if sqlite_support.SQLITE_ERRORS or sqlite_support.SQLITE_OPERATIONAL_ERRORS: + raise SystemExit("the sqlite3 error tuples are not empty without sqlite3") +if sqlite_support.sqlite3_available(): + raise SystemExit("sqlite3_available() lied") + +# Importing must not open (or create) the default run-history database either. +db = pathlib.Path.home() / ".je_auto_control" / "run_history.sqlite" +if db.exists(): + raise SystemExit("importing the facade created %s" % db) + +print("ok", len(je_auto_control.__all__)) +''' + + +def test_facade_imports_without_sqlite3(tmp_path): + """The facade imports on an interpreter with no ``_sqlite3``.""" + home = tmp_path / "home" + home.mkdir() + env = dict(os.environ, PYTHONPATH=str(REPO_ROOT), + HOME=str(home), USERPROFILE=str(home)) + env.pop("HOMEDRIVE", None) + env.pop("HOMEPATH", None) + # argv is this interpreter plus a module-level literal probe. No shell. + result = subprocess.run( # nosec B603 # nosemgrep # reason: literal argv, no shell + [sys.executable, "-c", _PROBE], + capture_output=True, text=True, timeout=180, check=False, env=env) + assert result.returncode == 0, ( + "import je_auto_control needs sqlite3:\n" + f"{result.stdout}\n{result.stderr}") + assert result.stdout.startswith("ok "), result.stdout + + +def test_require_sqlite3_reports_it_the_way_backends_do(monkeypatch): + """Without the module, using a store raises the unsupported-operation type. + + That type is what the GUI tabs, the REST handler and the executor already + translate into "not available here"; a bare ``ImportError`` would escape + every one of those boundaries. + """ + monkeypatch.setattr(sqlite_support, "_sqlite3", None) + assert sqlite_support.sqlite3_available() is False + with pytest.raises(AutoControlUnsupportedOperationException) as caught: + sqlite_support.require_sqlite3() + assert isinstance(caught.value, AutoControlException) + assert "databases/py-sqlite3" in str(caught.value) + + +def test_error_tuples_hold_the_real_classes_when_sqlite3_is_there(): + """With the module present the catch tuples are its exception classes. + + They are tuples rather than the classes themselves so that the same + ``except`` clauses stay valid, and catch nothing, on a build without it -- + which the subprocess probe above asserts on the empty side. + """ + sqlite3 = pytest.importorskip("sqlite3") + assert sqlite_support.sqlite3_available() is True + assert sqlite_support.SQLITE_ERRORS == (sqlite3.Error,) + assert sqlite_support.SQLITE_OPERATIONAL_ERRORS == (sqlite3.OperationalError,) + + +def test_history_store_opens_the_database_on_first_use(tmp_path): + """Constructing the store must not touch the disk; using it must. + + The module-level ``default_history_store`` is built while the facade is + importing, so anything its constructor does happens on every ``import + je_auto_control``. + """ + pytest.importorskip("sqlite3") + db_path = tmp_path / "nested" / "run_history.sqlite" + store = HistoryStore(path=db_path) + assert not db_path.parent.exists(), "constructing the store made directories" + + run_id = store.start_run("scheduler", "job-1", "script.json") + assert db_path.exists() + assert store.get_run(run_id) is not None + store.close() + + +def test_closing_a_store_that_never_opened_is_a_no_op(tmp_path): + """``close()`` before any query neither connects nor raises.""" + db_path = tmp_path / "unused.sqlite" + store = HistoryStore(path=db_path) + store.close() + assert not db_path.exists() + + +def test_a_closed_store_does_not_silently_reopen(tmp_path): + """After ``close()``, a query still fails instead of starting a new file. + + The lazy connection could otherwise resurrect a closed store -- and for an + in-memory store that means quietly returning an empty history. + """ + sqlite3 = pytest.importorskip("sqlite3") + store = HistoryStore(path=tmp_path / "closed.sqlite") + store.start_run("scheduler", "job-1", "script.json") + store.close() + with pytest.raises(sqlite3.ProgrammingError): + store.count() diff --git a/test/unit_test/headless/test_usb_devices.py b/test/unit_test/headless/test_usb_devices.py index fa7213a8..487c6b77 100644 --- a/test/unit_test/headless/test_usb_devices.py +++ b/test/unit_test/headless/test_usb_devices.py @@ -46,3 +46,34 @@ def test_vendor_and_product_ids_are_4_hex_chars_when_present(): def test_result_to_dict_count_matches_devices(): result = list_usb_devices() assert result.to_dict()["count"] == len(result.devices) + + +# --- system_profiler id parsing (macOS shapes, checked on every platform) --- + + +def test_symbolic_vendor_id_is_not_smuggled_through_as_hex(): + """Apple's own devices report ``apple_vendor_id`` instead of a number. + + Its leading "a" is a valid hex digit, so a lenient parse turns it into + ``000a`` and a pass-through returns the word itself — both of which break + the 4-hex-digit shape ``UsbDevice`` documents. Only real macOS hardware + produces this, which is why it went unseen while the suite ran on + Windows alone. + """ + from je_auto_control.utils.usb.usb_devices import _hex4_from_macos + + assert _hex4_from_macos("apple_vendor_id") is None + + +def test_hex_ids_from_system_profiler_are_normalised(): + from je_auto_control.utils.usb.usb_devices import _hex4_from_macos + + assert _hex4_from_macos("0x05ac") == "05ac" + assert _hex4_from_macos("0x8600") == "8600" + # system_profiler appends the vendor name to some entries. + assert _hex4_from_macos("0x05AC (Apple Inc.)") == "05ac" + # A bare id, and one that needs padding out to four digits. + assert _hex4_from_macos("05ac") == "05ac" + assert _hex4_from_macos("0x1") == "0001" + assert _hex4_from_macos(None) is None + assert _hex4_from_macos("") is None diff --git a/test/unit_test/headless/test_wayland_dbus_client.py b/test/unit_test/headless/test_wayland_dbus_client.py index 2be6a515..e5d15d85 100644 --- a/test/unit_test/headless/test_wayland_dbus_client.py +++ b/test/unit_test/headless/test_wayland_dbus_client.py @@ -20,8 +20,12 @@ import pytest -from je_auto_control.linux_wayland import _dbus_client -from je_auto_control.linux_wayland._dbus_client import ( +# These exercise the marshalling internals, so they import the client +# where it lives rather than through the Wayland package's re-export: +# a shim that forwarded private names too would be a second copy of +# the module's surface to keep in step. +from je_auto_control.utils.dbus_client import session_bus as _dbus_client +from je_auto_control.utils.dbus_client.session_bus import ( DBusError, SessionBus, Variant, _Reader, _SignatureReader, _Writer, ) @@ -159,9 +163,38 @@ def test_a_signature_that_wants_more_members_than_it_was_given_is_rejected(): def test_a_type_this_does_not_marshal_says_so(): + """``h`` is the one fixed-width type this refuses, and it refuses on purpose. + + A UNIX_FD is a 32-bit index into a descriptor array carried in the + message's control data, which this client does not receive. Marshalling + the index alone would hand a caller a number that addresses nothing. + """ writer = _Writer() with pytest.raises(DBusError, match="cannot marshal"): - writer.value("d", 1.5) + writer.value("h", 0) + + +def test_signed_and_wide_numbers_round_trip(): + """AT-SPI reports extents as signed integers, because coordinates can be. + + A monitor left of or above the primary one puts a window at a negative + coordinate, so reading these as unsigned would turn -1280 into + 4293967296 and send every click to the wrong screen. + """ + for signature, value in (("i", -1280), ("i", 42), ("n", -5), ("q", 5), + ("x", -(2 ** 40)), ("t", 2 ** 40), ("d", 1.5)): + writer = _Writer() + writer.value(signature, value) + reader = _dbus_client._Reader(writer.data) + assert reader.value(signature) == value, signature + + +def test_a_struct_of_signed_integers_round_trips(): + """The exact shape AT-SPI's Component.GetExtents returns.""" + writer = _Writer() + writer.value("(iiii)", [-1280, 0, 640, 480]) + reader = _dbus_client._Reader(writer.data) + assert reader.value("(iiii)") == (-1280, 0, 640, 480) # === Whole messages ======================================================== diff --git a/test/unit_test/headless/test_window_backends.py b/test/unit_test/headless/test_window_backends.py new file mode 100644 index 00000000..94d9aa08 --- /dev/null +++ b/test/unit_test/headless/test_window_backends.py @@ -0,0 +1,228 @@ +"""Headless tests for the window-management backend seam. No Qt. + +The old window tests are Windows-only, because the facade was: it branched on +``sys.platform`` and raised everywhere else, so there was nothing to test on +another platform. Now that the platform detail sits behind a backend, the +facade's own logic — substring matching, ordering, "move without restating the +size", waiting — is platform-neutral and can be checked anywhere, which is +what these do by driving a fake backend. +""" +import sys + +import pytest + +from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, AutoControlException, + AutoControlUnsupportedOperationException, +) +from je_auto_control.wrapper import auto_control_window as w +from je_auto_control.wrapper.window_backends import ( + NullWindowBackend, WindowManageBackend, get_backend, reset_backend_cache, +) + + +class FakeBackend(WindowManageBackend): + """A backend that records what it was asked and answers predictably.""" + + name = "fake" + + def __init__(self): + self.available = True + self.calls = [] + self.windows = [(11, "Editor"), (12, " "), (13, "Browser")] + self.rect = (10, 20, 110, 220) + self.minimized = False + + def list_windows(self): + return list(self.windows) + + def foreground_window(self): + return 13 + + def window_rect(self, window_id): + self.calls.append(("window_rect", window_id)) + return self.rect + + def window_process_id(self, window_id): + return {11: 111, 12: 112, 13: 113}.get(window_id, 0) + + def is_minimized(self, window_id): + return self.minimized + + def set_foreground(self, window_id): + self.calls.append(("set_foreground", window_id)) + + def restore(self, window_id): + self.calls.append(("restore", window_id)) + + def show(self, window_id, cmd_show): + self.calls.append(("show", window_id, cmd_show)) + + def close(self, window_id): + self.calls.append(("close", window_id)) + return True + + def minimize(self, window_id): + self.calls.append(("minimize", window_id)) + return True + + def move(self, window_id, x, y, width, height): + self.calls.append(("move", window_id, x, y, width, height)) + return True + + def post_key(self, window_id, keycode, character=""): + self.calls.append(("post_key", window_id, keycode, character)) + return True + + def post_click(self, window_id, button, x, y): + self.calls.append(("post_click", window_id, button, x, y)) + return True + + +@pytest.fixture() +def backend(monkeypatch): + """Point the facade at a fake backend, on every platform.""" + fake = FakeBackend() + monkeypatch.setattr(w, "get_backend", lambda: fake) + return fake + + +# --- the refusal contract -------------------------------------------------- + + +def test_unsupported_is_catchable_as_both_families(): + """A bare NotImplementedError escaped every containment boundary. + + The GUI tabs and the REST handler catch ``NotImplementedError`` to say + "not on this platform", so that has to keep working. The executor and the + background loops catch ``AutoControlException``, and a bare + ``NotImplementedError`` slipped straight past them — aborting a whole + script where one action should have been reported as failed. + """ + assert issubclass(AutoControlUnsupportedOperationException, + NotImplementedError) + assert issubclass(AutoControlUnsupportedOperationException, + AutoControlException) + + +def test_base_backend_refuses_by_name(): + base = WindowManageBackend() + with pytest.raises(AutoControlUnsupportedOperationException) as caught: + base.close(1) + assert "close" in str(caught.value) + assert "abstract" in str(caught.value) + + +def test_null_backend_lists_nothing_but_refuses_actions(): + """Listing is answerable without a backend; acting is not. + + "There are no windows I can see" lets a caller iterate and move on. A + silent False from ``close`` would read as "the window refused", which is + a different thing and would have callers retrying forever. + """ + null = NullWindowBackend("no display here") + assert null.list_windows() == [] + assert not null.available + assert "no display here" in null.name + with pytest.raises(AutoControlUnsupportedOperationException): + null.close(1) + + +def test_selection_is_cached_and_resettable(): + reset_backend_cache() + first = get_backend() + assert get_backend() is first + reset_backend_cache() + assert get_backend() is not first + + +def test_every_platform_gets_a_backend_that_imports(): + """No platform may fail to import, whatever it can or cannot do.""" + reset_backend_cache() + chosen = get_backend() + assert isinstance(chosen, WindowManageBackend) + assert chosen.name + + +@pytest.mark.skipif(sys.platform not in ("win32", "cygwin", "msys"), + reason="the Win32 backend only builds on Windows") +def test_windows_selects_the_win32_backend(): + reset_backend_cache() + assert get_backend().name == "win32" + + +# --- the facade's platform-neutral logic ----------------------------------- + + +def test_list_windows_can_drop_the_untitled(backend): + assert w.list_windows() == backend.windows + assert w.list_windows(titled_only=True) == [(11, "Editor"), (13, "Browser")] + + +def test_find_window_matches_case_insensitively_by_default(backend): + assert w.find_window("edit") == (11, "Editor") + assert w.find_window("EDIT") == (11, "Editor") + assert w.find_window("EDIT", case_sensitive=True) is None + assert w.find_window("nothing") is None + + +def test_focus_window_restores_only_when_minimised(backend): + """SW_RESTORE on a maximised window un-maximises it. + + So focusing must not restore unconditionally: that would shrink a + maximised window as a side effect of being asked to focus it. + """ + w.focus_window("Editor") + assert ("restore", 11) not in backend.calls + assert ("set_foreground", 11) in backend.calls + + backend.calls.clear() + backend.minimized = True + w.focus_window("Editor") + assert ("restore", 11) in backend.calls + + +def test_focus_window_reports_a_missing_window_as_an_action_failure(backend): + with pytest.raises(AutoControlActionException): + w.focus_window("nothing matches this") + + +def test_move_without_a_size_keeps_the_current_size(backend): + """Otherwise a plain reposition has to restate dimensions it must look up.""" + assert w.move_window_by_title("Editor", 5, 6) + assert ("move", 11, 5, 6, 100, 200) in backend.calls + + +def test_move_with_a_size_passes_it_through(backend): + assert w.move_window_by_title("Editor", 5, 6, 70, 80) + assert ("move", 11, 5, 6, 70, 80) in backend.calls + + +def test_close_and_minimize_report_no_match_as_false(backend): + assert w.close_window_by_title("nothing") is False + assert w.minimize_window_by_title("nothing") is False + assert backend.calls == [] + + +def test_windows_for_process_id_filters_by_owner(backend): + assert w.windows_for_process_id(111) == [(11, "Editor")] + assert w.windows_for_process_id(999) == [] + + +def test_minimize_windows_for_process_counts_what_it_minimised(backend): + assert w.minimize_windows_for_process(113) == 1 + assert ("minimize", 13) in backend.calls + + +def test_foreground_window_pairs_the_id_with_its_title(backend): + assert w.foreground_window() == (13, "Browser") + assert w.foreground_window_process_id() == 113 + + +def test_wait_for_window_returns_as_soon_as_it_appears(backend): + assert w.wait_for_window("Browser", timeout=1.0, poll=0.05) == 13 + + +def test_wait_for_window_times_out_as_an_action_failure(backend): + with pytest.raises(AutoControlActionException): + w.wait_for_window("never", timeout=0.2, poll=0.05) diff --git a/test/verify/freebsd_verify.py b/test/verify/freebsd_verify.py new file mode 100644 index 00000000..c05bca5c --- /dev/null +++ b/test/verify/freebsd_verify.py @@ -0,0 +1,409 @@ +"""Drive real X11 input from a real FreeBSD, and read it back off the server. + +The X11 backend was gated on ``sys.platform`` being ``linux``/``linux2``, so it +refused to load on a FreeBSD desktop running the same X server, the same +python-Xlib and the same code. Relaxing that guard is only worth something if a +BSD actually runs it, and no hosted runner is one — so +``.github/workflows/platform-smoke.yml`` boots a FreeBSD VM inside an Ubuntu +runner and runs this. + +For a while this could only check the *decision* — that ``sys.platform`` really +looks like ``freebsd14``, and that the classification every relaxed guard asks +answers correctly on it — because importing anything under ``je_auto_control`` +ran the facade, and the facade imported OpenCV and cryptography at module scope. +Neither publishes a FreeBSD wheel, and building them from ports had not finished +after fifty minutes. + +That was the wrong thing to fix. Moving a mouse needs neither package, and the +facade had no business insisting on them: they are imported by the functions +that use them now (``test_facade_import_is_light.py`` keeps it that way), which +leaves this VM needing python-Xlib and defusedxml — both pure Python — and an X +server. So the whole backend runs here, not just the guard. + +Running it then found the same mistake in a second place, one no wheel-shaped +reasoning would have caught: FreeBSD's ``python311`` has no ``sqlite3`` (it is +the separate ``databases/py-sqlite3`` package), and ten subsystems imported it +at module scope, so the facade still would not import. That is why the checks +below assert the module is *absent* here and that the package works anyway. + +Ground truth is the X server answering for itself, never this codebase: +``query_pointer`` for where the cursor actually is, its button mask for which +buttons the server believes are down, and ``query_keymap`` — a bitmap of every +physically-pressed key — for whether an injected key press really landed. That +last one is what makes a second X client unnecessary here; the Linux +``x11-verification`` job already reads events back out of ``xev``, and what a +BSD is uniquely needed to answer is whether this code drives the same server on +a different kernel. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import os +import sys +import time +import traceback +from typing import Any, Callable, List, Tuple + +#: Points the cursor is driven to. The corners matter: an off-by-one in the +#: coordinate space shows up at (0, 0) and at the far edge, not in the middle. +PROBE_POINTS: Tuple[Tuple[int, int], ...] = ( + (321, 123), (0, 0), (1, 1), (640, 480), (1279, 1023), +) + +#: How long the server is given to settle after an injected event. +SETTLE = 0.05 + +_results: List[Tuple[str, bool, str]] = [] + + +def check(name: str, fn: Callable[[], Any]) -> Any: + """Run one check, record pass/fail, and keep going either way.""" + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a failed check must not stop the rest + _results.append((name, False, traceback.format_exc(limit=3).strip())) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True, str(detail))) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _assert_eq(actual: Any, expected: Any) -> str: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + return f"{actual!r}" + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +# --- the decision only a BSD can answer --------------------------------- + + +def check_platform_identity() -> None: + """What ``sys.platform`` is here, and what every relaxed guard makes of it.""" + from je_auto_control.utils import platform_id + + def _is_freebsd() -> str: + _assert_true(sys.platform.startswith("freebsd"), + f"not a FreeBSD: sys.platform is {sys.platform!r}") + return sys.platform + check("sys.platform really is a FreeBSD", _is_freebsd) + + # Before the guards were relaxed the answer to the second of these was + # False on exactly this platform, and the package refused to import at all. + check("is_bsd() recognises it", + lambda: _assert_true(platform_id.is_bsd(), "not recognised as a BSD")) + check("is_x11_unix() recognises it", + lambda: _assert_true(platform_id.is_x11_unix(), "not an X11 unix")) + check("is_windows() does not", + lambda: _assert_true(not platform_id.is_windows(), "claimed Windows")) + check("is_macos() does not", + lambda: _assert_true(not platform_id.is_macos(), "claimed macOS")) + check("current_family() is bsd", + lambda: _assert_eq(platform_id.current_family(), "bsd")) + + # The version suffix is the trap: sys.platform is freebsd14 here, never a + # bare "freebsd", so an equality check would match no real system at all. + def _suffixed() -> str: + _assert_true( + sys.platform != "freebsd", + "this release stopped carrying a version suffix; the prefix match " + "still works, but the comment explaining why it exists no longer " + "describes reality") + return sys.platform + check("sys.platform still carries the major version", _suffixed) + + +def check_facade_is_importable() -> None: + """The facade imports here — which is the thing that used to be impossible. + + And it imports *without* the wheels FreeBSD has none of. Asserting they are + genuinely absent is the point: if some future image happens to have OpenCV + installed, this job would quietly stop testing the property it exists for. + """ + def _absent() -> str: + import importlib.util + + present = [name for name in ("cv2", "PIL", "cryptography", "je_open_cv") + if importlib.util.find_spec(name) is not None] + _assert_true(not present, + f"expected these to be absent on the VM, found: {present}") + return "cv2, PIL, cryptography, je_open_cv all absent" + check("the heavy wheels really are not installed here", _absent) + + # Not a wheel — a piece of the standard library the OS packages + # separately. Same trap, and asserting it is absent is what keeps this + # job testing the property: if a later image ships py311-sqlite3, the + # check goes red rather than quietly passing on an easier machine. + def _no_sqlite3() -> str: + import importlib.util + + _assert_true( + importlib.util.find_spec("_sqlite3") is None, + "this image has sqlite3, so importing the facade here no longer " + "proves the package works without it") + return "no _sqlite3, as FreeBSD's python311 ships it" + check("sqlite3 really is not installed here", _no_sqlite3) + + def _facade() -> str: + import je_auto_control + + return f"{len(je_auto_control.__all__)} public names" + check("import je_auto_control", _facade) + + def _backend() -> str: + from je_auto_control.wrapper import platform_wrapper + + module = platform_wrapper.mouse.__name__ + _assert_true("linux_with_x11" in module, + f"expected the X11 backend, got {module}") + return module + check("the wrapper selects the X11 backend on a BSD", _backend) + + +def check_sqlite_backed_features_degrade() -> None: + """Without sqlite3, the stores fail on use — not on import, and not fatally. + + The failure has to be the type every containment boundary already + understands. A bare ``ImportError`` escapes all of them, which is how one + absent database module took the mouse down with it. + """ + from je_auto_control.utils import sqlite_support + + check("sqlite3_available() reports the truth here", + lambda: _assert_eq(sqlite_support.sqlite3_available(), False)) + + def _typed() -> str: + from je_auto_control.utils.exception.exceptions import ( + AutoControlException, AutoControlUnsupportedOperationException, + ) + from je_auto_control.utils.run_history.history_store import ( + default_history_store, + ) + + try: + default_history_store.count() + except AutoControlUnsupportedOperationException as error: + _assert_true(isinstance(error, AutoControlException), + "not in the family the executor catches") + return type(error).__name__ + raise AssertionError("reading run history did not raise without sqlite3") + check("using a store raises the unsupported-operation error", _typed) + + +# --- input, driven and read back off the server ------------------------- + + +def check_mouse_position() -> None: + """Every point the backend is told to move to is where the server puts it.""" + from je_auto_control.linux_with_x11.mouse import x11_linux_mouse_control as m + + for x, y in PROBE_POINTS: + def _round_trip(x=x, y=y) -> str: + m.set_position(x, y) + time.sleep(SETTLE) + return _assert_eq(m.position(), (x, y)) + check(f"set_position({x}, {y}) lands where it was asked", _round_trip) + + +def check_mouse_buttons() -> None: + """A pressed button is a button the *server* reports as down.""" + from je_auto_control.linux_with_x11.core.utils.x11_linux_display import ( + display, + ) + from je_auto_control.linux_with_x11.mouse import x11_linux_mouse_control as m + + def _mask() -> int: + return display.screen().root.query_pointer()._data["mask"] + + #: Button 1 in the pointer mask reported by the server. + button_1 = 1 << 8 + + def _press() -> str: + m.set_position(400, 300) + m.press_mouse(m.x11_linux_mouse_left) + time.sleep(SETTLE) + held = _mask() + m.release_mouse(m.x11_linux_mouse_left) + time.sleep(SETTLE) + _assert_true(bool(held & button_1), + f"the server did not report button 1 down (mask {held:#x})") + _assert_true(not _mask() & button_1, + "button 1 stayed down after release_mouse") + return "pressed and released" + check("press_mouse reaches the server, release_mouse clears it", _press) + + def _click() -> str: + m.click_mouse(m.x11_linux_mouse_left, 500, 260) + time.sleep(SETTLE) + _assert_eq(m.position(), (500, 260)) + _assert_true(not _mask() & button_1, + "click_mouse left button 1 held down") + return "moved and clicked cleanly" + check("click_mouse moves and leaves no button held", _click) + + +def check_keyboard() -> None: + """An injected key press is a key the server reports as physically down. + + ``query_keymap`` is a 32-byte bitmap of every key currently held, straight + out of the server, so this needs no second client to read the event back. + """ + from je_auto_control.linux_with_x11.core.utils.x11_linux_display import ( + display, + ) + from je_auto_control.linux_with_x11.keyboard import ( + x11_linux_keyboard_control as k, + ) + + def _is_down(keycode: int) -> bool: + keymap = display.query_keymap() + return bool(keymap[keycode // 8] & (1 << (keycode % 8))) + + # Keycode 38 is 'a' on the standard PC layout Xvfb comes up with. The + # check does not depend on which character it produces — only on the + # server agreeing that this physical key went down and came back up. + keycode = 38 + + def _press() -> str: + _assert_true(not _is_down(keycode), "the key was already down") + k.press_key(keycode) + time.sleep(SETTLE) + down = _is_down(keycode) + k.release_key(keycode) + time.sleep(SETTLE) + _assert_true(down, "the server never saw the key go down") + _assert_true(not _is_down(keycode), "the key stayed down after release") + return f"keycode {keycode} down then up" + check("press_key/release_key reach the server", _press) + + +def check_scroll() -> None: + """Scrolling is the one BSD-only defect this job exists to catch. + + ``mouse_scroll`` matched Windows, then macOS, then a literal + ``["linux", "linux2"]`` — so on a BSD it fell off the end of the chain, + raised nothing and scrolled nothing. Nothing above would notice: the + pointer mask never shows a wheel button, because X11 delivers a scroll as a + press *and* release of button 4/5/6/7 too fast to sample. + + So this maps a real X window that has asked for button events, puts the + cursor inside it, and reads the buttons back out of the event queue — the + same ground truth the Linux job gets from ``xev``, without needing a second + process. It also holds the sign contract that was settled for all three + platforms: a negative count reverses the direction. + """ + from Xlib import X + + from je_auto_control.linux_with_x11.core.utils.x11_linux_display import ( + display, + ) + + screen = display.screen() + window = screen.root.create_window( + 0, 0, 400, 400, 0, screen.root_depth, + X.InputOutput, X.CopyFromParent, + background_pixel=screen.white_pixel, + event_mask=X.ButtonPressMask | X.ButtonReleaseMask) + window.map() + display.sync() + + def _drain() -> None: + while display.pending_events(): + display.next_event() + + def _collect(count: int) -> List[int]: + """The button of every ButtonPress that arrives, up to ``count``.""" + buttons: List[int] = [] + deadline = time.time() + 5.0 + while len(buttons) < count and time.time() < deadline: + if not display.pending_events(): + time.sleep(0.01) + continue + event = display.next_event() + if event.type == X.ButtonPress: + buttons.append(event.detail) + return buttons + + def _scroll(value: int, direction: str, expected: int) -> str: + import je_auto_control as ac + + ac.set_mouse_position(200, 200) + time.sleep(SETTLE) + _drain() + ac.mouse_scroll(value, scroll_direction=direction) + buttons = _collect(abs(value)) + _assert_eq(buttons, [expected] * abs(value)) + return f"{value:+d} {direction} arrived as button {expected}" + + # A positive count scrolls the direction it names... + check("scroll_down arrives as button 5", + lambda: _scroll(2, "scroll_down", 5)) + check("scroll_up arrives as button 4", + lambda: _scroll(2, "scroll_up", 4)) + # ...and a negative one reverses it, on this platform as on the others. + check("a negative count reverses scroll_down into button 4", + lambda: _scroll(-2, "scroll_down", 4)) + check("a negative count reverses scroll_up into button 5", + lambda: _scroll(-2, "scroll_up", 5)) + + window.destroy() + display.sync() + + +def check_facade_input() -> None: + """The same thing through the public API, so the binding is covered too.""" + def _through_facade() -> str: + import je_auto_control as ac + + ac.set_mouse_position(210, 340) + time.sleep(SETTLE) + return _assert_eq(tuple(ac.get_mouse_position()), (210, 340)) + check("set_mouse_position/get_mouse_position through the facade", + _through_facade) + + +def report_environment() -> None: + """Print what this is running on, so a failure has context in the log.""" + import platform + + print(f"uname : {' '.join(platform.uname())}") + print(f"sys.platform : {sys.platform}") + print(f"python : {sys.version.split()[0]}") + print(f"DISPLAY : {os.environ.get('DISPLAY', '(unset)')}") + print() + + +def summarise() -> int: + """Print the tally; return the number of failed checks.""" + failed = [name for name, ok, _ in _results if not ok] + print() + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + return len(failed) + + +def main() -> int: + report_environment() + check_platform_identity() + check_facade_is_importable() + check_sqlite_backed_features_degrade() + check_mouse_position() + check_mouse_buttons() + check_scroll() + check_keyboard() + check_facade_input() + return summarise() + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/test/verify/macos_verify.py b/test/verify/macos_verify.py new file mode 100644 index 00000000..c2879d8e --- /dev/null +++ b/test/verify/macos_verify.py @@ -0,0 +1,373 @@ +"""Measure what a real macOS runner lets AutoControl's backend actually do. + +macOS is the one supported platform with no container to put it in, so this +runs directly on a ``macos-14`` GitHub runner (see +``.github/workflows/platform-smoke.yml``). It exists because the capability +matrix said "implementation" for every macOS row: the code was there and +nothing had ever run it on a Mac. + +Two of the capabilities here are gated by TCC — macOS asks the *user* to +grant Screen Recording and Accessibility, and a headless CI runner has no +user to ask. Which of them a runner grants by default is not something to +guess at, and guessing is how the Wayland work lost time twice by recording a +desktop's refusal as a container's limitation. So this was measured first, +and the measurement was a surprise: **a macos-14 runner grants both**, and +every probe below passes on one. See :data:`EXPECTED`. + +``--measure`` + Run every probe, print what happened, and exit 0. Nothing is asserted; + the output is the measurement. This is how :data:`EXPECTED` is + (re)populated when a runner image changes. + +default + Assert :data:`EXPECTED` — what the measurement showed the runner permits. + Exit status is the number of checks that did not match, so a capability + appearing or disappearing turns CI red and names it. This is what the + workflow runs. + +Assert mode refuses to pass while :data:`EXPECTED` is empty, because a gate +that asserts nothing reads as coverage that does not exist. +""" +from __future__ import annotations + +import argparse +import platform +import sys +import time +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +#: What a ``macos-14`` runner was measured to permit, on 2026-08-19. Keys are +#: probe names; values are ``True`` (works), ``False`` (silently does nothing +#: or is refused), or a string naming the exception type it raises. +#: +#: The measurement was a surprise worth writing down: **a GitHub macOS runner +#: grants both Screen Recording and Accessibility to the interpreter**, so +#: every capability here works. Capture returns real pixels rather than the +#: black rectangle a refusal produces, ``CGEventPost`` moves the cursor and +#: the move reads back exactly, and the AX walk returns real elements. The +#: usual assumption — that CI cannot exercise a TCC-gated macOS API — is +#: wrong for this runner, which is why this is measured and not reasoned +#: about. +#: +#: If a future runner image tightens any of that, the mismatch turns this job +#: red and names the capability that changed, which is the point. +EXPECTED: Dict[str, Any] = { + "backend-selection": True, + "screen-size": True, + "screenshot": True, + "get-pixel": True, + "mouse-position": True, + "mouse-move": True, + "keyboard-post": True, + "accessibility-tree": True, + # The one row here that a CI run measured rather than a local one: there + # is no Mac in the loop where this is written. It is asserted True from + # the same grant the rows above it were measured to have — an event tap + # needs exactly the Accessibility permission `keyboard-post` already + # proved is granted. If that reasoning is wrong the job says so, names + # this probe, and prints how many events the tap actually saw. + "recorder": True, + # True means "the code answered", not "the runner had windows". Measured: + # Quartz reports 5 on-screen windows on a macos-14 runner and *none* of + # them is at the application layer — they are menu bar and system UI. So + # the count is reported and not asserted. + "window-management": True, +} + +#: How long the window server is given to reflect a posted modifier. +KEY_STATE_TIMEOUT = 3.0 + +#: How long the recorder's tap thread is given to see the posted events. Its +#: run loop advances in slices, so one slice is not enough to rely on. +RECORDER_SETTLE_SECONDS = 1.0 + +_results: List[Tuple[str, bool, str]] = [] + + +def note(message: str) -> None: + """Print an indented remark that is not a probe.""" + print(f" {message}") + + +class Outcome: + """What one probe did, in a form both modes can use.""" + + def __init__(self, worked: bool, detail: str, + error: Optional[str] = None) -> None: + self.worked = worked + self.detail = detail + self.error = error + + @property + def key(self) -> Any: + """The value :data:`EXPECTED` records for this outcome.""" + return self.error if self.error else self.worked + + def __str__(self) -> str: + if self.error: + return f"raised {self.error}: {self.detail}" + return ("works — " if self.worked else "no effect — ") + self.detail + + +def probe(name: str, fn: Callable[[], Outcome]) -> Outcome: + """Run one probe and record what it did, without judging it yet.""" + try: + outcome = fn() + except Exception as error: # noqa: BLE001 # reason: measuring, not asserting + outcome = Outcome(False, traceback.format_exc(limit=2).strip().replace( + "\n", " | "), type(error).__name__) + _results.append((name, outcome.worked, str(outcome))) + print(f" {name}: {outcome}") + return outcome + + +# --- probes ---------------------------------------------------------------- + + +def probe_backend() -> Outcome: + """The wrapper must land on the macOS backend at all.""" + from je_auto_control.wrapper import platform_wrapper + + module = platform_wrapper.mouse.__name__ + return Outcome("osx" in module, module) + + +def probe_screen_size() -> Outcome: + """Quartz reports the display size without any TCC grant.""" + from je_auto_control import screen_size + + width, height = screen_size() + return Outcome(width > 0 and height > 0, f"{width}x{height}") + + +def probe_screenshot() -> Outcome: + """Capture needs Screen Recording on 10.15+; a refusal is silent.""" + from je_auto_control import screen_size, screenshot + + width, height = screen_size() + frame = screenshot() + if frame is None or getattr(frame, "size", 0) == 0: + return Outcome(False, "returned an empty frame") + shape = f"{frame.shape[1]}x{frame.shape[0]}" + # A refused capture comes back as a correctly-sized black rectangle + # rather than an error, so size alone proves nothing. + blank = bool((frame == 0).all()) + return Outcome(not blank, + f"{shape} against a {width}x{height} display" + + (", but every pixel is black" if blank else "")) + + +def probe_get_pixel() -> Outcome: + """The single-pixel read goes through the same capture permission.""" + from je_auto_control import get_pixel + + pixel = get_pixel(10, 10) + return Outcome(pixel is not None, repr(pixel)) + + +def probe_mouse_position() -> Outcome: + """Reading the cursor needs no grant; only moving it does.""" + from je_auto_control import get_mouse_position + + position = get_mouse_position() + return Outcome(position is not None, repr(position)) + + +def probe_mouse_move() -> Outcome: + """CGEventPost is accepted whether or not it is allowed to take effect. + + So the move is measured by reading the cursor back, not by whether the + call returned — a refused post raises nothing at all. + """ + from je_auto_control import get_mouse_position, screen_size, set_mouse_position + + width, height = screen_size() + target = (min(width - 5, 137), min(height - 5, 211)) + set_mouse_position(*target) + landed = tuple(get_mouse_position() or (-1, -1)) + return Outcome(landed == target, f"asked {target}, cursor at {landed}") + + +def probe_keyboard() -> Outcome: + """Key posting is the most restricted of all; measure, do not assume. + + Nothing here has focus, so what is measured is whether the post is + accepted and the modifier state changes — not where the character went. + + The read is polled rather than taken once. Measured on a macos-14 runner, + an immediate ``check_key_is_press`` after the post returned True on one + run and False on the next: ``CGEventSourceKeyState`` reflects the window + server's state, and the posted event has to reach it first. Reading once + makes this probe a coin toss, and a gate that is a coin toss is worse + than no gate. + """ + from je_auto_control import check_key_is_press, press_keyboard_key, release_keyboard_key + + press_keyboard_key("shift") + try: + deadline = time.monotonic() + KEY_STATE_TIMEOUT + held = False + while time.monotonic() < deadline: + held = bool(check_key_is_press("shift")) + if held: + break + time.sleep(0.05) + finally: + release_keyboard_key("shift") + return Outcome(held, f"check_key_is_press('shift') became {held!r} within " + f"{KEY_STATE_TIMEOUT}s") + + +def probe_accessibility() -> Outcome: + """The AX tree needs Accessibility; without it the walk returns nothing.""" + from je_auto_control.utils.accessibility.backends import get_backend + + backend = get_backend() + if not backend.available: + return Outcome(False, f"backend {backend.name!r} reports unavailable") + elements = backend.list_elements(max_results=5) + return Outcome(bool(elements), + f"backend {backend.name!r} returned {len(elements)} elements") + + +def probe_recorder() -> Outcome: + """Recording needs Accessibility, a live event tap, and real events in it. + + macOS shipped without a recorder for as long as the code did exist: the + old listener built an ``NSApplication`` at import and stopped recording + with ``AppHelper.runEventLoop()``, so wiring it up would have put both on + the path of ``import je_auto_control``. It now captures through a + listen-only ``CGEventTap`` on its own thread, and this is where that gets + exercised against a real window server rather than a fake. + + Three things have to hold together and only a Mac can answer any of them: + the tap can be created at all (that is the Accessibility grant), events + posted into the session reach it, and what comes back out carries the + coordinates and the release — not just the press. + """ + import je_auto_control as ac + from je_auto_control.wrapper import platform_wrapper + + if platform_wrapper.recorder is None: + return Outcome(False, "no recorder is selected on this platform") + + target = (321, 123) + ac.record() + try: + # Posted through the public API, so this exercises the same path a + # user's session does rather than a private Quartz call. + ac.set_mouse_position(*target) + ac.click_mouse("mouse_left", *target) + ac.press_keyboard_key("a") + ac.release_keyboard_key("a") + # The tap thread runs the loop in slices; give it more than one. + time.sleep(RECORDER_SETTLE_SECONDS) + finally: + events = ac.stop_record_timeline() + + operations = [event.get("op") for event in events] + clicks = [event for event in events if event.get("op") == "mouse_down"] + landed = [(event.get("x"), event.get("y")) for event in clicks] + return Outcome( + "mouse_down" in operations and "mouse_up" in operations + and "key_down" in operations and target in landed, + f"{len(events)} event(s): {operations}; clicks landed at {landed}, " + f"posted {target}") + + +def probe_window_management() -> Outcome: + """Quartz lists windows without a grant; acting on one needs Accessibility. + + What is asserted is that the *code* answers — the backend is selected, the + Quartz query runs, and every window it returns can be described. What is + only *reported* is how many there are, because that is a property of the + runner's session rather than of this project: a GitHub macOS runner was + measured to have no ordinary application windows at all. Gating on a + count would go red the day the runner image happens to open one. + """ + import je_auto_control as ac + from je_auto_control.wrapper.window_backends import get_backend + + backend = get_backend() + if not backend.available: + return Outcome(False, f"backend {backend.name!r} reports unavailable") + # Before the layer filter, so "the session is empty" and "the filter + # dropped everything" are told apart rather than guessed at. + raw = len(backend._window_info()) + windows = ac.list_windows() + described = [] + for window_id, title in windows[:3]: + described.append((title, backend.window_rect(window_id), + backend.window_process_id(window_id))) + complete = all(rect is not None and pid > 0 + for _title, rect, pid in described) + return Outcome(complete, + f"{raw} on-screen window(s) from Quartz, {len(windows)} at " + f"the application layer; described {described}") + + +PROBES: List[Tuple[str, Callable[[], Outcome]]] = [ + ("backend-selection", probe_backend), + ("screen-size", probe_screen_size), + ("screenshot", probe_screenshot), + ("get-pixel", probe_get_pixel), + ("mouse-position", probe_mouse_position), + ("mouse-move", probe_mouse_move), + ("keyboard-post", probe_keyboard), + ("accessibility-tree", probe_accessibility), + ("recorder", probe_recorder), + ("window-management", probe_window_management), +] + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--measure", action="store_true", + help="report what the runner permits and exit 0, asserting nothing") + options = parser.parse_args(argv) + + print("=" * 72) + print("AutoControl macOS verification — real macOS, real window server") + print("=" * 72) + note(f"{platform.platform()} python {platform.python_version()}") + print("-" * 72) + + outcomes = {name: probe(name, fn) for name, fn in PROBES} + + print("-" * 72) + if options.measure: + print("measurement only — nothing asserted. EXPECTED would be:") + print() + for name, outcome in outcomes.items(): + print(f' "{name}": {outcome.key!r},') + print() + print("Paste that into EXPECTED and drop --measure to make it a gate.") + print("=" * 72) + return 0 + + if not EXPECTED: + print("EXPECTED is empty, so there is nothing to assert. Run with") + print("--measure first and fill it in; passing here would mean") + print("nothing and would read as coverage that does not exist.") + print("=" * 72) + return 1 + + failed = [] + for name, outcome in outcomes.items(): + if name not in EXPECTED: + failed.append(f"{name}: not in EXPECTED (a new probe?)") + elif outcome.key != EXPECTED[name]: + failed.append( + f"{name}: expected {EXPECTED[name]!r}, measured {outcome.key!r}") + print(f"{len(outcomes) - len(failed)}/{len(outcomes)} probes match " + f"what this runner was measured to permit") + for line in failed: + print(f" CHANGED: {line}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main())