diff --git a/docker/.dockerignore b/.dockerignore similarity index 100% rename from docker/.dockerignore rename to .dockerignore diff --git a/.gitattributes b/.gitattributes index dfe07704..e1207c57 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,8 @@ # Auto detect text files and perform LF normalization * text=auto + +# Shell scripts keep LF whatever the checkout platform. They are executed by +# /bin/sh inside a Linux container, and a CRLF shebang makes the kernel look +# for an interpreter literally named "/bin/sh\r": the image builds, then every +# container dies with "no such file or directory" on the entrypoint. +*.sh text eol=lf diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index d0905b11..d76d7236 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -108,3 +108,257 @@ jobs: echo "REST health probe never succeeded" >&2 exit 1 fi + + wayland-verification: + name: Wayland backend against a real compositor + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the Wayland verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.wayland + tags: autocontrol-wayland:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # Two halves, both of which mocks structurally cannot cover. + # + # Capture: sway's headless backend needs no GPU, no seat and no display, + # so a plain ubuntu runner can host a genuine wlroots session. grim's + # argv and -g geometry, wlr-randr's undocumented output format, and the + # whole screenshot -> screen_grabber -> capture -> grab_image chain are + # checked against pixels the compositor actually painted. Twice: once + # with the outputs side by side from the origin, and once with the + # left-hand one at x=-1280, which is the layout of any desktop with a + # monitor left of the primary. The whole-screen capture then starts at + # a negative coordinate, and a size, a crop or a located hit that + # assumes (0, 0) is wrong by the width of that monitor. + # + # libei: every entry point the ctypes binding names is resolved against + # the real libei.so — a misspelled symbol passes every mock and fails + # only on a user's machine — and the fail-closed chain is driven end to + # end. It also re-checks whether ei_unref still segfaults upstream, so + # the workaround in LibeiBackend._teardown gets removed once it can be. + # + # The container exits with the number of failed checks. + - name: Verify against headless sway and the real libei + run: docker run --rm autocontrol-wayland:ci + + eis-verification: + name: libei sender against a real EIS server + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the EIS verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.eis + tags: autocontrol-eis:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # The other half of the input path, and the half no mock can reach: a + # wrong capability value, a mis-marshalled variadic bind or a wrong + # scroll unit is *accepted* by a fake symbol table and only rejected by + # something that speaks the protocol. libeis is that something — the + # server side of libei's own protocol, packaged by Debian — so + # docker/eis_server.py runs a real EIS implementation on a Unix socket + # and records what AutoControl's real sender does to it. No compositor + # and no desktop session are involved. + # + # It also re-checks whether ei_unref is still safe on a live context, + # which is what lets _teardown release instead of leaking. + # + # The container exits with the number of failed checks. + - name: Verify the libei sender against libeis + run: docker run --rm autocontrol-eis:ci + + portal-verification: + name: RemoteDesktop portal handshake against a real liboeffis + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the portal verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.portal + tags: autocontrol-portal:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # How a client gets an EIS socket on GNOME and KDE: not a path on disk + # but a file descriptor handed over D-Bus at the end of the + # org.freedesktop.portal.RemoteDesktop dance. That was recorded as + # needing a GNOME VM because xdg-desktop-portal-wlr has no RemoteDesktop + # interface — but the portal is a D-Bus interface, not a compositor + # feature, so docker/portal_server.py owns the well-known name on a + # private session bus and answers the four calls for real. + # + # Its ConnectToEIS hands back a live connection to the same real libeis + # server the eis job uses, so the whole chain is checked at once: the + # call order and predicted request paths, the device mask the user would + # be consenting to, that the descriptor carries a real EI session, and + # that input emitted through it is recorded by an independent + # implementation. + # + # And every way a portal says no — a dismissed dialog, a dialog left + # open, a withheld descriptor, a closed session, a portal too old to + # have ConnectToEIS, no portal at all — has to come back as a refusal on + # this project's own clock rather than a hang or a silent downgrade. + # + # The container exits with the number of failed checks. + - name: Verify the portal handshake against liboeffis + run: docker run --rm autocontrol-portal:ci + + seat-verification: + name: ydotool absolute move against a seat that consumes it + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the seat verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.seat + tags: autocontrol-seat:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + - name: Load the uinput and evdev kernel modules + run: | + sudo modprobe uinput + sudo modprobe evdev || true + test -e /dev/uinput || { + echo "::error::/dev/uinput is missing after modprobe; this runner's" + echo "::error::kernel cannot host the seat verification." + exit 1 + } + + # The join between the two images above, and the one every earlier note + # in this file said needed a VM. The wayland job runs a compositor that + # consumes no input; the ydotool job reads ydotool's events off the + # kernel with no compositor. Neither can say where the cursor ends up. + # + # wlroots can: WLR_BACKENDS=headless,libinput keeps the outputs virtual + # while running the real libinput backend, libseat's builtin backend + # opens the device without logind, and SEATD_VTBOUND=0 stops it + # reaching for a VT no container owns. ydotoold's device is then an + # ordinary seat device, and grim -c draws the cursor into a screenshot. + # + # That settles what --absolute is absolute *to* — the top-left of the + # output layout, not layout (0, 0), which is the translation + # linux_wayland/mouse.py now applies — and what pointer acceleration + # does to it, which is double the distance asked for under libinput's + # default profile. It runs over the same two layouts as the wayland + # job, and the negative-origin one is where an untranslated request + # lands on the wrong monitor entirely. + # + # The container exits with the number of failed checks. + - name: Verify the absolute move against a real seat + run: | + docker run --rm --device /dev/uinput --device-cgroup-rule 'c 13:* rmw' autocontrol-seat:ci + + ydotool-verification: + name: ydotool argv against a real uinput device + needs: build-image + runs-on: ubuntu-22.04 + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/setup-buildx-action@v3 # NOSONAR githubactions:S7637 + + - name: Build the ydotool verification image + # nosemgrep: yaml.github-actions.security.third-party-action-not-pinned-to-commit-sha.third-party-action-not-pinned-to-commit-sha + uses: docker/build-push-action@v5 # NOSONAR githubactions:S7637 + with: + context: . + file: docker/Dockerfile.ydotool + tags: autocontrol-ydotool:ci + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + + # uinput is what ydotoold writes into; evdev is the handler that turns + # the device it creates into the /dev/input/eventN this verification + # reads back. Both ship with the runner's kernel as modules. The + # explicit check is here so a kernel without them fails saying so, + # rather than the container reporting an empty device list. + - name: Load the uinput and evdev kernel modules + run: | + sudo modprobe uinput + sudo modprobe evdev || true + test -e /dev/uinput || { + echo "::error::/dev/uinput is missing after modprobe; this runner's" + echo "::error::kernel cannot host the ydotool verification." + exit 1 + } + ls -l /dev/uinput + + # The half neither other image can reach. sway's headless backend + # consumes no libinput devices, so an injected event has nowhere to + # arrive there — but arriving is not what is being checked. ydotoold + # creates an ordinary uinput device and the kernel publishes it as an + # evdev node, so reading that node returns the exact input_event structs + # ydotool wrote, with no compositor and no seat in the picture. + # + # That settles the click bitmasks, the split press / release edges drag + # depends on, what --absolute really puts on the wire, and the wheel + # signs this project had assumed from the kernel's REL_WHEEL convention + # and never measured. The last check drives the backend's own functions + # rather than a hand-written argv, so the two halves meet. + # + # --device covers /dev/uinput, which exists before the container starts. + # The input node does not — ydotoold creates it afterwards — so the + # cgroup rule grants character major 13 and nothing else, which is much + # narrower than --privileged. + # + # The container exits with the number of failed checks. + - name: Verify the ydotool argv against the kernel + run: | + docker run --rm \ + --device /dev/uinput \ + --device-cgroup-rule 'c 13:* rmw' \ + autocontrol-ydotool:ci diff --git a/CHANGELOG.md b/CHANGELOG.md index aa0064d1..5aeea62a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,14 +49,53 @@ only when documented here with a migration path. `ac_foreground_window`, `ac_window_rect`). `list_windows` takes `titled_only`, and `move_window_by_title` keeps the window's current size when width/height are omitted. +- Window ownership: `foreground_window_process_id` and `window_process_id` + (`AC_foreground_window_pid`, `AC_window_pid`; `ac_foreground_window_pid`, + `ac_window_pid`; two Script Builder specs), on the Windows backend + `get_window_process_id`. A title is whatever the application decides to + display, so it cannot answer "which program is the user actually in front + of" — the process id can. Unavailable reads as `None` (`{"pid": 0}` on the + JSON surfaces) rather than a bare `0`, which a caller could otherwise match + against a process list and hit the System Idle Process. +- Windows by owning process: `windows_for_process_id` and + `minimize_windows_for_process` (`AC_windows_for_pid`, + `AC_minimize_windows_for_pid`; `ac_windows_for_pid`, + `ac_minimize_windows_for_pid`; two Script Builder specs). A multi-process + application cannot be addressed by title — its windows are named after + whatever they display and several of its processes have no window at all — + so ownership is the stable key. +- Input posted to a window without focusing it: `post_key_to_window` and + `post_click_to_window` (`AC_post_key_to_window`, `AC_post_click_to_window`; + `ac_post_key_to_window`, `ac_post_click_to_window`; two Script Builder specs), + on the Windows backend `get_focused_control`, `deepest_child_at`, `post_key` + and `post_click`. They resolve the window by title *substring* like every + other function here, and post to the control that actually has keyboard + focus — or, for a click, to the deepest child under the point, in that + child's client coordinates. Posting to the top-level frame (what the older + `send_key_event_to_window` does) types nothing in any application with child + controls: measured on Character Map, the frame swallowed the key while the + focused edit accepted it. Both return whether the messages were queued, and + posting remains best effort — applications reading raw input or checking the + foreground ignore posted messages. - `utils/url_canon` reaches its delivery surfaces: `canonicalize_url`, `normalize_url`, `urls_equal`, `build_query` and `parse_query` are exported from the facade, with `AC_canonicalize_url` / `AC_normalize_url` / `AC_urls_equal`, the matching `ac_*` MCP tools, and three Script Builder specs. The module and its tests already existed; only the wiring is new. +- `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` — how an operator declares what the + library cannot read back. `flat` says pointer acceleration is off for the + ydotoold device, so an absolute move through the ydotool fallback is exact + and needs no warning; `strict` refuses that move instead of letting a click + land somewhere else; unset (or any unrecognised value, which says so and + falls back) keeps the existing warn-once-and-move behaviour. The libei path + is absolute at the protocol level and is not affected either way. ### Removed +- `je_auto_control.linux_wayland._detect.WAYLAND_GDBUS` is gone, along with the + `gdbus` probe it named: the desktop-portal capture tier no longer shells out + to any binary. `_detect` is a private module and nothing else referenced the + constant. - **Breaking — `je_auto_control.windows.listener` is gone**, with its `Win32KeyboardListener` and `Win32MouseListener` classes. Recording moved to `windows/record/win32_input_hook.py`, after which nothing in the package or @@ -71,6 +110,73 @@ only when documented here with a migration path. ### Changed +- **The `xdg-desktop-portal` capture tier no longer needs `gdbus` installed.** + It speaks D-Bus directly, so `linux_wayland.portal.is_available()` now + reports whether a session bus address is set rather than whether the `gdbus` + binary is on `PATH`. This widens where the last-resort tier runs; the install + hint in the "no capture tool found" error and the `screen_capture` + diagnostics check were reworded to match. +- **`LibeiBackend.scroll()` sends whole wheel clicks, not raw detent counts.** + libei measures discrete scroll in 120ths of a click, so the previous call + asked for 1/120th of the scroll requested and libei logged it as a client + bug. Measured against a real EIS server (`docker/eis_verify.py`). +- **Wayland `mouse.scroll()` goes through libei where libei is up, instead of + always shelling out to ydotool.** Motion, buttons and keys already did; + scroll was held back because its sign was a guess. The two paths count + wheel detents in opposite directions — this repository's + `wayland_scroll_direction_*` constants are in the kernel's `REL_WHEEL` + frame, which is what ydotool writes (positive is up), and libei is in the + `wl_pointer` frame (positive is down) — so the vertical axis is negated on + the way to libei and the horizontal one is not. No API change: scrolling on + a libei host no longer needs ydotool or a uinput daemon at all. +- **A libei emission that a live backend refuses now falls back to the CLI, + as `libei`'s own docstring already claimed it did.** Only the *connection* + degraded; a compositor that paused a device, or a session that ended + between two calls, raised out of `set_position` / `press_key` / `hotkey` + instead of reaching ydotool. A chord refused part-way releases the keys it + already pressed before handing over, so no modifier is left held. +- **`LibeiUnavailable` derives from `AutoControlException`** (as well as + `RuntimeError`, which existing probes catch). It was a bare `RuntimeError`, + so it escaped every `except AutoControlException` containment boundary — + the executor, the poll loops, the request handlers and the GUI slots. +- **A libei session that completed its handshake is released instead of + abandoned.** `ei_unref` segfaults on libei 1.3.901 only for a context whose + backend opened and whose handshake never progressed; with an EIS peer to + test against, the live case is measurably safe. Teardown no longer leaks a + context and a file descriptor per process. + +- **`POST /execute` and `POST /execute_file` answer `400`, not `500`, for a + command name the executor does not know.** Both used to funnel every + executor failure into `500 {"error": "execute_action failed"}`, so a client + could not tell a typo in its own request from a broken server. Every name in + the list — nested flow-control bodies included — is now checked before + anything runs; an unrecognised one comes back as + `400 {"error": ..., "unknown_commands": [...]}` naming all of them, and + nothing was executed. `/execute_file` answers the same way for a path that + is unreadable or holds something that is not an action list. A client that + keyed off `500` to detect a bad request must key off `400` instead. +- **Windows clipboard calls wait out a clipboard another process is holding + open** instead of failing immediately. Only one process may have it open at + a time, so `RuntimeError: OpenClipboard failed` used to escape whenever + anything else was mid-copy — roughly one call in a thousand on a live + desktop. `win32_clipboard_api.open_clipboard()` is the one place that opens + it now, retrying for about 200 ms; the failure is still raised after that. + Callers that relied on an immediate failure will see a call take up to + 200 ms longer in the contended case. + +- **`send_key_event_to_window` / `send_mouse_event_to_window` now actually + reach the target.** They posted to the top-level frame, but keyboard messages + go to the control that *has focus* and a click belongs to the child under the + point in that child's client coordinates — so for any window with child + controls they did nothing at all while still reporting success. They delegate + to `post_key_to_window` / `post_click_to_window`. Two visible consequences: + the key sender now matches the window title as a *substring* (it required an + exact title before, via `FindWindowW`), and the mouse sender accepts a title + string as well as the hwnd it always took. +- `save_window_layout` now snapshots only titled windows (its documented + behaviour). Untitled entries could never be restored — `restore_window_layout` + addresses a window by title and skips blank ones — so they only inflated the + saved count, by roughly half on a real desktop. - `set_clipboard_image` accepts PNG bytes **or** a path to any Pillow-readable image, and `get_clipboard_image` / `set_clipboard_image` are now exported from `je_auto_control.utils.clipboard` and the top-level facade, with @@ -131,11 +237,121 @@ only when documented here with a migration path. ### Deprecated +- `send_key_event_to_window` and `send_mouse_event_to_window` — use + `post_key_to_window` / `post_click_to_window`. Both now emit a + `DeprecationWarning` and delegate to the working implementation; see Changed + for the behaviour that changes. + - New integrations should avoid the eager, historical top-level import surface and import stable entry points from `je_auto_control.api`. ### Fixed +- **Wayland: an absolute mouse move through the ydotool fallback counted from + the wrong origin.** `ydotool mousemove --absolute` emits no absolute event — + it drives the cursor into the corner the compositor clamps to and then moves + relative to it, and that corner is the top-left of the output layout rather + than layout `(0, 0)`. On a layout with a monitor left of the primary one the + two differ by the layout origin, so `set_position(x, y)` landed a monitor's + width away from the coordinate the capture path had located. It now + subtracts `layout_origin()`, the same correction `grab_image` applies. + Measured against a real wlroots session consuming the real ydotool device + (`docker/Dockerfile.seat`, the new `seat-verification` job). Layouts whose + outputs all sit at non-negative positions are unaffected. +- **Wayland: the same call is only pixel-accurate where pointer acceleration + is off.** The displacement ydotool sends is relative motion, so the + compositor accelerates it — libinput's default adaptive profile moves the + cursor exactly twice as far as asked. This cannot be corrected from inside + the library, because the factor is the compositor's setting; the backend now + logs the caveat once per process rather than mispositioning in silence. + Disable acceleration for the ydotoold device (sway: `input type:pointer + accel_profile flat` and `pointer_accel 0`), or install `liboeffis` so the + libei path — absolute at the protocol level — is used instead. Once it is + off, `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` silences the warning, and + `=strict` refuses the move rather than warn about it. + +- **The Wayland `xdg-desktop-portal` screen-capture tier could never have + succeeded.** `org.freedesktop.portal.Screenshot` returns a request handle and + delivers the image later as a `Response` signal **directed at the connection + that made the call**; the bus routes a directed message to its destination + and nowhere else. The implementation listened on a `gdbus monitor` + subprocess and called from a separate `gdbus` invocation — two connections, + so the listener was never the addressee. Against a real `dbus-daemon` the + capture ran out its full 30-second timeout every time. The tier now speaks + D-Bus itself on a single connection, subscribing to the request path it + predicts before it calls. No API changed; a path that always failed now + works. +- **On Wayland, a monitor placed left of or above the primary one made every + capture path read the wrong pixels.** The compositor lays its outputs out on + one plane, and that plane starts at a negative coordinate as soon as an + output sits left of (or above) the origin — a `-1280,0` + `0,0` pair is one + 2560x720 layout whose top-left pixel is at x=-1280. Three places assumed the + layout began at `(0, 0)`: `screen.size()` returned `max(x + width)`, the + layout's *right edge* (1280) rather than its width (2560), so everything + that composes size with a capture — the mss-shaped shim's monitor list, + `enumerate_monitors`, the recorder, the WebRTC host, the MCP monitor grab — + asked for half the desktop and called it the whole screen; the region crop + taken when the capture tier cannot apply one itself (gnome-screenshot, + spectacle, the portal, an operator's own command) cropped in layout + coordinates on a layout-origin image, which returns black padding instead of + the left-hand monitor; and `grab_logical` reported an origin of `(0, 0)`, so + a template or OCR match found on that monitor was reported 1280 px to the + right of where it was seen and the click landed on the wrong screen. The + Wayland backend now publishes `layout_origin()`, `size()` returns the + bounding box's size, the crop subtracts the origin, and the generic capture + layer exposes `screen_grabber.backend_layout_origin()` for the paths that + map a pixel back to a screen coordinate. Verified against a real headless + sway session laid out that way — the `wayland-verification` job now runs its + 27 checks over both layouts. + +- **On Wayland, `set_position` could move nothing at all and report success.** + libei accepts absolute motion only inside the regions the compositor + advertises for the pointer, and it discards a point outside every one of + them without a return code, an event or an error — so the move was lost in + silence and never reached the ydotool fallback that could have made it. + Region offsets are `uint32`, so no compositor can advertise a region left of + or above the origin, while the layout space this project addresses starts at + `layout_origin()` and goes negative on the same "monitor left of the primary" + desktop fixed above: on such a layout the input and capture halves named + different pixels, and the pointer went nowhere rather than to the wrong + screen. The libei sender now reads the device's regions, sends a covered + coordinate unchanged, retries an uncovered one normalised by the layout + origin, and refuses what neither covers so `_select_input` hands the move to + ydotool. A device that advertises no region is unaffected. Verified against a + real EIS peer — the `eis-verification` job now runs 20 checks, five of them + on this coordinate space. The ydotool path's own origin remains unverified + and unchanged; see `Progress.md`. + +- **The Wayland ydotool fallback reported success while sending nothing on + Debian and Ubuntu.** ydotool 1.0 replaced its entire command line, and every + argument this backend builds arrived in that release (`mousemove + --absolute`, `mousemove --wheel`, hex `click` bitmasks, `key CODE:STATE`). + Debian bookworm, Ubuntu 22.04 and Ubuntu 24.04 all ship 0.1.8 under the name + `ydotool`, and 0.1.8 exits **0** for those arguments while emitting no + events at all — including for the ones it rejects with `unrecognised + option`. Since the backend runs ydotool with `check=True`, nothing raised: + clicks, keystrokes and cursor moves silently did nothing and every call + reported success. AutoControl now classifies the installed ydotool once per + process and raises `AutoControlException` naming the fix instead of + emitting. **Migration**: install ydotool 1.0+ (Arch, Fedora and Debian + unstable package it; Debian trixie packages none at all), or set + `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` to drive XWayland. A version the + probe does not recognise is allowed through, so this cannot block a future + release. The two `ydotool` install hints no longer suggest `apt install + ydotool`, which is what produced the broken version. + +- **Four clipboard writers never worked on 64-bit Windows**: + `set_clipboard_files`, `set_clipboard_html`, `set_clipboard_rtf` and + `set_clipboard_csv` all raised `OverflowError: int too long to convert` on + every call, and the matching readers failed whenever that format was actually + present. Each module declared `restype` but not `argtypes`, so ctypes passed + the pointer-width memory handle as `c_int`. The prototypes and the + open/alloc/lock dance now live once in + `je_auto_control/utils/clipboard/win32_clipboard_api.py`, on private `WinDLL` + handles so the declarations cannot leak into other user32 callers, and + `rich_clipboard`, `clipboard_rich_formats`, `clipboard_files` and + `clipboard_formats` all go through it. + - `write` failing a whole string on the first character outside the 192-entry virtual-key table — on a US layout that includes `, . / : ? ! _ + @ %` and every CJK character, so URLs and non-English text could not be typed at all. diff --git a/CLAUDE.md b/CLAUDE.md index 23638098..dc90059c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -60,13 +60,20 @@ The map is only useful while it matches the tree, so **update it in the same cha python -c "from je_auto_control.utils.mcp_server.tools import build_default_tool_registry as b; print(len(b()))" # MCP tools ``` - Module and line counts come from walking the tree with `ast`; recompute §1, the affected §5.4 theme totals, and the §8 size appendix together so they stay consistent. + Every line count in the map — §1's totals, the §5.4 theme tables, the §5.4.17 file tables, the `####` headings and the §8 appendix — is rewritten in one pass by: + + ```bash + python test/unit_test/headless/test_doc_line_counts.py --fix # then review the diff + ``` + + Never adjust one by hand: the counts are `len(text.splitlines())` (what `wc -l` reports), and hand-editing is how the document ended up quoting the same subsystem at two sizes at once — most tables had been counting a phantom trailing line per file while §1 and §8 counted correctly. - A new `utils/` subpackage needs a row in **exactly one** §5.4 theme table — the tables partition all 308 subpackages; appearing twice or not at all is a defect. - A new subsystem over ~1,000 lines also needs a file-level table in §5.4.17. - Keep the header's scan date, version, and branch current. - `README.md` and both translations under `README/` cite the same figures (command / subpackage / tab / MCP-tool / example counts) — update all three alongside the map. - **`test/unit_test/headless/test_doc_counts.py` enforces this and fails CI on a mismatch.** It re-measures the command, MCP-tool, `utils/` subpackage and `examples/` counts and compares them against every place the four documents quote them, so code and docs have to move in the same commit. If you reword a sentence that holds one of those numbers, update the test's pattern — it fails loudly when a citation disappears rather than passing on a document it can no longer read. The GUI tab count is guarded the same way but from `test_actions_menu_gui.py`, whose subprocess probe already builds the widget that count needs. +- **`test_doc_line_counts.py` does the same for every line count**, and is also the `--fix` tool above — so any change in module size reddens CI until the map is re-measured, and re-measuring is one command. ### Outstanding work goes in `Progress.md` @@ -94,10 +101,28 @@ Anything agreed but not done — deferred follow-ups, known gaps, half-delivered - **Pin dependency versions**, including transitive ones that can change return shapes (`opencv-python` is bounded `<6` for exactly this reason). Review new dependencies for known vulnerabilities. - Common logic belongs in `wrapper/` or `utils/`, never duplicated across platform backends. -### Limits enforced by CI +### Size and complexity limits Cyclomatic complexity ≤ 10 · cognitive complexity ≤ 15 · function ≤ 75 lines · parameters ≤ 7 · nesting ≤ 4 · file ≤ 750 lines · line ≤ 120 chars · no duplicated block ≥ 10 lines. +**What actually enforces these.** `quality.yml` runs ruff and bandit only, so line length is the only limit a CI job rejects. Complexity is measured by `radon cc -nc` in the pre-commit list below and read by a human. The file-length limit is enforced by nobody — treat this section as a review standard, not a gate, and do not describe it as CI-enforced. + +**Scope of the file-length limit.** It applies to: + +- every **new** file, without exception — if it does not fit in 750 lines it is more than one module; +- any existing file being **substantially rewritten** — reorganising is the moment to split; +- any existing file that would **grow**: an over-limit file may be edited and may shrink, but a change that pushes it further over needs the split to come first, or a `Progress.md` entry saying why not. + +It does not apply to **flat data tables**: a file that is one mapping or list of entries, with no branching logic, where splitting would fragment a single lookup structure. Today that means the `AC_*` dispatch table, the MCP tool registry, the Script Builder command schema, the façade's re-exports, and the per-language string catalogues. The test is structural, not a label — if a file has to be *read* as a table, it is one. Logic that grew alongside a table does not inherit the exemption. + +Everything else currently over the limit is **grandfathered and listed in `Progress.md`**, which is the only place a standing exception may live. An over-limit file with no entry there is a defect, and the list is measured, not remembered: + +```bash +find je_auto_control -name '*.py' -not -path '*__pycache__*' -exec awk 'END{if(NR>750)print NR"\t"FILENAME}' {} \; | sort -rn +``` + +Adding a file to that list is a last resort, not a routine escape. + Docstrings on every public module, class, and function (one-line summary minimum; type hints replace parameter-type prose). Type hints on all public signatures. Import order stdlib → third-party → first-party; no wildcard imports outside the `__init__.py` façade. ### Automated verification diff --git a/Progress.md b/Progress.md index 7088a115..42c030d8 100644 --- a/Progress.md +++ b/Progress.md @@ -14,16 +14,122 @@ --- -## [TODO] `windows_backend.py` 915 行，超過 750 行上限 +## 750 行上限的既有豁免清單 -`je_auto_control/utils/accessibility/backends/windows_backend.py` 目前 915 行，超出 -`CLAUDE.md` §Limits 的 750 行。拆出 `windows_query.py`（170）與 `windows_state.py`（98）之後 -仍然超標——這個檔在拆之前就已經是 772 行，後續補視窗限定搜尋與控制項模式又長回來。 +`CLAUDE.md` §Size and complexity limits 規定:超標檔案只能列在這裡,列不進來的就是缺陷。 +清單上的檔案**可以改、可以變短,但不得再變長**——要再長就得先拆。 +行數為 2026-08-19 實測（`len(text.splitlines())`）。 -- **注意**：目前**沒有任何 CI job 在檢查行數與複雜度**（`quality.yml` 只有 ruff 與 bandit）， - 所以這條上限實際上靠自律；`action_executor.py` 8,021 行、`_factories.py` 8,866 行同樣超標。 -- **待決**：是要真的拆這個檔、把上限改成符合現況的數字，還是把這條規則的適用範圍寫清楚。 +| 檔案 | 行數 | 為何還沒拆 | +| --- | ---: | --- | +| `utils/mcp_server/tools/_handlers.py` | 4,789 | 676 個 MCP 工具的處理函式本體。與 `_factories.py`（表）不同,這裡是邏輯,應該依主題拆成 `_handlers/` 套件（input／screen／window／file／agent…）。拆點清楚,純粹是量大。 | +| `gui/remote_desktop/webrtc_panel.py` | 2,555 | 單一 Qt 面板,但已含連線、監視器選擇、頻寬自適應、麥克風、錄影五組互動狀態。應拆成 panel + 各控制器。 | +| `utils/accessibility/backends/windows_backend.py` | 915 | 已拆出 `windows_query.py`（170）與 `windows_state.py`（98）。剩下的是同一套 UIA COM 生命週期管理,再拆會把 `CoInitialize`／介面釋放的配對邏輯切散。 | + +**本質豁免（依 `CLAUDE.md` 的「flat data tables」條款,不算既有豁免）**: +`utils/mcp_server/tools/_factories.py`（8,968,MCP 工具註冊表）、 +`utils/executor/action_executor.py`（8,125,`AC_*` 分派表）、 +`gui/script_builder/command_schema.py`（5,051,每個 `AC_*` 的參數 schema）、 +`je_auto_control/__init__.py`（1,970,門面 re-export）、 +`gui/language_wrapper/{english,japanese,traditional_chinese,simplified_chinese}.py` +（1,316／1,203／1,189／1,188,語系字串表）。 + +### 2026-08-19 決議:上表的實測行數就是新的上限 + +2026-08-18 重新實測時,表上原有的七列**全部**變長,而 `CLAUDE.md` 明寫 +「列上的檔案不得再變長,要再長就得先拆」,所以這裡曾標成 `[DECIDE]`。 +**維護者已於 2026-08-19 拍板:接受實測數字當新基準**——不為了回到舊數字而去拆 +`_handlers.py`（4,789）與 `webrtc_panel.py`（2,555）。上表的行數即是各自的新上限, +規則不變:只准變短,再變長就得先拆。 + +同一批裡有六個檔案在 2026-08-19 已經拆回線內、從表上移除,做法寫在 +[WHATS_NEW.md](WHATS_NEW.md)。 + +行數沒有任何 CI 在把關（`quality.yml` 只跑 ruff 與 bandit,而 ruff 只管行寬), +所以這張表只會在有人手動實測時才會被發現對不上——上次就是。 --- -（目前沒有其他待辦。） +## Wayland:剩下的都不是「缺一台機器」 + +這一項曾經三度寫成「要一台 VM」——先是 portal 交握,再是 ydotool 的絕對移動落點, +中間還有負原點的擷取。三次都不是,三次都是同一個誤判:**把「合成器／桌面做不到的事」 +當成了「容器做不到的事」**。portal 是 D-Bus 介面,誰佔住那個名字誰就是 portal; +「會吃 libinput 裝置的 seat」是 wlroots 的 `WLR_BACKENDS=headless,libinput` 加 +`LIBSEAT_BACKEND=builtin` 加 `SEATD_VTBOUND=0`（第四個條件是 udev 要比 ydotoold 早起 +來)。都已經是 CI job 了,見下面「已經有答案的」與 [WHATS_NEW.md](WHATS_NEW.md)。 + +**下次要往這裡加「需要一台 VM／真桌面」之前,先問這件事到底是誰做不到。** + +### 還沒有答案的 + +- **`eis_device_pause()` 在 libeis 1.3.901 對 sender client 沒有送出任何東西。** + 對兩個 live device 呼叫 pause 再 dispatch,client 端的 ei fd 4 秒內完全沒有可讀資料。 + 所以 `LibeiBackend._on_event` 的 `DEVICE_PAUSED`／`DEVICE_REMOVED` 分支**仍然沒有 + peer 可以驅動**。`eis_verify.py` 把這件事寫成「要嘛有反應,要嘛根本沒被通知」, + 若哪天 libeis 開始送了,client 忽略它就會當場失敗。真的合成器上會不會不一樣,未知。 +- **`ei_device_start_emulating()` 的 sequence number 沒有被送到對面。** + 刻意送 4242 過去,server 讀回來是 0。我們這邊的計數本身符合標頭檔的約定 + （每次呼叫至少 +1）,所以不影響正確性,只是從對面驗不到。 +- **同意對話框「長什麼樣子、真人要按多久」。** portal 這一層現在驗到的是對話框 + *產生的東西*:准（Response 0）、拒（Response 1）、以及一直不回答。三種我們都在真的 + bus 上跑過,三種都得在自己的時限內收斂。至於真的 mutter 對話框長什麼樣、真人猶豫 + 三十秒會不會撞到別的東西,那是 mutter 的事,CI 裡沒有人可以去按它。 + +### 已經有答案的（都在 CI 裡,做法見 WHATS_NEW） + +五個 job 都在 GitHub runner 上跑過了（2026-08-19,PR #481）。`modprobe uinput evdev` +在 runner 上載得起來,`systemd-udevd` 在容器裡也收得到 kernel uevent——這兩件事原本 +只在本機（Docker Desktop 的 WSL2 kernel）驗過,曾經記在上面當待辦,現在有答案了。 +job 一律寫成模組載不起來就明講失敗,不會靜默跳過,所以哪天 runner 的 kernel 變了會 +當場紅掉。 + +| 面向 | 怎麼驗的 | job | +| --- | --- | --- | +| 擷取路徑 | 真的 wlroots 合成器（sway headless,兩個上不同純色的 output）,27 項 × 2 種版面 | `wayland-verification` | +| libei 協定層 | 真的 `libeis.so.1` server 在 Unix socket 上,20 項 | `eis-verification` | +| RemoteDesktop portal 交握 | 真的 `dbus-daemon` + 真的 `liboeffis`,對面是自己實作的 portal,`ConnectToEIS` 交出通往真 libeis 的活 fd,20 項 | `portal-verification` | +| ydotool CLI | 真的 uinput 裝置,直接讀回 `/dev/input/eventN`,12 項 | `ydotool-verification` | +| ydotool 的絕對移動落在哪 | 真的 wlroots session 吃真的 ydotool 裝置（`headless,libinput` + builtin seat）,游標位置從 `grim -c` 的像素讀回,14 項 × 2 種版面 | `seat-verification` | + +擷取那一列的第二種版面是**負原點**:`output HEADLESS-1 position -1280 0`, +也就是「第二台螢幕在主螢幕左邊」的桌面。sway headless 收這個座標,grim 也收負的 +`-g`,所以這件事根本不必等 GNOME VM——原本記在這裡說測不到,是把「合成器做得到的事」 +當成了「容器做不到的事」。跑起來當場抓到三個真的錯:`size()` 回的是版面右緣不是寬度、 +非 grim 層級的裁切用版面座標去裁一張以版面原點為 (0,0) 的圖、`grab_logical()` 一律回 +原點 (0,0) 所以比對到的座標整個偏掉。修法見 [WHATS_NEW.md](WHATS_NEW.md)。 + +portal 那一列是同一個錯誤犯第二次的結果,而它抓到的東西比前一次更嚴重: +`portal.py` 那條「先開 `gdbus monitor`、再用 `gdbus call` 發請求」的路 +**在任何真的 bus 上都不可能成功**——portal 的 `Response` 是**指名送給發出呼叫的那條 +連線**,兩個 gdbus 行程是兩條連線,監聽的那條永遠不是收件人。在真的 `dbus-daemon` 上 +量到的就是這樣:呼叫看得到,回答永遠等不到,每次都走到 30 秒逾時。修法見 +[WHATS_NEW.md](WHATS_NEW.md)。 + +五者都不需要合成器以外的東西,更不需要 GNOME VM。libei 這一層驗掉的包含 +capability enum 值與 variadic `ei_seat_bind_capabilities`、event-type enum 值、 +`start_emulating` → 事件 → `frame` 的實際上線內容、live context 的 teardown +安全性（原本每個行程漏一個 context + 一個 fd,已修）、以及絕對指標的座標空間 +（region offset 讀得回來且含在座標裡、region 外的移動被靜靜丟掉、負原點的版面要 +正規化）。portal 這一層驗掉的是四個呼叫的順序與 client 自己預測的 request path、 +`SelectDevices` 收到的裝置遮罩（也就是使用者被要求同意的範圍）、交回來的 fd 真的 +承載得起一個 EI session,以及六種拒絕路徑各自都要 fail closed。ydotool 這一層驗掉的是 +`click` 位元遮罩、拆邊的 press／release、`mousemove --absolute` 的實際上線內容、 +捲動正負號與軸向,以及 `mouse`／`keyboard` 自己組出來的 argv。seat 這一層驗掉的是 +`--absolute` 到底相對於哪裡（版面左上角,不是版面座標的 `(0, 0)`)、關掉加速度後 +一像素對一像素、沒轉換的 `(0, 0)` 會打到隔壁螢幕、`set_position` 減掉的正好是原點、 +以及預設 profile 下的 2 倍加速。 + +### 一件關於發行版的事實,會影響使用者拿到什麼 + +- **`liboeffis` 是獨立的二進位套件,`libei1` 不會把它帶進來。** Debian trixie + **有** `liboeffis1`（1.3.901-1,`liboeffis.so.1`,連 libsystemd 的 sd-bus）—— + 這裡原本寫「Debian trixie 沒有」,是錯的,已實測更正。Arch（1.6.0）與 Fedora 也有。 + 但因為它不是 `libei1` 的相依,只裝 libei 的機器上 portal 快速路徑仍然是關閉的, + `connect()` 會退到 `$XDG_RUNTIME_DIR/eis-0` socket,GNOME／KDE 不開那個 socket + → 退回 ydotool。**所以要用 libei 快速路徑,`liboeffis` 得自己裝。** +- 而那條退路本身,在同一批發行版上原本是壞的——0.1.x 對本專案送的 argv 回傳 0 + 卻不送任何事件。已於 2026-08-19 擋掉,見 CHANGELOG 與 WHATS_NEW;此處無待辦。 + +**緩解**:驗不到的擷取部分有逃生門——`JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` 讓操作者 +直接指定自己的擷取指令（`{output}` 會被換成暫存 PNG 路徑）,優先於所有偵測。 diff --git a/README.md b/README.md index 676e2b75..18066393 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ from JSON files / CLI / servers, and a **GUI tab**. Nothing is GUI-only. - **One API, six platforms.** `wrapper/platform_wrapper.py` picks the backend at import time; your script does not change between Windows, macOS, X11, and Wayland. -- **Scriptable without Python.** 767 `AC_*` commands cover the whole feature set, so a +- **Scriptable without Python.** 773 `AC_*` commands cover the whole feature set, so a JSON file can do anything the library can — including loops, branches, try/catch, macros, and variables. - **Headless by default.** `import je_auto_control` never loads Qt. The GUI is an @@ -134,7 +134,7 @@ desktop app; tab commands live in the window's **Actions** menu. | Natural-language planner | `plan_actions`, `run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`, `run_agent` | `AC_run_agent` | Computer Use | | Record & replay | `record`, `stop_record` | `AC_record`, `AC_stop_record` | Record | -| JSON scripting | `execute_action`, `execute_files` | all 767 commands | Script, Script Builder | +| JSON scripting | `execute_action`, `execute_files` | all 773 commands | Script, Script Builder | | Variables & flow control | `execute_action_with_vars` | `AC_set_var`, `AC_loop`, `AC_for_each`, `AC_try`, `AC_retry` | Variables | | Data-driven runs | — | `AC_for_each_row` (CSV / JSON / SQLite / Excel) | Data Sources | | Assertions | `assert_text`, `assert_image` | `AC_assert_text` + 20 more | Assertions | @@ -185,7 +185,7 @@ entry point still works. | Surface | Start it with | Notes | |---|---|---| -| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 670 tools for Claude Desktop / Claude Code / custom tool loops. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | +| **MCP server** | `je_auto_control_mcp` (stdio) or `AC_start_mcp_http_server` | 676 tools for Claude Desktop / Claude Code / custom tool loops. Bearer auth, TLS, audit log, rate limit, plugin hot-reload, CI fake backend. | | **REST API** | `je_auto_control start-rest` | Bearer token, per-IP rate limit + lockout, SQLite audit hook, `/metrics`, `/openapi.json`, `/docs` Swagger UI, `/dashboard`. | | **TCP socket server** | `je_auto_control start-server` | Newline-framed JSON action lists. Binds `127.0.0.1` by default. | | **pytest plugin** | installed automatically | Fixtures plus a Gherkin step library for pytest-bdd / behave. | @@ -241,10 +241,63 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Windows 10 / 11 | Win32 ctypes (+ optional Interception driver) | ✅ | ✅ | ✅ | ✅ | | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | | Linux X11 | python-Xlib (+ optional `uinput`) | ✅ | ✅ | ✅ | ❌ | -| Linux Wayland | libei, or ydotool / wtype / grim | ✅ | ✅ | ❌ | ❌ | +| Linux Wayland | libei via the desktop portal, or ydotool / wtype + a capture tool | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +Wayland input falls back to the `ydotool` CLI wherever libei is not +reachable, and that fallback needs **ydotool 1.0 or newer**. Every argument +AutoControl builds arrived in that release; 0.1.x — which is what Debian +bookworm and every current Ubuntu still ship under that name, and Debian +trixie ships not at all — answers the same arguments with exit code 0 and no +events. AutoControl detects it and refuses rather than reporting success for +input it never sent. Arch, Fedora and Debian unstable package 1.0. + +That fallback also positions the pointer accurately **only where the +compositor's pointer acceleration is off**. `ydotool mousemove --absolute` +sends no absolute event: it drives the cursor into the corner the compositor +clamps to and then moves relative to it, so the compositor accelerates the +move — measured against a real wlroots session, libinput's default profile +travels exactly twice the distance asked for. ydotool's own `--help` says the +same; AutoControl logs it once per process rather than mispositioning in +silence. Turn acceleration off for the ydotoold device (sway: `input +type:pointer accel_profile flat` and `pointer_accel 0`), or install +`liboeffis` so the libei path — absolute at the protocol level — is used. + +The factor is the compositor's own setting and no client can read it back, so +only you know whether it is off. `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` +says it is, and moves silently; `=strict` refuses the move rather than let a +click land somewhere else; leaving it unset keeps the warn-and-move default. + +Wayland screen capture needs the tool your compositor supports, because no +single one covers them all: `grim` on wlroots compositors (sway, Hyprland, +river), `gnome-screenshot` on GNOME, `spectacle` on KDE. Install one and every +capture path — screenshots, image and anchor locators, OCR, screen recording, +remote desktop — goes through it. With none of them installed, `gdbus` is +enough: `xdg-desktop-portal` is tried last, though it may ask for consent the +first time. Failing that, capture fails loudly with an install hint rather than +returning the blank XWayland root. The `screen_capture` check in +`je_auto_control.api.run_diagnostics()` (and the GUI Diagnostics tab) reports +which tier is in use. + +One Wayland-only difference to plan around: **a capture may contain the mouse +cursor.** Nothing here asks for it, but wlroots composites a *software* cursor +into the output buffer whenever the backend has no cursor plane — which +includes any session run with `WLR_NO_HARDWARE_CURSORS=1` — and that buffer is +what screen capture hands back. Windows and X11 never include the pointer, so a +locator, a template match or an OCR read can find a pointer-shaped hole in the +middle of its target here and nowhere else. Wayland does not let a client read +the cursor position, so there is nothing to reliably mask or move around it: +park the pointer away from what you are about to capture. The `screen_capture` +check reports this as `cursor_may_be_captured`. + +For a setup none of that fits, name your own command — it wins over every +detected tool, and `{output}` is replaced with a temporary PNG path: + +```bash +export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" +``` + Wayland forbids global input recording for unprivileged clients — set `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` to record on an X11 session. Window management is currently Windows-only and raises a clear `NotImplementedError` diff --git a/README/README_zh-CN.md b/README/README_zh-CN.md index d23c68ca..48447f7b 100644 --- a/README/README_zh-CN.md +++ b/README/README_zh-CN.md @@ -20,7 +20,7 @@ - **一套 API，六个平台。** `wrapper/platform_wrapper.py` 在导入时挑选后端；同一份脚本在 Windows、macOS、X11 与 Wayland 上都不需要改写。 -- **不写 Python 也能脚本化。** 767 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 +- **不写 Python 也能脚本化。** 773 个 `AC_*` 命令覆盖全部功能，因此一个 JSON 文件能做到库 能做的任何事——包含循环、分支、try/catch、宏与变量。 - **默认无头运行。** `import je_auto_control` 绝不会加载 Qt。GUI 是可选包，包在同一个无头内核之外。 - **四种定位方式。** 模板匹配、OCR、无障碍树、视觉语言模型——可通过锚点定位器与自愈回退串接组合。 @@ -128,7 +128,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 自然语言规划 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 录制与回放 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 脚本 | `execute_action`、`execute_files` | 全部 767 个命令 | Script、Script Builder | +| JSON 脚本 | `execute_action`、`execute_files` | 全部 773 个命令 | Script、Script Builder | | 变量与流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 数据驱动执行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 断言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 个 | Assertions | @@ -177,7 +177,7 @@ je_auto_control version | 接口 | 启动方式 | 说明 | |---|---|---| -| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 670 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | +| **MCP 服务器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 676 个工具，供 Claude Desktop／Claude Code／自定义 tool loop 使用。Bearer 认证、TLS、审计日志、限流、插件热重载、CI 假后端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、按 IP 限流与锁定、SQLite 审计 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 服务器** | `je_auto_control start-server` | 以换行分隔的 JSON 动作列表。默认绑定 `127.0.0.1`。 | | **pytest 插件** | 安装后自动生效 | 提供 fixture 与供 pytest-bdd／behave 使用的 Gherkin step library。 | @@ -230,10 +230,52 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Windows 10 / 11 | Win32 ctypes（可选 Interception 驱动） | ✅ | ✅ | ✅ | ✅ | | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | | Linux X11 | python-Xlib（可选 `uinput`） | ✅ | ✅ | ✅ | ❌ | -| Linux Wayland | libei，或 ydotool／wtype／grim | ✅ | ✅ | ❌ | ❌ | +| Linux Wayland | 经桌面 portal 的 libei，或 ydotool／wtype ＋ 截图工具 | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +Wayland 的输入在 libei 走不通时会退回 `ydotool` CLI，而这条退路需要 +**ydotool 1.0 以上**。AutoControl 送的每一个参数都是那一版才有的；0.1.x +（Debian bookworm 与目前所有 Ubuntu 仍以这个名字提供，Debian trixie 则根本没有） +对同一批参数返回 0 却不送出任何事件。AutoControl 会检测并直接拒绝， +而不是为根本没送出的输入回报成功。Arch、Fedora 与 Debian unstable 提供的是 1.0。 + +这条退路要能**准确定位**，还有一个前提:合成器的指针加速度必须是关的。 +`ydotool mousemove --absolute` 并不发任何绝对事件——它先把光标推到合成器夹取的 +那个角落，再发相对位移，所以这段位移会被合成器加速。对真的 wlroots session 量到的是: +libinput 的默认 profile 让光标走的距离正好是请求的两倍。ydotool 自己的 `--help` +也是这样写的；AutoControl 每个进程会记一次警告，而不是安静地把点击放到错的地方。 +请对 ydotoold 的设备关掉加速度（sway:`input type:pointer accel_profile flat` +加上 `pointer_accel 0`），或者装上 `liboeffis`，改走协议层本来就是绝对坐标的 libei。 + +倍率是合成器自己的设置，客户端读不回来，所以只有你知道它关了没有： +`JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示已经关掉，移动就不再出声； +`=strict` 则宁可拒绝这次移动，也不让点击落在别的地方；不设置就维持 +“警告一次后照样移动”的默认。 + +Wayland 的屏幕截取需要合成器对应的工具，因为没有单一工具能覆盖全部：wlroots 系 +（sway、Hyprland、river）用 `grim`，GNOME 用 `gnome-screenshot`，KDE 用 `spectacle`。 +装好其中一个之后，所有截取路径——截图、图像与锚点定位、OCR、屏幕录制、远程桌面——都会 +经由它。三个都没装也还有 `gdbus`：最后会尝试 `xdg-desktop-portal`，只是第一次可能会弹 +同意对话框。再不行，截取会带着安装提示明确失败，而不是返回空白的 XWayland root； +`je_auto_control.api.run_diagnostics()`（以及 GUI 的 Diagnostics 标签页）的 `screen_capture` +检查会报告当前使用的是哪一层。 + +有一件只在 Wayland 出现、需要事先规划的事：**截取回来的图里可能有鼠标光标。** +这里没有任何一条截取请求光标，但只要 backend 没有光标平面（包含任何以 +`WLR_NO_HARDWARE_CURSORS=1` 运行的 session），wlroots 就会画**软件光标**并把它 +合成进输出缓冲区，而截取交回来的正是那一份。Windows 与 X11 都不含光标，所以 +“定位器、模板匹配或 OCR 在目标中间看到一个光标形状的洞”只会在这里发生。 +Wayland 不让客户端读光标位置，所以没有东西可以可靠地遮或避开：请在截取之前 +把指针移离要拍的区域。`screen_capture` 检查会以 `cursor_may_be_captured` 报告这件事。 + +如果以上都不适用你的环境，可以直接指定自己的命令——它优先于所有检测，`{output}` 会被 +换成临时 PNG 路径： + +```bash +export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" +``` + Wayland 禁止非特权客户端进行全局输入录制——若要录制，请设置 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 并在 X11 会话下运行。窗口管理目前仅 Windows 有实现，其他平台会抛出明确的 `NotImplementedError`。对于会忽略合成输入的应用， diff --git a/README/README_zh-TW.md b/README/README_zh-TW.md index d75feed8..b9f04444 100644 --- a/README/README_zh-TW.md +++ b/README/README_zh-TW.md @@ -20,7 +20,7 @@ - **一套 API，六個平台。** `wrapper/platform_wrapper.py` 在匯入時挑選後端；同一份腳本在 Windows、macOS、X11 與 Wayland 上都不需要改寫。 -- **不寫 Python 也能腳本化。** 767 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 +- **不寫 Python 也能腳本化。** 773 個 `AC_*` 指令涵蓋全部功能，因此一個 JSON 檔能做到函式庫 能做的任何事——包含迴圈、分支、try/catch、巨集與變數。 - **預設無頭執行。** `import je_auto_control` 絕不會載入 Qt。GUI 是選用套件，包在同一個無頭核心之外。 - **四種定位方式。** 樣板比對、OCR、無障礙樹、視覺語言模型——可透過錨點定位器與自癒後備串接組合。 @@ -128,7 +128,7 @@ python -m je_auto_control # 或：je_auto_control.start_autocontrol_gui | 自然語言規劃 | `plan_actions`、`run_from_description` | `AC_llm_plan` | LLM Planner | | Computer-use agent | `AgentLoop`、`run_agent` | `AC_run_agent` | Computer Use | | 錄製與重播 | `record`、`stop_record` | `AC_record`、`AC_stop_record` | Record | -| JSON 腳本 | `execute_action`、`execute_files` | 全部 767 個指令 | Script、Script Builder | +| JSON 腳本 | `execute_action`、`execute_files` | 全部 773 個指令 | Script、Script Builder | | 變數與流程控制 | `execute_action_with_vars` | `AC_set_var`、`AC_loop`、`AC_for_each`、`AC_try`、`AC_retry` | Variables | | 資料驅動執行 | — | `AC_for_each_row`（CSV／JSON／SQLite／Excel） | Data Sources | | 斷言 | `assert_text`、`assert_image` | `AC_assert_text` 等 21 個 | Assertions | @@ -177,7 +177,7 @@ je_auto_control version | 介面 | 啟動方式 | 說明 | |---|---|---| -| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 670 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | +| **MCP 伺服器** | `je_auto_control_mcp`（stdio）或 `AC_start_mcp_http_server` | 676 個工具，供 Claude Desktop／Claude Code／自訂 tool loop 使用。Bearer 驗證、TLS、稽核記錄、限流、外掛熱重載、CI 假後端。 | | **REST API** | `je_auto_control start-rest` | Bearer token、逐 IP 限流與鎖定、SQLite 稽核 hook、`/metrics`、`/openapi.json`、`/docs` Swagger UI、`/dashboard`。 | | **TCP socket 伺服器** | `je_auto_control start-server` | 以換行分隔的 JSON 動作清單。預設綁 `127.0.0.1`。 | | **pytest 外掛** | 安裝後自動生效 | 提供 fixture 與供 pytest-bdd／behave 使用的 Gherkin step library。 | @@ -231,10 +231,52 @@ RemoteDesktopHost(token="tok", ip_allowlist=["10.0.0.0/8", "192.168.1.100"]) | Windows 10 / 11 | Win32 ctypes（可選 Interception 驅動） | ✅ | ✅ | ✅ | ✅ | | macOS 10.15+ | pyobjc / Quartz | ✅ | ✅ | ❌ | ❌ | | Linux X11 | python-Xlib（可選 `uinput`） | ✅ | ✅ | ✅ | ❌ | -| Linux Wayland | libei，或 ydotool／wtype／grim | ✅ | ✅ | ❌ | ❌ | +| Linux Wayland | 經桌面 portal 的 libei，或 ydotool／wtype ＋ 擷取工具 | ✅ | ✅ | ❌ | ❌ | | Android | adb + uiautomator2 | ✅ | ✅ | — | — | | iOS | WebDriverAgent / facebook-wda | ✅ | ✅ | — | — | +Wayland 的輸入在 libei 走不通時會退回 `ydotool` CLI，而這條退路需要 +**ydotool 1.0 以上**。AutoControl 送的每一個參數都是那一版才有的；0.1.x +（Debian bookworm 與目前所有 Ubuntu 仍以這個名字提供，Debian trixie 則根本沒有） +對同一批參數回傳 0 卻不送出任何事件。AutoControl 會偵測並直接拒絕， +而不是為根本沒送出的輸入回報成功。Arch、Fedora 與 Debian unstable 提供的是 1.0。 + +這條退路要能**準確定位**，還有一個前提:合成器的指標加速度必須是關的。 +`ydotool mousemove --absolute` 並不送任何絕對事件——它先把游標推到合成器夾取的 +那個角落，再送相對位移，所以這段位移會被合成器加速。對真的 wlroots session 量到的是: +libinput 的預設 profile 讓游標走的距離正好是要求的兩倍。ydotool 自己的 `--help` +也是這樣寫的；AutoControl 每個行程會記一次警告，而不是安靜地把點擊放到錯的地方。 +請對 ydotoold 的裝置關掉加速度（sway:`input type:pointer accel_profile flat` +加上 `pointer_accel 0`），或是裝上 `liboeffis`，改走協定層本來就是絕對座標的 libei。 + +倍率是合成器自己的設定，用戶端讀不回來，所以只有你知道它關了沒有： +`JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示已經關掉，移動就不再出聲； +`=strict` 則寧可拒絕這次移動，也不讓點擊落在別的地方；不設定就維持 +「警告一次後照樣移動」的預設。 + +Wayland 的螢幕擷取需要合成器對應的工具，因為沒有單一工具能涵蓋全部：wlroots 系 +（sway、Hyprland、river）用 `grim`，GNOME 用 `gnome-screenshot`，KDE 用 `spectacle`。 +裝好其中一個之後，所有擷取路徑——截圖、影像與錨點定位、OCR、螢幕錄影、遠端桌面——都會 +經由它。三個都沒裝也還有 `gdbus`：最後會嘗試 `xdg-desktop-portal`，只是第一次可能會跳 +同意對話框。再不行，擷取會帶著安裝提示明確失敗，而不是回傳空白的 XWayland root； +`je_auto_control.api.run_diagnostics()`（以及 GUI 的 Diagnostics 分頁）的 `screen_capture` +檢查會回報目前使用哪一層。 + +有一件只在 Wayland 出現、需要事先規劃的事：**擷取回來的圖裡可能有滑鼠游標。** +這裡沒有任何一條擷取要求游標，但只要 backend 沒有游標平面（包含任何以 +`WLR_NO_HARDWARE_CURSORS=1` 執行的 session），wlroots 就會畫**軟體游標**並把它 +合成進輸出緩衝區，而擷取交回來的正是那一份。Windows 與 X11 都不含游標，所以 +「定位器、樣板比對或 OCR 在目標中間看到一個游標形狀的洞」只會在這裡發生。 +Wayland 不讓用戶端讀游標位置，所以沒有東西可以可靠地遮或閃避：請在擷取之前 +把指標移離要拍的區域。`screen_capture` 檢查會以 `cursor_may_be_captured` 回報這件事。 + +如果以上都不適用你的環境，可以直接指定自己的指令——它優先於所有偵測，`{output}` 會被 +換成暫存 PNG 路徑： + +```bash +export JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycapture --png {output}" +``` + Wayland 禁止非特權用戶端進行全域輸入錄製——若要錄製，請設定 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11` 並在 X11 session 下執行。視窗管理目前僅 Windows 有實作，其他平台會拋出明確的 `NotImplementedError`。對於會忽略合成輸入的應用程式， diff --git a/README/WHATS_NEW_zh-CN.md b/README/WHATS_NEW_zh-CN.md index ba7e4e4b..b42b6033 100644 --- a/README/WHATS_NEW_zh-CN.md +++ b/README/WHATS_NEW_zh-CN.md @@ -1,5 +1,490 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-19) — Wayland 两个等人拍板的取舍,拍板了 + +`Progress.md` 上挂着两个 `DECIDE`:缺的不是活，是决定。两件事其实是同一个问题犯两次 +——一个合成器的设置,库**量得到**却**读不回来**,所以它必须停止假装自己知道。 + +- **指针加速度改成由操作者声明,库照信。** 量到的事实不变: + `ydotool mousemove --absolute` 发的是相对移动,libinput 默认的 adaptive profile 会 + 把它加成两倍,而倍率没有任何客户端读得回来。没拍板的是「那要怎么办」——继续警告后 + 照发、直接拒绝、还是让操作者自己讲。直接拒绝会让每一台没有 `liboeffis` 的 Wayland + 机器整条 `set_position` 不能用,所以答案是声明: + `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示 ydotoold 的设备已经关掉加速度, + 移动就静静地、准确地发出去;`=strict` 表示宁可拒绝这次移动,也不让点击落在别的地方; + 不设置就维持现在的「警告一次后照样移动」,所以今天能跑的东西明天照样能跑。 + 值打错会退回警告模式,**而且会讲出来**——shell profile 里的一个错字,不可以静静地 + 把一次移动升级成「可信赖的精确」。整个判断只挂在 ydotool 这条路上;libei 在协议层 + 本来就是绝对坐标。 +- **Wayland 截取里的软件光标,写进文档,不绕过。** 这是 `seat-verification` 顺手量到 + 的:项目里没有任何一条截取带 `-c`,也就是没有人请求光标,而光标还是在图里。原因不在 + 我们这边——只要 backend 没有光标平面,wlroots 就会画**软件光标**,而软件光标是合成进 + 输出缓冲区的,`wlr-screencopy` 交出来的正是那一份。headless 永远处在这个状态; + 真桌面上只要跑 `WLR_NO_HARDWARE_CURSORS=1` 也一样。Windows 的 BitBlt 与 X11 那条路 + 都不含光标,所以这是**只在 Wayland 出现的不一致**;指针压在目标上的时候,定位器、 + 模板匹配与 OCR 看到的就是目标中间有一个光标形状的洞。两条绕法——截取前把指针挪开再 + 挪回来、或匹配时把指针周围遮掉——都得先知道指针在哪,而 Wayland 不让客户端读光标 + 位置;靠进程内的记录去猜,用户一动实体鼠标就过期,而遮错位置比看得见光标更糟。 + 所以改成写下来:写进 capability matrix、三份 README,以及诊断包——`screen_capture` + 检查现在会带 `cursor_may_be_captured`,让那份「解释定位器为何失败」的报告自己说出 + 原因。检查本身照量到的样子断言,哪天 wlroots 开始尊重 `overlay_cursor`,CI 会当场 + 红掉来通知我们。 + +## 本次更新 (2026-08-19) — 「会吃 libinput 设备的合成器」只差三个环境变量 + +`ydotool mousemove --absolute` 不是绝对移动。它不发任何 ABS 事件:先在两轴各发一次 +`INT32_MIN` 的相对移动,把光标推到合成器夹取的那个角落,再把目标当成相对位移发出去。 +**那个角落到底在哪**、以及**位移在路上被合成器做了什么**,是 Wayland 输入路径最后 +两个没有答案的问题——而且两个都被记成「需要一台跑真桌面、会吃 libinput 设备的 VM」。 + +- **两个都不需要 VM。** wlroots 吃 `WLR_BACKENDS=headless,libinput`:输出保持虚拟, + 输入那一半是真的 libinput backend。libseat 的 builtin backend 不靠 logind 就能打开 + 设备,而 `SEATD_VTBOUND=0` 让它不要去抢一个容器根本没有的 VT。第四个条件最容易漏: + libinput 是通过 udev 枚举设备,不是通过 `/dev`,所以 `systemd-udevd` 必须在 ydotoold + 创建设备**之前**就已经在跑。四个都到位之后,ydotoold 的 uinput 设备就是一个普通的 + seat 设备,`grim -c` 会把光标合成进截图里,合成器就能用布局坐标回答问题。 +- **那个角落是布局的左上角,不是布局坐标的 `(0, 0)`。** 只有在所有输出都在非负位置时 + 这两点才是同一点。任何「主屏左边还有一台显示器」的桌面,两者就差一个布局原点——所以 + 在原点为 `-1280` 的布局上,没有转换的「布局 `(0, 0)`」会把光标送到**隔壁那台屏幕**, + 差 1,280 像素。`mouse.set_position` 现在会先减掉 `layout_origin()` 再交给 ydotool, + 这正是抓图路径早就在做的同一个修正;两条输入路径共用的那个查询搬进了 + `linux_wayland/_layout.py`,libei 与 ydotool 不会再对「原点是什么」各说各话。 +- **剩下的距离则被指针加速度放大。** 发出去的位移是相对移动,所以 libinput 会加速它: + 对着真的 wlroots session 量到的是,默认的 adaptive profile 让光标离那个角落的距离 + 正好是请求的**两倍**——因为 `--absolute` 的两个事件发在同一个 frame 里,速度直接把 + profile 打到上限。把 `accel_profile flat` 加上 `pointer_accel 0` 之后,同一个调用 + 就精确到像素。ydotool 自己的 `--help` 一直都写着「You need to disable mouse speed + acceleration for correct absolute movement」;后端现在会每个进程记一次这个警告,而不是 + 让点击安静地落在错的地方。库这一侧再多做不了什么——那个倍率是合成器的设置, + 调用端读不回来。 +- **新的 `seat-verification` job 把这些全部钉住。** `docker/Dockerfile.seat` 跑的是 + 抓图镜像的同两种布局,每种 14 项:sway 真的握着 ydotool 设备、`--absolute (0, 0)` + 会把光标画在布局的第一个像素上、关掉加速度后移动是一像素对一像素、没转换的 + `(0, 0)` 会打不中它指名的那台屏幕、`set_position` 减掉的正好是原点而且在两台屏幕上 + 都落在指定的像素、以及加速倍率就是量到的 2 倍。里面没有任何一项依赖光标主题:每个 + 主张都是两张截图之间的差,光标图片相对于热点的偏移会自己抵消掉。 +- **顺手抓到的一件事。** 在这台合成器上,`grim` 明明没有要求叠上光标,截回来的图里 + 还是有光标——因为只要 backend 没有光标平面,wlroots 就会画**软件光标**(headless + 永远如此,任何驱动不给平面、或用户设了 `WLR_NO_HARDWARE_CURSORS=1` 的 session + 也是)。所有定位器、模板匹配与 OCR 都走同一条抓图,所以指针会在它压着的东西上挖出 + 一个指针形状的洞。该检查照量到的样子记下这个行为,要怎么处理则列进 `Progress.md`。 + +## 本次更新 (2026-08-19) — 抓图用的 portal 本来就不可能成功,真的 bus 讲出来了 + +- **`xdg-desktop-portal` 的回答是「指名发给发出调用的那条连接」的信号。** + `Screenshot` 返回的是一个 *request handle*,不是图;图稍后才以 + `org.freedesktop.portal.Request::Response` 送达,收件人是调用者的 unique bus name。 + bus 对指名消息只发给收件人,所以别的连接再怎么加 match rule 也收不到。 +- **旧的做法是两条连接。** 先起一个 `gdbus monitor` 子进程,再用第二次 `gdbus` + 调用发请求,然后拿两条正则去读 monitor 的 stdout。每次 `gdbus` 都会用自己的 + unique name 开一条自己的连接,所以在听的那个进程从来就不是收件的那个。在真的 + `dbus-daemon` 上量到的:monitor 看得到调用经过,之后什么都不再打印,抓图每次都走完 + 整整 30 秒超时。唯一看得到指名信号的是完整的 bus monitor（`dbus-monitor`,它会向 + bus 要 `BecomeMonitor`）——为了一张兜底截图去换「观察用户 session bus 上每一条 + 消息」的权限,不划算。 +- **这一层现在自己讲 D-Bus,而且只用一条连接。** `linux_wayland/_dbus_client.py` + 是只用标准库写的 session bus 客户端:连接、SASL EXTERNAL 认证、`Hello`、 + `AddMatch`、一次方法调用,然后读到对得上的信号为止。它刻意不是通用绑定——没有 + property、没有 introspection、不导出对象、不传文件描述符（需要那一项的那一个调用 + 仍然交给 liboeffis）。`portal.py` 会**先**用自己的 unique name 推算出 request path + 并订阅,再发调用;portal 若不理会 `handle_token`,返回的 handle 也会一起跟。 +- **而且这是拿掉一个依赖,不是加一个。** 这一层原本需要装 `gdbus`（glib2),现在只要 + 有 session bus 就行,所以这条最后兜底的抓图路径能用的桌面比以前**更多**。安装提示 + 与诊断检查都改成这么说了。 +- **在真的 bus 上整条验过。** 新的 `portal-verification` job 跑真的 `dbus-daemon`、 + 真的 portal 实现与真的 client:抓回来的 PNG 字节解得开,而且就是 portal 画的 + 那几个像素,路径是含空格、percent-escape 过的,读完之后 portal 那个文件不见了。 + portal 没能交出图的每一种收场也都跑过——对话框被关掉、对话框一直开着、成功但没有 + URI、URI 不是本机文件——每一种都必须在 AutoControl 自己的时限内 fail closed。 + +## 本次更新 (2026-08-19) — RemoteDesktop portal 握手也从来不需要 GNOME VM + +- **portal 是 D-Bus 接口,不是合成器功能。** 在 GNOME 与 KDE 上要接到 libei,得跑 + `CreateSession` → `SelectDevices` → `Start` → `ConnectToEIS`,最后由 bus 交出一个 + EIS 文件描述符。这件事本来被记成「没有 GNOME VM 就验不到」,理由是 + `xdg-desktop-portal-wlr` 只做 ScreenCast 与 Screenshot、没有 RemoteDesktop——那是把 + 「没有容器附一个」当成了「没有容器当得了一个」。对 `liboeffis` 而言,谁占住 + `org.freedesktop.portal.Desktop`、谁回答那四个调用,谁就**是** portal。 +- **所以验证自己去占那个名字。** `docker/portal_server.py` 是一个跑在私有 session bus + 上的真 D-Bus 服务,它的 `ConnectToEIS` 交回去的是连到 `eis` 那个镜像用的同一台真 + `libeis` server 的活连接。真的 `liboeffis` 跑完真的握手;出来的描述符承载得起一个 + 真的 EI session;通过它发出的按键、绝对移动与按钮边沿,由对面一个独立实现记录下来。 +- **它确定了什么。** 四个调用按规范的顺序、发到 client 自己推算出来的 request path; + `SelectDevices` 要到的是键盘与指针、不多要——所以用户被要求同意的范围就是这个后端 + 真正需要的范围,`OEFFIS_DEVICE_DEFAULT` 也不是那个 `= 0` 的 all-devices 哨兵值; + 交回来的描述符是一个活的、由调用端持有并负责关闭的 socket——这正是把它交给 + `ei_setup_backend_fd`（一个会接管所有权的函数）之所以正确、而不是双重关闭的原因。 +- **以及每一种拒绝。** 同意对话框被关掉、对话框一直开着、描述符被扣住、portal 把 + session 关掉、portal 旧到根本没有 `ConnectToEIS`、bus 上根本没有 portal:每一种都 + 必须在本项目自己的时限内变成拒绝,而不是卡住或悄悄降级。`OEFFIS_EVENT_CLOSED` 这条 + 分支从来没有 peer 驱动得了,现在有了。 +- **仍然没有宣称的是什么。** 同意对话框作为「对话框」本身。CI 里没有人去按它,所以 + 真的 mutter 对话框长什么样、真人会让它开着多久,那仍然是 mutter 的事。对话框 + *产生的东西*——准、拒、沉默——三种都跑过了。 + +## 本次更新 (2026-08-19) — 一件记错了的打包事实 + +- **Debian trixie 有 `liboeffis`。** `Progress.md` 原本写没有,并据此推论 libei 快速 + 路径在 Debian／Ubuntu 上等于是关的。实测:`liboeffis1` 1.3.901-1 就在 trixie/main, + 提供 `liboeffis.so.1`。真正成立、而且对用户真正有影响的是另一件事:它是**独立的 + 二进制包**,`libei1` 并不依赖它——所以只装 libei 的机器上 portal 这条路仍然是关 + 的,`connect()` 会退回 GNOME 与 KDE 都不会开的 `eis-0` socket。要走快速路径, + `liboeffis` 得自己装。 + +## 本次更新 (2026-08-19) — 主屏幕左边多一块屏幕,Wayland 的抓取整条都读错 + +- **Wayland 没有「单块屏幕」这回事,而唯一那个平面也不一定从 `(0, 0)` 开始。** + 合成器把所有 output 排在同一个平面上,只要有一块在原点的左边或上面,这个平面的 + 原点就是负的——那正是「我的第二块屏幕在左边」对合成器的意思。sway 的 headless + backend 收 `output HEADLESS-1 position -1280 0`,所以这是 CI 立得起来的布局: + 两个 1280x720 的 output、一张 2560x720 的抓取、左上角那个像素在 x=-1280。 +- **`size()` 返回的是布局的右缘,不是宽度。** 它算的是各 output 的 `max(x + width)`, + 在上面那个布局是 1280,而 `grab_image()` 回来的画面宽 2560。凡是把这两者兜在一起 + 的人都信了小的那个:mss shim 的 monitor 列表（连带 `enumerate_monitors`）、屏幕 + 录制、WebRTC host、MCP 的 monitor 抓取,全都去要了一块只有桌面一半大的矩形, + 然后当成整个画面回报。 +- **不能自己套区域的层级,裁切裁在错的坐标空间。** 只有 grim 吃几何参数; + gnome-screenshot、spectacle、portal 与 `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` + 都是把整个布局交回来、由 AutoControl 事后裁切。那个裁切用的是布局坐标,而图的 + (0,0) 是布局原点,所以要 `[-1275, 5, -1175, 55]`（左边那块屏幕上的一块）等于向 + Pillow 要一个在画面左外侧 1275 px 的方框,拿回来的是黑色填充。 +- **而且比对到的东西会被回报在错的屏幕上。** `grab_logical()`——模板搜索、OCR 与 + visual match 背后的那个抓取——的原点是读 `GetSystemMetrics` 的,在 Windows 以外 + 无话可说,于是一律返回 `(0, 0)`,每个命中都被回报在实际位置的右边 1280 px 处。 + 这种错的表现是「点到别块屏幕」,比「找不到」更难查。 +- **修在接缝上,不是修在每个调用端。** Wayland backend 发布 `layout_origin()`; + `size()` 返回 bounding box 的**大小**;`grab_image` 裁切前先扣掉原点; + `screen_grabber.backend_layout_origin()` 则是 `grab_logical` 与 mss shim 去问的 + 那一个,让「自己抓取屏幕的后端」有办法说出这张画面从哪里开始。通用库本来就 + 看得见的后端（Windows／macOS／X11）什么都不用发布,行为完全不变——原点只会被 + 问到,不会被猜。 +- **对真的合成器两种布局都验过。** `wayland-verification` job 现在把那 27 项检查 + 跑两遍:一遍是两个 output 从原点并排,一遍是左边那个在 x=-1280。第二遍才是把 + 负原点整条钉死的那一遍——grim 的负 `-g`、回报的尺寸、`layout_origin()`、 + mss shim 的 monitor 矩形、`grab_logical()` 的原点,以及把操作者自定义指令指向 grim + 之后、让一张**真的**整布局 PNG 走过那条必须位移的裁切路径。 + +## 本次更新 (2026-08-19) — 同一个布局问题的输入侧:libei 把移动悄悄丢掉了 + +- **落在所有 region 之外的绝对移动,libei 会直接丢掉,而且什么都不说。** + 没有返回码、没有事件、调用方看不到任何错误——`ei_device_pointer_motion_absolute` + 就是不把这个事件送上线。`set_position` 于是像光标真的移动过一样正常返回。 + 这是对真的 EIS 对端量出来的,不是推论:设备只提供一个 `(0, 0, 1920, 1080)` + 的 region 时,`(1919, 1079)` 会到,`(1920, 1080)` 在 server 端连一个事件都没有。 +- **而那些 region 所在的坐标空间,不一定就是布局的坐标空间。** region 的 offset + 是 `uint32`,所以没有任何合成器**能**声明一个在原点左边或上面的 region——但本项目 + 的布局空间是从 `layout_origin()` 开始的,只要有一块屏幕在主屏幕左边就会变成负的。 + 那正是抓取那一半刚修好的同一种桌面。于是两半差了整整一个原点,也就是 + `get_pixel(x, y)` 与 `set_position(x, y)` 指向不同像素的那个情况——而本来会跑到 + 隔壁屏幕的光标,实际上是安安静静地哪里都没去。 +- **`LibeiBackend` 现在会读设备的 region,再把坐标映射进去。** 绑上了 + `ei_device_get_region` 与四个 `ei_region_get_*`;`_region_point` 对有覆盖的坐标 + 原样送出,对没覆盖的改用减掉布局原点后的坐标再试一次,两者都不覆盖就拒绝。 + 没有声明任何 region 的设备接受任何坐标,原样通过——这一点同样是量出来的,而那 + 就是单屏幕的常见情况,一分钱都不多花。 +- **拒绝是有用的结果,不是失败。** 它是 `LibeiUnavailable`,所以 + `_select_input.emitted` 会像处理「设备被暂停」那样,把这次移动交给 ydotool 那条路。 + libei 一直被写成快速路径而非唯一路径;这个 bug 的真正问题是:被丢掉的移动从来 + 到不了后备路径,因为没有人知道它被丢掉了。拒绝时也不会送 frame——什么都没有被 + 缓冲,那里送一个 frame 只会把上一次发送留在设备上的东西提交出去。 +- **布局原点只在坐标没中的时候才会去问。** 它要花一个 `wlr-randr` 子进程,所以不会 + 出现在每一次普通鼠标移动的路径上;在 GNOME 与 KDE 上它返回 `(0, 0)`——那在那里是 + 正确答案而不是退路,因为那些合成器自己就会把布局归一化。 +- **`eis-verification` job 多了五项对真协议的检查。** client 读回来的 offset 就是 + 合成器声明的那个;位于 `x=1280` 的 region,往内 100 px 的点要送 `1380` 而不是 + `100`;libei 到今天仍然一声不吭地丢掉 region 外的移动——这是整个防护所依据的测量, + 所以哪天 libei 改成夹取,这项会当场说出来;AutoControl 会拒绝这种移动而不是弄丢它; + 以及在原点为 `-1280` 的布局上,`(-1280, 10)` 到达 server 时是 `(0, 10)`。 + 这个 job 现在跑 20 项。 +- **这件事没有解决的部分。** ydotool 的 `mousemove --absolute` 有它自己的原点—— + 它夹到合成器的左上角,再把目标当成相对位移送出——那个角落是不是布局原点,仍然需要 + 一台真的会吃 libinput 设备的合成器才验得到。它继续留在 `Progress.md`,不会靠猜测 + 去改。 + +## 本次更新 (2026-08-19) — ydotool 一直回报成功,其实什么都没做 + +- **`apt install ydotool`——这个 backend 自己打印的安装提示——装到的版本跑不动它送的 + 命令行,而且是用「返回 0」来表达的。** ydotool 1.0 把整套 CLI 换掉了,而 Wayland + backend 送的每一个参数都是那一版才有的:`mousemove --absolute`、`mousemove + --wheel`、`click` 的十六进制位掩码（拆得开 press 与 release,拖拽整个建立在 + 这上面）、以及吃数字 evdev 码的 `key CODE:STATE`。Debian bookworm、Ubuntu 22.04 + 与 24.04 到今天都还是把 0.1.8 叫做 `ydotool`。对真的 uinput 设备实测:0.1.8 收到 + `click 0x40` **什么事件都不送,返回 0**;收到 `mousemove --absolute` 打印 + `unrecognised option`,**一样返回 0**。而 backend 是用 `check=True` 判断成败的, + 只有非零才会抛。所以在这些发行版上,脚本没点到、没打到、没移动,而每一次调用都回报成功。 +- **旧版 CLI 现在在送出任何东西之前就被挡下。** `linux_wayland/_ydotool_cli.py` + 每个进程只判定一次（鼠标与按键派送付不起每个事件一次 subprocess）,并在错误信息里 + 给出三条路:装 1.0+ 的包、自己编、或 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`。 + 两个版本都没有 `--version`,而 1.x 不启 daemon 连 `--help` 都不回答,所以探测读的是 + 两边都会打印、不需要 daemon、也没有副作用的那一样东西:无参数时的命令清单。认不出来的 + 版本一律放行而不是挡掉,免得将来改了字样的新版被过期的检测器锁在门外。 +- **两处安装提示还错在第二件事上**:Debian trixie 根本没有 `ydotool` 包。 + 提示已改成点名真的有的发行版。 + +## 本次更新 (2026-08-19) — 验 ydotool 从来不需要桌面,只需要一个读取端 + +- **两个验证镜像记下来的那个缺口,记错了。** `Dockerfile.wayland` 与 `Dockerfile.eis` + 结尾都写 ydotool「需要 /dev/uinput 以及一个会消费它的 seat」,而 `Progress.md` 把它 + 排在「先建一台 GNOME VM」后面。seat 决定的是注入的事件**会不会送达某处**,不是它 + **能不能被观察**:ydotoold 建的是普通的 uinput 设备,kernel 会挂成 + `/dev/input/eventN`,读那个节点就拿得回 ydotool 写进去的 `input_event`。 + 不需要合成器,不需要桌面 session,不需要 VM。 +- **`docker/Dockerfile.ydotool` 与 `docker/ydotool_verify.py` 就是在 CI 里做这件事, + 12 项检查。** 过去只对着 mock 断言的东西现在有答案了:`0xc0`／`0xc1`／`0xc2` 真的是 + BTN_LEFT／BTN_RIGHT／BTN_MIDDLE;拆边的 `0x40` 与 `0x80` 真的只送 press 或只送 + release(`press_mouse` 与拖拽整个建立在这上面);`key 30:1 30:0` 真的带的是数字 + evdev 码;而**滚动正负号是量出来的,不再是假设的**——`-y 1` 到 kernel 是 + `REL_WHEEL +1`、`-y -1` 是 `-1`、`-x 2` 是 `REL_HWHEEL +2`,而且轴没有互换。 + 最后这一项正是滚动那次改动之后,`Progress.md` 一直标着「未实测」的假设。 +- **第 12 项检查驱动的是 backend 自己的函数,不是手写 argv**,所以「ydotool 拿到这串 + 命令行会做什么」与「AutoControl 送的是什么」是接起来的,不只是并排。 +- **`mousemove --absolute` 送的并不是绝对事件**,这件事在信任它之前值得知道。 + ydotool 1.x 的设备上没有 ABS 轴:它先在两条相对轴上送 `INT32_MIN`,靠合成器把它夹到 + 左上角,再把目标当成相对位移送出去。所以 `set_position` 会落在你要的像素,**是因为 + 那个夹取**。kernel 这一侧现在钉住了;夹取本身是合成器的行为,仍然是开放项。 +- **容器拿到的是 `/dev/uinput` 加字符主号 13,不是 `--privileged`。** ydotoold 是在 + 容器起来**之后**才建输入节点,`--device` 覆盖不到,所以那个 job 只授予 + `--device-cgroup-rule 'c 13:* rmw'`,别的都没有。 + +## 本次更新 (2026-08-19) — Wayland 的滚动不再需要 uinput 常驻程序 + +光标移动、按键、按钮早就走 libei 了,只剩滚动每一格都还要 fork 一次 `ydotool`, +而 `Progress.md` 也写了原因:正负号是猜的,而滚错方向不会报错,只会安静地错。 +现在接上了,猜测也换成了两份独立佐证加一次实测。 + +- **两条路径对「一格」的正负号定义相反。** 本项目的 `wayland_scroll_direction_*` + 常数是 kernel `REL_WHEEL` 那一套,因为 ydotool 写进 `/dev/uinput` 的就是它:正值为上。 + libei 是 `wl_pointer`／libinput 那一套,正值为下——libinput 自己的 evdev 读取端 + 就是把 `REL_WHEEL` 取负号换过去的,而另一个有写明正负号的 libei sender（enigo） + 则是把「正值往下滚」的值原封不动送进 `scroll_discrete`。水平轴不用翻: + `REL_HWHEEL` 与 libinput 都以右为正。所以送往 libei 时垂直轴取负、水平轴不动—— + 这就是 `Progress.md` 在等的那个决定。 +- **这个翻转有对着真的 EIS server 验证,连负值一起。** `docker/eis_verify.py` 多了第 15 项 + 检查,驱动的是**公开的** `mouse.scroll()`（不是上一项检查驱动的 backend 方法）, + 从 server 端读回来:上是 `(0, -120)`、下是 `(0, 120)`、右是 `(120, 0)`。这也是唯一 + 一次把**负的**离散值送上线;先前那项检查只送过正值,正负号的 marshalling 若有毛病 + 根本没有地方会现形。 +- **被拒绝的发送现在会退回 CLI,也就是代码一直宣称的行为。** `libei` 的模块 + docstring 写着每个失败都抛 `LibeiUnavailable`,「`keyboard`／`mouse` 已经把它当成 + *改用 ydotool CLI*」。实际上只有**连接**是这样处理的。backend 交出去之后,合成器 + 暂停了设备、或 session 在两次调用之间结束,都会直接从 `set_position`、`press_key`、 + `hotkey` 抛出去。把滚动也接上 libei 等于再多一条「明明旁边就有可用后备却让脚本死掉」 + 的路,所以这个后备被补成真的:和弦中途被拒会先把已经按下的键反序放开,不会留下卡住的 + 修饰键;按钮的**放开**被拒则改由 ydotool 放开,不会整个 session 都按着。 +- **`LibeiUnavailable` 原本会穿过所有的拦截边界。** 它只继承 `RuntimeError`,而 + `CLAUDE.md` 写得很明白:不是 `AutoControlException` 的框架错误「会安静地逃出每一个 + 边界」——executor、后台轮询循环、请求处理器、GUI slot。现在两个都继承,原本接 + `RuntimeError` 的探测照旧能用,边界也终于看得到它。 + +## 本次更新 (2026-08-18) — libei 输入路径终于有对手可以说话了 + +Wayland 的**撷取**路径已经对着真的合成器验过;**输入**路径没有,而且被记成「需要一台 +GNOME VM」。其实不需要。libeis 就是 libei 自己那套协议的 server 端,Debian 有打包, +两个库可以直接在一条 Unix socket 上对话——所以 `docker/eis_server.py` 起一个真的 EIS +实作,`docker/eis_verify.py` 把 AutoControl 真正的 sender 对着它跑。不需要合成器, +不需要桌面 session,14 项检查,已接进 CI。 + +- **离散捲动差了 120 倍。** libei 的离散捲动以「一格的 120 分之一」为单位——跟 Windows + `WHEEL_DELTA` 同一套惯例——而 `scroll()` 直接送格数,等于一格只送了 1/120 的捲动量。 + libei 自己在执行期就会讲（`suspicious discrete event value 1, did you mean 120?`）, + 而这句话 mock 永远不会印出来。现在 `scroll(0, 1)` 到对面是 `(0, 120)`:一格、正确的轴、 + 正确的正负号。这条路径原本因为「正负号是猜的」而刻意没接线,结果正负号是这题里比较小的一半。 +- **拆除不再每个行程漏一个 context。** `ei_unref` 在 libei 1.3.901 会 segfault——但只在 + 「backend 开了、握手从未推进」那个状态。有了可以完成握手的对手,live 的情况终于测得到, + 而它是安全的。现在拆除会正常释放 device 与 context,只有真的会炸的那个状态才放弃。 +- **mock 检查不到的值都检查了。** server 端 offer 六个 capability,再把 client 真正绑定的 + 读回来:正好是 AutoControl 要的那四个,所以 `EI_DEVICE_CAP_*` 位元遮罩与 variadic + `ei_seat_bind_capabilities` 的编组都是对的。keycode、绝对座标、按键码都从线上读回来比对。 + 每次发送都确认有 frame,每个 device 都确认有先开 emulation transaction——没开的话 + libei 会把事件丢掉。 +- **两件量到但不是我们能修的事**,如实记下而不是含混带过:libeis 1.3.901 的 + `eis_device_pause()` 对 sender client 没有送出任何东西,所以 client 的 `DEVICE_PAUSED` + 处理仍然没有对手可以驱动;`start_emulating` 的 sequence number 也没有被送到对面 + （刻意送 4242,读回来是 0）。检查写成「要嘛有反应,要嘛根本没被通知」, + 哪天 libeis 开始送了而 client 忽略它,就会当场失败。 + +## 本次更新 (2026-08-18) — 架构地图的行数重新变成实测值 + +- **同一个子系统在地图里被写成两个不同的大小。** `CLAUDE.md` 规定 + `architecture_explore.md` 的每个数字都是实测的,但没有任何东西在检查,于是长出了 + 两套并存的计数惯例:§5.4 主题表与 §5.4.17 档案表数了一行不存在的结尾行—— + `len(text.split("\n"))` 会比以换行结尾的档案实际行数多一行,套件则是每个档案多一行 + ——而 §1 的总计与 §8 附录数的是对的。结果 `utils/executor/` 在一节里是 8,811 行、 + 在另一节里是 9,001 行,而 §8 那一栏加起来也不等于它自己的总计。除此之外还有大约 + 五十列根本是旧的,好几个 `####` 标题差了几百行（`linux_wayland/` 还写着 + 10 档／1,093 行,实际是 14／2,235）,而且有一张主题表多了两个子套件,它的摘要行 + 完全不知情。 +- **413 个数字用同一套惯例重新实测**——`len(text.splitlines())`,也就是 `wc -l` 的 + 结果,以及 `CLAUDE.md` 自己那段「超过 750 行」判断所用的算法。§5.4、§5.4.17 与 §8 + 现在对每个子系统都一致,§8 各列加起来也等于它写的总计。 +- **`test_doc_line_counts.py` 既是闸门也是修复工具。** 任何被引用的行数跟树上对不上 + 就让 CI 失败,并指出是哪几行;加 `--fix` 就一次全部就地改写。行数是地图里唯一没有 + 闸门的部分——指令、MCP 工具、子套件与范例数早就有 `test_doc_counts.py` 在管—— + 这正是它会漂掉的原因。 + +## 本次更新 (2026-08-18) — 剪贴板不再因为别的程序正在复制而失败 + +- **Windows 剪贴板同一时间只允许一个进程打开,而 AutoControl 的每一个剪贴板调用 + 在别人打开时都立刻放弃。** 资源管理器、Office 和每个浏览器复制时都会占住剪贴板 + 几毫秒,这段时间里 `OpenClipboard` 一律返回 false,而六个调用点——文字、图片、 + HTML、RTF、CSV、文件拖放、格式枚举——都把它直接翻成 + `RuntimeError: OpenClipboard failed`。在真实桌面上开一个进程循环复制实测:大约 + 千分之一的打开会失败,也就是脚本会因为操作者既看不见也复现不了的原因直接死掉。 + Win32 文档明写这种情况应该重试,而一个专门用来驱动「别的程序本来就很忙」的机器 + 的库,不能把「别人正在复制」当成错误。 +- **`win32_clipboard_api.open_clipboard()` 现在是唯一打开剪贴板的地方**,忙碌时会 + 等约 200 毫秒才报错,而且不论区块怎么结束都会关闭。三个自己手写 open/close 的 + 模块——包括两个比共享模块更早写的——都改走它,所以新的调用点不可能漏掉重试。 +- **剪贴板 round-trip 测试不再依赖机器上其他进程在做什么。** 这些测试跑的是真实的 + Win32 调用,而那正是四个历史 writer bug 唯一能被看见的地方,所以换成假的后端等于 + 删掉覆盖率,而不是让它稳定。改成读 Win32 剪贴板序号:写入与读回之间序号没变,就 + 代表这段时间没有别人写过,断言测的就只有 AutoControl 自己的代码。已在另一个进程 + 于测试期间写入 4,112 次剪贴板的情况下验证通过。 + +## 本次更新 (2026-08-18) — 打错指令名字回 400,不是 500 + +- **`POST /execute` 对不存在的 `AC_*` 名字回 `500 {"error": "execute_action failed"}`**, + 跟伺服器自己坏掉时的回应一模一样。调用端无法区分「我自己打错字」和「服务挂了」, + 错误讯息也没说是哪个名字不认得。 +- **所有指令名字现在都在执行前先校验**,不认得的回 `400`,并在 `unknown_commands` + 里列出**全部**——包含嵌在流程控制区块里的——让调用端一次改完所有拼错的名字。 + `POST /execute_file` 对读不到的档案、不是动作清单的内容、以及不认得的指令名, + 用同样的方式回应。OpenAPI 规格两者都写明了,并说明被拒绝的请求没有执行任何动作。 +- 校验与收集共用同一次走访,所以「嵌套动作清单可能藏在哪里」仍然只有一份定义。 + +## 本次更新 (2026-08-18) — libei 拆除时的 segfault,以及对真实库的核对 + +- **`ei_unref` 在 libei 1.3.901 上,只要 backend 已打开就会让进程崩溃,而我们的 + fallback 路径正好一头撞进去。** libei 握手的每一种失败都会走到 `_teardown()`, + 所以在任何装了 libei 而握手没完成的机器上,AutoControl 会直接 SIGSEGV,而不是 + 安静地改用 ydotool——跟 fail-closed 的承诺完全相反。逐个调用实测的结果:没有 + setup backend 时 `ei_unref` 安全、setup **失败**后安全、setup **成功**后必炸。 + `ei_disconnect` 在同样状态也炸,所以不是我们 refcount 用错;而头文件明写两种结果 + 都该用 `ei_unref`,因此这是上游的 bug。现在已打开的 backend 会被**放弃**而不是 + unref——代价是每个进程漏一个 context,换掉一个会驱动用户桌面的库直接崩溃。 + 验证程序里有一个哨兵会每次重新确认上游状态,修好时会提示可以移除 workaround。 +- **绑定里每一个入口点现在都对真的 `libei.so` 解析过。** 拼错的符号或错的 `argtypes` + 会毫无阻碍地通过假的符号表,只在用户的机器上才爆——22 个 prototype 加上 variadic + 的 `ei_seat_bind_capabilities` 现在都是真的验过。 +- **整条 fail-closed 链端到端跑过**:连到一个不说 EI 的 socket → 握手超时 → + `LibeiUnavailable` → `active_backend()` 返回 None → `press_key` 落到 ydotool CLI。 +- **`liboeffis` 不是每个发行版都有。** Arch 与 Fedora 有包,**Debian trixie 没有**。 + 没有它就没有 portal 路径,`connect()` 会退到众所周知的 EIS socket——而 GNOME 与 KDE + 根本不开那个 socket。所以在那些系统上 libei 快速路径是关闭的,而且会讲出来,不会 + 看起来莫名其妙地没作用。 + +## 本次更新 (2026-08-18) — Wayland 截取路径对上真的合成器了 + +- **`screen.size()` 报单一屏幕,`grab_image()` 却返回整个布局。** `wlr-randr` 的 + parser 抓文档里第一个 `WxH`,也就是第一个 output 的当前模式。双屏布局下那只有 + 半个画面——而 mss shim 正是把这两者组起来用的,所以录制、WebRTC 与 MCP 的屏幕路径 + 会去要一个只有半个画面大的区域,而且真的拿到了。`size()` 现在返回布局的 bounding + box,parser 会读每个启用中 output 的模式**与位置**。这个 bug 是跑真机才发现的: + 没有任何 mock 有第二块屏幕。 +- **`docker/Dockerfile.wayland` 让后端在 headless sway 下跑。** wlroots 的 headless + backend 不需要 GPU、seat 或显示器,所以一个真正的 Wayland session 塞得进容器——现在 + 也进了 CI,就是 `wayland-verification` job。两个 output 涂上不同纯色,因为在单一颜色 + 的画面上,区域抓错位置抓不出来,红蓝通道对调更是完全抓不出来。 +- **21 项此前只能对 mock 验的检查,现在对的是合成器真的画出来的像素**:grim 的 argv + 与 `-g` 几何、RGB 通道顺序、`wlr-randr` 没有文档记载的输出格式、 + `size()`／`grab_image()`／`get_pixel()`／`screenshot()`、 + `je_auto_control.screenshot()` 的 BGR 输出、`grab_logical()`(定位器与 OCR 路径)、 + mss shim、以及 `wtype`。 +- **容器答不了的部分直接写明,不含糊带过。** ydotool 需要 `/dev/uinput`,而 headless + sway 不吃 libinput 设备;`xdg-desktop-portal-wlr` 没有实现 RemoteDesktop,所以根本 + 没有 `ConnectToEIS` 可测。两者都留在 `Progress.md`。 + +## 本次更新 (2026-08-18) — libei 输入,整条打通 + +- **完整的 portal 握手做完了。** libei 不是“调用一个函数就按下一个键”的库: + sender 必须打开 EIS backend、绑定 seat 的能力、**从事件里**取得 device、在其上 + `start_emulating`,而且每次发送之后都要 `ei_device_frame`,否则什么都不会送达。 + 这些现在全部有了,所以 `press_key`、`set_position` 与鼠标按键不再需要每个事件 + spawn 一次进程。 +- **EIS socket 来自桌面 portal。** 在 GNOME 与 KDE 上它不是磁盘上的路径,而是由 + `org.freedesktop.portal.RemoteDesktop.ConnectToEIS` 经 D-Bus 交出的 file + descriptor,前面还有三次异步的 session 调用。**没有任何命令行工具能把 fd 交进 + 这个进程**,所以截图那条 `gdbus` 路在这里走不通;改用 `liboeffis`(libei 项目正是 + 为此附上它)。liboeffis 不存在时,仍会尝试众所周知的 `$XDG_RUNTIME_DIR/eis-0`。 +- **device 一律来自事件,不再由 context 冒充。** 先前的绑定把 `struct ei *` context + 传给收 `struct ei_device *` 的入口点——C 库里的指针类型混淆。现在从结构上 + 不可能发生。 +- **探测失败只付一次代价,不是每次按键。** 握手包含一次 portal 往返,可能还有同意 + 对话框。结果会在进程内缓存,所以“装了 libei 但不可用”的机器不会每次按键重试。 +- **所有情况都仍会回退到 ydotool。** 库缺失、用户拒绝、能力只给一半、device 被 + 暂停、握手没完成——每一种都抛 `LibeiUnavailable`,而 keyboard／mouse 本来就把它 + 当成“改用 CLI”。scroll 刻意留在 ydotool:它的方向约定有测试钉住,而 libei 的正负号 + 一旦猜错是**静默**的错误行为,不是明确的失败。 +- **ABI 常量是对过上游头文件的,不是猜的。** 对完发现三个错: + `enum ei_device_capability` 是**位掩码**,所以 `EI_DEVICE_CAP_KEYBOARD` 是 + `1 << 2` 而不是 3;`OEFFIS_EVENT_CLOSED` 排在 `OEFFIS_EVENT_DISCONNECTED` + **之前**;`OEFFIS_DEVICE_ALL_DEVICES` 是 `= 0` 的哨兵值,不是各设备位的 OR。 + 第一个代价最大——没有 device 会报告那个能力,所以每次连接都会超时然后静默改用 CLI。 + 核对过的值现在有测试钉住。另外 session 只申请实际会用到的键盘与指针,同意对话框 + 不会再要一个没人用的触摸屏授权。 + +## 本次更新 (2026-08-18) — Wayland 截取有了兜底 + +- **`xdg-desktop-portal` 为三个 CLI 工具兜底。** `grim`／`gnome-screenshot`／`spectacle` + 没有任何一个保证会装（GNOME 自 42 起不再默认安装 `gnome-screenshot`），所以最后会经由 + `gdbus` 尝试 `org.freedesktop.portal.Screenshot`，而不是直接放弃。它天生麻烦：portal + 返回的是 request handle、结果之后才以 signal 送达，所以监听必须在调用**之前**启动； + 而且同意对话框可能挡在前面，因此等待有 30 秒上限。 +- **操作者可以指定自己的截取命令。** + `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` 优先于所有检测。 + `{output}` 会换成临时 PNG 路径，在 `shlex.split` 之后逐个参数替换、不经过 shell， + 所以含空格的路径仍然是单个参数。这是给内置分层都不适用的环境用的逃生门——包含我们对 + 某个工具的 argv 猜错的情况。 +- **libei 主动拒绝它根本送不出输入的连接。** 这个绑定从头到尾只持有 `ei` context， + 但每个 device 入口点收的都是 `ei_device`，而且完全没有跑 libei 的 seat／device／ + `start_emulating`／`frame` 握手——所以它送不出输入，却**可能**在会开 + `$XDG_RUNTIME_DIR/eis-0` 的机器上把错误的指针传进 C 库。现在它会在 `connect()` + 就停下并说明原因。调用端本来就把这个情况当成“改用 ydotool CLI”，而那正是所有真实 + 桌面环境一直在做的事。 +- **portal 监听器不会在收尾时卡死。** 它的 pipe 只在读取线程放手之后才关闭；在有线程 + 仍阻塞在 `read()` 时关闭流可能卡在 buffer lock 上，那会让“超时”变成它本来要防止的 + “卡住”。 + +## 本次更新 (2026-08-18) — 容器镜像在 Windows 签出下也能构建、能启动 + +- **在 Windows 上 clone 再 build 出来的容器，一启动就死。** `.gitattributes` 只写了 + `* text=auto`，于是 `docker/entrypoint.sh` 与 `docker/entrypoint-xfce.sh` 被签出成 CRLF。 + shebang 就变成 `#!/bin/sh`，kernel 去找一个名字末尾带回车的解释器，镜像 build 得完全正常， + 一跑就抛出 `exec /usr/local/bin/autocontrol-entrypoint: no such file or directory`—— + 而那个文件明明就在。CI 永远看不到：Linux runner 签出来就是 LF。现在以 + `*.sh text eol=lf` 钉住，并有测试确认没有任何 entrypoint 带 CRLF。 +- **`.dockerignore` 放在 `docker/`，Docker 根本不会去那里读。** Docker 只读 build *context* + 根目录的那一份，而文档里每一条 build 指令都以仓库根目录为 context + （`docker build -f docker/Dockerfile .`），所以那些排除规则一条都没生效：`.git`、 + `.venv`、`test/` 跟各种缓存每次都被丢给 daemon。已移到根目录。 +- **`mss` 垫片的测试量的是开发机的屏幕，不是它自己的假件。** + `test_screen_grabber.py` 只换掉 `backend_grab_image`，`_backend_screen_size` 还是读真的 + `platform_wrapper.screen`，所以 `monitors[0]` 报的是开发者手边那台屏幕。在 1920x1080 + 的桌面上会过，在 1280x800 的 Xvfb 下就失败，而这跟被测代码无关。现在假件同时接管接缝的两半。 + +## 本次更新 (2026-08-17) — Wayland 看得见屏幕了 + +- **所有截取路径改走平台后端。** `screenshot()`、图像与锚点定位、OCR、smart waits、 + 视觉回归、屏幕录制、MCP 屏幕工具与远程桌面，此前各自直接调用 `PIL.ImageGrab` 或 + `mss`。这两者在 Linux 都是读 X11 root window——在 Wayland 下那个 root 属于 XWayland， + 不会合成原生 Wayland 窗口。Pillow 确实有回退到 `gnome-screenshot`／`grim`／`spectacle` + 的路径，但它只在 X11 截取**抛出异常**时才走（`except OSError`）——也就是根本没有 X + display 的情况；只要 XWayland 在跑（GNOME、KDE、sway 的默认）就不会触发。`mss` 则是 + 任何情况下都没有兜底。现在 `utils/cv2_utils/screen_grabber.py` 是唯一决定“这台机器怎么 + 读屏幕”的地方：后端若发布 `grab_image`，就把它包成调用端已在用的形状。Windows、 + macOS 与 Linux X11 不发布任何东西，仍然使用真正的库，行为完全不变。 +- **Wayland 截取覆盖三种合成器家族，而不只一种。** `grim` 只会说 `wlr-screencopy`， + GNOME 与 KDE 都没有实现——所以 `linux_wayland/capture.py` 依次尝试 `grim` + （sway、Hyprland、river）、`gnome-screenshot`、`spectacle`，并报告用了哪一个。只有 + `grim` 能自己接受区域参数，其余的截取整个屏幕后再裁剪。 +- **没有任何截取工具时明确失败。** 错误信息会列出各合成器该装什么，而不是返回空白的 + XWayland 截取——后者在下游只会表现成“找不到模板”。新的 `screen_capture` 诊断检查 + 会在任何失败发生前先说清楚当前用的是哪一个工具。 +- **`screen.size()` 与 `get_pixel` 在 GNOME／KDE 也能用。** 分辨率从 `wlr-randr` 回退 + 到以截取结果测量；`get_pixel` 从可用的截取路径裁出 1x1 区域。 + +**仍未完成**：libei 原生输入路径与 Wayland 真机验证，见 [Progress.md](../Progress.md)。 + ## 本次更新 (2026-07-03) — 稳定 API、失败诊断包与发布工程 给新集成用的版本化入口点、便携式失败诊断格式,以及强化后的发布管线。完整参考:[`docs/API_LIFECYCLE.md`](../docs/API_LIFECYCLE.md) 与 [`docs/CAPABILITY_MATRIX.md`](../docs/CAPABILITY_MATRIX.md)。 diff --git a/README/WHATS_NEW_zh-TW.md b/README/WHATS_NEW_zh-TW.md index 3ba94c5a..1de15dd2 100644 --- a/README/WHATS_NEW_zh-TW.md +++ b/README/WHATS_NEW_zh-TW.md @@ -1,5 +1,490 @@ # 本次更新 — AutoControl +## 本次更新 (2026-08-19) — Wayland 兩個等人拍板的取捨,拍板了 + +`Progress.md` 上掛著兩個 `DECIDE`:缺的不是工，是決定。兩件事其實是同一個問題犯兩次 +——一個合成器的設定,函式庫**量得到**卻**讀不回來**,所以它必須停止假裝自己知道。 + +- **指標加速度改成由操作者宣告,函式庫照信。** 量到的事實不變: + `ydotool mousemove --absolute` 送的是相對移動,libinput 預設的 adaptive profile 會 + 把它加成兩倍,而倍率沒有任何用戶端讀得回來。沒拍板的是「那要怎麼辦」——繼續警告後 + 照送、直接拒絕、還是讓操作者自己講。直接拒絕會讓每一台沒有 `liboeffis` 的 Wayland + 機器整條 `set_position` 不能用,所以答案是宣告: + `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` 表示 ydotoold 的裝置已經關掉加速度, + 移動就靜靜地、準確地送出去;`=strict` 表示寧可拒絕這次移動,也不讓點擊落在別的地方; + 不設定就維持現在的「警告一次後照樣移動」,所以今天能跑的東西明天照樣能跑。 + 值打錯會退回警告模式,**而且會講出來**——shell profile 裡的一個錯字,不可以靜靜地 + 把一次移動升級成「可信賴的精確」。整個判斷只掛在 ydotool 這條路上;libei 在協定層 + 本來就是絕對座標。 +- **Wayland 擷取裡的軟體游標,寫進文件,不繞過。** 這是 `seat-verification` 順手量到 + 的:專案裡沒有任何一條擷取帶 `-c`,也就是沒有人要求游標,而游標還是在圖裡。原因不在 + 我們這邊——只要 backend 沒有游標平面,wlroots 就會畫**軟體游標**,而軟體游標是合成進 + 輸出緩衝區的,`wlr-screencopy` 交出來的正是那一份。headless 永遠處在這個狀態; + 真桌面上只要跑 `WLR_NO_HARDWARE_CURSORS=1` 也一樣。Windows 的 BitBlt 與 X11 那條路 + 都不含游標,所以這是**只在 Wayland 出現的不一致**;指標壓在目標上的時候,定位器、 + 樣板比對與 OCR 看到的就是目標中間有一個游標形狀的洞。兩條繞法——擷取前把指標移開再 + 移回來、或比對時把指標周圍遮掉——都得先知道指標在哪,而 Wayland 不讓用戶端讀游標 + 位置;靠行程內的記錄去猜,使用者一動實體滑鼠就過期,而遮錯位置比看得見游標更糟。 + 所以改成寫下來:寫進 capability matrix、三份 README,以及診斷包——`screen_capture` + 檢查現在會帶 `cursor_may_be_captured`,讓那份「解釋定位器為何失敗」的報告自己說出 + 原因。檢查本身照量到的樣子斷言,哪天 wlroots 開始尊重 `overlay_cursor`,CI 會當場 + 紅掉來通知我們。 + +## 本次更新 (2026-08-19) — 「會吃 libinput 裝置的合成器」只差三個環境變數 + +`ydotool mousemove --absolute` 不是絕對移動。它不送任何 ABS 事件:先在兩軸各送一次 +`INT32_MIN` 的相對移動,把游標推到合成器夾取的那個角落,再把目標當成相對位移送出去。 +**那個角落到底是哪裡**、以及**位移在路上被合成器做了什麼**,是 Wayland 輸入路徑最後 +兩個沒有答案的問題——而且兩個都被記成「需要一台跑真桌面、會吃 libinput 裝置的 VM」。 + +- **兩個都不需要 VM。** wlroots 吃 `WLR_BACKENDS=headless,libinput`:輸出維持虛擬, + 輸入那一半是真的 libinput backend。libseat 的 builtin backend 不靠 logind 就能開 + 裝置,而 `SEATD_VTBOUND=0` 讓它不要去搶一個容器根本沒有的 VT。第四個條件最容易漏: + libinput 是透過 udev 列舉裝置,不是透過 `/dev`,所以 `systemd-udevd` 必須在 ydotoold + 建立裝置**之前**就已經在跑。四個都到位之後,ydotoold 的 uinput 裝置就是一個普通的 + seat 裝置,`grim -c` 會把游標合成進截圖裡,合成器就能用版面座標回答問題。 +- **那個角落是版面的左上角,不是版面座標的 `(0, 0)`。** 只有在所有輸出都在非負位置時 + 這兩點才是同一點。任何「主螢幕左邊還有一台螢幕」的桌面,兩者就差一個版面原點——所以 + 在原點為 `-1280` 的版面上,沒有轉換的「版面 `(0, 0)`」會把游標送到**隔壁那台螢幕**, + 差 1,280 像素。`mouse.set_position` 現在會先減掉 `layout_origin()` 再交給 ydotool, + 這正是擷取路徑早就在做的同一個修正;兩條輸入路徑共用的那個查詢搬進了 + `linux_wayland/_layout.py`,libei 與 ydotool 不會再對「原點是什麼」各說各話。 +- **剩下的距離則被指標加速度放大。** 送出去的位移是相對移動,所以 libinput 會加速它: + 對著真的 wlroots session 量到的是,預設的 adaptive profile 讓游標離那個角落的距離 + 正好是要求的**兩倍**——因為 `--absolute` 的兩個事件送在同一個 frame 裡,速度直接把 + profile 打到上限。把 `accel_profile flat` 加上 `pointer_accel 0` 之後,同一個呼叫 + 就精準到像素。ydotool 自己的 `--help` 一直都寫著「You need to disable mouse speed + acceleration for correct absolute movement」;後端現在會每個行程記一次這個警告,而不是 + 讓點擊安靜地落在錯的地方。函式庫這一側再多做不了什麼——那個倍率是合成器的設定, + 呼叫端讀不回來。 +- **新的 `seat-verification` job 把這些全部釘住。** `docker/Dockerfile.seat` 跑的是 + 擷取映像的同兩種版面,每種 14 項:sway 真的握著 ydotool 裝置、`--absolute (0, 0)` + 會把游標畫在版面的第一個像素上、關掉加速度後移動是一像素對一像素、沒轉換的 + `(0, 0)` 會打不中它指名的那台螢幕、`set_position` 減掉的正好是原點而且在兩台螢幕上 + 都落在指定的像素、以及加速倍率就是量到的 2 倍。裡面沒有任何一項依賴游標主題:每個 + 主張都是兩張截圖之間的差,游標圖片相對於熱點的偏移會自己抵銷掉。 +- **順手抓到的一件事。** 在這台合成器上,`grim` 明明沒有要求疊上游標,截回來的圖裡 + 還是有游標——因為只要 backend 沒有游標平面,wlroots 就會畫**軟體游標**(headless + 永遠如此,任何驅動不給平面、或使用者設了 `WLR_NO_HARDWARE_CURSORS=1` 的 session + 也是)。所有定位器、樣板比對與 OCR 都走同一條擷取,所以指標會在它壓著的東西上挖出 + 一個指標形狀的洞。該檢查照量到的樣子記下這個行為,要怎麼處理則列進 `Progress.md`。 + +## 本次更新 (2026-08-19) — 擷取用的 portal 本來就不可能成功,真的 bus 講出來了 + +- **`xdg-desktop-portal` 的回答是「指名送給發出呼叫的那條連線」的訊號。** + `Screenshot` 回的是一個 *request handle*,不是圖;圖稍後才以 + `org.freedesktop.portal.Request::Response` 送達,收件人是呼叫者的 unique bus name。 + bus 對指名訊息只送給收件人,所以別條連線再怎麼加 match rule 也接不到。 +- **舊的做法是兩條連線。** 先起一個 `gdbus monitor` 子行程,再用第二次 `gdbus` + 呼叫發請求,然後拿兩條正規表示式去讀 monitor 的 stdout。每次 `gdbus` 都會用自己的 + unique name 開一條自己的連線,所以在聽的那個行程從來就不是收件的那個。在真的 + `dbus-daemon` 上量到的:monitor 看得到呼叫經過,之後什麼都不再印,擷取每次都走完 + 整整 30 秒逾時。唯一看得到指名訊號的是完整的 bus monitor（`dbus-monitor`,它會向 + bus 要 `BecomeMonitor`）——為了一張備援截圖去換「觀察使用者 session bus 上每一則 + 訊息」的權限,不划算。 +- **這一層現在自己講 D-Bus,而且只用一條連線。** `linux_wayland/_dbus_client.py` + 是只用標準函式庫寫的 session bus 客戶端:連線、SASL EXTERNAL 認證、`Hello`、 + `AddMatch`、一次方法呼叫,然後讀到對得上的訊號為止。它刻意不是通用綁定——沒有 + property、沒有 introspection、不匯出物件、不傳檔案描述子（需要那一項的那一個呼叫 + 仍然交給 liboeffis）。`portal.py` 會**先**用自己的 unique name 推算出 request path + 並訂閱,再發呼叫;portal 若不理會 `handle_token`,回傳的 handle 也會一起跟。 +- **而且這是拿掉一個相依,不是加一個。** 這一層原本需要裝 `gdbus`（glib2),現在只要 + 有 session bus 就行,所以這條最後備援的擷取路徑能用的桌面比以前**更多**。安裝提示 + 與診斷檢查都改成這麼說了。 +- **在真的 bus 上整條驗過。** 新的 `portal-verification` job 跑真的 `dbus-daemon`、 + 真的 portal 實作與真的 client:擷取回來的 PNG 位元組解得開,而且就是 portal 畫的 + 那幾個像素,路徑是含空白、percent-escape 過的,讀完之後 portal 那個檔案不見了。 + portal 沒能交出圖的每一種收場也都跑過——對話框被關掉、對話框一直開著、成功但沒有 + URI、URI 不是本機檔案——每一種都必須在 AutoControl 自己的時限內 fail closed。 + +## 本次更新 (2026-08-19) — RemoteDesktop portal 交握也從來不需要 GNOME VM + +- **portal 是 D-Bus 介面,不是合成器功能。** 在 GNOME 與 KDE 上要接到 libei,得跑 + `CreateSession` → `SelectDevices` → `Start` → `ConnectToEIS`,最後由 bus 交出一個 + EIS 檔案描述子。這件事本來被記成「沒有 GNOME VM 就驗不到」,理由是 + `xdg-desktop-portal-wlr` 只做 ScreenCast 與 Screenshot、沒有 RemoteDesktop——那是把 + 「沒有容器附一個」當成了「沒有容器當得了一個」。對 `liboeffis` 而言,誰佔住 + `org.freedesktop.portal.Desktop`、誰回答那四個呼叫,誰就**是** portal。 +- **所以驗證自己去佔那個名字。** `docker/portal_server.py` 是一個跑在私有 session bus + 上的真 D-Bus 服務,它的 `ConnectToEIS` 交回去的是連到 `eis` 那個映像用的同一台真 + `libeis` server 的活連線。真的 `liboeffis` 跑完真的交握;出來的描述子承載得起一個 + 真的 EI session;透過它送出的按鍵、絕對移動與按鈕邊緣,由對面一個獨立實作記錄下來。 +- **它確定了什麼。** 四個呼叫依規範的順序、送到 client 自己推算出來的 request path; + `SelectDevices` 要到的是鍵盤與指標、不多要——所以使用者被要求同意的範圍就是這個後端 + 真正需要的範圍,`OEFFIS_DEVICE_DEFAULT` 也不是那個 `= 0` 的 all-devices 哨兵值; + 交回來的描述子是一個活的、由呼叫端持有並負責關閉的 socket——這正是把它交給 + `ei_setup_backend_fd`（一個會接管所有權的函式）之所以正確、而不是雙重關閉的原因。 +- **以及每一種拒絕。** 同意對話框被關掉、對話框一直開著、描述子被扣住、portal 把 + session 關掉、portal 舊到根本沒有 `ConnectToEIS`、bus 上根本沒有 portal:每一種都 + 必須在本專案自己的時限內變成拒絕,而不是卡住或悄悄降級。`OEFFIS_EVENT_CLOSED` 這條 + 分支從來沒有 peer 驅動得了,現在有了。 +- **仍然沒有宣稱的是什麼。** 同意對話框作為「對話框」本身。CI 裡沒有人去按它,所以 + 真的 mutter 對話框長什麼樣、真人會讓它開著多久,那仍然是 mutter 的事。對話框 + *產生的東西*——准、拒、沉默——三種都跑過了。 + +## 本次更新 (2026-08-19) — 一件記錯了的套件事實 + +- **Debian trixie 有 `liboeffis`。** `Progress.md` 原本寫沒有,並據此推論 libei 快速 + 路徑在 Debian／Ubuntu 上等於是關的。實測:`liboeffis1` 1.3.901-1 就在 trixie/main, + 提供 `liboeffis.so.1`。真正成立、而且對使用者真正有影響的是另一件事:它是**獨立的 + 二進位套件**,`libei1` 並不相依於它——所以只裝 libei 的機器上 portal 這條路仍然是關 + 的,`connect()` 會退回 GNOME 與 KDE 都不會開的 `eis-0` socket。要走快速路徑, + `liboeffis` 得自己裝。 + +## 本次更新 (2026-08-19) — 主螢幕左邊多一台螢幕,Wayland 的擷取整條都讀錯 + +- **Wayland 沒有「單一螢幕」這回事,而唯一那個平面也不一定從 `(0, 0)` 開始。** + 合成器把所有 output 排在同一個平面上,只要有一台在原點的左邊或上面,這個平面的 + 原點就是負的——那正是「我的第二台螢幕在左邊」對合成器的意思。sway 的 headless + backend 收 `output HEADLESS-1 position -1280 0`,所以這是 CI 立得起來的版面: + 兩個 1280x720 的 output、一張 2560x720 的擷取、左上角那個像素在 x=-1280。 +- **`size()` 回的是版面的右緣,不是寬度。** 它算的是各 output 的 `max(x + width)`, + 在上面那個版面是 1280,而 `grab_image()` 回來的畫面寬 2560。凡是把這兩者兜在一起 + 的人都信了小的那個:mss shim 的 monitor 清單（連帶 `enumerate_monitors`）、螢幕 + 錄影、WebRTC host、MCP 的 monitor 擷取,全都去要了一塊只有桌面一半大的矩形, + 然後當成整個畫面回報。 +- **不能自己套區域的層級,裁切裁在錯的座標空間。** 只有 grim 吃幾何參數; + gnome-screenshot、spectacle、portal 與 `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` + 都是把整個版面交回來、由 AutoControl 事後裁切。那個裁切用的是版面座標,而圖的 + (0,0) 是版面原點,所以要 `[-1275, 5, -1175, 55]`（左邊那台螢幕上的一塊）等於向 + Pillow 要一個在畫面左外側 1275 px 的方框,拿回來的是黑色填充。 +- **而且比對到的東西會被回報在錯的螢幕上。** `grab_logical()`——樣板搜尋、OCR 與 + visual match 背後的那個擷取——的原點是讀 `GetSystemMetrics` 的,在 Windows 以外 + 無話可說,於是一律回 `(0, 0)`,每個命中都被回報在實際位置的右邊 1280 px 處。 + 這種錯的表現是「點到別台螢幕」,比「找不到」更難查。 +- **修在接縫上,不是修在每個呼叫端。** Wayland backend 發布 `layout_origin()`; + `size()` 回 bounding box 的**大小**;`grab_image` 裁切前先扣掉原點; + `screen_grabber.backend_layout_origin()` 則是 `grab_logical` 與 mss shim 去問的 + 那一個,讓「自己擷取螢幕的後端」有辦法說出這張畫面從哪裡開始。通用函式庫本來就 + 看得見的後端（Windows／macOS／X11）什麼都不用發布,行為完全不變——原點只會被 + 問到,不會被猜。 +- **對真的合成器兩種版面都驗過。** `wayland-verification` job 現在把那 27 項檢查 + 跑兩遍:一遍是兩個 output 從原點並排,一遍是左邊那個在 x=-1280。第二遍才是把 + 負原點整條釘死的那一遍——grim 的負 `-g`、回報的尺寸、`layout_origin()`、 + mss shim 的 monitor 矩形、`grab_logical()` 的原點,以及把操作者自訂指令指向 grim + 之後、讓一張**真的**整版面 PNG 走過那條必須位移的裁切路徑。 + +## 本次更新 (2026-08-19) — 同一個版面問題的輸入側:libei 把移動悄悄丟掉了 + +- **落在所有 region 之外的絕對移動,libei 會直接丟掉,而且什麼都不說。** + 沒有回傳碼、沒有事件、呼叫端看不到任何錯誤——`ei_device_pointer_motion_absolute` + 就是不把這個事件送上線。`set_position` 於是像游標真的移動過一樣正常返回。 + 這是對真的 EIS 對端量出來的,不是推論:裝置只提供一個 `(0, 0, 1920, 1080)` + 的 region 時,`(1919, 1079)` 會到,`(1920, 1080)` 在 server 端連一個事件都沒有。 +- **而那些 region 所在的座標空間,不一定就是版面的座標空間。** region 的 offset + 是 `uint32`,所以沒有任何合成器**能**宣告一個在原點左邊或上面的 region——但本專案 + 的版面空間是從 `layout_origin()` 開始的,只要有一台螢幕在主螢幕左邊就會變成負的。 + 那正是擷取那一半剛修好的同一種桌面。於是兩半差了整整一個原點,也就是 + `get_pixel(x, y)` 與 `set_position(x, y)` 指到不同像素的那個情況——而本來會跑到 + 隔壁螢幕的游標,實際上是安安靜靜地哪裡都沒去。 +- **`LibeiBackend` 現在會讀裝置的 region,再把座標映射進去。** 綁上了 + `ei_device_get_region` 與四個 `ei_region_get_*`;`_region_point` 對有涵蓋的座標 + 原樣送出,對沒涵蓋的改用扣掉版面原點後的座標再試一次,兩者都不涵蓋就拒絕。 + 沒有宣告任何 region 的裝置接受任何座標,原樣通過——這一點同樣是量出來的,而那 + 就是單螢幕的常見情況,一毛錢都不多花。 +- **拒絕是有用的結果,不是失敗。** 它是 `LibeiUnavailable`,所以 + `_select_input.emitted` 會像處理「裝置被暫停」那樣,把這次移動交給 ydotool 那條路。 + libei 一直被寫成快速路徑而非唯一路徑;這個 bug 的真正問題是:被丟掉的移動從來 + 到不了後備路徑,因為沒有人知道它被丟掉了。拒絕時也不會送 frame——什麼都沒有被 + 緩衝,那裡送一個 frame 只會把上一次發送留在裝置上的東西提交出去。 +- **版面原點只在座標沒中的時候才會去問。** 它要花一個 `wlr-randr` 子行程,所以不會 + 出現在每一次普通滑鼠移動的路徑上;在 GNOME 與 KDE 上它回 `(0, 0)`——那在那裡是 + 正確答案而不是退路,因為那些合成器自己就會把版面正規化。 +- **`eis-verification` job 多了五項對真協定的檢查。** client 讀回來的 offset 就是 + 合成器宣告的那個;位在 `x=1280` 的 region,往內 100 px 的點要送 `1380` 而不是 + `100`;libei 到今天仍然一聲不吭地丟掉 region 外的移動——這是整個防護所依據的量測, + 所以哪天 libei 改成夾取,這項會當場說出來;AutoControl 會拒絕這種移動而不是弄丟它; + 以及在原點為 `-1280` 的版面上,`(-1280, 10)` 到達 server 時是 `(0, 10)`。 + 這個 job 現在跑 20 項。 +- **這件事沒有解決的部分。** ydotool 的 `mousemove --absolute` 有它自己的原點—— + 它夾到合成器的左上角,再把目標當成相對位移送出——那個角落是不是版面原點,仍然需要 + 一台真的會吃 libinput 裝置的合成器才驗得到。它繼續留在 `Progress.md`,不會靠猜測 + 去改。 + +## 本次更新 (2026-08-19) — ydotool 一直回報成功,其實什麼都沒做 + +- **`apt install ydotool`——這個 backend 自己印的安裝提示——裝到的版本跑不動它送的 + 命令列,而且是用「回傳 0」來表達的。** ydotool 1.0 把整套 CLI 換掉了,而 Wayland + backend 送的每一個參數都是那一版才有的:`mousemove --absolute`、`mousemove + --wheel`、`click` 的十六進位位元遮罩（拆得開 press 與 release,拖曳整個建立在 + 這上面）、以及吃數字 evdev 碼的 `key CODE:STATE`。Debian bookworm、Ubuntu 22.04 + 與 24.04 到今天都還是把 0.1.8 叫做 `ydotool`。對真的 uinput 裝置實測:0.1.8 收到 + `click 0x40` **什麼事件都不送,回傳 0**;收到 `mousemove --absolute` 印 + `unrecognised option`,**一樣回傳 0**。而 backend 是用 `check=True` 判斷成敗的, + 只有非零才會拋。所以在這些發行版上,腳本沒點到、沒打到、沒移動,而每一次呼叫都回報成功。 +- **舊版 CLI 現在在送出任何東西之前就被擋下。** `linux_wayland/_ydotool_cli.py` + 每個行程只判定一次（滑鼠與按鍵派送付不起每個事件一次 subprocess）,並在錯誤訊息裡 + 給出三條路:裝 1.0+ 的套件、自己編、或 `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`。 + 兩個版本都沒有 `--version`,而 1.x 沒起 daemon 連 `--help` 都不回答,所以探測讀的是 + 兩邊都會印、不需要 daemon、也沒有副作用的那一樣東西:無參數時的指令清單。認不出來的 + 版本一律放行而不是擋掉,免得將來改了字樣的新版被過期的偵測器鎖在門外。 +- **兩處安裝提示還錯在第二件事上**:Debian trixie 根本沒有 `ydotool` 套件。 + 提示已改成點名真的有的發行版。 + +## 本次更新 (2026-08-19) — 驗 ydotool 從來不需要桌面,只需要一個讀取端 + +- **兩個驗證映像記下來的那個缺口,記錯了。** `Dockerfile.wayland` 與 `Dockerfile.eis` + 結尾都寫 ydotool「需要 /dev/uinput 以及一個會消費它的 seat」,而 `Progress.md` 把它 + 排在「先建一台 GNOME VM」後面。seat 決定的是注入的事件**會不會送達某處**,不是它 + **能不能被觀察**:ydotoold 建的是普通的 uinput 裝置,kernel 會掛成 + `/dev/input/eventN`,讀那個節點就拿得回 ydotool 寫進去的 `input_event`。 + 不需要合成器,不需要桌面 session,不需要 VM。 +- **`docker/Dockerfile.ydotool` 與 `docker/ydotool_verify.py` 就是在 CI 裡做這件事, + 12 項檢查。** 過去只對著 mock 斷言的東西現在有答案了:`0xc0`／`0xc1`／`0xc2` 真的是 + BTN_LEFT／BTN_RIGHT／BTN_MIDDLE;拆邊的 `0x40` 與 `0x80` 真的只送 press 或只送 + release(`press_mouse` 與拖曳整個建立在這上面);`key 30:1 30:0` 真的帶的是數字 + evdev 碼;而**捲動正負號是量出來的,不再是假設的**——`-y 1` 到 kernel 是 + `REL_WHEEL +1`、`-y -1` 是 `-1`、`-x 2` 是 `REL_HWHEEL +2`,而且軸沒有互換。 + 最後這一項正是捲動那次改動之後,`Progress.md` 一直標著「未實測」的假設。 +- **第 12 項檢查驅動的是 backend 自己的函式,不是手寫 argv**,所以「ydotool 拿到這串 + 命令列會做什麼」與「AutoControl 送的是什麼」是接起來的,不只是並排。 +- **`mousemove --absolute` 送的並不是絕對事件**,這件事在信任它之前值得知道。 + ydotool 1.x 的裝置上沒有 ABS 軸:它先在兩條相對軸上送 `INT32_MIN`,靠合成器把它夾到 + 左上角,再把目標當成相對位移送出去。所以 `set_position` 會落在你要的像素,**是因為 + 那個夾取**。kernel 這一側現在釘住了;夾取本身是合成器的行為,仍然是開放項。 +- **容器拿到的是 `/dev/uinput` 加字元主號 13,不是 `--privileged`。** ydotoold 是在 + 容器起來**之後**才建輸入節點,`--device` 涵蓋不到,所以那個 job 只授予 + `--device-cgroup-rule 'c 13:* rmw'`,別的都沒有。 + +## 本次更新 (2026-08-19) — Wayland 的捲動不再需要 uinput 常駐程式 + +游標移動、按鍵、按鈕早就走 libei 了,只剩捲動每一格都還要 fork 一次 `ydotool`, +而 `Progress.md` 也寫了原因:正負號是猜的,而捲錯方向不會噴錯,只會安靜地錯。 +現在接上了,猜測也換成了兩份獨立佐證加一次實測。 + +- **兩條路徑對「一格」的正負號定義相反。** 本專案的 `wayland_scroll_direction_*` + 常數是 kernel `REL_WHEEL` 那一套,因為 ydotool 寫進 `/dev/uinput` 的就是它:正值為上。 + libei 是 `wl_pointer`／libinput 那一套,正值為下——libinput 自己的 evdev 讀取端 + 就是把 `REL_WHEEL` 取負號換過去的,而另一個有寫明正負號的 libei sender（enigo） + 則是把「正值往下捲」的值原封不動送進 `scroll_discrete`。水平軸不用翻: + `REL_HWHEEL` 與 libinput 都以右為正。所以送往 libei 時垂直軸取負、水平軸不動—— + 這就是 `Progress.md` 在等的那個決定。 +- **這個翻轉有對著真的 EIS server 驗,連負值一起。** `docker/eis_verify.py` 多了第 15 項 + 檢查,驅動的是**公開的** `mouse.scroll()`（不是上一項檢查驅動的 backend 方法）, + 從 server 端讀回來:上是 `(0, -120)`、下是 `(0, 120)`、右是 `(120, 0)`。這也是唯一 + 一次把**負的**離散值送上線;先前那項檢查只送過正值,正負號的 marshalling 若有毛病 + 根本沒有地方會現形。 +- **被拒絕的發送現在會退回 CLI,也就是程式碼一直宣稱的行為。** `libei` 的模組 + docstring 寫著每個失敗都拋 `LibeiUnavailable`,「`keyboard`／`mouse` 已經把它當成 + *改用 ydotool CLI*」。實際上只有**連線**是這樣處理的。backend 交出去之後,合成器 + 暫停了裝置、或 session 在兩次呼叫之間結束,都會直接從 `set_position`、`press_key`、 + `hotkey` 拋出去。把捲動也接上 libei 等於再多一條「明明旁邊就有可用後備卻讓腳本死掉」 + 的路,所以這個後備被補成真的:和弦中途被拒會先把已經按下的鍵反序放開,不會留下卡住的 + 修飾鍵;按鈕的**放開**被拒則改由 ydotool 放開,不會整個 session 都按著。 +- **`LibeiUnavailable` 原本會穿過所有的攔截邊界。** 它只繼承 `RuntimeError`,而 + `CLAUDE.md` 寫得很明白:不是 `AutoControlException` 的框架錯誤「會安靜地逃出每一個 + 邊界」——executor、背景輪詢迴圈、請求處理器、GUI slot。現在兩個都繼承,原本接 + `RuntimeError` 的探測照舊能用,邊界也終於看得到它。 + +## 本次更新 (2026-08-18) — libei 輸入路徑終於有對手可以說話了 + +Wayland 的**擷取**路徑已經對著真的合成器驗過;**輸入**路徑沒有,而且被記成「需要一台 +GNOME VM」。其實不需要。libeis 就是 libei 自己那套協定的 server 端,Debian 有打包, +兩個函式庫可以直接在一條 Unix socket 上對話——所以 `docker/eis_server.py` 起一個真的 EIS +實作,`docker/eis_verify.py` 把 AutoControl 真正的 sender 對著它跑。不需要合成器, +不需要桌面 session,14 項檢查,已接進 CI。 + +- **離散捲動差了 120 倍。** libei 的離散捲動以「一格的 120 分之一」為單位——跟 Windows + `WHEEL_DELTA` 同一套慣例——而 `scroll()` 直接送格數,等於一格只送了 1/120 的捲動量。 + libei 自己在執行期就會講（`suspicious discrete event value 1, did you mean 120?`）, + 而這句話 mock 永遠不會印出來。現在 `scroll(0, 1)` 到對面是 `(0, 120)`:一格、正確的軸、 + 正確的正負號。這條路徑原本因為「正負號是猜的」而刻意沒接線,結果正負號是這題裡比較小的一半。 +- **拆除不再每個行程漏一個 context。** `ei_unref` 在 libei 1.3.901 會 segfault——但只在 + 「backend 開了、握手從未推進」那個狀態。有了可以完成握手的對手,live 的情況終於測得到, + 而它是安全的。現在拆除會正常釋放 device 與 context,只有真的會炸的那個狀態才放棄。 +- **mock 檢查不到的值都檢查了。** server 端 offer 六個 capability,再把 client 真正綁定的 + 讀回來:正好是 AutoControl 要的那四個,所以 `EI_DEVICE_CAP_*` 位元遮罩與 variadic + `ei_seat_bind_capabilities` 的編組都是對的。keycode、絕對座標、按鍵碼都從線上讀回來比對。 + 每次發送都確認有 frame,每個 device 都確認有先開 emulation transaction——沒開的話 + libei 會把事件丟掉。 +- **兩件量到但不是我們能修的事**,如實記下而不是含混帶過:libeis 1.3.901 的 + `eis_device_pause()` 對 sender client 沒有送出任何東西,所以 client 的 `DEVICE_PAUSED` + 處理仍然沒有對手可以驅動;`start_emulating` 的 sequence number 也沒有被送到對面 + （刻意送 4242,讀回來是 0）。檢查寫成「要嘛有反應,要嘛根本沒被通知」, + 哪天 libeis 開始送了而 client 忽略它,就會當場失敗。 + +## 本次更新 (2026-08-18) — 架構地圖的行數重新變成實測值 + +- **同一個子系統在地圖裡被寫成兩個不同的大小。** `CLAUDE.md` 規定 + `architecture_explore.md` 的每個數字都是實測的,但沒有任何東西在檢查,於是長出了 + 兩套並存的計數慣例:§5.4 主題表與 §5.4.17 檔案表數了一行不存在的結尾行—— + `len(text.split("\n"))` 會比以換行結尾的檔案實際行數多一行,套件則是每個檔案多一行 + ——而 §1 的總計與 §8 附錄數的是對的。結果 `utils/executor/` 在一節裡是 8,811 行、 + 在另一節裡是 9,001 行,而 §8 那一欄加起來也不等於它自己的總計。除此之外還有大約 + 五十列根本是舊的,好幾個 `####` 標題差了幾百行（`linux_wayland/` 還寫著 + 10 檔／1,093 行,實際是 14／2,235）,而且有一張主題表多了兩個子套件,它的摘要行 + 完全不知情。 +- **413 個數字用同一套慣例重新實測**——`len(text.splitlines())`,也就是 `wc -l` 的 + 結果,以及 `CLAUDE.md` 自己那段「超過 750 行」判斷所用的算法。§5.4、§5.4.17 與 §8 + 現在對每個子系統都一致,§8 各列加起來也等於它寫的總計。 +- **`test_doc_line_counts.py` 既是閘門也是修復工具。** 任何被引用的行數跟樹上對不上 + 就讓 CI 失敗,並指出是哪幾行;加 `--fix` 就一次全部就地改寫。行數是地圖裡唯一沒有 + 閘門的部分——指令、MCP 工具、子套件與範例數早就有 `test_doc_counts.py` 在管—— + 這正是它會漂掉的原因。 + +## 本次更新 (2026-08-18) — 剪貼簿不再因為別的程式正在複製而失敗 + +- **Windows 剪貼簿同一時間只允許一個行程開啟,而 AutoControl 的每一個剪貼簿呼叫 + 在別人開著時都立刻放棄。** 檔案總管、Office 和每個瀏覽器複製時都會佔住剪貼簿 + 幾毫秒,這段時間裡 `OpenClipboard` 一律回傳 false,而六個呼叫點——文字、影像、 + HTML、RTF、CSV、檔案拖放、格式列舉——都把它直接翻成 + `RuntimeError: OpenClipboard failed`。在真實桌面上開一個行程迴圈複製實測:大約 + 千分之一的開啟會失敗,也就是腳本會因為操作者既看不見也重現不了的原因直接死掉。 + Win32 文件明寫這種情況應該重試,而一個專門用來驅動「別的程式本來就很忙」的機器 + 的函式庫,不能把「別人正在複製」當成錯誤。 +- **`win32_clipboard_api.open_clipboard()` 現在是唯一開啟剪貼簿的地方**,忙碌時會 + 等約 200 毫秒才報錯,而且不論區塊怎麼結束都會關閉。三個自己手寫 open/close 的 + 模組——包括兩個比共用模組更早寫的——都改走它,所以新的呼叫點不可能漏掉重試。 +- **剪貼簿 round-trip 測試不再依賴機器上其他行程在做什麼。** 這些測試跑的是真實的 + Win32 呼叫,而那正是四個歷史 writer bug 唯一能被看見的地方,所以換成假的後端等於 + 刪掉覆蓋率,而不是讓它穩定。改成讀 Win32 剪貼簿序號:寫入與讀回之間序號沒變,就 + 代表這段時間沒有別人寫過,斷言測的就只有 AutoControl 自己的程式碼。已在另一個行程 + 於測試期間寫入 4,112 次剪貼簿的情況下驗證通過。 + +## 本次更新 (2026-08-18) — 打錯指令名字回 400,不是 500 + +- **`POST /execute` 對不存在的 `AC_*` 名字回 `500 {"error": "execute_action failed"}`**, + 跟伺服器自己壞掉時的回應一模一樣。呼叫端無法區分「我自己打錯字」和「服務掛了」, + 錯誤訊息也沒說是哪個名字不認得。 +- **所有指令名字現在都在執行前先校驗**,不認得的回 `400`,並在 `unknown_commands` + 裡列出**全部**——包含嵌在流程控制區塊裡的——讓呼叫端一次改完所有拼錯的名字。 + `POST /execute_file` 對讀不到的檔案、不是動作清單的內容、以及不認得的指令名, + 用同樣的方式回應。OpenAPI 規格兩者都寫明了,並說明被拒絕的請求沒有執行任何動作。 +- 校驗與收集共用同一次走訪,所以「巢狀動作清單可能藏在哪裡」仍然只有一份定義。 + +## 本次更新 (2026-08-18) — libei 拆除時的 segfault,以及對真實函式庫的核對 + +- **`ei_unref` 在 libei 1.3.901 上,只要 backend 已開啟就會讓行程崩潰,而我們的 + fallback 路徑正好一頭撞進去。** libei 握手的每一種失敗都會走到 `_teardown()`, + 所以在任何裝了 libei 而握手沒完成的機器上,AutoControl 會直接 SIGSEGV,而不是 + 安靜地改用 ydotool——跟 fail-closed 的承諾完全相反。逐個呼叫實測的結果:沒有 + setup backend 時 `ei_unref` 安全、setup **失敗**後安全、setup **成功**後必炸。 + `ei_disconnect` 在同樣狀態也炸,所以不是我們 refcount 用錯;而標頭檔明寫兩種結果 + 都該用 `ei_unref`,因此這是上游的 bug。現在已開啟的 backend 會被**放棄**而不是 + unref——代價是每個行程漏一個 context,換掉一個會驅動使用者桌面的函式庫直接崩潰。 + 驗證程式裡有一個哨兵會每次重新確認上游狀態,修好時會提示可以移除 workaround。 +- **綁定裡每一個進入點現在都對真的 `libei.so` 解析過。** 拼錯的符號或錯的 `argtypes` + 會毫無阻礙地通過假的符號表,只在使用者的機器上才爆——22 個 prototype 加上 variadic + 的 `ei_seat_bind_capabilities` 現在都是真的驗過。 +- **整條 fail-closed 鏈端到端跑過**:連到一個不說 EI 的 socket → 握手逾時 → + `LibeiUnavailable` → `active_backend()` 回 None → `press_key` 落到 ydotool CLI。 +- **`liboeffis` 不是每個發行版都有。** Arch 與 Fedora 有包,**Debian trixie 沒有**。 + 沒有它就沒有 portal 路徑,`connect()` 會退到眾所周知的 EIS socket——而 GNOME 與 KDE + 根本不開那個 socket。所以在那些系統上 libei 快速路徑是關閉的,而且會講出來,不會 + 看起來莫名其妙地沒作用。 + +## 本次更新 (2026-08-18) — Wayland 擷取路徑對上真的合成器了 + +- **`screen.size()` 報單一螢幕,`grab_image()` 卻回傳整個佈局。** `wlr-randr` 的 + parser 抓文件裡第一個 `WxH`,也就是第一個 output 的當前模式。雙螢幕佈局下那只有 + 半個畫面——而 mss shim 正是把這兩者組起來用的,所以錄影、WebRTC 與 MCP 的螢幕路徑 + 會去要一個只有半個畫面大的區域,而且真的拿到了。`size()` 現在回傳佈局的 bounding + box,parser 會讀每個啟用中 output 的模式**與位置**。這個 bug 是跑真機才發現的: + 沒有任何 mock 有第二個螢幕。 +- **`docker/Dockerfile.wayland` 讓後端在 headless sway 底下跑。** wlroots 的 headless + backend 不需要 GPU、seat 或顯示器,所以一個真正的 Wayland session 塞得進容器——現在 + 也進了 CI,就是 `wayland-verification` job。兩個 output 塗上不同純色,因為在單一顏色 + 的畫面上,區域抓錯位置抓不出來,紅藍通道對調更是完全抓不出來。 +- **21 項先前只能對 mock 驗的檢查,現在對的是合成器真的畫出來的像素**:grim 的 argv + 與 `-g` 幾何、RGB 通道順序、`wlr-randr` 沒有文件記載的輸出格式、 + `size()`／`grab_image()`／`get_pixel()`／`screenshot()`、 + `je_auto_control.screenshot()` 的 BGR 輸出、`grab_logical()`(定位器與 OCR 路徑)、 + mss shim、以及 `wtype`。 +- **容器答不了的部分直接寫明,不含糊帶過。** ydotool 需要 `/dev/uinput`,而 headless + sway 不吃 libinput 裝置;`xdg-desktop-portal-wlr` 沒有實作 RemoteDesktop,所以根本 + 沒有 `ConnectToEIS` 可測。兩者都留在 `Progress.md`。 + +## 本次更新 (2026-08-18) — libei 輸入,整條打通 + +- **完整的 portal 握手做完了。** libei 不是「呼叫一個函式就按下一個鍵」的函式庫: + sender 必須開啟 EIS backend、綁定 seat 的能力、**從事件裡**取得 device、在其上 + `start_emulating`,而且每次發送之後都要 `ei_device_frame`,否則什麼都不會送達。 + 這些現在全部有了,所以 `press_key`、`set_position` 與滑鼠按鍵不再需要每個事件 + spawn 一次行程。 +- **EIS socket 來自桌面 portal。** 在 GNOME 與 KDE 上它不是磁碟上的路徑,而是由 + `org.freedesktop.portal.RemoteDesktop.ConnectToEIS` 經 D-Bus 交出的 file + descriptor,前面還有三次非同步的 session 呼叫。**沒有任何命令列工具能把 fd 交進 + 這個行程**,所以截圖那條 `gdbus` 路在這裡走不通;改用 `liboeffis`(libei 專案正是 + 為此附上它)。liboeffis 不存在時,仍會嘗試眾所周知的 `$XDG_RUNTIME_DIR/eis-0`。 +- **device 一律來自事件,不再由 context 冒充。** 先前的綁定把 `struct ei *` context + 傳給收 `struct ei_device *` 的進入點——C 函式庫裡的指標型別混淆。現在從結構上 + 不可能發生。 +- **探測失敗只付一次代價,不是每次按鍵。** 握手包含一次 portal 往返,可能還有同意 + 對話框。結果會在行程內快取,所以「裝了 libei 但不可用」的機器不會每次按鍵重試。 +- **所有情況都仍會退回 ydotool。** 函式庫缺失、使用者拒絕、能力只給一半、device 被 + 暫停、握手沒完成——每一種都丟 `LibeiUnavailable`,而 keyboard／mouse 本來就把它 + 當成「改用 CLI」。scroll 刻意留在 ydotool:它的方向約定有測試釘住,而 libei 的正負號 + 一旦猜錯是**靜默**的錯誤行為,不是明確的失敗。 +- **ABI 常數是對過上游標頭檔的,不是猜的。** 對完發現三個錯: + `enum ei_device_capability` 是**位元遮罩**,所以 `EI_DEVICE_CAP_KEYBOARD` 是 + `1 << 2` 而不是 3;`OEFFIS_EVENT_CLOSED` 排在 `OEFFIS_EVENT_DISCONNECTED` + **之前**;`OEFFIS_DEVICE_ALL_DEVICES` 是 `= 0` 的哨兵值,不是各裝置位元的 OR。 + 第一個代價最大——沒有 device 會回報那個能力,所以每次連線都會逾時然後靜默改用 CLI。 + 核對過的值現在有測試釘住。另外 session 只申請實際會用到的鍵盤與指標,同意對話框 + 不會再要一個沒人用的觸控螢幕授權。 + +## 本次更新 (2026-08-18) — Wayland 擷取有了保底 + +- **`xdg-desktop-portal` 為三個 CLI 工具兜底。** `grim`／`gnome-screenshot`／`spectacle` + 沒有任何一個保證會裝（GNOME 自 42 起不再預設安裝 `gnome-screenshot`），所以最後會經由 + `gdbus` 嘗試 `org.freedesktop.portal.Screenshot`，而不是直接放棄。它天生麻煩：portal + 回傳的是 request handle、結果之後才以 signal 送達，所以監聽必須在呼叫**之前**啟動； + 而且同意對話框可能擋在前面，因此等待有 30 秒上限。 +- **操作者可以指定自己的擷取指令。** + `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` 優先於所有偵測。 + `{output}` 會換成暫存 PNG 路徑，在 `shlex.split` 之後逐一參數替換、不經過 shell， + 所以含空白的路徑仍然是單一參數。這是給內建分層都不適用的環境用的逃生門——包含我們對 + 某個工具的 argv 猜錯的情況。 +- **libei 主動拒絕它根本送不出輸入的連線。** 這個綁定從頭到尾只握有 `ei` context， + 但每個 device 進入點收的都是 `ei_device`，而且完全沒有跑 libei 的 seat／device／ + `start_emulating`／`frame` 握手——所以它送不出輸入，卻**可能**在會開 + `$XDG_RUNTIME_DIR/eis-0` 的機器上把錯誤的指標傳進 C 函式庫。現在它會在 `connect()` + 就停下並說明原因。呼叫端本來就把這個情況當成「改用 ydotool CLI」，而那正是所有真實 + 桌面環境一直在做的事。 +- **portal 監聽器不會在收尾時卡死。** 它的 pipe 只在讀取執行緒放手之後才關閉；在有執行緒 + 仍阻塞在 `read()` 時關閉串流可能卡在 buffer lock 上，那會讓「逾時」變成它本來要防止的 + 「卡住」。 + +## 本次更新 (2026-08-18) — 容器映像在 Windows 簽出下也能建、能啟 + +- **在 Windows 上 clone 再 build 出來的容器，一啟動就死。** `.gitattributes` 只寫了 + `* text=auto`，於是 `docker/entrypoint.sh` 與 `docker/entrypoint-xfce.sh` 被簽出成 CRLF。 + shebang 就變成 `#!/bin/sh`，kernel 去找一個名字末尾帶回車的直譯器，映像 build 得完全正常， + 一跑就丟出 `exec /usr/local/bin/autocontrol-entrypoint: no such file or directory`—— + 而那個檔明明就在。CI 永遠看不到：Linux runner 簽出來就是 LF。現在以 + `*.sh text eol=lf` 釘住，並有測試確認沒有任何 entrypoint 帶 CRLF。 +- **`.dockerignore` 放在 `docker/`，Docker 根本不會去那裡讀。** Docker 只讀 build *context* + 根目錄的那一份，而文件裡每一道 build 指令都以倉庫根目錄為 context + （`docker build -f docker/Dockerfile .`），所以那些排除規則一條都沒生效：`.git`、 + `.venv`、`test/` 跟各種快取每次都被丟給 daemon。已移到根目錄。 +- **`mss` 墊片的測試量的是開發機的螢幕，不是它自己的假件。** + `test_screen_grabber.py` 只換掉 `backend_grab_image`，`_backend_screen_size` 還是讀真的 + `platform_wrapper.screen`，所以 `monitors[0]` 報的是開發者手邊那台螢幕。在 1920x1080 + 的桌面上會過，在 1280x800 的 Xvfb 下就失敗，而這跟被測程式碼無關。現在假件同時接管接縫的兩半。 + +## 本次更新 (2026-08-17) — Wayland 看得見螢幕了 + +- **所有擷取路徑改走平台後端。** `screenshot()`、影像與錨點定位、OCR、smart waits、 + 視覺回歸、螢幕錄影、MCP 螢幕工具與遠端桌面，先前各自直接呼叫 `PIL.ImageGrab` 或 + `mss`。這兩者在 Linux 都是讀 X11 root window——在 Wayland 下那個 root 屬於 XWayland， + 不會合成原生 Wayland 視窗。Pillow 確實有退回 `gnome-screenshot`／`grim`／`spectacle` + 的路徑，但它只在 X11 擷取**拋出例外**時才走（`except OSError`）——也就是根本沒有 X + display 的情況；只要 XWayland 在跑（GNOME、KDE、sway 的預設）就不會觸發。`mss` 則是 + 任何情況下都沒有後備。現在 `utils/cv2_utils/screen_grabber.py` 是唯一決定「這台機器怎麼 + 讀螢幕」的地方：後端若發布 `grab_image`，就把它包成呼叫端已在用的形狀。Windows、 + macOS 與 Linux X11 不發布任何東西，仍然使用真正的函式庫，行為完全不變。 +- **Wayland 擷取涵蓋三種合成器家族，而不只一種。** `grim` 只會說 `wlr-screencopy`， + GNOME 與 KDE 都沒有實作——所以 `linux_wayland/capture.py` 依序嘗試 `grim` + （sway、Hyprland、river）、`gnome-screenshot`、`spectacle`，並回報用了哪一個。只有 + `grim` 能自己接受區域參數，其餘的擷取整個螢幕後再裁切。 +- **沒有任何擷取工具時明確失敗。** 錯誤訊息會列出各合成器該裝什麼，而不是回傳空白的 + XWayland 擷取——後者在下游只會表現成「找不到樣板」。新的 `screen_capture` 診斷檢查 + 會在任何失敗發生前先講清楚目前用的是哪一個工具。 +- **`screen.size()` 與 `get_pixel` 在 GNOME／KDE 也能用。** 解析度從 `wlr-randr` 退回 + 以擷取結果量測；`get_pixel` 從可用的擷取路徑裁出 1x1 區域。 + +**仍未完成**：libei 原生輸入路徑與 Wayland 真機驗證，見 [Progress.md](../Progress.md)。 + ## 本次更新 (2026-07-03) — 穩定 API、失敗診斷包與發佈工程 給新整合用的版本化進入點、可攜式失敗診斷格式,以及強化後的發佈管線。完整參考:[`docs/API_LIFECYCLE.md`](../docs/API_LIFECYCLE.md) 與 [`docs/CAPABILITY_MATRIX.md`](../docs/CAPABILITY_MATRIX.md)。 diff --git a/WHATS_NEW.md b/WHATS_NEW.md index fe09f4fc..fac41b3f 100644 --- a/WHATS_NEW.md +++ b/WHATS_NEW.md @@ -1,5 +1,739 @@ # What's New — AutoControl +## What's new (2026-08-19) + +### Two Wayland Judgement Calls, Settled + +Two items had been sitting in `Progress.md` marked `DECIDE`: not missing work, +missing decisions. They turned out to be the same problem twice — a compositor +setting the library can *measure* but cannot *read*, and therefore must stop +pretending to know. + +- **Pointer acceleration is now something the operator declares, and the + library believes.** The measurement stands: `ydotool mousemove --absolute` + sends relative motion, libinput's default adaptive profile doubles it, and no + client can read the factor back. What was undecided was what to do about it — + keep warning and move anyway, refuse outright, or let the operator say. + Refusing outright would have taken `set_position` away from every Wayland + machine without `liboeffis`, so the answer is a declaration: + `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat` means acceleration is off for the + ydotoold device, and the move goes out silently and exactly; `=strict` means + refuse the move rather than let a click land somewhere else; unset keeps + today's warn-once-then-move, so nothing that works now stops working. An + unrecognised value falls back to the warning *and says that it did* — a typo + in a shell profile must not quietly promote a move to trusted-exact. The + whole gate is on the ydotool path; libei is absolute at the protocol level. +- **The software cursor in a Wayland capture is documented, not worked + around.** `seat-verification` measured it in passing: no capture in this + project passes `grim -c`, so none asks for the pointer, and the pointer is in + the image anyway. The reason is not ours — wlroots draws a *software* cursor + whenever the backend has no cursor plane, and a software cursor is composited + into the output buffer, which is exactly the buffer `wlr-screencopy` hands + back. Headless is permanently in that state; so is any real desktop running + `WLR_NO_HARDWARE_CURSORS=1`. Windows' BitBlt and the X11 path never include + the pointer, so this is a Wayland-only inconsistency, and with the pointer + resting on its target a locator, a template match or an OCR read sees a + pointer-shaped hole in the middle of it. Both ways out — move the pointer + away and back, or mask around it — need to know where the pointer is, and + Wayland does not let a client read that; an in-process guess goes stale the + moment the user touches their own mouse, and masking the wrong place is worse + than a visible cursor. So it is written down instead: in the capability + matrix, in all three READMEs, and in the diagnostics bundle, where the + `screen_capture` check now carries `cursor_may_be_captured` so the report + that explains a failed locator names the reason. `seat-verification` asserts + the behaviour as measured, so if wlroots ever honours `overlay_cursor` for + software cursors, CI goes red and tells us. + +### A Compositor That Consumes Input Was Three Environment Variables Away + +`ydotool mousemove --absolute` is not absolute. It emits no absolute event: +it sends `INT32_MIN` on both axes to drive the cursor into whatever corner the +compositor clamps to, then sends the target as a relative displacement. What +that corner *is*, and what the compositor does to the displacement on the way, +were the last two open questions on the Wayland input path — and both were +recorded as needing a VM running a desktop that consumes libinput devices. + +- **They needed no VM.** wlroots takes `WLR_BACKENDS=headless,libinput`: the + outputs stay virtual while the input half is the real libinput backend. + libseat's builtin backend opens the devices without logind, and + `SEATD_VTBOUND=0` stops it reaching for a VT no container owns. The fourth + requirement is the one that is easy to miss — libinput enumerates through + udev rather than through `/dev`, so `systemd-udevd` has to be running before + ydotoold creates its device. With those four in place, ydotoold's uinput + device is an ordinary seat device, and `grim -c` composites the cursor into + a screenshot, so the compositor answers in layout coordinates. +- **The corner is the layout's top-left, not layout `(0, 0)`.** Those are the + same point only while every output sits at a non-negative position. On the + layout every desktop with a monitor left of the primary one has, they differ + by the layout origin — so on a `-1280` layout an untranslated request for + layout `(0, 0)` put the cursor on the *other monitor*, 1,280 pixels away. + `mouse.set_position` now subtracts `layout_origin()` before handing the + coordinate to ydotool, which is the same correction the capture path already + applies; the lookup both input paths share moved into + `linux_wayland/_layout.py` so libei and ydotool cannot drift apart on it. +- **And pointer acceleration scales the rest.** The displacement is relative + motion, so libinput accelerates it: measured against a real wlroots session, + the default adaptive profile lands the cursor exactly twice as far from that + corner as asked, because `--absolute` sends both of its events in one frame + and the velocity saturates the profile. With `accel_profile flat` and + `pointer_accel 0` the same call is pixel-exact. ydotool's own `--help` has + said "You need to disable mouse speed acceleration for correct absolute + movement" all along; the backend now logs that caveat once per process + instead of letting a click land silently in the wrong place. Nothing else + can be done from inside the library — the factor is the compositor's + setting, not something a caller can read back. +- **A new `seat-verification` job holds all of it.** `docker/Dockerfile.seat` + runs the two layouts the capture image runs, and 14 checks each: that sway + really is holding the ydotool device, that `--absolute (0, 0)` draws the + cursor flush into the layout's first pixel, that with acceleration off the + move is one pixel per pixel, that an untranslated `(0, 0)` misses the + monitor it names, that `set_position` subtracts exactly the origin and lands + on the pixel it was given on both monitors, and that the acceleration factor + is the 2x this was measured at. Nothing in it depends on the cursor theme: + every claim is a difference between two captures, which the image's offset + from its hotspot cancels out of. +- **One thing it found on the way.** On this compositor a `grim` capture that + asks for no cursor overlay contains one anyway, because wlroots draws a + *software* cursor whenever the backend has no cursor plane — always on + headless, and on any session where the driver refuses one or the user set + `WLR_NO_HARDWARE_CURSORS=1`. Every locator, template match and OCR read goes + through that capture, so the pointer punches a pointer-shaped hole in + whatever it is sitting on. The check records the behaviour as measured, and + what to do about it is an open item in `Progress.md`. + +### The Screen-Capture Portal Could Never Have Worked, and a Real Bus Said So + +- **`xdg-desktop-portal` answers with a signal directed at the connection that + called it.** `Screenshot` returns a *request handle*, not an image; the image + arrives later as `org.freedesktop.portal.Request::Response`, addressed to the + caller's unique bus name. The bus routes a directed message to its + destination and nowhere else, so no match rule on any other connection can + make it arrive somewhere else. +- **The old implementation was two connections.** It started `gdbus monitor` in + one subprocess, made the call from a second `gdbus` invocation, and read the + monitor's stdout with a pair of regular expressions. Each `gdbus` invocation + opens its own connection under its own unique name, so the process listening + was never the process addressed. Measured against a real `dbus-daemon`: the + monitor sees the call go past and prints nothing else, and the capture runs + out its full 30-second timeout, every time. The only listener that can see a + directed signal is a full bus monitor — `dbus-monitor`, which asks the bus + for `BecomeMonitor` — and needing permission to observe every message on the + user's session bus is a poor price for a fallback screenshot. +- **The tier now speaks D-Bus itself, on one connection.** + `linux_wayland/_dbus_client.py` is a session-bus client in the standard + library alone: connect, SASL EXTERNAL authentication, `Hello`, `AddMatch`, + one method call, then read until the matching signal arrives. It is + deliberately not a general binding — no properties, no introspection, no + object export, no descriptor passing (liboeffis still does the one call that + needs that). `portal.py` subscribes to the request path it predicts from its + own unique name *before* it calls, and follows the returned handle as well + when a portal ignores `handle_token`. +- **Which also removes a dependency rather than adding one.** The tier used to + need `gdbus` (glib2) installed; it now needs nothing but a session bus, so + the last-resort capture path is available on strictly more desktops than + before. The install hint and the diagnostics check say so. +- **Verified end to end on a real bus.** A new `portal-verification` job runs a + real `dbus-daemon`, a real portal implementation and the real client: the + capture comes back as PNG bytes that decode to the pixels the portal painted, + at a percent-escaped path with a space in it, and the portal's file is gone + afterwards. Every way a portal ends without an image is driven too — a + dismissed dialog, a dialog left open, a success carrying no URI, a URI that + is not a local file — and each has to fail closed on AutoControl's own clock. + +### The RemoteDesktop Portal Handshake Never Needed a GNOME VM Either + +- **The portal is a D-Bus interface, not a compositor feature.** Reaching libei + on GNOME and KDE means `CreateSession` → `SelectDevices` → `Start` → + `ConnectToEIS`, ending in an EIS file descriptor passed over the bus. That + was recorded as unverifiable without a GNOME VM because + `xdg-desktop-portal-wlr` implements ScreenCast and Screenshot but not + RemoteDesktop — which confuses "no container ships one" with "no container + can host one". Whatever owns `org.freedesktop.portal.Desktop` and answers + those four calls *is* the portal, as far as `liboeffis` is concerned. +- **So the verification owns the name itself.** `docker/portal_server.py` is a + real D-Bus service on a private session bus, and its `ConnectToEIS` hands + back a live connection to the same real `libeis` server the `eis` image uses. + The real `liboeffis` runs the real handshake; the descriptor that comes out + carries a real EI session; and the key presses, absolute motion and button + edges emitted through it are recorded by an independent implementation at the + far end. +- **What it settles.** That the four calls arrive in the prescribed order at the + request paths the client predicted; that `SelectDevices` is asked for keyboard + and pointer and nothing wider, so the grant a user consents to is the one this + backend needs and `OEFFIS_DEVICE_DEFAULT` is not the `= 0` all-devices + sentinel; that the descriptor is a live socket the caller owns and must close, + which is what makes handing it to `ei_setup_backend_fd` — a function that + takes ownership — correct rather than a double close. +- **And every refusal.** A dismissed consent dialog, a dialog left open, a + withheld descriptor, a session the portal closes, a portal too old to have + `ConnectToEIS` at all, and no portal on the bus: each has to come back as a + refusal on this project's own clock rather than a hang or a silent downgrade. + The `OEFFIS_EVENT_CLOSED` branch had never had a peer able to drive it; it + does now. +- **What is still not claimed.** The consent dialog as a dialog. Nobody + dismisses anything in CI, so what a real mutter dialog looks like, and how + long a real person leaves it open, stays mutter's business. What a dialog + *produces* — a grant, a refusal, silence — is all exercised. + +### One Packaging Fact That Was Recorded Wrong + +- **Debian trixie does ship `liboeffis`.** `Progress.md` said it did not, and + concluded that the libei fast path was effectively off across Debian and + Ubuntu. Measured: `liboeffis1` 1.3.901-1 is in trixie/main, providing + `liboeffis.so.1`. What is true, and what actually matters to a user, is that + it is a *separate binary package* which `libei1` does not depend on — so + installing libei alone still leaves the portal route off and `connect()` + falls back to the `eis-0` socket that GNOME and KDE do not open. Install + `liboeffis` to get the fast path. + +### A Monitor Left of the Primary Broke Every Wayland Capture Path + +- **Wayland has no per-monitor screen, and the one it does have need not start + at `(0, 0)`.** The compositor lays every output out on a single plane, and + that plane starts at a negative coordinate the moment an output sits left of + or above the origin — which is what "my second monitor is on the left" means + to a compositor. sway's headless backend accepts `output HEADLESS-1 position + -1280 0`, so this is now a layout CI can stand up: two 1280x720 outputs, one + 2560x720 capture, top-left pixel at x=-1280. +- **`screen.size()` was returning the layout's right edge, not its width.** It + computed `max(x + width)` over the outputs, which is 1280 on that layout + while `grab_image()` returns a frame 2560 wide. Everything that composes the + two believed the smaller number: the mss-shaped shim's monitor list (and so + `enumerate_monitors`), the screen recorder, the WebRTC host and the MCP + monitor grab all asked for a rectangle half the size of the desktop and + reported it as the whole screen. +- **The crop for the tiers that cannot take a region cropped in the wrong + space.** Only grim accepts a geometry; gnome-screenshot, spectacle, the + portal and `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` all hand back the whole + layout and AutoControl crops afterwards. That crop used layout coordinates + against an image whose origin is the layout origin, so asking for + `[-1275, 5, -1175, 55]` — a rectangle on the left-hand monitor — asked + Pillow for a box 1275 px left of the frame and got black padding. +- **And a match found on that monitor was reported on the wrong one.** + `grab_logical`, the capture behind template search, OCR and visual match, + reads its origin from `GetSystemMetrics`, which says nothing off Windows — + so it returned `(0, 0)` and every hit came back 1280 px to the right of + where it was seen. That reads as "the click lands on the wrong screen" + rather than as a failure to find, which is the worse of the two. +- **Fixed at the seam, not at the call sites.** The Wayland backend publishes + `layout_origin()`; `size()` returns the bounding box's *size*; `grab_image` + subtracts the origin before cropping; and + `screen_grabber.backend_layout_origin()` is what `grab_logical` and the mss + shim ask, so a backend that captures its own screen can say where that + capture starts. Backends the generic libraries can already see (Windows, + macOS, X11) publish nothing and are unchanged — the origin is only ever + asked for, never guessed. +- **Verified against a real compositor, both ways round.** The + `wayland-verification` job now runs its 27 checks twice: once with the + outputs side by side from the origin, once with the left-hand one at + x=-1280. The second run is what pins the negative case end to end — grim's + negative `-g`, the reported size, `layout_origin()`, the mss shim's monitor + rectangle, `grab_logical`'s origin, and the fallback crop driven through the + operator override pointed at grim so a *real* whole-layout PNG goes through + the code path that has to shift it. + +### The Same Layout Problem, on the Input Side — a Move libei Was Dropping in Silence + +- **libei discards an absolute motion that lands in no region, and reports + nothing about it.** No return code, no event, no error the caller can see — + `ei_device_pointer_motion_absolute` simply does not put the event on the + wire. `set_position` then returned as though the pointer had moved. Measured + against a real EIS peer, not inferred: with the device offering one + `(0, 0, 1920, 1080)` region, `(1919, 1079)` arrives and `(1920, 1080)` + produces no server-side event at all. +- **And the space those regions live in need not be the layout's.** A region's + offset is a `uint32`, so no compositor *can* advertise one left of or above + the origin — while this project's layout space starts at `layout_origin()` + and goes negative the moment a monitor sits left of the primary. That is the + exact desktop the capture half was just fixed for. The two halves therefore + disagreed by the origin, which is the case where `get_pixel(x, y)` and + `set_position(x, y)` name different pixels — and the pointer that would have + gone to the wrong monitor instead went nowhere, quietly. +- **`LibeiBackend` now reads the device's regions and maps the point into + them.** `ei_device_get_region` and the four `ei_region_get_*` getters are + bound; `_region_point` sends a covered coordinate unchanged, retries an + uncovered one normalised by the layout origin, and refuses what neither + covers. A device that declared no region accepts anything and is passed + through untouched, which is measured too — that is the common single-monitor + case, and it costs nothing. +- **A refusal is the useful outcome, not a failure.** It is a + `LibeiUnavailable`, so `_select_input.emitted` hands the move to the ydotool + path exactly as it already does for a paused device. libei is documented as + the fast path and never the only one; the bug was that a dropped move never + reached the fallback because nothing knew it had been dropped. The frame is + not sent on a refusal either — nothing was buffered, and a frame there would + commit whatever the previous emission left on the device. +- **The layout origin is only consulted when a point misses.** It costs a + `wlr-randr` subprocess, so it stays off the path every ordinary mouse move + takes, and it answers `(0, 0)` on GNOME and KDE — which is the right answer + there rather than a fallback, because those compositors normalise the layout + themselves. +- **Five new checks in the `eis-verification` job, against the real + protocol.** That the client reads back the offsets the compositor + advertised; that a region at `x=1280` takes `1380` for a point 100 px into + it rather than `100`; that libei still drops an out-of-region motion without + a word — the measurement the whole guard rests on, so a future libei that + clamps instead says so; that AutoControl refuses such a move rather than + losing it; and that `(-1280, 10)` on a layout starting at `-1280` reaches + the server as `(0, 10)`. The job now runs 20 checks. +- **What this does not settle.** ydotool's `mousemove --absolute` has an + origin of its own — it clamps to the compositor's top-left corner and sends + the target as a relative delta — and whether that corner is the layout + origin still needs a compositor that consumes libinput devices. It stays + open in `Progress.md` rather than being changed on a guess. + +### The ydotool Path Was Reporting Success While Doing Nothing + +- **`apt install ydotool` — the hint this backend printed — installs a + version whose command line cannot run it, and which says so by exiting + zero.** ydotool 1.0 replaced the whole CLI, and every argument the Wayland + backend builds arrived in that release: `mousemove --absolute`, `mousemove + --wheel`, hex `click` bitmasks (which is what lets a press and a release be + sent separately, and therefore what makes drag possible), and `key + CODE:STATE` taking numeric evdev codes. Debian bookworm, Ubuntu 22.04 and + Ubuntu 24.04 all still ship 0.1.8 under that name. Measured against a real + uinput device, 0.1.8 answers `click 0x40` with **no events and exit code + 0**, and answers `mousemove --absolute` with `unrecognised option` — also + **exit code 0**. The backend runs ydotool with `check=True`, so a non-zero + status was the only thing that would have raised. On those distributions a + script clicked nothing, typed nothing and moved nothing, and every call + reported success. +- **The legacy CLI is now refused before anything is sent.** + `linux_wayland/_ydotool_cli.py` classifies the installed ydotool once per + process — mouse and key dispatch cannot afford a subprocess per event — and + raises with the three routes out: a 1.0+ package, a source build, or + `JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11`. Neither series implements + `--version` and 1.x will not answer `--help` without its daemon running, so + the probe reads the one thing both print with no daemon and no side + effects: the no-argument command list. A version it does not recognise is + allowed through rather than blocked, so a future release that changes that + banner cannot be locked out by a stale detector. +- **Both install hints were wrong in a second way.** Debian trixie ships no + `ydotool` package at all. The hints now name the distributions that do. + +### ydotool Never Needed a Desktop to Verify — Only a Reader + +- **The gap both verification images recorded turned out to be the wrong + gap.** `Dockerfile.wayland` and `Dockerfile.eis` each closed by saying + ydotool "needs /dev/uinput and a seat that consumes it", and `Progress.md` + filed that behind building a GNOME VM. A seat is what makes an injected + event *arrive somewhere*. It is not what makes one *observable*: ydotoold + creates an ordinary uinput device, the kernel publishes it as + `/dev/input/eventN`, and reading that node returns the exact `input_event` + structs ydotool wrote. No compositor, no session, no VM. +- **`docker/Dockerfile.ydotool` and `docker/ydotool_verify.py` do that, in + CI, in twelve checks.** They settle what had only ever been asserted + against mocks: `0xc0` / `0xc1` / `0xc2` really are BTN_LEFT / BTN_RIGHT / + BTN_MIDDLE; the split edges `0x40` and `0x80` really do send a press with + no release and a release with no press, which is the entire basis of + `press_mouse` and drag; `key 30:1 30:0` really does carry numeric evdev + codes; and **the wheel signs are measured rather than assumed** — `-y 1` + reaches the kernel as `REL_WHEEL +1`, `-y -1` as `-1`, and `-x 2` as + `REL_HWHEEL +2` with the axes not swapped. That last one is the assumption + `Progress.md` had flagged as untested since the scroll work landed. +- **The twelfth check drives the backend's own functions rather than a + hand-written argv**, so "what ydotool does with this command line" and + "what AutoControl sends" are joined rather than merely adjacent. +- **`mousemove --absolute` does not emit absolute events**, which is worth + knowing before trusting it. ydotool 1.x has no ABS axes on its device: it + sends `INT32_MIN` on both relative axes first, relies on the compositor + clamping that to the top-left corner, and then sends the target as a + relative delta. So `set_position` lands on the requested pixel *because of + that clamp*. The kernel side is now pinned; the clamp is the compositor's + behaviour and stays open. +- **The container gets `/dev/uinput` and character major 13, not + `--privileged`.** ydotoold creates its input node *after* the container + starts, which `--device` cannot cover, so the job grants + `--device-cgroup-rule 'c 13:* rmw'` and nothing else. + +### Scrolling on Wayland Stops Needing a uinput Daemon + +Motion, buttons and keys had already moved onto libei. Scroll was the one +input left shelling out to `ydotool` for every notch, and `Progress.md` said +why: the sign was a guess, and a scroll that goes the wrong way fails +silently. It is wired now, and the guess has been replaced with two +independent readings plus a measurement. + +- **The two paths count wheel detents in opposite directions.** This + repository's `wayland_scroll_direction_*` constants are in the kernel's + `REL_WHEEL` frame, because that is what ydotool writes into `/dev/uinput`: + positive is up. libei is in the `wl_pointer` / libinput frame, where + positive is down — libinput's own evdev reader negates `REL_WHEEL` to get + there, and the other libei sender that documents its sign (enigo) passes a + "positive scrolls down" value straight through to `scroll_discrete`. + Horizontal needs no flip: `REL_HWHEEL` and libinput both count right as + positive. So the vertical axis is negated on the way to libei and the + horizontal one is not, which is the decision `Progress.md` was waiting on. +- **The flip is checked against the real EIS server, negative value and + all.** A fifteenth check in `docker/eis_verify.py` drives the *public* + `mouse.scroll()` — not the backend method the earlier check drives — and + reads back `(0, -120)` for up, `(0, 120)` for down and `(120, 0)` for + right. It is also the only place a negative discrete value reaches the + wire; the earlier check only ever sent a positive one, so a marshalling + fault on the sign had nowhere to show up. +- **A refused emission now falls back to the CLI, which is what the code + always claimed.** `libei`'s module docstring says every failure raises + `LibeiUnavailable`, "which `keyboard` / `mouse` already treat as *use the + ydotool CLI*". Only the *connection* was treated that way. Once a backend + was handed over, a compositor that paused a device — or a session that + ended between two calls — raised straight out of `set_position`, + `press_key` or `hotkey`. Routing scroll through libei would have added a + fourth way for a script to die on a path that has a working fallback + sitting next to it, so the fallback was made real: a chord refused + part-way releases what it already pressed before handing over, so no + modifier is left held, and a button whose *release* is refused is released + by ydotool rather than staying down for the rest of the session. +- **`LibeiUnavailable` was escaping every containment boundary.** It + inherited `RuntimeError` alone, and `CLAUDE.md` is explicit that a + framework error which is not an `AutoControlException` "silently escapes + every boundary" — the executor, the background poll loops, the request + handlers, the GUI slots. It now inherits both, so the probes that catch + `RuntimeError` keep working and the boundaries finally see it. + +## What's new (2026-08-18) + +### The libei Input Path Now Has Something to Talk To + +The Wayland capture path was verified against a real compositor; the *input* +path was not, and was recorded as needing a GNOME VM. It does not. libeis is +the server side of libei's own protocol, Debian packages it, and the two +libraries will talk to each other over a plain Unix socket — so +`docker/eis_server.py` runs a real EIS implementation and +`docker/eis_verify.py` drives AutoControl's real sender against it. No +compositor, no desktop session, 14 checks, wired into CI. + +- **Discrete scroll was off by a factor of 120.** libei measures discrete + scroll in 120ths of a wheel click — the same convention as Windows' + `WHEEL_DELTA` — and `scroll()` was passing raw detent counts, so one click + asked for 1/120th of a scroll. libei says so at runtime ("suspicious + discrete event value 1, did you mean 120?"), which no mock was ever going to + print. `scroll(0, 1)` now arrives at the server as `(0, 120)`: one click, + right axis, right sign. This was the path `Progress.md` left deliberately + unwired because the *sign* was a guess; the sign turned out to be the + smaller half of the question. +- **The teardown no longer leaks a context per process.** `ei_unref` + segfaults on libei 1.3.901 — but only on a context whose backend opened and + whose handshake never progressed. With a peer to complete a handshake + against, the live case is finally testable, and it is safe. Teardown now + releases the devices and the context normally and abandons only the state + that actually crashes, instead of abandoning every opened backend on the + suspicion that it might. +- **The values a mock cannot check are checked.** The server offers six + capabilities and reads back what the client actually bound: exactly the four + AutoControl asks for, so both the `EI_DEVICE_CAP_*` bitmask and the variadic + `ei_seat_bind_capabilities` marshalling are right. Key codes, absolute + coordinates and button codes are read off the wire and compared. Every + emission is confirmed to carry a frame, and every device to open an + emulation transaction first — libei drops events from one that has not. +- **Two things measured but not ours to fix**, recorded rather than papered + over: `eis_device_pause()` puts nothing on the wire for a sender client on + libeis 1.3.901, so the client's `DEVICE_PAUSED` handling still has no peer + to exercise it; and the `start_emulating` sequence number does not survive + the trip (an explicit 4242 reads back as 0), so AutoControl's counter cannot + be checked from the far side. The check is written so that a libeis which + starts sending pauses will fail loudly if the client ignores them. + +### The Architecture Map's Line Counts Are Measured Again + +- **The map quoted the same subsystem at two different sizes.** `CLAUDE.md` + says every figure in `architecture_explore.md` is measured, but nothing + checked it, and two counting conventions had grown up side by side: the §5.4 + theme tables and the §5.4.17 file tables counted a phantom trailing line — + `len(text.split("\n"))` reports one line more than a file that ends in a + newline actually has, and one more *per file* for a package — while §1's + totals and the §8 appendix counted correctly. So `utils/executor/` was 8,811 + lines in one section and 9,001 in another, and the §8 column did not add up + to its own total. On top of that about fifty rows were simply stale, several + `####` headings were hundreds of lines out (`linux_wayland/` was still + quoted at 10 files / 1,093 lines against a real 14 / 2,235), and one theme + table had gained two subpackages its summary line never heard about. +- **413 figures re-measured** on one convention — `len(text.splitlines())`, + what `wc -l` reports and what `CLAUDE.md`'s own over-750-lines snippet + counts. §5.4, §5.4.17 and §8 now agree with each other for every subsystem, + and §8's rows sum to its stated total. +- **`test_doc_line_counts.py` is both the gate and the fix.** It fails CI when + any quoted line count stops matching the tree, naming the offending lines, + and rewrites all of them in place with `--fix`. The line counts were the one + part of the map with no gate — the command, MCP-tool, subpackage and example + counts already had `test_doc_counts.py` — which is exactly why they were the + part that drifted. + +### The Clipboard No Longer Fails Because Another Application Was Copying + +- **One process at a time may hold the Windows clipboard open, and every + clipboard call in AutoControl gave up the instant one did.** Explorer, + Office and every browser own the clipboard for a few milliseconds at a time + while they copy; `OpenClipboard` returns false for that whole window and all + six call sites — text, image, HTML, RTF, CSV, file drops, format + enumeration — turned it straight into `RuntimeError: OpenClipboard failed`. + Measured on a live desktop with a second process copying in a loop: about + one open in a thousand failed, which is a script that dies for no reason the + operator can see or reproduce. Win32 documents this as the condition to + retry, and a library whose job is driving a machine that other applications + are busy on cannot treat "somebody else was copying" as an error. +- **`win32_clipboard_api.open_clipboard()` is now the single place that opens + it**, waiting out a busy clipboard for roughly 200 ms before reporting + failure, and closing it however the block ends. The three modules that + hand-rolled the open/close pair — including the two that predated the shared + module — go through it, so the retry cannot be forgotten at a new call site. +- **The clipboard round-trip tests no longer depend on what the rest of the + machine is doing.** They exercise the real Win32 calls, which is the only + place the four historical writer bugs could ever be seen, so faking the + backend would have deleted the coverage instead of stabilising it. They now + read the Win32 clipboard sequence number instead: unchanged between the + write and the read means nothing else wrote in that window, so the assertion + is about AutoControl's code and nothing else. Verified against a process + making 4,112 competing clipboard writes during the run. + +### A Misspelled Command Name Is a 400, Not a 500 + +- **`POST /execute` answered `500 {"error": "execute_action failed"}` for an + `AC_*` name that does not exist**, which is the same answer it gives when + the server itself breaks. A client could not tell a typo in its own request + from an outage, and the message did not say which name was unrecognised. +- **Every command name is now checked before anything runs**, and an + unrecognised one comes back as `400` listing *all* of them in + `unknown_commands` — nested flow-control bodies included — so a client fixes + every typo in one round trip. `POST /execute_file` answers the same way for + a file that is unreadable, is not an action list, or names an unknown + command. The OpenAPI spec documents both, and states that a rejected request + executed nothing. +- Validation and collection share one traversal in `action_schema`, so there + is still exactly one definition of where a nested action list may hide. + +### A Segfault in the libei Teardown, and the Binding Checked Against the Real Library + +- **`ei_unref` crashes the process on libei 1.3.901 once a backend is open, + and the fallback path ran straight into it.** Every failure mode of the + libei handshake ends in `_teardown()`, so on any host where libei is + installed and the handshake does not complete, AutoControl died with + SIGSEGV instead of quietly using ydotool — the exact opposite of the + fail-closed promise. Measured one call at a time against the real library: + `ei_unref` is safe with no backend set up and safe after a *failed* setup, + and segfaults after a successful one. `ei_disconnect` crashes in the same + state, so it is not a refcounting mistake here; the header documents + `ei_unref` as correct for both outcomes, which makes this an upstream bug. + An opened backend is now abandoned rather than unreffed — a bounded leak of + one context per process, against a crash in a library that drives a desktop. + A sentinel in the verification re-checks the upstream state on every run and + says so when the workaround can go. +- **Every entry point the binding names is now resolved against the real + `libei.so`.** A misspelled symbol or a wrong `argtypes` sails past a mocked + symbol table and only surfaces on a user's machine; all 22 prototypes plus + the variadic `ei_seat_bind_capabilities` are checked for real. +- **The whole fail-closed chain runs end to end**: connect to a socket that + speaks no EI → handshake times out → `LibeiUnavailable` → `active_backend()` + returns None → `press_key` falls through to the ydotool CLI. +- **`liboeffis` is not packaged everywhere.** Arch and Fedora ship it; Debian + trixie does not. Without it there is no portal route, so `connect()` falls + back to the well-known EIS socket — which GNOME and KDE do not create. The + libei fast path is therefore unavailable on those systems, and says so + rather than looking mysteriously idle. + +### The Wayland Capture Path Now Meets a Real Compositor + +- **`screen.size()` reported one monitor while `grab_image()` returned the + whole layout.** The `wlr-randr` parser took the first `WxH` anywhere in the + document, which is the first output's current mode. On a two-monitor layout + that is half the screen — and the two are composed by the mss-shaped shim, + so the recorder, WebRTC and MCP monitor paths asked for a region half the + size of the screen and got it. `size()` is now the layout bounding box, from + a parser that reads every enabled output's mode *and* position. Found by + running against a real compositor; no mock had a second monitor. +- **`docker/Dockerfile.wayland` runs the backend under headless sway.** The + wlroots headless backend needs no GPU, no seat and no display, so a genuine + Wayland session fits in a container — and now in CI, as the + `wayland-verification` job. Two outputs are painted different solid colours, + because on a uniform screen a region grab cannot be caught reading the wrong + rectangle, and a red/blue swap cannot be caught at all. +- **21 checks that were previously mock-only now run against pixels the + compositor painted**: grim's argv and `-g` geometry, RGB channel order, + `wlr-randr`'s undocumented output format, `size()` / `grab_image()` / + `get_pixel()` / `screenshot()`, `je_auto_control.screenshot()`'s BGR output, + `grab_logical()` (the locator and OCR path), the mss shim, and `wtype`. +- **What the container cannot answer is stated rather than glossed over.** + ydotool needs `/dev/uinput` and headless sway consumes no libinput devices; + `xdg-desktop-portal-wlr` implements no RemoteDesktop, so there is no + `ConnectToEIS` to test. Both remain open in `Progress.md`. + +### libei Input, End to End + +- **The full portal handshake is implemented.** libei is not a + call-a-function-and-a-key-is-pressed library: a sender has to open an EIS + backend, bind a seat's capabilities, take a device *out of an event*, start + emulating on it, and follow every emission with `ei_device_frame` or nothing + is delivered. All of that now happens, so `press_key`, `set_position` and the + mouse buttons can emit without spawning a process per event. +- **The EIS socket comes from the desktop portal.** On GNOME and KDE it is not + a path on disk — it is a file descriptor handed over D-Bus by + `org.freedesktop.portal.RemoteDesktop.ConnectToEIS`, after a three-call + asynchronous session dance. No command-line tool can pass a file descriptor + into this process, so the `gdbus` route used for screenshots cannot work + here; `liboeffis` (which ships with libei for exactly this) does the dance. + Where liboeffis is absent, the well-known `$XDG_RUNTIME_DIR/eis-0` socket is + still tried. +- **Devices come from events, never from the context.** The previous binding + passed the `struct ei *` context to entry points that take a + `struct ei_device *` — pointer type confusion in a C library. That is now + impossible by construction. +- **A failed probe is paid once, not per keystroke.** The handshake involves a + portal round trip and possibly a consent dialog. The result is cached for the + process, so a host where libei is installed but unusable does not re-attempt + it on every key press. +- **Everything still falls back to ydotool.** Missing library, declined + consent, a partial capability grant, a paused device, a handshake that does + not complete — each raises `LibeiUnavailable`, which the keyboard and mouse + modules already treat as "use the CLI". Scroll deliberately stays on ydotool: + its direction convention is pinned by tests, and a wrong sign would fail + silently rather than loudly. +- **The ABI constants were checked against the upstream headers**, not + guessed. Three were wrong. `enum ei_device_capability` is a *bitmask*, so + `EI_DEVICE_CAP_KEYBOARD` is `1 << 2`, not 3; `OEFFIS_EVENT_CLOSED` comes + *before* `OEFFIS_EVENT_DISCONNECTED`; and `OEFFIS_DEVICE_ALL_DEVICES` is a + `= 0` sentinel rather than the OR of the device bits. The capability error + was the expensive one — no device would ever have reported the capability, + so every session would have timed out and silently used the CLI. The + verified values are now pinned by tests. The session also asks only for the + keyboard and pointer it actually drives, so the consent dialog does not + request a touchscreen grant nothing uses. + +### Wayland Capture Has a Floor Under It + +- **`xdg-desktop-portal` backs up the three CLI helpers.** None of `grim`, + `gnome-screenshot` or `spectacle` is guaranteed to be installed — GNOME has + not shipped `gnome-screenshot` by default since 42 — so + `org.freedesktop.portal.Screenshot` is tried last through `gdbus` rather than + giving up. It is awkward by nature: the portal returns a request handle and + answers later with a signal, so the listener starts before the call is made, + and the wait is bounded (30s) because a consent dialog can sit in front of it. +- **An operator can name their own capture command.** + `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND="mycap --png {output}"` wins over + every detected tool. `{output}` becomes a temporary PNG path, substituted per + argument after `shlex.split` and run without a shell, so a path with spaces + stays one argument. This is the escape hatch for a setup none of the built-in + tiers fit — including one where our argv guess for a helper turns out wrong. +- **libei refuses a connection it cannot emit through.** The binding only ever + holds an `ei` context, while every device entry point takes an `ei_device`, + and it runs none of libei's seat / device / `start_emulating` / `frame` + handshake — so it could not deliver input, but *could* pass the wrong pointer + into a C library on a host that opens `$XDG_RUNTIME_DIR/eis-0`. It now stops + at `connect()` with an explanation. Callers already treated that as "use the + ydotool CLI", which is what every real desktop was doing anyway. +- **The portal listener cannot wedge on shutdown.** Its pipe is closed only + after the reader thread lets go of it; closing a stream out from under a + blocked `read()` can hang on the buffer lock, which would have turned a + timed-out capture into the hang the timeout exists to prevent. + +### The Container Image Builds and Starts From a Windows Checkout + +- **Every container built on a Windows clone died on startup.** `.gitattributes` + said `* text=auto`, so `docker/entrypoint.sh` and `docker/entrypoint-xfce.sh` + were checked out with CRLF. The shebang then reads `#!/bin/sh`, the kernel + looks for an interpreter whose name ends in a carriage return, and the image + builds perfectly and then exits with `exec /usr/local/bin/autocontrol-entrypoint: + no such file or directory` — a message that names the file it just failed to + find. CI never saw it: a Linux runner checks the same file out with LF. + `*.sh text eol=lf` now pins it, and a test asserts no entrypoint carries CRLF. +- **`.dockerignore` was in `docker/`, where Docker does not look.** Docker reads + it from the build *context* root, and every documented build passes the + repository root (`docker build -f docker/Dockerfile .`), so the exclusions did + nothing: `.git`, `.venv`, `test/` and the caches were all being shipped to the + daemon on every build. Moved to the root, where it takes effect. +- **The `mss` shim test measured the host's monitor, not its own fake.** + `test_screen_grabber.py` patched `backend_grab_image` but left + `_backend_screen_size` reading the real `platform_wrapper.screen`, so + `monitors[0]` reported whatever display the developer had. It passed on a + 1920x1080 desktop and failed under a 1280x800 Xvfb, for reasons unrelated to + the code under test. The fake now owns both halves of the seam. + +## What's new (2026-08-17) + +### Wayland Sees the Screen + +- **Every capture path now goes through the platform backend.** `screenshot()`, + the image and anchor locators, OCR, smart waits, visual regression, screen + recording, the MCP monitor tools and remote desktop each reached for + `PIL.ImageGrab` or `mss` directly. Both read the X11 root window on Linux, + which under Wayland belongs to XWayland and does not composite native Wayland + windows. Pillow does fall back to `gnome-screenshot` / `grim` / `spectacle`, + but only inside `except OSError` around its X11 grab — so it fires when there + is no X display at all, and *not* while XWayland is up, which is the default + on GNOME, KDE and sway. `mss` has no fallback in any configuration. + `utils/cv2_utils/screen_grabber.py` is now the one place that decides how + pixels are read: a backend that publishes `grab_image` gets wrapped in + whichever library shape the caller already uses. Windows, macOS and Linux X11 + publish nothing and keep the real libraries, so their behaviour is + byte-for-byte unchanged. +- **Wayland capture covers three compositor families, not one.** `grim` only + speaks `wlr-screencopy`, which GNOME and KDE do not implement — so + `linux_wayland/capture.py` tries `grim` (sway, Hyprland, river), then + `gnome-screenshot`, then `spectacle`, and reports which one it used. Only + `grim` can take a region itself; the others capture the screen and the region + is cropped from it. +- **A missing capture tool fails loudly.** It raises with the install command + for each compositor rather than handing back an empty XWayland grab that + reads downstream as "template not found". The new `screen_capture` + diagnostics check names the tool in use before anything has to fail. +- **`screen.size()` and `get_pixel` work off GNOME/KDE too.** Resolution falls + back from `wlr-randr` to measuring a capture, and `get_pixel` crops a 1x1 + region from whichever capture path is available. + +### Which Program Owns That Window + +- **`foreground_window_process_id` / `window_process_id`** (`AC_foreground_window_pid`, + `AC_window_pid`, `ac_foreground_window_pid`, `ac_window_pid`, two Script Builder + specs; Windows backend `get_window_process_id`): a window title is whatever the + application decides to display, and unrelated programs share titles like + `Settings`, so "which program is the user in front of" had no reliable answer. + It does now, and callers can join it against a process list. Unavailable reads + as `None` — never a bare `0`, which would match the System Idle Process. +- **The two older window-input functions are deprecated and now work.** + `send_key_event_to_window` / `send_mouse_event_to_window` posted to the frame, + which reaches nothing in a window with child controls; they delegate to the + new pair, warn once, and keep their old argument types working. +- **`get_pixel` no longer risks a truncated device context.** The Windows screen + backend declared no prototypes, so an HDC — pointer-width — came back through + ctypes' default `c_int`, and the same truncated value was passed on to + `GetPixel` and `ReleaseDC`. Every prototype is declared and the module owns + private DLL handles, so the declarations cannot leak into other callers. +- **Act on a process's windows, not on a title** + (`windows_for_process_id`, `minimize_windows_for_process`): a browser names + its windows after the page they show and runs a dozen processes without any + window, so "minimise that application" was previously hand-rolled Win32 in + every caller — enumerate windows, ask each one which process owns it, filter, + minimise. That loop now lives here once. +- **Typing into a window without stealing focus actually works now** + (`post_key_to_window`, `post_click_to_window`, `AC_post_key_to_window`, + `AC_post_click_to_window`, `ac_post_key_to_window`, `ac_post_click_to_window`). + The existing `send_key_event_to_window` posted to the top-level frame, and + keyboard messages go to the control that *has focus* — so it silently did + nothing in any application with child controls while still reporting success. + Measured on Character Map: posting to the frame typed nothing, posting to the + focused edit typed the character; on Windows 11 Notepad the new path types + into a background window while the foreground window keeps focus. Clicks + resolve the deepest child under the point and convert to its client + coordinates. This is still best effort by nature — games and anything reading + raw input ignore posted messages — so both functions return whether the + messages were queued rather than claiming the application acted. +- **Half the clipboard was broken on 64-bit Windows, and nothing noticed.** + `set_clipboard_files`, `set_clipboard_html`, `set_clipboard_rtf` and + `set_clipboard_csv` raised `OverflowError` on every call — four writers that + had never once worked. Each module declared `restype` but not `argtypes`, so + a pointer-width memory handle went through ctypes' default `c_int`. The + pure-function tests could not see it (the byte packing was always right) and + nothing exercised the Win32 half. The prototypes now live once in + `utils/clipboard/win32_clipboard_api.py`, every clipboard module goes through + it, and a new test round-trips text, HTML, RTF, CSV and file lists through the + real clipboard — plus a static check that no module hand-rolls those calls + without declaring prototypes again. +- **`save_window_layout` no longer records windows it cannot restore.** Its + docstring promised titled windows; the default lister returned all of them, + while `restore_window_layout` addresses a window by title and skips blank ones. + On a real desktop that was 28 entries saved against 15 restorable — a caller + reporting the saved count was over-promising by a factor of two. The lister now + passes `titled_only=True`, and a round-trip test pins save and restore to the + same set. + ## What's new (2026-08-15) ### Text Entry and On-Screen Location That Match What You See diff --git a/architecture_explore.md b/architecture_explore.md index 52cb1a4e..c90b5cfd 100644 --- a/architecture_explore.md +++ b/architecture_explore.md @@ -6,7 +6,7 @@ > 擷取每個模組的 docstring 與頂層公開名稱；統計數字取自實際檔案，非估算。 > 指令數與公開 API 數以 `executor.known_commands()` 與 `je_auto_control.__all__` 在工作樹上實測取得。 > -> **掃描時間**：2026-08-16　**版本**：`pyproject.toml` version `0.0.195`　**分支**：`feat/desktop-automation-gaps` +> **掃描時間**：2026-08-19　**版本**：`pyproject.toml` version `0.0.218`　**分支**：`fix/clipboard-handles-and-window-input` --- @@ -19,14 +19,14 @@ iOS（WebDriverAgent）。核心能力是滑鼠／鍵盤控制、影像辨識、 | 指標 | 數值 | | --- | ---: | -| Python 模組總數（含周邊子專案） | 998 | -| 程式碼總行數 | 133,534 | +| Python 模組總數（含周邊子專案） | 1,016 | +| 程式碼總行數 | 137,517 | | `je_auto_control/utils/` 子套件數 | 308 | -| `AC_*` 動作指令數（`known_commands()` 實測） | 767 | -| 套件門面 `__all__` 公開名稱數 | 1,221 | +| `AC_*` 動作指令數（`known_commands()` 實測） | 773 | +| 套件門面 `__all__` 公開名稱數 | 1,238 | | GUI 分頁數（`main_widget` 註冊） | 48 | -| MCP 工具數（`build_default_tool_registry()` 實測） | 670 | -| `test_*.py` 測試檔／測試函式 | 458 / 4,319 | +| MCP 工具數（`build_default_tool_registry()` 實測） | 676 | +| `test_*.py` 測試檔／測試函式 | 466 / 4,443 | | 範例腳本 | 27 | **技術基線**：Python ≥ 3.10、MIT 授權、必要相依只有 `je_open_cv`／`opencv-python`／`pillow`／`mss`／ @@ -49,7 +49,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 │ 全部只呼叫下面這一層，不含業務邏輯 ┌───────────────────────────────▼──────────────────────────────────────────┐ │ 執行核心 Execution Core │ -│ utils/executor/action_executor.py ── Executor.event_dict（767 個 AC_*） │ +│ utils/executor/action_executor.py ── Executor.event_dict（773 個 AC_*） │ │ utils/executor/flow_control.py ── 34 個區塊指令（迴圈/分支/try/巨集） │ │ utils/script_vars ── ${var} 插值 │ utils/json ── action 檔 I/O │ └───────────────────────────────┬──────────────────────────────────────────┘ @@ -71,7 +71,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 │ windows/ (ctypes Win32 + Interception 驅動) │ │ osx/ (pyobjc Quartz) │ │ linux_with_x11/ (python-Xlib + 選用 uinput) │ -│ linux_wayland/ (libei ctypes + ydotool/wtype/grim CLI) │ +│ linux_wayland/ (libei via portal/EIS + ydotool/wtype + 擷取工具分層) │ │ android/ (ADB + uiautomator2) │ ios/ (WebDriverAgent) │ └──────────────────────────────────────────────────────────────────────────┘ ``` @@ -97,6 +97,7 @@ USB/IP 協定、Prometheus 指標），以維持這條輕相依基線。 | **Observer** | `utils/callback/`、`utils/observer/`、`utils/triggers/` | 動作完成後觸發回呼；畫面出現／消失／變化與外部事件（webhook／IMAP／檔案）驅動腳本。 | | **Template Method** | `utils/generate_report/` | HTML／JSON／XML 三個產生器共用「收集紀錄 → 格式化 → 寫檔」骨架，各自實作渲染。 | | **Adapter / Backend seam** | `accessibility/backends/`、`ocr/backends/`、`vision/backends/`、`llm/backends/`、`agent/backends/`、`hotkey/backends/`、`usb/passthrough/*_backend.py`、`usbip/backend.py` | 每個外部能力都有抽象基底 + 具體實作 + null fallback，讓無相依環境仍可載入與測試。 | +| **Adapter（螢幕擷取）** | `utils/cv2_utils/screen_grabber.py` | 全框架唯一決定「這台機器怎麼讀螢幕」的地方。平台後端若發布 `grab_image`（目前只有 Wayland 需要），就把它包成呼叫端已在用的形狀（`ImageGrab` 或 `mss`）；否則原樣交還真正的函式庫，Windows／macOS／X11 行為完全不變。後端另可發布 `layout_origin`，由 `backend_layout_origin()` 轉給需要把畫面上的點換回螢幕座標的路徑（`grab_logical`、mss shim 的 monitor 矩形）。 | | **Registry / Singleton** | `remote_desktop/registry.py`、`rest_api/rest_registry.py`、`profiler`、`run_history`、`secrets` | 行程級單例，讓 `AC_*` 指令能操作長生命週期的伺服器與狀態。 | --- @@ -151,13 +152,13 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `je_auto_control/__init__.py` | 1,927 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | -| `je_auto_control/__main__.py` | 71 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | -| `je_auto_control/cli.py` | 327 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | -| `je_auto_control/api/__init__.py` | 23 | 版本化整合進入點。 | -| `je_auto_control/api/core.py` | 20 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | -| `je_auto_control/utils/deprecation.py` | 36 | 公開 API 的一致性棄用警告。 | -| `je_auto_control/utils/http_headers.py` | 33 | 入站 HTTP 標頭的共用防禦式解析。 | +| `je_auto_control/__init__.py` | 1,970 | **套件門面**。集中匯入並再匯出 1,200 個公開名稱，以功能區塊註解分段（callback／exception／executor／a11y／vision／clipboard…）。 | +| `je_auto_control/__main__.py` | 70 | 舊版 argparse 進入點：`-e` 執行單檔、`-d` 執行整個目錄、`--execute_str` 執行 JSON 字串、`-c` 建立專案。 | +| `je_auto_control/cli.py` | 326 | **主 CLI**（`je_auto_control` console script）。子命令：`run`（含 `--var`／`--dry-run`）、`validate`／`lint`、`list-commands`、`fmt`、`record`、`codegen`、`failure-bundle`、`list-jobs`、`start-server`、`start-rest`、`version`。所有子命令延遲匯入，確保不碰 Qt。 | +| `je_auto_control/api/__init__.py` | 22 | 版本化整合進入點。 | +| `je_auto_control/api/core.py` | 19 | **穩定無頭 API 門面**：只暴露 `execute_action`、`execute_action_with_vars`、`generate_code`、`run_diagnostics`、`create_failure_bundle`、`failure_bundle_on_error`、`FailureBundleOptions`。mypy 型別契約只針對這一面。 | +| `je_auto_control/utils/deprecation.py` | 35 | 公開 API 的一致性棄用警告。 | +| `je_auto_control/utils/http_headers.py` | 32 | 入站 HTTP 標頭的共用防禦式解析。 | ### 5.2 wrapper 抽象層 @@ -165,21 +166,21 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `wrapper/platform_wrapper.py` | 60 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | -| `wrapper/_platform_windows.py` | 326 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | -| `wrapper/_platform_osx.py` | 150 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | -| `wrapper/_platform_linux.py` | 268 | X11 後端組裝（python-Xlib + 選用 uinput）。 | -| `wrapper/_platform_wayland.py` | 58 | Wayland 後端組裝（libei／ydotool／grim）。 | -| `wrapper/auto_control_mouse.py` | 304 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | -| `wrapper/auto_control_keyboard.py` | 228 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | -| `wrapper/auto_control_screen.py` | 98 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | +| `wrapper/platform_wrapper.py` | 59 | **Strategy 樞紐**。依 `sys.platform` 匯入唯一後端並匯出 `keyboard`、`keyboard_check`、`keyboard_keys_table`、`mouse`、`mouse_keys_table`、`special_mouse_keys_table`、`screen`、`recorder`；載入失敗直接拋 `AutoControlException`（fail fast）。 | +| `wrapper/_platform_windows.py` | 325 | Windows 後端組裝：Win32 ctypes 模組 + 虛擬鍵表 + 選用 Interception 驅動。 | +| `wrapper/_platform_osx.py` | 149 | macOS 後端組裝（Quartz 事件 + osx 虛擬鍵表）。 | +| `wrapper/_platform_linux.py` | 267 | X11 後端組裝（python-Xlib + 選用 uinput）。 | +| `wrapper/_platform_wayland.py` | 57 | Wayland 後端組裝（libei／ydotool／grim）。 | +| `wrapper/auto_control_mouse.py` | 346 | 滑鼠 API：位置讀寫、按下／放開／點擊、捲動、座標前處理、送訊息給指定視窗。 | +| `wrapper/auto_control_keyboard.py` | 273 | 鍵盤 API：鍵表查詢、按下／放開／敲擊、`write` 字串、`hotkey` 組合鍵、按鍵狀態偵測。 | +| `wrapper/auto_control_screen.py` | 97 | 螢幕 API：`screen_size`、`screenshot`（可指定區域）、`get_pixel`。 | | `wrapper/auto_control_image.py` | 83 | 影像 API：`locate_all_image`、`locate_image_center`、`locate_and_click`。 | -| `wrapper/auto_control_record.py` | 76 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | -| `wrapper/auto_control_window.py` | 94 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態（目前僅 Windows 實作）。 | +| `wrapper/auto_control_record.py` | 106 | 錄製 API：`record`／`stop_record`／`record_to_json`（支援 stop event 與逾時）。 | +| `wrapper/auto_control_window.py` | 293 | 視窗管理門面：列舉、尋找、聚焦、等待、關閉、顯示狀態、幾何、所屬行程 PID、依行程列舉／最小化視窗、不搶焦點的投遞式輸入（目前僅 Windows 實作）。 | ### 5.3 平台後端 -#### Windows（`windows/`，26 檔／1,939 行） +#### Windows（`windows/`，23 檔／1,995 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -190,14 +191,14 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `keyboard/win32_ctype_keyboard_control.py` | 55 | 鍵盤事件產生。 | | `record/win32_input_hook.py` | 223 | 單一一組低階鍵鼠 hook（`WH_KEYBOARD_LL`／`WH_MOUSE_LL`）＋訊息迴圈，產生帶時間戳的事件時間軸；停止時以 `PostThreadMessageW(WM_QUIT)` 收掉執行緒，不會每錄一次就漏一條。 | | `record/win32_record.py` | 126 | 把 `win32_input_hook` 的時間軸轉成 action list（含按鍵放開、滾輪與間隔）。 | -| `screen/win32_screen.py` | 53 | 螢幕尺寸與像素讀取。 | -| `window/windows_window_manage.py` | 208 | 視窗列舉／聚焦／關閉／最小化／幾何（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | +| `screen/win32_screen.py` | 89 | 螢幕尺寸與像素讀取。**每支 Win32 函式都明寫 argtypes/restype**（HDC 是指標寬度，走預設的 c_int 會截斷，錯誤會沉默地擴散到 GetPixel／ReleaseDC），並持有自己的 user32／gdi32 handle。import 時呼叫 `SetProcessDPIAware()`——**行程層級且不可還原**，實體↔邏輯座標換算請走 `utils/monitor_layout`。 | +| `window/windows_window_manage.py` | 366 | 視窗列舉／聚焦／關閉／最小化／幾何／所屬行程 PID／投遞式輸入（`auto_control_window` 的實作）。**每支 Win32 函式都明寫 argtypes/restype**，並持有自己的 user32 handle，避免把原型外溢到別的模組；hwnd 一律是 int。 | | `message/window_message.py` | 97 | 直接對視窗送 `WM_*` 訊息（背景輸入）。 | | `interception/_dll.py` | 231 | `interception.dll` 的延遲 ctypes 載入與結構定義。 | | `interception/keyboard.py` | 71 | 經 Interception 驅動的鍵盤輸入（繞過部分反自動化偵測）。 | | `interception/mouse.py` | 161 | 經 Interception 驅動的滑鼠輸入。 | -#### macOS（`osx/`，17 檔／771 行） +#### macOS（`osx/`，17 檔／761 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -210,7 +211,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `screen/osx_screen.py` | 143 | 螢幕擷取與尺寸（含 Retina 座標處理）。 | | `pid/pid_control.py` | 64 | 以 PID 操作應用程式。 | -#### Linux X11（`linux_with_x11/`，19 檔／1,189 行） +#### Linux X11（`linux_with_x11/`，19 檔／1,175 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | @@ -225,30 +226,37 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `uinput/keyboard.py` | 33 | uinput 鍵盤後端，介面與 X11 版一致。 | | `uinput/mouse.py` | 116 | uinput 滑鼠後端。 | -#### Linux Wayland（`linux_wayland/`，10 檔／1,093 行） +#### Linux Wayland（`linux_wayland/`，17 檔／3,431 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `_detect.py` | 66 | Wayland session 偵測與 CLI 工具探測。 | -| `_select_input.py` | 50 | 決定使用原生 libei 或 CLI shim。 | -| `libei.py` | 243 | libei（Wayland HID 層輸入模擬）的 ctypes 綁定。 | -| `mouse.py` | 168 | 經 ydotool 的滑鼠後端。 | -| `keyboard.py` | 149 | 經 ydotool + wtype 的鍵盤後端。 | -| `keymap.py` | 156 | 友善鍵名 → evdev key code。 | -| `screen.py` | 136 | 經 grim + wlr-randr 的螢幕後端。 | -| `listener.py` / `record.py` | 49 / 35 | 監聽與錄製 stub（Wayland 限制）。 | +| `_detect.py` | 77 | Wayland session 偵測與 CLI 工具探測。 | +| `_ydotool_cli.py` | 134 | 判定安裝的是哪一代 ydotool 命令列，擋掉會靜默失效的 0.1.x（對本專案送的 argv 回傳 0 卻不送任何事件）。 | +| `_ctypes_bind.py` | 75 | libei／liboeffis 共用的 ctypes 載入與 prototype 綁定。 | +| `_dbus_client.py` | 625 | 只用標準函式庫的 D-Bus session bus 客戶端（連線／認證／`Hello`／`AddMatch`／一次方法呼叫／等訊號）。portal 的回應是**指名送給發出呼叫的那條連線**,所以訂閱與呼叫必須同一條連線——這是 `gdbus monitor` + `gdbus call` 兩個行程做不到的事。 | +| `_select_input.py` | 85 | 決定使用原生 libei 或 CLI shim；`active_backend()` 是 keyboard／mouse 的唯一入口，`emitted()` 讓被拒絕的單次發送退回 CLI。 | +| `_layout.py` | 83 | 版面原點的共用查詢。擷取與輸入不是同一個座標空間,差的就是這個原點:libei 的 region offset 是 `uint32`（描述不了負原點）,`ydotool mousemove --absolute` 的原點是合成器夾取的那個角落——兩條路都要減掉它,所以放在這裡而不是各自複製。讀數快取一秒——擷取那一側刻意不快取,但 ydotool 每次絕對移動都會問,不快取等於每次移動多開一個 `wlr-randr` 行程。 | +| `oeffis.py` | 196 | liboeffis 綁定：跑完 RemoteDesktop portal 交握，交出 EIS fd。 | +| `libei.py` | 610 | libei 綁定與完整握手（seat 綁定能力 → 由事件取得 device → start_emulating → 每次發送後 frame）。另負責絕對指標的座標空間:讀回裝置的 region,把版面座標映射進去,沒有任何 region 涵蓋就拒絕（libei 對這種移動是靜靜丟掉的）。 | +| `mouse.py` | 379 | 滑鼠後端：移動、按鈕與捲動都 libei 優先，退回 ydotool；送往 libei 時垂直捲動軸取負（kernel `REL_WHEEL` 與 `wl_pointer` 正負號相反）。退到 ydotool 的絕對移動會先減掉版面原點（`--absolute` 是相對於版面左上角,不是版面座標的 `(0, 0)`),並依 `pointer_accel_mode()` 處理指標加速度——倍率讀不回來,只有操作者知道,所以由 `JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` 宣告:未設定＝每個行程警告一次後照送、`flat`＝已關掉加速度故靜靜送出、`strict`＝拒絕這次移動。 | +| `keyboard.py` | 173 | 鍵盤後端：libei 優先，退回 ydotool／wtype。 | +| `keymap.py` | 155 | 友善鍵名 → evdev key code。 | +| `capture.py` | 236 | 擷取分層：操作者自訂指令 → grim → gnome-screenshot → spectacle → portal。 | +| `portal.py` | 207 | `org.freedesktop.portal.Screenshot` 最後備援,經 `_dbus_client` 直接講 D-Bus（不再需要安裝 `gdbus`,只要有 session bus）。 | +| `screen.py` | 266 | 螢幕後端；發布 `grab_image` 與 `layout_origin`（擷取畫面左上角的版面座標，有螢幕在主螢幕左側／上方時為負），全框架的擷取都經由它。 | +| `listener.py` / `record.py` | 48 / 34 | 監聽與錄製 stub（Wayland 限制）。 | #### 行動裝置 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `android/adb_client.py` | 184 | `adb` CLI 的薄封裝。 | -| `android/client.py` | 92 | `uiautomator2.Device` 的延遲封裝。 | -| `android/find.py` | 105 | uiautomator2 widget 樹的元素查詢。 | -| `ios/client.py` | 95 | `facebook-wda`（WebDriverAgent）封裝。 | -| `ios/find.py` | 87 | XCUITest 無障礙查詢。 | -| `ios/input.py` | 47 | iOS 觸控與按鍵原語。 | -| `ios/screen.py` | 33 | iOS 裝置螢幕擷取與尺寸。 | +| `android/adb_client.py` | 183 | `adb` CLI 的薄封裝。 | +| `android/client.py` | 91 | `uiautomator2.Device` 的延遲封裝。 | +| `android/find.py` | 104 | uiautomator2 widget 樹的元素查詢。 | +| `ios/client.py` | 94 | `facebook-wda`（WebDriverAgent）封裝。 | +| `ios/find.py` | 86 | XCUITest 無障礙查詢。 | +| `ios/input.py` | 46 | iOS 觸控與按鍵原語。 | +| `ios/screen.py` | 32 | iOS 裝置螢幕擷取與尺寸。 | ### 5.4 能力層 `utils/`（308 個子套件） @@ -257,505 +265,513 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 ### 5.4.1 執行引擎與腳本資產 -> 24 個套件、約 12,443 行。 +> 24 個套件、約 12,655 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/action_lint/` | 332 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | -| `utils/action_signing/` | 232 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | -| `utils/checkpoint/` | 117 | 流程檢查點與續跑，讓長 action list 具持久性 | -| `utils/codegen/` | 159 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | -| `utils/dag/` | 478 | 跨主機 DAG 編排器（圖模型 + runner） | -| `utils/decision_table/` | 105 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | -| `utils/deterministic/` | 98 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | -| `utils/executor/` | 8,931 | **核心**。`Executor` 指令分派表（767 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | -| `utils/flow_debugger/` | 138 | action list 的單步除錯器與追蹤器 | -| `utils/input_macro/` | 129 | 定時輸入事件重播與宣告式輸入序列 DSL | -| `utils/json/` | 75 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | -| `utils/json_store/` | 63 | JSON 字典檔持久化的共用小工具（內部管線） | -| `utils/loop_guard/` | 142 | 機械式卡死迴圈偵測（agent loop 用） | -| `utils/plugin_loader/` | 87 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | -| `utils/plugin_sdk/` | 70 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | -| `utils/project/` | 186 | 專案腳手架：建立目錄結構與範本 action 檔 | -| `utils/recording_edit/` | 152 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | -| `utils/saga/` | 95 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | -| `utils/script_vars/` | 193 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | -| `utils/skill_library/` | 118 | 具名可重用 action 序列（skill）的持久化倉庫 | -| `utils/state_machine/` | 183 | 宣告式有限狀態機驅動 action JSON | -| `utils/stubs/` | 239 | 為 `AC_*` 指令面產生型別 stub | -| `utils/test_record/` | 65 | 全域測試紀錄單例，記錄每個動作的參數與例外 | -| `utils/work_queue/` | 176 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | +| `utils/action_lint/` | 328 | action 檔 linter 與 JSON Schema 產生器（CI 用 `python -m` 進入點） | +| `utils/action_signing/` | 229 | action 檔 HMAC-SHA256 簽章與 Fernet 加密，`execute_files` 會強制驗簽 | +| `utils/checkpoint/` | 115 | 流程檢查點與續跑，讓長 action list 具持久性 | +| `utils/codegen/` | 157 | 由 action list 產生可執行的 pytest / python / robot 測試碼 | +| `utils/dag/` | 475 | 跨主機 DAG 編排器（圖模型 + runner） | +| `utils/decision_table/` | 103 | DMN 風格決策表：規則 + 命中策略，把分支外部化 | +| `utils/deterministic/` | 96 | 決定性執行控制：固定亂數種子 + 凍結時鐘 | +| `utils/executor/` | 9,075 | **核心**。`Executor` 指令分派表（773 個 `AC_*`）、參數插值、乾跑、逐步 callback；`flow_control` 提供 34 個區塊指令（迴圈／分支／try／巨集／變數） | +| `utils/flow_debugger/` | 136 | action list 的單步除錯器與追蹤器 | +| `utils/input_macro/` | 127 | 定時輸入事件重播與宣告式輸入序列 DSL | +| `utils/json/` | 74 | action JSON 檔讀寫與正規化格式化（`fmt --check` 的後端） | +| `utils/json_store/` | 61 | JSON 字典檔持久化的共用小工具（內部管線） | +| `utils/loop_guard/` | 140 | 機械式卡死迴圈偵測（agent loop 用） | +| `utils/plugin_loader/` | 85 | 掃描外部 Python 外掛目錄並註冊其 `AC_` callable | +| `utils/plugin_sdk/` | 68 | 外掛 SDK：透過 entry points 發佈／載入第三方 `AC_*` 指令 | +| `utils/project/` | 182 | 專案腳手架：建立目錄結構與範本 action 檔 | +| `utils/recording_edit/` | 150 | 不重錄的前提下裁切／過濾／縮放已錄製的 action list | +| `utils/saga/` | 93 | Saga 協調器：失敗時以 LIFO 補償動作回滾 | +| `utils/script_vars/` | 190 | 執行期變數作用域與 `${var}` / `${secrets.*}` 插值 | +| `utils/skill_library/` | 116 | 具名可重用 action 序列（skill）的持久化倉庫 | +| `utils/state_machine/` | 181 | 宣告式有限狀態機驅動 action JSON | +| `utils/stubs/` | 236 | 為 `AC_*` 指令面產生型別 stub | +| `utils/test_record/` | 64 | 全域測試紀錄單例，記錄每個動作的參數與例外 | +| `utils/work_queue/` | 174 | 交易式工作佇列（dispatcher／performer），支撐大量批次執行 | ### 5.4.2 框架基礎設施 -> 12 個套件、約 1,848 行。 +> 12 個套件、約 1,881 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/callback/` | 201 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | -| `utils/config_bundle/` | 402 | 使用者設定的單檔匯出／匯入 | -| `utils/critical_exit/` | 98 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | -| `utils/diagnostics/` | 272 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | -| `utils/exception/` | 186 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | -| `utils/failure_bundle/` | 189 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | -| `utils/file_process/` | 27 | 目錄檔案列舉（`execute_dir` 的後端） | -| `utils/logging/` | 72 | `autocontrol_logger` 單例 + 輪替檔案 handler | -| `utils/package_manager/` | 99 | 動態載入套件並把 executor 注入其中 | -| `utils/path_guard/` | 101 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | -| `utils/shell_process/` | 161 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | -| `utils/start_exe/` | 40 | 啟動另一個執行檔行程 | +| `utils/callback/` | 200 | Observer 模式：`callback_executor` 以字串名觸發功能，執行後呼叫回呼 | +| `utils/config_bundle/` | 399 | 使用者設定的單檔匯出／匯入 | +| `utils/critical_exit/` | 97 | 監看緊急停止鍵的守護執行緒，用於中止失控腳本 | +| `utils/diagnostics/` | 312 | 跨子系統的「一切正常嗎」健檢，附 `python -m` 進入點 | +| `utils/exception/` | 194 | **例外階層根**。所有錯誤繼承 `AutoControlException`，加上集中式錯誤訊息字串（`exception_tags`） | +| `utils/failure_bundle/` | 187 | 可攜、已遮蔽的失敗診斷 ZIP（截圖 + 診斷 + log 尾段） | +| `utils/file_process/` | 26 | 目錄檔案列舉（`execute_dir` 的後端） | +| `utils/logging/` | 71 | `autocontrol_logger` 單例 + 輪替檔案 handler | +| `utils/package_manager/` | 98 | 動態載入套件並把 executor 注入其中 | +| `utils/path_guard/` | 99 | 命令列傳入路徑的正規化與邊界檢查（防路徑穿越） | +| `utils/shell_process/` | 159 | `ShellManager`：以 argv list 執行外部命令（禁用 `shell=True`） | +| `utils/start_exe/` | 39 | 啟動另一個執行檔行程 | ### 5.4.3 排程、觸發與背景監看 -> 11 個套件、約 3,574 行。 +> 11 個套件、約 3,544 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/hotkey/` | 734 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | -| `utils/idle_keepawake/` | 214 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | -| `utils/lock_session/` | 165 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | -| `utils/observer/` | 222 | 反應式畫面觀察者，在出現／消失／變化時觸發 | -| `utils/recurrence/` | 326 | RFC 5545 重複規則解析與發生時間展開 | -| `utils/scheduler/` | 355 | 間隔式與 cron 式的 action JSON 排程器 | -| `utils/session_guard/` | 64 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | -| `utils/triggers/` | 1,150 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | -| `utils/voice/` | 89 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | -| `utils/watchdog/` | 175 | 背景彈窗／中斷看門狗，供無人值守自動化 | -| `utils/watcher/` | 80 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | +| `utils/hotkey/` | 727 | 全域熱鍵守護行程，把 OS 層熱鍵綁到 action 檔（Win/macOS/X11 三後端） | +| `utils/idle_keepawake/` | 212 | 偵測使用者閒置時間並在無人值守執行期間阻止系統睡眠 | +| `utils/lock_session/` | 163 | 鎖定工作站、等待解鎖並分類鎖定狀態轉換 | +| `utils/observer/` | 220 | 反應式畫面觀察者，在出現／消失／變化時觸發 | +| `utils/recurrence/` | 324 | RFC 5545 重複規則解析與發生時間展開 | +| `utils/scheduler/` | 352 | 間隔式與 cron 式的 action JSON 排程器 | +| `utils/session_guard/` | 62 | 驅動輸入前先偵測工作階段是否已鎖定／非互動 | +| `utils/triggers/` | 1,146 | 事件驅動觸發引擎：影像／視窗／像素／檔案／webhook／IMAP 郵件 | +| `utils/voice/` | 87 | 語音指令路由：把辨識到的語句對應到 `AC_*` action list | +| `utils/watchdog/` | 173 | 背景彈窗／中斷看門狗，供無人值守自動化 | +| `utils/watcher/` | 78 | 無頭輪詢原語：滑鼠位置、像素顏色、log tail | ### 5.4.4 輸入模擬與動作品質 -> 20 個套件、約 2,325 行。 +> 22 個套件、約 2,610 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/act_in_view/` | 78 | 先把目標捲進視野，待其可操作後再動作 | -| `utils/act_modes/` | 69 | actionability 閘門之上的 trial／force 動作模式 | -| `utils/action_effect/` | 112 | 判定一個動作是否真的產生效果，並歸因到目標區域 | -| `utils/action_grounding/` | 82 | 動作前的接地守衛（邊界檢查 + 吸附到元素） | -| `utils/actionability/` | 165 | 動作前就緒閘門（可見 + 穩定 + 啟用 + 未被遮擋） | -| `utils/ensure_state/` | 74 | 冪等地把控制項／設定帶到期望狀態 | -| `utils/field_entry/` | 78 | 清空再輸入的欄位填寫慣用法（Playwright `fill`） | -| `utils/gamepad/` | 313 | 虛擬遊戲手把後端（Windows ViGEmBus 驅動） | -| `utils/humanize/` | 186 | 擬人輸入：貝茲曲線滑鼠路徑 + 抖動打字節奏 | -| `utils/ime_state/` | 146 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | -| `utils/key_hold/` | 109 | 按住按鍵一段時間，或以固定頻率自動重複 | -| `utils/modifier_state/` | 78 | 跨一組動作按住修飾鍵，並保證安全釋放 | -| `utils/mouse_path/` | 94 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | -| `utils/mouse_relative/` | 61 | 相對位移滑鼠移動 | -| `utils/postcondition/` | 140 | 宣告式的動作預期結果規格，對照畫面驗證 | -| `utils/step_repair/` | 116 | 失敗／無效動作的修復策略（自我修正迴圈） | -| `utils/table_grid_fill/` | 143 | 以 OCR 文字填滿格線表格，取得可定址的表格 | +| `utils/act_in_view/` | 76 | 先把目標捲進視野，待其可操作後再動作 | +| `utils/act_modes/` | 67 | actionability 閘門之上的 trial／force 動作模式 | +| `utils/action_effect/` | 110 | 判定一個動作是否真的產生效果，並歸因到目標區域 | +| `utils/action_grounding/` | 80 | 動作前的接地守衛（邊界檢查 + 吸附到元素） | +| `utils/actionability/` | 163 | 動作前就緒閘門（可見 + 穩定 + 啟用 + 未被遮擋） | +| `utils/ensure_state/` | 72 | 冪等地把控制項／設定帶到期望狀態 | +| `utils/field_entry/` | 76 | 清空再輸入的欄位填寫慣用法（Playwright `fill`） | +| `utils/gamepad/` | 311 | 虛擬遊戲手把後端（Windows ViGEmBus 驅動） | +| `utils/humanize/` | 183 | 擬人輸入：貝茲曲線滑鼠路徑 + 抖動打字節奏 | +| `utils/ime_state/` | 144 | 讀取即時 IME 組字／轉換狀態，確保 CJK 輸入安全 | +| `utils/key_hold/` | 107 | 按住按鍵一段時間，或以固定頻率自動重複 | +| `utils/modifier_state/` | 76 | 跨一組動作按住修飾鍵，並保證安全釋放 | +| `utils/mouse_path/` | 92 | 多路徑點滑鼠手勢（沿折線移動或拖曳） | +| `utils/mouse_relative/` | 59 | 相對位移滑鼠移動 | +| `utils/postcondition/` | 138 | 宣告式的動作預期結果規格，對照畫面驗證 | +| `utils/step_repair/` | 114 | 失敗／無效動作的修復策略（自我修正迴圈） | +| `utils/table_grid_fill/` | 141 | 以 OCR 文字填滿格線表格，取得可定址的表格 | | `utils/input_reach/` | 111 | 送出去的輸入到不到得了：桌面鎖定查詢（免費）＋ 實際送一個 F13 確認沒有被過濾（有副作用，只給診斷用） | | `utils/keyboard_layout/` | 148 | 向系統問「這個鍵盤配置下每個鍵印出什麼字」（`ToUnicodeEx`），問不到退回 US 對照表 | | `utils/text_unicode/` | 135 | 輸入任意 Unicode（emoji／CJK／重音字）：優先送字元按鍵事件，不支援時退回剪貼簿貼上 | -| `utils/tween_drag/` | 97 | 沿曲線的緩動插值拖曳 | -| `utils/verify_field/` | 114 | 打字後讀回欄位，確認內容確實落地 | +| `utils/tween_drag/` | 95 | 沿曲線的緩動插值拖曳 | +| `utils/verify_field/` | 112 | 打字後讀回欄位，確認內容確實落地 | ### 5.4.5 影像辨識與畫面分析 -> 37 個套件、約 4,572 行。 +> 37 個套件、約 4,999 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/annotate/` | 109 | 截圖標註：畫框、highlight、箭頭、標籤 | -| `utils/barcode/` | 55 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | -| `utils/color_match/` | 105 | 在 HSV 通道上做顏色感知的樣板比對 | -| `utils/color_region/` | 81 | 以顏色定位畫面區域（遮罩 + 連通元件） | -| `utils/color_stats/` | 91 | 區域顏色統計：平均色與主色 | -| `utils/coordinate_space/` | 86 | 模型網格座標與實體像素之間的座標空間對映 | -| `utils/cv2_utils/` | 355 | OpenCV 基礎層：截圖（mss/Pillow）、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | -| `utils/edge_lines/` | 122 | 以 Hough 轉換偵測線條／格線／分隔線 | -| `utils/edge_match/` | 114 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | -| `utils/feature_match/` | 131 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | -| `utils/hsv_segment/` | 93 | HSV 色彩空間分割（抗光照的顏色遮罩 + blob 框） | -| `utils/icon_classify/` | 115 | 從像素形狀判斷一個框是哪一類元件 | -| `utils/image_dedup/` | 85 | 感知雜湊影像去重（Pillow aHash/dHash） | -| `utils/image_quality/` | 79 | 在 OCR／比對前評分影像品質（銳利度／對比／亮度） | -| `utils/img_histogram/` | 101 | 顏色直方圖指紋與變化偵測（抗光照） | -| `utils/marks_layout/` | 126 | Set-of-Marks 標籤的不重疊排版與可讀配色 | -| `utils/match_autothresh/` | 107 | Otsu 自動門檻，免去手動調 `min_score` | -| `utils/match_ensemble/` | 65 | 多樣板共識比對（多張參考圖投票到同一位置） | -| `utils/match_stability/` | 70 | 比對前的靜止閘門與跨影格的比對持續性 | -| `utils/match_trust/` | 138 | 樣板比對可信度評分（次峰比 + peak-to-sidelobe） | -| `utils/monitor_layout/` | 290 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | -| `utils/motion_regions/` | 75 | 兩影格間的局部變化／活動偵測（absdiff） | -| `utils/perceptual_diff/` | 102 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | -| `utils/preprocess/` | 187 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | -| `utils/qr/` | 55 | 從影像或螢幕區域解碼 QR code（OpenCV） | -| `utils/rotated_match/` | 147 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | -| `utils/saliency/` | 109 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | -| `utils/scale_detect/` | 86 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | -| `utils/screen_grid/` | 145 | 供 VLM 接地用的粗粒度標號網格（點 ↔ 格對映） | -| `utils/set_of_marks/` | 152 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | -| `utils/shape_locator/` | 107 | 以邊緣／輪廓偵測定位元件（矩形／形狀，免樣板） | -| `utils/ssim/` | 142 | 結構相似度比較：感知分數 + 變化區域 | -| `utils/subpixel_match/` | 103 | 以二次曲面擬合做次像素級比對精修 | -| `utils/theme_normalize/` | 94 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | -| `utils/video_report/` | 135 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | +| `utils/annotate/` | 107 | 截圖標註：畫框、highlight、箭頭、標籤 | +| `utils/barcode/` | 53 | 一維條碼（EAN／UPC）解碼，解碼器可注入 | +| `utils/color_match/` | 103 | 在 HSV 通道上做顏色感知的樣板比對 | +| `utils/color_region/` | 79 | 以顏色定位畫面區域（遮罩 + 連通元件） | +| `utils/color_stats/` | 89 | 區域顏色統計：平均色與主色 | +| `utils/coordinate_space/` | 84 | 模型網格座標與實體像素之間的座標空間對映 | +| `utils/cv2_utils/` | 592 | OpenCV 基礎層：擷取後端選擇（`screen_grabber`，Pillow／mss 或平台後端）、截圖、樣板比對（走 `grab_logical`，涵蓋所有螢幕）、螢幕錄影、影片錄製、連通元件 | +| `utils/edge_lines/` | 120 | 以 Hough 轉換偵測線條／格線／分隔線 | +| `utils/edge_match/` | 112 | 邊緣形狀（Chamfer／距離轉換）樣板比對 | +| `utils/feature_match/` | 129 | ORB 特徵比對：在旋轉／縮放／主題變更下定位樣板 | +| `utils/hsv_segment/` | 91 | HSV 色彩空間分割（抗光照的顏色遮罩 + blob 框） | +| `utils/icon_classify/` | 113 | 從像素形狀判斷一個框是哪一類元件 | +| `utils/image_dedup/` | 83 | 感知雜湊影像去重（Pillow aHash/dHash） | +| `utils/image_quality/` | 77 | 在 OCR／比對前評分影像品質（銳利度／對比／亮度） | +| `utils/img_histogram/` | 99 | 顏色直方圖指紋與變化偵測（抗光照） | +| `utils/marks_layout/` | 124 | Set-of-Marks 標籤的不重疊排版與可讀配色 | +| `utils/match_autothresh/` | 105 | Otsu 自動門檻，免去手動調 `min_score` | +| `utils/match_ensemble/` | 63 | 多樣板共識比對（多張參考圖投票到同一位置） | +| `utils/match_stability/` | 68 | 比對前的靜止閘門與跨影格的比對持續性 | +| `utils/match_trust/` | 136 | 樣板比對可信度評分（次峰比 + peak-to-sidelobe） | +| `utils/monitor_layout/` | 317 | 多螢幕／虛擬桌面幾何（在哪個螢幕、位置、重映射）＋ `logical_frame` 以滑鼠座標空間擷取畫面 | +| `utils/motion_regions/` | 73 | 兩影格間的局部變化／活動偵測（absdiff） | +| `utils/perceptual_diff/` | 100 | 感知式（YIQ）影像差異，抑制反鋸齒邊緣誤報 | +| `utils/preprocess/` | 185 | OCR／比對前的影像前處理（灰階、二值化、去傾斜…） | +| `utils/qr/` | 53 | 從影像或螢幕區域解碼 QR code（OpenCV） | +| `utils/rotated_match/` | 145 | 容忍旋轉與縮放的樣板比對（尺度空間 × 角度掃描） | +| `utils/saliency/` | 107 | 頻譜殘差視覺顯著性：顯著圖與排序後的顯著區域 | +| `utils/scale_detect/` | 84 | 偵測樣板實際渲染的顯示縮放／視覺 DPI | +| `utils/screen_grid/` | 143 | 供 VLM 接地用的粗粒度標號網格（點 ↔ 格對映） | +| `utils/set_of_marks/` | 150 | Set-of-Marks 疊圖：為畫面元素編號供 VLM 指認 | +| `utils/shape_locator/` | 105 | 以邊緣／輪廓偵測定位元件（矩形／形狀，免樣板） | +| `utils/ssim/` | 140 | 結構相似度比較：感知分數 + 變化區域 | +| `utils/subpixel_match/` | 101 | 以二次曲面擬合做次像素級比對精修 | +| `utils/theme_normalize/` | 92 | 主題無關的影像正規化，讓亮色樣板能配對深色模式 | +| `utils/video_report/` | 133 | 影片步驟疊圖報告：把截圖加字幕串成操作導覽影片 | | `utils/visual_match/` | 427 | 會回傳信心值的樣板比對（分數、多尺度、find-all + NMS）；擷取走 `grab_logical`，命中座標已加回虛擬桌面原點，單色樣板直接拒收 | -| `utils/visual_regression/` | 218 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | +| `utils/visual_regression/` | 217 | 桌面 GUI 的視覺回歸測試（黃金圖比對） | ### 5.4.6 OCR 與文字理解 -> 19 個套件、約 3,098 行。 +> 19 個套件、約 3,180 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/bidi_check/` | 118 | 雙向文字 QA（bidi 控制碼、巢狀平衡、Trojan-source 掃描） | -| `utils/column_layout/` | 152 | 從垂直空白推斷欄位，處理無框線表格 | -| `utils/confusables/` | 114 | 易混淆／同形字偵測（Unicode 欺騙骨架） | -| `utils/form_fields/` | 130 | 多方向關聯表單標籤與值，並讀取核取方塊狀態 | -| `utils/fuzzy/` | 96 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | -| `utils/grid_locator/` | 73 | 以 (row, column) 從邊界框定址表格／網格儲存格 | -| `utils/guardrail/` | 110 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | -| `utils/heading_segment/` | 71 | 判定 OCR 行是標題或內文，建出文件大綱 | -| `utils/near_dup/` | 107 | 近似重複文字偵測（SimHash／MinHash） | -| `utils/ocr/` | 1,105 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | -| `utils/pii_text/` | 100 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | -| `utils/readability/` | 139 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | -| `utils/reading_flow/` | 121 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | -| `utils/search_index/` | 142 | 記憶體內 BM25／TF-IDF 全文檢索 | -| `utils/text_blocks/` | 90 | 把 OCR 行組成段落與項目符號／編號清單 | -| `utils/text_diff/` | 150 | unified diff 產生、套用與三方合併 | -| `utils/text_normalize/` | 65 | Unicode 正規化與 slug 產生 | -| `utils/text_regions/` | 159 | 免模型的畫面文字區域偵測（MSER）：區域與行 | -| `utils/text_similarity/` | 167 | 字串距離度量（文字比對用） | +| `utils/bidi_check/` | 116 | 雙向文字 QA（bidi 控制碼、巢狀平衡、Trojan-source 掃描） | +| `utils/column_layout/` | 150 | 從垂直空白推斷欄位，處理無框線表格 | +| `utils/confusables/` | 112 | 易混淆／同形字偵測（Unicode 欺騙骨架） | +| `utils/form_fields/` | 128 | 多方向關聯表單標籤與值，並讀取核取方塊狀態 | +| `utils/fuzzy/` | 94 | 模糊字串比對與去重（預設 difflib，有 rapidfuzz 則優先） | +| `utils/grid_locator/` | 71 | 以 (row, column) 從邊界框定址表格／網格儲存格 | +| `utils/guardrail/` | 108 | 針對畫面／OCR 文字的啟發式 prompt-injection 防護 | +| `utils/heading_segment/` | 69 | 判定 OCR 行是標題或內文，建出文件大綱 | +| `utils/near_dup/` | 105 | 近似重複文字偵測（SimHash／MinHash） | +| `utils/ocr/` | 1,112 | OCR 引擎門面 + 三個後端（Tesseract／EasyOCR／PaddleOCR）、版面結構化與跨詞比對（`text_span`） | +| `utils/pii_text/` | 98 | 自由文字中的 PII 偵測與遮蔽（email／電話／SSN／卡號／IP／IBAN） | +| `utils/readability/` | 137 | 可讀性評分（Flesch、Flesch-Kincaid、Gunning Fog、SMOG、ARI） | +| `utils/reading_flow/` | 119 | 以遞迴 XY-cut 推導欄位感知的閱讀順序 | +| `utils/search_index/` | 140 | 記憶體內 BM25／TF-IDF 全文檢索 | +| `utils/text_blocks/` | 88 | 把 OCR 行組成段落與項目符號／編號清單 | +| `utils/text_diff/` | 148 | unified diff 產生、套用與三方合併 | +| `utils/text_normalize/` | 63 | Unicode 正規化與 slug 產生 | +| `utils/text_regions/` | 157 | 免模型的畫面文字區域偵測（MSER）：區域與行 | +| `utils/text_similarity/` | 165 | 字串距離度量（文字比對用） | ### 5.4.7 無障礙樹與原生控制項 -> 16 個套件、約 3,287 行。 +> 16 個套件、約 3,851 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/a11y_audit/` | 358 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | +| `utils/a11y_audit/` | 355 | 以無障礙樹 + OCR 進行無障礙與 i18n 稽核 | | `utils/accessibility/` | 2,390 | 跨平台無障礙樹定位與錄製；Windows UIA／macOS AX／null 三後端。支援限定視窗（換搜尋起點，不是過濾）、逐節點可中斷走訪、`IUIAutomation2` 連線逾時、名稱子字串比對與排序、`control_get_state` 一次讀完值／勾選／選取／數值（密碼欄位不回內容） | -| `utils/ax_events/` | 31 | 反應式 UIA 事件等待（focus-changed） | -| `utils/ax_props/` | 46 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | -| `utils/ax_text/` | 104 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | -| `utils/ax_tree_walk/` | 120 | 可讀、可定址的無障礙樹後處理（角色名 + 節點路徑） | -| `utils/contrast_map/` | 122 | 取樣實際顏色以評定畫面文字的可讀性（WCAG） | -| `utils/control_patterns/` | 90 | 延伸 UIA 控制項模式動作（Expand／Select／Range／Scroll） | -| `utils/cvd_simulate/` | 127 | 模擬色覺缺陷並標示在該狀況下會撞色的顏色 | -| `utils/element_repository/` | 107 | 原生 UI 元素的具名定位器倉庫（object repository） | -| `utils/focus_order/` | 97 | 鍵盤焦點順序：預期 Tab 序列、WCAG 稽核與設定焦點 | -| `utils/legacy_accessible/` | 47 | MSAA 橋接，處理 UIA 無法建模的舊控制項 | -| `utils/selection_view/` | 59 | 容器選取狀態與檢視切換（Selection／MultipleView 模式） | -| `utils/table_pattern/` | 67 | 原生表格的表頭與儲存格定址（UIA TablePattern／GridItem） | -| `utils/transform_window/` | 72 | 以 UIA Transform／Window 模式移動、調整大小與視窗狀態 | -| `utils/virtualized/` | 45 | 實體化虛擬化清單／網格中的離屏項目 | +| `utils/ax_events/` | 29 | 反應式 UIA 事件等待（focus-changed） | +| `utils/ax_props/` | 44 | 讀取豐富 UIA 屬性（enabled／offscreen／help／status／快捷鍵） | +| `utils/ax_text/` | 102 | 透過 UIA TextPattern 取得原生文字（讀取／尋找／選取／屬性） | +| `utils/ax_tree_walk/` | 118 | 可讀、可定址的無障礙樹後處理（角色名 + 節點路徑） | +| `utils/contrast_map/` | 120 | 取樣實際顏色以評定畫面文字的可讀性（WCAG） | +| `utils/control_patterns/` | 88 | 延伸 UIA 控制項模式動作（Expand／Select／Range／Scroll） | +| `utils/cvd_simulate/` | 125 | 模擬色覺缺陷並標示在該狀況下會撞色的顏色 | +| `utils/element_repository/` | 105 | 原生 UI 元素的具名定位器倉庫（object repository） | +| `utils/focus_order/` | 95 | 鍵盤焦點順序：預期 Tab 序列、WCAG 稽核與設定焦點 | +| `utils/legacy_accessible/` | 45 | MSAA 橋接，處理 UIA 無法建模的舊控制項 | +| `utils/selection_view/` | 57 | 容器選取狀態與檢視切換（Selection／MultipleView 模式） | +| `utils/table_pattern/` | 65 | 原生表格的表頭與儲存格定址（UIA TablePattern／GridItem） | +| `utils/transform_window/` | 70 | 以 UIA Transform／Window 模式移動、調整大小與視窗狀態 | +| `utils/virtualized/` | 43 | 實體化虛擬化清單／網格中的離屏項目 | ### 5.4.8 元素定位、自我修復與智慧等待 -> 23 個套件、約 4,044 行。 +> 23 個套件、約 3,995 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/ab_locator/` | 339 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | -| `utils/adaptive_timeout/` | 86 | 由觀測到的步驟耗時推導等待逾時，而非硬猜 | -| `utils/anchor_locator/` | 440 | 錨點定位器：以空間關係組合 影像／OCR／VLM／a11y 四種來源 | -| `utils/app_idle/` | 110 | 等應用程式不再忙碌，再驅動下一步 | -| `utils/change_localize/` | 82 | 把畫面變化歸因到實際改變的元素框 | -| `utils/critic_features/` | 87 | 每步的 critic 特徵集合與規則式步驟評分 | -| `utils/element_diff/` | 90 | 跨影格的幾何感知元素比對（穩定 ID、移動追蹤） | -| `utils/element_parse/` | 108 | 融合並排序畫面元素框（IoU、合併、多來源融合、閱讀順序） | -| `utils/element_proposal/` | 88 | 免樣板、免模型地從原始像素提出乾淨元素清單 | -| `utils/element_scoring/` | 107 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | -| `utils/expect_poll/` | 139 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | -| `utils/grounding_consensus/` | 129 | 對同一目標的多個接地提案做自我一致性投票 | -| `utils/heal_analytics/` | 79 | 自癒事件記錄的分析（治癒率、脆弱定位器） | -| `utils/locator_chain/` | 114 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | -| `utils/locator_repair/` | 119 | 自癒回寫：把修正後的定位器持久化 | -| `utils/observation/` | 94 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | -| `utils/observation_delta/` | 105 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | -| `utils/screen_state/` | 145 | 語義畫面狀態：快照／差異與結構化畫面描述 | -| `utils/scroll_find/` | 86 | 捲動直到目標影像／文字可見 | -| `utils/self_healing/` | 345 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | -| `utils/semantic_recording/` | 427 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | -| `utils/settle_detector/` | 78 | 以純函式介面判定 UI 是否已靜止 | -| `utils/smart_waits/` | 647 | 智慧等待：以影格差異取代 `time.sleep` | +| `utils/ab_locator/` | 336 | A/B 定位器框架：同時競速 N 種策略並記錄各自勝率 | +| `utils/adaptive_timeout/` | 84 | 由觀測到的步驟耗時推導等待逾時，而非硬猜 | +| `utils/anchor_locator/` | 438 | 錨點定位器：以空間關係組合 影像／OCR／VLM／a11y 四種來源 | +| `utils/app_idle/` | 108 | 等應用程式不再忙碌，再驅動下一步 | +| `utils/change_localize/` | 80 | 把畫面變化歸因到實際改變的元素框 | +| `utils/critic_features/` | 85 | 每步的 critic 特徵集合與規則式步驟評分 | +| `utils/element_diff/` | 88 | 跨影格的幾何感知元素比對（穩定 ID、移動追蹤） | +| `utils/element_parse/` | 106 | 融合並排序畫面元素框（IoU、合併、多來源融合、閱讀順序） | +| `utils/element_proposal/` | 86 | 免樣板、免模型地從原始像素提出乾淨元素清單 | +| `utils/element_scoring/` | 105 | 加權候選評分（角色 + 名稱相似度 + 鄰近度 + 啟用狀態） | +| `utils/expect_poll/` | 137 | 反覆取值直到符合條件（Playwright `expect.poll` 風格） | +| `utils/grounding_consensus/` | 127 | 對同一目標的多個接地提案做自我一致性投票 | +| `utils/heal_analytics/` | 77 | 自癒事件記錄的分析（治癒率、脆弱定位器） | +| `utils/locator_chain/` | 112 | 可組合／可過濾的候選定位器（chained-locator 慣用法） | +| `utils/locator_repair/` | 117 | 自癒回寫：把修正後的定位器持久化 | +| `utils/observation/` | 92 | 供 VLM／agent 接地用的 token 預算內、帶索引的 a11y 文字觀察 | +| `utils/observation_delta/` | 103 | token 預算內的觀察差異：兩個 UI 影格之間變了什麼 | +| `utils/screen_state/` | 143 | 語義畫面狀態：快照／差異與結構化畫面描述 | +| `utils/scroll_find/` | 84 | 捲動直到目標影像／文字可見 | +| `utils/self_healing/` | 342 | 自癒定位器：先影像樣板、失敗改用 VLM，並留稽核記錄 | +| `utils/semantic_recording/` | 423 | 為錄製內容加上語義錨點，支援換機重播與自癒重播 | +| `utils/settle_detector/` | 76 | 以純函式介面判定 UI 是否已靜止 | +| `utils/smart_waits/` | 646 | 智慧等待：以影格差異取代 `time.sleep` | ### 5.4.9 AI / Agent / LLM -> 13 個套件、約 19,764 行。 +> 13 個套件、約 20,132 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/a2a/` | 94 | A2A（agent-to-agent）agent card 產生 | -| `utils/agent/` | 1,258 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | -| `utils/agent_memory/` | 148 | agent 的持久化情節記憶（goal → trajectory → outcome） | -| `utils/agent_replay/` | 65 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | -| `utils/agent_trace/` | 131 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | -| `utils/cost_telemetry/` | 295 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | -| `utils/cua_action/` | 129 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | -| `utils/llm/` | 363 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | -| `utils/mcp_registry/` | 94 | MCP registry `server.json` 資訊清單產生（可被發現） | -| `utils/mcp_server/` | 16,441 | **無頭 MCP 伺服器**（16K LOC，預設註冊 670 個工具＝651 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | -| `utils/tool_use_schema/` | 182 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | -| `utils/trajectory_eval/` | 108 | agent 軌跡評估：依評分規準為一次執行打分 | -| `utils/vision/` | 456 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | +| `utils/a2a/` | 92 | A2A（agent-to-agent）agent card 產生 | +| `utils/agent/` | 1,250 | 閉環 Computer-Use Agent 主迴圈 + Anthropic／OpenAI／Computer-Use 三後端 | +| `utils/agent_memory/` | 146 | agent 的持久化情節記憶（goal → trajectory → outcome） | +| `utils/agent_replay/` | 63 | 可攜的 agent 軌跡追蹤（記錄 observation→action 並重播） | +| `utils/agent_trace/` | 129 | agent 可觀測性：OpenTelemetry GenAI 慣例的 LLM span | +| `utils/cost_telemetry/` | 292 | 每次呼叫的 LLM 成本遙測：token 數 + 估算美金 | +| `utils/cua_action/` | 127 | 標準化 computer-use 動作結構（Anthropic／OpenAI → `AC_*`） | +| `utils/llm/` | 357 | 自然語言 → action list 規劃器 + Anthropic／null 後端 | +| `utils/mcp_registry/` | 92 | MCP registry `server.json` 資訊清單產生（可被發現） | +| `utils/mcp_server/` | 16,850 | **無頭 MCP 伺服器**（16K LOC，預設註冊 676 個工具＝657 個 `ac_*` + 19 個別名）：stdio + HTTP 傳輸、工具工廠與處理器、資源、prompt、稽核、限流、外掛熱重載 | +| `utils/tool_use_schema/` | 180 | 把 `AC_*` 指令匯出成 Claude／OpenAI 的 tool-use schema | +| `utils/trajectory_eval/` | 106 | agent 軌跡評估：依評分規準為一次執行打分 | +| `utils/vision/` | 448 | VLM 元素定位器（依描述找元素）+ Anthropic／OpenAI／null 後端 | ### 5.4.10 遠端桌面與 USB -> 6 個套件、約 17,622 行。 +> 6 個套件、約 17,703 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/admin/` | 329 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | -| `utils/config_sync/` | 247 | 透過訊令伺服器做跨機器設定同步 | -| `utils/device_matrix/` | 140 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | -| `utils/remote_desktop/` | 11,726 | **遠端桌面子系統**（51 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | -| `utils/usb/` | 4,255 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | -| `utils/usbip/` | 925 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | +| `utils/admin/` | 327 | 多主機管理主控台：平行輪詢 N 個 AutoControl REST 端點 | +| `utils/config_sync/` | 245 | 透過訊令伺服器做跨機器設定同步 | +| `utils/device_matrix/` | 138 | 行動裝置矩陣：同一 action list 於多台裝置平行執行 | +| `utils/remote_desktop/` | 11,835 | **遠端桌面子系統**（56 檔／11.7K LOC）：TCP／WebSocket／WebRTC 三條傳輸路徑、主機與檢視端、訊令伺服器、TURN／中繼、多檢視者、錄影、信任清單、TOTP、稽核鏈 | +| `utils/usb/` | 4,238 | 跨平台 USB 列舉／熱插拔／裝置直通（WinUSB、IOKit、libusb 後端 + ACL + WebRTC DataChannel 通道） | +| `utils/usbip/` | 920 | USB/IP 線路協定主機端（協定封包、TCP 伺服器、libusb URB 後端） | ### 5.4.11 伺服器、網路協定與外部整合 -> 24 個套件、約 5,889 行。 +> 24 個套件、約 5,881 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/acme_v2/` | 589 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | -| `utils/chatops/` | 632 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | -| `utils/cookie_jar/` | 105 | RFC 6265 cookie jar | -| `utils/email_send/` | 118 | SMTP 寄信（email 觸發器的發送端搭檔） | -| `utils/events/` | 84 | 對外 CloudEvents 發送（執行生命週期事件） | -| `utils/http_cassette/` | 112 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | -| `utils/http_client/` | 134 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | -| `utils/http_conditional/` | 89 | 條件式 HTTP 請求與快取驗證器 | -| `utils/http_content/` | 105 | HTTP 內容協商與回應解壓縮 | -| `utils/http_problem/` | 118 | RFC 9457 problem+json 解析 | -| `utils/jwt/` | 174 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | -| `utils/link_header/` | 114 | RFC 8288 Link header 解析與分頁 | -| `utils/multipart/` | 141 | multipart/form-data 建構與解析 | -| `utils/notify/` | 97 | 跨平台桌面通知 | -| `utils/notify_channels/` | 102 | 對外聊天／webhook 通知（Slack／Discord／Teams／raw） | -| `utils/otp/` | 39 | TOTP 一次性密碼產生（自動化 2FA 登入） | -| `utils/outbox/` | 94 | 交易式 outbox，保證至少一次的事件投遞 | -| `utils/pytest_plugin/` | 377 | pytest 外掛 + BDD step library（`pytest11` entry point） | -| `utils/rest_api/` | 1,693 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | -| `utils/socket_server/` | 133 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | -| `utils/sse_client/` | 114 | Server-Sent Events 用戶端解析 | -| `utils/tls_acme/` | 445 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | -| `utils/url_canon/` | 117 | RFC 3986 URL 正規化與查詢字串工具 | -| `utils/webrunner_bridge/` | 163 | 把 action JSON 橋接到 WebRunner（`je_web_runner`） | +| `utils/acme_v2/` | 586 | 完整 ACME v2 用戶端（RFC 8555），不依賴 certbot | +| `utils/chatops/` | 628 | Chat-ops bot：接收 Slack／Discord／webhook 的 slash 指令並路由到動作 | +| `utils/cookie_jar/` | 103 | RFC 6265 cookie jar | +| `utils/email_send/` | 116 | SMTP 寄信（email 觸發器的發送端搭檔） | +| `utils/events/` | 82 | 對外 CloudEvents 發送（執行生命週期事件） | +| `utils/http_cassette/` | 110 | 錄製／重播 HTTP 互動，做離線決定性 API 測試 | +| `utils/http_client/` | 132 | 零依賴 HTTP(S) 用戶端，供 action 步驟呼叫 API | +| `utils/http_conditional/` | 87 | 條件式 HTTP 請求與快取驗證器 | +| `utils/http_content/` | 103 | HTTP 內容協商與回應解壓縮 | +| `utils/http_problem/` | 116 | RFC 9457 problem+json 解析 | +| `utils/jwt/` | 172 | JWT（HMAC 家族）編碼、解碼與 claim 驗證 | +| `utils/link_header/` | 112 | RFC 8288 Link header 解析與分頁 | +| `utils/multipart/` | 139 | multipart/form-data 建構與解析 | +| `utils/notify/` | 95 | 跨平台桌面通知 | +| `utils/notify_channels/` | 100 | 對外聊天／webhook 通知（Slack／Discord／Teams／raw） | +| `utils/otp/` | 37 | TOTP 一次性密碼產生（自動化 2FA 登入） | +| `utils/outbox/` | 92 | 交易式 outbox，保證至少一次的事件投遞 | +| `utils/pytest_plugin/` | 373 | pytest 外掛 + BDD step library（`pytest11` entry point） | +| `utils/rest_api/` | 1,738 | 純標準庫 REST 前端：路由、Bearer 驗證、限流、Prometheus 指標、OpenAPI 3.1 產生 | +| `utils/socket_server/` | 131 | 執行 action JSON 的執行緒式 TCP 指令伺服器（預設綁 127.0.0.1） | +| `utils/sse_client/` | 112 | Server-Sent Events 用戶端解析 | +| `utils/tls_acme/` | 441 | TLS 自動化：HTTP-01 挑戰伺服器、金鑰／CSR、自動續期 | +| `utils/url_canon/` | 115 | RFC 3986 URL 正規化與查詢字串工具 | +| `utils/webrunner_bridge/` | 161 | 把 action JSON 橋接到 WebRunner（`je_web_runner`） | ### 5.4.12 報表、可觀測性與測試治理 -> 34 個套件、約 6,942 行。 +> 34 個套件、約 6,865 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/anomaly/` | 109 | 單一序列異常偵測 | -| `utils/approval/` | 104 | Approval testing：以核可基準線驗證產出物 | -| `utils/assertion/` | 866 | 斷言 DSL：畫面狀態驗證 + 組合子 | -| `utils/baggage/` | 113 | W3C Baggage 傳遞 | -| `utils/canonical_log/` | 92 | canonical log line 與結構化 JSON 日誌 | -| `utils/ci_annotations/` | 64 | 由執行結果輸出 CI 工作流程註記（GitHub Actions） | -| `utils/compliance/` | 138 | 合規：把治理證據對應到 SOC2／ISO 27001 控制項 | -| `utils/failure_hooks/` | 400 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | -| `utils/failure_signature/` | 76 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | -| `utils/flake_cluster/` | 105 | 以共同失敗 Jaccard 相似度為易碎測試分群 | -| `utils/flakiness/` | 152 | 以執行歷史分析不穩定測試 | -| `utils/generate_report/` | 311 | HTML／JSON／XML 三種報表產生器（Template Method） | -| `utils/media_assert/` | 235 | 媒體斷言：音訊活動與影片動態檢查 | -| `utils/observability/` | 665 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | -| `utils/otlp_export/` | 83 | OTLP/JSON span 匯出 | -| `utils/percentiles/` | 105 | 可合併的串流延遲摘要與精確百分位數 | -| `utils/process_doc/` | 87 | 由錄製的 action list 產生逐步 SOP 文件 | -| `utils/process_mining/` | 112 | 流程探勘：從動作日誌挖掘可自動化的候選 | -| `utils/profiler/` | 425 | 逐動作效能剖析器 + 資源剖析器 | -| `utils/quarantine/` | 192 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | -| `utils/run_diff/` | 125 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | -| `utils/run_history/` | 349 | 執行歷史儲存與產出物管理 | -| `utils/sarif/` | 136 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | -| `utils/slo/` | 114 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | -| `utils/smoothing/` | 69 | 數列移動平均平滑 | -| `utils/soft_assert/` | 64 | 軟斷言：累積檢查並在區塊結束時一次拋出 | -| `utils/stats/` | 215 | 描述統計與 A/B 顯著性檢定（純標準庫） | -| `utils/step_timeline/` | 83 | 每次執行的步驟瀑布圖與瓶頸（關鍵路徑）步驟排名 | -| `utils/test_select/` | 125 | 以執行歷史做風險導向的測試選取 | -| `utils/test_shard/` | 89 | 以耗時為權重的套件切分與分片結果合併 | -| `utils/test_suite/` | 446 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | -| `utils/time_travel/` | 384 | 錄製 session 的時光回溯除錯（控制器 + 播放器） | -| `utils/timeseries/` | 145 | 時間序列轉換（rate／降採樣／重採樣） | -| `utils/trace_context/` | 164 | W3C Trace Context 傳遞 | +| `utils/anomaly/` | 107 | 單一序列異常偵測 | +| `utils/approval/` | 102 | Approval testing：以核可基準線驗證產出物 | +| `utils/assertion/` | 863 | 斷言 DSL：畫面狀態驗證 + 組合子 | +| `utils/baggage/` | 111 | W3C Baggage 傳遞 | +| `utils/canonical_log/` | 90 | canonical log line 與結構化 JSON 日誌 | +| `utils/ci_annotations/` | 62 | 由執行結果輸出 CI 工作流程註記（GitHub Actions） | +| `utils/compliance/` | 136 | 合規：把治理證據對應到 SOC2／ISO 27001 控制項 | +| `utils/failure_hooks/` | 396 | 失敗 → 工單自動化：開 Jira／Linear／GitHub issue | +| `utils/failure_signature/` | 74 | 把錯誤訊息正規化成穩定的 SHA-256 失敗簽章並分群 | +| `utils/flake_cluster/` | 103 | 以共同失敗 Jaccard 相似度為易碎測試分群 | +| `utils/flakiness/` | 150 | 以執行歷史分析不穩定測試 | +| `utils/generate_report/` | 310 | HTML／JSON／XML 三種報表產生器（Template Method） | +| `utils/media_assert/` | 233 | 媒體斷言：音訊活動與影片動態檢查 | +| `utils/observability/` | 661 | Prometheus 格式指標 + OpenTelemetry 相容 trace + `/metrics` 匯出伺服器 | +| `utils/otlp_export/` | 81 | OTLP/JSON span 匯出 | +| `utils/percentiles/` | 103 | 可合併的串流延遲摘要與精確百分位數 | +| `utils/process_doc/` | 85 | 由錄製的 action list 產生逐步 SOP 文件 | +| `utils/process_mining/` | 110 | 流程探勘：從動作日誌挖掘可自動化的候選 | +| `utils/profiler/` | 422 | 逐動作效能剖析器 + 資源剖析器 | +| `utils/quarantine/` | 190 | 易碎測試隔離區，讓套件執行器跳過已知不穩定案例 | +| `utils/run_diff/` | 123 | 兩次執行軌跡的差異（LCS 對齊：新增／移除／狀態翻轉／退化） | +| `utils/run_history/` | 346 | 執行歷史儲存與產出物管理 | +| `utils/sarif/` | 134 | 以 SARIF 2.1.0 匯出發現項，供 GitHub／Azure code scanning | +| `utils/slo/` | 112 | SLO 評估：SLI、錯誤預算與多視窗燃燒率告警 | +| `utils/smoothing/` | 67 | 數列移動平均平滑 | +| `utils/soft_assert/` | 62 | 軟斷言：累積檢查並在區塊結束時一次拋出 | +| `utils/stats/` | 213 | 描述統計與 A/B 顯著性檢定（純標準庫） | +| `utils/step_timeline/` | 81 | 每次執行的步驟瀑布圖與瓶頸（關鍵路徑）步驟排名 | +| `utils/test_select/` | 123 | 以執行歷史做風險導向的測試選取 | +| `utils/test_shard/` | 87 | 以耗時為權重的套件切分與分片結果合併 | +| `utils/test_suite/` | 442 | QA 套件編排：把扁平 action list 評分為測試案例 + CI 報表 | +| `utils/time_travel/` | 381 | 錄製 session 的時光回溯除錯（控制器 + 播放器） | +| `utils/timeseries/` | 143 | 時間序列轉換（rate／降採樣／重採樣） | +| `utils/trace_context/` | 162 | W3C Trace Context 傳遞 | ### 5.4.13 資料來源、結構驗證與 i18n -> 24 個套件、約 3,937 行。 +> 24 個套件、約 3,886 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/checksum/` | 134 | 檢查碼演算法：Luhn、Verhoeff、Damm、ISO 7064 MOD 97-10 | -| `utils/config_schema/` | 111 | 型別化設定結構驗證 | -| `utils/data_drift/` | 127 | 分布漂移偵測 | -| `utils/data_profile/` | 123 | 資料剖析與結構推斷 | -| `utils/data_quality/` | 187 | 資料品質：列結構驗證、欄位擷取、遮蔽 | -| `utils/data_source/` | 182 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | -| `utils/dataset_diff/` | 91 | 表格資料列差異比對（CDC 風格） | -| `utils/gettext_catalog/` | 298 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | -| `utils/i18n_test/` | 132 | 國際化／在地化測試輔助 | -| `utils/json_contract/` | 137 | JSON 契約／快照比對：`match_json`、`diff_json`、`snapshot_json` | -| `utils/json_patch/` | 314 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | -| `utils/json_schema/` | 376 | JSON Schema（Draft 2020-12 子集）驗證 | -| `utils/jsonpath/` | 181 | 精簡 JSONPath 查詢 | -| `utils/list_format/` | 74 | 地區感知清單格式化（CLDR 風格的「A、B 和 C」） | -| `utils/locale_collation/` | 130 | 地區感知字串排序（決定性多層排序鍵） | -| `utils/locale_parse/` | 70 | 地區感知數字／貨幣／日期解析與格式化（選用 babel） | -| `utils/message_format/` | 238 | ICU-lite MessageFormat（plural／select／selectordinal） | -| `utils/office/` | 164 | Office 文件無頭讀寫（Excel／Word／PowerPoint） | -| `utils/pdf/` | 89 | PDF 讀取與斷言（選用 pypdf 後端） | -| `utils/referential/` | 77 | 跨資料集的參照完整性檢查 | -| `utils/schema_compat/` | 164 | JSON Schema 相容性分級 | -| `utils/sql/` | 76 | 對 SQLite 的臨時唯讀 SQL 查詢 | -| `utils/test_data/` | 207 | 帶種子的合成測試資料產生（純標準庫） | -| `utils/xml/` | 255 | XML 檔讀寫與結構變更（`defusedxml`） | +| `utils/checksum/` | 132 | 檢查碼演算法：Luhn、Verhoeff、Damm、ISO 7064 MOD 97-10 | +| `utils/config_schema/` | 109 | 型別化設定結構驗證 | +| `utils/data_drift/` | 125 | 分布漂移偵測 | +| `utils/data_profile/` | 121 | 資料剖析與結構推斷 | +| `utils/data_quality/` | 185 | 資料品質：列結構驗證、欄位擷取、遮蔽 | +| `utils/data_source/` | 180 | 資料驅動執行：從 CSV／JSON／SQLite／Excel 載入資料列 | +| `utils/dataset_diff/` | 89 | 表格資料列差異比對（CDC 風格） | +| `utils/gettext_catalog/` | 296 | GNU gettext 目錄 I/O（解析 .po、編譯／讀取 .mo、訊息查詢） | +| `utils/i18n_test/` | 130 | 國際化／在地化測試輔助 | +| `utils/json_contract/` | 135 | JSON 契約／快照比對：`match_json`、`diff_json`、`snapshot_json` | +| `utils/json_patch/` | 312 | JSON Pointer（6901）、JSON Patch（6902）與 Merge Patch（7386） | +| `utils/json_schema/` | 374 | JSON Schema（Draft 2020-12 子集）驗證 | +| `utils/jsonpath/` | 179 | 精簡 JSONPath 查詢 | +| `utils/list_format/` | 72 | 地區感知清單格式化（CLDR 風格的「A、B 和 C」） | +| `utils/locale_collation/` | 128 | 地區感知字串排序（決定性多層排序鍵） | +| `utils/locale_parse/` | 68 | 地區感知數字／貨幣／日期解析與格式化（選用 babel） | +| `utils/message_format/` | 236 | ICU-lite MessageFormat（plural／select／selectordinal） | +| `utils/office/` | 162 | Office 文件無頭讀寫（Excel／Word／PowerPoint） | +| `utils/pdf/` | 87 | PDF 讀取與斷言（選用 pypdf 後端） | +| `utils/referential/` | 75 | 跨資料集的參照完整性檢查 | +| `utils/schema_compat/` | 162 | JSON Schema 相容性分級 | +| `utils/sql/` | 74 | 對 SQLite 的臨時唯讀 SQL 查詢 | +| `utils/test_data/` | 205 | 帶種子的合成測試資料產生（純標準庫） | +| `utils/xml/` | 250 | XML 檔讀寫與結構變更（`defusedxml`） | ### 5.4.14 安全、機密與合規 -> 13 個套件、約 2,290 行。 +> 13 個套件、約 2,261 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/config_redaction/` | 77 | 設定結構與 log 字串的機密遮蔽 | -| `utils/egress/` | 116 | 無頭 HTTP 用戶端的網路外連允許清單守衛 | -| `utils/governance/` | 202 | 治理：maker-checker 核准閘門與即時憑證租約 | -| `utils/license_policy/` | 141 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | -| `utils/provenance/` | 106 | SLSA 建置來源證明（in-toto v1） | -| `utils/rbac/` | 274 | 角色型存取控制與逐使用者稽核歸因 | -| `utils/redaction/` | 461 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | -| `utils/sbom/` | 110 | SBOM（CycloneDX）產生 | -| `utils/secret_ref/` | 128 | URI scheme 形式的值參照解析 | -| `utils/secrets/` | 253 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | -| `utils/secrets_scan/` | 100 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | -| `utils/vex/` | 132 | OpenVEX 陳述撰寫與漏洞分類處置 | -| `utils/vuln_scan/` | 190 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | +| `utils/config_redaction/` | 75 | 設定結構與 log 字串的機密遮蔽 | +| `utils/egress/` | 114 | 無頭 HTTP 用戶端的網路外連允許清單守衛 | +| `utils/governance/` | 199 | 治理：maker-checker 核准閘門與即時憑證租約 | +| `utils/license_policy/` | 139 | 以 SBOM 元件評估 SPDX 授權允許／拒絕政策 | +| `utils/provenance/` | 104 | SLSA 建置來源證明（in-toto v1） | +| `utils/rbac/` | 272 | 角色型存取控制與逐使用者稽核歸因 | +| `utils/redaction/` | 457 | 截圖遮蔽層：規則偵測 + 政策 + 協調器（上傳 VLM 前先遮） | +| `utils/sbom/` | 108 | SBOM（CycloneDX）產生 | +| `utils/secret_ref/` | 126 | URI scheme 形式的值參照解析 | +| `utils/secrets/` | 251 | 加密機密儲存庫，供 `${secrets.NAME}` 解析 | +| `utils/secrets_scan/` | 98 | 掃描 action JSON／資料中應入庫卻硬編碼的機密 | +| `utils/vex/` | 130 | OpenVEX 陳述撰寫與漏洞分類處置 | +| `utils/vuln_scan/` | 188 | 以 OSV 比對 SBOM 元件的漏洞（純標準庫） | ### 5.4.15 韌性、流量控制與設定 -> 14 個套件、約 1,734 行。 +> 14 個套件、約 1,706 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/artifact_store/` | 116 | S3 相容產出物儲存（報表／截圖／錄影） | -| `utils/assets/` | 157 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | -| `utils/bulkhead/` | 136 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | -| `utils/chaos/` | 155 | 決定性混沌實驗（穩態假說 + 故障注入） | -| `utils/dedup_window/` | 65 | 時間視窗內的訊息去重 | -| `utils/dotenv/` | 103 | `.env` 檔解析與序列化 | -| `utils/feature_flags/` | 175 | 功能旗標評估，含目標規則與決定性灰度 | -| `utils/idempotency/` | 116 | 冪等鍵儲存與已存回應重放 | -| `utils/layered_config/` | 112 | 分層設定解析 | -| `utils/optimistic/` | 107 | 樂觀併發的版本化儲存 | -| `utils/rate_limit/` | 164 | 用戶端限流：token bucket、滑動視窗、throttle | -| `utils/resilience/` | 112 | 韌性原語：退避重試與斷路器 | -| `utils/retry_budget/` | 149 | 重試預算：以牆鐘期限與 full jitter 約束重試 | -| `utils/sequence_gap/` | 67 | 逐串流的序號缺口偵測 | +| `utils/artifact_store/` | 114 | S3 相容產出物儲存（報表／截圖／錄影） | +| `utils/assets/` | 155 | 環境範圍的型別化資產／設定儲存（UiPath Assets 風格） | +| `utils/bulkhead/` | 134 | Bulkhead 併發隔離 + 伺服器限流標頭解析 | +| `utils/chaos/` | 153 | 決定性混沌實驗（穩態假說 + 故障注入） | +| `utils/dedup_window/` | 63 | 時間視窗內的訊息去重 | +| `utils/dotenv/` | 101 | `.env` 檔解析與序列化 | +| `utils/feature_flags/` | 173 | 功能旗標評估，含目標規則與決定性灰度 | +| `utils/idempotency/` | 114 | 冪等鍵儲存與已存回應重放 | +| `utils/layered_config/` | 110 | 分層設定解析 | +| `utils/optimistic/` | 105 | 樂觀併發的版本化儲存 | +| `utils/rate_limit/` | 162 | 用戶端限流：token bucket、滑動視窗、throttle | +| `utils/resilience/` | 110 | 韌性原語：退避重試與斷路器 | +| `utils/retry_budget/` | 147 | 重試預算：以牆鐘期限與 full jitter 約束重試 | +| `utils/sequence_gap/` | 65 | 逐串流的序號缺口偵測 | ### 5.4.16 系統、視窗與剪貼簿 -> 16 個套件、約 2,411 行。 +> 16 個套件、約 2,403 行。 | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `utils/clipboard/` | 353 | 跨平台無頭剪貼簿存取（文字 + 影像）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | -| `utils/clipboard_files/` | 119 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | -| `utils/clipboard_formats/` | 152 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | -| `utils/clipboard_history/` | 111 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | -| `utils/clipboard_rich_formats/` | 279 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | -| `utils/file_assoc/` | 94 | 解析哪個應用程式被註冊來開啟某副檔名 | -| `utils/file_dialog/` | 62 | 驅動原生檔案 開啟／儲存／資料夾選擇 對話框 | -| `utils/file_drop/` | 98 | 以 WM_DROPFILES 把檔案拖放到視窗 | -| `utils/rich_clipboard/` | 151 | 豐富剪貼簿格式 — HTML（CF_HTML）建構／解析／存取 | -| `utils/shell_open/` | 99 | 以預設應用開啟檔案，或以預設瀏覽器開啟 URL | -| `utils/system_volume/` | 196 | 讀取與控制系統主音量與靜音狀態 | -| `utils/trash/` | 90 | 把檔案移到系統資源回收筒（可復原刪除） | -| `utils/window_capture/` | 249 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | -| `utils/window_geometry/` | 83 | 視窗客戶區幾何（外框內縮、client→screen 對映） | -| `utils/window_layout/` | 136 | 視窗拼貼／版面規劃器（左右半、四象限、網格、層疊） | -| `utils/window_zorder/` | 78 | 視窗 z 序控制（最上層／移到最前／送到最後） | +| `utils/clipboard/` | 489 | 跨平台無頭剪貼簿存取（文字 + 影像）＋ `win32_clipboard_api.py`：**所有剪貼簿格式共用的 Win32 原型與 open/alloc/lock 流程**（`open_clipboard()` 會等過短暫被別的行程佔住的剪貼簿——Win32 一次只允許一個行程開啟，別人正在複製就必然失敗）（`argtypes` 只宣告一半曾讓四支 writer 在 64 位元上必然丟 `OverflowError`，見 CHANGELOG）。`set_clipboard_image` 同時接受 PNG 位元組與檔案路徑——先前這個名字在本子套件裡有**兩份不同簽章的實作**（`clipboard.py` 吃 bytes、`clipboard_image.py` 吃路徑），匯錯來源只會在執行期才炸，已合併成一支 | +| `utils/clipboard_files/` | 96 | 剪貼簿檔案清單（CF_HDROP）：純 DROPFILES 封裝 + Win32 存取 | +| `utils/clipboard_formats/` | 151 | 檢視與分類剪貼簿可用格式（純分類／差異 + Win32 列舉） | +| `utils/clipboard_history/` | 109 | 剪貼簿歷史：環形緩衝 + 背景輪詢器 | +| `utils/clipboard_rich_formats/` | 254 | 豐富剪貼簿格式 — RTF 與 CSV/TSV 編解碼 + Windows 存取 | +| `utils/file_assoc/` | 92 | 解析哪個應用程式被註冊來開啟某副檔名 | +| `utils/file_dialog/` | 60 | 驅動原生檔案 開啟／儲存／資料夾選擇 對話框 | +| `utils/file_drop/` | 96 | 以 WM_DROPFILES 把檔案拖放到視窗 | +| `utils/rich_clipboard/` | 131 | 豐富剪貼簿格式 — HTML（CF_HTML）建構／解析／存取 | +| `utils/shell_open/` | 97 | 以預設應用開啟檔案，或以預設瀏覽器開啟 URL | +| `utils/system_volume/` | 194 | 讀取與控制系統主音量與靜音狀態 | +| `utils/trash/` | 88 | 把檔案移到系統資源回收筒（可復原刪除） | +| `utils/window_capture/` | 255 | 逐視窗截圖、視窗版面儲存／還原、貼齊與排列 | +| `utils/window_geometry/` | 81 | 視窗客戶區幾何（外框內縮、client→screen 對映） | +| `utils/window_layout/` | 134 | 視窗拼貼／版面規劃器（左右半、四象限、網格、層疊） | +| `utils/window_zorder/` | 76 | 視窗 z 序控制（最上層／移到最前／送到最後） | ### 5.4.17 大型子系統的檔案級剖析 上表以子套件為單位；以下把行數最大的幾個子系統展開到檔案層。 -#### `utils/executor/`（8,811 行）— 執行核心 +#### `utils/executor/`（9,075 行）— 執行核心 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `action_executor.py` | 8,042 | `Executor` 類別與 `event_dict` 分派表（767 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | -| `flow_control.py` | 757 | 34 個區塊指令：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`、`AC_*_to_var`）／`AC_assert_var`。`LoopBreak`／`LoopContinue` 以例外實作。 | -| `action_schema.py` | 94 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。 | -| `mouse_aliases.py` | 40 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | +| `action_executor.py` | 8,125 | `Executor` 類別與 `event_dict` 分派表（773 個指令），另含數百個把 utils 能力接成指令的 adapter 函式；全域單例 `executor` 與 `add_command_to_executor()` 擴充點。 | +| `flow_control.py` | 530 | 真正的流程控制：`AC_loop`／`AC_for_each`／`AC_while_*`／`AC_if_*`／`AC_try`／`AC_retry`／`AC_parallel`／`AC_define_macro`／`AC_call_macro`／變數指令（`AC_set_var`／`AC_get_var`／`AC_inc_var`）。`LoopBreak`／`LoopContinue` 以例外實作。34 個區塊指令的分派表 `BLOCK_COMMANDS` 也在這裡，含下一列匯入的資料來源指令。 | +| `flow_data_commands.py` | 253 | `AC_*_to_var` 資料來源與轉換指令：shell、時鐘、亂數、PDF、TOTP、SQL、檔案、HTTP、OCR，加上 `AC_assert_var`／`AC_assert_db`／`AC_assert_duration`／`AC_transform_var`。都不執行巢狀 action list，所以沒有迴圈／分支語意。 | +| `action_schema.py` | 128 | action list 的結構驗證：形狀、參數型別、未知指令拒絕。單一走訪同時支援兩種消費方式：`validate_actions()` 遇到第一個問題就拋、`unknown_command_names()` 收齊全部不認得的名字（REST `/execute` 用它回 400）。 | +| `mouse_aliases.py` | 39 | 單鍵點擊別名（`AC_click_left` 等），executor 與 callback executor 共用。 | -#### `utils/mcp_server/`（16,441 行，670 個工具）— 最大子系統 +#### `utils/mcp_server/`（16,850 行，676 個工具）— 最大子系統 | 檔案 | 行數 | 職責 | | --- | ---: | --- | | `tools/_factories.py` | 8,739 | 工具工廠：每個函式回傳一個領域的 `MCPTool` 清單（把 `AC_*` 能力包成 MCP 工具）。 | | `tools/_handlers.py` | 4,651 | 把 MCP 工具呼叫橋接到 AutoControl 無頭 API 的 adapter。 | -| `server.py` | 995 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器。 | +| `server.py` | 669 | JSON-RPC 2.0 over stdio 的最小 MCP 伺服器：連線範圍狀態、行內／併發分派、工具與 resource／prompt 處理器。 | | `http_transport.py` | 323 | MCP 的 HTTP 傳輸。 | -| `resources.py` | 304 | MCP resource 提供者。 | -| `prompts.py` | 221 | MCP prompt 目錄。 | -| `fake_backend.py` | 185 | CI／無頭測試用的記憶體內假後端。 | -| `plugin_watcher.py` | 150 | 檔案變更時熱重載外掛工具的背景 watcher。 | +| `_client_requests.py` | 217 | 伺服器主動送出的請求：`roots/list`／`elicitation/create`／`sampling/createMessage`,對應表與回應路由,以及破壞性工具的確認交握。 | +| `_protocol.py` | 165 | JSON-RPC 線路格式：版本與識別常數、`_MCPError`、決定失敗工具行為的錯誤 tuple、envelope 產生器、工具回傳值轉 `content` 區塊。不碰伺服器狀態。 | +| `resources.py` | 303 | MCP resource 提供者。 | +| `prompts.py` | 220 | MCP prompt 目錄。 | +| `fake_backend.py` | 184 | CI／無頭測試用的記憶體內假後端。 | +| `plugin_watcher.py` | 149 | 檔案變更時熱重載外掛工具的背景 watcher。 | | `tools/_base.py` | 147 | 工具註冊表的共用型別與輔助。 | | `tools/_validation.py` | 107 | MCP 工具用到的 JSON Schema 子集驗證器。 | | `tools/plugin_tools.py` | 90 | 把外掛載入的 `AC_*` callable 包成 `MCPTool`。 | -| `log_bridge.py` | 91 | 把 Python logging 記錄橋接成 MCP `notifications/message`。 | -| `audit.py` | 79 | MCP 工具呼叫稽核記錄。 | -| `context.py` | 72 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | -| `rate_limit.py` | 49 | 工具呼叫的 token bucket 限流。 | -| `__main__.py` | 88 | `je_auto_control_mcp` console script 進入點。 | +| `log_bridge.py` | 90 | 把 Python logging 記錄橋接成 MCP `notifications/message`。 | +| `audit.py` | 78 | MCP 工具呼叫稽核記錄。 | +| `context.py` | 71 | 傳給 opt-in 工具處理器的每次呼叫上下文。 | +| `rate_limit.py` | 48 | 工具呼叫的 token bucket 限流。 | +| `__main__.py` | 87 | `je_auto_control_mcp` console script 進入點。 | -#### `utils/remote_desktop/`（11,726 行／51 檔） +#### `utils/remote_desktop/`（11,835 行／56 檔） 三條傳輸路徑並存：**TCP**（JPEG 影格）、**WebSocket**（同協定換傳輸）、**WebRTC**（aiortc 視訊 + DataChannel）。 | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `host.py` | 1,319 | TCP 主機：串流 JPEG 影格並套用檢視端輸入。 | -| `webrtc_host.py` | 996 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入。 | -| `webrtc_viewer.py` | 639 | WebRTC 檢視端：接收視訊並送出輸入。 | -| `viewer.py` | 624 | TCP 檢視端。 | -| `host_service.py` | 543 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | -| `registry.py` | 371 | `AC_remote_*` 指令使用的行程級單例。 | -| `webrtc_transport.py` | 357 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | -| `multi_viewer.py` | 315 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | -| `signaling_server.py` | 298 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | -| `audit_log.py` | 284 | SQLite 雜湊鏈稽核記錄。 | -| `ws_protocol.py` | 278 | 最小 RFC 6455 WebSocket 框架與握手。 | -| `file_transfer.py` | 274 | 分塊檔案傳輸。 | -| `relay.py` | 271 | NAT 穿透失敗時的 TCP 中繼。 | -| `fingerprint.py` | 251 | TOFU 主機指紋驗證。 | -| `turn_config.py` | 235 | coturn 設定產生器。 | -| `presence.py` | 222 | 多檢視者的執行緒安全在場註冊表。 | -| `jpeg_recorder_encrypted.py` | 218 | AES-GCM 加密版 session 錄影。 | -| `address_book.py` | 210 | 檢視端的主機通訊錄。 | +| `webrtc_host.py` | 683 | WebRTC 主機：串流螢幕視訊並接受檢視端輸入;session 生命週期、DataChannel 接線、檔案收發。 | +| `webrtc_viewer.py` | 638 | WebRTC 檢視端：接收視訊並送出輸入。 | +| `host.py` | 625 | TCP 主機：接受迴圈、TLS 包裝、連線／認證握手、音訊與剪貼簿廣播、檔案推送、單次 token。 | +| `viewer.py` | 623 | TCP 檢視端。 | +| `host_service.py` | 542 | 無頭 WebRTC 主機執行器 + 多平台服務安裝器。 | +| `host_client.py` | 406 | TCP 主機的每連線處理器：一個檢視端一個實例,擁有它的認證交換、sender／audio／receiver 三條執行緒,以及入站訊息的路由表。 | +| `registry.py` | 370 | `AC_remote_*` 指令使用的行程級單例。 | +| `webrtc_transport.py` | 360 | 共用 WebRTC 管線：asyncio 橋接執行緒、螢幕視訊軌、設定。 | +| `multi_viewer.py` | 314 | 每個連入檢視端各跑一個 `WebRTCDesktopHost` 的協調器。 | +| `signaling_server.py` | 297 | 獨立的 WebRTC SDP 交換 rendezvous 服務。 | +| `audit_log.py` | 283 | SQLite 雜湊鏈稽核記錄。 | +| `host_capture.py` | 280 | TCP 主機的影格與游標產生：螢幕列舉、監視器索引轉擷取區域、預設 JPEG／游標 provider,以及 `FrameProductionMixin`（游標輪詢、擷取迴圈、上線編碼）。 | +| `ws_protocol.py` | 277 | 最小 RFC 6455 WebSocket 框架與握手。 | +| `file_transfer.py` | 273 | 分塊檔案傳輸。 | +| `relay.py` | 270 | NAT 穿透失敗時的 TCP 中繼。 | +| `fingerprint.py` | 250 | TOFU 主機指紋驗證。 | +| `turn_config.py` | 234 | coturn 設定產生器。 | +| `presence.py` | 221 | 多檢視者的執行緒安全在場註冊表。 | +| `jpeg_recorder_encrypted.py` | 217 | AES-GCM 加密版 session 錄影。 | +| `address_book.py` | 209 | 檢視端的主機通訊錄。 | | `audio.py` / `webrtc_audio.py` / `webrtc_mic.py` | 206 / 190 / 152 | 音訊擷取播放、音訊軌、麥克風上行。 | -| `webrtc_files.py` | 206 | 專屬 DataChannel 的分塊檔案傳輸。 | -| `lan_discovery.py` | 190 | mDNS／Zeroconf 區網探索。 | -| `video_codec.py` | 183 | TCP／WS 路徑的可插拔視訊編解碼。 | -| `hw_codec.py` | 170 | 硬體 H.264 編碼偵測與啟用。 | -| `webrtc_stats.py` | 164 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | -| `webrtc_inspector.py` | 139 | 行程級的 `StatsSnapshot` 滾動視窗。 | -| `adaptive_bitrate.py` | 149 | 依統計調整主機擷取 FPS。 | -| `connect_coordinator.py` | 150 | 由使用者輸入的目標決定該用哪條傳輸。 | -| `signaling_client.py` | 146 | 純標準庫的訊令用戶端。 | -| `trust_list.py` | 145 | 自動接受的檢視端信任清單。 | -| `input_dispatch.py` | 134 | 在主機端套用輸入訊息。 | -| `session_recorder.py` | 130 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | -| `totp.py` | 130 | RFC 6238 TOTP（零外部相依）。 | -| `file_sync.py` | 127 | 輪詢式資料夾鏡像。 | -| `transport.py` | 124 | 可插拔的型別化訊息傳輸。 | -| `protocol.py` | 97 | 長度前綴的 TCP 框架。 | +| `webrtc_files.py` | 205 | 專屬 DataChannel 的分塊檔案傳輸。 | +| `webrtc_host_auth.py` | 195 | 檢視端認證與核准：token 檢查、信任清單／IP 白名單自動放行、手動接受／拒絕、SAS、逾時關閉。 | +| `lan_discovery.py` | 189 | mDNS／Zeroconf 區網探索。 | +| `video_codec.py` | 182 | TCP／WS 路徑的可插拔視訊編解碼。 | +| `webrtc_host_media.py` | 172 | 重新協商與 recvonly 軌管理。aiortc 沒有 `removeTransceiver`,所以開／關不對稱——開是加軌重新 offer,關只能設 inactive 並停掉 receiver。 | +| `hw_codec.py` | 169 | 硬體 H.264 編碼偵測與啟用。 | +| `webrtc_stats.py` | 163 | 把 aiortc 的 `RTCStats` 報告輪詢成精簡 dict。 | +| `connect_coordinator.py` | 149 | 由使用者輸入的目標決定該用哪條傳輸。 | +| `adaptive_bitrate.py` | 148 | 依統計調整主機擷取 FPS。 | +| `signaling_client.py` | 145 | 純標準庫的訊令用戶端。 | +| `trust_list.py` | 144 | 自動接受的檢視端信任清單。 | +| `webrtc_inspector.py` | 138 | 行程級的 `StatsSnapshot` 滾動視窗。 | +| `input_dispatch.py` | 133 | 在主機端套用輸入訊息。 | +| `session_recorder.py` | 129 | 以 PyAV 把 WebRTC 影格錄成 mp4。 | +| `totp.py` | 129 | RFC 6238 TOTP（零外部相依）。 | +| `file_sync.py` | 126 | 輪詢式資料夾鏡像。 | +| `transport.py` | 123 | 可插拔的型別化訊息傳輸。 | +| `host_access.py` | 105 | TCP 主機的檢視端核准與存取控制：`PendingViewer`、權限字串、分享碼的 TOTP 候選值、IP 白名單。`host` 與 `host_client` 共用,所以獨立成模組。 | +| `protocol.py` | 96 | 長度前綴的 TCP 框架。 | | `resume_tokens.py` / `session_quality_cache.py` / `rate_limit.py` | 95 / 86 / 85 | 快速重連 token、每 session 品質快取、檢視端限流。 | | `host_id.py` / `viewer_id.py` | 82 / 78 | 主機與檢視端的持久身分。 | | `permissions.py` / `clipboard_sync.py` / `wake_on_lan.py` / `session_actions.py` / `auth.py` | 65 / 73 / 57 / 41 / 29 | 逐 session 權限、剪貼簿同步、WOL、SAS 注入與螢幕遮蔽、HMAC 挑戰回應。 | | `ws_host.py` / `ws_viewer.py` / `jpeg_recorder.py` | 41 / 30 / 139 | WebSocket 傳輸變體與 TCP 路徑錄影。 | -#### `utils/usb/`（4,255 行）與 `utils/usbip/`（925 行） +#### `utils/usb/`（4,238 行）與 `utils/usbip/`（920 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | @@ -778,17 +794,17 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `usbip/libusb_backend.py` | 209 | 以 PyUSB／libusb 執行 URB 的正式後端。 | | `usbip/backend.py` | 88 | 可插拔 URB 執行後端。 | -#### `utils/rest_api/`（1,693 行） +#### `utils/rest_api/`（1,738 行） | 檔案 | 行數 | 職責 | | --- | ---: | --- | -| `rest_server.py` | 468 | HTTP 前端主體。 | -| `rest_handlers.py` | 451 | 端點實作。 | -| `rest_openapi.py` | 406 | 走訪路由表產生 OpenAPI 3.1 規格。 | -| `rest_auth.py` | 144 | Bearer token 驗證 + 逐 client 限流閘門。 | -| `rest_metrics.py` | 76 | Prometheus 曝露端點。 | -| `rest_registry.py` | 76 | 保存執行中 REST 伺服器的行程級單例。 | -| `__main__.py` | 57 | `python -m je_auto_control.utils.rest_api` 進入點。 | +| `rest_server.py` | 467 | HTTP 前端主體。 | +| `rest_handlers.py` | 486 | 端點實作。 | +| `rest_openapi.py` | 422 | 走訪路由表產生 OpenAPI 3.1 規格。 | +| `rest_auth.py` | 143 | Bearer token 驗證 + 逐 client 限流閘門。 | +| `rest_metrics.py` | 75 | Prometheus 曝露端點。 | +| `rest_registry.py` | 75 | 保存執行中 REST 伺服器的行程級單例。 | +| `__main__.py` | 56 | `python -m je_auto_control.utils.rest_api` 進入點。 | #### 其他多檔子套件 @@ -809,7 +825,7 @@ socket server 有 8 MiB 讀取上限與 30 秒 handler timeout。 | `semantic_recording/` | `enrich.py`（加錨點）、`replay.py`（換機重播）、`self_healing.py`（自癒重播） | | `tls_acme/` | `challenge.py`、`keys.py`、`renewal.py` | | `pytest_plugin/` | `plugin.py`（pytest11 進入點）、`keywords.py`、`bdd_steps.py`（Gherkin） | -| `cv2_utils/` | `screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`blobs.py` | +| `cv2_utils/` | `screen_grabber.py`、`screenshot.py`、`template_detection.py`、`screen_record.py`、`video_recording.py`、`blobs.py` | | `action_lint/` | `linter.py`、`schema.py`、`__main__.py`（CI 使用） | | `time_travel/` | `controller.py`、`player.py` | | `dag/` | `graph.py`、`runner.py` | @@ -838,10 +854,14 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `gui/__init__.py` | 24 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | +| `gui/__init__.py` | 23 | `start_autocontrol_gui()`：**唯一**會延遲匯入 PySide6 的地方，維持頂層套件 Qt-free。 | | `main_window.py` | 290 | `QMainWindow`：選單列（File／Actions／View／…）、可關閉分頁、即時語言切換、字級預設、qt-material 主題。分頁分為 core／editing／detection／automation／system 五類。 | -| `main_widget.py` | 780 | 擁有 `QTabWidget`，註冊 48 個分頁，並暴露 show/hide/list API 給選單列。核心分頁在註冊時直接宣告 `(label_key, handler)` 動作對。 | +| `main_widget.py` | 423 | 擁有 `QTabWidget`，註冊 48 個分頁，並暴露 show/hide/list API 給選單列。核心分頁在註冊時直接宣告 `(label_key, handler)` 動作對；分頁本體都在下列 mixin。 | | `_auto_click_tab.py` | 270 | 自動點擊分頁的 mixin 建構器。 | +| `_screenshot_tab.py` | 127 | 截圖／取像素分頁 mixin。 | +| `_image_detect_tab.py` | 106 | 影像偵測分頁 mixin。 | +| `_script_tab.py` | 105 | 腳本執行分頁 mixin。 | +| `_record_tab.py` | 101 | 錄製／回放分頁 mixin。 | | `_report_tab.py` | 81 | 報表分頁 mixin。 | | `_i18n_helpers.py` | 67 | 需要即時語言切換的分頁共用的翻譯註冊 mixin。 | | `language_wrapper/` | 4,977 | 四語系字典（英／日／簡中／繁中）+ `multi_language_wrapper` 執行期切換器與監聽註冊表。 | @@ -856,12 +876,12 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | 分頁 | 模組 | 行數 | 職責 | | --- | --- | ---: | --- | | auto_click | `_auto_click_tab.py` | 270 | 自動點擊：座標、間隔、熱鍵、`write`、捲動。 | -| screenshot | `main_widget.py` 內建 | — | 截圖、選區、螢幕尺寸、取像素色。 | -| image_detect | `main_widget.py` 內建 | — | 樣板裁切、定位、定位全部、定位並點擊。 | -| record | `main_widget.py` 內建 | — | 錄製／停止／回放／存檔／載入。 | +| screenshot | `_screenshot_tab.py` | 127 | 截圖、選區、螢幕尺寸、取像素色。 | +| image_detect | `_image_detect_tab.py` | 106 | 樣板裁切、定位、定位全部、定位並點擊。 | +| record | `_record_tab.py` | 101 | 錄製／停止／回放／存檔／載入。 | | script_builder | `script_builder/` | 5,708 | **視覺化腳本編輯器**：`command_schema.py`（4,924 行 `AC_*` 參數綱要）、`step_model.py`（步驟模型與 AC JSON 序列化）、`step_list_view.py`（含巢狀 body 的樹狀檢視）、`step_form_view.py`（綱要驅動表單）、`builder_tab.py`。 | | flow_editor | `flow_editor/` | 490 | 節點式流程圖檢視：`layout.py`（純 Python 佈局演算法，可單測）、`scene.py`（Qt 場景繪製）、`tab.py`。 | -| script | `main_widget.py` 內建 | — | 載入／執行單檔或整個目錄、內建編輯器執行。 | +| script | `_script_tab.py` | 105 | 載入／執行單檔或整個目錄、內建編輯器執行。 | | recording_editor | `recording_editor_tab.py` | 244 | 裁切、過濾、重新縮放錄製內容。 | | variables | `variables_tab.py` | 166 | 檢視、灌入、清除 executor 執行期作用域。 | | secrets | `secrets_tab.py` | 188 | 解鎖保險庫並管理 `${secrets.NAME}`。 | @@ -892,7 +912,7 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | dag_runner | `dag_tab.py` | 188 | 編輯、驗證、執行跨主機 DAG。 | | chatops | `chatops_tab.py` | 108 | 在接上 Slack 前先本機測試 slash 指令。 | | trace_replay | `trace_replay_tab.py` | 187 | 拖曳捲動時光回溯錄製內容。 | -| remote_desktop | `remote_desktop/`（16 檔） | 6,240 | 見下。 | +| remote_desktop | `remote_desktop/`（17 檔） | 6,240 | 見下。 | | presence | `presence_tab.py` | 152 | 多檢視者在場名單。 | | rest_api | `rest_api_tab.py` | 198 | 啟停 HTTP 前端並顯示 URL 與 token。 | | admin_console | `admin_console_tab.py` | 313 | 管理多個遠端 AutoControl REST 端點。 | @@ -904,25 +924,26 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | diagnostics | `diagnostics_tab.py` | 91 | 執行子系統檢查並顯示結果。 | | report | `_report_tab.py` | 81 | 產生 HTML／JSON／XML 報表。 | -#### 遠端桌面 GUI（`gui/remote_desktop/`，16 檔／6,240 行） +#### 遠端桌面 GUI（`gui/remote_desktop/`，17 檔／6,254 行） | 模組 | 行數 | 職責 | | --- | ---: | --- | -| `webrtc_panel.py` | 2,556 | WebRTC 子分頁主體。 | -| `webrtc_dialogs.py` | 845 | WebRTC GUI 用的自訂對話框與清單元件。 | -| `connection_screen.py` | 673 | Quick Connect —— AnyDesk 風格單畫面入口。 | -| `viewer_panel.py` | 543 | 「控制另一台機器」子分頁。 | -| `host_panel.py` | 335 | 「分享這台機器」子分頁。 | -| `frame_display.py` | 229 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | -| `webrtc_workers.py` | 196 | 訊令流程的背景 `QThread` worker。 | -| `tab.py` | 166 | 外層容器分頁。 | -| `_helpers.py` | 149 | 面板共用輔助。 | -| `remote_screen_window.py` | 141 | 檢視端的彈出視窗。 | -| `tray_icon.py` | 99 | WebRTC 主機的系統匣圖示。 | -| `annotation_overlay.py` | 89 | 主機端標註的透明最上層覆蓋。 | -| `sparkline.py` | 78 | WebRTC 統計面板的迷你走勢圖。 | -| `blanking_overlay.py` | 72 | 遠端連線期間的隱私遮蔽全螢幕覆蓋。 | -| `viewer_screen_window.py` | 47 | 顯示連入檢視端分享畫面的彈出視窗。 | +| `webrtc_panel.py` | 2,555 | WebRTC 子分頁主體。 | +| `webrtc_dialogs.py` | 493 | WebRTC GUI 用的自訂對話框與清單元件（待審檢視者、信任清單、通訊錄、遠端檔案表、稽核記錄、LAN 瀏覽）。 | +| `connection_screen.py` | 672 | Quick Connect —— AnyDesk 風格單畫面入口。 | +| `viewer_panel.py` | 542 | 「控制另一台機器」子分頁。 | +| `webrtc_known_hosts.py` | 340 | TOFU 釘選庫瀏覽器：`KnownHostsDialog` 與帶外釘選用的小表單。由 `webrtc_dialogs` 再匯出。 | +| `host_panel.py` | 334 | 「分享這台機器」子分頁。 | +| `frame_display.py` | 228 | 繪製 JPEG 影格並發出遠端輸入事件的元件。 | +| `webrtc_workers.py` | 195 | 訊令流程的背景 `QThread` worker。 | +| `tab.py` | 165 | 外層容器分頁。 | +| `_helpers.py` | 189 | 面板共用輔助：翻譯、Qt→AC 鍵滑鼠對應、TLS context、狀態徽章、指紋與時間格式化。 | +| `remote_screen_window.py` | 140 | 檢視端的彈出視窗。 | +| `tray_icon.py` | 98 | WebRTC 主機的系統匣圖示。 | +| `annotation_overlay.py` | 88 | 主機端標註的透明最上層覆蓋。 | +| `sparkline.py` | 77 | WebRTC 統計面板的迷你走勢圖。 | +| `blanking_overlay.py` | 71 | 遠端連線期間的隱私遮蔽全螢幕覆蓋。 | +| `viewer_screen_window.py` | 46 | 顯示連入檢視端分享畫面的彈出視窗。 | ### 5.6 周邊子專案與資產 @@ -935,12 +956,12 @@ GUI 是**選用 extra**（`pip install je_auto_control[gui]`，PySide6 + qt-mate | `benchmarks/core_latency.py` | 32 行 | 對穩定無頭進入點的可重複煙霧基準測試。 | | `examples/` | 27 個腳本 | 從截圖點擊、OCR、排程、遠端桌面、agent loop、可觀測性，一路到 computer-use、Wayland、跨主機 DAG、chatops、pytest/BDD、anchor locator。 | | `browser-extension/` | manifest v3 擴充 | 瀏覽器端配合元件（background／content script／popup）。 | -| `docker/` | Dockerfile ×2 + compose | 無頭容器與帶 XFCE 桌面的容器。 | +| `docker/` | Dockerfile ×6 + compose + 8 支驗證／伺服器腳本 | 無頭容器（`Dockerfile`）、帶 XFCE 桌面的容器（`Dockerfile.xfce`),以及四個**驗證用**映像:`Dockerfile.wayland`（sway headless,擷取路徑 + `libei_verify.py` 對真的 libei.so 解析符號）、`Dockerfile.eis`（`eis_server.py` 用 ctypes 綁 libeis 起一個真的 EIS server,`eis_verify.py` 把 libei sender 對著它跑完整握手與發送）、`Dockerfile.portal`（`portal_server.py` 自己佔住 `org.freedesktop.portal.Desktop`,真的 `dbus-daemon` + 真的 liboeffis 跑完 RemoteDesktop 交握）、`Dockerfile.ydotool`（真的 uinput 裝置,`ydotool_verify.py` 直接讀回 `/dev/input/eventN`）、`Dockerfile.seat`（`headless,libinput` + builtin seat,合成器真的吃下 ydotool 裝置,`seat_verify.py` 從 `grim -c` 的像素讀回游標落點）。全部接在 `.github/workflows/docker.yml`。 | | `k8s/helm/` | Helm chart | Kubernetes 部署。 | | `ci_templates/.gitlab-ci.yml` | — | 供使用者專案複製的 GitLab CI 範本。 | | `docs/` | Sphinx（`API`／`Eng`／`Zh`／`getting_started`） | Read the Docs 文件。 | | `architecture_diagram/` | drawio + png | 既有的架構圖原始檔。 | -| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 460 個 `test_*.py`／4,333 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 則守住文件引用的數字與實測值一致。 | +| `test/` | `unit_test/headless`（主要）、`unit_test/flow_control`、`integrated_test`、`gui_test`、`manual_test`、`test_source` | 466 個 `test_*.py`／4,443 個測試函式。**注意**：`test/unit_test/` 下的 `*_test.py` 是會真的驅動滑鼠鍵盤的手動示範腳本，因此 `pyproject.toml` 把 `python_files` 釘成 `test_*.py`。`unit_test/headless/conftest.py` 有一個 autouse fixture，每個測試結束都沖掉 Qt 排隊中的 `deleteLater()`——不沖會讓殘留的 widget 在後面某個不相干的測試裡被銷毀，曾經整個直譯器 `__fastfail`。`test_doc_counts.py` 守住文件引用的指令／工具／子套件／範例數,`test_doc_line_counts.py` 守住所有行數（`--fix` 可一次重新產生）。 | --- @@ -994,26 +1015,26 @@ socket 預設綁 `127.0.0.1`；資源一律用 `with`。 | 層／子系統 | 檔案數 | 行數 | | --- | ---: | ---: | -| `gui/` | 84 | 26,367 | -| `utils/mcp_server/` | 18 | 16,441 | -| `utils/remote_desktop/` | 51 | 11,726 | -| `utils/executor/` | 5 | 8,811 | -| `utils/usb/` | 17 | 4,255 | -| `je_auto_control/`（頂層 3 檔） | 3 | 2,325 | +| `gui/` | 89 | 26,542 | +| `utils/mcp_server/` | 20 | 16,850 | +| `utils/remote_desktop/` | 56 | 11,835 | +| `utils/executor/` | 6 | 9,075 | +| `utils/usb/` | 17 | 4,238 | +| `je_auto_control/`（頂層 3 檔） | 3 | 2,366 | | `utils/accessibility/` | 12 | 2,390 | -| `wrapper/` | 12 | 1,747 | -| `utils/rest_api/` | 8 | 1,693 | -| `windows/` | 26 | 1,939 | -| `utils/agent/` | 8 | 1,258 | -| `linux_with_x11/` | 19 | 1,189 | -| `utils/triggers/` | 4 | 1,150 | -| `linux_wayland/` | 10 | 1,093 | -| `utils/ocr/` | 9 | 1,105 | -| `utils/usbip/` | 5 | 925 | -| `utils/assertion/` | 3 | 866 | -| `osx/` | 17 | 771 | -| `autocontrol-lsp/` | 8 | 752 | -| `utils/hotkey/` | 7 | 734 | -| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,047 | -| **總計** | **989** | **132,598** | +| `wrapper/` | 12 | 2,056 | +| `windows/` | 23 | 1,995 | +| `utils/rest_api/` | 8 | 1,738 | +| `utils/agent/` | 8 | 1,250 | +| `linux_with_x11/` | 19 | 1,175 | +| `linux_wayland/` | 17 | 3,431 | +| `utils/triggers/` | 4 | 1,146 | +| `utils/ocr/` | 9 | 1,112 | +| `utils/usbip/` | 5 | 920 | +| `utils/assertion/` | 3 | 863 | +| `osx/` | 17 | 761 | +| `autocontrol-lsp/` | 8 | 744 | +| `utils/hotkey/` | 7 | 727 | +| 其餘模組（約 286 個 `utils/` 子套件 + `android/`／`ios/`／周邊小工具） | 667 | 46,238 | +| **總計** | **1,010** | **137,452** | diff --git a/docker/Dockerfile.eis b/docker/Dockerfile.eis new file mode 100644 index 00000000..72c8369b --- /dev/null +++ b/docker/Dockerfile.eis @@ -0,0 +1,83 @@ +# AutoControl EIS verification image — libei talking to libeis, no desktop. +# +# The Wayland image (docker/Dockerfile.wayland) answers the capture half and +# closes by listing what it cannot: the libei *input* path needs a peer that +# speaks the EI protocol, and xdg-desktop-portal-wlr has no RemoteDesktop +# interface to provide one. That was recorded as needing a GNOME VM. +# +# It does not. libeis is the server side of the same protocol, Debian packages +# it, and the two libraries will talk to each other over a plain Unix socket. +# So this image runs a real EIS server (docker/eis_server.py) and drives +# AutoControl's real sender against it — which settles the capability and +# event-type enum values, the variadic ei_seat_bind_capabilities call, seat +# grants, device pause/resume, whether start_emulating -> frame actually puts +# events on the wire, the discrete-scroll sign, and the absolute pointer's +# coordinate space: that region offsets are part of the coordinate, and that +# libei drops a motion landing outside every region without saying a word. +# +# What it does not cover, and where that lives instead: +# * how a client *gets* this socket on GNOME and KDE — a file descriptor +# handed over D-Bus by org.freedesktop.portal.RemoteDesktop rather than a +# path. That was recorded here as needing a GNOME VM too; it does not, +# because the portal is a D-Bus interface rather than a compositor +# feature. docker/Dockerfile.portal owns the well-known name and runs the +# real liboeffis through the real handshake, ending in a live connection +# to this same server. +# +# ydotool used to be listed here too, as needing /dev/uinput "and a seat that +# consumes it". The seat was never the requirement: docker/Dockerfile.ydotool +# reads the injected events straight off the kernel device. +# +# Build: docker build -f docker/Dockerfile.eis -t autocontrol-eis:latest . +# Run: docker run --rm autocontrol-eis:latest + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# libei1 is the client this project binds; libeis1 is the server side of the +# same protocol and the whole reason this image exists. +# +# grim / wtype / wlr-randr are not used by anything below, but the Wayland +# backend refuses to import without them and `import je_auto_control` then +# falls back to the X11 backend — which dies reaching for DISPLAY=:0 before a +# line of this verification can run. +# +# libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libei1 libeis1 \ + grim wlr-randr wtype \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/eis_server.py /opt/verify/eis_server.py +COPY docker/eis_verify.py /opt/verify/eis_verify.py + +# Runs as root on purpose: one verification, then exit. Not a service image. +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/opt/verify \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=wayland + +ENTRYPOINT ["python", "/opt/verify/eis_verify.py"] diff --git a/docker/Dockerfile.portal b/docker/Dockerfile.portal new file mode 100644 index 00000000..e23c7f22 --- /dev/null +++ b/docker/Dockerfile.portal @@ -0,0 +1,98 @@ +# AutoControl portal verification image — the RemoteDesktop D-Bus handshake. +# +# Three images already answer three quarters of the Linux input and capture +# story: docker/Dockerfile.wayland (a real wlroots compositor), +# docker/Dockerfile.eis (a real libeis peer) and docker/Dockerfile.ydotool (a +# real uinput device). All three close by naming the same remaining gap in the +# same words — "the portal consent dialog and the EIS fd handover — that is +# xdg-desktop-portal + GNOME/mutter, not the EI protocol itself" — and it was +# recorded as needing a GNOME VM. +# +# It does not. The portal is a *D-Bus interface*, not a compositor feature: +# whatever owns org.freedesktop.portal.Desktop and answers CreateSession -> +# SelectDevices -> Start -> ConnectToEIS is a portal as far as liboeffis is +# concerned. docker/portal_server.py is that, on a private session bus, and +# its ConnectToEIS hands back a live connection to the same real EIS server +# the eis image uses. So the real liboeffis runs the real handshake, the +# descriptor it produces carries a real EI session, and the input emitted +# through it is recorded by an independent implementation. +# +# The one thing it still does not claim is the consent dialog itself — no user +# dismisses anything here. What a dialog produces is a Response code or +# silence, and docker/portal_verify.py drives a grant, a refusal, a dialog +# left open, a withheld descriptor, a closed session, a portal too old to have +# ConnectToEIS, and no portal at all. +# +# Two interpreters on purpose. Debian's python3 has GDBus, which is what can +# pass a file descriptor over D-Bus from Python; the image's own python3.12 +# has AutoControl. The portal has to be a separate process from its client in +# any case, so the split costs nothing. +# +# Build: docker build -f docker/Dockerfile.portal -t autocontrol-portal:latest . +# Run: docker run --rm autocontrol-portal:latest + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# liboeffis1 is the subject: the portal half of the libei handshake, and the +# library je_auto_control/linux_wayland/oeffis.py binds. Note it is a separate +# binary package that libei1 does not depend on — it is packaged on trixie, +# but installing libei1 alone does not bring it, which is the difference +# between the portal fast path being available and being quietly off. +# +# libei1 / libeis1 are the two ends of the EI protocol the descriptor carries. +# dbus supplies dbus-daemon, and python3-gi supplies the GDBus bindings the +# mock portal needs to put a file descriptor on the bus. +# +# Note what is deliberately NOT installed: gdbus. The Screenshot tier used to +# shell out to it, which could never work — the portal's answer is a signal +# directed at the connection that asked, and two gdbus processes are two +# connections. portal.py speaks D-Bus itself now, so this image is also the +# proof that the tier needs no binary beyond a session bus. +# +# grim / wtype / wlr-randr are not used by anything below, but the Wayland +# backend refuses to import without them and `import je_auto_control` then +# falls back to the X11 backend — which dies reaching for DISPLAY=:0 before a +# line of this verification can run. +# +# libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + libei1 libeis1 liboeffis1 \ + dbus gir1.2-glib-2.0 python3-gi \ + grim wlr-randr wtype \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/eis_server.py /opt/verify/eis_server.py +COPY docker/portal_server.py /opt/verify/portal_server.py +COPY docker/portal_verify.py /opt/verify/portal_verify.py + +# Runs as root on purpose: one verification, then exit. Not a service image. +ENV PYTHONUNBUFFERED=1 \ + PYTHONPATH=/opt/verify \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=wayland + +ENTRYPOINT ["python", "/opt/verify/portal_verify.py"] diff --git a/docker/Dockerfile.seat b/docker/Dockerfile.seat new file mode 100644 index 00000000..9a813f41 --- /dev/null +++ b/docker/Dockerfile.seat @@ -0,0 +1,118 @@ +# AutoControl seat verification image — a wlroots compositor that really +# consumes the ydotool device. +# +# The four images before this one each stop at the same wall. +# docker/Dockerfile.wayland runs a real compositor but sets +# WLR_LIBINPUT_NO_DEVICES=1, so nothing is injected into it; +# docker/Dockerfile.ydotool reads ydotool's events off the kernel with no +# compositor at all. Between them, what `mousemove --absolute` puts on the +# wire is settled and what a compositor does with it was not — and that was +# recorded in Progress.md as needing a VM running a desktop that consumes +# libinput devices. +# +# It does not. A "seat that consumes libinput devices" is three environment +# variables away from the same headless sway the capture image already runs: +# +# WLR_BACKENDS=headless,libinput virtual outputs, real input backend +# LIBSEAT_BACKEND=builtin open the devices directly, no logind +# SEATD_VTBOUND=0 and without reaching for a VT +# +# udev is the fourth requirement and the one that is easy to miss: libinput +# enumerates devices through udev, not through /dev, so udevd has to be +# running before ydotoold creates its uinput device. The entrypoint starts +# them in that order and fails loudly if `libinput list-devices` cannot see +# the result, rather than letting sway come up with an empty seat and every +# measurement below silently become a measurement of nothing. +# +# What it then answers, in layout coordinates read back from pixels the +# compositor painted (`grim -c` composites the cursor): +# * that --absolute counts from the top-left corner of the output layout, +# which is layout (0, 0) only while no monitor sits left of or above the +# primary one — the translation je_auto_control/linux_wayland/mouse.py +# now applies, measured rather than assumed; +# * that the displacement is relative motion and so is scaled by the +# compositor's pointer acceleration, which is what ydotool's own --help +# means by "You need to disable mouse speed acceleration for correct +# absolute movement"; +# * that with that acceleration off, AutoControl's own set_position lands +# on the layout pixel it was given, on both outputs, and that get_pixel +# reads the same pixel back. +# +# Build: docker build -f docker/Dockerfile.seat -t autocontrol-seat:latest . +# Run: modprobe uinput evdev +# docker run --rm --device /dev/uinput \ +# --device-cgroup-rule 'c 13:* rmw' autocontrol-seat:latest + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# ydotool 1.0.4 comes from unstable for the reason docker/Dockerfile.ydotool +# gives: trixie ships none and bookworm's 0.1.8 answers this argv with exit +# code 0 and no events. The pin keeps every other package on trixie. +# +# - sway + wlr-randr + grim: the compositor and the two tools this reads it +# with. grim's -c is what draws the cursor into the capture. +# - udev: systemd-udevd, so libinput can enumerate the device at all. +# - libinput-tools: the entrypoint's own precondition check, before sway. +# - dmz-cursor-theme: a cursor that is certain to be drawn. libwayland has a +# built-in fallback, but a theme on disk makes the pixels this measures the +# compositor's decision rather than a fallback's. +# - wtype: unused here, but the Wayland backend refuses to import without it +# and `import je_auto_control` then falls back to X11 and dies on DISPLAY. +# - libgl1 + libglib2.0-0: opencv-python's hard import-time requirements. +RUN printf 'deb https://deb.debian.org/debian sid main\n' \ + > /etc/apt/sources.list.d/sid.list \ + && printf 'Package: *\nPin: release a=unstable\nPin-Priority: 100\n' \ + > /etc/apt/preferences.d/no-sid-by-default \ + && apt-get update \ + && apt-get install -y --no-install-recommends -t sid ydotool \ + && apt-get install -y --no-install-recommends \ + grim sway wlr-randr wtype \ + dmz-cursor-theme libinput-tools udev \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/seat_verify.py /opt/verify/seat_verify.py +COPY docker/entrypoint-seat.sh /usr/local/bin/autocontrol-seat-verify +RUN chmod +x /usr/local/bin/autocontrol-seat-verify + +# Runs as root on purpose: creating the input node, opening it as a seat +# device and writing to /dev/uinput all need it, and this image exists to run +# one verification and exit. It is not a deployable service image — +# docker/Dockerfile is. +# +# The CLI input path is forced: libei would otherwise be preferred, and this +# image is entirely about the ydotool fallback. +ENV PYTHONUNBUFFERED=1 \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + YDOTOOL_SOCKET=/tmp/.ydotool_socket \ + JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli \ + WLR_BACKENDS=headless,libinput \ + WLR_LIBINPUT_NO_DEVICES=1 \ + WLR_RENDERER=pixman \ + WLR_HEADLESS_OUTPUTS=2 \ + LIBSEAT_BACKEND=builtin \ + SEATD_VTBOUND=0 + +ENTRYPOINT ["/usr/local/bin/autocontrol-seat-verify"] diff --git a/docker/Dockerfile.wayland b/docker/Dockerfile.wayland new file mode 100644 index 00000000..c36524af --- /dev/null +++ b/docker/Dockerfile.wayland @@ -0,0 +1,88 @@ +# AutoControl Wayland verification image — a real wlroots compositor, headless. +# +# The Wayland backend's CLI argv, the wlr-randr output format and the whole +# screenshot -> screen_grabber -> capture -> grab_image chain had only ever +# been exercised against mocks: no CI runner offers a Wayland session, and a +# developer machine running Windows cannot provide one either. sway's headless +# backend needs no GPU, no seat and no display, so it runs in a container and +# answers those questions for real. +# +# The entrypoint runs the verification over two output layouts: the outputs +# side by side from the origin, and the left-hand one moved to x=-1280. sway +# accepts a negative `position` and grim accepts a negative `-g`, so the +# negative-origin layout every multi-monitor desktop has is a real compositor +# answering rather than a second mock. +# +# What this image CANNOT answer, and why: +# * ydotool — needs /dev/uinput, and sway's headless backend consumes no +# libinput devices, so an injected event would have nowhere to arrive. +# Arriving turned out not to be what needed checking: docker/ +# Dockerfile.ydotool reads the events back off the kernel device ydotoold +# creates, with no compositor and no seat involved. +# * libei / the RemoteDesktop portal — xdg-desktop-portal-wlr implements +# ScreenCast and Screenshot but not RemoteDesktop, so there is no +# ConnectToEIS here. That was read as needing a GNOME (mutter) session, +# and it does not: the protocol half is docker/Dockerfile.eis and the +# portal half is docker/Dockerfile.portal, which owns the well-known +# D-Bus name itself rather than waiting for a desktop to provide one. +# +# Build: docker build -f docker/Dockerfile.wayland -t autocontrol-wayland:latest . +# Run: docker run --rm autocontrol-wayland:latest + +# Build every wheel in a throwaway stage, as docker/Dockerfile does, so the +# runtime layer installs binaries only and the dependency set is fixed here +# rather than re-resolved against PyPI at run time. +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# - sway: the wlroots compositor. WLR_BACKENDS=headless gives it a virtual +# output with no DRM device and no seat. +# - grim / wtype / wlr-randr: the three wlroots-protocol helpers the Wayland +# backend shells out to, and the actual subjects of this verification. +# - libgl1 + libglib2.0-0: opencv-python hard-requires libGL.so.1 and +# libgthread-2.0.so.0 at import, so the package cannot even load without them. +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + grim sway wlr-randr wtype \ + libei1 \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/wayland_verify.py /opt/verify/wayland_verify.py +COPY docker/libei_verify.py /opt/verify/libei_verify.py +COPY docker/entrypoint-wayland.sh /usr/local/bin/autocontrol-wayland-verify +RUN chmod +x /usr/local/bin/autocontrol-wayland-verify + +# Runs as root on purpose: this image exists to run one verification and +# exit, and root removes the XDG_RUNTIME_DIR ownership fiddling that would +# otherwise be the only thing standing between here and an answer. It is not +# a deployable service image — docker/Dockerfile is. +ENV PYTHONUNBUFFERED=1 \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + WLR_BACKENDS=headless \ + WLR_LIBINPUT_NO_DEVICES=1 \ + WLR_RENDERER=pixman \ + WLR_HEADLESS_OUTPUTS=2 + +ENTRYPOINT ["/usr/local/bin/autocontrol-wayland-verify"] diff --git a/docker/Dockerfile.ydotool b/docker/Dockerfile.ydotool new file mode 100644 index 00000000..757dc916 --- /dev/null +++ b/docker/Dockerfile.ydotool @@ -0,0 +1,102 @@ +# AutoControl ydotool verification image — real /dev/uinput, no compositor. +# +# The Wayland image (docker/Dockerfile.wayland) answers the capture half and +# the EIS image (docker/Dockerfile.eis) answers libei's. Both close by naming +# the same remaining gap in the same words — ydotool "needs /dev/uinput and a +# seat that consumes it" — and that was recorded as needing a GNOME VM. +# +# It does not. A seat is what makes an injected event *arrive somewhere*; it +# is not what makes it *observable*. ydotoold creates an ordinary uinput +# device, the kernel publishes it as /dev/input/eventN, and reading that node +# returns the exact input_event structs ydotool wrote. So this image runs +# AutoControl's real mouse and keyboard backends and reads back what the +# kernel got: the click bitmasks, the split press / release edges that drag +# depends on, what --absolute actually puts on the wire, the wheel signs this +# project had only ever assumed, and numeric key codes. +# +# Why ydotool comes from unstable: 1.0 replaced the entire command line, and +# every argument this backend builds arrived in it. Trixie ships no ydotool +# at all and bookworm ships 0.1.8, which answers this argv with exit code 0 +# and no events — see je_auto_control/linux_wayland/_ydotool_cli.py. Pulling +# one package from sid, pinned so nothing else follows it, is what gets the +# CLI under test onto the same Debian base as the other two images. +# +# What this image still cannot answer, and why: +# * whether a compositor clamps the INT32_MIN reset that --absolute relies +# on. The kernel side of that is checked here; the clamp is the +# compositor's behaviour, and it remains open in Progress.md rather than +# being quietly claimed. +# The portal consent dialog and the EIS fd handover used to be listed here +# too. They are xdg-desktop-portal's, not ydotool's, and they are now covered +# by docker/Dockerfile.portal against the real liboeffis. +# +# Build: docker build -f docker/Dockerfile.ydotool -t autocontrol-ydotool:latest . +# Run: modprobe uinput evdev +# docker run --rm --device /dev/uinput \ +# --device-cgroup-rule 'c 13:* rmw' autocontrol-ydotool:latest +# +# The cgroup rule is what lets the container open the input node ydotoold +# creates *after* it starts, which --device cannot cover because the node does +# not exist yet at run time. It grants character major 13 (input) and nothing +# else, which is a good deal narrower than --privileged. + +FROM python:3.12-slim AS builder + +WORKDIR /src +COPY pyproject.toml README.md ./ +COPY je_auto_control ./je_auto_control +COPY autocontrol-lsp ./autocontrol-lsp +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip wheel --no-cache-dir --wheel-dir /wheels . + + +FROM python:3.12-slim AS runtime + +ARG DEBIAN_FRONTEND=noninteractive + +# ydotool 1.0.4 comes from unstable; the pin keeps every other package on +# trixie, so this is one binary and its already-satisfied dependencies rather +# than a half-upgraded base. +# +# grim / wtype / wlr-randr are not used by anything below, but the Wayland +# backend refuses to import without them and `import je_auto_control` then +# falls back to the X11 backend — which dies reaching for DISPLAY=:0 before a +# line of this verification can run. +# +# libgl1 and libglib2.0-0 are opencv-python's hard import-time requirements. +RUN printf 'deb https://deb.debian.org/debian sid main\n' \ + > /etc/apt/sources.list.d/sid.list \ + && printf 'Package: *\nPin: release a=unstable\nPin-Priority: 100\n' \ + > /etc/apt/preferences.d/no-sid-by-default \ + && apt-get update \ + && apt-get install -y --no-install-recommends -t sid ydotool \ + && apt-get install -y --no-install-recommends \ + grim wlr-randr wtype \ + libgl1 libglib2.0-0 \ + ca-certificates \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY --from=builder /wheels /wheels +RUN pip install --no-cache-dir --only-binary :all: --upgrade "pip==26.0.1" \ + && pip install --no-cache-dir --only-binary :all: --no-index \ + --find-links=/wheels /wheels/je_auto_control-*.whl \ + && rm -rf /wheels + +COPY docker/ydotool_verify.py /opt/verify/ydotool_verify.py + +# Runs as root on purpose: writing to /dev/uinput and creating the input node +# both need it, and this image exists to run one verification and exit. It is +# not a deployable service image — docker/Dockerfile is. +# +# The Wayland backend is selected by XDG_SESSION_TYPE alone here: there is no +# compositor, and none is needed, because every function under test shells out +# to ydotool rather than talking to a Wayland socket. The CLI path is forced +# because libei would otherwise win and this image is about the fallback. +ENV PYTHONUNBUFFERED=1 \ + XDG_RUNTIME_DIR=/tmp/xdg \ + XDG_SESSION_TYPE=wayland \ + JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND=cli + +ENTRYPOINT ["python3", "/opt/verify/ydotool_verify.py"] diff --git a/docker/eis_server.py b/docker/eis_server.py new file mode 100644 index 00000000..3a7ae7be --- /dev/null +++ b/docker/eis_server.py @@ -0,0 +1,431 @@ +"""A minimal EIS server, bound to the real ``libeis.so`` with ctypes. + +AutoControl's libei sender had been verified as far as a peer-less container +can go: every prototype resolves, every call behaves, and the whole +fail-closed chain runs. What could not be checked without something that +speaks the protocol was the half where a wrong value is silently accepted — +the capability and event-type enums, the variadic +``ei_seat_bind_capabilities`` marshalling, seat capability grants, device +pause/resume, and whether ``start_emulating`` → event → ``frame`` actually +puts anything on the wire. + +``libeis`` is the server side of the same protocol and Debian packages it +(``libeis1`` / ``libeis-dev``), so that peer can just be built here: this +module runs a real EIS implementation on a Unix socket, in a thread, and +records everything a client does to it. No compositor, no desktop session, no +GNOME VM — the two libraries talk to each other and the recording is the +evidence. + +It is a verification fixture, not shipped library code, which is why it lives +under ``docker/`` next to the image that runs it. +""" +from __future__ import annotations + +import ctypes +import select +import threading +from typing import Dict, List, Optional, Sequence, Set, Tuple + +from je_auto_control.linux_wayland._ctypes_bind import BoundSymbols, bind + +_LIBRARY_CANDIDATES = ("eis", "libeis", "libeis.so.1", "libeis.so.0") + +# enum eis_device_capability — the server side of the same bitmask libei.py +# declares. Deliberately re-declared from libeis.h rather than imported from +# libei.py: if both sides read the same wrong constant the test passes while +# the protocol is broken, and this file exists to catch exactly that. +EIS_DEVICE_CAP_POINTER = 1 << 0 +EIS_DEVICE_CAP_POINTER_ABSOLUTE = 1 << 1 +EIS_DEVICE_CAP_KEYBOARD = 1 << 2 +EIS_DEVICE_CAP_TOUCH = 1 << 3 +EIS_DEVICE_CAP_SCROLL = 1 << 4 +EIS_DEVICE_CAP_BUTTON = 1 << 5 + +CAPABILITY_NAMES = { + EIS_DEVICE_CAP_POINTER: "POINTER", + EIS_DEVICE_CAP_POINTER_ABSOLUTE: "POINTER_ABSOLUTE", + EIS_DEVICE_CAP_KEYBOARD: "KEYBOARD", + EIS_DEVICE_CAP_TOUCH: "TOUCH", + EIS_DEVICE_CAP_SCROLL: "SCROLL", + EIS_DEVICE_CAP_BUTTON: "BUTTON", +} + +# enum eis_event_type, from libeis.h. +EIS_EVENT_CLIENT_CONNECT = 1 +EIS_EVENT_CLIENT_DISCONNECT = 2 +EIS_EVENT_SEAT_BIND = 3 +EIS_EVENT_DEVICE_CLOSED = 4 +EIS_EVENT_FRAME = 100 +EIS_EVENT_DEVICE_START_EMULATING = 200 +EIS_EVENT_DEVICE_STOP_EMULATING = 201 +EIS_EVENT_POINTER_MOTION = 300 +EIS_EVENT_POINTER_MOTION_ABSOLUTE = 400 +EIS_EVENT_BUTTON_BUTTON = 500 +EIS_EVENT_SCROLL_DISCRETE = 603 +EIS_EVENT_KEYBOARD_KEY = 700 + +#: What this server offers a seat. Wider than AutoControl asks for on +#: purpose — a client that binds a capability it never requested, or misses +#: one it did, then shows up in ``bound_capabilities``. +OFFERED_CAPABILITIES = ( + EIS_DEVICE_CAP_POINTER, EIS_DEVICE_CAP_POINTER_ABSOLUTE, + EIS_DEVICE_CAP_KEYBOARD, EIS_DEVICE_CAP_TOUCH, + EIS_DEVICE_CAP_SCROLL, EIS_DEVICE_CAP_BUTTON, +) + +#: The absolute pointer needs a region or the compositor has no coordinate +#: space to place motion in. +REGION_SIZE = (1920, 1080) + +#: Default shape of that space: one region at the origin, as ``((x, y), +#: (width, height))``. ``regions=`` overrides it, which is how the offset +#: layout that a monitor left of the primary produces gets exercised. +DEFAULT_REGIONS = (((0, 0), REGION_SIZE),) + +_VOID = ctypes.c_void_p +_U32 = ctypes.c_uint32 +_PROTOTYPES = ( + ("eis_new", _VOID, (_VOID,)), + ("eis_unref", _VOID, (_VOID,)), + ("eis_setup_backend_socket", ctypes.c_int, (_VOID, ctypes.c_char_p)), + ("eis_get_fd", ctypes.c_int, (_VOID,)), + ("eis_dispatch", None, (_VOID,)), + ("eis_get_event", _VOID, (_VOID,)), + ("eis_event_unref", _VOID, (_VOID,)), + ("eis_event_get_type", ctypes.c_int, (_VOID,)), + ("eis_event_get_client", _VOID, (_VOID,)), + ("eis_event_get_seat", _VOID, (_VOID,)), + ("eis_event_get_device", _VOID, (_VOID,)), + ("eis_event_seat_has_capability", ctypes.c_bool, (_VOID, ctypes.c_int)), + ("eis_client_connect", None, (_VOID,)), + ("eis_client_disconnect", None, (_VOID,)), + ("eis_client_is_sender", ctypes.c_bool, (_VOID,)), + ("eis_client_get_name", ctypes.c_char_p, (_VOID,)), + ("eis_client_new_seat", _VOID, (_VOID, ctypes.c_char_p)), + ("eis_seat_configure_capability", None, (_VOID, ctypes.c_int)), + ("eis_seat_add", None, (_VOID,)), + ("eis_seat_new_device", _VOID, (_VOID,)), + ("eis_device_configure_name", None, (_VOID, ctypes.c_char_p)), + ("eis_device_configure_capability", None, (_VOID, ctypes.c_int)), + ("eis_device_new_region", _VOID, (_VOID,)), + ("eis_region_set_size", None, (_VOID, _U32, _U32)), + ("eis_region_set_offset", None, (_VOID, _U32, _U32)), + ("eis_region_add", None, (_VOID,)), + ("eis_device_add", None, (_VOID,)), + ("eis_device_remove", None, (_VOID,)), + ("eis_device_pause", None, (_VOID,)), + ("eis_device_resume", None, (_VOID,)), + ("eis_device_get_name", ctypes.c_char_p, (_VOID,)), + ("eis_event_emulating_get_sequence", _U32, (_VOID,)), + ("eis_event_keyboard_get_key", _U32, (_VOID,)), + ("eis_event_keyboard_get_key_is_press", ctypes.c_bool, (_VOID,)), + ("eis_event_pointer_get_absolute_x", ctypes.c_double, (_VOID,)), + ("eis_event_pointer_get_absolute_y", ctypes.c_double, (_VOID,)), + ("eis_event_button_get_button", _U32, (_VOID,)), + ("eis_event_button_get_is_press", ctypes.c_bool, (_VOID,)), + ("eis_event_scroll_get_discrete_dx", ctypes.c_int32, (_VOID,)), + ("eis_event_scroll_get_discrete_dy", ctypes.c_int32, (_VOID,)), +) + + +class EisUnavailable(RuntimeError): + """libeis is missing, or a server cannot be brought up here.""" + + +def load_symbols() -> Optional[BoundSymbols]: + """Resolve every libeis entry point, or None if one is missing.""" + return bind(_LIBRARY_CANDIDATES, _PROTOTYPES) + + +class Recording: + """Everything one client did, as the server saw it happen.""" + + def __init__(self) -> None: + self.clients: List[str] = [] + self.sender_flags: List[bool] = [] + self.bound_capabilities: Set[int] = set() + self.seat_binds = 0 + self.devices: List[str] = [] + self.emulating_sequences: List[Tuple[str, int]] = [] + self.stopped_emulating = 0 + self.frames = 0 + self.keys: List[Tuple[int, bool]] = [] + self.absolute_motions: List[Tuple[float, float]] = [] + self.buttons: List[Tuple[int, bool]] = [] + self.scrolls: List[Tuple[int, int]] = [] + self.disconnects = 0 + #: Device labels this server asked to pause, for diagnosis. + self.paused: List[str] = [] + #: ``(event type, device name)`` for everything, in arrival order — + #: what a failing check needs to explain itself. + self.event_log: List[Tuple[int, str]] = [] + + def capability_names(self) -> List[str]: + return sorted(CAPABILITY_NAMES.get(cap, str(cap)) + for cap in self.bound_capabilities) + + +class RecordingEisServer: + """A real EIS server that grants a seat and records what arrives. + + Runs its whole libeis context on one background thread: libeis is not + thread-safe, and the client under test drives the main thread, so the two + contexts never share one. + """ + + def __init__(self, socket_path: str, + *, symbols: Optional[BoundSymbols] = None, + regions: Optional[Sequence[Tuple[Tuple[int, int], + Tuple[int, int]]]] = None, + ) -> None: + self.socket_path = socket_path + self.recording = Recording() + self.regions = tuple(DEFAULT_REGIONS if regions is None else regions) + self._symbols = symbols if symbols is not None else load_symbols() + self._eis: Optional[int] = None + self._devices: Dict[str, int] = {} + self._seat_devices_created = False + self._stop = threading.Event() + self._thread: Optional[threading.Thread] = None + self._error: Optional[BaseException] = None + #: Work handed to the serving thread; a list because append and pop + #: are atomic and this needs no more locking than that. + self._pending: List[tuple] = [] + + # --- lifecycle -------------------------------------------------------- + + def start(self) -> None: + """Bring the server up and begin answering clients.""" + if self._symbols is None: + raise EisUnavailable("libeis.so.* not found on the loader path") + context = self._symbols.eis_new(None) + if not context: + raise EisUnavailable("eis_new returned NULL") + self._eis = context + code = self._symbols.eis_setup_backend_socket( + context, self.socket_path.encode("utf-8")) + if code != 0: + raise EisUnavailable(f"eis_setup_backend_socket returned {code}") + self._thread = threading.Thread(target=self._serve, daemon=True, + name="eis-server") + self._thread.start() + + def stop(self, timeout: float = 2.0) -> None: + """Ask the loop to finish and wait for it.""" + self._stop.set() + if self._thread is not None: + self._thread.join(timeout) + self._thread = None + + @property + def error(self) -> Optional[BaseException]: + """Whatever took the serving thread down, if anything did.""" + return self._error + + def pause_devices(self) -> None: + """Suspend every device, so a client must stop emulating.""" + self._on_server_thread(self._pause_all) + + def _pause_all(self) -> None: + for label, device in self._devices.items(): + name = self._symbols.eis_device_get_name(device) + self.recording.paused.append( + f"{label}={name.decode() if name else ''}") + self._symbols.eis_device_pause(device) + # libeis buffers outgoing messages; a dispatch is what puts them on + # the wire, and nothing else here would trigger one while the client + # is only *sending*. + self._symbols.eis_dispatch(self._eis) + + def resume_devices(self) -> None: + """Resume every device, so a client may emulate again.""" + self._on_server_thread( + lambda: [self._symbols.eis_device_resume(device) + for device in self._devices.values()]) + + def _on_server_thread(self, action, timeout: float = 2.0) -> None: + """Run ``action`` on the serving thread and wait for it. + + libeis is not thread-safe, and every other call into this context + already happens on that thread. Pausing a device from the caller's + thread appeared to work and then simply never reached the client — + which reads exactly like the client ignoring the event, so it is + worth not being able to make that mistake. + """ + done = threading.Event() + self._pending.append((action, done)) + if not done.wait(timeout): + raise EisUnavailable("the server thread did not run a queued action") + + # --- event loop ------------------------------------------------------- + + def _serve(self) -> None: + try: + poll_fd = int(self._symbols.eis_get_fd(self._eis)) + while not self._stop.is_set(): + self._run_pending() + ready, _, _ = select.select([poll_fd], [], [], 0.05) + if not ready: + continue + self._symbols.eis_dispatch(self._eis) + self._drain() + except Exception as error: # noqa: BLE001 # reason: a fixture thread must report, not vanish + self._error = error + + def _run_pending(self) -> None: + while self._pending: + action, done = self._pending.pop(0) + try: + action() + finally: + done.set() + + def _drain(self) -> None: + while True: + event = self._symbols.eis_get_event(self._eis) + if not event: + return + try: + self._on_event(event) + finally: + self._symbols.eis_event_unref(event) + + def _on_event(self, event: int) -> None: + event_type = int(self._symbols.eis_event_get_type(event)) + self.recording.event_log.append((event_type, self._device_name(event))) + handler = _HANDLERS.get(event_type) + if handler is not None: + handler(self, event) + + def _device_name(self, event: int) -> str: + """The device an event belongs to, for the log. ``''`` when it has none.""" + device = self._symbols.eis_event_get_device(event) + if not device: + return "" + name = self._symbols.eis_device_get_name(device) + return name.decode("utf-8", "replace") if name else f"<{device:#x}>" + + # --- handlers --------------------------------------------------------- + + def _on_client_connect(self, event: int) -> None: + """Accept a sender client and offer it one fully-capable seat.""" + client = self._symbols.eis_event_get_client(event) + name = self._symbols.eis_client_get_name(client) + self.recording.clients.append( + name.decode("utf-8", "replace") if name else "") + self.recording.sender_flags.append( + bool(self._symbols.eis_client_is_sender(client))) + self._symbols.eis_client_connect(client) + seat = self._symbols.eis_client_new_seat(client, b"autocontrol-seat") + for capability in OFFERED_CAPABILITIES: + self._symbols.eis_seat_configure_capability(seat, capability) + self._symbols.eis_seat_add(seat) + + def _on_seat_bind(self, event: int) -> None: + """Record what the client asked this seat for, then hand it devices. + + This is the whole point of the fixture. ``eis_event_seat_has_capability`` + reads back what the client's variadic ``ei_seat_bind_capabilities`` + actually put on the wire, so a wrong enum value or a mis-marshalled + argument shows up as a missing or unexpected capability rather than as + a handshake that quietly never completes. + """ + self.recording.seat_binds += 1 + seat = self._symbols.eis_event_get_seat(event) + bound = {cap for cap in OFFERED_CAPABILITIES + if self._symbols.eis_event_seat_has_capability(event, cap)} + self.recording.bound_capabilities |= bound + if not bound or self._seat_devices_created: + return + self._seat_devices_created = True + if EIS_DEVICE_CAP_KEYBOARD in bound: + self._add_device("keyboard", seat, (EIS_DEVICE_CAP_KEYBOARD,)) + pointer_caps = tuple( + cap for cap in (EIS_DEVICE_CAP_POINTER_ABSOLUTE, + EIS_DEVICE_CAP_BUTTON, EIS_DEVICE_CAP_SCROLL) + if cap in bound) + if pointer_caps: + self._add_device("pointer", seat, pointer_caps, region=True) + + def _add_device(self, label: str, seat: int, capabilities: tuple, + *, region: bool = False) -> None: + device = self._symbols.eis_seat_new_device(seat) + self._symbols.eis_device_configure_name( + device, f"autocontrol-{label}".encode("utf-8")) + for capability in capabilities: + self._symbols.eis_device_configure_capability(device, capability) + if region: + for offset, size in self.regions: + shape = self._symbols.eis_device_new_region(device) + self._symbols.eis_region_set_offset(shape, *offset) + self._symbols.eis_region_set_size(shape, *size) + self._symbols.eis_region_add(shape) + self._symbols.eis_device_add(device) + self._symbols.eis_device_resume(device) + self._devices[label] = device + self.recording.devices.append(label) + + def _on_start_emulating(self, event: int) -> None: + device = self._symbols.eis_event_get_device(event) + name = self._symbols.eis_device_get_name(device) if device else None + self.recording.emulating_sequences.append(( + name.decode("utf-8", "replace") if name else "", + int(self._symbols.eis_event_emulating_get_sequence(event)), + )) + + def _on_stop_emulating(self, _event: int) -> None: + self.recording.stopped_emulating += 1 + + def _on_frame(self, _event: int) -> None: + self.recording.frames += 1 + + def _on_keyboard_key(self, event: int) -> None: + self.recording.keys.append(( + int(self._symbols.eis_event_keyboard_get_key(event)), + bool(self._symbols.eis_event_keyboard_get_key_is_press(event)), + )) + + def _on_absolute_motion(self, event: int) -> None: + self.recording.absolute_motions.append(( + float(self._symbols.eis_event_pointer_get_absolute_x(event)), + float(self._symbols.eis_event_pointer_get_absolute_y(event)), + )) + + def _on_button(self, event: int) -> None: + self.recording.buttons.append(( + int(self._symbols.eis_event_button_get_button(event)), + bool(self._symbols.eis_event_button_get_is_press(event)), + )) + + def _on_scroll_discrete(self, event: int) -> None: + self.recording.scrolls.append(( + int(self._symbols.eis_event_scroll_get_discrete_dx(event)), + int(self._symbols.eis_event_scroll_get_discrete_dy(event)), + )) + + def _on_client_disconnect(self, _event: int) -> None: + self.recording.disconnects += 1 + + +_HANDLERS = { + EIS_EVENT_CLIENT_CONNECT: RecordingEisServer._on_client_connect, + EIS_EVENT_CLIENT_DISCONNECT: RecordingEisServer._on_client_disconnect, + EIS_EVENT_SEAT_BIND: RecordingEisServer._on_seat_bind, + EIS_EVENT_DEVICE_START_EMULATING: RecordingEisServer._on_start_emulating, + EIS_EVENT_DEVICE_STOP_EMULATING: RecordingEisServer._on_stop_emulating, + EIS_EVENT_FRAME: RecordingEisServer._on_frame, + EIS_EVENT_KEYBOARD_KEY: RecordingEisServer._on_keyboard_key, + EIS_EVENT_POINTER_MOTION_ABSOLUTE: RecordingEisServer._on_absolute_motion, + EIS_EVENT_BUTTON_BUTTON: RecordingEisServer._on_button, + EIS_EVENT_SCROLL_DISCRETE: RecordingEisServer._on_scroll_discrete, +} + + +__all__ = [ + "CAPABILITY_NAMES", "EisUnavailable", "OFFERED_CAPABILITIES", + "DEFAULT_REGIONS", "REGION_SIZE", + "Recording", "RecordingEisServer", "load_symbols", + "EIS_DEVICE_CAP_POINTER", "EIS_DEVICE_CAP_POINTER_ABSOLUTE", + "EIS_DEVICE_CAP_KEYBOARD", "EIS_DEVICE_CAP_TOUCH", + "EIS_DEVICE_CAP_SCROLL", "EIS_DEVICE_CAP_BUTTON", +] diff --git a/docker/eis_verify.py b/docker/eis_verify.py new file mode 100644 index 00000000..acd3edb6 --- /dev/null +++ b/docker/eis_verify.py @@ -0,0 +1,663 @@ +"""Verify AutoControl's libei sender against a real EIS peer. + +``libei_verify.py`` goes as far as a peer-less container can: every prototype +resolves, every call behaves, and the fail-closed chain runs end to end. It +closes by listing what it cannot answer — the capability and event-type enum +values, and whether the variadic ``ei_seat_bind_capabilities`` call is +marshalled correctly — because those need something that speaks the protocol. + +``docker/eis_server.py`` is that something. With a real EIS implementation on +the other end of the socket, every one of those questions has an answer, and +they are answered here: + + * the seat records exactly the four capabilities AutoControl asks for, so + both the ``EI_DEVICE_CAP_*`` bitmask values and the variadic call are + right — a wrong value binds the wrong capability or none at all; + * ``SEAT_ADDED`` / ``DEVICE_ADDED`` / ``DEVICE_RESUMED`` are recognised, so + the ``EI_EVENT_*`` values are right — a wrong one leaves the handshake + waiting until it times out; + * ``start_emulating`` → event → ``frame`` puts real events on the wire, with + the key codes, coordinates, button codes and scroll signs intended — + including the axis flip ``mouse.scroll()`` applies between the kernel's + ``REL_WHEEL`` frame and libei's, and the negative value that flip makes; + * tearing down a *live* context is finally testable — it is safe, which is + what lets ``_teardown`` release a completed session instead of leaking a + context and an fd per process, and this file is the sentinel for that. + +Two things it measures but cannot settle, and says so rather than claiming +either way: ``eis_device_pause()`` puts nothing on the wire for a sender +client on libeis 1.3.901, so ``DEVICE_PAUSED`` handling still has no peer to +drive it; and the ``start_emulating`` sequence number does not survive the +trip, so the client's counter cannot be read back from this side. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import faulthandler +import os +import subprocess # nosec B404 # reason: runs this interpreter to isolate a known segfault +import sys +import tempfile +import time +import traceback +from typing import Any, Callable, List, Tuple + +# A wrong prototype in a ctypes binding is a segfault, not an exception. +faulthandler.enable() + +_results: List[Tuple[str, bool]] = [] + +#: evdev codes the emission checks use. KEY_A, BTN_LEFT. +KEY_A = 30 +BTN_LEFT = 272 +TARGET_POSITION = (640, 400) + + +def _scratch_dir() -> str: + """A private directory to write into, created 0700 if it is not there. + + The images set ``XDG_RUNTIME_DIR``, so that is what this returns inside + one. Run this script by hand without it and the answer is a fresh + ``mkdtemp`` rather than the ``/tmp`` root itself, which any user on the + host can create entries in. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-verify-") + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _wait_for(predicate: Callable[[], bool], timeout: float, + description: str) -> None: + """Spin until the server thread has recorded something, or give up.""" + import time + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.02) + raise AssertionError(f"timed out waiting for {description}") + + +def _socket_path() -> str: + runtime = _scratch_dir() + os.makedirs(runtime, exist_ok=True) + path = os.path.join(runtime, "eis-verify") + if os.path.exists(path): + os.unlink(path) + return path + + +def _connected_backend(libei, server, path): + """Drive a real handshake to completion against the recording server.""" + backend = libei.LibeiBackend() + backend.connect(timeout=5.0, socket_path=path.encode("utf-8")) + _require(backend.is_connected, + "connect() returned but the backend reports no live device") + _require(server.error is None, f"the server thread died: {server.error!r}") + return backend + + +def _check_handshake(server) -> str: + record = server.recording + _require(record.seat_binds >= 1, "the client never bound a seat") + _require(all(record.sender_flags), + "the client did not present itself as a sender") + return (f"client={record.clients[0]!r} seat binds={record.seat_binds} " + f"devices={record.devices}") + + +def _check_capabilities(server, libei) -> str: + """The enum values and the variadic bind call, read back off the wire.""" + from eis_server import CAPABILITY_NAMES + expected = set(libei._WANTED_CAPS) + bound = server.recording.bound_capabilities + _require(bound == expected, ( + "the seat was bound with " + f"{sorted(CAPABILITY_NAMES.get(c, c) for c in bound)} but AutoControl " + f"asks for {sorted(CAPABILITY_NAMES.get(c, c) for c in expected)} — " + "either an EI_DEVICE_CAP_* value is wrong or the variadic " + "ei_seat_bind_capabilities call is mis-marshalled")) + return ", ".join(server.recording.capability_names()) + + +def _check_keyboard(backend, server) -> str: + backend.press_key(KEY_A) + backend.release_key(KEY_A) + _wait_for(lambda: len(server.recording.keys) >= 2, 2.0, "two key events") + _require(server.recording.keys[:2] == [(KEY_A, True), (KEY_A, False)], + f"the server saw {server.recording.keys[:2]}") + return f"press+release of key {KEY_A} arrived in order" + + +def _check_pointer(backend, server) -> str: + backend.set_position(*TARGET_POSITION) + _wait_for(lambda: server.recording.absolute_motions, 2.0, "absolute motion") + got = server.recording.absolute_motions[-1] + _require(got == TARGET_POSITION, + f"asked for {TARGET_POSITION}, the server saw {got}") + return f"absolute motion landed on {got}" + + +def _check_button(backend, server) -> str: + backend.click_button(BTN_LEFT) + _wait_for(lambda: len(server.recording.buttons) >= 2, 2.0, "button events") + _require(server.recording.buttons[:2] == [(BTN_LEFT, True), + (BTN_LEFT, False)], + f"the server saw {server.recording.buttons[:2]}") + return f"press+release of button {BTN_LEFT} arrived in order" + + +#: libei.h: "A discrete scroll event is based logical scroll units (equivalent +#: to one mouse wheel click). The value for one scroll unit is 120." +SCROLL_UNIT = 120 + + +def _check_scroll_unit(backend, server) -> str: + """One wheel click has to arrive as one wheel click. + + This is the path Progress.md left deliberately unwired because the sign + was a guess. The sign turns out to be the smaller half of the question: + libei measures discrete scroll in 120ths of a click, so a raw detent count + is 1/120th of a scroll and libei logs it as a client bug. + """ + backend.scroll(0, 1) + _wait_for(lambda: server.recording.scrolls, 2.0, "a discrete scroll") + got = server.recording.scrolls[-1] + _require(got == (0, SCROLL_UNIT), ( + f"scroll(0, 1) — one wheel click down — reached the server as {got}, " + f"but libei measures discrete scroll in units of {SCROLL_UNIT} per " + "click, so this is 1/120th of the scroll that was asked for")) + return (f"scroll(0, 1) arrives as {got}: one click, positive on the y " + "axis, no axis swap") + + +def _check_scroll_wiring(backend, server) -> str: + """The public ``mouse.scroll()`` has to land the axis flip on the wire. + + ``_check_scroll_unit`` drives ``LibeiBackend.scroll`` directly, so it says + nothing about the frame conversion the mouse module does on top: this + repository's ``wayland_scroll_direction_*`` constants are in the kernel's + ``REL_WHEEL`` frame (positive is up, which is what ydotool writes) and + libei is in the ``wl_pointer`` frame (positive is down), so the vertical + axis is negated on the way out and the horizontal one is not. + + Running it through the real server is also the only place a *negative* + discrete value is put on the wire — the direct check only ever sent a + positive one, so a marshalling fault on the sign would have gone unseen. + """ + from unittest.mock import patch + + from je_auto_control.linux_wayland import mouse as wayland_mouse + + def _next_scroll(call) -> tuple: + before = len(server.recording.scrolls) + call() + _wait_for(lambda: len(server.recording.scrolls) > before, 2.0, + "a discrete scroll from mouse.scroll()") + return server.recording.scrolls[-1] + + with patch.object(wayland_mouse, "_try_libei", return_value=backend): + up = _next_scroll(lambda: wayland_mouse.scroll( + 1, wayland_mouse.wayland_scroll_direction_up)) + down = _next_scroll(lambda: wayland_mouse.scroll( + 1, wayland_mouse.wayland_scroll_direction_down)) + right = _next_scroll(lambda: wayland_mouse.scroll( + 1, wayland_mouse.wayland_scroll_direction_right)) + + _require(up == (0, -SCROLL_UNIT), ( + f"scrolling up reached the server as {up}, not (0, {-SCROLL_UNIT}) — " + "libei counts positive y as down, so up has to arrive negative")) + _require(down == (0, SCROLL_UNIT), + f"scrolling down reached the server as {down}") + _require(right == (SCROLL_UNIT, 0), ( + f"scrolling right reached the server as {right} — the horizontal " + "axis must not be flipped, both frames count right as positive")) + return (f"up arrives as {up}, down as {down}, right as {right}: the " + "vertical flip survives, including the negative value") + + +def _check_frames(server) -> str: + """No frame means nothing was delivered, however many events were sent.""" + frames = server.recording.frames + _require(frames >= 4, f"only {frames} frames for 9 emissions") + return f"{frames} frames — every emission was committed" + + +def _check_emulating(server) -> str: + """Every device the client kept has to open an emulation transaction. + + libei is explicit that "sending events before ei_device_start_emulating() + ... is a client bug", so a device that receives events without one is a + protocol violation this peer happens to tolerate and a stricter + compositor need not. + """ + offered = {f"autocontrol-{label}" for label in server.recording.devices} + + def _all_started() -> bool: + return {name for name, _seq in server.recording.emulating_sequences} \ + >= offered + + # The handshake returns as soon as the *required* devices are live, so the + # last device's transaction can still be in flight when this runs. + try: + _wait_for(_all_started, 2.0, "every device to start emulating") + except AssertionError: + pass + started = {name for name, _sequence in server.recording.emulating_sequences} + _require(started == offered, ( + f"devices {sorted(offered)} were handed over but only " + f"{sorted(started)} started emulating.\n server saw: " + f"{_format_log(server.recording.event_log)}")) + return f"{sorted(started)}, sequences " \ + f"{[seq for _n, seq in server.recording.emulating_sequences]}" + + +_EVENT_NAMES = { + 1: "CLIENT_CONNECT", 2: "CLIENT_DISCONNECT", 3: "SEAT_BIND", + 4: "DEVICE_CLOSED", 100: "FRAME", 200: "START_EMULATING", + 201: "STOP_EMULATING", 300: "POINTER_MOTION", 400: "MOTION_ABSOLUTE", + 500: "BUTTON", 603: "SCROLL_DISCRETE", 700: "KEYBOARD_KEY", +} + + +def _format_log(entries) -> str: + return ", ".join( + f"{_EVENT_NAMES.get(kind, kind)}{'@' + name if name else ''}" + for kind, name in entries) + + +def _check_sequence_reaches_the_server(backend, server, libei) -> str: + """Does the start/stop sequence number survive the trip at all? + + libei documents it as identifying one start→stop transaction and requires + it to rise by at least one per call. AutoControl counts correctly, but the + server reads 0 for every transaction — so either the number is not put on + the wire by this libei, or it is not read back by this libeis. Sending an + unmistakable value settles which end to blame. + """ + symbols = libei._load_symbols() + device = backend._devices[libei.EI_DEVICE_CAP_KEYBOARD] + symbols.ei_device_stop_emulating(device) + symbols.ei_device_start_emulating(device, 4242) + _wait_for(lambda: any(seq == 4242 for _n, seq + in server.recording.emulating_sequences) + or server.recording.stopped_emulating > 0, 2.0, + "the restarted emulation transaction") + seen = [seq for _n, seq in server.recording.emulating_sequences] + if 4242 in seen: + return "an explicit 4242 arrives intact, so the number is carried" + return (f"an explicit 4242 arrives as {seen[-1]} — libei {_libei_version()} " + "does not carry the sequence number, so AutoControl's counter " + "cannot be checked from this side (it satisfies the contract " + "regardless: it rises by one per call)") + + +def _libei_version() -> str: + import ctypes.util + return str(ctypes.util.find_library("ei")) + + +def _check_pause_fails_closed(backend, server) -> str: + """A paused device must take its capability out of service, not misfire. + + ``is_connected`` is not polled here on purpose: it only reads cached + state, and the client learns about a pause when it next pumps — which is + inside ``_emit``. Asking for a keystroke is therefore the only honest way + to ask whether the pause was noticed. + """ + import select + import time + symbols = libei_module()._load_symbols() + seen: List[int] = [] + original = backend._on_event + + def spy(event: int) -> None: + seen.append(int(symbols.ei_event_get_type(event))) + original(event) + + backend._on_event = spy + try: + server.pause_devices() + deadline = time.monotonic() + 3.0 + attempts = 0 + while time.monotonic() < deadline: + attempts += 1 + try: + backend.press_key(KEY_A) + except Exception as error: # noqa: BLE001 # reason: the type is the finding + _require(type(error).__name__ == "LibeiUnavailable", + f"a paused device raised {error!r}, not a fail-closed") + return (f"the keystroke {attempts} calls after the pause " + f"failed closed: {str(error)[:60]}") + time.sleep(0.05) + # Nothing arrived. Before blaming the client, ask whether anything was + # sent at all: an idle fd means the pause never left the server. + poll_fd = int(symbols.ei_get_fd(backend._ei)) + readable = bool(select.select([poll_fd], [], [], 1.0)[0]) + finally: + backend._on_event = original + _require(not readable and not seen, ( + f"the client was told about the pause (events {seen}, fd " + f"{'readable' if readable else 'idle'}) and still emitted {attempts} " + "keystrokes, so DEVICE_PAUSED is received and ignored")) + return ("UNTESTED here: libeis 1.3.901 put nothing on the wire for " + f"eis_device_pause on {len(server.recording.paused)} live devices " + "(client fd stayed idle for 4s), so the client's DEVICE_PAUSED " + "handling still has no peer to exercise it — see Progress.md") + + +def libei_module(): + from je_auto_control.linux_wayland import libei + return libei + + +def _check_resume_recovers(backend, server) -> str: + """And a device must still be usable after a pause/resume round trip.""" + before = len(server.recording.keys) + server.resume_devices() + _wait_for(lambda: server.recording.frames >= 0, 0.3, "the resume to land") + backend.press_key(KEY_A) + backend.release_key(KEY_A) + _wait_for(lambda: len(server.recording.keys) >= before + 2, 2.0, + "keys after the resume") + return "the device came back and emitted again" + + +def _live_teardown_sentinel(path: str) -> str: + """Is ``ei_unref`` safe once the handshake has actually completed? + + ``libei_verify.py`` established that it segfaults on a context whose + backend opened but never handshook. Whether a *live* context is safe was + untestable without a peer; it is testable now, and it is safe — which is + what lets ``_teardown`` release a completed session rather than leaking + its context. This is therefore a regression sentinel for that decision, + and it runs in a subprocess because the answer is a signal, not an + exception. + """ + program = ( + "import sys; sys.path.insert(0, '/opt/verify');" + "from eis_server import RecordingEisServer;" + "from je_auto_control.linux_wayland import libei;" + f"srv = RecordingEisServer({path + '-teardown'!r}); srv.start();" + "b = libei.LibeiBackend();" + f"b.connect(timeout=5.0, socket_path={(path + '-teardown').encode()!r});" + "assert b.is_connected;" + "sym = libei._load_symbols();" + "sym.ei_unref(b._ei);" + "print('survived')" + ) + # This interpreter, running a program built from literals above; no shell. + finished = subprocess.run([sys.executable, "-c", program], # nosec B603 # nosemgrep + capture_output=True, timeout=60) + if finished.returncode == 0: + return ("safe, which is what _teardown now relies on to release a " + "completed session instead of leaking its context") + if finished.returncode == -11: + print() + print(" *** REVISIT *** ei_unref now SEGFAULTS on a live") + print(" context too. LibeiBackend._teardown releases completed") + print(" sessions on the measurement that it is safe, so that") + print(" must go back to abandoning them — this is a crash in a") + print(" library that drives the user's desktop. See Progress.md.") + print() + raise AssertionError( + "ei_unref segfaults on a live context (rc=-11); _teardown's " + "release path is no longer safe on this libei") + detail = finished.stderr.decode("utf-8", "replace").strip().splitlines() + return f"inconclusive (rc={finished.returncode}): {detail[-1] if detail else ''}" + + +#: An offset region layout: the shape a compositor must advertise for a +#: desktop whose left-most monitor sits at a negative layout coordinate. +#: Region offsets are ``uint32``, so it cannot advertise the negative +#: coordinate itself — which is the whole reason the two spaces can differ. +OFFSET_REGIONS = (((0, 0), (1280, 1024)), ((1280, 0), (1920, 1080))) + + +def _offset_region_session(path, libei, server_class): + """Bring up a peer whose pointer regions are the OFFSET_REGIONS layout.""" + if os.path.exists(path): + os.unlink(path) + server = server_class(path, regions=OFFSET_REGIONS) + server.start() + backend = libei.LibeiBackend() + backend.connect(timeout=5.0, socket_path=path.encode("utf-8")) + _require(backend.is_connected, "the offset-region handshake never landed") + return backend, server + + +def _pointer_device(backend, libei): + device = backend._devices.get(libei.EI_DEVICE_CAP_POINTER_ABSOLUTE) + _require(bool(device), "no absolute pointer device was granted") + return device + + +def _check_regions_are_read_back(backend, libei) -> str: + """The client sees the regions the server configured, offsets included.""" + got = backend._device_regions(_pointer_device(backend, libei)) + expected = [(x, y, w, h) for (x, y), (w, h) in OFFSET_REGIONS] + _require(got == expected, + f"the server configured {expected}, the client reads {got}") + return f"{got} — offsets survive the trip, so they are part of the space" + + +def _check_offset_region_takes_absolute_coordinates(backend, server, + _libei) -> str: + """A region at x=1280 takes 1380 for a point 100 pixels into it.""" + before = len(server.recording.absolute_motions) + backend.set_position(1380, 100) + _wait_for(lambda: len(server.recording.absolute_motions) > before, 2.0, + "absolute motion into the offset region") + got = server.recording.absolute_motions[-1] + _require(got == (1380.0, 100.0), + f"asked for (1380, 100) inside a region at x=1280, saw {got}") + return "region-space coordinates carry the offset, they are not local" + + +def _check_libei_drops_out_of_region_motion(backend, server, libei) -> str: + """The measurement the whole guard rests on. + + Called under the guard rather than through it: ``set_position`` now + refuses this point, so the raw entry point is used to find out what libei + does when nothing stops it. If some future libei clamps instead of + dropping, this is the check that says so. + """ + device = _pointer_device(backend, libei) + symbols = backend._symbols + before = list(server.recording.absolute_motions) + symbols.ei_device_pointer_motion_absolute(device, 9000.0, 9000.0) + symbols.ei_device_frame(device, symbols.ei_now(backend._ei)) + time.sleep(0.5) + after = server.recording.absolute_motions + _require(after == before, ( + "libei no longer drops an absolute motion outside every region — it " + f"delivered {after[len(before):]}. Re-read LibeiBackend._region_point: " + "its refusal exists because this was silent")) + return ("(9000, 9000) reached the server as nothing at all — no event, " + "no error, no return code: the silent no-op _region_point " + "replaces with a refusal") + + +def _check_out_of_region_move_is_refused(backend, server, libei) -> str: + """AutoControl turns that silence into something the CLI path can act on.""" + before = len(server.recording.absolute_motions) + try: + backend.set_position(9000, 9000) + except libei.LibeiUnavailable as error: + _require("outside every region" in str(error), + f"refused, but with an unhelpful message: {error}") + time.sleep(0.3) + _require(len(server.recording.absolute_motions) == before, + "the refusal still put a motion on the wire") + return f"refused with {str(error)[:60]}... — _select_input.emitted "\ + "hands it to ydotool" + raise AssertionError( + "set_position(9000, 9000) returned as though the pointer had moved; " + "libei dropped it and nobody was told") + + +def _check_negative_origin_is_normalised(backend, server, libei, + monkey) -> str: + """The layout half of the same problem, end to end against the peer. + + A monitor left of the primary puts this project's layout origin at + ``(-1280, 0)`` while the compositor's regions still start at 0. Asking + for the top-left pixel of that monitor means asking for ``(-1280, 0)``, + which no region covers — so it has to arrive as ``(0, 0)``. + """ + before = len(server.recording.absolute_motions) + # The name bound inside ``libei`` by its ``from _layout import + # layout_origin``, which is what ``_region_point`` actually calls. + with monkey(libei, "layout_origin", lambda: (-1280, 0)): + backend.set_position(-1280, 10) + _wait_for(lambda: len(server.recording.absolute_motions) > before, 2.0, + "the normalised motion") + got = server.recording.absolute_motions[-1] + _require(got == (0.0, 10.0), + f"(-1280, 10) on a layout starting at -1280 arrived as {got}, " + "not the (0, 10) the region space calls that pixel") + return "(-1280, 10) arrives as (0, 10): input and capture name one pixel" + + +class _swap: + """Minimal context-managed attribute swap; no pytest in this image.""" + + def __init__(self, target, name, value): + self._target, self._name, self._value = target, name, value + + def __enter__(self): + self._old = getattr(self._target, self._name) + setattr(self._target, self._name, self._value) + return self + + def __exit__(self, *_exc): + setattr(self._target, self._name, self._old) + return False + + +def _run_region_checks(path, libei, server_class) -> None: + """Every region check, on a peer of their own. + + They need a differently-shaped device than the rest of the file, and a + device's regions are fixed when the compositor adds it. + """ + backend, server = _offset_region_session(path, libei, server_class) + try: + check("the client reads back the regions the compositor advertised", + lambda: _check_regions_are_read_back(backend, libei)) + check("an offset region takes absolute coordinates, not local ones", + lambda: _check_offset_region_takes_absolute_coordinates( + backend, server, libei)) + check("libei still drops out-of-region motion without a word", + lambda: _check_libei_drops_out_of_region_motion( + backend, server, libei)) + check("an out-of-region move is refused rather than silently lost", + lambda: _check_out_of_region_move_is_refused( + backend, server, libei)) + check("a negative layout origin is normalised into region space", + lambda: _check_negative_origin_is_normalised( + backend, server, libei, _swap)) + finally: + backend.disconnect() + server.stop() + + +def main() -> int: + print("=" * 72) + print("AutoControl libei sender — against a real EIS server (libeis)") + print("=" * 72) + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from eis_server import RecordingEisServer, load_symbols + from je_auto_control.linux_wayland import libei + + if load_symbols() is None: + print("libeis.so.* not found — install libeis1. Nothing to verify.") + return 1 + + path = _socket_path() + server = RecordingEisServer(path) + server.start() + print(f" EIS server listening at {path}") + print("-" * 72) + + backend = check("the handshake completes against a real EIS peer", + lambda: _connected_backend(libei, server, path)) + if backend is None: + server.stop() + print("the handshake never completed; the rest cannot run") + return 1 + sum(1 for _n, ok in _results if not ok) + + check("the client connects, binds a seat and is handed devices", + lambda: _check_handshake(server)) + check("the seat binds exactly the capabilities AutoControl asks for", + lambda: _check_capabilities(server, libei)) + check("a device starts emulating with a rising sequence number", + lambda: _check_emulating(server)) + check("press_key / release_key arrive as the right evdev code", + lambda: _check_keyboard(backend, server)) + check("set_position arrives as absolute motion at the right point", + lambda: _check_pointer(backend, server)) + check("press_button / release_button arrive as the right BTN_ code", + lambda: _check_button(backend, server)) + check("scroll() arrives as whole wheel clicks, on the right axis and sign", + lambda: _check_scroll_unit(backend, server)) + check("mouse.scroll()'s kernel-to-libei axis flip reaches the server", + lambda: _check_scroll_wiring(backend, server)) + check("every emission was committed with a frame", + lambda: _check_frames(server)) + check("the emulation sequence number, end to end", + lambda: _check_sequence_reaches_the_server(backend, server, libei)) + check("a paused device is either acted on, or never announced", + lambda: _check_pause_fails_closed(backend, server)) + check("a resumed device can emit again", + lambda: _check_resume_recovers(backend, server)) + + check("disconnect() after a live session does not crash the process", + lambda: (backend.disconnect(), backend.disconnect(), + "torn down twice")[-1]) + check("ei_unref on a live context is still safe — teardown depends on it", + lambda: _live_teardown_sentinel(path)) + + # The absolute pointer's coordinate space, on a peer whose regions are + # shaped like the desktop that made it a question: two monitors, the + # left one at a negative layout coordinate the region space cannot hold. + _run_region_checks(path + "-regions", libei, RecordingEisServer) + + server.stop() + if server.error is not None: + print(f" NOTE: the server thread ended with {server.error!r}") + + failed = [name for name, ok in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/entrypoint-seat.sh b/docker/entrypoint-seat.sh new file mode 100644 index 00000000..91c747ab --- /dev/null +++ b/docker/entrypoint-seat.sh @@ -0,0 +1,111 @@ +#!/bin/sh +# Bring up a wlroots session that really consumes the ydotool device, and ask +# it where an absolute move lands. +# +# Three things have to be true before the question can even be asked, and each +# was once the reason this was recorded as needing a VM: +# +# * the compositor must run libinput. WLR_BACKENDS=headless,libinput keeps +# the outputs virtual — no GPU, no DRM — while the input half is the real +# libinput backend rather than nothing at all. +# * libinput must be allowed to open the device. libseat's builtin backend +# does that without logind, but it also tries to take over a VT; a +# container has none, and SEATD_VTBOUND=0 is what stops it trying. +# * udev must know the device. ydotoold's uinput node is published by the +# kernel, but libinput enumerates through udev, so udevd has to be running +# when the device appears. It is started first for that reason. +# +# The device node itself is created here: udevd inside a container does not +# get to populate /dev, so the minor number is computed from sysfs the same +# way docker/ydotool_verify.py does it. +# +# The verification runs twice, over the same two layouts as +# docker/entrypoint-wayland.sh — side by side from the origin, and with +# HEADLESS-1 moved to x=-1280. The second is the one that separates "the +# layout corner" from "layout (0, 0)"; the first is the control where the two +# coincide and every check has to pass anyway. +set -eu + +RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg}" +mkdir -p "$RUNTIME_DIR" /run/udev +chmod 700 "$RUNTIME_DIR" + +LOG=/tmp/verify.log +RC=/tmp/verify.rc +SWAY_LOG=/tmp/sway.log + +echo "starting systemd-udevd" +/usr/lib/systemd/systemd-udevd --daemon +sleep 1 + +echo "starting ydotoold" +ydotoold --socket-path="$YDOTOOL_SOCKET" --socket-own="$(id -u):$(id -g)" \ + >/tmp/ydotoold.log 2>&1 & +sleep 3 + +# Input event nodes are character major 13, minor 64 + N. +mkdir -p /dev/input +for sysfs in /sys/class/input/event*; do + [ -e "$sysfs" ] || continue + node=$(basename "$sysfs") + [ -e "/dev/input/$node" ] || \ + mknod "/dev/input/$node" c 13 $((64 + ${node#event})) +done + +if [ -z "$(ls -A /dev/input 2>/dev/null)" ]; then + echo "No input device appeared. ydotoold could not create one:" + cat /tmp/ydotoold.log + echo "The container needs --device /dev/uinput and" + echo "--device-cgroup-rule 'c 13:* rmw', and the host needs the uinput" + echo "module loaded." + exit 1 +fi +ls -l /dev/input + +if ! libinput list-devices | grep -qi ydotool; then + echo "libinput cannot see the ydotool device, so sway will not either:" + libinput list-devices || true + exit 1 +fi + +# Both outputs carry a flat colour so that anything else in a capture is the +# cursor. The values are mirrored in seat_verify.py as OUTPUT_COLOURS. +cat > /tmp/sway.cfg.in <<'SWAYCFG' +default_border none +output HEADLESS-1 position @POS1@ bg #123456 solid_color +output HEADLESS-2 position 0 0 bg #abcdef solid_color +exec sh -c 'python3 /opt/verify/seat_verify.py >/tmp/verify.log 2>&1; echo $? >/tmp/verify.rc; swaymsg exit' +SWAYCFG + +# Runs one sway session over one layout; echoes the number of failed checks. +run_session() { + label="$1" + position="$2" + rm -f "$LOG" "$RC" "$SWAY_LOG" + sed "s/@POS1@/$position/" /tmp/sway.cfg.in > /tmp/sway.cfg + + echo "========================================================================" + echo "layout: $label (HEADLESS-1 at $position)" + echo "========================================================================" + + if ! sway -c /tmp/sway.cfg >"$SWAY_LOG" 2>&1; then + echo "sway exited non-zero; its log follows:" + cat "$SWAY_LOG" + fi + + if [ -s "$LOG" ]; then + cat "$LOG" + else + echo "The verification produced no output. sway log:" + cat "$SWAY_LOG" + return 1 + fi + return "$(cat "$RC" 2>/dev/null || echo 1)" +} + +rc=0 +run_session "side by side from the origin" "1280 0" || rc=$((rc + $?)) +echo +run_session "negative origin" "-1280 0" || rc=$((rc + $?)) + +exit "$rc" diff --git a/docker/entrypoint-wayland.sh b/docker/entrypoint-wayland.sh new file mode 100644 index 00000000..120dcdc2 --- /dev/null +++ b/docker/entrypoint-wayland.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# Bring up a headless sway session and run the Wayland verification in it. +# +# The verification has to run *inside* the compositor's session, because +# WAYLAND_DISPLAY is what makes grim, wtype and wlr-randr able to talk to it +# at all — and what makes AutoControl's platform wrapper select the Wayland +# backend. sway execs the script, the script writes its log and exit status +# to files, and this wrapper reports both once sway is gone. +# +# It does that twice, over two output layouts: +# +# * side by side from the origin — sway's own default for two headless +# outputs, and what a single-monitor desktop looks like too; +# * HEADLESS-1 moved to (-1280, 0) — what a desktop looks like whenever a +# monitor sits left of (or above) the primary one. The whole-layout +# capture then starts at a negative coordinate, so every mapping between +# a pixel and a screen coordinate has to subtract that origin. sway's +# headless backend accepts a negative `position`, and grim accepts a +# negative `-g`, so this is a real compositor answering rather than a +# mock agreeing with itself. +set -eu + +RUNTIME_DIR="${XDG_RUNTIME_DIR:-/tmp/xdg}" +mkdir -p "$RUNTIME_DIR" +chmod 700 "$RUNTIME_DIR" + +LOG=/tmp/verify.log +RC=/tmp/verify.rc +SWAY_LOG=/tmp/sway.log + +# The two outputs are painted different solid colours. A uniform screen would +# let a wrong region grab look right, and identical colours would let a +# red/blue swap through — these do neither. The values are mirrored in +# wayland_verify.py as OUTPUT_COLOURS. +cat > /tmp/sway.cfg.in <<'SWAYCFG' +default_border none +output HEADLESS-1 position @POS1@ bg #123456 solid_color +output HEADLESS-2 position 0 0 bg #abcdef solid_color +exec sh -c 'python3 /opt/verify/wayland_verify.py >/tmp/verify.log 2>&1; echo $? >/tmp/verify.rc; swaymsg exit' +SWAYCFG + +# Runs one sway session over one layout; echoes the number of failed checks. +run_session() { + label="$1" + position="$2" + rm -f "$LOG" "$RC" "$SWAY_LOG" + sed "s/@POS1@/$position/" /tmp/sway.cfg.in > /tmp/sway.cfg + + echo "========================================================================" + echo "layout: $label (HEADLESS-1 at $position)" + echo "========================================================================" + + # sway's own chatter goes to a file: it is only interesting when the + # session fails to come up at all, and it would otherwise bury the + # verification. + if ! sway -c /tmp/sway.cfg >"$SWAY_LOG" 2>&1; then + echo "sway exited non-zero; its log follows:" + cat "$SWAY_LOG" + fi + + if [ -s "$LOG" ]; then + cat "$LOG" + else + echo "The verification produced no output. sway log:" + cat "$SWAY_LOG" + return 1 + fi + return "$(cat "$RC" 2>/dev/null || echo 1)" +} + +rc=0 +run_session "side by side from the origin" "1280 0" || rc=$((rc + $?)) +echo +run_session "negative origin" "-1280 0" || rc=$((rc + $?)) + +# The libei half needs no compositor — it drives the binding against the real +# libei.so — so it runs after sway is gone rather than inside a session. +echo +python3 /opt/verify/libei_verify.py || rc=$((rc + $?)) + +exit "$rc" diff --git a/docker/libei_verify.py b/docker/libei_verify.py new file mode 100644 index 00000000..8a8eae76 --- /dev/null +++ b/docker/libei_verify.py @@ -0,0 +1,326 @@ +"""Verify AutoControl's libei binding against the real ``libei.so``. + +No compositor is needed for this half. What the unit tests cannot check — +because they inject a fake symbol table — is whether the entry points this +binding names actually exist in the shared object, with the signatures it +declares. A single misspelled name or a wrong ``argtypes`` would sail past +every mock and only surface on a user's machine. + +So: resolve every prototype against the installed library, then drive +``connect()`` at a socket that accepts the connection but speaks no EI. The +handshake cannot complete, and that is the point — the fail-closed promise +("anything short of a live device means use the ydotool CLI") is checked +here against the real library rather than asserted about a mock. + +What this half cannot answer is anything a peer has to *agree* with: the +capability and event-type enum values, the variadic +``ei_seat_bind_capabilities`` call, and whether emission puts anything on the +wire. ``docker/eis_verify.py`` answers those by running a real libeis server +on the other end of the socket. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import ctypes.util +import faulthandler +import os +import socket +import sys +import tempfile +import threading +import traceback +from typing import Any, Callable, List, Tuple + +# A wrong prototype in a ctypes binding shows up as a segfault, not as an +# exception, and a segfault with no traceback is the hardest kind of bug to +# act on. faulthandler turns it into a Python stack ending at the exact call. +faulthandler.enable() + +_results: List[Tuple[str, bool]] = [] + + +def _scratch_dir() -> str: + """A private directory to write into, created 0700 if it is not there. + + The images set ``XDG_RUNTIME_DIR``, so that is what this returns inside + one. Run this script by hand without it and the answer is a fresh + ``mkdtemp`` rather than the ``/tmp`` root itself, which any user on the + host can create entries in. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-verify-") + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def serve_silent_socket(path: str) -> socket.socket: + """Accept connections at ``path`` and then say nothing at all. + + libei will connect and begin its handshake; nothing answers, so the + client has to give up on its own deadline rather than hang. + """ + if os.path.exists(path): + os.unlink(path) + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + server.bind(path) + server.listen(4) + + def accept_forever() -> None: + held = [] + while True: + try: + conn, _ = server.accept() + except OSError: + return + held.append(conn) # keep it open; never write + + threading.Thread(target=accept_forever, daemon=True).start() + return server + + +def _check_every_entry_point_resolves() -> str: + """The check a mock structurally cannot make: does the .so have these?""" + from je_auto_control.linux_wayland import libei + symbols = libei._load_symbols() + if symbols is None: + raise AssertionError( + "not one prototype resolved — either libei.so is absent or a " + "name in _PROTOTYPES does not exist in it") + missing = [name for name, _, _ in libei._PROTOTYPES + if not hasattr(symbols, name)] + if missing: + raise AssertionError(f"unresolved entry points: {missing}") + # The variadic one is bound separately, without argtypes. + if not hasattr(symbols, "ei_seat_bind_capabilities"): + raise AssertionError("ei_seat_bind_capabilities did not resolve") + return f"{len(libei._PROTOTYPES)} prototypes + 1 variadic, all resolved" + + +def _check_each_call_in_isolation(socket_path: str) -> str: + """Walk connect()'s library calls by hand, printing as it goes. + + connect() is half a dozen calls deep; walking them with flushed output + means a crash names the call that caused it rather than the function + that contained it. + """ + from je_auto_control.linux_wayland import libei + symbols = libei._load_symbols() + + def step(message: str) -> None: + print(f" · {message}", flush=True) + + step("ei_new_sender(None) ...") + handle = symbols.ei_new_sender(None) + step(f" -> {handle!r}") + if not handle: + raise AssertionError("ei_new_sender returned NULL") + + step(f"ei_setup_backend_socket(handle, {socket_path!r}) ...") + code = symbols.ei_setup_backend_socket( + handle, socket_path.encode("utf-8")) + step(f" -> {code}") + + step("ei_get_fd(handle) ...") + poll_fd = symbols.ei_get_fd(handle) + step(f" -> {poll_fd}") + + step("ei_dispatch(handle) ...") + symbols.ei_dispatch(handle) + step(" -> returned") + + step("ei_get_event(handle) ...") + event = symbols.ei_get_event(handle) + step(f" -> {event!r}") + while event: + kind = symbols.ei_event_get_type(event) + step(f" event type {kind}") + symbols.ei_event_unref(event) + event = symbols.ei_get_event(handle) + step(f" next -> {event!r}") + + # ei_unref is NOT called here: on this libei it segfaults once the + # backend is open. The sentinel below establishes that separately, + # in a subprocess, so it cannot take this run down with it. + step("(context abandoned — see the ei_unref sentinel)") + return "every call up to teardown behaves" + + +def _check_unref_sentinel(socket_path: str) -> str: + """Is the upstream ei_unref crash this binding works around still there?""" + import subprocess # nosec B404 # reason: argv list, no shell + program = ( + "import ctypes, ctypes.util, os, socket, threading;" + "lib = ctypes.CDLL(ctypes.util.find_library('ei'));" + "lib.ei_new_sender.restype = ctypes.c_void_p;" + "lib.ei_new_sender.argtypes = (ctypes.c_void_p,);" + "lib.ei_setup_backend_socket.restype = ctypes.c_int;" + "lib.ei_setup_backend_socket.argtypes = " + "(ctypes.c_void_p, ctypes.c_char_p);" + "lib.ei_unref.restype = ctypes.c_void_p;" + "lib.ei_unref.argtypes = (ctypes.c_void_p,);" + f"p = {socket_path!r};" + "s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM);" + "s.connect(p);" + "h = lib.ei_new_sender(None);" + "rc = lib.ei_setup_backend_socket(h, p.encode());" + "assert rc == 0, rc;" + "lib.ei_unref(h)" + ) + # This interpreter, running a program built from literals above; no shell. + finished = subprocess.run([sys.executable, "-c", program], # nosec B603 # nosemgrep + capture_output=True) + if finished.returncode == -11: + return ("still segfaults (rc=-11), so the abandon-on-teardown " + "workaround in libei.py::_teardown is still required") + print() + print(" *** REVISIT *** ei_unref no longer crashes on this") + print(" libei (rc=%s). The workaround in LibeiBackend._teardown" + % finished.returncode) + print(" can probably go; see Progress.md.") + print() + return f"no longer crashes (rc={finished.returncode}) — see above" + + +def _check_connect_fails_closed(socket_path: str) -> str: + """A peer that sends nothing must not be reported as a live session.""" + from je_auto_control.linux_wayland import libei + backend = libei.LibeiBackend() + try: + backend.connect(timeout=1.0, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable as error: + return f"LibeiUnavailable: {str(error)[:90]}" + raise AssertionError( + "connect() reported success against a peer that sent nothing, so " + "the handshake is not actually gating on a live device") + + +def _check_teardown_survives(socket_path: str) -> str: + """disconnect() has to be safe after a failed connect, and idempotent.""" + from je_auto_control.linux_wayland import libei + backend = libei.LibeiBackend() + try: + backend.connect(timeout=0.5, + socket_path=socket_path.encode("utf-8")) + except libei.LibeiUnavailable: + pass + backend.disconnect() # must be safe after a failed connect + backend.disconnect() # and idempotent + return "teardown survived a failed connect, twice" + + +def _check_keyboard_falls_back() -> str: + """With no libei and no ydotool, the CLI path must surface its hint. + + ydotool is deliberately not installed in this image, so what comes back + must be the install hint — not a libei error and not a silent no-op. + """ + from je_auto_control.linux_wayland import keyboard as wl_keyboard + try: + wl_keyboard.press_key(30) + except Exception as error: # noqa: BLE001 # reason: any type is informative + if "ydotool" in str(error): + return f"{type(error).__name__}: {str(error)[:60]}" + raise + raise AssertionError("press_key claimed success with no libei and no " + "ydotool") + + +def main() -> int: + print("=" * 72) + print("AutoControl libei binding — against the real libei.so") + print("=" * 72) + + resolved = ctypes.util.find_library("ei") + print(f"find_library('ei') = {resolved!r}") + print(f"find_library('oeffis') = {ctypes.util.find_library('oeffis')!r}") + print("-" * 72) + + from je_auto_control.linux_wayland import _select_input, libei, oeffis + + # --- a real sender against a socket that speaks no EI ---------------- + runtime = _scratch_dir() + socket_path = os.path.join(runtime, "eis-0") + server = serve_silent_socket(socket_path) + print(f" silent EIS stand-in listening at {socket_path}") + + check("every libei entry point this binding names exists", + _check_every_entry_point_resolves) + + check("LibeiBackend reports the library as available", + lambda: _assert_true(libei.LibeiBackend().is_available, + "is_available was False with libei installed")) + + # --- liboeffis: packaged separately, so easily absent ---------------- + # Debian does package it (liboeffis1 on trixie), but as its own binary + # package that libei1 does not depend on — so installing libei alone + # leaves the portal route off, which is the state this image is in and + # the state a user who installed one package would be in. + available = oeffis.is_available() + print(f" liboeffis available here: {available}") + if not available: + print(" (liboeffis is not installed here, so the portal route is") + print(" unavailable and connect() falls back to the socket —") + print(" which is exactly the path exercised below. The portal") + print(" route itself is covered by docker/portal_verify.py.)") + + check("each libei call in isolation", + lambda: _check_each_call_in_isolation(socket_path)) + check("ei_unref after a successful setup — upstream state", + lambda: _check_unref_sentinel(socket_path)) + check("connect() against a silent peer fails closed, not open", + lambda: _check_connect_fails_closed(socket_path)) + check("teardown after a failed handshake does not crash the process", + lambda: _check_teardown_survives(socket_path)) + + # --- the fallback the whole design rests on -------------------------- + libei.reset_default_backend() + check("active_backend() gives up and hands over to the CLI", + lambda: _assert_true(_select_input.active_backend() is None, + "active_backend() returned a backend that " + "cannot emit")) + check("press_key falls through to the ydotool CLI path", + _check_keyboard_falls_back) + + server.close() + + print("-" * 72) + print("Not covered here, because it needs a peer that speaks EI:") + print(" the capability / event-type enum values, the variadic") + print(" ei_seat_bind_capabilities call, seat grants, emission and") + print(" the live-context teardown. docker/eis_verify.py covers all") + print(" of that against a real libeis server — run it too.") + + failed = [name for name, ok in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/portal_server.py b/docker/portal_server.py new file mode 100644 index 00000000..d98619f4 --- /dev/null +++ b/docker/portal_server.py @@ -0,0 +1,461 @@ +"""A mock ``org.freedesktop.portal.Desktop`` that implements RemoteDesktop. + +The libei input path has two halves. ``docker/eis_server.py`` answers the +protocol half — a real ``libeis`` peer on a Unix socket, which settles the +enum values, the variadic bind and what actually goes on the wire. The other +half is how a client *gets* that socket on GNOME and KDE: it is not a path on +disk but a file descriptor handed over D-Bus at the end of the +``org.freedesktop.portal.RemoteDesktop`` session dance, and +:mod:`je_auto_control.linux_wayland.oeffis` drives ``liboeffis`` to perform it. + +That half was recorded as needing a GNOME VM, on the grounds that +``xdg-desktop-portal-wlr`` implements ScreenCast and Screenshot but not +RemoteDesktop. What that reasoning missed is that the portal is *a D-Bus +interface*, not a compositor feature: anything that owns the well-known name +and answers the four calls is a portal as far as ``liboeffis`` is concerned. +So this module is that — a real D-Bus service on a real session bus, driving +the real ``liboeffis`` through the real handshake — and ``ConnectToEIS`` +hands back a live connection to the real EIS server next door, which makes +the whole chain end at pixels-equivalent evidence: input emitted through a +portal-obtained fd, recorded by an independent implementation. + +What it deliberately does *not* claim to be: a consent dialog. There is no +user here to dismiss anything. What a dialog produces, though — a grant, a +refusal, and a wait that never ends — are all just Response codes and silence +on the bus, so ``--behaviour`` produces each of them and the caller's +fail-closed handling is verified against all three. + +Runs on the system interpreter, not the one that has AutoControl installed: +GDBus is what can pass a file descriptor over D-Bus from Python, it comes +from Debian's ``python3-gi``, and it is bound to Debian's ``python3``. That +suits it anyway — the portal has to be a separate process from the client +that calls it. +""" +from __future__ import annotations + +import argparse +import json +import os +import socket +import struct +import sys +import tempfile +import zlib +from typing import Any, Dict, Optional + +import gi + +gi.require_version("Gio", "2.0") +from gi.repository import Gio, GLib # noqa: E402 # reason: gi.require_version must run first + + +BUS_NAME = "org.freedesktop.portal.Desktop" +OBJECT_PATH = "/org/freedesktop/portal/desktop" +REQUEST_INTERFACE = "org.freedesktop.portal.Request" +SESSION_INTERFACE = "org.freedesktop.portal.Session" + +#: Portal response codes, from the XDG portal specification. +RESPONSE_SUCCESS = 0 +RESPONSE_CANCELLED = 1 + +#: Everything a compositor could offer: keyboard | pointer | touchscreen. +ALL_DEVICE_TYPES = 7 + +#: ``grant`` is the happy path; the rest are the ways a real portal says no. +BEHAVIOURS = ("grant", "deny", "stall", "no-fd", "close") + +#: The same idea for the Screenshot interface, which is a different portal +#: with a different client: :mod:`je_auto_control.linux_wayland.portal` speaks +#: D-Bus itself, so what is under test there is hand-written marshalling and a +#: directed signal arriving on a path the client predicted. +SCREENSHOT_BEHAVIOURS = ("grant", "deny", "stall", "no-uri", "not-a-file") + +#: A filename with a space in it on purpose: the URI the portal returns is +#: percent-encoded, and unquoting it is a step the client has to get right. +SCREENSHOT_NAME = "autocontrol portal shot.png" + +#: What the mock paints, so the client can be checked on the bytes it got +#: rather than merely on getting some. +SHOT_SIZE = (4, 3) +SHOT_RGB = (0, 128, 255) + +PORTAL_XML = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + +REQUEST_XML = """ + + + + + + + + + +""" + +SESSION_XML = """ + + + + + + + + + +""" + + +def sender_token(sender: str) -> str: + """The client's unique bus name as the portal spec spells it in a path. + + Request and session objects live at a path the *client* predicts so it can + subscribe before it calls, which only works if both sides mangle the name + the same way: drop the leading colon, turn dots into underscores. + """ + return sender.lstrip(":").replace(".", "_") + + +def _interface(xml: str, name: str = "") -> Gio.DBusInterfaceInfo: + """One interface out of an introspection document, by name or the first.""" + interfaces = Gio.DBusNodeInfo.new_for_xml(xml).interfaces + if not name: + return interfaces[0] + return next(info for info in interfaces if info.name == name) + + +def encode_png(width: int, height: int, rgb: tuple) -> bytes: + """A valid PNG of one flat colour, with nothing but the standard library. + + The mock runs on Debian's interpreter, which has no imaging package, and + handing the client bytes that are not really a PNG would check the + transport while quietly skipping whether the result is usable. + """ + row = bytes((0,)) + bytes(rgb) * width + raw = zlib.compress(row * height, 9) + + def chunk(kind: bytes, payload: bytes) -> bytes: + body = kind + payload + return (struct.pack(">I", len(payload)) + body + + struct.pack(">I", zlib.crc32(body) & 0xFFFFFFFF)) + + header = struct.pack(">IIBBBBB", width, height, 8, 2, 0, 0, 0) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", header) + + chunk(b"IDAT", raw) + chunk(b"IEND", b"")) + + +class MockPortal: + """Owns the portal bus name and answers the RemoteDesktop calls. + + :param eis_socket: the EIS server socket ``ConnectToEIS`` connects to. + :param behaviour: one of :data:`BEHAVIOURS`. + :param record_path: JSON file the driver reads to see what arrived. + :param version: what to report as the RemoteDesktop interface version; + ``ConnectToEIS`` only exists from 2, so 1 is a portal too old to use. + """ + + def __init__(self, eis_socket: str, behaviour: str, record_path: str, + version: int = 2, screenshot: str = "grant", + shot_dir: Optional[str] = None) -> None: + self.eis_socket = eis_socket + self.behaviour = behaviour + self.record_path = record_path + self.version = version + self.screenshot = screenshot + # The driver always passes one; without it, a private + # directory rather than the world-writable /tmp root. + self.shot_dir = shot_dir or tempfile.mkdtemp( + prefix="autocontrol-portal-") + self.record: Dict[str, Any] = { + "calls": [], "device_types": None, "properties": [], + "session_closed_by_client": False, "shot_path": "", + } + self.connection: Optional[Gio.DBusConnection] = None + #: Registration ids are kept only so the objects stay exported. + self._exported: list = [] + self._remote_desktop_info = _interface( + PORTAL_XML, "org.freedesktop.portal.RemoteDesktop") + self._screenshot_info = _interface( + PORTAL_XML, "org.freedesktop.portal.Screenshot") + self._request_info = _interface(REQUEST_XML) + self._session_info = _interface(SESSION_XML) + + # --- recording -------------------------------------------------------- + + def _flush(self) -> None: + """Write the record out now; the driver reads it while we still run.""" + with open(self.record_path, "w", encoding="utf-8") as handle: + json.dump(self.record, handle) + + def _note(self, method: str, detail: Any = None) -> None: + self.record["calls"].append({"method": method, "detail": detail}) + self._flush() + + # --- lifecycle -------------------------------------------------------- + + def run(self) -> None: + """Take the bus name and serve until killed.""" + self.connection = Gio.bus_get_sync(Gio.BusType.SESSION, None) + for info in (self._remote_desktop_info, self._screenshot_info): + self.connection.register_object( + OBJECT_PATH, info, + self._on_method_call, self._on_get_property, None) + Gio.bus_own_name_on_connection( + self.connection, BUS_NAME, Gio.BusNameOwnerFlags.NONE, + lambda *_args: print("portal: owns " + BUS_NAME, flush=True), + lambda *_args: print("portal: lost " + BUS_NAME, flush=True)) + self._flush() + print(f"portal: ready (behaviour={self.behaviour}, " + f"screenshot={self.screenshot}, version={self.version})", + flush=True) + GLib.MainLoop().run() + + # --- properties ------------------------------------------------------- + + def _on_get_property(self, _connection: Gio.DBusConnection, _sender: str, + _path: str, _interface_name: str, + name: str) -> Optional[GLib.Variant]: + """``version`` is the first thing liboeffis reads, before any call.""" + self.record["properties"].append(name) + self._flush() + if name == "version": + return GLib.Variant("u", self.version) + if name == "AvailableDeviceTypes": + return GLib.Variant("u", ALL_DEVICE_TYPES) + return None + + # --- method dispatch -------------------------------------------------- + + def _on_method_call(self, _connection: Gio.DBusConnection, sender: str, + _path: str, _interface_name: str, method: str, + parameters: GLib.Variant, + invocation: Gio.DBusMethodInvocation) -> None: + print(f"portal: {method}{parameters}", flush=True) + handler = getattr(self, f"_do_{method}", None) + if not callable(handler): + invocation.return_error_literal( + Gio.dbus_error_quark(), Gio.DBusError.UNKNOWN_METHOD, method) + return + # pylint: disable=not-callable + # reason: callable() above is the guard; pylint infers the type of a + # dynamic getattr from its default and cannot narrow past the check. + handler(sender, parameters, invocation) + + def _request_path(self, sender: str, options: Dict[str, Any]) -> str: + """The object path the client already subscribed to.""" + token = options.get("handle_token", "unnamed") + return f"{OBJECT_PATH}/request/{sender_token(sender)}/{token}" + + def _export_request(self, path: str) -> None: + """Export the request object so ``Close()`` on it is answerable.""" + self._exported.append(self.connection.register_object( + path, self._request_info, self._answer_empty, None, None)) + + def _answer_empty(self, _connection: Gio.DBusConnection, _sender: str, + _path: str, _interface_name: str, _method: str, + _parameters: GLib.Variant, + invocation: Gio.DBusMethodInvocation) -> None: + invocation.return_value(None) + + def _on_session_call(self, _connection: Gio.DBusConnection, _sender: str, + _path: str, _interface_name: str, method: str, + _parameters: GLib.Variant, + invocation: Gio.DBusMethodInvocation) -> None: + """Record whether the client ends its grant explicitly, or just leaves.""" + if method == "Close": + self.record["session_closed_by_client"] = True + self._flush() + invocation.return_value(None) + + def _respond(self, sender: str, path: str, code: int, + results: Dict[str, GLib.Variant]) -> bool: + """Emit the ``Response`` signal the whole portal protocol turns on.""" + self.connection.emit_signal( + sender, path, REQUEST_INTERFACE, "Response", + GLib.Variant("(ua{sv})", (code, results))) + return False # reason: one-shot, so GLib.idle_add does not repeat it + + # --- the four RemoteDesktop calls ------------------------------------- + + def _do_CreateSession(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + options = parameters.unpack()[0] + self._note("CreateSession", options) + request = self._request_path(sender, options) + self._export_request(request) + token = options.get("session_handle_token", "unnamed") + session = f"{OBJECT_PATH}/session/{sender_token(sender)}/{token}" + self._exported.append(self.connection.register_object( + session, self._session_info, self._on_session_call, + lambda *_args: GLib.Variant("u", 2), None)) + invocation.return_value(GLib.Variant("(o)", (request,))) + if self.behaviour == "stall": + # A consent dialog nobody answers: the handle was returned, the + # Response never comes. The client must give up on its own clock. + return + GLib.idle_add(self._respond, sender, request, RESPONSE_SUCCESS, + {"session_handle": GLib.Variant("s", session)}) + if self.behaviour == "close": + GLib.idle_add(self._close_session, sender, session) + + def _close_session(self, sender: str, session: str) -> bool: + """End the grant the way a compositor does when the session stops.""" + self.connection.emit_signal( + sender, session, SESSION_INTERFACE, "Closed", + GLib.Variant("(a{sv})", ({},))) + return False + + def _do_SelectDevices(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + session, options = parameters.unpack() + self._note("SelectDevices", {"session": session, "options": options}) + types = options.get("types") + if types is not None: + self.record["device_types"] = int(types) + self._flush() + request = self._request_path(sender, options) + self._export_request(request) + invocation.return_value(GLib.Variant("(o)", (request,))) + GLib.idle_add(self._respond, sender, request, RESPONSE_SUCCESS, {}) + + def _do_Start(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + session, parent_window, options = parameters.unpack() + self._note("Start", {"session": session, "parent": parent_window, + "options": options}) + request = self._request_path(sender, options) + self._export_request(request) + invocation.return_value(GLib.Variant("(o)", (request,))) + # ``deny`` is the consent dialog the user dismissed: the portal + # answers, and the answer is no. + denied = self.behaviour == "deny" + code = RESPONSE_CANCELLED if denied else RESPONSE_SUCCESS + results = {} if denied else {"devices": GLib.Variant("u", 3)} + GLib.idle_add(self._respond, sender, request, code, results) + + def _do_ConnectToEIS(self, _sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + session, options = parameters.unpack() + self._note("ConnectToEIS", {"session": session, "options": options}) + if self.behaviour == "no-fd": + invocation.return_error_literal( + Gio.dbus_error_quark(), Gio.DBusError.FAILED, + "this portal will not hand over an EIS fd") + return + try: + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.connect(self.eis_socket) + except OSError as error: + invocation.return_error_literal( + Gio.dbus_error_quark(), Gio.DBusError.FAILED, str(error)) + return + # GDBus dups what it appends, so the local end is closed here and the + # client gets its own descriptor — which is exactly the ownership a + # real portal hands over. + fd_list = Gio.UnixFDList.new() + index = fd_list.append(client.fileno()) + client.close() + invocation.return_value_with_unix_fd_list( + GLib.Variant("(h)", (index,)), fd_list) + + + # --- the Screenshot call ---------------------------------------------- + + def _do_Screenshot(self, sender: str, parameters: GLib.Variant, # noqa: N802 # reason: D-Bus method name + invocation: Gio.DBusMethodInvocation) -> None: + """Answer the interface every desktop implements, however it captures. + + The client here is not a C library but AutoControl's own D-Bus + marshalling, in :mod:`je_auto_control.linux_wayland._dbus_client`. + Answering on the path the client predicted, with a signal directed at + the connection that called, is what that code has to cope with — and + is exactly what the ``gdbus monitor`` design it replaced could not. + """ + parent_window, options = parameters.unpack() + self._note("Screenshot", {"parent": parent_window, "options": options}) + request = self._request_path(sender, options) + self._export_request(request) + invocation.return_value(GLib.Variant("(o)", (request,))) + if self.screenshot == "stall": + return + GLib.idle_add(self._respond_screenshot, sender, request) + + def _respond_screenshot(self, sender: str, request: str) -> bool: + """Every outcome a real Screenshot portal can end on.""" + if self.screenshot == "deny": + return self._respond(sender, request, RESPONSE_CANCELLED, {}) + if self.screenshot == "no-uri": + return self._respond(sender, request, RESPONSE_SUCCESS, {}) + if self.screenshot == "not-a-file": + return self._respond(sender, request, RESPONSE_SUCCESS, + {"uri": GLib.Variant( + "s", "https://example.invalid/shot.png")}) + path = os.path.join(self.shot_dir, SCREENSHOT_NAME) + with open(path, "wb") as image: + image.write(encode_png(*SHOT_SIZE, SHOT_RGB)) + self.record["shot_path"] = path + self._flush() + uri = "file://" + GLib.uri_escape_string(path, "/", False) + return self._respond(sender, request, RESPONSE_SUCCESS, + {"uri": GLib.Variant("s", uri)}) + + +def main() -> int: + """Parse the arguments and serve until the driver kills the process.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--eis-socket", required=True, + help="EIS server socket that ConnectToEIS connects to") + parser.add_argument("--behaviour", default="grant", choices=BEHAVIOURS) + parser.add_argument("--record", required=True, + help="JSON file recording what the client asked for") + parser.add_argument("--version", type=int, default=2, + help="RemoteDesktop interface version to advertise") + parser.add_argument("--screenshot", default="grant", + choices=SCREENSHOT_BEHAVIOURS) + parser.add_argument("--shot-dir", default=None, + help="directory the mock writes its capture into") + arguments = parser.parse_args() + os.umask(0o077) + MockPortal(arguments.eis_socket, arguments.behaviour, arguments.record, + arguments.version, arguments.screenshot, + arguments.shot_dir).run() + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/portal_verify.py b/docker/portal_verify.py new file mode 100644 index 00000000..a6f4c377 --- /dev/null +++ b/docker/portal_verify.py @@ -0,0 +1,588 @@ +"""Verify AutoControl's portal handshake against a real ``liboeffis``. + +``eis_verify.py`` drives the libei sender against a real EIS peer, but it +reaches that peer the way no GNOME or KDE desktop offers: by handing +``connect()`` a socket path. The route those desktops actually use is +``org.freedesktop.portal.RemoteDesktop`` — a D-Bus session dance ending in a +file descriptor passed over SCM_RIGHTS — and it was recorded as unverifiable +without a GNOME VM, because ``xdg-desktop-portal-wlr`` has no RemoteDesktop +interface. + +The portal is a D-Bus interface, though, not a compositor feature. +``docker/portal_server.py`` owns the well-known name and answers the four +calls, so the real ``liboeffis`` runs the real handshake here, and its +``ConnectToEIS`` returns a live connection to the real EIS server from +``eis_server.py``. That makes the whole path checkable end to end: + + * the four calls arrive in the order the specification prescribes, at the + request paths the client predicted — a mismatch there is a client that + subscribes to a signal nobody sends and then times out; + * ``SelectDevices`` receives the keyboard-and-pointer mask this project + asks for, so :data:`~je_auto_control.linux_wayland.oeffis.OEFFIS_DEVICE_DEFAULT` + is neither wider than the grant the user consents to nor the ``= 0`` + all-devices sentinel it would be if the constant were wrong; + * the descriptor that comes back carries a real EI session: the sender + completes its handshake on it and the input it emits is recorded by an + independent implementation at the far end; + * and every way a portal says no — a dismissed dialog, a dialog left open, + a refused descriptor, a closed session, a portal too old to have + ``ConnectToEIS`` at all, no portal on the bus — produces a refusal on + this project's own clock rather than a hang or a silent downgrade. + +What is still not claimed: the consent dialog itself. No user dismisses +anything here. What a dialog *produces* is a Response code or silence, and +all three outcomes are exercised; what it looks like is mutter's business. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import contextlib +import faulthandler +import json +import os +import shutil +import stat +import subprocess # nosec B404 # reason: launches dbus-daemon and the mock portal, argv lists, no shell +import sys +import tempfile +import threading +import time +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +# A wrong prototype in a ctypes binding is a segfault, not an exception. +faulthandler.enable() + +_results: List[Tuple[str, bool]] = [] + +#: evdev codes the emission check uses. KEY_A, BTN_LEFT. +KEY_A = 30 +BTN_LEFT = 272 +TARGET_POSITION = (640, 400) + +#: The order the XDG portal specification prescribes, and the order a client +#: that got it wrong would never complete in. +EXPECTED_CALLS = ("CreateSession", "SelectDevices", "Start", "ConnectToEIS") + +#: What the mock paints. Re-declared rather than imported from +#: ``portal_server``, which runs on the other interpreter — and which is the +#: right call anyway: two sides agreeing on one wrong constant is a test that +#: passes while the thing under it is broken. +SHOT_SIZE = (4, 3) +SHOT_RGB = (0, 128, 255) + +#: The file name the mock writes into the ``--shot-dir`` it is given. Both +#: halves of the capture path are therefore this script's own, so the +#: cleanup check below never has to build a path out of what came back over +#: the bus; the equality assertion there is what fails loudly if +#: ``portal_server.SCREENSHOT_NAME`` and this ever drift apart. +SHOT_NAME = "autocontrol portal shot.png" + +#: Long enough for a portal that is going to answer; short enough that the +#: ones written not to answer do not hold the image up. +GRANT_TIMEOUT = 10.0 +REFUSAL_TIMEOUT = 3.0 + + +def check(name: str, function: Callable[[], Any]) -> Any: + """Run one check, record the outcome, and never let it stop the rest.""" + try: + detail = function() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _runtime_dir() -> str: + """A private directory for the sockets and the capture, created 0700. + + The image sets ``XDG_RUNTIME_DIR``, so that is what this returns inside + one. Run this by hand without it and the answer is a fresh ``mkdtemp`` + rather than the ``/tmp`` root itself, which any user can create entries + in — including the socket names this script predicts. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-portal-verify-") + + +def _fresh_socket_path(name: str) -> str: + path = os.path.join(_runtime_dir(), name) + with contextlib.suppress(FileNotFoundError): + os.unlink(path) + return path + + +class SessionBus: + """A private ``dbus-daemon`` this run owns, exported to the environment. + + liboeffis reads ``DBUS_SESSION_BUS_ADDRESS`` from the environment of the + process it is loaded into, so the address has to land in :data:`os.environ` + and not only in the child's. + """ + + def __init__(self) -> None: + self._process: Optional[subprocess.Popen] = None + self.address = "" + + def start(self) -> str: + binary = shutil.which("dbus-daemon") + if binary is None: + raise RuntimeError("dbus-daemon is not installed") + # argv is a private allow-list; no shell and no user-supplied component. + self._process = subprocess.Popen( # nosec B603 # nosemgrep + [binary, "--session", "--print-address", "--nofork"], + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True) + address = (self._process.stdout.readline() or "").strip() + if not address: + raise RuntimeError("dbus-daemon printed no bus address") + self.address = address + os.environ["DBUS_SESSION_BUS_ADDRESS"] = address + return address + + def stop(self) -> None: + if self._process is None: + return + with contextlib.suppress(OSError, ValueError): + self._process.terminate() + with contextlib.suppress(subprocess.TimeoutExpired, OSError, ValueError): + self._process.wait(timeout=5) + self._process = None + + +class Portal: + """The mock portal, running as a child process for one scenario. + + Used as a context manager so the bus name is released before the next + scenario asks for it. + """ + + def __init__(self, eis_socket: str, behaviour: str = "grant", + version: int = 2, screenshot: str = "grant") -> None: + self.eis_socket = eis_socket + self.behaviour = behaviour + self.version = version + self.screenshot = screenshot + self.record_path = os.path.join( + _runtime_dir(), + f"portal-record-{behaviour}-{screenshot}-{version}.json") + self.lines: List[str] = [] + self._process: Optional[subprocess.Popen] = None + self._pump: Optional[threading.Thread] = None + + def __enter__(self) -> "Portal": + server = os.path.join(os.path.dirname(os.path.abspath(__file__)), + "portal_server.py") + # Debian's python3 is the one with GDBus; this interpreter is the one + # with AutoControl. The portal has to be a separate process anyway. + argv = ["/usr/bin/python3", server, + "--eis-socket", self.eis_socket, + "--behaviour", self.behaviour, + "--record", self.record_path, + "--version", str(self.version), + "--screenshot", self.screenshot, + "--shot-dir", _runtime_dir()] + # argv is a private allow-list; no shell and no user-supplied component. + self._process = subprocess.Popen( # nosec B603 # nosemgrep + argv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + self._pump = threading.Thread(target=self._read, daemon=True, + name="portal-stdout") + self._pump.start() + self._await_name() + return self + + def __exit__(self, *_exception: Any) -> None: + if self._process is None: + return + with contextlib.suppress(OSError, ValueError): + self._process.terminate() + with contextlib.suppress(subprocess.TimeoutExpired, OSError, ValueError): + self._process.wait(timeout=5) + if self._pump is not None: + self._pump.join(timeout=2) + self._process = None + + def _read(self) -> None: + stream = self._process.stdout if self._process else None + if stream is None: + return + with contextlib.suppress(OSError, ValueError): + for line in stream: + self.lines.append(line.rstrip()) + + def _await_name(self, timeout: float = 15.0) -> None: + """Wait until the portal owns the name, not merely until it started.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + # Snapshot: the reader thread appends to this list as we read. + seen = list(self.lines) + if any("owns " in line for line in seen): + return + if self._process is not None and self._process.poll() is not None: + raise RuntimeError( + "the mock portal exited before taking the bus name:\n" + + "\n".join(self.lines)) + time.sleep(0.02) + raise RuntimeError("the mock portal never took the bus name") + + def recorded(self) -> Dict[str, Any]: + """What the portal has seen so far.""" + try: + with open(self.record_path, "r", encoding="utf-8") as handle: + return json.load(handle) + except (OSError, ValueError): + return {"calls": [], "device_types": None, "properties": []} + + def methods(self) -> List[str]: + return [call["method"] for call in self.recorded()["calls"]] + + def await_call(self, method: str, timeout: float = 5.0) -> None: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if method in self.methods(): + return + time.sleep(0.02) + raise AssertionError( + f"the portal never saw {method}; it saw {self.methods()}") + + +def _wait_for(predicate: Callable[[], bool], timeout: float, + description: str) -> None: + """Spin until the server thread has recorded something, or give up.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if predicate(): + return + time.sleep(0.02) + raise AssertionError(f"timed out waiting for {description}") + + +def _refused(action: Callable[[], Any], expected: type, + budget: float) -> str: + """Assert an action fails closed, promptly, and return the reason it gave.""" + started = time.monotonic() + try: + action() + except expected as error: + elapsed = time.monotonic() - started + _require(elapsed <= budget, + f"the refusal took {elapsed:.1f}s, longer than the {budget:g}s " + f"budget — that is a hang wearing a refusal's clothes") + return f"{str(error)[:96]} (in {elapsed:.1f}s)" + raise AssertionError( + f"expected {expected.__name__}; the call returned instead") + + +# --- checks --------------------------------------------------------------- + + +def _check_liboeffis_binds(oeffis) -> str: + """Every entry point the binding names, resolved against the real library.""" + symbols = oeffis.load_symbols() + _require(symbols is not None, + "liboeffis.so.* did not resolve — install liboeffis1") + _require(oeffis.is_available(), "is_available() disagrees with load_symbols()") + _require(oeffis.OEFFIS_DEVICE_DEFAULT != oeffis.OEFFIS_DEVICE_ALL_DEVICES, + "the default device mask is the all-devices sentinel") + return f"default device mask = {oeffis.OEFFIS_DEVICE_DEFAULT}" + + +def _check_no_portal_is_refused(oeffis) -> str: + """The state of every desktop that has no RemoteDesktop portal.""" + return _refused(lambda: oeffis.connect_eis_fd(timeout=REFUSAL_TIMEOUT), + oeffis.OeffisUnavailable, REFUSAL_TIMEOUT + 3.0) + + +def _check_call_order(portal: Portal) -> str: + calls = portal.methods() + _require(tuple(calls[:len(EXPECTED_CALLS)]) == EXPECTED_CALLS, + f"the portal saw {calls}, not {list(EXPECTED_CALLS)}") + _require("version" in portal.recorded()["properties"], + "liboeffis never read the RemoteDesktop version property") + return " -> ".join(calls) + + +def _check_device_mask(portal: Portal, oeffis) -> str: + """What the user is actually asked to consent to.""" + types = portal.recorded()["device_types"] + _require(types is not None, "SelectDevices carried no `types` option") + _require(types == oeffis.OEFFIS_DEVICE_DEFAULT, + f"the portal was asked for {types}, not " + f"{oeffis.OEFFIS_DEVICE_DEFAULT}") + _require(not types & oeffis.OEFFIS_DEVICE_TOUCHSCREEN, + "the grant includes a touchscreen this backend never emits on") + return f"types={types} (keyboard|pointer)" + + +def _check_fd_is_live_and_owned(oeffis) -> str: + """The descriptor is a real socket, and it is the caller's to close. + + ``oeffis_get_eis_fd`` is documented as returning a ``dup()`` whose owner + is the caller. Everything downstream depends on that: libei takes + ownership of what it is handed and closes it when torn down, so a second + owner here would be a double close. + """ + eis_fd, session = oeffis.connect_eis_fd(timeout=GRANT_TIMEOUT) + try: + _require(eis_fd >= 0, f"the portal handed over fd {eis_fd}") + info = os.fstat(eis_fd) + _require(stat.S_ISSOCK(info.st_mode), + f"fd {eis_fd} is not a socket (mode {info.st_mode:#o})") + os.close(eis_fd) + with contextlib.suppress(OSError): + os.fstat(eis_fd) + raise AssertionError( + f"fd {eis_fd} survived close() — it was not ours to own") + finally: + session.close() + session.close() # idempotent: the teardown path calls it too + return f"fd {eis_fd} was a socket, and closing it was ours to do" + + +def _connected_through_the_portal(libei): + """A full sender handshake with no socket path — the portal's route.""" + backend = libei.LibeiBackend() + backend.connect(timeout=GRANT_TIMEOUT) + _require(backend.is_connected, + "connect() returned but the backend reports no live device") + return backend + + +def _check_session_reaches_eis(server) -> str: + record = server.recording + _wait_for(lambda: bool(record.devices), 5.0, "the seat to hand over devices") + _require(record.seat_binds >= 1, "the client never bound a seat") + _require(all(record.sender_flags), + "the client did not present itself as a sender") + return (f"client={record.clients[0]!r} seat binds={record.seat_binds} " + f"devices={record.devices}") + + +def _check_input_over_the_portal_fd(backend, server) -> str: + record = server.recording + backend.press_key(KEY_A) + backend.release_key(KEY_A) + backend.set_position(*TARGET_POSITION) + backend.press_button(BTN_LEFT) + backend.release_button(BTN_LEFT) + _wait_for(lambda: len(record.keys) >= 2, 5.0, "the key press and release") + _wait_for(lambda: bool(record.absolute_motions), 5.0, "the absolute motion") + _wait_for(lambda: len(record.buttons) >= 2, 5.0, "the button edges") + _require(record.keys[:2] == [(KEY_A, True), (KEY_A, False)], + f"the server recorded {record.keys[:2]}") + landed = record.absolute_motions[-1] + _require(tuple(int(value) for value in landed) == TARGET_POSITION, + f"the pointer landed at {landed}, not {TARGET_POSITION}") + _require(record.buttons[:2] == [(BTN_LEFT, True), (BTN_LEFT, False)], + f"the server recorded {record.buttons[:2]}") + return (f"keys={record.keys[:2]} motion={landed} " + f"buttons={record.buttons[:2]}") + + +def _check_disconnect_ends_the_grant(backend, server, portal: Portal) -> str: + """Tearing down must revoke the grant, not merely stop emitting.""" + before = server.recording.disconnects + backend.disconnect() + backend.disconnect() + _wait_for(lambda: server.recording.disconnects > before, 5.0, + "the EIS server to see the client go away") + closed = portal.recorded().get("session_closed_by_client", False) + return ("the portal session was closed explicitly" if closed else + "the D-Bus connection was dropped, which revokes the grant") + + +def _check_portal_refuses(oeffis) -> str: + return _refused(lambda: oeffis.connect_eis_fd(timeout=GRANT_TIMEOUT), + oeffis.OeffisUnavailable, GRANT_TIMEOUT) + + +def _check_open_dialog_times_out(oeffis) -> str: + """A consent dialog nobody answers must end on our clock, not never.""" + reason = _refused(lambda: oeffis.connect_eis_fd(timeout=REFUSAL_TIMEOUT), + oeffis.OeffisUnavailable, REFUSAL_TIMEOUT + 3.0) + _require("consent" in reason, + f"the timeout did not name the consent dialog: {reason}") + return reason + + +def _check_backend_surfaces_the_refusal(libei) -> str: + """A refused portal must reach the caller as this project's own error.""" + return _refused(lambda: libei.LibeiBackend().connect(timeout=GRANT_TIMEOUT), + libei.LibeiUnavailable, GRANT_TIMEOUT + 3.0) + + +def _check_capture_returns_the_portal_bytes(wayland_portal, portal: Portal) -> str: + """The Screenshot tier, end to end, on hand-marshalled D-Bus. + + Nothing here is mocked: a real ``dbus-daemon`` routes real messages that + :mod:`je_auto_control.linux_wayland._dbus_client` marshalled itself, to a + real portal that answers with a signal directed at the caller — which is + the property that made the previous ``gdbus monitor`` design unable to work + at all, and which nothing short of a real bus would have shown. + """ + import numpy + import cv2 + + payload = wayland_portal.capture_png(timeout=15.0) + _require(payload.startswith(b"\x89PNG"), + f"the portal tier returned {payload[:8]!r}, which is not a PNG") + decoded = cv2.imdecode(numpy.frombuffer(payload, dtype=numpy.uint8), + cv2.IMREAD_COLOR) + _require(decoded is not None, "the portal's PNG did not decode") + height, width = decoded.shape[:2] + _require((width, height) == SHOT_SIZE, + f"decoded {width}x{height}, expected {SHOT_SIZE}") + # OpenCV decodes to BGR; the mock painted RGB. + _require(tuple(int(value) for value in decoded[0][0]) == SHOT_RGB[::-1], + f"the first pixel is {decoded[0][0]}, not {SHOT_RGB[::-1]}") + shot = portal.recorded().get("shot_path", "") + _require(bool(shot), "the portal never recorded where it wrote the capture") + expected = os.path.join(_runtime_dir(), SHOT_NAME) + _require(shot == expected, + f"the portal wrote {shot!r}, not the {expected!r} it was told to") + _require(not os.path.exists(expected), + f"the portal's file at {expected!r} was left behind") + return (f"{len(payload)} bytes, decoded {width}x{height}, " + f"and {SHOT_NAME!r} was cleaned up") + + +def _check_capture_refused(wayland_portal, needle: str, + timeout: float = 15.0) -> str: + """A capture that cannot succeed must say why, not return half an image.""" + reason = _refused(lambda: wayland_portal.capture_png(timeout=timeout), + wayland_portal.AutoControlScreenException, timeout + 3.0) + _require(needle in reason, + f"the failure did not mention {needle!r}: {reason}") + return reason + + +# --- scenarios ------------------------------------------------------------ + + +def _run_grant_scenario(portal: Portal, server, libei, oeffis, + wayland_portal) -> None: + """Everything that needs a portal which says yes.""" + backend = check("libei completes its handshake over a portal-obtained fd", + lambda: _connected_through_the_portal(libei)) + check("the portal dance runs the four calls in the prescribed order", + lambda: _check_call_order(portal)) + check("SelectDevices is asked for keyboard and pointer, nothing wider", + lambda: _check_device_mask(portal, oeffis)) + check("the EIS fd is a live socket the caller owns and must close", + lambda: _check_fd_is_live_and_owned(oeffis)) + if backend is None: + return + check("the portal's descriptor carries a real EI session", + lambda: _check_session_reaches_eis(server)) + check("input emitted through the portal path reaches the EIS server", + lambda: _check_input_over_the_portal_fd(backend, server)) + check("disconnect() revokes the grant instead of leaving it open", + lambda: _check_disconnect_ends_the_grant(backend, server, portal)) + check("the Screenshot tier reaches a real portal and returns real pixels", + lambda: _check_capture_returns_the_portal_bytes( + wayland_portal, portal)) + + +def _run_refusal_scenarios(socket_path: str, libei, oeffis) -> None: + """Every way a portal says no, and this project's answer to each.""" + with Portal(socket_path, behaviour="deny"): + check("a dismissed consent dialog is refused, not worked around", + lambda: _check_portal_refuses(oeffis)) + check("a refused portal reaches the caller as LibeiUnavailable", + lambda: _check_backend_surfaces_the_refusal(libei)) + with Portal(socket_path, behaviour="stall"): + check("a consent dialog left open times out on this project's clock", + lambda: _check_open_dialog_times_out(oeffis)) + with Portal(socket_path, behaviour="no-fd"): + check("a portal that withholds the descriptor fails closed", + lambda: _check_portal_refuses(oeffis)) + with Portal(socket_path, behaviour="close"): + check("a portal that closes the session fails closed", + lambda: _check_portal_refuses(oeffis)) + with Portal(socket_path, version=1): + check("a portal too old to have ConnectToEIS fails closed", + lambda: _check_portal_refuses(oeffis)) + + +def _run_capture_scenarios(socket_path: str, wayland_portal) -> None: + """Every way the Screenshot portal ends without an image.""" + with Portal(socket_path, screenshot="deny"): + check("a dismissed screenshot dialog is named as dismissed", + lambda: _check_capture_refused(wayland_portal, "dismissed")) + with Portal(socket_path, screenshot="stall"): + check("a screenshot dialog left open times out on our clock", + lambda: _check_capture_refused( + wayland_portal, "did not answer", REFUSAL_TIMEOUT)) + with Portal(socket_path, screenshot="no-uri"): + check("a success carrying no image URI is treated as a failure", + lambda: _check_capture_refused(wayland_portal, "no image URI")) + with Portal(socket_path, screenshot="not-a-file"): + check("a URI that is not a local file is refused", + lambda: _check_capture_refused(wayland_portal, "non-file URI")) + + +def main() -> int: + """Run every check and return the number that failed.""" + print("=" * 72) + print("AutoControl RemoteDesktop portal — against a real liboeffis") + print("=" * 72) + + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from eis_server import RecordingEisServer, load_symbols + from je_auto_control.linux_wayland import libei, oeffis + from je_auto_control.linux_wayland import portal as wayland_portal + + if load_symbols() is None: + print("libeis.so.* not found — install libeis1. Nothing to verify.") + return 1 + + bus = SessionBus() + print(f" session bus at {bus.start()}") + + socket_path = _fresh_socket_path("eis-portal-verify") + server = RecordingEisServer(socket_path) + server.start() + print(f" EIS server listening at {socket_path}") + print("-" * 72) + + try: + check("liboeffis resolves and every entry point binds", + lambda: _check_liboeffis_binds(oeffis)) + check("no portal on the bus is refused rather than waited on", + lambda: _check_no_portal_is_refused(oeffis)) + with Portal(socket_path) as portal: + _run_grant_scenario(portal, server, libei, oeffis, wayland_portal) + _run_refusal_scenarios(socket_path, libei, oeffis) + _run_capture_scenarios(socket_path, wayland_portal) + finally: + server.stop() + bus.stop() + + if server.error is not None: + print(f" NOTE: the server thread ended with {server.error!r}") + + failed = [name for name, ok in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/seat_verify.py b/docker/seat_verify.py new file mode 100644 index 00000000..526aadbb --- /dev/null +++ b/docker/seat_verify.py @@ -0,0 +1,510 @@ +"""Verify where ydotool's absolute move actually puts the cursor. + +The other four verification images each answer one half of the Wayland input +story and stop at the same wall. ``docker/Dockerfile.wayland`` runs a real +wlroots compositor but consumes no input devices; ``docker/Dockerfile.ydotool`` +reads ydotool's events straight off the kernel with no compositor at all. So +what ``mousemove --absolute`` *puts on the wire* is settled, and what a +compositor *does with it* was recorded in ``Progress.md`` as needing a VM with +"a compositor that consumes libinput devices". + +It does not need one, for the third time in this project's history. wlroots +takes ``WLR_BACKENDS=headless,libinput``: the outputs stay virtual while the +input side is the real libinput backend, and libseat's builtin backend opens +the devices directly once ``SEATD_VTBOUND=0`` stops it reaching for a VT that +a container has no business owning. ydotoold's uinput device is then an +ordinary seat device like any mouse, and ``grim -c`` composites the cursor +into a screenshot — so the compositor answers, in layout coordinates, the +question this file exists to ask: + + **Does ``ydotool mousemove --absolute -x X -y Y`` put the cursor at layout + ``(X, Y)``?** + +No, twice over: + +* Its origin is the corner the compositor clamps to, which is the top-left of + the *output layout*. That is layout ``(0, 0)`` only while every output sits + at a non-negative position; on the layout every desktop with a monitor left + of the primary one has, the two differ by the layout origin. That is the + translation :func:`je_auto_control.linux_wayland.mouse._ydotool_point` now + applies, and this file is where the number comes from. +* The displacement is relative motion, so the compositor's pointer + acceleration scales it. Under libinput's default adaptive profile the + cursor moves twice as far from that corner as asked. ydotool's own + ``--help`` says "You need to disable mouse speed acceleration for correct + absolute movement"; this measures what ignoring that costs. + +**What a screenshot can see is the cursor's image, not its hotspot**, and the +two differ by whatever offset the cursor theme declares. Nothing here depends +on that offset: every claim is a difference between two captures, a distance +that the offset cancels out of, or an output the cursor is unambiguously +inside. The one absolute reading — that a move to ``(0, 0)`` draws the cursor +flush into the layout's very first pixel — is the clamp pinning it there, +which is exactly the claim being made. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import json +import os +import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell +import sys +import tempfile +import time +import traceback +from typing import Any, Callable, List, Optional, Tuple + +import numpy +from PIL import Image + +_results: List[Tuple[str, bool]] = [] + +Point = Tuple[int, int] + +#: The solid colour ``entrypoint-seat.sh`` paints each output. Anything else +#: in a capture is the cursor: nothing else is ever on screen. Keyed by name +#: because sway lists its outputs in whatever order it likes. +OUTPUT_COLOURS = { + "HEADLESS-1": (0x12, 0x34, 0x56), + "HEADLESS-2": (0xAB, 0xCD, 0xEF), +} + +#: How long to let a move settle before capturing. ydotool's own start delay +#: is 100 ms; the rest is the compositor's next frame. +_SETTLE_SECONDS = 0.6 + +#: The cursor image is drawn from its hotspot by at most this many pixels on +#: either axis, so a reading is "the same pixel" within it. Only used where a +#: check compares a drawn position against a requested coordinate directly; +#: every other check is a difference, which the offset cancels out of. +_CURSOR_IMAGE_SLACK = 2 + +#: libinput's maximum acceleration factor for the adaptive profile at the +#: default speed of 0. ``--absolute`` sends both of its events in a single +#: frame, so the velocity saturates the profile and the factor lands here. +_ADAPTIVE_MAX_FACTOR = 2 + + +def _scratch_dir() -> str: + """A private directory for the capture, created 0700 if it is not there. + + The image sets ``XDG_RUNTIME_DIR``; without it — running this by hand — + a fresh ``mkdtemp``, rather than a predictable name under the ``/tmp`` + root that any user on the host can create entries in. + """ + runtime = os.environ.get("XDG_RUNTIME_DIR") + if runtime: + os.makedirs(runtime, mode=0o700, exist_ok=True) + return runtime + return tempfile.mkdtemp(prefix="autocontrol-seat-verify-") + + +CAPTURE = os.path.join(_scratch_dir(), "seat-capture.png") + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +def _run(argv: List[str], *, timeout: float = 10.0) -> str: + """Run a tool from this image and return its stdout.""" + # argv is assembled from literals in this file; no shell, no user input. + completed = subprocess.run(argv, check=True, # nosec B603 # nosemgrep + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return completed.stdout.decode("utf-8", errors="replace") + + +def _minus(left: Point, right: Point) -> Point: + return (left[0] - right[0], left[1] - right[1]) + + +def _plus(left: Point, right: Point) -> Point: + return (left[0] + right[0], left[1] + right[1]) + + +# -------------------------------------------------------------------------- +# The layout, straight out of the compositor +# -------------------------------------------------------------------------- + +class Layout: + """The output layout sway reports, as this verification's ground truth. + + Read from sway's IPC rather than from AutoControl's own ``wlr-randr`` + parser, so that agreeing with it is a check rather than a tautology. + """ + + def __init__(self, outputs: List[Tuple[str, int, int, int, int]]) -> None: + _require(bool(outputs), "sway reported no active outputs") + self.outputs = outputs + self.rects = [rect for _, *rect in outputs] + self.origin: Point = (min(x for x, _, _, _ in self.rects), + min(y for _, y, _, _ in self.rects)) + + def output_at(self, x: int, y: int) -> Optional[str]: + """Name of the output containing ``(x, y)``, or None.""" + for name, left, top, width, height in self.outputs: + if left <= x < left + width and top <= y < top + height: + return name + return None + + def far_from(self, x: int, y: int) -> Point: + """A point on another output, to park the cursor well out of the way.""" + for name, left, top, _, _ in self.outputs: + if name != self.output_at(x, y): + return (left + 600, top + 500) + return (x + 400, y + 400) + + def __str__(self) -> str: + return " + ".join(f"{name} {w}x{h}@({x},{y})" + for name, x, y, w, h in self.outputs) + + +def read_layout() -> Layout: + """Ask sway for its outputs.""" + outputs = json.loads(_run(["swaymsg", "-r", "-t", "get_outputs"])) + return Layout([ + (str(item["name"]), + int(item["rect"]["x"]), int(item["rect"]["y"]), + int(item["rect"]["width"]), int(item["rect"]["height"])) + for item in outputs if item.get("active") + ]) + + +def pointer_devices() -> List[str]: + """Names of every pointer sway's libinput backend is holding.""" + inputs = json.loads(_run(["swaymsg", "-r", "-t", "get_inputs"])) + return [item.get("name", "") for item in inputs + if item.get("type") == "pointer"] + + +def set_pointer_acceleration(profile: str, speed: str) -> None: + """Reconfigure every pointer, then let the change settle.""" + _run(["swaymsg", f"input type:pointer accel_profile {profile}"]) + _run(["swaymsg", f"input type:pointer pointer_accel {speed}"]) + time.sleep(0.2) + + +# -------------------------------------------------------------------------- +# Where the cursor was drawn, in layout coordinates +# -------------------------------------------------------------------------- + +def drawn_cursor(layout: Layout) -> Point: + """Layout coordinate of the drawn cursor's top-left pixel. + + ``grim -c`` composites the cursor into a capture of the whole layout, + whose first pixel is the layout origin. The outputs carry two flat + colours, so every other pixel belongs to the cursor. + """ + _run(["grim", "-c", CAPTURE]) + with Image.open(CAPTURE) as raw: + frame = numpy.asarray(raw.convert("RGB")) + background = numpy.zeros(frame.shape[:2], dtype=bool) + for colour in OUTPUT_COLOURS.values(): + background |= (frame == numpy.array(colour, dtype=frame.dtype)).all(-1) + rows, columns = numpy.nonzero(~background) + _require(rows.size > 0, + "no cursor was drawn: the capture is entirely background, so " + "either the compositor has no cursor theme or grim -c did not " + "composite one") + return _plus(layout.origin, (int(columns.min()), int(rows.min()))) + + +# -------------------------------------------------------------------------- +# The two ways to ask for a move +# -------------------------------------------------------------------------- + +def ydotool_absolute(x: int, y: int) -> None: + """The raw CLI call, with no translation of any kind.""" + _run(["ydotool", "mousemove", "--absolute", "-x", str(x), "-y", str(y)]) + + +def autocontrol_set_position(x: int, y: int) -> None: + """The backend's own entry point, translation included.""" + from je_auto_control.linux_wayland import mouse + mouse.set_position(x, y) + + +def land(move: Callable[[int, int], None], layout: Layout, + point: Point) -> Point: + """Ask ``move`` for ``point`` and report where the cursor was drawn.""" + move(*point) + time.sleep(_SETTLE_SECONDS) + return drawn_cursor(layout) + + +# -------------------------------------------------------------------------- +# Checks +# -------------------------------------------------------------------------- + +def _seat_holds_the_device(names: List[str]) -> str: + _require(any("ydotool" in name.lower() for name in names), + f"sway is holding no ydotool pointer; it has {names}. Without " + "one the libinput backend never picked the device up and " + "nothing below would be measuring a compositor at all.") + return ", ".join(names) + + +def _origin_is_the_layout_corner(layout: Layout) -> str: + """``--absolute 0 0`` drives the cursor into the layout's first pixel. + + The clamp holds it there and clips the image against the edge, so the + drawn position is the corner itself with no theme offset in the way. + """ + drawn = land(ydotool_absolute, layout, (0, 0)) + _require(drawn == layout.origin, + f"--absolute -x 0 -y 0 drew the cursor at {drawn}, expected it " + f"flush into the layout corner {layout.origin}") + return f"(0, 0) -> {drawn}, the layout's first pixel" + + +def _moves_one_pixel_per_pixel(layout: Layout, near: Point, + far: Point) -> str: + """With acceleration off, the displacement is the one that was asked for.""" + drawn_near = land(ydotool_absolute, layout, near) + drawn_far = land(ydotool_absolute, layout, far) + _require(_minus(drawn_far, drawn_near) == _minus(far, near), + f"asking for {near} then {far} moved the cursor " + f"{_minus(drawn_far, drawn_near)}, expected {_minus(far, near)}") + return f"{near} -> {far} moved {_minus(drawn_far, drawn_near)}" + + +def _counts_from_the_corner(layout: Layout, point: Point) -> str: + """The requested offset is measured from the corner, not from (0, 0).""" + drawn = land(ydotool_absolute, layout, point) + expected = _plus(layout.origin, point) + distance = _minus(drawn, expected) + _require(max(abs(distance[0]), abs(distance[1])) <= _CURSOR_IMAGE_SLACK, + f"--absolute -x {point[0]} -y {point[1]} drew the cursor at " + f"{drawn}, which is {distance} from the corner plus the offset " + f"{expected}") + return f"{point} -> {drawn}, corner + {point} within {distance}" + + +def _an_untranslated_zero_misses_its_monitor(layout: Layout) -> str: + """The failure this translation exists to prevent, made concrete.""" + if layout.origin == (0, 0): + return "not applicable: this layout's corner is layout (0, 0)" + drawn = land(ydotool_absolute, layout, (0, 0)) + reached = layout.output_at(*drawn) + intended = layout.output_at(0, 0) + _require(reached is not None and intended is not None + and reached != intended, + f"expected an untranslated (0, 0) to reach a different output " + f"than layout (0, 0); it drew at {drawn} on output {reached} " + f"while layout (0, 0) is on output {intended}") + return (f"reaches output {reached} at {drawn}, while layout (0, 0) is on " + f"output {intended} — {abs(layout.origin[0])} px apart") + + +def _acceleration_scales_the_move(layout: Layout, near: Point, + far: Point) -> str: + """Under the default profile the displacement is not the one asked for.""" + drawn_near = land(ydotool_absolute, layout, near) + drawn_far = land(ydotool_absolute, layout, far) + moved = _minus(drawn_far, drawn_near) + asked = _minus(far, near) + _require(moved != asked, + f"asking for {near} then {far} moved the cursor exactly {moved} " + "under the default adaptive profile, so this image is no longer " + "measuring pointer acceleration at all") + return f"asked {asked}, moved {moved}" + + +def _acceleration_factor_is_two(layout: Layout, near: Point, + far: Point) -> str: + """Pin today's number, so a libinput change is loud rather than silent.""" + drawn_near = land(ydotool_absolute, layout, near) + drawn_far = land(ydotool_absolute, layout, far) + moved = _minus(drawn_far, drawn_near) + asked = _minus(far, near) + expected = (asked[0] * _ADAPTIVE_MAX_FACTOR, + asked[1] * _ADAPTIVE_MAX_FACTOR) + _require(moved == expected, + f"expected libinput's adaptive profile to saturate at " + f"{_ADAPTIVE_MAX_FACTOR}x — {asked} becoming {expected} — but " + f"the cursor moved {moved}") + return f"{asked} became {moved}, {_ADAPTIVE_MAX_FACTOR}x on both axes" + + +def _backend_agrees_with_the_compositor(layout: Layout) -> str: + """AutoControl reads the same origin the compositor reports.""" + from je_auto_control.linux_wayland import screen + reported = screen.layout_origin() + _require(reported == layout.origin, + f"screen.layout_origin() says {reported}, sway says " + f"{layout.origin}") + return f"{reported}" + + +def _set_position_subtracts_exactly_the_origin(layout: Layout, + point: Point) -> str: + """The backend's translation is the layout origin and nothing else. + + Comparing the two paths against each other rather than against a + coordinate keeps the cursor theme out of it entirely: whatever offset the + drawn image carries is the same in both captures. + """ + through_backend = land(autocontrol_set_position, layout, point) + raw = land(ydotool_absolute, layout, _minus(point, layout.origin)) + _require(through_backend == raw, + f"set_position{point} drew at {through_backend} while the raw " + f"CLI asked for {_minus(point, layout.origin)} drew at {raw}") + return f"set_position{point} == --absolute{_minus(point, layout.origin)}" + + +def _set_position_lands_on_the_layout_pixel(layout: Layout, + point: Point) -> str: + """Ask in layout coordinates, land on that pixel.""" + drawn = land(autocontrol_set_position, layout, point) + distance = _minus(drawn, point) + _require(max(abs(distance[0]), abs(distance[1])) <= _CURSOR_IMAGE_SLACK, + f"mouse.set_position{point} drew the cursor at {drawn}, " + f"{distance} away") + _require(layout.output_at(*drawn) == layout.output_at(*point), + f"mouse.set_position{point} landed on output " + f"{layout.output_at(*drawn)}, not {layout.output_at(*point)}") + return f"{point} -> drawn at {drawn}" + + +def _software_cursor_lands_in_the_capture(layout: Layout, + point: Point) -> str: + """Record that a capture asked for no cursor contains one anyway. + + ``grim`` is invoked without ``-c`` here and everywhere else in the + project, so the request is right; what comes back is not. wlroots draws a + *software* cursor whenever the backend has no cursor plane — always on + headless, and on any DRM session where the driver refuses one or the user + set ``WLR_NO_HARDWARE_CURSORS=1`` — and a software cursor is composited + into the output buffer that ``wlr-screencopy`` then hands over. Every + locator, template match and OCR read in this project goes through that + capture, so on such a session the pointer punches a pointer-shaped hole + in whatever it is sitting on. + + Asserted the way it measures rather than the way it ought to be: a + wlroots that starts honouring ``overlay_cursor`` for software cursors + will fail this check, which is the notification this project wants. + """ + from je_auto_control.linux_wayland import screen + land(autocontrol_set_position, layout, point) + under_cursor = tuple(screen.get_pixel(*point)) + land(autocontrol_set_position, layout, layout.far_from(*point)) + uncovered = tuple(screen.get_pixel(*point)) + named = layout.output_at(*point) + _require(uncovered == OUTPUT_COLOURS[named], + f"with the pointer parked elsewhere, get_pixel{point} returned " + f"{uncovered} rather than {named}'s {OUTPUT_COLOURS[named]}") + _require(under_cursor != uncovered, + f"get_pixel{point} returned {uncovered} with the pointer both " + "on that pixel and away from it. If this compositor has started " + "honouring the no-overlay request for software cursors, the " + "caveat recorded in docs/CAPABILITY_MATRIX.md and the READMEs " + "can go, along with cursor_may_be_captured in the diagnostics " + "bundle.") + return (f"{under_cursor} under the pointer vs {uncovered} without it — " + "the capture is not cursor-free") + + +def _input_and_capture_address_one_pixel(layout: Layout, point: Point) -> str: + """``set_position`` and ``get_pixel`` have to mean the same point. + + The pointer is parked elsewhere before the pixel is read: on a compositor + drawing a software cursor the capture would otherwise return the cursor's + own colour, which is a different measurement from this one. + """ + from je_auto_control.linux_wayland import screen + drawn = land(autocontrol_set_position, layout, point) + reached = layout.output_at(*drawn) + land(autocontrol_set_position, layout, layout.far_from(*point)) + colour = tuple(screen.get_pixel(*point)) + named = layout.output_at(*point) + _require(named is not None, f"{point} is on no output") + _require(reached == named, + f"the pointer reached {reached} while the pixel was read on " + f"{named}") + _require(colour == OUTPUT_COLOURS[named], + f"get_pixel{point} returned {colour}, but {named} is painted " + f"{OUTPUT_COLOURS[named]}") + return f"both reach {named}, painted {colour}" + + +# -------------------------------------------------------------------------- + +def _interior_points(layout: Layout) -> List[Point]: + """One point per output, far enough inside that no edge can clip it.""" + return [(x + 120, y + 90) for x, y, _, _ in layout.rects] + + +def main() -> int: + os.environ.setdefault("JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", "cli") + + layout = read_layout() + print(f"layout: {layout} origin {layout.origin}\n") + + check("sway's libinput backend is holding the ydotool device", + lambda: _seat_holds_the_device(pointer_devices())) + + # Everything about the origin is measured with acceleration switched off, + # because a scaled move cannot show where it counted from. + set_pointer_acceleration("flat", "0") + + check("--absolute counts from the layout corner, not layout (0, 0)", + lambda: _origin_is_the_layout_corner(layout)) + check("with acceleration off the move is one pixel per pixel", + lambda: _moves_one_pixel_per_pixel(layout, (50, 50), (600, 400))) + check("a 500x300 request lands 500x300 from that corner", + lambda: _counts_from_the_corner(layout, (500, 300))) + check("an untranslated (0, 0) misses the monitor it names", + lambda: _an_untranslated_zero_misses_its_monitor(layout)) + + check("screen.layout_origin() matches the compositor's own layout", + lambda: _backend_agrees_with_the_compositor(layout)) + for point in _interior_points(layout): + check(f"set_position{point} subtracts exactly the layout origin", + lambda p=point: _set_position_subtracts_exactly_the_origin( + layout, p)) + check(f"set_position{point} lands on that layout pixel", + lambda p=point: _set_position_lands_on_the_layout_pixel( + layout, p)) + check("set_position and get_pixel address the same monitor", + lambda: _input_and_capture_address_one_pixel( + layout, _interior_points(layout)[0])) + check("a software cursor lands in a capture that asked for none", + lambda: _software_cursor_lands_in_the_capture( + layout, _interior_points(layout)[0])) + + # And with the compositor's default profile back, what the option name + # promises stops being true at all. + set_pointer_acceleration("adaptive", "0") + check("the default profile scales the move, so it is not absolute", + lambda: _acceleration_scales_the_move(layout, (50, 50), (300, 200))) + check("that scaling is libinput's 2x adaptive ceiling today", + lambda: _acceleration_factor_is_two(layout, (50, 50), (300, 200))) + + failed = [name for name, ok in _results if not ok] + print("\n" + "=" * 70) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" failed: {name}") + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/wayland_verify.py b/docker/wayland_verify.py new file mode 100644 index 00000000..c074a33a --- /dev/null +++ b/docker/wayland_verify.py @@ -0,0 +1,389 @@ +"""Verify AutoControl's Wayland backend against a real wlroots compositor. + +Runs inside a headless sway session (see ``Dockerfile.wayland``). Everything +the unit tests can only assert against a mock is checked here against the +compositor itself: the argv grim actually accepts, the geometry string it +actually honours, the output format wlr-randr actually prints, and whether +the whole ``screenshot()`` -> ``screen_grabber`` -> ``capture`` -> +``grab_image`` chain really returns the pixels on screen. + +Ground truth comes from ``swaymsg -t get_outputs`` — the compositor's own +report — rather than from an assumption about how sway lays two headless +outputs out. That matters: sway puts HEADLESS-2 at x=0 and HEADLESS-1 to its +right, so anything keyed on list order rather than on the reported geometry +tests the wrong pixel. The two outputs are painted different solid colours so +a region grab has something to get *wrong*: on a uniform screen any rectangle +looks correct. + +The entrypoint runs this twice, against two sway configs. The second one +moves HEADLESS-1 to ``position -1280 0``, which is the layout a desktop has +whenever a monitor sits left of (or above) the primary one: the capture then +starts at a negative coordinate, and everything that maps a pixel to a screen +coordinate has to subtract that origin rather than assume ``(0, 0)``. Nothing +below is written for one layout or the other — every coordinate is derived +from what the compositor reports. + +Exit status is the number of failed checks, so the container's exit code +says whether this passed. +""" +from __future__ import annotations + +import json +import os +import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell +import sys +import tempfile +import traceback +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Tuple + +# Painted onto the named outputs by the sway config. Deliberately asymmetric +# in every channel so a red/blue swap cannot pass. Which one ends up on the +# left is sway's business, so everything below keys on the *name*. +OUTPUT_COLOURS = { + "HEADLESS-1": (0x12, 0x34, 0x56), + "HEADLESS-2": (0xAB, 0xCD, 0xEF), +} + +_results: List[Tuple[str, bool, str]] = [] + + +@dataclass(frozen=True) +class Layout: + """The compositor's output layout, as this run found it.""" + + origin_x: int + origin_y: int + width: int + height: int + outputs: List[dict] + + @property + def left(self) -> dict: + """The left-most output.""" + return self.outputs[0] + + @property + def right(self) -> dict: + """The right-most output (the left-most one when there is only one).""" + return self.outputs[-1] + + @property + def multi(self) -> bool: + """Whether there is more than one output to tell apart.""" + return len(self.outputs) > 1 + + def colour(self, output: dict) -> Tuple[int, int, int]: + """The solid colour sway was told to paint ``output``.""" + return OUTPUT_COLOURS[output["name"]] + + def image_point(self, x: int, y: int) -> Tuple[int, int]: + """Layout coordinate ``(x, y)`` as a pixel index into a full capture.""" + return (x - self.origin_x, y - self.origin_y) + + def inside(self, output: dict, dx: int = 10, dy: int = 10) -> Tuple[int, int]: + """A layout coordinate ``(dx, dy)`` into ``output``.""" + rect = output["rect"] + return (rect["x"] + dx, rect["y"] + dy) + + +def check(name: str, fn: Callable[[], Any]) -> Any: + """Run one check, record pass/fail, and keep going either way.""" + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: a failed check must not stop the rest + _results.append((name, False, traceback.format_exc(limit=3).strip())) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=3).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True, str(detail))) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def note(message: str) -> None: + print(f" {message}") + + +def sway_outputs() -> List[dict]: + """The compositor's own description of its outputs.""" + raw = subprocess.run( # nosec B603 B607 # nosemgrep + ["swaymsg", "-t", "get_outputs", "-r"], + check=True, stdout=subprocess.PIPE).stdout + return json.loads(raw) + + +def read_layout() -> Layout: + """Describe the live layout, sorted so "left" and "right" mean what they say.""" + outputs = sorted(sway_outputs(), key=lambda o: o["rect"]["x"]) + origin_x = min(o["rect"]["x"] for o in outputs) + origin_y = min(o["rect"]["y"] for o in outputs) + width = max(o["rect"]["x"] + o["rect"]["width"] for o in outputs) - origin_x + height = max(o["rect"]["y"] + o["rect"]["height"] for o in outputs) - origin_y + return Layout(origin_x, origin_y, width, height, outputs) + + +def report_environment(layout: Layout) -> None: + """Print what session this is and what the compositor says it is showing.""" + print(f"WAYLAND_DISPLAY = {os.environ.get('WAYLAND_DISPLAY')!r}") + print(f"XDG_SESSION_TYPE = {os.environ.get('XDG_SESSION_TYPE')!r}") + print(f"DISPLAY = {os.environ.get('DISPLAY')!r} " + f"(absent means no XWayland, so nothing can silently fall back to it)") + for out in layout.outputs: + rect = out["rect"] + print(f"output {out['name']}: {rect['width']}x{rect['height']} " + f"at ({rect['x']},{rect['y']}) " + f"painted {OUTPUT_COLOURS.get(out['name'])}") + print(f"layout = {layout.width}x{layout.height} at " + f"({layout.origin_x},{layout.origin_y})") + print("-" * 72) + + +def check_detection(modules: Dict[str, Any]) -> None: + """The session is seen as Wayland and the capture tier resolves to grim.""" + detect = modules["detect"] + capture = modules["capture"] + screen_grabber = modules["screen_grabber"] + check("session detected as wayland", + lambda: _assert_eq(detect.select_display_server(), "wayland")) + check("capture tier resolves to grim", + lambda: _assert_eq(capture.available_tool(), "grim")) + check("platform wrapper publishes grab_image", + lambda: _assert_true(screen_grabber.backend_grab_image() is not None, + "backend_grab_image() returned None, so every " + "capture would have gone to Pillow/mss")) + + +def check_geometry(screen: Any, layout: Layout) -> None: + """wlr-randr's format, the reported size and the origin it implies.""" + def _wlr_randr_raw(): + raw = subprocess.run( # nosec B603 B607 # nosemgrep + ["wlr-randr"], check=True, + stdout=subprocess.PIPE).stdout.decode() + print(" wlr-randr prints:") + for line in raw.splitlines()[:8]: + print(f" | {line}") + return "captured" + check("wlr-randr runs", _wlr_randr_raw) + check("screen.size() reports the whole layout, not one output", + lambda: _assert_eq(screen.size(), (layout.width, layout.height))) + # On the shifted layout these two differ by 1280: the size is the width + # of the bounding box, never the coordinate of its right edge. + check("layout_origin() matches what the compositor reports", + lambda: _assert_eq(screen.layout_origin(), + (layout.origin_x, layout.origin_y))) + full = check("grab_image() full frame", + lambda: _size_of(screen.grab_image())) + check("full frame matches the layout bounding box", + lambda: _assert_eq(full, (layout.width, layout.height))) + + +def check_pixels(screen: Any, layout: Layout) -> None: + """Colour fidelity, grim's -g geometry and get_pixel, in layout coordinates.""" + image = screen.grab_image() + left_rgb = layout.colour(layout.left) + right_rgb = layout.colour(layout.right) + check(f"left output {layout.left['name']} reads its own colour (RGB order)", + lambda: _assert_eq(image.getpixel( + layout.image_point(*layout.inside(layout.left))), left_rgb)) + if layout.multi: + check(f"right output {layout.right['name']} reads its own colour " + f"(proves the x offset is real)", + lambda: _assert_eq(image.getpixel( + layout.image_point(*layout.inside(layout.right))), right_rgb)) + + rx, ry = layout.inside(layout.right, 5, 5) + region = [rx, ry, rx + 100, ry + 50] + cropped = check("grab_image(region) honours grim -g size", + lambda: _size_of(screen.grab_image(region))) + check("region is 100x50 as asked", lambda: _assert_eq(cropped, (100, 50))) + check("region landed on the right-hand output, not at the origin", + lambda: _assert_eq(screen.grab_image(region).getpixel((0, 0)), + right_rgb)) + lx, ly = layout.inside(layout.left, 5, 5) + check("a region on the left-hand output reads the left colour", + lambda: _assert_eq( + screen.grab_image([lx, ly, lx + 100, ly + 50]).getpixel((0, 0)), + left_rgb)) + + check("get_pixel() on the left-hand output", + lambda: _assert_eq(screen.get_pixel(*layout.inside(layout.left)), + left_rgb)) + if layout.multi: + check("get_pixel() on the right-hand output", + lambda: _assert_eq(screen.get_pixel(*layout.inside(layout.right)), + right_rgb)) + + +def check_public_paths(modules: Dict[str, Any], layout: Layout) -> None: + """The chain every locator, recorder and remote-desktop frame goes through.""" + screen = modules["screen"] + screen_grabber = modules["screen_grabber"] + left_rgb = layout.colour(layout.left) + left_point = layout.image_point(*layout.inside(layout.left)) + + def _screenshot_file(): + from PIL import Image + path = os.path.join( + os.environ.get("XDG_RUNTIME_DIR") + or tempfile.mkdtemp(prefix="autocontrol-wayland-verify-"), + "shot.png") + returned = screen.screenshot(path) + _assert_eq(returned, path) + with Image.open(path) as saved: + return f"{saved.width}x{saved.height} {saved.format}" + check("screen.screenshot() writes a real PNG", _screenshot_file) + + def _public_screenshot(): + import je_auto_control as ac + frame = ac.screenshot() + _assert_eq(frame.shape, (layout.height, layout.width, 3)) + # The wrapper converts to BGR for OpenCV, so the channels reverse. + pixel = tuple(int(v) for v in frame[left_point[1]][left_point[0]]) + _assert_eq(pixel, tuple(reversed(left_rgb))) + return f"shape={frame.shape} BGR pixel={pixel}" + check("je_auto_control.screenshot() returns the real screen in BGR", + _public_screenshot) + + def _pil_screenshot(): + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + return _size_of(pil_screenshot()) + check("pil_screenshot() goes through the backend", _pil_screenshot) + + def _grab_logical(): + from je_auto_control.utils.monitor_layout.logical_frame import grab_logical + frame, ox, oy = grab_logical() + _assert_eq(frame.getpixel(left_point), left_rgb) + # The origin is what a caller adds to a hit found in the frame, so on + # the shifted layout it has to be the negative one — a locator that + # gets (0, 0) here clicks 1280 px to the right of what it matched. + _assert_eq((ox, oy), (layout.origin_x, layout.origin_y)) + return f"{frame.width}x{frame.height} origin=({ox},{oy})" + check("grab_logical() — the locator and OCR capture path", _grab_logical) + + def _mss_shim(): + with screen_grabber.mss_grabber() as sct: + monitors = sct.monitors + _assert_eq((monitors[1]["left"], monitors[1]["top"]), + (layout.origin_x, layout.origin_y)) + shot = sct.grab(monitors[1]) + _assert_eq(shot.size, (layout.width, layout.height)) + # mss consumers read .bgra; four bytes per pixel, blue first. + offset = 4 * (left_point[1] * layout.width + left_point[0]) + _assert_eq(tuple(shot.bgra[offset:offset + 3]), tuple(reversed(left_rgb))) + return f"{len(monitors)} monitors, monitor[1] at {monitors[1]['left']},{monitors[1]['top']}" + check("mss-shaped shim (recorder / WebRTC / MCP path)", _mss_shim) + + +def check_fallback_crop(modules: Dict[str, Any], layout: Layout) -> None: + """The tiers that cannot apply a region themselves, cropped by us. + + grim is the only helper that takes a geometry; gnome-screenshot, + spectacle, the portal and an operator's own command all return the whole + layout and :func:`screen.grab_image` crops afterwards. Pointing the + operator override at grim is what puts a *real* whole-layout PNG through + that crop — and on the shifted layout a crop that forgets the origin + silently returns black padding instead of the left-hand monitor. + """ + capture = modules["capture"] + screen = modules["screen"] + if not layout.multi: + return + os.environ[capture.CAPTURE_COMMAND_ENV] = "grim {output}" + try: + check("operator override tier is selected", + lambda: _assert_eq(capture.available_tool(), + f"${capture.CAPTURE_COMMAND_ENV}")) + check("override returns the whole layout, region unapplied", + lambda: _assert_eq( + (lambda c: (_size_of_png(c.data), c.region_applied))( + capture.grab_png([0, 0, 10, 10])), + ((layout.width, layout.height), False))) + for output in (layout.left, layout.right): + x, y = layout.inside(output, 5, 5) + check(f"cropped region on {output['name']} reads its own colour", + lambda x=x, y=y, output=output: _assert_eq( + screen.grab_image([x, y, x + 100, y + 50]).getpixel((0, 0)), + layout.colour(output))) + check("get_pixel() through the cropping tier", + lambda: _assert_eq(screen.get_pixel(*layout.inside(layout.left)), + layout.colour(layout.left))) + finally: + del os.environ[capture.CAPTURE_COMMAND_ENV] + + +def main() -> int: + print("=" * 72) + print("AutoControl Wayland verification — real sway session") + print("=" * 72) + + layout = read_layout() + report_environment(layout) + + from je_auto_control.linux_wayland import _detect, capture, screen + from je_auto_control.linux_wayland import keyboard as wl_keyboard + from je_auto_control.utils.cv2_utils import screen_grabber + modules = {"detect": _detect, "capture": capture, "screen": screen, + "screen_grabber": screen_grabber} + + check_detection(modules) + check_geometry(screen, layout) + check_pixels(screen, layout) + check_public_paths(modules, layout) + check_fallback_crop(modules, layout) + + def _wtype(): + # Nothing is focused, so the keystrokes go nowhere. What is being + # checked is that wtype accepts this argv and exits cleanly — the + # part the unit tests could only assert against a mock. + wl_keyboard.write("autocontrol") + return "wtype accepted `wtype -- TEXT`" + check("wtype argv is accepted by the real binary", _wtype) + + # --- what this environment cannot answer ----------------------------- + print("-" * 72) + print("NOT verifiable in this container, and why:") + note("ydotool — needs /dev/uinput, and sway's headless backend consumes") + note(" no libinput devices, so an injected event has nowhere to land.") + note("libei / RemoteDesktop portal — xdg-desktop-portal-wlr implements") + note(" ScreenCast and Screenshot but not RemoteDesktop, so there is no") + note(" ConnectToEIS here. Covered instead by docker/eis_verify.py (the") + note(" protocol) and docker/portal_verify.py (the D-Bus handshake).") + + # --- summary --------------------------------------------------------- + failed = [name for name, ok, _ in _results if not ok] + print("=" * 72) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" FAILED: {name}") + print("=" * 72) + return len(failed) + + +def _assert_eq(actual: Any, expected: Any) -> str: + if actual != expected: + raise AssertionError(f"expected {expected!r}, got {actual!r}") + return repr(actual) + + +def _assert_true(value: bool, message: str) -> str: + if not value: + raise AssertionError(message) + return "yes" + + +def _size_of(image: Any) -> Tuple[int, int]: + return (image.width, image.height) + + +def _size_of_png(data: bytes) -> Tuple[int, int]: + from io import BytesIO + + from PIL import Image + with Image.open(BytesIO(data)) as image: + return (image.width, image.height) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docker/ydotool_verify.py b/docker/ydotool_verify.py new file mode 100644 index 00000000..f0ffec9d --- /dev/null +++ b/docker/ydotool_verify.py @@ -0,0 +1,414 @@ +"""Verify AutoControl's ydotool argv against the events the kernel really gets. + +The Wayland image answers the capture half and the EIS image answers libei's +half. Both close by naming the same remaining gap — ydotool, which "needs +/dev/uinput and a seat that consumes it" — and that gap was recorded as +needing a GNOME VM. + +It does not need one. A seat is required for an injected event to *arrive +somewhere*, but not for it to be *observed*: ydotoold creates an ordinary +uinput device, the kernel exposes it as ``/dev/input/eventN``, and reading +that node gives back the exact ``input_event`` structs ydotool wrote. No +compositor is involved, so a container with ``/dev/uinput`` and the input +device cgroup can settle every claim this backend makes about the CLI: + + * ``click`` bitmasks — ``0xc0`` / ``0xc1`` / ``0xc2`` really are BTN_LEFT / + BTN_RIGHT / BTN_MIDDLE, and the split edges ``0x40`` / ``0x80`` really do + send a press without a release and a release without a press, which is + the whole basis of :func:`press_mouse` and drag; + * ``mousemove --absolute`` — what it puts on the wire, which is not what + the option name suggests; + * ``mousemove --wheel`` signs — the direction this project assumed from the + kernel's ``REL_WHEEL`` convention and never measured, plus that the axes + are not swapped; + * ``key CODE:STATE`` — numeric evdev codes, both edges. + +It also pins the reason :mod:`je_auto_control.linux_wayland._ydotool_cli` +exists: the same argv is replayed against whatever ydotool is installed, and +the legacy 0.1.x CLI answers ``0`` while emitting nothing. + +Exit status is the number of failed checks. +""" +from __future__ import annotations + +import glob +import os +import struct +import subprocess # nosec B404 # reason: argv-list, fixed tool names, no shell +import sys +import time +import traceback +from typing import Any, Callable, Dict, List, Optional, Tuple + +_results: List[Tuple[str, bool]] = [] + +#: ``struct input_event``: two ``long`` for the timeval, then type/code/value. +_EVENT_FORMAT = "llHHi" +_EVENT_SIZE = struct.calcsize(_EVENT_FORMAT) + +EV_SYN, EV_KEY, EV_REL, EV_ABS = 0x00, 0x01, 0x02, 0x03 + +BTN_LEFT, BTN_RIGHT, BTN_MIDDLE = 272, 273, 274 +KEY_A, KEY_LEFTCTRL = 30, 29 +REL_X, REL_Y, REL_HWHEEL, REL_WHEEL = 0x00, 0x01, 0x06, 0x08 + +#: Input event nodes are character major 13, minor 64 + N. +_INPUT_MAJOR = 13 +_INPUT_MINOR_BASE = 64 + +#: ydotoold names its device this; matched case-insensitively. +_DEVICE_NAME_FRAGMENT = "ydotool" + +#: Long enough for ydotool's own default 100 ms start delay plus slack. +_SETTLE_SECONDS = 0.45 + + +def check(name: str, fn: Callable[[], Any]) -> Any: + try: + detail = fn() + except Exception: # noqa: BLE001 # reason: one failed check must not stop the rest + _results.append((name, False)) + print(f"FAIL {name}") + print(" " + traceback.format_exc(limit=4).strip().replace( + "\n", "\n ")) + return None + _results.append((name, True)) + print(f"ok {name}" + (f" — {detail}" if detail else "")) + return detail + + +def _require(condition: bool, message: str) -> None: + if not condition: + raise AssertionError(message) + + +# -------------------------------------------------------------------------- +# Reading the virtual device +# -------------------------------------------------------------------------- + +def _device_names() -> Dict[str, str]: + """Map ``eventN`` -> device name, straight out of sysfs.""" + names = {} + for path in glob.glob("/sys/class/input/event*"): + try: + with open(os.path.join(path, "device", "name"), + encoding="utf-8") as handle: + names[os.path.basename(path)] = handle.read().strip() + except OSError: + continue + return names + + +class VirtualDevice: + """Every ydotoold input node, opened non-blocking and read as a group. + + ydotoold may expose more than one node (a stale daemon from an earlier + run leaves one behind), and which one carries a given event is not + something this project should depend on, so they are drained together. + """ + + def __init__(self) -> None: + self._fds: Dict[str, int] = {} + + def open_all(self) -> List[str]: + os.makedirs("/dev/input", exist_ok=True) + opened = [] + for node_name, device_name in _device_names().items(): + if _DEVICE_NAME_FRAGMENT not in device_name.lower(): + continue + if node_name in self._fds: + continue + path = f"/dev/input/{node_name}" + if not os.path.exists(path): + minor = _INPUT_MINOR_BASE + int(node_name.replace("event", "")) + os.mknod(path, 0o600 | 0o020000, + os.makedev(_INPUT_MAJOR, minor)) + self._fds[node_name] = os.open(path, os.O_RDONLY | os.O_NONBLOCK) + opened.append(path) + return opened + + def drain(self) -> List[Tuple[int, int, int]]: + """Return every pending ``(type, code, value)``, SYN frames dropped.""" + events: List[Tuple[int, int, int]] = [] + for fd in self._fds.values(): + events.extend(self._drain_one(fd)) + return events + + @staticmethod + def _drain_one(fd: int) -> List[Tuple[int, int, int]]: + """Read one input node dry. + + One OSError branch covers both endings: BlockingIOError is a subclass + of it and means nothing more is queued, and anything else means the + node went away — either way this descriptor is done for now. + """ + events: List[Tuple[int, int, int]] = [] + while True: + try: + data = os.read(fd, _EVENT_SIZE) + except OSError: + break + if not data or len(data) < _EVENT_SIZE: + break + _, _, etype, code, value = struct.unpack(_EVENT_FORMAT, data) + if etype != EV_SYN: + events.append((etype, code, value)) + return events + + def close(self) -> None: + for fd in self._fds.values(): + try: + os.close(fd) + except OSError: + pass + self._fds.clear() + + +def _emit(device: VirtualDevice, argv: List[str], + ) -> Tuple[int, str, List[Tuple[int, int, int]]]: + """Run one ydotool command and return ``(rc, stderr, events)``.""" + device.drain() + completed = subprocess.run( # nosec B603 # nosemgrep + argv, capture_output=True, timeout=20, check=False, + ) + time.sleep(_SETTLE_SECONDS) + stderr = completed.stderr.decode("utf-8", errors="replace").strip() + return completed.returncode, stderr, device.drain() + + +def _keys(events: List[Tuple[int, int, int]]) -> List[Tuple[int, int]]: + return [(code, value) for etype, code, value in events if etype == EV_KEY] + + +def _rel(events: List[Tuple[int, int, int]]) -> List[Tuple[int, int]]: + return [(code, value) for etype, code, value in events if etype == EV_REL] + + +# -------------------------------------------------------------------------- +# The checks +# -------------------------------------------------------------------------- + +def _button_check(device: VirtualDevice, tool: str, code: str, + expected: List[Tuple[int, int]]) -> str: + rc, stderr, events = _emit(device, [tool, "click", code]) + _require(rc == 0, f"ydotool click {code} exited {rc}: {stderr}") + seen = _keys(events) + _require(seen == expected, + f"click {code} produced {seen}, expected {expected}") + return f"{code} -> {seen}" + + +def _wheel_check(device: VirtualDevice, tool: str, delta_x: int, delta_y: int, + expected: List[Tuple[int, int]]) -> str: + rc, stderr, events = _emit(device, [ + tool, "mousemove", "--wheel", "-x", str(delta_x), "-y", str(delta_y)]) + _require(rc == 0, f"ydotool mousemove --wheel exited {rc}: {stderr}") + seen = [pair for pair in _rel(events) + if pair[0] in (REL_WHEEL, REL_HWHEEL)] + _require(seen == expected, + f"wheel ({delta_x}, {delta_y}) produced {seen}, " + f"expected {expected}") + return f"(-x {delta_x} -y {delta_y}) -> {seen}" + + +def _absolute_move_check(device: VirtualDevice, tool: str) -> str: + """``--absolute`` is relative-with-a-reset, and that is worth pinning. + + ydotool 1.x has no ABS axes on its device. It fakes an absolute move by + sending ``INT32_MIN`` on both axes first — which every compositor clamps + to the top-left corner — and then the target coordinates as a relative + delta from there. So AutoControl's ``set_position`` lands on the pixel it + asked for only because of that clamp, and the reset pair has to be + present: without it the move would be relative to wherever the cursor + already was. + """ + target_x, target_y = 640, 400 + rc, stderr, events = _emit(device, [ + tool, "mousemove", "--absolute", + "-x", str(target_x), "-y", str(target_y)]) + _require(rc == 0, f"ydotool mousemove --absolute exited {rc}: {stderr}") + moves = _rel(events) + _require(len(moves) == 4, + f"expected a reset pair then the target pair, got {moves}") + reset_x, reset_y, move_x, move_y = moves + int32_min = -(2 ** 31) + _require(reset_x == (REL_X, int32_min) and reset_y == (REL_Y, int32_min), + f"expected an INT32_MIN reset on both axes, got " + f"{reset_x}, {reset_y}") + _require(move_x == (REL_X, target_x) and move_y == (REL_Y, target_y), + f"expected the target as a relative delta, got {move_x}, {move_y}") + return "reset to origin, then REL_X 640 / REL_Y 400" + + +def _key_check(device: VirtualDevice, tool: str) -> str: + rc, stderr, events = _emit(device, [tool, "key", "30:1", "30:0"]) + _require(rc == 0, f"ydotool key exited {rc}: {stderr}") + seen = _keys(events) + _require(seen == [(KEY_A, 1), (KEY_A, 0)], + f"key 30:1 30:0 produced {seen}") + return f"30:1 30:0 -> {seen}" + + +def _backend_argv_check(device: VirtualDevice) -> str: + """Drive the real backend functions, not a hand-written argv. + + Everything above proves what ydotool does with a given command line. This + proves AutoControl builds that command line — the two halves are only + worth something together. + """ + os.environ["JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND"] = "cli" + from je_auto_control.linux_wayland import keyboard, mouse + + device.drain() + mouse.click_mouse(mouse.wayland_mouse_right) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [(BTN_RIGHT, 1), (BTN_RIGHT, 0)], + "mouse.click_mouse(right) did not reach the kernel as BTN_RIGHT") + + device.drain() + mouse.press_mouse(mouse.wayland_mouse_left) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [(BTN_LEFT, 1)], + "mouse.press_mouse(left) sent something other than a lone press") + mouse.release_mouse(mouse.wayland_mouse_left) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [(BTN_LEFT, 0)], + "mouse.release_mouse(left) sent something other than a lone " + "release") + + device.drain() + mouse.scroll(1, mouse.wayland_scroll_direction_up) + time.sleep(_SETTLE_SECONDS) + _require(_rel(device.drain()) == [(REL_WHEEL, 1)], + "mouse.scroll(up) did not reach the kernel as REL_WHEEL +1") + + device.drain() + keyboard.hotkey([KEY_LEFTCTRL, KEY_A]) + time.sleep(_SETTLE_SECONDS) + _require(_keys(device.drain()) == [ + (KEY_LEFTCTRL, 1), (KEY_A, 1), (KEY_A, 0), (KEY_LEFTCTRL, 0)], + "keyboard.hotkey did not press in order and release in reverse") + return "click / press / release / scroll / hotkey all land as intended" + + +def _legacy_cli_detection_check(tool: str) -> str: + """The installed CLI must be classified as the modern one. + + This is the sentinel for :mod:`_ydotool_cli`: if a future ydotool changes + its usage banner, the probe starts answering ``unknown``, the guard stops + protecting anyone, and this check is what says so. + """ + from je_auto_control.linux_wayland import _ydotool_cli + + _ydotool_cli.reset_cache() + generation = _ydotool_cli.cli_generation(tool) + _require(generation == _ydotool_cli.MODERN, + f"the probe classified this ydotool as {generation!r}, not " + f"{_ydotool_cli.MODERN!r}; the 0.1.x guard is now blind") + _require(_ydotool_cli.reject_legacy_cli(tool) == tool, + "reject_legacy_cli refused a modern ydotool") + return generation + + +def _start_daemon() -> subprocess.Popen: + # Debian's ydotoold binds $XDG_RUNTIME_DIR/.ydotool_socket, and creates + # neither the directory nor a fallback: without it the daemon still comes + # up and still creates the uinput device, but every client exits 2 with an + # empty stderr. Upstream's default is /tmp/.ydotool_socket, so which path + # is in use is a packaging detail — making the directory is what keeps + # both working. + runtime_dir = os.environ.get("XDG_RUNTIME_DIR") + if runtime_dir: + os.makedirs(runtime_dir, mode=0o700, exist_ok=True) + os.chmod(runtime_dir, 0o700) # nosemgrep # reason: owner-only is the least ydotoold needs + daemon = subprocess.Popen( # nosec B603 B607 # nosemgrep + ["ydotoold"], stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, + ) + deadline = time.monotonic() + 15.0 + while time.monotonic() < deadline: + if any(_DEVICE_NAME_FRAGMENT in name.lower() + for name in _device_names().values()): + time.sleep(0.5) # let the node settle before the first read + return daemon + if daemon.poll() is not None: + raise RuntimeError(f"ydotoold exited {daemon.returncode}") + time.sleep(0.1) + raise RuntimeError("ydotoold created no input device within 15s") + + +def _tool_path() -> str: + from je_auto_control.linux_wayland._detect import binary_path + path = binary_path("ydotool") + if path is None: + raise RuntimeError("ydotool is not on PATH") + return path + + +def main() -> int: + print("AutoControl ydotool verification — real /dev/uinput, no compositor") + print("=" * 70) + + if not os.path.exists("/dev/uinput"): + print("FAIL /dev/uinput is missing. Run the container with " + "--device /dev/uinput (and `modprobe uinput evdev` on the host).") + return 1 + + tool = _tool_path() + daemon: Optional[subprocess.Popen] = None + device = VirtualDevice() + try: + daemon = _start_daemon() + nodes = device.open_all() + if not nodes: + print("FAIL ydotoold's device has no evdev node. The host kernel " + "needs the evdev handler (`modprobe evdev`), and the " + "container needs --device-cgroup-rule 'c 13:* rmw'.") + return 1 + print(f"reading {', '.join(nodes)}\n") + + check("click 0xc0 is BTN_LEFT down then up", + lambda: _button_check(device, tool, "0xc0", + [(BTN_LEFT, 1), (BTN_LEFT, 0)])) + check("click 0x40 is a press with no release", + lambda: _button_check(device, tool, "0x40", [(BTN_LEFT, 1)])) + check("click 0x80 is a release with no press", + lambda: _button_check(device, tool, "0x80", [(BTN_LEFT, 0)])) + check("click 0xc1 is BTN_RIGHT", + lambda: _button_check(device, tool, "0xc1", + [(BTN_RIGHT, 1), (BTN_RIGHT, 0)])) + check("click 0xc2 is BTN_MIDDLE", + lambda: _button_check(device, tool, "0xc2", + [(BTN_MIDDLE, 1), (BTN_MIDDLE, 0)])) + check("mousemove --absolute resets to the origin first", + lambda: _absolute_move_check(device, tool)) + check("wheel +y is REL_WHEEL positive (up, per the kernel convention)", + lambda: _wheel_check(device, tool, 0, 1, [(REL_WHEEL, 1)])) + check("wheel -y is REL_WHEEL negative (down)", + lambda: _wheel_check(device, tool, 0, -1, [(REL_WHEEL, -1)])) + check("wheel +x is REL_HWHEEL positive (right), axes not swapped", + lambda: _wheel_check(device, tool, 2, 0, [(REL_HWHEEL, 2)])) + check("key CODE:STATE sends numeric evdev codes", + lambda: _key_check(device, tool)) + check("the version probe recognises this CLI as modern", + lambda: _legacy_cli_detection_check(tool)) + check("the backend's own argv lands as intended", + lambda: _backend_argv_check(device)) + finally: + device.close() + if daemon is not None: + daemon.terminate() + try: + daemon.wait(timeout=5) + except subprocess.TimeoutExpired: + daemon.kill() + + failed = [name for name, ok in _results if not ok] + print("\n" + "=" * 70) + print(f"{len(_results) - len(failed)}/{len(_results)} checks passed") + for name in failed: + print(f" failed: {name}") + return len(failed) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/CAPABILITY_MATRIX.md b/docs/CAPABILITY_MATRIX.md index e4c7e063..7a18a802 100644 --- a/docs/CAPABILITY_MATRIX.md +++ b/docs/CAPABILITY_MATRIX.md @@ -6,9 +6,9 @@ without a compatibility window. | Capability | Status | Windows | Linux X11 | Linux Wayland | macOS | |---|---|---:|---:|---:|---:| -| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb | partial | implementation | +| Mouse, keyboard, screenshot | stable | CI | CI/Xvfb | CI/sway + libeis | implementation | | JSON executor and variables | stable | CI | CI | CI | platform-neutral | -| Image and anchor locators | beta | CI | CI | screenshot-only | implementation | +| Image and anchor locators | beta | CI | CI | implementation | implementation | | Accessibility locator | beta | CI | backend tests | unavailable | backend tests | | Recorder | beta | CI | implementation | unavailable | unavailable | | Reports, trace, failure bundle | stable | CI | CI | CI | platform-neutral | @@ -21,3 +21,121 @@ without a compatibility window. “Implementation” means code exists but the repository does not currently run a real OS runner for it. It must not be interpreted as a production guarantee. Hardware-backed results and known limitations should be attached to releases. + +Linux Wayland is split: **capture is exercised by CI against a real +compositor; input is exercised by CI against a real EI peer and a real portal.** + +Screen capture runs through the compositor's own tool (`grim` on wlroots, +`gnome-screenshot` on GNOME, `spectacle` on KDE), falling back to +`xdg-desktop-portal` over the session bus, instead of the X11-only Pillow/mss +path — +and `JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND` covers a setup none of those fit. +The `wayland-verification` job in `docker/` runs the whole capture path inside +a headless sway session and checks it against pixels the compositor painted, +which is why this row says CI rather than “implementation”. It runs twice, over +two output layouts: side by side from the origin, and with the left-hand +output at x=-1280 — the layout of any desktop with a monitor left of the +primary one, where the compositor's plane starts at a negative coordinate and +a size, a crop or a located hit that assumes `(0, 0)` is wrong by the width of +that monitor. + +One cross-platform difference falls out of the same job, and it is not one this +project can fix: **a Wayland capture may contain the mouse cursor.** No capture +here passes `grim -c`, so none of them asks for the pointer — but wlroots draws +a *software* cursor whenever the backend has no cursor plane, and a software +cursor is composited into the output buffer, which is the buffer +`wlr-screencopy` hands back. Headless is permanently in that state, and so is a +real desktop whose driver offers no cursor plane or whose user set +`WLR_NO_HARDWARE_CURSORS=1`, a common workaround. Windows' BitBlt and the X11 +Pillow/mss path never include the pointer, so this is a Wayland-only +inconsistency rather than something callers already expect: with the pointer +resting on its target, a locator, a template match or an OCR read sees a +pointer-shaped hole in the middle of it. Both ways out need to know where the +pointer is — move it away and back, or mask around it — and Wayland does not let +a client read the cursor position, so the only source would be an in-process +record that goes stale the moment the user touches their own mouse; masking the +wrong place is worse than a visible cursor. So this is documented rather than +worked around: park the pointer away from the region of interest before +capturing. The `seat-verification` job asserts the behaviour as measured, so if +wlroots ever honours `overlay_cursor` for software cursors, CI goes red and says +so. + +Input is verified in four parts, all of which are CI jobs. + +The `eis-verification` job in `docker/` runs AutoControl's real `libei` sender +against a real EIS server — libeis, over a Unix socket, with no compositor +involved — and reads back off the wire what arrived: the capability and +event-type enum values, the variadic seat bind, the key codes, the absolute +coordinates, the button codes, the scroll unit and sign, and a frame per +emission. It also settles the absolute pointer's coordinate space, which is +where the negative-origin layout above reaches the input half: a region's +offset is part of the coordinate rather than something to subtract, and a +motion landing outside every region is dropped by libei without a return code, +an event or an error — so the sender maps the point into region space and +refuses what no region covers, which is what lets the `ydotool` path take it. + +The `ydotool-verification` job covers the CLI fallback, which the `libei` path +drops to at every failure point. A seat is what makes an injected event arrive +somewhere; it is not what makes one observable, so no compositor is needed: +`ydotoold` creates an ordinary uinput device, the kernel publishes it as +`/dev/input/eventN`, and the job reads the `input_event` structs back off it. +That covers the `click` bitmasks, the split press / release edges drag depends +on, what `mousemove --absolute` really puts on the wire, the wheel signs and +axes, numeric key codes, and — in the last check — the argv the mouse and +keyboard backends build for themselves. + +The `seat-verification` job is where an injected event finally reaches a +compositor, and it settles what `mousemove --absolute` is absolute *to*. That +had been recorded as needing a VM running a desktop that consumes libinput +devices; it needs three environment variables instead. wlroots takes +`WLR_BACKENDS=headless,libinput`, so the outputs stay virtual while the input +half is the real libinput backend; libseat's builtin backend opens the device +without logind; and `SEATD_VTBOUND=0` stops it reaching for a VT no container +owns. `grim -c` then draws the cursor into a screenshot, so the compositor +answers in layout coordinates. Two findings come out of it, over the same two +layouts the capture job uses. The origin `--absolute` counts from is the +top-left of the *output layout*, not layout `(0, 0)` — the same distinction +the capture path already makes, and the reason `set_position` now subtracts +`layout_origin()` before calling ydotool. And the displacement is relative +motion, so the compositor's pointer acceleration scales it: libinput's default +adaptive profile moves the cursor twice as far as asked, which is what +ydotool's own `--help` means by "You need to disable mouse speed acceleration +for correct absolute movement". **The ydotool fallback is therefore only +pixel-accurate on a session whose pointer acceleration is off**; the libei +path is absolute at the protocol level and is unaffected. + +The factor is compositor configuration and no client can read it back, so the +library cannot compensate for it — only the operator knows whether it is off. +`JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL` is how they say so, and it applies to +the ydotool path alone: unset (or set to anything unrecognised, which says so +and falls back) warns once per process and sends the move anyway, `flat` +declares acceleration off and moves silently, and `strict` refuses the move +rather than let a click land somewhere else. + +The `portal-verification` job covers how a client reaches libei on GNOME and +KDE, which is not a socket path but a file descriptor handed over D-Bus at the +end of the `org.freedesktop.portal.RemoteDesktop` session dance. That had been +recorded as needing a real desktop, on the grounds that no container ships a +RemoteDesktop portal — but the portal is a D-Bus interface, so the job owns the +well-known name itself and runs the real `liboeffis` through the real +handshake, ending in a live connection to the same `libeis` server the +`eis-verification` job uses. It settles the call order and the predicted +request paths, the device mask a user would be consenting to, that the +descriptor carries a real EI session, and that input emitted through it is +recorded at the far end. Every refusal is covered too — a dismissed dialog, a +dialog left open, a withheld descriptor, a closed session, a portal too old for +`ConnectToEIS`, no portal at all — each of which has to fail closed on +AutoControl's own clock. + +What is still not covered is the consent dialog as a *dialog*: no user +dismisses anything in CI, so what a real dialog looks like and how long a real +one blocks stay mutter's business. The compositor also refuses global input +recording, key hooks, cursor-position reads and per-window injection outright; +those are Wayland design decisions, not gaps. See `Progress.md`. + +One packaging note that affects users more than any of the above: ydotool 1.0 +replaced its entire command line, and everything this backend builds arrived +in that release. Debian trixie ships no `ydotool` package; bookworm and every +current Ubuntu ship 0.1.8, which answers this argv with exit code 0 and no +events. AutoControl refuses that version up front rather than reporting +success for input it never sent. diff --git a/examples/22_wayland_backend.py b/examples/22_wayland_backend.py index 2b516cd7..ec9f6c12 100644 --- a/examples/22_wayland_backend.py +++ b/examples/22_wayland_backend.py @@ -21,6 +21,21 @@ Recording, global key listening, and per-window event injection are *not* available on Wayland by design — those calls raise ``NotImplementedError`` with a hint pointing at the X11 fallback. + +Two Wayland-only caveats are worth knowing before a script leans on them: + +* ``ydotool mousemove --absolute`` is relative motion the compositor + accelerates, so an absolute move through that fallback is pixel-exact only + where pointer acceleration is off. The factor cannot be read back, so turn + acceleration off for the ydotoold device and say so with + ``JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=flat``; ``=strict`` refuses such a + move rather than mispositioning, and leaving it unset warns once per + process and moves anyway. The libei path is absolute and unaffected. +* A capture may contain the mouse cursor. Nothing here asks for it, but + wlroots composites a *software* cursor into the buffer screen capture hands + back whenever the backend has no cursor plane. Windows and X11 never + include the pointer, so park it away from whatever a locator, a template + match or an OCR read is about to look at. """ from je_auto_control.linux_wayland import ( is_wayland_session, missing_dependencies, select_display_server, diff --git a/je_auto_control/__init__.py b/je_auto_control/__init__.py index f6c48481..4f6fa121 100644 --- a/je_auto_control/__init__.py +++ b/je_auto_control/__init__.py @@ -1269,8 +1269,10 @@ # Cross-platform window manager (headless) from je_auto_control.wrapper.auto_control_window import ( close_window_by_title, find_window, focus_window, foreground_window, - list_windows, minimize_window_by_title, move_window_by_title, - show_window_by_title, wait_for_window, window_rect, + foreground_window_process_id, list_windows, minimize_window_by_title, + minimize_windows_for_process, move_window_by_title, post_click_to_window, + post_key_to_window, show_window_by_title, wait_for_window, + window_process_id, window_rect, windows_for_process_id, ) # Windows-only modules (ctypes.WINFUNCTYPE / Win32 API) — gated so # ``import je_auto_control`` keeps working on macOS / Linux. Kept last @@ -1329,7 +1331,9 @@ def start_autocontrol_gui(*args, **kwargs): "list_windows", "find_window", "focus_window", "wait_for_window", "close_window_by_title", "show_window_by_title", "minimize_window_by_title", "foreground_window", "window_rect", - "move_window_by_title", + "move_window_by_title", "foreground_window_process_id", + "window_process_id", "post_key_to_window", "post_click_to_window", + "windows_for_process_id", "minimize_windows_for_process", # Clipboard "get_clipboard", "set_clipboard", "get_clipboard_image", "set_clipboard_image", diff --git a/je_auto_control/gui/_image_detect_tab.py b/je_auto_control/gui/_image_detect_tab.py new file mode 100644 index 00000000..983ae4e3 --- /dev/null +++ b/je_auto_control/gui/_image_detect_tab.py @@ -0,0 +1,106 @@ +"""Image-detection tab builder (extracted mixin).""" +from PySide6.QtGui import QDoubleValidator +from PySide6.QtWidgets import ( + QCheckBox, QFileDialog, QGridLayout, QLabel, QLineEdit, QMessageBox, + QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.selector import crop_template_to_file +from je_auto_control.wrapper.auto_control_image import ( + locate_all_image, locate_image_center, locate_and_click, +) + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class ImageDetectTabMixin: + """Provides the image-detection tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``). + ``_locate_click`` reuses the auto-click tab's ``mouse_button_combo`` when + the host also mixes in :class:`AutoClickTabMixin`, falling back to the + left button otherwise. + """ + + def _build_image_detect_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Detection inputs; locate/crop commands run from the Actions menu. + grid = QGridLayout() + grid.addWidget(self._tr(QLabel(), "template_image"), 0, 0) + self.img_path_input = QLineEdit() + grid.addWidget(self.img_path_input, 0, 1) + + grid.addWidget(self._tr(QLabel(), "threshold_label"), 1, 0) + self.threshold_input = QLineEdit("0.8") + self.threshold_input.setValidator(QDoubleValidator(0.0, 1.0, 2)) + grid.addWidget(self.threshold_input, 1, 1) + self.draw_check = self._tr(QCheckBox(), "draw_image_check") + grid.addWidget(self.draw_check, 1, 2) + + layout.addLayout(grid) + + layout.addWidget(self._tr(QLabel(), "detection_result")) + self.detect_result_text = QTextEdit() + self.detect_result_text.setReadOnly(True) + layout.addWidget(self.detect_result_text) + tab.setLayout(layout) + return tab + + def _browse_img(self): + path, _ = QFileDialog.getOpenFileName(self, _t("template_image"), "", "Images (*.png *.jpg *.bmp);;All (*)") + if path: + self.img_path_input.setText(path) + + def _crop_template(self): + save_path, _ = QFileDialog.getSaveFileName( + self, _t("crop_template"), "", "PNG (*.png)" + ) + if not save_path: + return + try: + region = crop_template_to_file(save_path, self) + if region is None: + return + self.img_path_input.setText(save_path) + self.detect_result_text.setText(f"Template saved: {save_path} region={region}") + except (OSError, ValueError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _get_detect_params(self): + path = self.img_path_input.text() + if not path: + raise ValueError("Template image path is empty") + threshold = float(self.threshold_input.text() or "0.8") + draw = self.draw_check.isChecked() + return path, threshold, draw + + def _locate_image(self): + try: + path, th, draw = self._get_detect_params() + result = locate_image_center(path, th, draw) + self.detect_result_text.setText(f"Center: {result}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.detect_result_text.setText(f"Error: {error}") + + def _locate_all(self): + try: + path, th, draw = self._get_detect_params() + result = locate_all_image(path, th, draw) + self.detect_result_text.setText(f"Found {len(result)} matches:\n{result}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.detect_result_text.setText(f"Error: {error}") + + def _locate_click(self): + try: + path, th, draw = self._get_detect_params() + btn = self.mouse_button_combo.currentText() if hasattr(self, "mouse_button_combo") else "mouse_left" + result = locate_and_click(path, btn, th, draw) + self.detect_result_text.setText(f"Clicked at: {result}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.detect_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/_record_tab.py b/je_auto_control/gui/_record_tab.py new file mode 100644 index 00000000..1e468c9e --- /dev/null +++ b/je_auto_control/gui/_record_tab.py @@ -0,0 +1,101 @@ +"""Record / playback tab builder (extracted mixin).""" +import json + +from PySide6.QtWidgets import ( + QFileDialog, QLabel, QMessageBox, QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.executor.action_executor import execute_action +from je_auto_control.utils.json.json_file import read_action_json, write_action_json +from je_auto_control.wrapper.auto_control_record import record, stop_record + +_JSON_FILE_FILTER = "JSON (*.json)" + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class RecordTabMixin: + """Provides the record/playback tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``, + ``self._translate(...)``) and a ``self._record_data`` list holding the + last recording. + """ + + def _build_record_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Record/playback/save/load all run from the Actions menu. + self._record_status_key = "record_idle" + self.record_status_label = QLabel() + self._apply_record_status_label() + layout.addWidget(self.record_status_label) + + layout.addWidget(self._tr(QLabel(), "record_list_label")) + self.record_list_text = QTextEdit() + self.record_list_text.setReadOnly(True) + layout.addWidget(self.record_list_text) + tab.setLayout(layout) + return tab + + def _apply_record_status_label(self) -> None: + if hasattr(self, "record_status_label"): + self.record_status_label.setText( + self._translate("record_status") + " " + + self._translate(self._record_status_key), + ) + + def _record_retranslate(self) -> None: + self._apply_record_status_label() + + def _start_record(self): + try: + record() + self._record_status_key = "record_recording" + self._apply_record_status_label() + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _stop_record(self): + try: + self._record_data = stop_record() or [] + self._record_status_key = "record_idle" + self._apply_record_status_label() + self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _playback_record(self): + try: + if not self._record_data: + QMessageBox.warning(self, "Warning", "No recorded data") + return + execute_action(self._record_data) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _save_record(self): + try: + if not self._record_data: + QMessageBox.warning(self, "Warning", "No recorded data") + return + path, _ = QFileDialog.getSaveFileName(self, _t("save_record"), "", _JSON_FILE_FILTER) + if path: + write_action_json(path, self._record_data) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _load_record(self): + try: + path, _ = QFileDialog.getOpenFileName(self, _t("load_record"), "", _JSON_FILE_FILTER) + if path: + self._record_data = read_action_json(path) + self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) diff --git a/je_auto_control/gui/_screenshot_tab.py b/je_auto_control/gui/_screenshot_tab.py new file mode 100644 index 00000000..a86c73d0 --- /dev/null +++ b/je_auto_control/gui/_screenshot_tab.py @@ -0,0 +1,127 @@ +"""Screenshot / pixel-probe tab builder (extracted mixin).""" +from PySide6.QtGui import QIntValidator +from PySide6.QtWidgets import ( + QFileDialog, QGridLayout, QGroupBox, QHBoxLayout, QLabel, QLineEdit, + QMessageBox, QTextEdit, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.gui.selector import open_region_selector +from je_auto_control.wrapper.auto_control_screen import screen_size, screenshot, get_pixel + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class ScreenshotTabMixin: + """Provides the screenshot tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``, + ``self._translate(...)``) so every label registers for live language + switching. + """ + + def _build_screenshot_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Screen size (read via Actions menu -> Get Screen Size) + size_group = self._tr(QGroupBox(), "screen_size_label") + sg = QHBoxLayout() + self.screen_size_label = QLabel("--") + sg.addWidget(self.screen_size_label) + size_group.setLayout(sg) + layout.addWidget(size_group) + + # Screenshot inputs; capture runs from the Actions menu. + ss_group = self._tr(QGroupBox(), "take_screenshot") + ss_grid = QGridLayout() + ss_grid.addWidget(self._tr(QLabel(), "file_path_label"), 0, 0) + self.ss_path_input = QLineEdit() + ss_grid.addWidget(self.ss_path_input, 0, 1) + + ss_grid.addWidget(self._tr(QLabel(), "region_label"), 1, 0) + self.ss_region_input = QLineEdit() + self.ss_region_input.setPlaceholderText("0, 0, 800, 600") + ss_grid.addWidget(self.ss_region_input, 1, 1) + ss_group.setLayout(ss_grid) + layout.addWidget(ss_group) + + # Pixel probe inputs; lookup runs from the Actions menu. + px_group = self._tr(QGroupBox(), "get_pixel_label") + px_grid = QGridLayout() + px_grid.addWidget(self._tr(QLabel(), "pixel_x"), 0, 0) + self.pixel_x_input = QLineEdit("0") + self.pixel_x_input.setValidator(QIntValidator()) + px_grid.addWidget(self.pixel_x_input, 0, 1) + px_grid.addWidget(self._tr(QLabel(), "pixel_y"), 0, 2) + self.pixel_y_input = QLineEdit("0") + self.pixel_y_input.setValidator(QIntValidator()) + px_grid.addWidget(self.pixel_y_input, 0, 3) + self.pixel_result_label = QLabel() + self._pixel_result_suffix = " --" + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) + px_grid.addWidget(self.pixel_result_label, 1, 0, 1, 4) + px_group.setLayout(px_grid) + layout.addWidget(px_group) + + self.ss_result_text = QTextEdit() + self.ss_result_text.setReadOnly(True) + self.ss_result_text.setMaximumHeight(100) + layout.addWidget(self.ss_result_text) + layout.addStretch() + tab.setLayout(layout) + return tab + + def _get_screen_size(self): + try: + w, h = screen_size() + self.screen_size_label.setText(f"{w} x {h}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + QMessageBox.warning(self, "Error", str(error)) + + def _browse_ss_path(self): + path, _ = QFileDialog.getSaveFileName(self, _t("save_screenshot"), "", "PNG (*.png);;All (*)") + if path: + self.ss_path_input.setText(path) + + def _pick_ss_region(self): + region = open_region_selector(self) + if region is None: + return + x, y, w, h = region + self.ss_region_input.setText(f"{x}, {y}, {x + w}, {y + h}") + + def _take_screenshot(self): + try: + path = self.ss_path_input.text() or None + region_text = self.ss_region_input.text().strip() + region = None + if region_text: + region = [int(x.strip()) for x in region_text.split(",")] + screenshot(file_path=path, screen_region=region) + self.ss_result_text.setText(f"Screenshot saved: {path or '(not saved)'}") + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.ss_result_text.setText(f"Error: {error}") + + def _get_pixel_color(self): + try: + x = int(self.pixel_x_input.text()) + y = int(self.pixel_y_input.text()) + color = get_pixel(x, y) + self._pixel_result_suffix = f" {color}" + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.pixel_result_label.setText(f"Error: {error}") + + def _screenshot_retranslate(self) -> None: + if hasattr(self, "pixel_result_label"): + self.pixel_result_label.setText( + self._translate("pixel_result") + self._pixel_result_suffix, + ) diff --git a/je_auto_control/gui/_script_tab.py b/je_auto_control/gui/_script_tab.py new file mode 100644 index 00000000..d9d7fa10 --- /dev/null +++ b/je_auto_control/gui/_script_tab.py @@ -0,0 +1,105 @@ +"""Script-executor tab builder (extracted mixin).""" +import json + +from PySide6.QtWidgets import ( + QFileDialog, QHBoxLayout, QLabel, QLineEdit, QTextEdit, QVBoxLayout, + QWidget, +) + +from je_auto_control.gui.language_wrapper.multi_language_wrapper import language_wrapper +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.executor.action_executor import execute_action, execute_files +from je_auto_control.utils.file_process.get_dir_file_list import get_dir_files_as_list +from je_auto_control.utils.json.json_file import read_action_json + +_JSON_FILE_FILTER = "JSON (*.json)" + + +def _t(key: str) -> str: + """language_wrapper shorthand""" + return language_wrapper.translate(key, key) + + +class ScriptTabMixin: + """Provides the script-executor tab builder/handlers. + + Host widget must expose the ``TranslatableMixin`` API (``self._tr(...)``). + """ + + def _build_script_tab(self) -> QWidget: + tab = QWidget() + layout = QVBoxLayout() + + # Load/execute commands run from the Actions menu; the tab keeps + # only the path inputs, the editor, and the result view. + file_h = QHBoxLayout() + file_h.addWidget(self._tr(QLabel(), "file_path_label")) + self.script_path_input = QLineEdit() + file_h.addWidget(self.script_path_input) + layout.addLayout(file_h) + + dir_h = QHBoxLayout() + dir_h.addWidget(self._tr(QLabel(), "execute_dir_label")) + self.script_dir_input = QLineEdit() + dir_h.addWidget(self.script_dir_input) + layout.addLayout(dir_h) + + layout.addWidget(self._tr(QLabel(), "script_content")) + self.script_editor = QTextEdit() + self.script_editor.setPlaceholderText('[["AC_type_keyboard", {"keycode": "a"}]]') + layout.addWidget(self.script_editor) + + layout.addWidget(self._tr(QLabel(), "execution_result")) + self.script_result_text = QTextEdit() + self.script_result_text.setReadOnly(True) + layout.addWidget(self.script_result_text) + tab.setLayout(layout) + return tab + + def _browse_script(self): + path, _ = QFileDialog.getOpenFileName(self, _t("load_script"), "", _JSON_FILE_FILTER) + if path: + self.script_path_input.setText(path) + try: + data = read_action_json(path) + self.script_editor.setText(json.dumps(data, indent=2, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error loading: {error}") + + def _execute_script(self): + try: + path = self.script_path_input.text() + if not path: + return + data = read_action_json(path) + result = execute_action(data) + self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error: {error}") + + def _browse_script_dir(self): + path = QFileDialog.getExistingDirectory(self, _t("execute_dir_label")) + if path: + self.script_dir_input.setText(path) + + def _execute_dir(self): + try: + path = self.script_dir_input.text() + if not path: + return + files = get_dir_files_as_list(path) + result = execute_files(files) + self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) + except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error: {error}") + + def _execute_manual_script(self): + try: + text = self.script_editor.toPlainText().strip() + if not text: + return + data = json.loads(text) + result = execute_action(data) + self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) + except (OSError, ValueError, TypeError, RuntimeError) as error: + self.script_result_text.setText(f"Error: {error}") diff --git a/je_auto_control/gui/main_widget.py b/je_auto_control/gui/main_widget.py index 6483303c..b84a7e25 100644 --- a/je_auto_control/gui/main_widget.py +++ b/je_auto_control/gui/main_widget.py @@ -3,15 +3,17 @@ from typing import Optional from PySide6.QtCore import QTimer, Signal, QObject -from PySide6.QtGui import QIntValidator, QDoubleValidator, QKeyEvent, Qt +from PySide6.QtGui import QKeyEvent, Qt from PySide6.QtWidgets import ( - QWidget, QLineEdit, QVBoxLayout, QLabel, - QGridLayout, QHBoxLayout, QMessageBox, - QTabWidget, QTextEdit, QFileDialog, QCheckBox, QGroupBox + QWidget, QVBoxLayout, QLabel, QTabWidget, ) from je_auto_control.gui._auto_click_tab import AutoClickTabMixin from je_auto_control.gui._i18n_helpers import TranslatableMixin +from je_auto_control.gui._image_detect_tab import ImageDetectTabMixin +from je_auto_control.gui._record_tab import RecordTabMixin +from je_auto_control.gui._screenshot_tab import ScreenshotTabMixin +from je_auto_control.gui._script_tab import ScriptTabMixin from je_auto_control.gui.accessibility_tab import AccessibilityTab from je_auto_control.gui.assertions_tab import AssertionsTab from je_auto_control.gui.data_source_tab import DataSourceTab @@ -58,7 +60,6 @@ from je_auto_control.gui.flow_editor import FlowEditorTab from je_auto_control.gui.script_builder import ScriptBuilderTab from je_auto_control.gui.self_healing_tab import SelfHealingTab -from je_auto_control.gui.selector import crop_template_to_file, open_region_selector from je_auto_control.gui.triggers_tab import TriggersTab from je_auto_control.gui.webhooks_tab import WebhooksTab from je_auto_control.gui.email_triggers_tab import EmailTriggersTab @@ -66,21 +67,7 @@ from je_auto_control.gui.vlm_tab import VLMTab from je_auto_control.gui.webrunner_tab import WebRunnerTab from je_auto_control.gui.window_tab import WindowManagerTab -from je_auto_control.wrapper.auto_control_screen import screen_size, screenshot, get_pixel -from je_auto_control.wrapper.auto_control_image import locate_all_image, locate_image_center, locate_and_click -from je_auto_control.wrapper.auto_control_record import record, stop_record -from je_auto_control.utils.exception.exceptions import AutoControlException -from je_auto_control.utils.executor.action_executor import execute_action, execute_files -from je_auto_control.utils.json.json_file import read_action_json, write_action_json -from je_auto_control.utils.file_process.get_dir_file_list import get_dir_files_as_list - - -_JSON_FILE_FILTER = "JSON (*.json)" - - -def _t(key: str) -> str: - """language_wrapper shorthand""" - return language_wrapper.translate(key, key) +from je_auto_control.utils.json.json_file import read_action_json class _WorkerSignals(QObject): @@ -102,7 +89,9 @@ class _TabEntry: # Main Widget # ============================================================================= class AutoControlGUIWidget( - TranslatableMixin, AutoClickTabMixin, ReportTabMixin, QWidget, + TranslatableMixin, AutoClickTabMixin, ScreenshotTabMixin, + ImageDetectTabMixin, RecordTabMixin, ScriptTabMixin, ReportTabMixin, + QWidget, ): """Owns the QTabWidget and exposes show/hide/list APIs for the menu bar.""" @@ -424,351 +413,6 @@ def open_script_file(self, path: str) -> None: if entry is not None: self.tabs.setCurrentWidget(entry.widget) - # ========================================================================= - # Tab 2: Screenshot - # ========================================================================= - def _build_screenshot_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Screen size (read via Actions menu -> Get Screen Size) - size_group = self._tr(QGroupBox(), "screen_size_label") - sg = QHBoxLayout() - self.screen_size_label = QLabel("--") - sg.addWidget(self.screen_size_label) - size_group.setLayout(sg) - layout.addWidget(size_group) - - # Screenshot inputs; capture runs from the Actions menu. - ss_group = self._tr(QGroupBox(), "take_screenshot") - ss_grid = QGridLayout() - ss_grid.addWidget(self._tr(QLabel(), "file_path_label"), 0, 0) - self.ss_path_input = QLineEdit() - ss_grid.addWidget(self.ss_path_input, 0, 1) - - ss_grid.addWidget(self._tr(QLabel(), "region_label"), 1, 0) - self.ss_region_input = QLineEdit() - self.ss_region_input.setPlaceholderText("0, 0, 800, 600") - ss_grid.addWidget(self.ss_region_input, 1, 1) - ss_group.setLayout(ss_grid) - layout.addWidget(ss_group) - - # Pixel probe inputs; lookup runs from the Actions menu. - px_group = self._tr(QGroupBox(), "get_pixel_label") - px_grid = QGridLayout() - px_grid.addWidget(self._tr(QLabel(), "pixel_x"), 0, 0) - self.pixel_x_input = QLineEdit("0") - self.pixel_x_input.setValidator(QIntValidator()) - px_grid.addWidget(self.pixel_x_input, 0, 1) - px_grid.addWidget(self._tr(QLabel(), "pixel_y"), 0, 2) - self.pixel_y_input = QLineEdit("0") - self.pixel_y_input.setValidator(QIntValidator()) - px_grid.addWidget(self.pixel_y_input, 0, 3) - self.pixel_result_label = QLabel() - self._pixel_result_suffix = " --" - self.pixel_result_label.setText( - self._translate("pixel_result") + self._pixel_result_suffix, - ) - px_grid.addWidget(self.pixel_result_label, 1, 0, 1, 4) - px_group.setLayout(px_grid) - layout.addWidget(px_group) - - self.ss_result_text = QTextEdit() - self.ss_result_text.setReadOnly(True) - self.ss_result_text.setMaximumHeight(100) - layout.addWidget(self.ss_result_text) - layout.addStretch() - tab.setLayout(layout) - return tab - - def _get_screen_size(self): - try: - w, h = screen_size() - self.screen_size_label.setText(f"{w} x {h}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _browse_ss_path(self): - path, _ = QFileDialog.getSaveFileName(self, _t("save_screenshot"), "", "PNG (*.png);;All (*)") - if path: - self.ss_path_input.setText(path) - - def _pick_ss_region(self): - region = open_region_selector(self) - if region is None: - return - x, y, w, h = region - self.ss_region_input.setText(f"{x}, {y}, {x + w}, {y + h}") - - def _take_screenshot(self): - try: - path = self.ss_path_input.text() or None - region_text = self.ss_region_input.text().strip() - region = None - if region_text: - region = [int(x.strip()) for x in region_text.split(",")] - screenshot(file_path=path, screen_region=region) - self.ss_result_text.setText(f"Screenshot saved: {path or '(not saved)'}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.ss_result_text.setText(f"Error: {error}") - - def _get_pixel_color(self): - try: - x = int(self.pixel_x_input.text()) - y = int(self.pixel_y_input.text()) - color = get_pixel(x, y) - self._pixel_result_suffix = f" {color}" - self.pixel_result_label.setText( - self._translate("pixel_result") + self._pixel_result_suffix, - ) - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.pixel_result_label.setText(f"Error: {error}") - - def _screenshot_retranslate(self) -> None: - if hasattr(self, "pixel_result_label"): - self.pixel_result_label.setText( - self._translate("pixel_result") + self._pixel_result_suffix, - ) - - # ========================================================================= - # Tab 3: Image Detection - # ========================================================================= - def _build_image_detect_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Detection inputs; locate/crop commands run from the Actions menu. - grid = QGridLayout() - grid.addWidget(self._tr(QLabel(), "template_image"), 0, 0) - self.img_path_input = QLineEdit() - grid.addWidget(self.img_path_input, 0, 1) - - grid.addWidget(self._tr(QLabel(), "threshold_label"), 1, 0) - self.threshold_input = QLineEdit("0.8") - self.threshold_input.setValidator(QDoubleValidator(0.0, 1.0, 2)) - grid.addWidget(self.threshold_input, 1, 1) - self.draw_check = self._tr(QCheckBox(), "draw_image_check") - grid.addWidget(self.draw_check, 1, 2) - - layout.addLayout(grid) - - layout.addWidget(self._tr(QLabel(), "detection_result")) - self.detect_result_text = QTextEdit() - self.detect_result_text.setReadOnly(True) - layout.addWidget(self.detect_result_text) - tab.setLayout(layout) - return tab - - def _browse_img(self): - path, _ = QFileDialog.getOpenFileName(self, _t("template_image"), "", "Images (*.png *.jpg *.bmp);;All (*)") - if path: - self.img_path_input.setText(path) - - def _crop_template(self): - save_path, _ = QFileDialog.getSaveFileName( - self, _t("crop_template"), "", "PNG (*.png)" - ) - if not save_path: - return - try: - region = crop_template_to_file(save_path, self) - if region is None: - return - self.img_path_input.setText(save_path) - self.detect_result_text.setText(f"Template saved: {save_path} region={region}") - except (OSError, ValueError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _get_detect_params(self): - path = self.img_path_input.text() - if not path: - raise ValueError("Template image path is empty") - threshold = float(self.threshold_input.text() or "0.8") - draw = self.draw_check.isChecked() - return path, threshold, draw - - def _locate_image(self): - try: - path, th, draw = self._get_detect_params() - result = locate_image_center(path, th, draw) - self.detect_result_text.setText(f"Center: {result}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.detect_result_text.setText(f"Error: {error}") - - def _locate_all(self): - try: - path, th, draw = self._get_detect_params() - result = locate_all_image(path, th, draw) - self.detect_result_text.setText(f"Found {len(result)} matches:\n{result}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.detect_result_text.setText(f"Error: {error}") - - def _locate_click(self): - try: - path, th, draw = self._get_detect_params() - btn = self.mouse_button_combo.currentText() if hasattr(self, "mouse_button_combo") else "mouse_left" - result = locate_and_click(path, btn, th, draw) - self.detect_result_text.setText(f"Clicked at: {result}") - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.detect_result_text.setText(f"Error: {error}") - - # ========================================================================= - # Tab 4: Record / Playback - # ========================================================================= - def _build_record_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Record/playback/save/load all run from the Actions menu. - self._record_status_key = "record_idle" - self.record_status_label = QLabel() - self._apply_record_status_label() - layout.addWidget(self.record_status_label) - - layout.addWidget(self._tr(QLabel(), "record_list_label")) - self.record_list_text = QTextEdit() - self.record_list_text.setReadOnly(True) - layout.addWidget(self.record_list_text) - tab.setLayout(layout) - return tab - - def _apply_record_status_label(self) -> None: - if hasattr(self, "record_status_label"): - self.record_status_label.setText( - self._translate("record_status") + " " - + self._translate(self._record_status_key), - ) - - def _record_retranslate(self) -> None: - self._apply_record_status_label() - - def _start_record(self): - try: - record() - self._record_status_key = "record_recording" - self._apply_record_status_label() - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _stop_record(self): - try: - self._record_data = stop_record() or [] - self._record_status_key = "record_idle" - self._apply_record_status_label() - self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _playback_record(self): - try: - if not self._record_data: - QMessageBox.warning(self, "Warning", "No recorded data") - return - execute_action(self._record_data) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _save_record(self): - try: - if not self._record_data: - QMessageBox.warning(self, "Warning", "No recorded data") - return - path, _ = QFileDialog.getSaveFileName(self, _t("save_record"), "", _JSON_FILE_FILTER) - if path: - write_action_json(path, self._record_data) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - def _load_record(self): - try: - path, _ = QFileDialog.getOpenFileName(self, _t("load_record"), "", _JSON_FILE_FILTER) - if path: - self._record_data = read_action_json(path) - self.record_list_text.setText(json.dumps(self._record_data, indent=2, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - QMessageBox.warning(self, "Error", str(error)) - - # ========================================================================= - # Tab 5: Script Executor - # ========================================================================= - def _build_script_tab(self) -> QWidget: - tab = QWidget() - layout = QVBoxLayout() - - # Load/execute commands run from the Actions menu; the tab keeps - # only the path inputs, the editor, and the result view. - file_h = QHBoxLayout() - file_h.addWidget(self._tr(QLabel(), "file_path_label")) - self.script_path_input = QLineEdit() - file_h.addWidget(self.script_path_input) - layout.addLayout(file_h) - - dir_h = QHBoxLayout() - dir_h.addWidget(self._tr(QLabel(), "execute_dir_label")) - self.script_dir_input = QLineEdit() - dir_h.addWidget(self.script_dir_input) - layout.addLayout(dir_h) - - layout.addWidget(self._tr(QLabel(), "script_content")) - self.script_editor = QTextEdit() - self.script_editor.setPlaceholderText('[["AC_type_keyboard", {"keycode": "a"}]]') - layout.addWidget(self.script_editor) - - layout.addWidget(self._tr(QLabel(), "execution_result")) - self.script_result_text = QTextEdit() - self.script_result_text.setReadOnly(True) - layout.addWidget(self.script_result_text) - tab.setLayout(layout) - return tab - - def _browse_script(self): - path, _ = QFileDialog.getOpenFileName(self, _t("load_script"), "", _JSON_FILE_FILTER) - if path: - self.script_path_input.setText(path) - try: - data = read_action_json(path) - self.script_editor.setText(json.dumps(data, indent=2, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error loading: {error}") - - def _execute_script(self): - try: - path = self.script_path_input.text() - if not path: - return - data = read_action_json(path) - result = execute_action(data) - self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error: {error}") - - def _browse_script_dir(self): - path = QFileDialog.getExistingDirectory(self, _t("execute_dir_label")) - if path: - self.script_dir_input.setText(path) - - def _execute_dir(self): - try: - path = self.script_dir_input.text() - if not path: - return - files = get_dir_files_as_list(path) - result = execute_files(files) - self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) - except (AutoControlException, OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error: {error}") - - def _execute_manual_script(self): - try: - text = self.script_editor.toPlainText().strip() - if not text: - return - data = json.loads(text) - result = execute_action(data) - self.script_result_text.setText(json.dumps(result, indent=2, default=str, ensure_ascii=False)) - except (OSError, ValueError, TypeError, RuntimeError) as error: - self.script_result_text.setText(f"Error: {error}") - # ========================================================================= # Global keyboard shortcut: Ctrl+4 to stop # ========================================================================= diff --git a/je_auto_control/gui/remote_desktop/_helpers.py b/je_auto_control/gui/remote_desktop/_helpers.py index bb16c2d8..08670268 100644 --- a/je_auto_control/gui/remote_desktop/_helpers.py +++ b/je_auto_control/gui/remote_desktop/_helpers.py @@ -146,3 +146,44 @@ def body(self) -> QWidget: def set_body_layout(self, layout) -> None: self._body.setLayout(layout) + + +def _short_fp(fp: Optional[str]) -> str: + if not fp: + return "" + return fp[:16] + ("..." if len(fp) > 16 else "") + + +def _iso_to_epoch(value: Optional[str]) -> float: + """Parse ISO; return Unix epoch (or 0 if invalid).""" + if not value: + return 0.0 + from datetime import datetime + try: + return datetime.fromisoformat(value).timestamp() + except (TypeError, ValueError): + return 0.0 + + +def _format_short_time(value: Optional[str]) -> str: + if not value: + return "" + from datetime import datetime + try: + dt = datetime.fromisoformat(value) + except (TypeError, ValueError): + return "" + return dt.astimezone().strftime("%m-%d %H:%M") + + +def _format_last_seen(value: Optional[str]) -> str: + if not value: + return "" + # Stored as ISO 8601 (UTC); render as local-readable "YYYY-MM-DD HH:MM" + from datetime import datetime + try: + dt = datetime.fromisoformat(value) + except (TypeError, ValueError): + return value + return dt.astimezone().strftime("%Y-%m-%d %H:%M") + diff --git a/je_auto_control/gui/remote_desktop/webrtc_dialogs.py b/je_auto_control/gui/remote_desktop/webrtc_dialogs.py index 68d3ff68..9905a843 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_dialogs.py +++ b/je_auto_control/gui/remote_desktop/webrtc_dialogs.py @@ -1,22 +1,28 @@ """Custom dialogs / list widgets used by the WebRTC GUI panels. Kept out of ``webrtc_panel.py`` so that file stays focused on layout -construction and signal wiring. +construction and signal wiring. The known-hosts browser lives in +``webrtc_known_hosts`` and is re-exported here, so callers keep importing +every dialog from one place. """ from __future__ import annotations from typing import Optional from PySide6.QtCore import Qt, Signal -from PySide6.QtGui import QColor from PySide6.QtWidgets import ( - QAbstractItemView, QDialog, QFileDialog, QFormLayout, QHBoxLayout, + QAbstractItemView, QDialog, QHBoxLayout, QHeaderView, QLabel, QLineEdit, QListWidget, QListWidgetItem, QMenu, - QMessageBox, QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, + QPushButton, QTableWidget, QTableWidgetItem, QVBoxLayout, QWidget, ) -from je_auto_control.gui.remote_desktop._helpers import _t +from je_auto_control.gui.remote_desktop._helpers import ( + _format_short_time, _iso_to_epoch, _t, +) +from je_auto_control.gui.remote_desktop.webrtc_known_hosts import ( + KnownHostsDialog, +) class PendingViewerDialog(QDialog): @@ -298,363 +304,6 @@ def _show_context_menu(self, position) -> None: self.copy_name_requested.emit(names[0]) -class KnownHostsDialog(QDialog): - """Browse + forget the persistent KnownHosts (TOFU app + DTLS fingerprints).""" - - def __init__(self, known_hosts, parent: Optional[QWidget] = None) -> None: - super().__init__(parent) - self._known = known_hosts - self.setWindowTitle(_t("rd_webrtc_known_hosts_title")) - self.setMinimumSize(720, 360) - layout = QVBoxLayout(self) - self._table = QTableWidget(0, 4) - self._table.setHorizontalHeaderLabels([ - _t("rd_webrtc_kh_col_host"), - _t("rd_webrtc_kh_col_app_fp"), - _t("rd_webrtc_kh_col_dtls_fp"), - _t("rd_webrtc_kh_col_last_seen"), - ]) - self._table.horizontalHeader().setSectionResizeMode( - 1, QHeaderView.ResizeMode.Stretch, - ) - self._table.horizontalHeader().setSectionResizeMode( - 2, QHeaderView.ResizeMode.Stretch, - ) - self._table.setSelectionBehavior( - QAbstractItemView.SelectionBehavior.SelectRows, - ) - self._table.setSelectionMode( - QAbstractItemView.SelectionMode.ExtendedSelection, - ) - self._table.setEditTriggers( - QAbstractItemView.EditTrigger.NoEditTriggers, - ) - layout.addWidget(self._table) - button_row = QHBoxLayout() - add_btn = QPushButton(_t("rd_webrtc_kh_add")) - add_btn.clicked.connect(self._on_add_manual) - button_row.addWidget(add_btn) - import_btn = QPushButton(_t("rd_webrtc_kh_import")) - import_btn.clicked.connect(self._on_import) - button_row.addWidget(import_btn) - export_btn = QPushButton(_t("rd_webrtc_kh_export")) - export_btn.clicked.connect(self._on_export) - button_row.addWidget(export_btn) - copy_app_btn = QPushButton(_t("rd_webrtc_kh_copy_app")) - copy_app_btn.clicked.connect(self._on_copy_app) - button_row.addWidget(copy_app_btn) - copy_dtls_btn = QPushButton(_t("rd_webrtc_kh_copy_dtls")) - copy_dtls_btn.clicked.connect(self._on_copy_dtls) - button_row.addWidget(copy_dtls_btn) - forget_btn = QPushButton(_t("rd_webrtc_kh_forget")) - forget_btn.clicked.connect(self._on_forget) - button_row.addWidget(forget_btn) - forget_stale_btn = QPushButton(_t("rd_webrtc_kh_forget_stale")) - forget_stale_btn.clicked.connect(self._on_forget_stale) - button_row.addWidget(forget_stale_btn) - clear_btn = QPushButton(_t("rd_webrtc_kh_clear_all")) - clear_btn.clicked.connect(self._on_clear_all) - button_row.addWidget(clear_btn) - button_row.addStretch() - close_btn = QPushButton(_t("rd_webrtc_kh_close")) - close_btn.clicked.connect(self.accept) - button_row.addWidget(close_btn) - layout.addLayout(button_row) - self._refresh() - - def _refresh(self) -> None: - from datetime import datetime, timedelta, timezone - stale_after = timedelta(days=90) - now = datetime.now(timezone.utc) - stale_color = QColor("#888") - entries = self._known.list_entries() - self._table.setRowCount(len(entries)) - for row, (host_id, fps) in enumerate(sorted(entries.items())): - self._populate_row(row, host_id, fps, - now=now, stale_after=stale_after, - stale_color=stale_color) - - def _populate_row(self, row: int, host_id: str, fps: dict, *, - now, stale_after, stale_color) -> None: - items = [ - QTableWidgetItem(host_id), - QTableWidgetItem(_short_fp(fps.get("app_fp"))), - QTableWidgetItem(_short_fp(fps.get("dtls_fp"))), - QTableWidgetItem(_format_last_seen(fps.get("last_seen"))), - ] - if self._is_stale(fps.get("last_seen"), now=now, - stale_after=stale_after): - tip = _t("rd_webrtc_kh_stale_tip") - for it in items: - it.setForeground(stale_color) - it.setToolTip(tip) - for col, item in enumerate(items): - self._table.setItem(row, col, item) - - @staticmethod - def _is_stale(last_seen, *, now, stale_after) -> bool: - if not last_seen: - return False - from datetime import datetime - try: - dt = datetime.fromisoformat(last_seen) - except (TypeError, ValueError): - return False - return now - dt > stale_after - - def _on_forget(self) -> None: - rows = sorted( - {i.row() for i in self._table.selectedIndexes()}, reverse=True, - ) - if not rows: - return - for row in rows: - item = self._table.item(row, 0) - if item is None: - continue - self._known.forget(item.text()) - self._refresh() - - def _on_add_manual(self) -> None: - dialog = _ManualKnownHostDialog(parent=self) - if dialog.exec() != QDialog.DialogCode.Accepted: - return - host_id, app_fp, dtls_fp = dialog.values() - if not host_id: - return - if app_fp: - self._known.remember(host_id, app_fp) - if dtls_fp: - self._known.remember_dtls_fingerprint(host_id, dtls_fp) - self._refresh() - - def _on_copy_app(self) -> None: - self._copy_selected_fingerprint("app_fp") - - def _on_copy_dtls(self) -> None: - self._copy_selected_fingerprint("dtls_fp") - - def _copy_selected_fingerprint(self, key: str) -> None: - from PySide6.QtWidgets import QApplication as _QApp - row = self._table.currentRow() - if row < 0: - return - host_item = self._table.item(row, 0) - if host_item is None: - return - entries = self._known.list_entries() - fps = entries.get(host_item.text()) - if not fps: - return - value = fps.get(key) or "" - clipboard = _QApp.clipboard() - if clipboard is not None: - clipboard.setText(value) - - def _on_export(self) -> None: - import json - path, _filter = QFileDialog.getSaveFileName( - self, _t("rd_webrtc_kh_export"), "known_hosts.json", - "JSON (*.json);;All (*)", - ) - if not path: - return - try: - with open(path, "w", encoding="utf-8") as fh: - json.dump(self._known.list_entries(), fh, - indent=2, ensure_ascii=False) - except OSError as error: - QMessageBox.warning(self, "WebRTC", str(error)) - - def _on_import(self) -> None: - data = self._prompt_import_data() - if data is None: - return - existing = self._known.list_entries() - added = 0 - skipped = 0 - for host_id, value in data.items(): - outcome = self._import_one(host_id, value, existing) - if outcome == "added": - added += 1 - elif outcome == "skipped": - skipped += 1 - QMessageBox.information( - self, "WebRTC", - _t("rd_webrtc_kh_import_done").format(added=added, skipped=skipped), - ) - self._refresh() - - def _prompt_import_data(self): - import json - path, _filter = QFileDialog.getOpenFileName( - self, _t("rd_webrtc_kh_import"), "", "JSON (*.json);;All (*)", - ) - if not path: - return None - try: - with open(path, "r", encoding="utf-8") as fh: - data = json.load(fh) - except (OSError, json.JSONDecodeError) as error: - QMessageBox.warning(self, "WebRTC", str(error)) - return None - if not isinstance(data, dict): - QMessageBox.warning( - self, "WebRTC", _t("rd_webrtc_kh_import_bad"), - ) - return None - return data - - def _import_one(self, host_id, value, existing) -> str: - """Return ``"added"``, ``"skipped"``, or ``"ignored"`` per entry.""" - if not isinstance(host_id, str): - return "ignored" - app_fp, dtls_fp = self._extract_fingerprints(value) - if app_fp is None and dtls_fp is None: - return "ignored" - if host_id in existing and not self._confirm_overwrite(host_id): - return "skipped" - if isinstance(app_fp, str) and app_fp: - self._known.remember(host_id, app_fp) - if isinstance(dtls_fp, str) and dtls_fp: - self._known.remember_dtls_fingerprint(host_id, dtls_fp) - return "added" - - @staticmethod - def _extract_fingerprints(value): - if isinstance(value, str): - return value, None - if isinstance(value, dict): - return value.get("app_fp"), value.get("dtls_fp") - return None, None - - def _confirm_overwrite(self, host_id: str) -> bool: - result = QMessageBox.question( - self, "WebRTC", - _t("rd_webrtc_kh_import_overwrite").format(host=host_id), - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - return result == QMessageBox.StandardButton.Yes - - def _on_forget_stale(self) -> None: - from datetime import datetime, timedelta, timezone - cutoff = datetime.now(timezone.utc) - timedelta(days=90) - stale_ids = [] - for host_id, fps in self._known.list_entries().items(): - last_seen = fps.get("last_seen") - if not last_seen: - continue - try: - if datetime.fromisoformat(last_seen) < cutoff: - stale_ids.append(host_id) - except (TypeError, ValueError): - continue - if not stale_ids: - QMessageBox.information( - self, "WebRTC", _t("rd_webrtc_kh_no_stale"), - ) - return - result = QMessageBox.question( - self, "WebRTC", - _t("rd_webrtc_kh_forget_stale_confirm").format(n=len(stale_ids)), - QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, - ) - if result != QMessageBox.StandardButton.Yes: - return - for host_id in stale_ids: - self._known.forget(host_id) - self._refresh() - - def _on_clear_all(self) -> None: - from PySide6.QtWidgets import QMessageBox as _QMB - result = _QMB.question( - self, "WebRTC", _t("rd_webrtc_kh_clear_confirm"), - _QMB.StandardButton.Yes | _QMB.StandardButton.No, - ) - if result != _QMB.StandardButton.Yes: - return - for host_id in list(self._known.list_entries().keys()): # NOSONAR python:S7504 # forget() mutates the underlying mapping — list() is required to avoid RuntimeError - self._known.forget(host_id) - self._refresh() - - -def _short_fp(fp: Optional[str]) -> str: - if not fp: - return "" - return fp[:16] + ("..." if len(fp) > 16 else "") - - -def _iso_to_epoch(value: Optional[str]) -> float: - """Parse ISO; return Unix epoch (or 0 if invalid).""" - if not value: - return 0.0 - from datetime import datetime - try: - return datetime.fromisoformat(value).timestamp() - except (TypeError, ValueError): - return 0.0 - - -def _format_short_time(value: Optional[str]) -> str: - if not value: - return "" - from datetime import datetime - try: - dt = datetime.fromisoformat(value) - except (TypeError, ValueError): - return "" - return dt.astimezone().strftime("%m-%d %H:%M") - - -def _format_last_seen(value: Optional[str]) -> str: - if not value: - return "" - # Stored as ISO 8601 (UTC); render as local-readable "YYYY-MM-DD HH:MM" - from datetime import datetime - try: - dt = datetime.fromisoformat(value) - except (TypeError, ValueError): - return value - return dt.astimezone().strftime("%Y-%m-%d %H:%M") - - -class _ManualKnownHostDialog(QDialog): - """Tiny form dialog for pinning a host fingerprint out-of-band.""" - - def __init__(self, parent: Optional[QWidget] = None) -> None: - super().__init__(parent) - self.setWindowTitle(_t("rd_webrtc_kh_add")) - self.setMinimumWidth(420) - layout = QVBoxLayout(self) - form = QFormLayout() - self._host_edit = QLineEdit() - self._host_edit.setPlaceholderText(_t("rd_webrtc_kh_add_host_ph")) - self._app_edit = QLineEdit() - self._app_edit.setPlaceholderText(_t("rd_webrtc_kh_add_app_ph")) - self._dtls_edit = QLineEdit() - self._dtls_edit.setPlaceholderText(_t("rd_webrtc_kh_add_dtls_ph")) - form.addRow(_t("rd_webrtc_kh_col_host"), self._host_edit) - form.addRow(_t("rd_webrtc_kh_col_app_fp"), self._app_edit) - form.addRow(_t("rd_webrtc_kh_col_dtls_fp"), self._dtls_edit) - layout.addLayout(form) - button_row = QHBoxLayout() - button_row.addStretch() - ok = QPushButton(_t("rd_webrtc_kh_add")) - ok.clicked.connect(self.accept) - cancel = QPushButton(_t("rd_webrtc_kh_close")) - cancel.clicked.connect(self.reject) - button_row.addWidget(cancel) - button_row.addWidget(ok) - layout.addLayout(button_row) - - def values(self) -> tuple: - return ( - self._host_edit.text().strip(), - self._app_edit.text().strip(), - self._dtls_edit.text().strip(), - ) - - class AuditLogDialog(QDialog): """Browse the SQLite audit log with filter on event_type / host_id.""" diff --git a/je_auto_control/gui/remote_desktop/webrtc_known_hosts.py b/je_auto_control/gui/remote_desktop/webrtc_known_hosts.py new file mode 100644 index 00000000..d3eb3f72 --- /dev/null +++ b/je_auto_control/gui/remote_desktop/webrtc_known_hosts.py @@ -0,0 +1,340 @@ +"""Known-hosts browser for the WebRTC GUI panels. + +Split out of ``webrtc_dialogs.py``: the TOFU pin store gets its own table +dialog plus a small out-of-band pinning form, and neither shares state with +the viewer / address-book / audit dialogs next door. The shared fingerprint +and timestamp formatters live in ``_helpers``. +""" +from __future__ import annotations + +from typing import Optional + +from PySide6.QtGui import QColor +from PySide6.QtWidgets import ( + QAbstractItemView, QDialog, QFileDialog, QFormLayout, QHBoxLayout, + QHeaderView, QLineEdit, QMessageBox, QPushButton, QTableWidget, + QTableWidgetItem, QVBoxLayout, QWidget, +) + +from je_auto_control.gui.remote_desktop._helpers import ( + _format_last_seen, _short_fp, _t, +) + + +class KnownHostsDialog(QDialog): + """Browse + forget the persistent KnownHosts (TOFU app + DTLS fingerprints).""" + + def __init__(self, known_hosts, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self._known = known_hosts + self.setWindowTitle(_t("rd_webrtc_known_hosts_title")) + self.setMinimumSize(720, 360) + layout = QVBoxLayout(self) + self._table = QTableWidget(0, 4) + self._table.setHorizontalHeaderLabels([ + _t("rd_webrtc_kh_col_host"), + _t("rd_webrtc_kh_col_app_fp"), + _t("rd_webrtc_kh_col_dtls_fp"), + _t("rd_webrtc_kh_col_last_seen"), + ]) + self._table.horizontalHeader().setSectionResizeMode( + 1, QHeaderView.ResizeMode.Stretch, + ) + self._table.horizontalHeader().setSectionResizeMode( + 2, QHeaderView.ResizeMode.Stretch, + ) + self._table.setSelectionBehavior( + QAbstractItemView.SelectionBehavior.SelectRows, + ) + self._table.setSelectionMode( + QAbstractItemView.SelectionMode.ExtendedSelection, + ) + self._table.setEditTriggers( + QAbstractItemView.EditTrigger.NoEditTriggers, + ) + layout.addWidget(self._table) + button_row = QHBoxLayout() + add_btn = QPushButton(_t("rd_webrtc_kh_add")) + add_btn.clicked.connect(self._on_add_manual) + button_row.addWidget(add_btn) + import_btn = QPushButton(_t("rd_webrtc_kh_import")) + import_btn.clicked.connect(self._on_import) + button_row.addWidget(import_btn) + export_btn = QPushButton(_t("rd_webrtc_kh_export")) + export_btn.clicked.connect(self._on_export) + button_row.addWidget(export_btn) + copy_app_btn = QPushButton(_t("rd_webrtc_kh_copy_app")) + copy_app_btn.clicked.connect(self._on_copy_app) + button_row.addWidget(copy_app_btn) + copy_dtls_btn = QPushButton(_t("rd_webrtc_kh_copy_dtls")) + copy_dtls_btn.clicked.connect(self._on_copy_dtls) + button_row.addWidget(copy_dtls_btn) + forget_btn = QPushButton(_t("rd_webrtc_kh_forget")) + forget_btn.clicked.connect(self._on_forget) + button_row.addWidget(forget_btn) + forget_stale_btn = QPushButton(_t("rd_webrtc_kh_forget_stale")) + forget_stale_btn.clicked.connect(self._on_forget_stale) + button_row.addWidget(forget_stale_btn) + clear_btn = QPushButton(_t("rd_webrtc_kh_clear_all")) + clear_btn.clicked.connect(self._on_clear_all) + button_row.addWidget(clear_btn) + button_row.addStretch() + close_btn = QPushButton(_t("rd_webrtc_kh_close")) + close_btn.clicked.connect(self.accept) + button_row.addWidget(close_btn) + layout.addLayout(button_row) + self._refresh() + + def _refresh(self) -> None: + from datetime import datetime, timedelta, timezone + stale_after = timedelta(days=90) + now = datetime.now(timezone.utc) + stale_color = QColor("#888") + entries = self._known.list_entries() + self._table.setRowCount(len(entries)) + for row, (host_id, fps) in enumerate(sorted(entries.items())): + self._populate_row(row, host_id, fps, + now=now, stale_after=stale_after, + stale_color=stale_color) + + def _populate_row(self, row: int, host_id: str, fps: dict, *, + now, stale_after, stale_color) -> None: + items = [ + QTableWidgetItem(host_id), + QTableWidgetItem(_short_fp(fps.get("app_fp"))), + QTableWidgetItem(_short_fp(fps.get("dtls_fp"))), + QTableWidgetItem(_format_last_seen(fps.get("last_seen"))), + ] + if self._is_stale(fps.get("last_seen"), now=now, + stale_after=stale_after): + tip = _t("rd_webrtc_kh_stale_tip") + for it in items: + it.setForeground(stale_color) + it.setToolTip(tip) + for col, item in enumerate(items): + self._table.setItem(row, col, item) + + @staticmethod + def _is_stale(last_seen, *, now, stale_after) -> bool: + if not last_seen: + return False + from datetime import datetime + try: + dt = datetime.fromisoformat(last_seen) + except (TypeError, ValueError): + return False + return now - dt > stale_after + + def _on_forget(self) -> None: + rows = sorted( + {i.row() for i in self._table.selectedIndexes()}, reverse=True, + ) + if not rows: + return + for row in rows: + item = self._table.item(row, 0) + if item is None: + continue + self._known.forget(item.text()) + self._refresh() + + def _on_add_manual(self) -> None: + dialog = _ManualKnownHostDialog(parent=self) + if dialog.exec() != QDialog.DialogCode.Accepted: + return + host_id, app_fp, dtls_fp = dialog.values() + if not host_id: + return + if app_fp: + self._known.remember(host_id, app_fp) + if dtls_fp: + self._known.remember_dtls_fingerprint(host_id, dtls_fp) + self._refresh() + + def _on_copy_app(self) -> None: + self._copy_selected_fingerprint("app_fp") + + def _on_copy_dtls(self) -> None: + self._copy_selected_fingerprint("dtls_fp") + + def _copy_selected_fingerprint(self, key: str) -> None: + from PySide6.QtWidgets import QApplication as _QApp + row = self._table.currentRow() + if row < 0: + return + host_item = self._table.item(row, 0) + if host_item is None: + return + entries = self._known.list_entries() + fps = entries.get(host_item.text()) + if not fps: + return + value = fps.get(key) or "" + clipboard = _QApp.clipboard() + if clipboard is not None: + clipboard.setText(value) + + def _on_export(self) -> None: + import json + path, _filter = QFileDialog.getSaveFileName( + self, _t("rd_webrtc_kh_export"), "known_hosts.json", + "JSON (*.json);;All (*)", + ) + if not path: + return + try: + with open(path, "w", encoding="utf-8") as fh: + json.dump(self._known.list_entries(), fh, + indent=2, ensure_ascii=False) + except OSError as error: + QMessageBox.warning(self, "WebRTC", str(error)) + + def _on_import(self) -> None: + data = self._prompt_import_data() + if data is None: + return + existing = self._known.list_entries() + added = 0 + skipped = 0 + for host_id, value in data.items(): + outcome = self._import_one(host_id, value, existing) + if outcome == "added": + added += 1 + elif outcome == "skipped": + skipped += 1 + QMessageBox.information( + self, "WebRTC", + _t("rd_webrtc_kh_import_done").format(added=added, skipped=skipped), + ) + self._refresh() + + def _prompt_import_data(self): + import json + path, _filter = QFileDialog.getOpenFileName( + self, _t("rd_webrtc_kh_import"), "", "JSON (*.json);;All (*)", + ) + if not path: + return None + try: + with open(path, "r", encoding="utf-8") as fh: + data = json.load(fh) + except (OSError, json.JSONDecodeError) as error: + QMessageBox.warning(self, "WebRTC", str(error)) + return None + if not isinstance(data, dict): + QMessageBox.warning( + self, "WebRTC", _t("rd_webrtc_kh_import_bad"), + ) + return None + return data + + def _import_one(self, host_id, value, existing) -> str: + """Return ``"added"``, ``"skipped"``, or ``"ignored"`` per entry.""" + if not isinstance(host_id, str): + return "ignored" + app_fp, dtls_fp = self._extract_fingerprints(value) + if app_fp is None and dtls_fp is None: + return "ignored" + if host_id in existing and not self._confirm_overwrite(host_id): + return "skipped" + if isinstance(app_fp, str) and app_fp: + self._known.remember(host_id, app_fp) + if isinstance(dtls_fp, str) and dtls_fp: + self._known.remember_dtls_fingerprint(host_id, dtls_fp) + return "added" + + @staticmethod + def _extract_fingerprints(value): + if isinstance(value, str): + return value, None + if isinstance(value, dict): + return value.get("app_fp"), value.get("dtls_fp") + return None, None + + def _confirm_overwrite(self, host_id: str) -> bool: + result = QMessageBox.question( + self, "WebRTC", + _t("rd_webrtc_kh_import_overwrite").format(host=host_id), + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + return result == QMessageBox.StandardButton.Yes + + def _on_forget_stale(self) -> None: + from datetime import datetime, timedelta, timezone + cutoff = datetime.now(timezone.utc) - timedelta(days=90) + stale_ids = [] + for host_id, fps in self._known.list_entries().items(): + last_seen = fps.get("last_seen") + if not last_seen: + continue + try: + if datetime.fromisoformat(last_seen) < cutoff: + stale_ids.append(host_id) + except (TypeError, ValueError): + continue + if not stale_ids: + QMessageBox.information( + self, "WebRTC", _t("rd_webrtc_kh_no_stale"), + ) + return + result = QMessageBox.question( + self, "WebRTC", + _t("rd_webrtc_kh_forget_stale_confirm").format(n=len(stale_ids)), + QMessageBox.StandardButton.Yes | QMessageBox.StandardButton.No, + ) + if result != QMessageBox.StandardButton.Yes: + return + for host_id in stale_ids: + self._known.forget(host_id) + self._refresh() + + def _on_clear_all(self) -> None: + from PySide6.QtWidgets import QMessageBox as _QMB + result = _QMB.question( + self, "WebRTC", _t("rd_webrtc_kh_clear_confirm"), + _QMB.StandardButton.Yes | _QMB.StandardButton.No, + ) + if result != _QMB.StandardButton.Yes: + return + for host_id in list(self._known.list_entries().keys()): # NOSONAR python:S7504 # forget() mutates the underlying mapping — list() is required to avoid RuntimeError + self._known.forget(host_id) + self._refresh() + + + + +class _ManualKnownHostDialog(QDialog): + """Tiny form dialog for pinning a host fingerprint out-of-band.""" + + def __init__(self, parent: Optional[QWidget] = None) -> None: + super().__init__(parent) + self.setWindowTitle(_t("rd_webrtc_kh_add")) + self.setMinimumWidth(420) + layout = QVBoxLayout(self) + form = QFormLayout() + self._host_edit = QLineEdit() + self._host_edit.setPlaceholderText(_t("rd_webrtc_kh_add_host_ph")) + self._app_edit = QLineEdit() + self._app_edit.setPlaceholderText(_t("rd_webrtc_kh_add_app_ph")) + self._dtls_edit = QLineEdit() + self._dtls_edit.setPlaceholderText(_t("rd_webrtc_kh_add_dtls_ph")) + form.addRow(_t("rd_webrtc_kh_col_host"), self._host_edit) + form.addRow(_t("rd_webrtc_kh_col_app_fp"), self._app_edit) + form.addRow(_t("rd_webrtc_kh_col_dtls_fp"), self._dtls_edit) + layout.addLayout(form) + button_row = QHBoxLayout() + button_row.addStretch() + ok = QPushButton(_t("rd_webrtc_kh_add")) + ok.clicked.connect(self.accept) + cancel = QPushButton(_t("rd_webrtc_kh_close")) + cancel.clicked.connect(self.reject) + button_row.addWidget(cancel) + button_row.addWidget(ok) + layout.addLayout(button_row) + + def values(self) -> tuple: + return ( + self._host_edit.text().strip(), + self._app_edit.text().strip(), + self._dtls_edit.text().strip(), + ) diff --git a/je_auto_control/gui/remote_desktop/webrtc_panel.py b/je_auto_control/gui/remote_desktop/webrtc_panel.py index d5c226cd..b4849caf 100644 --- a/je_auto_control/gui/remote_desktop/webrtc_panel.py +++ b/je_auto_control/gui/remote_desktop/webrtc_panel.py @@ -620,8 +620,8 @@ def _on_toggle_adaptive(self, value: bool) -> None: def _populate_monitor_combo(self) -> None: try: - import mss - with mss.mss() as sct: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as sct: monitors = sct.monitors for idx, mon in enumerate(monitors): if idx == 0: diff --git a/je_auto_control/gui/script_builder/command_schema.py b/je_auto_control/gui/script_builder/command_schema.py index f250aa11..e9ff2656 100644 --- a/je_auto_control/gui/script_builder/command_schema.py +++ b/je_auto_control/gui/script_builder/command_schema.py @@ -1097,6 +1097,45 @@ def _add_window_specs(specs: List[CommandSpec]) -> None: fields=(FieldSpec("title_substring", FieldType.STRING),), description="Screen rectangle (left, top, right, bottom) of a window.", )) + specs.append(CommandSpec( + "AC_foreground_window_pid", "Window", "Foreground Window PID", + description="PID owning the foreground window (0 when unavailable).", + )) + specs.append(CommandSpec( + "AC_windows_for_pid", "Window", "Windows by Process ID", + fields=(FieldSpec("pid", FieldType.INT),), + description="Visible top-level windows owned by a process.", + )) + specs.append(CommandSpec( + "AC_minimize_windows_for_pid", "Window", "Minimize Windows by PID", + fields=(FieldSpec("pid", FieldType.INT),), + description="Minimise every window a process owns.", + )) + specs.append(CommandSpec( + "AC_post_key_to_window", "Window", "Post Key to Window", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("key", FieldType.STRING), + ), + description="Type one key into a window without focusing it " + "(best effort: some applications ignore posted input).", + )) + specs.append(CommandSpec( + "AC_post_click_to_window", "Window", "Post Click to Window", + fields=( + FieldSpec("title_substring", FieldType.STRING), + FieldSpec("button", FieldType.STRING, optional=True), + FieldSpec("x", FieldType.INT, optional=True), + FieldSpec("y", FieldType.INT, optional=True), + ), + description="Click inside a window without focusing it; x / y are " + "relative to the window's top-left corner.", + )) + specs.append(CommandSpec( + "AC_window_pid", "Window", "Window PID by Title", + fields=(FieldSpec("title_substring", FieldType.STRING),), + description="PID owning the first matching window (0 when none).", + )) specs.append(CommandSpec( "AC_move_window", "Window", "Move Window by Title", fields=( diff --git a/je_auto_control/gui/selector/template_cropper.py b/je_auto_control/gui/selector/template_cropper.py index 35e06426..50d0045c 100644 --- a/je_auto_control/gui/selector/template_cropper.py +++ b/je_auto_control/gui/selector/template_cropper.py @@ -4,7 +4,6 @@ import cv2 import numpy as np -from PIL import ImageGrab from PySide6.QtWidgets import QWidget from je_auto_control.gui.selector.region_selector import open_region_selector @@ -13,8 +12,9 @@ def _capture_region(region: Tuple[int, int, int, int]) -> np.ndarray: """Grab the given (x, y, w, h) region as a BGR numpy array.""" + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber x, y, w, h = region - pil_image = ImageGrab.grab(bbox=(x, y, x + w, y + h), all_screens=True) + pil_image = image_grabber().grab(bbox=(x, y, x + w, y + h), all_screens=True) return cv2.cvtColor(np.array(pil_image), cv2.COLOR_RGB2BGR) diff --git a/je_auto_control/linux_wayland/__init__.py b/je_auto_control/linux_wayland/__init__.py index d49b0f5b..ce073388 100644 --- a/je_auto_control/linux_wayland/__init__.py +++ b/je_auto_control/linux_wayland/__init__.py @@ -7,9 +7,16 @@ * **wtype** — keyboard input via the ``wlr-virtual-keyboard-v1`` protocol (works on wlroots compositors: sway, hyprland, river); * **ydotool** — keyboard + mouse via ``/dev/uinput`` (works on - GNOME / KDE / wlroots, but the daemon needs uinput permission); + GNOME / KDE / wlroots, but the daemon needs uinput permission, and + it must be **1.0 or newer**: 0.1.x answers this backend's argv with + exit code 0 and no events, so ``_ydotool_cli`` refuses it); +* **libei** — the compositor's own input-emulation protocol, reached + through the ``RemoteDesktop`` desktop portal (see ``libei`` and + ``oeffis``). Preferred where it comes up, since it emits without + spawning a process per keystroke; falls back to ydotool otherwise; * **grim** — screenshot via the ``wlr-screencopy`` protocol (wlroots); - ``gnome-screenshot`` is used as a fallback on GNOME / KDE. + ``gnome-screenshot``, ``spectacle`` and the desktop portal back it up + (see ``capture`` and ``portal``). Each helper module probes for the matching binary lazily, so importing this package on a non-Linux host (e.g. CI on Windows / macOS) does not @@ -24,17 +31,18 @@ select_display_server, ) from je_auto_control.linux_wayland._select_input import ( - select_input_backend, + active_backend, select_input_backend, ) from je_auto_control.linux_wayland.libei import ( - LibeiBackend, LibeiUnavailable, get_default_backend, + LibeiBackend, LibeiUnavailable, connected_backend, get_default_backend, ) __all__ = [ "LibeiBackend", "LibeiUnavailable", "WAYLAND_GRIM", "WAYLAND_WTYPE", "WAYLAND_YDOTOOL", - "binary_path", "get_default_backend", "is_wayland_session", + "active_backend", "binary_path", "connected_backend", + "get_default_backend", "is_wayland_session", "missing_dependencies", "select_display_server", "select_input_backend", ] diff --git a/je_auto_control/linux_wayland/_ctypes_bind.py b/je_auto_control/linux_wayland/_ctypes_bind.py new file mode 100644 index 00000000..573bf434 --- /dev/null +++ b/je_auto_control/linux_wayland/_ctypes_bind.py @@ -0,0 +1,75 @@ +"""Shared ctypes plumbing for the Wayland native bindings. + +``libei`` and ``liboeffis`` are reached the same way: find the shared object +on the loader path, resolve a table of entry points with explicit prototypes, +and expose them by name so a test can substitute any object carrying the same +attributes. Keeping one copy matters more than the few lines it saves — a +second copy is a second place for a prototype to drift away from the header +it was written from. +""" +from __future__ import annotations + +import ctypes +import ctypes.util +from typing import Any, Iterable, Optional, Sequence, Tuple + + +#: ``(name, restype, argtypes)`` as ctypes wants them. +Prototype = Tuple[str, Any, tuple] + + +class BoundSymbols: + """Entry points resolved out of one shared object, addressed by name.""" + + def __init__(self, lib: ctypes.CDLL, prototypes: Iterable[Prototype], *, + unchecked: Sequence[str] = ()) -> None: + self._fn = {} + self.lib = lib + for name, restype, argtypes in prototypes: + entry = getattr(lib, name) + entry.restype = restype + entry.argtypes = argtypes + self._fn[name] = entry + for name in unchecked: + # Variadic entry points: ctypes has to infer each argument's type + # at the call site, so an argtypes tuple would be wrong here. + self._fn[name] = getattr(lib, name) + + def __getattr__(self, name: str): + try: + return self._fn[name] + except KeyError: + raise AttributeError(name) from None + + +def load_library(candidates: Iterable[str]) -> Optional[ctypes.CDLL]: + """Return the first of ``candidates`` that loads, or None if none do.""" + for name in candidates: + resolved = ctypes.util.find_library(name) + if resolved is None: + continue + try: + return ctypes.CDLL(resolved, use_errno=True) + except (OSError, RuntimeError): + continue + return None + + +def bind(candidates: Iterable[str], prototypes: Iterable[Prototype], *, + unchecked: Sequence[str] = ()) -> Optional[BoundSymbols]: + """Load a library and resolve its prototypes, or None if either fails. + + A missing entry point means the installed library is not the one these + prototypes were written for, which is a "not available" rather than an + error the caller should have to handle separately. + """ + lib = load_library(candidates) + if lib is None: + return None + try: + return BoundSymbols(lib, prototypes, unchecked=unchecked) + except AttributeError: + return None + + +__all__ = ["BoundSymbols", "Prototype", "bind", "load_library"] diff --git a/je_auto_control/linux_wayland/_dbus_client.py b/je_auto_control/linux_wayland/_dbus_client.py new file mode 100644 index 00000000..6a9d91a5 --- /dev/null +++ b/je_auto_control/linux_wayland/_dbus_client.py @@ -0,0 +1,625 @@ +"""A minimal D-Bus session-bus client, in the standard library alone. + +This exists because of one property of the XDG portal protocol: a portal call +returns only a *request handle*, and the answer arrives later as a ``Response`` +signal — **directed at the unique bus name that made the call**. The bus routes +a directed message to its destination and nowhere else, so a listener on a +second connection never receives it, whatever match rules it adds. + +That rules out the obvious shell-out. ``gdbus call`` and ``gdbus monitor`` are +two processes with two unique names, so the monitor is not the caller and the +Response is not addressed to it; measured against a real ``dbus-daemon``, it +sees the call go past and the answer never arrive. The only listener that +does see it is a full bus monitor (``dbus-monitor``, which asks the bus for +``BecomeMonitor``), and making a screenshot require permission to observe every +message on the user's session bus is a poor trade for a fallback tier. + +So the subscription and the call happen on one connection here, which is what +every portal client library does and what the protocol is designed for. The +cost is marshalling D-Bus by hand; the scope is kept to exactly what a portal +conversation needs — connect, authenticate, ``Hello``, ``AddMatch``, one method +call, and read signals until the matching one arrives. It is deliberately not a +general D-Bus binding: no properties, no introspection, no object export, no +file-descriptor passing (:mod:`je_auto_control.linux_wayland.oeffis` uses +liboeffis for the one call that needs that). + +Every failure is an :class:`AutoControlException` subclass, so the containment +boundaries that catch the family keep working. +""" +from __future__ import annotations + +import contextlib +import os +import socket +import struct +import time +from typing import Any, Dict, List, Optional, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: Message types, from the D-Bus specification. +METHOD_CALL = 1 +METHOD_RETURN = 2 +ERROR = 3 +SIGNAL = 4 + +#: Header field codes. +FIELD_PATH = 1 +FIELD_INTERFACE = 2 +FIELD_MEMBER = 3 +FIELD_ERROR_NAME = 4 +FIELD_REPLY_SERIAL = 5 +FIELD_DESTINATION = 6 +FIELD_SENDER = 7 +FIELD_SIGNATURE = 8 + +#: ``NO_REPLY_EXPECTED``. Declared for completeness; nothing here uses it, +#: because every call this makes is one whose answer is worth waiting for. +FLAG_NO_REPLY_EXPECTED = 1 + +BUS_NAME = "org.freedesktop.DBus" +BUS_PATH = "/org/freedesktop/DBus" +BUS_INTERFACE = "org.freedesktop.DBus" + +_ADDRESS_ENV = "DBUS_SESSION_BUS_ADDRESS" +_MAX_MESSAGE = 128 * 1024 * 1024 # the specification's own ceiling +_ALIGNMENT = {"y": 1, "b": 4, "n": 2, "q": 2, "i": 4, "u": 4, + "x": 8, "t": 8, "d": 8, "s": 4, "o": 4, "g": 1, + "a": 4, "(": 8, "{": 8, "v": 1, "h": 4} + + +class DBusError(AutoControlException): + """The session bus is unreachable, or answered with an error.""" + + +class Variant: + """A value with an explicit D-Bus type, for the ``a{sv}`` option maps.""" + + __slots__ = ("signature", "value") + + def __init__(self, signature: str, value: Any) -> None: + self.signature = signature + self.value = value + + def __repr__(self) -> str: + return f"Variant({self.signature!r}, {self.value!r})" + + +# --- marshalling ---------------------------------------------------------- + + +class _Writer: + """Little-endian marshaller. Alignment is relative to the message start.""" + + def __init__(self, offset: int = 0) -> None: + self._parts: List[bytes] = [] + self._length = offset + + def align(self, boundary: int) -> None: + padding = (-self._length) % boundary + if padding: + self._parts.append(bytes(padding)) + self._length += padding + + def raw(self, data: bytes) -> None: + self._parts.append(data) + self._length += len(data) + + def byte(self, value: int) -> None: + self.raw(struct.pack(" None: + self.align(4) + self.raw(struct.pack(" None: + encoded = value.encode("utf-8") + self.uint32(len(encoded)) + self.raw(encoded + b"\x00") + + def signature(self, value: str) -> None: + encoded = value.encode("ascii") + self.byte(len(encoded)) + self.raw(encoded + b"\x00") + + def value(self, sig: str, value: Any) -> None: + """Write one complete value of the given single signature.""" + _write_value(self, _SignatureReader(sig), value) + + @property + def data(self) -> bytes: + return b"".join(self._parts) + + def __len__(self) -> int: + return self._length + + +class _SignatureReader: + """Walks a signature string one complete type at a time.""" + + def __init__(self, text: str) -> None: + self.text = text + self.index = 0 + + def done(self) -> bool: + return self.index >= len(self.text) + + def peek(self) -> str: + return self.text[self.index] + + def take(self) -> str: + code = self.text[self.index] + self.index += 1 + return code + + def take_complete(self) -> str: + """Take one whole type, following containers to their close.""" + start = self.index + code = self.take() + if code == "a": + self.take_complete() + elif code in "({": + closing = ")" if code == "(" else "}" + while self.peek() != closing: + self.take_complete() + self.take() + return self.text[start:self.index] + + +def _write_value(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + code = reader.take() + if code == "y": + writer.byte(int(value)) + elif code == "b": + writer.uint32(1 if value else 0) + elif code == "u": + writer.uint32(int(value)) + elif code in "so": + writer.string(str(value)) + elif code == "g": + writer.signature(str(value)) + elif code == "v": + _write_variant(writer, value) + elif code == "a": + _write_array(writer, reader, value) + elif code in "({": + _write_struct(writer, reader, code, value) + else: + raise DBusError(f"cannot marshal D-Bus type {code!r}") + + +def _write_variant(writer: _Writer, value: Any) -> None: + variant = value if isinstance(value, Variant) else _guess_variant(value) + writer.signature(variant.signature) + writer.value(variant.signature, variant.value) + + +def _guess_variant(value: Any) -> Variant: + if isinstance(value, bool): + return Variant("b", value) + if isinstance(value, int): + return Variant("u", value) + if isinstance(value, str): + return Variant("s", value) + raise DBusError(f"no obvious D-Bus type for {type(value).__name__}") + + +def _write_array(writer: _Writer, reader: _SignatureReader, value: Any) -> None: + element = reader.take_complete() + writer.align(4) + # The length prefix counts the elements only, and it is written before + # the padding that aligns the first one — so the body is built separately + # at the offset it will actually occupy. + body_start = len(writer) + 4 + padding = (-body_start) % _ALIGNMENT.get(element[0], 1) + body = _Writer(body_start + padding) + items = value.items() if isinstance(value, dict) else value + for item in items: + body.align(_ALIGNMENT.get(element[0], 1)) + _write_value(body, _SignatureReader(element), item) + writer.uint32(len(body) - body_start - padding) + writer.raw(bytes(padding)) + writer.raw(body.data) + + +def _write_struct(writer: _Writer, reader: _SignatureReader, code: str, + value: Any) -> None: + closing = ")" if code == "(" else "}" + writer.align(8) + for item in value: + if reader.peek() == closing: + raise DBusError("too many members for this struct signature") + _write_value(writer, reader, item) + if reader.peek() != closing: + raise DBusError("too few members for this struct signature") + reader.take() + + +class _Reader: + """Little-endian demarshaller over one complete message.""" + + def __init__(self, data: bytes, offset: int = 0) -> None: + self.data = data + self.offset = offset + + def align(self, boundary: int) -> None: + self.offset += (-self.offset) % boundary + + def take(self, count: int) -> bytes: + if self.offset + count > len(self.data): + raise DBusError("truncated D-Bus message") + chunk = self.data[self.offset:self.offset + count] + self.offset += count + return chunk + + def byte(self) -> int: + return self.take(1)[0] + + def uint32(self) -> int: + self.align(4) + return struct.unpack(" str: + length = self.uint32() + text = self.take(length).decode("utf-8", errors="replace") + self.take(1) + return text + + def signature(self) -> str: + length = self.byte() + text = self.take(length).decode("ascii", errors="replace") + self.take(1) + return text + + def value(self, sig: str) -> Any: + return _read_value(self, _SignatureReader(sig)) + + +def _read_value(reader: _Reader, signature: _SignatureReader) -> Any: + code = signature.take() + if code == "y": + return reader.byte() + if code == "b": + return bool(reader.uint32()) + if code == "u": + return reader.uint32() + if code in "so": + return reader.string() + if code == "g": + return reader.signature() + if code == "v": + return reader.value(reader.signature()) + if code == "a": + return _read_array(reader, signature) + if code in "({": + return _read_struct(reader, signature, code) + raise DBusError(f"cannot demarshal D-Bus type {code!r}") + + +def _read_array(reader: _Reader, signature: _SignatureReader) -> Any: + element = signature.take_complete() + length = reader.uint32() + reader.align(_ALIGNMENT.get(element[0], 1)) + end = reader.offset + length + items = [] + while reader.offset < end: + reader.align(_ALIGNMENT.get(element[0], 1)) + items.append(_read_value(reader, _SignatureReader(element))) + if element.startswith("{"): + return dict(items) + return items + + +def _read_struct(reader: _Reader, signature: _SignatureReader, + code: str) -> tuple: + closing = ")" if code == "(" else "}" + reader.align(8) + members = [] + while signature.peek() != closing: + members.append(_read_value(reader, signature)) + signature.take() + return tuple(members) + + +class Message: + """One decoded D-Bus message: the header fields that matter, and the body.""" + + __slots__ = ("type", "serial", "fields", "body") + + def __init__(self, message_type: int, serial: int, + fields: Dict[int, Any], body: List[Any]) -> None: + self.type = message_type + self.serial = serial + self.fields = fields + self.body = body + + @property + def path(self) -> str: + return self.fields.get(FIELD_PATH, "") + + @property + def interface(self) -> str: + return self.fields.get(FIELD_INTERFACE, "") + + @property + def member(self) -> str: + return self.fields.get(FIELD_MEMBER, "") + + @property + def reply_serial(self) -> int: + return self.fields.get(FIELD_REPLY_SERIAL, 0) + + @property + def error_name(self) -> str: + return self.fields.get(FIELD_ERROR_NAME, "") + + +# --- the connection ------------------------------------------------------- + + +def session_address() -> Optional[str]: + """The session bus address, or None when this process has no session bus.""" + address = os.environ.get(_ADDRESS_ENV, "").strip() + return address or None + + +def is_available() -> bool: + """Whether a session bus address is set, so connecting is worth trying.""" + return session_address() is not None + + +def _socket_target(address: str) -> Tuple[str, bool]: + """Pick a connectable ``unix:`` transport out of a bus address. + + :return: the socket path and whether it is in the abstract namespace. + """ + for candidate in address.split(";"): + if not candidate.startswith("unix:"): + continue + fields = [part.split("=", 1) + for part in candidate[len("unix:"):].split(",") + if "=" in part] + options = dict(fields) + if "path" in options: + return options["path"], False + if "abstract" in options: + return options["abstract"], True + raise DBusError(f"no usable unix transport in {address!r}") + + +class SessionBus: + """An authenticated connection to the session bus, as a context manager.""" + + def __init__(self, address: Optional[str] = None) -> None: + self.address = address or session_address() + self.unique_name = "" + self._socket: Optional[socket.socket] = None + self._serial = 0 + self._buffer = b"" + #: Messages read while waiting for a method reply. A portal's Response + #: signal and its method return come from two different senders, so + #: the bus gives no ordering between them and the signal can arrive + #: first — dropping it here would be a wait that never ends. + self._queued: List[Message] = [] + + # --- lifecycle -------------------------------------------------------- + + def __enter__(self) -> "SessionBus": + self.connect() + return self + + def __exit__(self, *_exception: Any) -> None: + self.close() + + def connect(self) -> None: + """Open the socket, authenticate, and say ``Hello``.""" + if self.address is None: + raise DBusError( + f"{_ADDRESS_ENV} is not set, so there is no session bus to " + f"reach the desktop portal on", + ) + path, abstract = _socket_target(self.address) + try: + self._socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + self._socket.connect(("\0" + path) if abstract else path) + except OSError as error: + self.close() + raise DBusError(f"cannot reach the session bus: {error}") from error + self._authenticate() + self.unique_name = self.call( + BUS_NAME, BUS_PATH, BUS_INTERFACE, "Hello", "", [])[0] + + def close(self) -> None: + """Drop the connection; never raise from teardown.""" + if self._socket is not None: + with contextlib.suppress(OSError): + self._socket.close() + self._socket = None + + @property + def sender_token(self) -> str: + """This connection's name as the portal spells it inside object paths.""" + return self.unique_name.lstrip(":").replace(".", "_") + + # --- authentication --------------------------------------------------- + + def _authenticate(self) -> None: + """The SASL EXTERNAL handshake, which is a uid in hex over a socket.""" + uid = str(os.getuid()).encode("ascii") + self._send_raw(b"\x00AUTH EXTERNAL " + uid.hex().encode("ascii") + + b"\r\n") + reply = self._read_line() + if not reply.startswith("OK"): + raise DBusError(f"the session bus refused authentication: {reply}") + self._send_raw(b"BEGIN\r\n") + + def _send_raw(self, data: bytes) -> None: + if self._socket is None: + raise DBusError("the session bus connection is closed") + try: + self._socket.sendall(data) + except OSError as error: + raise DBusError(f"writing to the session bus failed: {error}") from error + + def _read_line(self, timeout: float = 10.0) -> str: + """Read one CRLF-terminated line of the auth conversation.""" + deadline = time.monotonic() + timeout + while b"\r\n" not in self._buffer: + self._fill(deadline - time.monotonic()) + line, _, self._buffer = self._buffer.partition(b"\r\n") + return line.decode("utf-8", errors="replace") + + # --- reading ---------------------------------------------------------- + + def _fill(self, timeout: float) -> None: + """Read whatever is available, or fail once the deadline has passed.""" + if self._socket is None: + raise DBusError("the session bus connection is closed") + if timeout <= 0: + raise DBusError("the session bus did not answer in time") + self._socket.settimeout(timeout) + try: + chunk = self._socket.recv(65536) + except socket.timeout as error: + raise DBusError("the session bus did not answer in time") from error + except OSError as error: + raise DBusError(f"reading from the session bus failed: {error}") from error + if not chunk: + raise DBusError("the session bus closed the connection") + self._buffer += chunk + + def read_message(self, deadline: float) -> Message: + """Read one whole message, or fail when the deadline passes.""" + while len(self._buffer) < 16: + self._fill(deadline - time.monotonic()) + endian, message_type, _flags, _version = struct.unpack( + " _MAX_MESSAGE: + raise DBusError("the session bus sent an implausibly large message") + while len(self._buffer) < total: + self._fill(deadline - time.monotonic()) + raw, self._buffer = self._buffer[:total], self._buffer[total:] + return _decode(raw, message_type, serial, fields_length, header_end) + + # --- writing ---------------------------------------------------------- + + def _next_serial(self) -> int: + self._serial += 1 + return self._serial + + def send(self, message_type: int, fields: Dict[int, Tuple[str, Any]], + signature: str, body: List[Any], flags: int = 0) -> int: + """Marshal and send one message; return its serial.""" + payload = _Writer() + for code, value in body_pairs(signature, body): + payload.value(code, value) + serial = self._next_serial() + header = _Writer() + header.raw(struct.pack(" List[Any]: + """Make a method call and return the reply body.""" + fields = { + FIELD_PATH: ("o", path), + FIELD_INTERFACE: ("s", interface), + FIELD_MEMBER: ("s", member), + FIELD_DESTINATION: ("s", destination), + } + if signature: + fields[FIELD_SIGNATURE] = ("g", signature) + serial = self.send(METHOD_CALL, fields, signature, body) + deadline = time.monotonic() + timeout + while True: + message = self.read_message(deadline) + if message.reply_serial != serial: + self._queued.append(message) + continue + if message.type == ERROR: + detail = message.body[0] if message.body else "" + raise DBusError(f"{message.error_name}: {detail}".strip(": ")) + if message.type == METHOD_RETURN: + return message.body + + def add_match(self, rule: str) -> None: + """Subscribe to signals, and wait for the bus to confirm the rule. + + Waiting matters twice over: a malformed rule is reported rather than + silently never matching, and the round trip proves the subscription is + in place before the call that provokes the signal is made. + """ + self.call(BUS_NAME, BUS_PATH, BUS_INTERFACE, "AddMatch", "s", [rule]) + + def wait_for_signal(self, paths: List[str], interface: str, member: str, + timeout: float) -> List[Any]: + """Read until the signal we subscribed to arrives, or time out.""" + def matches(message: Message) -> bool: + return (message.type == SIGNAL and message.member == member + and message.interface == interface + and message.path in paths) + + for index, message in enumerate(self._queued): + if matches(message): + del self._queued[index] + return message.body + self._queued.clear() + deadline = time.monotonic() + max(0.0, timeout) + while True: + message = self.read_message(deadline) + if matches(message): + return message.body + + +def body_pairs(signature: str, body: List[Any]): + """Pair each top-level type in a signature with its argument.""" + reader = _SignatureReader(signature) + for value in body: + if reader.done(): + raise DBusError("more arguments than the signature declares") + yield reader.take_complete(), value + if not reader.done(): + raise DBusError("fewer arguments than the signature declares") + + +def _decode(raw: bytes, message_type: int, serial: int, fields_length: int, + header_end: int) -> Message: + """Turn one complete message's bytes into a :class:`Message`.""" + reader = _Reader(raw, 12) + fields: Dict[int, Any] = {} + reader.uint32() # the header array's own length, already read + end = 16 + fields_length + while reader.offset < end: + reader.align(8) + code = reader.byte() + fields[code] = reader.value(reader.signature()) + reader.offset = header_end + ((-header_end) % 8) + body: List[Any] = [] + signature = fields.get(FIELD_SIGNATURE, "") + if signature: + walker = _SignatureReader(signature) + while not walker.done(): + body.append(_read_value(reader, _SignatureReader( + walker.take_complete()))) + return Message(message_type, serial, fields, body) + + +__all__ = [ + "BUS_INTERFACE", "BUS_NAME", "BUS_PATH", "DBusError", "ERROR", + "METHOD_CALL", "METHOD_RETURN", "Message", "SIGNAL", "SessionBus", + "Variant", "is_available", "session_address", +] diff --git a/je_auto_control/linux_wayland/_detect.py b/je_auto_control/linux_wayland/_detect.py index e7e00d61..98bfff4d 100644 --- a/je_auto_control/linux_wayland/_detect.py +++ b/je_auto_control/linux_wayland/_detect.py @@ -13,7 +13,18 @@ WAYLAND_WTYPE = "wtype" WAYLAND_YDOTOOL = "ydotool" + +# Screen capture is the one place where no single tool covers Wayland: +# grim speaks wlr-screencopy (sway / Hyprland / river), GNOME answers +# through gnome-screenshot and KDE through spectacle. See +# :mod:`je_auto_control.linux_wayland.capture` for the order they are tried in. WAYLAND_GRIM = "grim" +WAYLAND_GNOME_SCREENSHOT = "gnome-screenshot" +WAYLAND_SPECTACLE = "spectacle" +WAYLAND_WLR_RANDR = "wlr-randr" +# xdg-desktop-portal needs no binary at all: je_auto_control.linux_wayland. +# portal speaks D-Bus itself, because the portal's answer is directed at the +# connection that asked and no separate CLI process can be that connection. _ENV_OVERRIDE = "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER" @@ -59,7 +70,8 @@ def missing_dependencies(required: Iterable[str]) -> List[str]: __all__ = [ - "WAYLAND_GRIM", "WAYLAND_WTYPE", "WAYLAND_YDOTOOL", - "binary_path", "is_wayland_session", "missing_dependencies", - "select_display_server", + "WAYLAND_GNOME_SCREENSHOT", "WAYLAND_GRIM", + "WAYLAND_SPECTACLE", "WAYLAND_WLR_RANDR", "WAYLAND_WTYPE", + "WAYLAND_YDOTOOL", "binary_path", "is_wayland_session", + "missing_dependencies", "select_display_server", ] diff --git a/je_auto_control/linux_wayland/_layout.py b/je_auto_control/linux_wayland/_layout.py new file mode 100644 index 00000000..fdf82dff --- /dev/null +++ b/je_auto_control/linux_wayland/_layout.py @@ -0,0 +1,83 @@ +"""The output layout's origin, for the input paths that must translate into it. + +Capture and input do not share a coordinate space on Wayland, and the gap is +the layout origin: :func:`je_auto_control.linux_wayland.screen.grab_image` +returns the whole output layout, whose top-left pixel is +:func:`~je_auto_control.linux_wayland.screen.layout_origin` and only ``(0, 0)`` +while every output sits at a non-negative position. Put a monitor left of the +primary one and the origin goes negative, while both input transports address +a space that starts at that corner: + +* libei advertises regions with ``uint32`` offsets, so it *cannot* describe a + point left of or above the origin; +* ``ydotool mousemove --absolute`` drives the cursor into the corner the + compositor clamps to and then moves relative to it, and that corner is the + layout's top-left. + +Both therefore need the same subtraction, which is why it lives here rather +than in either of them. It is a separate module because +:mod:`je_auto_control.linux_wayland.screen` imports Pillow at module scope and +the input paths must keep working on a host without it — a missing capture +tool means no correction, never a failed move. +""" +from __future__ import annotations + +import time +from typing import Dict, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + +#: How long a reading stays good for. +#: +#: :mod:`je_auto_control.linux_wayland.screen` deliberately caches nothing — +#: an output can move between two captures and a stale layout is worse than +#: the ``shutil.which`` miss it costs. That reasoning does not survive the move +#: to the input path: the ydotool backend consults this on *every* absolute +#: move, so an uncached lookup spawns a ``wlr-randr`` process per move and +#: doubles the cost of a path that already spawns one. A window this short is +#: orders of magnitude below any real monitor rearrangement and orders of +#: magnitude above the gap between two moves in a drag. +_CACHE_SECONDS = 1.0 + +#: ``{"origin": (monotonic_stamp, (x, y))}``. A dict rather than a pair of +#: module globals so refreshing it needs no ``global`` statement. +_CACHE: Dict[str, Tuple[float, Tuple[int, int]]] = {} + + +def layout_origin() -> Tuple[int, int]: + """Return the layout coordinate of the capture's top-left pixel. + + ``(0, 0)`` whenever the answer cannot be obtained — no ``wlr-randr``, no + Pillow, no capture tool — which is also the correct answer for every + layout those hosts are likely to have, and leaves the caller sending the + coordinate it would have sent before this translation existed. + + Cached for :data:`_CACHE_SECONDS`; see there for why this one caches and + the capture path does not. + + :return: (origin_x, origin_y) + """ + now = time.monotonic() + cached = _CACHE.get("origin") + if cached is not None and now - cached[0] < _CACHE_SECONDS: + return cached[1] + origin = _read_layout_origin() + _CACHE["origin"] = (now, origin) + return origin + + +def _read_layout_origin() -> Tuple[int, int]: + """Ask the screen backend, tolerating every way it can be unavailable.""" + try: + from je_auto_control.linux_wayland import screen + return screen.layout_origin() + except (ImportError, OSError, AutoControlException): + return (0, 0) + + +def reset_cache() -> None: + """Drop the cached reading, so the next call measures again.""" + _CACHE.clear() + + +__all__ = ["layout_origin", "reset_cache"] diff --git a/je_auto_control/linux_wayland/_select_input.py b/je_auto_control/linux_wayland/_select_input.py index a37b66c7..60bff5ea 100644 --- a/je_auto_control/linux_wayland/_select_input.py +++ b/je_auto_control/linux_wayland/_select_input.py @@ -46,4 +46,40 @@ def _libei_loadable() -> bool: return False -__all__ = ["select_input_backend"] +def active_backend(): + """Return a connected libei backend, or None to use the ydotool CLI. + + The single entry point ``keyboard`` and ``mouse`` use. It lives here + rather than in :mod:`libei` so the dependency runs one way — this module + decides which input path is wanted, :mod:`libei` only knows how to bring + one up. None is the answer on any host where libei is absent, the + portal declines, or the handshake does not complete. + """ + try: + if select_input_backend() != "libei": + return None + from je_auto_control.linux_wayland.libei import connected_backend + return connected_backend() + except (ImportError, OSError, RuntimeError): + return None + + +def emitted(backend, send) -> bool: + """Run one emission on ``backend``; False means the CLI has to take it. + + A backend that finished its handshake can still refuse a single + emission: the compositor pauses a device, or the session ends between + two calls. :mod:`libei` is documented as the fast path and never the + only one, so a refusal falls through to the ``ydotool`` / ``wtype`` + shims here rather than reaching the caller — and if those are missing + too, *they* raise. Nothing is swallowed; the failure just changes hands. + """ + from je_auto_control.linux_wayland.libei import LibeiUnavailable + try: + send(backend) + except LibeiUnavailable: + return False + return True + + +__all__ = ["active_backend", "emitted", "select_input_backend"] diff --git a/je_auto_control/linux_wayland/_ydotool_cli.py b/je_auto_control/linux_wayland/_ydotool_cli.py new file mode 100644 index 00000000..1bdb1ab5 --- /dev/null +++ b/je_auto_control/linux_wayland/_ydotool_cli.py @@ -0,0 +1,134 @@ +"""Which ydotool command line is installed — and refusing the one that lies. + +ydotool 1.0 replaced its command line wholesale. Everything this backend +builds arrived in that release: ``mousemove --absolute``, ``mousemove +--wheel``, hex button bitmasks for ``click`` (so a press and a release can be +sent separately, which is what makes a drag possible), and ``key CODE:STATE`` +taking numeric evdev codes instead of key names. + +The 0.1.x series is still what ``apt install ydotool`` — the hint this +backend used to print — installs on Debian bookworm and on every current +Ubuntu. It does not merely reject the newer argv. Measured against a real +uinput device (``docker/ydotool_verify.py`` runs the same comparison in CI): + +=========================== ==== ======================================= +AutoControl's call rc what reached the kernel +=========================== ==== ======================================= +``click 0xc0`` (left) 0 BTN_LEFT down+up — right by coincidence +``click 0x40`` (press only) 0 *nothing* +``click 0xc1`` (right) 0 *nothing* +``mousemove --absolute ...`` 0 *nothing* — "unrecognised option" +``mousemove --wheel ...`` 0 *nothing* — "unrecognised option" +``key 30:1`` 0 *nothing* +=========================== ==== ======================================= + +Every one exits **0**, including the two that print ``unrecognised option``. +The backends run ydotool with ``check=True``, so a non-zero status is the +only thing that would raise — which means on those distributions a script +clicked nothing, typed nothing and moved nothing while every call reported +success. A silent no-op is the one failure mode a GUI automation library +cannot ship, so the legacy CLI is refused up front instead. + +Neither series implements ``--version``, and 1.x needs its daemon running +before it will answer ``--help``, so the probe uses the one thing both print +without a daemon and without side effects: the no-argument command list. +""" +from __future__ import annotations + +import subprocess # nosec B404 # reason: argv-list, path from shutil.which, no shell +import threading +from typing import Dict, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: ydotool 0.1.x listed a ``recorder`` command; 1.0 dropped it. +_LEGACY_COMMAND = "recorder" + +#: 1.x is daemon-only and advertises the socket override in its usage banner. +#: 0.1.x, which talks to ``/dev/uinput`` directly, has no such line. +_MODERN_MARKER = "ydotool_socket" + +MODERN = "modern" +LEGACY = "legacy" +UNKNOWN = "unknown" + +#: Probing costs a subprocess, and mouse / key dispatch must not pay that per +#: event, so the answer is cached per resolved binary path. +_cache: Dict[str, str] = {} +_cache_lock = threading.Lock() + +_PROBE_TIMEOUT = 5.0 + +_LEGACY_HINT = ( + "ydotool 0.1.x is installed at {path}, and its command line cannot " + "express what this backend needs: absolute cursor moves, separate " + "button press and release edges, wheel events, and numeric evdev key " + "codes all arrived in ydotool 1.0. Worse, 0.1.x exits 0 while emitting " + "nothing for those arguments, so every call would report success and do " + "nothing. Install ydotool 1.0 or newer (Debian: from unstable, which " + "packages 1.0.4; Arch: `pacman -S ydotool`; Fedora: `dnf install " + "ydotool`; or build from https://github.com/ReimuNotMoe/ydotool), or " + "set JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 to drive XWayland instead." +) + + +def _probe(path: str) -> str: + """Classify the installed ydotool by its no-argument usage banner.""" + try: + completed = subprocess.run( # nosec B603 # nosemgrep + [path], capture_output=True, timeout=_PROBE_TIMEOUT, check=False, + ) + except (OSError, subprocess.SubprocessError): + # An unreadable probe is not evidence of the broken version; let the + # real call fail with its own message rather than blaming the CLI. + return UNKNOWN + banner = (completed.stdout + completed.stderr).decode( + "utf-8", errors="replace").lower() + if _MODERN_MARKER in banner: + return MODERN + if _LEGACY_COMMAND in banner.split(): + return LEGACY + return UNKNOWN + + +def cli_generation(path: str) -> str: + """Return ``MODERN`` / ``LEGACY`` / ``UNKNOWN`` for the ydotool at ``path``. + + The result is cached per path for the life of the process. + """ + with _cache_lock: + cached = _cache.get(path) + if cached is not None: + return cached + generation = _probe(path) + with _cache_lock: + _cache[path] = generation + return generation + + +def reject_legacy_cli(path: str) -> str: + """Return ``path``, or raise when it is the 0.1.x CLI that silently no-ops. + + ``UNKNOWN`` is allowed through on purpose: only the version measured to + fail silently is refused, so a future release that drops the markers this + probe reads is not blocked by a stale detector. + """ + if cli_generation(path) == LEGACY: + raise AutoControlException(_LEGACY_HINT.format(path=path)) + return path + + +def reset_cache(path: Optional[str] = None) -> None: + """Forget probe results — for tests, and after installing a new ydotool.""" + with _cache_lock: + if path is None: + _cache.clear() + else: + _cache.pop(path, None) + + +__all__ = [ + "LEGACY", "MODERN", "UNKNOWN", "cli_generation", "reject_legacy_cli", + "reset_cache", +] diff --git a/je_auto_control/linux_wayland/capture.py b/je_auto_control/linux_wayland/capture.py new file mode 100644 index 00000000..9f454737 --- /dev/null +++ b/je_auto_control/linux_wayland/capture.py @@ -0,0 +1,236 @@ +"""Wayland screen capture, tried against each compositor's own tool. + +No Wayland compositor exposes a readable root window the way X11 does, and +which helper *can* read the screen differs by desktop: wlroots compositors +(sway, Hyprland, river) implement ``wlr-screencopy``, which ``grim`` speaks; +GNOME answers through ``gnome-screenshot``; KDE through ``spectacle``. None of +them is guaranteed to be present, so ``xdg-desktop-portal`` backs them all up +(see :mod:`portal`), and an operator whose setup none of that fits can name +their own command. The tiers, in order: + +1. ``JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND`` — an explicit operator override. +2. ``grim`` · 3. ``gnome-screenshot`` · 4. ``spectacle``. +5. ``xdg-desktop-portal``, over the session bus. + +Only ``grim`` accepts a region itself. Every other tier returns the whole +screen, so :class:`Capture` reports which of the two happened and the caller +crops when it has to. +""" +from __future__ import annotations + +import contextlib +import os +import shlex +import subprocess # nosec B404 # reason: argv-list from a private allow-list, no shell +import tempfile +from dataclasses import dataclass +from typing import Callable, List, Optional, Sequence, Tuple + +from je_auto_control.linux_wayland import portal +from je_auto_control.linux_wayland._detect import ( + WAYLAND_GNOME_SCREENSHOT, WAYLAND_GRIM, WAYLAND_SPECTACLE, binary_path, +) +from je_auto_control.utils.exception.exceptions import AutoControlScreenException + + +CAPTURE_TIMEOUT = 15.0 + +#: Operator override: a full command line whose ``{output}`` placeholder is +#: replaced with a temporary PNG path. Split with ``shlex``, run without a +#: shell. Always a whole-screen capture — regions are cropped afterwards, so +#: the command never has to understand geometry. +CAPTURE_COMMAND_ENV = "JE_AUTOCONTROL_WAYLAND_CAPTURE_COMMAND" + +_OVERRIDE_LABEL = f"${CAPTURE_COMMAND_ENV}" +_PORTAL_LABEL = "xdg-desktop-portal" + +_MISSING_TOOL_HINT = ( + "No Wayland screen-capture tool found. Install the one your compositor " + "supports: grim (sway / Hyprland / river, or any wlr-screencopy " + "compositor), gnome-screenshot (GNOME), or spectacle (KDE). Installing " + "none of them, the xdg-desktop-portal fallback needs only a session bus " + "and is tried automatically. Or name " + "your own capture command in " + CAPTURE_COMMAND_ENV + ", using {output} " + "for the file to write. To capture through XWayland set " + "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11 — note XWayland cannot see " + "native Wayland windows, so that capture is blank for most of the desktop." +) + + +@dataclass(frozen=True) +class Capture: + """One capture's PNG bytes, plus whether the tool applied the region. + + ``region_applied`` is False whenever the helper could only grab the + whole screen, which tells the caller it still has to crop. + """ + + data: bytes + region_applied: bool + + +def run_tool(argv: List[str], *, timeout: float = CAPTURE_TIMEOUT) -> bytes: + """Run one capture helper and return its stdout. + + :param argv: absolute binary path plus arguments, never a shell string. + :param timeout: seconds before the helper is considered hung. + :return: the helper's stdout (empty bytes when it writes to a file). + """ + # argv comes from a private allow-list (grim / gnome-screenshot / + # spectacle / wlr-randr resolved through shutil.which), never user + # input; no shell=True. + try: + completed = subprocess.run(argv, check=True, # nosec B603 # nosemgrep + timeout=timeout, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except subprocess.CalledProcessError as error: + message = (error.stderr or b"").decode("utf-8", errors="replace") + raise AutoControlScreenException( + f"{argv[0]} exited {error.returncode}: {message.strip()}", + ) from error + except subprocess.TimeoutExpired as error: + raise AutoControlScreenException( + f"{argv[0]} timed out after {timeout}s", + ) from error + return completed.stdout or b"" + + +def geometry(screen_region: Sequence[int]) -> str: + """Format ``[x1, y1, x2, y2]`` as grim's ``-g`` argument (``"x,y WxH"``).""" + x1, y1, x2, y2 = (int(value) for value in screen_region) + if x2 <= x1 or y2 <= y1: + raise AutoControlScreenException( + f"screen_region must have positive width and height; got " + f"[{x1}, {y1}, {x2}, {y2}]", + ) + return f"{x1},{y1} {x2 - x1}x{y2 - y1}" + + +def _gnome_argv(executable: str, output_path: str) -> List[str]: + return [executable, "-f", output_path] + + +def _spectacle_argv(executable: str, output_path: str) -> List[str]: + # -b background (no GUI), -n no notification, -f full screen, -o output. + return [executable, "-b", "-n", "-f", "-o", output_path] + + +# Helpers that can only write a full-screen PNG to a file, in the order they +# are tried once grim is absent. +_FILE_TOOLS: Tuple[Tuple[str, Callable[[str, str], List[str]]], ...] = ( + (WAYLAND_GNOME_SCREENSHOT, _gnome_argv), + (WAYLAND_SPECTACLE, _spectacle_argv), +) + + +def _override_template() -> str: + """The operator's capture command line, or "" when unset.""" + return (os.environ.get(CAPTURE_COMMAND_ENV) or "").strip() + + +def _override_argv(output_path: str) -> List[str]: + """Build the operator's capture command for one output path. + + ``shlex.split`` then per-token substitution — never a shell, and never + string-concatenating the path into a command line, so a path containing + spaces or quotes stays one argument. + """ + template = _override_template() + argv = shlex.split(template) + if not argv: + raise AutoControlScreenException( + f"{CAPTURE_COMMAND_ENV} is set but empty", + ) + if not any("{output}" in token for token in argv): + raise AutoControlScreenException( + f"{CAPTURE_COMMAND_ENV} must contain {{output}}, the path the " + f"command should write the capture to; got {template!r}", + ) + return [token.replace("{output}", output_path) for token in argv] + + +def available_tool() -> Optional[str]: + """Return the capture tier that would be used, or None if there is none. + + Reported by the diagnostics bundle so an operator can see *why* a capture + failed without reproducing it. + """ + if _override_template(): + return _OVERRIDE_LABEL + for name in (WAYLAND_GRIM, *(tool for tool, _ in _FILE_TOOLS)): + if binary_path(name) is not None: + return name + return _PORTAL_LABEL if portal.is_available() else None + + +def _grim_capture(executable: str, + screen_region: Optional[Sequence[int]]) -> Capture: + argv = [executable] + if screen_region is not None: + argv.extend(["-g", geometry(screen_region)]) + argv.append("-") + data = run_tool(argv) + if not data: + raise AutoControlScreenException("grim produced no output") + return Capture(data, screen_region is not None) + + +def _write_to_temp_png(write: Callable[[str], None], label: str) -> Capture: + """Let ``write`` fill a temporary PNG, then read and delete it.""" + handle, output_path = tempfile.mkstemp(prefix="je_autocontrol_", + suffix=".png") + os.close(handle) + try: + write(output_path) + with open(output_path, "rb") as captured: + data = captured.read() + finally: + with contextlib.suppress(OSError): + os.unlink(output_path) + if not data: + raise AutoControlScreenException(f"{label} produced no output") + return Capture(data, False) + + +def _file_capture(executable: str, + build_argv: Callable[[str, str], List[str]]) -> Capture: + """Run a helper that writes a PNG to a path we choose.""" + return _write_to_temp_png( + lambda output_path: run_tool(build_argv(executable, output_path)), + executable, + ) + + +def _override_capture() -> Capture: + """Run the operator's own capture command.""" + return _write_to_temp_png( + lambda output_path: run_tool(_override_argv(output_path)), + _OVERRIDE_LABEL, + ) + + +def grab_png(screen_region: Optional[Sequence[int]] = None) -> Capture: + """Capture the screen as PNG bytes using the first tier that can. + + :param screen_region: ``[x1, y1, x2, y2]`` to capture, or None for all. + :return: the PNG bytes and whether the region was applied by the tier. + """ + if _override_template(): + return _override_capture() + grim = binary_path(WAYLAND_GRIM) + if grim is not None: + return _grim_capture(grim, screen_region) + for name, build_argv in _FILE_TOOLS: + executable = binary_path(name) + if executable is not None: + return _file_capture(executable, build_argv) + if portal.is_available(): + return Capture(portal.capture_png(), False) + raise AutoControlScreenException(_MISSING_TOOL_HINT) + + +__all__ = [ + "CAPTURE_COMMAND_ENV", "CAPTURE_TIMEOUT", "Capture", "available_tool", + "geometry", "grab_png", "run_tool", +] diff --git a/je_auto_control/linux_wayland/keyboard.py b/je_auto_control/linux_wayland/keyboard.py index 2ce527cd..9766e876 100644 --- a/je_auto_control/linux_wayland/keyboard.py +++ b/je_auto_control/linux_wayland/keyboard.py @@ -20,6 +20,8 @@ from je_auto_control.linux_wayland._detect import ( WAYLAND_WTYPE, WAYLAND_YDOTOOL, binary_path, ) +from je_auto_control.linux_wayland._select_input import emitted +from je_auto_control.linux_wayland._ydotool_cli import reject_legacy_cli from je_auto_control.utils.exception.exceptions import AutoControlException @@ -28,9 +30,11 @@ "Install with your package manager (e.g. `apt install wtype`)." ) _INSTALL_HINT_YDOTOOL = ( - "ydotool is required for Wayland key events. " - "Install with your package manager (e.g. `apt install ydotool`) " - "and ensure ydotoold is running with /dev/uinput permission." + "ydotool 1.0 or newer is required for Wayland key events. " + "Install it with your package manager (Arch: `pacman -S ydotool`; " + "Fedora: `dnf install ydotool`; Debian: from unstable — trixie ships no " + "package and bookworm's 0.1.8 is too old) and ensure ydotoold is running " + "with /dev/uinput permission." ) @@ -41,6 +45,11 @@ def _require(name: str, hint: str) -> str: return path +def _require_ydotool() -> str: + """Resolve ydotool, refusing the 0.1.x CLI that fails silently.""" + return reject_legacy_cli(_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL)) + + def _run(argv: list, *, timeout: float = 5.0) -> None: # argv comes from a private allow-list (wtype / ydotool absolute # paths via shutil.which), never user input; no shell=True. @@ -64,38 +73,29 @@ def press_key(keycode: int) -> None: """Press one evdev key code. Uses libei when available, ydotool otherwise.""" _validate_keycode(keycode) libei = _try_libei() - if libei is not None: - libei.press_key(int(keycode)) + if libei is not None and emitted( + libei, lambda device: device.press_key(int(keycode))): return time.sleep(0.01) - _run([_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL), - "key", f"{int(keycode)}:1"]) + _run([_require_ydotool(), "key", f"{int(keycode)}:1"]) def release_key(keycode: int) -> None: """Release one evdev key code. Uses libei when available, ydotool otherwise.""" _validate_keycode(keycode) libei = _try_libei() - if libei is not None: - libei.release_key(int(keycode)) + if libei is not None and emitted( + libei, lambda device: device.release_key(int(keycode))): return time.sleep(0.01) - _run([_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL), - "key", f"{int(keycode)}:0"]) + _run([_require_ydotool(), "key", f"{int(keycode)}:0"]) def _try_libei(): """Return a connected :class:`LibeiBackend`, or None when CLI should win.""" try: - from je_auto_control.linux_wayland import select_input_backend - if select_input_backend() != "libei": - return None - from je_auto_control.linux_wayland.libei import get_default_backend - backend = get_default_backend() - if backend is None: - return None - backend.connect() - return backend + from je_auto_control.linux_wayland._select_input import active_backend + return active_backend() except (ImportError, RuntimeError, OSError): return None @@ -111,12 +111,37 @@ def hotkey(keycodes: Iterable[int]) -> None: codes = [int(code) for code in keycodes] if not codes: raise ValueError("hotkey requires at least one keycode") - args = [_require(WAYLAND_YDOTOOL, _INSTALL_HINT_YDOTOOL), "key"] + libei = _try_libei() + if libei is not None and _libei_chord(libei, codes): + return + args = [_require_ydotool(), "key"] args.extend(f"{code}:1" for code in codes) args.extend(f"{code}:0" for code in reversed(codes)) _run(args) +def _libei_chord(libei, codes: list) -> bool: + """Send a whole chord through libei; False means the CLI has to redo it. + + Whatever went down is released in reverse before giving up, so a refusal + part-way through cannot leave a modifier held. The CLI then replays the + chord from a clean state — and if it was a *release* that was refused, + replaying is also what unsticks the key. + """ + pressed = [] + taken = True + for code in codes: + if not emitted(libei, lambda device, key=code: device.press_key(key)): + taken = False + break + pressed.append(code) + for code in reversed(pressed): + if not emitted(libei, + lambda device, key=code: device.release_key(key)): + taken = False + return taken + + def write(text: str) -> None: """Type a UTF-8 string via wtype (Unicode-aware, no key-code conversion).""" if not isinstance(text, str): diff --git a/je_auto_control/linux_wayland/libei.py b/je_auto_control/linux_wayland/libei.py index 3fd87223..b37491ce 100644 --- a/je_auto_control/linux_wayland/libei.py +++ b/je_auto_control/linux_wayland/libei.py @@ -1,242 +1,610 @@ -"""ctypes binding for libei — Wayland's HID-layer input emulation library. - -libei runs at the kernel-input-event layer, which is *much* faster -than spawning ``wtype`` / ``ydotool`` per keystroke (a few µs vs. a -few ms). Compositors that expose the -``zwp_input_method_unstable_v2`` portal or the libei sender protocol -let an unprivileged client emit pointer / key events without the -sandbox veto. - -The binding is opt-in: callers go through :class:`LibeiBackend`, and -:func:`LibeiBackend.is_available` probes for ``libei.so.1`` on the -loader path. When the library is missing, callers should fall through -to the CLI shims in :mod:`keyboard` / :mod:`mouse` — that's what the -backend selector in :func:`select_input_backend` does. - -This module deliberately stays pure-Python so the test suite can run -on hosts without libei: the ctypes bindings are introduced lazily -through :class:`_LibeiSymbols`, and every external call is wrapped in -a method that's easy to mock. +"""ctypes binding for libei — Wayland's input-emulation protocol. + +libei is not a "call a function and a key is pressed" library. A sender has +to complete a handshake before it may emit anything: + +1. open a backend — an EIS **file descriptor** from the desktop portal + (:mod:`oeffis`), or a socket path where a compositor exposes one; +2. pump ``ei_dispatch`` / ``ei_get_event`` and answer what arrives; +3. on ``SEAT_ADDED``, bind the capabilities this sender wants; +4. on ``DEVICE_ADDED``, keep the device that carries each capability — + **devices only ever come from an event**, never from the context; +5. on ``DEVICE_RESUMED``, call ``ei_device_start_emulating``; +6. per emission, send the event *and* ``ei_device_frame``, or nothing is + delivered. + +The previous binding did none of that. It held only the ``struct ei *`` +context and passed it to entry points that take a ``struct ei_device *`` — +pointer type confusion in a C library — and without ``frame()`` no event +would have arrived even had the pointers been right. + +**Fail-closed by construction.** Every failure below raises +:class:`LibeiUnavailable`, which ``keyboard`` / ``mouse`` already treat as +"use the ydotool CLI". The handshake is bounded by a deadline and the result +is cached per process (:func:`connected_backend`), so a host where libei is +installed but unusable pays the probe once, not once per keystroke. + +The enum values are libei's, from ``libei.h``. They are the one part of this +module that a wrong guess would silently change — see ``Progress.md``. A +mismatch degrades safely rather than misfiring: capabilities that do not +match mean no device ever reports them, the handshake times out, and the CLI +takes over. """ from __future__ import annotations import ctypes -import ctypes.util +import os +import select import threading -from dataclasses import dataclass -from typing import Optional +import time +from typing import Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.linux_wayland import oeffis +from je_auto_control.linux_wayland._ctypes_bind import BoundSymbols, bind +from je_auto_control.linux_wayland._layout import layout_origin +from je_auto_control.utils.exception.exceptions import AutoControlException _LIBRARY_CANDIDATES = ("ei", "libei", "libei.so.1", "libei.so.0") +# enum ei_device_capability — a **bitmask**, not a sequence. Verified against +# libei.h (Debian 1.5.0-3 and upstream main, which agree on these six). +EI_DEVICE_CAP_POINTER = 1 << 0 +EI_DEVICE_CAP_POINTER_ABSOLUTE = 1 << 1 +EI_DEVICE_CAP_KEYBOARD = 1 << 2 +EI_DEVICE_CAP_TOUCH = 1 << 3 +EI_DEVICE_CAP_SCROLL = 1 << 4 +EI_DEVICE_CAP_BUTTON = 1 << 5 + +# enum ei_event_type — only the ones this sender has to answer. The block +# starts at 1 and increments implicitly through DEVICE_RESUMED before jumping +# to 90; the values below are that first run, verified against libei.h. +EI_EVENT_CONNECT = 1 +EI_EVENT_DISCONNECT = 2 +EI_EVENT_SEAT_ADDED = 3 +EI_EVENT_SEAT_REMOVED = 4 +EI_EVENT_DEVICE_ADDED = 5 +EI_EVENT_DEVICE_REMOVED = 6 +EI_EVENT_DEVICE_PAUSED = 7 +EI_EVENT_DEVICE_RESUMED = 8 + +#: Capabilities bound on every seat. Buttons and scroll ride along with the +#: pointer device wherever the compositor grants them. +_WANTED_CAPS = (EI_DEVICE_CAP_KEYBOARD, EI_DEVICE_CAP_POINTER_ABSOLUTE, + EI_DEVICE_CAP_BUTTON, EI_DEVICE_CAP_SCROLL) + +#: Without these two, half the public API would silently do nothing, so a +#: partial grant is treated as no grant at all. +_REQUIRED_CAPS = (EI_DEVICE_CAP_KEYBOARD, EI_DEVICE_CAP_POINTER_ABSOLUTE) + +#: Handshake budget once the EIS fd is in hand. The portal's own consent +#: wait already happened in :mod:`oeffis`; this is just protocol round trips. +HANDSHAKE_TIMEOUT = 3.0 + +#: One region of an absolute pointer's coordinate space: ``(x, y, w, h)``. +Region = Tuple[int, int, int, int] + +#: Ceiling on ``ei_device_get_region`` enumeration. libei ends the list with +#: NULL and a desktop has a handful of outputs; this only stops a library +#: whose contract differs from hanging the caller's mouse move forever. +_MAX_REGIONS = 64 + +#: One wheel click, in the units ``ei_device_scroll_discrete`` takes. From +#: libei.h: "A discrete scroll event is based logical scroll units (equivalent +#: to one mouse wheel click). The value for one scroll unit is 120." +SCROLL_UNIT = 120 + +_VOID = ctypes.c_void_p +_PROTOTYPES = ( + ("ei_new_sender", _VOID, (_VOID,)), + ("ei_unref", _VOID, (_VOID,)), + ("ei_setup_backend_fd", ctypes.c_int, (_VOID, ctypes.c_int)), + ("ei_setup_backend_socket", ctypes.c_int, (_VOID, ctypes.c_char_p)), + ("ei_get_fd", ctypes.c_int, (_VOID,)), + ("ei_dispatch", None, (_VOID,)), + ("ei_get_event", _VOID, (_VOID,)), + ("ei_event_unref", _VOID, (_VOID,)), + ("ei_event_get_type", ctypes.c_int, (_VOID,)), + ("ei_event_get_seat", _VOID, (_VOID,)), + ("ei_event_get_device", _VOID, (_VOID,)), + ("ei_device_ref", _VOID, (_VOID,)), + ("ei_device_unref", _VOID, (_VOID,)), + ("ei_device_has_capability", ctypes.c_bool, (_VOID, ctypes.c_int)), + ("ei_device_start_emulating", None, (_VOID, ctypes.c_uint32)), + ("ei_device_stop_emulating", None, (_VOID,)), + ("ei_device_frame", None, (_VOID, ctypes.c_uint64)), + ("ei_device_keyboard_key", None, (_VOID, ctypes.c_uint32, ctypes.c_bool)), + ("ei_device_pointer_motion_absolute", None, + (_VOID, ctypes.c_double, ctypes.c_double)), + ("ei_device_get_region", _VOID, (_VOID, ctypes.c_size_t)), + ("ei_region_get_x", ctypes.c_uint32, (_VOID,)), + ("ei_region_get_y", ctypes.c_uint32, (_VOID,)), + ("ei_region_get_width", ctypes.c_uint32, (_VOID,)), + ("ei_region_get_height", ctypes.c_uint32, (_VOID,)), + ("ei_device_button_button", None, (_VOID, ctypes.c_uint32, ctypes.c_bool)), + ("ei_device_scroll_discrete", None, + (_VOID, ctypes.c_int32, ctypes.c_int32)), + ("ei_now", ctypes.c_uint64, (_VOID,)), +) + + +class LibeiUnavailable(AutoControlException, RuntimeError): + """libei is missing, or a sender cannot be brought up on this session. + + Inherits both bases on purpose. ``AutoControlException`` is what every + containment boundary in the framework catches, and a sibling of it + escapes all of them — see that class's docstring. ``RuntimeError`` is + kept because the backend probes here and in ``_select_input`` were + written to catch it, and because "libei will not come up" is a runtime + environment fact rather than a caller mistake. + """ -class LibeiUnavailable(RuntimeError): - """Raised when libei isn't installed or the sender can't connect.""" +def _load_symbols() -> Optional[BoundSymbols]: + """Resolve every libei entry point, or None if one is missing. -@dataclass(frozen=True) -class _LibeiSymbols: - """Function pointers we resolve out of ``libei.so.*``.""" + ``ei_seat_bind_capabilities`` is bound unchecked because it is variadic: + ctypes has to infer each argument's type at the call site. + """ + return bind(_LIBRARY_CANDIDATES, _PROTOTYPES, + unchecked=("ei_seat_bind_capabilities",)) - lib: ctypes.CDLL - ei_new_sender: ctypes.CFUNCTYPE - ei_setup_backend_socket: ctypes.CFUNCTYPE - ei_device_keyboard_key: ctypes.CFUNCTYPE - ei_device_pointer_motion_absolute: ctypes.CFUNCTYPE - ei_device_button_button: ctypes.CFUNCTYPE - ei_device_scroll: ctypes.CFUNCTYPE - ei_unref: ctypes.CFUNCTYPE +def _default_socket_path() -> bytes: + """Where a compositor that exposes EIS on disk would put the socket.""" + runtime = os.environ.get("XDG_RUNTIME_DIR") + if not runtime: + return b"/run/user/1000/eis-0" + return f"{runtime}/eis-0".encode("utf-8") -def _try_load_library() -> Optional[ctypes.CDLL]: - """Probe ``libei.so.*`` on the loader path; return None if absent.""" - for name in _LIBRARY_CANDIDATES: - resolved = ctypes.util.find_library(name) - if resolved is None: - continue - try: - return ctypes.CDLL(resolved, use_errno=True) - except (OSError, RuntimeError): - continue - return None +def _in_any_region(regions: Sequence[Region], x: int, y: int) -> bool: + """Whether ``(x, y)`` falls inside one of ``regions``. -def _bind_symbols(lib: ctypes.CDLL) -> Optional[_LibeiSymbols]: - """Pull every libei function we use out of the shared object.""" - try: - new_sender = lib.ei_new_sender - setup_socket = lib.ei_setup_backend_socket - device_key = lib.ei_device_keyboard_key - device_motion = lib.ei_device_pointer_motion_absolute - device_button = lib.ei_device_button_button - device_scroll = lib.ei_device_scroll - unref = lib.ei_unref - except AttributeError: - return None - # Argument / return types match the upstream API. libei uses opaque - # handles + ints + uint32 booleans, so the binding stays compact. - new_sender.restype = ctypes.c_void_p - new_sender.argtypes = (ctypes.c_char_p,) - setup_socket.restype = ctypes.c_int - setup_socket.argtypes = (ctypes.c_void_p, ctypes.c_char_p) - device_key.restype = ctypes.c_int - device_key.argtypes = (ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32) - device_motion.restype = ctypes.c_int - device_motion.argtypes = (ctypes.c_void_p, ctypes.c_double, ctypes.c_double) - device_button.restype = ctypes.c_int - device_button.argtypes = (ctypes.c_void_p, ctypes.c_uint32, ctypes.c_uint32) - device_scroll.restype = ctypes.c_int - device_scroll.argtypes = (ctypes.c_void_p, ctypes.c_double, ctypes.c_double) - unref.restype = None - unref.argtypes = (ctypes.c_void_p,) - return _LibeiSymbols( - lib=lib, ei_new_sender=new_sender, - ei_setup_backend_socket=setup_socket, - ei_device_keyboard_key=device_key, - ei_device_pointer_motion_absolute=device_motion, - ei_device_button_button=device_button, - ei_device_scroll=device_scroll, ei_unref=unref, - ) + The right and bottom edges are outside, which is measured rather than + assumed: against a real EIS peer offering one ``(0, 0, 1920, 1080)`` + region, ``(1919, 1079)`` arrives and ``(1920, 1080)`` does not. + """ + return any(left <= x < left + width and top <= y < top + height + for left, top, width, height in regions) class LibeiBackend: - """Native libei sender — thread-safe, lazy initialisation. + """A libei sender that has completed the handshake and can emit input. - ``LibeiBackend()`` cheaply probes for libei but does NOT connect; - call :meth:`connect` to open the socket. Tests inject a fake - :class:`_LibeiSymbols` via the ``symbols=`` keyword so the - behaviour can be exercised without ctypes. + ``LibeiBackend()`` only probes for the library; :meth:`connect` performs + the portal request and the protocol handshake. Tests inject a fake + ``symbols=`` object exposing the same entry-point names. """ - def __init__(self, *, sender_name: bytes = b"je_auto_control", - symbols: Optional[_LibeiSymbols] = None) -> None: - self._sender_name = sender_name + def __init__(self, *, symbols: Optional[BoundSymbols] = None, + portal_connect: Optional[Callable[..., tuple]] = None) -> None: self._symbols = symbols if symbols is not None else _load_symbols() - self._handle: Optional[int] = None - self._lock = threading.Lock() + self._portal_connect = portal_connect or oeffis.connect_eis_fd + self._ei: Optional[int] = None + self._backend_open = False + self._handshake_complete = False + self._session = None + self._devices: Dict[int, int] = {} + self._emulating: Dict[int, bool] = {} + self._sequence = 0 + self._lock = threading.RLock() @property def is_available(self) -> bool: + """Whether libei itself resolved. Says nothing about the session.""" return self._symbols is not None - def connect(self, *, socket_path: Optional[bytes] = None) -> None: - """Open the libei sender socket (default: $XDG_RUNTIME_DIR/eis-0).""" + @property + def is_connected(self) -> bool: + """Whether the handshake finished and a device is emulating.""" + return self._ei is not None and self._has_required_devices() + + def connect(self, *, timeout: float = HANDSHAKE_TIMEOUT, + socket_path: Optional[bytes] = None) -> None: + """Open a backend and run the handshake through to a live device. + + :param timeout: seconds for the protocol handshake, once connected. + :param socket_path: bypass the portal and use this EIS socket. + """ if not self.is_available: - raise LibeiUnavailable( - "libei.so.* not found on the loader path", - ) + raise LibeiUnavailable("libei.so.* not found on the loader path") with self._lock: - if self._handle is not None: + if self.is_connected: return - sender = self._symbols.ei_new_sender(self._sender_name) + sender = self._symbols.ei_new_sender(None) if not sender: raise LibeiUnavailable("ei_new_sender returned NULL") - chosen_socket = socket_path or _default_socket_path() - rc = self._symbols.ei_setup_backend_socket(sender, chosen_socket) - if rc != 0: - self._symbols.ei_unref(sender) - raise LibeiUnavailable( - f"ei_setup_backend_socket returned {rc}", - ) - self._handle = sender + self._ei = sender + try: + self._open_backend(socket_path) + self._handshake(time.monotonic() + max(0.0, timeout)) + # Reaching a live device is what makes ei_unref safe on this + # libei, so _teardown reads this rather than guessing. + self._handshake_complete = True + except BaseException: + self._teardown() + raise def disconnect(self) -> None: + """Release the devices, the sender, and the portal session.""" with self._lock: - if self._handle is None: - return - if self._symbols is not None: - self._symbols.ei_unref(self._handle) - self._handle = None + self._teardown() + + # --- emission --------------------------------------------------------- def press_key(self, keycode: int) -> None: - """Send a keydown for one evdev key code via libei.""" - self._require_connected() - self._symbols.ei_device_keyboard_key(self._handle, int(keycode), 1) + """Send a keydown for one evdev key code.""" + self._emit(EI_DEVICE_CAP_KEYBOARD, lambda device: + self._symbols.ei_device_keyboard_key( + device, int(keycode), True)) def release_key(self, keycode: int) -> None: - self._require_connected() - self._symbols.ei_device_keyboard_key(self._handle, int(keycode), 0) + """Send a keyup for one evdev key code.""" + self._emit(EI_DEVICE_CAP_KEYBOARD, lambda device: + self._symbols.ei_device_keyboard_key( + device, int(keycode), False)) def set_position(self, x: int, y: int) -> None: - self._require_connected() - self._symbols.ei_device_pointer_motion_absolute( - self._handle, float(x), float(y), - ) + """Move the pointer to an absolute screen position. - def click_button(self, button_code: int) -> None: - """Press + release one BTN_* code (e.g. 272 for left click).""" - self._require_connected() - self._symbols.ei_device_button_button(self._handle, int(button_code), 1) - self._symbols.ei_device_button_button(self._handle, int(button_code), 0) + The coordinate is mapped into the device's own space first — see + :meth:`_region_point`, which is also where a point no region covers + is turned into a refusal instead of a silent no-op. + """ + self._emit(EI_DEVICE_CAP_POINTER_ABSOLUTE, lambda device: + self._symbols.ei_device_pointer_motion_absolute( + device, *self._region_point(device, x, y))) def press_button(self, button_code: int) -> None: - self._require_connected() - self._symbols.ei_device_button_button(self._handle, int(button_code), 1) + """Press one BTN_* code (272 is BTN_LEFT).""" + self._emit(EI_DEVICE_CAP_BUTTON, lambda device: + self._symbols.ei_device_button_button( + device, int(button_code), True)) def release_button(self, button_code: int) -> None: - self._require_connected() - self._symbols.ei_device_button_button(self._handle, int(button_code), 0) - - def scroll(self, dx: int, dy: int) -> None: - self._require_connected() - self._symbols.ei_device_scroll(self._handle, float(dx), float(dy)) + """Release one BTN_* code.""" + self._emit(EI_DEVICE_CAP_BUTTON, lambda device: + self._symbols.ei_device_button_button( + device, int(button_code), False)) - def _require_connected(self) -> None: - if not self.is_available: - raise LibeiUnavailable("libei not loaded") - if self._handle is None: - raise LibeiUnavailable( - "libei sender not connected; call connect() first", - ) + def click_button(self, button_code: int) -> None: + """Press then release one BTN_* code.""" + self.press_button(button_code) + self.release_button(button_code) + def scroll(self, dx: int, dy: int) -> None: + """Scroll by whole wheel clicks on either axis. + + libei measures discrete scroll in 120ths of a click — the same + convention as Windows' ``WHEEL_DELTA`` — so a raw detent count is a + 120th of the scroll the caller asked for. Measured against a real EIS + peer, which is also where libei's own "suspicious discrete event value + 1, did you mean 120?" bug warning showed up. + """ + self._emit(EI_DEVICE_CAP_SCROLL, lambda device: + self._symbols.ei_device_scroll_discrete( + device, int(dx) * SCROLL_UNIT, int(dy) * SCROLL_UNIT)) + + # --- handshake -------------------------------------------------------- + + def _open_backend(self, socket_path: Optional[bytes]) -> None: + """Attach the sender to an EIS fd from the portal, or to a socket. + + The portal is the route that works on GNOME and KDE. Where + liboeffis is not installed there is still the well-known socket + some compositors publish, so that is tried rather than giving up. + """ + if socket_path is None and not oeffis.is_available(): + socket_path = _default_socket_path() + if socket_path is not None: + code = self._symbols.ei_setup_backend_socket(self._ei, socket_path) + if code != 0: + raise LibeiUnavailable( + f"ei_setup_backend_socket returned {code}", + ) + self._backend_open = True + return + try: + eis_fd, session = self._portal_connect() + except oeffis.OeffisUnavailable as error: + raise LibeiUnavailable(str(error)) from error + self._session = session + code = self._symbols.ei_setup_backend_fd(self._ei, int(eis_fd)) + if code != 0: + raise LibeiUnavailable(f"ei_setup_backend_fd returned {code}") + self._backend_open = True + + def _handshake(self, deadline: float) -> None: + """Pump events until the devices we need are emulating.""" + while not self._has_required_devices(): + remaining = deadline - time.monotonic() + if remaining <= 0: + raise LibeiUnavailable( + "libei connected but no seat offered both a keyboard and " + "an absolute pointer within the handshake timeout", + ) + self._pump(min(remaining, 0.25)) + + def _pump(self, timeout: float) -> None: + """Dispatch libei and answer every event that is waiting.""" + poll_fd = int(self._symbols.ei_get_fd(self._ei)) + if poll_fd < 0: + raise LibeiUnavailable("ei_get_fd returned no pollable fd") + if timeout > 0: + ready, _, _ = select.select([poll_fd], [], [], timeout) + if not ready: + return + self._symbols.ei_dispatch(self._ei) + while True: + event = self._symbols.ei_get_event(self._ei) + if not event: + return + try: + self._on_event(event) + finally: + self._symbols.ei_event_unref(event) + + def _on_event(self, event: int) -> None: + """Answer one libei event.""" + event_type = int(self._symbols.ei_event_get_type(event)) + if event_type == EI_EVENT_SEAT_ADDED: + self._bind_seat(self._symbols.ei_event_get_seat(event)) + elif event_type == EI_EVENT_DEVICE_ADDED: + self._remember_device(self._symbols.ei_event_get_device(event)) + elif event_type == EI_EVENT_DEVICE_RESUMED: + self._start_emulating(self._symbols.ei_event_get_device(event)) + elif event_type in (EI_EVENT_DEVICE_PAUSED, EI_EVENT_DEVICE_REMOVED): + self._forget_device(self._symbols.ei_event_get_device(event)) + elif event_type == EI_EVENT_DISCONNECT: + raise LibeiUnavailable("the compositor disconnected the sender") + + def _bind_seat(self, seat: int) -> None: + """Ask a seat for the capabilities this sender emits. + + ``ei_seat_bind_capabilities`` is variadic and NULL-terminated, so the + arguments are passed as explicit ctypes values — an ``argtypes`` + tuple cannot describe a variadic call. + """ + if not seat: + return + args = [ctypes.c_void_p(seat)] + args.extend(ctypes.c_int(cap) for cap in _WANTED_CAPS) + args.append(ctypes.c_void_p(None)) + self._symbols.ei_seat_bind_capabilities(*args) + + def _remember_device(self, device: int) -> None: + """Keep a reference to a device for each capability it carries.""" + if not device: + return + kept = False + for cap in _WANTED_CAPS: + if not self._symbols.ei_device_has_capability(device, cap): + continue + if not kept: + self._symbols.ei_device_ref(device) + kept = True + self._devices[cap] = device + + def _start_emulating(self, device: int) -> None: + """Mark a resumed device ready and open its emulation sequence.""" + if not device or device not in self._devices.values(): + return + self._sequence += 1 + self._symbols.ei_device_start_emulating(device, self._sequence) + self._emulating[device] = True + + def _forget_device(self, device: int) -> None: + """Drop a paused or removed device; emissions then fail closed.""" + if not device: + return + self._emulating.pop(device, None) + stale = [cap for cap, known in self._devices.items() if known == device] + for cap in stale: + del self._devices[cap] + self._symbols.ei_device_unref(device) + + def _has_required_devices(self) -> bool: + return all(self._emulating.get(self._devices.get(cap, 0), False) + for cap in _REQUIRED_CAPS) + + # --- coordinate space ------------------------------------------------- + + def _device_regions(self, device: int) -> List[Region]: + """The regions this device accepts absolute motion in. + + libei reports them in its own space, offsets included: a device whose + region sits at ``x=1280`` takes ``1380`` for a point 100 pixels into + it, not ``100``. An empty list means the device declared none, and a + device with no regions accepts any coordinate — both measured against + a real EIS peer in ``docker/eis_verify.py``. + """ + regions: List[Region] = [] + for index in range(_MAX_REGIONS): + region = self._symbols.ei_device_get_region(device, index) + if not region: + break + regions.append(( + int(self._symbols.ei_region_get_x(region)), + int(self._symbols.ei_region_get_y(region)), + int(self._symbols.ei_region_get_width(region)), + int(self._symbols.ei_region_get_height(region)), + )) + return regions + + def _region_point(self, device: int, x: int, y: int) -> Tuple[float, float]: + """Map a layout coordinate into the space ``device`` accepts. + + **libei discards an absolute motion that lands in no region, and says + nothing about it** — no return code, no event, no log the caller can + see. ``set_position`` would return as though the pointer had moved. + That is the failure this method exists to prevent, and it is measured, + not inferred: against a real EIS peer, a point outside every region + produced no server-side event at all. + + The two spaces can genuinely differ. Region offsets are ``uint32``, so + no compositor *can* advertise a region left of or above the origin, + while this project's layout space starts at + :func:`je_auto_control.linux_wayland._layout.layout_origin` and + goes negative the moment a monitor sits left of the primary one — the + exact layout ``docker/wayland_verify.py`` exercises on the capture + side. Where the raw point misses, the origin-normalised one is tried, + which is the translation that keeps input and capture addressing the + same pixel. If that misses too, the caller gets a refusal and + ``_select_input.emitted`` hands the move to the ydotool path. + """ + regions = self._device_regions(device) + if not regions or _in_any_region(regions, x, y): + return (float(x), float(y)) + origin_x, origin_y = layout_origin() + moved_x, moved_y = x - origin_x, y - origin_y + if (origin_x or origin_y) and _in_any_region(regions, moved_x, moved_y): + return (float(moved_x), float(moved_y)) + raise LibeiUnavailable( + f"({x}, {y}) lies outside every region this pointer accepts " + f"{regions}; libei drops such a motion without reporting it, so " + "the move is refused here for the CLI path to take", + ) -def _load_symbols() -> Optional[_LibeiSymbols]: - lib = _try_load_library() - if lib is None: - return None - return _bind_symbols(lib) + # --- plumbing --------------------------------------------------------- + def _emit(self, capability: int, send: Callable[[int], None]) -> None: + """Send one event on the device carrying ``capability``, then frame. -def _default_socket_path() -> bytes: - import os - runtime = os.environ.get("XDG_RUNTIME_DIR") - if not runtime: - return b"/run/user/1000/eis-0" - return f"{runtime}/eis-0".encode("utf-8") + Without the trailing ``ei_device_frame`` libei buffers the event and + the compositor never sees it, so the two belong in one place. + """ + with self._lock: + if self._ei is None: + raise LibeiUnavailable("libei sender is not connected") + # Pause / resume arrive asynchronously; read them before deciding + # the device is still usable. + self._pump(0.0) + device = self._devices.get(capability) + if not device or not self._emulating.get(device, False): + raise LibeiUnavailable( + f"no libei device is emulating capability {capability}", + ) + send(device) + self._symbols.ei_device_frame(device, self._symbols.ei_now(self._ei)) + + def _teardown(self) -> None: + """Release what is safe to release; abandon what is not. + + ``ei_unref`` **segfaults** on libei 1.3.901 on a context whose backend + was opened but whose handshake never progressed. Measured, not + inferred — ``docker/libei_verify.py`` walks the states one call at a + time against the real library, and ``docker/eis_verify.py`` adds the + live one by running a real EIS peer: + + ========================================= ========== + state ``ei_unref`` + ========================================= ========== + context created, no backend set up safe + ``ei_setup_backend_socket`` failed (-2) safe + backend open, handshake never progressed **SIGSEGV** + handshake completed, devices emulating safe + ========================================= ========== + + ``ei_disconnect`` crashes in the same state, so it is not a + refcounting mistake on our side — it is tearing down a connection + whose EI handshake never progressed. The header documents ``unref`` + as the correct release for every outcome, so this reads as an + upstream bug rather than misuse. + + So the release is chosen by state. A session that reached a live + device is unreffed normally, which is the common path and no longer + leaks. Only the crashing state is abandoned — its context and fd are + dropped without unref, a few hundred bytes and one descriptor per + process, since :func:`connected_backend` probes only once. A segfault + in a library that drives someone's desktop is far worse than that. + """ + if self._ei is not None and self._safe_to_unref(): + for device in set(self._devices.values()): + _quietly(lambda handle=device: + self._symbols.ei_device_unref(handle)) + _quietly(lambda: self._symbols.ei_unref(self._ei)) + self._devices.clear() + self._emulating.clear() + self._ei = None + self._backend_open = False + self._handshake_complete = False + if self._session is not None: + # liboeffis is a different library and closing the portal session + # is what actually revokes the grant, so this always runs. + _quietly(self._session.close) + self._session = None + + def _safe_to_unref(self) -> bool: + """Whether this context is in a state libei survives being unreffed in. + + The two safe states are "no backend was ever opened" and "the + handshake completed". Everything in between is the measured crash. + """ + return not self._backend_open or self._handshake_complete + + +def _quietly(action: Callable[[], object]) -> None: + """Run a cleanup step; a failing one must not mask the real error.""" + try: + action() + except (AttributeError, OSError, ValueError, RuntimeError): + pass -# Cached default sender — created on first use, kept for the process -# lifetime so input latency isn't dominated by sender setup. _DEFAULT_BACKEND: Optional[LibeiBackend] = None +_PROBE_FAILED = False _DEFAULT_LOCK = threading.Lock() -def get_default_backend() -> Optional[LibeiBackend]: - """Return the cached :class:`LibeiBackend`, or None when libei is absent.""" - global _DEFAULT_BACKEND +def connected_backend() -> Optional[LibeiBackend]: + """Return a connected backend, or None — probing at most once. + + The probe involves a portal round trip and a consent dialog, so a host + where libei cannot be used must pay for that discovery once rather than + on every keystroke. Callers treat None as "use the ydotool CLI". + """ + global _DEFAULT_BACKEND, _PROBE_FAILED with _DEFAULT_LOCK: if _DEFAULT_BACKEND is not None: return _DEFAULT_BACKEND + if _PROBE_FAILED: + return None backend = LibeiBackend() if not backend.is_available: + _PROBE_FAILED = True + return None + try: + backend.connect() + except (LibeiUnavailable, OSError, ValueError, AttributeError): + _PROBE_FAILED = True return None _DEFAULT_BACKEND = backend - return _DEFAULT_BACKEND + return _DEFAULT_BACKEND + + +def get_default_backend() -> Optional[LibeiBackend]: + """Return the cached backend if libei resolved, without connecting.""" + with _DEFAULT_LOCK: + if _DEFAULT_BACKEND is not None: + return _DEFAULT_BACKEND + backend = LibeiBackend() + return backend if backend.is_available else None def reset_default_backend() -> None: - """Test hook — drop the cached default so probe runs fresh.""" - global _DEFAULT_BACKEND + """Test hook — drop the cached backend so the probe runs fresh.""" + global _DEFAULT_BACKEND, _PROBE_FAILED with _DEFAULT_LOCK: if _DEFAULT_BACKEND is not None: - try: - _DEFAULT_BACKEND.disconnect() - except LibeiUnavailable: - pass + _quietly(_DEFAULT_BACKEND.disconnect) _DEFAULT_BACKEND = None + _PROBE_FAILED = False __all__ = [ - "LibeiBackend", "LibeiUnavailable", - "get_default_backend", "reset_default_backend", + "EI_DEVICE_CAP_BUTTON", "EI_DEVICE_CAP_KEYBOARD", + "EI_DEVICE_CAP_POINTER_ABSOLUTE", "EI_DEVICE_CAP_SCROLL", + "HANDSHAKE_TIMEOUT", "LibeiBackend", "LibeiUnavailable", + "connected_backend", "get_default_backend", "reset_default_backend", ] diff --git a/je_auto_control/linux_wayland/mouse.py b/je_auto_control/linux_wayland/mouse.py index 0d94e617..82308146 100644 --- a/je_auto_control/linux_wayland/mouse.py +++ b/je_auto_control/linux_wayland/mouse.py @@ -8,12 +8,18 @@ """ from __future__ import annotations +import os import subprocess # nosec B404 # reason: argv-list, no shell interpolation import time -from typing import Optional, Tuple +from functools import lru_cache +from typing import Mapping, Optional, Tuple from je_auto_control.linux_wayland._detect import WAYLAND_YDOTOOL, binary_path +from je_auto_control.linux_wayland._layout import layout_origin +from je_auto_control.linux_wayland._select_input import emitted +from je_auto_control.linux_wayland._ydotool_cli import reject_legacy_cli from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger # ydotool ``click`` accepts hex bitmasks: the low nibble selects the @@ -33,9 +39,11 @@ wayland_scroll_direction_right = 2 _INSTALL_HINT = ( - "ydotool is required for Wayland mouse input. " - "Install with your package manager (e.g. `apt install ydotool`) " - "and ensure ydotoold runs with /dev/uinput permission." + "ydotool 1.0 or newer is required for Wayland mouse input. " + "Install it with your package manager (Arch: `pacman -S ydotool`; " + "Fedora: `dnf install ydotool`; Debian: from unstable — trixie ships no " + "package and bookworm's 0.1.8 is too old) and ensure ydotoold runs with " + "/dev/uinput permission." ) @@ -43,7 +51,7 @@ def _require_ydotool() -> str: path = binary_path(WAYLAND_YDOTOOL) if path is None: raise AutoControlException(_INSTALL_HINT) - return path + return reject_legacy_cli(path) def _run(argv: list, *, timeout: float = 5.0) -> None: @@ -73,32 +81,173 @@ def position() -> Tuple[int, int]: ) +POINTER_ACCEL_ENV = "JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL" +POINTER_ACCEL_MODES = ("warn", "flat", "strict") + +_ACCELERATION_ADVICE = ( + "Disable pointer acceleration for the ydotoold device (sway: `input " + "type:pointer accel_profile flat` plus `pointer_accel 0`) or install " + "liboeffis so the libei path can be used." +) + +_ACCELERATION_WARNING = ( + "Wayland absolute move fell back to ydotool. `mousemove --absolute` is " + "relative motion under the hood, so the compositor's pointer " + "acceleration scales it: measured against a real wlroots session, " + "libinput's default adaptive profile lands the cursor twice as far from " + f"the corner as asked. {_ACCELERATION_ADVICE} With acceleration off, set " + f"{POINTER_ACCEL_ENV}=flat to silence this; set it to strict to have the " + "move refused instead of landing somewhere else." +) + +_ACCELERATION_REFUSAL = ( + f"{POINTER_ACCEL_ENV}=strict, and this absolute move fell back to " + "ydotool, whose `mousemove --absolute` is relative motion the compositor " + f"scales — so the cursor would not land where asked. {_ACCELERATION_ADVICE} " + f"Then set {POINTER_ACCEL_ENV}=flat, or unset it to warn and move anyway." +) + + +def pointer_accel_mode(environ: Optional[Mapping[str, str]] = None) -> str: + """Return the operator's declared pointer-acceleration policy. + + The compositor's acceleration factor cannot be read back by the client, + so this backend cannot compensate for it — only the operator knows + whether it is switched off. This is how they say so: + + * ``warn`` — the default, and what an unset or unrecognised value means: + log the caveat once per process and send the move regardless. + * ``flat`` — acceleration is off for the ydotoold device, the move is + pixel-accurate, and nothing needs saying. + * ``strict`` — refuse to send an absolute move the compositor would + scale, rather than let a click land silently in the wrong place. + """ + env = environ if environ is not None else os.environ + declared = (env.get(POINTER_ACCEL_ENV) or "").strip().lower() + if not declared: + return "warn" + if declared in POINTER_ACCEL_MODES: + return declared + _warn_once( + f"{POINTER_ACCEL_ENV}={declared} is not one of " + f"{', '.join(POINTER_ACCEL_MODES)}; treating it as warn.", + ) + return "warn" + + +def _apply_accel_policy() -> None: + """Gate a ydotool absolute move on :func:`pointer_accel_mode`.""" + mode = pointer_accel_mode() + if mode == "flat": + return + if mode == "strict": + raise AutoControlException(_ACCELERATION_REFUSAL) + _warn_about_acceleration() + + +def _ydotool_point(x: int, y: int) -> Tuple[int, int]: + """Translate a layout coordinate into ydotool's absolute space. + + ``mousemove --absolute`` emits no absolute event. It sends ``INT32_MIN`` + on both axes to drive the cursor into the corner the compositor clamps + to, then sends the target as a relative displacement — so its origin is + the top-left of the *output layout*, not layout ``(0, 0)``. The two are + the same point only while every output sits at a non-negative position; + put a monitor left of the primary one and they differ by + :func:`~je_auto_control.linux_wayland._layout.layout_origin`, which is + exactly the correction the capture path already applies. + + Measured rather than reasoned: on a real wlroots session whose left-hand + output sits at ``x=-1280`` (``docker/Dockerfile.seat``), with the + pointer's acceleration disabled, ``--absolute -x 10 -y 10`` puts the + cursor at layout ``(-1270, 10)``. Without this subtraction a caller + asking for layout ``(-1270, 10)`` would land 1,280 pixels away, on the + other monitor. + """ + origin_x, origin_y = layout_origin() + return x - origin_x, y - origin_y + + def set_position(x: int, y: int) -> None: - """Move the cursor to absolute (x, y). Uses libei when available.""" + """Move the cursor to absolute (x, y). Uses libei when available. + + The libei path is absolute at the protocol level and lands exactly. The + ydotool fallback cannot: see :func:`_ydotool_point` for the origin it + counts from, and note that the compositor's pointer acceleration is + applied to the relative motion it really sends, so the move is only + pixel-accurate where that acceleration is switched off. ydotool's own + ``--help`` says as much; by default this backend logs it once per + process rather than letting a click land silently in the wrong place. + :func:`pointer_accel_mode` documents the environment variable an + operator sets to silence that (acceleration is off) or to have the move + refused outright. + + :raises AutoControlException: on the ydotool path when the operator has + set ``JE_AUTOCONTROL_WAYLAND_POINTER_ACCEL=strict``. + """ libei = _try_libei() - if libei is not None: - libei.set_position(int(x), int(y)) + if libei is not None and emitted( + libei, lambda device: device.set_position(int(x), int(y))): return + _apply_accel_policy() time.sleep(0.01) + target_x, target_y = _ydotool_point(int(x), int(y)) _run([_require_ydotool(), "mousemove", "--absolute", - "-x", str(int(x)), "-y", str(int(y))]) + "-x", str(target_x), "-y", str(target_y)]) + + +@lru_cache(maxsize=32) +def _warn_once(message: str) -> None: + """Log ``message`` at warning level, once per process per distinct text. + + Cached rather than latched on a module global: these caveats are worth + saying and not worth repeating on every move of a script that makes + thousands. ``_warn_once.cache_clear()`` re-arms them. The bound is there + because the key includes an operator-supplied env value; evicting only + costs a repeated line. + """ + autocontrol_logger.warning(message) + + +def _warn_about_acceleration() -> None: + """Log the ydotool absolute-move caveat, once per process.""" + _warn_once(_ACCELERATION_WARNING) def _try_libei(): + """Return a connected :class:`LibeiBackend`, or None when CLI should win.""" try: - from je_auto_control.linux_wayland import select_input_backend - if select_input_backend() != "libei": - return None - from je_auto_control.linux_wayland.libei import get_default_backend - backend = get_default_backend() - if backend is None: - return None - backend.connect() - return backend + from je_auto_control.linux_wayland._select_input import active_backend + return active_backend() except (ImportError, RuntimeError, OSError): return None +# ydotool's ``click`` verb and libei speak different button numbers: the +# constants above are ydotool's bitmask nibbles, libei takes raw evdev +# BTN_* codes. Scroll needs a conversion of its own — see +# ``_LIBEI_VERTICAL_SIGN``. +_BTN_LEFT = 272 +_BTN_RIGHT = 273 +_BTN_MIDDLE = 274 + +_EVDEV_BUTTONS = { + wayland_mouse_left: _BTN_LEFT, + wayland_mouse_right: _BTN_RIGHT, + wayland_mouse_middle: _BTN_MIDDLE, +} + + +def _evdev_button(mouse_keycode: int) -> Optional[int]: + """Map this module's public button code onto an evdev BTN_* code. + + A press- or release-only code has one edge bit cleared; restoring both + gives back the canonical constant the table is keyed on. + """ + canonical = int(mouse_keycode) | _YDOTOOL_DOWN_BIT | _YDOTOOL_UP_BIT + return _EVDEV_BUTTONS.get(canonical) + + def _press_code(mouse_keycode: int) -> int: """Button-down only: set the down-bit, clear the up-bit.""" return (int(mouse_keycode) | _YDOTOOL_DOWN_BIT) & ~_YDOTOOL_UP_BIT @@ -111,25 +260,78 @@ def _release_code(mouse_keycode: int) -> int: def press_mouse(mouse_keycode: int) -> None: """Press a mouse button and hold it (down edge only).""" + if _emit_button(mouse_keycode, press=True): + return time.sleep(0.01) _run([_require_ydotool(), "click", f"{_press_code(mouse_keycode):#x}"]) def release_mouse(mouse_keycode: int) -> None: """Release a held mouse button (up edge only).""" + if _emit_button(mouse_keycode, press=False): + return time.sleep(0.01) _run([_require_ydotool(), "click", f"{_release_code(mouse_keycode):#x}"]) +def _emit_button(mouse_keycode: int, *, press: bool) -> bool: + """Send one button edge through libei; False means fall back to the CLI.""" + button = _evdev_button(mouse_keycode) + if button is None: + return False + libei = _try_libei() + if libei is None: + return False + if press: + return emitted(libei, lambda device: device.press_button(button)) + return emitted(libei, lambda device: device.release_button(button)) + + def click_mouse(mouse_keycode: int, x: Optional[int] = None, y: Optional[int] = None) -> None: """Press + release a mouse button, optionally moving first.""" if x is not None and y is not None: set_position(int(x), int(y)) + if _emit_button(mouse_keycode, press=True): + # Not _emit_button: a refused release has to reach the CLI, or the + # button stays down for the rest of the session. + release_mouse(mouse_keycode) + return time.sleep(0.01) _run([_require_ydotool(), "click", f"{int(mouse_keycode):#x}"]) +#: Sign applied to the vertical axis on the way to libei. +#: +#: The two sides count wheels in opposite directions. This module's +#: ``wayland_scroll_direction_*`` constants are in the kernel's ``REL_WHEEL`` +#: frame, because that is what ydotool writes into ``/dev/uinput``: positive +#: is up. libei is in the ``wl_pointer`` / libinput frame, where positive is +#: down — libinput's own evdev reader negates ``REL_WHEEL`` to get there, and +#: the other libei sender that documents its sign (enigo) passes a +#: "positive scrolls down" value straight through to ``scroll_discrete``. +#: +#: Horizontal needs no flip: ``REL_HWHEEL`` and libinput both count right as +#: positive, and libinput passes that axis through unnegated. +#: +#: What ``docker/eis_verify.py`` measured is the half either frame agrees on: +#: ``scroll(0, 1)`` reaches a real EIS server as ``(0, 120)`` — one whole +#: click, on the y axis, sign preserved end to end, no axis swap. The frame +#: the compositor reads that in is the half above. +_LIBEI_VERTICAL_SIGN = -1 + + +def _wheel_deltas(scroll_value: int, scroll_direction: int) -> Tuple[int, int]: + """Split ``(value, direction)`` into the ``(x, y)`` detents to send. + + ``scroll_direction`` carries axis and sign, per this module's + ``wayland_scroll_direction_*`` constants: +-1 vertical, +-2 horizontal. + """ + direction = int(scroll_direction) + amount = abs(int(scroll_value)) * (1 if direction > 0 else -1) + return (0, amount) if abs(direction) == 1 else (amount, 0) + + def scroll(scroll_value: int, scroll_direction: int = wayland_scroll_direction_down) -> None: """Scroll ``scroll_value`` notches in ``scroll_direction``. @@ -140,13 +342,22 @@ def scroll(scroll_value: int, previous ``(direction, x, y)`` shape silently bound the wrapper's direction to ``x`` and then dropped it, so every scroll went the same way. - ``scroll_direction`` carries axis and sign, per this module's - ``wayland_scroll_direction_*`` constants: +-1 vertical, +-2 horizontal. + Prefers libei, which delivers whole wheel clicks without a uinput daemon; + the vertical axis is negated on the way out (``_LIBEI_VERTICAL_SIGN``). + + On the ydotool fallback both axes are always passed, as ydotool's own + documented example does (``ydotool mousemove -w -x 0 -y -1``) — leaving + one out would rely on it defaulting to zero, which the tool does not + promise. Positive is up / right there, matching the kernel's + ``REL_WHEEL`` convention that ydotool writes. """ - direction = int(scroll_direction) - axis = "-y" if abs(direction) == 1 else "-x" - amount = abs(int(scroll_value)) * (1 if direction > 0 else -1) - _run([_require_ydotool(), "mousemove", "--wheel", axis, str(amount)]) + delta_x, delta_y = _wheel_deltas(scroll_value, scroll_direction) + libei = _try_libei() + if libei is not None and emitted(libei, lambda device: device.scroll( + delta_x, _LIBEI_VERTICAL_SIGN * delta_y)): + return + _run([_require_ydotool(), "mousemove", "--wheel", + "-x", str(delta_x), "-y", str(delta_y)]) def send_mouse_event_to_window(*_args, **_kwargs) -> None: @@ -159,7 +370,8 @@ def send_mouse_event_to_window(*_args, **_kwargs) -> None: __all__ = [ - "click_mouse", "position", "press_mouse", "release_mouse", + "POINTER_ACCEL_ENV", "POINTER_ACCEL_MODES", "click_mouse", + "pointer_accel_mode", "position", "press_mouse", "release_mouse", "scroll", "send_mouse_event_to_window", "set_position", "wayland_mouse_left", "wayland_mouse_middle", "wayland_mouse_right", "wayland_scroll_direction_down", "wayland_scroll_direction_left", diff --git a/je_auto_control/linux_wayland/oeffis.py b/je_auto_control/linux_wayland/oeffis.py new file mode 100644 index 00000000..174dfa0a --- /dev/null +++ b/je_auto_control/linux_wayland/oeffis.py @@ -0,0 +1,196 @@ +"""ctypes binding for liboeffis — the portal half of the libei handshake. + +libei needs an EIS socket, and on GNOME / KDE that socket is not a path on +disk: it is a file descriptor handed over D-Bus by +``org.freedesktop.portal.RemoteDesktop.ConnectToEIS``, at the end of a +three-call asynchronous session dance (``CreateSession`` → ``SelectDevices`` +→ ``Start``). A file descriptor cannot be received through the ``gdbus`` +command line, so the CLI trick that works for the Screenshot portal (see +:mod:`portal`) cannot work here — it would need a D-Bus client that speaks +SCM_RIGHTS. + +``liboeffis`` ships with libei precisely so clients do not have to write +that. It performs the whole portal dance and hands back the EIS fd, which is +what :mod:`libei` then passes to ``ei_setup_backend_fd``. + +Everything here is fail-closed: if the library is missing, the portal denies +the request, or the user dismisses the consent dialog, the caller gets None +or an exception and falls back to the ydotool CLI. +""" +from __future__ import annotations + +import ctypes +import select +import time +from typing import Optional, Tuple + +from je_auto_control.linux_wayland._ctypes_bind import BoundSymbols, bind + + +_LIBRARY_CANDIDATES = ("oeffis", "liboeffis", "liboeffis.so.1", "liboeffis.so.0") + +# enum oeffis_device — a bitmask of what the session may do. Verified against +# liboeffis.h (Debian 1.5.0-3 and upstream main agree). +OEFFIS_DEVICE_ALL_DEVICES = 0 # the header's "everything" sentinel +OEFFIS_DEVICE_KEYBOARD = 1 << 0 +OEFFIS_DEVICE_POINTER = 1 << 1 +OEFFIS_DEVICE_TOUCHSCREEN = 1 << 2 + +#: What this backend actually emits. Asking for the touchscreen we never use +#: would widen the grant the user is consenting to for no benefit, and an +#: explicit mask cannot be misread as the ``= 0`` sentinel either way. +OEFFIS_DEVICE_DEFAULT = OEFFIS_DEVICE_KEYBOARD | OEFFIS_DEVICE_POINTER + +# enum oeffis_event_type — note CLOSED precedes DISCONNECTED, which is the +# opposite of the order the names suggest. +OEFFIS_EVENT_NONE = 0 +OEFFIS_EVENT_CONNECTED_TO_EIS = 1 +OEFFIS_EVENT_CLOSED = 2 +OEFFIS_EVENT_DISCONNECTED = 3 + +#: The consent dialog is a human in the loop, so this is a human-scale wait. +DEFAULT_TIMEOUT = 30.0 + +_PROTOTYPES = ( + ("oeffis_new", ctypes.c_void_p, (ctypes.c_void_p,)), + ("oeffis_unref", ctypes.c_void_p, (ctypes.c_void_p,)), + ("oeffis_create_session", None, (ctypes.c_void_p, ctypes.c_uint32)), + ("oeffis_get_fd", ctypes.c_int, (ctypes.c_void_p,)), + ("oeffis_dispatch", None, (ctypes.c_void_p,)), + ("oeffis_get_event", ctypes.c_int, (ctypes.c_void_p,)), + ("oeffis_get_eis_fd", ctypes.c_int, (ctypes.c_void_p,)), + ("oeffis_get_error_message", ctypes.c_char_p, (ctypes.c_void_p,)), +) + + +class OeffisUnavailable(RuntimeError): + """liboeffis is missing, or the portal refused to hand over an EIS fd.""" + + +def load_symbols() -> Optional[BoundSymbols]: + """Resolve liboeffis, or None when it is not installed.""" + return bind(_LIBRARY_CANDIDATES, _PROTOTYPES) + + +def is_available() -> bool: + """Whether the portal route to an EIS fd can be attempted at all.""" + return load_symbols() is not None + + +def _describe(symbols: BoundSymbols, handle: int) -> str: + """The library's own error text, when it has one.""" + try: + message = symbols.oeffis_get_error_message(handle) + except (AttributeError, OSError, ValueError): + return "" + if not message: + return "" + if isinstance(message, bytes): + return message.decode("utf-8", errors="replace") + return str(message) + + +def _pump(symbols: BoundSymbols, handle: int, deadline: float) -> int: + """Wait for one oeffis event, or 0 when the deadline passes.""" + poll_fd = symbols.oeffis_get_fd(handle) + if poll_fd < 0: + raise OeffisUnavailable("oeffis_get_fd returned no pollable fd") + while True: + remaining = deadline - time.monotonic() + if remaining <= 0: + return OEFFIS_EVENT_NONE + ready, _, _ = select.select([poll_fd], [], [], remaining) + if not ready: + return OEFFIS_EVENT_NONE + symbols.oeffis_dispatch(handle) + event = int(symbols.oeffis_get_event(handle)) + if event != OEFFIS_EVENT_NONE: + return event + + +def connect_eis_fd(devices: int = OEFFIS_DEVICE_DEFAULT, + timeout: float = DEFAULT_TIMEOUT, + symbols: Optional[BoundSymbols] = None) -> Tuple[int, object]: + """Run the portal session and return ``(eis_fd, session_handle)``. + + The session handle must be kept alive for as long as the EIS fd is used: + dropping it tears the portal session down and the compositor stops + accepting input from it. + + :param devices: bitmask of ``OEFFIS_DEVICE_*`` to request. + :param timeout: seconds to wait, including any consent dialog. + :param symbols: injected entry points, for tests. + :return: the EIS file descriptor and the handle owning the session. + """ + resolved = symbols if symbols is not None else load_symbols() + if resolved is None: + raise OeffisUnavailable("liboeffis.so.* not found on the loader path") + handle = resolved.oeffis_new(None) + if not handle: + raise OeffisUnavailable("oeffis_new returned NULL") + try: + resolved.oeffis_create_session(handle, int(devices)) + event = _pump(resolved, handle, time.monotonic() + max(0.0, timeout)) + _require_connected(resolved, handle, event, timeout) + eis_fd = int(resolved.oeffis_get_eis_fd(handle)) + if eis_fd < 0: + raise OeffisUnavailable( + "the portal reported success but handed over no EIS fd", + ) + except BaseException: + _release(resolved, handle) + raise + return eis_fd, _Session(resolved, handle) + + +def _require_connected(symbols: BoundSymbols, handle: int, event: int, + timeout: float) -> None: + """Turn anything but a successful connection into a clear failure.""" + if event == OEFFIS_EVENT_CONNECTED_TO_EIS: + return + if event == OEFFIS_EVENT_NONE: + raise OeffisUnavailable( + f"the desktop portal did not answer within {timeout:g}s " + f"(a consent dialog may be waiting)", + ) + detail = _describe(symbols, handle) + reason = ("the desktop portal closed the remote-desktop session" + if event == OEFFIS_EVENT_CLOSED + else "the desktop portal disconnected the remote-desktop session") + raise OeffisUnavailable(f"{reason}{': ' + detail if detail else ''}") + + +def _release(symbols: BoundSymbols, handle: int) -> None: + """Drop the oeffis context; never raise from cleanup.""" + try: + symbols.oeffis_unref(handle) + except (AttributeError, OSError, ValueError): + pass + + +class _Session: + """Keeps the portal session alive; releasing it ends the grant.""" + + def __init__(self, symbols: BoundSymbols, handle: int) -> None: + self._symbols = symbols + self._handle: Optional[int] = handle + + def close(self) -> None: + """End the portal session.""" + if self._handle is None: + return + _release(self._symbols, self._handle) + self._handle = None + + def __del__(self) -> None: + self.close() + + +__all__ = [ + "DEFAULT_TIMEOUT", "OEFFIS_DEVICE_ALL_DEVICES", "OEFFIS_DEVICE_DEFAULT", + "OEFFIS_DEVICE_KEYBOARD", "OEFFIS_DEVICE_POINTER", + "OEFFIS_DEVICE_TOUCHSCREEN", "OEFFIS_EVENT_CLOSED", + "OEFFIS_EVENT_CONNECTED_TO_EIS", "OEFFIS_EVENT_DISCONNECTED", + "OEFFIS_EVENT_NONE", "OeffisUnavailable", "connect_eis_fd", + "is_available", "load_symbols", +] diff --git a/je_auto_control/linux_wayland/portal.py b/je_auto_control/linux_wayland/portal.py new file mode 100644 index 00000000..bc4e446c --- /dev/null +++ b/je_auto_control/linux_wayland/portal.py @@ -0,0 +1,207 @@ +"""Last-resort screen capture through ``xdg-desktop-portal``. + +The three CLI helpers in :mod:`capture` each cover one compositor family and +none of them is guaranteed to be installed — GNOME has not shipped +``gnome-screenshot`` by default since 42. ``org.freedesktop.portal.Screenshot`` +is the one interface every modern desktop implements, so it is tried after +them all rather than not at all. + +It is awkward enough to deserve explaining. The portal does not return the +image: ``Screenshot`` returns a *request* object path and the result arrives +later as a ``Response`` signal on it, **directed at the unique bus name that +made the call**. So the subscription has to be in place before the call, and +it has to be on the same connection — a listener anywhere else is not the +destination, and the bus routes a directed message to its destination only. + +That is why this does not shell out. The obvious implementation — ``gdbus +monitor`` in one process, ``gdbus call`` in another — cannot work and did not: +each invocation opens its own connection under its own unique name, so the +monitor is never the caller. Measured against a real ``dbus-daemon``, the +monitor saw the call go past and the answer never arrived, and the capture +timed out every time. :mod:`_dbus_client` speaks the protocol directly instead, +which also drops the ``gdbus`` binary from what a desktop has to have +installed for this tier to work. + +Two consequences the caller has to live with, both documented at the tier that +uses this: the portal may show a consent dialog the first time (so the wait is +bounded by :data:`DEFAULT_TIMEOUT`, not instant), and it always captures the +whole screen, so a region is cropped afterwards. + +Every failure here is an ordinary capture failure — the caller reports the +install hint for the CLI helpers, which is the actionable advice. +""" +from __future__ import annotations + +import contextlib +import os +import threading +import urllib.parse +from typing import Any, Dict, List, Tuple + +from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.utils.exception.exceptions import AutoControlScreenException + + +PORTAL_BUS = "org.freedesktop.portal.Desktop" +PORTAL_PATH = "/org/freedesktop/portal/desktop" +SCREENSHOT_INTERFACE = "org.freedesktop.portal.Screenshot" +REQUEST_INTERFACE = "org.freedesktop.portal.Request" +PORTAL_METHOD = f"{SCREENSHOT_INTERFACE}.Screenshot" + +# The portal may put a consent dialog in front of the response the first time, +# so this is a human-scale wait, not a machine-scale one. +DEFAULT_TIMEOUT = 30.0 +_CALL_TIMEOUT = 10.0 + +# 0 success, 1 cancelled by the user, 2 ended some other way. +_RESPONSE_MEANING = { + 1: "the user dismissed the desktop portal's screenshot dialog", + 2: "the desktop portal ended the screenshot request", +} + + +def is_available() -> bool: + """Whether a session bus exists, so the portal tier is worth trying.""" + return _dbus_client.is_available() + + +def capture_png(timeout: float = DEFAULT_TIMEOUT) -> bytes: + """Ask the desktop portal for a full-screen capture; return PNG bytes. + + :param timeout: seconds to wait for the portal's Response signal, + including any time the user spends on a consent dialog. + :return: the captured image file's bytes. + """ + try: + with _dbus_client.SessionBus() as bus: + uri = _request_and_await(bus, timeout) + except _dbus_client.DBusError as error: + raise AutoControlScreenException( + f"the desktop portal could not be reached: {error}", + ) from error + return _read_and_discard(_path_from_uri(uri)) + + +def _request_and_await(bus: "_dbus_client.SessionBus", timeout: float) -> str: + """Subscribe, call, and wait — in that order, on the one connection.""" + token = _handle_token() + predicted = _request_path(bus.sender_token, token) + bus.add_match(_match_rule(predicted)) + handle = _screenshot(bus, token) + paths = [predicted] + # A portal that ignores handle_token answers on a path of its own choosing. + # The specification tells clients to follow the returned handle, and the + # subscription for it can only be added once the call has returned it — + # which is exactly why the predicted one is subscribed to first. + if handle and handle != predicted: + bus.add_match(_match_rule(handle)) + paths.append(handle) + try: + body = bus.wait_for_signal(paths, REQUEST_INTERFACE, "Response", + timeout) + except _dbus_client.DBusError as error: + raise AutoControlScreenException( + f"desktop portal did not answer within {timeout:g}s " + f"(a consent dialog may be waiting)", + ) from error + return _uri_from_response(body) + + +def _screenshot(bus: "_dbus_client.SessionBus", token: str) -> str: + """Make the Screenshot call; its return value is only a request handle.""" + options = { + "interactive": _dbus_client.Variant("b", False), + "handle_token": _dbus_client.Variant("s", token), + } + body = bus.call(PORTAL_BUS, PORTAL_PATH, SCREENSHOT_INTERFACE, + "Screenshot", "sa{sv}", ["", options], + timeout=_CALL_TIMEOUT) + return str(body[0]) if body else "" + + +#: Guards the counter below. Captures can be started from the callback +#: executor and a GUI thread at once, and ``+= 1`` is not atomic — two threads +#: reading the same value would put both portal answers on one object path. +_TOKEN_LOCK = threading.Lock() +_REQUEST_COUNT = 0 + + +def _handle_token() -> str: + """A token unique to this request, which the request path is built from. + + Only ``[A-Za-z0-9_]`` is legal in the object path element it becomes, and + reusing one across two concurrent captures would put both answers on one + path, so the process and a counter both go in. + """ + global _REQUEST_COUNT # noqa: PLW0603 # reason: one counter per process is the point + with _TOKEN_LOCK: + _REQUEST_COUNT += 1 + count = _REQUEST_COUNT + return f"je_auto_control_{os.getpid()}_{count}" + + +def _request_path(sender_token: str, handle_token: str) -> str: + """Where the portal will emit ``Response``, by the specification's rule.""" + return f"{PORTAL_PATH}/request/{sender_token}/{handle_token}" + + +def _match_rule(path: str) -> str: + """A match rule narrow enough that only this request's answer arrives.""" + return (f"type='signal',sender='{PORTAL_BUS}',path='{path}'," + f"interface='{REQUEST_INTERFACE}',member='Response'") + + +def _uri_from_response(body: List[Any]) -> str: + """Read the screenshot URI out of one ``Response`` signal body.""" + code, results = _response_parts(body) + if code != 0: + raise AutoControlScreenException( + _RESPONSE_MEANING.get(code, f"desktop portal returned {code}"), + ) + uri = results.get("uri", "") + if not uri: + raise AutoControlScreenException( + "desktop portal reported success but returned no image URI", + ) + return str(uri) + + +def _response_parts(body: List[Any]) -> Tuple[int, Dict[str, Any]]: + """Split a ``(u, a{sv})`` signal body, rejecting anything else.""" + if len(body) < 2 or not isinstance(body[1], dict): + raise AutoControlScreenException( + f"the desktop portal sent a Response this cannot read: {body!r}", + ) + return int(body[0]), body[1] + + +def _path_from_uri(uri: str) -> str: + """Convert the portal's ``file://`` URI into a local path.""" + parsed = urllib.parse.urlparse(uri) + if parsed.scheme != "file": + raise AutoControlScreenException( + f"desktop portal returned a non-file URI: {uri!r}", + ) + return urllib.parse.unquote(parsed.path) + + +def _read_and_discard(path: str) -> bytes: + """Read the portal's file, then remove it — it is ours to clean up.""" + try: + with open(path, "rb") as captured: + data = captured.read() + except OSError as error: + raise AutoControlScreenException( + f"could not read the portal's screenshot at {path!r}: {error}", + ) from error + finally: + with contextlib.suppress(OSError): + os.unlink(path) + if not data: + raise AutoControlScreenException("desktop portal wrote an empty file") + return data + + +__all__ = ["DEFAULT_TIMEOUT", "PORTAL_BUS", "PORTAL_METHOD", "PORTAL_PATH", + "REQUEST_INTERFACE", "SCREENSHOT_INTERFACE", "capture_png", + "is_available"] diff --git a/je_auto_control/linux_wayland/screen.py b/je_auto_control/linux_wayland/screen.py index 4ae4dc18..9879514b 100644 --- a/je_auto_control/linux_wayland/screen.py +++ b/je_auto_control/linux_wayland/screen.py @@ -1,135 +1,266 @@ -"""Wayland screen backend (grim + wlr-randr CLI bridges). +"""Wayland screen backend (compositor capture tools + wlr-randr). -Screen capture goes through ``grim`` — the wlroots screencopy tool — -because the xdg-desktop-portal ScreenCast path needs a user-consent -dialog every call. Resolution comes from ``wlr-randr`` when present; -otherwise the GNOME ``gnome-screenshot`` fallback is consulted. +Everything here funnels through :func:`grab_image`, which is also the hook +the framework's generic capture layer looks for +(:mod:`je_auto_control.utils.cv2_utils.screen_grabber`). That indirection is +the point: Pillow's ``ImageGrab`` and ``mss`` both read the X11 root window +on Linux, and under Wayland that root belongs to XWayland, which does not +composite native Wayland windows. Pillow's own fallback to the same capture +tools only runs when the X11 grab *raises*, which it does not while XWayland +is up; ``mss`` has no fallback at all. Publishing ``grab_image`` here is what +makes locators, OCR, screenshots, recording and remote-desktop frames see the +real screen — and see it the same way regardless of whether XWayland is +running. + +Resolution comes from ``wlr-randr`` when it is present and from the capture +itself otherwise, so the call works on GNOME / KDE without extra tools. """ from __future__ import annotations import re -import subprocess # nosec B404 # reason: argv-list, no shell interpolation -from typing import List, Optional, Tuple +from io import BytesIO +from typing import List, NamedTuple, Optional, Sequence, Tuple from PIL import Image -from je_auto_control.linux_wayland._detect import WAYLAND_GRIM, binary_path -from je_auto_control.utils.exception.exceptions import AutoControlException - - -_RESOLUTION_RE = re.compile( - # Bounded quantifiers (max 5 digits per side, more than enough for - # any real monitor resolution) keep this regex provably linear-time. - r"(\d{1,5})x(\d{1,5})", -) -_INSTALL_HINT_GRIM = ( - "grim is required for Wayland screenshots. " - "Install with your package manager (e.g. `apt install grim`)." -) +from je_auto_control.linux_wayland import capture as wayland_capture +from je_auto_control.linux_wayland._detect import WAYLAND_WLR_RANDR, binary_path +from je_auto_control.utils.exception.exceptions import AutoControlScreenException -def _require_grim() -> str: - path = binary_path(WAYLAND_GRIM) - if path is None: - raise AutoControlException(_INSTALL_HINT_GRIM) - return path +# Bounded quantifiers (max 5 digits per side, more than enough for any real +# monitor resolution) keep these regexes provably linear-time. +_MODE_RE = re.compile(r"(\d{1,5})x(\d{1,5})") +_POSITION_RE = re.compile(r"^\s*Position:\s*(-?\d{1,5}),(-?\d{1,5})") +_ENABLED_RE = re.compile(r"^\s*Enabled:\s*(\w+)") -def _run(argv: list, *, timeout: float = 10.0) -> bytes: - # argv comes from a private allow-list (grim / wlr-randr absolute - # paths via shutil.which), never user input; no shell=True. +def _validate_region(screen_region: Sequence[int]) -> Tuple[int, int, int, int]: + """Reject a region that would crop to an empty image.""" try: - completed = subprocess.run( # nosec B603 # nosemgrep - argv, check=True, timeout=timeout, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, - ) - except subprocess.CalledProcessError as error: - message = (error.stderr or b"").decode("utf-8", errors="replace") - raise AutoControlException( - f"{argv[0]} exited {error.returncode}: {message.strip()}", - ) from error - except subprocess.TimeoutExpired as error: - raise AutoControlException( - f"{argv[0]} timed out after {timeout}s", + x1, y1, x2, y2 = (int(value) for value in screen_region) + except (TypeError, ValueError) as error: + raise AutoControlScreenException( + f"screen_region must be 4 ints [left, top, right, bottom]; " + f"got {screen_region!r}", ) from error - return completed.stdout or b"" + if x2 <= x1 or y2 <= y1: + raise AutoControlScreenException( + f"screen_region must have positive width and height; got " + f"[{x1}, {y1}, {x2}, {y2}]", + ) + return x1, y1, x2, y2 + + +def grab_image(screen_region: Optional[Sequence[int]] = None) -> Image.Image: + """Capture the screen and return it as an RGB :class:`PIL.Image.Image`. + + ``screen_region`` is ``[left, top, right, bottom]`` — the same bbox + convention as ``ImageGrab.grab``, so the generic capture layer can hand + this straight to callers written against Pillow. + + Coordinates are the compositor's layout coordinates, which is what + ``grim -g`` takes and what ``wlr-randr`` reports positions in. The + whole-screen capture spans the entire output layout rather than one + monitor, so its top-left pixel is :func:`layout_origin` — ``(0, 0)`` + only while no output sits left of or above the origin. The crop below + subtracts that origin; grim applies the region itself and needs no + help, but every other tier hands back the whole layout. + + :param screen_region: region to capture, or None for the whole layout. + :return: the captured image, always in RGB mode. + """ + region = _validate_region(screen_region) if screen_region is not None else None + captured = wayland_capture.grab_png(region) + with Image.open(BytesIO(captured.data)) as raw: + # convert() reads the frame in, so the result outlives the file object. + image = raw.convert("RGB") + if region is None or captured.region_applied: + return image + origin_x, origin_y = layout_origin() + x1, y1, x2, y2 = region + return image.crop((x1 - origin_x, y1 - origin_y, + x2 - origin_x, y2 - origin_y)) def size() -> Tuple[int, int]: - """Return the primary monitor's pixel size. + """Return the size of the whole output layout, in pixels. Named ``size`` to match the backend contract the wrapper calls (``screen.size()``), as the windows / osx / x11 backends all do. - Tries ``wlr-randr`` first (sway / hyprland) then falls back to - grim's PNG header so the call still works on GNOME / KDE without - extra dependencies. + This is the size of the layout bounding box, not one monitor and not + its right/bottom edge: it has to agree with :func:`grab_image`, because + ``get_pixel`` addresses the same coordinate space and the mss-shaped + shim composes the two into its monitor list. On a layout whose left-most + output starts at a negative x the two differ — the edge is smaller than + the width by the origin — and reporting the edge would make every + consumer of ``size()`` grab a frame narrower than the screen. + + Tries ``wlr-randr`` first (sway / Hyprland) and falls back to measuring + a capture, so the call still works on GNOME / KDE. + + :return: (width, height) """ coords = _size_from_wlr_randr() if coords is not None: return coords - return _size_from_grim_capture() + image = grab_image() + return int(image.width), int(image.height) + + +def layout_origin() -> Tuple[int, int]: + """Return the layout coordinate of :func:`grab_image`'s top-left pixel. + + ``(0, 0)`` on a single-monitor layout and on any layout whose outputs + all sit at non-negative coordinates — but a monitor placed left of or + above the primary one gives the layout a negative origin, and a capture + of the whole layout then starts there rather than at the origin of the + coordinate space. + + Callers that map a position found *in* a frame back to a screen + coordinate have to add this; :func:`grab_image` subtracts it when it + crops. ``(0, 0)`` where ``wlr-randr`` cannot say (GNOME / KDE), which + is also the answer for the layouts it would otherwise report. + + :return: (origin_x, origin_y) + """ + rects = _wlr_randr_rects() + if not rects: + return (0, 0) + return (min(x for x, _, _, _ in rects), min(y for _, y, _, _ in rects)) def get_pixel(x: int, y: int) -> Tuple[int, int, int]: """Return the ``(r, g, b)`` colour at ``(x, y)``. - grim can capture an arbitrary region, so a 1x1 grab at the requested - point is the cheapest way to read a single pixel. Returns RGB to match - the x11 backend. + Captures a 1x1 region at the point — grim grabs exactly that, and the + file-based helpers grab the screen and crop. Returns RGB to match the + x11 / windows / osx backends. + + :param x: X coordinate. + :param y: Y coordinate. + :return: (R, G, B) """ - grim = _require_grim() - data = _run([grim, "-g", f"{int(x)},{int(y)} 1x1", "-"], timeout=10.0) - if not data: - raise AutoControlException("grim produced no output") - from io import BytesIO - with Image.open(BytesIO(data)) as image: - return image.convert("RGB").getpixel((0, 0)) + image = grab_image([int(x), int(y), int(x) + 1, int(y) + 1]) + return image.getpixel((0, 0)) def screenshot(file_path: Optional[str] = None, screen_region: Optional[List[int]] = None) -> Optional[str]: - """Capture the screen with ``grim``. + """Capture the screen, saving to ``file_path`` when one is given. + + ``screen_region`` is ``[x1, y1, x2, y2]``, matching the X11 backend's + calling convention. - ``screen_region`` is ``[x1, y1, x2, y2]`` (matching the X11 - backend's calling convention). When ``file_path`` is omitted the - capture is returned as PNG bytes via grim's stdout but discarded; - callers should pass an explicit path to keep the file. + :param file_path: where to write the PNG, or None to discard the capture. + :param screen_region: region to capture, or None for the whole layout. + :return: ``file_path`` unchanged, so callers can chain on it. """ - grim = _require_grim() - argv = [grim] - if screen_region is not None: - x1, y1, x2, y2 = (int(v) for v in screen_region) - argv.extend(["-g", f"{x1},{y1} {x2 - x1}x{y2 - y1}"]) - argv.append(file_path if file_path else "-") - _run(argv) + image = grab_image(screen_region) + if file_path: + try: + image.save(file_path) + except (OSError, ValueError) as error: + raise AutoControlScreenException( + f"Failed to save screenshot to {file_path!r}: {error}", + ) from error return file_path -def _size_from_wlr_randr() -> Optional[Tuple[int, int]]: - if binary_path("wlr-randr") is None: - return None +class _OutputBlock(NamedTuple): + """One ``wlr-randr`` output block while its fields are being read.""" + + mode: Optional[Tuple[int, int]] = None + position: Optional[Tuple[int, int]] = None + enabled: bool = True + + +def _read_field(block: _OutputBlock, line: str) -> _OutputBlock: + """Fold one indented ``wlr-randr`` field line into the block it belongs to. + + Lines that name none of the three fields leave the block untouched, which + is most of them — modes other than the current one, refresh rates, scale. + """ + enabled_match = _ENABLED_RE.match(line) + if enabled_match: + return block._replace(enabled=enabled_match.group(1).lower() == "yes") + position_match = _POSITION_RE.match(line) + if position_match: + return block._replace(position=(int(position_match.group(1)), + int(position_match.group(2)))) + mode_match = _MODE_RE.search(line) if "current" in line else None + if mode_match: + return block._replace(mode=(int(mode_match.group(1)), + int(mode_match.group(2)))) + return block + + +def parse_wlr_randr(text: str) -> List[Tuple[int, int, int, int]]: + """Parse ``wlr-randr`` into ``(x, y, width, height)`` per enabled output. + + An output block starts at a line with no leading whitespace and its + fields are indented under it:: + + HEADLESS-2 "Headless output 1" + Enabled: yes + Modes: + 1280x720 px (current) + Position: 0,0 + + Taking the first ``WxH`` in the whole document — which is what this used + to do — reads one monitor's mode and calls it the screen size. On any + multi-output layout that disagrees with :func:`grab_image`, which returns + the whole layout, and the two are composed by the mss-shaped shim. + """ + rects: List[Tuple[int, int, int, int]] = [] + block = _OutputBlock() + + def flush(finished: _OutputBlock) -> None: + if finished.enabled and finished.mode is not None: + x, y = finished.position or (0, 0) + rects.append((x, y, finished.mode[0], finished.mode[1])) + + for line in text.splitlines(): + if line and not line[0].isspace(): + flush(block) + block = _OutputBlock() + continue + block = _read_field(block, line) + flush(block) + return rects + + +def _wlr_randr_rects() -> List[Tuple[int, int, int, int]]: + """Enabled outputs as ``(x, y, width, height)``, empty when unknown. + + Not cached: an output can be plugged in, unplugged or moved between two + captures, and a stale layout is worse than the ``shutil.which`` miss + this costs on the desktops that have no ``wlr-randr`` at all. + """ + executable = binary_path(WAYLAND_WLR_RANDR) + if executable is None: + return [] try: - output = _run(["wlr-randr"], timeout=5.0).decode( + output = wayland_capture.run_tool([executable], timeout=5.0).decode( "utf-8", errors="replace", ) - except AutoControlException: - return None - for line in output.splitlines(): - match = _RESOLUTION_RE.search(line) - if match: - return int(match.group(1)), int(match.group(2)) - return None + except AutoControlScreenException: + return [] + return parse_wlr_randr(output) -def _size_from_grim_capture() -> Tuple[int, int]: - grim = _require_grim() - data = _run([grim, "-"], timeout=10.0) - if not data: - raise AutoControlException("grim produced no output") - from io import BytesIO - with Image.open(BytesIO(data)) as image: - return int(image.width), int(image.height) +def _size_from_wlr_randr() -> Optional[Tuple[int, int]]: + """Layout bounding box size from ``wlr-randr``, or None when it cannot say.""" + rects = _wlr_randr_rects() + if not rects: + return None + left = min(x for x, _, _, _ in rects) + top = min(y for _, y, _, _ in rects) + right = max(x + width for x, _, width, _ in rects) + bottom = max(y + height for _, y, _, height in rects) + return (right - left, bottom - top) -__all__ = ["size", "get_pixel", "screenshot"] +__all__ = ["get_pixel", "grab_image", "layout_origin", "parse_wlr_randr", + "screenshot", "size"] diff --git a/je_auto_control/utils/clipboard/clipboard.py b/je_auto_control/utils/clipboard/clipboard.py index 18ecce03..3759f5e5 100644 --- a/je_auto_control/utils/clipboard/clipboard.py +++ b/je_auto_control/utils/clipboard/clipboard.py @@ -17,8 +17,6 @@ from io import BytesIO from typing import Optional, Union -_OPEN_CLIPBOARD_FAILED = "OpenClipboard failed" - def get_clipboard() -> str: """Return the current clipboard text (empty string if empty).""" @@ -101,6 +99,8 @@ def _win_get() -> str: import ctypes from ctypes import wintypes + from je_auto_control.utils.clipboard.win32_clipboard_api import open_clipboard + user32 = ctypes.WinDLL("user32", use_last_error=True) kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) cf_unicodetext = 13 @@ -114,9 +114,7 @@ def _win_get() -> str: kernel32.GlobalLock.restype = ctypes.c_void_p kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_CLIPBOARD_FAILED) - try: + with open_clipboard(user32): handle = user32.GetClipboardData(cf_unicodetext) if not handle: return "" @@ -127,14 +125,14 @@ def _win_get() -> str: return ctypes.wstring_at(pointer) finally: kernel32.GlobalUnlock(handle) - finally: - user32.CloseClipboard() def _win_set(text: str) -> None: import ctypes from ctypes import wintypes + from je_auto_control.utils.clipboard.win32_clipboard_api import open_clipboard + user32 = ctypes.WinDLL("user32", use_last_error=True) kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) cf_unicodetext = 13 @@ -162,14 +160,10 @@ def _win_set(text: str) -> None: raise RuntimeError("GlobalLock failed") ctypes.memmove(pointer, ctypes.addressof(data), size) # NOSONAR S5655 false positive — Array is accepted by addressof kernel32.GlobalUnlock(handle) - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_CLIPBOARD_FAILED) - try: + with open_clipboard(user32): user32.EmptyClipboard() if not user32.SetClipboardData(cf_unicodetext, handle): raise RuntimeError("SetClipboardData failed") - finally: - user32.CloseClipboard() # === macOS backend =========================================================== @@ -262,6 +256,10 @@ def _win_set_image(png_bytes: bytes) -> None: import ctypes # noqa: PLC0415 from ctypes import wintypes # noqa: PLC0415 + from je_auto_control.utils.clipboard.win32_clipboard_api import ( # noqa: PLC0415 + open_clipboard, + ) + user32 = ctypes.WinDLL("user32", use_last_error=True) kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) cf_dib = 8 @@ -287,14 +285,10 @@ def _win_set_image(png_bytes: bytes) -> None: raise RuntimeError("GlobalLock failed") ctypes.memmove(pointer, dib, len(dib)) kernel32.GlobalUnlock(handle) - if not user32.OpenClipboard(None): - raise RuntimeError(_OPEN_CLIPBOARD_FAILED) - try: + with open_clipboard(user32): user32.EmptyClipboard() if not user32.SetClipboardData(cf_dib, handle): raise RuntimeError("SetClipboardData(CF_DIB) failed") - finally: - user32.CloseClipboard() def _mac_get_image() -> Optional[bytes]: diff --git a/je_auto_control/utils/clipboard/win32_clipboard_api.py b/je_auto_control/utils/clipboard/win32_clipboard_api.py new file mode 100644 index 00000000..028def63 --- /dev/null +++ b/je_auto_control/utils/clipboard/win32_clipboard_api.py @@ -0,0 +1,142 @@ +"""Win32 clipboard prototypes and the open/alloc/lock dance, declared once. + +Every clipboard format — text, image, HTML, RTF, CSV, file drops — goes through +the same four Win32 calls, and every module that reimplemented them got the same +detail wrong: ``argtypes``. A memory handle is pointer-width, ctypes defaults an +undeclared parameter to ``c_int``, and so ``GlobalLock(handle)`` raises +``OverflowError: int too long to convert`` for any real handle on 64-bit +Windows. Declaring only ``restype`` fixes the value coming *back* and leaves the +argument going *in* broken, which is exactly what three modules did — their +``set_clipboard_*`` functions failed on every single call. + +Handles are private (``WinDLL``, not the process-wide cached ``windll``): +prototypes live on the function objects, so a shared handle would leak these +declarations into every other user32 caller in the process. +""" +import ctypes +import sys +import time +from contextlib import contextmanager +from ctypes import wintypes +from typing import Iterator, Optional, Tuple + +GMEM_MOVEABLE = 0x0002 +_OPEN_FAILED = "OpenClipboard failed" + +# Only one process may hold the clipboard open at a time, so OpenClipboard +# fails outright whenever another application is mid-copy — Explorer, Office +# and every browser own it for a few milliseconds at a time. Reporting that as +# the caller's failure is wrong twice over: Win32 documents it as the condition +# to retry, and a library whose job is driving machines that are busy by +# definition cannot treat "somebody else was copying" as an error. Roughly +# 200 ms of waiting covers the transient owners without hanging a script +# behind one that keeps the clipboard for real. +_OPEN_ATTEMPTS = 10 +_OPEN_RETRY_SECONDS = 0.02 + + +def _require_windows() -> None: + if not sys.platform.startswith("win"): + raise RuntimeError("the Win32 clipboard API is only available on Windows") + + +def clipboard_api() -> Tuple[object, object]: + """``(user32, kernel32)`` with every clipboard prototype declared.""" + _require_windows() + user32 = ctypes.WinDLL("user32", use_last_error=True) + kernel32 = ctypes.WinDLL("kernel32", use_last_error=True) + + user32.OpenClipboard.argtypes = [wintypes.HWND] + user32.OpenClipboard.restype = wintypes.BOOL + user32.EmptyClipboard.argtypes = [] + user32.EmptyClipboard.restype = wintypes.BOOL + user32.CloseClipboard.argtypes = [] + user32.CloseClipboard.restype = wintypes.BOOL + user32.SetClipboardData.argtypes = [wintypes.UINT, wintypes.HANDLE] + user32.SetClipboardData.restype = wintypes.HANDLE + user32.GetClipboardData.argtypes = [wintypes.UINT] + user32.GetClipboardData.restype = wintypes.HANDLE + user32.RegisterClipboardFormatW.argtypes = [wintypes.LPCWSTR] + user32.RegisterClipboardFormatW.restype = wintypes.UINT + user32.EnumClipboardFormats.argtypes = [wintypes.UINT] + user32.EnumClipboardFormats.restype = wintypes.UINT + user32.GetClipboardFormatNameW.argtypes = [wintypes.UINT, wintypes.LPWSTR, + ctypes.c_int] + user32.GetClipboardFormatNameW.restype = ctypes.c_int + kernel32.GlobalAlloc.argtypes = [wintypes.UINT, ctypes.c_size_t] + kernel32.GlobalAlloc.restype = wintypes.HGLOBAL + kernel32.GlobalLock.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalLock.restype = ctypes.c_void_p + kernel32.GlobalUnlock.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalUnlock.restype = wintypes.BOOL + kernel32.GlobalSize.argtypes = [wintypes.HGLOBAL] + kernel32.GlobalSize.restype = ctypes.c_size_t + return user32, kernel32 + + +@contextmanager +def open_clipboard(user32: Optional[object] = None) -> Iterator[object]: + """Own the clipboard for the block, waiting out a transiently busy one. + + Pass the ``user32`` handle you already prototyped to keep the declarations + private to your module; omit it and one is built here. The clipboard is + closed on the way out however the block ends — leaving it open locks every + other process on the desktop out of it. + """ + api = clipboard_api()[0] if user32 is None else user32 + for attempt in range(_OPEN_ATTEMPTS): + if api.OpenClipboard(None): + break + if attempt == _OPEN_ATTEMPTS - 1: + raise RuntimeError(_OPEN_FAILED) + time.sleep(_OPEN_RETRY_SECONDS) + try: + yield api + finally: + api.CloseClipboard() + + +def register_format(name: str) -> int: + """Register (or look up) a named clipboard format id.""" + user32, _kernel32 = clipboard_api() + return int(user32.RegisterClipboardFormatW(name)) + + +def set_clipboard_format(format_id: int, payload: bytes, *, + empty_first: bool = True) -> None: + """Put raw bytes on the clipboard under ``format_id``. + + The buffer is allocated and filled *before* the clipboard is opened, so a + failure part-way leaves the user's clipboard untouched rather than emptied. + On success the clipboard owns the handle and must not free it here. + """ + user32, kernel32 = clipboard_api() + handle = kernel32.GlobalAlloc(GMEM_MOVEABLE, len(payload)) + if not handle: + raise RuntimeError("GlobalAlloc failed") + pointer = kernel32.GlobalLock(handle) + if not pointer: + raise RuntimeError("GlobalLock failed") + ctypes.memmove(pointer, payload, len(payload)) + kernel32.GlobalUnlock(handle) + with open_clipboard(user32): + if empty_first: + user32.EmptyClipboard() + if not user32.SetClipboardData(int(format_id), handle): + raise RuntimeError(f"SetClipboardData({format_id}) failed") + + +def get_clipboard_format(format_id: int) -> Optional[bytes]: + """Read the clipboard's ``format_id`` payload, or ``None`` when absent.""" + user32, kernel32 = clipboard_api() + with open_clipboard(user32): + handle = user32.GetClipboardData(int(format_id)) + if not handle: + return None + pointer = kernel32.GlobalLock(handle) + if not pointer: + return None + try: + return ctypes.string_at(pointer, kernel32.GlobalSize(handle)) + finally: + kernel32.GlobalUnlock(handle) diff --git a/je_auto_control/utils/clipboard_files/clipboard_files.py b/je_auto_control/utils/clipboard_files/clipboard_files.py index a3be087d..8a1ff45b 100644 --- a/je_auto_control/utils/clipboard_files/clipboard_files.py +++ b/je_auto_control/utils/clipboard_files/clipboard_files.py @@ -66,43 +66,22 @@ def get_clipboard_files() -> Optional[List[str]]: def _win_set_hdrop(blob: bytes) -> None: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - user32.EmptyClipboard() - handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(blob)) - if not handle: - raise RuntimeError("GlobalAlloc failed") - pointer = kernel32.GlobalLock(handle) - ctypes.memmove(pointer, blob, len(blob)) - kernel32.GlobalUnlock(handle) - if not user32.SetClipboardData(_CF_HDROP, handle): - raise RuntimeError("SetClipboardData(CF_HDROP) failed") - finally: - user32.CloseClipboard() + """Delegates to the one place that declares the Win32 prototypes. + + This module used to hand-roll the calls with ``restype`` but no + ``argtypes``: a memory handle is pointer-width, ctypes defaults an + undeclared parameter to ``c_int``, and so ``GlobalLock(handle)`` raised + ``OverflowError: int too long to convert`` on 64-bit Windows — + ``set_clipboard_files`` failed on every call. + """ + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + set_clipboard_format, + ) + set_clipboard_format(_CF_HDROP, blob) def _win_get_hdrop() -> Optional[bytes]: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - user32.GetClipboardData.restype = wintypes.HANDLE - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - handle = user32.GetClipboardData(_CF_HDROP) - if not handle: - return None - pointer = kernel32.GlobalLock(handle) - size = kernel32.GlobalSize(handle) - data = ctypes.string_at(pointer, size) - kernel32.GlobalUnlock(handle) - return data - finally: - user32.CloseClipboard() + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + get_clipboard_format, + ) + return get_clipboard_format(_CF_HDROP) diff --git a/je_auto_control/utils/clipboard_formats/clipboard_formats.py b/je_auto_control/utils/clipboard_formats/clipboard_formats.py index 0527b7e2..b25e3148 100644 --- a/je_auto_control/utils/clipboard_formats/clipboard_formats.py +++ b/je_auto_control/utils/clipboard_formats/clipboard_formats.py @@ -115,11 +115,14 @@ def list_clipboard_formats() -> List[Dict[str, Any]]: import sys if not sys.platform.startswith("win"): raise RuntimeError("list_clipboard_formats is only supported on Windows") - import ctypes - user32 = ctypes.windll.user32 - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: + # Prototypes come from the shared module, on a *private* handle: declaring + # them on the process-wide ``ctypes.windll.user32`` would leak into every + # other caller in this process. + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + clipboard_api, open_clipboard, + ) + user32, _kernel32 = clipboard_api() + with open_clipboard(user32): formats: List[Dict[str, Any]] = [] format_id = user32.EnumClipboardFormats(0) while format_id: @@ -127,8 +130,6 @@ def list_clipboard_formats() -> List[Dict[str, Any]]: "name": _format_name(user32, format_id)}) format_id = user32.EnumClipboardFormats(format_id) return formats - finally: - user32.CloseClipboard() def clipboard_formats() -> Dict[str, Any]: diff --git a/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py b/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py index 8e277b24..30b07010 100644 --- a/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py +++ b/je_auto_control/utils/clipboard_rich_formats/clipboard_rich_formats.py @@ -166,53 +166,30 @@ def csv_to_rows(text: str, *, delimiter: str = ",") -> List[List[str]]: # --- Win32 clipboard I/O --------------------------------------------------- def _format_id(name: str) -> int: - import ctypes - return ctypes.windll.user32.RegisterClipboardFormatW(name) + from je_auto_control.utils.clipboard.win32_clipboard_api import register_format + return register_format(name) def _win_set_format(format_id: int, payload: bytes, *, empty_first: bool = True) -> None: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - if empty_first: - user32.EmptyClipboard() - handle = kernel32.GlobalAlloc(_GMEM_MOVEABLE, len(payload)) - if not handle: - raise RuntimeError("GlobalAlloc failed") - pointer = kernel32.GlobalLock(handle) - ctypes.memmove(pointer, payload, len(payload)) - kernel32.GlobalUnlock(handle) - if not user32.SetClipboardData(format_id, handle): - raise RuntimeError("SetClipboardData failed") - finally: - user32.CloseClipboard() + """Delegates to the one place that declares the Win32 prototypes. + + This used to hand-roll the calls with ``restype`` but no ``argtypes``, so + ``GlobalLock`` raised ``OverflowError`` on 64-bit Windows and + ``set_clipboard_rtf`` / ``set_clipboard_csv`` had never worked. + """ + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + set_clipboard_format, + ) + set_clipboard_format(format_id, payload, empty_first=empty_first) def _win_get_format(format_id: int) -> Optional[bytes]: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - user32.GetClipboardData.restype = wintypes.HANDLE - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - handle = user32.GetClipboardData(format_id) - if not handle: - return None - pointer = kernel32.GlobalLock(handle) - size = kernel32.GlobalSize(handle) - data = ctypes.string_at(pointer, size) - kernel32.GlobalUnlock(handle) - return data.split(b"\x00", 1)[0] - finally: - user32.CloseClipboard() + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + get_clipboard_format, + ) + data = get_clipboard_format(format_id) + return None if data is None else data.split(b"\x00", 1)[0] def _seed_plaintext(text: str) -> None: diff --git a/je_auto_control/utils/cv2_utils/screen_grabber.py b/je_auto_control/utils/cv2_utils/screen_grabber.py new file mode 100644 index 00000000..7c977ac3 --- /dev/null +++ b/je_auto_control/utils/cv2_utils/screen_grabber.py @@ -0,0 +1,229 @@ +"""One place that decides how this platform's pixels are read. + +Pillow's ``ImageGrab`` and ``mss`` both read the X11 root window on Linux. +Under a Wayland session that root belongs to XWayland, which does not +composite native Wayland windows, so what comes back is not the desktop the +user sees. + +Pillow does have a fallback — but a narrower one than it first appears. +Reading ``PIL/ImageGrab.py``: the ``gnome-screenshot`` / ``grim`` / +``spectacle`` path runs only inside ``except OSError`` around +``grabscreen_x11``, so it fires when Pillow lacks XCB or there is no X +display at all. With XWayland running — the default on GNOME, KDE and sway — +``DISPLAY`` is set and the X11 grab *succeeds*, so the fallback never +triggers and the caller gets the XWayland root. ``mss`` has no fallback in +any configuration. + +Every locator, OCR call, screenshot, recording and remote-desktop frame in +the framework reached for one of those two libraries directly, so on the +common Wayland setup none of them saw the real screen while the Wayland +backend's own capture went unused. Routing through the backend also makes +the behaviour deterministic rather than dependent on whether XWayland +happens to be running, lets ``grim`` apply a region natively, and gives a +failure an actionable message instead of a blank frame. + +A platform backend that the generic libraries cannot see through publishes +a ``grab_image(screen_region=None) -> PIL.Image.Image`` function (only +:mod:`je_auto_control.linux_wayland.screen` needs to today). This module +wraps it in whichever library shape the caller already uses, so call sites +stay as they were: + +* :func:`image_grabber` — an ``ImageGrab``-shaped object (``.grab(bbox=...)``). +* :func:`mss_grabber` — an ``mss.mss()``-shaped context manager + (``.monitors``, ``.grab(monitor)``). + +A backend may also publish ``layout_origin() -> (x, y)``, the coordinate its +whole-screen capture starts at. It is ``(0, 0)`` for most desktops and +negative on any layout with a monitor left of or above the primary one, which +is why :func:`backend_layout_origin` reports it rather than letting each +caller assume the frame begins at the origin. + +Where the backend publishes no ``grab_image`` — Windows, macOS, Linux X11 — +both functions hand back the real library untouched, so nothing about those +platforms changes. +""" +from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple + +from je_auto_control.utils.exception.exceptions import AutoControlException + +Region = Sequence[int] +GrabImage = Callable[..., Any] + + +def backend_grab_image() -> Optional[GrabImage]: + """Return the active platform backend's ``grab_image``, or None. + + None means the generic libraries can see this session and should be + used as-is. + """ + try: + from je_auto_control.wrapper.platform_wrapper import screen + except (ImportError, AutoControlException): + return None + return getattr(screen, "grab_image", None) + + +class _BackendImageGrab: # pylint: disable=too-few-public-methods # reason: must expose exactly ImageGrab's surface + """``ImageGrab``-shaped adapter over a backend's ``grab_image``.""" + + def __init__(self, grab_image: GrabImage) -> None: + self._grab_image = grab_image + + def grab(self, bbox: Optional[Region] = None, **_kwargs: Any) -> Any: + """Capture ``bbox`` (left, top, right, bottom), or everything. + + ``ImageGrab``'s other keywords (``all_screens``, ``xdisplay``, + ``include_layered_windows``) are accepted and ignored: a Wayland + capture already spans the whole output layout and has no X display + to name, so honouring them would mean promising a distinction the + compositor does not offer. + """ + return self._grab_image(screen_region=list(bbox) if bbox is not None else None) + + +class _BackendShot: + """``mss.ScreenShot``-shaped view of one captured image.""" + + def __init__(self, image: Any, left: int, top: int) -> None: + self._image = image if image.mode == "RGB" else image.convert("RGB") + self.width = int(self._image.width) + self.height = int(self._image.height) + self.left = int(left) + self.top = int(top) + self.bgra = self._image.convert("RGBA").tobytes("raw", "BGRA") + # numpy reads this to build an array, the same way mss's own + # ScreenShot exposes its BGRA buffer. + self.__array_interface__ = { + "version": 3, + "shape": (self.height, self.width, 4), + "typestr": "|u1", + "data": self.bgra, + } + + @property + def size(self) -> Tuple[int, int]: + """``(width, height)``, as ``mss.ScreenShot.size`` reports it.""" + return self.width, self.height + + @property + def pos(self) -> Tuple[int, int]: + """``(left, top)`` of the captured rectangle.""" + return self.left, self.top + + @property + def rgb(self) -> bytes: + """The frame as packed RGB bytes.""" + return self._image.tobytes() + + +class _BackendMss: + """``mss.mss()``-shaped adapter over a backend's ``grab_image``. + + The compositor reports one seamless output layout rather than the + per-monitor rectangles mss enumerates, so ``monitors`` holds the + combined desktop twice: index 0 by mss's convention, and index 1 so + callers that default to "the first real screen" still get a frame. + """ + + def __init__(self, grab_image: GrabImage, + screen_size: Callable[[], Tuple[int, int]], + layout_origin: Optional[Callable[[], Tuple[int, int]]] = None) -> None: + self._grab_image = grab_image + self._screen_size = screen_size + self._layout_origin = layout_origin or (lambda: (0, 0)) + self._monitors: Optional[List[Dict[str, int]]] = None + + @property + def monitors(self) -> List[Dict[str, int]]: + """Monitor rectangles, mss-style: index 0 spans the whole desktop. + + ``left`` / ``top`` are the layout's origin rather than a hardcoded + ``(0, 0)``: mss reports a negative left for a monitor placed to the + left of the primary one, and :meth:`grab` feeds these straight back + as a capture region — so a layout with a negative origin described + as starting at ``(0, 0)`` grabs a rectangle that is off the screen + on one side and misses that much of the desktop on the other. + """ + if self._monitors is None: + width, height = self._screen_size() + left, top = self._layout_origin() + layout = {"left": int(left), "top": int(top), + "width": int(width), "height": int(height)} + self._monitors = [dict(layout), dict(layout)] + return self._monitors + + def grab(self, monitor: Dict[str, int]) -> _BackendShot: + """Capture one monitor rectangle and return an mss-shaped shot.""" + left = int(monitor["left"]) + top = int(monitor["top"]) + right = left + int(monitor["width"]) + bottom = top + int(monitor["height"]) + image = self._grab_image(screen_region=[left, top, right, bottom]) + return _BackendShot(image, left, top) + + def close(self) -> None: + """No handle to release; present so the mss shape is complete.""" + + def __enter__(self) -> "_BackendMss": + return self + + def __exit__(self, *_exc: Any) -> None: + self.close() + + +def _backend_screen_size() -> Tuple[int, int]: + from je_auto_control.wrapper.platform_wrapper import screen + return screen.size() + + +def backend_layout_origin() -> Tuple[int, int]: + """Top-left of the backend's whole-screen capture, in screen coordinates. + + ``(0, 0)`` for a backend that publishes no ``layout_origin`` — which is + every backend the generic libraries can already see (Windows, macOS, + X11), and the right answer for any layout that starts at the origin + anyway. Only the origin is reported, never the size: a caller that has + the frame already knows how big it is, and asking a backend for its + size can cost a whole extra capture where no tool can report it. + """ + try: + from je_auto_control.wrapper.platform_wrapper import screen + except (ImportError, AutoControlException): + return (0, 0) + origin = getattr(screen, "layout_origin", None) + if not callable(origin): + return (0, 0) + # pylint: disable=not-callable # reason: pylint resolves the getattr to its + # None default against whichever backend this host imports, and every + # backend but Wayland publishes no layout_origin; callable() is the check. + left, top = origin() + return (int(left), int(top)) + + +def image_grabber() -> Any: + """Return an ``ImageGrab``-shaped grabber for the current platform.""" + grab_image = backend_grab_image() + if grab_image is None: + from PIL import ImageGrab + return ImageGrab + return _BackendImageGrab(grab_image) + + +def mss_grabber() -> Any: + """Return an ``mss.mss()``-shaped grabber for the current platform. + + Use in place of ``mss.mss()``; the result is a context manager either + way, so ``with mss_grabber() as sct:`` works unchanged. + """ + grab_image = backend_grab_image() + if grab_image is None: + import mss + # mss 10 deprecated the lowercase factory in favour of MSS; both + # yield the same platform class, so prefer the one that is current. + return (getattr(mss, "MSS", None) or mss.mss)() + return _BackendMss(grab_image, _backend_screen_size, + backend_layout_origin) + + +__all__ = ["backend_grab_image", "backend_layout_origin", "image_grabber", + "mss_grabber"] diff --git a/je_auto_control/utils/cv2_utils/screenshot.py b/je_auto_control/utils/cv2_utils/screenshot.py index 0c10ccbd..3c6c84a4 100644 --- a/je_auto_control/utils/cv2_utils/screenshot.py +++ b/je_auto_control/utils/cv2_utils/screenshot.py @@ -1,6 +1,7 @@ -from PIL import ImageGrab, Image +from PIL import Image from typing import List, Optional +from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber from je_auto_control.utils.exception.exceptions import AutoControlScreenException @@ -31,8 +32,14 @@ def _validate_region(screen_region: List[int]) -> None: def pil_screenshot(file_path: Optional[str] = None, screen_region: Optional[List[int]] = None) -> Image.Image: """ - Take a screenshot using PIL (Pillow). - 使用 PIL (Pillow) 擷取螢幕畫面 + Take a screenshot through the platform's capture backend. + 透過平台擷取後端擷取螢幕畫面 + + Kept named ``pil_screenshot`` (and still returning a Pillow image) + because it is public API, but the grabber is now chosen per platform: + Pillow's ``ImageGrab`` reads the X11 root window, which under Wayland + belongs to XWayland and holds none of the native Wayland windows. See + :mod:`je_auto_control.utils.cv2_utils.screen_grabber`. :param file_path: (str | None) Path to save the screenshot. If None, do not save. 螢幕截圖的存檔路徑，若為 None 則不存檔 @@ -41,11 +48,12 @@ def pil_screenshot(file_path: Optional[str] = None, screen_region: Optional[List :return: PIL.Image.Image object 擷取到的影像物件 """ # 擷取螢幕畫面 Capture screen + grabber = image_grabber() if screen_region is not None: _validate_region(screen_region) - image = ImageGrab.grab(bbox=screen_region) + image = grabber.grab(bbox=screen_region) else: - image = ImageGrab.grab() + image = grabber.grab() # 如果指定了存檔路徑，則存檔 Save if file_path is provided. # Fail fast: a swallowed save error would leave the caller believing the diff --git a/je_auto_control/utils/cv2_utils/video_recording.py b/je_auto_control/utils/cv2_utils/video_recording.py index b2efe91a..3128b8c2 100644 --- a/je_auto_control/utils/cv2_utils/video_recording.py +++ b/je_auto_control/utils/cv2_utils/video_recording.py @@ -1,8 +1,8 @@ import threading import cv2 import numpy as np -from mss import mss +from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -10,7 +10,7 @@ class RecordingThread(threading.Thread): """ RecordingThread 螢幕錄影執行緒 - - 使用 mss 擷取螢幕畫面 + - 透過平台擷取後端取得畫面 (mss，Wayland 則走合成器的擷取工具) - 使用 OpenCV VideoWriter 寫入影片檔案 """ @@ -44,7 +44,7 @@ def run(self): 執行錄影迴圈 Run recording loop """ - with mss() as sct: + with mss_grabber() as sct: resolution = sct.monitors[0] output_file = self.video_name + ".mp4" diff --git a/je_auto_control/utils/diagnostics/diagnostics.py b/je_auto_control/utils/diagnostics/diagnostics.py index ddbecb4e..4b214995 100644 --- a/je_auto_control/utils/diagnostics/diagnostics.py +++ b/je_auto_control/utils/diagnostics/diagnostics.py @@ -136,6 +136,48 @@ def _check_audit_chain() -> Check: ) +def _check_screen_capture() -> Check: + """Report which grabber reads the screen, and on Wayland which tool. + + A Wayland session with no capture helper installed is the one case where + the framework can still move the mouse but sees nothing, so it is worth + naming before a locator fails with "template not found". + + The Wayland answer also carries ``cursor_may_be_captured``. wlroots + composites a *software* cursor into the output buffer whenever the + backend has no cursor plane, and that buffer is what ``wlr-screencopy`` + hands back, so the pointer can appear in a capture that never asked for + it — unlike BitBlt on Windows or Pillow/mss on X11. A pointer resting on + a target puts a pointer-shaped hole in the middle of it, and the bundle + that explains a failed locator should say so. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import backend_grab_image + if backend_grab_image() is None: + return Check( + name="screen_capture", ok=True, severity=_SEVERITY_INFO, + detail="generic capture (Pillow / mss)", + extra={"grabber": "generic"}, + ) + from je_auto_control.linux_wayland.capture import available_tool + tool = available_tool() + if tool is None: + return Check( + name="screen_capture", ok=False, severity=_SEVERITY_ERROR, + detail="Wayland session with no capture tool and no session " + "bus: install grim (wlroots), gnome-screenshot (GNOME) or " + "spectacle (KDE); the xdg-desktop-portal fallback needs " + "DBUS_SESSION_BUS_ADDRESS set", + extra={"grabber": "wayland", "tool": None}, + ) + return Check( + name="screen_capture", ok=True, severity=_SEVERITY_INFO, + detail=f"Wayland capture via {tool}; the pointer may be composited " + "into the image, so park it away from the target", + extra={"grabber": "wayland", "tool": tool, + "cursor_may_be_captured": True}, + ) + + def _check_screenshot() -> Check: from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot image = pil_screenshot() @@ -218,6 +260,7 @@ def _check_executor() -> Check: _check_optional_deps, _check_executor, _check_audit_chain, + _check_screen_capture, _check_screenshot, _check_mouse, _check_disk_space, diff --git a/je_auto_control/utils/executor/action_executor.py b/je_auto_control/utils/executor/action_executor.py index fb919e02..ee3cc775 100644 --- a/je_auto_control/utils/executor/action_executor.py +++ b/je_auto_control/utils/executor/action_executor.py @@ -24,7 +24,9 @@ from je_auto_control.utils.clipboard.clipboard import ( get_clipboard, set_clipboard, ) -from je_auto_control.utils.executor.action_schema import validate_actions +from je_auto_control.utils.executor.action_schema import ( + unknown_command_names, validate_actions, +) from je_auto_control.utils.executor.flow_control import ( BLOCK_COMMANDS, LoopBreak, LoopContinue, ) @@ -81,9 +83,11 @@ from je_auto_control.wrapper.auto_control_record import record, stop_record from je_auto_control.wrapper.auto_control_screen import screenshot, screen_size from je_auto_control.wrapper.auto_control_window import ( - close_window_by_title, focus_window, foreground_window, list_windows, - minimize_window_by_title, move_window_by_title, wait_for_window, - window_rect, + close_window_by_title, focus_window, foreground_window, + foreground_window_process_id, list_windows, minimize_window_by_title, + minimize_windows_for_process, move_window_by_title, post_click_to_window, + post_key_to_window, wait_for_window, window_process_id, window_rect, + windows_for_process_id, ) @@ -6657,6 +6661,46 @@ def _window_rect(title_substring: str, return {"rect": list(rect) if rect is not None else None} +def _foreground_window_process_id() -> Dict[str, Any]: + """Adapter: the PID owning the foreground window (``0`` when unknown).""" + return {"pid": foreground_window_process_id() or 0} + + +def _window_process_id(title_substring: str, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: the PID owning the first matching window (``0`` when none).""" + return {"pid": window_process_id(title_substring, bool(case_sensitive)) or 0} + + +def _windows_for_process_id(pid: int, + titled_only: bool = False) -> Dict[str, Any]: + """Adapter: every visible top-level window owned by a process.""" + found = windows_for_process_id(int(pid), bool(titled_only)) + return {"windows": [{"hwnd": hwnd, "title": title} + for hwnd, title in found]} + + +def _minimize_windows_for_process(pid: int) -> Dict[str, Any]: + """Adapter: minimise every window a process owns.""" + return {"minimized": minimize_windows_for_process(int(pid))} + + +def _post_key_to_window(title_substring: str, key: str, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: post one key to a window without focusing it.""" + return {"posted": bool(post_key_to_window(title_substring, key, + bool(case_sensitive)))} + + +def _post_click_to_window(title_substring: str, button: str = "left", + x: int = 0, y: int = 0, + case_sensitive: bool = False) -> Dict[str, Any]: + """Adapter: post one click into a window without focusing it.""" + return {"posted": bool(post_click_to_window(title_substring, button, + int(x), int(y), + bool(case_sensitive)))} + + def _canonicalize_url(url: str) -> Dict[str, Any]: """Adapter: opinionated canonical form of a URL, for equality checks.""" from je_auto_control.utils.url_canon import canonicalize_url @@ -7063,6 +7107,12 @@ def __init__(self): "AC_close_window": close_window_by_title, "AC_minimize_window": minimize_window_by_title, "AC_foreground_window": _foreground_window, + "AC_foreground_window_pid": _foreground_window_process_id, + "AC_window_pid": _window_process_id, + "AC_windows_for_pid": _windows_for_process_id, + "AC_minimize_windows_for_pid": _minimize_windows_for_process, + "AC_post_key_to_window": _post_key_to_window, + "AC_post_click_to_window": _post_click_to_window, "AC_window_rect": _window_rect, "AC_move_window": move_window_by_title, @@ -7861,6 +7911,17 @@ def known_commands(self) -> set: """Return the set of all command names the executor recognises.""" return set(self.event_dict.keys()) | set(self._block_commands.keys()) + def unknown_commands_in(self, action_list: Union[list, dict]) -> List[str]: + """Return the unrecognised command names in ``action_list``, in order. + + Nothing is executed. Structural problems raise exactly as + :meth:`execute_action` would, so a caller-facing boundary can tell + "you named a command that does not exist" apart from "the run + failed" *before* the first action moves anything. + """ + return unknown_command_names(self._unwrap_action_list(action_list), + self.known_commands()) + def _resolve_runtime_args(self, args: Any) -> Any: """Interpolate ``${var}`` placeholders against the current scope. diff --git a/je_auto_control/utils/executor/action_schema.py b/je_auto_control/utils/executor/action_schema.py index 1bb9ae79..f719b888 100644 --- a/je_auto_control/utils/executor/action_schema.py +++ b/je_auto_control/utils/executor/action_schema.py @@ -4,7 +4,7 @@ Validates the outer shape (``[name]`` / ``[name, params]``), that names are in the executor allowlist, and that flow-control nested bodies are themselves valid lists. """ -from typing import Any, Iterable, Set +from typing import Any, Iterable, Iterator, List, Tuple from je_auto_control.utils.exception.exceptions import AutoControlActionException @@ -42,52 +42,87 @@ def validate_actions(actions: Any, known_commands: Iterable[str]) -> None: """Validate an action list recursively; raise on the first problem.""" known = set(known_commands) - _validate_list(actions, known, trail="root") + for trail, name in _iter_actions(actions, "root"): + if not isinstance(name, str) or name not in known: + raise AutoControlActionException(f"{trail}: unknown command {name!r}") -def _validate_list(actions: Any, known: Set[str], trail: str) -> None: +def unknown_command_names(actions: Any, + known_commands: Iterable[str]) -> List[str]: + """Return every unrecognised command name in ``actions``, in order. + + Structural problems still raise, exactly as :func:`validate_actions` does; + the only difference is that an unknown *name* is collected instead of + ending the walk. A boundary that has to answer a caller — the REST API — + reports the whole list, so a client fixes every typo in one round trip + rather than one per request. + """ + known = set(known_commands) + unknown: List[str] = [] + for _trail, name in _iter_actions(actions, "root"): + if isinstance(name, str) and name in known: + continue + label = name if isinstance(name, str) else repr(name) + if label not in unknown: + unknown.append(label) + return unknown + + +def _iter_actions(actions: Any, trail: str) -> Iterator[Tuple[str, Any]]: + """Yield ``(trail, command_name)`` for every action in the tree, in order. + + Structural problems raise as they are met, but whether the name is in the + allowlist is left to the caller. That is what lets "reject the first + unknown name" and "collect every unknown name" share one traversal — and, + more importantly, one definition of where a nested action list may hide. + + Laziness is load-bearing: the caller inspects the name at the ``yield`` + before this walk goes on to check the params, so the order in which the + two complaints surface is unchanged from when it was all one function. + """ if not isinstance(actions, list): raise AutoControlActionException( f"{trail}: action list must be a list, got {type(actions).__name__}" ) for idx, action in enumerate(actions): - _validate_single(action, known, f"{trail}[{idx}]") - - -def _validate_single(action: Any, known: Set[str], trail: str) -> None: - if not isinstance(action, list) or not 1 <= len(action) <= 2: - raise AutoControlActionException( - f"{trail}: must be [name] or [name, params]" - ) - name = action[0] - if not isinstance(name, str) or name not in known: - raise AutoControlActionException(f"{trail}: unknown command {name!r}") - if len(action) == 2 and not isinstance(action[1], (dict, list)): - raise AutoControlActionException( - f"{trail}: params must be dict or list" - ) - _validate_nested_bodies(name, action, known, trail) - - -def _validate_nested_bodies(name: str, action: list, known: Set[str], trail: str) -> None: + node = f"{trail}[{idx}]" + if not isinstance(action, list) or not 1 <= len(action) <= 2: + raise AutoControlActionException( + f"{node}: must be [name] or [name, params]" + ) + yield node, action[0] + if len(action) == 2 and not isinstance(action[1], (dict, list)): + raise AutoControlActionException( + f"{node}: params must be dict or list" + ) + yield from _iter_nested_actions(action[0], action, node) + + +def _iter_nested_actions(name: Any, action: list, + trail: str) -> Iterator[Tuple[str, Any]]: + """Yield the actions held in a flow-control command's nested body keys.""" + # ``name`` arrives unvalidated — the collector does not stop on a bad one — + # and an unhashable name would blow up the two lookups below. + if not isinstance(name, str): + return if len(action) < 2 or not isinstance(action[1], dict): return params = action[1] for body_key in FLOW_BODY_KEYS.get(name, ()): body = params.get(body_key) if body is not None: - _validate_list(body, known, f"{trail}.{body_key}") + yield from _iter_actions(body, f"{trail}.{body_key}") for list_key in FLOW_BRANCH_LIST_KEYS.get(name, ()): - _validate_branch_list(params.get(list_key), known, f"{trail}.{list_key}") - - -def _validate_branch_list(branches: Any, known: Set[str], trail: str) -> None: - """Validate a list of action lists (e.g. ``AC_parallel``'s ``branches``).""" - if branches is None: - return - # The visual builder may pass a JSON string, which the runtime parses via - # _as_list. Leave that shape to the runtime rather than reject it here. - if not isinstance(branches, list): - return - for idx, branch in enumerate(branches): - _validate_list(branch, known, f"{trail}[{idx}]") + branches = params.get(list_key) + # The visual builder may pass a JSON string, which the runtime parses + # via _as_list. Leave that shape to the runtime rather than reject it. + if not isinstance(branches, list): + continue + for idx, branch in enumerate(branches): + yield from _iter_actions(branch, f"{trail}.{list_key}[{idx}]") + + +__all__ = [ + "FLOW_BODY_KEYS", "FLOW_BRANCH_LIST_KEYS", + "validate_actions", "unknown_command_names", +] diff --git a/je_auto_control/utils/executor/flow_control.py b/je_auto_control/utils/executor/flow_control.py index 87a816ec..aeb1b462 100644 --- a/je_auto_control/utils/executor/flow_control.py +++ b/je_auto_control/utils/executor/flow_control.py @@ -13,6 +13,12 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlActionException, AutoControlException, ImageNotFoundException, ) +from je_auto_control.utils.executor.flow_data_commands import ( + exec_assert_db, exec_assert_duration, exec_assert_var, exec_http_to_var, + exec_now_to_var, exec_ocr_to_var, exec_otp_to_var, exec_pdf_to_var, + exec_random_to_var, exec_read_file_to_var, exec_shell_to_var, + exec_sql_to_var, exec_transform_var, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.wrapper.auto_control_image import locate_image_center from je_auto_control.wrapper.auto_control_screen import get_pixel @@ -389,239 +395,6 @@ def exec_for_each_row(executor: Any, args: Mapping[str, Any]) -> int: return iterations -def exec_shell_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Run a shell command and store its stdout in a flow variable. - - The command is split into an argv list (never ``shell=True``) and its - captured stdout is bound under ``var`` (default ``shell_output``) for - later ``${var}`` use — the shell counterpart of ``AC_ocr_to_var``. - """ - import os - import shlex - import subprocess # nosec B404 — argv list only, no shell - command = args.get("command", args.get("shell_command")) - argv = ([str(part) for part in command] if isinstance(command, list) - else shlex.split(str(command), posix=(os.name != "nt"))) - timeout_s = float(args.get("timeout", 30.0)) - try: - completed = subprocess.run( # nosec B603 — argv list, no shell # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit - argv, capture_output=True, check=False, timeout=timeout_s, - ) - except subprocess.TimeoutExpired as error: - # TimeoutExpired subclasses SubprocessError (not AutoControlException - # nor OSError), so without this it escapes every executor containment - # boundary and aborts the whole script. - raise AutoControlActionException( - f"AC_shell_to_var: command timed out after {timeout_s}s" - ) from error - output = completed.stdout.decode("utf-8", errors="replace").strip() - var_name = args.get("var", "shell_output") - executor.variables.set(var_name, output) - return {"var": var_name, "output": output, - "returncode": completed.returncode} - - -def _now(): - import datetime as _dt - return _dt.datetime.now() - - -def exec_now_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Store the current local time (strftime format) in a flow variable.""" - value = _now().strftime(str(args.get("format", "%Y-%m-%d %H:%M:%S"))) - var_name = args.get("var", "now") - executor.variables.set(var_name, value) - return {"var": var_name, "value": value} - - -def exec_random_to_var(executor: Any, - args: Mapping[str, Any]) -> Dict[str, Any]: - """Store a random value (int / float / choice) in a flow variable.""" - import random - rng = random.Random(args.get("seed")) # nosec B311 # reason: non-crypto test data - kind = str(args.get("kind", "int")) - if kind == "choice": - value: Any = rng.choice(list(args.get("choices") or [None])) # NOSONAR S2245 non-crypto seeded - elif kind == "float": - value = rng.uniform(float(args.get("min", 0.0)), - float(args.get("max", 1.0))) - else: - value = rng.randint(int(args.get("min", 0)), int(args.get("max", 100))) # NOSONAR S2245 non-crypto - var_name = args.get("var", "random") - executor.variables.set(var_name, value) - return {"var": var_name, "value": value} - - -def exec_assert_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert a flow variable satisfies a condition (assertion DSL).""" - from je_auto_control.utils.assertion import assert_variable - name = args["name"] - return assert_variable( - executor.variables.get_value(name), op=str(args.get("op", "eq")), - expected=args.get("value"), name=name, - raise_on_fail=bool(args.get("raise_on_fail", True)), - ).to_dict() - - -def exec_pdf_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Extract a PDF's text (all pages or one page) into a flow variable.""" - from je_auto_control.utils.pdf.pdf_reader import extract_pdf_text - text = extract_pdf_text(args["path"], pages=args.get("page")) - var_name = args.get("var", "pdf_text") - executor.variables.set(var_name, text) - return {"var": var_name, "length": len(text)} - - -def exec_otp_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Generate a TOTP code from a base32 secret into a flow variable (2FA).""" - from je_auto_control.utils.otp import generate_totp - code = generate_totp(args["secret"], step=int(args.get("step", 30)), - digits=int(args.get("digits", 6))) - var_name = args.get("var", "otp") - executor.variables.set(var_name, code) - return {"var": var_name} - - -def exec_sql_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Run a read-only SQLite query and store its result in a flow variable.""" - from je_auto_control.utils.sql.sql_query import query_sqlite - fetch = str(args.get("fetch", "all")) - result = query_sqlite(args["database"], args["query"], - params=args.get("params"), fetch=fetch) - var_name = args.get("var", "sql_result") - executor.variables.set(var_name, result) - return {"var": var_name, "fetch": fetch} - - -def exec_assert_db(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert a scalar SQLite query result satisfies a condition.""" - from je_auto_control.utils.assertion import assert_variable - from je_auto_control.utils.sql.sql_query import query_sqlite - value = query_sqlite(args["database"], args["query"], - params=args.get("params"), fetch="scalar") - return assert_variable( - value, op=str(args.get("op", "eq")), expected=args.get("expected"), - name="AC_assert_db", - raise_on_fail=bool(args.get("raise_on_fail", True)), - ).to_dict() - - -def exec_read_file_to_var(executor: Any, - args: Mapping[str, Any]) -> Dict[str, Any]: - """Read a file's text content into a flow variable.""" - from pathlib import Path - text = Path(args["path"]).read_text(encoding=args.get("encoding", "utf-8")) - var_name = args.get("var", "file_content") - executor.variables.set(var_name, text) - return {"var": var_name, "length": len(text)} - - -def _dig_json(body: str, path: str) -> Any: - """Navigate a dotted JSON path, e.g. ``data.0.name``.""" - data = json.loads(body) - for part in str(path).split("."): - data = data[int(part)] if isinstance(data, list) else data[part] - return data - - -def exec_http_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Request a URL and store the body (or a JSON field) in a flow variable. - - Supports method/headers/json_body/data/auth via the shared HTTP client, - so the same command drives plain GET reads and POST/PUT API calls. - """ - from je_auto_control.utils.http_client.http_client import http_request - response = http_request( - args["url"], method=str(args.get("method", "GET")), - headers=args.get("headers"), json_body=args.get("json_body"), - data=args.get("data"), auth=args.get("auth"), - timeout=float(args.get("timeout", 30.0)), - ) - json_path = args.get("json_path") - body = response["text"] - value = _dig_json(body, json_path) if json_path else body - var_name = args.get("var", "http_response") - executor.variables.set(var_name, value) - return {"var": var_name, "status": response["status"]} - - -_SIMPLE_TRANSFORMS: Dict[str, Callable[[str], str]] = { - "upper": str.upper, "lower": str.lower, "strip": str.strip, - "title": str.title, "lstrip": str.lstrip, "rstrip": str.rstrip, -} - - -def _regex_extract(text: str, args: Mapping[str, Any]) -> str: - import re - match = re.search(str(args.get("pattern", "")), text) - return match.group(int(args.get("group", 0))) if match else "" - - -def _slice_text(text: str, args: Mapping[str, Any]) -> str: - start = args.get("start") - end = args.get("end") - return text[(int(start) if start is not None else None): - (int(end) if end is not None else None)] - - -def _transform_string(text: str, op: str, args: Mapping[str, Any]) -> str: - simple = _SIMPLE_TRANSFORMS.get(op) - if simple is not None: - return simple(text) - if op == "replace": - return text.replace(str(args.get("find", "")), - str(args.get("replace_with", ""))) - if op == "regex": - return _regex_extract(text, args) - if op == "slice": - return _slice_text(text, args) - raise AutoControlActionException(f"AC_transform_var: unknown op {op!r}") - - -def exec_transform_var(executor: Any, - args: Mapping[str, Any]) -> Dict[str, Any]: - """Apply a string transform to a variable (in place or into ``into``).""" - name = args["name"] - value = str(executor.variables.get_value(name, "")) - result = _transform_string(value, str(args.get("op", "strip")), args) - target = args.get("into", name) - executor.variables.set(target, result) - return {"var": target, "value": result} - - -def exec_ocr_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Read OCR text from a screen region into a flow variable. - - Binds the recognised text under ``var`` (default ``ocr_text``) so later - steps can read it as ``${var}`` — the bridge between OCR and the - variable scope for data-driven flows. - """ - from je_auto_control.utils.ocr.ocr_engine import read_text_in_region - region = args.get("region") - if isinstance(region, str): - region = json.loads(region) if region.strip() else None - matches = read_text_in_region( - region=region, lang=args.get("lang", "eng"), - min_confidence=float(args.get("min_confidence", 60.0)), - ) - text = " ".join(match.text for match in matches).strip() - var_name = args.get("var", "ocr_text") - executor.variables.set(var_name, text) - return {"var": var_name, "text": text} - - -def exec_assert_duration(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: - """Assert ``body`` completes within ``max_ms`` (a performance budget).""" - from je_auto_control.utils.assertion import assert_duration - body = args.get("body") or [] - return assert_duration( - lambda: executor.execute_action(body, _validated=True), - max_ms=float(args.get("max_ms", 1000.0)), - min_ms=float(args.get("min_ms", 0.0)), - raise_on_fail=bool(args.get("raise_on_fail", True)), - ).to_dict() - - def _as_list(value: Any) -> list: """Accept a native list or a JSON-string list (for the visual builder).""" if isinstance(value, str): diff --git a/je_auto_control/utils/executor/flow_data_commands.py b/je_auto_control/utils/executor/flow_data_commands.py new file mode 100644 index 00000000..e574b410 --- /dev/null +++ b/je_auto_control/utils/executor/flow_data_commands.py @@ -0,0 +1,253 @@ +"""Data-source and transform block commands for the action executor. + +These are the ``*_to_var`` steps plus their variable assertions: they read +from a shell command, a clock, a PRNG, a PDF, a TOTP secret, a database, +a file, an HTTP endpoint or OCR, and store the result in the executor's +variable scope. Unlike :mod:`flow_control` they never execute a nested +action list, so they carry no loop or branch semantics. + +They are registered in ``flow_control.BLOCK_COMMANDS`` alongside the real +flow-control commands, which is the only table the executor reads. +""" +import json +from typing import Any, Callable, Dict, Mapping + +from je_auto_control.utils.exception.exceptions import AutoControlActionException + + +def exec_shell_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Run a shell command and store its stdout in a flow variable. + + The command is split into an argv list (never ``shell=True``) and its + captured stdout is bound under ``var`` (default ``shell_output``) for + later ``${var}`` use — the shell counterpart of ``AC_ocr_to_var``. + """ + import os + import shlex + import subprocess # nosec B404 — argv list only, no shell + command = args.get("command", args.get("shell_command")) + argv = ([str(part) for part in command] if isinstance(command, list) + else shlex.split(str(command), posix=(os.name != "nt"))) + timeout_s = float(args.get("timeout", 30.0)) + try: + completed = subprocess.run( # nosec B603 — argv list, no shell # nosemgrep: python.lang.security.audit.dangerous-subprocess-use-audit.dangerous-subprocess-use-audit + argv, capture_output=True, check=False, timeout=timeout_s, + ) + except subprocess.TimeoutExpired as error: + # TimeoutExpired subclasses SubprocessError (not AutoControlException + # nor OSError), so without this it escapes every executor containment + # boundary and aborts the whole script. + raise AutoControlActionException( + f"AC_shell_to_var: command timed out after {timeout_s}s" + ) from error + output = completed.stdout.decode("utf-8", errors="replace").strip() + var_name = args.get("var", "shell_output") + executor.variables.set(var_name, output) + return {"var": var_name, "output": output, + "returncode": completed.returncode} + + +def _now(): + import datetime as _dt + return _dt.datetime.now() + + +def exec_now_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Store the current local time (strftime format) in a flow variable.""" + value = _now().strftime(str(args.get("format", "%Y-%m-%d %H:%M:%S"))) + var_name = args.get("var", "now") + executor.variables.set(var_name, value) + return {"var": var_name, "value": value} + + +def exec_random_to_var(executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Store a random value (int / float / choice) in a flow variable.""" + import random + rng = random.Random(args.get("seed")) # nosec B311 # reason: non-crypto test data + kind = str(args.get("kind", "int")) + if kind == "choice": + value: Any = rng.choice(list(args.get("choices") or [None])) # NOSONAR S2245 non-crypto seeded + elif kind == "float": + value = rng.uniform(float(args.get("min", 0.0)), + float(args.get("max", 1.0))) + else: + value = rng.randint(int(args.get("min", 0)), int(args.get("max", 100))) # NOSONAR S2245 non-crypto + var_name = args.get("var", "random") + executor.variables.set(var_name, value) + return {"var": var_name, "value": value} + + +def exec_assert_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert a flow variable satisfies a condition (assertion DSL).""" + from je_auto_control.utils.assertion import assert_variable + name = args["name"] + return assert_variable( + executor.variables.get_value(name), op=str(args.get("op", "eq")), + expected=args.get("value"), name=name, + raise_on_fail=bool(args.get("raise_on_fail", True)), + ).to_dict() + + +def exec_pdf_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Extract a PDF's text (all pages or one page) into a flow variable.""" + from je_auto_control.utils.pdf.pdf_reader import extract_pdf_text + text = extract_pdf_text(args["path"], pages=args.get("page")) + var_name = args.get("var", "pdf_text") + executor.variables.set(var_name, text) + return {"var": var_name, "length": len(text)} + + +def exec_otp_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Generate a TOTP code from a base32 secret into a flow variable (2FA).""" + from je_auto_control.utils.otp import generate_totp + code = generate_totp(args["secret"], step=int(args.get("step", 30)), + digits=int(args.get("digits", 6))) + var_name = args.get("var", "otp") + executor.variables.set(var_name, code) + return {"var": var_name} + + +def exec_sql_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Run a read-only SQLite query and store its result in a flow variable.""" + from je_auto_control.utils.sql.sql_query import query_sqlite + fetch = str(args.get("fetch", "all")) + result = query_sqlite(args["database"], args["query"], + params=args.get("params"), fetch=fetch) + var_name = args.get("var", "sql_result") + executor.variables.set(var_name, result) + return {"var": var_name, "fetch": fetch} + + +def exec_assert_db(_executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert a scalar SQLite query result satisfies a condition. + + Takes the executor every ``BLOCK_COMMANDS`` entry is called with and + does not need it: this command asserts on the query, storing nothing. + """ + from je_auto_control.utils.assertion import assert_variable + from je_auto_control.utils.sql.sql_query import query_sqlite + value = query_sqlite(args["database"], args["query"], + params=args.get("params"), fetch="scalar") + return assert_variable( + value, op=str(args.get("op", "eq")), expected=args.get("expected"), + name="AC_assert_db", + raise_on_fail=bool(args.get("raise_on_fail", True)), + ).to_dict() + + +def exec_read_file_to_var(executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Read a file's text content into a flow variable.""" + from pathlib import Path + text = Path(args["path"]).read_text(encoding=args.get("encoding", "utf-8")) + var_name = args.get("var", "file_content") + executor.variables.set(var_name, text) + return {"var": var_name, "length": len(text)} + + +def _dig_json(body: str, path: str) -> Any: + """Navigate a dotted JSON path, e.g. ``data.0.name``.""" + data = json.loads(body) + for part in str(path).split("."): + data = data[int(part)] if isinstance(data, list) else data[part] + return data + + +def exec_http_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Request a URL and store the body (or a JSON field) in a flow variable. + + Supports method/headers/json_body/data/auth via the shared HTTP client, + so the same command drives plain GET reads and POST/PUT API calls. + """ + from je_auto_control.utils.http_client.http_client import http_request + response = http_request( + args["url"], method=str(args.get("method", "GET")), + headers=args.get("headers"), json_body=args.get("json_body"), + data=args.get("data"), auth=args.get("auth"), + timeout=float(args.get("timeout", 30.0)), + ) + json_path = args.get("json_path") + body = response["text"] + value = _dig_json(body, json_path) if json_path else body + var_name = args.get("var", "http_response") + executor.variables.set(var_name, value) + return {"var": var_name, "status": response["status"]} + + +_SIMPLE_TRANSFORMS: Dict[str, Callable[[str], str]] = { + "upper": str.upper, "lower": str.lower, "strip": str.strip, + "title": str.title, "lstrip": str.lstrip, "rstrip": str.rstrip, +} + + +def _regex_extract(text: str, args: Mapping[str, Any]) -> str: + import re + match = re.search(str(args.get("pattern", "")), text) + return match.group(int(args.get("group", 0))) if match else "" + + +def _slice_text(text: str, args: Mapping[str, Any]) -> str: + start = args.get("start") + end = args.get("end") + return text[(int(start) if start is not None else None): + (int(end) if end is not None else None)] + + +def _transform_string(text: str, op: str, args: Mapping[str, Any]) -> str: + simple = _SIMPLE_TRANSFORMS.get(op) + if simple is not None: + return simple(text) + if op == "replace": + return text.replace(str(args.get("find", "")), + str(args.get("replace_with", ""))) + if op == "regex": + return _regex_extract(text, args) + if op == "slice": + return _slice_text(text, args) + raise AutoControlActionException(f"AC_transform_var: unknown op {op!r}") + + +def exec_transform_var(executor: Any, + args: Mapping[str, Any]) -> Dict[str, Any]: + """Apply a string transform to a variable (in place or into ``into``).""" + name = args["name"] + value = str(executor.variables.get_value(name, "")) + result = _transform_string(value, str(args.get("op", "strip")), args) + target = args.get("into", name) + executor.variables.set(target, result) + return {"var": target, "value": result} + + +def exec_ocr_to_var(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Read OCR text from a screen region into a flow variable. + + Binds the recognised text under ``var`` (default ``ocr_text``) so later + steps can read it as ``${var}`` — the bridge between OCR and the + variable scope for data-driven flows. + """ + from je_auto_control.utils.ocr.ocr_engine import read_text_in_region + region = args.get("region") + if isinstance(region, str): + region = json.loads(region) if region.strip() else None + matches = read_text_in_region( + region=region, lang=args.get("lang", "eng"), + min_confidence=float(args.get("min_confidence", 60.0)), + ) + text = " ".join(match.text for match in matches).strip() + var_name = args.get("var", "ocr_text") + executor.variables.set(var_name, text) + return {"var": var_name, "text": text} + + +def exec_assert_duration(executor: Any, args: Mapping[str, Any]) -> Dict[str, Any]: + """Assert ``body`` completes within ``max_ms`` (a performance budget).""" + from je_auto_control.utils.assertion import assert_duration + body = args.get("body") or [] + return assert_duration( + lambda: executor.execute_action(body, _validated=True), + max_ms=float(args.get("max_ms", 1000.0)), + min_ms=float(args.get("min_ms", 0.0)), + raise_on_fail=bool(args.get("raise_on_fail", True)), + ).to_dict() diff --git a/je_auto_control/utils/mcp_server/_client_requests.py b/je_auto_control/utils/mcp_server/_client_requests.py new file mode 100644 index 00000000..5d2e0119 --- /dev/null +++ b/je_auto_control/utils/mcp_server/_client_requests.py @@ -0,0 +1,217 @@ +"""Server-initiated requests to the MCP client. + +MCP is bidirectional: besides answering the client, the server asks *it* +for things — ``roots/list`` to learn the workspace, ``elicitation/create`` +to put a question to the user, ``sampling/createMessage`` to borrow the +model. Each one writes a request out and blocks on a slot until the reply +arrives on the inbound side, so the correlation table and the response +router belong together with the senders that populate it. + +Destructive-tool confirmation lives here too: it is an elicitation +round-trip, not a tool-execution step. +""" +import json +import threading +from typing import Any, Dict, List, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server._protocol import ( + _confirm_destructive_enabled, _file_uri_to_path, _MCPError, +) +from je_auto_control.utils.mcp_server.tools import MCPTool + + +class ClientRequestMixin: + """Outbound half of the MCP session, mixed into :class:`MCPServer`. + + Requires the host to provide ``_writer``, ``_client_capabilities``, + ``_resources``, ``_outbound_lock``, ``_pending_outbound``, + ``_outbound_id_counter`` and ``_sampling_id_counter``. + """ + + @staticmethod + def _is_outbound_response(method: Optional[str], msg_id: Any, + message: Dict[str, Any]) -> bool: + """True when ``message`` is a reply to a server-initiated request.""" + return ( + method is None + and msg_id is not None + and ("result" in message or "error" in message) + ) + + def _dispatch_outbound_response(self, msg_id: Any, + message: Dict[str, Any]) -> None: + """Route a JSON-RPC response to the matching pending request.""" + with self._outbound_lock: + slot = self._pending_outbound.get(msg_id) + if slot is None: + autocontrol_logger.debug( + "MCP outbound response for unknown id %r", msg_id, + ) + return + if "error" in message: + slot["error"] = message["error"] + else: + slot["result"] = message.get("result") + slot["event"].set() + + def _maybe_request_roots_async(self) -> None: + """Fire a roots/list request when the client supports it.""" + if "roots" not in self._client_capabilities: + return + if self._writer is None: + return + threading.Thread( + target=self._refresh_roots_safely, daemon=True, + name="MCPRootsRefresh", + ).start() + + def _refresh_roots_safely(self) -> None: + try: + self.refresh_roots(timeout=5.0) + except (RuntimeError, TimeoutError) as error: + autocontrol_logger.info("MCP roots refresh skipped: %r", error) + + def refresh_roots(self, timeout: float = 10.0) -> List[Dict[str, Any]]: + """Send ``roots/list`` to the client and apply the first root.""" + result = self._send_outbound_request( + "roots/list", params={}, timeout=timeout, + ) + roots_list = (result or {}).get("roots") or [] + if not isinstance(roots_list, list) or not roots_list: + return [] + first_uri = roots_list[0].get("uri") if isinstance(roots_list[0], + dict) else None + if isinstance(first_uri, str): + local_path = _file_uri_to_path(first_uri) + if local_path: + self._resources.set_workspace_root(local_path) + autocontrol_logger.info("MCP workspace root → %s", local_path) + return roots_list + + def _send_outbound_request(self, method: str, + params: Dict[str, Any], + timeout: float = 10.0) -> Dict[str, Any]: + """Send a server-initiated request and wait for the response.""" + writer = self._writer + if writer is None: + raise RuntimeError(f"{method} requires an outbound writer") + request_id = f"srv-{next(self._outbound_id_counter)}" + slot = {"event": threading.Event()} + with self._outbound_lock: + self._pending_outbound[request_id] = slot + envelope = json.dumps({ + "jsonrpc": "2.0", "id": request_id, + "method": method, "params": params, + }, ensure_ascii=False, default=str) + try: + writer(envelope) + if not slot["event"].wait(timeout=timeout): + raise TimeoutError(f"{method} timed out after {timeout}s") + finally: + with self._outbound_lock: + self._pending_outbound.pop(request_id, None) + if "error" in slot: + raise RuntimeError(f"{method} failed: {slot['error']}") + return slot.get("result") or {} + + def request_elicitation(self, message: str, + requested_schema: Optional[Dict[str, Any]] = None, + timeout: float = 60.0) -> Dict[str, Any]: + """Ask the connected client to elicit a response from the user. + + Returns the raw payload (typically ``{"action": "accept" | "decline" | "cancel", ...}``). + Requires the client to advertise the ``elicitation`` capability. + """ + params: Dict[str, Any] = {"message": str(message)} + if requested_schema is not None: + params["requestedSchema"] = requested_schema + return self._send_outbound_request( + "elicitation/create", params=params, timeout=timeout, + ) + + def request_sampling(self, messages: List[Dict[str, Any]], + system_prompt: Optional[str] = None, + max_tokens: int = 1024, + model_preferences: Optional[Dict[str, Any]] = None, + timeout: float = 120.0) -> Dict[str, Any]: + """Ask the connected client to run an LLM sampling request. + + Tools that need the model's help (e.g. an OCR fallback that + wants the model to identify a UI element from a screenshot) + can call this and receive the assistant's reply. Requires the + server to be running in concurrent mode with an outbound + writer set — typically meaning ``serve_stdio`` or the HTTP + SSE transport. + """ + writer = self._writer + if writer is None: + raise RuntimeError( + "request_sampling requires an outbound writer; " + "start serve_stdio or call set_writer() first", + ) + request_id = f"sampling-{next(self._sampling_id_counter)}" + params: Dict[str, Any] = { + "messages": list(messages), + "maxTokens": int(max_tokens), + } + if system_prompt is not None: + params["systemPrompt"] = str(system_prompt) + if model_preferences is not None: + params["modelPreferences"] = dict(model_preferences) + slot = {"event": threading.Event()} + with self._outbound_lock: + self._pending_outbound[request_id] = slot + envelope = json.dumps({ + "jsonrpc": "2.0", "id": request_id, + "method": "sampling/createMessage", "params": params, + }, ensure_ascii=False, default=str) + try: + writer(envelope) + if not slot["event"].wait(timeout=timeout): + raise TimeoutError( + f"sampling request {request_id} timed out after {timeout}s" + ) + finally: + with self._outbound_lock: + self._pending_outbound.pop(request_id, None) + if "error" in slot: + raise RuntimeError(f"sampling failed: {slot['error']}") + return slot.get("result") or {} + + def _maybe_confirm_destructive(self, name: str, tool: MCPTool, + arguments: Dict[str, Any]) -> None: + """Ask the client to confirm before running a destructive tool.""" + if not _confirm_destructive_enabled(): + return + annotations = tool.annotations + if annotations.read_only or not annotations.destructive: + return + if "elicitation" not in self._client_capabilities: + autocontrol_logger.info( + "MCP confirmation requested for %s but client lacks " + "elicitation capability — proceeding without prompt", name, + ) + return + if self._writer is None: + return + prompt = (f"AutoControl is about to run a destructive tool " + f"'{name}'. Continue?") + try: + response = self.request_elicitation( + message=prompt, requested_schema={"type": "object", + "properties": {}}, + timeout=60.0, + ) + except (RuntimeError, TimeoutError) as error: + autocontrol_logger.info( + "MCP elicitation for %s failed (%r) — refusing call", + name, error, + ) + raise _MCPError( + -32000, f"User confirmation unavailable for {name}", + ) from error + action = response.get("action") if isinstance(response, dict) else None + if action != "accept": + raise _MCPError(-32000, f"User declined to run {name}: action={action!r}") + del arguments # available for future per-arg confirmation policies diff --git a/je_auto_control/utils/mcp_server/_protocol.py b/je_auto_control/utils/mcp_server/_protocol.py new file mode 100644 index 00000000..f5e1502e --- /dev/null +++ b/je_auto_control/utils/mcp_server/_protocol.py @@ -0,0 +1,165 @@ +"""JSON-RPC 2.0 wire format for the MCP server. + +Everything here is about the protocol rather than the server: the version +and identity constants, the error classes and the error tuples that decide +what a failing tool is allowed to do, the envelope builders, and the pure +functions that normalise a tool's return value into MCP ``content`` blocks. + +None of it touches server state, so it is importable and testable without +starting a server. +""" +import json +import os +import sqlite3 +import subprocess # nosec B404 # reason: only its TimeoutExpired type is referenced +import sys +import time +from typing import Any, Dict, List, Optional + +from je_auto_control.utils.exception.exceptions import AutoControlException +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.mcp_server.tools import MCPContent + + +PROTOCOL_VERSION = "2025-06-18" +SERVER_NAME = "je_auto_control" +SERVER_VERSION = "0.1.0" +_TOOLS_CALL_METHOD = "tools/call" + +# Framework and external-library errors a tool handler may raise. They all +# subclass ``Exception`` directly (not OSError/RuntimeError/…), so without +# listing them here a failing tool would escape both containment layers — the +# stdio worker thread dies and the client waits forever, or the HTTP +# connection aborts with no JSON-RPC reply. ``AutoControlException`` is the +# family base every ``AutoControl*Exception``/``ImageNotFoundException`` now +# derives from. +_FRAMEWORK_TOOL_ERRORS = ( + AutoControlException, subprocess.TimeoutExpired, sqlite3.Error, +) +_BUILTIN_DISPATCH_ERRORS = ( + OSError, RuntimeError, ValueError, TypeError, KeyError, +) +_DISPATCH_ERRORS = _BUILTIN_DISPATCH_ERRORS + _FRAMEWORK_TOOL_ERRORS +_TOOL_INVOKE_ERRORS = ( + _BUILTIN_DISPATCH_ERRORS + (AttributeError,) + _FRAMEWORK_TOOL_ERRORS +) + + +class _MCPError(Exception): + """Raised inside the dispatcher to surface a JSON-RPC error response.""" + + def __init__(self, code: int, message: str) -> None: + super().__init__(message) + self.code = code + self.message = message + + +def _to_content_blocks(result: Any) -> List[Dict[str, Any]]: + """Normalise a tool's return value into MCP ``content`` blocks.""" + if isinstance(result, MCPContent): + return [result.to_dict()] + if isinstance(result, list) and result and \ + all(isinstance(item, MCPContent) for item in result): + return [item.to_dict() for item in result] + return [{"type": "text", "text": _stringify_result(result)}] + + +def _stringify_result(value: Any) -> str: + """Convert a tool return value into a model-readable string.""" + if isinstance(value, str): + return value + try: + return json.dumps(value, ensure_ascii=False, default=str) + except (TypeError, ValueError): + return repr(value) + + +def _confirm_destructive_enabled() -> bool: + """Return True when the operator wants destructive tools gated on user OK.""" + raw = os.environ.get("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "") + return raw.strip().lower() in {"1", "true", "yes", "on"} + + +def _capture_error_screenshot(tool_name: str) -> Optional[str]: + """Save a debug screenshot when JE_AUTOCONTROL_MCP_ERROR_SHOTS is set.""" + debug_dir = os.environ.get("JE_AUTOCONTROL_MCP_ERROR_SHOTS") + if not debug_dir: + return None + target_dir = os.path.realpath(os.fspath(debug_dir)) + try: + os.makedirs(target_dir, exist_ok=True) + except OSError as error: + autocontrol_logger.info( + "MCP error-screenshot dir unavailable: %r", error, + ) + return None + filename = f"{tool_name}_{int(time.time() * 1000)}.png" + path = os.path.join(target_dir, filename) + try: + from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot + pil_screenshot(file_path=path) + except (OSError, RuntimeError, ValueError, AttributeError, + ImportError) as error: + autocontrol_logger.info( + "MCP failed to capture error screenshot: %r", error, + ) + return None + return path + + +def _file_uri_to_path(uri: str) -> Optional[str]: + """Convert a ``file://`` URI to a local filesystem path; ``None`` otherwise.""" + if not isinstance(uri, str) or not uri.startswith("file://"): + return None + from urllib.parse import unquote, urlparse + parsed = urlparse(uri) + raw_path = unquote(parsed.path) + # Windows: file:///C:/foo strips the leading slash before the drive letter. + if sys.platform.startswith("win") and raw_path.startswith("/") and \ + len(raw_path) > 2 and raw_path[2] == ":": + raw_path = raw_path[1:] + return raw_path or None + + +def _is_hashable(value: Any) -> bool: + """Return True when ``value`` can be used as a dict key.""" + try: + hash(value) + except TypeError: + return False + return True + + +def _coerce_params(raw: Any, msg_id: Any) -> tuple: + """Normalise JSON-RPC ``params`` to a dict. + + Returns ``(params, error_line)``. Every handler here expects an object; + a non-object ``params`` yields a ``-32602`` error line for a request and + an empty dict for a notification (which cannot carry an error reply). + """ + if raw is None: + return {}, None + if isinstance(raw, dict): + return raw, None + if msg_id is None: + return {}, None + return {}, _error_response(msg_id, -32602, "Invalid params: expected an object") + + +def _notification_message(method: str, params: Dict[str, Any]) -> str: + return json.dumps({"jsonrpc": "2.0", "method": method, "params": params}, + ensure_ascii=False, default=str) + + +def _result_response(msg_id: Any, result: Any) -> str: + return json.dumps( + {"jsonrpc": "2.0", "id": msg_id, "result": result}, + ensure_ascii=False, default=str, + ) + + +def _error_response(msg_id: Any, code: int, message: str) -> str: + return json.dumps({ + "jsonrpc": "2.0", "id": msg_id, + "error": {"code": code, "message": message}, + }, ensure_ascii=False) diff --git a/je_auto_control/utils/mcp_server/http_transport.py b/je_auto_control/utils/mcp_server/http_transport.py index 8db697b3..46c667fd 100644 --- a/je_auto_control/utils/mcp_server/http_transport.py +++ b/je_auto_control/utils/mcp_server/http_transport.py @@ -20,9 +20,10 @@ from je_auto_control.utils.http_headers import parse_content_length from je_auto_control.utils.logging.logging_instance import autocontrol_logger -from je_auto_control.utils.mcp_server.server import ( - MCPServer, _notification_message, +from je_auto_control.utils.mcp_server._protocol import ( + _notification_message, ) +from je_auto_control.utils.mcp_server.server import MCPServer DEFAULT_PATH = "/mcp" _MAX_BODY = 1_000_000 diff --git a/je_auto_control/utils/mcp_server/server.py b/je_auto_control/utils/mcp_server/server.py index d3021eb1..f85c4122 100644 --- a/je_auto_control/utils/mcp_server/server.py +++ b/je_auto_control/utils/mcp_server/server.py @@ -9,15 +9,11 @@ import contextlib import itertools import json -import os -import sqlite3 -import subprocess # nosec B404 # reason: only its TimeoutExpired type is referenced import sys import threading import time from typing import Any, Callable, Dict, List, Optional, TextIO -from je_auto_control.utils.exception.exceptions import AutoControlException from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.mcp_server.audit import AuditLogger from je_auto_control.utils.mcp_server.context import ( @@ -34,46 +30,23 @@ ResourceProvider, default_resource_provider, ) from je_auto_control.utils.mcp_server.tools import ( - MCPContent, MCPTool, build_default_tool_registry, + MCPTool, build_default_tool_registry, ) from je_auto_control.utils.mcp_server.tools._validation import ( validate_arguments, ) - -PROTOCOL_VERSION = "2025-06-18" -SERVER_NAME = "je_auto_control" -SERVER_VERSION = "0.1.0" -_TOOLS_CALL_METHOD = "tools/call" - -# Framework and external-library errors a tool handler may raise. They all -# subclass ``Exception`` directly (not OSError/RuntimeError/…), so without -# listing them here a failing tool would escape both containment layers — the -# stdio worker thread dies and the client waits forever, or the HTTP -# connection aborts with no JSON-RPC reply. ``AutoControlException`` is the -# family base every ``AutoControl*Exception``/``ImageNotFoundException`` now -# derives from. -_FRAMEWORK_TOOL_ERRORS = ( - AutoControlException, subprocess.TimeoutExpired, sqlite3.Error, -) -_BUILTIN_DISPATCH_ERRORS = ( - OSError, RuntimeError, ValueError, TypeError, KeyError, +from je_auto_control.utils.mcp_server._client_requests import ( + ClientRequestMixin, ) -_DISPATCH_ERRORS = _BUILTIN_DISPATCH_ERRORS + _FRAMEWORK_TOOL_ERRORS -_TOOL_INVOKE_ERRORS = ( - _BUILTIN_DISPATCH_ERRORS + (AttributeError,) + _FRAMEWORK_TOOL_ERRORS +from je_auto_control.utils.mcp_server._protocol import ( + PROTOCOL_VERSION, SERVER_NAME, SERVER_VERSION, _capture_error_screenshot, + _coerce_params, _DISPATCH_ERRORS, _error_response, _is_hashable, + _MCPError, _notification_message, _result_response, _to_content_blocks, + _TOOL_INVOKE_ERRORS, _TOOLS_CALL_METHOD, ) -class _MCPError(Exception): - """Raised inside the dispatcher to surface a JSON-RPC error response.""" - - def __init__(self, code: int, message: str) -> None: - super().__init__(message) - self.code = code - self.message = message - - -class MCPServer: +class MCPServer(ClientRequestMixin): """JSON-RPC 2.0 MCP server with a configurable tool registry.""" def __init__(self, tools: Optional[List[MCPTool]] = None, @@ -390,32 +363,6 @@ def _handle_line_safely(self, line: str) -> Optional[str]: autocontrol_logger.exception("MCP handle_line failed; line skipped") return None - @staticmethod - def _is_outbound_response(method: Optional[str], msg_id: Any, - message: Dict[str, Any]) -> bool: - """True when ``message`` is a reply to a server-initiated request.""" - return ( - method is None - and msg_id is not None - and ("result" in message or "error" in message) - ) - - def _dispatch_outbound_response(self, msg_id: Any, - message: Dict[str, Any]) -> None: - """Route a JSON-RPC response to the matching pending request.""" - with self._outbound_lock: - slot = self._pending_outbound.get(msg_id) - if slot is None: - autocontrol_logger.debug( - "MCP outbound response for unknown id %r", msg_id, - ) - return - if "error" in message: - slot["error"] = message["error"] - else: - slot["result"] = message.get("result") - slot["event"].set() - def _dispatch_tools_call_async(self, msg_id: Any, params: Dict[str, Any]) -> None: """Run a tools/call on a worker thread; the worker writes the reply.""" @@ -463,66 +410,6 @@ def _handle_notification(self, method: Optional[str], return autocontrol_logger.debug("MCP notification ignored: %s", method) - def _maybe_request_roots_async(self) -> None: - """Fire a roots/list request when the client supports it.""" - if "roots" not in self._client_capabilities: - return - if self._writer is None: - return - threading.Thread( - target=self._refresh_roots_safely, daemon=True, - name="MCPRootsRefresh", - ).start() - - def _refresh_roots_safely(self) -> None: - try: - self.refresh_roots(timeout=5.0) - except (RuntimeError, TimeoutError) as error: - autocontrol_logger.info("MCP roots refresh skipped: %r", error) - - def refresh_roots(self, timeout: float = 10.0) -> List[Dict[str, Any]]: - """Send ``roots/list`` to the client and apply the first root.""" - result = self._send_outbound_request( - "roots/list", params={}, timeout=timeout, - ) - roots_list = (result or {}).get("roots") or [] - if not isinstance(roots_list, list) or not roots_list: - return [] - first_uri = roots_list[0].get("uri") if isinstance(roots_list[0], - dict) else None - if isinstance(first_uri, str): - local_path = _file_uri_to_path(first_uri) - if local_path: - self._resources.set_workspace_root(local_path) - autocontrol_logger.info("MCP workspace root → %s", local_path) - return roots_list - - def _send_outbound_request(self, method: str, - params: Dict[str, Any], - timeout: float = 10.0) -> Dict[str, Any]: - """Send a server-initiated request and wait for the response.""" - writer = self._writer - if writer is None: - raise RuntimeError(f"{method} requires an outbound writer") - request_id = f"srv-{next(self._outbound_id_counter)}" - slot = {"event": threading.Event()} - with self._outbound_lock: - self._pending_outbound[request_id] = slot - envelope = json.dumps({ - "jsonrpc": "2.0", "id": request_id, - "method": method, "params": params, - }, ensure_ascii=False, default=str) - try: - writer(envelope) - if not slot["event"].wait(timeout=timeout): - raise TimeoutError(f"{method} timed out after {timeout}s") - finally: - with self._outbound_lock: - self._pending_outbound.pop(request_id, None) - if "error" in slot: - raise RuntimeError(f"{method} failed: {slot['error']}") - return slot.get("result") or {} - def _cancel_active_call(self, params: Dict[str, Any]) -> None: """Mark the matching active tool call as cancelled, if any.""" request_id = params.get("requestId") @@ -764,107 +651,6 @@ def _handle_tools_call(self, msg_id: Any, response["structuredContent"] = result return response - def request_elicitation(self, message: str, - requested_schema: Optional[Dict[str, Any]] = None, - timeout: float = 60.0) -> Dict[str, Any]: - """Ask the connected client to elicit a response from the user. - - Returns the raw payload (typically ``{"action": "accept" | "decline" | "cancel", ...}``). - Requires the client to advertise the ``elicitation`` capability. - """ - params: Dict[str, Any] = {"message": str(message)} - if requested_schema is not None: - params["requestedSchema"] = requested_schema - return self._send_outbound_request( - "elicitation/create", params=params, timeout=timeout, - ) - - def request_sampling(self, messages: List[Dict[str, Any]], - system_prompt: Optional[str] = None, - max_tokens: int = 1024, - model_preferences: Optional[Dict[str, Any]] = None, - timeout: float = 120.0) -> Dict[str, Any]: - """Ask the connected client to run an LLM sampling request. - - Tools that need the model's help (e.g. an OCR fallback that - wants the model to identify a UI element from a screenshot) - can call this and receive the assistant's reply. Requires the - server to be running in concurrent mode with an outbound - writer set — typically meaning ``serve_stdio`` or the HTTP - SSE transport. - """ - writer = self._writer - if writer is None: - raise RuntimeError( - "request_sampling requires an outbound writer; " - "start serve_stdio or call set_writer() first", - ) - request_id = f"sampling-{next(self._sampling_id_counter)}" - params: Dict[str, Any] = { - "messages": list(messages), - "maxTokens": int(max_tokens), - } - if system_prompt is not None: - params["systemPrompt"] = str(system_prompt) - if model_preferences is not None: - params["modelPreferences"] = dict(model_preferences) - slot = {"event": threading.Event()} - with self._outbound_lock: - self._pending_outbound[request_id] = slot - envelope = json.dumps({ - "jsonrpc": "2.0", "id": request_id, - "method": "sampling/createMessage", "params": params, - }, ensure_ascii=False, default=str) - try: - writer(envelope) - if not slot["event"].wait(timeout=timeout): - raise TimeoutError( - f"sampling request {request_id} timed out after {timeout}s" - ) - finally: - with self._outbound_lock: - self._pending_outbound.pop(request_id, None) - if "error" in slot: - raise RuntimeError(f"sampling failed: {slot['error']}") - return slot.get("result") or {} - - def _maybe_confirm_destructive(self, name: str, tool: MCPTool, - arguments: Dict[str, Any]) -> None: - """Ask the client to confirm before running a destructive tool.""" - if not _confirm_destructive_enabled(): - return - annotations = tool.annotations - if annotations.read_only or not annotations.destructive: - return - if "elicitation" not in self._client_capabilities: - autocontrol_logger.info( - "MCP confirmation requested for %s but client lacks " - "elicitation capability — proceeding without prompt", name, - ) - return - if self._writer is None: - return - prompt = (f"AutoControl is about to run a destructive tool " - f"'{name}'. Continue?") - try: - response = self.request_elicitation( - message=prompt, requested_schema={"type": "object", - "properties": {}}, - timeout=60.0, - ) - except (RuntimeError, TimeoutError) as error: - autocontrol_logger.info( - "MCP elicitation for %s failed (%r) — refusing call", - name, error, - ) - raise _MCPError( - -32000, f"User confirmation unavailable for {name}", - ) from error - action = response.get("action") if isinstance(response, dict) else None - if action != "accept": - raise _MCPError(-32000, f"User declined to run {name}: action={action!r}") - del arguments # available for future per-arg confirmation policies - def _build_call_context(self, msg_id: Any, params: Dict[str, Any]) -> ToolCallContext: meta = params.get("_meta") if isinstance(params.get("_meta"), @@ -876,117 +662,6 @@ def _build_call_context(self, msg_id: Any, ) -def _to_content_blocks(result: Any) -> List[Dict[str, Any]]: - """Normalise a tool's return value into MCP ``content`` blocks.""" - if isinstance(result, MCPContent): - return [result.to_dict()] - if isinstance(result, list) and result and \ - all(isinstance(item, MCPContent) for item in result): - return [item.to_dict() for item in result] - return [{"type": "text", "text": _stringify_result(result)}] - - -def _stringify_result(value: Any) -> str: - """Convert a tool return value into a model-readable string.""" - if isinstance(value, str): - return value - try: - return json.dumps(value, ensure_ascii=False, default=str) - except (TypeError, ValueError): - return repr(value) - - -def _confirm_destructive_enabled() -> bool: - """Return True when the operator wants destructive tools gated on user OK.""" - raw = os.environ.get("JE_AUTOCONTROL_MCP_CONFIRM_DESTRUCTIVE", "") - return raw.strip().lower() in {"1", "true", "yes", "on"} - - -def _capture_error_screenshot(tool_name: str) -> Optional[str]: - """Save a debug screenshot when JE_AUTOCONTROL_MCP_ERROR_SHOTS is set.""" - debug_dir = os.environ.get("JE_AUTOCONTROL_MCP_ERROR_SHOTS") - if not debug_dir: - return None - target_dir = os.path.realpath(os.fspath(debug_dir)) - try: - os.makedirs(target_dir, exist_ok=True) - except OSError as error: - autocontrol_logger.info( - "MCP error-screenshot dir unavailable: %r", error, - ) - return None - filename = f"{tool_name}_{int(time.time() * 1000)}.png" - path = os.path.join(target_dir, filename) - try: - from je_auto_control.utils.cv2_utils.screenshot import pil_screenshot - pil_screenshot(file_path=path) - except (OSError, RuntimeError, ValueError, AttributeError, - ImportError) as error: - autocontrol_logger.info( - "MCP failed to capture error screenshot: %r", error, - ) - return None - return path - - -def _file_uri_to_path(uri: str) -> Optional[str]: - """Convert a ``file://`` URI to a local filesystem path; ``None`` otherwise.""" - if not isinstance(uri, str) or not uri.startswith("file://"): - return None - from urllib.parse import unquote, urlparse - parsed = urlparse(uri) - raw_path = unquote(parsed.path) - # Windows: file:///C:/foo strips the leading slash before the drive letter. - if sys.platform.startswith("win") and raw_path.startswith("/") and \ - len(raw_path) > 2 and raw_path[2] == ":": - raw_path = raw_path[1:] - return raw_path or None - - -def _is_hashable(value: Any) -> bool: - """Return True when ``value`` can be used as a dict key.""" - try: - hash(value) - except TypeError: - return False - return True - - -def _coerce_params(raw: Any, msg_id: Any) -> tuple: - """Normalise JSON-RPC ``params`` to a dict. - - Returns ``(params, error_line)``. Every handler here expects an object; - a non-object ``params`` yields a ``-32602`` error line for a request and - an empty dict for a notification (which cannot carry an error reply). - """ - if raw is None: - return {}, None - if isinstance(raw, dict): - return raw, None - if msg_id is None: - return {}, None - return {}, _error_response(msg_id, -32602, "Invalid params: expected an object") - - -def _notification_message(method: str, params: Dict[str, Any]) -> str: - return json.dumps({"jsonrpc": "2.0", "method": method, "params": params}, - ensure_ascii=False, default=str) - - -def _result_response(msg_id: Any, result: Any) -> str: - return json.dumps( - {"jsonrpc": "2.0", "id": msg_id, "result": result}, - ensure_ascii=False, default=str, - ) - - -def _error_response(msg_id: Any, code: int, message: str) -> str: - return json.dumps({ - "jsonrpc": "2.0", "id": msg_id, - "error": {"code": code, "message": message}, - }, ensure_ascii=False) - - def start_mcp_stdio_server() -> MCPServer: """Start a stdio MCP server in the foreground; blocks until EOF.""" server = MCPServer() diff --git a/je_auto_control/utils/mcp_server/tools/_factories.py b/je_auto_control/utils/mcp_server/tools/_factories.py index 9105f38a..17960fb9 100644 --- a/je_auto_control/utils/mcp_server/tools/_factories.py +++ b/je_auto_control/utils/mcp_server/tools/_factories.py @@ -320,6 +320,76 @@ def window_tools() -> List[MCPTool]: handler=h.window_rect, annotations=READ_ONLY, ), + MCPTool( + name="ac_foreground_window_pid", + description=("PID of the process owning the foreground window as " + "{pid}; 0 when unavailable. Titles are not identity — " + "use this to tell which program is in front."), + input_schema=schema({}), + handler=h.foreground_window_pid, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_windows_for_pid", + description=("Visible top-level windows owned by a process, as " + "{windows: [{hwnd, title}]}. Titles cannot address a " + "multi-process application; ownership can."), + input_schema=schema({ + "pid": {"type": "integer"}, + "titled_only": {"type": "boolean"}, + }, required=["pid"]), + handler=h.windows_for_pid, + annotations=READ_ONLY, + ), + MCPTool( + name="ac_minimize_windows_for_pid", + description=("Minimise every window a process owns, as " + "{minimized}."), + input_schema=schema({"pid": {"type": "integer"}}, required=["pid"]), + handler=h.minimize_windows_for_pid, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_post_key_to_window", + description=("Post one key to a window without focusing it, as " + "{posted}. Goes to the control that has keyboard " + "focus. Best effort: applications reading raw input " + "or checking the foreground ignore posted messages."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "key": {"type": "string"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring", "key"]), + handler=h.post_key_to_window, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_post_click_to_window", + description=("Post one click into a window without focusing it, " + "as {posted}. x / y are relative to the window's " + "top-left corner. Same best-effort caveat as " + "ac_post_key_to_window."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "button": {"type": "string"}, + "x": {"type": "integer"}, + "y": {"type": "integer"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring"]), + handler=h.post_click_to_window, + annotations=NON_DESTRUCTIVE, + ), + MCPTool( + name="ac_window_pid", + description=("PID of the process owning the first matching window " + "as {pid}; 0 when nothing matched."), + input_schema=schema({ + "title_substring": {"type": "string"}, + "case_sensitive": {"type": "boolean"}, + }, required=["title_substring"]), + handler=h.window_pid, + annotations=READ_ONLY, + ), MCPTool( name="ac_window_move", description=("Move and resize the first matching window to " diff --git a/je_auto_control/utils/mcp_server/tools/_handlers.py b/je_auto_control/utils/mcp_server/tools/_handlers.py index 76f2d695..d5a5d39b 100644 --- a/je_auto_control/utils/mcp_server/tools/_handlers.py +++ b/je_auto_control/utils/mcp_server/tools/_handlers.py @@ -145,8 +145,8 @@ def screenshot(file_path: Optional[str] = None, def list_monitors() -> List[Dict[str, Any]]: """Return every monitor's geometry. Index 0 spans all monitors.""" - import mss - with mss.mss() as sct: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as sct: return [ { "index": index, "left": int(monitor["left"]), @@ -160,10 +160,10 @@ def list_monitors() -> List[Dict[str, Any]]: def _grab_monitor(index: int): - """Capture a single monitor via ``mss`` and return a PIL Image.""" - import mss + """Capture a single monitor through the platform grabber; returns a PIL Image.""" + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber from PIL import Image - with mss.mss() as sct: + with mss_grabber() as sct: if index < 0 or index >= len(sct.monitors): raise ValueError( f"monitor index {index} out of range " @@ -451,6 +451,55 @@ def window_rect(title_substring: str, return {"rect": list(rect) if rect is not None else None} +def foreground_window_pid() -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + foreground_window_process_id as _pid, + ) + return {"pid": _pid() or 0} + + +def window_pid(title_substring: str, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + window_process_id as _pid, + ) + return {"pid": _pid(title_substring, case_sensitive=case_sensitive) or 0} + + +def windows_for_pid(pid: int, titled_only: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + windows_for_process_id as _windows, + ) + return {"windows": [{"hwnd": hwnd, "title": title} + for hwnd, title in _windows(int(pid), titled_only)]} + + +def minimize_windows_for_pid(pid: int) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + minimize_windows_for_process as _minimize, + ) + return {"minimized": _minimize(int(pid))} + + +def post_key_to_window(title_substring: str, key: str, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + post_key_to_window as _post, + ) + return {"posted": bool(_post(title_substring, key, + case_sensitive=case_sensitive))} + + +def post_click_to_window(title_substring: str, button: str = "left", + x: int = 0, y: int = 0, + case_sensitive: bool = False) -> Dict[str, Any]: + from je_auto_control.wrapper.auto_control_window import ( + post_click_to_window as _post, + ) + return {"posted": bool(_post(title_substring, button, int(x), int(y), + case_sensitive=case_sensitive))} + + def _resolve_window_hwnd(title_substring: str, case_sensitive: bool) -> int: from je_auto_control.wrapper.auto_control_window import find_window diff --git a/je_auto_control/utils/monitor_layout/logical_frame.py b/je_auto_control/utils/monitor_layout/logical_frame.py index 3fd3de7b..04da7a5e 100644 --- a/je_auto_control/utils/monitor_layout/logical_frame.py +++ b/je_auto_control/utils/monitor_layout/logical_frame.py @@ -19,6 +19,14 @@ desktop starts at negative coordinates whenever a monitor sits left of or above the primary one. +Wayland has the same requirement without the DPI half: its capture spans the +compositor's whole output layout, and that layout starts at a negative +coordinate whenever an output sits left of or above the origin. There is no +``GetSystemMetrics`` to ask, so the origin comes from the backend itself +(``screen_grabber.backend_layout_rect``) — without it a hit found in the frame +is reported 1920 px (or whatever the left-hand monitor is wide) to the right of +where it was matched. + The arithmetic (:func:`needs_rescale`, :func:`logical_scale`) is pure and unit-testable; the OS reader and the grabber are both injectable. Imports no ``PySide6``. @@ -72,10 +80,29 @@ def needs_rescale(physical: Tuple[int, int], logical: Tuple[int, int]) -> bool: return bool(logical[0] and logical[1]) and tuple(physical) != tuple(logical) +def _backend_frame_origin() -> Tuple[int, int]: + """Where a self-capturing backend's frame starts, ``(0, 0)`` by default. + + ``logical_virtual_rect`` reads ``GetSystemMetrics``, so off Windows it + has nothing to say — but the Wayland backend captures the compositor's + whole output layout, and that layout starts at a negative coordinate + whenever an output sits left of or above the origin. Treating the frame + as starting at ``(0, 0)`` there offsets every located hit by the origin, + which reads as "the click lands on the wrong monitor" rather than as a + failure to find. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import backend_layout_origin + return backend_layout_origin() + + def _load_image_grab(): - """Import Pillow's ``ImageGrab`` lazily (optional dependency at runtime).""" - from PIL import ImageGrab - return ImageGrab + """Load the platform's ``ImageGrab``-shaped grabber lazily. + + Pillow off Wayland, the compositor's capture tool on it — see + :mod:`je_auto_control.utils.cv2_utils.screen_grabber`. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber + return image_grabber() def _resample(): @@ -105,7 +132,7 @@ def grab_logical(region: Optional[Sequence[int]] = None, *, image = image_grab.grab(all_screens=True) rect = logical_virtual_rect(metrics) - origin_x, origin_y = (rect[0], rect[1]) if rect else (0, 0) + origin_x, origin_y = (rect[0], rect[1]) if rect else _backend_frame_origin() if rect and needs_rescale((image.width, image.height), (rect[2], rect[3])): image = image.resize((rect[2], rect[3]), _resample()) if region is None: diff --git a/je_auto_control/utils/monitor_layout/monitor_layout.py b/je_auto_control/utils/monitor_layout/monitor_layout.py index 2fb46f32..99ba0ed9 100644 --- a/je_auto_control/utils/monitor_layout/monitor_layout.py +++ b/je_auto_control/utils/monitor_layout/monitor_layout.py @@ -130,8 +130,8 @@ def remap_point(src: Monitor, dst: Monitor, local_x: int, def _mss_rows() -> List[Dict[str, Any]]: - import mss - with mss.mss() as screen: + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + with mss_grabber() as screen: monitors = screen.monitors return [{"x": int(m["left"]), "y": int(m["top"]), "width": int(m["width"]), "height": int(m["height"])} diff --git a/je_auto_control/utils/ocr/ocr_engine.py b/je_auto_control/utils/ocr/ocr_engine.py index 0d13b3c2..2b15cb06 100644 --- a/je_auto_control/utils/ocr/ocr_engine.py +++ b/je_auto_control/utils/ocr/ocr_engine.py @@ -27,17 +27,24 @@ def _load_image_grab(): + """Cache the platform's ``ImageGrab``-shaped grabber. + + Not Pillow directly: under Wayland the screen has to come from the + compositor's own capture tool, or OCR reads a blank XWayland root. + """ global _image_grab if _image_grab is not None: return _image_grab + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber try: - from PIL import ImageGrab + # image_grabber imports Pillow lazily, so a missing Pillow surfaces + # here rather than on the import above. + _image_grab = image_grabber() except ImportError as error: raise RuntimeError( "OCR requires Pillow for screen capture. Install with: pip install Pillow" ) from error - _image_grab = ImageGrab - return ImageGrab + return _image_grab @dataclass(frozen=True) diff --git a/je_auto_control/utils/remote_desktop/__init__.py b/je_auto_control/utils/remote_desktop/__init__.py index c6c45839..f700ef90 100644 --- a/je_auto_control/utils/remote_desktop/__init__.py +++ b/je_auto_control/utils/remote_desktop/__init__.py @@ -19,8 +19,11 @@ from je_auto_control.utils.remote_desktop.file_transfer import ( FileReceiver, FileSendResult, FileTransferError, send_file, ) -from je_auto_control.utils.remote_desktop.host import ( - PendingViewer, PendingViewerCallback, RemoteDesktopHost, +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import ( + PendingViewer, PendingViewerCallback, +) +from je_auto_control.utils.remote_desktop.host_capture import ( list_host_monitors, ) from je_auto_control.utils.remote_desktop.host_id import ( diff --git a/je_auto_control/utils/remote_desktop/host.py b/je_auto_control/utils/remote_desktop/host.py index e1ba0863..c9c055f2 100644 --- a/je_auto_control/utils/remote_desktop/host.py +++ b/je_auto_control/utils/remote_desktop/host.py @@ -1,23 +1,20 @@ """TCP host that streams JPEG frames and applies viewer input.""" -import collections import json import socket import ssl import threading import time -from dataclasses import dataclass -from io import BytesIO -from typing import Any, Callable, Deque, Dict, List, Mapping, Optional, Sequence +from typing import Any, Callable, List, Mapping, Optional, Sequence from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop.audio import ( AudioCapture, AudioCaptureConfig, ) from je_auto_control.utils.remote_desktop.auth import ( - make_nonce, verify_response, + verify_response, ) from je_auto_control.utils.remote_desktop.clipboard_sync import ( - ClipboardSyncError, decode as decode_clipboard, encode_image, encode_text, + encode_image, encode_text, ) from je_auto_control.utils.remote_desktop.file_transfer import ( FileReceiver, FileTransferError, send_file, @@ -26,87 +23,42 @@ load_or_create_host_id, validate_host_id, ) from je_auto_control.utils.remote_desktop.input_dispatch import ( - InputDispatchError, dispatch_input, + dispatch_input, ) from je_auto_control.utils.remote_desktop.protocol import ( - AuthenticationError, MessageType, ProtocolError, + MessageType, ) from je_auto_control.utils.remote_desktop.resume_tokens import ( ResumeTokenStore, ) from je_auto_control.utils.remote_desktop.video_codec import ( - CODEC_JPEG, CodecProvider, JpegPassthrough, codec_tag, + CodecProvider, JpegPassthrough, ) from je_auto_control.utils.remote_desktop.transport import ( MessageChannel, TcpMessageChannel, ) +from je_auto_control.utils.remote_desktop.host_access import ( + PendingViewerCallback, _AUTH_TIMEOUT_S, _candidate_totp_codes, + _compile_ip_allowlist, _ip_in_allowlist, +) +from je_auto_control.utils.remote_desktop.host_capture import ( + CursorProvider, FrameProductionMixin, FrameProvider, + _DEFAULT_QUALITY, _default_frame_provider, + _resolve_cursor_provider, _resolve_monitor_region, +) +from je_auto_control.utils.remote_desktop.host_client import ( + _ClientHandler, +) -FrameProvider = Callable[[], bytes] InputDispatcher = Callable[[Mapping[str, Any]], Any] -CursorProvider = Callable[[], Optional[Sequence[int]]] -"""Return ``(x, y)`` in host screen coordinates, or ``None`` to skip a tick.""" - -_CURSOR_POLL_INTERVAL_S = 1.0 / 30.0 # 30 Hz: smooth, low CPU on idle. - - -@dataclass(frozen=True) -class PendingViewer: - """Snapshot of an authenticated viewer awaiting host approval. - - Passed to the ``on_pending_viewer`` callback after the HMAC handshake - succeeds but before the host starts streaming frames. The callback's - return value is interpreted as: - - * ``True`` / ``"full"`` → admit with full control - * ``"view_only"`` → admit, but drop incoming INPUT messages - * ``False`` / ``None`` / etc. → reject - """ - address: tuple - host_id: str - transport: str = "tcp" - - -PendingViewerCallback = Callable[[PendingViewer], Any] -"""Callback signature: see :class:`PendingViewer` for return value semantics.""" - - -PERMISSION_FULL = "full" -PERMISSION_VIEW_ONLY = "view_only" -PERMISSION_DENIED = "denied" - - -def _interpret_approval(result: Any) -> str: - """Map an approval-callback return value to a permission string. - - Backward compatibility: any truthy value other than the literal - ``"view_only"`` / ``"denied"`` strings is treated as full-control - admit. Falsy values are denied. - """ - if result == PERMISSION_VIEW_ONLY: - return PERMISSION_VIEW_ONLY - if result == PERMISSION_DENIED or not result: - return PERMISSION_DENIED - return PERMISSION_FULL - - -_AUTH_TIMEOUT_S = 60.0 # accept() 的輪詢間隔，用來定期檢查 _shutdown 旗標。 # How long accept() blocks before re-checking the _shutdown flag. _ACCEPT_POLL_TIMEOUT_S = 0.5 -_DEFAULT_QUALITY = 70 _FILE_MSG_TYPES = frozenset({ MessageType.FILE_BEGIN, MessageType.FILE_CHUNK, MessageType.FILE_END, }) -def _candidate_totp_codes(secret: str): - """Yield TOTP codes within ±1 step of the current 30-second window.""" - from je_auto_control.utils.remote_desktop.totp import generate_code - now = time.time() - for delta in (-1, 0, 1): - yield generate_code(secret, at=now + (delta * 30.0)) - - def _validate_host_args(token: str, fps: float, quality: int) -> None: """Throw early on bad constructor args so the host never starts broken.""" if not isinstance(token, str) or not token: @@ -117,522 +69,7 @@ def _validate_host_args(token: str, fps: float, quality: int) -> None: raise ValueError("quality must be in [1, 95]") -def list_host_monitors() -> List[Dict[str, Any]]: - """Headless helper: return every monitor's geometry. - - Index 0 spans all monitors (the ``mss`` convention). Returns an - empty list if ``mss`` is not installed, so GUI callers can show a - disabled control instead of crashing. - """ - try: - import mss - except ImportError: - return [] - with mss.mss() as sct: - return [ - { - "index": index, "left": int(monitor["left"]), - "top": int(monitor["top"]), - "width": int(monitor["width"]), - "height": int(monitor["height"]), - "is_combined": index == 0, - } - for index, monitor in enumerate(sct.monitors) - ] - - -def _resolve_monitor_region( - monitor_index: int) -> Optional[Sequence[int]]: - """Map an ``mss`` monitor index to ``(x, y, width, height)``. - - Returns ``None`` (full-screen capture fallback) when ``mss`` is - not available so a stock install still works. - """ - try: - import mss - except ImportError: - autocontrol_logger.warning( - "remote_desktop monitor_index=%d ignored: mss not installed", - monitor_index, - ) - return None - with mss.mss() as sct: - if monitor_index < 0 or monitor_index >= len(sct.monitors): - raise ValueError( - f"monitor_index {monitor_index} out of range " - f"(0..{len(sct.monitors) - 1})" - ) - mon = sct.monitors[monitor_index] - return ( - int(mon["left"]), int(mon["top"]), - int(mon["width"]), int(mon["height"]), - ) - - -def _compile_ip_allowlist( - entries: Optional[Sequence[str]]) -> Optional[List[Any]]: - """Pre-parse ``entries`` into ``ip_address`` / ``ip_network`` objects. - - ``None`` or an empty list → no filtering (allow all). Entries are - plain IPs (``"192.168.1.10"``) or CIDR ranges (``"10.0.0.0/8"``); - unparseable entries are dropped with a warning so a typo doesn't - silently broaden access. - """ - if not entries: - return None - import ipaddress - compiled: List[Any] = [] - for entry in entries: - text = str(entry).strip() - if not text: - continue - try: - if "/" in text: - compiled.append(ipaddress.ip_network(text, strict=False)) - else: - compiled.append(ipaddress.ip_address(text)) - except ValueError: - autocontrol_logger.warning( - "remote_desktop ip_allowlist entry rejected: %r", text, - ) - return compiled or None - - -def _ip_in_allowlist(allowlist: Optional[List[Any]], peer_ip: str) -> bool: - """Return True when ``peer_ip`` matches any allowlist entry (or no list).""" - if not allowlist: - return True - import ipaddress - try: - addr = ipaddress.ip_address(peer_ip) - except ValueError: - return False - for entry in allowlist: - if isinstance(entry, (ipaddress.IPv4Network, ipaddress.IPv6Network)): - if addr in entry: - return True - elif entry == addr: - return True - return False - - -def _resolve_cursor_provider( - explicit: Optional[CursorProvider], - enabled: bool) -> Optional[CursorProvider]: - """Pick the cursor provider — explicit > default > disabled.""" - if explicit is not None: - return explicit - return _default_cursor_provider() if enabled else None - - -def _default_cursor_provider() -> CursorProvider: - """Build a cursor-position poller using the project's mouse wrapper. - - The wrapper is imported lazily inside the closure so importing this - module on platforms where mouse capture is unavailable does not - blow up. Returns ``None`` on read failures so the broadcast loop - silently skips the tick instead of crashing the host. - """ - def provide() -> Optional[Sequence[int]]: - try: - from je_auto_control.wrapper.auto_control_mouse import ( - get_mouse_position, - ) - except ImportError: - return None - try: - return get_mouse_position() - except (OSError, RuntimeError, AttributeError): - return None - return provide - - -def _default_frame_provider(region: Optional[Sequence[int]] = None, - quality: int = _DEFAULT_QUALITY) -> FrameProvider: - """Build a JPEG frame producer using PIL.ImageGrab.""" - def provide() -> bytes: - from PIL import ImageGrab # local import: not needed for unit tests - if region is not None: - x, y, width, height = (int(v) for v in region) - bbox = (x, y, x + width, y + height) - image = ImageGrab.grab(bbox=bbox, all_screens=True) - else: - image = ImageGrab.grab(all_screens=True) - if image.mode != "RGB": - image = image.convert("RGB") - buffer = BytesIO() - image.save(buffer, format="JPEG", quality=int(quality)) - return buffer.getvalue() - return provide - - -class _ClientHandler: - """Per-connection auth + input-receive + frame-send state.""" - - _AUDIO_QUEUE_MAXLEN = 50 # ~2.5 s of buffered chunks at 50 ms each - - def __init__(self, host: "RemoteDesktopHost", - channel: MessageChannel, address) -> None: - self._host = host - self._channel = channel - self._address = address - self._shutdown = threading.Event() - self._sender_thread: Optional[threading.Thread] = None - self._receiver_thread: Optional[threading.Thread] = None - self._audio_queue: Deque[bytes] = collections.deque( - maxlen=self._AUDIO_QUEUE_MAXLEN, - ) - self._audio_lock = threading.Lock() - self._audio_event = threading.Event() - self._audio_sender_thread: Optional[threading.Thread] = None - self.authenticated = False - # Phase 5.3: per-client permission set by the approval callback. - # Default is full control so legacy callers (no callback) keep - # the prior behaviour. - self.permission = PERMISSION_FULL - - @property - def address(self): - return self._address - - def start(self) -> None: - """Run auth (with optional host approval), then start the loops.""" - try: - self._authenticate() - except (AuthenticationError, ProtocolError, OSError) as error: - autocontrol_logger.info( - "remote_desktop client %s rejected: %r", self._address, error, - ) - self._close() - return - self.authenticated = True - # The initial cursor + frame are seeded from _send_loop (the - # per-client sender thread), not here. start() runs on the shared - # accept thread with the socket timeout already cleared, so sending a - # full-screen JPEG to a viewer that authenticates then stops reading - # would block every new accept until its send buffer drains. - self._sender_thread = threading.Thread( - target=self._send_loop, name="rd-sender", daemon=True, - ) - self._receiver_thread = threading.Thread( - target=self._recv_loop, name="rd-recv", daemon=True, - ) - self._sender_thread.start() - self._receiver_thread.start() - if self._host._audio_config.enabled: - self._audio_sender_thread = threading.Thread( - target=self._audio_send_loop, name="rd-audio", daemon=True, - ) - self._audio_sender_thread.start() - - def _send_initial_frame(self) -> None: - """Forward the most recent encoded frame so new clients aren't blank. - - Motion-dedup in :meth:`_capture_loop` means a static desktop - only bumps ``_latest_seq`` once; replaying that frame to the - new client keeps them from sitting on a black popup until the - host moves something. - """ - with self._host._frame_cond: - frame = self._host._latest_frame - if frame is None: - return - try: - self._channel.send_typed(MessageType.FRAME, frame) - except OSError: - pass - - def push_audio(self, chunk: bytes) -> None: - """Enqueue a PCM chunk for delivery; oldest dropped if queue is full.""" - if self._shutdown.is_set() or not self.authenticated: - return - with self._audio_lock: - self._audio_queue.append(chunk) - self._audio_event.set() - - def stop(self) -> None: - """Signal threads and close the socket.""" - self._shutdown.set() - with self._host._frame_cond: - self._host._frame_cond.notify_all() - self._audio_event.set() - self._close() - - def _resolve_permission(self) -> str: - """Run the host's optional approval callback after token auth. - - Returns one of :data:`PERMISSION_FULL` / :data:`PERMISSION_VIEW_ONLY` - (admit) or :data:`PERMISSION_DENIED` (reject). The caller is - expected to send ``AUTH_FAIL`` and raise on denial — keeping - that wire-level handling inside :meth:`_authenticate` so the - viewer sees the rejection before it has a chance to flip into - the post-handshake state where ``AUTH_FAIL`` is ignored. - """ - callback = self._host._on_pending_viewer - if callback is None: - return PERMISSION_FULL - pending = PendingViewer( - address=tuple(self._address) if self._address else (), - host_id=self._host.host_id, - transport=self._host._transport_name(), - ) - try: - return _interpret_approval(callback(pending)) - except (RuntimeError, ValueError, TypeError) as error: - autocontrol_logger.info( - "remote_desktop approval callback raised for %s: %r", - self._address, error, - ) - return PERMISSION_DENIED - - def _authenticate(self) -> None: - nonce = make_nonce() - self._channel.settimeout(_AUTH_TIMEOUT_S) - self._channel.send_typed(MessageType.AUTH_CHALLENGE, nonce) - msg_type, payload = self._channel.read_typed() - if msg_type is not MessageType.AUTH_RESPONSE: - self._channel.send_typed(MessageType.AUTH_FAIL, - b"expected AUTH_RESPONSE") - raise AuthenticationError( - f"expected AUTH_RESPONSE, got {msg_type.name}" - ) - # Phase 6.6: a viewer reconnecting with a valid resume token - # signs with that token directly — host short-circuits the - # approval popup and reuses the saved permission. - resumed = self._host._try_consume_resume(nonce, payload) - if resumed is not None: - self.permission = resumed - else: - if not self._host._verify_token(nonce, payload): - self._channel.send_typed(MessageType.AUTH_FAIL, b"bad token") - raise AuthenticationError("bad token") - # Host operator gates the session *before* AUTH_OK so the - # viewer surfaces the rejection as an AuthenticationError - # instead of connecting and then mysteriously disconnecting. - permission = self._resolve_permission() - if permission == PERMISSION_DENIED: - self._channel.send_typed( - MessageType.AUTH_FAIL, b"rejected by host", - ) - raise AuthenticationError("rejected by host") - self.permission = permission - # Issue a fresh resume token so the viewer can reconnect - # within the store's TTL without the approval popup. - resume_token = self._host._resume_store.issue(self.permission) - ok_payload = json.dumps( - {"host_id": self._host.host_id, - "resume_token": resume_token, - "resume_ttl": self._host._resume_store.ttl, - "codec": self._host._codec_provider.name}, - ensure_ascii=False, - ).encode("utf-8") - self._channel.send_typed(MessageType.AUTH_OK, ok_payload) - self._channel.settimeout(None) - - def _send_loop(self) -> None: - # Seed the viewer with the latest cursor + frame on this per-client - # sender thread (Phase 2.3: motion-aware capture only bumps the seq on - # change, so a viewer joining a static desktop would otherwise sit - # blank until the host moves). Doing it here rather than in start() - # keeps a slow-reading viewer from stalling the accept thread. - self._host._send_initial_cursor(self) - self._send_initial_frame() - last_sent = 0 - while not self._shutdown.is_set(): - with self._host._frame_cond: - while (not self._shutdown.is_set() - and self._host._latest_seq <= last_sent): - self._host._frame_cond.wait(timeout=0.5) - if self._shutdown.is_set(): - return - frame = self._host._latest_frame - seq = self._host._latest_seq - if frame is None: - continue - try: - self._channel.send_typed(MessageType.FRAME, frame) - except OSError as error: - autocontrol_logger.info( - "remote_desktop send to %s failed: %r", - self._address, error, - ) - self.stop() - return - last_sent = seq - - def _audio_send_loop(self) -> None: - while not self._shutdown.is_set(): - self._audio_event.wait(timeout=0.5) - if self._shutdown.is_set(): - return - while True: - with self._audio_lock: - if not self._audio_queue: - self._audio_event.clear() - break - chunk = self._audio_queue.popleft() - try: - self._channel.send_typed(MessageType.AUDIO, chunk) - except OSError as error: - autocontrol_logger.info( - "remote_desktop audio send to %s failed: %r", - self._address, error, - ) - self.stop() - return - - def _recv_loop(self) -> None: - while not self._shutdown.is_set(): - try: - msg_type, payload = self._channel.read_typed() - except (OSError, ProtocolError) as error: - if not self._shutdown.is_set(): - autocontrol_logger.info( - "remote_desktop recv from %s ended: %r", - self._address, error, - ) - self.stop() - return - self._route_incoming(msg_type, payload) - - def _route_incoming(self, msg_type: MessageType, payload: bytes) -> None: - """Dispatch one received message to the matching handler.""" - if msg_type is MessageType.PING: - return - if msg_type is MessageType.INPUT: - # Phase 5.3: drop input from view-only viewers so they can - # watch but cannot drive the mouse / keyboard. - if self.permission != PERMISSION_VIEW_ONLY: - self._handle_input_payload(payload) - return - if msg_type is MessageType.CLIPBOARD: - self._handle_clipboard_payload(payload) - return - if msg_type is MessageType.CHAT: - self._handle_chat_payload(payload) - return - if msg_type is MessageType.USB_LIST_REQUEST: - self._handle_usb_list_request() - return - if msg_type in _FILE_MSG_TYPES: - self._handle_file_payload(msg_type, payload) - return - autocontrol_logger.info( - "remote_desktop unexpected msg %s from %s", - msg_type.name, self._address, - ) - - def _handle_file_payload(self, msg_type: MessageType, - payload: bytes) -> None: - receiver = self._host._ensure_file_receiver() - try: - if msg_type is MessageType.FILE_BEGIN: - receiver.handle_begin(payload) - elif msg_type is MessageType.FILE_CHUNK: - receiver.handle_chunk(payload) - elif msg_type is MessageType.FILE_END: - receiver.handle_end(payload) - except FileTransferError as error: - autocontrol_logger.info( - "remote_desktop bad file message from %s: %r", - self._address, error, - ) - - def _handle_usb_list_request(self) -> None: - """Phase 6.9: enumerate the host's USB devices and ship the list back. - - Uses the existing :func:`list_usb_devices` helper so we get the - same cross-platform behaviour as the standalone USB module. - Errors fall back to an empty payload — the viewer should - treat that as "host has no usable USB backend" rather than - crashing. - """ - try: - from je_auto_control.utils.usb import list_usb_devices - result = list_usb_devices() - body = { - "backend": result.backend, - "devices": [d.to_dict() for d in result.devices], - } - except (ImportError, OSError, RuntimeError) as error: - autocontrol_logger.info( - "usb_list from %s failed: %r", self._address, error, - ) - body = {"backend": "unavailable", "devices": []} - try: - self._channel.send_typed( - MessageType.USB_LIST_RESPONSE, - json.dumps(body, ensure_ascii=False).encode("utf-8"), - ) - except OSError: - pass - - def _handle_chat_payload(self, payload: bytes) -> None: - """Forward viewer-originated chat to the host's optional callback.""" - callback = self._host._on_chat - if callback is None: - return - try: - body = json.loads(payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - return - if not isinstance(body, dict): - return - text = body.get("text") - sender = body.get("sender", "viewer") - if not isinstance(text, str) or not text: - return - try: - callback(str(sender), text) - except Exception: # noqa: BLE001 callback isolation - autocontrol_logger.exception( - "remote_desktop on_chat callback raised" - ) - - def _handle_clipboard_payload(self, payload: bytes) -> None: - try: - kind, data = decode_clipboard(payload) - except ClipboardSyncError as error: - autocontrol_logger.info( - "remote_desktop bad CLIPBOARD from %s: %r", - self._address, error, - ) - return - try: - self._host._apply_clipboard(kind, data) - except (OSError, RuntimeError, TypeError, ValueError) as error: - autocontrol_logger.warning( - "remote_desktop clipboard apply failed for %s: %r", - self._address, error, - ) - - def _handle_input_payload(self, payload: bytes) -> None: - try: - message = json.loads(payload.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError) as error: - autocontrol_logger.info( - "remote_desktop bad INPUT from %s: %r", - self._address, error, - ) - return - try: - self._host._dispatch(message) - except InputDispatchError as error: - autocontrol_logger.info( - "remote_desktop rejected INPUT from %s: %r", - self._address, error, - ) - except (OSError, RuntimeError, ValueError, TypeError) as error: - autocontrol_logger.warning( - "remote_desktop input apply failed for %s: %r", - self._address, error, - ) - - def _close(self) -> None: - self._channel.close() - - -class RemoteDesktopHost: +class RemoteDesktopHost(FrameProductionMixin): """Stream the screen to authenticated viewers and apply their input. The instance owns three kinds of threads: one accept loop, one @@ -1030,67 +467,6 @@ def _apply_clipboard(self, kind: str, data: Any) -> None: # internals ----------------------------------------------------------- - def _cursor_loop(self) -> None: - """Poll cursor position at ~30 Hz and push it to viewers as JSON.""" - provider = self._cursor_provider - if provider is None: - return - while not self._shutdown.is_set(): - position = provider() - if position is not None and len(position) >= 2: - payload = json.dumps( - {"x": int(position[0]), "y": int(position[1]), - "visible": True}, - ).encode("utf-8") - with self._cursor_lock: - is_new = payload != self._latest_cursor_payload - self._latest_cursor_payload = payload - if is_new: - self._broadcast_cursor(payload) - if self._shutdown.wait(timeout=_CURSOR_POLL_INTERVAL_S): - return - - def _broadcast_cursor(self, payload: bytes) -> None: - """Send a CURSOR message to every authenticated client. - - Errors per-client are swallowed — a flaky viewer should not - kill the cursor stream to healthy peers. - """ - with self._clients_lock: - clients = [c for c in self._clients - if c.authenticated and not c._shutdown.is_set()] - for client in clients: - try: - client._channel.send_typed(MessageType.CURSOR, payload) - except OSError: - continue - - def broadcast_viewer_cursor(self, viewer_id: str, - x: int, y: int) -> int: - """Phase 5.1: relay one viewer's cursor position to every other viewer. - - Typically called by :class:`MultiViewerHost` when several - viewers share a session so each viewer's overlay can show the - other operators' pointers (Figma / Google Docs style). The - viewer_id is opaque to the host — viewers use it to colour-key - their overlay. - """ - payload = json.dumps( - {"x": int(x), "y": int(y), "visible": True, - "viewer_id": str(viewer_id)}, - ).encode("utf-8") - with self._clients_lock: - clients = [c for c in self._clients - if c.authenticated and not c._shutdown.is_set()] - sent = 0 - for client in clients: - try: - client._channel.send_typed(MessageType.CURSOR, payload) - sent += 1 - except OSError: - continue - return sent - def broadcast_chat(self, text: str, sender: str = "host") -> int: """Phase 5.2: send a chat message to every connected viewer. @@ -1117,23 +493,6 @@ def broadcast_chat(self, text: str, sender: str = "host") -> int: ) return sent - def _send_initial_cursor(self, client: "_ClientHandler") -> None: - """Push the latest known cursor position to a fresh client. - - Sending unconditionally on auth means the viewer sees a cursor - immediately instead of waiting up to ~1 s for the next position - change. Safe to call when the cursor loop is disabled — we - only send if there's a payload to send. - """ - with self._cursor_lock: - payload = self._latest_cursor_payload - if payload is None: - return - try: - client._channel.send_typed(MessageType.CURSOR, payload) - except OSError: - pass - def _ip_allowed(self, address) -> bool: """Apply the Phase 4.3 allowlist; log + reject silently otherwise.""" peer_ip = address[0] if address else "" @@ -1251,58 +610,6 @@ def _maybe_wrap_tls(self, client_sock: socket.socket, pass return None - def _capture_loop(self) -> None: - next_tick = time.monotonic() - last_frame_hash: Optional[int] = None - while not self._shutdown.is_set(): - try: - frame = self._frame_provider() - except (OSError, RuntimeError, ValueError) as error: - autocontrol_logger.warning( - "remote_desktop frame capture failed: %r", error, - ) - self._shutdown.wait(self._period) - continue - # Phase 2.3: drop frames that are byte-identical to the - # previous capture. A static desktop produces the same JPEG - # every tick (JPEG is deterministic for identical input), - # so this skip costs nothing extra on motion-heavy - # workloads and saves a full FPS-worth of TCP / encoder - # bandwidth at idle. - frame_hash = hash(frame) - if frame_hash != last_frame_hash: - # Phase 6.8: hand the JPEG to the configured codec. - # JpegPassthrough yields the bytes unchanged so the - # wire format stays identical for stock clients. - for encoded in self._encode_for_wire(frame): - with self._frame_cond: - self._latest_frame = encoded - self._latest_seq += 1 - self._frame_cond.notify_all() - last_frame_hash = frame_hash - next_tick += self._period - sleep_for = max(0.0, next_tick - time.monotonic()) - if sleep_for <= 0.0: - next_tick = time.monotonic() - self._shutdown.wait(sleep_for) - - def _encode_for_wire(self, jpeg_bytes: bytes): - """Wrap codec output with a 1-byte tag (skipped for JPEG).""" - provider = self._codec_provider - if provider.name == CODEC_JPEG: - yield jpeg_bytes # legacy wire format: no tag, raw JPEG - return - tag = bytes([codec_tag(provider.name)]) - try: - packets = provider.encode_jpeg(jpeg_bytes) - except (OSError, RuntimeError, ValueError) as error: - autocontrol_logger.warning( - "remote_desktop codec %s failed: %r", provider.name, error, - ) - return - for packet in packets: - yield tag + bytes(packet) - def _reap_dead_clients(self) -> None: with self._clients_lock: self._clients = [c for c in self._clients diff --git a/je_auto_control/utils/remote_desktop/host_access.py b/je_auto_control/utils/remote_desktop/host_access.py new file mode 100644 index 00000000..2dce4d79 --- /dev/null +++ b/je_auto_control/utils/remote_desktop/host_access.py @@ -0,0 +1,105 @@ +"""Viewer approval and access control for the TCP remote-desktop host. + +The gate a viewer passes through after the HMAC handshake and before any +frame is sent: the snapshot handed to the approval callback, how that +callback's return value maps to a permission, the TOTP codes accepted for +a share code, and the IP allowlist. Shared by :mod:`host` and +:mod:`host_client`, which is why it is its own module rather than living +in either. +""" +import time +from dataclasses import dataclass +from typing import Any, Callable, List, Optional, Sequence + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger + + +@dataclass(frozen=True) +class PendingViewer: + """Snapshot of an authenticated viewer awaiting host approval. + + Passed to the ``on_pending_viewer`` callback after the HMAC handshake + succeeds but before the host starts streaming frames. The callback's + return value is interpreted as: + + * ``True`` / ``"full"`` → admit with full control + * ``"view_only"`` → admit, but drop incoming INPUT messages + * ``False`` / ``None`` / etc. → reject + """ + address: tuple + host_id: str + transport: str = "tcp" + +PendingViewerCallback = Callable[[PendingViewer], Any] +"""Callback signature: see :class:`PendingViewer` for return value semantics.""" + +PERMISSION_FULL = "full" +PERMISSION_VIEW_ONLY = "view_only" +PERMISSION_DENIED = "denied" + +def _interpret_approval(result: Any) -> str: + """Map an approval-callback return value to a permission string. + + Backward compatibility: any truthy value other than the literal + ``"view_only"`` / ``"denied"`` strings is treated as full-control + admit. Falsy values are denied. + """ + if result == PERMISSION_VIEW_ONLY: + return PERMISSION_VIEW_ONLY + if result == PERMISSION_DENIED or not result: + return PERMISSION_DENIED + return PERMISSION_FULL + +_AUTH_TIMEOUT_S = 60.0 + +def _candidate_totp_codes(secret: str): + """Yield TOTP codes within ±1 step of the current 30-second window.""" + from je_auto_control.utils.remote_desktop.totp import generate_code + now = time.time() + for delta in (-1, 0, 1): + yield generate_code(secret, at=now + (delta * 30.0)) + +def _compile_ip_allowlist( + entries: Optional[Sequence[str]]) -> Optional[List[Any]]: + """Pre-parse ``entries`` into ``ip_address`` / ``ip_network`` objects. + + ``None`` or an empty list → no filtering (allow all). Entries are + plain IPs (``"192.168.1.10"``) or CIDR ranges (``"10.0.0.0/8"``); + unparseable entries are dropped with a warning so a typo doesn't + silently broaden access. + """ + if not entries: + return None + import ipaddress + compiled: List[Any] = [] + for entry in entries: + text = str(entry).strip() + if not text: + continue + try: + if "/" in text: + compiled.append(ipaddress.ip_network(text, strict=False)) + else: + compiled.append(ipaddress.ip_address(text)) + except ValueError: + autocontrol_logger.warning( + "remote_desktop ip_allowlist entry rejected: %r", text, + ) + return compiled or None + +def _ip_in_allowlist(allowlist: Optional[List[Any]], peer_ip: str) -> bool: + """Return True when ``peer_ip`` matches any allowlist entry (or no list).""" + if not allowlist: + return True + import ipaddress + try: + addr = ipaddress.ip_address(peer_ip) + except ValueError: + return False + for entry in allowlist: + if isinstance(entry, (ipaddress.IPv4Network, ipaddress.IPv6Network)): + if addr in entry: + return True + elif entry == addr: + return True + return False diff --git a/je_auto_control/utils/remote_desktop/host_capture.py b/je_auto_control/utils/remote_desktop/host_capture.py new file mode 100644 index 00000000..01ae4f1c --- /dev/null +++ b/je_auto_control/utils/remote_desktop/host_capture.py @@ -0,0 +1,280 @@ +"""Frame and cursor production for the TCP remote-desktop host. + +What the host streams, as opposed to how it streams it: monitor +enumeration, mapping a monitor index to a capture region, and the default +providers that turn a screen grab into JPEG bytes and read the cursor +position. Every one of them degrades to a working fallback when the +optional dependency (``mss``, Pillow) is missing, so a stock install +still hosts. +""" +import json +import time +from io import BytesIO +from typing import ( + TYPE_CHECKING, Any, Callable, Dict, List, Optional, Sequence, +) + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.protocol import MessageType +from je_auto_control.utils.remote_desktop.video_codec import ( + CODEC_JPEG, codec_tag, +) + +if TYPE_CHECKING: # avoids a runtime cycle: host_client is a sibling + from je_auto_control.utils.remote_desktop.host_client import ( + _ClientHandler, + ) + + +FrameProvider = Callable[[], bytes] + +CursorProvider = Callable[[], Optional[Sequence[int]]] +"""Return ``(x, y)`` in host screen coordinates, or ``None`` to skip a tick.""" + +_DEFAULT_QUALITY = 70 + +_CURSOR_POLL_INTERVAL_S = 1.0 / 30.0 # 30 Hz: smooth, low CPU on idle. + +def list_host_monitors() -> List[Dict[str, Any]]: + """Headless helper: return every monitor's geometry. + + Index 0 spans all monitors (the ``mss`` convention). Returns an + empty list if ``mss`` is not installed, so GUI callers can show a + disabled control instead of crashing. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + try: + grabber = mss_grabber() + except ImportError: + return [] + with grabber as sct: + return [ + { + "index": index, "left": int(monitor["left"]), + "top": int(monitor["top"]), + "width": int(monitor["width"]), + "height": int(monitor["height"]), + "is_combined": index == 0, + } + for index, monitor in enumerate(sct.monitors) + ] + +def _resolve_monitor_region( + monitor_index: int) -> Optional[Sequence[int]]: + """Map an ``mss`` monitor index to ``(x, y, width, height)``. + + Returns ``None`` (full-screen capture fallback) when ``mss`` is + not available so a stock install still works. + """ + from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + try: + grabber = mss_grabber() + except ImportError: + autocontrol_logger.warning( + "remote_desktop monitor_index=%d ignored: mss not installed", + monitor_index, + ) + return None + with grabber as sct: + if monitor_index < 0 or monitor_index >= len(sct.monitors): + raise ValueError( + f"monitor_index {monitor_index} out of range " + f"(0..{len(sct.monitors) - 1})" + ) + mon = sct.monitors[monitor_index] + return ( + int(mon["left"]), int(mon["top"]), + int(mon["width"]), int(mon["height"]), + ) + +def _resolve_cursor_provider( + explicit: Optional[CursorProvider], + enabled: bool) -> Optional[CursorProvider]: + """Pick the cursor provider — explicit > default > disabled.""" + if explicit is not None: + return explicit + return _default_cursor_provider() if enabled else None + +def _default_cursor_provider() -> CursorProvider: + """Build a cursor-position poller using the project's mouse wrapper. + + The wrapper is imported lazily inside the closure so importing this + module on platforms where mouse capture is unavailable does not + blow up. Returns ``None`` on read failures so the broadcast loop + silently skips the tick instead of crashing the host. + """ + def provide() -> Optional[Sequence[int]]: + try: + from je_auto_control.wrapper.auto_control_mouse import ( + get_mouse_position, + ) + except ImportError: + return None + try: + return get_mouse_position() + except (OSError, RuntimeError, AttributeError): + return None + return provide + +def _default_frame_provider(region: Optional[Sequence[int]] = None, + quality: int = _DEFAULT_QUALITY) -> FrameProvider: + """Build a JPEG frame producer using the platform's screen grabber.""" + def provide() -> bytes: + # local import: not needed for unit tests + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber + grabber = image_grabber() + if region is not None: + x, y, width, height = (int(v) for v in region) + bbox = (x, y, x + width, y + height) + image = grabber.grab(bbox=bbox, all_screens=True) + else: + image = grabber.grab(all_screens=True) + if image.mode != "RGB": + image = image.convert("RGB") + buffer = BytesIO() + image.save(buffer, format="JPEG", quality=int(quality)) + return buffer.getvalue() + return provide + + +class FrameProductionMixin: + """Frame and cursor production half of :class:`RemoteDesktopHost`. + + The loops that turn the host screen into what viewers receive: the + ~30 Hz cursor poll, the capture loop that paces frames at the + configured fps, and the codec step between a JPEG frame and the bytes + that go on the wire. Requires the host to provide ``_shutdown``, + ``_clients``/``_clients_lock``, ``_frame_provider``, + ``_cursor_provider``, ``_codec``, ``_fps``, ``_latest_frame`` and + ``_frame_lock``. + """ + + def _cursor_loop(self) -> None: + """Poll cursor position at ~30 Hz and push it to viewers as JSON.""" + provider = self._cursor_provider + if provider is None: + return + while not self._shutdown.is_set(): + position = provider() + if position is not None and len(position) >= 2: + payload = json.dumps( + {"x": int(position[0]), "y": int(position[1]), + "visible": True}, + ).encode("utf-8") + with self._cursor_lock: + is_new = payload != self._latest_cursor_payload + self._latest_cursor_payload = payload + if is_new: + self._broadcast_cursor(payload) + if self._shutdown.wait(timeout=_CURSOR_POLL_INTERVAL_S): + return + + def _broadcast_cursor(self, payload: bytes) -> None: + """Send a CURSOR message to every authenticated client. + + Errors per-client are swallowed — a flaky viewer should not + kill the cursor stream to healthy peers. + """ + with self._clients_lock: + clients = [c for c in self._clients + if c.authenticated and not c._shutdown.is_set()] + for client in clients: + try: + client._channel.send_typed(MessageType.CURSOR, payload) + except OSError: + continue + + def broadcast_viewer_cursor(self, viewer_id: str, + x: int, y: int) -> int: + """Phase 5.1: relay one viewer's cursor position to every other viewer. + + Typically called by :class:`MultiViewerHost` when several + viewers share a session so each viewer's overlay can show the + other operators' pointers (Figma / Google Docs style). The + viewer_id is opaque to the host — viewers use it to colour-key + their overlay. + """ + payload = json.dumps( + {"x": int(x), "y": int(y), "visible": True, + "viewer_id": str(viewer_id)}, + ).encode("utf-8") + with self._clients_lock: + clients = [c for c in self._clients + if c.authenticated and not c._shutdown.is_set()] + sent = 0 + for client in clients: + try: + client._channel.send_typed(MessageType.CURSOR, payload) + sent += 1 + except OSError: + continue + return sent + + def _send_initial_cursor(self, client: "_ClientHandler") -> None: + """Push the latest known cursor position to a fresh client. + + Sending unconditionally on auth means the viewer sees a cursor + immediately instead of waiting up to ~1 s for the next position + change. Safe to call when the cursor loop is disabled — we + only send if there's a payload to send. + """ + with self._cursor_lock: + payload = self._latest_cursor_payload + if payload is None: + return + try: + client._channel.send_typed(MessageType.CURSOR, payload) + except OSError: + pass + + def _capture_loop(self) -> None: + next_tick = time.monotonic() + last_frame_hash: Optional[int] = None + while not self._shutdown.is_set(): + try: + frame = self._frame_provider() + except (OSError, RuntimeError, ValueError) as error: + autocontrol_logger.warning( + "remote_desktop frame capture failed: %r", error, + ) + self._shutdown.wait(self._period) + continue + # Phase 2.3: drop frames that are byte-identical to the + # previous capture. A static desktop produces the same JPEG + # every tick (JPEG is deterministic for identical input), + # so this skip costs nothing extra on motion-heavy + # workloads and saves a full FPS-worth of TCP / encoder + # bandwidth at idle. + frame_hash = hash(frame) + if frame_hash != last_frame_hash: + # Phase 6.8: hand the JPEG to the configured codec. + # JpegPassthrough yields the bytes unchanged so the + # wire format stays identical for stock clients. + for encoded in self._encode_for_wire(frame): + with self._frame_cond: + self._latest_frame = encoded + self._latest_seq += 1 + self._frame_cond.notify_all() + last_frame_hash = frame_hash + next_tick += self._period + sleep_for = max(0.0, next_tick - time.monotonic()) + if sleep_for <= 0.0: + next_tick = time.monotonic() + self._shutdown.wait(sleep_for) + + def _encode_for_wire(self, jpeg_bytes: bytes): + """Wrap codec output with a 1-byte tag (skipped for JPEG).""" + provider = self._codec_provider + if provider.name == CODEC_JPEG: + yield jpeg_bytes # legacy wire format: no tag, raw JPEG + return + tag = bytes([codec_tag(provider.name)]) + try: + packets = provider.encode_jpeg(jpeg_bytes) + except (OSError, RuntimeError, ValueError) as error: + autocontrol_logger.warning( + "remote_desktop codec %s failed: %r", provider.name, error, + ) + return + for packet in packets: + yield tag + bytes(packet) diff --git a/je_auto_control/utils/remote_desktop/host_client.py b/je_auto_control/utils/remote_desktop/host_client.py new file mode 100644 index 00000000..f1ddc6db --- /dev/null +++ b/je_auto_control/utils/remote_desktop/host_client.py @@ -0,0 +1,406 @@ +"""Per-connection handler for the TCP remote-desktop host. + +One instance per connected viewer, owning that viewer's auth exchange, +sender thread, audio sender thread and receiver thread, plus the routing +table that turns an inbound message type into the right handler. The +owning :class:`~je_auto_control.utils.remote_desktop.host.RemoteDesktopHost` +is referenced only through the instance passed to ``__init__``, so this +module does not import it. +""" +import collections +import json +import threading +from typing import TYPE_CHECKING, Deque, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.auth import make_nonce +from je_auto_control.utils.remote_desktop.clipboard_sync import ( + ClipboardSyncError, decode as decode_clipboard, +) +from je_auto_control.utils.remote_desktop.file_transfer import ( + FileTransferError, +) +from je_auto_control.utils.remote_desktop.host_access import ( + PERMISSION_DENIED, PERMISSION_FULL, PERMISSION_VIEW_ONLY, + PendingViewer, _AUTH_TIMEOUT_S, _interpret_approval, +) +from je_auto_control.utils.remote_desktop.input_dispatch import ( + InputDispatchError, +) +from je_auto_control.utils.remote_desktop.protocol import ( + AuthenticationError, MessageType, ProtocolError, +) +from je_auto_control.utils.remote_desktop.transport import MessageChannel + +if TYPE_CHECKING: # avoids a runtime cycle: host imports this module + from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost + +_FILE_MSG_TYPES = frozenset({ + MessageType.FILE_BEGIN, MessageType.FILE_CHUNK, MessageType.FILE_END, +}) + + +class _ClientHandler: + """Per-connection auth + input-receive + frame-send state.""" + + _AUDIO_QUEUE_MAXLEN = 50 # ~2.5 s of buffered chunks at 50 ms each + + def __init__(self, host: "RemoteDesktopHost", + channel: MessageChannel, address) -> None: + self._host = host + self._channel = channel + self._address = address + self._shutdown = threading.Event() + self._sender_thread: Optional[threading.Thread] = None + self._receiver_thread: Optional[threading.Thread] = None + self._audio_queue: Deque[bytes] = collections.deque( + maxlen=self._AUDIO_QUEUE_MAXLEN, + ) + self._audio_lock = threading.Lock() + self._audio_event = threading.Event() + self._audio_sender_thread: Optional[threading.Thread] = None + self.authenticated = False + # Phase 5.3: per-client permission set by the approval callback. + # Default is full control so legacy callers (no callback) keep + # the prior behaviour. + self.permission = PERMISSION_FULL + + @property + def address(self): + return self._address + + def start(self) -> None: + """Run auth (with optional host approval), then start the loops.""" + try: + self._authenticate() + except (AuthenticationError, ProtocolError, OSError) as error: + autocontrol_logger.info( + "remote_desktop client %s rejected: %r", self._address, error, + ) + self._close() + return + self.authenticated = True + # The initial cursor + frame are seeded from _send_loop (the + # per-client sender thread), not here. start() runs on the shared + # accept thread with the socket timeout already cleared, so sending a + # full-screen JPEG to a viewer that authenticates then stops reading + # would block every new accept until its send buffer drains. + self._sender_thread = threading.Thread( + target=self._send_loop, name="rd-sender", daemon=True, + ) + self._receiver_thread = threading.Thread( + target=self._recv_loop, name="rd-recv", daemon=True, + ) + self._sender_thread.start() + self._receiver_thread.start() + if self._host._audio_config.enabled: + self._audio_sender_thread = threading.Thread( + target=self._audio_send_loop, name="rd-audio", daemon=True, + ) + self._audio_sender_thread.start() + + def _send_initial_frame(self) -> None: + """Forward the most recent encoded frame so new clients aren't blank. + + Motion-dedup in :meth:`_capture_loop` means a static desktop + only bumps ``_latest_seq`` once; replaying that frame to the + new client keeps them from sitting on a black popup until the + host moves something. + """ + with self._host._frame_cond: + frame = self._host._latest_frame + if frame is None: + return + try: + self._channel.send_typed(MessageType.FRAME, frame) + except OSError: + pass + + def push_audio(self, chunk: bytes) -> None: + """Enqueue a PCM chunk for delivery; oldest dropped if queue is full.""" + if self._shutdown.is_set() or not self.authenticated: + return + with self._audio_lock: + self._audio_queue.append(chunk) + self._audio_event.set() + + def stop(self) -> None: + """Signal threads and close the socket.""" + self._shutdown.set() + with self._host._frame_cond: + self._host._frame_cond.notify_all() + self._audio_event.set() + self._close() + + def _resolve_permission(self) -> str: + """Run the host's optional approval callback after token auth. + + Returns one of :data:`PERMISSION_FULL` / :data:`PERMISSION_VIEW_ONLY` + (admit) or :data:`PERMISSION_DENIED` (reject). The caller is + expected to send ``AUTH_FAIL`` and raise on denial — keeping + that wire-level handling inside :meth:`_authenticate` so the + viewer sees the rejection before it has a chance to flip into + the post-handshake state where ``AUTH_FAIL`` is ignored. + """ + callback = self._host._on_pending_viewer + if callback is None: + return PERMISSION_FULL + pending = PendingViewer( + address=tuple(self._address) if self._address else (), + host_id=self._host.host_id, + transport=self._host._transport_name(), + ) + try: + return _interpret_approval(callback(pending)) + except (RuntimeError, ValueError, TypeError) as error: + autocontrol_logger.info( + "remote_desktop approval callback raised for %s: %r", + self._address, error, + ) + return PERMISSION_DENIED + + def _authenticate(self) -> None: + nonce = make_nonce() + self._channel.settimeout(_AUTH_TIMEOUT_S) + self._channel.send_typed(MessageType.AUTH_CHALLENGE, nonce) + msg_type, payload = self._channel.read_typed() + if msg_type is not MessageType.AUTH_RESPONSE: + self._channel.send_typed(MessageType.AUTH_FAIL, + b"expected AUTH_RESPONSE") + raise AuthenticationError( + f"expected AUTH_RESPONSE, got {msg_type.name}" + ) + # Phase 6.6: a viewer reconnecting with a valid resume token + # signs with that token directly — host short-circuits the + # approval popup and reuses the saved permission. + resumed = self._host._try_consume_resume(nonce, payload) + if resumed is not None: + self.permission = resumed + else: + if not self._host._verify_token(nonce, payload): + self._channel.send_typed(MessageType.AUTH_FAIL, b"bad token") + raise AuthenticationError("bad token") + # Host operator gates the session *before* AUTH_OK so the + # viewer surfaces the rejection as an AuthenticationError + # instead of connecting and then mysteriously disconnecting. + permission = self._resolve_permission() + if permission == PERMISSION_DENIED: + self._channel.send_typed( + MessageType.AUTH_FAIL, b"rejected by host", + ) + raise AuthenticationError("rejected by host") + self.permission = permission + # Issue a fresh resume token so the viewer can reconnect + # within the store's TTL without the approval popup. + resume_token = self._host._resume_store.issue(self.permission) + ok_payload = json.dumps( + {"host_id": self._host.host_id, + "resume_token": resume_token, + "resume_ttl": self._host._resume_store.ttl, + "codec": self._host._codec_provider.name}, + ensure_ascii=False, + ).encode("utf-8") + self._channel.send_typed(MessageType.AUTH_OK, ok_payload) + self._channel.settimeout(None) + + def _send_loop(self) -> None: + # Seed the viewer with the latest cursor + frame on this per-client + # sender thread (Phase 2.3: motion-aware capture only bumps the seq on + # change, so a viewer joining a static desktop would otherwise sit + # blank until the host moves). Doing it here rather than in start() + # keeps a slow-reading viewer from stalling the accept thread. + self._host._send_initial_cursor(self) + self._send_initial_frame() + last_sent = 0 + while not self._shutdown.is_set(): + with self._host._frame_cond: + while (not self._shutdown.is_set() + and self._host._latest_seq <= last_sent): + self._host._frame_cond.wait(timeout=0.5) + if self._shutdown.is_set(): + return + frame = self._host._latest_frame + seq = self._host._latest_seq + if frame is None: + continue + try: + self._channel.send_typed(MessageType.FRAME, frame) + except OSError as error: + autocontrol_logger.info( + "remote_desktop send to %s failed: %r", + self._address, error, + ) + self.stop() + return + last_sent = seq + + def _audio_send_loop(self) -> None: + while not self._shutdown.is_set(): + self._audio_event.wait(timeout=0.5) + if self._shutdown.is_set(): + return + while True: + with self._audio_lock: + if not self._audio_queue: + self._audio_event.clear() + break + chunk = self._audio_queue.popleft() + try: + self._channel.send_typed(MessageType.AUDIO, chunk) + except OSError as error: + autocontrol_logger.info( + "remote_desktop audio send to %s failed: %r", + self._address, error, + ) + self.stop() + return + + def _recv_loop(self) -> None: + while not self._shutdown.is_set(): + try: + msg_type, payload = self._channel.read_typed() + except (OSError, ProtocolError) as error: + if not self._shutdown.is_set(): + autocontrol_logger.info( + "remote_desktop recv from %s ended: %r", + self._address, error, + ) + self.stop() + return + self._route_incoming(msg_type, payload) + + def _route_incoming(self, msg_type: MessageType, payload: bytes) -> None: + """Dispatch one received message to the matching handler.""" + if msg_type is MessageType.PING: + return + if msg_type is MessageType.INPUT: + # Phase 5.3: drop input from view-only viewers so they can + # watch but cannot drive the mouse / keyboard. + if self.permission != PERMISSION_VIEW_ONLY: + self._handle_input_payload(payload) + return + if msg_type is MessageType.CLIPBOARD: + self._handle_clipboard_payload(payload) + return + if msg_type is MessageType.CHAT: + self._handle_chat_payload(payload) + return + if msg_type is MessageType.USB_LIST_REQUEST: + self._handle_usb_list_request() + return + if msg_type in _FILE_MSG_TYPES: + self._handle_file_payload(msg_type, payload) + return + autocontrol_logger.info( + "remote_desktop unexpected msg %s from %s", + msg_type.name, self._address, + ) + + def _handle_file_payload(self, msg_type: MessageType, + payload: bytes) -> None: + receiver = self._host._ensure_file_receiver() + try: + if msg_type is MessageType.FILE_BEGIN: + receiver.handle_begin(payload) + elif msg_type is MessageType.FILE_CHUNK: + receiver.handle_chunk(payload) + elif msg_type is MessageType.FILE_END: + receiver.handle_end(payload) + except FileTransferError as error: + autocontrol_logger.info( + "remote_desktop bad file message from %s: %r", + self._address, error, + ) + + def _handle_usb_list_request(self) -> None: + """Phase 6.9: enumerate the host's USB devices and ship the list back. + + Uses the existing :func:`list_usb_devices` helper so we get the + same cross-platform behaviour as the standalone USB module. + Errors fall back to an empty payload — the viewer should + treat that as "host has no usable USB backend" rather than + crashing. + """ + try: + from je_auto_control.utils.usb import list_usb_devices + result = list_usb_devices() + body = { + "backend": result.backend, + "devices": [d.to_dict() for d in result.devices], + } + except (ImportError, OSError, RuntimeError) as error: + autocontrol_logger.info( + "usb_list from %s failed: %r", self._address, error, + ) + body = {"backend": "unavailable", "devices": []} + try: + self._channel.send_typed( + MessageType.USB_LIST_RESPONSE, + json.dumps(body, ensure_ascii=False).encode("utf-8"), + ) + except OSError: + pass + + def _handle_chat_payload(self, payload: bytes) -> None: + """Forward viewer-originated chat to the host's optional callback.""" + callback = self._host._on_chat + if callback is None: + return + try: + body = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return + if not isinstance(body, dict): + return + text = body.get("text") + sender = body.get("sender", "viewer") + if not isinstance(text, str) or not text: + return + try: + callback(str(sender), text) + except Exception: # noqa: BLE001 callback isolation + autocontrol_logger.exception( + "remote_desktop on_chat callback raised" + ) + + def _handle_clipboard_payload(self, payload: bytes) -> None: + try: + kind, data = decode_clipboard(payload) + except ClipboardSyncError as error: + autocontrol_logger.info( + "remote_desktop bad CLIPBOARD from %s: %r", + self._address, error, + ) + return + try: + self._host._apply_clipboard(kind, data) + except (OSError, RuntimeError, TypeError, ValueError) as error: + autocontrol_logger.warning( + "remote_desktop clipboard apply failed for %s: %r", + self._address, error, + ) + + def _handle_input_payload(self, payload: bytes) -> None: + try: + message = json.loads(payload.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as error: + autocontrol_logger.info( + "remote_desktop bad INPUT from %s: %r", + self._address, error, + ) + return + try: + self._host._dispatch(message) + except InputDispatchError as error: + autocontrol_logger.info( + "remote_desktop rejected INPUT from %s: %r", + self._address, error, + ) + except (OSError, RuntimeError, ValueError, TypeError) as error: + autocontrol_logger.warning( + "remote_desktop input apply failed for %s: %r", + self._address, error, + ) + + def _close(self) -> None: + self._channel.close() diff --git a/je_auto_control/utils/remote_desktop/webrtc_host.py b/je_auto_control/utils/remote_desktop/webrtc_host.py index e8f9d374..fa8670d9 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_host.py +++ b/je_auto_control/utils/remote_desktop/webrtc_host.py @@ -21,9 +21,6 @@ from je_auto_control.utils.logging.logging_instance import autocontrol_logger from je_auto_control.utils.remote_desktop.audit_log import default_audit_log -from je_auto_control.utils.remote_desktop.fingerprint import ( - load_or_create_host_fingerprint, -) from je_auto_control.utils.remote_desktop.input_dispatch import dispatch_input from je_auto_control.utils.remote_desktop.permissions import SessionPermissions from je_auto_control.utils.remote_desktop.rate_limit import ( @@ -34,6 +31,12 @@ RTCPeerConnection, RTCSessionDescription, ScreenVideoTrack, WebRTCConfig, get_bridge, wait_for_ice_gathering, ) +from je_auto_control.utils.remote_desktop.webrtc_host_auth import ( + ViewerAuthMixin, +) +from je_auto_control.utils.remote_desktop.webrtc_host_media import ( + MediaNegotiationMixin, +) _AUTH_GRACE_S = 5.0 @@ -45,7 +48,7 @@ ConsentCallback = Callable[[str], bool] -class WebRTCDesktopHost: +class WebRTCDesktopHost(ViewerAuthMixin, MediaNegotiationMixin): """Single-viewer WebRTC host with manual SDP signaling. Multiple simultaneous viewers would require one ``RTCPeerConnection`` @@ -559,155 +562,6 @@ async def _async_apply_renegotiate_answer(self, sdp: str) -> None: self._maybe_resubscribe_viewer_video() self._maybe_resubscribe_viewer_audio() - def _maybe_resubscribe_viewer_video(self) -> None: - if not (self._config.accept_viewer_video - and self._viewer_video_task is None - and self._pc is not None): - return - video_ts = [ - t for t in self._pc.getTransceivers() if t.kind == "video" - ] - for transceiver in video_ts[1:]: # skip our outbound slot - track = self._receiver_track(transceiver) - if track is None: - continue - self._viewer_video_task = self._spawn_bg( - self._consume_viewer_video(track), - ) - autocontrol_logger.info( - "webrtc host: re-spawned viewer video consume task", - ) - return - - def _maybe_resubscribe_viewer_audio(self) -> None: - if not (self._config.accept_viewer_audio_opus - and self._opus_audio_receiver is None - and self._pc is not None): - return - for transceiver in self._pc.getTransceivers(): - if transceiver.kind != "audio": - continue - track = self._receiver_track(transceiver) - if track is None: - continue - self._start_opus_audio_receive(track) - return - - @staticmethod - def _receiver_track(transceiver): - receiver = transceiver.receiver - return receiver.track if receiver is not None else None - - async def _async_renegotiate(self) -> None: - """Host-initiated renegotiation: new offer → viewer over ctrl channel.""" - if self._pc is None: - return - try: - offer = await self._pc.createOffer() - await self._pc.setLocalDescription(offer) - await wait_for_ice_gathering(self._pc) - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("renegotiate offer: %r", error) - return - self._send_ctrl({ - "type": "renegotiate_offer", - "sdp": self._pc.localDescription.sdp, - }) - autocontrol_logger.info("webrtc host: sent renegotiate offer") - - def request_renegotiation(self) -> None: - """Public sync entry: kick off a fresh SDP exchange over ctrl channel.""" - if self._pc is None: - return - get_bridge().call_soon( - lambda: self._spawn_bg(self._async_renegotiate()), - ) - - def enable_accept_viewer_video(self) -> None: - """Live-add a recvonly video transceiver and renegotiate. - - ``enable_*`` only adds capacity — aiortc has no ``removeTransceiver``, - so disabling needs a reconnect (or set the transceiver to inactive). - """ - if self._pc is None: - return - self._config.accept_viewer_video = True - get_bridge().call_soon(self._add_recvonly_video_and_renegotiate) - - def enable_accept_viewer_audio_opus(self) -> None: - """Live-add a recvonly audio transceiver and renegotiate.""" - if self._pc is None: - return - self._config.accept_viewer_audio_opus = True - get_bridge().call_soon(self._add_recvonly_audio_and_renegotiate) - - def _add_recvonly_video_and_renegotiate(self) -> None: - if self._pc is None: - return - already = sum( - 1 for t in self._pc.getTransceivers() if t.kind == "video" - ) - if already < 2: - self._pc.addTransceiver("video", direction="recvonly") - self._spawn_bg(self._async_renegotiate()) - - def _add_recvonly_audio_and_renegotiate(self) -> None: - if self._pc is None: - return - already = sum( - 1 for t in self._pc.getTransceivers() if t.kind == "audio" - ) - if already < 1: - self._pc.addTransceiver("audio", direction="recvonly") - self._spawn_bg(self._async_renegotiate()) - - def disable_accept_viewer_video(self) -> None: - """Mark the recvonly video slot inactive + stop the consume task.""" - if self._pc is None: - return - self._config.accept_viewer_video = False - get_bridge().call_soon(self._deactivate_recvonly_video) - - def disable_accept_viewer_audio_opus(self) -> None: - """Mark the recvonly audio slot inactive + stop the Opus receiver.""" - if self._pc is None: - return - self._config.accept_viewer_audio_opus = False - get_bridge().call_soon(self._deactivate_recvonly_audio) - - def _deactivate_recvonly_video(self) -> None: - if self._pc is None: - return - # Find the second video transceiver (the recvonly one); first is our - # outbound screen track. - video_ts = [t for t in self._pc.getTransceivers() if t.kind == "video"] - if len(video_ts) >= 2: - try: - video_ts[1].direction = "inactive" - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("inactivate video: %r", error) - if self._viewer_video_task is not None: - self._viewer_video_task.cancel() - self._viewer_video_task = None - self._spawn_bg(self._async_renegotiate()) - - def _deactivate_recvonly_audio(self) -> None: - if self._pc is None: - return - audio_ts = [t for t in self._pc.getTransceivers() if t.kind == "audio"] - if audio_ts: - try: - audio_ts[0].direction = "inactive" - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("inactivate audio: %r", error) - if self._opus_audio_receiver is not None: - try: - self._opus_audio_receiver.stop() - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("opus receiver stop: %r", error) - self._opus_audio_receiver = None - self._spawn_bg(self._async_renegotiate()) - def _ensure_files_receiver(self): from je_auto_control.utils.remote_desktop.webrtc_files import ( FileTransferReceiver, @@ -803,172 +657,6 @@ def read_only(self) -> bool: def permissions(self) -> SessionPermissions: return self._permissions - def _handle_send_sas(self) -> None: - try: - from je_auto_control.utils.remote_desktop.session_actions import ( - send_secure_attention_sequence, - ) - send_secure_attention_sequence() - self._send_ctrl({"type": "sas_ok"}) - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("SendSAS: %r", error) - self._send_ctrl({"type": "sas_fail", "error": str(error)}) - - def _handle_auth(self, data: Mapping[str, Any]) -> None: - token = data.get("token") - if not isinstance(token, str) or token != self._token: - self._reject_auth(data) - return - viewer_id = data.get("viewer_id") - self._pending_viewer_id = ( - viewer_id if isinstance(viewer_id, str) else None - ) - if self._auto_approve_via_trust(): - return - if self._auto_approve_via_whitelist(): - return - if self._on_pending_viewer is None: - self._approve_pending_viewer() - return - self._has_pending_viewer = True - try: - self._on_pending_viewer() - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("pending viewer cb: %r", error) - - def _reject_auth(self, data: Mapping[str, Any]) -> None: - self._send_ctrl({"type": "auth_fail"}) - try: - default_audit_log().log( - "auth_fail", - viewer_id=str(data.get("viewer_id", "")) or None, - detail=f"remote_ip={self._remote_ip}", - ) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("audit log auth_fail: %r", error) - get_bridge().call_soon(self._schedule_close_after_fail) - - def _auto_approve_via_trust(self) -> bool: - if not self._is_trusted_viewer(self._pending_viewer_id): - return False - autocontrol_logger.info( - "webrtc host: viewer_id %s is trusted; auto-approving", - self._pending_viewer_id, - ) - if self._trust_list is not None: - try: - self._trust_list.touch(self._pending_viewer_id) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("trust touch: %r", error) - self._approve_pending_viewer() - return True - - def _auto_approve_via_whitelist(self) -> bool: - if not self._is_ip_whitelisted(self._remote_ip): - return False - autocontrol_logger.info( - "webrtc host: remote ip %s matches whitelist; auto-approving", - self._remote_ip, - ) - self._approve_pending_viewer() - return True - - def _is_ip_whitelisted(self, ip: Optional[str]) -> bool: - if not ip or not self._ip_whitelist: - return False - import ipaddress - try: - addr = ipaddress.ip_address(ip) - except ValueError: - return False - for cidr in self._ip_whitelist: - try: - if addr in ipaddress.ip_network(cidr.strip(), strict=False): - return True - except ValueError: - continue - return False - - def _is_trusted_viewer(self, viewer_id: Optional[str]) -> bool: - if self._trust_list is None or not viewer_id: - return False - try: - return self._trust_list.is_trusted(viewer_id) - except (OSError, RuntimeError) as error: - autocontrol_logger.warning("trust list check: %r", error) - return False - - def trust_pending_viewer(self, label: str = "") -> None: - """Add the current pending viewer to the trust list, then approve.""" - viewer_id = self._pending_viewer_id - if self._trust_list is not None and viewer_id: - try: - self._trust_list.add(viewer_id, label=label) - except (OSError, ValueError, RuntimeError) as error: - autocontrol_logger.warning("trust list add: %r", error) - self.approve_pending_viewer() - - @property - def pending_viewer_id(self) -> Optional[str]: - return self._pending_viewer_id - - def approve_pending_viewer(self) -> None: - """Thread-safe accept; call from GUI when user clicks Accept.""" - get_bridge().call_soon(self._approve_pending_viewer) - - def reject_pending_viewer(self) -> None: - """Thread-safe reject; call from GUI when user clicks Reject.""" - get_bridge().call_soon(self._reject_pending_viewer) - - def _approve_pending_viewer(self) -> None: - if not self._has_pending_viewer and self._authenticated: - return - self._has_pending_viewer = False - self._authenticated = True - self._send_ctrl({ - "type": "auth_ok", - "read_only": not self._permissions.allow_input, - "permissions": self._permissions.to_dict(), - "fingerprint": load_or_create_host_fingerprint(), - }) - try: - default_audit_log().log( - "auth_ok", - viewer_id=self._pending_viewer_id, - detail=f"remote_ip={self._remote_ip}", - ) - except (RuntimeError, OSError) as error: - autocontrol_logger.debug("audit log auth_ok: %r", error) - if self._auth_deadline_handle is not None: - self._auth_deadline_handle.cancel() - self._auth_deadline_handle = None - if self._on_authenticated is not None: - try: - self._on_authenticated() - except (RuntimeError, OSError) as error: - autocontrol_logger.warning("auth cb: %r", error) - - def _reject_pending_viewer(self) -> None: - self._has_pending_viewer = False - self._send_ctrl({"type": "auth_fail"}) - get_bridge().call_soon(self._schedule_close_after_fail) - - @property - def has_pending_viewer(self) -> bool: - return self._has_pending_viewer - - def _schedule_close_after_fail(self) -> None: - loop = asyncio.get_event_loop() - loop.call_later(0.5, lambda: self._spawn_bg(self._async_stop())) - - def _enforce_auth_deadline(self) -> None: - if self._authenticated: - return - autocontrol_logger.warning( - "webrtc host: viewer failed to authenticate within grace period", - ) - self._spawn_bg(self._async_stop()) - def _dispatch_input_safely(self, payload: Any) -> None: if not isinstance(payload, dict): return diff --git a/je_auto_control/utils/remote_desktop/webrtc_host_auth.py b/je_auto_control/utils/remote_desktop/webrtc_host_auth.py new file mode 100644 index 00000000..434501e9 --- /dev/null +++ b/je_auto_control/utils/remote_desktop/webrtc_host_auth.py @@ -0,0 +1,195 @@ +"""Viewer authentication and approval for the WebRTC host. + +Everything between a viewer's first ``auth`` message and the moment it is +allowed to send input: token check, the auto-approve paths (trust list, +IP whitelist), the manual accept/reject prompt the GUI drives, the SAS +exchange, and the grace-period deadline that closes an unauthenticated +peer. Kept apart from ``webrtc_host`` so the media session lifecycle is +readable on its own. +""" +from __future__ import annotations + +import asyncio +from typing import Any, Mapping, Optional + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.audit_log import default_audit_log +from je_auto_control.utils.remote_desktop.fingerprint import ( + load_or_create_host_fingerprint, +) +from je_auto_control.utils.remote_desktop.webrtc_transport import get_bridge + + +class ViewerAuthMixin: + """Auth half of :class:`WebRTCDesktopHost`. + + Requires the host to provide ``_token``, ``_trust_list``, + ``_ip_whitelist``, ``_authenticated``, ``_pending_viewer_id``, + ``_send_ctrl``, ``_spawn_bg`` and ``_async_stop``. + """ + + def _handle_send_sas(self) -> None: + try: + from je_auto_control.utils.remote_desktop.session_actions import ( + send_secure_attention_sequence, + ) + send_secure_attention_sequence() + self._send_ctrl({"type": "sas_ok"}) + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("SendSAS: %r", error) + self._send_ctrl({"type": "sas_fail", "error": str(error)}) + + def _handle_auth(self, data: Mapping[str, Any]) -> None: + token = data.get("token") + if not isinstance(token, str) or token != self._token: + self._reject_auth(data) + return + viewer_id = data.get("viewer_id") + self._pending_viewer_id = ( + viewer_id if isinstance(viewer_id, str) else None + ) + if self._auto_approve_via_trust(): + return + if self._auto_approve_via_whitelist(): + return + if self._on_pending_viewer is None: + self._approve_pending_viewer() + return + self._has_pending_viewer = True + try: + self._on_pending_viewer() + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("pending viewer cb: %r", error) + + def _reject_auth(self, data: Mapping[str, Any]) -> None: + self._send_ctrl({"type": "auth_fail"}) + try: + default_audit_log().log( + "auth_fail", + viewer_id=str(data.get("viewer_id", "")) or None, + detail=f"remote_ip={self._remote_ip}", + ) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("audit log auth_fail: %r", error) + get_bridge().call_soon(self._schedule_close_after_fail) + + def _auto_approve_via_trust(self) -> bool: + if not self._is_trusted_viewer(self._pending_viewer_id): + return False + autocontrol_logger.info( + "webrtc host: viewer_id %s is trusted; auto-approving", + self._pending_viewer_id, + ) + if self._trust_list is not None: + try: + self._trust_list.touch(self._pending_viewer_id) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("trust touch: %r", error) + self._approve_pending_viewer() + return True + + def _auto_approve_via_whitelist(self) -> bool: + if not self._is_ip_whitelisted(self._remote_ip): + return False + autocontrol_logger.info( + "webrtc host: remote ip %s matches whitelist; auto-approving", + self._remote_ip, + ) + self._approve_pending_viewer() + return True + + def _is_ip_whitelisted(self, ip: Optional[str]) -> bool: + if not ip or not self._ip_whitelist: + return False + import ipaddress + try: + addr = ipaddress.ip_address(ip) + except ValueError: + return False + for cidr in self._ip_whitelist: + try: + if addr in ipaddress.ip_network(cidr.strip(), strict=False): + return True + except ValueError: + continue + return False + + def _is_trusted_viewer(self, viewer_id: Optional[str]) -> bool: + if self._trust_list is None or not viewer_id: + return False + try: + return self._trust_list.is_trusted(viewer_id) + except (OSError, RuntimeError) as error: + autocontrol_logger.warning("trust list check: %r", error) + return False + + def trust_pending_viewer(self, label: str = "") -> None: + """Add the current pending viewer to the trust list, then approve.""" + viewer_id = self._pending_viewer_id + if self._trust_list is not None and viewer_id: + try: + self._trust_list.add(viewer_id, label=label) + except (OSError, ValueError, RuntimeError) as error: + autocontrol_logger.warning("trust list add: %r", error) + self.approve_pending_viewer() + + @property + def pending_viewer_id(self) -> Optional[str]: + return self._pending_viewer_id + + def approve_pending_viewer(self) -> None: + """Thread-safe accept; call from GUI when user clicks Accept.""" + get_bridge().call_soon(self._approve_pending_viewer) + + def reject_pending_viewer(self) -> None: + """Thread-safe reject; call from GUI when user clicks Reject.""" + get_bridge().call_soon(self._reject_pending_viewer) + + def _approve_pending_viewer(self) -> None: + if not self._has_pending_viewer and self._authenticated: + return + self._has_pending_viewer = False + self._authenticated = True + self._send_ctrl({ + "type": "auth_ok", + "read_only": not self._permissions.allow_input, + "permissions": self._permissions.to_dict(), + "fingerprint": load_or_create_host_fingerprint(), + }) + try: + default_audit_log().log( + "auth_ok", + viewer_id=self._pending_viewer_id, + detail=f"remote_ip={self._remote_ip}", + ) + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("audit log auth_ok: %r", error) + if self._auth_deadline_handle is not None: + self._auth_deadline_handle.cancel() + self._auth_deadline_handle = None + if self._on_authenticated is not None: + try: + self._on_authenticated() + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("auth cb: %r", error) + + def _reject_pending_viewer(self) -> None: + self._has_pending_viewer = False + self._send_ctrl({"type": "auth_fail"}) + get_bridge().call_soon(self._schedule_close_after_fail) + + @property + def has_pending_viewer(self) -> bool: + return self._has_pending_viewer + + def _schedule_close_after_fail(self) -> None: + loop = asyncio.get_event_loop() + loop.call_later(0.5, lambda: self._spawn_bg(self._async_stop())) + + def _enforce_auth_deadline(self) -> None: + if self._authenticated: + return + autocontrol_logger.warning( + "webrtc host: viewer failed to authenticate within grace period", + ) + self._spawn_bg(self._async_stop()) diff --git a/je_auto_control/utils/remote_desktop/webrtc_host_media.py b/je_auto_control/utils/remote_desktop/webrtc_host_media.py new file mode 100644 index 00000000..2921f3b3 --- /dev/null +++ b/je_auto_control/utils/remote_desktop/webrtc_host_media.py @@ -0,0 +1,172 @@ +"""Renegotiation and recvonly-track management for the WebRTC host. + +aiortc has no ``removeTransceiver``, so turning viewer video or Opus audio +on and off is not symmetric: enabling adds a recvonly transceiver and +re-offers, disabling can only set the existing one inactive and stop the +receiver. That asymmetry, the host-initiated offer that carries it, and the +re-subscription that runs after each answer are the whole of this mixin. +""" +from __future__ import annotations + +from je_auto_control.utils.logging.logging_instance import autocontrol_logger +from je_auto_control.utils.remote_desktop.webrtc_transport import ( + get_bridge, wait_for_ice_gathering, +) + + +class MediaNegotiationMixin: + """Media-track half of :class:`WebRTCDesktopHost`. + + Requires the host to provide ``_pc``, ``_config``, ``_viewer_video_task``, + ``_opus_audio_receiver``, ``_send_ctrl``, ``_spawn_bg``, + ``_consume_viewer_video`` and ``_start_opus_audio_receive``. + """ + + def _maybe_resubscribe_viewer_video(self) -> None: + if not (self._config.accept_viewer_video + and self._viewer_video_task is None + and self._pc is not None): + return + video_ts = [ + t for t in self._pc.getTransceivers() if t.kind == "video" + ] + for transceiver in video_ts[1:]: # skip our outbound slot + track = self._receiver_track(transceiver) + if track is None: + continue + self._viewer_video_task = self._spawn_bg( + self._consume_viewer_video(track), + ) + autocontrol_logger.info( + "webrtc host: re-spawned viewer video consume task", + ) + return + + def _maybe_resubscribe_viewer_audio(self) -> None: + if not (self._config.accept_viewer_audio_opus + and self._opus_audio_receiver is None + and self._pc is not None): + return + for transceiver in self._pc.getTransceivers(): + if transceiver.kind != "audio": + continue + track = self._receiver_track(transceiver) + if track is None: + continue + self._start_opus_audio_receive(track) + return + + @staticmethod + def _receiver_track(transceiver): + receiver = transceiver.receiver + return receiver.track if receiver is not None else None + + async def _async_renegotiate(self) -> None: + """Host-initiated renegotiation: new offer → viewer over ctrl channel.""" + if self._pc is None: + return + try: + offer = await self._pc.createOffer() + await self._pc.setLocalDescription(offer) + await wait_for_ice_gathering(self._pc) + except (RuntimeError, OSError) as error: + autocontrol_logger.warning("renegotiate offer: %r", error) + return + self._send_ctrl({ + "type": "renegotiate_offer", + "sdp": self._pc.localDescription.sdp, + }) + autocontrol_logger.info("webrtc host: sent renegotiate offer") + + def request_renegotiation(self) -> None: + """Public sync entry: kick off a fresh SDP exchange over ctrl channel.""" + if self._pc is None: + return + get_bridge().call_soon( + lambda: self._spawn_bg(self._async_renegotiate()), + ) + + def enable_accept_viewer_video(self) -> None: + """Live-add a recvonly video transceiver and renegotiate. + + ``enable_*`` only adds capacity — aiortc has no ``removeTransceiver``, + so disabling needs a reconnect (or set the transceiver to inactive). + """ + if self._pc is None: + return + self._config.accept_viewer_video = True + get_bridge().call_soon(self._add_recvonly_video_and_renegotiate) + + def enable_accept_viewer_audio_opus(self) -> None: + """Live-add a recvonly audio transceiver and renegotiate.""" + if self._pc is None: + return + self._config.accept_viewer_audio_opus = True + get_bridge().call_soon(self._add_recvonly_audio_and_renegotiate) + + def _add_recvonly_video_and_renegotiate(self) -> None: + if self._pc is None: + return + already = sum( + 1 for t in self._pc.getTransceivers() if t.kind == "video" + ) + if already < 2: + self._pc.addTransceiver("video", direction="recvonly") + self._spawn_bg(self._async_renegotiate()) + + def _add_recvonly_audio_and_renegotiate(self) -> None: + if self._pc is None: + return + already = sum( + 1 for t in self._pc.getTransceivers() if t.kind == "audio" + ) + if already < 1: + self._pc.addTransceiver("audio", direction="recvonly") + self._spawn_bg(self._async_renegotiate()) + + def disable_accept_viewer_video(self) -> None: + """Mark the recvonly video slot inactive + stop the consume task.""" + if self._pc is None: + return + self._config.accept_viewer_video = False + get_bridge().call_soon(self._deactivate_recvonly_video) + + def disable_accept_viewer_audio_opus(self) -> None: + """Mark the recvonly audio slot inactive + stop the Opus receiver.""" + if self._pc is None: + return + self._config.accept_viewer_audio_opus = False + get_bridge().call_soon(self._deactivate_recvonly_audio) + + def _deactivate_recvonly_video(self) -> None: + if self._pc is None: + return + # Find the second video transceiver (the recvonly one); first is our + # outbound screen track. + video_ts = [t for t in self._pc.getTransceivers() if t.kind == "video"] + if len(video_ts) >= 2: + try: + video_ts[1].direction = "inactive" + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("inactivate video: %r", error) + if self._viewer_video_task is not None: + self._viewer_video_task.cancel() + self._viewer_video_task = None + self._spawn_bg(self._async_renegotiate()) + + def _deactivate_recvonly_audio(self) -> None: + if self._pc is None: + return + audio_ts = [t for t in self._pc.getTransceivers() if t.kind == "audio"] + if audio_ts: + try: + audio_ts[0].direction = "inactive" + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("inactivate audio: %r", error) + if self._opus_audio_receiver is not None: + try: + self._opus_audio_receiver.stop() + except (RuntimeError, OSError) as error: + autocontrol_logger.debug("opus receiver stop: %r", error) + self._opus_audio_receiver = None + self._spawn_bg(self._async_renegotiate()) diff --git a/je_auto_control/utils/remote_desktop/webrtc_transport.py b/je_auto_control/utils/remote_desktop/webrtc_transport.py index 9fd3cb8b..b84d1722 100644 --- a/je_auto_control/utils/remote_desktop/webrtc_transport.py +++ b/je_auto_control/utils/remote_desktop/webrtc_transport.py @@ -30,10 +30,14 @@ ) from exc try: - import mss # type: ignore + import mss # type: ignore # noqa: F401 # reason: annotations + install probe except ImportError as exc: # pragma: no cover - mss is a base dep raise ImportError("mss is required for screen capture") from exc +# Frames come from the platform grabber rather than mss directly: on Wayland +# mss reads the XWayland root, which holds none of the native Wayland windows. +from je_auto_control.utils.cv2_utils.screen_grabber import mss_grabber + _DEFAULT_STUN = "stun:stun.l.google.com:19302" _DEFAULT_STUN_SERVERS = ( @@ -215,7 +219,7 @@ def _resolve_monitor(sct: "mss.base.MSSBase", index: int) -> dict: def _capture_frame(monitor: dict) -> "np.ndarray": sct = getattr(_capture_local, "sct", None) if sct is None: - sct = mss.mss() + sct = mss_grabber() _capture_local.sct = sct img = sct.grab(monitor) arr = np.frombuffer(img.bgra, dtype=np.uint8).reshape( @@ -279,7 +283,7 @@ def _resolve(self) -> dict: else: sct = getattr(_capture_local, "sct", None) if sct is None: - sct = mss.mss() + sct = mss_grabber() _capture_local.sct = sct self._monitor = _resolve_monitor(sct, self._monitor_index) return self._monitor diff --git a/je_auto_control/utils/rest_api/rest_handlers.py b/je_auto_control/utils/rest_api/rest_handlers.py index 708f2b7b..7fd8c933 100644 --- a/je_auto_control/utils/rest_api/rest_handlers.py +++ b/je_auto_control/utils/rest_api/rest_handlers.py @@ -13,6 +13,10 @@ from typing import Any, Dict, List, Optional, Tuple from urllib.parse import parse_qs +from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, AutoControlActionNullException, + AutoControlException, AutoControlJsonActionException, +) from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -138,12 +142,37 @@ def handle_commands(_ctx: RouteContext) -> HandlerResult: return 200, {"commands": names, "count": len(names)} +def _reject_bad_action_list(actions: Any) -> Optional[HandlerResult]: + """Return a 4xx result if ``actions`` is not dispatchable, else ``None``. + + A misspelt ``AC_*`` name is the caller's mistake, so it has to come back + as a 400 naming the command rather than the blanket 500 that any other + executor failure produces — otherwise a client cannot tell a typo from a + broken server. The whole list is checked before anything runs, which is + also what keeps a half-executed action list from being possible here. + """ + from je_auto_control.utils.executor.action_executor import executor + try: + unknown = executor.unknown_commands_in(actions) + except AutoControlException as error: + return 400, {"error": str(error)} + if unknown: + return 400, { + "error": "unknown command name(s): " + ", ".join(unknown), + "unknown_commands": unknown, + } + return None + + def handle_execute(ctx: RouteContext) -> HandlerResult: if not isinstance(ctx.body, dict): return 400, {"error": "body must be JSON object"} actions = ctx.body.get("actions") if actions is None: return 400, {"error": "missing 'actions' field"} + rejection = _reject_bad_action_list(actions) + if rejection is not None: + return rejection try: from je_auto_control.utils.executor.action_executor import execute_action result = execute_action(actions) @@ -162,6 +191,13 @@ def handle_execute_file(ctx: RouteContext) -> HandlerResult: try: from je_auto_control.utils.executor.action_executor import execute_files result = execute_files([path]) + except (AutoControlActionException, AutoControlActionNullException, + AutoControlJsonActionException) as error: + # The caller chose the path, so an unreadable file or an action list + # naming a command that does not exist is their input being wrong — + # same reasoning as _reject_bad_action_list, reported the same way. + autocontrol_logger.info("rest execute_file rejected: %r", error) + return 400, {"error": str(error)} except Exception as error: # noqa: BLE001 # pylint: disable=broad-except # reason: REST boundary must always return JSON, never drop the HTTP response autocontrol_logger.error("rest execute_file failed: %r", error) return 500, {"error": "execute_files failed"} diff --git a/je_auto_control/utils/rest_api/rest_openapi.py b/je_auto_control/utils/rest_api/rest_openapi.py index f6747507..11cf6221 100644 --- a/je_auto_control/utils/rest_api/rest_openapi.py +++ b/je_auto_control/utils/rest_api/rest_openapi.py @@ -218,6 +218,13 @@ }, }, }, + "errors": { + "400": ("Malformed body, or a command name the executor does " + "not know. The whole list is checked before anything " + "runs, so nothing was executed; 'unknown_commands' " + "lists every unrecognised name."), + "500": "The action list was valid but the run failed.", + }, }, ("POST", "/execute_file"): { "summary": "Run a JSON action file by absolute path.", @@ -229,6 +236,11 @@ "path": {"type": "string"}, }, }, + "errors": { + "400": ("Missing 'path', or the file is unreadable, is not a " + "valid action list, or names an unknown command."), + "500": "The action file was valid but the run failed.", + }, }, ("POST", "/config/export"): { "summary": "Export AutoControl user config as a JSON bundle.", @@ -367,6 +379,11 @@ def _build_responses(meta: Dict[str, Any]) -> Dict[str, Any]: "description": "Bad request body.", "content": {_JSON_MEDIA_TYPE: {"schema": _error_schema()}}, } + for code, description in meta.get("errors", {}).items(): + responses[code] = { + "description": description, + "content": {_JSON_MEDIA_TYPE: {"schema": _error_schema()}}, + } return responses diff --git a/je_auto_control/utils/rich_clipboard/rich_clipboard.py b/je_auto_control/utils/rich_clipboard/rich_clipboard.py index 0719d486..289d450c 100644 --- a/je_auto_control/utils/rich_clipboard/rich_clipboard.py +++ b/je_auto_control/utils/rich_clipboard/rich_clipboard.py @@ -91,52 +91,34 @@ def get_clipboard_html() -> Optional[str]: def _html_format_id(): - import ctypes - return ctypes.windll.user32.RegisterClipboardFormatW(_HTML_FORMAT_NAME) + from je_auto_control.utils.clipboard.win32_clipboard_api import register_format + return register_format(_HTML_FORMAT_NAME) def _win_set_html(cf_html: bytes, fragment_plaintext: Optional[str]) -> None: - import ctypes - from ctypes import wintypes + """Put CF_HTML on the clipboard, optionally seeding a plain-text fallback. + + The Win32 dance (prototypes, alloc, lock, open) lives in + ``utils/clipboard/win32_clipboard_api.py``; this module only decides *what* + bytes go on the clipboard. It used to hand-roll those calls with ``restype`` + but no ``argtypes``, so every call raised ``OverflowError`` on 64-bit + Windows — the function had never worked. + """ from je_auto_control.utils.clipboard.clipboard import set_clipboard - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - kernel32.GlobalAlloc.restype = wintypes.HGLOBAL - kernel32.GlobalLock.restype = ctypes.c_void_p + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + set_clipboard_format, + ) if fragment_plaintext is not None: set_clipboard(fragment_plaintext) # seeds CF_UNICODETEXT first - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - if fragment_plaintext is None: - user32.EmptyClipboard() - handle = kernel32.GlobalAlloc(0x0002, len(cf_html) + 1) - if not handle: - raise RuntimeError("GlobalAlloc failed") - pointer = kernel32.GlobalLock(handle) - ctypes.memmove(pointer, cf_html + b"\x00", len(cf_html) + 1) - kernel32.GlobalUnlock(handle) - if not user32.SetClipboardData(_html_format_id(), handle): - raise RuntimeError("SetClipboardData(CF_HTML) failed") - finally: - user32.CloseClipboard() + # Keep that seed when there is one: emptying the clipboard here would throw + # the plain-text fallback away again. + set_clipboard_format(_html_format_id(), cf_html + b"\x00", + empty_first=fragment_plaintext is None) def _win_get_html() -> Optional[bytes]: - import ctypes - from ctypes import wintypes - user32, kernel32 = ctypes.windll.user32, ctypes.windll.kernel32 - user32.GetClipboardData.restype = wintypes.HANDLE - kernel32.GlobalLock.restype = ctypes.c_void_p - if not user32.OpenClipboard(None): - raise RuntimeError("OpenClipboard failed") - try: - handle = user32.GetClipboardData(_html_format_id()) - if not handle: - return None - pointer = kernel32.GlobalLock(handle) - size = kernel32.GlobalSize(handle) - data = ctypes.string_at(pointer, size) - kernel32.GlobalUnlock(handle) - return data.split(b"\x00", 1)[0] - finally: - user32.CloseClipboard() + from je_auto_control.utils.clipboard.win32_clipboard_api import ( + get_clipboard_format, + ) + data = get_clipboard_format(_html_format_id()) + return None if data is None else data.split(b"\x00", 1)[0] diff --git a/je_auto_control/utils/smart_waits/waits.py b/je_auto_control/utils/smart_waits/waits.py index 2e08b12f..ce79d247 100644 --- a/je_auto_control/utils/smart_waits/waits.py +++ b/je_auto_control/utils/smart_waits/waits.py @@ -54,15 +54,16 @@ class Frame: def _default_sampler(region: Optional[Sequence[int]]) -> Frame: - """Use PIL ImageGrab to snapshot once. Fails closed on missing dep.""" + """Snapshot once through the platform's grabber. Fails closed on missing dep.""" + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber try: - from PIL import ImageGrab + grabber = image_grabber() except ImportError as error: raise RuntimeError( "Smart waits require Pillow for screen capture.", ) from error bbox = tuple(int(v) for v in region) if region else None - image = ImageGrab.grab(bbox=bbox).convert("RGB") + image = grabber.grab(bbox=bbox).convert("RGB") return Frame(width=image.width, height=image.height, pixels=image.tobytes()) diff --git a/je_auto_control/utils/visual_regression/compare.py b/je_auto_control/utils/visual_regression/compare.py index 9a4d52f0..d26d5f1b 100644 --- a/je_auto_control/utils/visual_regression/compare.py +++ b/je_auto_control/utils/visual_regression/compare.py @@ -89,13 +89,14 @@ def take_golden(path, def _grab(region: Optional[Sequence[int]]) -> Image.Image: - """Screen capture via PIL.ImageGrab; raises if not available.""" - from PIL import ImageGrab + """Screen capture via the platform's grabber; raises if not available.""" + from je_auto_control.utils.cv2_utils.screen_grabber import image_grabber + grabber = image_grabber() if region is not None: x, y, width, height = (int(v) for v in region) - return ImageGrab.grab(bbox=(x, y, x + width, y + height), - all_screens=True) - return ImageGrab.grab(all_screens=True) + return grabber.grab(bbox=(x, y, x + width, y + height), + all_screens=True) + return grabber.grab(all_screens=True) def image_difference(actual: Image.Image, expected: Image.Image, diff --git a/je_auto_control/utils/window_capture/window_capture.py b/je_auto_control/utils/window_capture/window_capture.py index 56053bb2..30fdf3d2 100644 --- a/je_auto_control/utils/window_capture/window_capture.py +++ b/je_auto_control/utils/window_capture/window_capture.py @@ -71,8 +71,16 @@ def capture_window(title: str, output_path: Union[str, Path], *, def _default_lister() -> List[Tuple[int, str]]: + """Titled windows only — an untitled one cannot be restored. + + ``restore_window_layout`` addresses a window by its title, so an entry with + a blank one is skipped there; saving them anyway made the two disagree on + how many windows a layout contains. On a real desktop that is roughly half + the entries (measured here: 28 saved, 15 restorable), so a caller reporting + the saved count was over-promising by a factor of two. + """ from je_auto_control.wrapper.auto_control_window import list_windows - return list_windows() + return list_windows(titled_only=True) def save_window_layout(path: Optional[Union[str, Path]] = None, *, diff --git a/je_auto_control/windows/screen/win32_screen.py b/je_auto_control/windows/screen/win32_screen.py index 88a5769d..e0aa268a 100644 --- a/je_auto_control/windows/screen/win32_screen.py +++ b/je_auto_control/windows/screen/win32_screen.py @@ -9,11 +9,47 @@ raise AutoControlException(windows_import_error_message) import ctypes +from ctypes import wintypes -# 初始化 Win32 API 函式 Initialize Win32 API functions -_user32 = ctypes.windll.user32 -_user32.SetProcessDPIAware() # 確保 DPI 感知，避免座標偏移 -_gdi32 = ctypes.windll.gdi32 +# 這個模組持有自己的 user32 / gdi32 handle，而不是共用 `ctypes.windll`： +# argtypes/restype 是設在**函式物件**上的，共用 handle 會讓這裡的原型外溢到 +# 別的呼叫者（`utils/window_capture/` 就用自己的 RECT 呼叫 GetWindowRect）。 +# +# This module owns its user32 / gdi32 handles rather than sharing +# ``ctypes.windll``: prototypes live on the function objects, so a shared handle +# would leak these declarations into every other caller in the process. +_user32 = ctypes.WinDLL("user32", use_last_error=True) +_gdi32 = ctypes.WinDLL("gdi32", use_last_error=True) + +# HDC 是**指標寬度**的 handle。ctypes 預設把回傳值與參數當成 c_int，在 64 位元 +# Windows 上會截斷——`GetDC` 回來就已經是壞的，再傳給 `GetPixel` / `ReleaseDC` +# 只會讓錯誤沉默地擴散（顏色讀錯、DC 沒被釋放）。與 `windows_window_manage` 的 +# HWND、剪貼簿的 HGLOBAL 是同一個陷阱，所以每支都明寫原型。 +# +# An HDC is a pointer-width handle and ctypes defaults to ``c_int``, which +# truncates it on 64-bit Windows: the value is already wrong coming out of +# ``GetDC``, and passing it on silently reads the wrong colour and leaks the DC. +_user32.SetProcessDPIAware.argtypes = [] +_user32.SetProcessDPIAware.restype = wintypes.BOOL +_user32.GetSystemMetrics.argtypes = [ctypes.c_int] +_user32.GetSystemMetrics.restype = ctypes.c_int +_user32.GetDC.argtypes = [wintypes.HWND] +_user32.GetDC.restype = wintypes.HDC +_user32.ReleaseDC.argtypes = [wintypes.HWND, wintypes.HDC] +_user32.ReleaseDC.restype = ctypes.c_int +_gdi32.GetPixel.argtypes = [wintypes.HDC, ctypes.c_int, ctypes.c_int] +_gdi32.GetPixel.restype = wintypes.COLORREF + +# 確保 DPI 感知，避免座標偏移。**這是行程層級的副作用**，而它發生在 import 時： +# 一旦設定就無法還原，之後所有 Win32 座標查詢都會拿到實體像素。這正是本模組被 +# import 的理由（螢幕尺寸與取色都必須是實體座標），但呼叫端要知道它會影響整個 +# 行程——擷取與滑鼠座標的換算請走 `utils/monitor_layout`。 +# +# Process-wide and irreversible, and it happens at import time; conversions +# between physical and logical coordinates belong to ``utils/monitor_layout``. +_user32.SetProcessDPIAware() + +_CLR_INVALID = 0xFFFFFFFF def size() -> List[int]: @@ -41,8 +77,8 @@ def get_pixel(x: int, y: int, hwnd: int = 0) -> Tuple[int, int, int]: raise AutoControlException("GetDC failed") try: - pixel = _gdi32.GetPixel(dc, x, y) - if pixel == 0xFFFFFFFF: # GetPixel 失敗時回傳 -1 (0xFFFFFFFF) + pixel = int(_gdi32.GetPixel(dc, int(x), int(y))) + if pixel == _CLR_INVALID: # GetPixel 失敗時回傳 CLR_INVALID raise AutoControlException("GetPixel failed") r = pixel & 0xFF @@ -50,4 +86,4 @@ def get_pixel(x: int, y: int, hwnd: int = 0) -> Tuple[int, int, int]: b = (pixel >> 16) & 0xFF return r, g, b finally: - _user32.ReleaseDC(hwnd, dc) \ No newline at end of file + _user32.ReleaseDC(hwnd, dc) diff --git a/je_auto_control/windows/window/windows_window_manage.py b/je_auto_control/windows/window/windows_window_manage.py index d922fdb2..5ea10eb8 100644 --- a/je_auto_control/windows/window/windows_window_manage.py +++ b/je_auto_control/windows/window/windows_window_manage.py @@ -62,6 +62,9 @@ _user32.MoveWindow.restype = wintypes.BOOL _user32.IsIconic.argtypes = [wintypes.HWND] _user32.IsIconic.restype = wintypes.BOOL +_user32.GetWindowThreadProcessId.argtypes = [wintypes.HWND, + ctypes.POINTER(wintypes.DWORD)] +_user32.GetWindowThreadProcessId.restype = wintypes.DWORD WM_CLOSE = 0x0010 SW_RESTORE = 9 @@ -122,6 +125,27 @@ def get_window_rect(hwnd: int) -> Optional[Tuple[int, int, int, int]]: return (int(rect.left), int(rect.top), int(rect.right), int(rect.bottom)) +def get_window_process_id(hwnd: int) -> int: + """ + 視窗所屬行程的 PID，查不到回 0 + The PID of the process owning the window, or 0 when unavailable + + 這是「畫面上那個視窗是哪個程式」的唯一可靠答案：視窗標題會被程式自己改成 + 任何字串，行程名不會。有了它，呼叫端才能把前景視窗接到 psutil 之類的行程 + 資訊上。 + + This is the only reliable answer to "which program owns that window": a + title is whatever the application decides to display, a process is not. + """ + pid = wintypes.DWORD(0) + # 回傳值是 thread id（0 代表失敗）；pid 由 out 參數帶回來。 + # The return value is the thread id (0 on failure); the pid comes back + # through the out parameter. + if not _user32.GetWindowThreadProcessId(hwnd, byref(pid)): + return 0 + return int(pid.value) + + def is_window_minimized(hwnd: int) -> bool: """ 視窗目前是否被最小化 @@ -206,3 +230,137 @@ def move_window(hwnd: int, x: int, y: int, width: int, height: int, return bool(_user32.MoveWindow(int(hwnd), int(x), int(y), int(width), int(height), bool(repaint))) + + +# --- Posting input to a window without focusing it -------------------------- +# +# Keyboard and mouse messages are delivered to the *control that has focus*, not +# to the top-level window. Posting to the top-level handle is therefore a no-op +# for anything with child controls — measured on Character Map: posting to the +# frame typed nothing, posting to the focused edit typed the character. The +# helpers below resolve that target before posting. + + +class _GUIThreadInfo(ctypes.Structure): + _fields_ = [ + ("cbSize", wintypes.DWORD), + ("flags", wintypes.DWORD), + ("hwndActive", wintypes.HWND), + ("hwndFocus", wintypes.HWND), + ("hwndCapture", wintypes.HWND), + ("hwndMenuOwner", wintypes.HWND), + ("hwndMoveSize", wintypes.HWND), + ("hwndCaret", wintypes.HWND), + ("rcCaret", wintypes.RECT), + ] + + +_user32.GetGUIThreadInfo.argtypes = [wintypes.DWORD, + ctypes.POINTER(_GUIThreadInfo)] +_user32.GetGUIThreadInfo.restype = wintypes.BOOL +_user32.ChildWindowFromPointEx.argtypes = [wintypes.HWND, wintypes.POINT, + wintypes.UINT] +_user32.ChildWindowFromPointEx.restype = wintypes.HWND +_user32.ScreenToClient.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.POINT)] +_user32.ScreenToClient.restype = wintypes.BOOL +_user32.ClientToScreen.argtypes = [wintypes.HWND, ctypes.POINTER(wintypes.POINT)] +_user32.ClientToScreen.restype = wintypes.BOOL + +WM_KEYDOWN = 0x0100 +WM_KEYUP = 0x0101 +WM_CHAR = 0x0102 +_CWP_SKIPINVISIBLE = 0x0001 +_CWP_SKIPTRANSPARENT = 0x0004 +_MOUSE_MESSAGES = { + "left": (0x0201, 0x0202, 0x0001), # WM_LBUTTONDOWN / UP / MK_LBUTTON + "right": (0x0204, 0x0205, 0x0002), # WM_RBUTTON* / MK_RBUTTON + "middle": (0x0207, 0x0208, 0x0010), # WM_MBUTTON* / MK_MBUTTON +} + + +def get_focused_control(hwnd: int) -> int: + """ + 視窗執行緒目前的焦點控制項，問不到就回原本的 hwnd + The control with keyboard focus in the window's thread, else ``hwnd`` + + 走 `GetGUIThreadInfo` 而不是 `AttachThreadInput` + `GetFocus`：後者會把兩個 + 執行緒的輸入狀態接在一起，附帶影響前景與焦點——那正是這條路徑要避免的。 + + Uses ``GetGUIThreadInfo`` rather than ``AttachThreadInput`` + ``GetFocus``: + attaching joins two threads' input state and disturbs focus, which is the + one thing this code path exists to avoid. + """ + thread_id = _user32.GetWindowThreadProcessId(hwnd, None) + if not thread_id: + return int(hwnd) + info = _GUIThreadInfo() + info.cbSize = ctypes.sizeof(_GUIThreadInfo) + if _user32.GetGUIThreadInfo(thread_id, byref(info)) and info.hwndFocus: + return int(info.hwndFocus) + return int(hwnd) + + +def deepest_child_at(hwnd: int, x: int, y: int) -> int: + """ + 視窗內某個點底下最深的子控制項（座標是相對視窗左上角） + The deepest child control under a window-relative point + + 只走 `hwnd` 自己的子樹，所以目標視窗被別的程式蓋住也一樣正確—— + `WindowFromPoint` 會回到最上層那個視窗，對背景操作是錯的答案。 + """ + rect = get_window_rect(hwnd) + if rect is None: + return int(hwnd) + screen = wintypes.POINT(rect[0] + int(x), rect[1] + int(y)) + current = int(hwnd) + for _depth in range(8): # 巢狀控制項的合理上限，兼作迴圈保險 + point = wintypes.POINT(screen.x, screen.y) + _user32.ScreenToClient(current, byref(point)) + child = _user32.ChildWindowFromPointEx( + current, point, _CWP_SKIPINVISIBLE | _CWP_SKIPTRANSPARENT) + if not child or int(child) == current: + return current + current = int(child) + return current + + +def post_key(hwnd: int, keycode: int, character: str = "") -> bool: + """ + 把一次按鍵投遞給視窗（不搶焦點）；回傳訊息是否都排進佇列 + Post one key press to a window without focusing it + + 可列印字元要送 `WM_CHAR`：控制項是靠它拿到文字的，只送 `WM_KEYDOWN` 對多數 + 編輯控制項不會產生任何字。 + + A printable character also needs ``WM_CHAR``: edit controls take their text + from that message, so ``WM_KEYDOWN`` alone types nothing in most of them. + """ + target = get_focused_control(hwnd) + posted = bool(_user32.PostMessageW(target, WM_KEYDOWN, int(keycode), 0)) + if character: + posted = bool(_user32.PostMessageW( + target, WM_CHAR, ord(character[0]), 0)) and posted + posted = bool(_user32.PostMessageW(target, WM_KEYUP, int(keycode), 0)) and posted + return posted + + +def post_click(hwnd: int, button: str, x: int, y: int) -> bool: + """ + 把一次點擊投遞給視窗內 `(x, y)` 的控制項（座標相對視窗左上角，不搶焦點） + Post one click at a window-relative point without focusing the window + """ + key = str(button).lower() + if key not in _MOUSE_MESSAGES: + raise ValueError( + f"unknown mouse button {button!r}; expected one of " + f"{sorted(_MOUSE_MESSAGES)}") + down, up, flag = _MOUSE_MESSAGES[key] + rect = get_window_rect(hwnd) + if rect is None: + return False + target = deepest_child_at(hwnd, x, y) + point = wintypes.POINT(rect[0] + int(x), rect[1] + int(y)) + _user32.ScreenToClient(target, byref(point)) + l_param = ((int(point.y) & 0xFFFF) << 16) | (int(point.x) & 0xFFFF) + posted = bool(_user32.PostMessageW(target, down, flag, l_param)) + return bool(_user32.PostMessageW(target, up, 0, l_param)) and posted diff --git a/je_auto_control/wrapper/auto_control_keyboard.py b/je_auto_control/wrapper/auto_control_keyboard.py index 859cc646..00e1c225 100644 --- a/je_auto_control/wrapper/auto_control_keyboard.py +++ b/je_auto_control/wrapper/auto_control_keyboard.py @@ -1,4 +1,5 @@ import sys +import warnings from typing import Optional, Union, Tuple from je_auto_control.utils.exception.exception_tags import ( @@ -231,33 +232,41 @@ def hotkey(key_code_list: list, is_shift: bool = False) -> Optional[Tuple[str, s def send_key_event_to_window(window_title: str, keycode: Union[int, str]) -> None: """ - 將鍵盤事件送到指定視窗 - Send a key event to a specific window - - :param window_title: 視窗標題 Window title + 將鍵盤事件送到指定視窗（**已棄用**，改用 ``post_key_to_window``） + Send a key event to a specific window. **Deprecated** — use + ``je_auto_control.post_key_to_window``. + + 這支原本把訊息投遞給**頂層視窗**，但鍵盤訊息是送給**有焦點的子控制項**的， + 所以對任何有子控制項的程式都等於什麼都沒做——而且照樣回報成功。實測（字元 + 對應表）：投遞給外框，一個字都沒進去；投遞給焦點控制項，字就進去了。現在 + 轉呼叫 ``post_key_to_window``，行為因此**改變**（會真的作用），並發出 + ``DeprecationWarning``。視窗標題也跟著改成**片段比對**，與其餘視窗函式一致。 + + This posted to the top-level frame, but keyboard messages go to the control + that *has focus*: it silently did nothing in any application with child + controls while still reporting success. It now delegates to + ``post_key_to_window``, so the behaviour changes — it works — and the title + is matched as a substring like every other window function. + + :param window_title: 視窗標題片段 Window title substring :param keycode: 鍵盤代碼或字串 Keycode or string """ + warnings.warn( + "send_key_event_to_window is deprecated; use post_key_to_window. The " + "old implementation posted to the top-level frame and silently did " + "nothing for windows with child controls.", + DeprecationWarning, stacklevel=2, + ) autocontrol_logger.info(f"send_key_event_to_window, window={window_title}, keycode={keycode}") + if sys.platform == "darwin": + return + from je_auto_control.wrapper.auto_control_window import post_key_to_window try: - # macOS 不支援直接送鍵盤事件 - if sys.platform == "darwin": - return - - # 解析 keycode Resolve keycode - if isinstance(keycode, int): - get_key_code = keycode - else: - get_key_code = keyboard_keys_table.get(keycode) - if get_key_code is None: - raise AutoControlKeyboardException(f"Key not found: {keycode}") - - # 呼叫底層 API Send event - keyboard.send_key_event_to_window(window_title, keycode=get_key_code) - - # 紀錄動作 Record action - record_action_to_list("send_key_event_to_window", {"window_title": window_title, "keycode": get_key_code}) - - except (OSError, RuntimeError, AttributeError, TypeError, ValueError) as error: + posted = post_key_to_window(window_title, keycode) + record_action_to_list( + "send_key_event_to_window", + {"window_title": window_title, "keycode": keycode, "posted": posted}) + except Exception as error: # noqa: BLE001 - preserved contract: never raises record_action_to_list("send_key_event_to_window", {"window_title": window_title, "keycode": keycode}, repr(error)) autocontrol_logger.error( f"send_key_event_to_window failed, window={window_title}, keycode={keycode}, error={repr(error)}" diff --git a/je_auto_control/wrapper/auto_control_mouse.py b/je_auto_control/wrapper/auto_control_mouse.py index 7e5e0a02..ab97cb09 100644 --- a/je_auto_control/wrapper/auto_control_mouse.py +++ b/je_auto_control/wrapper/auto_control_mouse.py @@ -1,5 +1,6 @@ import ctypes import sys +import warnings from typing import Tuple, Union from je_auto_control.utils.exception.exception_tags import ( @@ -279,25 +280,67 @@ def mouse_scroll(scroll_value: int, x: int = None, y: int = None, def send_mouse_event_to_window(window, mouse_keycode: Union[int, str], x: int = None, y: int = None) -> None: """ - 將滑鼠事件送到指定視窗 - Send mouse event to a specific window - - :param window: 視窗 handle Window handle + 將滑鼠事件送到指定視窗（**已棄用**，改用 ``post_click_to_window``） + Send mouse event to a specific window. **Deprecated** — use + ``je_auto_control.post_click_to_window``. + + 原本把訊息投遞給傳進來的那個 handle，也就是**頂層視窗**；但點擊要送給座標 + 底下的**子控制項**、而且座標要換算成那個控制項的 client 座標，否則點不到 + 任何東西。這支現在轉呼叫修好的路徑：`window` 傳字串就當**標題片段**（與其餘 + 視窗函式一致），傳整數仍當 hwnd（維持舊呼叫端的型別）。 + + Posted to whatever handle it was given — the top-level frame — but a click + belongs to the child control under the point, in that control's client + coordinates. It now delegates to the fixed path: a string ``window`` is a + title substring (consistent with every other window function), an int is + still an hwnd. + + :param window: 視窗 handle 或標題片段 Window handle or title substring :param mouse_keycode: 滑鼠按鍵代碼 Mouse keycode - :param x: X 座標 X position + :param x: X 座標（相對視窗左上角）X position, relative to the window :param y: Y 座標 Y position """ + warnings.warn( + "send_mouse_event_to_window is deprecated; use post_click_to_window. " + "The old implementation posted to the top-level frame, so the click " + "landed on nothing in any window with child controls.", + DeprecationWarning, stacklevel=2, + ) autocontrol_logger.info(f"send_mouse_event_to_window, window={window}, keycode={mouse_keycode}, x={x}, y={y}") param = {"window": window, "keycode": mouse_keycode, "x": x, "y": y} + if sys.platform == "darwin": + autocontrol_logger.warning("send_mouse_event_to_window not supported on macOS") + return try: - if sys.platform == "darwin": - autocontrol_logger.warning("send_mouse_event_to_window not supported on macOS") - return - - mouse_keycode, x, y = mouse_preprocess(mouse_keycode, x, y) - mouse.send_mouse_event_to_window(window, mouse_keycode=mouse_keycode, x=x, y=y) - record_action_to_list("send_mouse_event_to_window", param) + button = _button_name_for_post(mouse_keycode) + if isinstance(window, str): + from je_auto_control.wrapper.auto_control_window import ( + post_click_to_window, + ) + posted = post_click_to_window(window, button, int(x or 0), int(y or 0)) + else: + from je_auto_control.windows.window import windows_window_manage as wm + posted = wm.post_click(int(window), button, int(x or 0), int(y or 0)) + record_action_to_list("send_mouse_event_to_window", {**param, "posted": posted}) - except (OSError, RuntimeError, AttributeError, TypeError, ValueError) as error: + except Exception as error: # noqa: BLE001 - preserved contract: never raises record_action_to_list("send_mouse_event_to_window", param, repr(error)) autocontrol_logger.error(f"send_mouse_event_to_window failed: {repr(error)}") + + +def _button_name_for_post(mouse_keycode: Union[int, str]) -> str: + """把舊介面收的按鍵代碼轉成投遞路徑用的按鍵名。 + + 舊介面同時吃名稱（``mouse_left``）與底層代碼元組；後者反查回名稱，查不到就 + 當左鍵並記一筆——這條是相容路徑，不值得為了它讓呼叫端爆掉。 + """ + if isinstance(mouse_keycode, str): + name = mouse_keycode.lower() + return name[len("mouse_"):] if name.startswith("mouse_") else name + for name, code in mouse_keys_table.items(): + if code == mouse_keycode: + return name[len("mouse_"):] + autocontrol_logger.warning( + "send_mouse_event_to_window: unknown keycode %r, assuming left", + mouse_keycode) + return "left" diff --git a/je_auto_control/wrapper/auto_control_window.py b/je_auto_control/wrapper/auto_control_window.py index 9469f727..20b20066 100644 --- a/je_auto_control/wrapper/auto_control_window.py +++ b/je_auto_control/wrapper/auto_control_window.py @@ -5,7 +5,7 @@ """ import sys import time -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, Union from je_auto_control.utils.exception.exceptions import AutoControlActionException from je_auto_control.utils.logging.logging_instance import autocontrol_logger @@ -122,6 +122,125 @@ def foreground_window() -> Optional[Tuple[int, str]]: return hwnd, titles.get(hwnd, "") +def post_key_to_window(title_substring: str, key: Union[int, str], + case_sensitive: bool = False) -> bool: + """Type one key into a window **without focusing it**. ``False`` if no match. + + Unlike :func:`send_key_event_to_window` this resolves the window by + substring (like every other function here) and posts to the control that + actually has keyboard focus. Posting to the top-level frame — what the older + function does — types nothing in any application with child controls; + measured on Character Map, the frame swallowed the key and the focused edit + accepted it. + + **This is best effort, not input.** ``PostMessage`` returning true means the + message reached a queue, not that the application acted on it: games, + anything reading raw input, and applications that check whether they are in + the foreground all ignore posted messages. Callers must say so rather than + reporting success. + """ + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + keycode, character = _resolve_key(key) + return wm.post_key(hit[0], keycode, character) + + +def post_click_to_window(title_substring: str, button: str = "left", + x: int = 0, y: int = 0, + case_sensitive: bool = False) -> bool: + """Click inside a window **without focusing it**; ``False`` if no match. + + ``x`` / ``y`` are relative to the window's top-left corner. The click is + posted to the deepest child control under that point, in that control's own + client coordinates — a click posted to the frame lands nowhere. Same + best-effort caveat as :func:`post_key_to_window`. + """ + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return False + from je_auto_control.windows.window import windows_window_manage as wm + return wm.post_click(hit[0], _mouse_button_name(button), int(x), int(y)) + + +def _resolve_key(key: Union[int, str]) -> Tuple[int, str]: + """``(virtual key code, character to also post as WM_CHAR)``.""" + if isinstance(key, int): + return int(key), "" + name = str(key) + from je_auto_control.wrapper.platform_wrapper import keyboard_keys_table + keycode = keyboard_keys_table.get(name) + if keycode is None: + raise AutoControlActionException(f"unknown key name: {name!r}") + # A one-character key is text: edit controls take their content from + # WM_CHAR, so posting only the virtual-key messages types nothing. + character = name if len(name) == 1 and name.isprintable() else "" + return int(keycode), character + + +def _mouse_button_name(button: str) -> str: + """Accept both plain and ``mouse_``-prefixed button names.""" + name = str(button).lower() + return name[len("mouse_"):] if name.startswith("mouse_") else name + + +def foreground_window_process_id() -> Optional[int]: + """The PID owning the foreground window, or ``None``. + + A window title is not identity: applications rewrite theirs at will, and + unrelated programs share titles like ``Settings``. Callers that need to know + *which program* the user is actually in front of — presence reporting, + activity probes, "is my automation target focused" — have to go through the + process id. + """ + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + hwnd = wm.get_foreground_window() + if not hwnd: + return None + return wm.get_window_process_id(hwnd) or None + + +def window_process_id(title_substring: str, + case_sensitive: bool = False) -> Optional[int]: + """The PID owning the first window whose title contains the substring.""" + _require_windows() + hit = find_window(title_substring, case_sensitive) + if hit is None: + return None + from je_auto_control.windows.window import windows_window_manage as wm + return wm.get_window_process_id(hit[0]) or None + + +def windows_for_process_id(pid: int, + titled_only: bool = False) -> List[Tuple[int, str]]: + """Every visible top-level window owned by ``pid``. + + Titles cannot address a multi-process application: a browser's windows are + named after whatever page they show, and several of its processes have no + window at all. Ownership is the stable key. + """ + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + target = int(pid) + return [(hwnd, title) for hwnd, title in list_windows(titled_only) + if wm.get_window_process_id(hwnd) == target] + + +def minimize_windows_for_process(pid: int) -> int: + """Minimise every visible top-level window owned by ``pid``; return the count.""" + _require_windows() + from je_auto_control.windows.window import windows_window_manage as wm + minimized = 0 + for hwnd, _title in windows_for_process_id(pid): + if wm.minimize_window(hwnd): + minimized += 1 + return minimized + + def window_rect(title_substring: str, case_sensitive: bool = False, ) -> Optional[Tuple[int, int, int, int]]: diff --git a/test/unit_test/headless/test_clipboard_win32_prototypes.py b/test/unit_test/headless/test_clipboard_win32_prototypes.py new file mode 100644 index 00000000..f7060a9c --- /dev/null +++ b/test/unit_test/headless/test_clipboard_win32_prototypes.py @@ -0,0 +1,193 @@ +"""Every clipboard format must survive a real round-trip, and no module may +hand-roll the Win32 calls again. No Qt. + +Three ``set_clipboard_*`` functions were broken on 64-bit Windows for their +whole existence — HTML, RTF and CSV — and so was the file-drop writer. All four +failed the same way: the module declared ``restype`` but not ``argtypes``, so +ctypes passed a pointer-width memory handle as ``c_int`` and ``GlobalLock`` +raised ``OverflowError: int too long to convert``. Pure-function tests could not +see it (the byte packing was always correct) and no test called the Win32 half. + +So this file tests the half that was untested: the actual clipboard. It skips +when the clipboard cannot be opened at all — a locked workstation, a session +without a window station, or a non-Windows CI runner — rather than reporting a +failure the environment made inevitable. + +That real clipboard is machine-global, which used to make these tests depend on +what every *other* process on the box was doing: one write from anything else +between our write and our read failed the run, and during a parallel Docker +build one did. :func:`_round_trip` closes that hole without giving up the real +Win32 calls — see its docstring for why faking the backend instead would delete +the only coverage this file has. +""" +import sys + +import pytest + +_WINDOWS = sys.platform.startswith("win") +pytestmark = pytest.mark.skipif(not _WINDOWS, reason="Windows clipboard only") + +# How many times a stolen round-trip is worth re-running before giving up. +_ATTEMPTS = 4 + + +def _clipboard_available() -> bool: + from je_auto_control.utils.clipboard.clipboard import get_clipboard + try: + get_clipboard() + return True + except Exception: # noqa: BLE001 - locked desktop / no window station + return False + + +def _sequence_number() -> int: + """Win32's clipboard change counter for this window station. + + It moves on every *modification*, by any process, and never on a read — + which is precisely the signal needed to tell "somebody stole my clipboard" + from "my writer is broken". + """ + import ctypes + from ctypes import wintypes + + user32 = ctypes.WinDLL("user32", use_last_error=True) + user32.GetClipboardSequenceNumber.argtypes = [] + user32.GetClipboardSequenceNumber.restype = wintypes.DWORD + return int(user32.GetClipboardSequenceNumber()) + + +def _round_trip(write, read): + """Run ``write`` then ``read`` as one uninterrupted pair; return the value. + + Faking the clipboard backend would make this file deterministic by + deleting the only thing it tests — the Win32 half, which is where all four + historical bugs were and which no pure-function test could reach. So the + calls stay real and the *interference* is detected instead: if the + sequence number has not moved between the end of our write and the end of + our read, nothing else wrote in that window, so the bytes we read are the + bytes we wrote and the assertion that follows is about our code alone. + + Nothing that looks like a bug is retried away: ``write`` raising is the + exact shape of the regression this file exists to catch, so it propagates + on the first attempt. The other half of the race — a clipboard another + process is holding *open* — is not handled here at all, because + ``win32_clipboard_api.open_clipboard`` waits that out for every caller of + the library, not just for this test. + """ + for _attempt in range(_ATTEMPTS): + write() + stamp = _sequence_number() + value = read() + if _sequence_number() == stamp: + return value + pytest.skip("another process kept overwriting the clipboard") + return None # unreachable; keeps the return type honest for linters + + +@pytest.fixture +def clipboard(): + """Skip when unusable, and put the user's clipboard back afterwards.""" + if not _clipboard_available(): + pytest.skip("clipboard cannot be opened in this session") + from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, set_clipboard, + ) + saved = get_clipboard() + try: + yield + finally: + try: + set_clipboard(saved) + except Exception: # noqa: BLE001 # nosec B110 + pass + + +def test_text_round_trip(clipboard): + from je_auto_control.utils.clipboard.clipboard import ( + get_clipboard, set_clipboard, + ) + read_back = _round_trip(lambda: set_clipboard("round-trip probe"), + get_clipboard) + assert read_back == "round-trip probe" + + +def test_html_round_trip(clipboard): + from je_auto_control.utils.rich_clipboard.rich_clipboard import ( + get_clipboard_html, set_clipboard_html, + ) + read_back = _round_trip(lambda: set_clipboard_html("hi"), + get_clipboard_html) + assert "hi" in (read_back or "") + + +def test_rtf_round_trip(clipboard): + from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( + build_rtf, get_clipboard_rtf, set_clipboard_rtf, + ) + read_back = _round_trip(lambda: set_clipboard_rtf(build_rtf("hello")), + get_clipboard_rtf) + assert "hello" in (read_back or "") + + +def test_csv_round_trip(clipboard): + from je_auto_control.utils.clipboard_rich_formats.clipboard_rich_formats import ( + get_clipboard_csv, set_clipboard_csv, + ) + read_back = _round_trip( + lambda: set_clipboard_csv([["a", "b"], ["c", "d"]]), + get_clipboard_csv, + ) + assert read_back == [["a", "b"], ["c", "d"]] + + +def test_file_list_round_trip(clipboard, tmp_path): + from je_auto_control.utils.clipboard_files.clipboard_files import ( + get_clipboard_files, set_clipboard_files, + ) + one = tmp_path / "one.png" + one.write_bytes(b"x") + read_back = _round_trip(lambda: set_clipboard_files([str(one)]), + get_clipboard_files) + assert read_back == [str(one)] + + +def test_format_enumeration_sees_what_was_written(clipboard): + from je_auto_control.utils.clipboard.clipboard import set_clipboard + from je_auto_control.utils.clipboard_formats.clipboard_formats import ( + clipboard_formats, + ) + summary = _round_trip(lambda: set_clipboard("text only"), clipboard_formats) + assert summary["has_text"] is True + assert summary["has_files"] is False + + +# --- static invariant ------------------------------------------------------- + +_HANDLE_CALLS = ("GlobalLock", "GlobalAlloc", "GlobalSize", "SetClipboardData", + "GetClipboardData") +_SHARED_MODULE = "win32_clipboard_api" + + +def test_no_module_hand_rolls_the_clipboard_calls_without_prototypes(): + """A handle call needs declared ``argtypes`` — in the file or via the shared module. + + This is the invariant the three broken modules violated. Any new clipboard + code either goes through ``utils/clipboard/win32_clipboard_api.py`` or + declares the prototypes itself; ``restype``-only is what shipped a function + that had never once worked. + """ + import pathlib + + root = pathlib.Path(__file__).resolve().parents[3] / "je_auto_control" + offenders = [] + for path in root.rglob("*.py"): + text = path.read_text(encoding="utf-8", errors="ignore") + if not any(call in text for call in _HANDLE_CALLS): + continue + if _SHARED_MODULE in text or "argtypes" in text: + continue + offenders.append(str(path.relative_to(root))) + assert not offenders, ( + "these modules call Win32 handle APIs without declaring argtypes and " + "without going through the shared clipboard module: " + repr(offenders) + ) diff --git a/test/unit_test/headless/test_diagnostics.py b/test/unit_test/headless/test_diagnostics.py index 2f207e35..4a7d4c64 100644 --- a/test/unit_test/headless/test_diagnostics.py +++ b/test/unit_test/headless/test_diagnostics.py @@ -1,6 +1,7 @@ """Tests for the system diagnostics runner (round 28).""" import subprocess import sys +from unittest.mock import patch from je_auto_control.utils.diagnostics.diagnostics import ( Check, DiagnosticsReport, run_diagnostics, @@ -52,3 +53,38 @@ def test_cli_exits_zero_when_all_green(): # cleanly with one of those codes and emit the summary line. assert completed.returncode in (0, 1), completed.returncode assert "Summary:" in completed.stdout + + +def _wayland_capture_check(tool="grim"): + """Run the capture check with the Wayland grabber and ``tool`` present.""" + from je_auto_control.linux_wayland import capture + from je_auto_control.utils.cv2_utils import screen_grabber + from je_auto_control.utils.diagnostics import diagnostics + with patch.object(screen_grabber, "backend_grab_image", + return_value=lambda *a, **k: None), patch.object(capture, "available_tool", return_value=tool): + return diagnostics._check_screen_capture() + + +def test_wayland_capture_reports_the_tool_it_found(): + check = _wayland_capture_check() + assert check.ok is True + assert check.extra["grabber"] == "wayland" + assert check.extra["tool"] == "grim" + + +def test_wayland_capture_warns_that_the_pointer_may_be_in_the_image(): + """wlroots composites a software cursor into the buffer screencopy hands + back, so a locator can find a pointer-shaped hole in its target. The + bundle that explains that failure has to carry the fact.""" + check = _wayland_capture_check() + assert check.extra["cursor_may_be_captured"] is True + assert "pointer" in check.detail + + +def test_the_generic_grabber_says_nothing_about_the_pointer(): + """BitBlt and Pillow/mss never include it; only Wayland does.""" + from je_auto_control.utils.cv2_utils import screen_grabber + from je_auto_control.utils.diagnostics import diagnostics + with patch.object(screen_grabber, "backend_grab_image", return_value=None): + check = diagnostics._check_screen_capture() + assert check.extra == {"grabber": "generic"} diff --git a/test/unit_test/headless/test_doc_line_counts.py b/test/unit_test/headless/test_doc_line_counts.py new file mode 100644 index 00000000..aa44bdb1 --- /dev/null +++ b/test/unit_test/headless/test_doc_line_counts.py @@ -0,0 +1,350 @@ +"""Every line count in `architecture_explore.md` is measured, not remembered. + +`CLAUDE.md` requires the map's figures to be re-measured rather than adjusted +by hand. The command / MCP-tool / subpackage / example counts have had a gate +in `test_doc_counts.py`; the *line* counts had none, and drifted two ways at +once. Most table rows counted a phantom trailing line — `len(text.split("\\n"))` +reports one more line than a file that ends in a newline actually has, and one +more *per file* for a package — while the §1 totals and the §8 appendix used +the real count. The same subsystem was therefore quoted at two different sizes +in one document, and roughly fifty rows were additionally just stale. + +This module measures the tree and compares. `len(text.splitlines())` is the +convention it enforces everywhere: it is what `wc -l` reports, what the §8 +appendix already used, and what `CLAUDE.md`'s own over-750-lines snippet +counts. + +Run it directly to rewrite every figure in place rather than hand-editing: + + python test/unit_test/headless/test_doc_line_counts.py --fix +""" +from __future__ import annotations + +import pathlib +import re +import sys +from typing import Dict, List, Optional, Tuple + +ROOT = pathlib.Path(__file__).resolve().parents[3] +PACKAGE = ROOT / "je_auto_control" +DOC = ROOT / "architecture_explore.md" + +FIX_COMMAND = "python test/unit_test/headless/test_doc_line_counts.py --fix" + +# The scan scope quoted in the document header, for the §1 totals. +SCAN_SCOPE = ("je_auto_control", "autocontrol-lsp", "autocontrol_driver", + "AutoControl", "exe", "benchmarks") + +# Only tables whose header names 行數 hold line counts; §1's 指標／數值 table +# holds subpackage and command counts that must not be touched. +_LINE_TABLE_HEADERS = ("| 模組 | 行數 | 職責 |", "| 檔案 | 行數 | 職責 |") +_SIZE_TABLE_HEADER = "| 層／子系統 | 檔案數 | 行數 |" + +_ROW = re.compile(r"^\|\s*`([^`]+)`") +_ONE_NUMBER = re.compile(r"^(\|[^|]*\|\s*)([\d,]+)(\s*\|)") +_TWO_NUMBERS = re.compile(r"^(\|[^|]*\|\s*)([\d,]+)(\s*\|\s*)([\d,]+)(\s*\|)") +_SUBSECTION = re.compile(r"^####\s") +_PATH_TICK = re.compile(r"`([^`]+/)`") +_PAREN = re.compile(r"（[^（）]*）") +_COUNT_OF = re.compile(r"(\d[\d,]*)(\s*(?:檔|行))") +_BLOCKQUOTE = re.compile(r"^> (\d[\d,]*) 個套件、約 (\d[\d,]*) 行。") +_METRIC_TOTAL_LINES = re.compile(r"^(\| 程式碼總行數 \| )([\d,]+)( \|)") +_METRIC_TOTAL_FILES = re.compile(r"^(\| Python 模組總數（含周邊子專案） \| )([\d,]+)( \|)") + +# §8 rows that are not a plain path measurement. +_TOP_LEVEL_ROW = "je_auto_control/" # 頂層 3 檔 — not recursive +_REMAINDER = "其餘模組" +_GRAND_TOTAL = "**總計**" + + +def _lines(path: pathlib.Path) -> int: + return len(path.read_text(encoding="utf-8", errors="ignore").splitlines()) + + +def _py_files(directory: pathlib.Path) -> List[pathlib.Path]: + return [p for p in sorted(directory.rglob("*.py")) + if "__pycache__" not in p.parts] + + +def _measure(target: pathlib.Path, *, recursive: bool = True) -> Tuple[int, int]: + """``(files, lines)`` for a directory tree or a single file.""" + if target.is_file(): + return 1, _lines(target) + files = _py_files(target) if recursive else sorted(target.glob("*.py")) + return len(files), sum(_lines(p) for p in files) + + +def _resolve(name: str, context: Optional[str]) -> Optional[pathlib.Path]: + """Find what a backticked name in the document refers to, or ``None``. + + Names are written relative to ``je_auto_control/`` in most tables, relative + to the repo root in a few (``autocontrol-lsp/``), and as a bare file name + in the §5.4.17 file tables — where the enclosing ``#### `utils/x/``` header + supplies the package. + """ + candidates = [PACKAGE / name, ROOT / name] + if context and "/" not in name: + candidates.insert(0, PACKAGE / context / name) + for candidate in candidates: + if candidate.is_file() or candidate.is_dir(): + return candidate + return None + + +def _replace_numbers(line: str, values: Tuple[int, ...]) -> str: + """Rewrite the one or two numeric cells of a table row.""" + if len(values) == 2: + return _TWO_NUMBERS.sub( + lambda m: f"{m.group(1)}{values[0]:,}{m.group(3)}{values[1]:,}{m.group(5)}", + line, count=1) + return _ONE_NUMBER.sub( + lambda m: f"{m.group(1)}{values[0]:,}{m.group(3)}", line, count=1) + + +def _header_group_package(line: str, group: str) -> Optional[str]: + """The package a ``（…）`` group on a heading describes, if any. + + The path is written either inside the group (``（`windows/`，23 檔…）``) or + immediately before it (`` `utils/usb/`（4,238 行） ``). + """ + inside = _PATH_TICK.search(group) + if inside: + return inside.group(1) + outside = _PATH_TICK.findall(line.split(group)[0]) + return outside[-1] if outside else None + + +def _rewrite_header(line: str) -> str: + """Rewrite the ``（… 檔／… 行）`` figures carried by a `####` heading.""" + result = line + for group in _PAREN.findall(line): + name = _header_group_package(result, group) + target = _resolve(name, None) if name else None + if target is None or not target.is_dir(): + continue + files, total = _measure(target) + result = result.replace(group, _fill_counts(group, files, total), 1) + return result + + +def _fill_counts(text: str, files: int, lines: int) -> str: + """Replace every ``N 檔`` with ``files`` and every ``N 行`` with ``lines``.""" + def _sub(match: "re.Match[str]") -> str: + unit = match.group(2) + return f"{files if '檔' in unit else lines:,}{unit}" + return _COUNT_OF.sub(_sub, text) + + +def _row_name(line: str) -> Optional[str]: + """The first cell of a table row, if it names something measurable.""" + match = _ROW.match(line) + if match: + return match.group(1) + if _REMAINDER in line or _GRAND_TOTAL in line: + return line.split("|")[1].strip() + return None + + +def _table_rows(doc_lines: List[str]) -> Dict[int, Tuple[str, Optional[str]]]: + """Map each measurable table row's index to ``(name, package context)``. + + Only tables headed 行數 are collected. §1's 指標／數值 table holds + subpackage and command counts, which belong to ``test_doc_counts.py`` and + must not be overwritten with a line count. + """ + rows: Dict[int, Tuple[str, Optional[str]]] = {} + context: Optional[str] = None + header: Optional[str] = None + for index, line in enumerate(doc_lines): + if _SUBSECTION.match(line): + paths = _PATH_TICK.findall(line) + context = paths[0].rstrip("/") if paths else None + if not line.startswith("|"): + header = None + continue + stripped = line.rstrip() + if stripped in _LINE_TABLE_HEADERS or stripped == _SIZE_TABLE_HEADER: + header = stripped + continue + name = _row_name(line) if header is not None else None + if name is not None: + rows[index] = (name, context) + return rows + + +def _section_totals(doc_lines: List[str], start: int) -> Optional[Tuple[int, int]]: + """``(packages, lines)`` of the theme table that follows a blockquote.""" + index = start + while index < len(doc_lines) and doc_lines[index].rstrip() not in _LINE_TABLE_HEADERS: + if doc_lines[index].startswith("### "): + return None + index += 1 + if index >= len(doc_lines): + return None + packages = total = 0 + for line in doc_lines[index + 2:]: + if not line.startswith("|"): + break + match = _ROW.match(line) + if not match: + continue + target = _resolve(match.group(1), None) + if target is None: + continue + packages += 1 + total += _measure(target)[1] + return packages, total + + +def _scope_totals() -> Tuple[int, int]: + files = total = 0 + for name in SCAN_SCOPE: + directory = ROOT / name + if not directory.is_dir(): + continue + count, lines = _measure(directory) + files += count + total += lines + return files, total + + +def _rewrite_table_rows(doc_lines: List[str]) -> Dict[str, object]: + """Rewrite every measurable row; report what §8's derived rows need.""" + named_files = named_lines = 0 + derived: Dict[str, object] = {"remainder": None, "total": None} + for index, (name, context) in _table_rows(doc_lines).items(): + line = doc_lines[index] + if name.startswith(_REMAINDER): + derived["remainder"] = index + continue + if name == _GRAND_TOTAL: + derived["total"] = index + continue + two_columns = _TWO_NUMBERS.match(line) is not None + target = _resolve(name, context) + if target is None: + continue + files, total = _measure( + target, recursive=not (two_columns and name == _TOP_LEVEL_ROW)) + if two_columns: + named_files += files + named_lines += total + doc_lines[index] = _replace_numbers( + line, (files, total) if two_columns else (total,)) + derived["named"] = (named_files, named_lines) + return derived + + +def _rewrite_appendix_totals(doc_lines: List[str], + derived: Dict[str, object]) -> None: + """Fill in §8's remainder and grand-total rows. + + Neither is measured on its own: the remainder is whatever the named rows + did not account for, and the total is the appendix's whole scope. Deriving + them is what makes the column actually add up, which it did not before. + """ + files, lines = _measure(PACKAGE) + lsp_files, lsp_lines = _measure(ROOT / "autocontrol-lsp") + files, lines = files + lsp_files, lines + lsp_lines + named_files, named_lines = derived["named"] # type: ignore[misc] + remainder = derived["remainder"] + if remainder is not None: + doc_lines[remainder] = _replace_numbers( + doc_lines[remainder], (files - named_files, lines - named_lines)) + total = derived["total"] + if total is not None: + doc_lines[total] = re.sub( + r"\*\*([\d,]+)\*\* \| \*\*([\d,]+)\*\*", + f"**{files:,}** | **{lines:,}**", doc_lines[total], count=1) + + +def _rewrite_headings(doc_lines: List[str]) -> None: + """Rewrite `####` headings and the ``> N 個套件、約 L 行。`` summaries.""" + for index, line in enumerate(doc_lines): + if _SUBSECTION.match(line): + doc_lines[index] = _rewrite_header(line) + elif _BLOCKQUOTE.match(line): + totals = _section_totals(doc_lines, index) + if totals is not None: + doc_lines[index] = f"> {totals[0]:,} 個套件、約 {totals[1]:,} 行。" + + +def _rewrite_scope_metrics(doc_lines: List[str]) -> None: + """Rewrite §1's module and line totals over the documented scan scope.""" + files, lines = _scope_totals() + for index, line in enumerate(doc_lines): + line = _METRIC_TOTAL_LINES.sub( + lambda m: f"{m.group(1)}{lines:,}{m.group(3)}", line) + doc_lines[index] = _METRIC_TOTAL_FILES.sub( + lambda m: f"{m.group(1)}{files:,}{m.group(3)}", line) + + +def rewrite(text: str) -> str: + """Return ``text`` with every measurable figure replaced by a measurement.""" + doc_lines = text.split("\n") + _rewrite_appendix_totals(doc_lines, _rewrite_table_rows(doc_lines)) + _rewrite_headings(doc_lines) + _rewrite_scope_metrics(doc_lines) + return "\n".join(doc_lines) + + +def mismatches() -> List[str]: + """One line per figure that does not match the tree. + + Only the figures are reported, never the surrounding prose: the document is + Chinese and a console on a cp950 code page cannot print it. + """ + original = DOC.read_text(encoding="utf-8") + fixed = rewrite(original) + if original == fixed: + return [] + found = [] + for number, (before, after) in enumerate( + zip(original.split("\n"), fixed.split("\n")), 1): + if before == after: + continue + was = " / ".join(re.findall(r"[\d,]{2,}", before)) or "?" + now = " / ".join(re.findall(r"[\d,]{2,}", after)) or "?" + found.append(f"line {number}: {was} -> {now}") + return found + + +def test_every_quoted_line_count_matches_the_tree(): + """`architecture_explore.md` quotes measurements, so they have to measure.""" + found = mismatches() + preview = "\n".join(found[:20]) + more = f"\n… and {len(found) - 20} more" if len(found) > 20 else "" + assert not found, ( + f"{len(found)} line-count figures in architecture_explore.md no longer " + f"match the tree. CLAUDE.md says to re-measure rather than adjust by " + f"hand:\n\n {FIX_COMMAND}\n\n{preview}{more}" + ) + + +def test_the_rewriter_is_idempotent(): + """A second pass must be a no-op, or --fix would churn the document.""" + once = rewrite(DOC.read_text(encoding="utf-8")) + assert rewrite(once) == once + + +def _main(argv: List[str]) -> int: + if "--fix" not in argv: + found = mismatches() + for entry in found: + print(entry) + print(f"{len(found)} figures out of date; rerun with --fix" + if found else "every figure matches the tree") + return 1 if found else 0 + original = DOC.read_text(encoding="utf-8") + fixed = rewrite(original) + if original == fixed: + print("every figure already matches the tree") + return 0 + DOC.write_text(fixed, encoding="utf-8") + changed = sum(1 for a, b in zip(original.split("\n"), fixed.split("\n")) if a != b) + print(f"rewrote {changed} lines in {DOC.name}") + return 0 + + +if __name__ == "__main__": + sys.exit(_main(sys.argv[1:])) diff --git a/test/unit_test/headless/test_docker_artifacts.py b/test/unit_test/headless/test_docker_artifacts.py index 7bb24cf8..5abf85c0 100644 --- a/test/unit_test/headless/test_docker_artifacts.py +++ b/test/unit_test/headless/test_docker_artifacts.py @@ -10,7 +10,11 @@ import pytest -_DOCKER_DIR = Path(__file__).resolve().parents[3] / "docker" +_REPO_ROOT = Path(__file__).resolve().parents[3] +_DOCKER_DIR = _REPO_ROOT / "docker" +#: Spelled from ``chr`` so this file stays byte-identical under any +#: checkout normalisation that might rewrite a literal in it. +CRLF = (chr(13) + chr(10)).encode() def test_dockerfile_exists_and_uses_python_base(): @@ -43,13 +47,47 @@ def test_compose_file_declares_three_services(): assert "8765:8765" in raw +def test_dockerignore_sits_at_the_build_context_root(): + """Docker reads ``.dockerignore`` from the build *context* root only. + + Every documented build passes the repository root as the context + (``docker build -f docker/Dockerfile .``), so a copy next to the + Dockerfile is never read: the whole tree -- .git, .venv, test/ -- is + shipped to the daemon and the exclusions do nothing. + """ + assert (_REPO_ROOT / ".dockerignore").is_file(), ( + ".dockerignore must live at the repository root, not in docker/") + assert not (_DOCKER_DIR / ".dockerignore").exists(), ( + "a docker/.dockerignore is dead weight; Docker never reads it") + + def test_dockerignore_keeps_build_context_lean(): - raw = (_DOCKER_DIR / ".dockerignore").read_text(encoding="utf-8") + raw = (_REPO_ROOT / ".dockerignore").read_text(encoding="utf-8") # The biggest space-wasters should all be excluded. for line in ("test/", "docs/", "__pycache__", "*.egg-info"): assert line in raw, f".dockerignore missing {line}" +@pytest.mark.parametrize("script", [ + "entrypoint.sh", "entrypoint-xfce.sh", "entrypoint-wayland.sh", + "entrypoint-seat.sh", +]) +def test_entrypoints_keep_unix_line_endings(script): + """A CRLF shebang makes an image that builds and then cannot start. + + A carriage return at the end of the shebang sends the kernel looking + for an interpreter whose name ends in one, and every container dies + with the useless ``exec ...: no such file or directory``. + ``.gitattributes`` pins ``*.sh text eol=lf`` so a Windows checkout + cannot reintroduce it; this fails if that pin is ever dropped. + """ + path = _DOCKER_DIR / script + if not path.exists(): # optional variants stay optional + pytest.skip(f"{script} not present") + assert CRLF not in path.read_bytes(), ( + f"{script} has CRLF line endings; its container cannot exec /bin/sh") + + @pytest.mark.parametrize("expected_port", ["9939", "9940", "8765"]) def test_dockerfile_exposes_each_service_port(expected_port): raw = (_DOCKER_DIR / "Dockerfile").read_text(encoding="utf-8") @@ -94,6 +132,113 @@ def test_github_actions_docker_workflow_exists(): assert "pytest" in raw +def test_ydotool_verification_image_and_script_exist(): + """The ydotool path is only verified while these two files are wired up.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.ydotool").read_text( + encoding="utf-8") + # ydotool 1.0 replaced the CLI; trixie ships none and bookworm ships + # 0.1.8, so the image has to pull the one under test from unstable. + assert "sid" in dockerfile + assert "ydotool" in dockerfile + assert "ydotool_verify.py" in dockerfile + + script = (_DOCKER_DIR / "ydotool_verify.py").read_text(encoding="utf-8") + # Reading the kernel device back is the whole point; a version that only + # checks exit codes would pass against the CLI that exits 0 and emits + # nothing, which is the bug this exists for. + assert "/dev/input" in script + assert "input_event" in script + + +def test_ydotool_verification_job_grants_uinput_and_the_input_cgroup(): + """--device cannot cover a node ydotoold creates after the container starts.""" + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "ydotool-verification" in raw + assert "modprobe uinput" in raw + assert "--device /dev/uinput" in raw + assert "c 13:* rmw" in raw + + +def test_seat_verification_image_and_script_exist(): + """The one image where an injected event reaches a compositor. + + The other two Wayland images each hold one half still: the capture image + runs a compositor that consumes no input, the ydotool image reads the + kernel with no compositor. This one needs all four of the settings that + join them, and dropping any of them turns the verification into a + measurement of an empty seat that still passes. + """ + dockerfile = (_DOCKER_DIR / "Dockerfile.seat").read_text(encoding="utf-8") + for setting in ("WLR_BACKENDS=headless,libinput", + "LIBSEAT_BACKEND=builtin", + "SEATD_VTBOUND=0"): + assert setting in dockerfile, f"Dockerfile.seat missing {setting}" + # libinput enumerates through udev, not through /dev, so udevd has to be + # in the image or sway comes up holding nothing. + assert "udev" in dockerfile + assert "seat_verify.py" in dockerfile + + entrypoint = (_DOCKER_DIR / "entrypoint-seat.sh").read_text( + encoding="utf-8") + # Coming up with an empty seat must fail here rather than downstream. + assert "libinput list-devices" in entrypoint + # Both layouts, as the capture image does: the negative-origin one is the + # only one where the layout corner and layout (0, 0) differ. + assert "-1280 0" in entrypoint + + script = (_DOCKER_DIR / "seat_verify.py").read_text(encoding="utf-8") + # grim -c is what makes the cursor visible to a screenshot at all. + assert '"-c"' in script + # And the two findings the image exists to hold. + assert "layout corner" in script + assert "acceleration" in script + + +def test_seat_verification_job_grants_uinput_and_the_input_cgroup(): + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "seat-verification" in raw + assert "docker/Dockerfile.seat" in raw + assert "autocontrol-seat:ci" in raw + + +def test_portal_verification_image_and_scripts_exist(): + """The portal path is only verified while these three files are wired up.""" + dockerfile = (_DOCKER_DIR / "Dockerfile.portal").read_text(encoding="utf-8") + # liboeffis is a separate binary package that libei1 does not depend on, + # so installing libei alone leaves the portal route quietly off. + assert "liboeffis1" in dockerfile + assert "portal_verify.py" in dockerfile + assert "portal_server.py" in dockerfile + # gdbus must stay out: the Screenshot tier speaks D-Bus itself now, and + # this image is the proof that it needs no binary beyond a session bus. + assert "libglib2.0-bin" not in dockerfile + + server = (_DOCKER_DIR / "portal_server.py").read_text(encoding="utf-8") + # A portal that answers only CreateSession is not a portal; the fd handover + # at the end of the dance is the whole point. + for method in ("CreateSession", "SelectDevices", "Start", "ConnectToEIS"): + assert method in server, f"mock portal missing {method}" + assert "UnixFDList" in server, "the mock must really pass a descriptor" + + verify = (_DOCKER_DIR / "portal_verify.py").read_text(encoding="utf-8") + # Refusals are half the value: a portal that says no must fail closed. + for behaviour in ("deny", "stall", "no-fd", "close"): + assert f'"{behaviour}"' in verify, f"no scenario drives {behaviour}" + + +def test_portal_verification_job_runs_the_image(): + raw = (_REPO_ROOT / ".github" / "workflows" / "docker.yml").read_text( + encoding="utf-8", + ) + assert "portal-verification" in raw + assert "docker/Dockerfile.portal" in raw + assert "autocontrol-portal:ci" in raw + + def test_gitlab_template_covers_build_test_smoke_stages(): raw = (_REPO_ROOT / "ci_templates" / ".gitlab-ci.yml").read_text( encoding="utf-8", diff --git a/test/unit_test/headless/test_flow_extensions.py b/test/unit_test/headless/test_flow_extensions.py index 2cc75b74..07dab00c 100644 --- a/test/unit_test/headless/test_flow_extensions.py +++ b/test/unit_test/headless/test_flow_extensions.py @@ -9,7 +9,10 @@ ) from je_auto_control.utils.executor.action_executor import Executor from je_auto_control.utils.executor.flow_control import ( - exec_assert_duration, exec_call_macro, exec_parallel, + exec_call_macro, exec_parallel, +) +from je_auto_control.utils.executor.flow_data_commands import ( + exec_assert_duration, ) diff --git a/test/unit_test/headless/test_flow_var_commands.py b/test/unit_test/headless/test_flow_var_commands.py index cc524238..2cff682d 100644 --- a/test/unit_test/headless/test_flow_var_commands.py +++ b/test/unit_test/headless/test_flow_var_commands.py @@ -7,9 +7,9 @@ from je_auto_control.utils.exception.exceptions import ( AutoControlAssertionException, ) -from je_auto_control.utils.executor import flow_control +from je_auto_control.utils.executor import flow_data_commands from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import ( +from je_auto_control.utils.executor.flow_data_commands import ( exec_assert_var, exec_http_to_var, exec_now_to_var, exec_random_to_var, exec_read_file_to_var, exec_transform_var, ) @@ -76,7 +76,7 @@ def test_transform_var_replace_and_slice(): def test_now_to_var_formats_injected_clock(monkeypatch): - monkeypatch.setattr(flow_control, "_now", + monkeypatch.setattr(flow_data_commands, "_now", lambda: datetime.datetime(2026, 1, 2, 3, 4, 5)) executor = Executor() result = exec_now_to_var(executor, {"format": "%Y-%m-%d", "var": "d"}) diff --git a/test/unit_test/headless/test_ocr_to_var.py b/test/unit_test/headless/test_ocr_to_var.py index 7796a637..83d1aafb 100644 --- a/test/unit_test/headless/test_ocr_to_var.py +++ b/test/unit_test/headless/test_ocr_to_var.py @@ -1,6 +1,6 @@ """Tests for AC_ocr_to_var (OCR a region into a flow variable).""" from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import exec_ocr_to_var +from je_auto_control.utils.executor.flow_data_commands import exec_ocr_to_var class _Match: diff --git a/test/unit_test/headless/test_pdf.py b/test/unit_test/headless/test_pdf.py index 05c93048..7700ad74 100644 --- a/test/unit_test/headless/test_pdf.py +++ b/test/unit_test/headless/test_pdf.py @@ -8,7 +8,7 @@ import je_auto_control as ac from je_auto_control.utils.exception.exceptions import AutoControlAssertionException from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import exec_pdf_to_var +from je_auto_control.utils.executor.flow_data_commands import exec_pdf_to_var from je_auto_control.utils.pdf import pdf_reader diff --git a/test/unit_test/headless/test_r3_gui_slot_exceptions.py b/test/unit_test/headless/test_r3_gui_slot_exceptions.py index 65b9feb1..d0d82268 100644 --- a/test/unit_test/headless/test_r3_gui_slot_exceptions.py +++ b/test/unit_test/headless/test_r3_gui_slot_exceptions.py @@ -55,11 +55,12 @@ def test_builder_run_slot_surfaces_autocontrol_exception(monkeypatch): def test_playback_record_slot_surfaces_autocontrol_exception(monkeypatch): import je_auto_control.gui.main_widget as mw + import je_auto_control.gui._record_tab as rt warned = {} - monkeypatch.setattr(mw.QMessageBox, "warning", + monkeypatch.setattr(rt.QMessageBox, "warning", lambda *a, **k: warned.setdefault("hit", True)) monkeypatch.setattr( - mw, "execute_action", + rt, "execute_action", _raiser(AutoControlExecuteActionException("boom")), ) stub = types.SimpleNamespace(_record_data=[["AC_ok"]]) @@ -70,10 +71,11 @@ def test_playback_record_slot_surfaces_autocontrol_exception(monkeypatch): def test_execute_script_slot_surfaces_autocontrol_exception(monkeypatch): import je_auto_control.gui.main_widget as mw + import je_auto_control.gui._script_tab as st captured = {} - monkeypatch.setattr(mw, "read_action_json", lambda _p: [["AC_ok"]]) + monkeypatch.setattr(st, "read_action_json", lambda _p: [["AC_ok"]]) monkeypatch.setattr( - mw, "execute_action", + st, "execute_action", _raiser(AutoControlExecuteActionException("boom")), ) editor = types.SimpleNamespace(text=lambda: "some.json") diff --git a/test/unit_test/headless/test_r3_rdusb_host.py b/test/unit_test/headless/test_r3_rdusb_host.py index 8dace307..7932f996 100644 --- a/test/unit_test/headless/test_r3_rdusb_host.py +++ b/test/unit_test/headless/test_r3_rdusb_host.py @@ -14,7 +14,7 @@ from je_auto_control.utils.remote_desktop import ( RemoteDesktopHost, RemoteDesktopViewer, ) -from je_auto_control.utils.remote_desktop.host import _ClientHandler +from je_auto_control.utils.remote_desktop.host_client import _ClientHandler def _wait_until(predicate, timeout: float = 5.0, diff --git a/test/unit_test/headless/test_r3_vision_capture.py b/test/unit_test/headless/test_r3_vision_capture.py index 8f157f4a..f8e36cd8 100644 --- a/test/unit_test/headless/test_r3_vision_capture.py +++ b/test/unit_test/headless/test_r3_vision_capture.py @@ -126,7 +126,9 @@ def grab(*_args, **_kwargs): def test_screenshot_save_failure_raises(monkeypatch, tmp_path): pytest.importorskip("PIL") - monkeypatch.setattr(ss, "ImageGrab", _FakeGrab) + # pil_screenshot now asks the platform layer for its grabber rather than + # importing ImageGrab itself, so Wayland gets the compositor's capture. + monkeypatch.setattr(ss, "image_grabber", lambda: _FakeGrab) bad_path = str(tmp_path / "no_such_dir" / "shot.png") # parent missing with pytest.raises(AutoControlScreenException): ss.pil_screenshot(file_path=bad_path) diff --git a/test/unit_test/headless/test_remote_desktop_ip_allowlist.py b/test/unit_test/headless/test_remote_desktop_ip_allowlist.py index 23e877dd..f3c5912f 100644 --- a/test/unit_test/headless/test_remote_desktop_ip_allowlist.py +++ b/test/unit_test/headless/test_remote_desktop_ip_allowlist.py @@ -5,8 +5,9 @@ import pytest -from je_auto_control.utils.remote_desktop.host import ( - RemoteDesktopHost, _compile_ip_allowlist, _ip_in_allowlist, +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import ( + _compile_ip_allowlist, _ip_in_allowlist, ) from je_auto_control.utils.remote_desktop.viewer import RemoteDesktopViewer diff --git a/test/unit_test/headless/test_remote_desktop_pending_viewer.py b/test/unit_test/headless/test_remote_desktop_pending_viewer.py index bbba16fb..538ed7c0 100644 --- a/test/unit_test/headless/test_remote_desktop_pending_viewer.py +++ b/test/unit_test/headless/test_remote_desktop_pending_viewer.py @@ -4,9 +4,8 @@ import pytest -from je_auto_control.utils.remote_desktop.host import ( - PendingViewer, RemoteDesktopHost, -) +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import PendingViewer from je_auto_control.utils.remote_desktop.protocol import ( AuthenticationError, ) diff --git a/test/unit_test/headless/test_remote_desktop_resume.py b/test/unit_test/headless/test_remote_desktop_resume.py index 9574cd62..4c0d5b85 100644 --- a/test/unit_test/headless/test_remote_desktop_resume.py +++ b/test/unit_test/headless/test_remote_desktop_resume.py @@ -4,9 +4,8 @@ import pytest -from je_auto_control.utils.remote_desktop.host import ( - PendingViewer, RemoteDesktopHost, -) +from je_auto_control.utils.remote_desktop.host import RemoteDesktopHost +from je_auto_control.utils.remote_desktop.host_access import PendingViewer from je_auto_control.utils.remote_desktop.resume_tokens import ( ResumeTokenStore, ) diff --git a/test/unit_test/headless/test_rest_server.py b/test/unit_test/headless/test_rest_server.py index b2f02085..93abf31c 100644 --- a/test/unit_test/headless/test_rest_server.py +++ b/test/unit_test/headless/test_rest_server.py @@ -68,17 +68,87 @@ def test_execute_rejects_missing_actions(rest_server): assert "actions" in payload.get("error", "") +def test_execute_rejects_unknown_command_with_400(rest_server): + """A misspelt AC_* name is the caller's error, so 4xx and not 500.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute", method="POST", + body={"actions": [["AC_bogus_command"]]}, + token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert payload.get("unknown_commands") == ["AC_bogus_command"] + assert "AC_bogus_command" in payload.get("error", "") + + +def test_execute_lists_every_unknown_command(rest_server): + """All bad names come back at once, including ones nested in a body.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute", method="POST", + body={"actions": [ + ["AC_typo_one"], + ["AC_loop", {"times": 1, "body": [["AC_typo_two"]]}], + ]}, + token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert payload.get("unknown_commands") == ["AC_typo_one", "AC_typo_two"] + + +def test_execute_rejects_empty_action_list_with_400(rest_server): + """An empty list is bad input, not a server fault.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute", method="POST", + body={"actions": []}, token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert "error" in payload + + +def test_execute_file_rejects_missing_file_with_400(rest_server, tmp_path): + """The caller named the path, so an unreadable file is their error.""" + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute_file", method="POST", + body={"path": str(tmp_path / "does_not_exist.json")}, + token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert "error" in payload + + +def test_execute_file_rejects_unknown_command_with_400(rest_server, tmp_path): + action_file = tmp_path / "actions.json" + action_file.write_text(json.dumps([["AC_bogus_command"]]), encoding="utf-8") + with pytest.raises(urllib.error.HTTPError) as exc_info: + _request(rest_server, "/execute_file", method="POST", + body={"path": str(action_file)}, token=rest_server.token) + assert exc_info.value.code == 400 + payload = json.loads(exc_info.value.read().decode("utf-8")) + assert "AC_bogus_command" in payload.get("error", "") + + def test_unknown_path_returns_404(rest_server): with pytest.raises(urllib.error.HTTPError) as exc_info: _request(rest_server, "/nope", token=rest_server.token) assert exc_info.value.code == 404 -def test_handler_crash_returns_500_not_dropped(rest_server): - """Sending an action list that raises must produce JSON, not RST.""" +def test_handler_crash_returns_500_not_dropped(rest_server, monkeypatch): + """A genuine server-side failure must produce JSON, not RST. + + The action list is valid, so the 400 gate lets it through and the crash + comes from the run itself — which is exactly the 500 that misspelt + command names used to be lumped in with. + """ + from je_auto_control.utils.executor import action_executor + + def _boom(_action_list): + raise MemoryError("simulated executor crash") + + monkeypatch.setattr(action_executor, "execute_action", _boom) with pytest.raises(urllib.error.HTTPError) as exc_info: _request(rest_server, "/execute", method="POST", - body={"actions": []}, token=rest_server.token) + body={"actions": [["AC_sleep", {"seconds": 0}]]}, + token=rest_server.token) assert exc_info.value.code == 500 payload = json.loads(exc_info.value.read().decode("utf-8")) assert "error" in payload diff --git a/test/unit_test/headless/test_screen_grabber.py b/test/unit_test/headless/test_screen_grabber.py new file mode 100644 index 00000000..352e156e --- /dev/null +++ b/test/unit_test/headless/test_screen_grabber.py @@ -0,0 +1,276 @@ +"""The capture layer must reach the platform backend, not Pillow directly. + +Every locator, OCR call, screenshot, recording and remote-desktop frame in +the framework used to call ``PIL.ImageGrab`` or ``mss`` itself. Both read the +X11 root window on Linux, which under Wayland belongs to XWayland and holds +none of the native Wayland windows — so all of them captured a blank screen +while the Wayland backend's own capture sat unused and unreachable. + +These tests pin the seam that fixes it: when the active backend publishes +``grab_image`` every capture path goes through it, and when it does not +(Windows, macOS, X11) the real libraries are handed back untouched. No Qt, +and no dependency on the host's display server. +""" +import contextlib +import sys +from unittest.mock import patch + +import pytest +from PIL import Image + +from je_auto_control.utils.cv2_utils import screen_grabber + + +class _FakeBackendScreen: + """Minimal stand-in for a backend screen module that can capture.""" + + def __init__(self, size=(8, 4), colour=(10, 20, 30), origin=(0, 0)): + self._size = size + self._colour = colour + self._origin = origin + self.calls = [] + + def grab_image(self, screen_region=None): + self.calls.append(screen_region) + if screen_region is None: + return Image.new("RGB", self._size, self._colour) + x1, y1, x2, y2 = screen_region + return Image.new("RGB", (x2 - x1, y2 - y1), self._colour) + + def size(self): + return self._size + + def layout_origin(self): + return self._origin + + +@contextlib.contextmanager +def _with_backend(backend): + """Patch the wrapper's ``screen`` so the grabbers see our fake backend. + + Both halves of the seam have to come from the fake. ``mss_grabber`` + reports its layout from ``platform_wrapper.screen.size()``, so patching + only ``grab_image`` would leave ``monitors`` describing whatever display + the test host happens to have -- passing on a 1920x1080 desktop and + failing under a 1280x800 Xvfb for reasons unrelated to the code here. + """ + with patch.object(screen_grabber, "backend_grab_image", + return_value=backend.grab_image): + with patch.object(screen_grabber, "_backend_screen_size", + side_effect=backend.size): + with patch.object(screen_grabber, "backend_layout_origin", + side_effect=backend.layout_origin): + yield + + +# === Backend detection ===================================================== + +def test_backend_grab_image_is_none_without_a_publishing_backend(): + """Windows / macOS / X11 publish no grab_image, and must keep using the + real libraries — this fix must change nothing on those platforms.""" + class _PlainScreen: + def size(self): + return (1920, 1080) + + stub = type(sys)("stub") + stub.screen = _PlainScreen() + with patch.dict(sys.modules, + {"je_auto_control.wrapper.platform_wrapper": stub}): + assert screen_grabber.backend_grab_image() is None + + +def test_image_grabber_returns_pillow_when_no_backend_publishes_capture(): + with patch.object(screen_grabber, "backend_grab_image", + return_value=None): + from PIL import ImageGrab + assert screen_grabber.image_grabber() is ImageGrab + + +# === ImageGrab-shaped adapter ============================================== + +def test_image_grabber_routes_full_screen_grab_to_the_backend(): + backend = _FakeBackendScreen() + with _with_backend(backend): + image = screen_grabber.image_grabber().grab() + assert backend.calls == [None] + assert image.size == (8, 4) + + +def test_image_grabber_passes_the_bbox_through_as_a_region(): + backend = _FakeBackendScreen() + with _with_backend(backend): + image = screen_grabber.image_grabber().grab(bbox=(2, 3, 6, 9)) + assert backend.calls == [[2, 3, 6, 9]] + assert image.size == (4, 6) + + +def test_image_grabber_tolerates_pillow_only_keywords(): + """Callers pass all_screens / xdisplay; a Wayland capture already spans + the whole layout, so these must be accepted and ignored rather than + raising TypeError deep inside a locator.""" + backend = _FakeBackendScreen() + with _with_backend(backend): + grabber = screen_grabber.image_grabber() + grabber.grab(all_screens=True) + grabber.grab(bbox=(0, 0, 2, 2), all_screens=True, xdisplay=None) + assert backend.calls == [None, [0, 0, 2, 2]] + + +def test_logical_frame_capture_goes_through_the_backend(): + """grab_logical feeds the template, OCR and visual-match locators; it + resolves its grabber lazily, so the seam has to hold there too.""" + from je_auto_control.utils.monitor_layout import logical_frame + backend = _FakeBackendScreen() + # Pin the virtual-desktop metrics so the assertion is about the grabber + # seam, not about whatever monitors this test host happens to have. + flat_desktop = {76: 0, 77: 0, 78: 8, 79: 4} + with _with_backend(backend): + image, origin_x, origin_y = logical_frame.grab_logical( + metrics=flat_desktop.__getitem__) + assert (origin_x, origin_y) == (0, 0) + assert image.size == (8, 4) + assert backend.calls == [None] + + +def test_pil_screenshot_goes_through_the_backend(tmp_path): + from je_auto_control.utils.cv2_utils import screenshot as screenshot_module + backend = _FakeBackendScreen() + target = tmp_path / "shot.png" + with _with_backend(backend): + image = screenshot_module.pil_screenshot(file_path=str(target)) + assert image.size == (8, 4) + assert target.exists() + assert backend.calls == [None] + + +# === mss-shaped adapter ==================================================== + +def test_mss_grabber_reports_the_layout_as_monitor_zero_and_one(): + backend = _FakeBackendScreen(size=(1920, 1080)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + monitors = sct.monitors + assert len(monitors) == 2 + assert monitors[0] == {"left": 0, "top": 0, + "width": 1920, "height": 1080} + # Callers that default to "the first real screen" must still get a frame. + assert monitors[1] == monitors[0] + + +def test_mss_grabber_reports_a_negative_layout_origin(): + """Regression: ``monitors`` was hardcoded to start at (0, 0). + + A Wayland layout starts at a negative coordinate whenever an output + sits left of or above the origin, and mss's own callers already expect + a negative ``left`` there. ``grab`` feeds the rectangle straight back as + a capture region, so claiming (0, 0) grabbed half of one monitor plus + the empty space past the desktop's right edge. + """ + backend = _FakeBackendScreen(size=(2560, 720), origin=(-1280, 0)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + monitors = sct.monitors + sct.grab(monitors[1]) + assert monitors[0] == {"left": -1280, "top": 0, + "width": 2560, "height": 720} + assert backend.calls == [[-1280, 0, 1280, 720]] + + +def test_backend_layout_origin_is_the_origin_without_a_publishing_backend(): + """Windows / macOS / X11 publish no layout_origin, and a guess would be worse.""" + class _PlainScreen: + def size(self): + return (1920, 1080) + + stub = type(sys)("stub") + stub.screen = _PlainScreen() + with patch.dict(sys.modules, + {"je_auto_control.wrapper.platform_wrapper": stub}): + assert screen_grabber.backend_layout_origin() == (0, 0) + + +def test_backend_layout_origin_reads_a_publishing_backend(): + backend = _FakeBackendScreen(size=(2560, 720), origin=(-1280, -200)) + stub = type(sys)("stub") + stub.screen = backend + with patch.dict(sys.modules, + {"je_auto_control.wrapper.platform_wrapper": stub}): + assert screen_grabber.backend_layout_origin() == (-1280, -200) + + +def test_mss_shot_exposes_the_bgra_buffer_callers_read(): + """mss consumers read .bgra / .size directly (the MCP monitor grab) and + hand the shot to numpy (the recorder), so both shapes must hold.""" + backend = _FakeBackendScreen(size=(2, 1), colour=(10, 20, 30)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + shot = sct.grab({"left": 0, "top": 0, "width": 2, "height": 1}) + assert shot.size == (2, 1) + assert (shot.width, shot.height) == (2, 1) + # BGRA order, so the blue channel comes first. + assert shot.bgra[:4] == bytes((30, 20, 10, 255)) + assert shot.rgb[:3] == bytes((10, 20, 30)) + rebuilt = Image.frombytes("RGB", shot.size, shot.bgra, "raw", "BGRX") + assert rebuilt.getpixel((0, 0)) == (10, 20, 30) + + +def test_mss_shot_converts_to_a_numpy_bgra_array(): + numpy = pytest.importorskip("numpy") + backend = _FakeBackendScreen(size=(3, 2)) + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + shot = sct.grab({"left": 0, "top": 0, "width": 3, "height": 2}) + array = numpy.array(shot) + assert array.shape == (2, 3, 4) + + +# === Whole chain, public API down to the compositor's tool ================= + +def _solid_png(size, rgb) -> bytes: + from io import BytesIO + buffer = BytesIO() + Image.new("RGB", size, rgb).save(buffer, format="PNG") + return buffer.getvalue() + + +def test_public_screenshot_returns_the_compositors_pixels_on_wayland(): + """The regression this whole seam exists for. + + ``je_auto_control.screenshot()`` used to call ImageGrab, so on Wayland it + returned the XWayland root — blank — while the backend's grim capture was + unreachable. Drive the real public entry point with a Wayland backend and + a stubbed grim, and the bytes that come back have to be grim's. + """ + import subprocess + + from je_auto_control.linux_wayland import capture as wayland_capture + from je_auto_control.linux_wayland import screen as wayland_screen + from je_auto_control.wrapper.auto_control_screen import screenshot + + png = _solid_png((3, 2), (10, 20, 30)) + + def fake_grim(argv, **_kwargs): + assert argv[0] == "/usr/bin/grim" + return subprocess.CompletedProcess(argv, 0, png, b"") # nosemgrep + + with patch.object(screen_grabber, "backend_grab_image", + return_value=wayland_screen.grab_image), \ + patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_capture.subprocess, "run", + side_effect=fake_grim): + frame = screenshot() + + # The wrapper hands back BGR for OpenCV, so RGB (10, 20, 30) arrives + # reversed — proof these are grim's pixels and not an empty grab. + assert frame.shape == (2, 3, 3) + assert tuple(int(v) for v in frame[0][0]) == (30, 20, 10) + + +def test_mss_grabber_grabs_the_requested_monitor_rectangle(): + backend = _FakeBackendScreen() + with _with_backend(backend): + with screen_grabber.mss_grabber() as sct: + shot = sct.grab({"left": 5, "top": 6, "width": 3, "height": 4}) + assert backend.calls == [[5, 6, 8, 10]] + assert (shot.left, shot.top) == (5, 6) diff --git a/test/unit_test/headless/test_shell_to_var.py b/test/unit_test/headless/test_shell_to_var.py index 6e7c85ba..ef56f7dc 100644 --- a/test/unit_test/headless/test_shell_to_var.py +++ b/test/unit_test/headless/test_shell_to_var.py @@ -2,7 +2,7 @@ import sys from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import exec_shell_to_var +from je_auto_control.utils.executor.flow_data_commands import exec_shell_to_var def test_shell_to_var_captures_stdout(): diff --git a/test/unit_test/headless/test_sql_steps.py b/test/unit_test/headless/test_sql_steps.py index 29aeffe4..9fe6aafd 100644 --- a/test/unit_test/headless/test_sql_steps.py +++ b/test/unit_test/headless/test_sql_steps.py @@ -9,7 +9,7 @@ import je_auto_control as ac from je_auto_control.utils.exception.exceptions import AutoControlAssertionException from je_auto_control.utils.executor.action_executor import Executor -from je_auto_control.utils.executor.flow_control import ( +from je_auto_control.utils.executor.flow_data_commands import ( exec_assert_db, exec_sql_to_var, ) from je_auto_control.utils.sql.sql_query import query_sqlite diff --git a/test/unit_test/headless/test_unattended_reliability.py b/test/unit_test/headless/test_unattended_reliability.py index d7677ef5..83960805 100644 --- a/test/unit_test/headless/test_unattended_reliability.py +++ b/test/unit_test/headless/test_unattended_reliability.py @@ -23,7 +23,7 @@ def test_generate_totp_deterministic_and_verifies(): def test_otp_to_var_command(): from je_auto_control.utils.executor.action_executor import Executor - from je_auto_control.utils.executor.flow_control import exec_otp_to_var + from je_auto_control.utils.executor.flow_data_commands import exec_otp_to_var executor = Executor() result = exec_otp_to_var( executor, {"secret": ac.generate_secret(), "var": "code"}) diff --git a/test/unit_test/headless/test_wayland_backend.py b/test/unit_test/headless/test_wayland_backend.py index c6a5ad93..dd6a6a29 100644 --- a/test/unit_test/headless/test_wayland_backend.py +++ b/test/unit_test/headless/test_wayland_backend.py @@ -12,13 +12,40 @@ import pytest from je_auto_control.linux_wayland import ( - _detect, keyboard as wayland_keyboard, mouse as wayland_mouse, + _detect, _ydotool_cli, capture as wayland_capture, + keyboard as wayland_keyboard, mouse as wayland_mouse, screen as wayland_screen, ) from je_auto_control.linux_wayland.keymap import keyboard_keys_table from je_auto_control.utils.exception.exceptions import AutoControlException +@pytest.fixture(autouse=True) +def _pin_the_cli_input_path(): + """Keep every dispatch test in this file on the wtype / ydotool argv. + + ``mouse`` and ``keyboard`` prefer libei wherever it is loadable, so on a + host that has it these tests would probe for a portal and then assert + against argv that was never produced. The libei path has its own file, + ``test_wayland_libei.py``. + + The ydotool generation cache is pre-seeded for the same reason. Before + building any argv both backends probe the installed ydotool once, because + 0.1.x answers this argv with exit code 0 and no events — and that probe is + a ``subprocess.run`` call too, so it would land in the captured argv list + of every dispatch test here. What it does instead is its own file, + ``test_wayland_ydotool_cli.py``. + """ + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + try: + with patch.object(wayland_mouse, "_try_libei", return_value=None), \ + patch.object(wayland_keyboard, "_try_libei", return_value=None): + yield + finally: + _ydotool_cli.reset_cache() + + # === Detection ============================================================== def test_is_wayland_session_reads_session_type(): @@ -227,39 +254,49 @@ def test_mouse_raises_when_ydotool_missing(): # === Screen dispatch ======================================================== -def test_screenshot_calls_grim_with_path(): +def _fake_capture(captured, png): + """Stand in for subprocess.run inside the capture module.""" + def runner(argv, **_kwargs): + captured.append(list(argv)) + return subprocess.CompletedProcess(argv, 0, png, b"") # nosemgrep + return runner + + +def test_screenshot_captures_through_grim_and_saves_with_pillow(tmp_path): + """grim writes PNG to stdout and Pillow saves it, so the same capture + can also be handed back in memory to the locators.""" captured: list = [] - # CompletedProcess constructor used to mock subprocess.run. - with patch.object(wayland_screen, "binary_path", + png = _one_pixel_png((1, 2, 3)) + target = tmp_path / "out.png" + with patch.object(wayland_capture, "binary_path", return_value="/usr/bin/grim"), \ - patch.object(wayland_screen.subprocess, "run", - side_effect=lambda argv, **kw: (captured.append(argv) - or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep - wayland_screen.screenshot("out.png") - assert captured == [["/usr/bin/grim", "out.png"]] + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture(captured, png)): + assert wayland_screen.screenshot(str(target)) == str(target) + assert captured == [["/usr/bin/grim", "-"]] + assert target.exists() def test_screenshot_passes_screen_region(): captured: list = [] - with patch.object(wayland_screen, "binary_path", + png = _one_pixel_png((1, 2, 3)) + with patch.object(wayland_capture, "binary_path", return_value="/usr/bin/grim"), \ - patch.object(wayland_screen.subprocess, "run", - side_effect=lambda argv, **kw: (captured.append(argv) - or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep - wayland_screen.screenshot("out.png", screen_region=[10, 20, 110, 220]) + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture(captured, png)): + wayland_screen.screenshot(None, screen_region=[10, 20, 110, 220]) assert captured == [[ - "/usr/bin/grim", "-g", "10,20 100x200", "out.png", + "/usr/bin/grim", "-g", "10,20 100x200", "-", ]] -@pytest.mark.parametrize("direction_name, expected_axis, expected_amount", [ - ("wayland_scroll_direction_down", "-y", "-5"), - ("wayland_scroll_direction_up", "-y", "5"), - ("wayland_scroll_direction_left", "-x", "-5"), - ("wayland_scroll_direction_right", "-x", "5"), +@pytest.mark.parametrize("direction_name, expected_x, expected_y", [ + ("wayland_scroll_direction_down", "0", "-5"), + ("wayland_scroll_direction_up", "0", "5"), + ("wayland_scroll_direction_left", "-5", "0"), + ("wayland_scroll_direction_right", "5", "0"), ]) -def test_scroll_honours_direction(direction_name, expected_axis, - expected_amount): +def test_scroll_honours_direction(direction_name, expected_x, expected_y): """Regression: scroll's signature was ``(direction, x, y)`` while the wrapper calls ``scroll(scroll_value, scroll_direction)``. The direction bound to ``x`` and was then dropped, so every direction scrolled the same @@ -272,8 +309,9 @@ def test_scroll_honours_direction(direction_name, expected_axis, patch.object(wayland_mouse.subprocess, "run", side_effect=_fake_run(captured)): wayland_mouse.scroll(5, direction) + # Both axes are always sent, per ydotool's documented example. assert captured[0][1:] == [ - "mousemove", "--wheel", expected_axis, expected_amount, + "mousemove", "--wheel", "-x", expected_x, "-y", expected_y, ] @@ -295,42 +333,281 @@ def test_get_pixel_grabs_a_one_by_one_region_at_the_point(): png = _one_pixel_png((10, 20, 30)) captured: list = [] - def _run(argv, **_kwargs): - captured.append(argv) - return subprocess.CompletedProcess(argv, 0, png, b"") # nosemgrep - - with patch.object(wayland_screen, "binary_path", + with patch.object(wayland_capture, "binary_path", return_value="/usr/bin/grim"), \ - patch.object(wayland_screen.subprocess, "run", side_effect=_run): + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture(captured, png)): assert wayland_screen.get_pixel(7, 9) == (10, 20, 30) assert "-g" in captured[0] and "7,9 1x1" in captured[0] def _one_pixel_png(rgb) -> bytes: + return _solid_png((1, 1), rgb) + + +def _solid_png(size, rgb) -> bytes: from io import BytesIO from PIL import Image buffer = BytesIO() - Image.new("RGB", (1, 1), rgb).save(buffer, format="PNG") + Image.new("RGB", size, rgb).save(buffer, format="PNG") return buffer.getvalue() +# Captured verbatim from `wlr-randr` inside the headless sway session in +# docker/Dockerfile.wayland — two outputs, the second one to the right of +# the first, no refresh rate on a headless mode. +_REAL_WLR_RANDR = """HEADLESS-2 "Headless output 1" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: 0,0 + Transform: normal + Scale: 1.000000 +HEADLESS-1 "Headless output 2" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: 1280,0 + Transform: normal + Scale: 1.000000 +""" + + +def test_wlr_randr_parses_every_output_not_just_the_first(): + """Regression, found by running against a real sway session. + + The old parser returned the first ``WxH`` anywhere in the document, so a + two-monitor layout reported one monitor's mode as the screen size — + while ``grab_image()`` returned the whole layout. The mss-shaped shim + composes the two, so it asked for a region half the size of the screen. + """ + rects = wayland_screen.parse_wlr_randr(_REAL_WLR_RANDR) + assert rects == [(0, 0, 1280, 720), (1280, 0, 1280, 720)] + + +def test_wlr_randr_skips_a_disabled_output(): + text = _REAL_WLR_RANDR.replace(" Enabled: yes\n Modes:\n" + " 1280x720 px (current)\n" + " Position: 1280,0\n", + " Enabled: no\n") + assert wayland_screen.parse_wlr_randr(text) == [(0, 0, 1280, 720)] + + +def test_wlr_randr_reads_a_mode_line_that_carries_a_refresh_rate(): + """A real monitor prints `1920x1080 px, 60.000000 Hz (preferred, current)`; + only the headless backend omits the rate.""" + text = ('DP-1 "Acme"\n Enabled: yes\n Modes:\n' + ' 1920x1080 px, 60.000000 Hz (preferred, current)\n' + ' 1280x720 px, 60.000000 Hz\n Position: 0,0\n') + assert wayland_screen.parse_wlr_randr(text) == [(0, 0, 1920, 1080)] + + def test_screen_size_uses_wlr_randr_when_available(): with patch.object(wayland_screen, "binary_path", side_effect=lambda name: "/usr/bin/" + name), \ patch.object( - wayland_screen.subprocess, "run", + wayland_capture.subprocess, "run", return_value=subprocess.CompletedProcess( # nosemgrep - ["wlr-randr"], 0, b" HDMI-A-1 1920x1080@60.000Hz\n", b"", + ["wlr-randr"], 0, _REAL_WLR_RANDR.encode(), b"", ), ): - assert wayland_screen.size() == (1920, 1080) + # The bounding box of both outputs, not the first output's mode. + assert wayland_screen.size() == (2560, 720) + + +def test_screen_size_falls_back_to_measuring_a_capture(): + """GNOME / KDE have no wlr-randr, so the size comes from the capture.""" + png = _one_pixel_png((0, 0, 0)) + with patch.object(wayland_screen, "binary_path", return_value=None), \ + patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_capture.subprocess, "run", + side_effect=_fake_capture([], png)): + assert wayland_screen.size() == (1, 1) + + +# Captured verbatim from the same sway session with HEADLESS-1 moved to +# `position -1280 0`, which sway's headless backend accepts. A monitor +# placed left of the primary one is what puts a real desktop here, and it +# is the layout every assertion below is about. +_NEGATIVE_ORIGIN_WLR_RANDR = """HEADLESS-2 "Headless output 1" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: 0,0 + Transform: normal + Scale: 1.000000 + Adaptive Sync: disabled +HEADLESS-1 "Headless output 2" + Make: (null) + Model: (null) + Serial: (null) + Enabled: yes + Modes: + 1280x720 px (current) + Position: -1280,0 + Transform: normal + Scale: 1.000000 + Adaptive Sync: disabled +""" + + +def _with_wlr_randr(reported): + """The two seams ``wlr-randr`` is read through, as a pair of patches.""" + return ( + patch.object(wayland_screen, "binary_path", + side_effect=lambda name: "/usr/bin/" + name), + patch.object( + wayland_capture.subprocess, "run", + return_value=subprocess.CompletedProcess( # nosemgrep + ["wlr-randr"], 0, reported.encode(), b"", + ), + ), + ) -def test_screenshot_raises_when_grim_missing(): +def test_wlr_randr_parses_a_negative_output_position(): + rects = wayland_screen.parse_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + assert rects == [(0, 0, 1280, 720), (-1280, 0, 1280, 720)] + + +def test_screen_size_is_the_layout_width_not_its_right_edge(): + """Regression: ``size()`` returned ``max(x + width)``. + + On the layout above that is 1280 — the right edge — while the capture + grim returns is 2560 wide. Everything that composes the two (the + mss-shaped shim's monitor list, the recorder, the WebRTC host) then + asked for half the desktop and called it the whole screen. + """ + binary, run = _with_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + with binary, run: + assert wayland_screen.size() == (2560, 720) + + +def test_layout_origin_is_the_left_most_output(): + binary, run = _with_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + with binary, run: + assert wayland_screen.layout_origin() == (-1280, 0) + + +def test_layout_origin_is_the_origin_on_an_ordinary_layout(): + binary, run = _with_wlr_randr(_REAL_WLR_RANDR) + with binary, run: + assert wayland_screen.layout_origin() == (0, 0) + + +def test_layout_origin_is_the_origin_when_wlr_randr_is_absent(): + """GNOME / KDE ship no wlr-randr, and a guess would be worse than (0, 0).""" with patch.object(wayland_screen, "binary_path", return_value=None): - with pytest.raises(AutoControlException, match="grim"): + assert wayland_screen.layout_origin() == (0, 0) + + +def test_grab_image_crops_relative_to_the_layout_origin(): + """Regression: the crop used layout coordinates on a layout-origin image. + + Only grim applies a region itself; gnome-screenshot, spectacle, the + portal and the operator's own command all hand back the whole layout, + and its top-left pixel is the layout origin rather than (0, 0). Cropping + ``[-1275, 5, -1175, 55]`` straight off that image asks Pillow for a box + starting 1275 px to the left of the frame — which pads with black + instead of returning the left-hand monitor. + """ + from io import BytesIO + + from PIL import Image + left, right = (0x12, 0x34, 0x56), (0xAB, 0xCD, 0xEF) + layout = Image.new("RGB", (2560, 720), right) + layout.paste(Image.new("RGB", (1280, 720), left), (0, 0)) + buffer = BytesIO() + layout.save(buffer, format="PNG") + + binary, run = _with_wlr_randr(_NEGATIVE_ORIGIN_WLR_RANDR) + whole_layout = wayland_capture.Capture(buffer.getvalue(), False) + with binary, run, patch.object(wayland_capture, "grab_png", + return_value=whole_layout): + cropped = wayland_screen.grab_image([-1275, 5, -1175, 55]) + # A region on the right-hand output still reads the right colour, + # so what changed is a shift and not a constant. + other = wayland_screen.grab_image([5, 5, 105, 55]) + assert cropped.size == (100, 50) + assert cropped.getpixel((0, 0)) == left + assert other.getpixel((0, 0)) == right + + +def test_screenshot_raises_when_no_capture_tool_is_installed(): + """The message has to name every tool that would have worked, because + which one is right depends on the compositor the operator is running.""" + with patch.object(wayland_capture, "binary_path", return_value=None): + with pytest.raises(AutoControlException) as error: wayland_screen.screenshot("out.png") + message = str(error.value) + for tool in ("grim", "gnome-screenshot", "spectacle"): + assert tool in message + + +def test_capture_falls_back_to_gnome_screenshot_when_grim_is_absent(tmp_path): + """grim only speaks wlr-screencopy; GNOME needs its own helper.""" + png = _one_pixel_png((4, 5, 6)) + captured: list = [] + + def _which(name): + return None if name == "grim" else "/usr/bin/" + name + + def _run(argv, **_kwargs): + captured.append(list(argv)) + # The helper writes a file rather than to stdout. + with open(argv[-1], "wb") as handle: + handle.write(png) + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_capture, "binary_path", side_effect=_which), \ + patch.object(wayland_capture.subprocess, "run", side_effect=_run): + image = wayland_screen.grab_image() + assert image.size == (1, 1) + assert captured[0][0] == "/usr/bin/gnome-screenshot" + + +def test_grab_image_crops_when_the_helper_cannot_take_a_region(): + """Only grim applies a region itself; the file-based helpers always + return the whole screen, so the region has to be cropped afterwards.""" + png = _solid_png((4, 2), (7, 8, 9)) + + def _which(name): + return None if name == "grim" else "/usr/bin/" + name + + def _run(argv, **_kwargs): + with open(argv[-1], "wb") as handle: + handle.write(png) + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_capture, "binary_path", side_effect=_which), \ + patch.object(wayland_capture.subprocess, "run", side_effect=_run): + image = wayland_screen.grab_image([1, 0, 3, 2]) + assert image.size == (2, 2) + + +def test_grab_image_rejects_an_empty_region(): + with pytest.raises(AutoControlException, match="positive width"): + wayland_screen.grab_image([10, 10, 10, 20]) + + +def test_available_tool_reports_the_helper_that_would_run(): + with patch.object(wayland_capture, "binary_path", + side_effect=lambda name: None if name == "grim" + else "/usr/bin/" + name): + assert wayland_capture.available_tool() == "gnome-screenshot" + with patch.object(wayland_capture, "binary_path", return_value=None): + assert wayland_capture.available_tool() is None # === Listener / record stubs =============================================== diff --git a/test/unit_test/headless/test_wayland_capture_tiers.py b/test/unit_test/headless/test_wayland_capture_tiers.py new file mode 100644 index 00000000..42d67559 --- /dev/null +++ b/test/unit_test/headless/test_wayland_capture_tiers.py @@ -0,0 +1,339 @@ +"""The capture tiers below the three compositor CLI helpers. + +No compositor helper is guaranteed to be installed — GNOME has not shipped +``gnome-screenshot`` by default since 42 — so the capture layer falls back to +``xdg-desktop-portal`` and, above everything, honours an operator-supplied +command. These tests drive both without a Wayland session: the session bus +connection is a fake that answers from a script, and the operator command is +any program that writes a file. + +The input side of the portal lives in ``test_wayland_oeffis.py``; it needs a +file descriptor rather than a file, so it cannot share this route. +""" +import subprocess +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import capture as wayland_capture +from je_auto_control.linux_wayland import portal as wayland_portal +from je_auto_control.linux_wayland import screen as wayland_screen +from je_auto_control.utils.exception.exceptions import AutoControlScreenException + + +def _png(size=(2, 2), rgb=(9, 9, 9)) -> bytes: + from io import BytesIO + + from PIL import Image + buffer = BytesIO() + Image.new("RGB", size, rgb).save(buffer, format="PNG") + return buffer.getvalue() + + +def _no_binaries(name): + return None + + +# === Operator override ===================================================== + +def test_override_command_wins_over_every_installed_tool(monkeypatch): + """An operator who names a command means it — detection does not get a + vote, or the override could not rescue a box where grim exists but is + broken.""" + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, + "/usr/bin/mycap --png {output}") + png = _png() + captured = [] + + def run(argv, **_kwargs): + captured.append(list(argv)) + with open(argv[-1], "wb") as handle: + handle.write(png) + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_capture.subprocess, "run", side_effect=run): + image = wayland_screen.grab_image() + + assert image.size == (2, 2) + assert captured[0][:2] == ["/usr/bin/mycap", "--png"] + assert captured[0][2].endswith(".png") + + +def test_override_keeps_a_spaced_path_as_one_argument(monkeypatch): + """The path is substituted per-token after shlex.split, never + concatenated into a command line, so a temp dir with a space in it + cannot split into two arguments (or be quoted into a shell).""" + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, "cap {output}") + argv = wayland_capture._override_argv("/tmp/a b/shot.png") + assert argv == ["cap", "/tmp/a b/shot.png"] + + +def test_override_without_the_output_placeholder_is_rejected(monkeypatch): + """A command with nowhere to write would silently produce an empty file; + say so at the boundary instead.""" + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, "grim -") + with pytest.raises(AutoControlScreenException, match=r"\{output\}"): + wayland_capture._override_argv("/tmp/x.png") + + +def test_override_is_reported_as_the_active_tool(monkeypatch): + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, "cap {output}") + assert wayland_capture.available_tool() == "$" + \ + wayland_capture.CAPTURE_COMMAND_ENV + + +def test_blank_override_is_ignored(monkeypatch): + monkeypatch.setenv(wayland_capture.CAPTURE_COMMAND_ENV, " ") + with patch.object(wayland_capture, "binary_path", side_effect=_no_binaries), \ + patch.object(wayland_portal, "is_available", return_value=False): + assert wayland_capture.available_tool() is None + + +# === Portal tier =========================================================== + +def test_portal_runs_only_after_every_cli_helper_is_absent(monkeypatch): + monkeypatch.delenv(wayland_capture.CAPTURE_COMMAND_ENV, raising=False) + png = _png() + with patch.object(wayland_capture, "binary_path", + return_value="/usr/bin/grim"), \ + patch.object(wayland_portal, "capture_png", + return_value=png) as portal_call, \ + patch.object(wayland_capture.subprocess, "run", + return_value=subprocess.CompletedProcess( # nosemgrep + ["grim"], 0, png, b"")): + wayland_capture.grab_png() + portal_call.assert_not_called() + + +def test_portal_is_used_when_no_helper_is_installed(monkeypatch): + monkeypatch.delenv(wayland_capture.CAPTURE_COMMAND_ENV, raising=False) + png = _png() + with patch.object(wayland_capture, "binary_path", side_effect=_no_binaries), \ + patch.object(wayland_portal, "is_available", return_value=True), \ + patch.object(wayland_portal, "capture_png", return_value=png): + result = wayland_capture.grab_png([0, 0, 1, 1]) + assert result.data == png + # The portal always captures everything, so the region must be cropped. + assert result.region_applied is False + + +def test_portal_is_reported_as_the_active_tool(monkeypatch): + monkeypatch.delenv(wayland_capture.CAPTURE_COMMAND_ENV, raising=False) + with patch.object(wayland_capture, "binary_path", side_effect=_no_binaries), \ + patch.object(wayland_portal, "is_available", return_value=True): + assert wayland_capture.available_tool() == "xdg-desktop-portal" + + +# === Portal response handling ============================================== +# +# The portal answers with a signal directed at the connection that called, so +# these drive the module the way the bus does: a fake connection that records +# the order things happened in. The real bus, the real marshalling and a real +# portal are exercised in ``docker/portal_verify.py`` — a mock cannot show +# that a directed signal never reaches a second connection, which is the +# defect this design replaced. + +_URI = "file:///run/user/1000/doc/ab/screenshot.png" + + +class _FakeBus: + """A session bus that answers from a script and records the order.""" + + sender_token = "1_9" + + def __init__(self, body=None, handle=None, error=None): + self.body = [0, {"uri": _URI}] if body is None else body + self.handle = handle + self.error = error + self.events = [] + self.rules = [] + self.awaited_paths = None + + def __enter__(self): + return self + + def __exit__(self, *_exception): + self.events.append("close") + + def add_match(self, rule): + self.events.append("add_match") + self.rules.append(rule) + + def call(self, _destination, _path, _interface, member, _signature, + _body, timeout=None): + del timeout + self.events.append(f"call:{member}") + return [self.handle] if self.handle else [] + + def wait_for_signal(self, paths, _interface, _member, _timeout): + self.events.append("wait") + self.awaited_paths = list(paths) + if self.error is not None: + raise self.error + return self.body + + +def _with_bus(bus): + """Patch the module's connection factory to hand back ``bus``.""" + return patch.object(wayland_portal._dbus_client, "SessionBus", + return_value=bus) + + +def test_portal_extracts_the_uri_from_a_response_signal(): + assert wayland_portal._uri_from_response([0, {"uri": _URI}]) == _URI + + +def test_portal_rejects_a_response_body_it_cannot_read(): + """A Response that is not ``(u, a{sv})`` must be named, not indexed into.""" + with pytest.raises(AutoControlScreenException, match="cannot read"): + wayland_portal._uri_from_response([0]) + + +def test_portal_reports_a_user_cancelled_request(): + """Response code 1 is the user dismissing the dialog; that has to read as + a cancellation, not as a parse failure.""" + with pytest.raises(AutoControlScreenException, match="dismissed"): + wayland_portal._uri_from_response([1, {}]) + + +def test_portal_rejects_a_success_with_no_uri(): + with pytest.raises(AutoControlScreenException, match="no image URI"): + wayland_portal._uri_from_response([0, {}]) + + +def test_portal_decodes_a_percent_escaped_file_uri(): + assert wayland_portal._path_from_uri( + "file:///run/user/1000/my%20shot.png", + ) == "/run/user/1000/my shot.png" + + +def test_portal_rejects_a_non_file_uri(): + with pytest.raises(AutoControlScreenException, match="non-file URI"): + wayland_portal._path_from_uri("https://example.invalid/x.png") + + +def test_portal_reads_then_deletes_the_file_it_was_handed(tmp_path): + """The portal hands over a real file; leaving one behind per capture + would fill the runtime dir over a long automation run.""" + target = tmp_path / "shot.png" + target.write_bytes(_png()) + assert wayland_portal._read_and_discard(str(target)) == _png() + assert not target.exists() + + +def test_portal_is_unavailable_without_a_session_bus(monkeypatch): + """No bus address is the one case where this tier cannot even be tried.""" + monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + assert wayland_portal.is_available() is False + with pytest.raises(AutoControlScreenException, match="session bus"): + wayland_portal.capture_png(timeout=0.1) + + +def test_portal_subscribes_before_it_calls(): + """The ordering the whole design turns on. + + The portal emits ``Response`` as soon as it has an answer, and it emits it + to the caller. Calling first and subscribing afterwards is a race that + loses the answer whenever the portal is quick — which, with no consent + dialog to show, it always is. + """ + bus = _FakeBus() + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + return_value=""), \ + patch.object(wayland_portal, "_read_and_discard", + return_value=_png()): + wayland_portal.capture_png(timeout=1.0) + assert bus.events[:3] == ["add_match", "call:Screenshot", "wait"] + + +def test_portal_subscribes_to_the_path_it_predicted(): + """The request path is built from our own unique name and handle token.""" + bus = _FakeBus() + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + return_value=""), \ + patch.object(wayland_portal, "_read_and_discard", + return_value=_png()): + wayland_portal.capture_png(timeout=1.0) + assert len(bus.awaited_paths) == 1 + predicted = bus.awaited_paths[0] + assert predicted.startswith( + "/org/freedesktop/portal/desktop/request/1_9/je_auto_control_") + assert predicted in bus.rules[0] + assert "member='Response'" in bus.rules[0] + + +def test_portal_also_follows_a_handle_that_differs_from_the_prediction(): + """A portal that ignores ``handle_token`` answers somewhere of its own. + + The specification tells clients to follow the returned handle, so both + paths have to be live: the predicted one could not be dropped (the answer + may already be on its way) and the returned one could not be subscribed + to any earlier than this. + """ + elsewhere = "/org/freedesktop/portal/desktop/request/1_9/portal_chose" + bus = _FakeBus(handle=elsewhere) + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + return_value=""), \ + patch.object(wayland_portal, "_read_and_discard", + return_value=_png()): + wayland_portal.capture_png(timeout=1.0) + assert bus.events[:4] == ["add_match", "call:Screenshot", "add_match", + "wait"] + assert elsewhere in bus.awaited_paths + assert len(bus.awaited_paths) == 2 + + +def test_portal_gives_each_request_its_own_token(): + """Two captures sharing a token would put both answers on one path.""" + first = wayland_portal._handle_token() + second = wayland_portal._handle_token() + assert first != second + for token in (first, second): + assert token.replace("_", "").isalnum(), \ + "the token becomes an object-path element, so it must be [A-Za-z0-9_]" + + +def test_portal_times_out_rather_than_waiting_forever(): + """A consent dialog nobody answers must not hang the caller's script.""" + bus = _FakeBus(error=wayland_portal._dbus_client.DBusError("no answer")) + with _with_bus(bus): + with pytest.raises(AutoControlScreenException, match="did not answer"): + wayland_portal.capture_png(timeout=0.05) + assert bus.events[-1] == "close", "the connection must not be left open" + + +def test_portal_reports_a_bus_that_went_away(): + """The bus dropping mid-request is a capture failure, not a crash.""" + bus = _FakeBus() + error = wayland_portal._dbus_client.DBusError("the session bus closed") + with _with_bus(bus), patch.object(bus, "call", side_effect=error): + with pytest.raises(AutoControlScreenException, + match="could not be reached"): + wayland_portal.capture_png(timeout=1.0) + + +def test_portal_captures_end_to_end_from_a_response_signal(tmp_path): + """The whole tier wired together: signal -> URI -> bytes -> cleanup. + + Only the URI-to-path step is stubbed, because a portal URI is an absolute + POSIX path that a Windows dev host cannot represent; that step has its own + tests above. + """ + target = tmp_path / "portal-shot.png" + target.write_bytes(_png()) + bus = _FakeBus(body=[0, {"uri": _URI}]) + seen = [] + + def resolve(value): + seen.append(value) + return str(target) + + with _with_bus(bus), patch.object(wayland_portal, "_path_from_uri", + side_effect=resolve): + data = wayland_portal.capture_png(timeout=5.0) + + assert seen == [_URI] + assert data == _png() + assert not target.exists() diff --git a/test/unit_test/headless/test_wayland_dbus_client.py b/test/unit_test/headless/test_wayland_dbus_client.py new file mode 100644 index 00000000..2be6a515 --- /dev/null +++ b/test/unit_test/headless/test_wayland_dbus_client.py @@ -0,0 +1,313 @@ +"""The hand-written D-Bus marshalling behind the xdg-desktop-portal tier. + +``je_auto_control.linux_wayland._dbus_client`` exists because a portal answers +with a signal *directed at the connection that asked*, so the subscription and +the call have to share one connection — which rules out shelling out to +``gdbus`` and leaves marshalling D-Bus by hand. + +Hand-written marshalling is exactly the kind of code that passes review and +fails on a real bus, so it is checked twice. Absolute correctness — that these +bytes are the bytes a real ``dbus-daemon`` accepts and that a real portal +answers — is settled in ``docker/portal_verify.py`` against a real bus. What +is checked here is everything a bus is not needed for: the alignment rules, +the signature walker, container types, address parsing, and that a message +this module writes is one it reads back unchanged. + +No session bus is touched: the socket is a loopback object that hands back +whatever was written to it. +""" +import struct + +import pytest + +from je_auto_control.linux_wayland import _dbus_client +from je_auto_control.linux_wayland._dbus_client import ( + DBusError, SessionBus, Variant, _Reader, _SignatureReader, _Writer, +) + + +class _LoopbackSocket: + """Whatever is written can be read back, so a message can round-trip.""" + + def __init__(self): + self.written = b"" + self.readable = b"" + self.closed = False + + def sendall(self, data): + self.written += data + + def settimeout(self, _timeout): + return None + + def recv(self, size): + chunk, self.readable = self.readable[:size], self.readable[size:] + return chunk + + def close(self): + self.closed = True + + +def _bus(): + """A connection whose socket is loopback, so nothing leaves the process.""" + bus = SessionBus(address="unix:path=/nonexistent") + bus._socket = _LoopbackSocket() + bus.unique_name = ":1.9" + return bus + + +# === Signatures ============================================================ + +@pytest.mark.parametrize("signature, expected", [ + ("s", ["s"]), + ("sa{sv}", ["s", "a{sv}"]), + ("ua{sv}", ["u", "a{sv}"]), + ("a(yv)", ["a(yv)"]), + ("aas", ["aas"]), + ("(us)o", ["(us)", "o"]), + ("a{sa{sv}}", ["a{sa{sv}}"]), +]) +def test_signature_walker_takes_one_whole_type_at_a_time(signature, expected): + """A container has to be followed to its close, however deeply it nests.""" + reader = _SignatureReader(signature) + taken = [] + while not reader.done(): + taken.append(reader.take_complete()) + assert taken == expected + + +# === Byte-exact marshalling ================================================ + +def test_a_string_is_length_then_bytes_then_a_nul(): + writer = _Writer() + writer.value("s", "abc") + assert writer.data == struct.pack("BBBBIII", ord("B"), _dbus_client.SIGNAL, 0, 1, 0, 1, 0) + deadline = _never() + with pytest.raises(DBusError, match="big-endian"): + bus.read_message(deadline=deadline) + + +# === Addresses ============================================================= + +@pytest.mark.parametrize("address, expected", [ + ("unix:path=/run/user/1000/bus", ("/run/user/1000/bus", False)), + ("unix:path=/tmp/dbus-x,guid=deadbeef", ("/tmp/dbus-x", False)), + ("unix:abstract=/tmp/dbus-y,guid=f00", ("/tmp/dbus-y", True)), + ("tcp:host=localhost,port=1;unix:path=/run/bus", ("/run/bus", False)), +]) +def test_a_bus_address_resolves_to_a_socket(address, expected): + assert _dbus_client._socket_target(address) == expected + + +def test_an_address_with_no_unix_transport_is_refused(): + with pytest.raises(DBusError, match="no usable unix transport"): + _dbus_client._socket_target("tcp:host=localhost,port=1234") + + +def test_availability_follows_the_environment(monkeypatch): + monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + assert _dbus_client.is_available() is False + assert _dbus_client.session_address() is None + monkeypatch.setenv("DBUS_SESSION_BUS_ADDRESS", "unix:path=/run/bus") + assert _dbus_client.is_available() is True + + +def test_connecting_without_an_address_says_which_variable_is_missing(monkeypatch): + monkeypatch.delenv("DBUS_SESSION_BUS_ADDRESS", raising=False) + bus = SessionBus() + with pytest.raises(DBusError, match="DBUS_SESSION_BUS_ADDRESS"): + bus.connect() + + +def test_the_sender_token_is_the_unique_name_as_a_path_element(): + """The portal builds the request path out of this, so both sides have to + mangle the name the same way or the client subscribes to the wrong path.""" + bus = SessionBus(address="unix:path=/x") + bus.unique_name = ":1.42" + assert bus.sender_token == "1_42" + + +def test_closing_twice_is_safe_and_closes_the_socket(): + bus = _bus() + socket = bus._socket + bus.close() + bus.close() + assert socket.closed + + +def test_a_dbus_error_is_part_of_the_framework_family(): + """Every containment boundary catches AutoControlException; a sibling of + it would escape all of them.""" + from je_auto_control.utils.exception.exceptions import AutoControlException + assert issubclass(DBusError, AutoControlException) + + +def _never() -> float: + """A deadline far enough away that the loopback socket always wins.""" + import time + return time.monotonic() + 60.0 diff --git a/test/unit_test/headless/test_wayland_libei.py b/test/unit_test/headless/test_wayland_libei.py index 00999e33..6756cfd6 100644 --- a/test/unit_test/headless/test_wayland_libei.py +++ b/test/unit_test/headless/test_wayland_libei.py @@ -1,29 +1,232 @@ -"""Tests for the libei backend selector + ctypes binding.""" -from unittest.mock import MagicMock +"""Tests for the libei backend selector, the portal hop, and the handshake. + +libei will not emit anything until a sender has bound a seat's capabilities, +taken a device *out of an event*, and started emulating on it — and every +emission has to be followed by a frame. These tests drive that state machine +against a fake libei whose entry points record what they were called with, so +the protocol order is pinned on any host. + +The one thing they cannot check is the ABI: the enum values and function +signatures come from ``libei.h`` and only a real Wayland session can confirm +them. What the tests *can* guarantee is the fail-closed property — every path +that does not reach a live device raises ``LibeiUnavailable``, which the +keyboard and mouse modules turn into "use the ydotool CLI". +""" +from unittest.mock import MagicMock, patch import pytest from je_auto_control.linux_wayland import ( - LibeiBackend, LibeiUnavailable, get_default_backend, - select_input_backend, + LibeiBackend, LibeiUnavailable, select_input_backend, ) from je_auto_control.linux_wayland import libei as libei_mod +from je_auto_control.linux_wayland import oeffis as oeffis_mod from je_auto_control.linux_wayland import ( _select_input as select_mod, ) +from je_auto_control.linux_wayland import _ydotool_cli + + +SEAT = 0x5EA7 +KEYBOARD_DEVICE = 0xD0001 +POINTER_DEVICE = 0xD0002 +SENDER = 0xEEEE + + +@pytest.fixture(autouse=True) +def _select_always_ready(): + """Report libei's fd readable without touching the real ``select``. + + On Linux ``ei_get_fd`` hands back a genuine epoll descriptor; the fake + returns a plain integer, and Windows' ``select`` accepts only sockets. + Stubbing the readiness check keeps these tests about the handshake + rather than about the host's poll implementation. + """ + with patch.object(libei_mod.select, "select", + side_effect=lambda r, _w, _x, _t: (list(r), [], [])): + yield + + +@pytest.fixture(autouse=True) +def _ydotool_generation_already_known(): + """Keep the ydotool 0.1.x probe out of the captured argv of these tests. + + The CLI fallback paths here assert on the exact argv the backend builds. + Before building any, both backends probe the installed ydotool once — + 0.1.x answers this argv with exit code 0 and no events — and that probe is + a ``subprocess.run`` call, so it would be captured alongside the argv + under test. ``test_wayland_ydotool_cli.py`` covers the probe itself. + """ + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + try: + yield + finally: + _ydotool_cli.reset_cache() + + +class FakeLibei: + """A libei that plays back a scripted event stream and records calls. + + Devices carry capabilities the way real ones do, so the backend has to + ask ``ei_device_has_capability`` rather than assume. + """ + + def __init__(self, events=None, capabilities=None, regions=None): + self.calls = [] + self.pending = list(events if events is not None else _full_handshake()) + self.capabilities = capabilities if capabilities is not None else { + KEYBOARD_DEVICE: {libei_mod.EI_DEVICE_CAP_KEYBOARD}, + POINTER_DEVICE: {libei_mod.EI_DEVICE_CAP_POINTER_ABSOLUTE, + libei_mod.EI_DEVICE_CAP_BUTTON, + libei_mod.EI_DEVICE_CAP_SCROLL}, + } + #: ``device -> [(x, y, width, height)]``. Empty by default, which is + #: the real "device declared no region" case: it accepts anything. + self.regions = dict(regions or {}) + self._queue = [] + self.lib = MagicMock() + # -- context --------------------------------------------------------- + def ei_new_sender(self, _user_data): + self.calls.append(("new_sender",)) + return SENDER -def _fake_symbols() -> object: - """Build a stand-in for :class:`_LibeiSymbols` we can introspect.""" - symbols = MagicMock() - symbols.ei_new_sender = MagicMock(return_value=0xDEADBEEF) - symbols.ei_setup_backend_socket = MagicMock(return_value=0) - symbols.ei_device_keyboard_key = MagicMock() - symbols.ei_device_pointer_motion_absolute = MagicMock() - symbols.ei_device_button_button = MagicMock() - symbols.ei_device_scroll = MagicMock() - symbols.ei_unref = MagicMock() - return symbols + def ei_setup_backend_fd(self, _ei, fd): + self.calls.append(("setup_backend_fd", fd)) + return 0 + + def ei_setup_backend_socket(self, _ei, path): + self.calls.append(("setup_backend_socket", path)) + return 0 + + def ei_get_fd(self, _ei): + return 0 # a real fd number is never polled: timeout is 0 + + def ei_dispatch(self, _ei): + # One dispatch moves the whole scripted batch into the event queue. + self._queue.extend(self.pending) + self.pending = [] + + def ei_get_event(self, _ei): + return self._queue.pop(0) if self._queue else None + + def ei_event_unref(self, _event): + return None + + def ei_now(self, _ei): + return 123456789 + + def ei_unref(self, _ei): + self.calls.append(("unref",)) + return None + + # -- events ---------------------------------------------------------- + def ei_event_get_type(self, event): + return event[0] + + def ei_event_get_seat(self, event): + return event[1] + + def ei_event_get_device(self, event): + return event[1] + + # -- seat / device --------------------------------------------------- + def ei_seat_bind_capabilities(self, *args): + self.calls.append(("bind", tuple( + getattr(a, "value", a) for a in args))) + + def ei_device_has_capability(self, device, cap): + return cap in self.capabilities.get(device, set()) + + def ei_device_ref(self, device): + self.calls.append(("device_ref", device)) + return device + + def ei_device_unref(self, device): + self.calls.append(("device_unref", device)) + return None + + def ei_device_start_emulating(self, device, sequence): + self.calls.append(("start_emulating", device, sequence)) + + def ei_device_stop_emulating(self, device): + self.calls.append(("stop_emulating", device)) + + def ei_device_frame(self, device, when): + self.calls.append(("frame", device, when)) + + def ei_device_keyboard_key(self, device, keycode, is_press): + self.calls.append(("key", device, keycode, is_press)) + + def ei_device_pointer_motion_absolute(self, device, x, y): + self.calls.append(("motion", device, x, y)) + + # -- regions --------------------------------------------------------- + # A region handle is ``(device, index)``: truthy, and enough to answer + # the four getters. libei ends the list by returning NULL. + def ei_device_get_region(self, device, index): + shapes = self.regions.get(device, ()) + return (device, index) if index < len(shapes) else None + + def _region(self, handle): + device, index = handle + return self.regions[device][index] + + def ei_region_get_x(self, handle): + return self._region(handle)[0] + + def ei_region_get_y(self, handle): + return self._region(handle)[1] + + def ei_region_get_width(self, handle): + return self._region(handle)[2] + + def ei_region_get_height(self, handle): + return self._region(handle)[3] + + def ei_device_button_button(self, device, button, is_press): + self.calls.append(("button", device, button, is_press)) + + def ei_device_scroll_discrete(self, device, x, y): + self.calls.append(("scroll", device, x, y)) + + +def _full_handshake(): + """The event stream a cooperating compositor produces.""" + return [ + (libei_mod.EI_EVENT_CONNECT, None), + (libei_mod.EI_EVENT_SEAT_ADDED, SEAT), + (libei_mod.EI_EVENT_DEVICE_ADDED, KEYBOARD_DEVICE), + (libei_mod.EI_EVENT_DEVICE_ADDED, POINTER_DEVICE), + (libei_mod.EI_EVENT_DEVICE_RESUMED, KEYBOARD_DEVICE), + (libei_mod.EI_EVENT_DEVICE_RESUMED, POINTER_DEVICE), + ] + + +def _portal_present(): + """Pretend liboeffis is installed, so the portal route is taken. + + Which route ``_open_backend`` picks is decided by that probe, not by the + injected connector — a host without liboeffis would otherwise fall + through to the well-known socket and these tests would silently stop + exercising the portal. + """ + return patch.object(libei_mod.oeffis, "is_available", return_value=True) + + +def _connected(fake=None, session=None): + """Return a backend that has completed the handshake over a fake fd.""" + fake = fake or FakeLibei() + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, session or MagicMock())) + with _portal_present(): + backend.connect(timeout=1.0) + return backend, fake + + +def _kinds(fake): + return [call[0] for call in fake.calls] # === Selector ============================================================= @@ -60,7 +263,390 @@ def test_select_input_backend_invalid_override_treated_as_auto(monkeypatch): }) == "cli" -# === LibeiBackend (mocked) ================================================ +# === ABI constants ======================================================== + +def test_capability_constants_match_libei_h(): + """``enum ei_device_capability`` is a bitmask, not a sequence. + + Checked against libei.h (Debian 1.5.0-3 and upstream main, which agree on + these six). This was wrong here — KEYBOARD had been guessed as 3 rather + than ``1 << 2`` — and the consequence was invisible: no device would ever + have reported the capability, so the handshake would have timed out and + every session silently fallen back to the ydotool CLI. + """ + assert libei_mod.EI_DEVICE_CAP_POINTER == 1 + assert libei_mod.EI_DEVICE_CAP_POINTER_ABSOLUTE == 2 + assert libei_mod.EI_DEVICE_CAP_KEYBOARD == 4 + assert libei_mod.EI_DEVICE_CAP_TOUCH == 8 + assert libei_mod.EI_DEVICE_CAP_SCROLL == 16 + assert libei_mod.EI_DEVICE_CAP_BUTTON == 32 + + +def test_event_type_constants_match_libei_h(): + """``enum ei_event_type`` starts at 1 and increments implicitly through + DEVICE_RESUMED before jumping to 90.""" + assert libei_mod.EI_EVENT_CONNECT == 1 + assert libei_mod.EI_EVENT_DISCONNECT == 2 + assert libei_mod.EI_EVENT_SEAT_ADDED == 3 + assert libei_mod.EI_EVENT_SEAT_REMOVED == 4 + assert libei_mod.EI_EVENT_DEVICE_ADDED == 5 + assert libei_mod.EI_EVENT_DEVICE_REMOVED == 6 + assert libei_mod.EI_EVENT_DEVICE_PAUSED == 7 + assert libei_mod.EI_EVENT_DEVICE_RESUMED == 8 + + +# === Handshake ============================================================ + +def test_connect_attaches_the_portal_fd_not_a_socket_path(): + """GNOME / KDE hand the EIS socket over as a file descriptor; a path + only exists on compositors that publish one.""" + _, fake = _connected() + assert ("setup_backend_fd", 7) in fake.calls + assert not any(call[0] == "setup_backend_socket" for call in fake.calls) + + +def test_connect_falls_back_to_the_well_known_socket_without_liboeffis(): + """Some compositors publish EIS at ``$XDG_RUNTIME_DIR/eis-0``; without + liboeffis that is the only route left, and it beats giving up.""" + fake = FakeLibei() + backend = LibeiBackend(symbols=fake) + with patch.object(libei_mod.oeffis, "is_available", return_value=False): + backend.connect(timeout=1.0) + setup = next(c for c in fake.calls if c[0] == "setup_backend_socket") + assert setup[1].endswith(b"eis-0") + + +def test_connect_binds_the_seat_capabilities_it_will_emit(): + _, fake = _connected() + bind = next(call for call in fake.calls if call[0] == "bind") + # seat first, then each capability, then the NULL terminator. + assert bind[1][0] == SEAT + assert set(bind[1][1:-1]) == set(libei_mod._WANTED_CAPS) + assert bind[1][-1] is None + + +def test_connect_starts_emulating_on_every_resumed_device(): + _, fake = _connected() + started = [call for call in fake.calls if call[0] == "start_emulating"] + assert {call[1] for call in started} == {KEYBOARD_DEVICE, POINTER_DEVICE} + # The sequence number increases per device, as libei requires. + assert [call[2] for call in started] == [1, 2] + + +def test_devices_are_taken_from_events_never_from_the_context(): + """The defect this rewrite exists for: the old binding passed the ``ei`` + context to entry points that take an ``ei_device``.""" + backend, fake = _connected() + backend.press_key(30) + for call in fake.calls: + if call[0] in ("key", "motion", "button", "scroll", "frame", + "start_emulating"): + assert call[1] in (KEYBOARD_DEVICE, POINTER_DEVICE) + assert call[1] != SENDER + + +def test_handshake_times_out_when_no_device_ever_resumes(): + """A seat that never resumes a device must not hang the caller.""" + fake = FakeLibei(events=[(libei_mod.EI_EVENT_SEAT_ADDED, SEAT)]) + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, MagicMock())) + with _portal_present(), pytest.raises(LibeiUnavailable, + match="handshake timeout"): + backend.connect(timeout=0.05) + + +def test_handshake_rejects_a_partial_grant(): + """Only a pointer and no keyboard would leave every key press silently + doing nothing, so a partial grant counts as no grant.""" + fake = FakeLibei( + events=[(libei_mod.EI_EVENT_SEAT_ADDED, SEAT), + (libei_mod.EI_EVENT_DEVICE_ADDED, POINTER_DEVICE), + (libei_mod.EI_EVENT_DEVICE_RESUMED, POINTER_DEVICE)], + ) + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, MagicMock())) + with _portal_present(), pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + + +def test_a_denied_portal_request_reads_as_libei_unavailable(): + """A dismissed consent dialog is an ordinary fall-back-to-CLI, not a + crash and not a hang.""" + def refuse(): + raise oeffis_mod.OeffisUnavailable("the user dismissed the dialog") + + fake = FakeLibei() + backend = LibeiBackend(symbols=fake, portal_connect=refuse) + with _portal_present(), pytest.raises(LibeiUnavailable, match="dismissed"): + backend.connect(timeout=1.0) + # The sender is released rather than leaked on the way out. + assert ("unref",) in fake.calls + + +def test_connect_abandons_an_opened_context_rather_than_unref_it(): + """``ei_unref`` segfaults on libei 1.3.901 once the backend is open. + + Measured against the real library in ``docker/libei_verify.py``: safe + before setup and after a *failed* setup, SIGSEGV after a successful one. + A crash in a library driving someone's desktop is worse than leaking one + context per process, so an opened backend is dropped without unref. + """ + fake = FakeLibei(events=[]) + session = MagicMock() + backend = LibeiBackend(symbols=fake, portal_connect=lambda: (7, session)) + with _portal_present(), pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + assert ("unref",) not in fake.calls + # The portal session is liboeffis, a different library, and closing it is + # what actually revokes the grant — so that still has to happen. + session.close.assert_called_once() + + +def test_a_context_that_never_opened_a_backend_is_unreffed(): + """The other half of the rule: before any backend is set up, ``unref`` is + the documented release and is safe, so it must still run.""" + fake = FakeLibei() + + def refuse(): + raise oeffis_mod.OeffisUnavailable("no portal") + + backend = LibeiBackend(symbols=fake, portal_connect=refuse) + with _portal_present(), pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + assert ("unref",) in fake.calls + + +def test_a_failed_backend_setup_is_unreffed(): + """``ei_setup_backend_socket`` returning non-zero leaves the context in + the state the header documents as releasable, and the probe confirms is + safe.""" + class _SetupFails(FakeLibei): + def ei_setup_backend_socket(self, _ei, path): + self.calls.append(("setup_backend_socket", path)) + return -2 + + fake = _SetupFails() + backend = LibeiBackend(symbols=fake) + with pytest.raises(LibeiUnavailable, match="returned -2"): + backend.connect(timeout=0.05, socket_path=b"/nope") + assert ("unref",) in fake.calls + + +# === Emission ============================================================= + +def test_every_emission_is_followed_by_a_frame(): + """libei buffers until ``ei_device_frame``; without it the compositor + sees nothing, which is how the old binding could look correct and emit + nothing at all.""" + backend, fake = _connected() + backend.press_key(30) + backend.set_position(4, 9) + backend.press_button(272) + backend.scroll(0, 3) + kinds = _kinds(fake) + for emission in ("key", "motion", "button", "scroll"): + index = kinds.index(emission) + assert kinds[index + 1] == "frame", f"{emission} was not framed" + + +def test_scroll_sends_whole_wheel_clicks_not_raw_detents(): + """libei measures discrete scroll in 120ths of a click, like WHEEL_DELTA. + + Passing a raw detent count is a 120th of the scroll asked for, and libei + says so at runtime — "suspicious discrete event value 1, did you mean + 120?" — which is how ``docker/eis_verify.py`` found it against a real EIS + peer. It reads the value back off the wire there; this pins the unit + without needing one.""" + backend, fake = _connected() + + def _last_scroll(): + return [call for call in fake.calls if call[0] == "scroll"][-1][2:] + + backend.scroll(0, 3) + assert _last_scroll() == (0, 3 * libei_mod.SCROLL_UNIT) + backend.scroll(-2, 0) + assert _last_scroll() == (-2 * libei_mod.SCROLL_UNIT, 0), \ + "the sign has to survive the conversion too" + + +def test_key_press_and_release_use_the_keyboard_device(): + backend, fake = _connected() + backend.press_key(30) + backend.release_key(30) + keys = [call for call in fake.calls if call[0] == "key"] + assert keys == [("key", KEYBOARD_DEVICE, 30, True), + ("key", KEYBOARD_DEVICE, 30, False)] + + +def test_pointer_motion_uses_the_absolute_pointer_device(): + backend, fake = _connected() + backend.set_position(120, 340) + assert ("motion", POINTER_DEVICE, 120.0, 340.0) in fake.calls + + +# === Absolute pointer regions ============================================= +# +# libei drops an absolute motion that lands in no region and reports nothing +# about it: no return code, no event, no error the caller can see. Measured +# against a real EIS peer, which is also where the rest of this section's +# expectations come from — see ``docker/eis_verify.py``. + +def _with_regions(regions): + """A connected backend whose absolute pointer advertises ``regions``.""" + return _connected(FakeLibei(regions={POINTER_DEVICE: regions})) + + +def test_a_device_with_no_region_takes_the_coordinate_unchanged(): + """The common case, and the one every other test here relies on: a + device that declared no region accepts anything, so nothing is mapped.""" + backend, fake = _connected() + backend.set_position(4000, 3000) + assert ("motion", POINTER_DEVICE, 4000.0, 3000.0) in fake.calls + + +def test_a_point_inside_a_region_is_sent_as_it_stands(): + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + backend.set_position(640, 400) + assert ("motion", POINTER_DEVICE, 640.0, 400.0) in fake.calls + + +def test_region_offsets_are_part_of_the_coordinate_not_stripped(): + """A region at ``x=1280`` takes 1380 for a point 100 pixels into it.""" + backend, fake = _with_regions([(1280, 0, 1920, 1080)]) + backend.set_position(1380, 100) + assert ("motion", POINTER_DEVICE, 1380.0, 100.0) in fake.calls + + +def test_the_far_edges_of_a_region_are_outside_it(monkeypatch): + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + backend.set_position(1919, 1079) + assert ("motion", POINTER_DEVICE, 1919.0, 1079.0) in fake.calls + with pytest.raises(LibeiUnavailable, match="outside every region"): + backend.set_position(1920, 1080) + + +def test_a_point_no_region_covers_is_refused_not_silently_dropped(monkeypatch): + """The bug this section exists for. Without the check libei swallows the + motion and ``set_position`` returns as though the pointer had moved.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + with pytest.raises(LibeiUnavailable, match=r"\(4000, 30\) lies outside"): + backend.set_position(4000, 30) + assert not [call for call in fake.calls if call[0] == "motion"] + + +def test_a_refused_motion_is_not_committed_with_a_frame(monkeypatch): + """Nothing was buffered, so nothing may be flushed — a frame here would + commit whatever the previous emission left on the device.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + with pytest.raises(LibeiUnavailable): + backend.set_position(4000, 30) + assert not [call for call in fake.calls if call[0] == "frame"] + + +def test_a_negative_layout_origin_is_normalised_into_region_space(monkeypatch): + """Region offsets are ``uint32``, so a compositor cannot advertise one + left of the origin — but a monitor placed left of the primary gives this + project's layout a negative origin. Input has to make the same shift + capture already makes, or ``get_pixel`` and ``set_position`` name + different pixels.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (-1280, 0)) + backend, fake = _with_regions([(0, 0, 1280, 1024), (1280, 0, 1920, 1080)]) + backend.set_position(-1280, 10) + assert ("motion", POINTER_DEVICE, 0.0, 10.0) in fake.calls + + +def test_normalising_is_not_attempted_on_a_layout_that_starts_at_zero( + monkeypatch): + """A zero origin makes the shift a no-op, so an out-of-region point must + still be refused rather than sent twice.""" + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + with pytest.raises(LibeiUnavailable): + backend.set_position(-5, -5) + assert not [call for call in fake.calls if call[0] == "motion"] + + +def test_a_point_outside_even_after_normalising_is_still_refused(monkeypatch): + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (-1280, 0)) + backend, fake = _with_regions([(0, 0, 1280, 1024)]) + with pytest.raises(LibeiUnavailable, match="outside every region"): + backend.set_position(9000, 9000) + assert not [call for call in fake.calls if call[0] == "motion"] + + +def test_the_layout_origin_is_only_consulted_when_a_point_misses(): + """It costs a ``wlr-randr`` subprocess, so it must stay off the path + every ordinary mouse move takes.""" + asked = [] + with patch.object(libei_mod, "layout_origin", + side_effect=lambda: asked.append(1) or (0, 0)): + backend, _fake = _with_regions([(0, 0, 1920, 1080)]) + backend.set_position(640, 400) + assert asked == [] + + +def test_layout_origin_is_zero_when_the_layout_cannot_be_read(): + """GNOME and KDE have no ``wlr-randr``; they also normalise the layout + themselves, so zero is the right answer rather than a fallback.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + side_effect=OSError("no wlr-randr")): + assert libei_mod.layout_origin() == (0, 0) + + +def test_region_enumeration_stops_at_the_ceiling(): + """A libei whose contract differs must not hang the caller's mouse.""" + fake = FakeLibei() + fake.ei_device_get_region = lambda _device, index: ("endless", index) + fake.ei_region_get_x = lambda _handle: 0 + fake.ei_region_get_y = lambda _handle: 0 + fake.ei_region_get_width = lambda _handle: 1 + fake.ei_region_get_height = lambda _handle: 1 + backend, _fake = _connected(fake) + assert len(backend._device_regions(POINTER_DEVICE)) == libei_mod._MAX_REGIONS + + +def test_a_motion_refused_for_its_region_reaches_the_cli(monkeypatch): + """End to end: the refusal is the *point*, because ``emitted`` turns it + into the ydotool move that libei would have swallowed.""" + from je_auto_control.linux_wayland import mouse as wayland_mouse + monkeypatch.setattr(libei_mod, "layout_origin", lambda: (0, 0)) + backend, fake = _with_regions([(0, 0, 1920, 1080)]) + captured = [] + + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.set_position(4000, 30) + assert not [call for call in fake.calls if call[0] == "motion"] + assert captured[0][1:] == ["mousemove", "--absolute", + "-x", "4000", "-y", "30"] + + +def test_click_button_presses_then_releases(): + backend, fake = _connected() + backend.click_button(272) + buttons = [call for call in fake.calls if call[0] == "button"] + assert buttons == [("button", POINTER_DEVICE, 272, True), + ("button", POINTER_DEVICE, 272, False)] + + +def test_a_paused_device_stops_accepting_emissions(): + """Pause arrives asynchronously; emitting into a paused device would be + silently dropped, so it has to raise and let the CLI take over.""" + backend, fake = _connected() + fake.pending = [(libei_mod.EI_EVENT_DEVICE_PAUSED, KEYBOARD_DEVICE)] + with pytest.raises(LibeiUnavailable, match="no libei device"): + backend.press_key(30) + + +def test_emitting_before_connect_raises(): + backend = LibeiBackend(symbols=FakeLibei()) + with pytest.raises(LibeiUnavailable, match="not connected"): + backend.press_key(30) + def test_backend_reports_unavailable_when_symbols_missing(): backend = LibeiBackend(symbols=None) @@ -69,168 +655,263 @@ def test_backend_reports_unavailable_when_symbols_missing(): backend.connect() -def test_backend_connect_calls_new_sender_and_setup_socket(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"./eis-test") - symbols.ei_new_sender.assert_called_once_with(b"je_auto_control") - symbols.ei_setup_backend_socket.assert_called_once_with( - 0xDEADBEEF, b"./eis-test", - ) +def test_disconnect_ends_the_portal_session_and_releases_a_live_context(): + """A completed handshake is released properly — devices, then the context. + ``ei_unref`` is only unsafe on a context whose backend opened but whose + handshake never progressed; on a live one it is safe, measured against a + real EIS peer in ``docker/eis_verify.py``. The portal session is a + different library and always closes, or the compositor keeps the + remote-desktop grant open.""" + session = MagicMock() + backend, fake = _connected(session=session) + backend.disconnect() + session.close.assert_called_once() + assert backend.is_connected is False + assert ("unref",) in fake.calls + devices = [call for call in fake.calls if call[0] == "device_unref"] + assert devices, "the devices were dropped without being unreffed" + assert _kinds(fake).index("unref") > _kinds(fake).index("device_unref"), \ + "the context was unreffed before its devices, so those are use-after-free" + + +def test_a_handshake_that_never_completed_abandons_the_context(): + """The one state where ``ei_unref`` segfaults must still be abandoned. + + ``docker/libei_verify.py`` measures it: a backend that opened but never + handshook takes the process down on unref, and on ``ei_disconnect`` too, + so it is not a refcount mistake here.""" + fake = FakeLibei(events=[]) # nothing arrives, so no device goes live + backend = LibeiBackend(symbols=fake, + portal_connect=lambda: (7, MagicMock())) + with _portal_present(): + with pytest.raises(LibeiUnavailable): + backend.connect(timeout=0.05) + assert ("unref",) not in fake.calls + + +# === Probe caching ======================================================== + +def test_a_failed_probe_is_not_retried_on_every_keystroke(): + """The probe costs a portal round trip and possibly a consent dialog. + Paying that per key press would be worse than not having libei at all.""" + libei_mod.reset_default_backend() + attempts = [] -def test_backend_connect_is_idempotent_within_one_instance(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.connect(socket_path=b"/x") - assert symbols.ei_new_sender.call_count == 1 + class _Failing(LibeiBackend): + def __init__(self, **_kwargs): + super().__init__(symbols=FakeLibei()) + def connect(self, **_kwargs): + attempts.append(1) + raise LibeiUnavailable("no portal here") -def test_backend_connect_unrefs_on_setup_failure(): - symbols = _fake_symbols() - symbols.ei_setup_backend_socket.return_value = -1 - backend = LibeiBackend(symbols=symbols) - with pytest.raises(LibeiUnavailable, match="setup_backend_socket"): - backend.connect(socket_path=b"/x") - symbols.ei_unref.assert_called_once_with(0xDEADBEEF) + try: + with patch.object(libei_mod, "LibeiBackend", _Failing): + assert libei_mod.connected_backend() is None + assert libei_mod.connected_backend() is None + assert libei_mod.connected_backend() is None + finally: + libei_mod.reset_default_backend() + assert len(attempts) == 1 -def test_backend_press_key_calls_keyboard_key_with_state_one(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.press_key(28) # KEY_ENTER - symbols.ei_device_keyboard_key.assert_called_once_with( - 0xDEADBEEF, 28, 1, - ) +def test_active_backend_is_none_when_the_selector_says_cli(monkeypatch): + monkeypatch.setenv("JE_AUTOCONTROL_WAYLAND_INPUT_BACKEND", "cli") + assert select_mod.active_backend() is None -def test_backend_release_key_uses_state_zero(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.release_key(28) - symbols.ei_device_keyboard_key.assert_called_once_with( - 0xDEADBEEF, 28, 0, - ) +# === Keyboard / mouse fall back rather than fail ========================== +def test_keyboard_falls_back_to_ydotool_when_libei_is_unavailable(): + import subprocess -def test_backend_set_position_uses_absolute_motion(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.set_position(120, 240) - symbols.ei_device_pointer_motion_absolute.assert_called_once_with( - 0xDEADBEEF, 120.0, 240.0, - ) + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + captured = [] + with patch.object(wayland_keyboard, "_try_libei", return_value=None), \ + patch.object(wayland_keyboard, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_keyboard.subprocess, "run", + side_effect=lambda argv, **kw: ( + captured.append(list(argv)) + or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep + wayland_keyboard.press_key(28) + assert captured == [["/usr/bin/ydotool", "key", "28:1"]] -def test_backend_click_button_sends_press_then_release(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.click_button(272) - calls = symbols.ei_device_button_button.call_args_list - assert len(calls) == 2 - assert calls[0].args == (0xDEADBEEF, 272, 1) - assert calls[1].args == (0xDEADBEEF, 272, 0) +def test_keyboard_uses_libei_when_it_is_connected(): + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + backend = MagicMock() + with patch.object(wayland_keyboard, "_try_libei", return_value=backend): + wayland_keyboard.press_key(28) + backend.press_key.assert_called_once_with(28) -def test_backend_scroll_forwards_float_deltas(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.scroll(0, 3) - symbols.ei_device_scroll.assert_called_once_with(0xDEADBEEF, 0.0, 3.0) +def test_mouse_button_maps_onto_an_evdev_code_for_libei(): + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() + with patch.object(wayland_mouse, "_try_libei", return_value=backend): + wayland_mouse.press_mouse(wayland_mouse.wayland_mouse_left) + # BTN_LEFT, not ydotool's 0xC0 bitmask. + backend.press_button.assert_called_once_with(272) -def test_backend_disconnect_releases_handle(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - backend.connect(socket_path=b"/x") - backend.disconnect() - symbols.ei_unref.assert_called_once_with(0xDEADBEEF) +def _scroll_calls(backend): + return [tuple(call.args) for call in backend.scroll.call_args_list] -def test_backend_method_before_connect_raises(): - symbols = _fake_symbols() - backend = LibeiBackend(symbols=symbols) - with pytest.raises(LibeiUnavailable, match="not connected"): - backend.press_key(28) +def test_mouse_scroll_flips_the_vertical_axis_for_libei(): + """The two paths count wheel detents in opposite directions. -def test_get_default_backend_returns_none_when_unavailable(monkeypatch): - libei_mod.reset_default_backend() - monkeypatch.setattr(libei_mod, "_load_symbols", lambda: None) - assert get_default_backend() is None - libei_mod.reset_default_backend() + This module's ``wayland_scroll_direction_*`` constants are in the + kernel's ``REL_WHEEL`` frame — what ydotool writes into ``/dev/uinput``, + where positive is up. libei is in the ``wl_pointer`` / libinput frame, + where positive is down: libinput's own evdev reader negates ``REL_WHEEL`` + to get there. Handing the constant over unchanged would scroll the wrong + way on every libei host, and scrolling the wrong way fails silently. + """ + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + patch.object(wayland_mouse, "binary_path", return_value=None): + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_up) + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_down) + # binary_path is None, so any fall-through to the CLI would have raised. + assert _scroll_calls(backend) == [(0, -5), (0, 5)] -def test_get_default_backend_caches_single_instance(monkeypatch): - libei_mod.reset_default_backend() - symbols = _fake_symbols() - monkeypatch.setattr(libei_mod, "_load_symbols", lambda: symbols) - a = get_default_backend() - b = get_default_backend() - assert a is b - libei_mod.reset_default_backend() +def test_mouse_scroll_does_not_flip_the_horizontal_axis_for_libei(): + """``REL_HWHEEL`` and libinput agree that right is positive, and + libinput passes that axis through unnegated — so flipping both axes + would be as wrong as flipping neither.""" + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() -# === Library probe ======================================================== + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + patch.object(wayland_mouse, "binary_path", return_value=None): + wayland_mouse.scroll(3, wayland_mouse.wayland_scroll_direction_right) + wayland_mouse.scroll(3, wayland_mouse.wayland_scroll_direction_left) + assert _scroll_calls(backend) == [(3, 0), (-3, 0)] -def test_try_load_library_returns_none_when_find_library_fails(monkeypatch): - monkeypatch.setattr(libei_mod.ctypes.util, "find_library", - lambda _name: None) - assert libei_mod._try_load_library() is None +def _cli_capture(module, captured): + """Patch ``module``'s ydotool lookup and subprocess.run, recording argv.""" + import subprocess -# === Keyboard integration ================================================ + return ( + patch.object(module, "binary_path", return_value="/usr/bin/ydotool"), + patch.object(module.subprocess, "run", + side_effect=lambda argv, **kw: ( + captured.append(list(argv)) + or subprocess.CompletedProcess(argv, 0, b"", b""))), # nosemgrep + ) -def test_keyboard_press_key_uses_libei_when_selected(monkeypatch): - from je_auto_control.linux_wayland import keyboard as wayland_kb - libei_mock = MagicMock() - monkeypatch.setattr(wayland_kb, "_try_libei", lambda: libei_mock) - wayland_kb.press_key(28) - libei_mock.press_key.assert_called_once_with(28) +def test_libei_unavailable_is_catchable_as_an_autocontrol_error(): + """It used to inherit ``RuntimeError`` alone, so every ``except + AutoControlException`` boundary — the executor, the poll loops, the + request handlers, the GUI slots — let it straight through.""" + from je_auto_control.utils.exception.exceptions import AutoControlException -def test_keyboard_falls_back_to_ydotool_when_libei_unavailable(monkeypatch): - from je_auto_control.linux_wayland import keyboard as wayland_kb - monkeypatch.setattr(wayland_kb, "_try_libei", lambda: None) - monkeypatch.setattr(wayland_kb, "binary_path", - lambda _name: "/usr/bin/ydotool") - monkeypatch.setattr(wayland_kb.subprocess, "run", - lambda *_args, **_kw: None) - # Should not raise. - wayland_kb.press_key(28) + assert issubclass(LibeiUnavailable, AutoControlException) + # The probes in _select_input and the two input modules catch + # RuntimeError; that has to keep working. + assert issubclass(LibeiUnavailable, RuntimeError) -def test_mouse_set_position_uses_libei_when_selected(monkeypatch): +def test_a_refused_emission_falls_back_to_the_cli(): + """A backend that finished its handshake can still refuse one emission — + a paused device, a session that ended between calls. libei is documented + as the fast path and never the only one, but the refusal used to escape + the module instead of reaching ydotool.""" from je_auto_control.linux_wayland import mouse as wayland_mouse - libei_mock = MagicMock() - monkeypatch.setattr(wayland_mouse, "_try_libei", lambda: libei_mock) - wayland_mouse.set_position(100, 200) - libei_mock.set_position.assert_called_once_with(100, 200) + backend = MagicMock() + backend.set_position.side_effect = LibeiUnavailable("device paused") + captured = [] + + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.set_position(120, 240) + assert captured[0][1:] == ["mousemove", "--absolute", + "-x", "120", "-y", "240"] -def test_mouse_falls_back_to_ydotool_when_libei_unavailable(monkeypatch): +def test_a_refused_scroll_falls_back_to_the_cli_in_ydotools_own_frame(): + """And it falls back *unflipped*: the vertical flip belongs to libei.""" from je_auto_control.linux_wayland import mouse as wayland_mouse - monkeypatch.setattr(wayland_mouse, "_try_libei", lambda: None) - monkeypatch.setattr(wayland_mouse, "binary_path", - lambda _name: "/usr/bin/ydotool") - monkeypatch.setattr(wayland_mouse.subprocess, "run", - lambda *_args, **_kw: None) - wayland_mouse.set_position(0, 0) + backend = MagicMock() + backend.scroll.side_effect = LibeiUnavailable("device paused") + captured = [] + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_up) + assert captured[0][1:] == ["mousemove", "--wheel", "-x", "0", "-y", "5"] -# === Facade =============================================================== -def test_facade_exports_libei_helpers(): - from je_auto_control.linux_wayland import ( - LibeiBackend as LB, LibeiUnavailable as LU, - get_default_backend as gdb, select_input_backend as sib, - ) - assert LB is LibeiBackend - assert LU is LibeiUnavailable - assert callable(gdb) and callable(sib) +def test_a_refused_button_release_still_reaches_the_cli(): + """The worst refusal to drop: the press landed, so giving up on the + release leaves the button held for the rest of the session.""" + from je_auto_control.linux_wayland import mouse as wayland_mouse + backend = MagicMock() + backend.release_button.side_effect = LibeiUnavailable("device paused") + captured = [] + + binary, run = _cli_capture(wayland_mouse, captured) + with patch.object(wayland_mouse, "_try_libei", return_value=backend), \ + binary, run: + wayland_mouse.click_mouse(wayland_mouse.wayland_mouse_left) + backend.press_button.assert_called_once_with(272) + # 0xC0 with the down-bit cleared: a release-only click. + assert captured[0][1:] == ["click", "0x80"] + + +def test_a_refused_key_press_falls_back_to_the_cli(): + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + backend = MagicMock() + backend.press_key.side_effect = LibeiUnavailable("device paused") + captured = [] + + binary, run = _cli_capture(wayland_keyboard, captured) + with patch.object(wayland_keyboard, "_try_libei", return_value=backend), \ + binary, run: + wayland_keyboard.press_key(28) + assert captured[0][1:] == ["key", "28:1"] + + +def test_a_chord_refused_part_way_releases_what_it_already_pressed(): + """Otherwise the fallback presses Ctrl again on top of a Ctrl that libei + never released, and the modifier is stuck once the CLI chord ends.""" + from je_auto_control.linux_wayland import keyboard as wayland_keyboard + backend = MagicMock() + backend.press_key.side_effect = [None, LibeiUnavailable("device paused")] + captured = [] + + binary, run = _cli_capture(wayland_keyboard, captured) + with patch.object(wayland_keyboard, "_try_libei", return_value=backend), \ + binary, run: + wayland_keyboard.hotkey([29, 42]) + backend.release_key.assert_called_once_with(29) + assert captured[0][1:] == ["key", "29:1", "42:1", "42:0", "29:0"] + + +def test_mouse_scroll_falls_back_to_ydotool_without_libei(): + """No libei means the ydotool argv, in ydotool's own frame — unflipped.""" + import subprocess + + from je_auto_control.linux_wayland import mouse as wayland_mouse + captured = [] + + with patch.object(wayland_mouse, "_try_libei", return_value=None), \ + patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=lambda argv, **kw: ( + captured.append(list(argv)) + or subprocess.CompletedProcess(argv, 0, b"", b""))): # nosemgrep + wayland_mouse.scroll(5, wayland_mouse.wayland_scroll_direction_down) + assert captured[0][1:] == ["mousemove", "--wheel", "-x", "0", "-y", "-5"] diff --git a/test/unit_test/headless/test_wayland_oeffis.py b/test/unit_test/headless/test_wayland_oeffis.py new file mode 100644 index 00000000..7b50939f --- /dev/null +++ b/test/unit_test/headless/test_wayland_oeffis.py @@ -0,0 +1,165 @@ +"""The portal hop that hands libei an EIS file descriptor. + +``ConnectToEIS`` returns a file descriptor over D-Bus, which no command-line +tool can pass into this process — so unlike the Screenshot portal there is no +``gdbus`` shortcut, and liboeffis does the session dance instead. These tests +drive its event loop against a fake library: the success path, the user +declining, the portal going away, and the timeout. All four have to end in a +released context, because a leaked oeffis handle keeps a remote-desktop grant +open on the user's session. +""" +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import oeffis as oeffis_mod + + +HANDLE = 0x0EFF15 + + +class FakeOeffis: + """A liboeffis whose event stream the test scripts.""" + + def __init__(self, events=(), eis_fd=11, error=b""): + self.calls = [] + self._events = list(events) + self._eis_fd = eis_fd + self._error = error + + def oeffis_new(self, _user_data): + self.calls.append(("new",)) + return HANDLE + + def oeffis_unref(self, handle): + self.calls.append(("unref", handle)) + return None + + def oeffis_create_session(self, handle, devices): + self.calls.append(("create_session", handle, devices)) + + def oeffis_get_fd(self, _handle): + return 0 + + def oeffis_dispatch(self, _handle): + self.calls.append(("dispatch",)) + + def oeffis_get_event(self, _handle): + return self._events.pop(0) if self._events else oeffis_mod.OEFFIS_EVENT_NONE + + def oeffis_get_eis_fd(self, _handle): + return self._eis_fd + + def oeffis_get_error_message(self, _handle): + return self._error + + +@pytest.fixture(autouse=True) +def _select_always_ready(): + """Report the oeffis fd readable; the fake's fd is not a real socket.""" + with patch.object(oeffis_mod.select, "select", + side_effect=lambda r, _w, _x, _t: (list(r), [], [])): + yield + + +def test_connect_returns_the_eis_fd_and_a_live_session(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS]) + eis_fd, session = oeffis_mod.connect_eis_fd(symbols=fake) + assert eis_fd == 11 + # The session must NOT be released yet: dropping it ends the grant. + assert ("unref", HANDLE) not in fake.calls + session.close() + assert ("unref", HANDLE) in fake.calls + + +def test_connect_requests_only_the_devices_this_backend_emits(): + """The consent dialog shows the user what is being granted, so asking for + a touchscreen this backend never drives would widen the grant for no + benefit.""" + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS]) + oeffis_mod.connect_eis_fd(symbols=fake) + created = next(c for c in fake.calls if c[0] == "create_session") + assert created[2] == (oeffis_mod.OEFFIS_DEVICE_KEYBOARD + | oeffis_mod.OEFFIS_DEVICE_POINTER) + assert not created[2] & oeffis_mod.OEFFIS_DEVICE_TOUCHSCREEN + + +def test_event_constants_match_liboeffis_h(): + """Checked against liboeffis.h (Debian 1.5.0-3 and upstream main, which + agree). CLOSED precedes DISCONNECTED — the opposite of what the names + suggest, and it was wrong here until the header was read. + """ + assert oeffis_mod.OEFFIS_EVENT_NONE == 0 + assert oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS == 1 + assert oeffis_mod.OEFFIS_EVENT_CLOSED == 2 + assert oeffis_mod.OEFFIS_EVENT_DISCONNECTED == 3 + + +def test_device_constants_match_liboeffis_h(): + assert oeffis_mod.OEFFIS_DEVICE_ALL_DEVICES == 0 + assert oeffis_mod.OEFFIS_DEVICE_KEYBOARD == 1 + assert oeffis_mod.OEFFIS_DEVICE_POINTER == 2 + assert oeffis_mod.OEFFIS_DEVICE_TOUCHSCREEN == 4 + + +def test_a_closed_session_reads_as_the_user_declining(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CLOSED], + error=b"permission denied") + with pytest.raises(oeffis_mod.OeffisUnavailable, match="closed"): + oeffis_mod.connect_eis_fd(symbols=fake) + assert ("unref", HANDLE) in fake.calls + + +def test_the_librarys_own_error_text_reaches_the_message(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CLOSED], + error=b"permission denied") + with pytest.raises(oeffis_mod.OeffisUnavailable, match="permission denied"): + oeffis_mod.connect_eis_fd(symbols=fake) + + +def test_a_disconnect_is_reported_and_the_handle_released(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_DISCONNECTED]) + with pytest.raises(oeffis_mod.OeffisUnavailable, match="disconnected"): + oeffis_mod.connect_eis_fd(symbols=fake) + assert ("unref", HANDLE) in fake.calls + + +def test_an_unanswered_consent_dialog_times_out(): + """Nobody clicking the dialog must not wedge the caller's script.""" + fake = FakeOeffis(events=[]) + with pytest.raises(oeffis_mod.OeffisUnavailable, match="did not answer"): + oeffis_mod.connect_eis_fd(symbols=fake, timeout=0.05) + assert ("unref", HANDLE) in fake.calls + + +def test_a_success_without_an_fd_is_still_a_failure(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS], + eis_fd=-1) + with pytest.raises(oeffis_mod.OeffisUnavailable, match="no EIS fd"): + oeffis_mod.connect_eis_fd(symbols=fake) + assert ("unref", HANDLE) in fake.calls + + +def test_a_null_context_is_reported(): + class _NoContext(FakeOeffis): + def oeffis_new(self, _user_data): + return 0 + + symbols = _NoContext() + with pytest.raises(oeffis_mod.OeffisUnavailable, match="NULL"): + oeffis_mod.connect_eis_fd(symbols=symbols) + + +def test_missing_library_reads_as_unavailable(): + with patch.object(oeffis_mod, "load_symbols", return_value=None): + assert oeffis_mod.is_available() is False + with pytest.raises(oeffis_mod.OeffisUnavailable, match="not found"): + oeffis_mod.connect_eis_fd() + + +def test_closing_a_session_twice_is_harmless(): + fake = FakeOeffis(events=[oeffis_mod.OEFFIS_EVENT_CONNECTED_TO_EIS]) + _, session = oeffis_mod.connect_eis_fd(symbols=fake) + session.close() + session.close() + assert [c for c in fake.calls if c[0] == "unref"] == [("unref", HANDLE)] diff --git a/test/unit_test/headless/test_wayland_pointer_accel.py b/test/unit_test/headless/test_wayland_pointer_accel.py new file mode 100644 index 00000000..bfbf5d2c --- /dev/null +++ b/test/unit_test/headless/test_wayland_pointer_accel.py @@ -0,0 +1,171 @@ +"""The operator declares whether the compositor's pointer acceleration is off. + +``mousemove --absolute`` is relative motion under the hood, so the compositor +scales it: measured against a real wlroots session, libinput's default +adaptive profile lands the cursor twice as far from the corner as asked. The +factor is compositor configuration and no client can read it back, so the +library cannot compensate — only the operator knows. ``POINTER_ACCEL_ENV`` is +how they say so, and these tests pin what each answer does. + +The libei path is absolute at the protocol level, so none of this applies to +it; the last test holds that line. +""" +import subprocess +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import ( + _layout, _ydotool_cli, mouse as wayland_mouse, +) +from je_auto_control.utils.exception.exceptions import AutoControlException + + +@pytest.fixture(autouse=True) +def _pin_the_cli_input_path(monkeypatch): + """Force the ydotool argv and re-arm the once-per-process warnings.""" + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + monkeypatch.delenv(wayland_mouse.POINTER_ACCEL_ENV, raising=False) + try: + with patch.object(wayland_mouse, "_try_libei", return_value=None): + yield + finally: + _ydotool_cli.reset_cache() + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + + +def _move(x=10, y=20): + """Run ``set_position`` on the CLI path; return (argv list, warnings).""" + captured: list = [] + warned: list = [] + + def runner(argv, **_kwargs): + captured.append(list(argv)) + # CompletedProcess is a *constructor* (not a process spawn). + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse, "layout_origin", return_value=(0, 0)), \ + patch.object(wayland_mouse.autocontrol_logger, "warning", + side_effect=lambda message, *a, **k: warned.append( + str(message))), \ + patch.object(wayland_mouse.subprocess, "run", side_effect=runner): + wayland_mouse.set_position(x, y) + return captured, warned + + +def test_unset_means_warn_once_and_move_anyway(): + """The default keeps every existing caller working, loudly.""" + argv, warned = _move() + assert argv == [["/usr/bin/ydotool", "mousemove", "--absolute", + "-x", "10", "-y", "20"]] + assert len(warned) == 1 + assert "acceleration" in warned[0] + + +def test_the_warning_names_the_way_out(): + """A warning nobody can act on is noise: it must name the variable.""" + _, warned = _move() + assert wayland_mouse.POINTER_ACCEL_ENV in warned[0] + assert "flat" in warned[0] + assert "strict" in warned[0] + + +def test_warn_does_not_repeat_on_every_move(): + """A script making thousands of moves gets one line, not thousands.""" + _move() + _, warned = _move() + assert warned == [] + + +def test_flat_moves_silently(monkeypatch): + """The operator has switched acceleration off; there is nothing to say.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "flat") + argv, warned = _move() + assert len(argv) == 1 + assert warned == [] + + +def test_strict_refuses_rather_than_landing_somewhere_else(monkeypatch): + """Fail fast: no argv is sent at all, so nothing lands anywhere.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "strict") + with pytest.raises(AutoControlException) as raised: + _move() + assert wayland_mouse.POINTER_ACCEL_ENV in str(raised.value) + + +def test_strict_sends_no_command(monkeypatch): + """The refusal happens before the subprocess, not after.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "strict") + captured: list = [] + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=lambda argv, **k: captured.append(argv)), \ + pytest.raises(AutoControlException): + wayland_mouse.set_position(10, 20) + assert captured == [] + + +@pytest.mark.parametrize("declared, expected", [ + ("flat", "flat"), + ("FLAT", "flat"), + (" strict ", "strict"), + ("warn", "warn"), +]) +def test_the_value_is_read_case_and_whitespace_insensitively(declared, + expected): + """Operators type these into shell profiles and CI YAML by hand.""" + assert wayland_mouse.pointer_accel_mode( + {wayland_mouse.POINTER_ACCEL_ENV: declared}) == expected + + +@pytest.mark.parametrize("declared", ["", " ", "off", "flatt", "0"]) +def test_an_unrecognised_value_falls_back_to_the_safe_answer(declared): + """A typo must not silently promote a move to trusted-exact.""" + assert wayland_mouse.pointer_accel_mode( + {wayland_mouse.POINTER_ACCEL_ENV: declared}) == "warn" + + +def test_a_typo_says_so_rather_than_being_swallowed(): + """Falling back silently is how an operator believes a lie.""" + warned: list = [] + with patch.object(wayland_mouse.autocontrol_logger, "warning", + side_effect=lambda message, *a, **k: warned.append( + str(message))): + wayland_mouse.pointer_accel_mode( + {wayland_mouse.POINTER_ACCEL_ENV: "flatt"}) + assert len(warned) == 1 + assert "flatt" in warned[0] + assert "warn" in warned[0] + + +def test_a_missing_variable_is_not_a_typo(): + """An absent variable is the default, and says nothing.""" + warned: list = [] + with patch.object(wayland_mouse.autocontrol_logger, "warning", + side_effect=lambda message, *a, **k: warned.append( + str(message))): + assert wayland_mouse.pointer_accel_mode({}) == "warn" + assert warned == [] + + +def test_strict_does_not_touch_the_libei_path(monkeypatch): + """libei is absolute at the protocol level; the caveat is ydotool's.""" + monkeypatch.setenv(wayland_mouse.POINTER_ACCEL_ENV, "strict") + moved: list = [] + + class _Device: + def set_position(self, x, y): + moved.append((x, y)) + + with patch.object(wayland_mouse, "_try_libei", return_value=_Device()), \ + patch.object(wayland_mouse, "emitted", + side_effect=lambda backend, call: (call(backend), True)[1]): + wayland_mouse.set_position(7, 9) + assert moved == [(7, 9)] diff --git a/test/unit_test/headless/test_wayland_ydotool_cli.py b/test/unit_test/headless/test_wayland_ydotool_cli.py new file mode 100644 index 00000000..18652deb --- /dev/null +++ b/test/unit_test/headless/test_wayland_ydotool_cli.py @@ -0,0 +1,207 @@ +"""The ydotool 0.1.x guard: what it classifies, refuses and lets through. + +The bug it exists for is measured in ``docker/ydotool_verify.py`` against a +real uinput device — 0.1.x answers this backend's argv with exit code 0 and +no events, so ``check=True`` sees success while nothing happened. These tests +cover the classification and the two backends' use of it without needing +ydotool installed, which is what makes them CI-gating on every platform. +""" +import subprocess +from unittest import mock + +import pytest + +from je_auto_control.linux_wayland import _ydotool_cli +from je_auto_control.utils.exception.exceptions import AutoControlException + + +#: The real no-argument banners, copied from ydotool 0.1.8 (Debian bookworm) +#: and 1.0.4 (Debian unstable / Arch). If a future release changes these the +#: probe answers "unknown" and stops refusing anything, which is the safe +#: direction — but docker/ydotool_verify.py fails loudly when it happens. +LEGACY_BANNER = b"""Usage: ydotool +Available commands: + type + recorder + mousemove + key + click +""" + +MODERN_BANNER = b"""Usage: ydotool +Available commands: + click + mousemove + type + key + debug + bakers +Use environment variable YDOTOOL_SOCKET to specify daemon socket. +""" + + +@pytest.fixture(autouse=True) +def _clear_probe_cache(): + """The probe caches per path for the process; tests must not share it.""" + _ydotool_cli.reset_cache() + yield + _ydotool_cli.reset_cache() + + +def _fake_run(stdout=b"", stderr=b"", returncode=0): + completed = subprocess.CompletedProcess( # nosemgrep # reason: a result object, not a launch + args=["ydotool"], returncode=returncode, stdout=stdout, stderr=stderr) + return mock.Mock(return_value=completed) + + +def test_legacy_banner_is_classified_legacy(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=LEGACY_BANNER)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.LEGACY + + +def test_modern_banner_is_classified_modern(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=MODERN_BANNER)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.MODERN + + +def test_banner_on_stderr_is_read_too(): + """0.1.8 prints its usage to stderr on some paths; both streams count.""" + with mock.patch.object(subprocess, "run", _fake_run(stderr=LEGACY_BANNER)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.LEGACY + + +def test_unrecognised_banner_is_unknown_not_legacy(): + """Only the version measured to fail silently is refused.""" + with mock.patch.object(subprocess, "run", + _fake_run(stdout=b"ydotool 3.0\nsomething new\n")): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_a_word_containing_recorder_does_not_trigger_the_legacy_verdict(): + """The marker is a whole command name, not a substring of prose.""" + banner = b"Usage: ydotool \nAvailable commands:\n screenrecorder\n" + with mock.patch.object(subprocess, "run", _fake_run(stdout=banner)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_probe_failure_is_unknown_rather_than_an_accusation(): + with mock.patch.object(subprocess, "run", + side_effect=OSError("no such binary")): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_probe_timeout_is_unknown(): + with mock.patch.object( + subprocess, "run", + side_effect=subprocess.TimeoutExpired( # nosemgrep # reason: an exception object, not a launch + cmd="ydotool", timeout=5.0)): + assert _ydotool_cli.cli_generation("/usr/bin/ydotool") == \ + _ydotool_cli.UNKNOWN + + +def test_the_probe_runs_once_per_path(): + """Mouse and key dispatch must not pay a subprocess per event.""" + runner = _fake_run(stdout=MODERN_BANNER) + with mock.patch.object(subprocess, "run", runner): + for _ in range(5): + _ydotool_cli.cli_generation("/usr/bin/ydotool") + assert runner.call_count == 1 + + +def test_the_cache_is_keyed_on_the_path(): + runner = _fake_run(stdout=MODERN_BANNER) + with mock.patch.object(subprocess, "run", runner): + _ydotool_cli.cli_generation("/usr/bin/ydotool") + _ydotool_cli.cli_generation("/opt/other/ydotool") + assert runner.call_count == 2 + + +def test_reject_legacy_cli_raises_with_an_actionable_message(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException) as error: + _ydotool_cli.reject_legacy_cli("/usr/bin/ydotool") + message = str(error.value) + assert "0.1.x" in message + assert "/usr/bin/ydotool" in message + # The three routes out: a newer ydotool, or the X11 backend. + assert "1.0" in message + assert "JE_AUTOCONTROL_LINUX_DISPLAY_SERVER=x11" in message + + +def test_reject_legacy_cli_returns_the_path_for_a_modern_tool(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=MODERN_BANNER)): + assert _ydotool_cli.reject_legacy_cli("/usr/bin/ydotool") == \ + "/usr/bin/ydotool" + + +def test_reject_legacy_cli_lets_an_unknown_version_through(): + with mock.patch.object(subprocess, "run", _fake_run(stdout=b"???")): + assert _ydotool_cli.reject_legacy_cli("/usr/bin/ydotool") == \ + "/usr/bin/ydotool" + + +def test_reset_cache_can_forget_one_path_only(): + runner = _fake_run(stdout=MODERN_BANNER) + with mock.patch.object(subprocess, "run", runner): + _ydotool_cli.cli_generation("/a/ydotool") + _ydotool_cli.cli_generation("/b/ydotool") + _ydotool_cli.reset_cache("/a/ydotool") + _ydotool_cli.cli_generation("/a/ydotool") + _ydotool_cli.cli_generation("/b/ydotool") + assert runner.call_count == 3 + + +# -------------------------------------------------------------------------- +# The two backends have to consult the guard, or it protects nobody. +# -------------------------------------------------------------------------- + +def test_mouse_refuses_to_emit_through_the_legacy_cli(): + from je_auto_control.linux_wayland import mouse + + with mock.patch("je_auto_control.linux_wayland.mouse.binary_path", + return_value="/usr/bin/ydotool"), \ + mock.patch.object(mouse, "_try_libei", return_value=None), \ + mock.patch.object(subprocess, "run", _fake_run( + stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException, match="0.1.x"): + mouse.click_mouse(mouse.wayland_mouse_left) + + +def test_keyboard_refuses_to_emit_through_the_legacy_cli(): + from je_auto_control.linux_wayland import keyboard + + with mock.patch("je_auto_control.linux_wayland.keyboard.binary_path", + return_value="/usr/bin/ydotool"), \ + mock.patch.object(keyboard, "_try_libei", return_value=None), \ + mock.patch.object(subprocess, "run", _fake_run( + stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException, match="0.1.x"): + keyboard.press_key(30) + + +def test_hotkey_refuses_to_emit_through_the_legacy_cli(): + """The chord path builds its argv separately, so it needs its own guard.""" + from je_auto_control.linux_wayland import keyboard + + with mock.patch("je_auto_control.linux_wayland.keyboard.binary_path", + return_value="/usr/bin/ydotool"), \ + mock.patch.object(keyboard, "_try_libei", return_value=None), \ + mock.patch.object(subprocess, "run", _fake_run( + stdout=LEGACY_BANNER)): + with pytest.raises(AutoControlException, match="0.1.x"): + keyboard.hotkey([29, 30]) + + +def test_the_install_hints_no_longer_recommend_a_broken_package(): + """`apt install ydotool` gives 0.1.8 on bookworm and nothing on trixie.""" + from je_auto_control.linux_wayland import keyboard, mouse + + for hint in (mouse._INSTALL_HINT, keyboard._INSTALL_HINT_YDOTOOL): + assert "apt install ydotool" not in hint + assert "1.0" in hint diff --git a/test/unit_test/headless/test_wayland_ydotool_origin.py b/test/unit_test/headless/test_wayland_ydotool_origin.py new file mode 100644 index 00000000..19712bfd --- /dev/null +++ b/test/unit_test/headless/test_wayland_ydotool_origin.py @@ -0,0 +1,175 @@ +"""The ydotool fallback counts absolute moves from the layout's corner. + +``mousemove --absolute`` sends no absolute event: it drives the cursor into +the corner the compositor clamps to and then moves relative to it. That corner +is the top-left of the output layout, which is layout ``(0, 0)`` only while +every output sits at a non-negative position — so on a desktop with a monitor +left of the primary one, a raw layout coordinate lands a monitor's width away. + +Measured against a real wlroots session in ``docker/Dockerfile.seat``; these +tests pin the translation that measurement produced, on any host, without a +compositor. +""" +import subprocess +from unittest.mock import patch + +import pytest + +from je_auto_control.linux_wayland import ( + _layout, _ydotool_cli, libei as wayland_libei, mouse as wayland_mouse, +) + + +@pytest.fixture(autouse=True) +def _pin_the_cli_input_path(): + """Keep every test here on the ydotool argv, as the sibling file does.""" + _ydotool_cli.reset_cache() + _ydotool_cli._cache["/usr/bin/ydotool"] = _ydotool_cli.MODERN + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + try: + with patch.object(wayland_mouse, "_try_libei", return_value=None): + yield + finally: + _ydotool_cli.reset_cache() + wayland_mouse._warn_once.cache_clear() + _layout.reset_cache() + + +def _fake_run(captured): + def runner(argv, **_kwargs): + captured.append(list(argv)) + # CompletedProcess is a *constructor* (not a process spawn). + return subprocess.CompletedProcess(argv, 0, b"", b"") # nosemgrep + return runner + + +def _argv_for(point, origin): + """The argv ``set_position(*point)`` builds on a layout at ``origin``.""" + captured: list = [] + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse, "layout_origin", return_value=origin), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=_fake_run(captured)): + wayland_mouse.set_position(*point) + assert len(captured) == 1 + return captured[0] + + +def test_a_layout_that_starts_at_the_origin_needs_no_translation(): + assert _argv_for((120, 240), (0, 0)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "120", "-y", "240", + ] + + +def test_a_negative_origin_is_subtracted_from_the_request(): + """A point on the primary monitor is 1,280 px into the layout.""" + assert _argv_for((120, 90), (-1280, 0)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "1400", "-y", "90", + ] + + +def test_a_point_on_the_left_hand_monitor_reaches_the_corner(): + """And the monitor left of the primary starts at the corner itself.""" + assert _argv_for((-1280, 0), (-1280, 0)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "0", "-y", "0", + ] + + +def test_a_negative_vertical_origin_is_subtracted_too(): + assert _argv_for((40, -300), (0, -400)) == [ + "/usr/bin/ydotool", "mousemove", "--absolute", "-x", "40", "-y", "100", + ] + + +def test_the_translation_is_the_difference_the_capture_path_applies(): + """``_ydotool_point`` is exactly ``point - layout_origin()``.""" + with patch.object(wayland_mouse, "layout_origin", return_value=(-1280, -20)): + assert wayland_mouse._ydotool_point(120, 90) == (1400, 110) + + +# === The shared origin lookup ============================================== + +def test_both_input_paths_share_one_origin_lookup(): + """libei and ydotool must not drift apart on what the origin is.""" + assert wayland_libei.layout_origin is _layout.layout_origin + + +def test_the_origin_is_zero_when_the_screen_module_cannot_answer(): + """A host without Pillow or wlr-randr still moves, just untranslated.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + side_effect=OSError("no capture tool")): + assert _layout.layout_origin() == (0, 0) + + +def test_the_origin_is_whatever_the_screen_module_reports(): + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)): + assert _layout.layout_origin() == (-1280, 0) + + +def test_the_origin_is_read_once_for_a_burst_of_moves(): + """A drag emits many moves; each must not spawn its own ``wlr-randr``.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)) as reader: + for _ in range(20): + assert _layout.layout_origin() == (-1280, 0) + assert reader.call_count == 1 + + +def test_a_reset_makes_the_next_call_measure_again(): + """The window is short so a rearranged desktop is picked up, not pinned.""" + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)): + assert _layout.layout_origin() == (-1280, 0) + _layout.reset_cache() + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(0, 0)): + assert _layout.layout_origin() == (0, 0) + + +def test_a_reading_older_than_the_window_is_measured_again(): + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(-1280, 0)): + assert _layout.layout_origin() == (-1280, 0) + stamp, value = _layout._CACHE["origin"] + _layout._CACHE["origin"] = (stamp - _layout._CACHE_SECONDS - 1, value) + with patch("je_auto_control.linux_wayland.screen.layout_origin", + return_value=(0, -400)) as reader: + assert _layout.layout_origin() == (0, -400) + assert reader.call_count == 1 + + +# === The acceleration caveat =============================================== + +def test_the_acceleration_caveat_is_logged_once_per_process(): + """Silently landing in the wrong place is the failure being prevented.""" + captured: list = [] + with patch.object(wayland_mouse, "binary_path", + return_value="/usr/bin/ydotool"), \ + patch.object(wayland_mouse, "layout_origin", return_value=(0, 0)), \ + patch.object(wayland_mouse.subprocess, "run", + side_effect=_fake_run(captured)), \ + patch.object(wayland_mouse.autocontrol_logger, "warning") as warned: + wayland_mouse.set_position(10, 10) + wayland_mouse.set_position(20, 20) + assert warned.call_count == 1 + assert "acceleration" in warned.call_args[0][0] + + +def test_the_libei_path_says_nothing_about_acceleration(): + """It is absolute at the protocol level, so the caveat does not apply.""" + sent: list = [] + + class _Device: + def set_position(self, x, y): + sent.append((x, y)) + + with patch.object(wayland_mouse, "_try_libei", return_value=_Device()), \ + patch.object(wayland_mouse, "emitted", + side_effect=lambda backend, send: (send(backend), True)[1]), \ + patch.object(wayland_mouse.autocontrol_logger, "warning") as warned: + wayland_mouse.set_position(120, 90) + assert sent == [(120, 90)] + assert warned.call_count == 0 diff --git a/test/unit_test/headless/test_window_capture.py b/test/unit_test/headless/test_window_capture.py index e17b2bc7..c9029249 100644 --- a/test/unit_test/headless/test_window_capture.py +++ b/test/unit_test/headless/test_window_capture.py @@ -115,3 +115,37 @@ def test_snap_window_unknown_position_raises(): with pytest.raises(ValueError): snap_window("E", "diagonal", screen_size=lambda: (1000, 800), mover=lambda *a: True) + + +def test_default_lister_only_offers_windows_restore_can_address(monkeypatch): + """Saving an untitled window is a promise the restore side cannot keep. + + ``restore_window_layout`` finds a window by title and skips blank ones, so + an untitled entry inflates the saved count without ever being restored — + on a real desktop that was 28 saved against 15 restorable. + """ + from je_auto_control.utils.window_capture import window_capture as wc + + seen = {} + + def _fake_list_windows(titled_only=False): + seen["titled_only"] = titled_only + return [(1, "Editor"), (2, " "), (3, "")] + + import je_auto_control.wrapper.auto_control_window as window_api + monkeypatch.setattr(window_api, "list_windows", _fake_list_windows) + assert wc._default_lister() == [(1, "Editor"), (2, " "), (3, "")] + assert seen["titled_only"] is True + + +def test_saved_entries_are_all_restorable(): + """Round-trip: every entry save produces must survive restore.""" + layout = save_window_layout( + lister=lambda: [(1, "Editor"), (2, "Browser")], + geometry=lambda title: (0, 0, 100, 100), + ) + moved = [] + restored = restore_window_layout( + layout, mover=lambda title, *rect: moved.append(title) or True) + assert restored == len(layout) == 2 + assert moved == ["Editor", "Browser"] diff --git a/test/unit_test/headless/test_window_manage.py b/test/unit_test/headless/test_window_manage.py index 3042a734..ae13b57e 100644 --- a/test/unit_test/headless/test_window_manage.py +++ b/test/unit_test/headless/test_window_manage.py @@ -127,3 +127,162 @@ def SetForegroundWindow(self, hwnd): # noqa: N802 # Win32 name seen.clear() module.show_window(7, 3) # SW_MAXIMIZE assert seen == [("show", 7, 3), ("front", 7)] + + +# --- process id ------------------------------------------------------------ + +def test_window_process_id_asks_the_backend_for_the_match(wm, monkeypatch): + monkeypatch.setattr(wm, "get_window_process_id", lambda h: 4242) + assert w.window_process_id("Editor") == 4242 + + +def test_window_process_id_is_none_when_nothing_matches(wm): + assert w.window_process_id("no such window") is None + + +def test_foreground_window_process_id_follows_the_foreground_hwnd( + wm, monkeypatch): + seen = [] + monkeypatch.setattr(wm, "get_window_process_id", + lambda h: seen.append(h) or 99) + assert w.foreground_window_process_id() == 99 + assert seen == [13] # the hwnd get_foreground_window reported + + +def test_process_id_zero_reads_as_unknown_not_as_pid_zero(wm, monkeypatch): + """0 is the backend's "could not tell", and it is never a real user pid. + + Returning it verbatim would let a caller compare `pid == 0` against a + process list and match the System Idle Process. + """ + monkeypatch.setattr(wm, "get_window_process_id", lambda h: 0) + assert w.foreground_window_process_id() is None + assert w.window_process_id("Editor") is None + + +def test_real_foreground_window_process_id_is_a_live_pid(): + """No stubs: the ctypes prototype has to survive a real call.""" + pid = w.foreground_window_process_id() + assert pid is None or (isinstance(pid, int) and pid > 0) + + +# --- posting input without focus ------------------------------------------- + +def test_post_key_targets_the_focused_control_not_the_frame(wm, monkeypatch): + """The whole point of the fix. + + Keyboard messages are delivered to the control that has focus. Measured on + Character Map: posting to the top-level frame typed nothing, posting to the + focused edit typed the character — so a "background typing" feature that + posts to the frame silently does nothing in any app with child controls. + """ + posted = [] + monkeypatch.setattr(wm, "get_focused_control", lambda hwnd: 999) + monkeypatch.setattr(wm, "post_key", + lambda hwnd, code, char="": posted.append( + (hwnd, code, char)) or True) + assert w.post_key_to_window("Editor", "a") is True + assert posted == [(11, 65, "a")] # hwnd of the matched window + + +def test_post_key_sends_a_character_for_printable_keys_only(): + assert w._resolve_key("a") == (65, "a") # WM_CHAR carries the text + assert w._resolve_key("f5")[1] == "" # a function key has no character + assert w._resolve_key(65) == (65, "") + + +def test_post_key_rejects_an_unknown_key_name(): + from je_auto_control.utils.exception.exceptions import ( + AutoControlActionException, + ) + with pytest.raises(AutoControlActionException): + w._resolve_key("wingdings") + + +def test_post_click_accepts_both_button_spellings(wm, monkeypatch): + seen = [] + monkeypatch.setattr(wm, "post_click", + lambda hwnd, button, x, y: seen.append( + (hwnd, button, x, y)) or True) + assert w.post_click_to_window("Editor", "mouse_right", 5, 6) is True + assert w.post_click_to_window("Editor", "LEFT", 1, 2) is True + assert seen == [(11, "right", 5, 6), (11, "left", 1, 2)] + + +def test_posting_to_a_missing_window_is_false_not_an_exception(wm): + assert w.post_key_to_window("no such window", "a") is False + assert w.post_click_to_window("no such window") is False + + +def test_post_click_rejects_unknown_buttons(): + from je_auto_control.windows.window import windows_window_manage as module + with pytest.raises(ValueError): + module.post_click(0, "scroll", 0, 0) + + +def test_focused_control_falls_back_to_the_window_itself(): + """A window whose thread reports no focus must still be a usable target.""" + from je_auto_control.windows.window import windows_window_manage as module + assert module.get_focused_control(0) == 0 + + +# --- windows by owning process --------------------------------------------- + +def test_windows_for_process_id_filters_by_owner(wm, monkeypatch): + monkeypatch.setattr(wm, "get_window_process_id", + lambda hwnd: {11: 4242, 12: 7, 13: 4242}.get(hwnd, 0)) + assert w.windows_for_process_id(4242) == [(11, "Editor"), (13, "Browser")] + + +def test_windows_for_process_id_can_keep_untitled_windows(wm, monkeypatch): + """A browser's helper windows are often untitled — and still worth acting on.""" + monkeypatch.setattr(wm, "get_window_process_id", lambda hwnd: 4242) + assert len(w.windows_for_process_id(4242)) == 3 + assert len(w.windows_for_process_id(4242, titled_only=True)) == 2 + + +def test_minimize_windows_for_process_counts_only_what_it_minimised( + wm, monkeypatch): + monkeypatch.setattr(wm, "get_window_process_id", lambda hwnd: 4242) + monkeypatch.setattr(wm, "minimize_window", lambda hwnd: hwnd != 12) + assert w.minimize_windows_for_process(4242) == 2 + + +# --- the deprecated pair now delegates instead of doing nothing ------------- + +def test_deprecated_key_sender_warns_and_delegates(wm, monkeypatch): + """It used to post to the frame and silently do nothing while reporting success.""" + from je_auto_control.wrapper import auto_control_keyboard as k + + posted = [] + monkeypatch.setattr(wm, "get_focused_control", lambda hwnd: hwnd) + monkeypatch.setattr(wm, "post_key", + lambda hwnd, code, char="": posted.append( + (hwnd, code, char)) or True) + with pytest.warns(DeprecationWarning): + k.send_key_event_to_window("Editor", "a") + assert posted == [(11, 65, "a")] # resolved by title substring + + +def test_deprecated_mouse_sender_accepts_an_hwnd_and_a_title(wm, monkeypatch): + """The old signature took an hwnd; keep that working while fixing the target.""" + from je_auto_control.wrapper import auto_control_mouse as m + + seen = [] + monkeypatch.setattr(wm, "post_click", + lambda hwnd, button, x, y: seen.append( + (hwnd, button, x, y)) or True) + with pytest.warns(DeprecationWarning): + m.send_mouse_event_to_window(11, "mouse_right", 5, 6) + with pytest.warns(DeprecationWarning): + m.send_mouse_event_to_window("Editor", "mouse_left", 1, 2) + assert seen == [(11, "right", 5, 6), (11, "left", 1, 2)] + + +def test_deprecated_mouse_sender_maps_raw_keycodes_back_to_a_button(): + from je_auto_control.wrapper.auto_control_mouse import _button_name_for_post + from je_auto_control.wrapper.platform_wrapper import mouse_keys_table + + assert _button_name_for_post("mouse_middle") == "middle" + assert _button_name_for_post(mouse_keys_table["mouse_right"]) == "right" + assert _button_name_for_post(object()) == "left" # unknown → safe default