Ship semantic I/O and repository hygiene

.gitattributes

@@ -0,0 +1,21 @@
+* text=auto eol=lf
+
+*.apk binary
+*.jpg binary
+*.jpeg binary
+*.mp4 binary
+*.p12 binary
+*.png binary
+*.wav binary
+
+*.svg text eol=lf
+*.toml text eol=lf
+*.json text eol=lf
+*.md text eol=lf
+*.rs text eol=lf
+*.java text eol=lf
+*.js text eol=lf
+*.mjs text eol=lf
+*.ts text eol=lf
+*.yml text eol=lf
+*.yaml text eol=lf

.github/workflows/ci.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
-        with: { node-version: '20.11.1', cache: npm }
+        with: { node-version: '22.18.0', cache: npm }
       - run: npm ci
       - run: npm test
       - run: npm run lint

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
           test "$GITHUB_REF_NAME" = "v$version"
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
         with:
-          node-version: '20.11.1'
+          node-version: '22.18.0'
           cache: npm
       - uses: actions/setup-java@c5195efecf7bdfc987ee8bae7a71cb8b11521c00
         with:

.github/workflows/security.yml

@@ -30,7 +30,7 @@ jobs:
         run: cargo deny check
       - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
         with:
-          node-version: '20.11.1'
+          node-version: '22.18.0'
       - run: npm ci
       - run: npm audit --audit-level=high
       - run: cargo xtask check

.gitignore

@@ -1,16 +1,31 @@
 /target/
 /device/.gradle/
+/device/.cxx/
+/device/local.properties
 /device/build/
 /device/app/build/
 /device/example/build/
-/artifacts/raw/
 /dist/
 
+# Local IDE, OS, and tool output.
+.gradle/
+.idea/
+.vscode/
+*.iml
+.DS_Store
+Thumbs.db
+*.swp
+*.swo
+*.log
+coverage/
+.nyc_output/
+
+# Local agent skill installations.
+/.agents/
+
 # Local release state, device evidence, and private signing material.
 /artifacts/
 /device/gradle-*/
-/device/*.keystore
-/device/*.jks
 /device/signing.properties
 *.token
 *.pem

.oxlintrc.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "./node_modules/oxlint/configuration_schema.json",
+  "ignorePatterns": [
+    ".agent/**",
+    ".agents/**",
+    ".claude/**",
+    ".codex/**",
+    ".codex-run/**",
+    ".continue/**",
+    ".cursor/**",
+    ".gemini/**",
+    ".opencode/**",
+    ".pi/**",
+    ".roo/**",
+    ".windsurf/**",
+    "tools/oxlint/anti-slop/**"
+  ],
+  "jsPlugins": [
+    {
+      "name": "anti-slop",
+      "specifier": "./tools/oxlint/anti-slop/index.ts"
+    }
+  ],
+  "rules": {
+    "anti-slop/no-chained-type-assertions": "error",
+    "anti-slop/no-conditional-empty-object-spread": "error",
+    "anti-slop/no-known-value-widening": "error",
+    "anti-slop/no-module-mocking": "error",
+    "anti-slop/no-object-parameters": "error",
+    "anti-slop/no-reflect-apply": "error",
+    "anti-slop/no-reflect-get": "error",
+    "anti-slop/no-runtime-typeof": "error",
+    "anti-slop/no-shape-in-symbol-names": "error",
+    "anti-slop/no-unknown-parameters": "error",
+    "anti-slop/no-unknown-returns": "error",
+    "anti-slop/no-unknown-type-aliases": "error",
+    "anti-slop/no-unsafe-dictionary-type": "error",
+    "anti-slop/no-widen-then-assert": "error",
+    "anti-slop/require-safety-comment-for-type-assertion": "error"
+  }
+}

AGENTS.md

@@ -1,28 +1,9 @@
 # Android Use agent guide
 
-Android Use gives an AI agent bounded control of one enrolled Android device. The preferred interface is the MCP server:
+Use the local MCP server `au serve --mcp`. It exposes exactly two tools, each with one required string: `android.read` for non-mutating commands and `android.act` for bounded actions.
 
-```text
-au serve --mcp
-```
+Read with commands such as `status`, `screen`, `page`, or `page text`. Act with runtime values such as `tap "TARGET"`, `type "TEXT" in "FIELD"`, `open app "DISPLAY NAME"`, or `page click "TARGET"`; nothing is typed unless the command requests it. Join a short sequence with `then`. The host owns observations, identity, target resolution, safety limits, journals, tabs, and image transport.
 
-Use `au serve --jsonl` only when the client does not support MCP. Do not drive the machine through raw ADB when the typed interface can complete the task.
+Read only when state is unknown. Prefer semantic labels. Use screenshots and `tap point X Y` only after a semantic miss. Retry only a stale pre-send failure; after `partial` or `unknown`, read and reconcile before mutating again. Ask before destructive, account, purchase, submission, notification, location, or camera/microphone/recording actions.
 
-## Operating loop
-
-1. Read `android.read` with `q=status`.
-2. Read `q=observe` for the semantic UI frontier.
-3. Act with `android.act`, passing the returned `g` generation and a unique operation `id`.
-4. Prefer integer refs from the latest observation. Keep plans short and linear.
-5. Include an immediate `wait` or `assert` when the outcome matters.
-6. On `stale`, observe and rebuild. On `partial` or `unknown`, observe before doing anything else. Never replay a mutation blindly.
-
-For Chrome content, read `q=browser` with `op=tabs|observe|text`, then use a browser-targeted plan. Use Android UI semantics for Chrome's own toolbar.
-
-Use semantic UI before screenshots. Request a screenshot only when layout, imagery, or an unlabeled control matters. Artifacts are private handles; fetch only the required range.
-
-Ask before deletion, account changes, purchases, submissions, camera or microphone capture, location-sensitive work, notification actions, or screen recording.
-
-Setup and recovery: [docs/agents/quickstart.md](docs/agents/quickstart.md)  
-Typed protocol: [docs/reference/agent-protocol.md](docs/reference/agent-protocol.md)  
-Security boundaries: [SECURITY.md](SECURITY.md)
+For setup see [quickstart](docs/agents/quickstart.md); for grammar see [agent protocol](docs/reference/agent-protocol.md); for security see [SECURITY.md](SECURITY.md).

CHANGELOG.md

@@ -4,6 +4,15 @@
 
 The first supported public release of Android Use.
 
+### Agent command interface
+
+- Added the bounded model-facing `command` string for the existing `android.read` and `android.act` tools. The host now owns observation generations, operation identity, semantic target resolution, app and tab selection, safety limits, journals, and image content.
+- Added plain-language receipts, ambiguity guidance, filtered page text, direct MCP image content, semantic-miss screenshots, allowlisted settings, safe links, point fallback, and bounded swipes.
+- Kept the structured CLI, JSONL, MCP, golden-wire, helper, artifact, browser, and visual forms operational as a deprecated compatibility path. Raw generations, refs, plans, artifact ranges, and package IDs are legacy-only for ordinary agents.
+- Browser actions reuse the active CDP connection, avoid unnecessary tab-list synchronization, track same-page DOM identity, use framework-friendly value events, and invalidate on meaningful DOM changes. Android text, content-description, and state changes invalidate semantic state.
+- No new runtime dependency or cloud service was added. The helper remains local-only, authenticated, bounded, no-root, and without `INTERNET` permission.
+- The automation source budget is now 1,250 lines (measured baseline 1,202) to cover the repository-owned documentation budget, parser-consistency, benchmark, and evaluation-status gates; production and authored-code limits remain unchanged.
+
 - Control one enrolled Android device through the `au` CLI, MCP, or JSONL.
 - Read compact semantic UI, act through generation-checked plans, control supported Chrome sessions, and keep screenshots and other large results as local artifacts.
 - Install the matching Android helper with `au setup`, then use `au doctor` for clear connection and permission diagnostics.

README.md

@@ -82,39 +82,13 @@ The release archive contains `au` and the Android Use helper together. Keep them
 
 ## Use Android Use in your agent
 
-If you want to use Android Use in Codex, Cursor, Claude Code, OpenClaw, Hermes, or another coding agent, paste this prompt into the agent. It installs the agent skill, finds the right host runtime, walks you through the Android-owned steps, and resumes after you approve them:
+If you want to use Android Use in Codex, Cursor, Claude Code, OpenClaw, Hermes, or another coding agent, paste this prompt into the agent:
 
 ```text
-Set up Android Use for this agent and connect one Android device.
-
-Use https://github.com/austinintelligence/android-use as the source of truth. Work through the setup in order and keep the conversation on rails:
-
-1. Inspect the computer, operating system, CPU architecture, existing `au` installation, Android platform tools, and current agent configuration. Reuse a working installation when possible. Do not delete or overwrite unrelated files, device data, agent settings, credentials, or an existing Android Use enrollment.
-
-2. Register the `android-use` Agent Skill for the agent I am using. Prefer the agent's native skill installer. For a skills.sh-compatible agent, use the matching agent id with:
-   `npx skills add austinintelligence/android-use --skill android-use -g -a <agent-id> --copy -y`
-   For OpenClaw, use:
-   `openclaw skills install git:austinintelligence/android-use@main --global`
-   Replace `<agent-id>` with the real id; do not run it literally. Reload the agent if its skill list is cached.
-
-3. Install the host runtime from an official source. First check whether `android-use` is actually published before using `npx android-use@latest`. If it is not published, download the matching archive from the latest official GitHub release, verify the archive against both `SHA256SUMS` and `release-manifest.json`, and extract it to a durable user-owned directory. Keep `au` and `aubridge.apk` together and use the absolute path to `au`. Do not use an unsigned or unexplained prerelease unless I approve it.
-
-4. Check readiness with `<absolute-au-path> doctor --json`. If Android platform tools are missing, use an already installed trusted `adb` when available; otherwise tell me exactly how to install platform-tools or set `AU_ADB`. If no authorized device is found, do not keep retrying. Tell me, in plain language:
-   - unlock the phone or tablet;
-   - use a USB cable that carries data;
-   - open Settings → About phone and tap Build number seven times if Developer options is not visible;
-   - open Developer options and turn on USB debugging;
-   - reconnect the device and tap Allow on “Allow USB debugging?”; choose Always allow only for my own computer.
-   Then wait for me and rerun `doctor --json`.
-
-5. Run `<absolute-au-path> setup --json` once the device is authorized. If it reports an Android permission step, tell me exactly what to tap: open Settings → Accessibility → Android Use, turn Android Use on, and approve Android's warning. Wait for me, then rerun `setup --json` or `doctor --json` to verify the change. If multiple devices are connected, show me their endpoints and ask me which one to enroll; never guess.
-
-6. When `doctor --json` reports ready, connect the local MCP server using the absolute executable path and the arguments `serve --mcp`. Preserve other MCP entries, keep the server on local stdio, and reload the agent. Then verify with `android.read` using `q=status` followed by `q=observe` without changing the device.
-
-At the end, report: the installed `au` path and version, the registered skill location, the enrolled device identity without exposing secrets, the MCP connection, required checks, optional capabilities, and the exact next action if anything is still waiting on me. If any step fails, read https://github.com/austinintelligence/android-use/blob/main/docs/agents/install.md and resume from the reported phase. Never bypass Android security prompts or replay an unknown device mutation.
+Set up Android Use from the official release or repository. Preserve unrelated files, credentials, enrollments, and agent settings. Keep au beside aubridge.apk. Run au setup with one unlocked, USB-debugging-authorized device; pause for Android's Accessibility approval and resume with au doctor. Configure a local stdio MCP server using the absolute au path and serve --mcp. Verify with android.read command status and android.read command screen. Use the two command-string tools for normal work. Never use raw ADB, bypass Android prompts, or replay a partial or unknown mutation.
 ```
 
-Then tell your agent what you want done on the device, such as: “Open Settings and tell me which Wi-Fi network is connected.”
+Then ask: “Open Settings and tell me which Wi-Fi network is connected.”
 
 The full fallback runbook and manual commands are in the [Agent installation and recovery guide](docs/agents/install.md).
 

computer/src/adapter.rs

@@ -1,7 +1,8 @@
 use crate::{
-    api::{parse_read, tool_schemas, BrowserPlan, BrowserRead, Code, Error, Plan, Read, Result, VisualPlan, VisualRead, MAX_FRAME},
-    engine::Engine,
+    api::{parse_act_command, parse_read, parse_read_command, tool_schemas, BrowserPlan, BrowserRead, Code, Error, Plan, Read, Result, VisualPlan, VisualRead, MAX_FRAME},
+    engine::{plain_error, Engine, ModelResponse},
 };
+use base64::{engine::general_purpose::STANDARD, Engine as _};
 use serde_json::{json, Value};
 use std::io::{self, BufRead, BufReader, Write};
 
@@ -46,9 +47,14 @@ fn rpc(engine: &mut Engine, v: Value) -> Option<Value> {
             let p = v.get("params").cloned().unwrap_or(Value::Null);
             let name = p.get("name").and_then(Value::as_str).unwrap_or("");
             let args = p.get("arguments").cloned().unwrap_or_else(|| json!({}));
-            call(engine, name, args)
-                .map(|data| json!({"structuredContent":data,"content":[],"isError":false}))
-                .or_else(|e| Ok(json!({"structuredContent":e.json(),"content":[],"isError":true})))
+            if args.get("command").is_some() {
+                let request_identity = id.as_ref().map(|value| serde_json::to_string(value).unwrap_or_default());
+                Ok(model_result(new_call(engine, name, args, request_identity.as_deref())))
+            } else {
+                call(engine, name, args)
+                    .map(|data| json!({"structuredContent":data,"content":[],"isError":false}))
+                    .or_else(|e| Ok(json!({"structuredContent":e.json(),"content":[],"isError":true})))
+            }
         }
         _ => Err(Error::new(Code::Unsupported, "unknown JSON-RPC method")),
     };
@@ -60,8 +66,40 @@ fn rpc(engine: &mut Engine, v: Value) -> Option<Value> {
 
 fn direct(engine: &mut Engine, v: Value) -> Option<Value> {
     let name = v.get("tool").and_then(Value::as_str).unwrap_or("").to_string();
+    let identity = v.get("id").map(|value| serde_json::to_string(value).unwrap_or_default());
     let args = v.get("arguments").cloned().unwrap_or(v);
-    Some(call(engine, &name, args).unwrap_or_else(|e| e.json()))
+    if args.get("command").is_some() {
+        Some(model_result(new_call(engine, &name, args, identity.as_deref())))
+    } else {
+        Some(call(engine, &name, args).unwrap_or_else(|e| e.json()))
+    }
+}
+
+fn new_call(engine: &mut Engine, name: &str, args: Value, request_identity: Option<&str>) -> Result<ModelResponse> {
+    if args.as_object().is_none_or(|object| object.len() != 1 || !object.contains_key("command")) {
+        return Err(Error::new(Code::Args, "new tool calls accept only the command string"));
+    }
+    let command = args.get("command").and_then(Value::as_str).ok_or_else(|| Error::new(Code::Args, "command must be a string"))?;
+    match name {
+        "android.read" => engine.model_read(parse_read_command(command)?),
+        "android.act" => {
+            let actions = parse_act_command(command)?;
+            engine.model_act(&actions, request_identity)
+        }
+        _ => Err(Error::new(Code::Args, "tool must be android.read or android.act")),
+    }
+}
+
+fn model_result(result: Result<ModelResponse>) -> Value {
+    result.map(|response| model_json(&response)).unwrap_or_else(|error| json!({"content":[{"type":"text","text":plain_error(&error)}],"isError":true}))
+}
+
+fn model_json(response: &ModelResponse) -> Value {
+    let mut content = vec![json!({"type":"text","text":response.text})];
+    if let Some(image) = &response.image {
+        content.push(json!({"type":"image","data":STANDARD.encode(image.bytes.as_ref()),"mimeType":image.mime_type}));
+    }
+    json!({"content":content,"isError":false})
 }
 fn call(engine: &mut Engine, name: &str, args: Value) -> Result<Value> {
     match name {
@@ -129,4 +167,26 @@ mod tests {
         let mut r = BufReader::new(bytes.as_slice());
         assert_eq!(bounded_line(&mut r).unwrap_err().code, Code::Bounds);
     }
+
+    #[test]
+    fn model_response_uses_text_and_native_image_content() {
+        let response = ModelResponse {
+            text: "Captured the screen. The image is attached.".into(),
+            image: Some(crate::engine::ModelImage { bytes: b"png".to_vec().into(), mime_type: "image/png" }),
+        };
+        let value = model_json(&response);
+        assert!(value.get("structuredContent").is_none());
+        assert_eq!(value["content"][0]["type"], "text");
+        assert_eq!(value["content"][1]["type"], "image");
+        assert_eq!(value["content"][1]["mimeType"], "image/png");
+        assert_eq!(value["isError"], false);
+    }
+
+    #[test]
+    fn legacy_call_shape_still_routes_separately() {
+        let value = tool_schemas();
+        assert_eq!(value.as_array().unwrap().iter().map(|tool| tool["name"].as_str().unwrap()).collect::<Vec<_>>(), vec!["android.read", "android.act"]);
+        assert!(parse_read(json!({"q":"status"})).is_ok());
+        assert!(parse_act_command("tap \"Save\"").is_ok());
+    }
 }