From 6e11d6357970908c15d0e1d4e6a4eff83b40d908 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 26 Oct 2026 21:03:18 -0700 Subject: [PATCH 01/36] Draft Signed-off-by: Lotus Li --- pyproject.toml | 5 + scripts/tactile_metadata/README.md | 185 ++ .../tactile_metadata_editor/__init__.py | 2 + .../tactile_metadata_editor/cli.py | 121 ++ .../tactile_metadata_editor/document.py | 264 +++ .../tactile_metadata_editor/geometry.py | 309 ++++ .../tactile_metadata_editor/sdk.py | 236 +++ .../tactile_metadata_editor/storage.py | 191 ++ .../tactile_metadata_editor/ui.py | 1558 +++++++++++++++++ src/core/python/requirements-ui.txt | 5 + 10 files changed, 2876 insertions(+) create mode 100644 scripts/tactile_metadata/README.md create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/__init__.py create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/cli.py create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/document.py create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/geometry.py create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/sdk.py create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/storage.py create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/ui.py diff --git a/pyproject.toml b/pyproject.toml index 49fc75eda3..15319553f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,6 +49,11 @@ dev = [ ] ui = [ "imgui[glfw]>=2.0.0", + "numpy>=2,<3", + "viser==1.1.1", + "yourdfpy==0.0.60", + "trimesh==5.1.1", + "rtree>=1.3,<2", ] retargeters = [ "dex-retargeting>=0.4.6,<0.6.0", diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md new file mode 100644 index 0000000000..0c6fe2ad94 --- /dev/null +++ b/scripts/tactile_metadata/README.md @@ -0,0 +1,185 @@ + + +# Tactile Metadata Editor + +A local Viser application for editing glove tactile metadata. It accepts URDF and +schema JSON files, preserves incomplete drafts, and exports checked numerical JSON +with an optional SDK binary. All runtime and device interactions are read-only. + +## Prepare the existing Python environment + +Use your existing Isaac Capture environment with Python 3.11–3.13. Viser provides +the browser interface; yourdfpy loads URDF and computes forward kinematics; +trimesh and rtree provide surface queries; NumPy supplies numerical operations. +These libraries are declared in Isaac Capture's shared UI requirements and package +extra. The editor does not create an environment, install packages, find another +interpreter, or build an SDK. + +From the repository root, using the Python environment you selected: + +```bash +cd scripts/tactile_metadata +python -m pip install -r ../../src/core/python/requirements-ui.txt +python -m tactile_metadata_editor.cli gui --urdf model/left/linkerhand_FFG_left.urdf --sides left +``` + +Open [the local editor](http://127.0.0.1:8080). Use `--port` to choose another port. +Offline editing does not import Isaac Capture or start OpenXR. SDK checking, +binary export and actual live reception require an SDK already importable in the +same environment. Live reception also requires a compatible runtime and HMD +system; see the [tactile device documentation](../../docs/source/device/tactile/index.rst). + +## Create a layout + +1. In **Model**, choose the wrist link (`base_link` in the supplied left model). + Enter allowed links, for example `base_link,index_distal`, and apply them. + Inspect the resource list and joint tree. Visibility and opacity affect only + the drawing. Package mappings are a JSON object of package names to local + directories, used to resolve `package://` resources without network access. +2. Use **Pick target** and **Pick on q0 model** to pick `Wrist O`, `Wrist F` + toward the forearm, and `Wrist D` toward the back of the hand. Preview and + confirm the wrist reference. Both hands use the same right-handed axes. +3. In **Segments**, adjust the free-joint slider or numeric input. Fixed and + mimic joints appear in the tree; mimic joints follow their source. Confirm + **Set shared binding pose q0**. Existing bind poses and weights remain + numerically unchanged when q0 or the wrist reference changes. +4. Add `WRIST` with the wrist link; its bind pose is identity. Add, for example, + `INDEX_DISTAL` with `index_distal`. New non-wrist segments start at the URDF + link frame when calibrated. Pick a segment center or use the native gizmo and + inspector to adjust its frame. URDF frames are initial estimates: calibrate + their axes to the HandJoint convention before production use. Associating an + imported segment with a link preserves its numerical bind pose. +5. In **Layout**, select the intended connected surface component. Set rows and + columns to `4`, and enter the new taxels' segment row list, for example `[1]`. + Pick the first center `O`, first-row last center `C`, and first-column last + center `R`. Preview the grid. Spans and spacing are reported in meters; spans + can be edited along the picked directions. +6. Choose scan order, reversal, optional excluded `[row,column]` cells and a + start output index. `-1` finds an unused contiguous interval. Failed + projections remain red candidates; excluded cells produce no candidate. + Apply or cancel. Applying creates independent taxels and clears the transient + generation parameters. For a single row use O/C; for a single column use O/R. + For one point, pick O and then **Single tangent** to provide the tangent. +7. Click draft points, toggle with Ctrl/Cmd, enable rectangle selection, or enter + an output-index list/range such as `0:15,20`. Selection defaults to visible + points; **Select through mesh** includes hidden points. Inspect lists in + pages of 25. Reproject, move, flip normals or twist tangents as needed. + Inspector fields replace only the edited component; mixed values are reported. +8. In **Binding**, edit the four slots or assign one to four segment rows with + equal initial weights. Repeated indices and slot order are retained. A slider + or numeric weight edit replaces its slot, then divides all four values by + their sum independently for each selected taxel. Invalid edits preserve the + rejected row and report the reason. Loading, recording, checking, saving and + exporting never normalize raw weights. +9. Preview motion with URDF controls. Return to q0 to resume surface picking. + Use **Preview / apply output indices** to review explicit index changes. + Moving or deleting taxels does not compact output indices; formal export + requires exactly `0..N-1`. Segment reordering remaps valid slot references, + including zero-weight slots. Deleted references remain invalid until repaired. + +Use the supplied right model in a separate document: + +```bash +python -m tactile_metadata_editor.cli gui --urdf model/right/linkerhand_FFG_right.urdf --sides right +``` + +Its wrist link is `hand_base_link`. For a real dual-hand URDF, choose `--sides +both`, then assign distinct wrist links and each hand's allowed surfaces. The +editor does not mirror a single-hand model. **New document** unlocks the model +hand selection after handling unsaved drafts. + +## Save, reopen, check and export + +**Export → Save all drafts** writes one editable JSON per hand, including an +optional `editor` extension with relative URDF/package paths, calibration, +mappings, stable entity IDs and output indices. Incomplete drafts can be saved +and are explicitly labeled. If any save fails, no side is marked saved. +**Info issue → Locate** selects the relevant hand/entity in the shared draft +inspector. Advanced schema repair can replace incomplete arrays, with a temporary +backup before replacement. + +```bash +python -m tactile_metadata_editor.cli gui --metadata left.metadata.json +python -m tactile_metadata_editor.cli check left.metadata.json --report check.json +python -m tactile_metadata_editor.cli check left.metadata.json --sdk +python -m tactile_metadata_editor.cli export left.metadata.json --format sdk --out package +python -m tactile_metadata_editor.cli export left.metadata.json right.metadata.json --out pair +``` + +For JSON without `editor.side`, choose **Assign JSON to** and **Open JSON**, or +provide `--sides left/right/both` on the CLI. Pure checking needs no side. Each +side must be unique; conflicting shared URDF configurations are rejected. + +The protocol JSON follows [tactile_metadata.fbs](../../src/core/schema/fbs/tactile_metadata.fbs): +`geometry_mapping_type: "LBS"`, nested `geometry_mapping` arrays, and +`segment_source_type: "HandSkeletonRef"` with nested `segment_source.joint_ids`. +Pose quaternions are XYZW; influence structs contain `values` arrays. There is no +additional metadata format. Numerical export omits `editor` and orders the taxel +arrays by the chosen output indices. Each package contains numerical JSON, +editable JSON, a draft-taxel-ID/export-index CSV and a revision-tagged report. +SDK packages also contain binaries constructed and size-checked with the official +bindings. All sides must succeed before the new or empty destination is published. +GUI exports capture a snapshot; later edits do not change that snapshot. + +## Live reception and the SDK-free demo + +Launch with `gui --demo` to use a synthetic independent layout and hand/force +stream. In **Live**, configure each side's tactile collection (default `tactile`), +HandTracker output and optional JointState collection, then connect. Save source +configuration and reconnect to apply source changes. JointState mappings specify +a source name, URDF target, scale and offset; the panel reports raw values, +converted values, limits and validity. Demo JointState names come from mappings +present when connecting. + +Only enabled JointState drives the live URDF; otherwise it is hidden. HandPose +supplies the tracked skeleton and current segment transforms relative to WRIST. +The live skeleton and taxels use the tracked wrist frame. The draft gizmo always +uses the calibrated q0 frame, independently of the animated drawing. + +**Play / pause** freezes the live drawing while receipt continues. Draft forms, +wire markers and manipulators remain editable. Resume uses the latest frame and +current draft. Returning to EDIT shows q0 without stopping receipt. Interrupted +streams retain the last geometry and mark forces invalid rather than zero. +Force colors cover 0–30 N by default and saturate at the selected maximum. +Multi-segment LBS supplies position, without inventing a mixed rigid orientation +or a world-space force arrow. + +Visualization defaults to **Received** metadata. Switching to **Draft** changes +only the drawing. **Record metadata groups** first backs up the draft to the +system temporary directory, logs its path, then replaces the selected groups +verbatim. Partial replacements may introduce count/reference errors, which remain +editable. Metadata/force alignment is the producer's and user's responsibility: +forces always use protocol row order. Source switching establishes no alignment, +and the export correspondence CSV does not map or permute force data. + +## Implementation and verification + +`document.py` owns raw arrays, IDs, explicit edits, validation and transactional +State revisions. `geometry.py` owns local resource resolution, FK, frame math, +projection and LBS. Its link attachments are derived caches, not metadata edits. +`storage.py` owns schema JSON, atomic saves, backups and package publication. +`sdk.py` imports SDK classes only inside children started with `sys.executable`: +checks use `LBS`, `HandSkeletonRef`, `TactileMetadata.to_bytes()`; live reception +uses `HandTracker`, `TactileTracker`, optional `JointStateTracker`, +`OpenXRSession` and `DeviceIOSession.run/update`. Views are copied before the +bounded local pipe, and only the latest frame is retained. `cli.py` and `ui.py` +share document, geometry and storage operations. The UI uses native Viser +controls and persistent scene nodes; document edits run on one event queue. + +Local development checks target this package only: + +```bash +ruff format --check tactile_metadata_editor +ruff check tactile_metadata_editor +python -m compileall -q tactile_metadata_editor +``` + +The inspection environment lacked the GUI/geometry libraries and SDK. Syntax, +lint, schema round trips, weight/reordering operations, rollback, atomic export, +and SDK-unavailable handling were checked locally. Viser call signatures were +checked against cached 1.1.1 source. Browser interaction, numerical mesh/LBS +execution, successful SDK encoding and actual hardware reception still require +verification in the configured environment; these checks are not claimed passed. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/__init__.py b/scripts/tactile_metadata/tactile_metadata_editor/__init__.py new file mode 100644 index 0000000000..52a7a9daf0 --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/tactile_metadata/tactile_metadata_editor/cli.py b/scripts/tactile_metadata/tactile_metadata_editor/cli.py new file mode 100644 index 0000000000..3b6937a0e2 --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/cli.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from . import storage +from .document import Document, State + + +def main(): + parser = argparse.ArgumentParser(description="Local tactile metadata editor") + commands = parser.add_subparsers(dest="command", required=True) + gui = commands.add_parser("gui") + gui.add_argument("--urdf") + gui.add_argument("--sides", choices=("left", "right", "both")) + gui.add_argument("--metadata") + gui.add_argument("--left-metadata") + gui.add_argument("--right-metadata") + gui.add_argument("--port", type=int, default=8080) + gui.add_argument("--demo", action="store_true") + check = commands.add_parser("check") + check.add_argument("metadata") + check.add_argument("--sdk", action="store_true") + check.add_argument("--report") + export = commands.add_parser("export") + export.add_argument("metadata", nargs="+") + export.add_argument("--sides", choices=("left", "right", "both")) + export.add_argument("--format", choices=("json", "sdk"), default="json") + export.add_argument("--out", required=True) + args = parser.parse_args() + try: + if args.command == "check": + d = storage.load(args.metadata) + result = dict(local=d.check(), sdk="not checked") + failed = any(v["severity"] == "error" for v in result["local"]) + if args.sdk: + from .sdk import check + + try: + result["sdk"] = check(d) + except (ValueError, RuntimeError) as exc: + result["sdk"], failed = dict(error=str(exc)), True + if args.report: + storage.write(args.report, result) + print(json.dumps(result, indent=2)) + return int(failed) + sides = ( + ["left", "right"] + if args.sides == "both" + else [args.sides] + if args.sides + else [] + ) + inputs = ( + [ + (p, sides[i] if len(sides) == len(args.metadata or []) else None) + for i, p in enumerate(args.metadata) + ] + if args.command == "export" + else ( + [(args.metadata, sides[0] if len(sides) == 1 else None)] + if args.metadata + else [] + ) + ) + if args.command == "gui": + inputs += [ + (getattr(args, s + "_metadata"), s) + for s in ("left", "right") + if getattr(args, s + "_metadata") + ] + documents, model, paths = {}, {}, {} + for path, side in inputs: + d = storage.load(path, side) + if not d.side: + if args.command == "gui": + continue + raise ValueError("Assign JSON sides with editor.side or --sides") + if d.side in documents: + raise ValueError("Duplicate metadata side") + cfg = d.editor.get("model", {}) + if model.get("urdf") and cfg.get("urdf") and model != cfg: + raise ValueError("Conflicting URDF configurations") + if cfg.get("urdf"): + model = cfg + documents[d.side], paths[d.side] = d, str(Path(path).resolve()) + if args.command == "export": + print( + json.dumps( + storage.export(documents, model, 0, args.out, args.format == "sdk"), + indent=2, + ) + ) + else: + from .ui import Editor + + for side in sides: + documents.setdefault(side, Document(side)) + if args.urdf: + model["urdf"] = str(Path(args.urdf).resolve()) + state = State(documents, model) + state.paths = paths + state.saved = { + s: state.fingerprint(s) + for s in paths + if documents[s].editor.get("side") == s + and documents[s].editor.get("model", {}) == state.model + } + Editor(state, args).run() + return 0 + except (ValueError, OSError, RuntimeError, ImportError, TypeError) as exc: + print(f"{type(exc).__name__}: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py new file mode 100644 index 0000000000..5767af0c8d --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -0,0 +1,264 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import copy +import json +import math +import uuid + +JOINTS = """PALM WRIST THUMB_METACARPAL THUMB_PROXIMAL THUMB_DISTAL THUMB_TIP +INDEX_METACARPAL INDEX_PROXIMAL INDEX_INTERMEDIATE INDEX_DISTAL INDEX_TIP +MIDDLE_METACARPAL MIDDLE_PROXIMAL MIDDLE_INTERMEDIATE MIDDLE_DISTAL MIDDLE_TIP +RING_METACARPAL RING_PROXIMAL RING_INTERMEDIATE RING_DISTAL RING_TIP +LITTLE_METACARPAL LITTLE_PROXIMAL LITTLE_INTERMEDIATE LITTLE_DISTAL LITTLE_TIP""".split() +FIELDS = ( + "segment_bind_poses", + "taxel_bind_poses", + "taxel_segment_indices", + "taxel_segment_weights", +) +GROUPS = { + "Skeleton binding poses": ("joints", FIELDS[0]), + "Taxel binding poses": FIELDS[1:2], + "Skinning": FIELDS[2:], +} +IDENTITY = [0, 0, 0, 0, 0, 0, 1] + + +def joint_number(joint): + number = JOINTS.index(joint) if isinstance(joint, str) else joint + if type(number) is not int or not 0 <= number < len(JOINTS): + raise ValueError("Invalid HandJoint identifier") + return number + + +def row_error(field, values, segments): + width = 7 if field in FIELDS[:2] else 4 + if not isinstance(values, list) or len(values) != width: + return f"Expected {width} components" + if any( + type(v) not in (int, float) or (type(v) is float and not math.isfinite(v)) + for v in values + ): + return "Components must be finite numbers" + if field == FIELDS[2]: + if any( + type(v) is not int or not 0 <= v < min(segments, 0x100000000) + for v in values + ): + return "All slots must index segment rows, including zero-weight slots" + elif field == FIELDS[3]: + if min(values) < 0 or abs(sum(values) - 1) > 1e-5: + return "Weights must be nonnegative and sum to one" + elif ( + max(map(abs, values)) > 3.4028234663852886e38 + or abs(sum(v * v for v in values[3:]) - 1) > 1e-4 + ): + return "Expected float32 coordinates and a unit XYZW quaternion" + return None + + +class Document: + def __init__(self, side=None): + self.side, self.joints = side, [] + self.arrays = {field: [] for field in FIELDS} + self.types = ("LBS", "HandSkeletonRef") + self.editor = {"taxel_ids": [], "segment_ids": [], "output_indices": []} + + def sync(self): + for key, field in (("taxel_ids", FIELDS[1]), ("segment_ids", FIELDS[0])): + count, ids = len(self.arrays[field]), self.editor.setdefault(key, []) + ids.extend(uuid.uuid4().hex for _ in range(count - len(ids))) + del ids[count:] + count, order = ( + len(self.arrays[FIELDS[1]]), + self.editor.setdefault("output_indices", []), + ) + order.extend(range(len(order), count)) + del order[count:] + + def set_components(self, field, selected, components, values): + width = 7 if field in FIELDS[:2] else 4 + for i in selected: + while len(self.arrays[field]) <= i: + self.arrays[field].append(None) + if not isinstance(self.arrays[field][i], list): + self.arrays[field][i] = [None] * width + row = self.arrays[field][i] + row.extend([None] * max(0, width - len(row))) + for component, value in zip(components, values): + row[component] = value + if field in FIELDS[:2] and selected: + self.editor["approximate"] = True + self.sync() + + def weight(self, selected, slot, value): + rejected = [] + for i in selected: + try: + values = list(self.arrays[FIELDS[3]][i]) + values[slot] = float(value) + if ( + len(values) != 4 + or min(values) < 0 + or not all(math.isfinite(v) for v in values) + ): + raise ValueError("Expected four finite nonnegative weights") + total = sum(values) + if total <= 0 or not math.isfinite(total): + raise ValueError("Weight sum must be finite and positive") + self.arrays[FIELDS[3]][i] = [v / total for v in values] + except (IndexError, ValueError, TypeError, OverflowError) as exc: + rejected.append(f"Taxel row {i}: {exc}") + return rejected + + def assign(self, selected, segments): + if not 1 <= len(segments) <= 4 or any( + type(i) is not int or not 0 <= i < len(self.arrays[FIELDS[0]]) + for i in segments + ): + raise ValueError("Choose one to four existing segment rows") + for i in selected: + self.set_components( + FIELDS[2], [i], range(4), segments + [segments[0]] * (4 - len(segments)) + ) + self.set_components( + FIELDS[3], + [i], + range(4), + [1 / len(segments)] * len(segments) + [0] * (4 - len(segments)), + ) + + def unused_start(self, count): + start = 0 + while any(start + i in self.editor["output_indices"] for i in range(count)): + start += 1 + return start + + def add(self, poses, segments, start=-1): + if any(len(self.arrays[f]) != len(self.arrays[FIELDS[1]]) for f in FIELDS[2:]): + raise ValueError("Repair skinning counts before adding taxels") + if not poses: + raise ValueError("No candidates to apply") + start = self.unused_start(len(poses)) if start < 0 else start + first = len(self.arrays[FIELDS[1]]) + self.arrays[FIELDS[1]].extend(copy.deepcopy(poses)) + self.assign(range(first, first + len(poses)), segments) + self.editor["output_indices"].extend(range(start, start + len(poses))) + self.sync() + self.editor["approximate"] = True + + def delete(self, selected): + for i in sorted(set(selected), reverse=True): + for values in ( + *[self.arrays[f] for f in FIELDS[1:]], + self.editor["taxel_ids"], + self.editor["output_indices"], + ): + if i < len(values): + values.pop(i) + + def segments(self, order): + count = len(self.arrays[FIELDS[0]]) + if ( + len(self.joints) != count + or len(set(order)) != len(order) + or any(type(i) is not int or not 0 <= i < count for i in order) + ): + raise ValueError("Repair skeleton counts and supply distinct segment rows") + mapping = {old: new for new, old in enumerate(order)} + self.joints = [self.joints[i] for i in order] + self.arrays[FIELDS[0]] = [self.arrays[FIELDS[0]][i] for i in order] + self.editor["segment_ids"] = [self.editor["segment_ids"][i] for i in order] + for values in self.arrays[FIELDS[2]]: + if isinstance(values, list): + values[:] = [ + mapping.get(i, -1) if type(i) is int and 0 <= i < count else i + for i in values + ] + + def record(self, received, groups): + for group in groups: + for field in GROUPS[group]: + if field == "joints": + self.joints = copy.deepcopy(received.joints) + else: + self.arrays[field] = copy.deepcopy(received.arrays[field]) + self.sync() + + def check(self): + problems = [] + + def issue(field, message, row=None, severity="error"): + problems.append( + dict(severity=severity, field=field, row=row, message=message) + ) + + if self.types != ("LBS", "HandSkeletonRef"): + issue("metadata", "Expected LBS and HandSkeletonRef discriminators") + j, n, seen = len(self.joints), len(self.arrays[FIELDS[1]]), set() + for field, expected in zip(FIELDS, (j, n, n, n)): + if len(self.arrays[field]) != expected: + issue(field, f"Expected {expected} rows") + for i, row in enumerate(self.arrays[field]): + error = row_error(field, row, len(self.arrays[FIELDS[0]])) + if ( + field == FIELDS[0] + and i < j + and self.joints[i] in ("WRIST", 1) + and not error + ): + if ( + any(abs(v) > 1e-6 for v in row[:6]) + or abs(abs(row[6]) - 1) > 1e-6 + ): + error = "WRIST bind pose must be identity" + if error: + issue(field, error, i) + for i, joint in enumerate(self.joints): + try: + number = joint_number(joint) + if number in seen: + raise ValueError("Duplicate HandJoint identifier") + seen.add(number) + except ValueError as exc: + issue("joint_ids", str(exc), i) + order = self.editor["output_indices"] + if ( + len(order) != n + or any(type(i) is not int for i in order) + or sorted(order) != list(range(n)) + ): + issue("output_indices", "Taxel indices must cover 0..N-1 exactly") + if self.editor.get("approximate"): + issue( + FIELDS[1], + "Surface/manual placement requires physical calibration", + severity="warning", + ) + return problems + + +class State: + def __init__(self, documents, model=None): + self.documents, self.model = documents, copy.deepcopy(model or {}) + self.revision, self.saved, self.paths = 0, {}, {} + + def fingerprint(self, side): + return json.dumps((vars(self.documents[side]), self.model), sort_keys=True) + + def edit(self, action, *args, **kwargs): + before = copy.deepcopy((self.documents, self.model)) + hashes = [self.fingerprint(s) for s in self.documents] + try: + result = action(*args, **kwargs) + except Exception: + self.documents, self.model = before + raise + if hashes != [self.fingerprint(s) for s in self.documents]: + self.revision += 1 + return result + + def dirty(self): + return any(self.saved.get(s) != self.fingerprint(s) for s in self.documents) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py new file mode 100644 index 0000000000..63c1f39a22 --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -0,0 +1,309 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +from pathlib import Path + +import numpy as np +import trimesh +from trimesh import transformations as tf +from yourdfpy import URDF + +from .document import FIELDS, joint_number, row_error + + +def matrix(p): + try: + a = np.asarray(p, dtype=float) + except OverflowError as exc: + raise ValueError("Pose components exceed floating-point range") from exc + if a.shape != (7,) or not np.isfinite(a).all() or abs(a[3:] @ a[3:] - 1) > 1e-4: + raise ValueError("Invalid XYZ/XYZW pose") + result = tf.quaternion_matrix(a[[6, 3, 4, 5]]) + result[:3, 3] = a[:3] + return result + + +def pose(transform): + q = tf.quaternion_from_matrix(transform) + return [*transform[:3, 3].tolist(), *q[[1, 2, 3, 0]].tolist()] + + +def unit(vector): + vector = np.asarray(vector, dtype=float) + length = np.linalg.norm(vector) + if not np.isfinite(length) or length < 1e-10: + raise ValueError("Reference directions must be finite and non-collinear") + return vector / length + + +def frame(origin, normal, tangent): + z = unit(normal) + x = unit(np.asarray(tangent) - z * (np.asarray(tangent) @ z)) + result = np.eye(4) + result[:3, :3], result[:3, 3] = np.column_stack((x, np.cross(z, x), z)), origin + return result + + +def frames(rows, root): + for i, p in enumerate(rows): + try: + yield i, root @ matrix(p) + except (ValueError, TypeError): + pass + + +class Model: + def __init__(self, path, packages): + self.path, self.missing, self.surfaces = Path(path).resolve(), [], {} + self.handles, self.style, self.attachments = {}, {}, {} + + # yourdfpy invokes this resolver using the keyword fname. + def resource(fname): + name = fname + if name.startswith("package://"): + package, relative = name[10:].split("/", 1) + resolved = ( + Path(packages.get(package, self.path.parent / package)) / relative + ) + elif "://" in name: + raise ValueError(f"Only local resources are supported: {name}") + else: + resolved = self.path.parent / name + if not resolved.is_file(): + self.missing.append(str(resolved)) + return str(resolved) + + self.urdf = URDF.load(str(self.path), filename_handler=resource) + self.links, self.joints = list(self.urdf.link_map), self.urdf.actuated_joints + scene = self.urdf.scene + parents = {b: a for a, b, _ in scene.graph.to_edgelist()} + for node in scene.graph.nodes_geometry: + link = node + while link not in self.links: + link = parents[link] + transform, name = scene.graph.get(frame_to=node, frame_from=link) + mesh = scene.geometry[name].copy() + mesh.apply_transform(transform) + for i, part in enumerate(mesh.split(only_watertight=False)): + self.surfaces[f"{link}/{node}/{i}"] = (link, part) + + def fk(self, cfg): + self.urdf.update_cfg({j.name: cfg.get(j.name, 0) for j in self.joints}) + return {link: self.urdf.get_transform(link).copy() for link in self.links} + + def root(self, d, fk): + reference = d.editor["root"] + local = np.asarray(reference["frame"], dtype=float) + if ( + local.shape != (4, 4) + or not np.isfinite(local).all() + or not np.allclose(local[3], [0, 0, 0, 1]) + or not np.allclose(local[:3, :3].T @ local[:3, :3], np.eye(3)) + or np.linalg.det(local[:3, :3]) < 0 + ): + raise ValueError("Invalid wrist reference frame") + return fk[reference["link"]] @ local + + def attach(self, d, fk): + root = self.root(d, fk) + attachments = self.attachments[d.side] = {} + for i, sid in enumerate(d.editor["segment_ids"]): + link = d.editor.get("segment_links", {}).get(sid) + try: + attachments[sid] = ( + np.linalg.inv(fk[link]) @ root @ matrix(d.arrays[FIELDS[0]][i]) + ) + except (KeyError, TypeError, ValueError): + pass + + def current(self, d, fk): + inverse = np.linalg.inv(self.root(d, fk)) + links, attachments = ( + d.editor.get("segment_links", {}), + self.attachments.get(d.side, {}), + ) + return [ + inverse @ fk[links[sid]] @ attachments[sid] + if sid in attachments and links.get(sid) in fk + else None + for sid in d.editor["segment_ids"] + ] + + def draw(self, scene, cfg, visible=True, highlight=None): + fk = self.fk(cfg) + for name, (link, mesh) in self.surfaces.items(): + if name not in self.handles: + self.handles[name] = scene.add_mesh_simple( + "/model/" + name, + mesh.vertices, + mesh.faces, + color=(170, 175, 180), + side="double", + ) + h = self.handles[name] + shown, opacity = self.style.get(link, (True, 1.0)) + h.visible = visible and shown + h.opacity = ( + opacity if not highlight or link in highlight else opacity * 0.15 + ) + h.position, h.wxyz = fk[link][:3, 3], tf.quaternion_from_matrix(fk[link]) + return fk + + def hit(self, origin, direction, surfaces, fk): + nearest = None + for name in surfaces: + link, mesh = self.surfaces[name] + inverse = np.linalg.inv(fk[link]) + local = (inverse @ np.r_[origin, 1])[:3] + points, _, faces = mesh.ray.intersects_location( + [local], [inverse[:3, :3] @ direction] + ) + for point, face in zip(points, faces): + world = (fk[link] @ np.r_[point, 1])[:3] + distance = np.linalg.norm(world - origin) + if nearest is None or distance < nearest[0]: + nearest = ( + distance, + world, + fk[link][:3, :3] @ mesh.face_normals[face], + name, + ) + return nearest + + def grid(self, d, fk, surface, points, rows, cols, scan, reverse, excluded): + if ( + rows < 1 + or cols < 1 + or points[0] is None + or (cols > 1 and points[1] is None) + or (rows > 1 and points[2] is None) + ): + raise ValueError( + "Pick required first/last centers and use positive row/column counts" + ) + if scan not in ( + "row-major", + "column-major", + "row-snake", + "column-snake", + ) or any( + len(v) != 2 + or any(type(k) is not int for k in v) + or not 0 <= v[0] < rows + or not 0 <= v[1] < cols + for v in excluded + ): + raise ValueError("Invalid scan or excluded cell coordinates") + origin = np.asarray(points[0]) + column = ( + np.asarray(points[1]) - origin if points[1] is not None else np.zeros(3) + ) + row = np.asarray(points[2]) - origin if points[2] is not None else np.zeros(3) + if ( + (cols > 1 and np.linalg.norm(column) < 1e-10) + or (rows > 1 and np.linalg.norm(row) < 1e-10) + or (rows > 1 and cols > 1 and np.linalg.norm(np.cross(column, row)) < 1e-10) + ): + raise ValueError( + "Required row/column spans must be nonzero and independent" + ) + tangent = unit(column if np.linalg.norm(column) > 0 else row) + link, mesh = self.surfaces[surface] + face = trimesh.proximity.closest_point( + mesh, [(np.linalg.inv(fk[link]) @ np.r_[origin, 1])[:3]] + )[2][0] + normal = fk[link][:3, :3] @ mesh.face_normals[face] + cells = [(r, c) for r in range(rows) for c in range(cols)] + major = 1 if scan.startswith("column") else 0 + cells.sort( + key=lambda rc: ( + rc[major], + -rc[1 - major] + if scan.endswith("snake") and rc[major] % 2 + else rc[1 - major], + ) + ) + depth, inverse = ( + max(np.linalg.norm(mesh.extents), 0.01), + np.linalg.inv(self.root(d, fk)), + ) + result = [] + for r, c in cells: + r, c = rows - 1 - r if reverse[0] else r, cols - 1 - c if reverse[1] else c + if [r, c] in excluded: + continue + point = ( + origin + + column * (c / (cols - 1) if cols > 1 else 0) + + row * (r / (rows - 1) if rows > 1 else 0) + ) + hit = self.hit(point + normal * depth, -normal, [surface], fk) + try: + candidate = ( + pose(inverse @ frame(hit[1], hit[2], tangent)) if hit else None + ) + except ValueError: + candidate = None + result.append((candidate, (inverse @ np.r_[point, 1])[:3])) + return result + + +def transform(d, field, selected, delta, local=False): + for i in selected: + old = matrix(d.arrays[field][i]) + d.arrays[field][i] = pose(old @ delta if local else delta @ old) + d.editor["approximate"] = True + + +def deform(d, current): + result = [] + for i, p in enumerate(d.arrays[FIELDS[1]]): + try: + indices, weights = d.arrays[FIELDS[2]][i], d.arrays[FIELDS[3]][i] + if any( + row_error(f, v, len(current)) + for f, v in zip(FIELDS[2:], (indices, weights)) + ): + raise ValueError("Invalid skinning row") + transforms = [ + current[j] @ np.linalg.inv(matrix(d.arrays[FIELDS[0]][j])) @ matrix(p) + for j, w in zip(indices, weights) + if w > 0 + ] + position = sum( + w * t[:3, 3] for w, t in zip([w for w in weights if w > 0], transforms) + ) + rigid = ( + transforms[0].copy() + if len({j for j, w in zip(indices, weights) if w > 0}) == 1 + else None + ) + if rigid is not None: + rigid[:3, 3] = position + result.append((position, rigid)) + except (TypeError, ValueError, KeyError, IndexError): + result.append((None, None)) + return result + + +def hand_current(d, hand): + values = [] + for p, q, valid in zip(hand["positions"], hand["orientations"], hand["is_valid"]): + try: + values.append(matrix([*p, *q]) if valid else None) + except (ValueError, TypeError): + values.append(None) + if len(values) != 26 or len(hand["is_valid"]) != 26 or values[1] is None: + raise ValueError("HandTracker WRIST or array shape is invalid") + inverse = np.linalg.inv(values[1]) + return ( + [ + inverse @ values[joint_number(j)] + if values[joint_number(j)] is not None + else None + for j in d.joints + ], + values, + inverse, + ) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py new file mode 100644 index 0000000000..87917635a9 --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py @@ -0,0 +1,236 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import json +import math +import multiprocessing +import subprocess +import sys +import tempfile +import threading +import time +from pathlib import Path + +from .document import Document, FIELDS, IDENTITY, joint_number +from .storage import load, metadata, write + +ROOT = Path(__file__).resolve().parent.parent + + +def check(document, output=None): + with tempfile.TemporaryDirectory(prefix="tactile-sdk-") as folder: + root = Path(folder) + write(root / "input.json", metadata(document, numerical=True)) + try: + result = subprocess.run( + [sys.executable, "-m", __package__ + ".sdk", "check", folder], + cwd=ROOT, + capture_output=True, + check=False, + timeout=30, + ) + except subprocess.TimeoutExpired as exc: + raise RuntimeError("SDK construction timed out after 30 seconds") from exc + report = root / "result.json" + if result.returncode or not report.exists(): + reason = ( + report.read_text() + if report.exists() + else result.stderr.decode(errors="replace")[-2000:] + ) + raise RuntimeError( + f"SDK construction failed (exit {result.returncode}): " + reason + ) + if output: + Path(output).write_bytes((root / "metadata.bin").read_bytes()) + return json.loads(report.read_text()) + + +def encode(folder): + import numpy as np + from isaaccapture import schema + from isaaccapture.deviceio_trackers import TactileTracker + + root = Path(folder) + d = load(root / "input.json") + arrays = [ + np.asarray( + d.arrays[f], dtype=np.uint32 if f == FIELDS[2] else np.float32 + ).reshape(-1, 7 if f in FIELDS[:2] else 4) + for f in FIELDS + ] + encoded = schema.TactileMetadata( + schema.LBS(*arrays), + schema.HandSkeletonRef([schema.HandJoint(joint_number(j)) for j in d.joints]), + ).to_bytes() + frame_size = len( + schema.TactileDataArray( + np.zeros((len(arrays[1]), 3), dtype=np.float32) + ).to_bytes() + ) + if ( + len(encoded) > schema.TRACKER_METADATA_MAX_SIZE + or frame_size > TactileTracker.DEFAULT_MAX_FLATBUFFER_SIZE + ): + raise ValueError( + "Encoded metadata or force frame exceeds the default tracker capacity" + ) + (root / "metadata.bin").write_bytes(encoded) + return dict( + status="SDK construction and size checks passed", + metadata_bytes=len(encoded), + force_bytes=frame_size, + ) + + +class Receiver: + def __init__(self, configs): + context = multiprocessing.get_context("spawn") + context.set_executable(sys.executable) + self.connection, child = context.Pipe(duplex=False) + self.latest, self.status, self.active = {}, "Connecting", True + self.process = context.Process(target=live, args=(child, configs), daemon=True) + self.process.start() + child.close() + self.thread = threading.Thread(target=self.receive, daemon=True) + self.thread.start() + + def receive(self): + try: + while True: + value = json.loads(self.connection.recv_bytes(16 * 1024 * 1024)) + self.status = value.get("status", "Connected") + self.active = "sides" in value + if "sides" in value: + self.latest = value["sides"] + except (EOFError, OSError, ValueError): + self.active = False + self.status += "; receiver stopped" + + def close(self): + self.process.terminate() + self.process.join(timeout=2) + if self.process.is_alive(): + self.process.kill() + self.process.join() + self.connection.close() + self.thread.join(timeout=2) + + +def owned(value, fields): + return ( + {f: getattr(value, f).copy().tolist() for f in fields} + if value is not None + else None + ) + + +def live(connection, configs): + try: + from isaaccapture.deviceio_session import DeviceIOSession + from isaaccapture.deviceio_trackers import ( + HandTracker, + JointStateTracker, + TactileTracker, + ) + from isaaccapture.oxr import OpenXRSession + + hand = HandTracker() + tactile = {s: TactileTracker(c["collection"]) for s, c in configs.items()} + joints = { + s: JointStateTracker(c["joint_state"]) + for s, c in configs.items() + if c.get("joint_state") + } + trackers = [hand, *tactile.values(), *joints.values()] + with OpenXRSession( + "TactileMetadataEditor", DeviceIOSession.get_required_extensions(trackers) + ) as oxr: + with DeviceIOSession.run(trackers, oxr.get_handles()) as session: + while True: + session.update() + sides = {} + for side, tracker in tactile.items(): + m, data = ( + tracker.get_metadata(session), + tracker.get_data(session), + ) + h = getattr(hand, "get_" + configs[side]["hand"] + "_hand")( + session + ) + j = joints[side].get_data(session) if side in joints else None + raw = None + if m is not None: + d = Document(side) + d.joints = [int(v) for v in m.segment_source.joint_ids] + d.arrays = owned(m.geometry_mapping, FIELDS) + d.sync() + raw = metadata(d) + raw.pop("editor") + sides[side] = dict( + metadata=raw, + forces=data.taxel_forces.copy().tolist() + if data is not None + else None, + hand=owned(h, ("positions", "orientations", "is_valid")), + joints={v.name: [v.position, v.valid] for v in j.joints} + if j is not None + else {}, + read_time=time.monotonic(), + config=configs[side], + ) + connection.send_bytes(json.dumps(dict(sides=sides)).encode()) + time.sleep(1 / 30) + except Exception as exc: + connection.send_bytes( + json.dumps(dict(status=f"{type(exc).__name__}: {exc}")).encode() + ) + finally: + connection.close() + + +def demo(configs): + sides, angle = {}, math.sin(time.monotonic()) * 0.4 + for side, config in configs.items(): + d = Document(side) + d.joints = ["WRIST", "INDEX_DISTAL"] + d.arrays[FIELDS[0]] = [IDENTITY[:], [0, 0.06, 0, 0, 0, 0, 1]] + d.add( + [ + [c * 0.008, 0.06 + r * 0.008, 0, 0, 0, 0, 1] + for r in range(4) + for c in range(4) + ], + [1], + ) + positions = [[0, 0, 0] for _ in range(26)] + orientations = [[0, 0, 0, 1] for _ in range(26)] + valid = [False] * 26 + positions[9] = [0, 0.06, 0] + orientations[9] = [math.sin(angle / 2), 0, 0, math.cos(angle / 2)] + valid[1] = valid[9] = True + sides[side] = dict( + metadata=metadata(d), + forces=[ + [0, 0, 15 + 15 * math.sin(time.monotonic() + i / 4)] for i in range(16) + ], + hand=dict(positions=positions, orientations=orientations, is_valid=valid), + joints={name: [angle, True] for name in config.get("mapping", {})} + if config.get("joint_state") + else {}, + read_time=time.monotonic(), + config=config, + ) + return sides + + +if __name__ == "__main__": + try: + write(Path(sys.argv[2]) / "result.json", encode(sys.argv[2])) + except Exception as exc: + write( + Path(sys.argv[2]) / "result.json", + dict(error=f"{type(exc).__name__}: {exc}"), + ) + sys.exit(1) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/storage.py b/scripts/tactile_metadata/tactile_metadata_editor/storage.py new file mode 100644 index 0000000000..3d5f90645f --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/storage.py @@ -0,0 +1,191 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import copy +import csv +import json +import os +import tempfile +from pathlib import Path + +from .document import Document, FIELDS + + +def parse(value, side=None): + if not isinstance(value, dict) or any( + not isinstance(value.get(k, {}), dict) + for k in ("editor", "geometry_mapping", "segment_source") + ): + raise ValueError( + "Expected a TactileMetadata object with object-valued sections" + ) + editor = value.get("editor", {}) + d = Document(side or editor.get("side")) + if d.side not in (None, "left", "right"): + raise ValueError("Assign metadata to left or right explicitly") + d.types = (value.get("geometry_mapping_type"), value.get("segment_source_type")) + d.joints = copy.deepcopy(value.get("segment_source", {}).get("joint_ids")) + if not isinstance(d.joints, list): + raise ValueError("joint_ids: expected an array") + for field in FIELDS: + rows = value.get("geometry_mapping", {}).get(field) + if not isinstance(rows, list): + raise ValueError(f"{field}: expected an array") + if field in FIELDS[:2] and any( + isinstance(v, dict) + and any( + not isinstance(v.get(k, {}), dict) for k in ("position", "orientation") + ) + for v in rows + ): + raise ValueError(f"{field}: position and orientation must be objects") + d.arrays[field] = [ + ( + [v.get("position", {}).get(k) for k in "xyz"] + + [v.get("orientation", {}).get(k) for k in "xyzw"] + if field in FIELDS[:2] + else v.get("values") + ) + if isinstance(v, dict) + else None + for v in rows + ] + d.editor.update(copy.deepcopy(editor)) + d.sync() + return d + + +def metadata(d, model=None, numerical=False): + if numerical and any(i["severity"] == "error" for i in d.check()): + raise ValueError("Local errors prevent export") + value = dict( + geometry_mapping_type=d.types[0], + geometry_mapping={}, + segment_source_type=d.types[1], + segment_source={"joint_ids": copy.deepcopy(d.joints)}, + ) + order = ( + sorted( + range(len(d.editor["output_indices"])), + key=lambda i: d.editor["output_indices"][i], + ) + if numerical + else None + ) + for field, rows in d.arrays.items(): + if numerical and field in FIELDS[1:]: + rows = [rows[i] for i in order] + value["geometry_mapping"][field] = [ + { + "position": dict(zip("xyz", r[:3])), + "orientation": dict(zip("xyzw", r[3:])), + } + if field in FIELDS[:2] and r is not None + else {"values": copy.deepcopy(r)} + if field in FIELDS[2:] + else None + for r in rows + ] + if not numerical: + value["editor"] = copy.deepcopy(d.editor) + value["editor"].update( + side=d.side, + model=copy.deepcopy( + model if model is not None else d.editor.get("model", {}) + ), + ) + value["editor"]["draft_status"] = ( + "Editing draft; not yet valid metadata" + if any(i["severity"] == "error" for i in d.check()) + else "Locally checked metadata" + ) + return value + + +def load(path, side=None): + path = Path(path).resolve() + with path.open() as stream: + d = parse(json.load(stream), side) + cfg = d.editor.get("model", {}) + if cfg.get("urdf"): + cfg["urdf"] = str((path.parent / cfg["urdf"]).resolve()) + cfg["packages"] = { + k: str((path.parent / v).resolve()) for k, v in cfg.get("packages", {}).items() + } + return d + + +def write(path, value): + path = Path(path).resolve() + fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".tactile-") + try: + with os.fdopen(fd, "w") as stream: + json.dump(value, stream, indent=2) + stream.write("\n") + os.replace(temporary, path) + finally: + Path(temporary).unlink(missing_ok=True) + + +def save(state, paths): + if len({Path(p).resolve() for p in paths.values()}) != len(paths): + raise ValueError("Each hand requires a different save path") + for side, path in paths.items(): + value = metadata(state.documents[side], state.model) + cfg, parent = value["editor"]["model"], Path(path).resolve().parent + if cfg.get("urdf"): + cfg["urdf"] = os.path.relpath(cfg["urdf"], parent) + cfg["packages"] = { + k: os.path.relpath(v, parent) for k, v in cfg.get("packages", {}).items() + } + write(path, value) + for side, path in paths.items(): + state.paths[side], state.saved[side] = str(path), state.fingerprint(side) + + +def backup(d, model): + with tempfile.NamedTemporaryFile( + prefix="tactile-backup-", suffix=".json", delete=False + ) as stream: + path = stream.name + write(path, metadata(d, model)) + return path + + +def export(documents, model, revision, destination, sdk=False): + destination = Path(destination).resolve() + if destination.exists() and ( + not destination.is_dir() or any(destination.iterdir()) + ): + raise ValueError("Choose a new or empty output directory") + with tempfile.TemporaryDirectory( + prefix=".tactile-", dir=destination.parent + ) as temporary: + root, report = Path(temporary), {"revision": revision, "sides": {}} + for side, d in documents.items(): + write(root / f"{side}.metadata.json", metadata(d, numerical=True)) + editable = metadata(d, model) + cfg = editable["editor"]["model"] + if cfg.get("urdf"): + cfg["urdf"] = os.path.relpath(cfg["urdf"], destination) + cfg["packages"] = { + k: os.path.relpath(v, destination) + for k, v in cfg.get("packages", {}).items() + } + write(root / f"{side}.editor.json", editable) + result = dict(local=d.check(), sdk="not checked") + if sdk: + from .sdk import check + + result["sdk"] = check(d, root / f"{side}.metadata.bin") + with (root / f"{side}.taxels.csv").open("w", newline="") as stream: + writer = csv.writer(stream) + writer.writerow(("draft_taxel", "exported_index")) + writer.writerows(zip(d.editor["taxel_ids"], d.editor["output_indices"])) + report["sides"][side] = result + write(root / "report.json", report) + if destination.exists(): + destination.rmdir() + os.replace(root, destination) + return report diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py new file mode 100644 index 0000000000..407856cd39 --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -0,0 +1,1558 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + +import copy +import json +import logging +import math +import queue +import time +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path + +import numpy as np +import trimesh +import viser +from trimesh import transformations as tf + +from . import geometry as geo, sdk, storage +from .document import Document, FIELDS, GROUPS, IDENTITY, JOINTS, State, joint_number + +LOG = logging.getLogger("isaaccapture.tactile_metadata.editor") + + +class Editor: + def __init__(self, state, args): + self.state, self.args, self.started = state, args, bool(state.documents) + if not state.documents: + state.documents["left"] = Document("left") + state.saved["left"] = state.fingerprint("left") + self.events, self.pool = queue.SimpleQueue(), ThreadPoolExecutor(max_workers=1) + self.model, self.receiver, self.job = None, None, None + self.selected, self.points, self.candidates = [], {}, [] + self.handles, self.q, self.logs, self.drive = {}, {}, [], {} + self.frame_forces, self.frame_colors, self.frame_counts, self.issues = ( + {}, + {}, + {}, + [], + ) + self.pick_target, self.slot, self.dragging, self.closing = ( + None, + None, + False, + False, + ) + self.sdk_result, self.sdk_revision = "not checked", None + self.server = viser.ViserServer(host="127.0.0.1", port=args.port) + g = self.server.gui + g.configure_theme(control_width="medium", show_share_button=False) + g.main_panel.dock_right() + self.header = g.add_markdown("") + self.side = g.add_dropdown("Edit hand", tuple(state.documents)) + self.both = g.add_checkbox("Show both hands", True) + self.mode = g.add_dropdown("Mode", ("EDIT", "PREVIEW", "LIVE", "PAUSED")) + self.watch(self.side, lambda _: self.switch()) + self.watch(self.mode, lambda _: None) + with g.add_tab_group() as tabs: + with tabs.add_tab("Model"): + self.urdf = g.add_text("URDF", state.model.get("urdf", "")) + self.packages = g.add_text( + "Local package map JSON", + json.dumps(state.model.get("packages", {})), + ) + self.sides = g.add_dropdown( + "New model hands", + ("left", "right", "both"), + initial_value="both" + if len(state.documents) == 2 + else self.side.value, + ) + self.button("New document", lambda: self.guard(self.reset)) + self.button("Load URDF", lambda: self.guard(self.load_new)) + self.open_path = g.add_text("Metadata JSON", args.metadata or "") + self.json_side = g.add_dropdown( + "Assign JSON to", ("left", "right"), initial_value=self.side.value + ) + self.button("Open JSON", lambda: self.guard(self.open)) + self.wrist = g.add_dropdown("Wrist link", ("",)) + self.allowed = g.add_text("Allowed links, comma separated", "") + self.button("Apply allowed links", self.set_links) + self.pick = g.add_dropdown( + "Pick target", + ( + "O", + "C", + "R", + "Wrist O", + "Wrist F", + "Wrist D", + "Segment", + "Move selected", + "Single tangent", + ), + ) + self.pick_button = self.button( + "Pick on q0 model", + lambda: setattr(self, "pick_target", self.pick.value), + ) + self.calibrate_button = self.button( + "Preview / set wrist reference", self.calibrate + ) + self.link = g.add_dropdown("Display link", ("",)) + self.opacity = g.add_slider("Link opacity", 0.0, 1.0, 0.05, 1.0) + self.visible = g.add_checkbox("Link visible", True) + self.watch(self.opacity, lambda _: self.link_style()) + self.watch(self.visible, lambda _: self.link_style()) + self.resources = g.add_markdown("") + self.button("Reset cameras", self.camera) + with tabs.add_tab("Segments"): + self.joint = g.add_dropdown("URDF free joint", ("",)) + self.angle = g.add_slider("Preview joint", -3.15, 3.15, 0.001, 0.0) + self.angle_number = g.add_number("Preview joint value", 0.0) + self.watch(self.joint, lambda _: self.joint_controls()) + for w in (self.angle, self.angle_number): + self.watch(w, self.preview_joint) + self.button("Set shared binding pose q0", self.confirm_q0) + self.button("Return to q0", lambda: setattr(self.mode, "value", "EDIT")) + self.segment = g.add_dropdown("Segment row", ("",)) + self.semantic = g.add_dropdown("HandJoint", tuple(JOINTS)) + self.segment_link = g.add_dropdown("Associated link", ("",)) + self.watch(self.segment, lambda _: self.select_segment()) + self.button("Add segment", self.add_segment) + self.button("Set semantic / link", self.associate) + self.order = g.add_text("Segment row order", "[]") + self.button( + "Reorder / delete omitted segments", + self.reorder_segments, + ) + self.segment_table = g.add_markdown("") + with tabs.add_tab("Layout"): + self.surface = g.add_dropdown("Projection surface", ("",)) + self.rows = g.add_number("Rows", 4, min=1) + self.cols = g.add_number("Columns", 4, min=1) + self.centers = [ + g.add_vector3(k + " world center", (0.0, 0.0, 0.0)) + for k in ("O", "C", "R") + ] + for key, w in zip(("O", "C", "R"), self.centers): + self.watch(w, lambda v, key=key: self.points.update({key: list(v)})) + self.spans = [ + g.add_number(k + " span (m)", 0.03) for k in ("Column", "Row") + ] + self.button("Apply spans along picked directions", self.spans_apply) + self.scan = g.add_dropdown( + "Scan", ("row-major", "column-major", "row-snake", "column-snake") + ) + self.reverse = [ + g.add_checkbox(k, False) + for k in ("Reverse rows", "Reverse columns") + ] + self.excluded = g.add_text("Exclude cells [row,column]", "[]") + self.start = g.add_number("Start output index (-1 = unused)", -1) + self.influences = g.add_text("New taxel segment rows (1–4)", "[]") + self.grid_button = self.button("Preview grid", self.grid) + self.button("Apply candidates", self.apply_grid) + self.button("Cancel candidates", self.clear_layout) + self.layout_info = g.add_markdown("") + self.selection = g.add_text("Select output indices / ranges", "") + self.button("Select indices", self.select_indices) + self.rectangle = g.add_checkbox("Rectangle selection (drag)", False) + self.through = g.add_checkbox("Select through mesh", False) + self.watch(self.rectangle, self.rectangle_mode) + self.button( + "Delete selected taxels", + self.delete_taxels, + ) + self.reproject_button = self.button( + "Reproject selected", self.reproject + ) + self.indices = g.add_text("New selected output indices", "[]") + self.button("Preview / apply output indices", self.reindex) + self.page = g.add_number("List page (25 rows)", 0, min=0) + self.watch(self.page, lambda _: None) + self.table = g.add_markdown("") + with tabs.add_tab("Binding"): + self.binding_info = g.add_markdown("") + self.button( + "Assign segment rows equally", + lambda: self.change( + self.d.assign, self.selected, json.loads(self.influences.value) + ), + ) + self.slots = [] + for k in range(4): + with g.add_folder(f"Slot {k}"): + segment = g.add_dropdown("Segment", ("",)) + index = g.add_number("Set raw index", 0) + slider = g.add_slider("Weight", 0.0, 1.0, 0.001, 0.0) + number = g.add_number("Set weight", 0.0) + self.watch( + segment, + lambda v, k=k: ( + self.set_slot(k, int(v.split(":")[0])) + if not v.startswith("Raw") + else None + ), + ) + self.watch(index, lambda v, k=k: self.set_slot(k, v)) + for w in (slider, number): + self.watch(w, lambda v, k=k: self.weight(k, v)) + self.slots.append((segment, index, slider, number)) + with tabs.add_tab("Live"): + self.collection = g.add_text("Tactile collection_id", "tactile") + self.hand = g.add_dropdown("HandTracker output", ("left", "right")) + self.joint_enabled = g.add_checkbox("Enable JointState", False) + self.joint_id = g.add_text("JointState collection_id", "") + self.button( + "Save this hand's source configuration", self.configure_source + ) + self.source_joint = g.add_text("Source joint name", "") + self.target_joint = g.add_dropdown("URDF target joint", ("",)) + self.scale = g.add_number("Scale", 1.0) + self.offset = g.add_number("Offset", 0.0) + self.button("Add / update mapping", self.mapping) + self.button("Remove mapping", lambda: self.mapping(True)) + self.mapping_info = g.add_markdown("") + self.button("Connect / disconnect", self.connect) + self.button( + "Play / pause", + lambda: setattr( + self.mode, + "value", + "PAUSED" if self.mode.value == "LIVE" else "LIVE", + ), + ) + self.source = g.add_dropdown( + "Visualization metadata", ("Received", "Draft") + ) + self.force_max = g.add_number( + "Force color maximum (N)", 30.0, min=0.001 + ) + self.groups = {name: g.add_checkbox(name, True) for name in GROUPS} + self.button("Record metadata groups", self.record) + self.live_info = g.add_markdown("") + with tabs.add_tab("Export"): + self.paths = { + s: g.add_text( + s + " save JSON", state.paths.get(s, s + ".metadata.json") + ) + for s in ("left", "right") + } + self.button("Save all drafts", self.save) + self.all_sides = g.add_checkbox("Check / export all hands", True) + self.destination = g.add_text("New output directory", "package") + self.binary = g.add_checkbox("Include SDK binary", False) + self.button("Local check", self.local_check) + self.button("SDK check", lambda: self.background(False)) + self.button("Export snapshot", lambda: self.background(True)) + self.check_info = g.add_markdown("") + with g.add_folder("Advanced schema repair"): + self.schema = g.add_text("Schema JSON", "", multiline=True) + self.button( + "Copy draft into repair field", + lambda: setattr( + self.schema, + "value", + json.dumps( + storage.metadata(self.d, self.state.model), indent=2 + ), + ), + ) + self.button("Replace draft from repair field", self.repair) + with g.add_folder("Draft property inspector"): + self.entity = g.add_dropdown("Edit entity", ("Taxels", "Segment")) + self.watch(self.entity, lambda _: None) + self.raw = g.add_markdown("") + self.pose_fields = [] + for axis, k in enumerate("XYZ"): + w = g.add_number("Set " + k + " (m)", 0.0) + self.watch(w, lambda v, axis=axis: self.components([axis], [v])) + self.pose_fields.append(w) + self.button("Flip normal", lambda: self.rotate(np.pi, [1, 0, 0])) + self.twist = g.add_number("Tangent twist (degrees)", 0.0) + self.button( + "Apply tangent twist", + lambda: self.rotate(np.deg2rad(self.twist.value), [0, 0, 1]), + ) + with g.add_folder("Advanced XYZW"): + for axis, k in enumerate("XYZW", 3): + w = g.add_number("Set Q" + k, float(axis == 6)) + self.watch(w, lambda v, axis=axis: self.components([axis], [v])) + self.pose_fields.append(w) + self.radius = g.add_number("Marker radius (m)", 0.002, min=0.0001) + g.add_markdown( + "Draft manipulators always edit q0. Wire markers are draft targets during playback. Ctrl/Cmd toggles selection. Ranges use `0:15,20`." + ) + self.info = g.add_dropdown("Info issue", ("",)) + self.button("Locate issue in inspector", self.locate) + self.log_text = g.add_markdown("") + self.layers = { + s: self.server.scene.add_frame("/view/" + s, show_axes=False) + for s in ("left", "right") + } + self.gizmo = self.server.scene.add_transform_controls( + "/draft-editor", scale=0.05, visible=False + ) + self.gizmo.on_update(self.gizmo_event) + for modifier in (None, "cmd/ctrl"): + self.server.scene.on_click(modifier=modifier)(self.scene_click) + if state.model.get("urdf"): + try: + self.install_model( + geo.Model(state.model["urdf"], state.model.get("packages", {})) + ) + except (OSError, ValueError) as exc: + self.resources.content = str(exc) + self.log( + "URDF unavailable; numerical draft editing remains available: " + + str(exc) + ) + self.switch() + + @property + def d(self): + return self.state.documents[self.side.value] + + def watch(self, widget, callback): + async def updated(event): + if event.client: + value = copy.deepcopy(event.target.value) + self.events.put(lambda: callback(value)) + + widget.on_update(updated) + + def button(self, label, callback): + handle = self.server.gui.add_button(label) + + async def clicked(_): + self.events.put(callback) + + handle.on_click(clicked) + return handle + + def log(self, message): + LOG.info("%s", message) + self.logs = (self.logs + [str(message)])[-8:] + self.log_text.content = "\n\n".join(self.logs) + + def confirm(self, message, action, unsaved=False, cleanup=None): + modal = self.server.gui.add_modal("Confirm") + + def close(): + if cleanup: + cleanup() + modal.close() + + def finish(save=False): + if save: + self.save() + action() + close() + + with modal: + self.server.gui.add_markdown(message) + if unsaved: + self.button("Save and continue", lambda: finish(True)) + self.button("Discard and continue" if unsaved else "Apply", finish) + self.button("Cancel", close) + + def guard(self, action): + if self.state.dirty(): + self.confirm("Unsaved drafts", action, True) + else: + action() + + def change(self, action, *args, **kwargs): + result = self.state.edit(action, *args, **kwargs) + self.refresh() + return result + + def install_model(self, model): + if self.model: + for h in self.model.handles.values(): + h.remove() + self.model = model + self.wrist.options = self.link.options = self.segment_link.options = tuple( + model.links + ) + self.joint.options = self.target_joint.options = tuple( + j.name for j in model.joints + ) or ("",) + self.q = copy.deepcopy(self.state.model.get("q0", {})) + self.resources.content = ( + "Missing resources: " + + str(model.missing) + + "\n\n" + + "\n\n".join( + f"{j.name}: {j.parent} → {j.child}; {j.type}; limits={j.limit}; mimic={j.mimic}" + for j in model.urdf.robot.joints + ) + ) + self.sides.disabled = True + self.sides.value = "both" if len(self.state.documents) == 2 else self.side.value + self.joint_controls() + + def reset(self): + if self.receiver: + self.connect() + for h in ( + *self.handles.values(), + *(self.model.handles.values() if self.model else ()), + ): + h.remove() + self.handles, self.model = {}, None + self.state, self.started = State({"left": Document("left")}), False + self.state.saved["left"] = self.state.fingerprint("left") + self.side.options, self.side.value = ("left",), "left" + self.sides.disabled, self.mode.value = False, "EDIT" + self.frame_forces, self.frame_colors, self.frame_counts, self.drive = ( + {}, + {}, + {}, + {}, + ) + self.q, self.resources.content = {}, "" + for w in ( + self.wrist, + self.link, + self.joint, + self.segment_link, + self.target_joint, + self.surface, + ): + w.options = ("",) + for side, w in self.paths.items(): + w.value = side + ".metadata.json" + self.switch() + + def load_new(self): + cfg = dict( + urdf=str(Path(self.urdf.value).resolve()), + packages={ + k: str(Path(v).resolve()) + for k, v in json.loads(self.packages.value).items() + }, + q0={}, + ) + model = geo.Model(cfg["urdf"], cfg["packages"]) + sides = ("left", "right") if self.sides.value == "both" else (self.sides.value,) + self.reset() + self.state, self.started = State({s: Document(s) for s in sides}, cfg), True + self.side.options, self.side.value = sides, sides[0] + self.install_model(model) + self.switch() + + def open(self): + side, path = self.json_side.value, self.open_path.value + d = storage.load(path, side) + if self.model and side not in self.state.documents: + raise ValueError("Create a new document to change the fixed model hands") + cfg = d.editor.get("model", {}) + model = ( + geo.Model(cfg["urdf"], cfg.get("packages", {})) + if not self.model and cfg.get("urdf") + else None + ) + + def replace(): + if not self.started: + self.state.documents = {} + self.state.documents[side] = d + if model: + self.state.model = copy.deepcopy(cfg) + + if self.receiver: + self.connect() + self.mode.value = "EDIT" + self.state.edit(replace) + self.started = True + self.side.options, self.side.value = tuple(self.state.documents), side + if model: + self.install_model(model) + self.state.paths[side] = self.paths[side].value = str(Path(path).resolve()) + if d.editor.get("side") == side and cfg == self.state.model: + self.state.saved[side] = self.state.fingerprint(side) + else: + self.state.saved.pop(side, None) + self.switch() + + def save(self): + storage.save(self.state, {s: self.paths[s].value for s in self.state.documents}) + self.log("Saved all drafts") + + def switch(self): + self.selected, self.points, self.candidates = [], {}, [] + self.pick_target, self.slot = None, None + self.segment.value = "" + self.influences.value = "[]" + self.allowed.value = ",".join(self.d.editor.get("allowed_links", [])) + if self.model: + link = self.d.editor.get("root", {}).get("link") + self.wrist.value = link if link in self.model.links else self.model.links[0] + source = self.d.editor.get("source", {}) + self.collection.value, self.hand.value = ( + source.get("collection", "tactile"), + source.get("hand", self.d.side), + ) + self.joint_id.value = source.get("joint_state", "") + self.joint_enabled.value = bool(self.joint_id.value) + self.refresh() + + def root(self, d=None, cfg=None): + d = d or self.d + if self.model and "root" in d.editor: + try: + return self.model.root( + d, + self.model.fk( + cfg if cfg is not None else self.state.model.get("q0", {}) + ), + ) + except (KeyError, ValueError, TypeError): + pass + return np.eye(4) + + def surfaces(self): + return [ + n + for n, (link, _) in self.model.surfaces.items() + if link in self.d.editor.get("allowed_links", []) + and self.model.style.get(link, (True, 1))[0] + ] + + def set_links(self): + links = [v.strip() for v in self.allowed.value.split(",") if v.strip()] + if not self.model or any(v not in self.model.links for v in links): + raise ValueError("Choose links from the loaded model") + self.change(self.d.editor.update, allowed_links=links) + + def link_style(self): + if self.model: + self.model.style[self.link.value] = (self.visible.value, self.opacity.value) + + def camera(self): + for client in self.server.get_clients().values(): + client.camera.position, client.camera.look_at = (0.3, -0.4, 0.3), (0, 0, 0) + + def calibrate(self): + o, f, d = [ + np.asarray(self.points[k]) for k in ("Wrist O", "Wrist F", "Wrist D") + ] + z = geo.unit(f - o) + y = geo.unit(d - o - z * ((d - o) @ z)) + reference = geo.frame(o, z, np.cross(y, z)) + for other in self.state.documents.values(): + if ( + other is not self.d + and other.editor.get("root", {}).get("link") == self.wrist.value + ): + raise ValueError( + "Both hands require distinct wrist links in the actual model" + ) + self.axes("/calibration-preview", [reference]) + local = ( + np.linalg.inv( + self.model.fk(self.state.model.get("q0", {}))[self.wrist.value] + ) + @ reference + ) + value = dict(link=self.wrist.value, frame=local.tolist()) + + def apply(): + self.change(self.d.editor.update, root=value) + self.clear_layout() + + self.confirm( + "Preserve all numerical bind poses and weights; recompute attachments at this wrist reference.", + apply, + cleanup=lambda: setattr( + self.handles["/calibration-preview"], "visible", False + ), + ) + + def joint_controls(self): + if self.model and self.joint.value: + j = self.model.urdf.joint_map[self.joint.value] + low = j.limit.lower if j.limit and j.limit.lower is not None else -np.pi + high = j.limit.upper if j.limit and j.limit.upper is not None else np.pi + self.angle.min, self.angle.max = low, high + self.angle_number.min = j.limit.lower if j.limit else None + self.angle_number.max = j.limit.upper if j.limit else None + self.angle_number.value = self.q.get(j.name, 0.0) + self.angle.value = float(np.clip(self.angle_number.value, low, high)) + + def preview_joint(self, value): + if self.joint.value: + self.q[self.joint.value] = value + self.angle_number.value = value + self.angle.value = float(np.clip(value, self.angle.min, self.angle.max)) + self.mode.value = "PREVIEW" + + def confirm_q0(self): + q0 = copy.deepcopy(self.q) + + def apply(): + self.change(self.state.model.update, q0=q0) + self.mode.value = "EDIT" + self.clear_layout() + + self.confirm( + "Update shared q0; bind poses and weights stay unchanged and points may leave the mesh. Affected hands: " + + ", ".join(self.state.documents), + apply, + ) + + def add_segment(self): + initial = IDENTITY[:] if self.semantic.value == "WRIST" else None + if self.semantic.value != "WRIST" and self.model and "root" in self.d.editor: + initial = geo.pose( + np.linalg.inv(self.root()) + @ self.model.fk(self.state.model.get("q0", {}))[self.segment_link.value] + ) + + def add(): + self.d.joints.append(self.semantic.value) + self.d.arrays[FIELDS[0]].append(initial) + self.d.sync() + self.d.editor.setdefault("segment_links", {})[ + self.d.editor["segment_ids"][-1] + ] = self.segment_link.value + self.d.editor["approximate"] = True + + self.change(add) + self.segment.value = str(len(self.d.arrays[FIELDS[0]]) - 1) + self.select_segment() + + def associate(self): + i = int(self.segment.value) + + def update(): + self.d.joints[i] = self.semantic.value + self.d.editor.setdefault("segment_links", {})[ + self.d.editor["segment_ids"][i] + ] = self.segment_link.value + + self.change(update) + + def select_segment(self): + self.entity.value, self.slot = "Segment", None + if self.segment.value: + i = int(self.segment.value) + if i < len(self.d.joints): + self.semantic.value = JOINTS[joint_number(self.d.joints[i])] + link = self.d.editor.get("segment_links", {}).get( + self.d.editor["segment_ids"][i] + ) + if self.model and link in self.model.links: + self.segment_link.value = link + + def targets(self): + if self.entity.value == "Segment" and self.segment.value: + return FIELDS[0], [int(self.segment.value)] + return FIELDS[1], self.selected + + def reorder_segments(self): + ids = self.d.editor["segment_ids"] + selected = ( + ids[int(self.segment.value)] + if self.segment.value and int(self.segment.value) < len(ids) + else None + ) + self.change(self.d.segments, json.loads(self.order.value)) + ids = self.d.editor["segment_ids"] + self.segment.value = str(ids.index(selected)) if selected in ids else "" + + def delete_taxels(self): + self.change(self.d.delete, self.selected) + self.selected = [] + + def components(self, components, values): + field, rows = self.targets() + if not rows: + raise ValueError("Select a draft segment or taxel first") + self.change(self.d.set_components, field, rows, components, values) + + def rotate(self, angle, axis): + field, rows = self.targets() + if not rows: + raise ValueError("Select a draft segment or taxel first") + self.change( + geo.transform, + self.d, + field, + rows, + tf.rotation_matrix(angle, axis), + local=True, + ) + + # Inline async callbacks preserve Viser's drag phase ordering. + async def gizmo_event(self, event): + if event.phase == "start" and event.client: + self.dragging = True + elif event.phase == "end": + if event.client: + self.drag_end(event) + else: + self.dragging = False + + def drag_end(self, event): + values = [*event.target.position, *np.asarray(event.target.wxyz)[[1, 2, 3, 0]]] + + def apply(): + try: + field, rows = self.targets() + if rows: + old = geo.matrix(self.d.arrays[field][rows[0]]) + delta = ( + np.linalg.inv(self.root()) + @ geo.matrix(values) + @ np.linalg.inv(old) + ) + self.change(geo.transform, self.d, field, rows, delta) + finally: + self.dragging = False + + self.events.put(apply) + + def set_slot(self, k, value): + self.slot = k + self.change(self.d.set_components, FIELDS[2], self.selected, [k], [value]) + + def weight(self, k, value): + self.slot = k + rejected = self.change(self.d.weight, self.selected, k, value) + if rejected: + self.log("; ".join(rejected)) + + def spans_apply(self): + o = np.asarray(self.points["O"]) + for key, control, span in zip(("C", "R"), self.centers[1:], self.spans): + if key in self.points: + self.points[key] = ( + o + geo.unit(np.asarray(self.points[key]) - o) * span.value + ).tolist() + control.value = tuple(self.points[key]) + + def grid(self): + if self.surface.value not in self.surfaces(): + raise ValueError("Choose an allowed projection surface") + self.candidates = self.model.grid( + self.d, + self.model.fk(self.state.model.get("q0", {})), + self.surface.value, + [self.points.get(k) for k in ("O", "C", "R")], + int(self.rows.value), + int(self.cols.value), + self.scan.value, + [w.value for w in self.reverse], + json.loads(self.excluded.value), + ) + start = int(self.start.value) + start = self.d.unused_start(len(self.candidates)) if start < 0 else start + spans = [ + np.linalg.norm(np.asarray(self.points[k]) - self.points["O"]) + if k in self.points + else 0 + for k in ("C", "R") + ] + spacing = [ + v / (n - 1) if n > 1 else 0 + for v, n in zip(spans, (self.cols.value, self.rows.value)) + ] + self.layout_info.content = f"Spans C/R: {spans} m; spacing C/R: {spacing} m. {len(self.candidates)} candidates; {sum(p is None for p, _ in self.candidates)} projection failures. Output indices: {list(range(start, start + len(self.candidates)))}" + + def apply_grid(self): + self.change( + self.d.add, + [p for p, _ in self.candidates], + json.loads(self.influences.value), + int(self.start.value), + ) + self.clear_layout() + + def clear_layout(self): + self.candidates, self.points = [], {} + self.rows.value = self.cols.value = 4 + self.scan.value, self.start.value, self.excluded.value = "row-major", -1, "[]" + for w in self.reverse: + w.value = False + for w in self.centers: + w.value = (0.0, 0.0, 0.0) + for w in self.spans: + w.value = 0.03 + self.layout_info.content = "" + + def select_indices(self): + indices = [] + for part in self.selection.value.split(","): + ends = [int(v) for v in part.strip().split(":")] + indices.extend(range(ends[0], ends[1] + 1) if len(ends) == 2 else ends) + self.selected = [ + i + for i, index in enumerate(self.d.editor["output_indices"]) + if index in indices + ] + self.entity.value = "Taxels" + + def reindex(self): + values, order = ( + json.loads(self.indices.value), + self.d.editor["output_indices"][:], + ) + if len(values) != len(self.selected) or any( + type(v) is not int or v < 0 for v in values + ): + raise ValueError("Supply one nonnegative integer per selected taxel") + preview = [(order[i], value) for i, value in zip(self.selected, values)] + for i, value in zip(self.selected, values): + order[i] = value + self.confirm( + str(preview), + lambda: self.change(self.d.editor.update, output_indices=order), + ) + + def reproject(self): + root, fk = self.root(), self.model.fk(self.state.model.get("q0", {})) + if self.surface.value not in self.surfaces(): + raise ValueError("Choose an allowed projection surface") + + def apply(): + for i in self.selected: + old = root @ geo.matrix(self.d.arrays[FIELDS[1]][i]) + hit = self.model.hit( + old[:3, 3] + old[:3, 2], -old[:3, 2], [self.surface.value], fk + ) + if not hit: + raise ValueError( + f"Taxel {i}: projection failed; selection unchanged" + ) + self.d.arrays[FIELDS[1]][i] = geo.pose( + np.linalg.inv(root) @ geo.frame(hit[1], hit[2], old[:3, 0]) + ) + self.d.editor["approximate"] = True + + self.change(apply) + + def configure_source(self): + source = dict( + collection=self.collection.value, + hand=self.hand.value, + joint_state=self.joint_id.value if self.joint_enabled.value else "", + ) + if not source["collection"].strip() or ( + self.joint_enabled.value and not source["joint_state"].strip() + ): + raise ValueError("Enabled sources require collection_id values") + self.change(self.d.editor.update, source=source) + if self.receiver: + self.log("Source configuration saved; reconnect to apply it") + + def mapping(self, remove=False): + mappings = dict(self.d.editor.get("mapping", {})) + if remove: + mappings.pop(self.source_joint.value, None) + else: + if ( + not self.source_joint.value + or not self.target_joint.value + or not all( + math.isfinite(v) for v in (self.scale.value, self.offset.value) + ) + ): + raise ValueError("Choose source/target joints and finite scale/offset") + mappings[self.source_joint.value] = [ + self.target_joint.value, + self.scale.value, + self.offset.value, + ] + self.change(self.d.editor.update, mapping=mappings) + + def connect(self): + if self.receiver: + if self.receiver is not True: + self.receiver.close() + self.receiver = None + self.log("Disconnected") + else: + self.configure_source() + self.configs = { + s: dict( + d.editor.get( + "source", dict(collection="tactile", hand=s, joint_state="") + ), + mapping=d.editor.get("mapping", {}), + ) + for s, d in self.state.documents.items() + } + self.receiver = True if self.args.demo else sdk.Receiver(self.configs) + self.mode.value = "LIVE" + + def latest(self): + return ( + sdk.demo(self.configs) + if self.receiver is True + else self.receiver.latest + if self.receiver + else {} + ) + + def record(self): + value = self.latest().get(self.side.value, {}).get("metadata") + if value is None: + raise ValueError("No received metadata") + received = storage.parse(value, self.side.value) + path = storage.backup(self.d, self.state.model) + self.log("Draft backup: " + path) + self.change( + self.d.record, received, [k for k, w in self.groups.items() if w.value] + ) + + def repair(self): + d = storage.parse(json.loads(self.schema.value), self.side.value) + self.log("Draft backup: " + storage.backup(self.d, self.state.model)) + self.change(self.state.documents.__setitem__, self.side.value, d) + + def local_check(self): + documents = ( + self.state.documents if self.all_sides.value else {self.side.value: self.d} + ) + self.log(str({s: d.check() for s, d in documents.items()})) + + def background(self, exporting): + if self.job and not self.job.done(): + raise ValueError("A check/export is already running") + documents = copy.deepcopy( + self.state.documents if self.all_sides.value else {self.side.value: self.d} + ) + self.job_revision = self.state.revision + self.sdk_revision = self.job_revision + if exporting: + self.job = self.pool.submit( + storage.export, + documents, + copy.deepcopy(self.state.model), + self.job_revision, + self.destination.value, + self.binary.value, + ) + else: + self.job = self.pool.submit( + lambda: {s: sdk.check(d) for s, d in documents.items()} + ) + self.sdk_result = f"Running for revision {self.job_revision}" + + def refresh(self): + d = self.d + self.issues = [ + dict(v, side=s) + for s, doc in self.state.documents.items() + for v in doc.check() + ] + if self.model: + fk = self.model.fk(self.state.model.get("q0", {})) + for doc in self.state.documents.values(): + try: + if "root" in doc.editor: + self.model.attach(doc, fk) + except (ValueError, KeyError, TypeError) as exc: + self.model.attachments.pop(doc.side, None) + self.issues.append( + dict( + side=doc.side, + severity="warning", + field="Model", + row=None, + message=str(exc), + ) + ) + self.surface.options = tuple(self.surfaces()) or ("",) + calibrated = bool(self.model and d.side in self.model.attachments) + self.pick_button.disabled = self.calibrate_button.disabled = self.model is None + self.grid_button.disabled = self.reproject_button.disabled = not calibrated + self.angle.disabled = self.angle_number.disabled = self.model is None + self.selected = [i for i in self.selected if 0 <= i < len(d.arrays[FIELDS[1]])] + self.side.visible = self.both.visible = len(self.state.documents) == 2 + self.header.content = f"{Path(self.state.model.get('urdf', 'No URDF')).name}; hands: {', '.join(self.state.documents)}; {self.mode.value}\n\n{self.state.paths.get(d.side, 'Unsaved draft')}{' *' if self.state.dirty() else ''}; revision {self.state.revision}" + self.segment.options = ( + "", + *map(str, range(max(len(d.joints), len(d.arrays[FIELDS[0]])))), + ) + links = d.editor.get("segment_links", {}) + self.segment_table.content = "\n\n".join( + f"row {i}: {j}; link={links.get(sid, 'unassigned')}; attachment={'ready' if self.model and sid in self.model.attachments.get(d.side, {}) else 'missing'}" + for i, (j, sid) in enumerate(zip(d.joints, d.editor["segment_ids"])) + ) + first = int(self.page.value) * 25 + self.table.content = "\n\n".join( + f"row {i}: output {d.editor['output_indices'][i]}; {p}" + for i, p in enumerate(d.arrays[FIELDS[1]]) + if first <= i < first + 25 + ) + field, rows = self.targets() + values = [ + d.arrays[field][i] if i < len(d.arrays[field]) else None for i in rows + ] + mixed = len({str(v) for v in values}) > 1 + self.raw.content = f"Draft {field}; rows {rows[:25]}; {'Multiple values' if mixed else str(values[:1])}. Set controls replace only the edited component." + for k, w in enumerate(self.pose_fields): + cells = [ + v[k] if isinstance(v, list) and k < len(v) else None for v in values + ] + if ( + cells + and all(v == cells[0] for v in cells) + and type(cells[0]) in (int, float) + and abs(cells[0]) <= 3.4028234663852886e38 + ): + w.value = float(cells[0]) + raw, sums = [], [] + for i in self.selected: + parts = [ + d.arrays[f][i] if i < len(d.arrays[f]) else None for f in FIELDS[2:] + ] + raw.append((i, *parts)) + sums.append( + sum(parts[1]) + if isinstance(parts[1], list) + and all(type(v) in (int, float) for v in parts[1]) + else "invalid" + ) + self.binding_info.content = f"{len(self.selected)} selected; weight sums (first 25): {sums[:25]}; {'Multiple values' if len({str(v[1:]) for v in raw}) > 1 else ''}\n\nRaw rows (index slots, weight slots, first 25): {raw[:25]}" + for k, (dropdown, index, slider, number) in enumerate(self.slots): + indices = [ + v[1][k] if isinstance(v[1], list) and len(v[1]) > k else None + for v in raw + ] + weights = [ + v[2][k] if isinstance(v[2], list) and len(v[2]) > k else None + for v in raw + ] + common = ( + indices[0] + if indices and all(v == indices[0] for v in indices) + else None + ) + options = tuple( + f"{i}: {d.joints[i] if i < len(d.joints) else 'missing joint_id'}" + for i in range(len(d.arrays[FIELDS[0]])) + ) + label = ( + options[common] + if type(common) is int and 0 <= common < len(options) + else f"Raw: {common if common is not None else 'mixed / missing'}" + ) + dropdown.options, dropdown.value = ( + tuple(dict.fromkeys((label, *options))), + label, + ) + index.label = ( + "Set raw index" + if type(common) is int + else "Set raw index (mixed / missing)" + ) + if type(common) is int and abs(common) <= 9007199254740991: + index.value = common + mixed_weight = not weights or any(v != weights[0] for v in weights) + slider.label = number.label = ( + "Set weight (mixed / missing)" if mixed_weight else "Set weight" + ) + if ( + weights + and all(v == weights[0] for v in weights) + and type(weights[0]) in (int, float) + and abs(weights[0]) <= 3.4028234663852886e38 + ): + number.value = float(weights[0]) + slider.value = float(np.clip(weights[0], 0, 1)) + self.info.options = tuple( + f"{i}: {v['side']} {v['severity']} {v['field']}[{v['row']}]: {v['message']}" + for i, v in enumerate(self.issues) + ) or ("No local errors",) + self.mapping_info.content = str(d.editor.get("mapping", {})) + self.check_info.content = f"Local revision {self.state.revision}: {sum(v['severity'] == 'error' for v in self.issues)} errors. SDK/export revision {self.sdk_revision}: {self.sdk_result}{' (stale)' if self.sdk_revision is not None and self.sdk_revision != self.state.revision else ''}" + + def locate(self): + if not self.issues: + return + issue = self.issues[int(self.info.value.split(":")[0])] + if issue["side"] != self.side.value: + self.side.value = issue["side"] + self.switch() + if issue["row"] is not None: + if issue["field"] in (FIELDS[0], "joint_ids"): + self.segment.value, self.entity.value = str(issue["row"]), "Segment" + else: + self.selected, self.entity.value = [issue["row"]], "Taxels" + self.log("Located in draft inspector: " + issue["field"]) + + def axes(self, name, transforms): + positions = np.array([t[:3, 3] for t in transforms]).reshape(-1, 3) + quaternions = np.array( + [tf.quaternion_from_matrix(t) for t in transforms] + ).reshape(-1, 4) + if name not in self.handles and len(positions): + self.handles[name] = self.server.scene.add_batched_axes( + name, quaternions, positions, axes_length=0.015, axes_radius=0.0006 + ) + if name in self.handles: + h = self.handles[name] + h.visible = bool(len(positions)) + if len(positions): + h.batched_positions, h.batched_wxyzs = positions, quaternions + + def balls(self, name, points, colors, outline=False): + positions = np.asarray(points, dtype=float).reshape(-1, 3) + rotations = np.tile([1.0, 0, 0, 0], (len(positions), 1)) + if name not in self.handles and len(positions): + mesh = trimesh.creation.icosphere(subdivisions=1) + self.handles[name] = self.server.scene.add_batched_meshes_simple( + name, mesh.vertices, mesh.faces, rotations, positions, wireframe=outline + ) + if name in self.handles: + h = self.handles[name] + h.visible = bool(len(positions)) + if len(positions): + h.batched_positions, h.batched_wxyzs = positions, rotations + h.batched_colors = np.asarray(colors, dtype=np.uint8) + h.batched_scales = np.full( + len(positions), self.radius.value * (1.5 if outline else 1) + ) + + def draft_points(self): + return [ + (i, t[:3, 3]) for i, t in geo.frames(self.d.arrays[FIELDS[1]], self.root()) + ] + + def rectangle_mode(self, enabled): + if enabled: + self.server.scene.on_rect_select()(self.box_event) + else: + self.server.scene.remove_rect_select_callback(self.box_event) + + async def box_event(self, event): + self.events.put(lambda: self.select_scene(event, True)) + + async def scene_click(self, event): + self.events.put(lambda: self.click(event)) + + def select_scene(self, event, box=False): + camera, chosen = event.client.camera, [] + rotation = tf.quaternion_matrix(camera.wxyz)[:3, :3] + for i, point in self.draft_points(): + origin = np.asarray(camera.position if box else event.ray_origin) + direction = ( + geo.unit(point - origin) if box else np.asarray(event.ray_direction) + ) + if box: + p = rotation.T @ (point - origin) + if p[2] >= 0: + continue + uv = [ + 0.5 - p[0] / p[2] / (2 * np.tan(camera.fov / 2) * camera.aspect), + 0.5 + p[1] / p[2] / (2 * np.tan(camera.fov / 2)), + ] + inside = all( + a <= b <= c + for a, b, c in zip(event.screen_min, uv, event.screen_max) + ) + else: + along = (point - origin) @ direction + inside = ( + along > 0 + and np.linalg.norm(point - origin - along * direction) + < self.radius.value * 2 + ) + if ( + inside + and self.model + and getattr(self, "display_model", False) + and not self.through.value + ): + surfaces = [ + n + for n, (link, _) in self.model.surfaces.items() + if all(self.model.style.get(link, (True, 1))) + ] + hit = self.model.hit(origin, direction, surfaces, self.display_fk) + inside = ( + hit is None + or hit[0] >= np.linalg.norm(point - origin) - self.radius.value * 2 + ) + if inside: + chosen.append((np.linalg.norm(point - origin), i)) + indices = [i for _, i in (chosen if box else sorted(chosen)[:1])] + self.selected = ( + sorted(set(self.selected) ^ set(indices)) if event.modifier else indices + ) + self.entity.value = "Taxels" + + def click(self, event): + if self.dragging: + return + if not self.pick_target: + self.select_scene(event) + return + target, self.pick_target = self.pick_target, None + if self.mode.value != "EDIT" or not self.model: + raise ValueError("Surface picking requires EDIT and a q0 model") + fk = self.model.fk(self.state.model.get("q0", {})) + surfaces = self.surfaces() + if target in ("O", "C", "R", "Move selected", "Single tangent"): + if self.surface.value not in surfaces: + raise ValueError("Choose an allowed projection surface") + surfaces = [self.surface.value] + hit = self.model.hit( + np.asarray(event.ray_origin), + np.asarray(event.ray_direction), + surfaces, + fk, + ) + if not hit: + raise ValueError("No hit on an allowed visible surface") + self.points[target] = hit[1].tolist() + if target in ("O", "C", "R"): + self.centers[("O", "C", "R").index(target)].value = tuple(hit[1]) + elif target == "Segment": + self.entity.value = "Segment" + self.components( + range(7), + geo.pose( + np.linalg.inv(self.root()) + @ geo.frame(hit[1], hit[2], fk[self.segment_link.value][:3, 0]) + ), + ) + elif target == "Move selected": + old = geo.matrix(self.d.arrays[FIELDS[1]][self.selected[0]]) + new = np.linalg.inv(self.root()) @ geo.frame( + hit[1], hit[2], (self.root() @ old)[:3, 0] + ) + self.change( + geo.transform, + self.d, + FIELDS[1], + self.selected, + new @ np.linalg.inv(old), + ) + elif target == "Single tangent": + self.points["C"] = hit[1].tolist() + self.rows.value = self.cols.value = 1 + self.grid() + + def render_side(self, side, d, root, current, forces, live): + positions, colors, frames, outlines, edges = [], [], [], [], [] + deformed = geo.deform(d, current) if current is not None else None + errors = { + v["row"] + for v in d.check() + if v["severity"] == "error" and v["field"] in FIELDS[1:] + } + for i, t in geo.frames(d.arrays[FIELDS[1]], root): + rigid, color = t, (140, 140, 140) + if deformed is not None: + point, orientation = deformed[i] + if point is None: + continue + t[:3, 3] = (root @ np.r_[point, 1])[:3] + rigid = root @ orientation if orientation is not None else None + if live and forces is not None and len(forces) == len(d.arrays[FIELDS[1]]): + force = np.asarray(forces[i], dtype=float) + if force.shape == (3,) and np.isfinite(force).all(): + fraction = np.clip( + np.linalg.norm(force) / self.force_max.value, 0, 1 + ) + color = (255 * fraction, 40, 255 * (1 - fraction)) + elif not live: + try: + indices, weights = d.arrays[FIELDS[2]][i], d.arrays[FIELDS[3]][i] + if i not in errors: + color = np.random.default_rng( + indices[int(np.argmax(weights))] + ).integers(70, 240, 3) + if self.slot is not None and i in self.selected and d is self.d: + color = ( + 255 * weights[self.slot], + 80, + 255 * (1 - weights[self.slot]), + ) + except (IndexError, TypeError, ValueError): + pass + positions.append(t[:3, 3]) + colors.append(color) + selected = d is self.d and i in self.selected + if selected or i in errors: + outlines.append(t[:3, 3]) + edges.append((255, 0, 0) if i in errors else (255, 220, 0)) + if selected and rigid is not None: + frames.append(rigid) + prefix = "/view/" + side + self.balls(prefix + "/taxels", positions, colors) + self.balls(prefix + "/outlines", outlines, edges, True) + self.axes(prefix + "/taxel-axes", frames) + self.axes(prefix + "/root", [root]) + segments = ( + [root @ t for t in current if t is not None] + if current is not None + else [t for _, t in geo.frames(d.arrays[FIELDS[0]], root)] + ) + self.axes(prefix + "/segments", segments) + self.balls( + prefix + "/segment-points", + [t[:3, 3] for t in segments], + [(90, 200, 250)] * len(segments), + ) + self.frame_colors[side] = np.asarray(colors, dtype=np.uint8) + + def draw(self): + latest, mode = self.latest(), self.mode.value + playing, status = mode in ("LIVE", "PAUSED"), [] + cfg = self.q if mode == "PREVIEW" else self.state.model.get("q0", {}) + highlight, lines, line_colors = set(), [], [] + for side, draft in self.state.documents.items(): + value = latest.get(side, {}) + unmapped = set(value.get("joints", {})) - set( + draft.editor.get("mapping", {}) + ) + status.append( + f"{side} raw JointState: {value.get('joints', {})}; unmapped: {sorted(unmapped)}" + ) + fresh = ( + bool(self.receiver) + and (self.receiver is True or self.receiver.active) + and time.monotonic() - value.get("read_time", 0) < 1 + ) + if ( + self.model + and self.receiver + and self.configs.get(side, {}).get("joint_state") + ): + for name, (target, scale, offset) in draft.editor.get( + "mapping", {} + ).items(): + raw, valid = value.get("joints", {}).get(name, (None, False)) + converted = ( + raw * scale + offset + if valid and raw is not None + else float("nan") + ) + joint = self.model.urdf.joint_map.get(target) + limit = joint.limit if joint else None + valid = ( + fresh and joint in self.model.joints and np.isfinite(converted) + ) + if limit: + valid &= (limit.lower is None or converted >= limit.lower) and ( + limit.upper is None or converted <= limit.upper + ) + status.append( + f"{side}/{name}: raw={raw}; converted={converted}; limits={limit}; valid={valid}" + ) + if valid and mode == "LIVE": + self.drive[target] = converted + self.layers[side].visible = side == self.side.value or self.both.value + if mode != "PAUSED": + try: + root, d, current, skeleton = self.root(draft, cfg), draft, None, [] + if mode == "LIVE": + if self.source.value == "Received": + d = storage.parse(value["metadata"], side) + current, joints, _ = geo.hand_current(d, value["hand"]) + root = joints[1] + skeleton = [t for t in joints if t is not None] + elif mode == "PREVIEW" and self.model and "root" in draft.editor: + current = self.model.current(draft, self.model.fk(cfg)) + self.render_side( + side, + d, + root, + current, + value.get("forces") if fresh else None, + playing, + ) + self.axes("/view/" + side + "/live-skeleton", skeleton) + self.frame_forces[side] = value.get("forces") + self.frame_counts[side] = len(d.arrays[FIELDS[1]]) + except (ValueError, TypeError, KeyError, IndexError) as exc: + status.append(f"{side}: invalid visualization: {exc}") + fresh = False + taxels = self.handles.get("/view/" + side + "/taxels") + if playing and taxels and taxels.visible: + taxels.batched_colors = ( + self.frame_colors.get(side, (140, 140, 140)) + if fresh + else (140, 140, 140) + ) + forces = ( + self.frame_forces.get(side) if mode == "PAUSED" else value.get("forces") + ) + force_array = ( + np.asarray(forces, dtype=float) + if forces is not None + else np.empty((0, 3)) + ) + valid_forces = ( + forces is not None + and force_array.ndim == 2 + and force_array.shape[1:] == (3,) + and np.isfinite(force_array).all() + and len(force_array) + == self.frame_counts.get(side, len(draft.arrays[FIELDS[1]])) + ) + status.append( + f"{side}: {value.get('config')}; local read={value.get('read_time')}; current={fresh}; force shape={force_array.shape}; valid/count-matched={valid_forces}; draft taxels={len(draft.arrays[FIELDS[1]])}" + ) + hand_flags = (value.get("hand") or {}).get("is_valid", []) + status.append( + f"{side} HandPose: {sum(bool(v) for v in hand_flags)} valid joints; WRIST flag={len(hand_flags) == 26 and bool(hand_flags[1])}" + ) + if side == self.side.value and fresh and valid_forces: + status.extend( + f"{'Paused display' if mode == 'PAUSED' else 'Latest'} row {i}: fx,fy,fz={forces[i]}; |f|={np.linalg.norm(forces[i]):.4g} N" + for i in self.selected[:25] + if i < len(force_array) + ) + root, d = self.root(), self.d + points = self.draft_points() if mode != "EDIT" else [] + self.balls( + "/draft-targets", + [p for _, p in points], + [ + (255, 220, 0) if i in self.selected else (140, 140, 140) + for i, _ in points + ], + True, + ) + field, rows = self.targets() + self.axes( + "/draft-axes", + [t for i, t in geo.frames(d.arrays[field], root) if i in rows], + ) + for i in self.selected: + try: + for j, w in zip(d.arrays[FIELDS[2]][i], d.arrays[FIELDS[3]][i]): + if ( + type(j) is int + and 0 <= j < len(d.joints) + and type(w) in (int, float) + and w <= 1 + and w > 0 + ): + lines.append( + [ + (root @ geo.matrix(d.arrays[FIELDS[0]][j]))[:3, 3], + (root @ geo.matrix(d.arrays[FIELDS[1]][i]))[:3, 3], + ] + ) + line_colors.append([(int(255 * min(w, 1)), 80, 30)] * 2) + if ( + self.slot is not None + and j == d.arrays[FIELDS[2]][i][self.slot] + ): + highlight.add( + d.editor.get("segment_links", {}).get( + d.editor["segment_ids"][j] + ) + ) + except (IndexError, TypeError, ValueError): + pass + if ( + self.entity.value == "Segment" + and self.segment.value + and int(self.segment.value) < len(d.editor["segment_ids"]) + ): + highlight.add( + d.editor.get("segment_links", {}).get( + d.editor["segment_ids"][int(self.segment.value)] + ) + ) + if "/draft-binding" not in self.handles and lines: + self.handles["/draft-binding"] = self.server.scene.add_line_segments( + "/draft-binding", + np.asarray(lines), + np.asarray(line_colors, dtype=np.uint8), + thickness=2, + thickness_units="screen", + ) + if "/draft-binding" in self.handles: + h = self.handles["/draft-binding"] + h.visible = bool(lines) + if lines: + h.points, h.colors = ( + np.asarray(lines), + np.asarray(line_colors, dtype=np.uint8), + ) + if self.model and mode != "PAUSED": + enabled = any( + bool(self.receiver) and self.configs.get(side, {}).get("joint_state") + for side in self.state.documents + ) + self.display_model = not playing or enabled + self.display_fk = self.model.draw( + self.server.scene, + self.drive if playing else cfg, + self.display_model, + highlight - {None}, + ) + connection = ( + self.receiver.status + if self.receiver and self.receiver is not True + else "Demo" + if self.receiver + else "Disconnected" + ) + self.live_info.content = connection + "\n\n" + "\n\n".join(status) + candidates = [ + (root @ np.r_[geo.matrix(p)[:3, 3] if p else fallback, 1])[:3] + for p, fallback in self.candidates + ] + self.balls( + "/candidates", + candidates, + [(50, 220, 90) if p else (255, 0, 0) for p, _ in self.candidates], + ) + if not self.dragging: + try: + t = root @ geo.matrix(d.arrays[field][rows[0]]) + self.gizmo.position, self.gizmo.wxyz = ( + t[:3, 3], + tf.quaternion_from_matrix(t), + ) + self.gizmo.visible = True + except (ValueError, TypeError, IndexError): + self.gizmo.visible = False + + def run(self): + self.camera() + last_error = None + try: + while not self.closing: + try: + while not self.events.empty(): + self.events.get()() + self.refresh() + if self.job and self.job.done(): + job, self.job = self.job, None + self.sdk_revision = self.job_revision + try: + self.sdk_result = str(job.result()) + except Exception as exc: + self.sdk_result = f"Failed: {exc}" + self.refresh() + with self.server.atomic(): + self.draw() + last_error = None + time.sleep(1 / 30) + except KeyboardInterrupt: + self.guard(lambda: setattr(self, "closing", True)) + except Exception as exc: + message = f"{type(exc).__name__}: {exc}" + if message != last_error: + self.log(message) + last_error = message + time.sleep(0.1) + finally: + if self.receiver and self.receiver is not True: + self.receiver.close() + self.pool.shutdown(wait=True) + self.server.stop() diff --git a/src/core/python/requirements-ui.txt b/src/core/python/requirements-ui.txt index 5bb07526e4..8da97507a2 100644 --- a/src/core/python/requirements-ui.txt +++ b/src/core/python/requirements-ui.txt @@ -1,2 +1,7 @@ # Optional UI dependencies for parameter tuning imgui[glfw]>=2.0.0 +numpy>=2,<3 +viser==1.1.1 +yourdfpy==0.0.60 +trimesh==5.1.1 +rtree>=1.3,<2 From 7b1b2e5eca737fe548d0a517c1a8dc2927648213 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 26 Oct 2026 21:26:23 -0700 Subject: [PATCH 02/36] Remove redundant link transparency control Signed-off-by: Lotus Li --- scripts/tactile_metadata/README.md | 7 ++++--- .../tactile_metadata_editor/geometry.py | 10 ++++----- .../tactile_metadata_editor/ui.py | 21 ++----------------- 3 files changed, 11 insertions(+), 27 deletions(-) diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 0c6fe2ad94..f3b804234e 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -36,9 +36,10 @@ system; see the [tactile device documentation](../../docs/source/device/tactile/ 1. In **Model**, choose the wrist link (`base_link` in the supplied left model). Enter allowed links, for example `base_link,index_distal`, and apply them. - Inspect the resource list and joint tree. Visibility and opacity affect only - the drawing. Package mappings are a JSON object of package names to local - directories, used to resolve `package://` resources without network access. + Inspect the resource list and joint tree. All URDF links use fixed opacity + 0.6; highlighting changes color. Package mappings are a JSON object of package + names to local directories, used to resolve `package://` resources without + network access. 2. Use **Pick target** and **Pick on q0 model** to pick `Wrist O`, `Wrist F` toward the forearm, and `Wrist D` toward the back of the hand. Preview and confirm the wrist reference. Both hands use the same right-handed axes. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index 63c1f39a22..51bf679a0c 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -56,7 +56,7 @@ def frames(rows, root): class Model: def __init__(self, path, packages): self.path, self.missing, self.surfaces = Path(path).resolve(), [], {} - self.handles, self.style, self.attachments = {}, {}, {} + self.handles, self.attachments = {}, {} # yourdfpy invokes this resolver using the keyword fname. def resource(fname): @@ -139,13 +139,13 @@ def draw(self, scene, cfg, visible=True, highlight=None): mesh.vertices, mesh.faces, color=(170, 175, 180), + opacity=0.6, side="double", ) h = self.handles[name] - shown, opacity = self.style.get(link, (True, 1.0)) - h.visible = visible and shown - h.opacity = ( - opacity if not highlight or link in highlight else opacity * 0.15 + h.visible = visible + h.color = ( + (255, 220, 0) if highlight and link in highlight else (170, 175, 180) ) h.position, h.wxyz = fk[link][:3, 3], tf.quaternion_from_matrix(fk[link]) return fk diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 407856cd39..9250db89f9 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -100,11 +100,6 @@ def __init__(self, state, args): self.calibrate_button = self.button( "Preview / set wrist reference", self.calibrate ) - self.link = g.add_dropdown("Display link", ("",)) - self.opacity = g.add_slider("Link opacity", 0.0, 1.0, 0.05, 1.0) - self.visible = g.add_checkbox("Link visible", True) - self.watch(self.opacity, lambda _: self.link_style()) - self.watch(self.visible, lambda _: self.link_style()) self.resources = g.add_markdown("") self.button("Reset cameras", self.camera) with tabs.add_tab("Segments"): @@ -374,9 +369,7 @@ def install_model(self, model): for h in self.model.handles.values(): h.remove() self.model = model - self.wrist.options = self.link.options = self.segment_link.options = tuple( - model.links - ) + self.wrist.options = self.segment_link.options = tuple(model.links) self.joint.options = self.target_joint.options = tuple( j.name for j in model.joints ) or ("",) @@ -416,7 +409,6 @@ def reset(self): self.q, self.resources.content = {}, "" for w in ( self.wrist, - self.link, self.joint, self.segment_link, self.target_joint, @@ -519,7 +511,6 @@ def surfaces(self): n for n, (link, _) in self.model.surfaces.items() if link in self.d.editor.get("allowed_links", []) - and self.model.style.get(link, (True, 1))[0] ] def set_links(self): @@ -528,10 +519,6 @@ def set_links(self): raise ValueError("Choose links from the loaded model") self.change(self.d.editor.update, allowed_links=links) - def link_style(self): - if self.model: - self.model.style[self.link.value] = (self.visible.value, self.opacity.value) - def camera(self): for client in self.server.get_clients().values(): client.camera.position, client.camera.look_at = (0.3, -0.4, 0.3), (0, 0, 0) @@ -1170,11 +1157,7 @@ def select_scene(self, event, box=False): and getattr(self, "display_model", False) and not self.through.value ): - surfaces = [ - n - for n, (link, _) in self.model.surfaces.items() - if all(self.model.style.get(link, (True, 1))) - ] + surfaces = list(self.model.surfaces) hit = self.model.hit(origin, direction, surfaces, self.display_fk) inside = ( hit is None From 33795e3d2351bc4a875600c30b379c6fb3abbc78 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 26 Oct 2026 21:37:01 -0700 Subject: [PATCH 03/36] Fix seven sources of redundant work in the tactile metadata editor 1. Refresh once after each queued event, including rejected edits. Update segment choices before selecting newly added or cross-hand rows. 2. Reuse FK snapshots through a bounded, instance-owned LRU cache. 3. Redraw static geometry only after changes while retaining live playback and connection-status updates. 4. Cache received metadata parsing and validation by content, and reuse draft validation results during drawing. 5. Share common-number inspection between pose and weight controls. 6. Remove redundant weight clamping after the existing range checks. 7. Remove the repeated SDK revision assignment when a background job ends. Validation: Ruff 0.15.1 lint and formatting checks, package syntax checks, and focused regression checks for refresh ordering, static/live updates, cache invalidation and lifetime, mixed inspector fields, and SDK revisions. Browser and numerical integration remain unverified because the inspection interpreter lacks numpy, trimesh, viser, and yourdfpy. Signed-off-by: Lotus Li --- .../tactile_metadata_editor/geometry.py | 16 ++- .../tactile_metadata_editor/ui.py | 113 +++++++++++------- 2 files changed, 86 insertions(+), 43 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index 51bf679a0c..c415b3d3f5 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations +from collections import OrderedDict from pathlib import Path import numpy as np @@ -57,6 +58,8 @@ class Model: def __init__(self, path, packages): self.path, self.missing, self.surfaces = Path(path).resolve(), [], {} self.handles, self.attachments = {}, {} + # Instance-owned caches must not keep replaced models alive. + self._fk_cache = OrderedDict() # yourdfpy invokes this resolver using the keyword fname. def resource(fname): @@ -89,8 +92,17 @@ def resource(fname): self.surfaces[f"{link}/{node}/{i}"] = (link, part) def fk(self, cfg): - self.urdf.update_cfg({j.name: cfg.get(j.name, 0) for j in self.joints}) - return {link: self.urdf.get_transform(link).copy() for link in self.links} + configuration = tuple(cfg.get(j.name, 0) for j in self.joints) + result = self._fk_cache.pop(configuration, None) + if result is None: + self.urdf.update_cfg( + dict(zip((j.name for j in self.joints), configuration)) + ) + result = {link: self.urdf.get_transform(link).copy() for link in self.links} + self._fk_cache[configuration] = result + if len(self._fk_cache) > 4: + self._fk_cache.popitem(last=False) + return result def root(self, d, fk): reference = d.editor["root"] diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 9250db89f9..b23d3b65ef 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -22,6 +22,17 @@ LOG = logging.getLogger("isaaccapture.tactile_metadata.editor") +def common_number(values): + if ( + values + and all(v == values[0] for v in values) + and type(values[0]) in (int, float) + and abs(values[0]) <= 3.4028234663852886e38 + ): + return float(values[0]) + return None + + class Editor: def __init__(self, state, args): self.state, self.args, self.started = state, args, bool(state.documents) @@ -32,6 +43,7 @@ def __init__(self, state, args): self.model, self.receiver, self.job = None, None, None self.selected, self.points, self.candidates = [], {}, [] self.handles, self.q, self.logs, self.drive = {}, {}, [], {} + self.received, self.redraw = {}, True self.frame_forces, self.frame_colors, self.frame_counts, self.issues = ( {}, {}, @@ -304,7 +316,10 @@ def __init__(self, state, args): "URDF unavailable; numerical draft editing remains available: " + str(exc) ) + for widget in (self.both, self.source, self.force_max, self.radius): + self.watch(widget, lambda _: None) self.switch() + self.refresh() @property def d(self): @@ -360,9 +375,13 @@ def guard(self, action): action() def change(self, action, *args, **kwargs): - result = self.state.edit(action, *args, **kwargs) - self.refresh() - return result + return self.state.edit(action, *args, **kwargs) + + def segment_options(self): + return ( + "", + *map(str, range(max(len(self.d.joints), len(self.d.arrays[FIELDS[0]])))), + ) def install_model(self, model): if self.model: @@ -490,7 +509,7 @@ def switch(self): ) self.joint_id.value = source.get("joint_state", "") self.joint_enabled.value = bool(self.joint_id.value) - self.refresh() + self.segment.options = self.segment_options() def root(self, d=None, cfg=None): d = d or self.d @@ -609,6 +628,8 @@ def add(): self.d.editor["approximate"] = True self.change(add) + # Register new rows before selection; the event loop refreshes the inspector. + self.segment.options = self.segment_options() self.segment.value = str(len(self.d.arrays[FIELDS[0]]) - 1) self.select_segment() @@ -931,6 +952,7 @@ def background(self, exporting): self.sdk_result = f"Running for revision {self.job_revision}" def refresh(self): + self.redraw = True d = self.d self.issues = [ dict(v, side=s) @@ -962,10 +984,7 @@ def refresh(self): self.selected = [i for i in self.selected if 0 <= i < len(d.arrays[FIELDS[1]])] self.side.visible = self.both.visible = len(self.state.documents) == 2 self.header.content = f"{Path(self.state.model.get('urdf', 'No URDF')).name}; hands: {', '.join(self.state.documents)}; {self.mode.value}\n\n{self.state.paths.get(d.side, 'Unsaved draft')}{' *' if self.state.dirty() else ''}; revision {self.state.revision}" - self.segment.options = ( - "", - *map(str, range(max(len(d.joints), len(d.arrays[FIELDS[0]])))), - ) + self.segment.options = self.segment_options() links = d.editor.get("segment_links", {}) self.segment_table.content = "\n\n".join( f"row {i}: {j}; link={links.get(sid, 'unassigned')}; attachment={'ready' if self.model and sid in self.model.attachments.get(d.side, {}) else 'missing'}" @@ -987,13 +1006,9 @@ def refresh(self): cells = [ v[k] if isinstance(v, list) and k < len(v) else None for v in values ] - if ( - cells - and all(v == cells[0] for v in cells) - and type(cells[0]) in (int, float) - and abs(cells[0]) <= 3.4028234663852886e38 - ): - w.value = float(cells[0]) + value = common_number(cells) + if value is not None: + w.value = value raw, sums = [], [] for i in self.selected: parts = [ @@ -1045,14 +1060,10 @@ def refresh(self): slider.label = number.label = ( "Set weight (mixed / missing)" if mixed_weight else "Set weight" ) - if ( - weights - and all(v == weights[0] for v in weights) - and type(weights[0]) in (int, float) - and abs(weights[0]) <= 3.4028234663852886e38 - ): - number.value = float(weights[0]) - slider.value = float(np.clip(weights[0], 0, 1)) + value = common_number(weights) + if value is not None: + number.value = value + slider.value = float(np.clip(value, 0, 1)) self.info.options = tuple( f"{i}: {v['side']} {v['severity']} {v['field']}[{v['row']}]: {v['message']}" for i, v in enumerate(self.issues) @@ -1223,12 +1234,20 @@ def click(self, event): self.rows.value = self.cols.value = 1 self.grid() - def render_side(self, side, d, root, current, forces, live): + def received_document(self, side, value): + cached = self.received.get(side) + if cached is None or cached[0] != value: + d = storage.parse(value, side) + cached = (copy.deepcopy(value), d, d.check()) + self.received[side] = cached + return cached[1:] + + def render_side(self, side, d, root, current, forces, live, issues): positions, colors, frames, outlines, edges = [], [], [], [], [] deformed = geo.deform(d, current) if current is not None else None errors = { v["row"] - for v in d.check() + for v in issues if v["severity"] == "error" and v["field"] in FIELDS[1:] } for i, t in geo.frames(d.arrays[FIELDS[1]], root): @@ -1290,6 +1309,7 @@ def render_side(self, side, d, root, current, forces, live): def draw(self): latest, mode = self.latest(), self.mode.value playing, status = mode in ("LIVE", "PAUSED"), [] + updating = self.redraw or mode == "LIVE" cfg = self.q if mode == "PREVIEW" else self.state.model.get("q0", {}) highlight, lines, line_colors = set(), [], [] for side, draft in self.state.documents.items(): @@ -1334,12 +1354,13 @@ def draw(self): if valid and mode == "LIVE": self.drive[target] = converted self.layers[side].visible = side == self.side.value or self.both.value - if mode != "PAUSED": + if updating and mode != "PAUSED": try: root, d, current, skeleton = self.root(draft, cfg), draft, None, [] + issues = [v for v in self.issues if v["side"] == side] if mode == "LIVE": if self.source.value == "Received": - d = storage.parse(value["metadata"], side) + d, issues = self.received_document(side, value["metadata"]) current, joints, _ = geo.hand_current(d, value["hand"]) root = joints[1] skeleton = [t for t in joints if t is not None] @@ -1352,6 +1373,7 @@ def draw(self): current, value.get("forces") if fresh else None, playing, + issues, ) self.axes("/view/" + side + "/live-skeleton", skeleton) self.frame_forces[side] = value.get("forces") @@ -1395,6 +1417,16 @@ def draw(self): for i in self.selected[:25] if i < len(force_array) ) + connection = ( + self.receiver.status + if self.receiver and self.receiver is not True + else "Demo" + if self.receiver + else "Disconnected" + ) + self.live_info.content = connection + "\n\n" + "\n\n".join(status) + if not updating: + return root, d = self.root(), self.d points = self.draft_points() if mode != "EDIT" else [] self.balls( @@ -1427,7 +1459,7 @@ def draw(self): (root @ geo.matrix(d.arrays[FIELDS[1]][i]))[:3, 3], ] ) - line_colors.append([(int(255 * min(w, 1)), 80, 30)] * 2) + line_colors.append([(int(255 * w), 80, 30)] * 2) if ( self.slot is not None and j == d.arrays[FIELDS[2]][i][self.slot] @@ -1477,14 +1509,6 @@ def draw(self): self.display_model, highlight - {None}, ) - connection = ( - self.receiver.status - if self.receiver and self.receiver is not True - else "Demo" - if self.receiver - else "Disconnected" - ) - self.live_info.content = connection + "\n\n" + "\n\n".join(status) candidates = [ (root @ np.r_[geo.matrix(p)[:3, 3] if p else fallback, 1])[:3] for p, fallback in self.candidates @@ -1512,18 +1536,25 @@ def run(self): while not self.closing: try: while not self.events.empty(): - self.events.get()() - self.refresh() + try: + self.events.get()() + finally: + self.refresh() if self.job and self.job.done(): job, self.job = self.job, None - self.sdk_revision = self.job_revision try: self.sdk_result = str(job.result()) except Exception as exc: self.sdk_result = f"Failed: {exc}" self.refresh() - with self.server.atomic(): - self.draw() + if ( + self.redraw + or self.receiver + or self.mode.value in ("LIVE", "PAUSED") + ): + with self.server.atomic(): + self.draw() + self.redraw = False last_error = None time.sleep(1 / 30) except KeyboardInterrupt: From 7d47d8f9d935a998ea849541df17234f49087a22 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Mon, 26 Oct 2026 23:11:24 -0700 Subject: [PATCH 04/36] Remove editor-specific metadata files and fields Use the existing schema JSON functions for loading, saving, backups, export, SDK input and saved-state comparisons. Remove editor-extension persistence, model restoration, resource-path rewriting and the editable/numerical mode. Drop editor JSON, correspondence CSVs, saved reports, --report and unused taxel-ID bookkeeping. Keep incomplete values saveable, preserve raw weights and array ordering, and validate formal exports separately. Update the README for values-only files and explicit hand assignment. Validation: Ruff 0.15.1 lint/format checks, syntax checks, actual JSON save/load and export checks, CLI checks, and focused UI ordering/repair checks. No new functions or classes were added. SDK transport and UI checks used stubs; actual SDK encoding and browser rendering remain unverified. Signed-off-by: Lotus Li --- scripts/tactile_metadata/README.md | 41 +++++----- .../tactile_metadata_editor/cli.py | 19 +---- .../tactile_metadata_editor/document.py | 19 ++--- .../tactile_metadata_editor/sdk.py | 5 +- .../tactile_metadata_editor/storage.py | 82 ++++--------------- .../tactile_metadata_editor/ui.py | 37 +++------ 6 files changed, 64 insertions(+), 139 deletions(-) diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index f3b804234e..825b2b8cfc 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -94,43 +94,43 @@ hand selection after handling unsaved drafts. ## Save, reopen, check and export -**Export → Save all drafts** writes one editable JSON per hand, including an -optional `editor` extension with relative URDF/package paths, calibration, -mappings, stable entity IDs and output indices. Incomplete drafts can be saved -and are explicitly labeled. If any save fails, no side is marked saved. +**Export → Save all metadata** writes schema metadata values only, using the same +JSON as backups and export. Model, calibration and source settings stay in memory; +reopening restores values and array order only. Incomplete values remain saveable, +with validation errors shown in the interface. If any save fails, no side is +marked saved. Only metadata-value changes mark the document unsaved. **Info issue → Locate** selects the relevant hand/entity in the shared draft inspector. Advanced schema repair can replace incomplete arrays, with a temporary backup before replacement. ```bash -python -m tactile_metadata_editor.cli gui --metadata left.metadata.json -python -m tactile_metadata_editor.cli check left.metadata.json --report check.json +python -m tactile_metadata_editor.cli gui --metadata left.metadata.json --sides left +python -m tactile_metadata_editor.cli check left.metadata.json python -m tactile_metadata_editor.cli check left.metadata.json --sdk -python -m tactile_metadata_editor.cli export left.metadata.json --format sdk --out package -python -m tactile_metadata_editor.cli export left.metadata.json right.metadata.json --out pair +python -m tactile_metadata_editor.cli export left.metadata.json --sides left --format sdk --out package +python -m tactile_metadata_editor.cli export left.metadata.json right.metadata.json --sides both --out pair ``` -For JSON without `editor.side`, choose **Assign JSON to** and **Open JSON**, or -provide `--sides left/right/both` on the CLI. Pure checking needs no side. Each -side must be unique; conflicting shared URDF configurations are rejected. +Choose **Assign JSON to** and **Open JSON**, or provide `--sides left/right/both` +on the CLI. Pure checking needs no side; export requires one file per specified +hand, left then right for `both`. The protocol JSON follows [tactile_metadata.fbs](../../src/core/schema/fbs/tactile_metadata.fbs): `geometry_mapping_type: "LBS"`, nested `geometry_mapping` arrays, and `segment_source_type: "HandSkeletonRef"` with nested `segment_source.joint_ids`. Pose quaternions are XYZW; influence structs contain `values` arrays. There is no -additional metadata format. Numerical export omits `editor` and orders the taxel -arrays by the chosen output indices. Each package contains numerical JSON, -editable JSON, a draft-taxel-ID/export-index CSV and a revision-tagged report. -SDK packages also contain binaries constructed and size-checked with the official -bindings. All sides must succeed before the new or empty destination is published. -GUI exports capture a snapshot; later edits do not change that snapshot. +additional metadata format. Taxel array row order expresses output order. +Exports contain only metadata JSON and optional SDK binaries; check results and +revisions appear in the interface or CLI output. All sides must succeed before +the new or empty destination is published. GUI exports capture a snapshot; +later edits do not change that snapshot. ## Live reception and the SDK-free demo Launch with `gui --demo` to use a synthetic independent layout and hand/force stream. In **Live**, configure each side's tactile collection (default `tactile`), -HandTracker output and optional JointState collection, then connect. Save source -configuration and reconnect to apply source changes. JointState mappings specify +HandTracker output and optional JointState collection, then connect. Apply source +configuration in the current session and reconnect. JointState mappings specify a source name, URDF target, scale and offset; the panel reports raw values, converted values, limits and validity. Demo JointState names come from mappings present when connecting. @@ -153,8 +153,7 @@ only the drawing. **Record metadata groups** first backs up the draft to the system temporary directory, logs its path, then replaces the selected groups verbatim. Partial replacements may introduce count/reference errors, which remain editable. Metadata/force alignment is the producer's and user's responsibility: -forces always use protocol row order. Source switching establishes no alignment, -and the export correspondence CSV does not map or permute force data. +forces always use protocol row order. Source switching establishes no alignment. ## Implementation and verification diff --git a/scripts/tactile_metadata/tactile_metadata_editor/cli.py b/scripts/tactile_metadata/tactile_metadata_editor/cli.py index 3b6937a0e2..29f1a6c146 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/cli.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/cli.py @@ -25,7 +25,6 @@ def main(): check = commands.add_parser("check") check.add_argument("metadata") check.add_argument("--sdk", action="store_true") - check.add_argument("--report") export = commands.add_parser("export") export.add_argument("metadata", nargs="+") export.add_argument("--sides", choices=("left", "right", "both")) @@ -44,8 +43,6 @@ def main(): result["sdk"] = check(d) except (ValueError, RuntimeError) as exc: result["sdk"], failed = dict(error=str(exc)), True - if args.report: - storage.write(args.report, result) print(json.dumps(result, indent=2)) return int(failed) sides = ( @@ -79,19 +76,14 @@ def main(): if not d.side: if args.command == "gui": continue - raise ValueError("Assign JSON sides with editor.side or --sides") + raise ValueError("Assign JSON sides with --sides") if d.side in documents: raise ValueError("Duplicate metadata side") - cfg = d.editor.get("model", {}) - if model.get("urdf") and cfg.get("urdf") and model != cfg: - raise ValueError("Conflicting URDF configurations") - if cfg.get("urdf"): - model = cfg documents[d.side], paths[d.side] = d, str(Path(path).resolve()) if args.command == "export": print( json.dumps( - storage.export(documents, model, 0, args.out, args.format == "sdk"), + storage.export(documents, 0, args.out, args.format == "sdk"), indent=2, ) ) @@ -104,12 +96,7 @@ def main(): model["urdf"] = str(Path(args.urdf).resolve()) state = State(documents, model) state.paths = paths - state.saved = { - s: state.fingerprint(s) - for s in paths - if documents[s].editor.get("side") == s - and documents[s].editor.get("model", {}) == state.model - } + state.saved = {s: state.fingerprint(s) for s in paths} Editor(state, args).run() return 0 except (ValueError, OSError, RuntimeError, ImportError, TypeError) as exc: diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index 5767af0c8d..ef416ecc98 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -64,17 +64,13 @@ def __init__(self, side=None): self.side, self.joints = side, [] self.arrays = {field: [] for field in FIELDS} self.types = ("LBS", "HandSkeletonRef") - self.editor = {"taxel_ids": [], "segment_ids": [], "output_indices": []} + self.editor = {"segment_ids": [], "output_indices": []} def sync(self): - for key, field in (("taxel_ids", FIELDS[1]), ("segment_ids", FIELDS[0])): - count, ids = len(self.arrays[field]), self.editor.setdefault(key, []) - ids.extend(uuid.uuid4().hex for _ in range(count - len(ids))) - del ids[count:] - count, order = ( - len(self.arrays[FIELDS[1]]), - self.editor.setdefault("output_indices", []), - ) + count, ids = len(self.arrays[FIELDS[0]]), self.editor["segment_ids"] + ids.extend(uuid.uuid4().hex for _ in range(count - len(ids))) + del ids[count:] + count, order = len(self.arrays[FIELDS[1]]), self.editor["output_indices"] order.extend(range(len(order), count)) del order[count:] @@ -153,7 +149,6 @@ def delete(self, selected): for i in sorted(set(selected), reverse=True): for values in ( *[self.arrays[f] for f in FIELDS[1:]], - self.editor["taxel_ids"], self.editor["output_indices"], ): if i < len(values): @@ -246,7 +241,9 @@ def __init__(self, documents, model=None): self.revision, self.saved, self.paths = 0, {}, {} def fingerprint(self, side): - return json.dumps((vars(self.documents[side]), self.model), sort_keys=True) + from .storage import metadata + + return json.dumps(metadata(self.documents[side]), sort_keys=True) def edit(self, action, *args, **kwargs): before = copy.deepcopy((self.documents, self.model)) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py index 87917635a9..3e1ae716d5 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py @@ -19,9 +19,11 @@ def check(document, output=None): + if any(v["severity"] == "error" for v in document.check()): + raise ValueError("Local errors prevent export") with tempfile.TemporaryDirectory(prefix="tactile-sdk-") as folder: root = Path(folder) - write(root / "input.json", metadata(document, numerical=True)) + write(root / "input.json", metadata(document)) try: result = subprocess.run( [sys.executable, "-m", __package__ + ".sdk", "check", folder], @@ -167,7 +169,6 @@ def live(connection, configs): d.arrays = owned(m.geometry_mapping, FIELDS) d.sync() raw = metadata(d) - raw.pop("editor") sides[side] = dict( metadata=raw, forces=data.taxel_forces.copy().tolist() diff --git a/scripts/tactile_metadata/tactile_metadata_editor/storage.py b/scripts/tactile_metadata/tactile_metadata_editor/storage.py index 3d5f90645f..e9b25e4fec 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/storage.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/storage.py @@ -3,7 +3,6 @@ from __future__ import annotations import copy -import csv import json import os import tempfile @@ -15,13 +14,12 @@ def parse(value, side=None): if not isinstance(value, dict) or any( not isinstance(value.get(k, {}), dict) - for k in ("editor", "geometry_mapping", "segment_source") + for k in ("geometry_mapping", "segment_source") ): raise ValueError( "Expected a TactileMetadata object with object-valued sections" ) - editor = value.get("editor", {}) - d = Document(side or editor.get("side")) + d = Document(side) if d.side not in (None, "left", "right"): raise ValueError("Assign metadata to left or right explicitly") d.types = (value.get("geometry_mapping_type"), value.get("segment_source_type")) @@ -51,30 +49,24 @@ def parse(value, side=None): else None for v in rows ] - d.editor.update(copy.deepcopy(editor)) d.sync() return d -def metadata(d, model=None, numerical=False): - if numerical and any(i["severity"] == "error" for i in d.check()): - raise ValueError("Local errors prevent export") +def metadata(d): value = dict( geometry_mapping_type=d.types[0], geometry_mapping={}, segment_source_type=d.types[1], segment_source={"joint_ids": copy.deepcopy(d.joints)}, ) - order = ( - sorted( - range(len(d.editor["output_indices"])), - key=lambda i: d.editor["output_indices"][i], - ) - if numerical - else None + order = sorted( + range(len(d.editor["output_indices"])), + key=lambda i: d.editor["output_indices"][i], ) + aligned = all(len(d.arrays[f]) == len(order) for f in FIELDS[1:]) for field, rows in d.arrays.items(): - if numerical and field in FIELDS[1:]: + if aligned and field in FIELDS[1:]: rows = [rows[i] for i in order] value["geometry_mapping"][field] = [ { @@ -87,33 +79,12 @@ def metadata(d, model=None, numerical=False): else None for r in rows ] - if not numerical: - value["editor"] = copy.deepcopy(d.editor) - value["editor"].update( - side=d.side, - model=copy.deepcopy( - model if model is not None else d.editor.get("model", {}) - ), - ) - value["editor"]["draft_status"] = ( - "Editing draft; not yet valid metadata" - if any(i["severity"] == "error" for i in d.check()) - else "Locally checked metadata" - ) return value def load(path, side=None): - path = Path(path).resolve() - with path.open() as stream: - d = parse(json.load(stream), side) - cfg = d.editor.get("model", {}) - if cfg.get("urdf"): - cfg["urdf"] = str((path.parent / cfg["urdf"]).resolve()) - cfg["packages"] = { - k: str((path.parent / v).resolve()) for k, v in cfg.get("packages", {}).items() - } - return d + with Path(path).open() as stream: + return parse(json.load(stream), side) def write(path, value): @@ -132,28 +103,21 @@ def save(state, paths): if len({Path(p).resolve() for p in paths.values()}) != len(paths): raise ValueError("Each hand requires a different save path") for side, path in paths.items(): - value = metadata(state.documents[side], state.model) - cfg, parent = value["editor"]["model"], Path(path).resolve().parent - if cfg.get("urdf"): - cfg["urdf"] = os.path.relpath(cfg["urdf"], parent) - cfg["packages"] = { - k: os.path.relpath(v, parent) for k, v in cfg.get("packages", {}).items() - } - write(path, value) + write(path, metadata(state.documents[side])) for side, path in paths.items(): state.paths[side], state.saved[side] = str(path), state.fingerprint(side) -def backup(d, model): +def backup(d): with tempfile.NamedTemporaryFile( prefix="tactile-backup-", suffix=".json", delete=False ) as stream: path = stream.name - write(path, metadata(d, model)) + write(path, metadata(d)) return path -def export(documents, model, revision, destination, sdk=False): +def export(documents, revision, destination, sdk=False): destination = Path(destination).resolve() if destination.exists() and ( not destination.is_dir() or any(destination.iterdir()) @@ -164,27 +128,15 @@ def export(documents, model, revision, destination, sdk=False): ) as temporary: root, report = Path(temporary), {"revision": revision, "sides": {}} for side, d in documents.items(): - write(root / f"{side}.metadata.json", metadata(d, numerical=True)) - editable = metadata(d, model) - cfg = editable["editor"]["model"] - if cfg.get("urdf"): - cfg["urdf"] = os.path.relpath(cfg["urdf"], destination) - cfg["packages"] = { - k: os.path.relpath(v, destination) - for k, v in cfg.get("packages", {}).items() - } - write(root / f"{side}.editor.json", editable) result = dict(local=d.check(), sdk="not checked") + if any(v["severity"] == "error" for v in result["local"]): + raise ValueError("Local errors prevent export") + write(root / f"{side}.metadata.json", metadata(d)) if sdk: from .sdk import check result["sdk"] = check(d, root / f"{side}.metadata.bin") - with (root / f"{side}.taxels.csv").open("w", newline="") as stream: - writer = csv.writer(stream) - writer.writerow(("draft_taxel", "exported_index")) - writer.writerows(zip(d.editor["taxel_ids"], d.editor["output_indices"])) report["sides"][side] = result - write(root / "report.json", report) if destination.exists(): destination.rmdir() os.replace(root, destination) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index b23d3b65ef..7c5468a94b 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -213,7 +213,7 @@ def __init__(self, state, args): self.joint_enabled = g.add_checkbox("Enable JointState", False) self.joint_id = g.add_text("JointState collection_id", "") self.button( - "Save this hand's source configuration", self.configure_source + "Apply this hand's source configuration", self.configure_source ) self.source_joint = g.add_text("Source joint name", "") self.target_joint = g.add_dropdown("URDF target joint", ("",)) @@ -247,7 +247,7 @@ def __init__(self, state, args): ) for s in ("left", "right") } - self.button("Save all drafts", self.save) + self.button("Save all metadata", self.save) self.all_sides = g.add_checkbox("Check / export all hands", True) self.destination = g.add_text("New output directory", "package") self.binary = g.add_checkbox("Include SDK binary", False) @@ -262,9 +262,7 @@ def __init__(self, state, args): lambda: setattr( self.schema, "value", - json.dumps( - storage.metadata(self.d, self.state.model), indent=2 - ), + json.dumps(storage.metadata(self.d), indent=2), ), ) self.button("Replace draft from repair field", self.repair) @@ -460,19 +458,11 @@ def open(self): d = storage.load(path, side) if self.model and side not in self.state.documents: raise ValueError("Create a new document to change the fixed model hands") - cfg = d.editor.get("model", {}) - model = ( - geo.Model(cfg["urdf"], cfg.get("packages", {})) - if not self.model and cfg.get("urdf") - else None - ) def replace(): if not self.started: self.state.documents = {} self.state.documents[side] = d - if model: - self.state.model = copy.deepcopy(cfg) if self.receiver: self.connect() @@ -480,18 +470,13 @@ def replace(): self.state.edit(replace) self.started = True self.side.options, self.side.value = tuple(self.state.documents), side - if model: - self.install_model(model) self.state.paths[side] = self.paths[side].value = str(Path(path).resolve()) - if d.editor.get("side") == side and cfg == self.state.model: - self.state.saved[side] = self.state.fingerprint(side) - else: - self.state.saved.pop(side, None) + self.state.saved[side] = self.state.fingerprint(side) self.switch() def save(self): storage.save(self.state, {s: self.paths[s].value for s in self.state.documents}) - self.log("Saved all drafts") + self.log("Saved all metadata") def switch(self): self.selected, self.points, self.candidates = [], {}, [] @@ -855,7 +840,7 @@ def configure_source(self): raise ValueError("Enabled sources require collection_id values") self.change(self.d.editor.update, source=source) if self.receiver: - self.log("Source configuration saved; reconnect to apply it") + self.log("Source configuration updated; reconnect to apply it") def mapping(self, remove=False): mappings = dict(self.d.editor.get("mapping", {})) @@ -911,7 +896,7 @@ def record(self): if value is None: raise ValueError("No received metadata") received = storage.parse(value, self.side.value) - path = storage.backup(self.d, self.state.model) + path = storage.backup(self.d) self.log("Draft backup: " + path) self.change( self.d.record, received, [k for k, w in self.groups.items() if w.value] @@ -919,7 +904,11 @@ def record(self): def repair(self): d = storage.parse(json.loads(self.schema.value), self.side.value) - self.log("Draft backup: " + storage.backup(self.d, self.state.model)) + order = d.editor["output_indices"] + d.editor = copy.deepcopy(self.d.editor) + d.editor["output_indices"] = order + d.sync() + self.log("Draft backup: " + storage.backup(self.d)) self.change(self.state.documents.__setitem__, self.side.value, d) def local_check(self): @@ -940,7 +929,6 @@ def background(self, exporting): self.job = self.pool.submit( storage.export, documents, - copy.deepcopy(self.state.model), self.job_revision, self.destination.value, self.binary.value, @@ -961,6 +949,7 @@ def refresh(self): ] if self.model: fk = self.model.fk(self.state.model.get("q0", {})) + self.model.attachments.clear() for doc in self.state.documents.values(): try: if "root" in doc.editor: From 68b7f542558f93868fd9c5193b34d5ca59fb9a64 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 11:27:29 -0700 Subject: [PATCH 05/36] tmp files Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 32 + scripts/tactile_metadata/DESIGN.md | 225 +++ scripts/tactile_metadata/DESIGN.md.license | 3 + .../tactile_metadata/IMPLEMENTATION_PROMPT.md | 38 + .../model/left/linkerhand_FFG_left.urdf | 1265 +++++++++++++++++ .../model/left/meshes/base_link.STL | 3 + .../model/left/meshes/index_distal.STL | 3 + .../model/left/meshes/index_metacarpals.STL | 3 + .../model/left/meshes/index_middle.STL | 3 + .../model/left/meshes/index_proximal.STL | 3 + .../model/left/meshes/middle_distal.STL | 3 + .../model/left/meshes/middle_metacarpals.STL | 3 + .../model/left/meshes/middle_middle.STL | 3 + .../model/left/meshes/middle_proximal.STL | 3 + .../model/left/meshes/pinky_distal.STL | 3 + .../model/left/meshes/pinky_metacarpals.STL | 3 + .../model/left/meshes/pinky_middle.STL | 3 + .../model/left/meshes/pinky_proximal.STL | 3 + .../model/left/meshes/ring_distal.STL | 3 + .../model/left/meshes/ring_metacarpals.STL | 3 + .../model/left/meshes/ring_middle.STL | 3 + .../model/left/meshes/ring_proximal.STL | 3 + .../model/left/meshes/thumb_distal.STL | 3 + .../model/left/meshes/thumb_metacarpals.STL | 3 + .../left/meshes/thumb_metacarpals_base1.STL | 3 + .../left/meshes/thumb_metacarpals_base2.STL | 3 + .../model/left/meshes/thumb_proximal.STL | 3 + .../model/right/linkerhand_FFG_right.urdf | 1265 +++++++++++++++++ .../model/right/meshes/hand_base_link.STL | 3 + .../model/right/meshes/index_distal.STL | 3 + .../model/right/meshes/index_metacarpals.STL | 3 + .../model/right/meshes/index_middle.STL | 3 + .../model/right/meshes/index_proximal.STL | 3 + .../right/meshes/linkerhand_FFG_right.urdf | 1265 +++++++++++++++++ .../model/right/meshes/middle_distal.STL | 3 + .../model/right/meshes/middle_metacarpals.STL | 3 + .../model/right/meshes/middle_middle.STL | 3 + .../model/right/meshes/middle_proximal.STL | 3 + .../model/right/meshes/pinky_distal.STL | 3 + .../model/right/meshes/pinky_metacarpals.STL | 3 + .../model/right/meshes/pinky_middle.STL | 3 + .../model/right/meshes/pinky_proximal.STL | 3 + .../model/right/meshes/ring_distal.STL | 3 + .../model/right/meshes/ring_metacarpals.STL | 3 + .../model/right/meshes/ring_middle.STL | 3 + .../model/right/meshes/ring_proximal.STL | 3 + .../model/right/meshes/thumb_distal.STL | 3 + .../model/right/meshes/thumb_metacarpals.STL | 3 + .../right/meshes/thumb_metacarpals_base1.STL | 3 + .../right/meshes/thumb_metacarpals_base2.STL | 3 + .../model/right/meshes/thumb_proximal.STL | 3 + 51 files changed, 4225 insertions(+) create mode 100644 scripts/tactile_metadata/AGENTS.md create mode 100644 scripts/tactile_metadata/DESIGN.md create mode 100644 scripts/tactile_metadata/DESIGN.md.license create mode 100644 scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md create mode 100644 scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf create mode 100644 scripts/tactile_metadata/model/left/meshes/base_link.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/index_distal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/index_middle.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/index_proximal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/middle_distal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/middle_middle.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/middle_proximal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_distal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_middle.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/ring_distal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/ring_middle.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/ring_proximal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_distal.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL create mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL create mode 100644 scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf create mode 100644 scripts/tactile_metadata/model/right/meshes/hand_base_link.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/index_distal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/index_middle.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/index_proximal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf create mode 100644 scripts/tactile_metadata/model/right/meshes/middle_distal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/middle_middle.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/middle_proximal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_distal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_middle.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/ring_distal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/ring_middle.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/ring_proximal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_distal.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL create mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md new file mode 100644 index 0000000000..df475a6c09 --- /dev/null +++ b/scripts/tactile_metadata/AGENTS.md @@ -0,0 +1,32 @@ + + +# Tactile metadata editor agent notes + +- 工具设计必须明确界面操作、持久化字段、坐标转换、绑定生成和 SDK 接入，不能把需求清单作为完整设计交给实现代理。 +- 设计按用户界面与功能、依赖、代码架构、模块内部实现及约定、代理执行规范分类；正确性、性能和扩展性归入相应类别，不与设计层级并列。代理执行规范与工具设计分开组织。 +- 用户要求完全重写时，从用户操作与数据流重新建立规格和实现提示词，不以搬运原段落或更换标题代替重写。 +- 多语言文档以 REUSE 实际识别的授权信息为准；钩子生成 `.license` 附属文件时，核对并随文档一起暂存，再执行完整检查。 +- 实现以本目录的 `DESIGN.md` 为依据；协议事实以仓库 schema、绑定和共享校验器为准。发现冲突时提供具体代码证据，并同步修正设计与实现提示词。 +- 首版的手套规则留在手部适配层；左右配置独立，通用编辑状态不使用 `HandJoint` 枚举作为内部标识。 +- 评审改变输入范围、术语或编辑实体时，同时更新界面、持久化、算法、验收与实现提示词；已取消的产品概念不能留作隐藏的数据模型。 +- 对设计文档的结构与表达要求应直接落实到目标文档；页面布局按区块列出具体控件，不以聊天中的补充说明代替文档修改。 +- Keep URDF driving separate from taxel deformation: HandPose supplies skeleton and LBS poses, while live URDF motion uses JointState only. Remove obsolete driver controls, persisted options, algorithms, and acceptance cases together. +- Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. +- Preserve slot order and repeated segment indices. Normalize all four weights only on explicit user weight edits and reject negative edit values; loading, recording received values, checking, saving, and exporting preserve raw weights. Distinguish receiver failures from sender behavior, and keep all editor operations local. +- SDK process isolation specifies where SDK classes are called; document the actual tracker/session calls required for live reception as well as export encoding. +- Keep tool implementation inside this subtree; add missing dependencies to Isaac Capture's existing requirements and matching package metadata. Run checks only from this directory or its descendants; never run repository-wide fixing hooks that can change other directories. +- Explain this standalone tool's setup from the user's prerequisites: define external libraries before packaging terms, and distinguish requirement files, generated version records, and installation commands. Identify inspected environments by their Python executable path; distinguish declared requirements from packages actually installed there. +- Reuse the launching Python environment and its SDK through sys.executable. Do not create tool-specific project metadata, lockfiles, environments or SDK installations, and do not infer runtime requirements from old local build artifacts. +- Missing packages in the agent's inspection environment limit verification; report the unavailable checks. Do not turn verification requirements into proposals to install dependencies, provision environments, or substitute interpreters; maintain Isaac Capture's shared dependency declarations instead. +- Use Viser's standard controls and public callbacks, with document changes serialized through State. Apply the same explicit weight-normalization rule to sliders and numeric fields. +- Keep metadata/force alignment with the producer and user; do not add source-row mappings, correspondence tracking across reordering, or alignment baselines to the editor. +- JSON follows tactile_metadata.fbs and the existing bindings. Treat array and enum conversion for encoding as routine implementation rather than a separate product-format decision. +- Persist only schema-defined tactile metadata fields. Keep model configuration, calibration, source mappings and internal IDs in session state; audit actual save/export output rather than inferring persistence requirements from editing needs. +- Simplify persistence by deleting unsupported branches in place; do not relocate serialization into Document or introduce new state abstractions for a narrower file format. +- For local editor checks using a cached Ruff executable, match the version in the repository hook configuration; older binaries may not understand the shared lint rules. Keep every check and formatter target inside this subtree. +- Keep the GUI a thin adapter over shared document, geometry, storage and SDK operations. Prefer explicit native Viser widgets and one reusable property inspector; avoid widget-type inference, duplicate field editors in dialogs, and rebuilding scene nodes on every frame. Moving logic between files alone does not reduce implementation size. +- Use fixed URDF opacity and color-based link highlighting; do not add per-link visibility or opacity controls or state. +- Under a hard code budget, finish and count a complete executable slice before expanding the UI. Refactoring dispatch or moving functions is not evidence that the implementation fits; verify the formatted total and retain budget for integration fixes. diff --git a/scripts/tactile_metadata/DESIGN.md b/scripts/tactile_metadata/DESIGN.md new file mode 100644 index 0000000000..70f6a3110a --- /dev/null +++ b/scripts/tactile_metadata/DESIGN.md @@ -0,0 +1,225 @@ + + +# Tactile Metadata Editor 设计 + +Tactile Metadata Editor 是供触觉手套厂商使用的本地 GUI 工具，用于生成、修改、验证 TactileMetadata，并结合 URDF 和 Isaac Capture 数据检查手部运动与触觉分布。首版支持手套；设备来源和几何映射分别通过适配接口接入，以便跟随 schema 扩展。 + +本文按用户界面和功能、依赖、代码架构、模块内部实现及约定、实现代理规范组织。以下是实施规格，尚不是已实现功能。 + +## 1 用户界面和用户功能设计 + +### 1.1 输入与结果 + +文件输入只支持 URDF 和 TactileMetadata JSON。STL/OBJ 可以是 URDF 引用的网格资源，不作为独立设备输入；不提供 CSV 或裸二进制导入。载入 URDF 后，可新建 metadata，或把已有 metadata JSON 分配到明确的手侧并继续编辑。 + +典型操作是：载入 URDF 并确定手侧，在 Segments 页配置关节与绑定姿态，在指腹标记范围并输入 4×4，批量生成 16 个 taxel，设置输出顺序及最多四个关节的权重，再通过模型运动或实时数据检查。不要求厂商预先持有逐 taxel 位姿、四元数或 skinning weights。 + +保存、备份与导出只写触觉 metadata 数值，使用同一种 schema JSON 表示；可选 SDK 二进制仅编码同一组 metadata 数值。JSON 本身可重新打开编辑，不区分“可编辑版”和“纯数值版”。不生成编辑器工程、`.editor.json`、内部 ID 对应表或检查报告文件。 + +JSON 仅采用 [`tactile_metadata.fbs`](../../src/core/schema/fbs/tactile_metadata.fbs) 定义的字段、嵌套结构及 union 类型标识：顶层为 `geometry_mapping_type`、`geometry_mapping`、`segment_source_type`、`segment_source`；内容为 segment 绑定位姿、taxel 绑定位姿、四槽 segment 索引与权重、`joint_ids`。不添加 `editor` 扩展或其他工具专用字段。Python 使用 `json.load()` 读取；需要 SDK 编码时，在隔离子进程中进行现有绑定所需的数组和枚举转换，调用 `LBS`、`HandSkeletonRef`、`TactileMetadata` 构造器及 `to_bytes()` 生成符合该 schema 的 FlatBuffer。转换复用[现有绑定](../../src/core/schema/python/tactile_metadata_bindings.h)，属于常规实现工作，不构成待定产品规格。 + +URDF 路径、package 映射、q0、腕部校准、segment 与链接关联、手侧分配、实时数据源与关节映射、内部标识和显示状态只存在于当前会话，不写入 metadata 或另存配置文件。检查状态、草稿标记和 revision 只显示在界面或 CLI 输出中。 + +实时输入首版接入 TactileDataArray/TactileMetadata、HandPose 和可选 JointStateOutput。真实设备协议接入、原始 ADC/压力到牛顿的转换、设备控制及物理标定不在工具内完成。工具只接收现有 Isaac Capture 数据；编辑、保存和导出均不向 runtime 或生产设备发送 metadata、控制命令或其他数据。 + +### 1.2 窗口布局与显示 + +一个浏览器窗口包含三维画布和右侧标准 medium 宽度面板。面板顶部显示文件、已固定的 URDF 手侧及当前模式；下方是模型、Segments、布局、绑定、实时、导出六个页签。长表单滚动，复用 Viser 客户端及标准控件，不维护独立前端。 + +```text +┌──────────────────────────────────┬──────────────────────────────┐ +│ │ URDF 名称；手侧：双手（只读） │ +│ 三维模型与手骨架 │ Metadata 文件 * 保存 │ +│ │ 编辑对象：[左手] [右手] │ +│ taxel、选中关节、法线和切线 │ 模式：编辑 / 预览 / 实时 / 暂停│ +│ ├──────────────────────────────┤ +│ 相机：旋转 平移 缩放 重置 │ 模型 Segments 布局 绑定 实时 导出│ +│ 选择：单选 / 框选 / 穿透选择 │ 当前页控件与选择属性 │ +│ ├──────────────────────────────┤ +│ Info：草稿错误状态及具体原因 │ 连接和数据状态 │ +└──────────────────────────────────┴──────────────────────────────┘ +``` + +模型为中性色，所有 URDF 链接使用固定不透明度（opacity）0.6，高亮只改变颜色。编辑时 taxel 按主影响关节着色，无绑定为灰色，选中轮廓为黄色，错误轮廓为红色。实时触觉用力向量模长映射蓝至红，默认范围 0–30 N，可调整上限，超限饱和；无有效数据为灰色，不能显示成零力。属性面板显示 fx、fy、fz 和模长，单位 N。 + +球形标记大小只影响显示，不改变 sensing location。选中 taxel 显示 +Z=外向法线 N、+X=传感器切线 T、+Y=N×T；X×Y=Z。双手使用同一右手坐标约定。多关节 LBS 动画只保证位置，不把混合旋转解释为协议定义的唯一方向。 + +六页表单始终显示并编辑 Document 中的草稿值，包括 PREVIEW、LIVE、PAUSED 及连接中断时。三维视图的 metadata 来源切换不改变表单数据；接收值只有通过“记录 metadata”显式替换所选草稿分组后才成为表单值。力值和连接状态是单独的只读实时信息。 + +允许草稿包含错误。出错字段及所属分组显示错误状态和原因，三维视图下方的紧凑 Info 区汇总当前手侧、分组、实体和原因，并可定位对应字段。保留能显示的有效几何；缺失或无效的位姿不以假位置绘制。错误不阻止继续编辑或替换数据，只阻止正式导出。 + +### 1.3 文件、URDF 与左右手 + +启动页提供“载入 URDF”和“打开 metadata JSON”。载入 URDF 时选择左手、右手或双手，完成后只读；替换 URDF 或改变手侧需要创建新文档，先处理未保存内容。URDF 不携带标准手侧字段，该选择是用户对模型的明确声明。 + +一个会话使用一个 URDF。双手选择要求模型实际包含左右手结构，随后分别指定腕部链接和关联链接；不能把单手模型自动镜像成双手。独立左右 URDF 分别打开文档。双手文档中的左右 metadata 独立，但共享同一 URDF 和绑定关节配置；共享关节的修改会明确列出受影响手侧。 + +顶部“编辑对象”只选择当前要编辑的 metadata，不修改模型手侧。单手不显示切侧控件；双手可以显示当前侧或双侧，另一侧 taxel 只读。任何显示排布偏移均不参与元数据计算。两侧数量与输出排列可以不同，各自的 taxel 索引从 0 开始。 + +新建 metadata 不要求输入预期数量，界面从当前 taxel 记录计算 N。检查与实时数据比较时才核对数量。全工具使用“taxel 数量”“taxel 索引”和“输出顺序”，不以 channel 表示 taxel。 + +打开 metadata JSON 时，用户通过界面或 `--sides` 明确分配左侧或右侧，不能从关节名称推断手侧。重新打开只恢复 metadata 数值及 segment、taxel 数组行序；需要表面操作和模型动画时，在当前会话重新载入 URDF、关联链接并设置腕部参考。仅有 JSON 时仍可查看和修改数值。 + +模型、JSON 和 package 资源均使用本地路径，不要求逐个上传网格。未保存标记只比较 metadata 数值；仅改变模型、校准、数据源或显示配置不构成 metadata 未保存。打开新文档、替换一侧 metadata 或退出时，有未保存数值则提供保存、放弃、取消。双手分别保存两个 metadata JSON；“保存全部”处理两侧，失败不得将任一侧误标为已保存。 + +### 1.4 模型页 + +页面提供 URDF 路径、package:// 的本地目录映射、资源状态。可检查模型树、关节类型与范围；缺失资源列出具体路径，不从网络寻找模型。 + +用户为每侧选择腕部链接，并拾取腕中心 O、朝前臂的参考 F、朝手背的参考 D，建立来源根坐标系。双手分别校准，界面显示两套参考轴，不按左/右自动变号。关联链接限制该侧拾取范围；共享掌臂链接可以作为参考，但另一侧表面不能成为默认拾取对象。 + +### 1.5 Segments 页 + +页面位于布局之前，包含“绑定姿态”和“Segments”两组控件。 + +绑定姿态组提供 URDF 自由关节滑块、数值输入和“设为绑定姿态”。fixed 不可编辑，mimic 跟随主关节，限位始终可见。此姿态 q0 是静态布局和 bind poses 的共同参考；实时角度及运动预览角度不会自动写入 q0。 + +Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_JOINTS。表格显示语义名称、输出行号、关联 URDF 链接和参考系状态。合法 metadata 每侧同名语义只出现一次，不要求完整骨架，WRIST 使用根恒等位姿；草稿违反这些要求时显示错误并保留值。选择一行会高亮关节球、坐标轴和关联链接。 + +新建 segment 时选择语义和链接，在模型上定位关节中心及参考方向，通过控制器调整位置与扭转。已有 metadata 的 B_j 已给出坐标：选择目标链接后计算固定局部修正，保持原 B_j，而不是把数值吸附到 URDF 原点。 + +用户可以调整有序 segment 列表。显式重排通过行号置换更新已有合法 skinning 索引，保持其稳定实体引用；语义枚举值不参与行号运算。允许删除仍被引用的 segment；未解决的引用留在草稿中，绑定页和 Info 区显示原因，不隐式重绑。 + +设置新 q0 或修改腕部参考系时，先预览模型和附件变化；现有 B_j、taxel 的绑定根位姿和权重保持数值不变，确认后重算关节局部附件并使检查过期。实际 B_j 的修改通过关节参考系控制器完成。工具不隐式重投影、蒙皮重绑定或修改生产端 metadata。偏离模型的点保留并提示用户调整。 + +### 1.6 布局页 + +页面提供“批量添加”“放置单点”和所选 taxel 的属性。没有区域名、区域列表或持久化批次；批量添加只是一次生成操作，导入 metadata 与新建结果拥有相同编辑能力。 + +批量添加只有一种方式：输入范围和行列数量。操作依次为： + +1. 选择允许投影的链接和表面，输入 rows、cols。 +2. 拾取首点中心 O、首行末点中心 C、首列末点中心 R。显示列跨度、行跨度及推导间距；可直接修改跨度数值，仍属于同一个生成模式。 +3. 选择 row-major、column-major、row-snake 或 column-snake、行列反向和起始 taxel 索引，预览后应用或取消。 + +范围以首末 taxel 中心定义，不是传感片外边缘。单行只需 O/C，单列只需 O/R，单点只需 O 和切线方向，避免除以零。可在预览中排除实际不存在的单元；被排除单元不产生 taxel，投影失败单元则保留为有问题的候选，不能混为一谈。 + +预览显示位置、数量和输出索引，错误标红。应用时原子创建本批独立 taxel，随后丢弃范围、行列和扫描参数。没有基于隐藏区域对象的“重新生成”；需要调整时多选并移动、重新投影或删除后重新批量添加。 + +默认从尚未使用的连续索引区间分配新点。移动、重投影和删除不重排已有输出；“重排所选 taxel”显示旧→新索引对应表后应用，检查要求最终覆盖 0..N-1。批量复制到另一侧只复用尚在界面的生成参数，必须重新拾取表面、绑定关节和分配索引，不镜像几何。 + +文件中的输出顺序由 taxel 数组行序表达；内部标识和临时编号不写入文件，重新打开后按文件行序建立编号。 + +选择提供三种入口：单击并用 Ctrl/Cmd 增减；框选画面范围；输入 taxel 索引范围或列表。框选模式暂时接管拖动，退出后恢复相机操作；默认只选择当前侧、可见且未被网格遮挡的点，“穿透选择”显式包含背面。多选属性中的不同值显示“多个值”，批量修改只作用于填写字段。 + +选中后可编辑绑定根中的位置、法线翻转、切线旋转和 taxel 索引。表面移动在指定表面重新拾取，自由移动标记为手工来源。高级折叠项提供 XYZW 四元数，不要求正常用户填写。列表每页 25 条，不创建逐 taxel 常驻控件。 + +### 1.7 绑定页 + +只有一套操作：编辑所选 taxel 的四个索引和权重槽，通常选择一至四个 segment。没有独立的刚性、两段过渡或手工模式。原始槽顺序始终保留，不按 segment 合并重复槽；权重在明确编辑时按下述规则更新。 + +右侧显示四槽，每槽包含 segment 下拉框、原始索引、颜色标记、0–1 权重滑块和独立的权重数值框；上方显示选中 taxel 数量和权重和。缺失、越界及重复索引仍显示原值。新选关节默认等分。通过滑块或数值框调整一个权重时，先替换该槽的值，再对该 taxel 的全部四槽执行 `w_k ← w_k / Σ_i w_i`，使权重和为 1；被调整槽的最终值也参与归一化。拒绝负权重输入；四槽含非有限值、负值或总和为零时，拒绝本次编辑、保留原行并显示原因。载入、显式记录接收值、检查、保存和导出保留原始权重，不自动归一化、累加或修复；其中的非法值仍须报告为错误。 + +选择或调整某槽权重时，通过颜色高亮对应关节、关联链接和当前 taxel，显示关节到 taxel 的连线，线宽或亮度表达权重。多选时保留黄色选择轮廓，以当前关节的权重给所选点着色；混合值显示为混合，不伪造统一值。批量调整对每个 taxel 独立执行四槽归一化。 + +“运动预览”使用草稿数组和 URDF 滑块检查绑定，草稿仍可编辑。“返回绑定姿态”结束预览。单段可显示运动坐标轴；多段只显示位置与力模长，不显示协议未定义的世界空间力箭头。偏离网格的点保持可见，不重新吸附来掩盖误差。 + +### 1.8 实时页 + +页面从上到下包含以下区块与控件： + +**数据源（每侧分别配置）** + +- 触觉 collection_id 输入框。By default "tactile". +- HandTracker 左手／右手输出选择. +- JointState 输入框 (optional, by default disabled). + +**URDF 驱动** + +实时 URDF 只能由启用的 JointState 输入驱动；未启用时隐藏 URDF，仅显示手骨架和 taxel。HandPose 不驱动 URDF，只提供实际手骨架和实时 taxel LBS 当前段位姿。EDIT/PREVIEW 中的 q0 编辑和手动运动预览不受此实时规则影响。 + +- JointState 映射表：源关节名、URDF 自由关节、比例、偏移。 +- 各关节的原始输入值、换算值、限位与映射状态。 + +**连接与播放** + +- 连接／断开按钮。 +- 播放／暂停及返回 q0 显示按钮；这些操作不改变草稿的可编辑性。 +- 记录 metadata 按钮，提供分组勾选项，默认全部选中。每组字段一并选择： + - Skeleton binding poses：`joint_ids`、`segment_bind_poses`。 + - Taxel binding poses：`taxel_bind_poses`。 + - Skinning：`taxel_segment_indices`、`taxel_segment_weights`。 +- 显式记录前先按第 1.1 节的相同 schema JSON 格式将当前 metadata 数值备份到临时位置，再替换所选分组；日志显示备份路径。 +- 三维可视化 metadata 来源切换：接收的 metadata／用户草稿，默认接收来源。所有页签中的 metadata 数值始终来自草稿。 +- 连接状态。各数据源的接入类型、配置标识、有效性和本地读取时间。 +- 工具操作事件、警告和错误日志，包括替换前的草稿备份路径。 + +实时接收和三维播放不自动替换草稿。每次显式记录仅以接收值替换所选分组，不合并旧值；替换后即使产生引用或数组长度错误，也保留结果并在对应区域显示原因。 + +metadata 与力值的实际对应关系由生产端和使用者负责。工具按当前 metadata 与力数组的协议行序显示，执行协议结构、数量及有效性检查；不维护独立的源行映射、重排前后对应关系或对齐基线，不为 metadata 编辑或重排重新排列力值。切换三维 metadata 来源不建立或修复它与力流的对应关系。 + +### 1.9 导出、检查与状态跳转 + +导出页选择当前侧或全部，提供“本地检查”“SDK 检查”和“导出”。问题按 error、warning、info 显示，点击后定位页面和实体。error 阻止正式导出；当前会话中的近似来源产生 warning，显示在检查结果中，不引入逐项确认门槛。 + +本地与 SDK 状态分开。SDK 探测成功只表示环境可用；未完成元数据构造不能显示官方通过。JSON 导出只要求本地无 error；SDK 导出还要求官方构造与大小检查通过。导出目录仅包含各侧 metadata JSON 及用户选择的 SDK 二进制；检查结果输出到界面或 CLI，不作为附属文件保存。 + +| 状态 | 显示与可用操作 | 跳转 | +| --- | --- | --- | +| EDIT | q0 下的 URDF、segments 和 taxel；允许全部静态编辑 | 运动预览进入 PREVIEW；连接并播放进入 LIVE | +| PREVIEW | 滑块驱动模型与 LBS；表单始终编辑草稿 | 返回绑定姿态恢复 EDIT | +| LIVE | 连续更新三维模型、骨架和力值；三维 metadata 来源可切换，所有表单仍显示并编辑草稿 | 暂停进入 PAUSED | +| PAUSED | 冻结实时三维层；接收继续；表单和草稿操纵器仍可编辑 | 恢复取最新帧；返回 q0 显示进入 EDIT | +| EDIT 且已连接 | q0 编辑；后台只保留最新数据，不覆盖文档 | 播放重新进入 LIVE | +| 数据中断 | 保留最后画面并标记失效；草稿继续可编辑，不能把旧力视为当前值 | 可回 EDIT；重连后重新检查数据有效性 | + +所有模式下，数值表单及草稿操纵器都编辑 q0 下的手根坐标，不反解实时混合蒙皮，也不把实时位姿写回草稿。三维操纵器针对明确选中的草稿实体；接收 metadata 的三维图形只读，不能因拾取它而替换表单。PAUSED 只冻结实时三维层，草稿操纵器随编辑更新；恢复后使用最新帧及当前草稿重新计算。播放、暂停、选择、相机和实时帧不修改 Document。 + +本地保存允许未完成的 metadata 数值，仍使用第 1.1 节的相同字段结构；“编辑草稿，尚非有效 metadata”仅为界面提示，不写入文件。正式导出必须完整，校验作为独立步骤，不引入另一种序列化格式。导出使用启动时的 metadata 数值快照，继续编辑不改变导出内容，界面或 CLI 结果显示对应 revision。目标目录非空时要求新目录；全部手侧成功后才发布数值文件。 + +### 1.10 命令行 + +GUI 和 CLI 共用计算、检查与导出，通过 Python 模块入口启动。以下命令使用调用者选定的 Python 环境，从工具目录执行；`python` 指该环境的解释器，工具不替换它： + +```bash +cd scripts/tactile_metadata +python -m tactile_metadata_editor.cli gui --urdf glove.urdf --sides left +python -m tactile_metadata_editor.cli gui --urdf hands.urdf --sides both --left-metadata left.metadata.json --right-metadata right.metadata.json +python -m tactile_metadata_editor.cli gui --metadata left.metadata.json --sides left +python -m tactile_metadata_editor.cli check left.metadata.json +python -m tactile_metadata_editor.cli check left.metadata.json --sdk +python -m tactile_metadata_editor.cli export left.metadata.json --sides left --format sdk --out package +``` + +GUI 不带参数时进入启动页，默认监听 127.0.0.1:8080，可用 --port 修改。JSON 不携带手侧或 URDF 配置：GUI 通过界面或 `--sides` 明确手侧，CLI export 通过 `--sides left/right/both` 分配输入文件，手侧重复明确拒绝；纯数值 CLI 检查不需要手侧。检查结果输出到界面或 CLI 标准输出，不提供 `--report` 文件输出。实时连接在当前 GUI 会话中配置；`--demo` 使用内置合成数据验证实时交互，不是额外文件导入。 + +## 2 依赖 + +| 项目 | 决策 | 原因与用途 | +| --- | --- | --- | +| 语言 | Python >=3.11,<3.14 | 与 SDK 生态衔接，工具本身无需原生编译。 | +| 浏览器 GUI | viser==1.1.1 | 复用三维客户端、表单、框选、射线事件和控制器。 | +| URDF | yourdfpy==0.0.60 | 资源解析、链接图、关节配置和 FK。 | +| 网格处理 | trimesh==5.1.1 | URDF 网格资源、法线、连通分量和射线。 | +| 数值运算 | numpy>=2,<3 | 批量变换、数组校验和 LBS 位置计算。 | +| 查询加速 | rtree>=1.3,<2 | 三角形空间查询，不要求 Embree。 | +| SDK 集成 | 启动工具的 Python 环境中可用的 Isaac Capture SDK | 官方编码以及实时 tracker/session；复用该环境中的 SDK，离线流程不导入 SDK。 | +| 本地通信 | Python 标准库 subprocess、socket、json | SDK 隔离与实时数据传输，不增加消息中间件。 | +| 依赖维护与运行 | Isaac Capture 现有环境依赖声明及 Python 模块入口 | 补齐缺失库，复用启动工具的环境；不建立工具专用项目、虚拟环境或锁文件。 | +| 开发检查 | 仓库已有检查，按用户指定范围执行 | 不新增测试文件或测试依赖，不运行越出工作目录或可能改写其他目录的检查。 | + +将缺失库及所需版本约束加入 Isaac Capture 现有的 [`src/core/python/requirements*.txt`](../../src/core/python/) 对应依赖分组，并同步[根 pyproject.toml](../../pyproject.toml) 等对应包依赖声明。已有依赖按所需版本复用或补充约束，不另建工具专用 `pyproject.toml`、`uv.lock` 或 `.venv`。依赖准备使用调用者选定的同一 Python 环境；工具启动时不自动安装库、创建环境或构建 SDK。 + +版本依据为 [Viser](https://pypi.org/project/viser/1.1.1/)、[yourdfpy](https://pypi.org/project/yourdfpy/0.0.60/) 和 [trimesh](https://pypi.org/project/trimesh/5.1.1/) 发布页。实施时核对共享依赖声明和实际安装版本，并验证模型加载。Viser 框选接口以 [v1.1.1 源码](https://github.com/nerfstudio-project/viser/blob/v1.1.1/src/viser/_scene_api.py) 为准，不把 main 文档新增接口当成指定版本保证。 + +离线编辑不依赖 ROS、CUDA、OpenXR 会话或模拟器。官方检查、SDK 二进制导出和实时接收使用启动工具的 Python 环境中可导入的 SDK；实时接收还需要对应 runtime/设备，当前 live OpenXR 路径需要可用 HMD system，依据[设备文档](../../docs/source/device/tactile/index.rst)。SDK 仅在以 `sys.executable` 启动、继承启动环境的本地子进程中导入。不可用时显示原因，离线编辑仍可运行；不查找旧构建目录、不切换解释器、不提供 SDK 路径配置，也不要求实现机器预先具备未来用户的运行环境。 + + +## 3 给 agent 的范围规定风格规范和 restriction + +### 3.1 范围与工作规范 + +实现第一节定义的完整工具，工具代码及模型示例位于 scripts/tactile_metadata/。允许为本工具补齐 Isaac Capture 现有环境依赖声明及对应包元数据；不建立工具专用环境。不改 schema、golden 或无关子系统；真实冲突提供代码证据并同步规格。设计工具架构以确保well structured and functionally correct，并且concise without redundant codes, strictly limited to the explicitly described scope。制定的实现计划行数预估必须小于2000行（不包括文档）。 + +修改前列出目录并读取全部适用 AGENTS.md，保护已有改动。本地检查，不连接远程计算机，不自动提交或推送。不把本机 SDK/构建/资源绝对路径硬编码到工具或可移植文件。 + +领域类型集中，GUI/CLI 共享数组准备、检查和导出；按模块契约实现，不另造通用框架。遵守仓库日志、SPDX、简短注释、格式化和 Mandatory learning loop。文件输入范围、术语和批量操作生命周期是产品约束，不能为实现便利恢复被移除入口。 + +### 3.2 实施顺序与交付 + +制定实施计划后，按合理的逻辑顺序执行实施。 + +交付复用启动 Python 环境的工具、README中包括使用工具的step to step instruction，以本目录现有model/文件夹内urdf为示例、面向开发者的技术说明（解释模块和数据流等）。不交付工具专用环境或锁文件，不要求新增 URDF、网格或测试文件，也不新增测试依赖。合成实时源可在无 SDK/硬件时运行，实际 live 接收仍须实现，不能用合成数据代替。README 逐步说明在已有 Isaac Capture 环境中准备依赖并启动工具，以及离线编辑、schema JSON 继续编辑、四槽权重归一化、taxel 输出顺序和实时播放/暂停/编辑/恢复。 diff --git a/scripts/tactile_metadata/DESIGN.md.license b/scripts/tactile_metadata/DESIGN.md.license new file mode 100644 index 0000000000..8cf194ccd1 --- /dev/null +++ b/scripts/tactile_metadata/DESIGN.md.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md new file mode 100644 index 0000000000..35be7c1d48 --- /dev/null +++ b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md @@ -0,0 +1,38 @@ + + +# Tactile Metadata Editor 实现提示词 + +将以下提示词交给能够读取本仓库的实现代理。完整规格见同目录 [DESIGN.md](DESIGN.md)。 + +```text + +请实现 /Users/lotusl/Codes/IsaacTeleop/scripts/tactile_metadata/DESIGN.md +定义的 Tactile Metadata Editor。All implementation should use English instead of Chinese. + +When implementation, make sure to only modify folder /Users/lotusl/Codes/IsaacTeleop/scripts/tactile_metadata and its decendent files. It's where your implementation should fully locate. Don't write to any other paths, except for /tmp and your own workspace, and the IsaacTeleop folder's library config only to add the library dependencies as guided. Don't connect to any remote computer or write to any remote end. You're allowed to read codes in the whole ~/Codes/IsaacTeleop. Don't run pre-commit or other whole-repo formatter/tests. It's OK to run local formatter/test within tactile_metadata. + +先完整读取设计、仓库根及所有修改路径适用的 AGENTS.md，检查已有改动。 +这是实现任务，按已确定的规格工作。编码前完整了解领域、数据流及已有 +能力，以现有 schema、公开绑定、共享校验函数和 tracker/session 核实 +协议。 + +The following product decisions are settled and must be reflected throughout the implementation: +1. Weight editing: after a slider or numeric-field edit, replace that slot and normalize all four weights by their sum, including the edited slot. Reject negative weights. Reject an edit whose resulting row contains non-finite or negative values or has a zero sum, preserving the previous row. Apply normalization independently to each selected taxel. Preserve slot order and repeated indices. Loading, explicitly recording received metadata, checking, saving, and exporting preserve raw weights and report invalid values without silently normalizing them. +2. Metadata persistence: saving, backups, and export write only tactile metadata values using the fields, nested structure, and union type discriminators defined by src/core/schema/fbs/tactile_metadata.fbs. The only top-level JSON fields are geometry_mapping_type, geometry_mapping, segment_source_type, and segment_source. Use one schema JSON representation for saving, reopening, and JSON export; optional SDK binaries encode the same values. Do not add an editor extension, tool-specific fields, project/configuration files, .editor.json, internal-ID correspondence tables, or saved check reports. Read with json.load(), perform the array and enum conversion required by the existing SDK constructors, and encode through LBS, HandSkeletonRef, TactileMetadata, and to_bytes() in the isolated SDK subprocess. Do not invent another metadata format or treat this conversion as an unresolved product decision. +3. Scope: metadata/force alignment is the producer's and user's responsibility. Display the current arrays in protocol row order and retain structural, count, and validity checks. Do not maintain source-row mappings, correspondence across reordering, alignment baselines, or force permutations to compensate for metadata edits. Switching visualization sources does not establish or repair force alignment. Persist taxel output order through array row order, without saving internal IDs or temporary numbering. +4. URDF display: all links use fixed opacity 0.6. Do not add per-link visibility or opacity controls or state. Highlight associated links through color without changing opacity; retain mode-level model visibility. +5. Session state: URDF/package paths, q0, wrist calibration, segment/link associations, hand assignments, live-source and joint mappings, internal IDs, and display settings exist only in the current session. Reopening restores only metadata values and array order; the user assigns the hand through the GUI or --sides and sets up the URDF associations and calibration again when needed. Unsaved state tracks metadata values only. Incomplete saves use the same schema fields; show draft status, validation results, and revisions in the GUI or CLI output, never in metadata or companion files. Validate formal exports separately from serialization. Do not implement separate editable/numerical formats, session restoration, or a --report file output. + +Before implementation, do three analysis: +1. whether there's any unresolved 规格矛盾、错误假设、风险、缺失功能或 SDK 能力缺口. +2. whether the implementation can be scoped to 2000 lines of codes or fewer (this impentation shouldn't delete or modify any existing lines, only will add lines). +3. whether any operation would require my manual permission. +If finding any of the gaps above, stop and clarify the gaps. + +Over-engineering is absolutely forbidden. 2000 lines is the hard limit. If ever you exceed this number, it's considered to be a failure, and you need to remove all your changes and analyze what you did wrong. +Now redo the task with the new 2000 line limit and strictly obey the minimal implementation rule. + +``` diff --git a/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf b/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf new file mode 100644 index 0000000000..128d40f2cb --- /dev/null +++ b/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf @@ -0,0 +1,1265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/tactile_metadata/model/left/meshes/base_link.STL b/scripts/tactile_metadata/model/left/meshes/base_link.STL new file mode 100644 index 0000000000..caa668eb53 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/base_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7c8168b89ad9f225f29bf409719dabab78885fefefffdf2820845937fa9bc542 +size 2470434 diff --git a/scripts/tactile_metadata/model/left/meshes/index_distal.STL b/scripts/tactile_metadata/model/left/meshes/index_distal.STL new file mode 100644 index 0000000000..b50fbfb6fc --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/index_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:719d02f83aa1b69693f1b478ca9de4afb804a02630cd6920977c3f75b9a45a99 +size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL new file mode 100644 index 0000000000..3596feb0e1 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:37c0c7070aa833f085bcf7120d903977abdafe3cbf95173eecd684be794afdd7 +size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/index_middle.STL b/scripts/tactile_metadata/model/left/meshes/index_middle.STL new file mode 100644 index 0000000000..c6066e326a --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/index_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60c01971d5558a913db3431b3186245bccec6cd296800685b5324faa5833c725 +size 188084 diff --git a/scripts/tactile_metadata/model/left/meshes/index_proximal.STL b/scripts/tactile_metadata/model/left/meshes/index_proximal.STL new file mode 100644 index 0000000000..b1dbb3e657 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/index_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87ddd901598bab2e64a11ec61193c3056ebfdd259d32fc4ed0da208e844b83a6 +size 652384 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_distal.STL b/scripts/tactile_metadata/model/left/meshes/middle_distal.STL new file mode 100644 index 0000000000..2a01de4d6c --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/middle_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a1bf0b18f6a6e77089cc8f392e78bb832415570e388f9c61e98f956d5604a61f +size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL new file mode 100644 index 0000000000..2d5e64465b --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f74d86cc7e7290ff7ed873ce038c7b33718b0d0d157deaaa4ff964ece0abc828 +size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_middle.STL b/scripts/tactile_metadata/model/left/meshes/middle_middle.STL new file mode 100644 index 0000000000..fe4baad770 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/middle_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:abd61a575ea84c33362107e90c8964b4bd6f544cbda8ac2098651be8482947db +size 188084 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_proximal.STL b/scripts/tactile_metadata/model/left/meshes/middle_proximal.STL new file mode 100644 index 0000000000..b4c4e99e78 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/middle_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da4562a6e80b940e89ce26c9a12828d2eadba1cd4d4987995c2d484de1b326ba +size 652384 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_distal.STL b/scripts/tactile_metadata/model/left/meshes/pinky_distal.STL new file mode 100644 index 0000000000..e9f861bf6b --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/pinky_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:502aa6ff2d997975546a0b244af2969046c3e5b286d13ae482f4498e0deeb638 +size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL new file mode 100644 index 0000000000..e98beab8ac --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0d00911724b3d736d22c79d2cc3b52cf4b7449e3635a4b328b90370452b9cb67 +size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_middle.STL b/scripts/tactile_metadata/model/left/meshes/pinky_middle.STL new file mode 100644 index 0000000000..5529fa5991 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/pinky_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e72ffa11712ece3a77441d11876dc8dd0c881570e49b6c32f909b6d6e3d0d7da +size 226484 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL b/scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL new file mode 100644 index 0000000000..a688ef8a46 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9fc1bf89257a7a0b6d735d10178c33ea9e3347c7419ce9ac45830b7b2e042486 +size 654284 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_distal.STL b/scripts/tactile_metadata/model/left/meshes/ring_distal.STL new file mode 100644 index 0000000000..557d08cfa2 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/ring_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e4a4001c3797a4294c43a5902c5efc495136eaf3d6e8555552ef8ed8c850b59 +size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL new file mode 100644 index 0000000000..98f858c212 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:819a47007b09596026a4b9454a8fe5c101a3e88c4c83eb3d8b448ea3bcf6a161 +size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_middle.STL b/scripts/tactile_metadata/model/left/meshes/ring_middle.STL new file mode 100644 index 0000000000..f0e71a8f3a --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/ring_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:888c84e49b57a86626cc797bf71c5a7a7cfc867d80ee0f5e8591fdad3164bcd1 +size 188084 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_proximal.STL b/scripts/tactile_metadata/model/left/meshes/ring_proximal.STL new file mode 100644 index 0000000000..0c95de5c7c --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/ring_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8df7a9d6a15fd254ef66307f2044457abad5d6b439617718ff03209d29584d8b +size 652384 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_distal.STL b/scripts/tactile_metadata/model/left/meshes/thumb_distal.STL new file mode 100644 index 0000000000..01ede11968 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/thumb_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:833a95573cf8de2e33b40896fc6de36ab9af8bccad9b8f6c8a9ac5c2699b5a62 +size 276184 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL new file mode 100644 index 0000000000..399e701edc --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7857b0bf11381e343c66ee63ec8c5289e294dd20adc04c78125e26ec45f8c083 +size 161184 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL new file mode 100644 index 0000000000..c0b399f3f6 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:90414281d61e1bad2073836dd754d5b0e0ce8b9179f3c23aefe95f0e0e3624dd +size 485784 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL new file mode 100644 index 0000000000..63b4666ea9 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b225b009ac9803e6f59476316e037710f0a86fa4d920272c2a4e865126630229 +size 628484 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL b/scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL new file mode 100644 index 0000000000..d6310ac610 --- /dev/null +++ b/scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6fbba678d35ba609a3abc62c4dececab1e3d0104ae5a6bdee918e89905eb65dc +size 210584 diff --git a/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf b/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf new file mode 100644 index 0000000000..ac4a83918e --- /dev/null +++ b/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf @@ -0,0 +1,1265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/tactile_metadata/model/right/meshes/hand_base_link.STL b/scripts/tactile_metadata/model/right/meshes/hand_base_link.STL new file mode 100644 index 0000000000..c2a62e2e7e --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/hand_base_link.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:94c50cfcc29be7179f9e6d8352ab5a3213c21e4795cb8d58a346ebfff74ee8bc +size 1220684 diff --git a/scripts/tactile_metadata/model/right/meshes/index_distal.STL b/scripts/tactile_metadata/model/right/meshes/index_distal.STL new file mode 100644 index 0000000000..eb1de0fb2d --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/index_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b806badc8765acb6ed9401a0fb23d90e7ae6ff2f3a77ff40b014e063140b8e06 +size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL new file mode 100644 index 0000000000..bc629465fb --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:df03f980f92a0c66662081fb4c147294c60b9acdf8cc7905e21a9790d6c7720d +size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/index_middle.STL b/scripts/tactile_metadata/model/right/meshes/index_middle.STL new file mode 100644 index 0000000000..4846357f98 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/index_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:17f81c7ad8297dac033dded0f0c89a521f766208a8efadb1b0f7c57d3e7c374a +size 188084 diff --git a/scripts/tactile_metadata/model/right/meshes/index_proximal.STL b/scripts/tactile_metadata/model/right/meshes/index_proximal.STL new file mode 100644 index 0000000000..5b902ded0d --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/index_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:57b1aa6fc7a303a6b18417eb078092c332fa577f548d5747b500253a9daecae1 +size 651984 diff --git a/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf b/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf new file mode 100644 index 0000000000..8dbd52cd30 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf @@ -0,0 +1,1265 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/scripts/tactile_metadata/model/right/meshes/middle_distal.STL b/scripts/tactile_metadata/model/right/meshes/middle_distal.STL new file mode 100644 index 0000000000..148ba11db8 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/middle_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:459a2b44a893cb42eb2b4671f17985f1dabaa9bce6371f190e69a60badd1299f +size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL new file mode 100644 index 0000000000..8a49231528 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2583b7237b2a33a7ad2ddc17f1ff61c6ec5d38335e8ba1340d43e6efecc1256 +size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/middle_middle.STL b/scripts/tactile_metadata/model/right/meshes/middle_middle.STL new file mode 100644 index 0000000000..d31133bc28 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/middle_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e533f80d298bf783b5a20c30bae383fe53fe1d3b4430268070655f363ace2ba +size 188084 diff --git a/scripts/tactile_metadata/model/right/meshes/middle_proximal.STL b/scripts/tactile_metadata/model/right/meshes/middle_proximal.STL new file mode 100644 index 0000000000..349a109f75 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/middle_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8cff8d8a4a6b719b77fc525351ad0fbb0c961a73b859b0b0a4c07190a0aa186a +size 651984 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_distal.STL b/scripts/tactile_metadata/model/right/meshes/pinky_distal.STL new file mode 100644 index 0000000000..9e86b13165 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/pinky_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ca40e877cf35deaea1ccdb6880ded2f1d85460d47d8c1e31cdd36f0ef1c1d05c +size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL new file mode 100644 index 0000000000..54b5a30e9d --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9d130c833b07a827a6ea194fcf3844dcc0947c7aad9f6c954d7ec7c3c9aefaa +size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_middle.STL b/scripts/tactile_metadata/model/right/meshes/pinky_middle.STL new file mode 100644 index 0000000000..4cca9d63fe --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/pinky_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45ce34033924e8a5392bbacbd9f7af752110606c68ff33f24daad8f3941fc8ab +size 226484 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL b/scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL new file mode 100644 index 0000000000..2e71e87d48 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:201bd33cc07460751846b8ca8e355c30d55c82d986874e8db73f8cdd40e33609 +size 654084 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_distal.STL b/scripts/tactile_metadata/model/right/meshes/ring_distal.STL new file mode 100644 index 0000000000..7ab5495a38 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/ring_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1953f24ce5d1b99658c64ef589c4ad5ae51f4538833d5dfa4b18514d3cc39fc7 +size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL new file mode 100644 index 0000000000..2b8bcc7138 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b4b7d52ccc032e6c3c3061bdfe2478ebf6d68bc0ddcbb37723f77e18e2059532 +size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_middle.STL b/scripts/tactile_metadata/model/right/meshes/ring_middle.STL new file mode 100644 index 0000000000..e68bcbb27e --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/ring_middle.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5e079aa92ed3362b0e0bdbf6d2467927804df65c2d2a333a70f57dc7c6a60ff8 +size 188084 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_proximal.STL b/scripts/tactile_metadata/model/right/meshes/ring_proximal.STL new file mode 100644 index 0000000000..169b959e80 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/ring_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f726495555c728c61759f710d864f4688c0d9a8c3df6fb07bca02840b9498c8 +size 651984 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_distal.STL b/scripts/tactile_metadata/model/right/meshes/thumb_distal.STL new file mode 100644 index 0000000000..3a24c5e2be --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/thumb_distal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4f857160f2c186bd105f29ac90a630d6918612cbd97a010654e12d873c0546cf +size 276184 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL new file mode 100644 index 0000000000..1c155faaed --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f5b61b2e29e6eaa5623980d00465fb3a4f93c3b6d978fc0f50a703827cb2642c +size 161384 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL new file mode 100644 index 0000000000..20919e64e3 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dfdaebb2b720a4949986740bbadc3da519fe46c09bff8e145ead0640977256bc +size 484884 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL new file mode 100644 index 0000000000..eac307de4f --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2d4a436027f4b5ade7c50d15a334a109a3a4562073118ef5382784748ef55600 +size 628484 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL b/scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL new file mode 100644 index 0000000000..bb116eb9e1 --- /dev/null +++ b/scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45237da8babd57d1b32b250d8bd6d313679e8fad313e1f9a793fba82c5e35a60 +size 210584 From e5116e806841bef16e420af714ef3f34c926d60f Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 11:46:03 -0700 Subject: [PATCH 06/36] fix wrong function call Signed-off-by: Lotus Li --- .../tactile_metadata_editor/geometry.py | 2 +- .../tactile_metadata_editor/ui.py | 386 +++++++++--------- 2 files changed, 194 insertions(+), 194 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index c415b3d3f5..adee21ee1e 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -92,7 +92,7 @@ def resource(fname): self.surfaces[f"{link}/{node}/{i}"] = (link, part) def fk(self, cfg): - configuration = tuple(cfg.get(j.name, 0) for j in self.joints) + configuration = tuple(float(cfg.get(j.name, 0.0)) for j in self.joints) result = self._fk_cache.pop(configuration, None) if result is None: self.urdf.update_cfg( diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 7c5468a94b..f14015cc60 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -67,205 +67,205 @@ def __init__(self, state, args): self.mode = g.add_dropdown("Mode", ("EDIT", "PREVIEW", "LIVE", "PAUSED")) self.watch(self.side, lambda _: self.switch()) self.watch(self.mode, lambda _: None) - with g.add_tab_group() as tabs: - with tabs.add_tab("Model"): - self.urdf = g.add_text("URDF", state.model.get("urdf", "")) - self.packages = g.add_text( - "Local package map JSON", - json.dumps(state.model.get("packages", {})), - ) - self.sides = g.add_dropdown( - "New model hands", - ("left", "right", "both"), - initial_value="both" - if len(state.documents) == 2 - else self.side.value, - ) - self.button("New document", lambda: self.guard(self.reset)) - self.button("Load URDF", lambda: self.guard(self.load_new)) - self.open_path = g.add_text("Metadata JSON", args.metadata or "") - self.json_side = g.add_dropdown( - "Assign JSON to", ("left", "right"), initial_value=self.side.value - ) - self.button("Open JSON", lambda: self.guard(self.open)) - self.wrist = g.add_dropdown("Wrist link", ("",)) - self.allowed = g.add_text("Allowed links, comma separated", "") - self.button("Apply allowed links", self.set_links) - self.pick = g.add_dropdown( - "Pick target", - ( - "O", - "C", - "R", - "Wrist O", - "Wrist F", - "Wrist D", - "Segment", - "Move selected", - "Single tangent", - ), - ) - self.pick_button = self.button( - "Pick on q0 model", - lambda: setattr(self, "pick_target", self.pick.value), - ) - self.calibrate_button = self.button( - "Preview / set wrist reference", self.calibrate - ) - self.resources = g.add_markdown("") - self.button("Reset cameras", self.camera) - with tabs.add_tab("Segments"): - self.joint = g.add_dropdown("URDF free joint", ("",)) - self.angle = g.add_slider("Preview joint", -3.15, 3.15, 0.001, 0.0) - self.angle_number = g.add_number("Preview joint value", 0.0) - self.watch(self.joint, lambda _: self.joint_controls()) - for w in (self.angle, self.angle_number): - self.watch(w, self.preview_joint) - self.button("Set shared binding pose q0", self.confirm_q0) - self.button("Return to q0", lambda: setattr(self.mode, "value", "EDIT")) - self.segment = g.add_dropdown("Segment row", ("",)) - self.semantic = g.add_dropdown("HandJoint", tuple(JOINTS)) - self.segment_link = g.add_dropdown("Associated link", ("",)) - self.watch(self.segment, lambda _: self.select_segment()) - self.button("Add segment", self.add_segment) - self.button("Set semantic / link", self.associate) - self.order = g.add_text("Segment row order", "[]") - self.button( - "Reorder / delete omitted segments", - self.reorder_segments, - ) - self.segment_table = g.add_markdown("") - with tabs.add_tab("Layout"): - self.surface = g.add_dropdown("Projection surface", ("",)) - self.rows = g.add_number("Rows", 4, min=1) - self.cols = g.add_number("Columns", 4, min=1) - self.centers = [ - g.add_vector3(k + " world center", (0.0, 0.0, 0.0)) - for k in ("O", "C", "R") - ] - for key, w in zip(("O", "C", "R"), self.centers): - self.watch(w, lambda v, key=key: self.points.update({key: list(v)})) - self.spans = [ - g.add_number(k + " span (m)", 0.03) for k in ("Column", "Row") - ] - self.button("Apply spans along picked directions", self.spans_apply) - self.scan = g.add_dropdown( - "Scan", ("row-major", "column-major", "row-snake", "column-snake") - ) - self.reverse = [ - g.add_checkbox(k, False) - for k in ("Reverse rows", "Reverse columns") - ] - self.excluded = g.add_text("Exclude cells [row,column]", "[]") - self.start = g.add_number("Start output index (-1 = unused)", -1) - self.influences = g.add_text("New taxel segment rows (1–4)", "[]") - self.grid_button = self.button("Preview grid", self.grid) - self.button("Apply candidates", self.apply_grid) - self.button("Cancel candidates", self.clear_layout) - self.layout_info = g.add_markdown("") - self.selection = g.add_text("Select output indices / ranges", "") - self.button("Select indices", self.select_indices) - self.rectangle = g.add_checkbox("Rectangle selection (drag)", False) - self.through = g.add_checkbox("Select through mesh", False) - self.watch(self.rectangle, self.rectangle_mode) - self.button( - "Delete selected taxels", - self.delete_taxels, - ) - self.reproject_button = self.button( - "Reproject selected", self.reproject - ) - self.indices = g.add_text("New selected output indices", "[]") - self.button("Preview / apply output indices", self.reindex) - self.page = g.add_number("List page (25 rows)", 0, min=0) - self.watch(self.page, lambda _: None) - self.table = g.add_markdown("") - with tabs.add_tab("Binding"): - self.binding_info = g.add_markdown("") - self.button( - "Assign segment rows equally", - lambda: self.change( - self.d.assign, self.selected, json.loads(self.influences.value) - ), - ) - self.slots = [] - for k in range(4): - with g.add_folder(f"Slot {k}"): - segment = g.add_dropdown("Segment", ("",)) - index = g.add_number("Set raw index", 0) - slider = g.add_slider("Weight", 0.0, 1.0, 0.001, 0.0) - number = g.add_number("Set weight", 0.0) - self.watch( - segment, - lambda v, k=k: ( - self.set_slot(k, int(v.split(":")[0])) - if not v.startswith("Raw") - else None - ), - ) - self.watch(index, lambda v, k=k: self.set_slot(k, v)) - for w in (slider, number): - self.watch(w, lambda v, k=k: self.weight(k, v)) - self.slots.append((segment, index, slider, number)) - with tabs.add_tab("Live"): - self.collection = g.add_text("Tactile collection_id", "tactile") - self.hand = g.add_dropdown("HandTracker output", ("left", "right")) - self.joint_enabled = g.add_checkbox("Enable JointState", False) - self.joint_id = g.add_text("JointState collection_id", "") - self.button( - "Apply this hand's source configuration", self.configure_source + tabs = g.add_tab_group() + with tabs.add_tab("Model"): + self.urdf = g.add_text("URDF", state.model.get("urdf", "")) + self.packages = g.add_text( + "Local package map JSON", + json.dumps(state.model.get("packages", {})), + ) + self.sides = g.add_dropdown( + "New model hands", + ("left", "right", "both"), + initial_value="both" + if len(state.documents) == 2 + else self.side.value, + ) + self.button("New document", lambda: self.guard(self.reset)) + self.button("Load URDF", lambda: self.guard(self.load_new)) + self.open_path = g.add_text("Metadata JSON", args.metadata or "") + self.json_side = g.add_dropdown( + "Assign JSON to", ("left", "right"), initial_value=self.side.value + ) + self.button("Open JSON", lambda: self.guard(self.open)) + self.wrist = g.add_dropdown("Wrist link", ("",)) + self.allowed = g.add_text("Allowed links, comma separated", "") + self.button("Apply allowed links", self.set_links) + self.pick = g.add_dropdown( + "Pick target", + ( + "O", + "C", + "R", + "Wrist O", + "Wrist F", + "Wrist D", + "Segment", + "Move selected", + "Single tangent", + ), + ) + self.pick_button = self.button( + "Pick on q0 model", + lambda: setattr(self, "pick_target", self.pick.value), + ) + self.calibrate_button = self.button( + "Preview / set wrist reference", self.calibrate + ) + self.resources = g.add_markdown("") + self.button("Reset cameras", self.camera) + with tabs.add_tab("Segments"): + self.joint = g.add_dropdown("URDF free joint", ("",)) + self.angle = g.add_slider("Preview joint", -3.15, 3.15, 0.001, 0.0) + self.angle_number = g.add_number("Preview joint value", 0.0) + self.watch(self.joint, lambda _: self.joint_controls()) + for w in (self.angle, self.angle_number): + self.watch(w, self.preview_joint) + self.button("Set shared binding pose q0", self.confirm_q0) + self.button("Return to q0", lambda: setattr(self.mode, "value", "EDIT")) + self.segment = g.add_dropdown("Segment row", ("",)) + self.semantic = g.add_dropdown("HandJoint", tuple(JOINTS)) + self.segment_link = g.add_dropdown("Associated link", ("",)) + self.watch(self.segment, lambda _: self.select_segment()) + self.button("Add segment", self.add_segment) + self.button("Set semantic / link", self.associate) + self.order = g.add_text("Segment row order", "[]") + self.button( + "Reorder / delete omitted segments", + self.reorder_segments, + ) + self.segment_table = g.add_markdown("") + with tabs.add_tab("Layout"): + self.surface = g.add_dropdown("Projection surface", ("",)) + self.rows = g.add_number("Rows", 4, min=1) + self.cols = g.add_number("Columns", 4, min=1) + self.centers = [ + g.add_vector3(k + " world center", (0.0, 0.0, 0.0)) + for k in ("O", "C", "R") + ] + for key, w in zip(("O", "C", "R"), self.centers): + self.watch(w, lambda v, key=key: self.points.update({key: list(v)})) + self.spans = [ + g.add_number(k + " span (m)", 0.03) for k in ("Column", "Row") + ] + self.button("Apply spans along picked directions", self.spans_apply) + self.scan = g.add_dropdown( + "Scan", ("row-major", "column-major", "row-snake", "column-snake") + ) + self.reverse = [ + g.add_checkbox(k, False) + for k in ("Reverse rows", "Reverse columns") + ] + self.excluded = g.add_text("Exclude cells [row,column]", "[]") + self.start = g.add_number("Start output index (-1 = unused)", -1) + self.influences = g.add_text("New taxel segment rows (1–4)", "[]") + self.grid_button = self.button("Preview grid", self.grid) + self.button("Apply candidates", self.apply_grid) + self.button("Cancel candidates", self.clear_layout) + self.layout_info = g.add_markdown("") + self.selection = g.add_text("Select output indices / ranges", "") + self.button("Select indices", self.select_indices) + self.rectangle = g.add_checkbox("Rectangle selection (drag)", False) + self.through = g.add_checkbox("Select through mesh", False) + self.watch(self.rectangle, self.rectangle_mode) + self.button( + "Delete selected taxels", + self.delete_taxels, + ) + self.reproject_button = self.button( + "Reproject selected", self.reproject + ) + self.indices = g.add_text("New selected output indices", "[]") + self.button("Preview / apply output indices", self.reindex) + self.page = g.add_number("List page (25 rows)", 0, min=0) + self.watch(self.page, lambda _: None) + self.table = g.add_markdown("") + with tabs.add_tab("Binding"): + self.binding_info = g.add_markdown("") + self.button( + "Assign segment rows equally", + lambda: self.change( + self.d.assign, self.selected, json.loads(self.influences.value) + ), + ) + self.slots = [] + for k in range(4): + with g.add_folder(f"Slot {k}"): + segment = g.add_dropdown("Segment", ("",)) + index = g.add_number("Set raw index", 0) + slider = g.add_slider("Weight", 0.0, 1.0, 0.001, 0.0) + number = g.add_number("Set weight", 0.0) + self.watch( + segment, + lambda v, k=k: ( + self.set_slot(k, int(v.split(":")[0])) + if not v.startswith("Raw") + else None + ), + ) + self.watch(index, lambda v, k=k: self.set_slot(k, v)) + for w in (slider, number): + self.watch(w, lambda v, k=k: self.weight(k, v)) + self.slots.append((segment, index, slider, number)) + with tabs.add_tab("Live"): + self.collection = g.add_text("Tactile collection_id", "tactile") + self.hand = g.add_dropdown("HandTracker output", ("left", "right")) + self.joint_enabled = g.add_checkbox("Enable JointState", False) + self.joint_id = g.add_text("JointState collection_id", "") + self.button( + "Apply this hand's source configuration", self.configure_source + ) + self.source_joint = g.add_text("Source joint name", "") + self.target_joint = g.add_dropdown("URDF target joint", ("",)) + self.scale = g.add_number("Scale", 1.0) + self.offset = g.add_number("Offset", 0.0) + self.button("Add / update mapping", self.mapping) + self.button("Remove mapping", lambda: self.mapping(True)) + self.mapping_info = g.add_markdown("") + self.button("Connect / disconnect", self.connect) + self.button( + "Play / pause", + lambda: setattr( + self.mode, + "value", + "PAUSED" if self.mode.value == "LIVE" else "LIVE", + ), + ) + self.source = g.add_dropdown( + "Visualization metadata", ("Received", "Draft") + ) + self.force_max = g.add_number( + "Force color maximum (N)", 30.0, min=0.001 + ) + self.groups = {name: g.add_checkbox(name, True) for name in GROUPS} + self.button("Record metadata groups", self.record) + self.live_info = g.add_markdown("") + with tabs.add_tab("Export"): + self.paths = { + s: g.add_text( + s + " save JSON", state.paths.get(s, s + ".metadata.json") ) - self.source_joint = g.add_text("Source joint name", "") - self.target_joint = g.add_dropdown("URDF target joint", ("",)) - self.scale = g.add_number("Scale", 1.0) - self.offset = g.add_number("Offset", 0.0) - self.button("Add / update mapping", self.mapping) - self.button("Remove mapping", lambda: self.mapping(True)) - self.mapping_info = g.add_markdown("") - self.button("Connect / disconnect", self.connect) + for s in ("left", "right") + } + self.button("Save all metadata", self.save) + self.all_sides = g.add_checkbox("Check / export all hands", True) + self.destination = g.add_text("New output directory", "package") + self.binary = g.add_checkbox("Include SDK binary", False) + self.button("Local check", self.local_check) + self.button("SDK check", lambda: self.background(False)) + self.button("Export snapshot", lambda: self.background(True)) + self.check_info = g.add_markdown("") + with g.add_folder("Advanced schema repair"): + self.schema = g.add_text("Schema JSON", "", multiline=True) self.button( - "Play / pause", + "Copy draft into repair field", lambda: setattr( - self.mode, + self.schema, "value", - "PAUSED" if self.mode.value == "LIVE" else "LIVE", + json.dumps(storage.metadata(self.d), indent=2), ), ) - self.source = g.add_dropdown( - "Visualization metadata", ("Received", "Draft") - ) - self.force_max = g.add_number( - "Force color maximum (N)", 30.0, min=0.001 - ) - self.groups = {name: g.add_checkbox(name, True) for name in GROUPS} - self.button("Record metadata groups", self.record) - self.live_info = g.add_markdown("") - with tabs.add_tab("Export"): - self.paths = { - s: g.add_text( - s + " save JSON", state.paths.get(s, s + ".metadata.json") - ) - for s in ("left", "right") - } - self.button("Save all metadata", self.save) - self.all_sides = g.add_checkbox("Check / export all hands", True) - self.destination = g.add_text("New output directory", "package") - self.binary = g.add_checkbox("Include SDK binary", False) - self.button("Local check", self.local_check) - self.button("SDK check", lambda: self.background(False)) - self.button("Export snapshot", lambda: self.background(True)) - self.check_info = g.add_markdown("") - with g.add_folder("Advanced schema repair"): - self.schema = g.add_text("Schema JSON", "", multiline=True) - self.button( - "Copy draft into repair field", - lambda: setattr( - self.schema, - "value", - json.dumps(storage.metadata(self.d), indent=2), - ), - ) - self.button("Replace draft from repair field", self.repair) + self.button("Replace draft from repair field", self.repair) with g.add_folder("Draft property inspector"): self.entity = g.add_dropdown("Edit entity", ("Taxels", "Segment")) self.watch(self.entity, lambda _: None) From 9863a24bde54451a29bd5122b83679923d5120ea Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 12:17:56 -0700 Subject: [PATCH 07/36] minor UI cleanup Signed-off-by: Lotus Li --- .../tactile_metadata_editor/ui.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index f14015cc60..2b5fd8d5ed 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -61,7 +61,6 @@ def __init__(self, state, args): g = self.server.gui g.configure_theme(control_width="medium", show_share_button=False) g.main_panel.dock_right() - self.header = g.add_markdown("") self.side = g.add_dropdown("Edit hand", tuple(state.documents)) self.both = g.add_checkbox("Show both hands", True) self.mode = g.add_dropdown("Mode", ("EDIT", "PREVIEW", "LIVE", "PAUSED")) @@ -112,7 +111,6 @@ def __init__(self, state, args): self.calibrate_button = self.button( "Preview / set wrist reference", self.calibrate ) - self.resources = g.add_markdown("") self.button("Reset cameras", self.camera) with tabs.add_tab("Segments"): self.joint = g.add_dropdown("URDF free joint", ("",)) @@ -309,7 +307,6 @@ def __init__(self, state, args): geo.Model(state.model["urdf"], state.model.get("packages", {})) ) except (OSError, ValueError) as exc: - self.resources.content = str(exc) self.log( "URDF unavailable; numerical draft editing remains available: " + str(exc) @@ -391,15 +388,6 @@ def install_model(self, model): j.name for j in model.joints ) or ("",) self.q = copy.deepcopy(self.state.model.get("q0", {})) - self.resources.content = ( - "Missing resources: " - + str(model.missing) - + "\n\n" - + "\n\n".join( - f"{j.name}: {j.parent} → {j.child}; {j.type}; limits={j.limit}; mimic={j.mimic}" - for j in model.urdf.robot.joints - ) - ) self.sides.disabled = True self.sides.value = "both" if len(self.state.documents) == 2 else self.side.value self.joint_controls() @@ -423,7 +411,7 @@ def reset(self): {}, {}, ) - self.q, self.resources.content = {}, "" + self.q = {} for w in ( self.wrist, self.joint, @@ -972,7 +960,6 @@ def refresh(self): self.angle.disabled = self.angle_number.disabled = self.model is None self.selected = [i for i in self.selected if 0 <= i < len(d.arrays[FIELDS[1]])] self.side.visible = self.both.visible = len(self.state.documents) == 2 - self.header.content = f"{Path(self.state.model.get('urdf', 'No URDF')).name}; hands: {', '.join(self.state.documents)}; {self.mode.value}\n\n{self.state.paths.get(d.side, 'Unsaved draft')}{' *' if self.state.dirty() else ''}; revision {self.state.revision}" self.segment.options = self.segment_options() links = d.editor.get("segment_links", {}) self.segment_table.content = "\n\n".join( @@ -1547,7 +1534,7 @@ def run(self): last_error = None time.sleep(1 / 30) except KeyboardInterrupt: - self.guard(lambda: setattr(self, "closing", True)) + self.closing = True except Exception as exc: message = f"{type(exc).__name__}: {exc}" if message != last_error: From 9aa956204111d8f827ebf98de64d914fcf201584 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 12:18:50 -0700 Subject: [PATCH 08/36] Open to remote connection Signed-off-by: Lotus Li --- scripts/tactile_metadata/DESIGN.md | 2 +- scripts/tactile_metadata/tactile_metadata_editor/ui.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/tactile_metadata/DESIGN.md b/scripts/tactile_metadata/DESIGN.md index 70f6a3110a..2ecbaca844 100644 --- a/scripts/tactile_metadata/DESIGN.md +++ b/scripts/tactile_metadata/DESIGN.md @@ -184,7 +184,7 @@ python -m tactile_metadata_editor.cli check left.metadata.json --sdk python -m tactile_metadata_editor.cli export left.metadata.json --sides left --format sdk --out package ``` -GUI 不带参数时进入启动页，默认监听 127.0.0.1:8080，可用 --port 修改。JSON 不携带手侧或 URDF 配置：GUI 通过界面或 `--sides` 明确手侧，CLI export 通过 `--sides left/right/both` 分配输入文件，手侧重复明确拒绝；纯数值 CLI 检查不需要手侧。检查结果输出到界面或 CLI 标准输出，不提供 `--report` 文件输出。实时连接在当前 GUI 会话中配置；`--demo` 使用内置合成数据验证实时交互，不是额外文件导入。 +GUI 不带参数时进入启动页，默认监听 0.0.0.0:8080，可用 --port 修改。JSON 不携带手侧或 URDF 配置：GUI 通过界面或 `--sides` 明确手侧，CLI export 通过 `--sides left/right/both` 分配输入文件，手侧重复明确拒绝；纯数值 CLI 检查不需要手侧。检查结果输出到界面或 CLI 标准输出，不提供 `--report` 文件输出。实时连接在当前 GUI 会话中配置；`--demo` 使用内置合成数据验证实时交互，不是额外文件导入。 ## 2 依赖 diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 2b5fd8d5ed..90c852aaf0 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -57,7 +57,7 @@ def __init__(self, state, args): False, ) self.sdk_result, self.sdk_revision = "not checked", None - self.server = viser.ViserServer(host="127.0.0.1", port=args.port) + self.server = viser.ViserServer(host="0.0.0.0", port=args.port) g = self.server.gui g.configure_theme(control_width="medium", show_share_button=False) g.main_panel.dock_right() From 60cea9d1f3ede2c20c9b7a3960410dfc1a5eec88 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 12:21:54 -0700 Subject: [PATCH 09/36] Start at initial camera pose Signed-off-by: lotusl --- scripts/tactile_metadata/tactile_metadata_editor/ui.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 90c852aaf0..28073a62d9 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -34,6 +34,8 @@ def common_number(values): class Editor: + INITIAL_CAMERA_POSE = ((0.3, -0.4, 0.3), (0, 0, 0)) + def __init__(self, state, args): self.state, self.args, self.started = state, args, bool(state.documents) if not state.documents: @@ -58,6 +60,10 @@ def __init__(self, state, args): ) self.sdk_result, self.sdk_revision = "not checked", None self.server = viser.ViserServer(host="0.0.0.0", port=args.port) + ( + self.server.initial_camera.position, + self.server.initial_camera.look_at, + ) = self.INITIAL_CAMERA_POSE g = self.server.gui g.configure_theme(control_width="medium", show_share_button=False) g.main_panel.dock_right() @@ -513,7 +519,7 @@ def set_links(self): def camera(self): for client in self.server.get_clients().values(): - client.camera.position, client.camera.look_at = (0.3, -0.4, 0.3), (0, 0, 0) + client.camera.position, client.camera.look_at = self.INITIAL_CAMERA_POSE def calibrate(self): o, f, d = [ From cc6d246fdcafef2b00382783d4e074e27bbbb867 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 17:38:18 -0700 Subject: [PATCH 10/36] Align coordinates, add coordinate indicator, prevent tactile from blocking live control, etc Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 5 + scripts/tactile_metadata/DESIGN.md | 9 +- scripts/tactile_metadata/README.md | 12 +- .../tactile_metadata_editor/document.py | 41 ++++- .../tactile_metadata_editor/geometry.py | 17 +- .../tactile_metadata_editor/sdk.py | 6 +- .../tactile_metadata_editor/ui.py | 101 +++++++----- .../tactile_metadata_editor/viewport_axes.js | 148 ++++++++++++++++++ 8 files changed, 290 insertions(+), 49 deletions(-) create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index df475a6c09..586e4cbca1 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -30,3 +30,8 @@ SPDX-License-Identifier: Apache-2.0 - Keep the GUI a thin adapter over shared document, geometry, storage and SDK operations. Prefer explicit native Viser widgets and one reusable property inspector; avoid widget-type inference, duplicate field editors in dialogs, and rebuilding scene nodes on every frame. Moving logic between files alone does not reduce implementation size. - Use fixed URDF opacity and color-based link highlighting; do not add per-link visibility or opacity controls or state. - Under a hard code budget, finish and count a complete executable slice before expanding the UI. Refactoring dispatch or moving functions is not evidence that the implementation fits; verify the formatted total and retain budget for integration fixes. +- Treat metadata wrist coordinates, Viser scene coordinates and camera screen orientation as distinct; diagnose an upside-down display from the scene/camera mapping, not URDF alignment. +- Separate OpenXR schema/reference-space contracts from provider basis conversion; verify the plugin conversion before claiming injected HandPose follows OpenXR joint axes. +- Rebase both the raw live skeleton and metadata deformation by the same full inverse WRIST pose; never reattach the world-space wrist transform when the live view is wrist-relative. +- Render viewport direction indicators in client-side screen space from the render camera; perspective scene nodes and server camera callbacks introduce distortion, delay and jumps. +- When reporting a captured protocol frame, include every row in protocol order; a summary may follow but must not replace the requested raw frame. diff --git a/scripts/tactile_metadata/DESIGN.md b/scripts/tactile_metadata/DESIGN.md index 2ecbaca844..5eabcda5e8 100644 --- a/scripts/tactile_metadata/DESIGN.md +++ b/scripts/tactile_metadata/DESIGN.md @@ -43,7 +43,7 @@ URDF 路径、package 映射、q0、腕部校准、segment 与链接关联、手 └──────────────────────────────────┴──────────────────────────────┘ ``` -模型为中性色，所有 URDF 链接使用固定不透明度（opacity）0.6，高亮只改变颜色。编辑时 taxel 按主影响关节着色，无绑定为灰色，选中轮廓为黄色，错误轮廓为红色。实时触觉用力向量模长映射蓝至红，默认范围 0–30 N，可调整上限，超限饱和；无有效数据为灰色，不能显示成零力。属性面板显示 fx、fy、fz 和模长，单位 N。 +三维场景采用右手坐标约定：+X 向左、+Y 向上、+Z 向前，长度单位为米。模型为中性色，所有 URDF 链接使用固定不透明度（opacity）0.6，高亮只改变颜色。编辑时 taxel 按主影响关节着色，无绑定为灰色，选中轮廓为黄色，错误轮廓为红色。实时触觉用力向量模长映射蓝至红，默认范围 0–30 N，可调整上限，超限饱和；无有效数据为灰色，不能显示成零力。属性面板显示 fx、fy、fz 和模长，单位 N。 球形标记大小只影响显示，不改变 sensing location。选中 taxel 显示 +Z=外向法线 N、+X=传感器切线 T、+Y=N×T；X×Y=Z。双手使用同一右手坐标约定。多关节 LBS 动画只保证位置，不把混合旋转解释为协议定义的唯一方向。 @@ -67,7 +67,7 @@ URDF 路径、package 映射、q0、腕部校准、segment 与链接关联、手 ### 1.4 模型页 -页面提供 URDF 路径、package:// 的本地目录映射、资源状态。可检查模型树、关节类型与范围；缺失资源列出具体路径，不从网络寻找模型。 +页面提供 URDF 路径、package:// 的本地目录映射、资源状态。URDF 初始映射将其 +Z、+Y、+X 分别映射到场景 +Y、+X、+Z。可检查模型树、关节类型与范围；缺失资源列出具体路径，不从网络寻找模型。 用户为每侧选择腕部链接，并拾取腕中心 O、朝前臂的参考 F、朝手背的参考 D，建立来源根坐标系。双手分别校准，界面显示两套参考轴，不按左/右自动变号。关联链接限制该侧拾取范围；共享掌臂链接可以作为参考，但另一侧表面不能成为默认拾取对象。 @@ -77,7 +77,7 @@ URDF 路径、package 映射、q0、腕部校准、segment 与链接关联、手 绑定姿态组提供 URDF 自由关节滑块、数值输入和“设为绑定姿态”。fixed 不可编辑，mimic 跟随主关节，限位始终可见。此姿态 q0 是静态布局和 bind poses 的共同参考；实时角度及运动预览角度不会自动写入 q0。 -Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_JOINTS。表格显示语义名称、输出行号、关联 URDF 链接和参考系状态。合法 metadata 每侧同名语义只出现一次，不要求完整骨架，WRIST 使用根恒等位姿；草稿违反这些要求时显示错误并保留值。选择一行会高亮关节球、坐标轴和关联链接。 +Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_JOINTS。新草稿以腕部为根，在中性张开姿态中初始化全部 26 个 OpenXR 手部关节；左右手使用镜像姿态。初始显示映射将手部 -Z、+X、+Y 分别映射到场景 +Y、-X、-Z。表格显示语义名称、输出行号、关联 URDF 链接和参考系状态。合法 metadata 每侧同名语义只出现一次，不要求完整骨架，WRIST 使用根恒等位姿；草稿违反这些要求时显示错误并保留值。选择一行会高亮关节球、坐标轴和关联链接。 新建 segment 时选择语义和链接，在模型上定位关节中心及参考方向，通过控制器调整位置与扭转。已有 metadata 的 B_j 已给出坐标：选择目标链接后计算固定局部修正，保持原 B_j，而不是把数值吸附到 URDF 原点。 @@ -131,7 +131,7 @@ Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_ 实时 URDF 只能由启用的 JointState 输入驱动；未启用时隐藏 URDF，仅显示手骨架和 taxel。HandPose 不驱动 URDF，只提供实际手骨架和实时 taxel LBS 当前段位姿。EDIT/PREVIEW 中的 q0 编辑和手动运动预览不受此实时规则影响。 -- JointState 映射表：源关节名、URDF 自由关节、比例、偏移。 +- JointState 映射 JSON 文本框：`{源关节名: [URDF 自由关节, 比例, 偏移]}`，整表应用。 - 各关节的原始输入值、换算值、限位与映射状态。 **连接与播放** @@ -144,6 +144,7 @@ Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_ - Skinning：`taxel_segment_indices`、`taxel_segment_weights`。 - 显式记录前先按第 1.1 节的相同 schema JSON 格式将当前 metadata 数值备份到临时位置，再替换所选分组；日志显示备份路径。 - 三维可视化 metadata 来源切换：接收的 metadata／用户草稿，默认接收来源。所有页签中的 metadata 数值始终来自草稿。 +- 接收 metadata 缺失或无效时，HandPose 与 JointState 继续更新，三维 metadata 临时使用草稿并显示原因。 - 连接状态。各数据源的接入类型、配置标识、有效性和本地读取时间。 - 工具操作事件、警告和错误日志，包括替换前的草稿备份路径。 diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 825b2b8cfc..9f726cfbab 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -47,12 +47,12 @@ system; see the [tactile device documentation](../../docs/source/device/tactile/ mimic joints appear in the tree; mimic joints follow their source. Confirm **Set shared binding pose q0**. Existing bind poses and weights remain numerically unchanged when q0 or the wrist reference changes. -4. Add `WRIST` with the wrist link; its bind pose is identity. Add, for example, - `INDEX_DISTAL` with `index_distal`. New non-wrist segments start at the URDF - link frame when calibrated. Pick a segment center or use the native gizmo and - inspector to adjust its frame. URDF frames are initial estimates: calibrate - their axes to the HandJoint convention before production use. Associating an - imported segment with a link preserves its numerical bind pose. +4. New drafts contain all OpenXR hand joints in a neutral open pose, with `WRIST` + at the identity. Associate segments with URDF links, then pick a segment center + or use the native gizmo and inspector to adjust its frame. URDF frames are + initial estimates: calibrate their axes to the HandJoint convention before + production use. Associating an imported segment with a link preserves its + numerical bind pose. 5. In **Layout**, select the intended connected surface component. Set rows and columns to `4`, and enter the new taxels' segment row list, for example `[1]`. Pick the first center `O`, first-row last center `C`, and first-column last diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index ef416ecc98..9831286441 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -24,6 +24,37 @@ "Skinning": FIELDS[2:], } IDENTITY = [0, 0, 0, 0, 0, 0, 1] +# Neutral right-hand OpenXR poses in wrist coordinates; left is mirrored below. +# fmt: off +DEFAULT_OPENXR_SEGMENT_POSES = ( + (-0.00244, 0.00122, -0.05917, 0.010296699, 0.02060433, -0.000212213, 0.999734662), + (0, 0, 0, 0, 0, 0, 1), + (-0.022, -0.012, -0.028, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.047208158, -0.020249943, -0.065583071, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.065292271, -0.02616838, -0.09254484, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.080088363, -0.031010737, -0.114604468, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.00528, 0, -0.02068, 0, 0.124664128, 0, 0.992199), + (-0.024, 0, -0.094, -0.017438446, 0.039983244, 0.00069791, 0.999047924), + (-0.027194641, -0.00139598, -0.133847779, -0.043584497, 0.039951273, 0.00174431, 0.99824909), + (-0.029105295, -0.003487718, -0.157679985, -0.060999707, 0.039914746, 0.00244129, 0.997336397), + (-0.030691675, -0.005925105, -0.177467419, -0.060999707, 0.039914746, 0.00244129, 0.997336397), + (-0.00088, 0.00044, -0.02134, 0.010296699, 0.02060433, -0.000212213, 0.999734662), + (-0.004, 0.002, -0.097, -0.017452406, 0, 0, 0.999847695), + (-0.004, 0.000429523, -0.141972587, -0.043619387, 0, 0, 0.999048222), + (-0.004, -0.002010838, -0.169866039, -0.06104854, 0, 0, 0.998134798), + (-0.004, -0.004691964, -0.191702054, -0.06104854, 0, 0, 0.998134798), + (0.0033, 0, -0.02002, 0, -0.081592271, 0, 0.996665792), + (0.015, 0, -0.091, -0.017444553, -0.029990932, -0.000523494, 0.999397797), + (0.017516954, -0.001465779, -0.132898883, -0.04359976, -0.029966951, -0.001308385, 0.998598684), + (0.019070086, -0.003731828, -0.158753338, -0.06102107, -0.029939553, -0.001831181, 0.997685671), + (0.020319944, -0.006291084, -0.1795593, -0.06102107, -0.029939553, -0.001831181, 0.997685671), + (0.00704, -0.00088, -0.01804, -0.022312091, -0.184914996, -0.004199362, 0.982492229), + (0.032, -0.004, -0.082, -0.017415551, -0.064944346, -0.001133608, 0.99773626), + (0.036145766, -0.005116784, -0.113710652, -0.043527274, -0.064892417, -0.002833264, 0.996938475), + (0.038599442, -0.006772743, -0.132478637, -0.06091962, -0.064833086, -0.003965361, 0.996026981), + (0.040915463, -0.008966391, -0.150193714, -0.06091962, -0.064833086, -0.003965361, 0.996026981), +) +# fmt: on def joint_number(joint): @@ -61,10 +92,18 @@ def row_error(field, values, segments): class Document: def __init__(self, side=None): - self.side, self.joints = side, [] + self.side = side + self.joints = JOINTS[:] if side in ("left", "right") else [] self.arrays = {field: [] for field in FIELDS} + if self.joints: + poses = [list(pose) for pose in DEFAULT_OPENXR_SEGMENT_POSES] + if side == "left": + for pose in poses: + pose[0], pose[4], pose[5] = -pose[0], -pose[4], -pose[5] + self.arrays[FIELDS[0]] = poses self.types = ("LBS", "HandSkeletonRef") self.editor = {"segment_ids": [], "output_indices": []} + self.sync() def sync(self): count, ids = len(self.arrays[FIELDS[0]]), self.editor["segment_ids"] diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index adee21ee1e..cbe576a06a 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -12,6 +12,17 @@ from .document import FIELDS, joint_number, row_error +# URDF +X/+Y/+Z to visualization +Z/+X/+Y. +VISUALIZATION_FROM_URDF = np.array( + [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 0, 1]], + dtype=float, +) +# Hand +X/+Y/-Z to visualization -X/-Z/+Y. +VISUALIZATION_FROM_HAND = np.array( + [[-1, 0, 0, 0], [0, 0, -1, 0], [0, -1, 0, 0], [0, 0, 0, 1]], + dtype=float, +) + def matrix(p): try: @@ -98,7 +109,11 @@ def fk(self, cfg): self.urdf.update_cfg( dict(zip((j.name for j in self.joints), configuration)) ) - result = {link: self.urdf.get_transform(link).copy() for link in self.links} + result = { + link: VISUALIZATION_FROM_URDF + @ self.urdf.get_transform(link).copy() + for link in self.links + } self._fk_cache[configuration] = result if len(self._fk_cache) > 4: self._fk_cache.popitem(last=False) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py index 3e1ae716d5..f4d050114d 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py @@ -174,7 +174,11 @@ def live(connection, configs): forces=data.taxel_forces.copy().tolist() if data is not None else None, - hand=owned(h, ("positions", "orientations", "is_valid")), + # Bulk hand views belong to the nested HandJoints object. + hand=owned( + h.joints if h is not None else None, + ("positions", "orientations", "is_valid"), + ), joints={v.name: [v.position, v.valid] for v in j.joints} if j is not None else {}, diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 28073a62d9..7d148cb3d8 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -15,6 +15,7 @@ import trimesh import viser from trimesh import transformations as tf +from viser import _messages from . import geometry as geo, sdk, storage from .document import Document, FIELDS, GROUPS, IDENTITY, JOINTS, State, joint_number @@ -34,7 +35,7 @@ def common_number(values): class Editor: - INITIAL_CAMERA_POSE = ((0.3, -0.4, 0.3), (0, 0, 0)) + INITIAL_CAMERA_POSE = ((0.3, 0.3, -0.4), (0, 0.08, 0)) def __init__(self, state, args): self.state, self.args, self.started = state, args, bool(state.documents) @@ -59,7 +60,14 @@ def __init__(self, state, args): False, ) self.sdk_result, self.sdk_revision = "not checked", None + if viser.__version__ != "1.1.1": + raise RuntimeError("Viewport axes require viser 1.1.1") + viewport_axes = Path(__file__).with_name("viewport_axes.js").read_text() self.server = viser.ViserServer(host="0.0.0.0", port=args.port) + self.server.scene.set_up_direction("+y") + self.server._websock_server.queue_message( + _messages.RunJavascriptMessage(source=viewport_axes) + ) ( self.server.initial_camera.position, self.server.initial_camera.look_at, @@ -219,13 +227,12 @@ def __init__(self, state, args): self.button( "Apply this hand's source configuration", self.configure_source ) - self.source_joint = g.add_text("Source joint name", "") - self.target_joint = g.add_dropdown("URDF target joint", ("",)) - self.scale = g.add_number("Scale", 1.0) - self.offset = g.add_number("Offset", 0.0) - self.button("Add / update mapping", self.mapping) - self.button("Remove mapping", lambda: self.mapping(True)) - self.mapping_info = g.add_markdown("") + self.mapping_json = g.add_text( + "Joint mappings JSON: source → [target, scale, offset]", + "{}", + multiline=True, + ) + self.button("Apply joint mappings", self.mapping) self.button("Connect / disconnect", self.connect) self.button( "Play / pause", @@ -390,9 +397,7 @@ def install_model(self, model): h.remove() self.model = model self.wrist.options = self.segment_link.options = tuple(model.links) - self.joint.options = self.target_joint.options = tuple( - j.name for j in model.joints - ) or ("",) + self.joint.options = tuple(j.name for j in model.joints) or ("",) self.q = copy.deepcopy(self.state.model.get("q0", {})) self.sides.disabled = True self.sides.value = "both" if len(self.state.documents) == 2 else self.side.value @@ -422,7 +427,6 @@ def reset(self): self.wrist, self.joint, self.segment_link, - self.target_joint, self.surface, ): w.options = ("",) @@ -488,6 +492,9 @@ def switch(self): ) self.joint_id.value = source.get("joint_state", "") self.joint_enabled.value = bool(self.joint_id.value) + self.mapping_json.value = json.dumps( + self.d.editor.get("mapping", {}), indent=2 + ) self.segment.options = self.segment_options() def root(self, d=None, cfg=None): @@ -502,7 +509,7 @@ def root(self, d=None, cfg=None): ) except (KeyError, ValueError, TypeError): pass - return np.eye(4) + return geo.VISUALIZATION_FROM_HAND.copy() def surfaces(self): return [ @@ -836,24 +843,25 @@ def configure_source(self): if self.receiver: self.log("Source configuration updated; reconnect to apply it") - def mapping(self, remove=False): - mappings = dict(self.d.editor.get("mapping", {})) - if remove: - mappings.pop(self.source_joint.value, None) - else: - if ( - not self.source_joint.value - or not self.target_joint.value - or not all( - math.isfinite(v) for v in (self.scale.value, self.offset.value) - ) - ): - raise ValueError("Choose source/target joints and finite scale/offset") - mappings[self.source_joint.value] = [ - self.target_joint.value, - self.scale.value, - self.offset.value, - ] + def mapping(self): + mappings = json.loads(self.mapping_json.value) + targets = {joint.name for joint in self.model.joints} if self.model else set() + if not isinstance(mappings, dict) or any( + not isinstance(source, str) + or not source + or not isinstance(values, list) + or len(values) != 3 + or not isinstance(values[0], str) + or values[0] not in targets + or any( + type(value) not in (int, float) or not math.isfinite(value) + for value in values[1:] + ) + for source, values in mappings.items() + ): + raise ValueError( + "Expected {source: [URDF target joint, finite scale, finite offset]}" + ) self.change(self.d.editor.update, mapping=mappings) def connect(self): @@ -1050,7 +1058,6 @@ def refresh(self): f"{i}: {v['side']} {v['severity']} {v['field']}[{v['row']}]: {v['message']}" for i, v in enumerate(self.issues) ) or ("No local errors",) - self.mapping_info.content = str(d.editor.get("mapping", {})) self.check_info.content = f"Local revision {self.state.revision}: {sum(v['severity'] == 'error' for v in self.issues)} errors. SDK/export revision {self.sdk_revision}: {self.sdk_result}{' (stale)' if self.sdk_revision is not None and self.sdk_revision != self.state.revision else ''}" def locate(self): @@ -1341,11 +1348,33 @@ def draw(self): root, d, current, skeleton = self.root(draft, cfg), draft, None, [] issues = [v for v in self.issues if v["side"] == side] if mode == "LIVE": + current, joints, world_to_wrist = geo.hand_current( + draft, value["hand"] + ) + root = geo.VISUALIZATION_FROM_HAND.copy() + skeleton = [ + root @ world_to_wrist @ t + for t in joints + if t is not None + ] if self.source.value == "Received": - d, issues = self.received_document(side, value["metadata"]) - current, joints, _ = geo.hand_current(d, value["hand"]) - root = joints[1] - skeleton = [t for t in joints if t is not None] + try: + received, received_issues = self.received_document( + side, value["metadata"] + ) + received_current, _, _ = geo.hand_current( + received, value["hand"] + ) + except (ValueError, TypeError, KeyError, IndexError) as exc: + status.append( + f"{side}: received metadata unavailable; using draft: {exc}" + ) + else: + d, issues, current = ( + received, + received_issues, + received_current, + ) elif mode == "PREVIEW" and self.model and "root" in draft.editor: current = self.model.current(draft, self.model.fk(cfg)) self.render_side( diff --git a/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js b/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js new file mode 100644 index 0000000000..b0a100a7fb --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +(() => { + "use strict"; + + const key = "__tactileMetadataViewportAxes"; + globalThis[key]?.stop(); + + const ns = "http://www.w3.org/2000/svg"; + const svg = document.createElementNS(ns, "svg"); + Object.assign(svg.style, { + position: "absolute", + top: "16px", + left: "16px", + width: "112px", + height: "112px", + zIndex: "10", + pointerEvents: "none", + overflow: "visible", + }); + svg.setAttribute("viewBox", "0 0 112 112"); + + const background = document.createElementNS(ns, "circle"); + background.setAttribute("cx", "56"); + background.setAttribute("cy", "56"); + background.setAttribute("r", "49"); + background.setAttribute("fill", "rgba(255,255,255,0.72)"); + background.setAttribute("stroke", "rgba(0,0,0,0.2)"); + svg.appendChild(background); + + const axes = [ + ["x", "#e53935", [1, 0, 0]], + ["y", "#43a047", [0, 1, 0]], + ["z", "#1e88e5", [0, 0, 1]], + ].map(([name, color, vector]) => { + const line = document.createElementNS(ns, "line"); + line.setAttribute("x1", "56"); + line.setAttribute("y1", "56"); + line.setAttribute("stroke", color); + line.setAttribute("stroke-width", "3"); + line.setAttribute("stroke-linecap", "round"); + svg.appendChild(line); + + const end = document.createElementNS(ns, "circle"); + end.setAttribute("r", "4"); + end.setAttribute("stroke", color); + end.setAttribute("stroke-width", "2"); + svg.appendChild(end); + + const label = document.createElementNS(ns, "text"); + label.textContent = name; + label.setAttribute("fill", color); + label.setAttribute("font-size", "15"); + label.setAttribute("font-family", "sans-serif"); + label.setAttribute("font-weight", "700"); + label.setAttribute("text-anchor", "middle"); + label.setAttribute("dominant-baseline", "central"); + svg.appendChild(label); + return { color, end, label, line, vector }; + }); + + const multiply = (a, b) => [ + a[3] * b[0] + a[0] * b[3] + a[1] * b[2] - a[2] * b[1], + a[3] * b[1] - a[0] * b[2] + a[1] * b[3] + a[2] * b[0], + a[3] * b[2] + a[0] * b[1] - a[1] * b[0] + a[2] * b[3], + a[3] * b[3] - a[0] * b[0] - a[1] * b[1] - a[2] * b[2], + ]; + + const rotate = (q, v) => { + const tx = 2 * (q[1] * v[2] - q[2] * v[1]); + const ty = 2 * (q[2] * v[0] - q[0] * v[2]); + const tz = 2 * (q[0] * v[1] - q[1] * v[0]); + return [ + v[0] + q[3] * tx + q[1] * tz - q[2] * ty, + v[1] + q[3] * ty + q[2] * tx - q[0] * tz, + v[2] + q[3] * tz + q[0] * ty - q[1] * tx, + ]; + }; + + const state = { + controls: null, + raf: 0, + stopped: false, + stop() { + this.stopped = true; + cancelAnimationFrame(this.raf); + this.controls?.removeEventListener("update", draw); + svg.remove(); + }, + }; + globalThis[key] = state; + + const draw = () => { + const mutable = globalThis.__viserMutable; + const camera = mutable?.camera; + const root = mutable?.nodeRefFromName?.[""]; + const parent = mutable?.canvas?.parentElement; + if (!camera || !root || !parent) return; + if (svg.parentElement !== parent) parent.appendChild(svg); + + const cameraQuaternion = camera.quaternion; + const rootQuaternion = root.quaternion; + const cameraFromWorld = [ + -cameraQuaternion.x, + -cameraQuaternion.y, + -cameraQuaternion.z, + cameraQuaternion.w, + ]; + const cameraFromVisualization = multiply(cameraFromWorld, [ + rootQuaternion.x, + rootQuaternion.y, + rootQuaternion.z, + rootQuaternion.w, + ]); + + for (const [index, axis] of axes.entries()) { + const direction = rotate(cameraFromVisualization, axis.vector); + const x = 56 + 36 * direction[0]; + const y = 56 - 36 * direction[1]; + const projected = Math.hypot(direction[0], direction[1]); + const fallback = [[10, -8], [-10, -8], [0, 12]][index]; + const labelOffset = + projected > 0.12 + ? [8 * direction[0] / projected, -8 * direction[1] / projected] + : fallback; + axis.line.setAttribute("x2", x.toFixed(2)); + axis.line.setAttribute("y2", y.toFixed(2)); + axis.end.setAttribute("cx", x.toFixed(2)); + axis.end.setAttribute("cy", y.toFixed(2)); + axis.end.setAttribute("fill", direction[2] >= 0 ? axis.color : "white"); + axis.label.setAttribute("x", (x + labelOffset[0]).toFixed(2)); + axis.label.setAttribute("y", (y + labelOffset[1]).toFixed(2)); + } + }; + + const frame = () => { + if (state.stopped) return; + const controls = globalThis.__viserMutable?.cameraControl; + if (controls !== state.controls) { + state.controls?.removeEventListener("update", draw); + controls?.addEventListener("update", draw); + state.controls = controls; + } + draw(); + state.raf = requestAnimationFrame(frame); + }; + frame(); +})(); From 57a323737828579de54f9427d8c7794a0db66678 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Tue, 27 Oct 2026 20:03:19 -0700 Subject: [PATCH 11/36] Fix Avatar hand coordinate basis Signed-off-by: Lotus Li --- .../sharpa_avatar/core/avatar_hand_tracking_plugin.cpp | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp b/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp index 9a7df1454d..32fd6ade2d 100644 --- a/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp +++ b/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp @@ -203,6 +203,12 @@ constexpr std::array, XR_HAND_JOINT_COUNT_EXT> kOpenXRSlot 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, std::nullopt, std::nullopt, 24, }; +// Avatar HUMAN uses +Z toward the fingertips and -X toward the back of the hand. +// OpenXR uses -Z toward the fingertips and +Y toward the back of the hand. +constexpr float kInvSqrt2 = 0.70710678f; +constexpr XrPosef kAvatarToOpenXR = { { kInvSqrt2, -kInvSqrt2, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }; +constexpr XrPosef kOpenXRToAvatar = { { -kInvSqrt2, kInvSqrt2, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }; + } // anonymous namespace // Wrist offsets for XR controllers: align the controller aim pose to a natural @@ -685,7 +691,9 @@ void AvatarTracker::map_landmarks_to_openxr(const std::vector<::avatar::Pose>& l local_pose.orientation.z = lp.orientation.z; local_pose.orientation.w = lp.orientation.w; - out_joints[j].pose = oxr_utils::multiply_poses(root_pose, local_pose); + const XrPosef openxr_pose = + oxr_utils::multiply_poses(kAvatarToOpenXR, oxr_utils::multiply_poses(local_pose, kOpenXRToAvatar)); + out_joints[j].pose = oxr_utils::multiply_poses(root_pose, openxr_pose); out_joints[j].radius = 0.01f; out_joints[j].locationFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; From a1138932a9c20f1178efd2b65891dbdfae73b69e Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 28 Oct 2026 11:15:41 -0700 Subject: [PATCH 12/36] add urdf scale UI Signed-off-by: Lotus Li --- scripts/tactile_metadata/DESIGN.md | 2 +- .../tactile_metadata_editor/geometry.py | 8 +++++- .../tactile_metadata_editor/ui.py | 26 +++++++++++++++++-- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/scripts/tactile_metadata/DESIGN.md b/scripts/tactile_metadata/DESIGN.md index 5eabcda5e8..3f8fe0642f 100644 --- a/scripts/tactile_metadata/DESIGN.md +++ b/scripts/tactile_metadata/DESIGN.md @@ -67,7 +67,7 @@ URDF 路径、package 映射、q0、腕部校准、segment 与链接关联、手 ### 1.4 模型页 -页面提供 URDF 路径、package:// 的本地目录映射、资源状态。URDF 初始映射将其 +Z、+Y、+X 分别映射到场景 +Y、+X、+Z。可检查模型树、关节类型与范围；缺失资源列出具体路径，不从网络寻找模型。 +页面提供 URDF 路径、统一正比例缩放、package:// 的本地目录映射和资源状态。缩放同时作用于网格与关节平移，不改变关节角或 metadata 数值。URDF 初始映射将其 +Z、+Y、+X 分别映射到场景 +Y、+X、+Z。可检查模型树、关节类型与范围；缺失资源列出具体路径，不从网络寻找模型。 用户为每侧选择腕部链接，并拾取腕中心 O、朝前臂的参考 F、朝手背的参考 D，建立来源根坐标系。双手分别校准，界面显示两套参考轴，不按左/右自动变号。关联链接限制该侧拾取范围；共享掌臂链接可以作为参考，但另一侧表面不能成为默认拾取对象。 diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index cbe576a06a..ce95205390 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -66,7 +66,10 @@ def frames(rows, root): class Model: - def __init__(self, path, packages): + def __init__(self, path, packages, scale=1.0): + if not np.isfinite(scale) or scale <= 0: + raise ValueError("URDF scale must be finite and positive") + self.scale = float(scale) self.path, self.missing, self.surfaces = Path(path).resolve(), [], {} self.handles, self.attachments = {}, {} # Instance-owned caches must not keep replaced models alive. @@ -99,6 +102,7 @@ def resource(fname): transform, name = scene.graph.get(frame_to=node, frame_from=link) mesh = scene.geometry[name].copy() mesh.apply_transform(transform) + mesh.apply_scale(self.scale) for i, part in enumerate(mesh.split(only_watertight=False)): self.surfaces[f"{link}/{node}/{i}"] = (link, part) @@ -114,6 +118,8 @@ def fk(self, cfg): @ self.urdf.get_transform(link).copy() for link in self.links } + for transform in result.values(): + transform[:3, 3] *= self.scale self._fk_cache[configuration] = result if len(self._fk_cache) > 4: self._fk_cache.popitem(last=False) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 7d148cb3d8..cf80828620 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -87,6 +87,9 @@ def __init__(self, state, args): "Local package map JSON", json.dumps(state.model.get("packages", {})), ) + self.urdf_scale = g.add_number( + "URDF scale", state.model.get("scale", 1.0), min=1e-6 + ) self.sides = g.add_dropdown( "New model hands", ("left", "right", "both"), @@ -96,6 +99,7 @@ def __init__(self, state, args): ) self.button("New document", lambda: self.guard(self.reset)) self.button("Load URDF", lambda: self.guard(self.load_new)) + self.button("Apply URDF scale", self.scale_model) self.open_path = g.add_text("Metadata JSON", args.metadata or "") self.json_side = g.add_dropdown( "Assign JSON to", ("left", "right"), initial_value=self.side.value @@ -317,7 +321,11 @@ def __init__(self, state, args): if state.model.get("urdf"): try: self.install_model( - geo.Model(state.model["urdf"], state.model.get("packages", {})) + geo.Model( + state.model["urdf"], + state.model.get("packages", {}), + state.model.get("scale", 1.0), + ) ) except (OSError, ValueError) as exc: self.log( @@ -441,9 +449,10 @@ def load_new(self): k: str(Path(v).resolve()) for k, v in json.loads(self.packages.value).items() }, + scale=self.urdf_scale.value, q0={}, ) - model = geo.Model(cfg["urdf"], cfg["packages"]) + model = geo.Model(cfg["urdf"], cfg["packages"], cfg["scale"]) sides = ("left", "right") if self.sides.value == "both" else (self.sides.value,) self.reset() self.state, self.started = State({s: Document(s) for s in sides}, cfg), True @@ -451,6 +460,19 @@ def load_new(self): self.install_model(model) self.switch() + def scale_model(self): + if not self.model: + raise ValueError("Load a URDF before applying its scale") + scale = self.urdf_scale.value + model = geo.Model( + self.state.model["urdf"], + self.state.model.get("packages", {}), + scale, + ) + self.change(self.state.model.update, scale=scale) + self.install_model(model) + self.clear_layout() + def open(self): side, path = self.json_side.value, self.open_path.value d = storage.load(path, side) From cef60dc13fa7fb7be56ad3c0520c3b8ffc00e759 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 28 Oct 2026 16:55:07 -0700 Subject: [PATCH 13/36] Full design of UI and feature Signed-off-by: lotusl --- scripts/tactile_metadata/AGENTS.md | 1 + scripts/tactile_metadata/UI&Feature Design.md | 80 +++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 scripts/tactile_metadata/UI&Feature Design.md diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 586e4cbca1..56a212e73c 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -12,6 +12,7 @@ SPDX-License-Identifier: Apache-2.0 - 实现以本目录的 `DESIGN.md` 为依据；协议事实以仓库 schema、绑定和共享校验器为准。发现冲突时提供具体代码证据，并同步修正设计与实现提示词。 - 首版的手套规则留在手部适配层；左右配置独立，通用编辑状态不使用 `HandJoint` 枚举作为内部标识。 - 评审改变输入范围、术语或编辑实体时，同时更新界面、持久化、算法、验收与实现提示词；已取消的产品概念不能留作隐藏的数据模型。 +- 评审替代性 UI 规格时，以当前稿为产品范围，只把 schema/API 约束视为阻塞；不要把旧功能、状态增强或可接受的瞬态覆盖升级为要求，并逐字核对交互手势。 - 对设计文档的结构与表达要求应直接落实到目标文档；页面布局按区块列出具体控件，不以聊天中的补充说明代替文档修改。 - Keep URDF driving separate from taxel deformation: HandPose supplies skeleton and LBS poses, while live URDF motion uses JointState only. Remove obsolete driver controls, persisted options, algorithms, and acceptance cases together. - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md new file mode 100644 index 0000000000..34e3794d56 --- /dev/null +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -0,0 +1,80 @@ +UI&Feature Design + +Global standard: unless improper for specific reason, for all float or int input fields, enable up/down arrow to adjust the value. For float input field, the step should be determined by the estimated range of the value. For int input field, the step should be 1. Only when a field has a clear specific limit range, the input field should reject illegal values. For sliders, when there's a clear specific limit range, use the limit range. +* When "Save", check the validation of metadata. If has error, pop up warning window "Metadata contain errors, are you sure to save?" for user to select continue or cancel. + +For field names, use title case for multi-word field names. For example, "URDF Path" instead of "urdf path". + +About data cooridanates: all UI editor use visualization space. When save to metadata file, do the coordinate transformation with Segments Alignment value. When load from metadata file, use the default VISUALIZATION_FROM_HAND value as Segments Alignement (and do transformation). Modifying Segments Alignment will cause values in editors also update. + +About LBS taxel rotation: determine by Polar Decomposition algorithm. At singularity, use taxel's bind rotation. Make sure to optimize for single-segment binding case, where the rotation can be directly determined without running Polar Decomposition. + +1. At top, always display general function: Toggle side, new/save/load metadata file, reset camera. A Mode single seletion from: Edit, Preview. And a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". + +2. This tool actually manages 3 copies of pose data (pose of segments and taxels), with different data sources. + a) Editing dataset: the data that the user is editing in the tool. It's the binding poses of the segments and taxels under editing. Plus the URDF joint values that's under Editing mode. + b) Saved dataset: the binding data that the metadata file stores. This doesn't contain URDF joint values. + c) Preview dataset: the segments poses, and a set of URDF joint values. It's not a binding data and can't be saved to the metadata file. + +3. When toggle the "Show Saved Binding Poses" on, the 3D scene will display the saved binding poses of the segments and taxels, at a opacity of 0.2. + +4. Two modes use different source of pose data: When switch to Edit mode, the Editing dataset will be used to render in the 3D scene, at the opacity of 1.0. When switch to Preview mode, the Preview dataset will be used to render in the 3D scene, at the opacity of 1.0. + +6. Two modes use different UI menu: When switch to Edit mode, enable "URDF", "Segments", "Taxels" 3 menu tabs. When switch to Preview mode, there won't be tabs. These tabs will be defined explicitly at following. + +7. URDF tab: contains following fields: + a) URDF Path: a text field to input the absolute path of the URDF file. + b) "Load" button, to load the URDF file at URDF Path in scene. + c) URDF Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation, and one float input field to scale the URDF model. + c) A field to modify each joint value in the loaded URDF file. Current UI field "URDF free joint" and "Preview joint" provide this function, but I want you to use a different UI that shows a spannable full list of URDF joints and their input sliders for the user to slide. + +8. Segments tab: + a) Segments Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation. + b) Add new segments: input fields of segment name (HandJoint enum, dropdown selection), position (3 float input fields). An "Add with Input" button. + c) Add new segments by 3D scene interaction: a button to "Add with URDF Click" button. After clicking this button, the button shows a "pressed" shape, the 3D scene will grey out, with only the URDF (of the Editing dataset) highlighted. The user can click on the URDF to add a new segment on the URDF surface position. Clicking on any other area will cancel the operation. + d) (spannable) A full list of all segments, with an id number (read-only, starting at 0, it's the index of the segment in the list), their name (dropdown list of HandJoint, display in enum name), editable position (3 float input), editable rotation (3 float slider). This is to modify the binding poses of the segments, display this info above this section. Each segment has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. +* It's invalid to have more than one segments to select the same HandJoint. At error, outline the repeated fields in red. +* Some fields in Taxels tab refer to segments in this Segments tab, and modifying segments may result in invalid taxel binding that's revealed in Taxels tab. +* Segments are displayed as yellow spheres of radius 0.01m. The three axes of each segment are not displayed by default.Clicking an segment on the 3D scene while in Edit mode will select the segment, highlight it, reveal its 3 axes. When a segment is newly added, it's selected, and deselect the previous selected object(s). Clicking on any other area in the 3D scene will deselect the segment; dragging 3D scene or clicking on the menu/other UI area will not deselect the segment. Clicking on the same selected segment will not do anything. + +9. Taxels tab: +* Taxels don't have their onw alignment, it uses the alignment of the segments. + a) Add new taxels: input fields of columns (int input), rows (int input), intersection width (float input, declare the unit is m), height (float input, declare the unit is m), taxels center position (3 float input fields). An "Add with Input" button. When adding taxels, it will generate columns*rows taxels, which aligned in a grid of the inputted area width and height, with the grid center at the inputted taxels center position, the grid orientation facing +Z axis of visualization space. Each taxel's orientation is originally identity. + b) Add new taxels by 3D scene interaction: a button to "Add with URDF Click" button with columns and rows int input fields. After clicking this button, the button shows a "pressed" shape, the 3D scene will grey out, with only the URDF (of the Editing dataset) highlighted. The user can click on the URDF to add a new array of taxel on the URDF surface, this will use the clicked URDF position as grid center position, and the clicked URDF normal (pointing out of the URDF surface) as grid orientation. Otherwise the grid-adding rule is the same as Add with Input.Clicking on any other area will cancel the operation. When taxels are newly created, they're selected and the previous selected objects are de-selected. + c) A series of buttons "Start Binding to Segments" "Select Segments" "Complete". At first, only "Start Binding to Segments" is clicable. When clicking this button, 3D scene will grey out except for taxels (from Edit dataset) is displayed normally (without highlight), the instruction field under the button will print: "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. Click "Select Segments" after done selecting taxels. Click on any other area in 3D scene to cancel.". When at least one taxel is selected, the button "Select Segments" becomes clickcable. The 3D scene stays greyed out and only segments (from Edit dataset) are displayed normally (without highlight). The instruction field changes to print: "Select 1-4 segment(s) to bind to the selected taxel(s) in 3D scene. Hold Shift to multi-select. Click "Complete" after done selecting segments. Click on any other area in 3D scene to cancel.". When at least one segment is selected, the "Complete" button becomes clickable. Clicking "Complete" button will update the Edit dataset's binding, each original weight is average (if selected 4 segments, then 0.25 each; if selected 3 segments, then 1/3 each.) (also update the following section). If more than 4 segments are selected, don't update the binding, and print in instruction field: "Too many segments selected, please select 1-4 segments. Click "Start Binding to Segments" to start over.". +* Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. + d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. Each taxel has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. +* "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. +* Taxels data can be invalid for multiple reasons. When invalid, line the invalid field in red. Reasons including: + a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. + b) the weights don't sum to 1. (Can happen when a segment is removed from Segments tab or by Preview apply to editor.) - weights input fields. + c) the weight is 0 for a non-None segment - weight input field. +* Taxels are displayed as light grey flat square of size 0.01m*0.01m. The three axes of each taxel are not displayed by default. Clicking an taxel on the 3D scene while in Edit mode will select the taxel, highlight it, reveal its 3 axes. The 3D scene modification rules are the same as segments. +* Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. +* When multiple objects are selected, the 3-axis visualizer should show in the center of the selected objects' area, with axis directions align with the visualiation space coordinates. +* An additional small menu for selected object(s)' modification, apply also to single object selection: +a) (read only) selected elements type and count (multi selection is stated in following part) +b) (editable) current position (defined above) +c) (editable) current rotation (defined above) +d) (editable single float) scaling (only for multi-selection, scale up/down the selected objects' position in space, based on the area center. For example, if one object was at (1, 2, 3) and the area center is at (1, 1, 1), scaling the area up to 2x will move the object to (1, 3, 5).) +e) Delete + +10. Preview menu: + a) A selection field to select the Binding Metadata Source from "Live" and "Editing", by default "Editing". "Live" is only available after the first live metadata is received. This selection affects the taxels pose rendered, because it's calculated based on the segments poses and binding metadata. + b) A checkbox toggling "Show Binding Poses". When checked, show the segments and taxels (without URDF) from the Editing dataset in the 3D scene at a opacity of 0.5. + c) A text input field "Tactile Collection ID", default to "tactile". + d) A button "Apply Preview Data to Editor". When clicked, apply the Preview dataset's segment poses and deferred taxel poses (segment poses + selected metadata binding) and the binding relationships, to the Editing dataset's segment binding poses and taxel binding data. Also update the urdf joint values to Editing dataset. Remind the user that this action will fully override segments and taxels. +* When deffering taxel poses or applying binding relationships to the Edit dataset, segments should be referred to by their HandJoint enum name instead of indice. It means received Metadata requires one layer of translation to turn the binding indice into HandJoint. +* If the segment list is incompatible to the metadata's taxels binding (e.g. taxels refer to a segment that's not in list), an error should show up at the top of Preview menu including information like the missing segment enum name, and "Apply Preview Data to Editor" should grey out not clickable. +Live section: + a) Checkbox "Use JointState to Drive URDF". When checked, enable the "JointState collection id" input field, and a Joint Mappings text field to receive a long json string to map the joint names in JointState to the URDF. The current field "Joint mappings JSON: source → [target, scale, offset]" is reusable. + b) A button "Connect Live" button. When clicked, update the Preview dataset's segments poses with received hand data in real time, and update the taxels poses based on the segments poses and the binding metadata (selected in above selection field). Manage a field to clearly state the connection statue: Live or Disconnected. After clicking "Connect Live", the button changes text to "Disconnect Live". When disconnected, stops updating the segments poses or URDF joint values. (If JointState is used, Live mode also updates the URDF joint values.) Show warning: Live will override values in Static section. +* If the metadata +* While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. +Static section: + a) A button "Restore to Editor Binding Values" + b) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. + c) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Exclude the reordering grabber UI. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. +* Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. + +Button info display: show error messages at button when action fails. \ No newline at end of file From 7785426b7c1f03c1c70388e9372bca45e57fad50 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 28 Oct 2026 17:33:54 -0700 Subject: [PATCH 14/36] implment complete UI feature design Signed-off-by: Lotus Li --- scripts/tactile_metadata/UI&Feature Design.md | 16 +- .../model/left/linkerhand_FFG_left.urdf | 4 +- .../model/right/linkerhand_FFG_right.urdf | 4 +- .../right/meshes/linkerhand_FFG_right.urdf | 4 +- .../tactile_metadata_editor/cli.py | 94 +- .../tactile_metadata_editor/document.py | 440 +-- .../editor_interactions.js | 90 + .../tactile_metadata_editor/geometry.py | 506 ++- .../tactile_metadata_editor/sdk.py | 263 +- .../tactile_metadata_editor/storage.py | 246 +- .../tactile_metadata_editor/ui.py | 2799 ++++++++--------- .../tactile_metadata_editor/viewport_axes.js | 148 - 12 files changed, 2052 insertions(+), 2562 deletions(-) create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js delete mode 100644 scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index 34e3794d56..bee503c32a 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -1,3 +1,9 @@ + + UI&Feature Design Global standard: unless improper for specific reason, for all float or int input fields, enable up/down arrow to adjust the value. For float input field, the step should be determined by the estimated range of the value. For int input field, the step should be 1. Only when a field has a clear specific limit range, the input field should reject illegal values. For sliders, when there's a clear specific limit range, use the limit range. @@ -26,7 +32,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) URDF Path: a text field to input the absolute path of the URDF file. b) "Load" button, to load the URDF file at URDF Path in scene. c) URDF Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation, and one float input field to scale the URDF model. - c) A field to modify each joint value in the loaded URDF file. Current UI field "URDF free joint" and "Preview joint" provide this function, but I want you to use a different UI that shows a spannable full list of URDF joints and their input sliders for the user to slide. + c) A field to modify each joint value in the loaded URDF file. Current UI field "URDF free joint" and "Preview joint" provide this function, but I want you to use a different UI that shows a spannable full list of URDF joints and their input sliders for the user to slide. 8. Segments tab: a) Segments Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation. @@ -67,14 +73,14 @@ e) Delete * When deffering taxel poses or applying binding relationships to the Edit dataset, segments should be referred to by their HandJoint enum name instead of indice. It means received Metadata requires one layer of translation to turn the binding indice into HandJoint. * If the segment list is incompatible to the metadata's taxels binding (e.g. taxels refer to a segment that's not in list), an error should show up at the top of Preview menu including information like the missing segment enum name, and "Apply Preview Data to Editor" should grey out not clickable. Live section: - a) Checkbox "Use JointState to Drive URDF". When checked, enable the "JointState collection id" input field, and a Joint Mappings text field to receive a long json string to map the joint names in JointState to the URDF. The current field "Joint mappings JSON: source → [target, scale, offset]" is reusable. + a) Checkbox "Use JointState to Drive URDF". When checked, enable the "JointState collection id" input field, and a Joint Mappings text field to receive a long json string to map the joint names in JointState to the URDF. The current field "Joint mappings JSON: source → [target, scale, offset]" is reusable. b) A button "Connect Live" button. When clicked, update the Preview dataset's segments poses with received hand data in real time, and update the taxels poses based on the segments poses and the binding metadata (selected in above selection field). Manage a field to clearly state the connection statue: Live or Disconnected. After clicking "Connect Live", the button changes text to "Disconnect Live". When disconnected, stops updating the segments poses or URDF joint values. (If JointState is used, Live mode also updates the URDF joint values.) Show warning: Live will override values in Static section. -* If the metadata +* If the metadata * While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. Static section: a) A button "Restore to Editor Binding Values" b) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. c) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Exclude the reordering grabber UI. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. -* Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. +* Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. -Button info display: show error messages at button when action fails. \ No newline at end of file +Button info display: show error messages at bottom when action fails. diff --git a/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf b/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf index 128d40f2cb..bf2ef60d8b 100644 --- a/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf +++ b/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf @@ -1,5 +1,5 @@ - - \ No newline at end of file + diff --git a/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf b/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf index ac4a83918e..341ce5cbb4 100644 --- a/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf +++ b/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf @@ -1,5 +1,5 @@ - - \ No newline at end of file + diff --git a/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf b/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf index 8dbd52cd30..c7e4515b77 100644 --- a/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf +++ b/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf @@ -1,5 +1,5 @@ - - \ No newline at end of file + diff --git a/scripts/tactile_metadata/tactile_metadata_editor/cli.py b/scripts/tactile_metadata/tactile_metadata_editor/cli.py index 29f1a6c146..b8eceb9b3c 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/cli.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/cli.py @@ -3,103 +3,21 @@ from __future__ import annotations import argparse -import json import sys -from pathlib import Path -from . import storage -from .document import Document, State +from .document import State -def main(): +def main() -> int: parser = argparse.ArgumentParser(description="Local tactile metadata editor") - commands = parser.add_subparsers(dest="command", required=True) - gui = commands.add_parser("gui") - gui.add_argument("--urdf") - gui.add_argument("--sides", choices=("left", "right", "both")) - gui.add_argument("--metadata") - gui.add_argument("--left-metadata") - gui.add_argument("--right-metadata") - gui.add_argument("--port", type=int, default=8080) - gui.add_argument("--demo", action="store_true") - check = commands.add_parser("check") - check.add_argument("metadata") - check.add_argument("--sdk", action="store_true") - export = commands.add_parser("export") - export.add_argument("metadata", nargs="+") - export.add_argument("--sides", choices=("left", "right", "both")) - export.add_argument("--format", choices=("json", "sdk"), default="json") - export.add_argument("--out", required=True) + parser.add_argument("--port", type=int, default=8080) args = parser.parse_args() try: - if args.command == "check": - d = storage.load(args.metadata) - result = dict(local=d.check(), sdk="not checked") - failed = any(v["severity"] == "error" for v in result["local"]) - if args.sdk: - from .sdk import check + from .ui import Editor - try: - result["sdk"] = check(d) - except (ValueError, RuntimeError) as exc: - result["sdk"], failed = dict(error=str(exc)), True - print(json.dumps(result, indent=2)) - return int(failed) - sides = ( - ["left", "right"] - if args.sides == "both" - else [args.sides] - if args.sides - else [] - ) - inputs = ( - [ - (p, sides[i] if len(sides) == len(args.metadata or []) else None) - for i, p in enumerate(args.metadata) - ] - if args.command == "export" - else ( - [(args.metadata, sides[0] if len(sides) == 1 else None)] - if args.metadata - else [] - ) - ) - if args.command == "gui": - inputs += [ - (getattr(args, s + "_metadata"), s) - for s in ("left", "right") - if getattr(args, s + "_metadata") - ] - documents, model, paths = {}, {}, {} - for path, side in inputs: - d = storage.load(path, side) - if not d.side: - if args.command == "gui": - continue - raise ValueError("Assign JSON sides with --sides") - if d.side in documents: - raise ValueError("Duplicate metadata side") - documents[d.side], paths[d.side] = d, str(Path(path).resolve()) - if args.command == "export": - print( - json.dumps( - storage.export(documents, 0, args.out, args.format == "sdk"), - indent=2, - ) - ) - else: - from .ui import Editor - - for side in sides: - documents.setdefault(side, Document(side)) - if args.urdf: - model["urdf"] = str(Path(args.urdf).resolve()) - state = State(documents, model) - state.paths = paths - state.saved = {s: state.fingerprint(s) for s in paths} - Editor(state, args).run() + Editor(State(), args).run() return 0 - except (ValueError, OSError, RuntimeError, ImportError, TypeError) as exc: + except (ImportError, OSError, RuntimeError, TypeError, ValueError) as exc: print(f"{type(exc).__name__}: {exc}", file=sys.stderr) return 1 diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index 9831286441..b6cd366c09 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -3,298 +3,194 @@ from __future__ import annotations import copy -import json import math -import uuid +from dataclasses import dataclass, field JOINTS = """PALM WRIST THUMB_METACARPAL THUMB_PROXIMAL THUMB_DISTAL THUMB_TIP INDEX_METACARPAL INDEX_PROXIMAL INDEX_INTERMEDIATE INDEX_DISTAL INDEX_TIP MIDDLE_METACARPAL MIDDLE_PROXIMAL MIDDLE_INTERMEDIATE MIDDLE_DISTAL MIDDLE_TIP RING_METACARPAL RING_PROXIMAL RING_INTERMEDIATE RING_DISTAL RING_TIP LITTLE_METACARPAL LITTLE_PROXIMAL LITTLE_INTERMEDIATE LITTLE_DISTAL LITTLE_TIP""".split() -FIELDS = ( - "segment_bind_poses", - "taxel_bind_poses", - "taxel_segment_indices", - "taxel_segment_weights", -) -GROUPS = { - "Skeleton binding poses": ("joints", FIELDS[0]), - "Taxel binding poses": FIELDS[1:2], - "Skinning": FIELDS[2:], -} -IDENTITY = [0, 0, 0, 0, 0, 0, 1] -# Neutral right-hand OpenXR poses in wrist coordinates; left is mirrored below. -# fmt: off -DEFAULT_OPENXR_SEGMENT_POSES = ( - (-0.00244, 0.00122, -0.05917, 0.010296699, 0.02060433, -0.000212213, 0.999734662), - (0, 0, 0, 0, 0, 0, 1), - (-0.022, -0.012, -0.028, -0.00692186, 0.30239121, 0.275661445, 0.912426661), - (-0.047208158, -0.020249943, -0.065583071, -0.00692186, 0.30239121, 0.275661445, 0.912426661), - (-0.065292271, -0.02616838, -0.09254484, -0.00692186, 0.30239121, 0.275661445, 0.912426661), - (-0.080088363, -0.031010737, -0.114604468, -0.00692186, 0.30239121, 0.275661445, 0.912426661), - (-0.00528, 0, -0.02068, 0, 0.124664128, 0, 0.992199), - (-0.024, 0, -0.094, -0.017438446, 0.039983244, 0.00069791, 0.999047924), - (-0.027194641, -0.00139598, -0.133847779, -0.043584497, 0.039951273, 0.00174431, 0.99824909), - (-0.029105295, -0.003487718, -0.157679985, -0.060999707, 0.039914746, 0.00244129, 0.997336397), - (-0.030691675, -0.005925105, -0.177467419, -0.060999707, 0.039914746, 0.00244129, 0.997336397), - (-0.00088, 0.00044, -0.02134, 0.010296699, 0.02060433, -0.000212213, 0.999734662), - (-0.004, 0.002, -0.097, -0.017452406, 0, 0, 0.999847695), - (-0.004, 0.000429523, -0.141972587, -0.043619387, 0, 0, 0.999048222), - (-0.004, -0.002010838, -0.169866039, -0.06104854, 0, 0, 0.998134798), - (-0.004, -0.004691964, -0.191702054, -0.06104854, 0, 0, 0.998134798), - (0.0033, 0, -0.02002, 0, -0.081592271, 0, 0.996665792), - (0.015, 0, -0.091, -0.017444553, -0.029990932, -0.000523494, 0.999397797), - (0.017516954, -0.001465779, -0.132898883, -0.04359976, -0.029966951, -0.001308385, 0.998598684), - (0.019070086, -0.003731828, -0.158753338, -0.06102107, -0.029939553, -0.001831181, 0.997685671), - (0.020319944, -0.006291084, -0.1795593, -0.06102107, -0.029939553, -0.001831181, 0.997685671), - (0.00704, -0.00088, -0.01804, -0.022312091, -0.184914996, -0.004199362, 0.982492229), - (0.032, -0.004, -0.082, -0.017415551, -0.064944346, -0.001133608, 0.99773626), - (0.036145766, -0.005116784, -0.113710652, -0.043527274, -0.064892417, -0.002833264, 0.996938475), - (0.038599442, -0.006772743, -0.132478637, -0.06091962, -0.064833086, -0.003965361, 0.996026981), - (0.040915463, -0.008966391, -0.150193714, -0.06091962, -0.064833086, -0.003965361, 0.996026981), -) -# fmt: on +IDENTITY_POSE = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0] +IDENTITY_MATRIX = [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0], +] -def joint_number(joint): - number = JOINTS.index(joint) if isinstance(joint, str) else joint - if type(number) is not int or not 0 <= number < len(JOINTS): - raise ValueError("Invalid HandJoint identifier") - return number +@dataclass +class Segment: + joint: str + pose: list[float] = field(default_factory=lambda: IDENTITY_POSE.copy()) -def row_error(field, values, segments): - width = 7 if field in FIELDS[:2] else 4 - if not isinstance(values, list) or len(values) != width: - return f"Expected {width} components" +@dataclass +class Taxel: + pose: list[float] = field(default_factory=lambda: IDENTITY_POSE.copy()) + segments: list[str | None] = field(default_factory=lambda: [None] * 4) + weights: list[float] = field(default_factory=lambda: [0.0] * 4) + + +def pose_error(values: object) -> str | None: + if not isinstance(values, list) or len(values) != 7: + return "Expected XYZ and XYZW" if any( - type(v) not in (int, float) or (type(v) is float and not math.isfinite(v)) - for v in values - ): - return "Components must be finite numbers" - if field == FIELDS[2]: - if any( - type(v) is not int or not 0 <= v < min(segments, 0x100000000) - for v in values - ): - return "All slots must index segment rows, including zero-weight slots" - elif field == FIELDS[3]: - if min(values) < 0 or abs(sum(values) - 1) > 1e-5: - return "Weights must be nonnegative and sum to one" - elif ( - max(map(abs, values)) > 3.4028234663852886e38 - or abs(sum(v * v for v in values[3:]) - 1) > 1e-4 + type(value) not in (int, float) or not math.isfinite(value) for value in values ): - return "Expected float32 coordinates and a unit XYZW quaternion" + return "Pose components must be finite numbers" + if abs(sum(value * value for value in values[3:]) - 1.0) > 1e-4: + return "Orientation must be a unit quaternion" return None class Document: - def __init__(self, side=None): - self.side = side - self.joints = JOINTS[:] if side in ("left", "right") else [] - self.arrays = {field: [] for field in FIELDS} - if self.joints: - poses = [list(pose) for pose in DEFAULT_OPENXR_SEGMENT_POSES] - if side == "left": - for pose in poses: - pose[0], pose[4], pose[5] = -pose[0], -pose[4], -pose[5] - self.arrays[FIELDS[0]] = poses - self.types = ("LBS", "HandSkeletonRef") - self.editor = {"segment_ids": [], "output_indices": []} - self.sync() - - def sync(self): - count, ids = len(self.arrays[FIELDS[0]]), self.editor["segment_ids"] - ids.extend(uuid.uuid4().hex for _ in range(count - len(ids))) - del ids[count:] - count, order = len(self.arrays[FIELDS[1]]), self.editor["output_indices"] - order.extend(range(len(order), count)) - del order[count:] - - def set_components(self, field, selected, components, values): - width = 7 if field in FIELDS[:2] else 4 - for i in selected: - while len(self.arrays[field]) <= i: - self.arrays[field].append(None) - if not isinstance(self.arrays[field][i], list): - self.arrays[field][i] = [None] * width - row = self.arrays[field][i] - row.extend([None] * max(0, width - len(row))) - for component, value in zip(components, values): - row[component] = value - if field in FIELDS[:2] and selected: - self.editor["approximate"] = True - self.sync() - - def weight(self, selected, slot, value): - rejected = [] - for i in selected: - try: - values = list(self.arrays[FIELDS[3]][i]) - values[slot] = float(value) - if ( - len(values) != 4 - or min(values) < 0 - or not all(math.isfinite(v) for v in values) - ): - raise ValueError("Expected four finite nonnegative weights") - total = sum(values) - if total <= 0 or not math.isfinite(total): - raise ValueError("Weight sum must be finite and positive") - self.arrays[FIELDS[3]][i] = [v / total for v in values] - except (IndexError, ValueError, TypeError, OverflowError) as exc: - rejected.append(f"Taxel row {i}: {exc}") - return rejected - - def assign(self, selected, segments): - if not 1 <= len(segments) <= 4 or any( - type(i) is not int or not 0 <= i < len(self.arrays[FIELDS[0]]) - for i in segments - ): - raise ValueError("Choose one to four existing segment rows") - for i in selected: - self.set_components( - FIELDS[2], [i], range(4), segments + [segments[0]] * (4 - len(segments)) - ) - self.set_components( - FIELDS[3], - [i], - range(4), - [1 / len(segments)] * len(segments) + [0] * (4 - len(segments)), - ) - - def unused_start(self, count): - start = 0 - while any(start + i in self.editor["output_indices"] for i in range(count)): - start += 1 - return start - - def add(self, poses, segments, start=-1): - if any(len(self.arrays[f]) != len(self.arrays[FIELDS[1]]) for f in FIELDS[2:]): - raise ValueError("Repair skinning counts before adding taxels") - if not poses: - raise ValueError("No candidates to apply") - start = self.unused_start(len(poses)) if start < 0 else start - first = len(self.arrays[FIELDS[1]]) - self.arrays[FIELDS[1]].extend(copy.deepcopy(poses)) - self.assign(range(first, first + len(poses)), segments) - self.editor["output_indices"].extend(range(start, start + len(poses))) - self.sync() - self.editor["approximate"] = True + """Editor-native data. Poses and semantic bindings never use FlatBuffer row indices.""" - def delete(self, selected): - for i in sorted(set(selected), reverse=True): - for values in ( - *[self.arrays[f] for f in FIELDS[1:]], - self.editor["output_indices"], + def __init__(self, side: str): + if side not in ("left", "right"): + raise ValueError("Side must be left or right") + self.side = side + self.segments: list[Segment] = [] + self.taxels: list[Taxel] = [] + self.q: dict[str, float] = {} + self.alignment = copy.deepcopy(IDENTITY_MATRIX) + + def clone(self) -> Document: + return copy.deepcopy(self) + + def add_segment(self, joint: str, pose: list[float]) -> int: + if joint not in JOINTS: + raise ValueError("Choose a HandJoint") + self.segments.append(Segment(joint, list(pose))) + return len(self.segments) - 1 + + def add_taxels(self, poses: list[list[float]]) -> list[int]: + first = len(self.taxels) + self.taxels.extend(Taxel(list(pose)) for pose in poses) + return list(range(first, len(self.taxels))) + + def delete(self, kind: str, rows: list[int]) -> None: + values = self.segments if kind == "segments" else self.taxels + for row in sorted(set(rows), reverse=True): + if 0 <= row < len(values): + values.pop(row) + + def move(self, kind: str, source: int, target: int) -> None: + values = self.segments if kind == "segments" else self.taxels + if not 0 <= source < len(values) or not 0 <= target < len(values): + raise ValueError("Reorder row is outside the list") + value = values.pop(source) + values.insert(target, value) + + def assign(self, taxels: list[int], segment_rows: list[int]) -> None: + if not 1 <= len(segment_rows) <= 4: + raise ValueError("Select one to four segments") + joints = [] + for row in segment_rows: + if not 0 <= row < len(self.segments): + raise ValueError("Selected segment no longer exists") + joints.append(self.segments[row].joint) + weights = [1.0 / len(joints)] * len(joints) + for row in taxels: + if 0 <= row < len(self.taxels): + self.taxels[row].segments = joints + [None] * (4 - len(joints)) + self.taxels[row].weights = weights + [0.0] * (4 - len(joints)) + + def set_weight(self, taxel_row: int, slot: int, value: float) -> None: + taxel = self.taxels[taxel_row] + if taxel.segments[slot] is None: + raise ValueError("Select a segment before editing its weight") + value = float(value) + if not math.isfinite(value) or not 0.0 <= value <= 1.0: + raise ValueError("Weight must be from 0 to 1") + active = [i for i, segment in enumerate(taxel.segments) if segment is not None] + if len(active) == 1: + taxel.weights = [1.0 if i == slot else 0.0 for i in range(4)] + return + others = [i for i in active if i != slot] + remainder = 1.0 - value + total = sum(taxel.weights[i] for i in others) + taxel.weights[slot] = value + for i in range(4): + if taxel.segments[i] is None: + taxel.weights[i] = 0.0 + if total > 0.0: + for i in others: + taxel.weights[i] = taxel.weights[i] / total * remainder + else: + for i in others: + taxel.weights[i] = remainder / len(others) + + def validate(self) -> list[dict]: + issues: list[dict] = [] + + def issue(kind: str, row: int | None, field: str, message: str) -> None: + issues.append(dict(kind=kind, row=row, field=field, message=message)) + + duplicate_joints = { + joint + for joint in {segment.joint for segment in self.segments} + if sum(segment.joint == joint for segment in self.segments) > 1 + } + joints = {segment.joint for segment in self.segments} + for row, segment in enumerate(self.segments): + if segment.joint not in JOINTS: + issue("segments", row, "joint", "Invalid HandJoint") + elif segment.joint in duplicate_joints: + issue("segments", row, "joint", "HandJoint is already used") + if error := pose_error(segment.pose): + issue("segments", row, "pose", error) + for row, taxel in enumerate(self.taxels): + if error := pose_error(taxel.pose): + issue("taxels", row, "pose", error) + if len(taxel.segments) != 4 or len(taxel.weights) != 4: + issue("taxels", row, "binding", "Expected four binding slots") + continue + for slot, (joint, weight) in enumerate(zip(taxel.segments, taxel.weights)): + valid_weight = type(weight) in (int, float) and math.isfinite(weight) + if joint is None: + if not valid_weight or weight != 0.0: + issue( + "taxels", row, f"weight{slot}", "None weight must be zero" + ) + elif joint not in joints: + issue("taxels", row, f"segment{slot}", f"Missing segment {joint}") + elif not valid_weight or weight <= 0.0: + issue( + "taxels", + row, + f"weight{slot}", + "Selected segment weight must be greater than zero", + ) + if not any( + joint in joints for joint in taxel.segments if joint is not None ): - if i < len(values): - values.pop(i) - - def segments(self, order): - count = len(self.arrays[FIELDS[0]]) - if ( - len(self.joints) != count - or len(set(order)) != len(order) - or any(type(i) is not int or not 0 <= i < count for i in order) - ): - raise ValueError("Repair skeleton counts and supply distinct segment rows") - mapping = {old: new for new, old in enumerate(order)} - self.joints = [self.joints[i] for i in order] - self.arrays[FIELDS[0]] = [self.arrays[FIELDS[0]][i] for i in order] - self.editor["segment_ids"] = [self.editor["segment_ids"][i] for i in order] - for values in self.arrays[FIELDS[2]]: - if isinstance(values, list): - values[:] = [ - mapping.get(i, -1) if type(i) is int and 0 <= i < count else i - for i in values - ] - - def record(self, received, groups): - for group in groups: - for field in GROUPS[group]: - if field == "joints": - self.joints = copy.deepcopy(received.joints) - else: - self.arrays[field] = copy.deepcopy(received.arrays[field]) - self.sync() - - def check(self): - problems = [] - - def issue(field, message, row=None, severity="error"): - problems.append( - dict(severity=severity, field=field, row=row, message=message) + issue("taxels", row, "binding", "Taxel is not bound to a segment") + numeric = all( + type(value) in (int, float) and math.isfinite(value) + for value in taxel.weights ) - - if self.types != ("LBS", "HandSkeletonRef"): - issue("metadata", "Expected LBS and HandSkeletonRef discriminators") - j, n, seen = len(self.joints), len(self.arrays[FIELDS[1]]), set() - for field, expected in zip(FIELDS, (j, n, n, n)): - if len(self.arrays[field]) != expected: - issue(field, f"Expected {expected} rows") - for i, row in enumerate(self.arrays[field]): - error = row_error(field, row, len(self.arrays[FIELDS[0]])) - if ( - field == FIELDS[0] - and i < j - and self.joints[i] in ("WRIST", 1) - and not error - ): - if ( - any(abs(v) > 1e-6 for v in row[:6]) - or abs(abs(row[6]) - 1) > 1e-6 - ): - error = "WRIST bind pose must be identity" - if error: - issue(field, error, i) - for i, joint in enumerate(self.joints): - try: - number = joint_number(joint) - if number in seen: - raise ValueError("Duplicate HandJoint identifier") - seen.add(number) - except ValueError as exc: - issue("joint_ids", str(exc), i) - order = self.editor["output_indices"] - if ( - len(order) != n - or any(type(i) is not int for i in order) - or sorted(order) != list(range(n)) - ): - issue("output_indices", "Taxel indices must cover 0..N-1 exactly") - if self.editor.get("approximate"): - issue( - FIELDS[1], - "Surface/manual placement requires physical calibration", - severity="warning", - ) - return problems + if not numeric or abs(sum(taxel.weights) - 1.0) > 1e-5: + issue("taxels", row, "weights", "Weights must sum to one") + return issues class State: - def __init__(self, documents, model=None): - self.documents, self.model = documents, copy.deepcopy(model or {}) - self.revision, self.saved, self.paths = 0, {}, {} - - def fingerprint(self, side): - from .storage import metadata - - return json.dumps(metadata(self.documents[side]), sort_keys=True) - - def edit(self, action, *args, **kwargs): - before = copy.deepcopy((self.documents, self.model)) - hashes = [self.fingerprint(s) for s in self.documents] - try: - result = action(*args, **kwargs) - except Exception: - self.documents, self.model = before - raise - if hashes != [self.fingerprint(s) for s in self.documents]: - self.revision += 1 - return result - - def dirty(self): - return any(self.saved.get(s) != self.fingerprint(s) for s in self.documents) + def __init__(self): + self.documents = {side: Document(side) for side in ("left", "right")} + self.saved: dict[str, Document | None] = {"left": None, "right": None} + self.paths: dict[str, str] = {} + self.side = "left" + + @property + def document(self) -> Document: + return self.documents[self.side] + + def replace(self, document: Document, path: str | None = None) -> None: + self.documents[document.side] = document + self.saved[document.side] = document.clone() + self.saved[document.side].q = {} + if path is not None: + self.paths[document.side] = path diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js new file mode 100644 index 0000000000..11b148bd1d --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -0,0 +1,90 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +(() => { + "use strict"; + + const key = "__tactileMetadataEditorInteractions"; + globalThis[key]?.stop(); + + let dragged = null; + let nonce = 0; + let observer = null; + + const bridge = () => { + for (const input of document.querySelectorAll("input")) { + let parent = input; + for (let depth = 0; parent && depth < 5; ++depth, parent = parent.parentElement) { + if (parent.textContent?.includes("__Tactile Reorder Event")) { + if (parent.parentElement) parent.parentElement.style.display = "none"; + return input; + } + } + } + return null; + }; + + const emit = (kind, source, target) => { + const input = bridge(); + if (!input) return; + const value = `${kind}:${source}:${target}:${++nonce}`; + const setter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + "value", + )?.set; + setter?.call(input, value); + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new Event("change", { bubbles: true })); + }; + + const bind = () => { + bridge(); + for (const label of document.querySelectorAll("label")) { + if (!label.textContent?.trim().startsWith("⚠")) continue; + const field = label.parentElement?.querySelector("input, button"); + if (field) { + field.style.outline = "1px solid #d32f2f"; + field.style.outlineOffset = "1px"; + } + } + for (const handle of document.querySelectorAll(".tm-sort-handle")) { + if (handle.dataset.bound === "true") continue; + handle.dataset.bound = "true"; + handle.style.cursor = "grab"; + handle.style.userSelect = "none"; + handle.addEventListener("dragstart", (event) => { + dragged = { + kind: handle.dataset.kind, + row: Number(handle.dataset.row), + }; + event.dataTransfer.effectAllowed = "move"; + }); + handle.addEventListener("dragover", (event) => { + if (dragged?.kind === handle.dataset.kind) { + event.preventDefault(); + event.dataTransfer.dropEffect = "move"; + } + }); + handle.addEventListener("drop", (event) => { + event.preventDefault(); + if (dragged?.kind === handle.dataset.kind) { + emit(dragged.kind, dragged.row, Number(handle.dataset.row)); + } + dragged = null; + }); + handle.addEventListener("dragend", () => { + dragged = null; + }); + } + }; + + observer = new MutationObserver(bind); + observer.observe(document.body, { childList: true, subtree: true }); + bind(); + globalThis[key] = { + stop() { + observer?.disconnect(); + observer = null; + dragged = null; + }, + }; +})(); diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index ce95205390..e6b5d57a19 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -10,333 +10,309 @@ from trimesh import transformations as tf from yourdfpy import URDF -from .document import FIELDS, joint_number, row_error +from .document import Document, JOINTS, Segment, Taxel -# URDF +X/+Y/+Z to visualization +Z/+X/+Y. -VISUALIZATION_FROM_URDF = np.array( - [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 0, 1]], - dtype=float, -) -# Hand +X/+Y/-Z to visualization -X/-Z/+Y. VISUALIZATION_FROM_HAND = np.array( [[-1, 0, 0, 0], [0, 0, -1, 0], [0, -1, 0, 0], [0, 0, 0, 1]], dtype=float, ) +VISUALIZATION_FROM_URDF = np.array( + [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 0, 1]], + dtype=float, +) -def matrix(p): - try: - a = np.asarray(p, dtype=float) - except OverflowError as exc: - raise ValueError("Pose components exceed floating-point range") from exc - if a.shape != (7,) or not np.isfinite(a).all() or abs(a[3:] @ a[3:] - 1) > 1e-4: +def matrix(values: list[float]) -> np.ndarray: + array = np.asarray(values, dtype=float) + if ( + array.shape != (7,) + or not np.isfinite(array).all() + or abs(array[3:] @ array[3:] - 1.0) > 1e-4 + ): raise ValueError("Invalid XYZ/XYZW pose") - result = tf.quaternion_matrix(a[[6, 3, 4, 5]]) - result[:3, 3] = a[:3] + result = tf.quaternion_matrix(array[[6, 3, 4, 5]]) + result[:3, 3] = array[:3] return result -def pose(transform): - q = tf.quaternion_from_matrix(transform) - return [*transform[:3, 3].tolist(), *q[[1, 2, 3, 0]].tolist()] +def pose(transform: np.ndarray) -> list[float]: + quaternion = tf.quaternion_from_matrix(transform) + return [ + *np.asarray(transform[:3, 3], dtype=float).tolist(), + *quaternion[[1, 2, 3, 0]].tolist(), + ] + + +def transform(position, rotation_degrees) -> np.ndarray: + result = tf.euler_matrix(*np.deg2rad(rotation_degrees), axes="sxyz") + result[:3, 3] = np.asarray(position, dtype=float) + return result + + +def components(value: np.ndarray) -> tuple[tuple[float, ...], tuple[float, ...]]: + return ( + tuple(value[:3, 3]), + tuple(np.rad2deg(tf.euler_from_matrix(value, axes="sxyz"))), + ) -def unit(vector): - vector = np.asarray(vector, dtype=float) - length = np.linalg.norm(vector) - if not np.isfinite(length) or length < 1e-10: - raise ValueError("Reference directions must be finite and non-collinear") - return vector / length +def unit(value) -> np.ndarray: + value = np.asarray(value, dtype=float) + norm = np.linalg.norm(value) + if not np.isfinite(norm) or norm < 1e-10: + raise ValueError("Expected a finite nonzero direction") + return value / norm -def frame(origin, normal, tangent): +def surface_frame(origin, normal) -> np.ndarray: z = unit(normal) - x = unit(np.asarray(tangent) - z * (np.asarray(tangent) @ z)) + x = np.array([1.0, 0.0, 0.0]) + x -= z * (x @ z) + if np.linalg.norm(x) < 1e-6: + x = np.array([0.0, 1.0, 0.0]) + x -= z * (x @ z) + x = unit(x) result = np.eye(4) - result[:3, :3], result[:3, 3] = np.column_stack((x, np.cross(z, x), z)), origin + result[:3, :3] = np.column_stack((x, np.cross(z, x), z)) + result[:3, 3] = origin return result -def frames(rows, root): - for i, p in enumerate(rows): - try: - yield i, root @ matrix(p) - except (ValueError, TypeError): - pass +def realign(document: Document, alignment: np.ndarray) -> np.ndarray: + old = np.asarray(document.alignment, dtype=float) + delta = alignment @ np.linalg.inv(old) + for value in (*document.segments, *document.taxels): + value.pose = pose(delta @ matrix(value.pose)) + document.alignment = alignment.tolist() + return delta + + +def apply_delta(values, rows: list[int], delta: np.ndarray) -> None: + for row in rows: + if 0 <= row < len(values): + values[row].pose = pose(delta @ matrix(values[row].pose)) + + +def scale_positions(values, rows: list[int], factor: float) -> None: + if not np.isfinite(factor) or factor <= 0: + raise ValueError("Scale must be finite and positive") + points = np.asarray([values[row].pose[:3] for row in rows], dtype=float) + center = points.mean(axis=0) + for row in rows: + values[row].pose[:3] = ( + center + (values[row].pose[:3] - center) * factor + ).tolist() + + +def grid_poses( + center, width: float, height: float, rows: int, columns: int, frame=None +) -> list[list[float]]: + if rows < 1 or columns < 1: + raise ValueError("Rows and Columns must be positive") + if not np.isfinite(width) or not np.isfinite(height) or width < 0 or height < 0: + raise ValueError("Grid dimensions must be finite and nonnegative") + basis = np.eye(4) if frame is None else np.asarray(frame, dtype=float).copy() + basis[:3, 3] = np.asarray(center, dtype=float) + result = [] + for row in range(rows): + y = 0.0 if rows == 1 else (row / (rows - 1) - 0.5) * height + for column in range(columns): + x = 0.0 if columns == 1 else (column / (columns - 1) - 0.5) * width + current = basis.copy() + current[:3, 3] += basis[:3, :3] @ [x, y, 0.0] + result.append(pose(current)) + return result class Model: - def __init__(self, path, packages, scale=1.0): - if not np.isfinite(scale) or scale <= 0: - raise ValueError("URDF scale must be finite and positive") - self.scale = float(scale) - self.path, self.missing, self.surfaces = Path(path).resolve(), [], {} - self.handles, self.attachments = {}, {} - # Instance-owned caches must not keep replaced models alive. - self._fk_cache = OrderedDict() - - # yourdfpy invokes this resolver using the keyword fname. + def __init__(self, path: str): + self.path = Path(path).resolve() + self.handles: dict[str, object] = {} + self._fk_cache: OrderedDict[tuple[float, ...], dict[str, np.ndarray]] = ( + OrderedDict() + ) + def resource(fname): - name = fname + name = str(fname) if name.startswith("package://"): package, relative = name[10:].split("/", 1) - resolved = ( - Path(packages.get(package, self.path.parent / package)) / relative - ) + root = self.path.parent + while root != root.parent and root.name != package: + root = root.parent + if root.name == package: + return str((root / relative).resolve()) + for root in self.path.parents: + candidate = root / relative + if candidate.is_file(): + return str(candidate) + raise ValueError(f"Cannot resolve package resource: {name}") elif "://" in name: - raise ValueError(f"Only local resources are supported: {name}") - else: - resolved = self.path.parent / name - if not resolved.is_file(): - self.missing.append(str(resolved)) - return str(resolved) + raise ValueError(f"Only local mesh resources are supported: {name}") + return str((self.path.parent / name).resolve()) self.urdf = URDF.load(str(self.path), filename_handler=resource) - self.links, self.joints = list(self.urdf.link_map), self.urdf.actuated_joints + self.joints = self.urdf.actuated_joints + self.links = list(self.urdf.link_map) + self.surfaces: dict[str, tuple[str, trimesh.Trimesh]] = {} scene = self.urdf.scene - parents = {b: a for a, b, _ in scene.graph.to_edgelist()} + parents = {child: parent for parent, child, _ in scene.graph.to_edgelist()} for node in scene.graph.nodes_geometry: link = node while link not in self.links: link = parents[link] - transform, name = scene.graph.get(frame_to=node, frame_from=link) - mesh = scene.geometry[name].copy() - mesh.apply_transform(transform) - mesh.apply_scale(self.scale) - for i, part in enumerate(mesh.split(only_watertight=False)): - self.surfaces[f"{link}/{node}/{i}"] = (link, part) - - def fk(self, cfg): - configuration = tuple(float(cfg.get(j.name, 0.0)) for j in self.joints) - result = self._fk_cache.pop(configuration, None) - if result is None: + local, geometry_name = scene.graph.get(frame_to=node, frame_from=link) + mesh = scene.geometry[geometry_name].copy() + mesh.apply_transform(local) + for index, part in enumerate(mesh.split(only_watertight=False)): + self.surfaces[f"{link}/{node}/{index}"] = (link, part) + + def raw_fk(self, q: dict[str, float]) -> dict[str, np.ndarray]: + configuration = tuple(float(q.get(joint.name, 0.0)) for joint in self.joints) + cached = self._fk_cache.pop(configuration, None) + if cached is None: self.urdf.update_cfg( dict(zip((j.name for j in self.joints), configuration)) ) - result = { - link: VISUALIZATION_FROM_URDF - @ self.urdf.get_transform(link).copy() - for link in self.links - } - for transform in result.values(): - transform[:3, 3] *= self.scale - self._fk_cache[configuration] = result + cached = {link: self.urdf.get_transform(link).copy() for link in self.links} + self._fk_cache[configuration] = cached if len(self._fk_cache) > 4: self._fk_cache.popitem(last=False) - return result + return cached - def root(self, d, fk): - reference = d.editor["root"] - local = np.asarray(reference["frame"], dtype=float) - if ( - local.shape != (4, 4) - or not np.isfinite(local).all() - or not np.allclose(local[3], [0, 0, 0, 1]) - or not np.allclose(local[:3, :3].T @ local[:3, :3], np.eye(3)) - or np.linalg.det(local[:3, :3]) < 0 - ): - raise ValueError("Invalid wrist reference frame") - return fk[reference["link"]] @ local - - def attach(self, d, fk): - root = self.root(d, fk) - attachments = self.attachments[d.side] = {} - for i, sid in enumerate(d.editor["segment_ids"]): - link = d.editor.get("segment_links", {}).get(sid) - try: - attachments[sid] = ( - np.linalg.inv(fk[link]) @ root @ matrix(d.arrays[FIELDS[0]][i]) - ) - except (KeyError, TypeError, ValueError): - pass - - def current(self, d, fk): - inverse = np.linalg.inv(self.root(d, fk)) - links, attachments = ( - d.editor.get("segment_links", {}), - self.attachments.get(d.side, {}), - ) - return [ - inverse @ fk[links[sid]] @ attachments[sid] - if sid in attachments and links.get(sid) in fk - else None - for sid in d.editor["segment_ids"] - ] + @staticmethod + def _rigid(raw: np.ndarray, alignment: np.ndarray, scale: float) -> np.ndarray: + result = alignment.copy() + result[:3, :3] = alignment[:3, :3] @ raw[:3, :3] + result[:3, 3] = alignment[:3, :3] @ (raw[:3, 3] * scale) + alignment[:3, 3] + return result - def draw(self, scene, cfg, visible=True, highlight=None): - fk = self.fk(cfg) + def draw( + self, + scene, + q: dict[str, float], + alignment: np.ndarray, + scale: float, + highlighted: bool = False, + opacity: float = 0.6, + ) -> None: + raw = self.raw_fk(q) for name, (link, mesh) in self.surfaces.items(): if name not in self.handles: self.handles[name] = scene.add_mesh_simple( - "/model/" + name, + "/urdf/" + name, mesh.vertices, mesh.faces, color=(170, 175, 180), - opacity=0.6, + opacity=opacity, side="double", ) - h = self.handles[name] - h.visible = visible - h.color = ( - (255, 220, 0) if highlight and link in highlight else (170, 175, 180) - ) - h.position, h.wxyz = fk[link][:3, 3], tf.quaternion_from_matrix(fk[link]) - return fk - - def hit(self, origin, direction, surfaces, fk): - nearest = None - for name in surfaces: - link, mesh = self.surfaces[name] - inverse = np.linalg.inv(fk[link]) - local = (inverse @ np.r_[origin, 1])[:3] + current = self._rigid(raw[link], alignment, scale) + handle = self.handles[name] + handle.position = current[:3, 3] + handle.wxyz = tf.quaternion_from_matrix(current) + handle.scale = scale + handle.opacity = opacity + handle.color = (255, 210, 45) if highlighted else (170, 175, 180) + handle.visible = True + + def hit( + self, + origin, + direction, + q: dict[str, float], + alignment: np.ndarray, + scale: float, + ): + nearest, transforms = None, self.raw_fk(q) + for link, mesh in self.surfaces.values(): + raw = transforms[link] + similarity = alignment @ np.diag([scale, scale, scale, 1.0]) @ raw + inverse = np.linalg.inv(similarity) + local_origin = (inverse @ np.r_[origin, 1.0])[:3] + local_direction = inverse[:3, :3] @ direction points, _, faces = mesh.ray.intersects_location( - [local], [inverse[:3, :3] @ direction] + [local_origin], [local_direction] ) for point, face in zip(points, faces): - world = (fk[link] @ np.r_[point, 1])[:3] + world = (similarity @ np.r_[point, 1.0])[:3] distance = np.linalg.norm(world - origin) if nearest is None or distance < nearest[0]: - nearest = ( - distance, - world, - fk[link][:3, :3] @ mesh.face_normals[face], - name, + normal = ( + np.linalg.inv(similarity[:3, :3]).T @ mesh.face_normals[face] ) + nearest = (distance, world, unit(normal)) return nearest - def grid(self, d, fk, surface, points, rows, cols, scan, reverse, excluded): - if ( - rows < 1 - or cols < 1 - or points[0] is None - or (cols > 1 and points[1] is None) - or (rows > 1 and points[2] is None) - ): - raise ValueError( - "Pick required first/last centers and use positive row/column counts" - ) - if scan not in ( - "row-major", - "column-major", - "row-snake", - "column-snake", - ) or any( - len(v) != 2 - or any(type(k) is not int for k in v) - or not 0 <= v[0] < rows - or not 0 <= v[1] < cols - for v in excluded - ): - raise ValueError("Invalid scan or excluded cell coordinates") - origin = np.asarray(points[0]) - column = ( - np.asarray(points[1]) - origin if points[1] is not None else np.zeros(3) - ) - row = np.asarray(points[2]) - origin if points[2] is not None else np.zeros(3) - if ( - (cols > 1 and np.linalg.norm(column) < 1e-10) - or (rows > 1 and np.linalg.norm(row) < 1e-10) - or (rows > 1 and cols > 1 and np.linalg.norm(np.cross(column, row)) < 1e-10) - ): - raise ValueError( - "Required row/column spans must be nonzero and independent" - ) - tangent = unit(column if np.linalg.norm(column) > 0 else row) - link, mesh = self.surfaces[surface] - face = trimesh.proximity.closest_point( - mesh, [(np.linalg.inv(fk[link]) @ np.r_[origin, 1])[:3]] - )[2][0] - normal = fk[link][:3, :3] @ mesh.face_normals[face] - cells = [(r, c) for r in range(rows) for c in range(cols)] - major = 1 if scan.startswith("column") else 0 - cells.sort( - key=lambda rc: ( - rc[major], - -rc[1 - major] - if scan.endswith("snake") and rc[major] % 2 - else rc[1 - major], - ) - ) - depth, inverse = ( - max(np.linalg.norm(mesh.extents), 0.01), - np.linalg.inv(self.root(d, fk)), - ) - result = [] - for r, c in cells: - r, c = rows - 1 - r if reverse[0] else r, cols - 1 - c if reverse[1] else c - if [r, c] in excluded: - continue - point = ( - origin - + column * (c / (cols - 1) if cols > 1 else 0) - + row * (r / (rows - 1) if rows > 1 else 0) - ) - hit = self.hit(point + normal * depth, -normal, [surface], fk) - try: - candidate = ( - pose(inverse @ frame(hit[1], hit[2], tangent)) if hit else None - ) - except ValueError: - candidate = None - result.append((candidate, (inverse @ np.r_[point, 1])[:3])) - return result - - -def transform(d, field, selected, delta, local=False): - for i in selected: - old = matrix(d.arrays[field][i]) - d.arrays[field][i] = pose(old @ delta if local else delta @ old) - d.editor["approximate"] = True + def remove(self) -> None: + for handle in self.handles.values(): + handle.remove() + self.handles.clear() -def deform(d, current): +def deform(binding: Document, current: list[Segment]) -> list[Taxel]: + bind_segments = { + segment.joint: matrix(segment.pose) for segment in binding.segments + } + current_segments = {segment.joint: matrix(segment.pose) for segment in current} + if len(bind_segments) != len(binding.segments) or len(current_segments) != len( + current + ): + raise ValueError("Segment HandJoint values must be unique") result = [] - for i, p in enumerate(d.arrays[FIELDS[1]]): - try: - indices, weights = d.arrays[FIELDS[2]][i], d.arrays[FIELDS[3]][i] - if any( - row_error(f, v, len(current)) - for f, v in zip(FIELDS[2:], (indices, weights)) - ): - raise ValueError("Invalid skinning row") - transforms = [ - current[j] @ np.linalg.inv(matrix(d.arrays[FIELDS[0]][j])) @ matrix(p) - for j, w in zip(indices, weights) - if w > 0 - ] - position = sum( - w * t[:3, 3] for w, t in zip([w for w in weights if w > 0], transforms) - ) - rigid = ( - transforms[0].copy() - if len({j for j, w in zip(indices, weights) if w > 0}) == 1 - else None + for taxel in binding.taxels: + active = [ + (joint, weight) + for joint, weight in zip(taxel.segments, taxel.weights) + if joint is not None and weight > 0 + ] + if not active: + raise ValueError("Taxel has no active segment") + transforms = [] + for joint, weight in active: + if joint not in bind_segments or joint not in current_segments: + raise ValueError(f"Missing segment {joint}") + transforms.append( + (weight, current_segments[joint] @ np.linalg.inv(bind_segments[joint])) ) - if rigid is not None: - rigid[:3, 3] = position - result.append((position, rigid)) - except (TypeError, ValueError, KeyError, IndexError): - result.append((None, None)) + bind_taxel = matrix(taxel.pose) + position = sum( + weight * (delta @ np.r_[bind_taxel[:3, 3], 1.0])[:3] + for weight, delta in transforms + ) + if len({joint for joint, _ in active}) == 1: + rotation = transforms[0][1][:3, :3] @ bind_taxel[:3, :3] + else: + blend = sum(weight * delta[:3, :3] for weight, delta in transforms) + u, singular, vh = np.linalg.svd(blend) + if singular[-1] < 1e-8: + rotation = bind_taxel[:3, :3] + else: + correction = np.eye(3) + correction[2, 2] = np.linalg.det(u @ vh) + rotation = u @ correction @ vh @ bind_taxel[:3, :3] + value = np.eye(4) + value[:3, :3], value[:3, 3] = rotation, position + result.append(Taxel(pose(value), list(taxel.segments), list(taxel.weights))) return result -def hand_current(d, hand): +def hand_segments(joints: list[str], hand: dict) -> list[Segment]: values = [] - for p, q, valid in zip(hand["positions"], hand["orientations"], hand["is_valid"]): - try: - values.append(matrix([*p, *q]) if valid else None) - except (ValueError, TypeError): - values.append(None) - if len(values) != 26 or len(hand["is_valid"]) != 26 or values[1] is None: - raise ValueError("HandTracker WRIST or array shape is invalid") - inverse = np.linalg.inv(values[1]) - return ( - [ - inverse @ values[joint_number(j)] - if values[joint_number(j)] is not None - else None - for j in d.joints - ], - values, - inverse, - ) + for position, orientation, valid in zip( + hand["positions"], hand["orientations"], hand["is_valid"] + ): + values.append(matrix([*position, *orientation]) if valid else None) + if len(values) != len(JOINTS) or values[1] is None: + raise ValueError("HandTracker WRIST pose is unavailable") + wrist_inverse = np.linalg.inv(values[1]) + result = [] + for joint in joints: + index = JOINTS.index(joint) + if values[index] is None: + raise ValueError(f"HandTracker {joint} pose is unavailable") + result.append( + Segment( + joint, pose(VISUALIZATION_FROM_HAND @ wrist_inverse @ values[index]) + ) + ) + return result diff --git a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py index f4d050114d..aad4367582 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/sdk.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/sdk.py @@ -3,114 +3,39 @@ from __future__ import annotations import json -import math import multiprocessing -import subprocess import sys -import tempfile import threading import time -from pathlib import Path -from .document import Document, FIELDS, IDENTITY, joint_number -from .storage import load, metadata, write - -ROOT = Path(__file__).resolve().parent.parent - - -def check(document, output=None): - if any(v["severity"] == "error" for v in document.check()): - raise ValueError("Local errors prevent export") - with tempfile.TemporaryDirectory(prefix="tactile-sdk-") as folder: - root = Path(folder) - write(root / "input.json", metadata(document)) - try: - result = subprocess.run( - [sys.executable, "-m", __package__ + ".sdk", "check", folder], - cwd=ROOT, - capture_output=True, - check=False, - timeout=30, - ) - except subprocess.TimeoutExpired as exc: - raise RuntimeError("SDK construction timed out after 30 seconds") from exc - report = root / "result.json" - if result.returncode or not report.exists(): - reason = ( - report.read_text() - if report.exists() - else result.stderr.decode(errors="replace")[-2000:] - ) - raise RuntimeError( - f"SDK construction failed (exit {result.returncode}): " + reason - ) - if output: - Path(output).write_bytes((root / "metadata.bin").read_bytes()) - return json.loads(report.read_text()) - - -def encode(folder): - import numpy as np - from isaaccapture import schema - from isaaccapture.deviceio_trackers import TactileTracker - - root = Path(folder) - d = load(root / "input.json") - arrays = [ - np.asarray( - d.arrays[f], dtype=np.uint32 if f == FIELDS[2] else np.float32 - ).reshape(-1, 7 if f in FIELDS[:2] else 4) - for f in FIELDS - ] - encoded = schema.TactileMetadata( - schema.LBS(*arrays), - schema.HandSkeletonRef([schema.HandJoint(joint_number(j)) for j in d.joints]), - ).to_bytes() - frame_size = len( - schema.TactileDataArray( - np.zeros((len(arrays[1]), 3), dtype=np.float32) - ).to_bytes() - ) - if ( - len(encoded) > schema.TRACKER_METADATA_MAX_SIZE - or frame_size > TactileTracker.DEFAULT_MAX_FLATBUFFER_SIZE - ): - raise ValueError( - "Encoded metadata or force frame exceeds the default tracker capacity" - ) - (root / "metadata.bin").write_bytes(encoded) - return dict( - status="SDK construction and size checks passed", - metadata_bytes=len(encoded), - force_bytes=frame_size, - ) +from .document import JOINTS class Receiver: - def __init__(self, configs): + def __init__(self, config: dict): context = multiprocessing.get_context("spawn") context.set_executable(sys.executable) self.connection, child = context.Pipe(duplex=False) self.latest, self.status, self.active = {}, "Connecting", True - self.process = context.Process(target=live, args=(child, configs), daemon=True) + self.process = context.Process(target=live, args=(child, config), daemon=True) self.process.start() child.close() - self.thread = threading.Thread(target=self.receive, daemon=True) + self.thread = threading.Thread(target=self._receive, daemon=True) self.thread.start() - def receive(self): + def _receive(self) -> None: try: while True: value = json.loads(self.connection.recv_bytes(16 * 1024 * 1024)) - self.status = value.get("status", "Connected") - self.active = "sides" in value - if "sides" in value: - self.latest = value["sides"] + self.status = value.get("status", "Live") + if "frame" in value: + self.latest = value["frame"] + self.active = True except (EOFError, OSError, ValueError): self.active = False - self.status += "; receiver stopped" + self.status = "Disconnected" - def close(self): + def close(self) -> None: self.process.terminate() self.process.join(timeout=2) if self.process.is_alive(): @@ -120,15 +45,52 @@ def close(self): self.thread.join(timeout=2) -def owned(value, fields): - return ( - {f: getattr(value, f).copy().tolist() for f in fields} - if value is not None - else None - ) +def _pose_rows(values) -> list[dict]: + return [ + { + "position": dict(zip("xyz", row[:3])), + "orientation": dict(zip("xyzw", row[3:])), + } + for row in values.copy().tolist() + ] -def live(connection, configs): +def _metadata(value) -> dict | None: + if value is None or value.geometry_mapping is None or value.segment_source is None: + return None + mapping = value.geometry_mapping + return { + "geometry_mapping_type": "LBS", + "geometry_mapping": { + "segment_bind_poses": _pose_rows(mapping.segment_bind_poses), + "taxel_bind_poses": _pose_rows(mapping.taxel_bind_poses), + "taxel_segment_indices": [ + {"values": row} for row in mapping.taxel_segment_indices.copy().tolist() + ], + "taxel_segment_weights": [ + {"values": row} for row in mapping.taxel_segment_weights.copy().tolist() + ], + }, + "segment_source_type": "HandSkeletonRef", + "segment_source": { + "joint_ids": [ + JOINTS[int(joint)] for joint in value.segment_source.joint_ids + ] + }, + } + + +def _hand(value) -> dict | None: + if value is None: + return None + joints = value.joints + return { + name: getattr(joints, name).copy().tolist() + for name in ("positions", "orientations", "is_valid") + } + + +def live(connection, config: dict) -> None: try: from isaaccapture.deviceio_session import DeviceIOSession from isaaccapture.deviceio_trackers import ( @@ -139,103 +101,42 @@ def live(connection, configs): from isaaccapture.oxr import OpenXRSession hand = HandTracker() - tactile = {s: TactileTracker(c["collection"]) for s, c in configs.items()} - joints = { - s: JointStateTracker(c["joint_state"]) - for s, c in configs.items() - if c.get("joint_state") - } - trackers = [hand, *tactile.values(), *joints.values()] + tactile = TactileTracker(config["collection"]) + joint = ( + JointStateTracker(config["joint_state"]) + if config.get("joint_state") + else None + ) + trackers = [hand, tactile, *([joint] if joint else [])] with OpenXRSession( "TactileMetadataEditor", DeviceIOSession.get_required_extensions(trackers) ) as oxr: with DeviceIOSession.run(trackers, oxr.get_handles()) as session: while True: session.update() - sides = {} - for side, tracker in tactile.items(): - m, data = ( - tracker.get_metadata(session), - tracker.get_data(session), - ) - h = getattr(hand, "get_" + configs[side]["hand"] + "_hand")( - session - ) - j = joints[side].get_data(session) if side in joints else None - raw = None - if m is not None: - d = Document(side) - d.joints = [int(v) for v in m.segment_source.joint_ids] - d.arrays = owned(m.geometry_mapping, FIELDS) - d.sync() - raw = metadata(d) - sides[side] = dict( - metadata=raw, - forces=data.taxel_forces.copy().tolist() - if data is not None - else None, - # Bulk hand views belong to the nested HandJoints object. - hand=owned( - h.joints if h is not None else None, - ("positions", "orientations", "is_valid"), - ), - joints={v.name: [v.position, v.valid] for v in j.joints} - if j is not None - else {}, - read_time=time.monotonic(), - config=configs[side], - ) - connection.send_bytes(json.dumps(dict(sides=sides)).encode()) + metadata = tactile.get_metadata(session) + data = tactile.get_data(session) + hand_value = getattr(hand, f"get_{config['side']}_hand")(session) + joint_value = joint.get_data(session) if joint else None + frame = { + "metadata": _metadata(metadata), + "forces": data.taxel_forces.copy().tolist() + if data is not None + else None, + "hand": _hand(hand_value), + "joints": { + item.name: [item.position, item.valid] + for item in joint_value.joints + } + if joint_value is not None + else {}, + "read_time": time.monotonic(), + } + connection.send_bytes(json.dumps({"frame": frame}).encode()) time.sleep(1 / 30) except Exception as exc: connection.send_bytes( - json.dumps(dict(status=f"{type(exc).__name__}: {exc}")).encode() + json.dumps({"status": f"{type(exc).__name__}: {exc}"}).encode() ) finally: connection.close() - - -def demo(configs): - sides, angle = {}, math.sin(time.monotonic()) * 0.4 - for side, config in configs.items(): - d = Document(side) - d.joints = ["WRIST", "INDEX_DISTAL"] - d.arrays[FIELDS[0]] = [IDENTITY[:], [0, 0.06, 0, 0, 0, 0, 1]] - d.add( - [ - [c * 0.008, 0.06 + r * 0.008, 0, 0, 0, 0, 1] - for r in range(4) - for c in range(4) - ], - [1], - ) - positions = [[0, 0, 0] for _ in range(26)] - orientations = [[0, 0, 0, 1] for _ in range(26)] - valid = [False] * 26 - positions[9] = [0, 0.06, 0] - orientations[9] = [math.sin(angle / 2), 0, 0, math.cos(angle / 2)] - valid[1] = valid[9] = True - sides[side] = dict( - metadata=metadata(d), - forces=[ - [0, 0, 15 + 15 * math.sin(time.monotonic() + i / 4)] for i in range(16) - ], - hand=dict(positions=positions, orientations=orientations, is_valid=valid), - joints={name: [angle, True] for name in config.get("mapping", {})} - if config.get("joint_state") - else {}, - read_time=time.monotonic(), - config=config, - ) - return sides - - -if __name__ == "__main__": - try: - write(Path(sys.argv[2]) / "result.json", encode(sys.argv[2])) - except Exception as exc: - write( - Path(sys.argv[2]) / "result.json", - dict(error=f"{type(exc).__name__}: {exc}"), - ) - sys.exit(1) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/storage.py b/scripts/tactile_metadata/tactile_metadata_editor/storage.py index e9b25e4fec..5c234ae720 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/storage.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/storage.py @@ -2,93 +2,150 @@ # SPDX-License-Identifier: Apache-2.0 from __future__ import annotations -import copy import json import os import tempfile from pathlib import Path -from .document import Document, FIELDS +import numpy as np +from .document import Document, JOINTS, Segment, State, Taxel +from .geometry import VISUALIZATION_FROM_HAND, matrix, pose -def parse(value, side=None): - if not isinstance(value, dict) or any( - not isinstance(value.get(k, {}), dict) - for k in ("geometry_mapping", "segment_source") - ): - raise ValueError( - "Expected a TactileMetadata object with object-valued sections" - ) - d = Document(side) - if d.side not in (None, "left", "right"): - raise ValueError("Assign metadata to left or right explicitly") - d.types = (value.get("geometry_mapping_type"), value.get("segment_source_type")) - d.joints = copy.deepcopy(value.get("segment_source", {}).get("joint_ids")) - if not isinstance(d.joints, list): - raise ValueError("joint_ids: expected an array") - for field in FIELDS: - rows = value.get("geometry_mapping", {}).get(field) - if not isinstance(rows, list): - raise ValueError(f"{field}: expected an array") - if field in FIELDS[:2] and any( - isinstance(v, dict) - and any( - not isinstance(v.get(k, {}), dict) for k in ("position", "orientation") - ) - for v in rows - ): - raise ValueError(f"{field}: position and orientation must be objects") - d.arrays[field] = [ - ( - [v.get("position", {}).get(k) for k in "xyz"] - + [v.get("orientation", {}).get(k) for k in "xyzw"] - if field in FIELDS[:2] - else v.get("values") - ) - if isinstance(v, dict) - else None - for v in rows - ] - d.sync() - return d +def _pose_row(value: object) -> list[float]: + if not isinstance(value, dict): + raise ValueError("Pose row must be an object") + position, orientation = value.get("position"), value.get("orientation") + if not isinstance(position, dict) or not isinstance(orientation, dict): + raise ValueError("Pose row requires position and orientation") + return [ + *[position.get(key) for key in "xyz"], + *[orientation.get(key) for key in "xyzw"], + ] -def metadata(d): - value = dict( - geometry_mapping_type=d.types[0], - geometry_mapping={}, - segment_source_type=d.types[1], - segment_source={"joint_ids": copy.deepcopy(d.joints)}, - ) - order = sorted( - range(len(d.editor["output_indices"])), - key=lambda i: d.editor["output_indices"][i], - ) - aligned = all(len(d.arrays[f]) == len(order) for f in FIELDS[1:]) - for field, rows in d.arrays.items(): - if aligned and field in FIELDS[1:]: - rows = [rows[i] for i in order] - value["geometry_mapping"][field] = [ - { - "position": dict(zip("xyz", r[:3])), - "orientation": dict(zip("xyzw", r[3:])), - } - if field in FIELDS[:2] and r is not None - else {"values": copy.deepcopy(r)} - if field in FIELDS[2:] - else None - for r in rows - ] + +def _rows(section: dict, name: str) -> list: + value = section.get(name) + if not isinstance(value, list): + raise ValueError(f"{name}: expected an array") return value -def load(path, side=None): +def _joint_name(value: object) -> str: + if isinstance(value, str): + return value + if type(value) is int and 0 <= value < len(JOINTS): + return JOINTS[value] + return f"INVALID_HAND_JOINT_{value}" + + +def parse( + value: object, + side: str, + alignment: np.ndarray | list[list[float]] | None = None, +) -> Document: + if not isinstance(value, dict): + raise ValueError("Expected a TactileMetadata object") + geometry, source = value.get("geometry_mapping"), value.get("segment_source") + if not isinstance(geometry, dict) or not isinstance(source, dict): + raise ValueError("Expected geometry_mapping and segment_source objects") + joints = source.get("joint_ids") + if not isinstance(joints, list): + raise ValueError("joint_ids: expected an array") + segment_rows = _rows(geometry, "segment_bind_poses") + taxel_rows = _rows(geometry, "taxel_bind_poses") + index_rows = _rows(geometry, "taxel_segment_indices") + weight_rows = _rows(geometry, "taxel_segment_weights") + if len(segment_rows) != len(joints): + raise ValueError("segment_bind_poses must match joint_ids") + if len({len(taxel_rows), len(index_rows), len(weight_rows)}) != 1: + raise ValueError("All taxel arrays must have the same length") + + transform = np.asarray( + VISUALIZATION_FROM_HAND if alignment is None else alignment, dtype=float + ) + document = Document(side) + document.alignment = transform.tolist() + names = [_joint_name(joint) for joint in joints] + document.segments = [ + Segment(joint, pose(transform @ matrix(_pose_row(row)))) + for joint, row in zip(names, segment_rows) + ] + for raw_pose, raw_indices, raw_weights in zip(taxel_rows, index_rows, weight_rows): + indices = raw_indices.get("values") if isinstance(raw_indices, dict) else None + weights = raw_weights.get("values") if isinstance(raw_weights, dict) else None + if not isinstance(indices, list) or not isinstance(weights, list): + raise ValueError("Taxel indices and weights require values arrays") + if len(indices) != 4 or len(weights) != 4: + raise ValueError("Taxel bindings require four slots") + bindings = [ + None + if type(weight) in (int, float) and weight == 0 + else names[index] + if type(index) is int and 0 <= index < len(names) + else f"INVALID_SEGMENT_INDEX_{index}" + for index, weight in zip(indices, weights) + ] + document.taxels.append( + Taxel( + pose(transform @ matrix(_pose_row(raw_pose))), + bindings, + list(weights), + ) + ) + return document + + +def _json_pose(values: list[float]) -> dict: + return { + "position": dict(zip("xyz", values[:3])), + "orientation": dict(zip("xyzw", values[3:])), + } + + +def metadata(document: Document) -> dict: + inverse = np.linalg.inv(np.asarray(document.alignment, dtype=float)) + joints = [segment.joint for segment in document.segments] + indices = {joint: row for row, joint in enumerate(joints)} + fallback = 0 + segment_poses = [ + _json_pose(pose(inverse @ matrix(segment.pose))) + for segment in document.segments + ] + taxel_poses = [ + _json_pose(pose(inverse @ matrix(taxel.pose))) for taxel in document.taxels + ] + binding_indices = [ + {"values": [indices.get(joint, fallback) for joint in taxel.segments]} + for taxel in document.taxels + ] + binding_weights = [{"values": list(taxel.weights)} for taxel in document.taxels] + return { + "geometry_mapping_type": "LBS", + "geometry_mapping": { + "segment_bind_poses": segment_poses, + "taxel_bind_poses": taxel_poses, + "taxel_segment_indices": binding_indices, + "taxel_segment_weights": binding_weights, + }, + "segment_source_type": "HandSkeletonRef", + "segment_source": {"joint_ids": joints}, + } + + +def load( + path: str | Path, + side: str, + alignment: np.ndarray | list[list[float]] | None = None, +) -> Document: with Path(path).open() as stream: - return parse(json.load(stream), side) + return parse(json.load(stream), side, alignment) -def write(path, value): +def write(path: str | Path, value: object) -> None: path = Path(path).resolve() + path.parent.mkdir(parents=True, exist_ok=True) fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".tactile-") try: with os.fdopen(fd, "w") as stream: @@ -99,45 +156,8 @@ def write(path, value): Path(temporary).unlink(missing_ok=True) -def save(state, paths): - if len({Path(p).resolve() for p in paths.values()}) != len(paths): - raise ValueError("Each hand requires a different save path") - for side, path in paths.items(): - write(path, metadata(state.documents[side])) - for side, path in paths.items(): - state.paths[side], state.saved[side] = str(path), state.fingerprint(side) - - -def backup(d): - with tempfile.NamedTemporaryFile( - prefix="tactile-backup-", suffix=".json", delete=False - ) as stream: - path = stream.name - write(path, metadata(d)) - return path - - -def export(documents, revision, destination, sdk=False): - destination = Path(destination).resolve() - if destination.exists() and ( - not destination.is_dir() or any(destination.iterdir()) - ): - raise ValueError("Choose a new or empty output directory") - with tempfile.TemporaryDirectory( - prefix=".tactile-", dir=destination.parent - ) as temporary: - root, report = Path(temporary), {"revision": revision, "sides": {}} - for side, d in documents.items(): - result = dict(local=d.check(), sdk="not checked") - if any(v["severity"] == "error" for v in result["local"]): - raise ValueError("Local errors prevent export") - write(root / f"{side}.metadata.json", metadata(d)) - if sdk: - from .sdk import check - - result["sdk"] = check(d, root / f"{side}.metadata.bin") - report["sides"][side] = result - if destination.exists(): - destination.rmdir() - os.replace(root, destination) - return report +def save(state: State, path: str | Path) -> None: + write(path, metadata(state.document)) + state.paths[state.side] = str(Path(path).resolve()) + state.saved[state.side] = state.document.clone() + state.saved[state.side].q = {} diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index cf80828620..83b579647a 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -8,7 +8,6 @@ import math import queue import time -from concurrent.futures import ThreadPoolExecutor from pathlib import Path import numpy as np @@ -18,1588 +17,1420 @@ from viser import _messages from . import geometry as geo, sdk, storage -from .document import Document, FIELDS, GROUPS, IDENTITY, JOINTS, State, joint_number +from .document import Document, IDENTITY_POSE, JOINTS, Segment, State, Taxel LOG = logging.getLogger("isaaccapture.tactile_metadata.editor") -def common_number(values): - if ( - values - and all(v == values[0] for v in values) - and type(values[0]) in (int, float) - and abs(values[0]) <= 3.4028234663852886e38 - ): - return float(values[0]) - return None - - class Editor: - INITIAL_CAMERA_POSE = ((0.3, 0.3, -0.4), (0, 0.08, 0)) - - def __init__(self, state, args): - self.state, self.args, self.started = state, args, bool(state.documents) - if not state.documents: - state.documents["left"] = Document("left") - state.saved["left"] = state.fingerprint("left") - self.events, self.pool = queue.SimpleQueue(), ThreadPoolExecutor(max_workers=1) - self.model, self.receiver, self.job = None, None, None - self.selected, self.points, self.candidates = [], {}, [] - self.handles, self.q, self.logs, self.drive = {}, {}, [], {} - self.received, self.redraw = {}, True - self.frame_forces, self.frame_colors, self.frame_counts, self.issues = ( - {}, - {}, - {}, - [], - ) - self.pick_target, self.slot, self.dragging, self.closing = ( - None, - None, - False, - False, - ) - self.sdk_result, self.sdk_revision = "not checked", None - if viser.__version__ != "1.1.1": - raise RuntimeError("Viewport axes require viser 1.1.1") - viewport_axes = Path(__file__).with_name("viewport_axes.js").read_text() + CAMERA_POSITION = (0.3, 0.3, -0.4) + CAMERA_LOOK_AT = (0.0, 0.08, 0.0) + + def __init__(self, state: State, args): + self.state = state + for document in state.documents.values(): + document.alignment = geo.VISUALIZATION_FROM_HAND.tolist() + self.preview = { + side: self._preview_from(document) + for side, document in state.documents.items() + } + self.live_document: Document | None = None + self.live_metadata_key = "" + self.receiver: sdk.Receiver | None = None + self.model: geo.Model | None = None + self.urdf_alignment = geo.VISUALIZATION_FROM_URDF.copy() + self.urdf_scale = 1.0 + self.events: queue.SimpleQueue = queue.SimpleQueue() + self.scene_handles: dict[str, object] = {} + self.segment_row_folder = None + self.taxel_row_folder = None + self.edit_joint_folder = None + self.preview_joint_folder = None + self.preview_segment_folder = None + self.selection_folder = None + self.segment_widgets, self.taxel_widgets = [], [] + self.edit_joint_widgets, self.preview_joint_widgets = {}, {} + self.preview_segment_widgets = [] + self.selected_kind: str | None = None + self.selected_rows: list[int] = [] + self.binding_phase: str | None = None + self.binding_taxels: list[int] = [] + self.binding_segments: list[int] = [] + self.pick_mode: str | None = None + self.selection_rotation = np.zeros(3) + self.selection_scale = 1.0 + self.redraw = True self.server = viser.ViserServer(host="0.0.0.0", port=args.port) self.server.scene.set_up_direction("+y") + source = Path(__file__).with_name("editor_interactions.js").read_text() self.server._websock_server.queue_message( - _messages.RunJavascriptMessage(source=viewport_axes) + _messages.RunJavascriptMessage(source=source) ) - ( - self.server.initial_camera.position, - self.server.initial_camera.look_at, - ) = self.INITIAL_CAMERA_POSE - g = self.server.gui - g.configure_theme(control_width="medium", show_share_button=False) - g.main_panel.dock_right() - self.side = g.add_dropdown("Edit hand", tuple(state.documents)) - self.both = g.add_checkbox("Show both hands", True) - self.mode = g.add_dropdown("Mode", ("EDIT", "PREVIEW", "LIVE", "PAUSED")) - self.watch(self.side, lambda _: self.switch()) - self.watch(self.mode, lambda _: None) - tabs = g.add_tab_group() - with tabs.add_tab("Model"): - self.urdf = g.add_text("URDF", state.model.get("urdf", "")) - self.packages = g.add_text( - "Local package map JSON", - json.dumps(state.model.get("packages", {})), - ) - self.urdf_scale = g.add_number( - "URDF scale", state.model.get("scale", 1.0), min=1e-6 - ) - self.sides = g.add_dropdown( - "New model hands", - ("left", "right", "both"), - initial_value="both" - if len(state.documents) == 2 - else self.side.value, - ) - self.button("New document", lambda: self.guard(self.reset)) - self.button("Load URDF", lambda: self.guard(self.load_new)) - self.button("Apply URDF scale", self.scale_model) - self.open_path = g.add_text("Metadata JSON", args.metadata or "") - self.json_side = g.add_dropdown( - "Assign JSON to", ("left", "right"), initial_value=self.side.value - ) - self.button("Open JSON", lambda: self.guard(self.open)) - self.wrist = g.add_dropdown("Wrist link", ("",)) - self.allowed = g.add_text("Allowed links, comma separated", "") - self.button("Apply allowed links", self.set_links) - self.pick = g.add_dropdown( - "Pick target", - ( - "O", - "C", - "R", - "Wrist O", - "Wrist F", - "Wrist D", - "Segment", - "Move selected", - "Single tangent", - ), - ) - self.pick_button = self.button( - "Pick on q0 model", - lambda: setattr(self, "pick_target", self.pick.value), - ) - self.calibrate_button = self.button( - "Preview / set wrist reference", self.calibrate - ) - self.button("Reset cameras", self.camera) - with tabs.add_tab("Segments"): - self.joint = g.add_dropdown("URDF free joint", ("",)) - self.angle = g.add_slider("Preview joint", -3.15, 3.15, 0.001, 0.0) - self.angle_number = g.add_number("Preview joint value", 0.0) - self.watch(self.joint, lambda _: self.joint_controls()) - for w in (self.angle, self.angle_number): - self.watch(w, self.preview_joint) - self.button("Set shared binding pose q0", self.confirm_q0) - self.button("Return to q0", lambda: setattr(self.mode, "value", "EDIT")) - self.segment = g.add_dropdown("Segment row", ("",)) - self.semantic = g.add_dropdown("HandJoint", tuple(JOINTS)) - self.segment_link = g.add_dropdown("Associated link", ("",)) - self.watch(self.segment, lambda _: self.select_segment()) - self.button("Add segment", self.add_segment) - self.button("Set semantic / link", self.associate) - self.order = g.add_text("Segment row order", "[]") - self.button( - "Reorder / delete omitted segments", - self.reorder_segments, - ) - self.segment_table = g.add_markdown("") - with tabs.add_tab("Layout"): - self.surface = g.add_dropdown("Projection surface", ("",)) - self.rows = g.add_number("Rows", 4, min=1) - self.cols = g.add_number("Columns", 4, min=1) - self.centers = [ - g.add_vector3(k + " world center", (0.0, 0.0, 0.0)) - for k in ("O", "C", "R") - ] - for key, w in zip(("O", "C", "R"), self.centers): - self.watch(w, lambda v, key=key: self.points.update({key: list(v)})) - self.spans = [ - g.add_number(k + " span (m)", 0.03) for k in ("Column", "Row") - ] - self.button("Apply spans along picked directions", self.spans_apply) - self.scan = g.add_dropdown( - "Scan", ("row-major", "column-major", "row-snake", "column-snake") - ) - self.reverse = [ - g.add_checkbox(k, False) - for k in ("Reverse rows", "Reverse columns") - ] - self.excluded = g.add_text("Exclude cells [row,column]", "[]") - self.start = g.add_number("Start output index (-1 = unused)", -1) - self.influences = g.add_text("New taxel segment rows (1–4)", "[]") - self.grid_button = self.button("Preview grid", self.grid) - self.button("Apply candidates", self.apply_grid) - self.button("Cancel candidates", self.clear_layout) - self.layout_info = g.add_markdown("") - self.selection = g.add_text("Select output indices / ranges", "") - self.button("Select indices", self.select_indices) - self.rectangle = g.add_checkbox("Rectangle selection (drag)", False) - self.through = g.add_checkbox("Select through mesh", False) - self.watch(self.rectangle, self.rectangle_mode) - self.button( - "Delete selected taxels", - self.delete_taxels, - ) - self.reproject_button = self.button( - "Reproject selected", self.reproject - ) - self.indices = g.add_text("New selected output indices", "[]") - self.button("Preview / apply output indices", self.reindex) - self.page = g.add_number("List page (25 rows)", 0, min=0) - self.watch(self.page, lambda _: None) - self.table = g.add_markdown("") - with tabs.add_tab("Binding"): - self.binding_info = g.add_markdown("") - self.button( - "Assign segment rows equally", - lambda: self.change( - self.d.assign, self.selected, json.loads(self.influences.value) - ), - ) - self.slots = [] - for k in range(4): - with g.add_folder(f"Slot {k}"): - segment = g.add_dropdown("Segment", ("",)) - index = g.add_number("Set raw index", 0) - slider = g.add_slider("Weight", 0.0, 1.0, 0.001, 0.0) - number = g.add_number("Set weight", 0.0) - self.watch( - segment, - lambda v, k=k: ( - self.set_slot(k, int(v.split(":")[0])) - if not v.startswith("Raw") - else None - ), - ) - self.watch(index, lambda v, k=k: self.set_slot(k, v)) - for w in (slider, number): - self.watch(w, lambda v, k=k: self.weight(k, v)) - self.slots.append((segment, index, slider, number)) - with tabs.add_tab("Live"): - self.collection = g.add_text("Tactile collection_id", "tactile") - self.hand = g.add_dropdown("HandTracker output", ("left", "right")) - self.joint_enabled = g.add_checkbox("Enable JointState", False) - self.joint_id = g.add_text("JointState collection_id", "") - self.button( - "Apply this hand's source configuration", self.configure_source - ) - self.mapping_json = g.add_text( - "Joint mappings JSON: source → [target, scale, offset]", - "{}", - multiline=True, - ) - self.button("Apply joint mappings", self.mapping) - self.button("Connect / disconnect", self.connect) - self.button( - "Play / pause", - lambda: setattr( - self.mode, - "value", - "PAUSED" if self.mode.value == "LIVE" else "LIVE", - ), + self.server.initial_camera.position = self.CAMERA_POSITION + self.server.initial_camera.look_at = self.CAMERA_LOOK_AT + self.server.gui.configure_theme(control_width="medium", show_share_button=False) + self.server.gui.main_panel.dock_right() + self._build_gui() + self._register_scene_events() + self._sync_side() + + @property + def document(self) -> Document: + return self.state.document + + @property + def preview_document(self) -> Document: + return self.preview[self.state.side] + + @staticmethod + def _preview_from(document: Document) -> Document: + preview = document.clone() + preview.taxels = [] + return preview + + def _build_gui(self) -> None: + gui = self.server.gui + self.side_text = gui.add_markdown("") + self._action_button("Toggle Side", self._toggle_side) + self.metadata_path = gui.add_text("Metadata Path", "") + self._action_button("New", self._new) + _, self.save_info = self._action_button("Save", self._save) + self._action_button("Load", self._load) + self._action_button("Reset Camera", self._reset_camera) + self.mode = gui.add_dropdown("Mode", ("Edit", "Preview")) + self.show_saved = gui.add_checkbox("Show Saved Binding Poses", False) + self._watch(self.mode, self._set_mode) + self._watch(self.show_saved, lambda _: self._request_draw()) + self.reorder_bridge = gui.add_text("__Tactile Reorder Event", "") + self._watch(self.reorder_bridge, self._reorder) + + self.edit_tabs = gui.add_tab_group() + self.urdf_tab = self.edit_tabs.add_tab("URDF") + with self.urdf_tab: + self.urdf_path = gui.add_text("URDF Path", "") + self._action_button("Load URDF", self._load_urdf) + with gui.add_folder("URDF Alignment"): + self.urdf_position = self._numbers("Position", (0.0, 0.0, 0.0), 0.001) + self.urdf_rotation = self._sliders("Rotation", (0.0, 0.0, 0.0)) + self.urdf_scale_widget = gui.add_number( + "Scale", 1.0, min=1e-6, step=0.01 + ) + for widget in (*self.urdf_position, *self.urdf_rotation): + self._watch(widget, lambda _: self._update_urdf_alignment()) + self._watch( + self.urdf_scale_widget, lambda _: self._update_urdf_alignment() + ) + self.edit_joint_folder = gui.add_folder("URDF Joints") + + self.segments_tab = self.edit_tabs.add_tab("Segments") + with self.segments_tab: + with gui.add_folder("Segments Alignment"): + self.segment_position = self._numbers( + "Position", (0.0, 0.0, 0.0), 0.001 + ) + self.segment_rotation = self._sliders("Rotation", (0.0, 0.0, 0.0)) + for widget in (*self.segment_position, *self.segment_rotation): + self._watch(widget, lambda _: self._update_segment_alignment()) + with gui.add_folder("Add Segment"): + self.new_segment_joint = gui.add_dropdown("Name", tuple(JOINTS)) + self.new_segment_position = self._numbers( + "Position", (0.0, 0.0, 0.0), 0.001 + ) + self._action_button("Add with Input", self._add_segment_input) + self.segment_click_button, _ = self._action_button( + "Add with URDF Click", self._pick_segment + ) + self.segment_row_folder = gui.add_folder("All Segments") + + self.taxels_tab = self.edit_tabs.add_tab("Taxels") + with self.taxels_tab: + with gui.add_folder("Add Taxels"): + self.grid_columns = gui.add_number("Columns", 4, min=1, step=1) + self.grid_rows = gui.add_number("Rows", 4, min=1, step=1) + self.grid_width = gui.add_number( + "Area Width (m)", 0.03, min=0.0, step=0.001 + ) + self.grid_height = gui.add_number( + "Area Height (m)", 0.03, min=0.0, step=0.001 + ) + self.grid_center = self._numbers( + "Center Position", (0.0, 0.0, 0.0), 0.001 + ) + self._action_button("Add with Input", self._add_taxels_input) + self.taxel_click_button, _ = self._action_button( + "Add with URDF Click", self._pick_taxels + ) + with gui.add_folder("Bind Taxels to Segments"): + self.binding_start = gui.add_button("Start Binding to Segments") + self.binding_next = gui.add_button("Select Segments", disabled=True) + self.binding_complete = gui.add_button("Complete", disabled=True) + self.binding_info = gui.add_markdown("") + self._button(self.binding_start, self._start_binding) + self._button(self.binding_next, self._select_binding_segments) + self._button(self.binding_complete, self._complete_binding) + self.taxel_row_folder = gui.add_folder("All Taxels") + + self.preview_panel = gui.add_folder("Preview", visible=False) + with self.preview_panel: + self.preview_error = gui.add_markdown("") + self.binding_source = gui.add_dropdown( + "Binding Metadata Source", ("Editing",) ) - self.source = g.add_dropdown( - "Visualization metadata", ("Received", "Draft") + self.show_binding = gui.add_checkbox("Show Binding Poses", False) + self.collection = gui.add_text("Tactile Collection ID", "tactile") + self.apply_preview_button, _ = self._action_button( + "Apply Preview Data to Editor", self._apply_preview ) - self.force_max = g.add_number( - "Force color maximum (N)", 30.0, min=0.001 + gui.add_markdown( + "Apply fully overrides the editor's segments, taxels, bindings, and URDF joint values." ) - self.groups = {name: g.add_checkbox(name, True) for name in GROUPS} - self.button("Record metadata groups", self.record) - self.live_info = g.add_markdown("") - with tabs.add_tab("Export"): - self.paths = { - s: g.add_text( - s + " save JSON", state.paths.get(s, s + ".metadata.json") + self._watch(self.binding_source, lambda _: self._binding_source_changed()) + self._watch(self.show_binding, lambda _: self._request_draw()) + with gui.add_folder("Live"): + self.joint_enabled = gui.add_checkbox( + "Use JointState to Drive URDF", False ) - for s in ("left", "right") - } - self.button("Save all metadata", self.save) - self.all_sides = g.add_checkbox("Check / export all hands", True) - self.destination = g.add_text("New output directory", "package") - self.binary = g.add_checkbox("Include SDK binary", False) - self.button("Local check", self.local_check) - self.button("SDK check", lambda: self.background(False)) - self.button("Export snapshot", lambda: self.background(True)) - self.check_info = g.add_markdown("") - with g.add_folder("Advanced schema repair"): - self.schema = g.add_text("Schema JSON", "", multiline=True) - self.button( - "Copy draft into repair field", - lambda: setattr( - self.schema, - "value", - json.dumps(storage.metadata(self.d), indent=2), - ), + self.joint_collection = gui.add_text( + "JointState Collection ID", "", disabled=True ) - self.button("Replace draft from repair field", self.repair) - with g.add_folder("Draft property inspector"): - self.entity = g.add_dropdown("Edit entity", ("Taxels", "Segment")) - self.watch(self.entity, lambda _: None) - self.raw = g.add_markdown("") - self.pose_fields = [] - for axis, k in enumerate("XYZ"): - w = g.add_number("Set " + k + " (m)", 0.0) - self.watch(w, lambda v, axis=axis: self.components([axis], [v])) - self.pose_fields.append(w) - self.button("Flip normal", lambda: self.rotate(np.pi, [1, 0, 0])) - self.twist = g.add_number("Tangent twist (degrees)", 0.0) - self.button( - "Apply tangent twist", - lambda: self.rotate(np.deg2rad(self.twist.value), [0, 0, 1]), - ) - with g.add_folder("Advanced XYZW"): - for axis, k in enumerate("XYZW", 3): - w = g.add_number("Set Q" + k, float(axis == 6)) - self.watch(w, lambda v, axis=axis: self.components([axis], [v])) - self.pose_fields.append(w) - self.radius = g.add_number("Marker radius (m)", 0.002, min=0.0001) - g.add_markdown( - "Draft manipulators always edit q0. Wire markers are draft targets during playback. Ctrl/Cmd toggles selection. Ranges use `0:15,20`." - ) - self.info = g.add_dropdown("Info issue", ("",)) - self.button("Locate issue in inspector", self.locate) - self.log_text = g.add_markdown("") - self.layers = { - s: self.server.scene.add_frame("/view/" + s, show_axes=False) - for s in ("left", "right") - } - self.gizmo = self.server.scene.add_transform_controls( - "/draft-editor", scale=0.05, visible=False - ) - self.gizmo.on_update(self.gizmo_event) - for modifier in (None, "cmd/ctrl"): - self.server.scene.on_click(modifier=modifier)(self.scene_click) - if state.model.get("urdf"): - try: - self.install_model( - geo.Model( - state.model["urdf"], - state.model.get("packages", {}), - state.model.get("scale", 1.0), - ) + self.joint_mapping = gui.add_text( + "Joint Mappings JSON", "{}", multiline=True, disabled=True + ) + self._watch(self.joint_enabled, self._joint_enabled) + self.connect_button, _ = self._action_button( + "Connect Live", self._connect ) - except (OSError, ValueError) as exc: - self.log( - "URDF unavailable; numerical draft editing remains available: " - + str(exc) + gui.add_markdown("Live input overrides values in the Static section.") + self.live_status = gui.add_markdown("Disconnected") + self.static_folder = gui.add_folder("Static") + with self.static_folder: + gui.add_markdown("The following values affect Preview only.") + self._action_button( + "Restore to Editor Binding Values", + self._restore_preview, ) - for widget in (self.both, self.source, self.force_max, self.radius): - self.watch(widget, lambda _: None) - self.switch() - self.refresh() + self.preview_joint_folder = gui.add_folder("URDF Joints") + self.preview_segment_folder = gui.add_folder("Segments") - @property - def d(self): - return self.state.documents[self.side.value] + self.selection_folder = gui.add_folder("Selected Objects", visible=False) + + def _numbers(self, prefix: str, values, step: float): + return tuple( + self.server.gui.add_number(f"{prefix} {axis}", float(value), step=step) + for axis, value in zip("XYZ", values) + ) - def watch(self, widget, callback): + def _sliders(self, prefix: str, values): + return tuple( + self.server.gui.add_slider( + f"{prefix} {axis} (deg)", -180.0, 180.0, 1.0, float(value) + ) + for axis, value in zip("XYZ", values) + ) + + def _watch(self, widget, callback) -> None: async def updated(event): - if event.client: + if event.client is not None: value = copy.deepcopy(event.target.value) - self.events.put(lambda: callback(value)) + self.events.put(lambda: self._run_action(callback, value)) widget.on_update(updated) - def button(self, label, callback): - handle = self.server.gui.add_button(label) - + def _button(self, widget, callback) -> None: async def clicked(_): - self.events.put(callback) + self.events.put(lambda: self._run_action(callback)) - handle.on_click(clicked) - return handle + widget.on_click(clicked) - def log(self, message): - LOG.info("%s", message) - self.logs = (self.logs + [str(message)])[-8:] - self.log_text.content = "\n\n".join(self.logs) + def _action_button(self, label: str, callback): + button = self.server.gui.add_button(label) + info = self.server.gui.add_markdown("") - def confirm(self, message, action, unsaved=False, cleanup=None): - modal = self.server.gui.add_modal("Confirm") + async def clicked(_): + self.events.put(lambda: self._run_action(callback, info=info)) - def close(): - if cleanup: - cleanup() - modal.close() + button.on_click(clicked) + return button, info - def finish(save=False): - if save: - self.save() - action() - close() + def _run_action(self, callback, value=None, info=None) -> None: + try: + callback() if value is None else callback(value) + if info is not None: + info.content = "" + except Exception as exc: + LOG.exception("Editor action failed") + if info is not None: + info.content = f'Error: {exc}' + else: + self.preview_error.content = ( + f'Error: {exc}' + ) + self.redraw = True + def _request_draw(self) -> None: + self.redraw = True + + def _set_mode(self, value: str) -> None: + self.edit_tabs.visible = value == "Edit" + self.preview_panel.visible = value == "Preview" + self.pick_mode = None + self.selected_kind, self.selected_rows = None, [] + self._cancel_binding() + self._rebuild_selection() + if value == "Preview": + self._update_preview_error() + + def _toggle_side(self) -> None: + self._disconnect() + self.state.side = "right" if self.state.side == "left" else "left" + self._sync_side() + + def _sync_side(self) -> None: + self.live_document, self.live_metadata_key = None, "" + self.pick_mode = None + self.segment_click_button.label = "Add with URDF Click" + self.taxel_click_button.label = "Add with URDF Click" + self.binding_source.options = ("Editing",) + self.binding_source.value = "Editing" + self.side_text.content = f"**Side: {self.state.side.title()}**" + self.metadata_path.value = self.state.paths.get(self.state.side, "") + position, rotation = geo.components( + np.asarray(self.document.alignment, dtype=float) + ) + self._set_values(self.segment_position, position) + self._set_values(self.segment_rotation, rotation) + urdf_position, urdf_rotation = geo.components(self.urdf_alignment) + self._set_values(self.urdf_position, urdf_position) + self._set_values(self.urdf_rotation, urdf_rotation) + self.urdf_scale_widget.value = self.urdf_scale + self.selected_kind, self.selected_rows = None, [] + self._cancel_binding() + self._rebuild_edit_lists() + self._rebuild_joint_controls() + self._rebuild_preview_segments() + self._rebuild_selection() + self.redraw = True + + @staticmethod + def _set_values(widgets, values) -> None: + for widget, value in zip(widgets, values): + widget.value = float(value) + + def _new(self) -> None: + document = Document(self.state.side) + document.alignment = geo.VISUALIZATION_FROM_HAND.tolist() + self.state.documents[self.state.side] = document + self.state.saved[self.state.side] = None + self.state.paths.pop(self.state.side, None) + self.preview[self.state.side] = self._preview_from(document) + self._sync_side() + + def _load(self) -> None: + path = self.metadata_path.value.strip() + if not path: + raise ValueError("Enter Metadata Path") + document = storage.load(path, self.state.side) + self.state.replace(document, path) + self.preview[self.state.side] = self._preview_from(document) + self._sync_side() + + def _save(self) -> None: + path = self.metadata_path.value.strip() + if not path: + raise ValueError("Enter Metadata Path") + issues = self.document.validate() + if not issues: + self._write_save(path) + return + modal = self.server.gui.add_modal("Save Metadata") with modal: - self.server.gui.add_markdown(message) - if unsaved: - self.button("Save and continue", lambda: finish(True)) - self.button("Discard and continue" if unsaved else "Apply", finish) - self.button("Cancel", close) - - def guard(self, action): - if self.state.dirty(): - self.confirm("Unsaved drafts", action, True) - else: - action() + self.server.gui.add_markdown( + f"Metadata contain errors, are you sure to save? ({len(issues)} error(s))" + ) + save_button = self.server.gui.add_button("Continue") + cancel_button = self.server.gui.add_button("Cancel") + + async def save(_): + self.events.put( + lambda: self._run_action( + lambda: (self._write_save(path), modal.close()), + info=self.save_info, + ) + ) - def change(self, action, *args, **kwargs): - return self.state.edit(action, *args, **kwargs) + async def cancel(_): + self.events.put(modal.close) - def segment_options(self): - return ( - "", - *map(str, range(max(len(self.d.joints), len(self.d.arrays[FIELDS[0]])))), - ) + save_button.on_click(save) + cancel_button.on_click(cancel) - def install_model(self, model): - if self.model: - for h in self.model.handles.values(): - h.remove() - self.model = model - self.wrist.options = self.segment_link.options = tuple(model.links) - self.joint.options = tuple(j.name for j in model.joints) or ("",) - self.q = copy.deepcopy(self.state.model.get("q0", {})) - self.sides.disabled = True - self.sides.value = "both" if len(self.state.documents) == 2 else self.side.value - self.joint_controls() - - def reset(self): - if self.receiver: - self.connect() - for h in ( - *self.handles.values(), - *(self.model.handles.values() if self.model else ()), - ): - h.remove() - self.handles, self.model = {}, None - self.state, self.started = State({"left": Document("left")}), False - self.state.saved["left"] = self.state.fingerprint("left") - self.side.options, self.side.value = ("left",), "left" - self.sides.disabled, self.mode.value = False, "EDIT" - self.frame_forces, self.frame_colors, self.frame_counts, self.drive = ( - {}, - {}, - {}, - {}, - ) - self.q = {} - for w in ( - self.wrist, - self.joint, - self.segment_link, - self.surface, - ): - w.options = ("",) - for side, w in self.paths.items(): - w.value = side + ".metadata.json" - self.switch() - - def load_new(self): - cfg = dict( - urdf=str(Path(self.urdf.value).resolve()), - packages={ - k: str(Path(v).resolve()) - for k, v in json.loads(self.packages.value).items() - }, - scale=self.urdf_scale.value, - q0={}, - ) - model = geo.Model(cfg["urdf"], cfg["packages"], cfg["scale"]) - sides = ("left", "right") if self.sides.value == "both" else (self.sides.value,) - self.reset() - self.state, self.started = State({s: Document(s) for s in sides}, cfg), True - self.side.options, self.side.value = sides, sides[0] - self.install_model(model) - self.switch() - - def scale_model(self): - if not self.model: - raise ValueError("Load a URDF before applying its scale") - scale = self.urdf_scale.value - model = geo.Model( - self.state.model["urdf"], - self.state.model.get("packages", {}), - scale, - ) - self.change(self.state.model.update, scale=scale) - self.install_model(model) - self.clear_layout() - - def open(self): - side, path = self.json_side.value, self.open_path.value - d = storage.load(path, side) - if self.model and side not in self.state.documents: - raise ValueError("Create a new document to change the fixed model hands") - - def replace(): - if not self.started: - self.state.documents = {} - self.state.documents[side] = d - - if self.receiver: - self.connect() - self.mode.value = "EDIT" - self.state.edit(replace) - self.started = True - self.side.options, self.side.value = tuple(self.state.documents), side - self.state.paths[side] = self.paths[side].value = str(Path(path).resolve()) - self.state.saved[side] = self.state.fingerprint(side) - self.switch() - - def save(self): - storage.save(self.state, {s: self.paths[s].value for s in self.state.documents}) - self.log("Saved all metadata") - - def switch(self): - self.selected, self.points, self.candidates = [], {}, [] - self.pick_target, self.slot = None, None - self.segment.value = "" - self.influences.value = "[]" - self.allowed.value = ",".join(self.d.editor.get("allowed_links", [])) - if self.model: - link = self.d.editor.get("root", {}).get("link") - self.wrist.value = link if link in self.model.links else self.model.links[0] - source = self.d.editor.get("source", {}) - self.collection.value, self.hand.value = ( - source.get("collection", "tactile"), - source.get("hand", self.d.side), - ) - self.joint_id.value = source.get("joint_state", "") - self.joint_enabled.value = bool(self.joint_id.value) - self.mapping_json.value = json.dumps( - self.d.editor.get("mapping", {}), indent=2 - ) - self.segment.options = self.segment_options() + def _write_save(self, path: str) -> None: + storage.save(self.state, path) + self.metadata_path.value = self.state.paths[self.state.side] + self.redraw = True - def root(self, d=None, cfg=None): - d = d or self.d - if self.model and "root" in d.editor: - try: - return self.model.root( - d, - self.model.fk( - cfg if cfg is not None else self.state.model.get("q0", {}) - ), - ) - except (KeyError, ValueError, TypeError): - pass - return geo.VISUALIZATION_FROM_HAND.copy() - - def surfaces(self): - return [ - n - for n, (link, _) in self.model.surfaces.items() - if link in self.d.editor.get("allowed_links", []) - ] - - def set_links(self): - links = [v.strip() for v in self.allowed.value.split(",") if v.strip()] - if not self.model or any(v not in self.model.links for v in links): - raise ValueError("Choose links from the loaded model") - self.change(self.d.editor.update, allowed_links=links) - - def camera(self): + def _reset_camera(self) -> None: for client in self.server.get_clients().values(): - client.camera.position, client.camera.look_at = self.INITIAL_CAMERA_POSE - - def calibrate(self): - o, f, d = [ - np.asarray(self.points[k]) for k in ("Wrist O", "Wrist F", "Wrist D") - ] - z = geo.unit(f - o) - y = geo.unit(d - o - z * ((d - o) @ z)) - reference = geo.frame(o, z, np.cross(y, z)) - for other in self.state.documents.values(): - if ( - other is not self.d - and other.editor.get("root", {}).get("link") == self.wrist.value - ): - raise ValueError( - "Both hands require distinct wrist links in the actual model" - ) - self.axes("/calibration-preview", [reference]) - local = ( - np.linalg.inv( - self.model.fk(self.state.model.get("q0", {}))[self.wrist.value] - ) - @ reference - ) - value = dict(link=self.wrist.value, frame=local.tolist()) - - def apply(): - self.change(self.d.editor.update, root=value) - self.clear_layout() - - self.confirm( - "Preserve all numerical bind poses and weights; recompute attachments at this wrist reference.", - apply, - cleanup=lambda: setattr( - self.handles["/calibration-preview"], "visible", False - ), + client.camera.position = self.CAMERA_POSITION + client.camera.look_at = self.CAMERA_LOOK_AT + + def _load_urdf(self) -> None: + path = self.urdf_path.value.strip() + if not path: + raise ValueError("Enter URDF Path") + if self.model is not None: + self.model.remove() + self.model = geo.Model(path) + self._rebuild_joint_controls() + + def _update_urdf_alignment(self) -> None: + self.urdf_alignment = geo.transform( + [widget.value for widget in self.urdf_position], + [widget.value for widget in self.urdf_rotation], ) + self.urdf_scale = float(self.urdf_scale_widget.value) + if not math.isfinite(self.urdf_scale) or self.urdf_scale <= 0: + raise ValueError("URDF Scale must be positive") + self.redraw = True - def joint_controls(self): - if self.model and self.joint.value: - j = self.model.urdf.joint_map[self.joint.value] - low = j.limit.lower if j.limit and j.limit.lower is not None else -np.pi - high = j.limit.upper if j.limit and j.limit.upper is not None else np.pi - self.angle.min, self.angle.max = low, high - self.angle_number.min = j.limit.lower if j.limit else None - self.angle_number.max = j.limit.upper if j.limit else None - self.angle_number.value = self.q.get(j.name, 0.0) - self.angle.value = float(np.clip(self.angle_number.value, low, high)) - - def preview_joint(self, value): - if self.joint.value: - self.q[self.joint.value] = value - self.angle_number.value = value - self.angle.value = float(np.clip(value, self.angle.min, self.angle.max)) - self.mode.value = "PREVIEW" - - def confirm_q0(self): - q0 = copy.deepcopy(self.q) - - def apply(): - self.change(self.state.model.update, q0=q0) - self.mode.value = "EDIT" - self.clear_layout() - - self.confirm( - "Update shared q0; bind poses and weights stay unchanged and points may leave the mesh. Affected hands: " - + ", ".join(self.state.documents), - apply, + def _update_segment_alignment(self) -> None: + alignment = geo.transform( + [widget.value for widget in self.segment_position], + [widget.value for widget in self.segment_rotation], ) - - def add_segment(self): - initial = IDENTITY[:] if self.semantic.value == "WRIST" else None - if self.semantic.value != "WRIST" and self.model and "root" in self.d.editor: - initial = geo.pose( - np.linalg.inv(self.root()) - @ self.model.fk(self.state.model.get("q0", {}))[self.segment_link.value] + delta = geo.realign(self.document, alignment) + saved = self.state.saved[self.state.side] + if saved is not None: + geo.apply_delta(saved.segments, list(range(len(saved.segments))), delta) + geo.apply_delta(saved.taxels, list(range(len(saved.taxels))), delta) + saved.alignment = alignment.tolist() + preview = self.preview_document + geo.apply_delta(preview.segments, list(range(len(preview.segments))), delta) + preview.alignment = alignment.tolist() + if self.live_document is not None: + geo.apply_delta( + self.live_document.segments, + list(range(len(self.live_document.segments))), + delta, + ) + geo.apply_delta( + self.live_document.taxels, + list(range(len(self.live_document.taxels))), + delta, ) + self.live_document.alignment = alignment.tolist() + self._rebuild_edit_lists() + self._rebuild_preview_segments() + + def _rebuild_joint_controls(self) -> None: + if self.edit_joint_folder is not None: + self.edit_joint_folder.remove() + if self.preview_joint_folder is not None: + self.preview_joint_folder.remove() + self.edit_joint_widgets, self.preview_joint_widgets = {}, {} + with self.urdf_tab: + self.edit_joint_folder = self.server.gui.add_folder("URDF Joints") + with self.edit_joint_folder: + self._add_joint_sliders(self.document, self.edit_joint_widgets) + with self.static_folder: + self.preview_joint_folder = self.server.gui.add_folder("URDF Joints") + with self.preview_joint_folder: + self._add_joint_sliders( + self.preview_document, self.preview_joint_widgets + ) - def add(): - self.d.joints.append(self.semantic.value) - self.d.arrays[FIELDS[0]].append(initial) - self.d.sync() - self.d.editor.setdefault("segment_links", {})[ - self.d.editor["segment_ids"][-1] - ] = self.segment_link.value - self.d.editor["approximate"] = True - - self.change(add) - # Register new rows before selection; the event loop refreshes the inspector. - self.segment.options = self.segment_options() - self.segment.value = str(len(self.d.arrays[FIELDS[0]]) - 1) - self.select_segment() - - def associate(self): - i = int(self.segment.value) - - def update(): - self.d.joints[i] = self.semantic.value - self.d.editor.setdefault("segment_links", {})[ - self.d.editor["segment_ids"][i] - ] = self.segment_link.value - - self.change(update) - - def select_segment(self): - self.entity.value, self.slot = "Segment", None - if self.segment.value: - i = int(self.segment.value) - if i < len(self.d.joints): - self.semantic.value = JOINTS[joint_number(self.d.joints[i])] - link = self.d.editor.get("segment_links", {}).get( - self.d.editor["segment_ids"][i] + def _add_joint_sliders(self, document: Document, output: dict) -> None: + if self.model is None: + self.server.gui.add_markdown("Load a URDF to edit joints.") + return + for joint in self.model.joints: + lower = ( + joint.limit.lower + if joint.limit and joint.limit.lower is not None + else -math.pi + ) + upper = ( + joint.limit.upper + if joint.limit and joint.limit.upper is not None + else math.pi + ) + value = min(upper, max(lower, document.q.get(joint.name, 0.0))) + widget = self.server.gui.add_slider( + joint.name, float(lower), float(upper), 0.001, float(value) + ) + self._watch( + widget, + lambda current, name=joint.name, document=document: ( + document.q.__setitem__(name, float(current)) + ), ) - if self.model and link in self.model.links: - self.segment_link.value = link - - def targets(self): - if self.entity.value == "Segment" and self.segment.value: - return FIELDS[0], [int(self.segment.value)] - return FIELDS[1], self.selected - - def reorder_segments(self): - ids = self.d.editor["segment_ids"] - selected = ( - ids[int(self.segment.value)] - if self.segment.value and int(self.segment.value) < len(ids) - else None + output[joint.name] = widget + + def _add_segment_input(self) -> None: + pose = IDENTITY_POSE.copy() + pose[:3] = [float(widget.value) for widget in self.new_segment_position] + row = self.document.add_segment(self.new_segment_joint.value, pose) + self.selected_kind, self.selected_rows = "segments", [row] + self._rebuild_edit_lists() + self._rebuild_selection() + + def _pick_segment(self) -> None: + self.pick_mode = None if self.pick_mode == "segment" else "segment" + self.segment_click_button.label = ( + "● Add with URDF Click" if self.pick_mode else "Add with URDF Click" ) - self.change(self.d.segments, json.loads(self.order.value)) - ids = self.d.editor["segment_ids"] - self.segment.value = str(ids.index(selected)) if selected in ids else "" - def delete_taxels(self): - self.change(self.d.delete, self.selected) - self.selected = [] - - def components(self, components, values): - field, rows = self.targets() - if not rows: - raise ValueError("Select a draft segment or taxel first") - self.change(self.d.set_components, field, rows, components, values) - - def rotate(self, angle, axis): - field, rows = self.targets() - if not rows: - raise ValueError("Select a draft segment or taxel first") - self.change( - geo.transform, - self.d, - field, - rows, - tf.rotation_matrix(angle, axis), - local=True, + def _grid_values(self): + return ( + [float(widget.value) for widget in self.grid_center], + float(self.grid_width.value), + float(self.grid_height.value), + int(self.grid_rows.value), + int(self.grid_columns.value), ) - # Inline async callbacks preserve Viser's drag phase ordering. - async def gizmo_event(self, event): - if event.phase == "start" and event.client: - self.dragging = True - elif event.phase == "end": - if event.client: - self.drag_end(event) - else: - self.dragging = False - - def drag_end(self, event): - values = [*event.target.position, *np.asarray(event.target.wxyz)[[1, 2, 3, 0]]] - - def apply(): - try: - field, rows = self.targets() - if rows: - old = geo.matrix(self.d.arrays[field][rows[0]]) - delta = ( - np.linalg.inv(self.root()) - @ geo.matrix(values) - @ np.linalg.inv(old) - ) - self.change(geo.transform, self.d, field, rows, delta) - finally: - self.dragging = False - - self.events.put(apply) - - def set_slot(self, k, value): - self.slot = k - self.change(self.d.set_components, FIELDS[2], self.selected, [k], [value]) - - def weight(self, k, value): - self.slot = k - rejected = self.change(self.d.weight, self.selected, k, value) - if rejected: - self.log("; ".join(rejected)) - - def spans_apply(self): - o = np.asarray(self.points["O"]) - for key, control, span in zip(("C", "R"), self.centers[1:], self.spans): - if key in self.points: - self.points[key] = ( - o + geo.unit(np.asarray(self.points[key]) - o) * span.value - ).tolist() - control.value = tuple(self.points[key]) - - def grid(self): - if self.surface.value not in self.surfaces(): - raise ValueError("Choose an allowed projection surface") - self.candidates = self.model.grid( - self.d, - self.model.fk(self.state.model.get("q0", {})), - self.surface.value, - [self.points.get(k) for k in ("O", "C", "R")], - int(self.rows.value), - int(self.cols.value), - self.scan.value, - [w.value for w in self.reverse], - json.loads(self.excluded.value), - ) - start = int(self.start.value) - start = self.d.unused_start(len(self.candidates)) if start < 0 else start - spans = [ - np.linalg.norm(np.asarray(self.points[k]) - self.points["O"]) - if k in self.points - else 0 - for k in ("C", "R") - ] - spacing = [ - v / (n - 1) if n > 1 else 0 - for v, n in zip(spans, (self.cols.value, self.rows.value)) - ] - self.layout_info.content = f"Spans C/R: {spans} m; spacing C/R: {spacing} m. {len(self.candidates)} candidates; {sum(p is None for p, _ in self.candidates)} projection failures. Output indices: {list(range(start, start + len(self.candidates)))}" - - def apply_grid(self): - self.change( - self.d.add, - [p for p, _ in self.candidates], - json.loads(self.influences.value), - int(self.start.value), + def _add_taxels_input(self) -> None: + center, width, height, rows, columns = self._grid_values() + selected = self.document.add_taxels( + geo.grid_poses(center, width, height, rows, columns) ) - self.clear_layout() - - def clear_layout(self): - self.candidates, self.points = [], {} - self.rows.value = self.cols.value = 4 - self.scan.value, self.start.value, self.excluded.value = "row-major", -1, "[]" - for w in self.reverse: - w.value = False - for w in self.centers: - w.value = (0.0, 0.0, 0.0) - for w in self.spans: - w.value = 0.03 - self.layout_info.content = "" - - def select_indices(self): - indices = [] - for part in self.selection.value.split(","): - ends = [int(v) for v in part.strip().split(":")] - indices.extend(range(ends[0], ends[1] + 1) if len(ends) == 2 else ends) - self.selected = [ - i - for i, index in enumerate(self.d.editor["output_indices"]) - if index in indices - ] - self.entity.value = "Taxels" - - def reindex(self): - values, order = ( - json.loads(self.indices.value), - self.d.editor["output_indices"][:], - ) - if len(values) != len(self.selected) or any( - type(v) is not int or v < 0 for v in values - ): - raise ValueError("Supply one nonnegative integer per selected taxel") - preview = [(order[i], value) for i, value in zip(self.selected, values)] - for i, value in zip(self.selected, values): - order[i] = value - self.confirm( - str(preview), - lambda: self.change(self.d.editor.update, output_indices=order), + self.selected_kind, self.selected_rows = "taxels", selected + self._rebuild_edit_lists() + self._rebuild_selection() + + def _pick_taxels(self) -> None: + self.pick_mode = None if self.pick_mode == "taxels" else "taxels" + self.taxel_click_button.label = ( + "● Add with URDF Click" if self.pick_mode else "Add with URDF Click" ) - def reproject(self): - root, fk = self.root(), self.model.fk(self.state.model.get("q0", {})) - if self.surface.value not in self.surfaces(): - raise ValueError("Choose an allowed projection surface") + def _rebuild_edit_lists(self) -> None: + self._rebuild_segment_list() + self._rebuild_taxel_list() + self._rebuild_selection() + self._update_preview_error() + self.redraw = True - def apply(): - for i in self.selected: - old = root @ geo.matrix(self.d.arrays[FIELDS[1]][i]) - hit = self.model.hit( - old[:3, 3] + old[:3, 2], -old[:3, 2], [self.surface.value], fk - ) - if not hit: - raise ValueError( - f"Taxel {i}: projection failed; selection unchanged" + def _rebuild_segment_list(self) -> None: + if self.segment_row_folder is not None: + self.segment_row_folder.remove() + self.segment_widgets = [] + issues = self.document.validate() + with self.segments_tab: + self.segment_row_folder = self.server.gui.add_folder("All Segments") + with self.segment_row_folder: + for row, segment in enumerate(self.document.segments): + row_issues = [ + issue + for issue in issues + if issue["kind"] == "segments" and issue["row"] == row + ] + self.server.gui.add_html( + f'
⠿ Drag
' ) - self.d.arrays[FIELDS[1]][i] = geo.pose( - np.linalg.inv(root) @ geo.frame(hit[1], hit[2], old[:3, 0]) - ) - self.d.editor["approximate"] = True + with self.server.gui.add_folder(f"{row}: {segment.joint}"): + joint = self.server.gui.add_dropdown( + "⚠ Name" + if any(issue["field"] == "joint" for issue in row_issues) + else "Name", + tuple(JOINTS), + initial_value=segment.joint, + ) + position, rotation = geo.components(geo.matrix(segment.pose)) + positions = self._numbers("Position", position, 0.001) + rotations = self._sliders("Rotation", rotation) + self._watch( + joint, + lambda value, row=row: self._set_segment_joint(row, value), + ) + for axis, widget in enumerate(positions): + self._watch( + widget, + lambda value, row=row, axis=axis: ( + self._set_pose_component( + "segments", row, axis, value + ) + ), + ) + for axis, widget in enumerate(rotations): + self._watch( + widget, + lambda value, row=row, axis=axis: ( + self._set_pose_rotation( + "segments", row, axis, value + ) + ), + ) + messages = [issue["message"] for issue in row_issues] + if messages: + self.server.gui.add_html( + '
' + + "
".join(messages) + + "
" + ) + self.segment_widgets.append((joint, positions, rotations)) + + def _rebuild_taxel_list(self) -> None: + if self.taxel_row_folder is not None: + self.taxel_row_folder.remove() + self.taxel_widgets = [] + issues = self.document.validate() + options = ("None", *[segment.joint for segment in self.document.segments]) + with self.taxels_tab: + self.taxel_row_folder = self.server.gui.add_folder("All Taxels") + with self.taxel_row_folder: + for row, taxel in enumerate(self.document.taxels): + row_issues = [ + issue + for issue in issues + if issue["kind"] == "taxels" and issue["row"] == row + ] + self.server.gui.add_html( + f'
⠿ Drag
' + ) + with self.server.gui.add_folder(f"Taxel {row}"): + position, rotation = geo.components(geo.matrix(taxel.pose)) + positions = self._numbers("Position", position, 0.001) + rotations = self._sliders("Rotation", rotation) + for axis, widget in enumerate(positions): + self._watch( + widget, + lambda value, row=row, axis=axis: ( + self._set_pose_component("taxels", row, axis, value) + ), + ) + for axis, widget in enumerate(rotations): + self._watch( + widget, + lambda value, row=row, axis=axis: ( + self._set_pose_rotation("taxels", row, axis, value) + ), + ) + slots = [] + for slot in range(4): + with self.server.gui.add_folder(f"Binding {slot}"): + selected = ( + taxel.segments[slot] + if taxel.segments[slot] in options + else "None" + ) + segment = self.server.gui.add_dropdown( + "⚠ Segment" + if any( + issue["field"] in (f"segment{slot}", "binding") + for issue in row_issues + ) + else "Segment", + options, + initial_value=selected, + ) + weight = self._weight_widget( + taxel.weights[slot], + selected == "None", + any( + issue["field"] in (f"weight{slot}", "weights") + for issue in row_issues + ), + ) + self._watch( + segment, + lambda value, row=row, slot=slot: ( + self._set_taxel_segment(row, slot, value) + ), + ) + self._watch( + weight, + lambda value, row=row, slot=slot: ( + self._set_taxel_weight(row, slot, value) + ), + ) + slots.append((segment, weight)) + messages = [issue["message"] for issue in row_issues] + if messages: + self.server.gui.add_html( + '
' + + "
".join(dict.fromkeys(messages)) + + "
" + ) + self.taxel_widgets.append((positions, rotations, slots)) + + def _weight_widget(self, value, disabled: bool, invalid: bool): + label = "⚠ Weight" if invalid else "Weight" + if type(value) in (int, float) and math.isfinite(value): + limits = {"min": 0.0, "max": 1.0} if 0.0 <= value <= 1.0 else {} + return self.server.gui.add_number( + label, float(value), step=0.01, disabled=disabled, **limits + ) + return self.server.gui.add_text(label, str(value), disabled=disabled) - self.change(apply) + def _set_segment_joint(self, row: int, value: str) -> None: + self.document.segments[row].joint = value + self._rebuild_edit_lists() - def configure_source(self): - source = dict( - collection=self.collection.value, - hand=self.hand.value, - joint_state=self.joint_id.value if self.joint_enabled.value else "", - ) - if not source["collection"].strip() or ( - self.joint_enabled.value and not source["joint_state"].strip() - ): - raise ValueError("Enabled sources require collection_id values") - self.change(self.d.editor.update, source=source) - if self.receiver: - self.log("Source configuration updated; reconnect to apply it") - - def mapping(self): - mappings = json.loads(self.mapping_json.value) - targets = {joint.name for joint in self.model.joints} if self.model else set() - if not isinstance(mappings, dict) or any( - not isinstance(source, str) - or not source - or not isinstance(values, list) - or len(values) != 3 - or not isinstance(values[0], str) - or values[0] not in targets - or any( - type(value) not in (int, float) or not math.isfinite(value) - for value in values[1:] - ) - for source, values in mappings.items() - ): - raise ValueError( - "Expected {source: [URDF target joint, finite scale, finite offset]}" - ) - self.change(self.d.editor.update, mapping=mappings) + def _values(self, kind: str): + return self.document.segments if kind == "segments" else self.document.taxels - def connect(self): - if self.receiver: - if self.receiver is not True: - self.receiver.close() - self.receiver = None - self.log("Disconnected") - else: - self.configure_source() - self.configs = { - s: dict( - d.editor.get( - "source", dict(collection="tactile", hand=s, joint_state="") - ), - mapping=d.editor.get("mapping", {}), - ) - for s, d in self.state.documents.items() - } - self.receiver = True if self.args.demo else sdk.Receiver(self.configs) - self.mode.value = "LIVE" + def _set_pose_component(self, kind: str, row: int, axis: int, value) -> None: + self._values(kind)[row].pose[axis] = float(value) + self.redraw = True - def latest(self): - return ( - sdk.demo(self.configs) - if self.receiver is True - else self.receiver.latest - if self.receiver - else {} - ) + def _set_pose_rotation(self, kind: str, row: int, axis: int, value) -> None: + current = geo.matrix(self._values(kind)[row].pose) + position, rotation = geo.components(current) + rotation = list(rotation) + rotation[axis] = float(value) + self._values(kind)[row].pose = geo.pose(geo.transform(position, rotation)) + self.redraw = True - def record(self): - value = self.latest().get(self.side.value, {}).get("metadata") - if value is None: - raise ValueError("No received metadata") - received = storage.parse(value, self.side.value) - path = storage.backup(self.d) - self.log("Draft backup: " + path) - self.change( - self.d.record, received, [k for k, w in self.groups.items() if w.value] - ) + def _set_taxel_segment(self, row: int, slot: int, value: str) -> None: + taxel = self.document.taxels[row] + taxel.segments[slot] = None if value == "None" else value + if taxel.segments[slot] is None: + taxel.weights[slot] = 0.0 + self._rebuild_taxel_list() + + def _set_taxel_weight(self, row: int, slot: int, value) -> None: + try: + self.document.set_weight(row, slot, float(value)) + finally: + self._rebuild_taxel_list() - def repair(self): - d = storage.parse(json.loads(self.schema.value), self.side.value) - order = d.editor["output_indices"] - d.editor = copy.deepcopy(self.d.editor) - d.editor["output_indices"] = order - d.sync() - self.log("Draft backup: " + storage.backup(self.d)) - self.change(self.state.documents.__setitem__, self.side.value, d) - - def local_check(self): - documents = ( - self.state.documents if self.all_sides.value else {self.side.value: self.d} + def _reorder(self, value: str) -> None: + if not value: + return + parts = value.split(":") + if len(parts) != 4: + return + kind, source, target, _ = parts + if kind not in ("segments", "taxels"): + return + self.document.move(kind, int(source), int(target)) + self._rebuild_edit_lists() + + def _start_binding(self) -> None: + self.pick_mode = None + self.segment_click_button.label = "Add with URDF Click" + self.taxel_click_button.label = "Add with URDF Click" + self.binding_phase = "taxels" + self.binding_taxels, self.binding_segments = [], [] + self.selected_kind, self.selected_rows = None, [] + self.binding_next.disabled = True + self.binding_complete.disabled = True + self.binding_info.content = ( + "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. " + 'Click "Select Segments" after selecting taxels. Click another area to cancel.' ) - self.log(str({s: d.check() for s, d in documents.items()})) - def background(self, exporting): - if self.job and not self.job.done(): - raise ValueError("A check/export is already running") - documents = copy.deepcopy( - self.state.documents if self.all_sides.value else {self.side.value: self.d} + def _select_binding_segments(self) -> None: + if not self.binding_taxels: + raise ValueError("Select at least one taxel") + self.binding_phase = "segments" + self.binding_info.content = ( + 'Select 1-4 segment(s). Hold Shift to multi-select. Click "Complete" ' + "after selecting segments. Click another area to cancel." ) - self.job_revision = self.state.revision - self.sdk_revision = self.job_revision - if exporting: - self.job = self.pool.submit( - storage.export, - documents, - self.job_revision, - self.destination.value, - self.binary.value, - ) - else: - self.job = self.pool.submit( - lambda: {s: sdk.check(d) for s, d in documents.items()} + + def _complete_binding(self) -> None: + if not 1 <= len(self.binding_segments) <= 4: + self.binding_info.content = ( + 'Too many segments selected. Select 1-4 segments and click "Start ' + 'Binding to Segments" to start over.' ) - self.sdk_result = f"Running for revision {self.job_revision}" + return + self.document.assign(self.binding_taxels, self.binding_segments) + self._cancel_binding() + self._rebuild_taxel_list() + + def _cancel_binding(self) -> None: + self.binding_phase = None + self.binding_taxels, self.binding_segments = [], [] + if hasattr(self, "binding_next"): + self.binding_next.disabled = True + self.binding_complete.disabled = True + self.binding_info.content = "" + + def _joint_enabled(self, enabled: bool) -> None: + self.joint_collection.disabled = not enabled + self.joint_mapping.disabled = not enabled + + def _connect(self) -> None: + if self.receiver is not None: + self._disconnect() + return + config = { + "side": self.state.side, + "collection": self.collection.value, + "joint_state": self.joint_collection.value + if self.joint_enabled.value + else "", + } + self.receiver = sdk.Receiver(config) + self.connect_button.label = "Disconnect Live" + self.live_status.content = "Connecting" - def refresh(self): + def _disconnect(self) -> None: + if self.receiver is not None: + self.receiver.close() + self.receiver = None + self.connect_button.label = "Connect Live" + self.live_status.content = "Disconnected" self.redraw = True - d = self.d - self.issues = [ - dict(v, side=s) - for s, doc in self.state.documents.items() - for v in doc.check() - ] - if self.model: - fk = self.model.fk(self.state.model.get("q0", {})) - self.model.attachments.clear() - for doc in self.state.documents.values(): - try: - if "root" in doc.editor: - self.model.attach(doc, fk) - except (ValueError, KeyError, TypeError) as exc: - self.model.attachments.pop(doc.side, None) - self.issues.append( - dict( - side=doc.side, - severity="warning", - field="Model", - row=None, - message=str(exc), - ) - ) - self.surface.options = tuple(self.surfaces()) or ("",) - calibrated = bool(self.model and d.side in self.model.attachments) - self.pick_button.disabled = self.calibrate_button.disabled = self.model is None - self.grid_button.disabled = self.reproject_button.disabled = not calibrated - self.angle.disabled = self.angle_number.disabled = self.model is None - self.selected = [i for i in self.selected if 0 <= i < len(d.arrays[FIELDS[1]])] - self.side.visible = self.both.visible = len(self.state.documents) == 2 - self.segment.options = self.segment_options() - links = d.editor.get("segment_links", {}) - self.segment_table.content = "\n\n".join( - f"row {i}: {j}; link={links.get(sid, 'unassigned')}; attachment={'ready' if self.model and sid in self.model.attachments.get(d.side, {}) else 'missing'}" - for i, (j, sid) in enumerate(zip(d.joints, d.editor["segment_ids"])) - ) - first = int(self.page.value) * 25 - self.table.content = "\n\n".join( - f"row {i}: output {d.editor['output_indices'][i]}; {p}" - for i, p in enumerate(d.arrays[FIELDS[1]]) - if first <= i < first + 25 - ) - field, rows = self.targets() - values = [ - d.arrays[field][i] if i < len(d.arrays[field]) else None for i in rows - ] - mixed = len({str(v) for v in values}) > 1 - self.raw.content = f"Draft {field}; rows {rows[:25]}; {'Multiple values' if mixed else str(values[:1])}. Set controls replace only the edited component." - for k, w in enumerate(self.pose_fields): - cells = [ - v[k] if isinstance(v, list) and k < len(v) else None for v in values - ] - value = common_number(cells) - if value is not None: - w.value = value - raw, sums = [], [] - for i in self.selected: - parts = [ - d.arrays[f][i] if i < len(d.arrays[f]) else None for f in FIELDS[2:] - ] - raw.append((i, *parts)) - sums.append( - sum(parts[1]) - if isinstance(parts[1], list) - and all(type(v) in (int, float) for v in parts[1]) - else "invalid" - ) - self.binding_info.content = f"{len(self.selected)} selected; weight sums (first 25): {sums[:25]}; {'Multiple values' if len({str(v[1:]) for v in raw}) > 1 else ''}\n\nRaw rows (index slots, weight slots, first 25): {raw[:25]}" - for k, (dropdown, index, slider, number) in enumerate(self.slots): - indices = [ - v[1][k] if isinstance(v[1], list) and len(v[1]) > k else None - for v in raw - ] - weights = [ - v[2][k] if isinstance(v[2], list) and len(v[2]) > k else None - for v in raw - ] - common = ( - indices[0] - if indices and all(v == indices[0] for v in indices) - else None - ) - options = tuple( - f"{i}: {d.joints[i] if i < len(d.joints) else 'missing joint_id'}" - for i in range(len(d.arrays[FIELDS[0]])) - ) - label = ( - options[common] - if type(common) is int and 0 <= common < len(options) - else f"Raw: {common if common is not None else 'mixed / missing'}" - ) - dropdown.options, dropdown.value = ( - tuple(dict.fromkeys((label, *options))), - label, + + def _binding_source_changed(self) -> None: + self._update_preview_error() + self.redraw = True + + def _binding_document(self) -> Document: + if self.binding_source.value == "Live": + if self.live_document is None: + raise ValueError("Live metadata is unavailable") + return self.live_document + return self.document + + def _preview_taxels(self) -> list[Taxel]: + return geo.deform(self._binding_document(), self.preview_document.segments) + + def _update_preview_error(self) -> None: + try: + self._preview_taxels() + except (ValueError, TypeError, KeyError, IndexError) as exc: + self.preview_error.content = ( + f'Binding error: {exc}' ) - index.label = ( - "Set raw index" - if type(common) is int - else "Set raw index (mixed / missing)" + self.apply_preview_button.disabled = True + else: + self.preview_error.content = "" + self.apply_preview_button.disabled = False + + def _apply_preview(self) -> None: + taxels = self._preview_taxels() + replacement = Document(self.state.side) + replacement.alignment = copy.deepcopy(self.document.alignment) + replacement.segments = copy.deepcopy(self.preview_document.segments) + replacement.taxels = copy.deepcopy(taxels) + replacement.q = copy.deepcopy(self.preview_document.q) + self.state.documents[self.state.side] = replacement + self._rebuild_edit_lists() + self._rebuild_joint_controls() + + def _restore_preview(self) -> None: + self.preview[self.state.side] = self._preview_from(self.document) + self._rebuild_joint_controls() + self._rebuild_preview_segments() + self._update_preview_error() + + def _rebuild_preview_segments(self) -> None: + if self.preview_segment_folder is not None: + self.preview_segment_folder.remove() + self.preview_segment_widgets = [] + with self.static_folder: + self.preview_segment_folder = self.server.gui.add_folder("Segments") + with self.preview_segment_folder: + for row, segment in enumerate(self.preview_document.segments): + with self.server.gui.add_folder(f"{row}: {segment.joint}"): + self.server.gui.add_markdown(f"HandJoint: `{segment.joint}`") + position, rotation = geo.components(geo.matrix(segment.pose)) + positions = self._numbers("Position", position, 0.001) + rotations = self._sliders("Rotation", rotation) + for axis, widget in enumerate(positions): + self._watch( + widget, + lambda value, row=row, axis=axis: ( + self._set_preview_pose(row, axis, value, False) + ), + ) + for axis, widget in enumerate(rotations): + self._watch( + widget, + lambda value, row=row, axis=axis: ( + self._set_preview_pose(row, axis, value, True) + ), + ) + self.preview_segment_widgets.append((positions, rotations)) + + def _set_preview_pose( + self, row: int, axis: int, value: float, rotation_field: bool + ) -> None: + segment = self.preview_document.segments[row] + if not rotation_field: + segment.pose[axis] = float(value) + else: + position, rotation = geo.components(geo.matrix(segment.pose)) + rotation = list(rotation) + rotation[axis] = float(value) + segment.pose = geo.pose(geo.transform(position, rotation)) + self._update_preview_error() + self.redraw = True + + def _update_live(self) -> None: + if self.receiver is None: + return + self.live_status.content = self.receiver.status + if not self.receiver.active: + return + frame = self.receiver.latest + raw_metadata = frame.get("metadata") + if raw_metadata is not None: + key = json.dumps(raw_metadata, sort_keys=True) + if key != self.live_metadata_key: + self.live_document = storage.parse( + raw_metadata, self.state.side, self.document.alignment + ) + self.live_metadata_key = key + self.binding_source.options = ("Editing", "Live") + try: + binding = self._binding_document() + hand = frame.get("hand") + if hand is not None: + joints = [segment.joint for segment in binding.segments] + updated = geo.hand_segments(joints, hand) + signature = [ + segment.joint for segment in self.preview_document.segments + ] + self.preview_document.segments = updated + if signature != joints: + self._rebuild_preview_segments() + if self.joint_enabled.value: + mapping = json.loads(self.joint_mapping.value or "{}") + for source, rule in mapping.items(): + raw, valid = frame.get("joints", {}).get(source, (None, False)) + if valid and isinstance(rule, list) and len(rule) == 3: + target, scale, offset = rule + self.preview_document.q[target] = raw * scale + offset + self._sync_preview_values() + self._update_preview_error() + except ( + ValueError, + TypeError, + KeyError, + IndexError, + json.JSONDecodeError, + ) as exc: + self.preview_error.content = ( + f'Live error: {exc}' ) - if type(common) is int and abs(common) <= 9007199254740991: - index.value = common - mixed_weight = not weights or any(v != weights[0] for v in weights) - slider.label = number.label = ( - "Set weight (mixed / missing)" if mixed_weight else "Set weight" + self.apply_preview_button.disabled = True + self.redraw = True + + def _sync_preview_values(self) -> None: + for row, widgets in enumerate(self.preview_segment_widgets): + if row >= len(self.preview_document.segments): + break + position, rotation = geo.components( + geo.matrix(self.preview_document.segments[row].pose) ) - value = common_number(weights) - if value is not None: - number.value = value - slider.value = float(np.clip(value, 0, 1)) - self.info.options = tuple( - f"{i}: {v['side']} {v['severity']} {v['field']}[{v['row']}]: {v['message']}" - for i, v in enumerate(self.issues) - ) or ("No local errors",) - self.check_info.content = f"Local revision {self.state.revision}: {sum(v['severity'] == 'error' for v in self.issues)} errors. SDK/export revision {self.sdk_revision}: {self.sdk_result}{' (stale)' if self.sdk_revision is not None and self.sdk_revision != self.state.revision else ''}" - - def locate(self): - if not self.issues: + self._set_values(widgets[0], position) + self._set_values(widgets[1], rotation) + for name, widget in self.preview_joint_widgets.items(): + widget.value = self.preview_document.q.get(name, 0.0) + + def _register_scene_events(self) -> None: + self.server.scene.on_click()(self._scene_click) + self.server.scene.on_click(modifier="shift")(self._scene_click) + self.server.scene.on_rect_select(modifier="shift")(self._scene_box) + + async def _scene_click(self, event) -> None: + self.events.put(lambda: self._handle_scene_event(event, False)) + + async def _scene_box(self, event) -> None: + self.events.put(lambda: self._handle_scene_event(event, True)) + + def _display_values(self): + if self.mode.value == "Preview": + return self.preview_document.segments, [] + return self.document.segments, self.document.taxels + + def _handle_scene_event(self, event, box: bool) -> None: + if self.pick_mode is not None and not box: + self._handle_pick(event) return - issue = self.issues[int(self.info.value.split(":")[0])] - if issue["side"] != self.side.value: - self.side.value = issue["side"] - self.switch() - if issue["row"] is not None: - if issue["field"] in (FIELDS[0], "joint_ids"): - self.segment.value, self.entity.value = str(issue["row"]), "Segment" + segments, taxels = self._display_values() + candidates = { + "segments": list(enumerate(segments)), + "taxels": list(enumerate(taxels)), + } + if self.binding_phase is not None: + candidates = { + key: values + for key, values in candidates.items() + if key == self.binding_phase + } + chosen = { + kind: self._box_rows(event, values) + if box + else self._ray_rows(event, kind, values) + for kind, values in candidates.items() + } + chosen = {kind: rows for kind, rows in chosen.items() if rows} + if not chosen: + if self.binding_phase is not None: + self._cancel_binding() else: - self.selected, self.entity.value = [issue["row"]], "Taxels" - self.log("Located in draft inspector: " + issue["field"]) - - def axes(self, name, transforms): - positions = np.array([t[:3, 3] for t in transforms]).reshape(-1, 3) - quaternions = np.array( - [tf.quaternion_from_matrix(t) for t in transforms] - ).reshape(-1, 4) - if name not in self.handles and len(positions): - self.handles[name] = self.server.scene.add_batched_axes( - name, quaternions, positions, axes_length=0.015, axes_radius=0.0006 + self.selected_kind, self.selected_rows = None, [] + self._rebuild_selection() + self.redraw = True + return + kind = ( + self.selected_kind + if self.selected_kind in chosen + else "taxels" + if "taxels" in chosen + else "segments" + ) + rows = chosen[kind] + if self.binding_phase == "taxels": + self.binding_taxels = ( + sorted(set(self.binding_taxels) | set(rows)) + if event.modifier == "shift" + else rows ) - if name in self.handles: - h = self.handles[name] - h.visible = bool(len(positions)) - if len(positions): - h.batched_positions, h.batched_wxyzs = positions, quaternions - - def balls(self, name, points, colors, outline=False): - positions = np.asarray(points, dtype=float).reshape(-1, 3) - rotations = np.tile([1.0, 0, 0, 0], (len(positions), 1)) - if name not in self.handles and len(positions): - mesh = trimesh.creation.icosphere(subdivisions=1) - self.handles[name] = self.server.scene.add_batched_meshes_simple( - name, mesh.vertices, mesh.faces, rotations, positions, wireframe=outline + self.binding_next.disabled = not self.binding_taxels + elif self.binding_phase == "segments": + self.binding_segments = ( + sorted(set(self.binding_segments) | set(rows)) + if event.modifier == "shift" + else rows ) - if name in self.handles: - h = self.handles[name] - h.visible = bool(len(positions)) - if len(positions): - h.batched_positions, h.batched_wxyzs = positions, rotations - h.batched_colors = np.asarray(colors, dtype=np.uint8) - h.batched_scales = np.full( - len(positions), self.radius.value * (1.5 if outline else 1) - ) - - def draft_points(self): - return [ - (i, t[:3, 3]) for i, t in geo.frames(self.d.arrays[FIELDS[1]], self.root()) - ] - - def rectangle_mode(self, enabled): - if enabled: - self.server.scene.on_rect_select()(self.box_event) - else: - self.server.scene.remove_rect_select_callback(self.box_event) - - async def box_event(self, event): - self.events.put(lambda: self.select_scene(event, True)) - - async def scene_click(self, event): - self.events.put(lambda: self.click(event)) + self.binding_complete.disabled = not self.binding_segments + elif event.modifier == "shift": + if self.selected_kind not in (None, kind): + return + self.selected_kind = kind + self.selected_rows = sorted(set(self.selected_rows) | set(rows)) + self._rebuild_selection() + elif self.selected_kind != kind or self.selected_rows != rows: + self.selected_kind, self.selected_rows = kind, rows + self._rebuild_selection() + self.redraw = True - def select_scene(self, event, box=False): - camera, chosen = event.client.camera, [] + @staticmethod + def _ray_rows(event, kind: str, values) -> list[int]: + origin = np.asarray(event.ray_origin) + direction = np.asarray(event.ray_direction) + hits = [] + for row, value in values: + point = np.asarray(value.pose[:3]) + if kind == "segments": + distance = (point - origin) @ direction + offset = np.linalg.norm(point - origin - distance * direction) + if distance > 0 and offset <= 0.01: + hits.append((distance, row)) + continue + inverse = np.linalg.inv(geo.matrix(value.pose)) + local_origin = (inverse @ np.r_[origin, 1.0])[:3] + local_direction = inverse[:3, :3] @ direction + if abs(local_direction[2]) < 1e-10: + continue + distance = -local_origin[2] / local_direction[2] + local_hit = local_origin + distance * local_direction + if distance > 0 and np.max(np.abs(local_hit[:2])) <= 0.005: + hits.append((distance, row)) + return [min(hits)[1]] if hits else [] + + @staticmethod + def _box_rows(event, values) -> list[int]: + camera = event.client.camera rotation = tf.quaternion_matrix(camera.wxyz)[:3, :3] - for i, point in self.draft_points(): - origin = np.asarray(camera.position if box else event.ray_origin) - direction = ( - geo.unit(point - origin) if box else np.asarray(event.ray_direction) + origin = np.asarray(camera.position) + rows = [] + for row, value in values: + point = np.asarray(value.pose[:3]) + local = rotation.T @ (point - origin) + if local[2] <= 0: + continue + uv = ( + 0.5 + + local[0] / local[2] / (2 * np.tan(camera.fov / 2) * camera.aspect), + 0.5 + local[1] / local[2] / (2 * np.tan(camera.fov / 2)), ) - if box: - p = rotation.T @ (point - origin) - if p[2] >= 0: - continue - uv = [ - 0.5 - p[0] / p[2] / (2 * np.tan(camera.fov / 2) * camera.aspect), - 0.5 + p[1] / p[2] / (2 * np.tan(camera.fov / 2)), - ] - inside = all( - a <= b <= c - for a, b, c in zip(event.screen_min, uv, event.screen_max) - ) - else: - along = (point - origin) @ direction - inside = ( - along > 0 - and np.linalg.norm(point - origin - along * direction) - < self.radius.value * 2 - ) - if ( - inside - and self.model - and getattr(self, "display_model", False) - and not self.through.value + if all( + lower <= value <= upper + for lower, value, upper in zip(event.screen_min, uv, event.screen_max) ): - surfaces = list(self.model.surfaces) - hit = self.model.hit(origin, direction, surfaces, self.display_fk) - inside = ( - hit is None - or hit[0] >= np.linalg.norm(point - origin) - self.radius.value * 2 - ) - if inside: - chosen.append((np.linalg.norm(point - origin), i)) - indices = [i for _, i in (chosen if box else sorted(chosen)[:1])] - self.selected = ( - sorted(set(self.selected) ^ set(indices)) if event.modifier else indices - ) - self.entity.value = "Taxels" - - def click(self, event): - if self.dragging: - return - if not self.pick_target: - self.select_scene(event) - return - target, self.pick_target = self.pick_target, None - if self.mode.value != "EDIT" or not self.model: - raise ValueError("Surface picking requires EDIT and a q0 model") - fk = self.model.fk(self.state.model.get("q0", {})) - surfaces = self.surfaces() - if target in ("O", "C", "R", "Move selected", "Single tangent"): - if self.surface.value not in surfaces: - raise ValueError("Choose an allowed projection surface") - surfaces = [self.surface.value] + rows.append(row) + return rows + + def _handle_pick(self, event) -> None: + mode, self.pick_mode = self.pick_mode, None + self.segment_click_button.label = "Add with URDF Click" + self.taxel_click_button.label = "Add with URDF Click" + if self.model is None: + raise ValueError("Load a URDF before using URDF Click") hit = self.model.hit( np.asarray(event.ray_origin), np.asarray(event.ray_direction), - surfaces, - fk, + self.document.q, + self.urdf_alignment, + self.urdf_scale, ) - if not hit: - raise ValueError("No hit on an allowed visible surface") - self.points[target] = hit[1].tolist() - if target in ("O", "C", "R"): - self.centers[("O", "C", "R").index(target)].value = tuple(hit[1]) - elif target == "Segment": - self.entity.value = "Segment" - self.components( - range(7), - geo.pose( - np.linalg.inv(self.root()) - @ geo.frame(hit[1], hit[2], fk[self.segment_link.value][:3, 0]) - ), + if hit is None: + return + if mode == "segment": + value = IDENTITY_POSE.copy() + value[:3] = hit[1].tolist() + row = self.document.add_segment(self.new_segment_joint.value, value) + self.selected_kind, self.selected_rows = "segments", [row] + else: + _, width, height, rows, columns = self._grid_values() + frame = geo.surface_frame(hit[1], hit[2]) + selected = self.document.add_taxels( + geo.grid_poses(hit[1], width, height, rows, columns, frame) ) - elif target == "Move selected": - old = geo.matrix(self.d.arrays[FIELDS[1]][self.selected[0]]) - new = np.linalg.inv(self.root()) @ geo.frame( - hit[1], hit[2], (self.root() @ old)[:3, 0] + self.selected_kind, self.selected_rows = "taxels", selected + self._rebuild_edit_lists() + + def _selection_values(self): + if self.selected_kind is None: + return [] + if self.mode.value == "Preview": + return self.preview_document.segments + return self._values(self.selected_kind) + + def _rebuild_selection(self) -> None: + if self.selection_folder is not None: + self.selection_folder.remove() + values = self._selection_values() + rows = [row for row in self.selected_rows if 0 <= row < len(values)] + self.selected_rows = rows + self.selection_folder = self.server.gui.add_folder( + "Selected Objects", visible=bool(rows) + ) + if not rows: + return + with self.selection_folder: + self.server.gui.add_markdown( + f"Type: **{self.selected_kind}** \nCount: **{len(rows)}**" ) - self.change( - geo.transform, - self.d, - FIELDS[1], - self.selected, - new @ np.linalg.inv(old), + positions = np.asarray([values[row].pose[:3] for row in rows]) + center = positions.mean(axis=0) + if len(rows) == 1: + _, rotation = geo.components(geo.matrix(values[rows[0]].pose)) + else: + rotation = (0.0, 0.0, 0.0) + self.selection_position_widgets = self._numbers("Position", center, 0.001) + self.selection_rotation_widgets = self._sliders("Rotation", rotation) + self.selection_rotation = np.asarray(rotation, dtype=float) + for axis, widget in enumerate(self.selection_position_widgets): + self._watch( + widget, + lambda value, axis=axis: self._set_selection_position(axis, value), + ) + for axis, widget in enumerate(self.selection_rotation_widgets): + self._watch( + widget, + lambda value, axis=axis: self._set_selection_rotation(axis, value), + ) + self.selection_scale = 1.0 + self.selection_scale_widget = self.server.gui.add_number( + "Scale", 1.0, min=1e-6, step=0.01, visible=len(rows) > 1 ) - elif target == "Single tangent": - self.points["C"] = hit[1].tolist() - self.rows.value = self.cols.value = 1 - self.grid() - - def received_document(self, side, value): - cached = self.received.get(side) - if cached is None or cached[0] != value: - d = storage.parse(value, side) - cached = (copy.deepcopy(value), d, d.check()) - self.received[side] = cached - return cached[1:] - - def render_side(self, side, d, root, current, forces, live, issues): - positions, colors, frames, outlines, edges = [], [], [], [], [] - deformed = geo.deform(d, current) if current is not None else None - errors = { - v["row"] - for v in issues - if v["severity"] == "error" and v["field"] in FIELDS[1:] - } - for i, t in geo.frames(d.arrays[FIELDS[1]], root): - rigid, color = t, (140, 140, 140) - if deformed is not None: - point, orientation = deformed[i] - if point is None: - continue - t[:3, 3] = (root @ np.r_[point, 1])[:3] - rigid = root @ orientation if orientation is not None else None - if live and forces is not None and len(forces) == len(d.arrays[FIELDS[1]]): - force = np.asarray(forces[i], dtype=float) - if force.shape == (3,) and np.isfinite(force).all(): - fraction = np.clip( - np.linalg.norm(force) / self.force_max.value, 0, 1 - ) - color = (255 * fraction, 40, 255 * (1 - fraction)) - elif not live: - try: - indices, weights = d.arrays[FIELDS[2]][i], d.arrays[FIELDS[3]][i] - if i not in errors: - color = np.random.default_rng( - indices[int(np.argmax(weights))] - ).integers(70, 240, 3) - if self.slot is not None and i in self.selected and d is self.d: - color = ( - 255 * weights[self.slot], - 80, - 255 * (1 - weights[self.slot]), - ) - except (IndexError, TypeError, ValueError): - pass - positions.append(t[:3, 3]) - colors.append(color) - selected = d is self.d and i in self.selected - if selected or i in errors: - outlines.append(t[:3, 3]) - edges.append((255, 0, 0) if i in errors else (255, 220, 0)) - if selected and rigid is not None: - frames.append(rigid) - prefix = "/view/" + side - self.balls(prefix + "/taxels", positions, colors) - self.balls(prefix + "/outlines", outlines, edges, True) - self.axes(prefix + "/taxel-axes", frames) - self.axes(prefix + "/root", [root]) - segments = ( - [root @ t for t in current if t is not None] - if current is not None - else [t for _, t in geo.frames(d.arrays[FIELDS[0]], root)] - ) - self.axes(prefix + "/segments", segments) - self.balls( - prefix + "/segment-points", - [t[:3, 3] for t in segments], - [(90, 200, 250)] * len(segments), + self._watch(self.selection_scale_widget, self._set_selection_scale) + self._action_button("Delete", self._delete_selection) + + def _set_selection_position(self, axis: int, value: float) -> None: + values = self._selection_values() + center = np.asarray([values[row].pose[:3] for row in self.selected_rows]).mean( + axis=0 ) - self.frame_colors[side] = np.asarray(colors, dtype=np.uint8) - - def draw(self): - latest, mode = self.latest(), self.mode.value - playing, status = mode in ("LIVE", "PAUSED"), [] - updating = self.redraw or mode == "LIVE" - cfg = self.q if mode == "PREVIEW" else self.state.model.get("q0", {}) - highlight, lines, line_colors = set(), [], [] - for side, draft in self.state.documents.items(): - value = latest.get(side, {}) - unmapped = set(value.get("joints", {})) - set( - draft.editor.get("mapping", {}) + delta = np.eye(4) + delta[axis, 3] = float(value) - center[axis] + geo.apply_delta(values, self.selected_rows, delta) + self._sync_selected_list_values() + + def _set_selection_rotation(self, axis: int, value: float) -> None: + values = self._selection_values() + if len(self.selected_rows) == 1: + row = self.selected_rows[0] + position, rotation = geo.components(geo.matrix(values[row].pose)) + rotation = list(rotation) + rotation[axis] = float(value) + values[row].pose = geo.pose(geo.transform(position, rotation)) + else: + points = np.asarray([values[row].pose[:3] for row in self.selected_rows]) + center = points.mean(axis=0) + angle = np.deg2rad(float(value) - self.selection_rotation[axis]) + rotation = tf.rotation_matrix(angle, np.eye(3)[axis], center) + geo.apply_delta(values, self.selected_rows, rotation) + self.selection_rotation[axis] = float(value) + self._sync_selected_list_values() + + def _set_selection_scale(self, value: float) -> None: + values = self._selection_values() + factor = float(value) / self.selection_scale + geo.scale_positions(values, self.selected_rows, factor) + self.selection_scale = float(value) + self._sync_selected_list_values() + + def _sync_selected_list_values(self) -> None: + if self.mode.value == "Edit": + widgets = ( + self.segment_widgets + if self.selected_kind == "segments" + else self.taxel_widgets ) - status.append( - f"{side} raw JointState: {value.get('joints', {})}; unmapped: {sorted(unmapped)}" + values = self._selection_values() + for row in self.selected_rows: + if row >= len(widgets): + continue + position, rotation = geo.components(geo.matrix(values[row].pose)) + position_widgets, rotation_widgets = ( + (widgets[row][1], widgets[row][2]) + if self.selected_kind == "segments" + else (widgets[row][0], widgets[row][1]) + ) + self._set_values(position_widgets, position) + self._set_values(rotation_widgets, rotation) + else: + self._sync_preview_values() + self.redraw = True + + def _delete_selection(self) -> None: + if self.mode.value == "Preview": + for row in sorted(self.selected_rows, reverse=True): + self.preview_document.segments.pop(row) + self._rebuild_preview_segments() + self._update_preview_error() + else: + self.document.delete(self.selected_kind, self.selected_rows) + self._rebuild_edit_lists() + self.selected_kind, self.selected_rows = None, [] + self._rebuild_selection() + + def _draw_instances( + self, name: str, kind: str, values, colors, opacity: float, wireframe=False + ) -> None: + matrices = [geo.matrix(value.pose) for value in values] + if not matrices: + if name in self.scene_handles: + self.scene_handles[name].visible = False + return + positions = np.asarray([value[:3, 3] for value in matrices]) + rotations = np.asarray([tf.quaternion_from_matrix(value) for value in matrices]) + mesh = ( + trimesh.creation.icosphere(subdivisions=1) + if kind == "segments" + else trimesh.Trimesh( + vertices=[ + [-0.5, -0.5, 0.0], + [0.5, -0.5, 0.0], + [0.5, 0.5, 0.0], + [-0.5, 0.5, 0.0], + ], + faces=[[0, 1, 2], [0, 2, 3]], + process=False, ) - fresh = ( - bool(self.receiver) - and (self.receiver is True or self.receiver.active) - and time.monotonic() - value.get("read_time", 0) < 1 + ) + if name not in self.scene_handles: + self.scene_handles[name] = self.server.scene.add_batched_meshes_simple( + name, + mesh.vertices, + mesh.faces, + rotations, + positions, + batched_scales=np.full(len(values), 0.01), + batched_colors=np.asarray(colors, dtype=np.uint8), + opacity=opacity, + wireframe=wireframe, + side="double", ) + handle = self.scene_handles[name] + handle.visible = True + handle.batched_positions = positions + handle.batched_wxyzs = rotations + handle.batched_scales = np.full(len(values), 0.01) + handle.batched_colors = np.asarray(colors, dtype=np.uint8) + handle.opacity = opacity + + def _hide_prefix(self, prefix: str) -> None: + for name, handle in self.scene_handles.items(): + if name.startswith(prefix): + handle.visible = False + + def _draw_dataset( + self, + prefix: str, + segments: list[Segment], + taxels: list[Taxel], + opacity: float, + active: bool, + forces=None, + ) -> None: + segment_colors = [] + for row in range(len(segments)): + color = (245, 200, 45) if active else (130, 130, 130) if ( - self.model - and self.receiver - and self.configs.get(side, {}).get("joint_state") + self.selected_kind == "segments" + and row in self.selected_rows + and active ): - for name, (target, scale, offset) in draft.editor.get( - "mapping", {} - ).items(): - raw, valid = value.get("joints", {}).get(name, (None, False)) - converted = ( - raw * scale + offset - if valid and raw is not None - else float("nan") - ) - joint = self.model.urdf.joint_map.get(target) - limit = joint.limit if joint else None - valid = ( - fresh and joint in self.model.joints and np.isfinite(converted) - ) - if limit: - valid &= (limit.lower is None or converted >= limit.lower) and ( - limit.upper is None or converted <= limit.upper - ) - status.append( - f"{side}/{name}: raw={raw}; converted={converted}; limits={limit}; valid={valid}" - ) - if valid and mode == "LIVE": - self.drive[target] = converted - self.layers[side].visible = side == self.side.value or self.both.value - if updating and mode != "PAUSED": - try: - root, d, current, skeleton = self.root(draft, cfg), draft, None, [] - issues = [v for v in self.issues if v["side"] == side] - if mode == "LIVE": - current, joints, world_to_wrist = geo.hand_current( - draft, value["hand"] - ) - root = geo.VISUALIZATION_FROM_HAND.copy() - skeleton = [ - root @ world_to_wrist @ t - for t in joints - if t is not None - ] - if self.source.value == "Received": - try: - received, received_issues = self.received_document( - side, value["metadata"] - ) - received_current, _, _ = geo.hand_current( - received, value["hand"] - ) - except (ValueError, TypeError, KeyError, IndexError) as exc: - status.append( - f"{side}: received metadata unavailable; using draft: {exc}" - ) - else: - d, issues, current = ( - received, - received_issues, - received_current, - ) - elif mode == "PREVIEW" and self.model and "root" in draft.editor: - current = self.model.current(draft, self.model.fk(cfg)) - self.render_side( - side, - d, - root, - current, - value.get("forces") if fresh else None, - playing, - issues, - ) - self.axes("/view/" + side + "/live-skeleton", skeleton) - self.frame_forces[side] = value.get("forces") - self.frame_counts[side] = len(d.arrays[FIELDS[1]]) - except (ValueError, TypeError, KeyError, IndexError) as exc: - status.append(f"{side}: invalid visualization: {exc}") - fresh = False - taxels = self.handles.get("/view/" + side + "/taxels") - if playing and taxels and taxels.visible: - taxels.batched_colors = ( - self.frame_colors.get(side, (140, 140, 140)) - if fresh - else (140, 140, 140) - ) - forces = ( - self.frame_forces.get(side) if mode == "PAUSED" else value.get("forces") - ) - force_array = ( - np.asarray(forces, dtype=float) - if forces is not None - else np.empty((0, 3)) - ) - valid_forces = ( - forces is not None - and force_array.ndim == 2 - and force_array.shape[1:] == (3,) - and np.isfinite(force_array).all() - and len(force_array) - == self.frame_counts.get(side, len(draft.arrays[FIELDS[1]])) - ) - status.append( - f"{side}: {value.get('config')}; local read={value.get('read_time')}; current={fresh}; force shape={force_array.shape}; valid/count-matched={valid_forces}; draft taxels={len(draft.arrays[FIELDS[1]])}" - ) - hand_flags = (value.get("hand") or {}).get("is_valid", []) - status.append( - f"{side} HandPose: {sum(bool(v) for v in hand_flags)} valid joints; WRIST flag={len(hand_flags) == 26 and bool(hand_flags[1])}" - ) - if side == self.side.value and fresh and valid_forces: - status.extend( - f"{'Paused display' if mode == 'PAUSED' else 'Latest'} row {i}: fx,fy,fz={forces[i]}; |f|={np.linalg.norm(forces[i]):.4g} N" - for i in self.selected[:25] - if i < len(force_array) - ) - connection = ( - self.receiver.status - if self.receiver and self.receiver is not True - else "Demo" - if self.receiver - else "Disconnected" + color = (255, 120, 20) + if self.binding_phase == "segments" and row in self.binding_segments: + color = (255, 120, 20) + if self.binding_phase == "taxels" and active: + color = (130, 130, 130) + if self.pick_mode is not None and active: + color = (130, 130, 130) + segment_colors.append(color) + taxel_colors = [(190, 190, 190)] * len(taxels) + array = np.asarray(forces, dtype=float) if forces is not None else None + if ( + array is not None + and array.shape == (len(taxels), 3) + and np.isfinite(array).all() + ): + fraction = np.clip(np.linalg.norm(array, axis=1) / 30.0, 0.0, 1.0) + taxel_colors = [ + (int(255 * value), 0, int(255 * (1.0 - value))) for value in fraction + ] + for row in range(len(taxels)): + if self.selected_kind == "taxels" and row in self.selected_rows and active: + taxel_colors[row] = (255, 120, 20) + if self.binding_phase == "taxels" and row in self.binding_taxels: + taxel_colors[row] = (255, 120, 20) + if self.binding_phase == "segments" and active: + taxel_colors[row] = (130, 130, 130) + if self.pick_mode is not None and active: + taxel_colors[row] = (130, 130, 130) + self._draw_instances( + prefix + "/segments", "segments", segments, segment_colors, opacity ) - self.live_info.content = connection + "\n\n" + "\n\n".join(status) - if not updating: - return - root, d = self.root(), self.d - points = self.draft_points() if mode != "EDIT" else [] - self.balls( - "/draft-targets", - [p for _, p in points], - [ - (255, 220, 0) if i in self.selected else (140, 140, 140) - for i, _ in points - ], + self._draw_instances( + prefix + "/taxels", "taxels", taxels, taxel_colors, opacity + ) + + def _draw_errors(self) -> None: + segment_rows = { + issue["row"] + for issue in self.document.validate() + if issue["kind"] == "segments" and issue["row"] is not None + } + taxel_rows = { + issue["row"] + for issue in self.document.validate() + if issue["kind"] == "taxels" and issue["row"] is not None + } + self._draw_instances( + "/errors/segments", + "segments", + [self.document.segments[row] for row in sorted(segment_rows)], + [(255, 0, 0)] * len(segment_rows), + 1.0, True, ) - field, rows = self.targets() - self.axes( - "/draft-axes", - [t for i, t in geo.frames(d.arrays[field], root) if i in rows], + self._draw_instances( + "/errors/taxels", + "taxels", + [self.document.taxels[row] for row in sorted(taxel_rows)], + [(255, 0, 0)] * len(taxel_rows), + 1.0, + True, ) - for i in self.selected: - try: - for j, w in zip(d.arrays[FIELDS[2]][i], d.arrays[FIELDS[3]][i]): - if ( - type(j) is int - and 0 <= j < len(d.joints) - and type(w) in (int, float) - and w <= 1 - and w > 0 - ): - lines.append( - [ - (root @ geo.matrix(d.arrays[FIELDS[0]][j]))[:3, 3], - (root @ geo.matrix(d.arrays[FIELDS[1]][i]))[:3, 3], - ] - ) - line_colors.append([(int(255 * w), 80, 30)] * 2) - if ( - self.slot is not None - and j == d.arrays[FIELDS[2]][i][self.slot] - ): - highlight.add( - d.editor.get("segment_links", {}).get( - d.editor["segment_ids"][j] - ) - ) - except (IndexError, TypeError, ValueError): - pass - if ( - self.entity.value == "Segment" - and self.segment.value - and int(self.segment.value) < len(d.editor["segment_ids"]) - ): - highlight.add( - d.editor.get("segment_links", {}).get( - d.editor["segment_ids"][int(self.segment.value)] - ) + + def _draw_selection_axes(self) -> None: + values = self._selection_values() + rows = [row for row in self.selected_rows if 0 <= row < len(values)] + if not rows: + if "/selection/axes" in self.scene_handles: + self.scene_handles["/selection/axes"].visible = False + return + if len(rows) == 1: + frames = [geo.matrix(values[rows[0]].pose)] + else: + frame = np.eye(4) + frame[:3, 3] = np.asarray([values[row].pose[:3] for row in rows]).mean( + axis=0 ) - if "/draft-binding" not in self.handles and lines: - self.handles["/draft-binding"] = self.server.scene.add_line_segments( - "/draft-binding", - np.asarray(lines), - np.asarray(line_colors, dtype=np.uint8), - thickness=2, - thickness_units="screen", + frames = [frame] + positions = np.asarray([frame[:3, 3] for frame in frames]) + rotations = np.asarray([tf.quaternion_from_matrix(frame) for frame in frames]) + if "/selection/axes" not in self.scene_handles: + self.scene_handles["/selection/axes"] = self.server.scene.add_batched_axes( + "/selection/axes", + rotations, + positions, + axes_length=0.03, + axes_radius=0.001, ) - if "/draft-binding" in self.handles: - h = self.handles["/draft-binding"] - h.visible = bool(lines) - if lines: - h.points, h.colors = ( - np.asarray(lines), - np.asarray(line_colors, dtype=np.uint8), - ) - if self.model and mode != "PAUSED": - enabled = any( - bool(self.receiver) and self.configs.get(side, {}).get("joint_state") - for side in self.state.documents + handle = self.scene_handles["/selection/axes"] + handle.visible = True + handle.batched_positions = positions + handle.batched_wxyzs = rotations + + def _draw(self) -> None: + if self.model is not None: + q = ( + self.document.q + if self.mode.value == "Edit" + else self.preview_document.q ) - self.display_model = not playing or enabled - self.display_fk = self.model.draw( + self.model.draw( self.server.scene, - self.drive if playing else cfg, - self.display_model, - highlight - {None}, + q, + self.urdf_alignment, + self.urdf_scale, + highlighted=self.pick_mode is not None, + opacity=1.0, ) - candidates = [ - (root @ np.r_[geo.matrix(p)[:3, 3] if p else fallback, 1])[:3] - for p, fallback in self.candidates - ] - self.balls( - "/candidates", - candidates, - [(50, 220, 90) if p else (255, 0, 0) for p, _ in self.candidates], - ) - if not self.dragging: + self._hide_prefix("/edit") + self._hide_prefix("/preview") + self._hide_prefix("/saved") + self._hide_prefix("/errors") + if self.mode.value == "Edit": + self._draw_dataset( + "/edit", + self.document.segments, + self.document.taxels, + 1.0, + True, + ) + self._draw_errors() + else: try: - t = root @ geo.matrix(d.arrays[field][rows[0]]) - self.gizmo.position, self.gizmo.wxyz = ( - t[:3, 3], - tf.quaternion_from_matrix(t), + preview_taxels = self._preview_taxels() + except (ValueError, TypeError, KeyError, IndexError): + preview_taxels = [] + forces = ( + self.receiver.latest.get("forces") + if self.receiver + and self.receiver.active + and time.monotonic() + - self.receiver.latest.get("read_time", float("-inf")) + < 0.5 + else None + ) + self._draw_dataset( + "/preview", + self.preview_document.segments, + preview_taxels, + 1.0, + True, + forces, + ) + if self.show_binding.value: + self._draw_dataset( + "/edit", + self.document.segments, + self.document.taxels, + 0.5, + False, ) - self.gizmo.visible = True - except (ValueError, TypeError, IndexError): - self.gizmo.visible = False + saved = self.state.saved[self.state.side] + if self.show_saved.value and saved is not None: + self._draw_dataset("/saved", saved.segments, saved.taxels, 0.2, False) + self._draw_selection_axes() - def run(self): - self.camera() - last_error = None + def run(self) -> None: try: - while not self.closing: - try: - while not self.events.empty(): - try: - self.events.get()() - finally: - self.refresh() - if self.job and self.job.done(): - job, self.job = self.job, None - try: - self.sdk_result = str(job.result()) - except Exception as exc: - self.sdk_result = f"Failed: {exc}" - self.refresh() - if ( - self.redraw - or self.receiver - or self.mode.value in ("LIVE", "PAUSED") - ): - with self.server.atomic(): - self.draw() - self.redraw = False - last_error = None - time.sleep(1 / 30) - except KeyboardInterrupt: - self.closing = True - except Exception as exc: - message = f"{type(exc).__name__}: {exc}" - if message != last_error: - self.log(message) - last_error = message - time.sleep(0.1) + while True: + while not self.events.empty(): + self.events.get_nowait()() + self._update_live() + if self.redraw or self.receiver is not None: + self._draw() + self.redraw = False + time.sleep(1 / 30) + except KeyboardInterrupt: + pass finally: - if self.receiver and self.receiver is not True: - self.receiver.close() - self.pool.shutdown(wait=True) + self._disconnect() self.server.stop() diff --git a/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js b/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js deleted file mode 100644 index b0a100a7fb..0000000000 --- a/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js +++ /dev/null @@ -1,148 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 -(() => { - "use strict"; - - const key = "__tactileMetadataViewportAxes"; - globalThis[key]?.stop(); - - const ns = "http://www.w3.org/2000/svg"; - const svg = document.createElementNS(ns, "svg"); - Object.assign(svg.style, { - position: "absolute", - top: "16px", - left: "16px", - width: "112px", - height: "112px", - zIndex: "10", - pointerEvents: "none", - overflow: "visible", - }); - svg.setAttribute("viewBox", "0 0 112 112"); - - const background = document.createElementNS(ns, "circle"); - background.setAttribute("cx", "56"); - background.setAttribute("cy", "56"); - background.setAttribute("r", "49"); - background.setAttribute("fill", "rgba(255,255,255,0.72)"); - background.setAttribute("stroke", "rgba(0,0,0,0.2)"); - svg.appendChild(background); - - const axes = [ - ["x", "#e53935", [1, 0, 0]], - ["y", "#43a047", [0, 1, 0]], - ["z", "#1e88e5", [0, 0, 1]], - ].map(([name, color, vector]) => { - const line = document.createElementNS(ns, "line"); - line.setAttribute("x1", "56"); - line.setAttribute("y1", "56"); - line.setAttribute("stroke", color); - line.setAttribute("stroke-width", "3"); - line.setAttribute("stroke-linecap", "round"); - svg.appendChild(line); - - const end = document.createElementNS(ns, "circle"); - end.setAttribute("r", "4"); - end.setAttribute("stroke", color); - end.setAttribute("stroke-width", "2"); - svg.appendChild(end); - - const label = document.createElementNS(ns, "text"); - label.textContent = name; - label.setAttribute("fill", color); - label.setAttribute("font-size", "15"); - label.setAttribute("font-family", "sans-serif"); - label.setAttribute("font-weight", "700"); - label.setAttribute("text-anchor", "middle"); - label.setAttribute("dominant-baseline", "central"); - svg.appendChild(label); - return { color, end, label, line, vector }; - }); - - const multiply = (a, b) => [ - a[3] * b[0] + a[0] * b[3] + a[1] * b[2] - a[2] * b[1], - a[3] * b[1] - a[0] * b[2] + a[1] * b[3] + a[2] * b[0], - a[3] * b[2] + a[0] * b[1] - a[1] * b[0] + a[2] * b[3], - a[3] * b[3] - a[0] * b[0] - a[1] * b[1] - a[2] * b[2], - ]; - - const rotate = (q, v) => { - const tx = 2 * (q[1] * v[2] - q[2] * v[1]); - const ty = 2 * (q[2] * v[0] - q[0] * v[2]); - const tz = 2 * (q[0] * v[1] - q[1] * v[0]); - return [ - v[0] + q[3] * tx + q[1] * tz - q[2] * ty, - v[1] + q[3] * ty + q[2] * tx - q[0] * tz, - v[2] + q[3] * tz + q[0] * ty - q[1] * tx, - ]; - }; - - const state = { - controls: null, - raf: 0, - stopped: false, - stop() { - this.stopped = true; - cancelAnimationFrame(this.raf); - this.controls?.removeEventListener("update", draw); - svg.remove(); - }, - }; - globalThis[key] = state; - - const draw = () => { - const mutable = globalThis.__viserMutable; - const camera = mutable?.camera; - const root = mutable?.nodeRefFromName?.[""]; - const parent = mutable?.canvas?.parentElement; - if (!camera || !root || !parent) return; - if (svg.parentElement !== parent) parent.appendChild(svg); - - const cameraQuaternion = camera.quaternion; - const rootQuaternion = root.quaternion; - const cameraFromWorld = [ - -cameraQuaternion.x, - -cameraQuaternion.y, - -cameraQuaternion.z, - cameraQuaternion.w, - ]; - const cameraFromVisualization = multiply(cameraFromWorld, [ - rootQuaternion.x, - rootQuaternion.y, - rootQuaternion.z, - rootQuaternion.w, - ]); - - for (const [index, axis] of axes.entries()) { - const direction = rotate(cameraFromVisualization, axis.vector); - const x = 56 + 36 * direction[0]; - const y = 56 - 36 * direction[1]; - const projected = Math.hypot(direction[0], direction[1]); - const fallback = [[10, -8], [-10, -8], [0, 12]][index]; - const labelOffset = - projected > 0.12 - ? [8 * direction[0] / projected, -8 * direction[1] / projected] - : fallback; - axis.line.setAttribute("x2", x.toFixed(2)); - axis.line.setAttribute("y2", y.toFixed(2)); - axis.end.setAttribute("cx", x.toFixed(2)); - axis.end.setAttribute("cy", y.toFixed(2)); - axis.end.setAttribute("fill", direction[2] >= 0 ? axis.color : "white"); - axis.label.setAttribute("x", (x + labelOffset[0]).toFixed(2)); - axis.label.setAttribute("y", (y + labelOffset[1]).toFixed(2)); - } - }; - - const frame = () => { - if (state.stopped) return; - const controls = globalThis.__viserMutable?.cameraControl; - if (controls !== state.controls) { - state.controls?.removeEventListener("update", draw); - controls?.addEventListener("update", draw); - state.controls = controls; - } - draw(); - state.raf = requestAnimationFrame(frame); - }; - frame(); -})(); From a072b24c244939f877234caad3ca34ad552610b9 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Wed, 28 Oct 2026 18:01:04 -0700 Subject: [PATCH 15/36] some adjustments Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 5 +- scripts/tactile_metadata/README.md | 319 +++++++++--------- scripts/tactile_metadata/UI&Feature Design.md | 6 +- .../tactile_metadata_editor/document.py | 32 ++ .../editor_interactions.js | 12 +- .../tactile_metadata_editor/geometry.py | 22 +- .../tactile_metadata_editor/ui.py | 70 ++-- 7 files changed, 276 insertions(+), 190 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 56a212e73c..7c3c52aa28 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -16,6 +16,7 @@ SPDX-License-Identifier: Apache-2.0 - 对设计文档的结构与表达要求应直接落实到目标文档；页面布局按区块列出具体控件，不以聊天中的补充说明代替文档修改。 - Keep URDF driving separate from taxel deformation: HandPose supplies skeleton and LBS poses, while live URDF motion uses JointState only. Remove obsolete driver controls, persisted options, algorithms, and acceptance cases together. - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. +- New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. - Preserve slot order and repeated segment indices. Normalize all four weights only on explicit user weight edits and reject negative edit values; loading, recording received values, checking, saving, and exporting preserve raw weights. Distinguish receiver failures from sender behavior, and keep all editor operations local. - SDK process isolation specifies where SDK classes are called; document the actual tracker/session calls required for live reception as well as export encoding. - Keep tool implementation inside this subtree; add missing dependencies to Isaac Capture's existing requirements and matching package metadata. Run checks only from this directory or its descendants; never run repository-wide fixing hooks that can change other directories. @@ -23,13 +24,15 @@ SPDX-License-Identifier: Apache-2.0 - Reuse the launching Python environment and its SDK through sys.executable. Do not create tool-specific project metadata, lockfiles, environments or SDK installations, and do not infer runtime requirements from old local build artifacts. - Missing packages in the agent's inspection environment limit verification; report the unavailable checks. Do not turn verification requirements into proposals to install dependencies, provision environments, or substitute interpreters; maintain Isaac Capture's shared dependency declarations instead. - Use Viser's standard controls and public callbacks, with document changes serialized through State. Apply the same explicit weight-normalization rule to sliders and numeric fields. +- Client-side helpers may hide only their exact bridge input and label; never hide an ancestor container that can contain the entire Viser control panel. +- Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. - Keep metadata/force alignment with the producer and user; do not add source-row mappings, correspondence tracking across reordering, or alignment baselines to the editor. - JSON follows tactile_metadata.fbs and the existing bindings. Treat array and enum conversion for encoding as routine implementation rather than a separate product-format decision. - Persist only schema-defined tactile metadata fields. Keep model configuration, calibration, source mappings and internal IDs in session state; audit actual save/export output rather than inferring persistence requirements from editing needs. - Simplify persistence by deleting unsupported branches in place; do not relocate serialization into Document or introduce new state abstractions for a narrower file format. - For local editor checks using a cached Ruff executable, match the version in the repository hook configuration; older binaries may not understand the shared lint rules. Keep every check and formatter target inside this subtree. - Keep the GUI a thin adapter over shared document, geometry, storage and SDK operations. Prefer explicit native Viser widgets and one reusable property inspector; avoid widget-type inference, duplicate field editors in dialogs, and rebuilding scene nodes on every frame. Moving logic between files alone does not reduce implementation size. -- Use fixed URDF opacity and color-based link highlighting; do not add per-link visibility or opacity controls or state. +- Use URDF base opacity 0.4 multiplied by the rendered dataset layer opacity, with color-based link highlighting; do not add per-link visibility or opacity controls or state. - Under a hard code budget, finish and count a complete executable slice before expanding the UI. Refactoring dispatch or moving functions is not evidence that the implementation fits; verify the formatted total and retain budget for integration fixes. - Treat metadata wrist coordinates, Viser scene coordinates and camera screen orientation as distinct; diagnose an upside-down display from the scene/camera mapping, not URDF alignment. - Separate OpenXR schema/reference-space contracts from provider basis conversion; verify the plugin conversion before claiming injected HandPose follows OpenXR joint axes. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 9f726cfbab..7f3f7fd9f1 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -5,181 +5,188 @@ SPDX-License-Identifier: Apache-2.0 # Tactile Metadata Editor -A local Viser application for editing glove tactile metadata. It accepts URDF and -schema JSON files, preserves incomplete drafts, and exports checked numerical JSON -with an optional SDK binary. All runtime and device interactions are read-only. +Tactile Metadata Editor is a local Viser application for creating, editing, and +previewing glove tactile metadata. It loads URDF models and +[`TactileMetadata`](../../src/core/schema/fbs/tactile_metadata.fbs) JSON files. +All edits remain local; live connections only receive Isaac Capture data. -## Prepare the existing Python environment +## Set up and launch -Use your existing Isaac Capture environment with Python 3.11–3.13. Viser provides -the browser interface; yourdfpy loads URDF and computes forward kinematics; -trimesh and rtree provide surface queries; NumPy supplies numerical operations. -These libraries are declared in Isaac Capture's shared UI requirements and package -extra. The editor does not create an environment, install packages, find another -interpreter, or build an SDK. - -From the repository root, using the Python environment you selected: +Use an existing Isaac Capture environment with Python 3.11–3.13. Install the +shared UI requirements into that environment: ```bash cd scripts/tactile_metadata python -m pip install -r ../../src/core/python/requirements-ui.txt -python -m tactile_metadata_editor.cli gui --urdf model/left/linkerhand_FFG_left.urdf --sides left +python -m tactile_metadata_editor.cli ``` -Open [the local editor](http://127.0.0.1:8080). Use `--port` to choose another port. -Offline editing does not import Isaac Capture or start OpenXR. SDK checking, -binary export and actual live reception require an SDK already importable in the -same environment. Live reception also requires a compatible runtime and HMD -system; see the [tactile device documentation](../../docs/source/device/tactile/index.rst). - -## Create a layout - -1. In **Model**, choose the wrist link (`base_link` in the supplied left model). - Enter allowed links, for example `base_link,index_distal`, and apply them. - Inspect the resource list and joint tree. All URDF links use fixed opacity - 0.6; highlighting changes color. Package mappings are a JSON object of package - names to local directories, used to resolve `package://` resources without - network access. -2. Use **Pick target** and **Pick on q0 model** to pick `Wrist O`, `Wrist F` - toward the forearm, and `Wrist D` toward the back of the hand. Preview and - confirm the wrist reference. Both hands use the same right-handed axes. -3. In **Segments**, adjust the free-joint slider or numeric input. Fixed and - mimic joints appear in the tree; mimic joints follow their source. Confirm - **Set shared binding pose q0**. Existing bind poses and weights remain - numerically unchanged when q0 or the wrist reference changes. -4. New drafts contain all OpenXR hand joints in a neutral open pose, with `WRIST` - at the identity. Associate segments with URDF links, then pick a segment center - or use the native gizmo and inspector to adjust its frame. URDF frames are - initial estimates: calibrate their axes to the HandJoint convention before - production use. Associating an imported segment with a link preserves its - numerical bind pose. -5. In **Layout**, select the intended connected surface component. Set rows and - columns to `4`, and enter the new taxels' segment row list, for example `[1]`. - Pick the first center `O`, first-row last center `C`, and first-column last - center `R`. Preview the grid. Spans and spacing are reported in meters; spans - can be edited along the picked directions. -6. Choose scan order, reversal, optional excluded `[row,column]` cells and a - start output index. `-1` finds an unused contiguous interval. Failed - projections remain red candidates; excluded cells produce no candidate. - Apply or cancel. Applying creates independent taxels and clears the transient - generation parameters. For a single row use O/C; for a single column use O/R. - For one point, pick O and then **Single tangent** to provide the tangent. -7. Click draft points, toggle with Ctrl/Cmd, enable rectangle selection, or enter - an output-index list/range such as `0:15,20`. Selection defaults to visible - points; **Select through mesh** includes hidden points. Inspect lists in - pages of 25. Reproject, move, flip normals or twist tangents as needed. - Inspector fields replace only the edited component; mixed values are reported. -8. In **Binding**, edit the four slots or assign one to four segment rows with - equal initial weights. Repeated indices and slot order are retained. A slider - or numeric weight edit replaces its slot, then divides all four values by - their sum independently for each selected taxel. Invalid edits preserve the - rejected row and report the reason. Loading, recording, checking, saving and - exporting never normalize raw weights. -9. Preview motion with URDF controls. Return to q0 to resume surface picking. - Use **Preview / apply output indices** to review explicit index changes. - Moving or deleting taxels does not compact output indices; formal export - requires exactly `0..N-1`. Segment reordering remaps valid slot references, - including zero-weight slots. Deleted references remain invalid until repaired. - -Use the supplied right model in a separate document: +Open [http://127.0.0.1:8080](http://127.0.0.1:8080). To use another port: ```bash -python -m tactile_metadata_editor.cli gui --urdf model/right/linkerhand_FFG_right.urdf --sides right +python -m tactile_metadata_editor.cli --port 8090 ``` -Its wrist link is `hand_base_link`. For a real dual-hand URDF, choose `--sides -both`, then assign distinct wrist links and each hand's allowed surfaces. The -editor does not mirror a single-hand model. **New document** unlocks the model -hand selection after handling unsaved drafts. +Offline editing uses Viser, NumPy, yourdfpy, trimesh, and rtree. Live reception +also requires the Isaac Capture SDK, a compatible OpenXR runtime, and an +available HMD system in the same Python environment. -## Save, reopen, check and export +## Data and coordinates -**Export → Save all metadata** writes schema metadata values only, using the same -JSON as backups and export. Model, calibration and source settings stay in memory; -reopening restores values and array order only. Incomplete values remain saveable, -with validation errors shown in the interface. If any save fails, no side is -marked saved. Only metadata-value changes mark the document unsaved. -**Info issue → Locate** selects the relevant hand/entity in the shared draft -inspector. Advanced schema repair can replace incomplete arrays, with a temporary -backup before replacement. +The editor maintains independent left- and right-hand documents. **Toggle Side** +selects the active document; each side has its own metadata path and data. +General controls are in a floating panel at the top of the view. Selecting a +segment or taxel opens its property panel on the left. -```bash -python -m tactile_metadata_editor.cli gui --metadata left.metadata.json --sides left -python -m tactile_metadata_editor.cli check left.metadata.json -python -m tactile_metadata_editor.cli check left.metadata.json --sdk -python -m tactile_metadata_editor.cli export left.metadata.json --sides left --format sdk --out package -python -m tactile_metadata_editor.cli export left.metadata.json right.metadata.json --sides both --out pair -``` +Three pose datasets are displayed: + +- **Editing** contains editable segment and taxel binding poses plus Edit-mode + URDF joint values. +- **Saved** is the last loaded or saved metadata snapshot. Enable **Show Saved + Binding Poses** to render it at opacity 0.2. +- **Preview** contains current segment poses and Preview-mode URDF joint values. + Preview taxel poses are derived from the selected binding metadata. + +All UI values and scene interactions use visualization-space coordinates. +**Segments Alignment** maps metadata wrist coordinates into visualization space. +Loading applies that transform; saving applies its inverse. Changing the +alignment transforms the visible editor values without changing the underlying +metadata coordinates. + +## Edit metadata + +The **Edit** mode contains three tabs. + +### URDF + +1. Enter an absolute **URDF Path** and select **Load URDF**. +2. Use **URDF Alignment** to translate, rotate, and scale the model. +3. Expand **URDF Joints** to adjust each movable joint. + +URDF configuration is session-only and is not saved in tactile metadata. Its +base opacity is 0.4, multiplied by the opacity of the dataset layer being +rendered. + +### Segments + +- Use **Segments Alignment** to align all metadata poses. +- Add a segment from a HandJoint and XYZ position, or select **Add with URDF + Click** and click the model surface. +- Segments are represented by yellow spheres with radius 0.005 m. +- Expand **All Segments** to edit HandJoint, position, and XYZ rotation. +- Drag the handle above a segment to reorder it. Displayed IDs follow list order. + +HandJoint values must be unique. Taxel bindings use HandJoint names internally, +so segment reordering does not change their semantic references. Removing or +renaming a referenced segment leaves the affected taxel field invalid and +outlined in red. + +### Taxels + +**Add with Input** creates a row-major planar grid from Rows, Columns, Area +Width, Area Height, and Center Position. **Add with URDF Click** uses the clicked +surface position and outward normal as the grid frame. + +To bind taxels: + +1. Select **Start Binding to Segments** and select taxels in the scene. +2. Select **Select Segments** and choose one to four segments. +3. Select **Complete**. Initial active weights are equal. + +Expand **All Taxels** to edit pose, four segment slots, and four weights. `None` +is an editor-only inactive slot with weight zero. Editing a weight preserves the +entered value and adjusts other active weights proportionally so the row sums to +one. If only one slot is active, its weight is one. + +Dragging taxel rows changes their IDs and metadata array order. + +## Scene selection + +- Click an object to select it. +- Shift-click adds objects of the same type. +- Shift-drag selects a screen rectangle and temporarily disables camera dragging. +- A selection cannot contain both segments and taxels. If an initial rectangle + contains both, taxels take priority. +- Clicking empty scene space clears the selection. -Choose **Assign JSON to** and **Open JSON**, or provide `--sides left/right/both` -on the CLI. Pure checking needs no side; export requires one file per specified -hand, left then right for `both`. - -The protocol JSON follows [tactile_metadata.fbs](../../src/core/schema/fbs/tactile_metadata.fbs): -`geometry_mapping_type: "LBS"`, nested `geometry_mapping` arrays, and -`segment_source_type: "HandSkeletonRef"` with nested `segment_source.joint_ids`. -Pose quaternions are XYZW; influence structs contain `values` arrays. There is no -additional metadata format. Taxel array row order expresses output order. -Exports contain only metadata JSON and optional SDK binaries; check results and -revisions appear in the interface or CLI output. All sides must succeed before -the new or empty destination is published. GUI exports capture a snapshot; -later edits do not change that snapshot. - -## Live reception and the SDK-free demo - -Launch with `gui --demo` to use a synthetic independent layout and hand/force -stream. In **Live**, configure each side's tactile collection (default `tactile`), -HandTracker output and optional JointState collection, then connect. Apply source -configuration in the current session and reconnect. JointState mappings specify -a source name, URDF target, scale and offset; the panel reports raw values, -converted values, limits and validity. Demo JointState names come from mappings -present when connecting. - -Only enabled JointState drives the live URDF; otherwise it is hidden. HandPose -supplies the tracked skeleton and current segment transforms relative to WRIST. -The live skeleton and taxels use the tracked wrist frame. The draft gizmo always -uses the calibrated q0 frame, independently of the animated drawing. - -**Play / pause** freezes the live drawing while receipt continues. Draft forms, -wire markers and manipulators remain editable. Resume uses the latest frame and -current draft. Returning to EDIT shows q0 without stopping receipt. Interrupted -streams retain the last geometry and mark forces invalid rather than zero. -Force colors cover 0–30 N by default and saturate at the selected maximum. -Multi-segment LBS supplies position, without inventing a mixed rigid orientation -or a world-space force arrow. - -Visualization defaults to **Received** metadata. Switching to **Draft** changes -only the drawing. **Record metadata groups** first backs up the draft to the -system temporary directory, logs its path, then replaces the selected groups -verbatim. Partial replacements may introduce count/reference errors, which remain -editable. Metadata/force alignment is the producer's and user's responsibility: -forces always use protocol row order. Source switching establishes no alignment. - -## Implementation and verification - -`document.py` owns raw arrays, IDs, explicit edits, validation and transactional -State revisions. `geometry.py` owns local resource resolution, FK, frame math, -projection and LBS. Its link attachments are derived caches, not metadata edits. -`storage.py` owns schema JSON, atomic saves, backups and package publication. -`sdk.py` imports SDK classes only inside children started with `sys.executable`: -checks use `LBS`, `HandSkeletonRef`, `TactileMetadata.to_bytes()`; live reception -uses `HandTracker`, `TactileTracker`, optional `JointStateTracker`, -`OpenXRSession` and `DeviceIOSession.run/update`. Views are copied before the -bounded local pipe, and only the latest frame is retained. `cli.py` and `ui.py` -share document, geometry and storage operations. The UI uses native Viser -controls and persistent scene nodes; document edits run on one event queue. - -Local development checks target this package only: +The **Selected Objects** panel edits position and rotation, deletes selected +objects, and scales multi-selection positions about their center. Edit mode +changes Editing data; Preview mode changes Preview segments. Preview taxels are +derived and cannot be selected. + +## Save and load + +Enter **Metadata Path**, then use **Load** or **Save**. Saving validates the +current document. If errors remain, the editor asks whether to continue; invalid +drafts can still be written. + +**New** resets the active side and initializes all OpenXR HandJoint segments in +the neutral hand pose. The left-hand defaults are mirrored before conversion to +visualization space. + +The JSON contains only schema fields: + +- `geometry_mapping_type` and `geometry_mapping` +- `segment_source_type` and `segment_source` + +Editor HandJoint references are converted to `taxel_segment_indices` when +saving and restored when loading. Pose orientations are XYZW quaternions, and +taxel array order is the tactile force order. URDF paths, alignments, joint +values, selections, and live settings are not persisted. + +The tool does not provide SDK binary export, a separate project format, or +standalone check/export CLI commands. + +## Preview and live data + +Use **Restore to Editor Binding Values** to initialize Preview from Editing. +Choose **Binding Metadata Source**: + +- **Editing** uses the current editor bindings. +- **Live** becomes available after metadata is received from the tactile stream. + +**Apply Preview Data to Editor** fully replaces Editing segments, taxels, +bindings, and URDF joint values. The button is disabled when Preview segments +are incompatible with the selected bindings. + +For live reception: + +1. Set **Tactile Collection ID**. +2. Optionally enable **Use JointState to Drive URDF**, set its collection ID, + and provide mappings: + + ```json + {"source_joint": ["urdf_joint", 1.0, 0.0]} + ``` + +3. Select **Connect Live**. + +HandPose updates Preview segments. Enabled JointState mappings update Preview +URDF joints. Live frames overwrite manual Static values while connected. +Tactile force magnitude maps from blue at 0 N to red at 30 N, with values above +30 N clamped. Missing or inactive force data is light grey. + +For multi-segment LBS, taxel orientation uses polar decomposition of the blended +rotation. Single-segment bindings use their direct rigid transform; singular +blends fall back to the taxel bind rotation. + +## Implementation + +- `document.py` defines editor-native segments, taxels, validation, and per-side + state. +- `storage.py` converts between editor data and schema JSON and performs atomic + saves. +- `geometry.py` handles URDF loading, scene transforms, picking, grid creation, + HandPose conversion, and LBS. +- `sdk.py` receives HandPose, TactileMetadata/TactileDataArray, and optional + JointState in a child process started with the launching interpreter. +- `ui.py` owns Viser controls, scene state, selection, and Edit/Preview workflow. +- `editor_interactions.js` supplies list drag-and-drop and invalid-field styling. + +Local source checks: ```bash ruff format --check tactile_metadata_editor ruff check tactile_metadata_editor python -m compileall -q tactile_metadata_editor ``` - -The inspection environment lacked the GUI/geometry libraries and SDK. Syntax, -lint, schema round trips, weight/reordering operations, rollback, atomic export, -and SDK-unavailable handling were checked locally. Viser call signatures were -checked against cached 1.1.1 source. Browser interaction, numerical mesh/LBS -execution, successful SDK encoding and actual hardware reception still require -verification in the configured environment; these checks are not claimed passed. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index bee503c32a..fc94536176 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -15,7 +15,7 @@ About data cooridanates: all UI editor use visualization space. When save to met About LBS taxel rotation: determine by Polar Decomposition algorithm. At singularity, use taxel's bind rotation. Make sure to optimize for single-segment binding case, where the rotation can be directly determined without running Polar Decomposition. -1. At top, always display general function: Toggle side, new/save/load metadata file, reset camera. A Mode single seletion from: Edit, Preview. And a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". +1. In a General panel at the top of the view, always display: Toggle side, new/save/load metadata file, reset camera, a Mode single selection from Edit and Preview, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". 2. This tool actually manages 3 copies of pose data (pose of segments and taxels), with different data sources. a) Editing dataset: the data that the user is editing in the tool. It's the binding poses of the segments and taxels under editing. Plus the URDF joint values that's under Editing mode. @@ -41,7 +41,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula d) (spannable) A full list of all segments, with an id number (read-only, starting at 0, it's the index of the segment in the list), their name (dropdown list of HandJoint, display in enum name), editable position (3 float input), editable rotation (3 float slider). This is to modify the binding poses of the segments, display this info above this section. Each segment has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. * It's invalid to have more than one segments to select the same HandJoint. At error, outline the repeated fields in red. * Some fields in Taxels tab refer to segments in this Segments tab, and modifying segments may result in invalid taxel binding that's revealed in Taxels tab. -* Segments are displayed as yellow spheres of radius 0.01m. The three axes of each segment are not displayed by default.Clicking an segment on the 3D scene while in Edit mode will select the segment, highlight it, reveal its 3 axes. When a segment is newly added, it's selected, and deselect the previous selected object(s). Clicking on any other area in the 3D scene will deselect the segment; dragging 3D scene or clicking on the menu/other UI area will not deselect the segment. Clicking on the same selected segment will not do anything. +* Segments are displayed as yellow spheres of radius 0.005m. The three axes of each segment are not displayed by default.Clicking an segment on the 3D scene while in Edit mode will select the segment, highlight it, reveal its 3 axes. When a segment is newly added, it's selected, and deselect the previous selected object(s). Clicking on any other area in the 3D scene will deselect the segment; dragging 3D scene or clicking on the menu/other UI area will not deselect the segment. Clicking on the same selected segment will not do anything. 9. Taxels tab: * Taxels don't have their onw alignment, it uses the alignment of the segments. @@ -58,7 +58,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula * Taxels are displayed as light grey flat square of size 0.01m*0.01m. The three axes of each taxel are not displayed by default. Clicking an taxel on the 3D scene while in Edit mode will select the taxel, highlight it, reveal its 3 axes. The 3D scene modification rules are the same as segments. * Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. * When multiple objects are selected, the 3-axis visualizer should show in the center of the selected objects' area, with axis directions align with the visualiation space coordinates. -* An additional small menu for selected object(s)' modification, apply also to single object selection: +* An additional small menu on the left side of the view for selected object(s)' modification, apply also to single object selection: a) (read only) selected elements type and count (multi selection is stated in following part) b) (editable) current position (defined above) c) (editable) current rotation (defined above) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index b6cd366c09..5421955fc2 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -18,6 +18,38 @@ [0.0, 0.0, 1.0, 0.0], [0.0, 0.0, 0.0, 1.0], ] +# Neutral right-hand OpenXR poses in wrist coordinates; left-hand mirroring is +# applied before conversion into visualization space. +# fmt: off +DEFAULT_OPENXR_SEGMENT_POSES = ( + (-0.00244, 0.00122, -0.05917, 0.010296699, 0.02060433, -0.000212213, 0.999734662), + (0, 0, 0, 0, 0, 0, 1), + (-0.022, -0.012, -0.028, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.047208158, -0.020249943, -0.065583071, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.065292271, -0.02616838, -0.09254484, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.080088363, -0.031010737, -0.114604468, -0.00692186, 0.30239121, 0.275661445, 0.912426661), + (-0.00528, 0, -0.02068, 0, 0.124664128, 0, 0.992199), + (-0.024, 0, -0.094, -0.017438446, 0.039983244, 0.00069791, 0.999047924), + (-0.027194641, -0.00139598, -0.133847779, -0.043584497, 0.039951273, 0.00174431, 0.99824909), + (-0.029105295, -0.003487718, -0.157679985, -0.060999707, 0.039914746, 0.00244129, 0.997336397), + (-0.030691675, -0.005925105, -0.177467419, -0.060999707, 0.039914746, 0.00244129, 0.997336397), + (-0.00088, 0.00044, -0.02134, 0.010296699, 0.02060433, -0.000212213, 0.999734662), + (-0.004, 0.002, -0.097, -0.017452406, 0, 0, 0.999847695), + (-0.004, 0.000429523, -0.141972587, -0.043619387, 0, 0, 0.999048222), + (-0.004, -0.002010838, -0.169866039, -0.06104854, 0, 0, 0.998134798), + (-0.004, -0.004691964, -0.191702054, -0.06104854, 0, 0, 0.998134798), + (0.0033, 0, -0.02002, 0, -0.081592271, 0, 0.996665792), + (0.015, 0, -0.091, -0.017444553, -0.029990932, -0.000523494, 0.999397797), + (0.017516954, -0.001465779, -0.132898883, -0.04359976, -0.029966951, -0.001308385, 0.998598684), + (0.019070086, -0.003731828, -0.158753338, -0.06102107, -0.029939553, -0.001831181, 0.997685671), + (0.020319944, -0.006291084, -0.1795593, -0.06102107, -0.029939553, -0.001831181, 0.997685671), + (0.00704, -0.00088, -0.01804, -0.022312091, -0.184914996, -0.004199362, 0.982492229), + (0.032, -0.004, -0.082, -0.017415551, -0.064944346, -0.001133608, 0.99773626), + (0.036145766, -0.005116784, -0.113710652, -0.043527274, -0.064892417, -0.002833264, 0.996938475), + (0.038599442, -0.006772743, -0.132478637, -0.06091962, -0.064833086, -0.003965361, 0.996026981), + (0.040915463, -0.008966391, -0.150193714, -0.06091962, -0.064833086, -0.003965361, 0.996026981), +) +# fmt: on @dataclass diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index 11b148bd1d..475a542523 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -15,7 +15,17 @@ let parent = input; for (let depth = 0; parent && depth < 5; ++depth, parent = parent.parentElement) { if (parent.textContent?.includes("__Tactile Reorder Event")) { - if (parent.parentElement) parent.parentElement.style.display = "none"; + input.style.position = "absolute"; + input.style.width = "1px"; + input.style.height = "1px"; + input.style.opacity = "0"; + input.style.pointerEvents = "none"; + input.tabIndex = -1; + for (const label of parent.querySelectorAll("label")) { + if (label.textContent?.includes("__Tactile Reorder Event")) { + label.style.display = "none"; + } + } return input; } } diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index e6b5d57a19..0dbddc127c 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -10,7 +10,13 @@ from trimesh import transformations as tf from yourdfpy import URDF -from .document import Document, JOINTS, Segment, Taxel +from .document import ( + DEFAULT_OPENXR_SEGMENT_POSES, + Document, + JOINTS, + Segment, + Taxel, +) VISUALIZATION_FROM_HAND = np.array( [[-1, 0, 0, 0], [0, 0, -1, 0], [0, -1, 0, 0], [0, 0, 0, 1]], @@ -20,6 +26,7 @@ [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 0, 1]], dtype=float, ) +URDF_OPACITY = 0.4 def matrix(values: list[float]) -> np.ndarray: @@ -43,6 +50,17 @@ def pose(transform: np.ndarray) -> list[float]: ] +def default_segments(side: str) -> list[Segment]: + poses = [list(value) for value in DEFAULT_OPENXR_SEGMENT_POSES] + if side == "left": + for value in poses: + value[0], value[4], value[5] = -value[0], -value[4], -value[5] + return [ + Segment(joint, pose(VISUALIZATION_FROM_HAND @ matrix(value))) + for joint, value in zip(JOINTS, poses) + ] + + def transform(position, rotation_degrees) -> np.ndarray: result = tf.euler_matrix(*np.deg2rad(rotation_degrees), axes="sxyz") result[:3, 3] = np.asarray(position, dtype=float) @@ -193,7 +211,7 @@ def draw( alignment: np.ndarray, scale: float, highlighted: bool = False, - opacity: float = 0.6, + opacity: float = URDF_OPACITY, ) -> None: raw = self.raw_fk(q) for name, (link, mesh) in self.surfaces.items(): diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 83b579647a..858bcc059e 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -90,19 +90,23 @@ def _preview_from(document: Document) -> Document: def _build_gui(self) -> None: gui = self.server.gui - self.side_text = gui.add_markdown("") - self._action_button("Toggle Side", self._toggle_side) - self.metadata_path = gui.add_text("Metadata Path", "") - self._action_button("New", self._new) - _, self.save_info = self._action_button("Save", self._save) - self._action_button("Load", self._load) - self._action_button("Reset Camera", self._reset_camera) - self.mode = gui.add_dropdown("Mode", ("Edit", "Preview")) - self.show_saved = gui.add_checkbox("Show Saved Binding Poses", False) - self._watch(self.mode, self._set_mode) - self._watch(self.show_saved, lambda _: self._request_draw()) - self.reorder_bridge = gui.add_text("__Tactile Reorder Event", "") - self._watch(self.reorder_bridge, self._reorder) + self.general_panel = gui.add_panel() + self.general_tab = self.general_panel.add_tab("General") + with self.general_tab: + self.side_text = gui.add_markdown("") + self._action_button("Toggle Side", self._toggle_side) + self.metadata_path = gui.add_text("Metadata Path", "") + _, self.new_info = self._action_button("New", self._new) + _, self.save_info = self._action_button("Save", self._save) + self._action_button("Load", self._load) + self._action_button("Reset Camera", self._reset_camera) + self.mode = gui.add_dropdown("Mode", ("Edit", "Preview")) + self.show_saved = gui.add_checkbox("Show Saved Binding Poses", False) + self._watch(self.mode, self._set_mode) + self._watch(self.show_saved, lambda _: self._request_draw()) + self.reorder_bridge = gui.add_text("__Tactile Reorder Event", "") + self._watch(self.reorder_bridge, self._reorder) + self.general_panel.float(x=0, y=0, width=320) self.edit_tabs = gui.add_tab_group() self.urdf_tab = self.edit_tabs.add_tab("URDF") @@ -212,7 +216,12 @@ def _build_gui(self) -> None: self.preview_joint_folder = gui.add_folder("URDF Joints") self.preview_segment_folder = gui.add_folder("Segments") - self.selection_folder = gui.add_folder("Selected Objects", visible=False) + self.selection_panel = gui.add_panel(visible=False) + self.selection_tab = self.selection_panel.add_tab("Selected Object") + with self.selection_tab: + self.selection_folder = gui.add_folder("Selection") + self.selection_panel.dock_left() + self.selection_panel.set_width(320) def _numbers(self, prefix: str, values, step: float): return tuple( @@ -253,10 +262,10 @@ async def clicked(_): return button, info def _run_action(self, callback, value=None, info=None) -> None: + if info is not None: + info.content = "" try: callback() if value is None else callback(value) - if info is not None: - info.content = "" except Exception as exc: LOG.exception("Editor action failed") if info is not None: @@ -319,11 +328,13 @@ def _set_values(widgets, values) -> None: def _new(self) -> None: document = Document(self.state.side) document.alignment = geo.VISUALIZATION_FROM_HAND.tolist() + document.segments = geo.default_segments(self.state.side) self.state.documents[self.state.side] = document self.state.saved[self.state.side] = None self.state.paths.pop(self.state.side, None) self.preview[self.state.side] = self._preview_from(document) self._sync_side() + self.new_info.content = f"New {self.state.side} metadata created." def _load(self) -> None: path = self.metadata_path.value.strip() @@ -1013,7 +1024,7 @@ def _ray_rows(event, kind: str, values) -> list[int]: if kind == "segments": distance = (point - origin) @ direction offset = np.linalg.norm(point - origin - distance * direction) - if distance > 0 and offset <= 0.01: + if distance > 0 and offset <= 0.005: hits.append((distance, row)) continue inverse = np.linalg.inv(geo.matrix(value.pose)) @@ -1092,9 +1103,9 @@ def _rebuild_selection(self) -> None: values = self._selection_values() rows = [row for row in self.selected_rows if 0 <= row < len(values)] self.selected_rows = rows - self.selection_folder = self.server.gui.add_folder( - "Selected Objects", visible=bool(rows) - ) + self.selection_panel.visible = bool(rows) + with self.selection_tab: + self.selection_folder = self.server.gui.add_folder("Selection") if not rows: return with self.selection_folder: @@ -1206,6 +1217,7 @@ def _draw_instances( return positions = np.asarray([value[:3, 3] for value in matrices]) rotations = np.asarray([tf.quaternion_from_matrix(value) for value in matrices]) + scale = 0.005 if kind == "segments" else 0.01 mesh = ( trimesh.creation.icosphere(subdivisions=1) if kind == "segments" @@ -1227,7 +1239,7 @@ def _draw_instances( mesh.faces, rotations, positions, - batched_scales=np.full(len(values), 0.01), + batched_scales=np.full(len(values), scale), batched_colors=np.asarray(colors, dtype=np.uint8), opacity=opacity, wireframe=wireframe, @@ -1237,7 +1249,7 @@ def _draw_instances( handle.visible = True handle.batched_positions = positions handle.batched_wxyzs = rotations - handle.batched_scales = np.full(len(values), 0.01) + handle.batched_scales = np.full(len(values), scale) handle.batched_colors = np.asarray(colors, dtype=np.uint8) handle.opacity = opacity @@ -1357,6 +1369,7 @@ def _draw_selection_axes(self) -> None: handle.batched_wxyzs = rotations def _draw(self) -> None: + active_opacity = 1.0 if self.model is not None: q = ( self.document.q @@ -1369,7 +1382,7 @@ def _draw(self) -> None: self.urdf_alignment, self.urdf_scale, highlighted=self.pick_mode is not None, - opacity=1.0, + opacity=geo.URDF_OPACITY * active_opacity, ) self._hide_prefix("/edit") self._hide_prefix("/preview") @@ -1380,7 +1393,7 @@ def _draw(self) -> None: "/edit", self.document.segments, self.document.taxels, - 1.0, + active_opacity, True, ) self._draw_errors() @@ -1402,7 +1415,7 @@ def _draw(self) -> None: "/preview", self.preview_document.segments, preview_taxels, - 1.0, + active_opacity, True, forces, ) @@ -1422,8 +1435,11 @@ def _draw(self) -> None: def run(self) -> None: try: while True: - while not self.events.empty(): - self.events.get_nowait()() + try: + while True: + self.events.get_nowait()() + except queue.Empty: + pass self._update_live() if self.redraw or self.receiver is not None: self._draw() From 3178fe9818d4d047ba159e98812cf1e3f8ee1df8 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 13:10:25 -0700 Subject: [PATCH 16/36] 1. Switch the places of Preview tab - URDF Joints and Segments 2. Put the "General" menu back to at first: the top of the right menu 3. Make the "Selected objects" menu always displayed on the left. WHen nothing is selected, grey out all input fields and show that type: None count: 0 4. Change the dimension of the taxel squre to 1/4 of current. 5. In Taxel menu, modify the Add Taxels section: Add Rotation fields (3*floats slider). Re-text "Add with Input" to "Add Taxels". Replace "Add with URDF Click" with a diffferent function, that when clicking the button and click one point of urdf (same as Add with URDF Click), replace the Position and Rotation fieds above with the captured urdf surface values but not adding taxel yet (different from Add with URDF Click). You determine the text on this button. And place this button above "Add Taxels" button. Signed-off-by: Lotus Li --- scripts/tactile_metadata/README.md | 16 ++-- scripts/tactile_metadata/UI&Feature Design.md | 13 ++- .../tactile_metadata_editor/ui.py | 87 +++++++++++-------- 3 files changed, 65 insertions(+), 51 deletions(-) diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 7f3f7fd9f1..3d4938c6e1 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -35,8 +35,8 @@ available HMD system in the same Python environment. The editor maintains independent left- and right-hand documents. **Toggle Side** selects the active document; each side has its own metadata path and data. -General controls are in a floating panel at the top of the view. Selecting a -segment or taxel opens its property panel on the left. +General controls are at the top of the right menu. The Selected Object panel is +always present on the left. Three pose datasets are displayed: @@ -83,9 +83,10 @@ outlined in red. ### Taxels -**Add with Input** creates a row-major planar grid from Rows, Columns, Area -Width, Area Height, and Center Position. **Add with URDF Click** uses the clicked -surface position and outward normal as the grid frame. +Set Rows, Columns, Area Width, Area Height, Center Position, and Rotation. +**Capture URDF Surface Pose** copies a clicked surface position and orientation +into those fields without creating taxels. **Add Taxels** then creates the +row-major planar grid. Taxels are light-grey squares with side length 0.0025 m. To bind taxels: @@ -112,7 +113,8 @@ Dragging taxel rows changes their IDs and metadata array order. The **Selected Objects** panel edits position and rotation, deletes selected objects, and scales multi-selection positions about their center. Edit mode changes Editing data; Preview mode changes Preview segments. Preview taxels are -derived and cannot be selected. +derived and cannot be selected. With no selection, the panel remains visible, +shows Type `None` and Count `0`, and disables its input fields. ## Save and load @@ -145,6 +147,8 @@ Choose **Binding Metadata Source**: - **Editing** uses the current editor bindings. - **Live** becomes available after metadata is received from the tactile stream. +The Preview Static section lists Segments before URDF Joints. + **Apply Preview Data to Editor** fully replaces Editing segments, taxels, bindings, and URDF joint values. The button is disabled when Preview segments are incompatible with the selected bindings. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index fc94536176..79f9c502b6 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -15,7 +15,7 @@ About data cooridanates: all UI editor use visualization space. When save to met About LBS taxel rotation: determine by Polar Decomposition algorithm. At singularity, use taxel's bind rotation. Make sure to optimize for single-segment binding case, where the rotation can be directly determined without running Polar Decomposition. -1. In a General panel at the top of the view, always display: Toggle side, new/save/load metadata file, reset camera, a Mode single selection from Edit and Preview, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". +1. At the top of the right menu, always display a General section containing: Toggle side, new/save/load metadata file, reset camera, a Mode single selection from Edit and Preview, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". 2. This tool actually manages 3 copies of pose data (pose of segments and taxels), with different data sources. a) Editing dataset: the data that the user is editing in the tool. It's the binding poses of the segments and taxels under editing. Plus the URDF joint values that's under Editing mode. @@ -45,8 +45,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula 9. Taxels tab: * Taxels don't have their onw alignment, it uses the alignment of the segments. - a) Add new taxels: input fields of columns (int input), rows (int input), intersection width (float input, declare the unit is m), height (float input, declare the unit is m), taxels center position (3 float input fields). An "Add with Input" button. When adding taxels, it will generate columns*rows taxels, which aligned in a grid of the inputted area width and height, with the grid center at the inputted taxels center position, the grid orientation facing +Z axis of visualization space. Each taxel's orientation is originally identity. - b) Add new taxels by 3D scene interaction: a button to "Add with URDF Click" button with columns and rows int input fields. After clicking this button, the button shows a "pressed" shape, the 3D scene will grey out, with only the URDF (of the Editing dataset) highlighted. The user can click on the URDF to add a new array of taxel on the URDF surface, this will use the clicked URDF position as grid center position, and the clicked URDF normal (pointing out of the URDF surface) as grid orientation. Otherwise the grid-adding rule is the same as Add with Input.Clicking on any other area will cancel the operation. When taxels are newly created, they're selected and the previous selected objects are de-selected. + a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), and rotation (3 float sliders). "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields; newly created taxels are selected and previous objects are deselected. c) A series of buttons "Start Binding to Segments" "Select Segments" "Complete". At first, only "Start Binding to Segments" is clicable. When clicking this button, 3D scene will grey out except for taxels (from Edit dataset) is displayed normally (without highlight), the instruction field under the button will print: "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. Click "Select Segments" after done selecting taxels. Click on any other area in 3D scene to cancel.". When at least one taxel is selected, the button "Select Segments" becomes clickcable. The 3D scene stays greyed out and only segments (from Edit dataset) are displayed normally (without highlight). The instruction field changes to print: "Select 1-4 segment(s) to bind to the selected taxel(s) in 3D scene. Hold Shift to multi-select. Click "Complete" after done selecting segments. Click on any other area in 3D scene to cancel.". When at least one segment is selected, the "Complete" button becomes clickable. Clicking "Complete" button will update the Edit dataset's binding, each original weight is average (if selected 4 segments, then 0.25 each; if selected 3 segments, then 1/3 each.) (also update the following section). If more than 4 segments are selected, don't update the binding, and print in instruction field: "Too many segments selected, please select 1-4 segments. Click "Start Binding to Segments" to start over.". * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. Each taxel has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. @@ -55,10 +54,10 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. b) the weights don't sum to 1. (Can happen when a segment is removed from Segments tab or by Preview apply to editor.) - weights input fields. c) the weight is 0 for a non-None segment - weight input field. -* Taxels are displayed as light grey flat square of size 0.01m*0.01m. The three axes of each taxel are not displayed by default. Clicking an taxel on the 3D scene while in Edit mode will select the taxel, highlight it, reveal its 3 axes. The 3D scene modification rules are the same as segments. +* Taxels are displayed as light grey flat squares of size 0.0025m*0.0025m. The three axes of each taxel are not displayed by default. Clicking an taxel on the 3D scene while in Edit mode will select the taxel, highlight it, reveal its 3 axes. The 3D scene modification rules are the same as segments. * Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. * When multiple objects are selected, the 3-axis visualizer should show in the center of the selected objects' area, with axis directions align with the visualiation space coordinates. -* An additional small menu on the left side of the view for selected object(s)' modification, apply also to single object selection: +* An additional small menu on the left side of the view is always displayed for selected object(s)' modification, applying also to single-object selection. With no selection it shows Type: None and Count: 0 and disables every input field: a) (read only) selected elements type and count (multi selection is stated in following part) b) (editable) current position (defined above) c) (editable) current rotation (defined above) @@ -79,8 +78,8 @@ Live section: * While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. Static section: a) A button "Restore to Editor Binding Values" - b) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. - c) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Exclude the reordering grabber UI. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. + b) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Exclude the reordering grabber UI. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. + c) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. * Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. Button info display: show error messages at bottom when action fails. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 858bcc059e..92bbbb515a 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -90,9 +90,7 @@ def _preview_from(document: Document) -> Document: def _build_gui(self) -> None: gui = self.server.gui - self.general_panel = gui.add_panel() - self.general_tab = self.general_panel.add_tab("General") - with self.general_tab: + with gui.add_folder("General"): self.side_text = gui.add_markdown("") self._action_button("Toggle Side", self._toggle_side) self.metadata_path = gui.add_text("Metadata Path", "") @@ -106,7 +104,6 @@ def _build_gui(self) -> None: self._watch(self.show_saved, lambda _: self._request_draw()) self.reorder_bridge = gui.add_text("__Tactile Reorder Event", "") self._watch(self.reorder_bridge, self._reorder) - self.general_panel.float(x=0, y=0, width=320) self.edit_tabs = gui.add_tab_group() self.urdf_tab = self.edit_tabs.add_tab("URDF") @@ -160,10 +157,11 @@ def _build_gui(self) -> None: self.grid_center = self._numbers( "Center Position", (0.0, 0.0, 0.0), 0.001 ) - self._action_button("Add with Input", self._add_taxels_input) - self.taxel_click_button, _ = self._action_button( - "Add with URDF Click", self._pick_taxels + self.grid_rotation = self._sliders("Rotation", (0.0, 0.0, 0.0)) + self.taxel_capture_button, _ = self._action_button( + "Capture URDF Surface Pose", self._capture_taxel_pose ) + self._action_button("Add Taxels", self._add_taxels_input) with gui.add_folder("Bind Taxels to Segments"): self.binding_start = gui.add_button("Start Binding to Segments") self.binding_next = gui.add_button("Select Segments", disabled=True) @@ -213,10 +211,10 @@ def _build_gui(self) -> None: "Restore to Editor Binding Values", self._restore_preview, ) - self.preview_joint_folder = gui.add_folder("URDF Joints") - self.preview_segment_folder = gui.add_folder("Segments") + self.preview_segment_folder = gui.add_folder("Segments", order=1) + self.preview_joint_folder = gui.add_folder("URDF Joints", order=2) - self.selection_panel = gui.add_panel(visible=False) + self.selection_panel = gui.add_panel() self.selection_tab = self.selection_panel.add_tab("Selected Object") with self.selection_tab: self.selection_folder = gui.add_folder("Selection") @@ -298,7 +296,7 @@ def _sync_side(self) -> None: self.live_document, self.live_metadata_key = None, "" self.pick_mode = None self.segment_click_button.label = "Add with URDF Click" - self.taxel_click_button.label = "Add with URDF Click" + self.taxel_capture_button.label = "Capture URDF Surface Pose" self.binding_source.options = ("Editing",) self.binding_source.value = "Editing" self.side_text.content = f"**Side: {self.state.side.title()}**" @@ -444,7 +442,9 @@ def _rebuild_joint_controls(self) -> None: with self.edit_joint_folder: self._add_joint_sliders(self.document, self.edit_joint_widgets) with self.static_folder: - self.preview_joint_folder = self.server.gui.add_folder("URDF Joints") + self.preview_joint_folder = self.server.gui.add_folder( + "URDF Joints", order=2 + ) with self.preview_joint_folder: self._add_joint_sliders( self.preview_document, self.preview_joint_widgets @@ -492,27 +492,31 @@ def _pick_segment(self) -> None: ) def _grid_values(self): + center = [float(widget.value) for widget in self.grid_center] return ( - [float(widget.value) for widget in self.grid_center], + center, float(self.grid_width.value), float(self.grid_height.value), int(self.grid_rows.value), int(self.grid_columns.value), + geo.transform(center, [widget.value for widget in self.grid_rotation]), ) def _add_taxels_input(self) -> None: - center, width, height, rows, columns = self._grid_values() + center, width, height, rows, columns, frame = self._grid_values() selected = self.document.add_taxels( - geo.grid_poses(center, width, height, rows, columns) + geo.grid_poses(center, width, height, rows, columns, frame) ) self.selected_kind, self.selected_rows = "taxels", selected self._rebuild_edit_lists() self._rebuild_selection() - def _pick_taxels(self) -> None: - self.pick_mode = None if self.pick_mode == "taxels" else "taxels" - self.taxel_click_button.label = ( - "● Add with URDF Click" if self.pick_mode else "Add with URDF Click" + def _capture_taxel_pose(self) -> None: + self.pick_mode = None if self.pick_mode == "taxel_pose" else "taxel_pose" + self.taxel_capture_button.label = ( + "● Capture URDF Surface Pose" + if self.pick_mode + else "Capture URDF Surface Pose" ) def _rebuild_edit_lists(self) -> None: @@ -721,7 +725,7 @@ def _reorder(self, value: str) -> None: def _start_binding(self) -> None: self.pick_mode = None self.segment_click_button.label = "Add with URDF Click" - self.taxel_click_button.label = "Add with URDF Click" + self.taxel_capture_button.label = "Capture URDF Surface Pose" self.binding_phase = "taxels" self.binding_taxels, self.binding_segments = [], [] self.selected_kind, self.selected_rows = None, [] @@ -835,7 +839,9 @@ def _rebuild_preview_segments(self) -> None: self.preview_segment_folder.remove() self.preview_segment_widgets = [] with self.static_folder: - self.preview_segment_folder = self.server.gui.add_folder("Segments") + self.preview_segment_folder = self.server.gui.add_folder( + "Segments", order=1 + ) with self.preview_segment_folder: for row, segment in enumerate(self.preview_document.segments): with self.server.gui.add_folder(f"{row}: {segment.joint}"): @@ -1034,7 +1040,7 @@ def _ray_rows(event, kind: str, values) -> list[int]: continue distance = -local_origin[2] / local_direction[2] local_hit = local_origin + distance * local_direction - if distance > 0 and np.max(np.abs(local_hit[:2])) <= 0.005: + if distance > 0 and np.max(np.abs(local_hit[:2])) <= 0.00125: hits.append((distance, row)) return [min(hits)[1]] if hits else [] @@ -1064,7 +1070,7 @@ def _box_rows(event, values) -> list[int]: def _handle_pick(self, event) -> None: mode, self.pick_mode = self.pick_mode, None self.segment_click_button.label = "Add with URDF Click" - self.taxel_click_button.label = "Add with URDF Click" + self.taxel_capture_button.label = "Capture URDF Surface Pose" if self.model is None: raise ValueError("Load a URDF before using URDF Click") hit = self.model.hit( @@ -1081,14 +1087,12 @@ def _handle_pick(self, event) -> None: value[:3] = hit[1].tolist() row = self.document.add_segment(self.new_segment_joint.value, value) self.selected_kind, self.selected_rows = "segments", [row] + self._rebuild_edit_lists() else: - _, width, height, rows, columns = self._grid_values() frame = geo.surface_frame(hit[1], hit[2]) - selected = self.document.add_taxels( - geo.grid_poses(hit[1], width, height, rows, columns, frame) - ) - self.selected_kind, self.selected_rows = "taxels", selected - self._rebuild_edit_lists() + position, rotation = geo.components(frame) + self._set_values(self.grid_center, position) + self._set_values(self.grid_rotation, rotation) def _selection_values(self): if self.selected_kind is None: @@ -1103,17 +1107,18 @@ def _rebuild_selection(self) -> None: values = self._selection_values() rows = [row for row in self.selected_rows if 0 <= row < len(values)] self.selected_rows = rows - self.selection_panel.visible = bool(rows) with self.selection_tab: self.selection_folder = self.server.gui.add_folder("Selection") - if not rows: - return with self.selection_folder: self.server.gui.add_markdown( - f"Type: **{self.selected_kind}** \nCount: **{len(rows)}**" + f"Type: **{self.selected_kind if rows else 'None'}** \n" + f"Count: **{len(rows)}**" + ) + center = ( + np.asarray([values[row].pose[:3] for row in rows]).mean(axis=0) + if rows + else np.zeros(3) ) - positions = np.asarray([values[row].pose[:3] for row in rows]) - center = positions.mean(axis=0) if len(rows) == 1: _, rotation = geo.components(geo.matrix(values[rows[0]].pose)) else: @@ -1121,6 +1126,11 @@ def _rebuild_selection(self) -> None: self.selection_position_widgets = self._numbers("Position", center, 0.001) self.selection_rotation_widgets = self._sliders("Rotation", rotation) self.selection_rotation = np.asarray(rotation, dtype=float) + for widget in ( + *self.selection_position_widgets, + *self.selection_rotation_widgets, + ): + widget.disabled = not rows for axis, widget in enumerate(self.selection_position_widgets): self._watch( widget, @@ -1133,10 +1143,11 @@ def _rebuild_selection(self) -> None: ) self.selection_scale = 1.0 self.selection_scale_widget = self.server.gui.add_number( - "Scale", 1.0, min=1e-6, step=0.01, visible=len(rows) > 1 + "Scale", 1.0, min=1e-6, step=0.01, disabled=len(rows) < 2 ) self._watch(self.selection_scale_widget, self._set_selection_scale) - self._action_button("Delete", self._delete_selection) + delete, _ = self._action_button("Delete", self._delete_selection) + delete.disabled = not rows def _set_selection_position(self, axis: int, value: float) -> None: values = self._selection_values() @@ -1217,7 +1228,7 @@ def _draw_instances( return positions = np.asarray([value[:3, 3] for value in matrices]) rotations = np.asarray([tf.quaternion_from_matrix(value) for value in matrices]) - scale = 0.005 if kind == "segments" else 0.01 + scale = 0.005 if kind == "segments" else 0.0025 mesh = ( trimesh.creation.icosphere(subdivisions=1) if kind == "segments" From d2167641e444ec98e81d1a3cd65ed710c3e98595 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 14:11:33 -0700 Subject: [PATCH 17/36] 1. All spandable fieds I indicated in the UI%Feature design doc, should be collapsed by default. 2. The index of segments, taxels and urdf joints, should start with 0 instead of 1. 3. I asked for error message display in the info section at the button of display. Your implementation is errors display in the general menu at the top, it's against my design. 4. Align "Start Binding to Segments" -> "Select Segments" -> "Complete" the 3 buttons horizantlly. And at each time only at most one button can be clickable. After clicking "Start", the button should grey out; so is after clicking "Select Segments", so at each time the user can only click the 3D scene or one of the buttons. 5. Add a button to Preview -> Static section's top: "Reset Values". This will reset the segments and urdf values to the edit dataset binding segment poses and edit dataset urdf values. 6. Add cli.py to take in metadata path, side and urdf path. All optional Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 8 +- scripts/tactile_metadata/README.md | 33 ++- scripts/tactile_metadata/UI&Feature Design.md | 12 +- .../tactile_metadata_editor/cli.py | 12 +- .../editor_interactions.js | 82 +++++++- .../tactile_metadata_editor/geometry.py | 128 +++++++----- .../tactile_metadata_editor/storage.py | 6 +- .../tactile_metadata_editor/ui.py | 196 +++++++++++------- 8 files changed, 340 insertions(+), 137 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 7c3c52aa28..182fc82560 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -15,6 +15,8 @@ SPDX-License-Identifier: Apache-2.0 - 评审替代性 UI 规格时，以当前稿为产品范围，只把 schema/API 约束视为阻塞；不要把旧功能、状态增强或可接受的瞬态覆盖升级为要求，并逐字核对交互手势。 - 对设计文档的结构与表达要求应直接落实到目标文档；页面布局按区块列出具体控件，不以聊天中的补充说明代替文档修改。 - Keep URDF driving separate from taxel deformation: HandPose supplies skeleton and LBS poses, while live URDF motion uses JointState only. Remove obsolete driver controls, persisted options, algorithms, and acceptance cases together. +- Live HandPose overwrites only valid joints. Invalid joints retain their previous pose, initialized from the binding pose; an invalid WRIST freezes all segment poses because no wrist-relative conversion is available. +- Preview deformation is per taxel: omit invalid taxels and report their row errors without suppressing valid taxels. Preserve original row indices when matching live force values, and block Apply while any preview taxel is invalid. - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. - New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. - Preserve slot order and repeated segment indices. Normalize all four weights only on explicit user weight edits and reject negative edit values; loading, recording received values, checking, saving, and exporting preserve raw weights. Distinguish receiver failures from sender behavior, and keep all editor operations local. @@ -24,15 +26,19 @@ SPDX-License-Identifier: Apache-2.0 - Reuse the launching Python environment and its SDK through sys.executable. Do not create tool-specific project metadata, lockfiles, environments or SDK installations, and do not infer runtime requirements from old local build artifacts. - Missing packages in the agent's inspection environment limit verification; report the unavailable checks. Do not turn verification requirements into proposals to install dependencies, provision environments, or substitute interpreters; maintain Isaac Capture's shared dependency declarations instead. - Use Viser's standard controls and public callbacks, with document changes serialized through State. Apply the same explicit weight-normalization rule to sliders and numeric fields. +- When product placement says top/before/below and any sibling uses explicit Viser `order`, assign explicit order to every constrained sibling; source creation order does not override mixed default and explicit ordering. +- Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. UI sections marked `(spannable)` in the product design start collapsed. - Client-side helpers may hide only their exact bridge input and label; never hide an ancestor container that can contain the entire Viser control panel. - Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. +- Call `/edit`, `/preview`, `/saved`, and `/errors` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. +- During live redraw, update the active scene-node group in place. Hide only groups that should be inactive; never hide and immediately re-show the active group each frame. - Keep metadata/force alignment with the producer and user; do not add source-row mappings, correspondence tracking across reordering, or alignment baselines to the editor. - JSON follows tactile_metadata.fbs and the existing bindings. Treat array and enum conversion for encoding as routine implementation rather than a separate product-format decision. - Persist only schema-defined tactile metadata fields. Keep model configuration, calibration, source mappings and internal IDs in session state; audit actual save/export output rather than inferring persistence requirements from editing needs. - Simplify persistence by deleting unsupported branches in place; do not relocate serialization into Document or introduce new state abstractions for a narrower file format. - For local editor checks using a cached Ruff executable, match the version in the repository hook configuration; older binaries may not understand the shared lint rules. Keep every check and formatter target inside this subtree. - Keep the GUI a thin adapter over shared document, geometry, storage and SDK operations. Prefer explicit native Viser widgets and one reusable property inspector; avoid widget-type inference, duplicate field editors in dialogs, and rebuilding scene nodes on every frame. Moving logic between files alone does not reduce implementation size. -- Use URDF base opacity 0.4 multiplied by the rendered dataset layer opacity, with color-based link highlighting; do not add per-link visibility or opacity controls or state. +- Use URDF base opacity 0.4 multiplied by the rendered dataset group's opacity, with color-based link highlighting; do not add per-link visibility or opacity controls or state. - Under a hard code budget, finish and count a complete executable slice before expanding the UI. Refactoring dispatch or moving functions is not evidence that the implementation fits; verify the formatted total and retain budget for integration fixes. - Treat metadata wrist coordinates, Viser scene coordinates and camera screen orientation as distinct; diagnose an upside-down display from the scene/camera mapping, not URDF alignment. - Separate OpenXR schema/reference-space contracts from provider basis conversion; verify the plugin conversion before claiming injected HandPose follows OpenXR joint axes. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 3d4938c6e1..f077ace1af 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -27,6 +27,17 @@ Open [http://127.0.0.1:8080](http://127.0.0.1:8080). To use another port: python -m tactile_metadata_editor.cli --port 8090 ``` +Metadata, side, and URDF inputs are optional: + +```bash +python -m tactile_metadata_editor.cli \ + --metadata ~/Downloads/right.metadata.json \ + --side right \ + --urdf "$(pwd)/model/right/linkerhand_FFG_right.urdf" +``` + +`--side` defaults to `left`. Paths beginning with `~` are expanded. + Offline editing uses Viser, NumPy, yourdfpy, trimesh, and rtree. Live reception also requires the Isaac Capture SDK, a compatible OpenXR runtime, and an available HMD system in the same Python environment. @@ -36,7 +47,7 @@ available HMD system in the same Python environment. The editor maintains independent left- and right-hand documents. **Toggle Side** selects the active document; each side has its own metadata path and data. General controls are at the top of the right menu. The Selected Object panel is -always present on the left. +always present as a floating window on the left. Three pose datasets are displayed: @@ -63,6 +74,9 @@ The **Edit** mode contains three tabs. 2. Use **URDF Alignment** to translate, rotate, and scale the model. 3. Expand **URDF Joints** to adjust each movable joint. +URDF Joints and the full Segment/Taxel lists are collapsed by default. Their +displayed indices start at 0. + URDF configuration is session-only and is not saved in tactile metadata. Its base opacity is 0.4, multiplied by the opacity of the dataset layer being rendered. @@ -94,6 +108,9 @@ To bind taxels: 2. Select **Select Segments** and choose one to four segments. 3. Select **Complete**. Initial active weights are equal. +The three buttons are horizontal, and at most one is enabled at a time. Between +steps, only the required 3D selection is available. + Expand **All Taxels** to edit pose, four segment slots, and four weights. `None` is an editor-only inactive slot with weight zero. Editing a weight preserves the entered value and adjusts other active weights proportionally so the row sums to @@ -116,6 +133,9 @@ changes Editing data; Preview mode changes Preview segments. Preview taxels are derived and cannot be selected. With no selection, the panel remains visible, shows Type `None` and Count `0`, and disables its input fields. +Action and validation errors appear in the fixed **Info** area at the bottom of +the 3D viewport. It cannot be moved. + ## Save and load Enter **Metadata Path**, then use **Load** or **Save**. Saving validates the @@ -141,7 +161,8 @@ standalone check/export CLI commands. ## Preview and live data -Use **Restore to Editor Binding Values** to initialize Preview from Editing. +Use **Reset Values** to restore Preview segment poses and URDF joint values from +Editing. Choose **Binding Metadata Source**: - **Editing** uses the current editor bindings. @@ -153,6 +174,11 @@ The Preview Static section lists Segments before URDF Joints. bindings, and URDF joint values. The button is disabled when Preview segments are incompatible with the selected bindings. +Each Preview taxel is evaluated independently. Invalid rows are reported and +omitted from the scene while valid taxels continue to render. Live force rows +remain matched by their original taxel indices. Apply remains disabled until all +taxels are valid. + For live reception: 1. Set **Tactile Collection ID**. @@ -167,6 +193,9 @@ For live reception: HandPose updates Preview segments. Enabled JointState mappings update Preview URDF joints. Live frames overwrite manual Static values while connected. +HandPose overwrites only valid joints; invalid joints retain their last pose, +starting from the binding pose. If WRIST is invalid, every segment remains at +its last pose because a wrist-relative transform cannot be computed. Tactile force magnitude maps from blue at 0 N to red at 30 N, with values above 30 N clamped. Missing or inactive force data is light grey. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index 79f9c502b6..e2684259c4 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -8,6 +8,8 @@ UI&Feature Design Global standard: unless improper for specific reason, for all float or int input fields, enable up/down arrow to adjust the value. For float input field, the step should be determined by the estimated range of the value. For int input field, the step should be 1. Only when a field has a clear specific limit range, the input field should reject illegal values. For sliders, when there's a clear specific limit range, use the limit range. * When "Save", check the validation of metadata. If has error, pop up warning window "Metadata contain errors, are you sure to save?" for user to select continue or cancel. +* Every field marked `(spannable)` is collapsed by default. +* Segment, taxel, and displayed URDF-joint indices start at 0. For field names, use title case for multi-word field names. For example, "URDF Path" instead of "urdf path". @@ -46,7 +48,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula 9. Taxels tab: * Taxels don't have their onw alignment, it uses the alignment of the segments. a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), and rotation (3 float sliders). "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields; newly created taxels are selected and previous objects are deselected. - c) A series of buttons "Start Binding to Segments" "Select Segments" "Complete". At first, only "Start Binding to Segments" is clicable. When clicking this button, 3D scene will grey out except for taxels (from Edit dataset) is displayed normally (without highlight), the instruction field under the button will print: "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. Click "Select Segments" after done selecting taxels. Click on any other area in 3D scene to cancel.". When at least one taxel is selected, the button "Select Segments" becomes clickcable. The 3D scene stays greyed out and only segments (from Edit dataset) are displayed normally (without highlight). The instruction field changes to print: "Select 1-4 segment(s) to bind to the selected taxel(s) in 3D scene. Hold Shift to multi-select. Click "Complete" after done selecting segments. Click on any other area in 3D scene to cancel.". When at least one segment is selected, the "Complete" button becomes clickable. Clicking "Complete" button will update the Edit dataset's binding, each original weight is average (if selected 4 segments, then 0.25 each; if selected 3 segments, then 1/3 each.) (also update the following section). If more than 4 segments are selected, don't update the binding, and print in instruction field: "Too many segments selected, please select 1-4 segments. Click "Start Binding to Segments" to start over.". + c) Horizontally align "Start Binding to Segments", "Select Segments", and "Complete". At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The scene instructions and equal initial weights otherwise follow the same three-step binding flow. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. Each taxel has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. * "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. @@ -57,7 +59,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula * Taxels are displayed as light grey flat squares of size 0.0025m*0.0025m. The three axes of each taxel are not displayed by default. Clicking an taxel on the 3D scene while in Edit mode will select the taxel, highlight it, reveal its 3 axes. The 3D scene modification rules are the same as segments. * Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. * When multiple objects are selected, the 3-axis visualizer should show in the center of the selected objects' area, with axis directions align with the visualiation space coordinates. -* An additional small menu on the left side of the view is always displayed for selected object(s)' modification, applying also to single-object selection. With no selection it shows Type: None and Count: 0 and disables every input field: +* An additional floating menu on the left side of the view is always displayed for selected object(s)' modification; it is not docked to the viewport edge. It applies also to single-object selection. With no selection it shows Type: None and Count: 0 and disables every input field: a) (read only) selected elements type and count (multi selection is stated in following part) b) (editable) current position (defined above) c) (editable) current rotation (defined above) @@ -71,15 +73,17 @@ e) Delete d) A button "Apply Preview Data to Editor". When clicked, apply the Preview dataset's segment poses and deferred taxel poses (segment poses + selected metadata binding) and the binding relationships, to the Editing dataset's segment binding poses and taxel binding data. Also update the urdf joint values to Editing dataset. Remind the user that this action will fully override segments and taxels. * When deffering taxel poses or applying binding relationships to the Edit dataset, segments should be referred to by their HandJoint enum name instead of indice. It means received Metadata requires one layer of translation to turn the binding indice into HandJoint. * If the segment list is incompatible to the metadata's taxels binding (e.g. taxels refer to a segment that's not in list), an error should show up at the top of Preview menu including information like the missing segment enum name, and "Apply Preview Data to Editor" should grey out not clickable. +* Preview deformation is independent per taxel. Invalid taxels are omitted and reported by row without preventing valid taxels from rendering. Applying Preview remains disabled until every taxel is valid. Live section: a) Checkbox "Use JointState to Drive URDF". When checked, enable the "JointState collection id" input field, and a Joint Mappings text field to receive a long json string to map the joint names in JointState to the URDF. The current field "Joint mappings JSON: source → [target, scale, offset]" is reusable. b) A button "Connect Live" button. When clicked, update the Preview dataset's segments poses with received hand data in real time, and update the taxels poses based on the segments poses and the binding metadata (selected in above selection field). Manage a field to clearly state the connection statue: Live or Disconnected. After clicking "Connect Live", the button changes text to "Disconnect Live". When disconnected, stops updating the segments poses or URDF joint values. (If JointState is used, Live mode also updates the URDF joint values.) Show warning: Live will override values in Static section. +* HandPose updates only valid joints. An invalid joint retains its previous pose, initialized from its binding pose. If WRIST is invalid, all segment poses remain stale because wrist-relative conversion is unavailable. * If the metadata * While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. Static section: - a) A button "Restore to Editor Binding Values" + a) A button "Reset Values" that resets Preview segments and URDF joint values from the Editing dataset binding poses and URDF values. b) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Exclude the reordering grabber UI. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. c) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. * Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. -Button info display: show error messages at bottom when action fails. +Button info display: show error messages in a fixed, non-movable Info area occupying the bottom of the 3D display when an action fails. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/cli.py b/scripts/tactile_metadata/tactile_metadata_editor/cli.py index b8eceb9b3c..b3d65b7b04 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/cli.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/cli.py @@ -4,18 +4,28 @@ import argparse import sys +from pathlib import Path +from . import storage from .document import State def main() -> int: parser = argparse.ArgumentParser(description="Local tactile metadata editor") parser.add_argument("--port", type=int, default=8080) + parser.add_argument("--metadata", help="Metadata JSON path") + parser.add_argument("--side", choices=("left", "right"), default="left") + parser.add_argument("--urdf", help="URDF path") args = parser.parse_args() try: from .ui import Editor - Editor(State(), args).run() + state = State() + state.side = args.side + if args.metadata: + path = Path(args.metadata).expanduser().resolve() + state.replace(storage.load(path, args.side), str(path)) + Editor(state, args).run() return 0 except (ImportError, OSError, RuntimeError, TypeError, ValueError) as exc: print(f"{type(exc).__name__}: {exc}", file=sys.stderr) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index 475a542523..b561a14fdf 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -9,12 +9,15 @@ let dragged = null; let nonce = 0; let observer = null; + let infoInput = null; + let infoElement = null; + let raf = 0; - const bridge = () => { + const findBridge = (name) => { for (const input of document.querySelectorAll("input")) { let parent = input; for (let depth = 0; parent && depth < 5; ++depth, parent = parent.parentElement) { - if (parent.textContent?.includes("__Tactile Reorder Event")) { + if (parent.textContent?.includes(name)) { input.style.position = "absolute"; input.style.width = "1px"; input.style.height = "1px"; @@ -22,7 +25,7 @@ input.style.pointerEvents = "none"; input.tabIndex = -1; for (const label of parent.querySelectorAll("label")) { - if (label.textContent?.includes("__Tactile Reorder Event")) { + if (label.textContent?.includes(name)) { label.style.display = "none"; } } @@ -33,6 +36,8 @@ return null; }; + const bridge = () => findBridge("__Tactile Reorder Event"); + const emit = (kind, source, target) => { const input = bridge(); if (!input) return; @@ -46,8 +51,74 @@ input.dispatchEvent(new Event("change", { bubbles: true })); }; + const alignBindingButtons = () => { + const names = new Set([ + "Start Binding to Segments", + "Select Segments", + "Complete", + ]); + const buttons = [...document.querySelectorAll("button")].filter((button) => + names.has(button.textContent?.trim()), + ); + if (buttons.length !== 3) return; + let common = buttons[0].parentElement; + while (common && !buttons.every((button) => common.contains(button))) { + common = common.parentElement; + } + if (!common || common.querySelectorAll("button").length !== 3) return; + const children = buttons.map((button) => { + let child = button; + while (child.parentElement !== common) child = child.parentElement; + return child; + }); + if (new Set(children).size !== 3) return; + common.style.display = "grid"; + common.style.gridTemplateColumns = "repeat(3, minmax(0, 1fr))"; + common.style.gap = "6px"; + for (const child of common.children) { + child.style.gridColumn = children.includes(child) ? "auto" : "1 / -1"; + } + for (const button of buttons) button.style.width = "100%"; + }; + + const syncInfo = () => { + const parent = globalThis.__viserMutable?.canvas?.parentElement; + if (!parent) return; + if (!infoElement) { + infoElement = document.createElement("div"); + Object.assign(infoElement.style, { + position: "absolute", + left: "12px", + right: "12px", + bottom: "12px", + minHeight: "64px", + maxHeight: "120px", + overflow: "auto", + zIndex: "20", + padding: "10px 14px", + border: "1px solid rgba(0,0,0,0.25)", + borderRadius: "6px", + background: "rgba(255,255,255,0.9)", + color: "#202020", + boxSizing: "border-box", + }); + } + if (infoElement.parentElement !== parent) parent.appendChild(infoElement); + if (!infoInput?.isConnected) { + infoInput = findBridge("__Tactile Info"); + } + infoElement.innerHTML = `Info
${infoInput?.value || " "}
`; + }; + + const frame = () => { + syncInfo(); + raf = requestAnimationFrame(frame); + }; + const bind = () => { bridge(); + findBridge("__Tactile Info"); + alignBindingButtons(); for (const label of document.querySelectorAll("label")) { if (!label.textContent?.trim().startsWith("⚠")) continue; const field = label.parentElement?.querySelector("input, button"); @@ -90,10 +161,15 @@ observer = new MutationObserver(bind); observer.observe(document.body, { childList: true, subtree: true }); bind(); + frame(); globalThis[key] = { stop() { observer?.disconnect(); observer = null; + cancelAnimationFrame(raf); + infoElement?.remove(); + infoElement = null; + infoInput = null; dragged = null; }, }; diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index 0dbddc127c..2a27e66a5d 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -267,7 +267,9 @@ def remove(self) -> None: self.handles.clear() -def deform(binding: Document, current: list[Segment]) -> list[Taxel]: +def deform( + binding: Document, current: list[Segment] +) -> tuple[list[Taxel | None], list[str]]: bind_segments = { segment.joint: matrix(segment.pose) for segment in binding.segments } @@ -276,61 +278,89 @@ def deform(binding: Document, current: list[Segment]) -> list[Taxel]: current ): raise ValueError("Segment HandJoint values must be unique") - result = [] - for taxel in binding.taxels: - active = [ - (joint, weight) - for joint, weight in zip(taxel.segments, taxel.weights) - if joint is not None and weight > 0 - ] - if not active: - raise ValueError("Taxel has no active segment") - transforms = [] - for joint, weight in active: - if joint not in bind_segments or joint not in current_segments: - raise ValueError(f"Missing segment {joint}") - transforms.append( - (weight, current_segments[joint] @ np.linalg.inv(bind_segments[joint])) + invalid_taxels: dict[int, list[str]] = {} + for issue in binding.validate(): + if issue["kind"] == "taxels" and issue["row"] is not None: + invalid_taxels.setdefault(issue["row"], []).append(issue["message"]) + result: list[Taxel | None] = [] + errors = [] + for row, taxel in enumerate(binding.taxels): + if row in invalid_taxels: + result.append(None) + errors.append( + f"Taxel {row}: " + "; ".join(dict.fromkeys(invalid_taxels[row])) ) - bind_taxel = matrix(taxel.pose) - position = sum( - weight * (delta @ np.r_[bind_taxel[:3, 3], 1.0])[:3] - for weight, delta in transforms - ) - if len({joint for joint, _ in active}) == 1: - rotation = transforms[0][1][:3, :3] @ bind_taxel[:3, :3] - else: - blend = sum(weight * delta[:3, :3] for weight, delta in transforms) - u, singular, vh = np.linalg.svd(blend) - if singular[-1] < 1e-8: - rotation = bind_taxel[:3, :3] + continue + try: + active = [ + (joint, weight) + for joint, weight in zip(taxel.segments, taxel.weights) + if joint is not None and weight > 0 + ] + if not active: + raise ValueError("no active segment") + transforms = [] + for joint, weight in active: + if joint not in bind_segments or joint not in current_segments: + raise ValueError(f"missing segment {joint}") + transforms.append( + ( + weight, + current_segments[joint] @ np.linalg.inv(bind_segments[joint]), + ) + ) + bind_taxel = matrix(taxel.pose) + position = sum( + weight * (delta @ np.r_[bind_taxel[:3, 3], 1.0])[:3] + for weight, delta in transforms + ) + if len({joint for joint, _ in active}) == 1: + rotation = transforms[0][1][:3, :3] @ bind_taxel[:3, :3] else: - correction = np.eye(3) - correction[2, 2] = np.linalg.det(u @ vh) - rotation = u @ correction @ vh @ bind_taxel[:3, :3] - value = np.eye(4) - value[:3, :3], value[:3, 3] = rotation, position - result.append(Taxel(pose(value), list(taxel.segments), list(taxel.weights))) - return result - - -def hand_segments(joints: list[str], hand: dict) -> list[Segment]: + blend = sum(weight * delta[:3, :3] for weight, delta in transforms) + u, singular, vh = np.linalg.svd(blend) + if singular[-1] < 1e-8: + rotation = bind_taxel[:3, :3] + else: + correction = np.eye(3) + correction[2, 2] = np.linalg.det(u @ vh) + rotation = u @ correction @ vh @ bind_taxel[:3, :3] + value = np.eye(4) + value[:3, :3], value[:3, 3] = rotation, position + result.append(Taxel(pose(value), list(taxel.segments), list(taxel.weights))) + except ( + ValueError, + TypeError, + KeyError, + IndexError, + np.linalg.LinAlgError, + ) as exc: + result.append(None) + errors.append(f"Taxel {row}: {exc}") + return result, errors + + +def hand_segments( + binding: list[Segment], hand: dict, stale: list[Segment] +) -> list[Segment]: values = [] for position, orientation, valid in zip( hand["positions"], hand["orientations"], hand["is_valid"] ): values.append(matrix([*position, *orientation]) if valid else None) - if len(values) != len(JOINTS) or values[1] is None: - raise ValueError("HandTracker WRIST pose is unavailable") - wrist_inverse = np.linalg.inv(values[1]) + if len(values) != len(JOINTS): + raise ValueError("HandTracker joint array has an invalid length") + stale_by_joint = {segment.joint: segment for segment in stale} + wrist_inverse = np.linalg.inv(values[1]) if values[1] is not None else None result = [] - for joint in joints: - index = JOINTS.index(joint) - if values[index] is None: - raise ValueError(f"HandTracker {joint} pose is unavailable") - result.append( - Segment( - joint, pose(VISUALIZATION_FROM_HAND @ wrist_inverse @ values[index]) + for bind_segment in binding: + joint = bind_segment.joint + tracked = values[JOINTS.index(joint)] + if wrist_inverse is None or tracked is None: + fallback = stale_by_joint.get(joint, bind_segment) + result.append(Segment(joint, list(fallback.pose))) + else: + result.append( + Segment(joint, pose(VISUALIZATION_FROM_HAND @ wrist_inverse @ tracked)) ) - ) return result diff --git a/scripts/tactile_metadata/tactile_metadata_editor/storage.py b/scripts/tactile_metadata/tactile_metadata_editor/storage.py index 5c234ae720..975407e3ec 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/storage.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/storage.py @@ -139,12 +139,12 @@ def load( side: str, alignment: np.ndarray | list[list[float]] | None = None, ) -> Document: - with Path(path).open() as stream: + with Path(path).expanduser().resolve().open() as stream: return parse(json.load(stream), side, alignment) def write(path: str | Path, value: object) -> None: - path = Path(path).resolve() + path = Path(path).expanduser().resolve() path.parent.mkdir(parents=True, exist_ok=True) fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=".tactile-") try: @@ -158,6 +158,6 @@ def write(path: str | Path, value: object) -> None: def save(state: State, path: str | Path) -> None: write(path, metadata(state.document)) - state.paths[state.side] = str(Path(path).resolve()) + state.paths[state.side] = str(Path(path).expanduser().resolve()) state.saved[state.side] = state.document.clone() state.saved[state.side].q = {} diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 92bbbb515a..7b61b4f15c 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -73,6 +73,9 @@ def __init__(self, state: State, args): self._build_gui() self._register_scene_events() self._sync_side() + if args.urdf: + self.urdf_path.value = str(Path(args.urdf).expanduser().resolve()) + self._load_urdf() @property def document(self) -> Document: @@ -94,8 +97,8 @@ def _build_gui(self) -> None: self.side_text = gui.add_markdown("") self._action_button("Toggle Side", self._toggle_side) self.metadata_path = gui.add_text("Metadata Path", "") - _, self.new_info = self._action_button("New", self._new) - _, self.save_info = self._action_button("Save", self._save) + self._action_button("New", self._new) + self._action_button("Save", self._save) self._action_button("Load", self._load) self._action_button("Reset Camera", self._reset_camera) self.mode = gui.add_dropdown("Mode", ("Edit", "Preview")) @@ -121,7 +124,9 @@ def _build_gui(self) -> None: self._watch( self.urdf_scale_widget, lambda _: self._update_urdf_alignment() ) - self.edit_joint_folder = gui.add_folder("URDF Joints") + self.edit_joint_folder = gui.add_folder( + "URDF Joints", expand_by_default=False + ) self.segments_tab = self.edit_tabs.add_tab("Segments") with self.segments_tab: @@ -138,10 +143,12 @@ def _build_gui(self) -> None: "Position", (0.0, 0.0, 0.0), 0.001 ) self._action_button("Add with Input", self._add_segment_input) - self.segment_click_button, _ = self._action_button( + self.segment_click_button = self._action_button( "Add with URDF Click", self._pick_segment ) - self.segment_row_folder = gui.add_folder("All Segments") + self.segment_row_folder = gui.add_folder( + "All Segments", expand_by_default=False + ) self.taxels_tab = self.edit_tabs.add_tab("Taxels") with self.taxels_tab: @@ -158,7 +165,7 @@ def _build_gui(self) -> None: "Center Position", (0.0, 0.0, 0.0), 0.001 ) self.grid_rotation = self._sliders("Rotation", (0.0, 0.0, 0.0)) - self.taxel_capture_button, _ = self._action_button( + self.taxel_capture_button = self._action_button( "Capture URDF Surface Pose", self._capture_taxel_pose ) self._action_button("Add Taxels", self._add_taxels_input) @@ -170,17 +177,18 @@ def _build_gui(self) -> None: self._button(self.binding_start, self._start_binding) self._button(self.binding_next, self._select_binding_segments) self._button(self.binding_complete, self._complete_binding) - self.taxel_row_folder = gui.add_folder("All Taxels") + self.taxel_row_folder = gui.add_folder( + "All Taxels", expand_by_default=False + ) self.preview_panel = gui.add_folder("Preview", visible=False) with self.preview_panel: - self.preview_error = gui.add_markdown("") self.binding_source = gui.add_dropdown( "Binding Metadata Source", ("Editing",) ) self.show_binding = gui.add_checkbox("Show Binding Poses", False) self.collection = gui.add_text("Tactile Collection ID", "tactile") - self.apply_preview_button, _ = self._action_button( + self.apply_preview_button = self._action_button( "Apply Preview Data to Editor", self._apply_preview ) gui.add_markdown( @@ -189,6 +197,7 @@ def _build_gui(self) -> None: self._watch(self.binding_source, lambda _: self._binding_source_changed()) self._watch(self.show_binding, lambda _: self._request_draw()) with gui.add_folder("Live"): + self.live_status = gui.add_markdown("Current Statue: Disconnected") self.joint_enabled = gui.add_checkbox( "Use JointState to Drive URDF", False ) @@ -199,27 +208,31 @@ def _build_gui(self) -> None: "Joint Mappings JSON", "{}", multiline=True, disabled=True ) self._watch(self.joint_enabled, self._joint_enabled) - self.connect_button, _ = self._action_button( - "Connect Live", self._connect - ) + self.connect_button = self._action_button("Connect Live", self._connect) gui.add_markdown("Live input overrides values in the Static section.") - self.live_status = gui.add_markdown("Disconnected") self.static_folder = gui.add_folder("Static") with self.static_folder: - gui.add_markdown("The following values affect Preview only.") self._action_button( - "Restore to Editor Binding Values", + "Reset Values", self._restore_preview, + + ) + gui.add_markdown("The following values affect Preview only.") + self.preview_segment_folder = gui.add_folder( + "Segments", expand_by_default=False ) - self.preview_segment_folder = gui.add_folder("Segments", order=1) - self.preview_joint_folder = gui.add_folder("URDF Joints", order=2) + self.preview_joint_folder = gui.add_folder( + "URDF Joints", expand_by_default=False + ) + + self.info = gui.add_text("__Tactile Info", "") + self.preview_error = self.info self.selection_panel = gui.add_panel() self.selection_tab = self.selection_panel.add_tab("Selected Object") with self.selection_tab: self.selection_folder = gui.add_folder("Selection") - self.selection_panel.dock_left() - self.selection_panel.set_width(320) + self.selection_panel.float(x=10, y=10, width=320, height=520) def _numbers(self, prefix: str, values, step: float): return tuple( @@ -249,29 +262,22 @@ async def clicked(_): widget.on_click(clicked) - def _action_button(self, label: str, callback): - button = self.server.gui.add_button(label) - info = self.server.gui.add_markdown("") + def _action_button(self, label: str, callback, *, order=None): + button = self.server.gui.add_button(label, order=order) async def clicked(_): - self.events.put(lambda: self._run_action(callback, info=info)) + self.events.put(lambda: self._run_action(callback)) button.on_click(clicked) - return button, info + return button - def _run_action(self, callback, value=None, info=None) -> None: - if info is not None: - info.content = "" + def _run_action(self, callback, value=None) -> None: + self.info.value = "" try: callback() if value is None else callback(value) except Exception as exc: LOG.exception("Editor action failed") - if info is not None: - info.content = f'Error: {exc}' - else: - self.preview_error.content = ( - f'Error: {exc}' - ) + self.info.value = f'Error: {exc}' self.redraw = True def _request_draw(self) -> None: @@ -332,7 +338,7 @@ def _new(self) -> None: self.state.paths.pop(self.state.side, None) self.preview[self.state.side] = self._preview_from(document) self._sync_side() - self.new_info.content = f"New {self.state.side} metadata created." + self.info.value = f"New {self.state.side} metadata created." def _load(self) -> None: path = self.metadata_path.value.strip() @@ -362,8 +368,7 @@ def _save(self) -> None: async def save(_): self.events.put( lambda: self._run_action( - lambda: (self._write_save(path), modal.close()), - info=self.save_info, + lambda: (self._write_save(path), modal.close()) ) ) @@ -438,12 +443,14 @@ def _rebuild_joint_controls(self) -> None: self.preview_joint_folder.remove() self.edit_joint_widgets, self.preview_joint_widgets = {}, {} with self.urdf_tab: - self.edit_joint_folder = self.server.gui.add_folder("URDF Joints") + self.edit_joint_folder = self.server.gui.add_folder( + "URDF Joints", expand_by_default=False + ) with self.edit_joint_folder: self._add_joint_sliders(self.document, self.edit_joint_widgets) with self.static_folder: self.preview_joint_folder = self.server.gui.add_folder( - "URDF Joints", order=2 + "URDF Joints", expand_by_default=False ) with self.preview_joint_folder: self._add_joint_sliders( @@ -454,7 +461,7 @@ def _add_joint_sliders(self, document: Document, output: dict) -> None: if self.model is None: self.server.gui.add_markdown("Load a URDF to edit joints.") return - for joint in self.model.joints: + for index, joint in enumerate(self.model.joints): lower = ( joint.limit.lower if joint.limit and joint.limit.lower is not None @@ -467,7 +474,11 @@ def _add_joint_sliders(self, document: Document, output: dict) -> None: ) value = min(upper, max(lower, document.q.get(joint.name, 0.0))) widget = self.server.gui.add_slider( - joint.name, float(lower), float(upper), 0.001, float(value) + f"{index}: {joint.name}", + float(lower), + float(upper), + 0.001, + float(value), ) self._watch( widget, @@ -532,7 +543,9 @@ def _rebuild_segment_list(self) -> None: self.segment_widgets = [] issues = self.document.validate() with self.segments_tab: - self.segment_row_folder = self.server.gui.add_folder("All Segments") + self.segment_row_folder = self.server.gui.add_folder( + "All Segments", expand_by_default=False + ) with self.segment_row_folder: for row, segment in enumerate(self.document.segments): row_issues = [ @@ -592,7 +605,9 @@ def _rebuild_taxel_list(self) -> None: issues = self.document.validate() options = ("None", *[segment.joint for segment in self.document.segments]) with self.taxels_tab: - self.taxel_row_folder = self.server.gui.add_folder("All Taxels") + self.taxel_row_folder = self.server.gui.add_folder( + "All Taxels", expand_by_default=False + ) with self.taxel_row_folder: for row, taxel in enumerate(self.document.taxels): row_issues = [ @@ -729,8 +744,7 @@ def _start_binding(self) -> None: self.binding_phase = "taxels" self.binding_taxels, self.binding_segments = [], [] self.selected_kind, self.selected_rows = None, [] - self.binding_next.disabled = True - self.binding_complete.disabled = True + self._set_binding_button(None) self.binding_info.content = ( "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. " 'Click "Select Segments" after selecting taxels. Click another area to cancel.' @@ -740,6 +754,7 @@ def _select_binding_segments(self) -> None: if not self.binding_taxels: raise ValueError("Select at least one taxel") self.binding_phase = "segments" + self._set_binding_button(None) self.binding_info.content = ( 'Select 1-4 segment(s). Hold Shift to multi-select. Click "Complete" ' "after selecting segments. Click another area to cancel." @@ -747,21 +762,27 @@ def _select_binding_segments(self) -> None: def _complete_binding(self) -> None: if not 1 <= len(self.binding_segments) <= 4: - self.binding_info.content = ( - 'Too many segments selected. Select 1-4 segments and click "Start ' - 'Binding to Segments" to start over.' + self._cancel_binding() + self.info.value = ( + 'Too many segments selected. ' + 'Select 1-4 segments and click "Start Binding to Segments" ' + "to start over." ) return self.document.assign(self.binding_taxels, self.binding_segments) self._cancel_binding() self._rebuild_taxel_list() + def _set_binding_button(self, active: str | None) -> None: + self.binding_start.disabled = active != "start" + self.binding_next.disabled = active != "next" + self.binding_complete.disabled = active != "complete" + def _cancel_binding(self) -> None: self.binding_phase = None self.binding_taxels, self.binding_segments = [], [] if hasattr(self, "binding_next"): - self.binding_next.disabled = True - self.binding_complete.disabled = True + self._set_binding_button("start") self.binding_info.content = "" def _joint_enabled(self, enabled: bool) -> None: @@ -781,14 +802,14 @@ def _connect(self) -> None: } self.receiver = sdk.Receiver(config) self.connect_button.label = "Disconnect Live" - self.live_status.content = "Connecting" + self.live_status.content = "Current Status: Connecting" def _disconnect(self) -> None: if self.receiver is not None: self.receiver.close() self.receiver = None self.connect_button.label = "Connect Live" - self.live_status.content = "Disconnected" + self.live_status.content = "Current Status: Disconnected" self.redraw = True def _binding_source_changed(self) -> None: @@ -802,27 +823,35 @@ def _binding_document(self) -> Document: return self.live_document return self.document - def _preview_taxels(self) -> list[Taxel]: + def _preview_taxels(self) -> tuple[list[Taxel | None], list[str]]: return geo.deform(self._binding_document(), self.preview_document.segments) def _update_preview_error(self) -> None: try: - self._preview_taxels() + _, errors = self._preview_taxels() except (ValueError, TypeError, KeyError, IndexError) as exc: - self.preview_error.content = ( + self.preview_error.value = ( f'Binding error: {exc}' ) self.apply_preview_button.disabled = True else: - self.preview_error.content = "" - self.apply_preview_button.disabled = False + self.preview_error.value = ( + '' + "
".join(errors) + "" + if errors + else "" + ) + self.apply_preview_button.disabled = bool(errors) def _apply_preview(self) -> None: - taxels = self._preview_taxels() + taxels, errors = self._preview_taxels() + if errors: + raise ValueError("Repair invalid preview taxels before applying") replacement = Document(self.state.side) replacement.alignment = copy.deepcopy(self.document.alignment) replacement.segments = copy.deepcopy(self.preview_document.segments) - replacement.taxels = copy.deepcopy(taxels) + replacement.taxels = copy.deepcopy( + [taxel for taxel in taxels if taxel is not None] + ) replacement.q = copy.deepcopy(self.preview_document.q) self.state.documents[self.state.side] = replacement self._rebuild_edit_lists() @@ -840,7 +869,7 @@ def _rebuild_preview_segments(self) -> None: self.preview_segment_widgets = [] with self.static_folder: self.preview_segment_folder = self.server.gui.add_folder( - "Segments", order=1 + "Segments", expand_by_default=False ) with self.preview_segment_folder: for row, segment in enumerate(self.preview_document.segments): @@ -882,7 +911,7 @@ def _set_preview_pose( def _update_live(self) -> None: if self.receiver is None: return - self.live_status.content = self.receiver.status + self.live_status.content = "Current Status: " + self.receiver.status if not self.receiver.active: return frame = self.receiver.latest @@ -900,7 +929,9 @@ def _update_live(self) -> None: hand = frame.get("hand") if hand is not None: joints = [segment.joint for segment in binding.segments] - updated = geo.hand_segments(joints, hand) + updated = geo.hand_segments( + binding.segments, hand, self.preview_document.segments + ) signature = [ segment.joint for segment in self.preview_document.segments ] @@ -923,7 +954,7 @@ def _update_live(self) -> None: IndexError, json.JSONDecodeError, ) as exc: - self.preview_error.content = ( + self.preview_error.value = ( f'Live error: {exc}' ) self.apply_preview_button.disabled = True @@ -947,10 +978,14 @@ def _register_scene_events(self) -> None: self.server.scene.on_rect_select(modifier="shift")(self._scene_box) async def _scene_click(self, event) -> None: - self.events.put(lambda: self._handle_scene_event(event, False)) + self.events.put( + lambda: self._run_action(lambda: self._handle_scene_event(event, False)) + ) async def _scene_box(self, event) -> None: - self.events.put(lambda: self._handle_scene_event(event, True)) + self.events.put( + lambda: self._run_action(lambda: self._handle_scene_event(event, True)) + ) def _display_values(self): if self.mode.value == "Preview": @@ -1001,14 +1036,14 @@ def _handle_scene_event(self, event, box: bool) -> None: if event.modifier == "shift" else rows ) - self.binding_next.disabled = not self.binding_taxels + self._set_binding_button("next" if self.binding_taxels else None) elif self.binding_phase == "segments": self.binding_segments = ( sorted(set(self.binding_segments) | set(rows)) if event.modifier == "shift" else rows ) - self.binding_complete.disabled = not self.binding_segments + self._set_binding_button("complete" if self.binding_segments else None) elif event.modifier == "shift": if self.selected_kind not in (None, kind): return @@ -1146,7 +1181,7 @@ def _rebuild_selection(self) -> None: "Scale", 1.0, min=1e-6, step=0.01, disabled=len(rows) < 2 ) self._watch(self.selection_scale_widget, self._set_selection_scale) - delete, _ = self._action_button("Delete", self._delete_selection) + delete = self._action_button("Delete", self._delete_selection) delete.disabled = not rows def _set_selection_position(self, axis: int, value: float) -> None: @@ -1395,11 +1430,8 @@ def _draw(self) -> None: highlighted=self.pick_mode is not None, opacity=geo.URDF_OPACITY * active_opacity, ) - self._hide_prefix("/edit") - self._hide_prefix("/preview") - self._hide_prefix("/saved") - self._hide_prefix("/errors") if self.mode.value == "Edit": + self._hide_prefix("/preview") self._draw_dataset( "/edit", self.document.segments, @@ -1409,11 +1441,18 @@ def _draw(self) -> None: ) self._draw_errors() else: + self._hide_prefix("/errors") try: - preview_taxels = self._preview_taxels() + preview_rows, _ = self._preview_taxels() except (ValueError, TypeError, KeyError, IndexError): - preview_taxels = [] - forces = ( + preview_rows = [] + valid_rows = [ + (row, taxel) + for row, taxel in enumerate(preview_rows) + if taxel is not None + ] + preview_taxels = [taxel for _, taxel in valid_rows] + raw_forces = ( self.receiver.latest.get("forces") if self.receiver and self.receiver.active @@ -1422,6 +1461,11 @@ def _draw(self) -> None: < 0.5 else None ) + forces = ( + [raw_forces[row] for row, _ in valid_rows] + if raw_forces is not None and len(raw_forces) == len(preview_rows) + else None + ) self._draw_dataset( "/preview", self.preview_document.segments, @@ -1438,9 +1482,13 @@ def _draw(self) -> None: 0.5, False, ) + else: + self._hide_prefix("/edit") saved = self.state.saved[self.state.side] if self.show_saved.value and saved is not None: self._draw_dataset("/saved", saved.segments, saved.taxels, 0.2, False) + else: + self._hide_prefix("/saved") self._draw_selection_axes() def run(self) -> None: From ad6390024cac232e09bc107e9a40d7f546c217a7 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 14:15:41 -0700 Subject: [PATCH 18/36] Remove reorder function and grabber Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 2 +- scripts/tactile_metadata/DESIGN.md | 6 +-- .../tactile_metadata/IMPLEMENTATION_PROMPT.md | 2 +- scripts/tactile_metadata/README.md | 8 +--- scripts/tactile_metadata/UI&Feature Design.md | 6 +-- .../tactile_metadata_editor/document.py | 7 --- .../editor_interactions.js | 48 ------------------- .../tactile_metadata_editor/ui.py | 21 -------- 8 files changed, 10 insertions(+), 90 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 182fc82560..bc38e86cd9 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -32,7 +32,7 @@ SPDX-License-Identifier: Apache-2.0 - Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. - Call `/edit`, `/preview`, `/saved`, and `/errors` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. - During live redraw, update the active scene-node group in place. Hide only groups that should be inactive; never hide and immediately re-show the active group each frame. -- Keep metadata/force alignment with the producer and user; do not add source-row mappings, correspondence tracking across reordering, or alignment baselines to the editor. +- Keep metadata/force alignment with the producer and user; do not add source-row mappings, alignment baselines, force permutations, or list-order controls to the editor. - JSON follows tactile_metadata.fbs and the existing bindings. Treat array and enum conversion for encoding as routine implementation rather than a separate product-format decision. - Persist only schema-defined tactile metadata fields. Keep model configuration, calibration, source mappings and internal IDs in session state; audit actual save/export output rather than inferring persistence requirements from editing needs. - Simplify persistence by deleting unsupported branches in place; do not relocate serialization into Document or introduce new state abstractions for a narrower file format. diff --git a/scripts/tactile_metadata/DESIGN.md b/scripts/tactile_metadata/DESIGN.md index 3f8fe0642f..2b912dc933 100644 --- a/scripts/tactile_metadata/DESIGN.md +++ b/scripts/tactile_metadata/DESIGN.md @@ -81,7 +81,7 @@ Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_ 新建 segment 时选择语义和链接，在模型上定位关节中心及参考方向，通过控制器调整位置与扭转。已有 metadata 的 B_j 已给出坐标：选择目标链接后计算固定局部修正，保持原 B_j，而不是把数值吸附到 URDF 原点。 -用户可以调整有序 segment 列表。显式重排通过行号置换更新已有合法 skinning 索引，保持其稳定实体引用；语义枚举值不参与行号运算。允许删除仍被引用的 segment；未解决的引用留在草稿中，绑定页和 Info 区显示原因，不隐式重绑。 +允许删除仍被引用的 segment；未解决的引用留在草稿中，绑定页和 Info 区显示原因，不隐式重绑。 设置新 q0 或修改腕部参考系时，先预览模型和附件变化；现有 B_j、taxel 的绑定根位姿和权重保持数值不变，确认后重算关节局部附件并使检查过期。实际 B_j 的修改通过关节参考系控制器完成。工具不隐式重投影、蒙皮重绑定或修改生产端 metadata。偏离模型的点保留并提示用户调整。 @@ -99,7 +99,7 @@ Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_ 预览显示位置、数量和输出索引，错误标红。应用时原子创建本批独立 taxel，随后丢弃范围、行列和扫描参数。没有基于隐藏区域对象的“重新生成”；需要调整时多选并移动、重新投影或删除后重新批量添加。 -默认从尚未使用的连续索引区间分配新点。移动、重投影和删除不重排已有输出；“重排所选 taxel”显示旧→新索引对应表后应用，检查要求最终覆盖 0..N-1。批量复制到另一侧只复用尚在界面的生成参数，必须重新拾取表面、绑定关节和分配索引，不镜像几何。 +当前 taxel 数组行序就是输出顺序，不维护另一套输出索引。批量复制到另一侧只复用尚在界面的生成参数，必须重新拾取表面、绑定关节和分配索引，不镜像几何。 文件中的输出顺序由 taxel 数组行序表达；内部标识和临时编号不写入文件，重新打开后按文件行序建立编号。 @@ -150,7 +150,7 @@ Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_ 实时接收和三维播放不自动替换草稿。每次显式记录仅以接收值替换所选分组，不合并旧值；替换后即使产生引用或数组长度错误，也保留结果并在对应区域显示原因。 -metadata 与力值的实际对应关系由生产端和使用者负责。工具按当前 metadata 与力数组的协议行序显示，执行协议结构、数量及有效性检查；不维护独立的源行映射、重排前后对应关系或对齐基线，不为 metadata 编辑或重排重新排列力值。切换三维 metadata 来源不建立或修复它与力流的对应关系。 +metadata 与力值的实际对应关系由生产端和使用者负责。工具按当前 metadata 与力数组的协议行序显示，执行协议结构、数量及有效性检查；不维护独立的源行映射或对齐基线，不为 metadata 编辑重新排列力值。切换三维 metadata 来源不建立或修复它与力流的对应关系。 ### 1.9 导出、检查与状态跳转 diff --git a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md index 35be7c1d48..1fe9a0c442 100644 --- a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md +++ b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md @@ -22,7 +22,7 @@ When implementation, make sure to only modify folder /Users/lotusl/Codes/IsaacTe The following product decisions are settled and must be reflected throughout the implementation: 1. Weight editing: after a slider or numeric-field edit, replace that slot and normalize all four weights by their sum, including the edited slot. Reject negative weights. Reject an edit whose resulting row contains non-finite or negative values or has a zero sum, preserving the previous row. Apply normalization independently to each selected taxel. Preserve slot order and repeated indices. Loading, explicitly recording received metadata, checking, saving, and exporting preserve raw weights and report invalid values without silently normalizing them. 2. Metadata persistence: saving, backups, and export write only tactile metadata values using the fields, nested structure, and union type discriminators defined by src/core/schema/fbs/tactile_metadata.fbs. The only top-level JSON fields are geometry_mapping_type, geometry_mapping, segment_source_type, and segment_source. Use one schema JSON representation for saving, reopening, and JSON export; optional SDK binaries encode the same values. Do not add an editor extension, tool-specific fields, project/configuration files, .editor.json, internal-ID correspondence tables, or saved check reports. Read with json.load(), perform the array and enum conversion required by the existing SDK constructors, and encode through LBS, HandSkeletonRef, TactileMetadata, and to_bytes() in the isolated SDK subprocess. Do not invent another metadata format or treat this conversion as an unresolved product decision. -3. Scope: metadata/force alignment is the producer's and user's responsibility. Display the current arrays in protocol row order and retain structural, count, and validity checks. Do not maintain source-row mappings, correspondence across reordering, alignment baselines, or force permutations to compensate for metadata edits. Switching visualization sources does not establish or repair force alignment. Persist taxel output order through array row order, without saving internal IDs or temporary numbering. +3. Scope: metadata/force alignment is the producer's and user's responsibility. Display the current arrays in protocol row order and retain structural, count, and validity checks. Do not maintain source-row mappings, alignment baselines, or force permutations to compensate for metadata edits. Switching visualization sources does not establish or repair force alignment. Persist taxel output order through array row order, without saving internal IDs or temporary numbering. 4. URDF display: all links use fixed opacity 0.6. Do not add per-link visibility or opacity controls or state. Highlight associated links through color without changing opacity; retain mode-level model visibility. 5. Session state: URDF/package paths, q0, wrist calibration, segment/link associations, hand assignments, live-source and joint mappings, internal IDs, and display settings exist only in the current session. Reopening restores only metadata values and array order; the user assigns the hand through the GUI or --sides and sets up the URDF associations and calibration again when needed. Unsaved state tracks metadata values only. Incomplete saves use the same schema fields; show draft status, validation results, and revisions in the GUI or CLI output, never in metadata or companion files. Validate formal exports separately from serialization. Do not implement separate editable/numerical formats, session restoration, or a --report file output. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index f077ace1af..91042f7a60 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -88,12 +88,10 @@ rendered. Click** and click the model surface. - Segments are represented by yellow spheres with radius 0.005 m. - Expand **All Segments** to edit HandJoint, position, and XYZ rotation. -- Drag the handle above a segment to reorder it. Displayed IDs follow list order. HandJoint values must be unique. Taxel bindings use HandJoint names internally, -so segment reordering does not change their semantic references. Removing or -renaming a referenced segment leaves the affected taxel field invalid and -outlined in red. +so removing or renaming a referenced segment leaves the affected taxel field +invalid and outlined in red. ### Taxels @@ -116,8 +114,6 @@ is an editor-only inactive slot with weight zero. Editing a weight preserves the entered value and adjusts other active weights proportionally so the row sums to one. If only one slot is active, its weight is one. -Dragging taxel rows changes their IDs and metadata array order. - ## Scene selection - Click an object to select it. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index e2684259c4..90c5f14ed2 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -40,7 +40,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) Segments Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation. b) Add new segments: input fields of segment name (HandJoint enum, dropdown selection), position (3 float input fields). An "Add with Input" button. c) Add new segments by 3D scene interaction: a button to "Add with URDF Click" button. After clicking this button, the button shows a "pressed" shape, the 3D scene will grey out, with only the URDF (of the Editing dataset) highlighted. The user can click on the URDF to add a new segment on the URDF surface position. Clicking on any other area will cancel the operation. - d) (spannable) A full list of all segments, with an id number (read-only, starting at 0, it's the index of the segment in the list), their name (dropdown list of HandJoint, display in enum name), editable position (3 float input), editable rotation (3 float slider). This is to modify the binding poses of the segments, display this info above this section. Each segment has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. + d) (spannable) A full list of all segments, with an id number (read-only, starting at 0, it's the index of the segment in the list), their name (dropdown list of HandJoint, display in enum name), editable position (3 float input), editable rotation (3 float slider). This is to modify the binding poses of the segments, display this info above this section. * It's invalid to have more than one segments to select the same HandJoint. At error, outline the repeated fields in red. * Some fields in Taxels tab refer to segments in this Segments tab, and modifying segments may result in invalid taxel binding that's revealed in Taxels tab. * Segments are displayed as yellow spheres of radius 0.005m. The three axes of each segment are not displayed by default.Clicking an segment on the 3D scene while in Edit mode will select the segment, highlight it, reveal its 3 axes. When a segment is newly added, it's selected, and deselect the previous selected object(s). Clicking on any other area in the 3D scene will deselect the segment; dragging 3D scene or clicking on the menu/other UI area will not deselect the segment. Clicking on the same selected segment will not do anything. @@ -50,7 +50,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), and rotation (3 float sliders). "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields; newly created taxels are selected and previous objects are deselected. c) Horizontally align "Start Binding to Segments", "Select Segments", and "Complete". At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The scene instructions and equal initial weights otherwise follow the same three-step binding flow. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. - d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. Each taxel has a grabber on the side that can be used to drag them and reorder, this will change the id numbers. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. + d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. * "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. * Taxels data can be invalid for multiple reasons. When invalid, line the invalid field in red. Reasons including: a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. @@ -82,7 +82,7 @@ Live section: * While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. Static section: a) A button "Reset Values" that resets Preview segments and URDF joint values from the Editing dataset binding poses and URDF values. - b) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Exclude the reordering grabber UI. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. + b) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. c) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. * Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index 5421955fc2..fb93f5961b 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -109,13 +109,6 @@ def delete(self, kind: str, rows: list[int]) -> None: if 0 <= row < len(values): values.pop(row) - def move(self, kind: str, source: int, target: int) -> None: - values = self.segments if kind == "segments" else self.taxels - if not 0 <= source < len(values) or not 0 <= target < len(values): - raise ValueError("Reorder row is outside the list") - value = values.pop(source) - values.insert(target, value) - def assign(self, taxels: list[int], segment_rows: list[int]) -> None: if not 1 <= len(segment_rows) <= 4: raise ValueError("Select one to four segments") diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index b561a14fdf..5c865e048e 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -6,8 +6,6 @@ const key = "__tactileMetadataEditorInteractions"; globalThis[key]?.stop(); - let dragged = null; - let nonce = 0; let observer = null; let infoInput = null; let infoElement = null; @@ -36,21 +34,6 @@ return null; }; - const bridge = () => findBridge("__Tactile Reorder Event"); - - const emit = (kind, source, target) => { - const input = bridge(); - if (!input) return; - const value = `${kind}:${source}:${target}:${++nonce}`; - const setter = Object.getOwnPropertyDescriptor( - HTMLInputElement.prototype, - "value", - )?.set; - setter?.call(input, value); - input.dispatchEvent(new Event("input", { bubbles: true })); - input.dispatchEvent(new Event("change", { bubbles: true })); - }; - const alignBindingButtons = () => { const names = new Set([ "Start Binding to Segments", @@ -116,7 +99,6 @@ }; const bind = () => { - bridge(); findBridge("__Tactile Info"); alignBindingButtons(); for (const label of document.querySelectorAll("label")) { @@ -127,35 +109,6 @@ field.style.outlineOffset = "1px"; } } - for (const handle of document.querySelectorAll(".tm-sort-handle")) { - if (handle.dataset.bound === "true") continue; - handle.dataset.bound = "true"; - handle.style.cursor = "grab"; - handle.style.userSelect = "none"; - handle.addEventListener("dragstart", (event) => { - dragged = { - kind: handle.dataset.kind, - row: Number(handle.dataset.row), - }; - event.dataTransfer.effectAllowed = "move"; - }); - handle.addEventListener("dragover", (event) => { - if (dragged?.kind === handle.dataset.kind) { - event.preventDefault(); - event.dataTransfer.dropEffect = "move"; - } - }); - handle.addEventListener("drop", (event) => { - event.preventDefault(); - if (dragged?.kind === handle.dataset.kind) { - emit(dragged.kind, dragged.row, Number(handle.dataset.row)); - } - dragged = null; - }); - handle.addEventListener("dragend", () => { - dragged = null; - }); - } }; observer = new MutationObserver(bind); @@ -170,7 +123,6 @@ infoElement?.remove(); infoElement = null; infoInput = null; - dragged = null; }, }; })(); diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 7b61b4f15c..793c24071b 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -105,8 +105,6 @@ def _build_gui(self) -> None: self.show_saved = gui.add_checkbox("Show Saved Binding Poses", False) self._watch(self.mode, self._set_mode) self._watch(self.show_saved, lambda _: self._request_draw()) - self.reorder_bridge = gui.add_text("__Tactile Reorder Event", "") - self._watch(self.reorder_bridge, self._reorder) self.edit_tabs = gui.add_tab_group() self.urdf_tab = self.edit_tabs.add_tab("URDF") @@ -215,7 +213,6 @@ def _build_gui(self) -> None: self._action_button( "Reset Values", self._restore_preview, - ) gui.add_markdown("The following values affect Preview only.") self.preview_segment_folder = gui.add_folder( @@ -553,9 +550,6 @@ def _rebuild_segment_list(self) -> None: for issue in issues if issue["kind"] == "segments" and issue["row"] == row ] - self.server.gui.add_html( - f'
⠿ Drag
' - ) with self.server.gui.add_folder(f"{row}: {segment.joint}"): joint = self.server.gui.add_dropdown( "⚠ Name" @@ -615,9 +609,6 @@ def _rebuild_taxel_list(self) -> None: for issue in issues if issue["kind"] == "taxels" and issue["row"] == row ] - self.server.gui.add_html( - f'
⠿ Drag
' - ) with self.server.gui.add_folder(f"Taxel {row}"): position, rotation = geo.components(geo.matrix(taxel.pose)) positions = self._numbers("Position", position, 0.001) @@ -725,18 +716,6 @@ def _set_taxel_weight(self, row: int, slot: int, value) -> None: finally: self._rebuild_taxel_list() - def _reorder(self, value: str) -> None: - if not value: - return - parts = value.split(":") - if len(parts) != 4: - return - kind, source, target, _ = parts - if kind not in ("segments", "taxels"): - return - self.document.move(kind, int(source), int(target)) - self._rebuild_edit_lists() - def _start_binding(self) -> None: self.pick_mode = None self.segment_click_button.label = "Add with URDF Click" From e467b9667b4f0b9ec9c586f1a383827a862c0547 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 15:50:29 -0700 Subject: [PATCH 19/36] Multiple updates on visualization and functions Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 14 +- scripts/tactile_metadata/README.md | 69 ++- scripts/tactile_metadata/UI&Feature Design.md | 34 +- .../editor_interactions.js | 140 ++++- .../tactile_metadata_editor/geometry.py | 29 +- .../tactile_metadata_editor/ui.py | 554 ++++++++++++------ .../tactile_metadata_editor/viewport_axes.js | 148 +++++ 7 files changed, 734 insertions(+), 254 deletions(-) create mode 100644 scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index bc38e86cd9..6cb6b470a9 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -15,8 +15,9 @@ SPDX-License-Identifier: Apache-2.0 - 评审替代性 UI 规格时，以当前稿为产品范围，只把 schema/API 约束视为阻塞；不要把旧功能、状态增强或可接受的瞬态覆盖升级为要求，并逐字核对交互手势。 - 对设计文档的结构与表达要求应直接落实到目标文档；页面布局按区块列出具体控件，不以聊天中的补充说明代替文档修改。 - Keep URDF driving separate from taxel deformation: HandPose supplies skeleton and LBS poses, while live URDF motion uses JointState only. Remove obsolete driver controls, persisted options, algorithms, and acceptance cases together. -- Live HandPose overwrites only valid joints. Invalid joints retain their previous pose, initialized from the binding pose; an invalid WRIST freezes all segment poses because no wrist-relative conversion is available. -- Preview deformation is per taxel: omit invalid taxels and report their row errors without suppressing valid taxels. Preserve original row indices when matching live force values, and block Apply while any preview taxel is invalid. +- Live Preview segments come only from `is_valid=True` HandPose joints in HandJoint order; invalid joints are absent, and an invalid WRIST yields an empty segment list because wrist-relative conversion is unavailable. The selected metadata source controls only taxel bindings and never seeds or overwrites Preview segments. +- Preview deformation is per taxel: omit invalid taxels and report their row errors without suppressing valid taxels. Preserve original row indices when matching live force values. Apply Taxels and Apply ALL copy exactly the valid subset returned for 3D rendering and never require every taxel to be valid. +- Preview apply operations remain independent and available despite taxel errors. Error reasons persist in the fixed Info area after disconnects and unrelated Preview actions. - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. - New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. - Preserve slot order and repeated segment indices. Normalize all four weights only on explicit user weight edits and reject negative edit values; loading, recording received values, checking, saving, and exporting preserve raw weights. Distinguish receiver failures from sender behavior, and keep all editor operations local. @@ -26,11 +27,15 @@ SPDX-License-Identifier: Apache-2.0 - Reuse the launching Python environment and its SDK through sys.executable. Do not create tool-specific project metadata, lockfiles, environments or SDK installations, and do not infer runtime requirements from old local build artifacts. - Missing packages in the agent's inspection environment limit verification; report the unavailable checks. Do not turn verification requirements into proposals to install dependencies, provision environments, or substitute interpreters; maintain Isaac Capture's shared dependency declarations instead. - Use Viser's standard controls and public callbacks, with document changes serialized through State. Apply the same explicit weight-normalization rule to sliders and numeric fields. +- The selected-object transform control enables only XYZ axis translation. Serialize drag phases through the event queue; single selections use object axes, multi-selections use visualization-space axes, and the same translation delta applies to every selected object. +- Selection side-highlighting resolves bindings by HandJoint name and includes only slots with positive weights. Keep `SIDE_HIGHLIGHT_COLOR` distinct from `SELECTED_COLOR`; selection color wins on selected objects and validation error color wins on invalid taxels. - When product placement says top/before/below and any sibling uses explicit Viser `order`, assign explicit order to every constrained sibling; source creation order does not override mixed default and explicit ordering. -- Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. UI sections marked `(spannable)` in the product design start collapsed. +- Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. UI sections marked `(spannable)` in the product design and all nested entity/joint/binding folders start collapsed. - Client-side helpers may hide only their exact bridge input and label; never hide an ancestor container that can contain the entire Viser control panel. +- Preserve focused numeric drafts such as `1.` client-side because Viser ignores partial numeric values and controlled rerenders otherwise erase the decimal point before the user finishes typing. Restore through the native input value setter and dispatch an `input` event; a raw DOM assignment leaves React/Mantine state stale and breaks the next digit. +- The right menu is a flex column: General is fixed, the Edit tab group fills the remaining height, and only the active tab body scrolls. Scope DOM styling to the root containing the General folder and URDF/Segments/Taxels dock group. - Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. -- Call `/edit`, `/preview`, `/saved`, and `/errors` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. +- Call `/edit`, `/preview`, and `/saved` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. - During live redraw, update the active scene-node group in place. Hide only groups that should be inactive; never hide and immediately re-show the active group each frame. - Keep metadata/force alignment with the producer and user; do not add source-row mappings, alignment baselines, force permutations, or list-order controls to the editor. - JSON follows tactile_metadata.fbs and the existing bindings. Treat array and enum conversion for encoding as routine implementation rather than a separate product-format decision. @@ -38,6 +43,7 @@ SPDX-License-Identifier: Apache-2.0 - Simplify persistence by deleting unsupported branches in place; do not relocate serialization into Document or introduce new state abstractions for a narrower file format. - For local editor checks using a cached Ruff executable, match the version in the repository hook configuration; older binaries may not understand the shared lint rules. Keep every check and formatter target inside this subtree. - Keep the GUI a thin adapter over shared document, geometry, storage and SDK operations. Prefer explicit native Viser widgets and one reusable property inspector; avoid widget-type inference, duplicate field editors in dialogs, and rebuilding scene nodes on every frame. Moving logic between files alone does not reduce implementation size. +- Segment cubes use flat shading with cast and receive shadows disabled; do not restore smooth vertex-normal gradients to semantic markers. - Use URDF base opacity 0.4 multiplied by the rendered dataset group's opacity, with color-based link highlighting; do not add per-link visibility or opacity controls or state. - Under a hard code budget, finish and count a complete executable slice before expanding the UI. Refactoring dispatch or moving functions is not evidence that the implementation fits; verify the formatted total and retain budget for integration fixes. - Treat metadata wrist coordinates, Viser scene coordinates and camera screen orientation as distinct; diagnose an upside-down display from the scene/camera mapping, not URDF alignment. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 91042f7a60..d0aecf51a4 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -48,6 +48,12 @@ The editor maintains independent left- and right-hand documents. **Toggle Side** selects the active document; each side has its own metadata path and data. General controls are at the top of the right menu. The Selected Object panel is always present as a floating window on the left. +The **Mode** section uses bold, mutually exclusive **Edit** and **Preview** +checkbox labels; one mode is always selected. +General and the URDF/Segments/Taxels tab selector stay fixed at the top of the +right menu. Only the active tab contents scroll. +Focused numeric fields preserve partial decimal input such as `1.` until the +next digit is entered or the field loses focus. Three pose datasets are displayed: @@ -64,6 +70,10 @@ Loading applies that transform; saving applies its inverse. Changing the alignment transforms the visible editor values without changing the underlying metadata coordinates. +A fixed three-axis indicator in the top-right of the 3D view shows visualization +space relative to the render camera. The Viser icon remains at the bottom-left, +positioned above the fixed Info area. + ## Edit metadata The **Edit** mode contains three tabs. @@ -74,8 +84,9 @@ The **Edit** mode contains three tabs. 2. Use **URDF Alignment** to translate, rotate, and scale the model. 3. Expand **URDF Joints** to adjust each movable joint. -URDF Joints and the full Segment/Taxel lists are collapsed by default. Their -displayed indices start at 0. +URDF Joints and the full Segment/Taxel lists are recursively collapsed by +default. Each joint/entity item and each nested Taxel Binding folder also starts +collapsed. Their displayed indices start at 0. URDF configuration is session-only and is not saved in tactile metadata. Its base opacity is 0.4, multiplied by the opacity of the dataset layer being @@ -86,7 +97,8 @@ rendered. - Use **Segments Alignment** to align all metadata poses. - Add a segment from a HandJoint and XYZ position, or select **Add with URDF Click** and click the model surface. -- Segments are represented by yellow spheres with radius 0.005 m. +- Segments are flat-shaded cubes with side length 0.004 m. Smooth color + gradients and cast/received shadows are disabled. - Expand **All Segments** to edit HandJoint, position, and XYZ rotation. HandJoint values must be unique. Taxel bindings use HandJoint names internally, @@ -98,7 +110,8 @@ invalid and outlined in red. Set Rows, Columns, Area Width, Area Height, Center Position, and Rotation. **Capture URDF Surface Pose** copies a clicked surface position and orientation into those fields without creating taxels. **Add Taxels** then creates the -row-major planar grid. Taxels are light-grey squares with side length 0.0025 m. +row-major planar grid. Taxels are squares with side length 0.002 m. A taxel with +any validation error uses `ERROR_COLOR` as its fill color. To bind taxels: @@ -106,8 +119,9 @@ To bind taxels: 2. Select **Select Segments** and choose one to four segments. 3. Select **Complete**. Initial active weights are equal. -The three buttons are horizontal, and at most one is enabled at a time. Between -steps, only the required 3D selection is available. +The three buttons are vertical with down arrows between adjacent steps, and at +most one is enabled at a time. Between steps, only the required 3D selection is +available. Expand **All Taxels** to edit pose, four segment slots, and four weights. `None` is an editor-only inactive slot with weight zero. Editing a weight preserves the @@ -123,12 +137,22 @@ one. If only one slot is active, its weight is one. contains both, taxels take priority. - Clicking empty scene space clears the selection. +Selected objects keep `SELECTED_COLOR`. Related objects use the separate +`SIDE_HIGHLIGHT_COLOR`: selecting segments highlights taxels with positive +weights to those HandJoints, while selecting taxels highlights their +positive-weight segments. Zero-weight slots are ignored. + The **Selected Objects** panel edits position and rotation, deletes selected objects, and scales multi-selection positions about their center. Edit mode changes Editing data; Preview mode changes Preview segments. Preview taxels are derived and cannot be selected. With no selection, the panel remains visible, shows Type `None` and Count `0`, and disables its input fields. +The selected-object 3-axis control supports axis-only translation. Dragging an +axis moves every selected object by the same delta. Single selections use the +object orientation; multi-selections use visualization-space axes. Rotation, +plane-translation, and scale handles are disabled. + Action and validation errors appear in the fixed **Info** area at the bottom of the 3D viewport. It cannot be moved. @@ -159,21 +183,26 @@ standalone check/export CLI commands. Use **Reset Values** to restore Preview segment poses and URDF joint values from Editing. -Choose **Binding Metadata Source**: +Choose the mutually exclusive checkbox in **Binding Metadata Source**: -- **Editing** uses the current editor bindings. -- **Live** becomes available after metadata is received from the tactile stream. +- **Editor** uses the current editor bindings and is selected by default. +- **Live** is disabled until metadata is received from the tactile stream. The Preview Static section lists Segments before URDF Joints. -**Apply Preview Data to Editor** fully replaces Editing segments, taxels, -bindings, and URDF joint values. The button is disabled when Preview segments -are incompatible with the selected bindings. +Preview provides four independent apply operations: + +- **Apply Segments** replaces HandJoint/segment poses. +- **Apply Taxels** replaces Editing taxels with exactly the valid derived taxels + currently rendered in Preview; invalid rows are omitted. +- **Apply URDF** replaces URDF joint values. +- **Apply ALL** applies Segments and URDF plus the same valid rendered taxel + subset used by Apply Taxels. Each Preview taxel is evaluated independently. Invalid rows are reported and omitted from the scene while valid taxels continue to render. Live force rows -remain matched by their original taxel indices. Apply remains disabled until all -taxels are valid. +remain matched by their original taxel indices. Invalid taxels are reported in +Info and omitted from Apply Taxels and Apply ALL; no Apply button is disabled. For live reception: @@ -187,11 +216,13 @@ For live reception: 3. Select **Connect Live**. -HandPose updates Preview segments. Enabled JointState mappings update Preview -URDF joints. Live frames overwrite manual Static values while connected. -HandPose overwrites only valid joints; invalid joints retain their last pose, -starting from the binding pose. If WRIST is invalid, every segment remains at -its last pose because a wrist-relative transform cannot be computed. +HandPose replaces Preview segments with only its valid joints, retained in +HandJoint order, and is the sole source of Live Preview segment identities and +poses. Invalid joints are absent. If WRIST is invalid, Preview contains no live +segments because a wrist-relative transform cannot be computed. Selected +binding metadata affects only taxel deformation; it never seeds Preview +segments. Enabled JointState mappings update Preview URDF joints. Live frames +overwrite manual Static values while connected. Tactile force magnitude maps from blue at 0 N to red at 30 N, with values above 30 N clamped. Missing or inactive force data is light grey. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index 90c5f14ed2..d8e8809e61 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -7,17 +7,21 @@ SPDX-License-Identifier: Apache-2.0 UI&Feature Design Global standard: unless improper for specific reason, for all float or int input fields, enable up/down arrow to adjust the value. For float input field, the step should be determined by the estimated range of the value. For int input field, the step should be 1. Only when a field has a clear specific limit range, the input field should reject illegal values. For sliders, when there's a clear specific limit range, use the limit range. +* Preserve partial focused float input such as `1.` while the user edits; controlled UI refreshes must not erase the decimal point before the next digit is entered. * When "Save", check the validation of metadata. If has error, pop up warning window "Metadata contain errors, are you sure to save?" for user to select continue or cancel. -* Every field marked `(spannable)` is collapsed by default. +* Every field marked `(spannable)` is fully collapsed recursively by default: the list container, each entity/joint item, and nested Taxel Binding folders all start collapsed. * Segment, taxel, and displayed URDF-joint indices start at 0. For field names, use title case for multi-word field names. For example, "URDF Path" instead of "urdf path". About data cooridanates: all UI editor use visualization space. When save to metadata file, do the coordinate transformation with Segments Alignment value. When load from metadata file, use the default VISUALIZATION_FROM_HAND value as Segments Alignement (and do transformation). Modifying Segments Alignment will cause values in editors also update. +* A fixed three-axis visualization-space direction indicator is displayed in the top-right of the 3D view and updates from the render camera. +* Place the Viser icon at the bottom-left of the view, above the fixed bottom Info area. About LBS taxel rotation: determine by Polar Decomposition algorithm. At singularity, use taxel's bind rotation. Make sure to optimize for single-segment binding case, where the rotation can be directly determined without running Polar Decomposition. -1. At the top of the right menu, always display a General section containing: Toggle side, new/save/load metadata file, reset camera, a Mode single selection from Edit and Preview, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". +1. At the top of the right menu, always display a General section containing: Toggle side, new/save/load metadata file, reset camera, a named "Mode" section containing bold Edit and Preview checkbox labels that are mutually exclusive and cannot both be unchecked, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". +* Freeze General and the URDF/Segments/Taxels tab selector at the top of the right menu. Scrolling affects only the active tab contents; the General section and tab selector do not scroll. 2. This tool actually manages 3 copies of pose data (pose of segments and taxels), with different data sources. a) Editing dataset: the data that the user is editing in the tool. It's the binding poses of the segments and taxels under editing. Plus the URDF joint values that's under Editing mode. @@ -43,12 +47,12 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula d) (spannable) A full list of all segments, with an id number (read-only, starting at 0, it's the index of the segment in the list), their name (dropdown list of HandJoint, display in enum name), editable position (3 float input), editable rotation (3 float slider). This is to modify the binding poses of the segments, display this info above this section. * It's invalid to have more than one segments to select the same HandJoint. At error, outline the repeated fields in red. * Some fields in Taxels tab refer to segments in this Segments tab, and modifying segments may result in invalid taxel binding that's revealed in Taxels tab. -* Segments are displayed as yellow spheres of radius 0.005m. The three axes of each segment are not displayed by default.Clicking an segment on the 3D scene while in Edit mode will select the segment, highlight it, reveal its 3 axes. When a segment is newly added, it's selected, and deselect the previous selected object(s). Clicking on any other area in the 3D scene will deselect the segment; dragging 3D scene or clicking on the menu/other UI area will not deselect the segment. Clicking on the same selected segment will not do anything. +* Segments are displayed as flat-shaded cubes with side length 0.004m, without smooth color gradients or cast/received shadows. The three axes of each segment are not displayed by default. Clicking a segment on the 3D scene while in Edit mode will select the segment, highlight it, and reveal its 3-axis translation control. When a segment is newly added, it is selected and the previous selected objects are deselected. Clicking any other area in the 3D scene deselects the segment; dragging the 3D scene or clicking menu/UI areas does not. Clicking the same selected segment does nothing. 9. Taxels tab: * Taxels don't have their onw alignment, it uses the alignment of the segments. a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), and rotation (3 float sliders). "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields; newly created taxels are selected and previous objects are deselected. - c) Horizontally align "Start Binding to Segments", "Select Segments", and "Complete". At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The scene instructions and equal initial weights otherwise follow the same three-step binding flow. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. + c) Vertically align "Start Binding to Segments", "Select Segments", and "Complete", with a centered down arrow between each adjacent pair. At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The scene instructions and equal initial weights otherwise follow the same three-step binding flow. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. * "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. @@ -56,9 +60,10 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. b) the weights don't sum to 1. (Can happen when a segment is removed from Segments tab or by Preview apply to editor.) - weights input fields. c) the weight is 0 for a non-None segment - weight input field. -* Taxels are displayed as light grey flat squares of size 0.0025m*0.0025m. The three axes of each taxel are not displayed by default. Clicking an taxel on the 3D scene while in Edit mode will select the taxel, highlight it, reveal its 3 axes. The 3D scene modification rules are the same as segments. +* Taxels are displayed as flat squares of size 0.002m*0.002m. A taxel with any validation error uses ERROR_COLOR as its square fill color. The three axes of each taxel are not displayed by default. Clicking a taxel on the 3D scene while in Edit mode will select the taxel, highlight it, and reveal its 3-axis translation control. The 3D scene modification rules are the same as segments. * Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. -* When multiple objects are selected, the 3-axis visualizer should show in the center of the selected objects' area, with axis directions align with the visualiation space coordinates. +* Selection also highlights related objects with the separate configured `SIDE_HIGHLIGHT_COLOR`: selecting segment(s) highlights taxels with a positive binding weight to any selected HandJoint; selecting taxel(s) highlights segments referenced by a positive binding weight. Zero-weight slots never produce this highlight, and selected objects retain `SELECTED_COLOR`. +* The 3-axis translation control is draggable only along its X/Y/Z axes and moves every selected object by the same translation. For one object it follows that object's orientation; for multiple objects it is centered on the selection with axes aligned to visualization-space coordinates. Rotation, plane translation, and scaling handles are disabled. * An additional floating menu on the left side of the view is always displayed for selected object(s)' modification; it is not docked to the viewport edge. It applies also to single-object selection. With no selection it shows Type: None and Count: 0 and disables every input field: a) (read only) selected elements type and count (multi selection is stated in following part) b) (editable) current position (defined above) @@ -67,17 +72,22 @@ d) (editable single float) scaling (only for multi-selection, scale up/down the e) Delete 10. Preview menu: - a) A selection field to select the Binding Metadata Source from "Live" and "Editing", by default "Editing". "Live" is only available after the first live metadata is received. This selection affects the taxels pose rendered, because it's calculated based on the segments poses and binding metadata. + a) A named "Binding Metadata Source" section containing bold Editor and Live checkbox labels. They are mutually exclusive and cannot both be unchecked; Editor is selected by default. Live remains disabled/grey until the first Live metadata is received. This selection affects rendered taxel poses because they are calculated from segment poses and the selected binding metadata. b) A checkbox toggling "Show Binding Poses". When checked, show the segments and taxels (without URDF) from the Editing dataset in the 3D scene at a opacity of 0.5. c) A text input field "Tactile Collection ID", default to "tactile". - d) A button "Apply Preview Data to Editor". When clicked, apply the Preview dataset's segment poses and deferred taxel poses (segment poses + selected metadata binding) and the binding relationships, to the Editing dataset's segment binding poses and taxel binding data. Also update the urdf joint values to Editing dataset. Remind the user that this action will fully override segments and taxels. + d) Four buttons: + - "Apply Segments" replaces only Editing HandJoint/segment poses from Preview. + - "Apply Taxels" replaces Editing taxels with exactly the valid derived Preview taxels currently rendered in the 3D scene; invalid taxels are omitted and do not block this operation. + - "Apply URDF" replaces only Editing URDF joint values. + - "Apply ALL" applies Preview Segments and URDF values plus exactly the same valid rendered taxel subset used by Apply Taxels. + Invalid Preview taxels do not disable any Apply operation; they are reported in Info and omitted from taxel application. * When deffering taxel poses or applying binding relationships to the Edit dataset, segments should be referred to by their HandJoint enum name instead of indice. It means received Metadata requires one layer of translation to turn the binding indice into HandJoint. -* If the segment list is incompatible to the metadata's taxels binding (e.g. taxels refer to a segment that's not in list), an error should show up at the top of Preview menu including information like the missing segment enum name, and "Apply Preview Data to Editor" should grey out not clickable. -* Preview deformation is independent per taxel. Invalid taxels are omitted and reported by row without preventing valid taxels from rendering. Applying Preview remains disabled until every taxel is valid. +* If the segment list is incompatible with the metadata's taxel bindings, show the missing HandJoint reason in the fixed Info area. Apply Taxels and Apply ALL remain available and use only the valid rendered subset. +* Preview deformation is independent per taxel. Invalid taxels are omitted and reported by row without preventing valid taxels from rendering. Live section: a) Checkbox "Use JointState to Drive URDF". When checked, enable the "JointState collection id" input field, and a Joint Mappings text field to receive a long json string to map the joint names in JointState to the URDF. The current field "Joint mappings JSON: source → [target, scale, offset]" is reusable. - b) A button "Connect Live" button. When clicked, update the Preview dataset's segments poses with received hand data in real time, and update the taxels poses based on the segments poses and the binding metadata (selected in above selection field). Manage a field to clearly state the connection statue: Live or Disconnected. After clicking "Connect Live", the button changes text to "Disconnect Live". When disconnected, stops updating the segments poses or URDF joint values. (If JointState is used, Live mode also updates the URDF joint values.) Show warning: Live will override values in Static section. -* HandPose updates only valid joints. An invalid joint retains its previous pose, initialized from its binding pose. If WRIST is invalid, all segment poses remain stale because wrist-relative conversion is unavailable. + b) A button "Connect Live" button. When clicked, replace the Preview segment list and poses with only the `is_valid=True` HandJoint entries from received HandPose, kept in HandJoint order. The selected binding metadata never seeds, filters, or overwrites Preview segments; it is used only to derive taxel poses from the valid HandPose segments. Manage a field to clearly state the connection state: Live or Disconnected. After clicking "Connect Live", the button changes text to "Disconnect Live". When disconnected, stop updating segment poses or URDF joint values. (If JointState is used, Live also updates URDF joint values.) Show warning: Live will override values in Static section. +* Invalid HandPose joints are absent from Preview. If WRIST is invalid, Preview has no live segments because wrist-relative conversion is unavailable. * If the metadata * While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. Static section: diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index 5c865e048e..5068f0fe90 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -10,6 +10,11 @@ let infoInput = null; let infoElement = null; let raf = 0; + const numberDrafts = new Map(); + const nativeInputValueSetter = Object.getOwnPropertyDescriptor( + HTMLInputElement.prototype, + "value", + )?.set; const findBridge = (name) => { for (const input of document.querySelectorAll("input")) { @@ -34,34 +39,13 @@ return null; }; - const alignBindingButtons = () => { - const names = new Set([ - "Start Binding to Segments", - "Select Segments", - "Complete", - ]); - const buttons = [...document.querySelectorAll("button")].filter((button) => - names.has(button.textContent?.trim()), - ); - if (buttons.length !== 3) return; - let common = buttons[0].parentElement; - while (common && !buttons.every((button) => common.contains(button))) { - common = common.parentElement; - } - if (!common || common.querySelectorAll("button").length !== 3) return; - const children = buttons.map((button) => { - let child = button; - while (child.parentElement !== common) child = child.parentElement; - return child; - }); - if (new Set(children).size !== 3) return; - common.style.display = "grid"; - common.style.gridTemplateColumns = "repeat(3, minmax(0, 1fr))"; - common.style.gap = "6px"; - for (const child of common.children) { - child.style.gridColumn = children.includes(child) ? "auto" : "1 / -1"; - } - for (const button of buttons) button.style.width = "100%"; + // Place Viser's logo above the fixed bottom Info area. + const positionViserLogo = () => { + const logo = document.querySelector('a[title="About Viser"]'); + if (!logo) return; + logo.style.top = "auto"; + logo.style.left = "16px"; + logo.style.bottom = "144px"; }; const syncInfo = () => { @@ -94,13 +78,110 @@ }; const frame = () => { + for (const [input, draft] of numberDrafts) { + if (!input.isConnected || document.activeElement !== input) { + numberDrafts.delete(input); + } else if (input.value !== draft) { + nativeInputValueSetter?.call(input, draft); + input.dispatchEvent(new Event("input", { bubbles: true })); + input.setSelectionRange(draft.length, draft.length); + } + } syncInfo(); raf = requestAnimationFrame(frame); }; + const boldChoiceLabels = () => { + for (const label of document.querySelectorAll("label")) { + if (["Edit", "Preview", "Editor", "Live"].includes(label.textContent?.trim())) { + label.style.fontWeight = "700"; + } + } + }; + + const folderFromLabel = (name) => { + const label = [...document.querySelectorAll("div")].find( + (element) => element.textContent?.trim() === name && element.querySelector("svg"), + ); + return label?.parentElement || null; + }; + + const directChild = (parent, descendant) => { + let child = descendant; + while (child?.parentElement && child.parentElement !== parent) { + child = child.parentElement; + } + return child?.parentElement === parent ? child : null; + }; + + const freezeMainMenu = () => { + const general = folderFromLabel("General"); + const preview = folderFromLabel("Preview"); + const tabNames = new Set(["URDF", "Segments", "Taxels"]); + const tabs = [...document.querySelectorAll("button")].filter((button) => + tabNames.has(button.textContent?.trim()), + ); + if (!general || tabs.length !== 3) return; + const tabGroup = tabs[0].closest("[data-dock-group]"); + if (!tabGroup) return; + let root = general.parentElement; + while (root && !root.contains(tabGroup)) root = root.parentElement; + if (!root) return; + const generalChild = directChild(root, general); + const tabsChild = directChild(root, tabGroup); + const previewChild = preview ? directChild(root, preview) : null; + if (!generalChild || !tabsChild) return; + + Object.assign(root.style, { + display: "flex", + flexDirection: "column", + height: "100%", + minHeight: "0", + overflow: "hidden", + }); + Object.assign(generalChild.style, { + flex: "0 0 auto", + zIndex: "30", + }); + Object.assign(tabsChild.style, { + flex: "1 1 0", + minHeight: "0", + overflow: "hidden", + }); + if (previewChild) { + Object.assign(previewChild.style, { + flex: "1 1 0", + minHeight: "0", + overflowY: "auto", + }); + } + }; + + const preservePartialNumbers = () => { + for (const input of document.querySelectorAll("input")) { + if (input.dataset.tactileNumberDraft === "true") continue; + if (!["decimal", "numeric"].includes(input.inputMode) && input.type !== "number") { + continue; + } + input.dataset.tactileNumberDraft = "true"; + input.addEventListener("input", () => { + const value = input.value; + if (/^[+-]?(?:\d+)?\.$/.test(value)) { + numberDrafts.set(input, value); + } else { + numberDrafts.delete(input); + } + }); + input.addEventListener("blur", () => numberDrafts.delete(input)); + } + }; + const bind = () => { findBridge("__Tactile Info"); - alignBindingButtons(); + positionViserLogo(); + boldChoiceLabels(); + freezeMainMenu(); + preservePartialNumbers(); for (const label of document.querySelectorAll("label")) { if (!label.textContent?.trim().startsWith("⚠")) continue; const field = label.parentElement?.querySelector("input, button"); @@ -123,6 +204,7 @@ infoElement?.remove(); infoElement = null; infoInput = null; + numberDrafts.clear(); }, }; })(); diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index 2a27e66a5d..018543f966 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -26,7 +26,6 @@ [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 0, 1]], dtype=float, ) -URDF_OPACITY = 0.4 def matrix(values: list[float]) -> np.ndarray: @@ -210,8 +209,8 @@ def draw( q: dict[str, float], alignment: np.ndarray, scale: float, + opacity: float, highlighted: bool = False, - opacity: float = URDF_OPACITY, ) -> None: raw = self.raw_fk(q) for name, (link, mesh) in self.surfaces.items(): @@ -340,9 +339,7 @@ def deform( return result, errors -def hand_segments( - binding: list[Segment], hand: dict, stale: list[Segment] -) -> list[Segment]: +def hand_segments(hand: dict) -> list[Segment]: values = [] for position, orientation, valid in zip( hand["positions"], hand["orientations"], hand["is_valid"] @@ -350,17 +347,11 @@ def hand_segments( values.append(matrix([*position, *orientation]) if valid else None) if len(values) != len(JOINTS): raise ValueError("HandTracker joint array has an invalid length") - stale_by_joint = {segment.joint: segment for segment in stale} - wrist_inverse = np.linalg.inv(values[1]) if values[1] is not None else None - result = [] - for bind_segment in binding: - joint = bind_segment.joint - tracked = values[JOINTS.index(joint)] - if wrist_inverse is None or tracked is None: - fallback = stale_by_joint.get(joint, bind_segment) - result.append(Segment(joint, list(fallback.pose))) - else: - result.append( - Segment(joint, pose(VISUALIZATION_FROM_HAND @ wrist_inverse @ tracked)) - ) - return result + if values[1] is None: + return [] + wrist_inverse = np.linalg.inv(values[1]) + return [ + Segment(joint, pose(VISUALIZATION_FROM_HAND @ wrist_inverse @ tracked)) + for joint, tracked in zip(JOINTS, values) + if tracked is not None + ] diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 793c24071b..30cfb49693 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -21,6 +21,27 @@ LOG = logging.getLogger("isaaccapture.tactile_metadata.editor") +SEGMENT_CUBE_SIDE_M = 0.004 +TAXEL_SIDE_M = 0.002 +URDF_OPACITY = 0.2 +SEGMENT_FLAT_SHADING = True +VISUALIZER_CAST_SHADOW = False +VISUALIZER_RECEIVE_SHADOW = False +SEGMENT_COLOR = (200, 120, 0) +TAXEL_COLOR = (200, 120, 0) +SELECTED_COLOR = (0, 255, 0) +SIDE_HIGHLIGHT_COLOR = (0, 180, 255) +INACTIVE_COLOR = (200, 200, 200) +ERROR_COLOR = (255, 0, 0) +FORCE_MIN_COLOR = (0, 0, 255) +FORCE_MAX_COLOR = (255, 0, 0) +FORCE_MAX_N = 30.0 +SELECTION_GIZMO_SCALE_M = 0.005 +SELECTION_GIZMO_LINE_WIDTH = 2.5 +ACTIVE_DATASET_OPACITY = 1 +BINDING_OVERLAY_OPACITY = 0.4 +SAVED_BINDING_OPACITY = 0.2 + class Editor: CAMERA_POSITION = (0.3, 0.3, -0.4) @@ -57,21 +78,35 @@ def __init__(self, state: State, args): self.binding_taxels: list[int] = [] self.binding_segments: list[int] = [] self.pick_mode: str | None = None + self.preview_has_errors = False self.selection_rotation = np.zeros(3) self.selection_scale = 1.0 + self.selection_gizmo_dragging = False + self.selection_gizmo_last_position: np.ndarray | None = None self.redraw = True self.server = viser.ViserServer(host="0.0.0.0", port=args.port) self.server.scene.set_up_direction("+y") - source = Path(__file__).with_name("editor_interactions.js").read_text() - self.server._websock_server.queue_message( - _messages.RunJavascriptMessage(source=source) - ) + for script in ("editor_interactions.js", "viewport_axes.js"): + source = Path(__file__).with_name(script).read_text() + self.server._websock_server.queue_message( + _messages.RunJavascriptMessage(source=source) + ) self.server.initial_camera.position = self.CAMERA_POSITION self.server.initial_camera.look_at = self.CAMERA_LOOK_AT self.server.gui.configure_theme(control_width="medium", show_share_button=False) self.server.gui.main_panel.dock_right() self._build_gui() self._register_scene_events() + self.selection_gizmo = self.server.scene.add_transform_controls( + "/selection/translation", + scale=SELECTION_GIZMO_SCALE_M, + line_width=SELECTION_GIZMO_LINE_WIDTH, + disable_sliders=True, + disable_rotations=True, + depth_test=False, + visible=False, + ) + self.selection_gizmo.on_update(self._selection_gizmo_event) self._sync_side() if args.urdf: self.urdf_path.value = str(Path(args.urdf).expanduser().resolve()) @@ -101,9 +136,14 @@ def _build_gui(self) -> None: self._action_button("Save", self._save) self._action_button("Load", self._load) self._action_button("Reset Camera", self._reset_camera) - self.mode = gui.add_dropdown("Mode", ("Edit", "Preview")) + with gui.add_folder("Mode"): + self.mode_edit = gui.add_checkbox("Edit", True) + self.mode_preview = gui.add_checkbox("Preview", False) self.show_saved = gui.add_checkbox("Show Saved Binding Poses", False) - self._watch(self.mode, self._set_mode) + self._watch(self.mode_edit, lambda checked: self._set_mode("Edit", checked)) + self._watch( + self.mode_preview, lambda checked: self._set_mode("Preview", checked) + ) self._watch(self.show_saved, lambda _: self._request_draw()) self.edit_tabs = gui.add_tab_group() @@ -169,7 +209,13 @@ def _build_gui(self) -> None: self._action_button("Add Taxels", self._add_taxels_input) with gui.add_folder("Bind Taxels to Segments"): self.binding_start = gui.add_button("Start Binding to Segments") + gui.add_html( + '
↓
' + ) self.binding_next = gui.add_button("Select Segments", disabled=True) + gui.add_html( + '
↓
' + ) self.binding_complete = gui.add_button("Complete", disabled=True) self.binding_info = gui.add_markdown("") self._button(self.binding_start, self._start_binding) @@ -181,18 +227,36 @@ def _build_gui(self) -> None: self.preview_panel = gui.add_folder("Preview", visible=False) with self.preview_panel: - self.binding_source = gui.add_dropdown( - "Binding Metadata Source", ("Editing",) - ) + with gui.add_folder("Binding Metadata Source"): + self.binding_source_editor = gui.add_checkbox("Editor", True) + self.binding_source_live = gui.add_checkbox( + "Live", False, disabled=True + ) self.show_binding = gui.add_checkbox("Show Binding Poses", False) self.collection = gui.add_text("Tactile Collection ID", "tactile") - self.apply_preview_button = self._action_button( - "Apply Preview Data to Editor", self._apply_preview + self.apply_segments_button = self._action_button( + "Apply Segments", self._apply_preview_segments + ) + self.apply_taxels_button = self._action_button( + "Apply Taxels", self._apply_preview_taxels + ) + self.apply_urdf_button = self._action_button( + "Apply URDF", self._apply_preview_urdf + ) + self.apply_all_button = self._action_button( + "Apply ALL", self._apply_preview_all ) gui.add_markdown( - "Apply fully overrides the editor's segments, taxels, bindings, and URDF joint values." + "Apply ALL fully overrides the editor's segments, taxels, bindings, and URDF joint values." + ) + self._watch( + self.binding_source_editor, + lambda checked: self._set_binding_source("Editor", checked), + ) + self._watch( + self.binding_source_live, + lambda checked: self._set_binding_source("Live", checked), ) - self._watch(self.binding_source, lambda _: self._binding_source_changed()) self._watch(self.show_binding, lambda _: self._request_draw()) with gui.add_folder("Live"): self.live_status = gui.add_markdown("Current Statue: Disconnected") @@ -229,7 +293,7 @@ def _build_gui(self) -> None: self.selection_tab = self.selection_panel.add_tab("Selected Object") with self.selection_tab: self.selection_folder = gui.add_folder("Selection") - self.selection_panel.float(x=10, y=10, width=320, height=520) + self.selection_panel.float(x=10, y=140, width=320, height=520) def _numbers(self, prefix: str, values, step: float): return tuple( @@ -275,19 +339,33 @@ def _run_action(self, callback, value=None) -> None: except Exception as exc: LOG.exception("Editor action failed") self.info.value = f'Error: {exc}' + else: + if self.mode_value == "Preview" and self.preview_has_errors: + self._update_preview_error() self.redraw = True def _request_draw(self) -> None: self.redraw = True - def _set_mode(self, value: str) -> None: - self.edit_tabs.visible = value == "Edit" - self.preview_panel.visible = value == "Preview" + @property + def mode_value(self) -> str: + return "Edit" if self.mode_edit.value else "Preview" + + def _set_mode(self, mode: str, checked: bool) -> None: + selected = self.mode_edit if mode == "Edit" else self.mode_preview + other = self.mode_preview if mode == "Edit" else self.mode_edit + if not checked: + if not other.value: + selected.value = True + return + other.value = False + self.edit_tabs.visible = mode == "Edit" + self.preview_panel.visible = mode == "Preview" self.pick_mode = None self.selected_kind, self.selected_rows = None, [] self._cancel_binding() self._rebuild_selection() - if value == "Preview": + if mode == "Preview": self._update_preview_error() def _toggle_side(self) -> None: @@ -300,8 +378,9 @@ def _sync_side(self) -> None: self.pick_mode = None self.segment_click_button.label = "Add with URDF Click" self.taxel_capture_button.label = "Capture URDF Surface Pose" - self.binding_source.options = ("Editing",) - self.binding_source.value = "Editing" + self.binding_source_editor.value = True + self.binding_source_live.value = False + self.binding_source_live.disabled = True self.side_text.content = f"**Side: {self.state.side.title()}**" self.metadata_path.value = self.state.paths.get(self.state.side, "") position, rotation = geo.components( @@ -470,13 +549,16 @@ def _add_joint_sliders(self, document: Document, output: dict) -> None: else math.pi ) value = min(upper, max(lower, document.q.get(joint.name, 0.0))) - widget = self.server.gui.add_slider( - f"{index}: {joint.name}", - float(lower), - float(upper), - 0.001, - float(value), - ) + with self.server.gui.add_folder( + f"{index}: {joint.name}", expand_by_default=False + ): + widget = self.server.gui.add_slider( + "Value", + float(lower), + float(upper), + 0.001, + float(value), + ) self._watch( widget, lambda current, name=joint.name, document=document: ( @@ -550,7 +632,9 @@ def _rebuild_segment_list(self) -> None: for issue in issues if issue["kind"] == "segments" and issue["row"] == row ] - with self.server.gui.add_folder(f"{row}: {segment.joint}"): + with self.server.gui.add_folder( + f"{row}: {segment.joint}", expand_by_default=False + ): joint = self.server.gui.add_dropdown( "⚠ Name" if any(issue["field"] == "joint" for issue in row_issues) @@ -609,7 +693,9 @@ def _rebuild_taxel_list(self) -> None: for issue in issues if issue["kind"] == "taxels" and issue["row"] == row ] - with self.server.gui.add_folder(f"Taxel {row}"): + with self.server.gui.add_folder( + f"Taxel {row}", expand_by_default=False + ): position, rotation = geo.components(geo.matrix(taxel.pose)) positions = self._numbers("Position", position, 0.001) rotations = self._sliders("Rotation", rotation) @@ -629,7 +715,9 @@ def _rebuild_taxel_list(self) -> None: ) slots = [] for slot in range(4): - with self.server.gui.add_folder(f"Binding {slot}"): + with self.server.gui.add_folder( + f"Binding {slot}", expand_by_default=False + ): selected = ( taxel.segments[slot] if taxel.segments[slot] in options @@ -789,14 +877,34 @@ def _disconnect(self) -> None: self.receiver = None self.connect_button.label = "Connect Live" self.live_status.content = "Current Status: Disconnected" + self._update_preview_error() self.redraw = True - def _binding_source_changed(self) -> None: + @property + def binding_source_value(self) -> str: + return "Live" if self.binding_source_live.value else "Editor" + + def _set_binding_source(self, source: str, checked: bool) -> None: + selected = ( + self.binding_source_editor + if source == "Editor" + else self.binding_source_live + ) + other = ( + self.binding_source_live + if source == "Editor" + else self.binding_source_editor + ) + if not checked: + if not other.value: + selected.value = True + return + other.value = False self._update_preview_error() self.redraw = True def _binding_document(self) -> Document: - if self.binding_source.value == "Live": + if self.binding_source_value == "Live": if self.live_document is None: raise ValueError("Live metadata is unavailable") return self.live_document @@ -805,32 +913,51 @@ def _binding_document(self) -> Document: def _preview_taxels(self) -> tuple[list[Taxel | None], list[str]]: return geo.deform(self._binding_document(), self.preview_document.segments) - def _update_preview_error(self) -> None: + def _rendered_preview_taxels( + self, + ) -> tuple[list[tuple[int, Taxel]], list[str], int]: try: - _, errors = self._preview_taxels() + rows, errors = self._preview_taxels() except (ValueError, TypeError, KeyError, IndexError) as exc: - self.preview_error.value = ( - f'Binding error: {exc}' - ) - self.apply_preview_button.disabled = True - else: - self.preview_error.value = ( - '' + "
".join(errors) + "" - if errors - else "" - ) - self.apply_preview_button.disabled = bool(errors) + return [], [f"Binding error: {exc}"], 0 + return ( + [(row, taxel) for row, taxel in enumerate(rows) if taxel is not None], + errors, + len(rows), + ) + + def _update_preview_error(self) -> None: + _, errors, _ = self._rendered_preview_taxels() + self.preview_has_errors = bool(errors) + self.preview_error.value = ( + '' + "
".join(errors) + "" + if errors + else "" + ) + self.apply_taxels_button.disabled = False + self.apply_all_button.disabled = False + self.apply_segments_button.disabled = False + self.apply_urdf_button.disabled = False + + def _apply_preview_segments(self) -> None: + self.document.segments = copy.deepcopy(self.preview_document.segments) + self._rebuild_edit_lists() - def _apply_preview(self) -> None: - taxels, errors = self._preview_taxels() - if errors: - raise ValueError("Repair invalid preview taxels before applying") + def _apply_preview_taxels(self) -> None: + rows, _, _ = self._rendered_preview_taxels() + self.document.taxels = copy.deepcopy([taxel for _, taxel in rows]) + self._rebuild_edit_lists() + + def _apply_preview_urdf(self) -> None: + self.document.q = copy.deepcopy(self.preview_document.q) + self._rebuild_joint_controls() + + def _apply_preview_all(self) -> None: replacement = Document(self.state.side) replacement.alignment = copy.deepcopy(self.document.alignment) replacement.segments = copy.deepcopy(self.preview_document.segments) - replacement.taxels = copy.deepcopy( - [taxel for taxel in taxels if taxel is not None] - ) + rows, _, _ = self._rendered_preview_taxels() + replacement.taxels = copy.deepcopy([taxel for _, taxel in rows]) replacement.q = copy.deepcopy(self.preview_document.q) self.state.documents[self.state.side] = replacement self._rebuild_edit_lists() @@ -852,7 +979,9 @@ def _rebuild_preview_segments(self) -> None: ) with self.preview_segment_folder: for row, segment in enumerate(self.preview_document.segments): - with self.server.gui.add_folder(f"{row}: {segment.joint}"): + with self.server.gui.add_folder( + f"{row}: {segment.joint}", expand_by_default=False + ): self.server.gui.add_markdown(f"HandJoint: `{segment.joint}`") position, rotation = geo.components(geo.matrix(segment.pose)) positions = self._numbers("Position", position, 0.001) @@ -902,15 +1031,12 @@ def _update_live(self) -> None: raw_metadata, self.state.side, self.document.alignment ) self.live_metadata_key = key - self.binding_source.options = ("Editing", "Live") + self.binding_source_live.disabled = False try: - binding = self._binding_document() hand = frame.get("hand") if hand is not None: - joints = [segment.joint for segment in binding.segments] - updated = geo.hand_segments( - binding.segments, hand, self.preview_document.segments - ) + updated = geo.hand_segments(hand) + joints = [segment.joint for segment in updated] signature = [ segment.joint for segment in self.preview_document.segments ] @@ -936,7 +1062,7 @@ def _update_live(self) -> None: self.preview_error.value = ( f'Live error: {exc}' ) - self.apply_preview_button.disabled = True + self.preview_has_errors = True self.redraw = True def _sync_preview_values(self) -> None: @@ -951,6 +1077,43 @@ def _sync_preview_values(self) -> None: for name, widget in self.preview_joint_widgets.items(): widget.value = self.preview_document.q.get(name, 0.0) + async def _selection_gizmo_event(self, event) -> None: + if event.client is None: + return + phase = event.phase + position = np.asarray(event.target.position, dtype=float) + self.events.put(lambda: self._apply_selection_gizmo_event(phase, position)) + + def _apply_selection_gizmo_event(self, phase: str, position: np.ndarray) -> None: + values = self._selection_values() + rows = [row for row in self.selected_rows if 0 <= row < len(values)] + if not rows: + self.selection_gizmo_dragging = False + self.selection_gizmo_last_position = None + return + if phase == "start": + self.selection_gizmo_dragging = True + self.selection_gizmo_last_position = position + return + if ( + not self.selection_gizmo_dragging + or self.selection_gizmo_last_position is None + ): + return + delta = position - self.selection_gizmo_last_position + if np.any(delta): + transform = np.eye(4) + transform[:3, 3] = delta + geo.apply_delta(values, rows, transform) + center = np.asarray([values[row].pose[:3] for row in rows]).mean(axis=0) + self._set_values(self.selection_position_widgets, center) + self._sync_selected_list_values() + self.selection_gizmo_last_position = position + if phase == "end": + self.selection_gizmo_dragging = False + self.selection_gizmo_last_position = None + self.redraw = True + def _register_scene_events(self) -> None: self.server.scene.on_click()(self._scene_click) self.server.scene.on_click(modifier="shift")(self._scene_click) @@ -967,7 +1130,7 @@ async def _scene_box(self, event) -> None: ) def _display_values(self): - if self.mode.value == "Preview": + if self.mode_value == "Preview": return self.preview_document.segments, [] return self.document.segments, self.document.taxels @@ -1040,24 +1203,42 @@ def _ray_rows(event, kind: str, values) -> list[int]: direction = np.asarray(event.ray_direction) hits = [] for row, value in values: - point = np.asarray(value.pose[:3]) - if kind == "segments": - distance = (point - origin) @ direction - offset = np.linalg.norm(point - origin - distance * direction) - if distance > 0 and offset <= 0.005: - hits.append((distance, row)) - continue inverse = np.linalg.inv(geo.matrix(value.pose)) local_origin = (inverse @ np.r_[origin, 1.0])[:3] local_direction = inverse[:3, :3] @ direction + if kind == "segments": + distance = Editor._ray_box_distance( + local_origin, local_direction, SEGMENT_CUBE_SIDE_M / 2 + ) + if distance is not None: + hits.append((distance, row)) + continue if abs(local_direction[2]) < 1e-10: continue distance = -local_origin[2] / local_direction[2] local_hit = local_origin + distance * local_direction - if distance > 0 and np.max(np.abs(local_hit[:2])) <= 0.00125: + if distance > 0 and np.max(np.abs(local_hit[:2])) <= TAXEL_SIDE_M / 2: hits.append((distance, row)) return [min(hits)[1]] if hits else [] + @staticmethod + def _ray_box_distance(origin, direction, half_extent: float) -> float | None: + near, far = -math.inf, math.inf + for axis in range(3): + if abs(direction[axis]) < 1e-10: + if abs(origin[axis]) > half_extent: + return None + continue + first = (-half_extent - origin[axis]) / direction[axis] + second = (half_extent - origin[axis]) / direction[axis] + near = max(near, min(first, second)) + far = min(far, max(first, second)) + if near > far: + return None + if far <= 0: + return None + return near if near > 0 else far + @staticmethod def _box_rows(event, values) -> list[int]: camera = event.client.camera @@ -1111,7 +1292,7 @@ def _handle_pick(self, event) -> None: def _selection_values(self): if self.selected_kind is None: return [] - if self.mode.value == "Preview": + if self.mode_value == "Preview": return self.preview_document.segments return self._values(self.selected_kind) @@ -1198,7 +1379,7 @@ def _set_selection_scale(self, value: float) -> None: self._sync_selected_list_values() def _sync_selected_list_values(self) -> None: - if self.mode.value == "Edit": + if self.mode_value == "Edit": widgets = ( self.segment_widgets if self.selected_kind == "segments" @@ -1221,7 +1402,7 @@ def _sync_selected_list_values(self) -> None: self.redraw = True def _delete_selection(self) -> None: - if self.mode.value == "Preview": + if self.mode_value == "Preview": for row in sorted(self.selected_rows, reverse=True): self.preview_document.segments.pop(row) self._rebuild_preview_segments() @@ -1233,7 +1414,12 @@ def _delete_selection(self) -> None: self._rebuild_selection() def _draw_instances( - self, name: str, kind: str, values, colors, opacity: float, wireframe=False + self, + name: str, + kind: str, + values, + colors, + opacity: float, ) -> None: matrices = [geo.matrix(value.pose) for value in values] if not matrices: @@ -1242,22 +1428,22 @@ def _draw_instances( return positions = np.asarray([value[:3, 3] for value in matrices]) rotations = np.asarray([tf.quaternion_from_matrix(value) for value in matrices]) - scale = 0.005 if kind == "segments" else 0.0025 - mesh = ( - trimesh.creation.icosphere(subdivisions=1) - if kind == "segments" - else trimesh.Trimesh( - vertices=[ - [-0.5, -0.5, 0.0], - [0.5, -0.5, 0.0], - [0.5, 0.5, 0.0], - [-0.5, 0.5, 0.0], - ], - faces=[[0, 1, 2], [0, 2, 3]], - process=False, - ) - ) + scale = SEGMENT_CUBE_SIDE_M if kind == "segments" else TAXEL_SIDE_M if name not in self.scene_handles: + mesh = ( + trimesh.creation.box(extents=(1.0, 1.0, 1.0)) + if kind == "segments" + else trimesh.Trimesh( + vertices=[ + [-0.5, -0.5, 0.0], + [0.5, -0.5, 0.0], + [0.5, 0.5, 0.0], + [-0.5, 0.5, 0.0], + ], + faces=[[0, 1, 2], [0, 2, 3]], + process=False, + ) + ) self.scene_handles[name] = self.server.scene.add_batched_meshes_simple( name, mesh.vertices, @@ -1267,8 +1453,10 @@ def _draw_instances( batched_scales=np.full(len(values), scale), batched_colors=np.asarray(colors, dtype=np.uint8), opacity=opacity, - wireframe=wireframe, + flat_shading=SEGMENT_FLAT_SHADING if kind == "segments" else False, side="double", + cast_shadow=VISUALIZER_CAST_SHADOW, + receive_shadow=VISUALIZER_RECEIVE_SHADOW, ) handle = self.scene_handles[name] handle.visible = True @@ -1283,6 +1471,44 @@ def _hide_prefix(self, prefix: str) -> None: if name.startswith(prefix): handle.visible = False + @staticmethod + def _side_highlight_rows( + segments: list[Segment], + taxels: list[Taxel], + selected_kind: str | None, + selected_rows: list[int], + ) -> tuple[set[int], set[int]]: + segment_rows: set[int] = set() + taxel_rows: set[int] = set() + if selected_kind == "segments": + selected_joints = { + segments[row].joint for row in selected_rows if 0 <= row < len(segments) + } + taxel_rows = { + row + for row, taxel in enumerate(taxels) + if any( + joint in selected_joints + and type(weight) in (int, float) + and weight > 0 + for joint, weight in zip(taxel.segments, taxel.weights) + ) + } + elif selected_kind == "taxels": + bound_joints = { + joint + for row in selected_rows + if 0 <= row < len(taxels) + for joint, weight in zip(taxels[row].segments, taxels[row].weights) + if joint is not None and type(weight) in (int, float) and weight > 0 + } + segment_rows = { + row + for row, segment in enumerate(segments) + if segment.joint in bound_joints + } + return segment_rows, taxel_rows + def _draw_dataset( self, prefix: str, @@ -1292,42 +1518,66 @@ def _draw_dataset( active: bool, forces=None, ) -> None: + side_segment_rows, side_taxel_rows = ( + self._side_highlight_rows( + segments, taxels, self.selected_kind, self.selected_rows + ) + if active + else (set(), set()) + ) segment_colors = [] for row in range(len(segments)): - color = (245, 200, 45) if active else (130, 130, 130) + color = SEGMENT_COLOR if active else INACTIVE_COLOR + if row in side_segment_rows: + color = SIDE_HIGHLIGHT_COLOR if ( self.selected_kind == "segments" and row in self.selected_rows and active ): - color = (255, 120, 20) + color = SELECTED_COLOR if self.binding_phase == "segments" and row in self.binding_segments: - color = (255, 120, 20) + color = SELECTED_COLOR if self.binding_phase == "taxels" and active: - color = (130, 130, 130) + color = INACTIVE_COLOR if self.pick_mode is not None and active: - color = (130, 130, 130) + color = INACTIVE_COLOR segment_colors.append(color) - taxel_colors = [(190, 190, 190)] * len(taxels) + taxel_colors = [TAXEL_COLOR] * len(taxels) array = np.asarray(forces, dtype=float) if forces is not None else None if ( array is not None and array.shape == (len(taxels), 3) and np.isfinite(array).all() ): - fraction = np.clip(np.linalg.norm(array, axis=1) / 30.0, 0.0, 1.0) + fraction = np.clip(np.linalg.norm(array, axis=1) / FORCE_MAX_N, 0.0, 1.0) taxel_colors = [ - (int(255 * value), 0, int(255 * (1.0 - value))) for value in fraction + tuple( + int(low + value * (high - low)) + for low, high in zip(FORCE_MIN_COLOR, FORCE_MAX_COLOR) + ) + for value in fraction ] for row in range(len(taxels)): + if row in side_taxel_rows: + taxel_colors[row] = SIDE_HIGHLIGHT_COLOR if self.selected_kind == "taxels" and row in self.selected_rows and active: - taxel_colors[row] = (255, 120, 20) + taxel_colors[row] = SELECTED_COLOR if self.binding_phase == "taxels" and row in self.binding_taxels: - taxel_colors[row] = (255, 120, 20) + taxel_colors[row] = SELECTED_COLOR if self.binding_phase == "segments" and active: - taxel_colors[row] = (130, 130, 130) + taxel_colors[row] = INACTIVE_COLOR if self.pick_mode is not None and active: - taxel_colors[row] = (130, 130, 130) + taxel_colors[row] = INACTIVE_COLOR + if prefix == "/edit": + error_rows = { + issue["row"] + for issue in self.document.validate() + if issue["kind"] == "taxels" and issue["row"] is not None + } + for row in error_rows: + if 0 <= row < len(taxel_colors): + taxel_colors[row] = ERROR_COLOR self._draw_instances( prefix + "/segments", "segments", segments, segment_colors, opacity ) @@ -1335,70 +1585,32 @@ def _draw_dataset( prefix + "/taxels", "taxels", taxels, taxel_colors, opacity ) - def _draw_errors(self) -> None: - segment_rows = { - issue["row"] - for issue in self.document.validate() - if issue["kind"] == "segments" and issue["row"] is not None - } - taxel_rows = { - issue["row"] - for issue in self.document.validate() - if issue["kind"] == "taxels" and issue["row"] is not None - } - self._draw_instances( - "/errors/segments", - "segments", - [self.document.segments[row] for row in sorted(segment_rows)], - [(255, 0, 0)] * len(segment_rows), - 1.0, - True, - ) - self._draw_instances( - "/errors/taxels", - "taxels", - [self.document.taxels[row] for row in sorted(taxel_rows)], - [(255, 0, 0)] * len(taxel_rows), - 1.0, - True, - ) - - def _draw_selection_axes(self) -> None: + def _draw_selection_gizmo(self) -> None: values = self._selection_values() rows = [row for row in self.selected_rows if 0 <= row < len(values)] if not rows: - if "/selection/axes" in self.scene_handles: - self.scene_handles["/selection/axes"].visible = False + self.selection_gizmo.visible = False + self.selection_gizmo_dragging = False + self.selection_gizmo_last_position = None return - if len(rows) == 1: - frames = [geo.matrix(values[rows[0]].pose)] - else: - frame = np.eye(4) - frame[:3, 3] = np.asarray([values[row].pose[:3] for row in rows]).mean( - axis=0 - ) - frames = [frame] - positions = np.asarray([frame[:3, 3] for frame in frames]) - rotations = np.asarray([tf.quaternion_from_matrix(frame) for frame in frames]) - if "/selection/axes" not in self.scene_handles: - self.scene_handles["/selection/axes"] = self.server.scene.add_batched_axes( - "/selection/axes", - rotations, - positions, - axes_length=0.03, - axes_radius=0.001, - ) - handle = self.scene_handles["/selection/axes"] - handle.visible = True - handle.batched_positions = positions - handle.batched_wxyzs = rotations + self.selection_gizmo.visible = True + if self.selection_gizmo_dragging: + return + self.selection_gizmo.position = np.asarray( + [values[row].pose[:3] for row in rows] + ).mean(axis=0) + self.selection_gizmo.wxyz = ( + tf.quaternion_from_matrix(geo.matrix(values[rows[0]].pose)) + if len(rows) == 1 + else np.array([1.0, 0.0, 0.0, 0.0]) + ) def _draw(self) -> None: - active_opacity = 1.0 + active_opacity = ACTIVE_DATASET_OPACITY if self.model is not None: q = ( self.document.q - if self.mode.value == "Edit" + if self.mode_value == "Edit" else self.preview_document.q ) self.model.draw( @@ -1407,9 +1619,9 @@ def _draw(self) -> None: self.urdf_alignment, self.urdf_scale, highlighted=self.pick_mode is not None, - opacity=geo.URDF_OPACITY * active_opacity, + opacity=URDF_OPACITY * active_opacity, ) - if self.mode.value == "Edit": + if self.mode_value == "Edit": self._hide_prefix("/preview") self._draw_dataset( "/edit", @@ -1418,18 +1630,12 @@ def _draw(self) -> None: active_opacity, True, ) - self._draw_errors() else: - self._hide_prefix("/errors") try: - preview_rows, _ = self._preview_taxels() + valid_rows, _, source_count = self._rendered_preview_taxels() except (ValueError, TypeError, KeyError, IndexError): - preview_rows = [] - valid_rows = [ - (row, taxel) - for row, taxel in enumerate(preview_rows) - if taxel is not None - ] + valid_rows = [] + source_count = 0 preview_taxels = [taxel for _, taxel in valid_rows] raw_forces = ( self.receiver.latest.get("forces") @@ -1442,7 +1648,7 @@ def _draw(self) -> None: ) forces = ( [raw_forces[row] for row, _ in valid_rows] - if raw_forces is not None and len(raw_forces) == len(preview_rows) + if raw_forces is not None and len(raw_forces) == source_count else None ) self._draw_dataset( @@ -1458,17 +1664,23 @@ def _draw(self) -> None: "/edit", self.document.segments, self.document.taxels, - 0.5, + BINDING_OVERLAY_OPACITY, False, ) else: self._hide_prefix("/edit") saved = self.state.saved[self.state.side] if self.show_saved.value and saved is not None: - self._draw_dataset("/saved", saved.segments, saved.taxels, 0.2, False) + self._draw_dataset( + "/saved", + saved.segments, + saved.taxels, + SAVED_BINDING_OPACITY, + False, + ) else: self._hide_prefix("/saved") - self._draw_selection_axes() + self._draw_selection_gizmo() def run(self) -> None: try: diff --git a/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js b/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js new file mode 100644 index 0000000000..ea144aa295 --- /dev/null +++ b/scripts/tactile_metadata/tactile_metadata_editor/viewport_axes.js @@ -0,0 +1,148 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 +(() => { + "use strict"; + + const key = "__tactileMetadataViewportAxes"; + globalThis[key]?.stop(); + + const ns = "http://www.w3.org/2000/svg"; + const svg = document.createElementNS(ns, "svg"); + Object.assign(svg.style, { + position: "absolute", + top: "16px", + right: "16px", + width: "112px", + height: "112px", + zIndex: "10", + pointerEvents: "none", + overflow: "visible", + }); + svg.setAttribute("viewBox", "0 0 112 112"); + + const background = document.createElementNS(ns, "circle"); + background.setAttribute("cx", "56"); + background.setAttribute("cy", "56"); + background.setAttribute("r", "49"); + background.setAttribute("fill", "rgba(255,255,255,0.72)"); + background.setAttribute("stroke", "rgba(0,0,0,0.2)"); + svg.appendChild(background); + + const axes = [ + ["x", "#e53935", [1, 0, 0]], + ["y", "#43a047", [0, 1, 0]], + ["z", "#1e88e5", [0, 0, 1]], + ].map(([name, color, vector]) => { + const line = document.createElementNS(ns, "line"); + line.setAttribute("x1", "56"); + line.setAttribute("y1", "56"); + line.setAttribute("stroke", color); + line.setAttribute("stroke-width", "3"); + line.setAttribute("stroke-linecap", "round"); + svg.appendChild(line); + + const end = document.createElementNS(ns, "circle"); + end.setAttribute("r", "4"); + end.setAttribute("stroke", color); + end.setAttribute("stroke-width", "2"); + svg.appendChild(end); + + const label = document.createElementNS(ns, "text"); + label.textContent = name; + label.setAttribute("fill", color); + label.setAttribute("font-size", "15"); + label.setAttribute("font-family", "sans-serif"); + label.setAttribute("font-weight", "700"); + label.setAttribute("text-anchor", "middle"); + label.setAttribute("dominant-baseline", "central"); + svg.appendChild(label); + return { color, end, label, line, vector }; + }); + + const multiply = (a, b) => [ + a[3] * b[0] + a[0] * b[3] + a[1] * b[2] - a[2] * b[1], + a[3] * b[1] - a[0] * b[2] + a[1] * b[3] + a[2] * b[0], + a[3] * b[2] + a[0] * b[1] - a[1] * b[0] + a[2] * b[3], + a[3] * b[3] - a[0] * b[0] - a[1] * b[1] - a[2] * b[2], + ]; + + const rotate = (q, v) => { + const tx = 2 * (q[1] * v[2] - q[2] * v[1]); + const ty = 2 * (q[2] * v[0] - q[0] * v[2]); + const tz = 2 * (q[0] * v[1] - q[1] * v[0]); + return [ + v[0] + q[3] * tx + q[1] * tz - q[2] * ty, + v[1] + q[3] * ty + q[2] * tx - q[0] * tz, + v[2] + q[3] * tz + q[0] * ty - q[1] * tx, + ]; + }; + + const state = { + controls: null, + raf: 0, + stopped: false, + stop() { + this.stopped = true; + cancelAnimationFrame(this.raf); + this.controls?.removeEventListener("update", draw); + svg.remove(); + }, + }; + globalThis[key] = state; + + const draw = () => { + const mutable = globalThis.__viserMutable; + const camera = mutable?.camera; + const root = mutable?.nodeRefFromName?.[""]; + const parent = mutable?.canvas?.parentElement; + if (!camera || !root || !parent) return; + if (svg.parentElement !== parent) parent.appendChild(svg); + + const cameraQuaternion = camera.quaternion; + const rootQuaternion = root.quaternion; + const cameraFromWorld = [ + -cameraQuaternion.x, + -cameraQuaternion.y, + -cameraQuaternion.z, + cameraQuaternion.w, + ]; + const cameraFromVisualization = multiply(cameraFromWorld, [ + rootQuaternion.x, + rootQuaternion.y, + rootQuaternion.z, + rootQuaternion.w, + ]); + + for (const [index, axis] of axes.entries()) { + const direction = rotate(cameraFromVisualization, axis.vector); + const x = 56 + 36 * direction[0]; + const y = 56 - 36 * direction[1]; + const projected = Math.hypot(direction[0], direction[1]); + const fallback = [[10, -8], [-10, -8], [0, 12]][index]; + const labelOffset = + projected > 0.12 + ? [8 * direction[0] / projected, -8 * direction[1] / projected] + : fallback; + axis.line.setAttribute("x2", x.toFixed(2)); + axis.line.setAttribute("y2", y.toFixed(2)); + axis.end.setAttribute("cx", x.toFixed(2)); + axis.end.setAttribute("cy", y.toFixed(2)); + axis.end.setAttribute("fill", direction[2] >= 0 ? axis.color : "white"); + axis.label.setAttribute("x", (x + labelOffset[0]).toFixed(2)); + axis.label.setAttribute("y", (y + labelOffset[1]).toFixed(2)); + } + }; + + const frame = () => { + if (state.stopped) return; + const controls = globalThis.__viserMutable?.cameraControl; + if (controls !== state.controls) { + state.controls?.removeEventListener("update", draw); + controls?.addEventListener("update", draw); + state.controls = controls; + } + draw(); + state.raf = requestAnimationFrame(frame); + }; + frame(); +})(); From d30373d3ee3019dc49dcf25a0075eba0feed8a51 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 17:43:04 -0700 Subject: [PATCH 20/36] draft Signed-off-by: Lotus Li --- .../editor_interactions.js | 44 ++++++ .../tactile_metadata_editor/geometry.py | 4 +- .../tactile_metadata_editor/ui.py | 144 +++++++++++------- 3 files changed, 137 insertions(+), 55 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index 5068f0fe90..953b442805 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -114,6 +114,49 @@ return child?.parentElement === parent ? child : null; }; + const layoutTaxelBindings = () => { + const names = [ + "Segment 0", + "Segment 1", + "Segment 2", + "Segment 3", + "Weight 0", + "Weight 1", + "Weight 2", + "Weight 3", + ]; + const clean = (label) => label.textContent?.trim().replace(/^⚠\s*/, ""); + for (const first of document.querySelectorAll("label")) { + if (clean(first) !== names[0]) continue; + let container = first.parentElement; + let fields = []; + while (container) { + const labels = [...container.querySelectorAll("label")]; + fields = names.map((name) => labels.find((label) => clean(label) === name)); + if (fields.every(Boolean)) break; + container = container.parentElement; + } + if (!container) continue; + const children = fields.map((field) => directChild(container, field)); + if (children.some((child) => !child) || new Set(children).size !== names.length) { + continue; + } + const bindingChildren = new Set(children); + Object.assign(container.style, { + display: "grid", + gridTemplateColumns: "repeat(4, minmax(0, 1fr))", + gap: "8px", + alignItems: "end", + }); + // Only bindings use the four columns; all other taxel controls stay full-width. + for (const child of container.children) { + child.style.gridColumn = bindingChildren.has(child) ? "auto" : "1 / -1"; + if (bindingChildren.has(child)) child.style.minWidth = "0"; + } + for (const field of fields) field.style.display = "none"; + } + }; + const freezeMainMenu = () => { const general = folderFromLabel("General"); const preview = folderFromLabel("Preview"); @@ -182,6 +225,7 @@ boldChoiceLabels(); freezeMainMenu(); preservePartialNumbers(); + layoutTaxelBindings(); for (const label of document.querySelectorAll("label")) { if (!label.textContent?.trim().startsWith("⚠")) continue; const field = label.parentElement?.querySelector("input, button"); diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index 018543f966..e982562d75 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -339,7 +339,7 @@ def deform( return result, errors -def hand_segments(hand: dict) -> list[Segment]: +def hand_segments(hand: dict, alignment: np.ndarray) -> list[Segment]: values = [] for position, orientation, valid in zip( hand["positions"], hand["orientations"], hand["is_valid"] @@ -351,7 +351,7 @@ def hand_segments(hand: dict) -> list[Segment]: return [] wrist_inverse = np.linalg.inv(values[1]) return [ - Segment(joint, pose(VISUALIZATION_FROM_HAND @ wrist_inverse @ tracked)) + Segment(joint, pose(alignment @ wrist_inverse @ tracked)) for joint, tracked in zip(JOINTS, values) if tracked is not None ] diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 30cfb49693..24d6a277c8 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -269,6 +269,10 @@ def _build_gui(self) -> None: self.joint_mapping = gui.add_text( "Joint Mappings JSON", "{}", multiline=True, disabled=True ) + gui.add_markdown( + 'Format: `{"source_joint": ["urdf_joint", scale, shift]}`; ' + "`value = raw * scale + shift`." + ) self._watch(self.joint_enabled, self._joint_enabled) self.connect_button = self._action_button("Connect Live", self._connect) gui.add_markdown("Live input overrides values in the Static section.") @@ -509,8 +513,30 @@ def _update_segment_alignment(self) -> None: delta, ) self.live_document.alignment = alignment.tolist() - self._rebuild_edit_lists() - self._rebuild_preview_segments() + for segment, (_, positions, rotations) in zip( + self.document.segments, self.segment_widgets + ): + position, rotation = geo.components(geo.matrix(segment.pose)) + self._set_values(positions, position) + self._set_values(rotations, rotation) + for taxel, (positions, rotations, _) in zip( + self.document.taxels, self.taxel_widgets + ): + position, rotation = geo.components(geo.matrix(taxel.pose)) + self._set_values(positions, position) + self._set_values(rotations, rotation) + self._sync_preview_values() + values = self._selection_values() + rows = [row for row in self.selected_rows if 0 <= row < len(values)] + if rows: + center = np.asarray([values[row].pose[:3] for row in rows]).mean(axis=0) + self._set_values(self.selection_position_widgets, center) + if len(rows) == 1: + _, rotation = geo.components(geo.matrix(values[rows[0]].pose)) + self._set_values(self.selection_rotation_widgets, rotation) + self.selection_rotation = np.asarray(rotation, dtype=float) + self._update_preview_error() + self.redraw = True def _rebuild_joint_controls(self) -> None: if self.edit_joint_folder is not None: @@ -549,16 +575,13 @@ def _add_joint_sliders(self, document: Document, output: dict) -> None: else math.pi ) value = min(upper, max(lower, document.q.get(joint.name, 0.0))) - with self.server.gui.add_folder( - f"{index}: {joint.name}", expand_by_default=False - ): - widget = self.server.gui.add_slider( - "Value", - float(lower), - float(upper), - 0.001, - float(value), - ) + widget = self.server.gui.add_slider( + f"{index}: {joint.name}", + float(lower), + float(upper), + 0.001, + float(value), + ) self._watch( widget, lambda current, name=joint.name, document=document: ( @@ -714,46 +737,47 @@ def _rebuild_taxel_list(self) -> None: ), ) slots = [] + segment_fields = [] for slot in range(4): - with self.server.gui.add_folder( - f"Binding {slot}", expand_by_default=False - ): - selected = ( - taxel.segments[slot] - if taxel.segments[slot] in options - else "None" - ) - segment = self.server.gui.add_dropdown( - "⚠ Segment" - if any( - issue["field"] in (f"segment{slot}", "binding") - for issue in row_issues - ) - else "Segment", - options, - initial_value=selected, - ) - weight = self._weight_widget( - taxel.weights[slot], - selected == "None", - any( - issue["field"] in (f"weight{slot}", "weights") - for issue in row_issues - ), - ) - self._watch( - segment, - lambda value, row=row, slot=slot: ( - self._set_taxel_segment(row, slot, value) - ), - ) - self._watch( - weight, - lambda value, row=row, slot=slot: ( - self._set_taxel_weight(row, slot, value) - ), + selected = ( + taxel.segments[slot] + if taxel.segments[slot] in options + else "None" + ) + segment = self.server.gui.add_dropdown( + f"⚠ Segment {slot}" + if any( + issue["field"] in (f"segment{slot}", "binding") + for issue in row_issues ) - slots.append((segment, weight)) + else f"Segment {slot}", + options, + initial_value=selected, + ) + segment_fields.append((segment, selected)) + for slot, (segment, selected) in enumerate(segment_fields): + weight = self._weight_widget( + taxel.weights[slot], + selected == "None", + any( + issue["field"] in (f"weight{slot}", "weights") + for issue in row_issues + ), + slot, + ) + self._watch( + segment, + lambda value, row=row, slot=slot: ( + self._set_taxel_segment(row, slot, value) + ), + ) + self._watch( + weight, + lambda value, row=row, slot=slot: ( + self._set_taxel_weight(row, slot, value) + ), + ) + slots.append((segment, weight)) messages = [issue["message"] for issue in row_issues] if messages: self.server.gui.add_html( @@ -763,8 +787,8 @@ def _rebuild_taxel_list(self) -> None: ) self.taxel_widgets.append((positions, rotations, slots)) - def _weight_widget(self, value, disabled: bool, invalid: bool): - label = "⚠ Weight" if invalid else "Weight" + def _weight_widget(self, value, disabled: bool, invalid: bool, slot: int): + label = f"{'⚠ ' if invalid else ''}Weight {slot}" if type(value) in (int, float) and math.isfinite(value): limits = {"min": 0.0, "max": 1.0} if 0.0 <= value <= 1.0 else {} return self.server.gui.add_number( @@ -1035,7 +1059,9 @@ def _update_live(self) -> None: try: hand = frame.get("hand") if hand is not None: - updated = geo.hand_segments(hand) + updated = geo.hand_segments( + hand, np.asarray(self.document.alignment, dtype=float) + ) joints = [segment.joint for segment in updated] signature = [ segment.joint for segment in self.preview_document.segments @@ -1309,6 +1335,18 @@ def _rebuild_selection(self) -> None: f"Type: **{self.selected_kind if rows else 'None'}** \n" f"Count: **{len(rows)}**" ) + names = [ + values[row].joint + if self.selected_kind == "segments" + else f"Taxel {row}" + for row in rows + ] + self.server.gui.add_text( + "Name", + names[0] if len(names) == 1 else json.dumps(names, indent=2), + multiline=True, + disabled=True, + ) center = ( np.asarray([values[row].pose[:3] for row in rows]).mean(axis=0) if rows From 702d9c9b08aac53623f0a8bebf3a1d1c69035000 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 18:52:27 -0700 Subject: [PATCH 21/36] multiple bugfixing and enhancement Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 11 +- .../tactile_metadata/IMPLEMENTATION_PROMPT.md | 2 +- scripts/tactile_metadata/README.md | 19 +- scripts/tactile_metadata/UI&Feature Design.md | 9 +- .../tactile_metadata_editor/document.py | 27 +- .../editor_interactions.js | 176 +++--- .../tactile_metadata_editor/ui.py | 539 +++++++++++++----- 7 files changed, 541 insertions(+), 242 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 6cb6b470a9..0abdfb2e21 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -20,20 +20,19 @@ SPDX-License-Identifier: Apache-2.0 - Preview apply operations remain independent and available despite taxel errors. Error reasons persist in the fixed Info area after disconnects and unrelated Preview actions. - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. - New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. -- Preserve slot order and repeated segment indices. Normalize all four weights only on explicit user weight edits and reject negative edit values; loading, recording received values, checking, saving, and exporting preserve raw weights. Distinguish receiver failures from sender behavior, and keep all editor operations local. +- Preserve slot order and repeated segment indices. The first selected Segment receives weight 1 and later selections receive 0; manual weight edits change only that slot. Reject values outside 0~1, and let validation report totals other than 1 without normalization. - SDK process isolation specifies where SDK classes are called; document the actual tracker/session calls required for live reception as well as export encoding. - Keep tool implementation inside this subtree; add missing dependencies to Isaac Capture's existing requirements and matching package metadata. Run checks only from this directory or its descendants; never run repository-wide fixing hooks that can change other directories. - Explain this standalone tool's setup from the user's prerequisites: define external libraries before packaging terms, and distinguish requirement files, generated version records, and installation commands. Identify inspected environments by their Python executable path; distinguish declared requirements from packages actually installed there. - Reuse the launching Python environment and its SDK through sys.executable. Do not create tool-specific project metadata, lockfiles, environments or SDK installations, and do not infer runtime requirements from old local build artifacts. - Missing packages in the agent's inspection environment limit verification; report the unavailable checks. Do not turn verification requirements into proposals to install dependencies, provision environments, or substitute interpreters; maintain Isaac Capture's shared dependency declarations instead. -- Use Viser's standard controls and public callbacks, with document changes serialized through State. Apply the same explicit weight-normalization rule to sliders and numeric fields. +- Use Viser's standard controls and public callbacks, with document changes serialized through State. Segment and weight fields in Add Taxels and Taxel rows share the same independent-slot behavior. - The selected-object transform control enables only XYZ axis translation. Serialize drag phases through the event queue; single selections use object axes, multi-selections use visualization-space axes, and the same translation delta applies to every selected object. -- Selection side-highlighting resolves bindings by HandJoint name and includes only slots with positive weights. Keep `SIDE_HIGHLIGHT_COLOR` distinct from `SELECTED_COLOR`; selection color wins on selected objects and validation error color wins on invalid taxels. +- Selection side-highlighting resolves bindings by HandJoint name and includes only slots with positive weights. Keep `SIDE_HIGHLIGHT_COLOR` distinct from `SELECTED_COLOR`; selection color overrides validation error color on selected taxels. - When product placement says top/before/below and any sibling uses explicit Viser `order`, assign explicit order to every constrained sibling; source creation order does not override mixed default and explicit ordering. -- Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. UI sections marked `(spannable)` in the product design and all nested entity/joint/binding folders start collapsed. +- Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. All Segments/Taxels lists, their entity folders, and Preview Segments start collapsed. - Client-side helpers may hide only their exact bridge input and label; never hide an ancestor container that can contain the entire Viser control panel. -- Preserve focused numeric drafts such as `1.` client-side because Viser ignores partial numeric values and controlled rerenders otherwise erase the decimal point before the user finishes typing. Restore through the native input value setter and dispatch an `input` event; a raw DOM assignment leaves React/Mantine state stale and breaks the next digit. -- The right menu is a flex column: General is fixed, the Edit tab group fills the remaining height, and only the active tab body scrolls. Scope DOM styling to the root containing the General folder and URDF/Segments/Taxels dock group. +- Preserve focused formatting-sensitive numeric drafts such as `0.0` and `0.` client-side because Viser canonicalizes them to floats during controlled rerenders. Restore through the native input value setter and dispatch an `input` event; a raw DOM assignment leaves React/Mantine state stale and breaks the next digit. - Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. - Call `/edit`, `/preview`, and `/saved` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. - During live redraw, update the active scene-node group in place. Hide only groups that should be inactive; never hide and immediately re-show the active group each frame. diff --git a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md index 1fe9a0c442..45bc6065ff 100644 --- a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md +++ b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md @@ -20,7 +20,7 @@ When implementation, make sure to only modify folder /Users/lotusl/Codes/IsaacTe 协议。 The following product decisions are settled and must be reflected throughout the implementation: -1. Weight editing: after a slider or numeric-field edit, replace that slot and normalize all four weights by their sum, including the edited slot. Reject negative weights. Reject an edit whose resulting row contains non-finite or negative values or has a zero sum, preserving the previous row. Apply normalization independently to each selected taxel. Preserve slot order and repeated indices. Loading, explicitly recording received metadata, checking, saving, and exporting preserve raw weights and report invalid values without silently normalizing them. +1. Weight editing: selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. A slider or numeric-field edit replaces only that slot and never modifies other weights. Reject non-finite values and values outside 0~1. Preserve slot order and repeated indices. Loading, explicitly recording received metadata, checking, saving, and exporting preserve raw weights; validation reports rows whose weights do not sum to 1. 2. Metadata persistence: saving, backups, and export write only tactile metadata values using the fields, nested structure, and union type discriminators defined by src/core/schema/fbs/tactile_metadata.fbs. The only top-level JSON fields are geometry_mapping_type, geometry_mapping, segment_source_type, and segment_source. Use one schema JSON representation for saving, reopening, and JSON export; optional SDK binaries encode the same values. Do not add an editor extension, tool-specific fields, project/configuration files, .editor.json, internal-ID correspondence tables, or saved check reports. Read with json.load(), perform the array and enum conversion required by the existing SDK constructors, and encode through LBS, HandSkeletonRef, TactileMetadata, and to_bytes() in the isolated SDK subprocess. Do not invent another metadata format or treat this conversion as an unresolved product decision. 3. Scope: metadata/force alignment is the producer's and user's responsibility. Display the current arrays in protocol row order and retain structural, count, and validity checks. Do not maintain source-row mappings, alignment baselines, or force permutations to compensate for metadata edits. Switching visualization sources does not establish or repair force alignment. Persist taxel output order through array row order, without saving internal IDs or temporary numbering. 4. URDF display: all links use fixed opacity 0.6. Do not add per-link visibility or opacity controls or state. Highlight associated links through color without changing opacity; retain mode-level model visibility. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index d0aecf51a4..e570b75519 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -109,24 +109,29 @@ invalid and outlined in red. Set Rows, Columns, Area Width, Area Height, Center Position, and Rotation. **Capture URDF Surface Pose** copies a clicked surface position and orientation -into those fields without creating taxels. **Add Taxels** then creates the -row-major planar grid. Taxels are squares with side length 0.002 m. A taxel with -any validation error uses `ERROR_COLOR` as its fill color. +into those fields without creating taxels. **Capture Segment Pose** copies the +complete pose of a clicked Editing segment. **Add Taxels** then creates the +row-major planar grid and copies the four Segment/Weight fields above the +capture buttons into every new Taxel. Taxels are squares with side length +0.002 m. A taxel with any validation error uses `ERROR_COLOR`; +`SELECTED_COLOR` overrides it while that taxel is selected. To bind taxels: 1. Select **Start Binding to Segments** and select taxels in the scene. 2. Select **Select Segments** and choose one to four segments. -3. Select **Complete**. Initial active weights are equal. +3. Select **Complete**. The first Segment receives weight one; later Segments + receive weight zero. The three buttons are vertical with down arrows between adjacent steps, and at most one is enabled at a time. Between steps, only the required 3D selection is available. Expand **All Taxels** to edit pose, four segment slots, and four weights. `None` -is an editor-only inactive slot with weight zero. Editing a weight preserves the -entered value and adjusts other active weights proportionally so the row sums to -one. If only one slot is active, its weight is one. +is an editor-only inactive slot with weight zero. Selecting the first active +segment sets its weight to one; later segment selections start at zero. Editing +a weight changes only that slot. Validation reports rows whose weights do not +sum to one. ## Scene selection diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index d8e8809e61..14d8da802a 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -51,16 +51,15 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula 9. Taxels tab: * Taxels don't have their onw alignment, it uses the alignment of the segments. - a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), and rotation (3 float sliders). "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields; newly created taxels are selected and previous objects are deselected. - c) Vertically align "Start Binding to Segments", "Select Segments", and "Complete", with a centered down arrow between each adjacent pair. At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The scene instructions and equal initial weights otherwise follow the same three-step binding flow. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. + a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), rotation (3 float sliders), and a left-labeled four-slot Segment/Weight binding grid matching each Taxel row. Each binding grid has no gaps between cells. Leave standard vertical spacing between this grid and "Capture URDF Surface Pose" without a visible spacer element. "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. "Capture Segment Pose" does the same using the clicked Editing segment's complete pose. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields and copies the binding grid into every new Taxel; newly created taxels are selected and previous objects are deselected. + c) Vertically align "Start Binding to Segments", "Select Segments", and "Complete", with a centered down arrow between each adjacent pair. At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The first selected Segment receives weight 1 and later selected Segments receive weight 0. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. - d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. After modifying a weight here, automatically update the other weights for the same taxel by their proportion to sum to 1 (unify them without changing just edited value. If all other weights are 0, treat them as equal proportion). Weight field reject numbers out of 0~1. If the modified weight is the only weight, adjust it to 1. + d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. Selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. Editing a weight changes only that slot and never modifies the other weights. Weight fields reject numbers out of 0~1, and row validation reports when the total is not 1. * "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. * Taxels data can be invalid for multiple reasons. When invalid, line the invalid field in red. Reasons including: a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. b) the weights don't sum to 1. (Can happen when a segment is removed from Segments tab or by Preview apply to editor.) - weights input fields. - c) the weight is 0 for a non-None segment - weight input field. -* Taxels are displayed as flat squares of size 0.002m*0.002m. A taxel with any validation error uses ERROR_COLOR as its square fill color. The three axes of each taxel are not displayed by default. Clicking a taxel on the 3D scene while in Edit mode will select the taxel, highlight it, and reveal its 3-axis translation control. The 3D scene modification rules are the same as segments. +* Taxels are displayed as flat squares of size 0.002m*0.002m. A taxel with any validation error uses ERROR_COLOR unless selected; selection uses SELECTED_COLOR with higher priority. The three axes of each taxel are not displayed by default. Clicking a taxel on the 3D scene while in Edit mode will select the taxel, highlight it, and reveal its 3-axis translation control. The 3D scene modification rules are the same as segments. * Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. * Selection also highlights related objects with the separate configured `SIDE_HIGHLIGHT_COLOR`: selecting segment(s) highlights taxels with a positive binding weight to any selected HandJoint; selecting taxel(s) highlights segments referenced by a positive binding weight. Zero-weight slots never produce this highlight, and selected objects retain `SELECTED_COLOR`. * The 3-axis translation control is draggable only along its X/Y/Z axes and moves every selected object by the same translation. For one object it follows that object's orientation; for multiple objects it is centered on the selection with axes aligned to visualization-space coordinates. Rotation, plane translation, and scaling handles are disabled. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index fb93f5961b..418c9366b8 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -117,36 +117,23 @@ def assign(self, taxels: list[int], segment_rows: list[int]) -> None: if not 0 <= row < len(self.segments): raise ValueError("Selected segment no longer exists") joints.append(self.segments[row].joint) - weights = [1.0 / len(joints)] * len(joints) + weights = [1.0] + [0.0] * (len(joints) - 1) for row in taxels: if 0 <= row < len(self.taxels): self.taxels[row].segments = joints + [None] * (4 - len(joints)) self.taxels[row].weights = weights + [0.0] * (4 - len(joints)) def set_weight(self, taxel_row: int, slot: int, value: float) -> None: - taxel = self.taxels[taxel_row] + self.set_taxel_weight(self.taxels[taxel_row], slot, value) + + @staticmethod + def set_taxel_weight(taxel: Taxel, slot: int, value: float) -> None: if taxel.segments[slot] is None: raise ValueError("Select a segment before editing its weight") value = float(value) if not math.isfinite(value) or not 0.0 <= value <= 1.0: raise ValueError("Weight must be from 0 to 1") - active = [i for i, segment in enumerate(taxel.segments) if segment is not None] - if len(active) == 1: - taxel.weights = [1.0 if i == slot else 0.0 for i in range(4)] - return - others = [i for i in active if i != slot] - remainder = 1.0 - value - total = sum(taxel.weights[i] for i in others) taxel.weights[slot] = value - for i in range(4): - if taxel.segments[i] is None: - taxel.weights[i] = 0.0 - if total > 0.0: - for i in others: - taxel.weights[i] = taxel.weights[i] / total * remainder - else: - for i in others: - taxel.weights[i] = remainder / len(others) def validate(self) -> list[dict]: issues: list[dict] = [] @@ -182,12 +169,12 @@ def issue(kind: str, row: int | None, field: str, message: str) -> None: ) elif joint not in joints: issue("taxels", row, f"segment{slot}", f"Missing segment {joint}") - elif not valid_weight or weight <= 0.0: + elif not valid_weight or weight < 0.0: issue( "taxels", row, f"weight{slot}", - "Selected segment weight must be greater than zero", + "Selected segment weight must be nonnegative", ) if not any( joint in joints for joint in taxel.segments if joint is not None diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index 953b442805..b968ef2dcc 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -17,26 +17,24 @@ )?.set; const findBridge = (name) => { - for (const input of document.querySelectorAll("input")) { - let parent = input; - for (let depth = 0; parent && depth < 5; ++depth, parent = parent.parentElement) { - if (parent.textContent?.includes(name)) { - input.style.position = "absolute"; - input.style.width = "1px"; - input.style.height = "1px"; - input.style.opacity = "0"; - input.style.pointerEvents = "none"; - input.tabIndex = -1; - for (const label of parent.querySelectorAll("label")) { - if (label.textContent?.includes(name)) { - label.style.display = "none"; - } - } - return input; - } - } + const label = [...document.querySelectorAll("label")].find( + (candidate) => candidate.textContent?.trim() === name, + ); + let input = label?.htmlFor ? document.getElementById(label.htmlFor) : null; + let row = label?.parentElement; + while (!input && row) { + input = row.querySelector("input"); + row = row.parentElement; } - return null; + if (!(input instanceof HTMLInputElement)) return null; + input.style.position = "absolute"; + input.style.width = "1px"; + input.style.height = "1px"; + input.style.opacity = "0"; + input.style.pointerEvents = "none"; + input.tabIndex = -1; + label.style.display = "none"; + return input; }; // Place Viser's logo above the fixed bottom Info area. @@ -99,13 +97,6 @@ } }; - const folderFromLabel = (name) => { - const label = [...document.querySelectorAll("div")].find( - (element) => element.textContent?.trim() === name && element.querySelector("svg"), - ); - return label?.parentElement || null; - }; - const directChild = (parent, descendant) => { let child = descendant; while (child?.parentElement && child.parentElement !== parent) { @@ -114,6 +105,36 @@ return child?.parentElement === parent ? child : null; }; + const layoutSelectedNames = () => { + for (const label of document.querySelectorAll("label")) { + if (label.textContent?.trim() !== "__Selected Object Names") continue; + let row = label.parentElement; + while (row && !row.querySelector("textarea")) row = row.parentElement; + if (!row) continue; + const textarea = row.querySelector("textarea"); + const labelColumn = directChild(row, label); + const valueColumn = directChild(row, textarea); + if (!labelColumn || !valueColumn) continue; + labelColumn.style.display = "none"; + valueColumn.style.flex = "1 1 100%"; + valueColumn.style.width = "100%"; + row.style.width = "100%"; + if (textarea.dataset.tactileFixedRows !== "true") { + const style = getComputedStyle(textarea); + const fontSize = Number.parseFloat(style.fontSize); + const lineHeight = + Number.parseFloat(style.lineHeight) || (Number.isFinite(fontSize) ? fontSize * 1.5 : 20); + const chrome = ["paddingTop", "paddingBottom", "borderTopWidth", "borderBottomWidth"] + .map((name) => Number.parseFloat(style[name]) || 0) + .reduce((sum, value) => sum + value, 0); + textarea.rows = 2; + textarea.style.height = `${Math.ceil(2 * lineHeight + chrome)}px`; + textarea.style.overflowY = "auto"; + textarea.dataset.tactileFixedRows = "true"; + } + } + }; + const layoutTaxelBindings = () => { const names = [ "Segment 0", @@ -141,62 +162,62 @@ if (children.some((child) => !child) || new Set(children).size !== names.length) { continue; } + const title = container.querySelector("[data-tactile-binding-title]"); + const titleChild = directChild(container, title); + if (!title || !titleChild) continue; const bindingChildren = new Set(children); Object.assign(container.style, { display: "grid", - gridTemplateColumns: "repeat(4, minmax(0, 1fr))", - gap: "8px", + gridTemplateColumns: "6em repeat(4, minmax(0, 1fr))", + gap: "0", alignItems: "end", }); // Only bindings use the four columns; all other taxel controls stay full-width. for (const child of container.children) { - child.style.gridColumn = bindingChildren.has(child) ? "auto" : "1 / -1"; - if (bindingChildren.has(child)) child.style.minWidth = "0"; + if (child === titleChild) { + child.style.gridColumn = "1"; + child.style.gridRow = "span 2"; + child.style.alignSelf = "center"; + child.style.margin = "0"; + child.style.padding = "0 0.5em"; + } else if (bindingChildren.has(child)) { + child.style.gridColumn = "auto"; + child.style.minWidth = "0"; + child.style.margin = "0"; + child.style.padding = "0"; + } else { + child.style.gridColumn = "1 / -1"; + } } - for (const field of fields) field.style.display = "none"; - } - }; - - const freezeMainMenu = () => { - const general = folderFromLabel("General"); - const preview = folderFromLabel("Preview"); - const tabNames = new Set(["URDF", "Segments", "Taxels"]); - const tabs = [...document.querySelectorAll("button")].filter((button) => - tabNames.has(button.textContent?.trim()), - ); - if (!general || tabs.length !== 3) return; - const tabGroup = tabs[0].closest("[data-dock-group]"); - if (!tabGroup) return; - let root = general.parentElement; - while (root && !root.contains(tabGroup)) root = root.parentElement; - if (!root) return; - const generalChild = directChild(root, general); - const tabsChild = directChild(root, tabGroup); - const previewChild = preview ? directChild(root, preview) : null; - if (!generalChild || !tabsChild) return; - - Object.assign(root.style, { - display: "flex", - flexDirection: "column", - height: "100%", - minHeight: "0", - overflow: "hidden", - }); - Object.assign(generalChild.style, { - flex: "0 0 auto", - zIndex: "30", - }); - Object.assign(tabsChild.style, { - flex: "1 1 0", - minHeight: "0", - overflow: "hidden", - }); - if (previewChild) { - Object.assign(previewChild.style, { - flex: "1 1 0", - minHeight: "0", - overflowY: "auto", + Object.assign(title.style, { + color: "var(--mantine-color-dimmed)", + fontSize: "0.875em", + fontWeight: "450", }); + for (const field of fields) { + let row = field.parentElement; + while (row && !row.querySelector("input, button")) row = row.parentElement; + if (!row) continue; + const control = row.querySelector("input, button"); + const labelColumn = directChild(row, field); + const valueColumn = directChild(row, control); + if (!labelColumn || !valueColumn) continue; + // Hiding only the label text leaves Viser's fixed-width label column empty. + labelColumn.style.display = "none"; + valueColumn.style.flex = "1 1 100%"; + valueColumn.style.width = "100%"; + valueColumn.style.minWidth = "0"; + row.style.width = "100%"; + row.style.margin = "0"; + control.style.borderRadius = "0"; + } + if (title.dataset.tactileBindingTitle === "add") { + const capture = [...container.querySelectorAll("button")].find( + (button) => button.textContent?.trim() === "Capture URDF Surface Pose", + ); + const captureChild = directChild(container, capture); + if (captureChild) captureChild.style.marginTop = "0.5em"; + } } }; @@ -209,7 +230,12 @@ input.dataset.tactileNumberDraft = "true"; input.addEventListener("input", () => { const value = input.value; - if (/^[+-]?(?:\d+)?\.$/.test(value)) { + const parsed = Number(value); + if ( + value === "" || + !Number.isFinite(parsed) || + String(parsed) !== value + ) { numberDrafts.set(input, value); } else { numberDrafts.delete(input); @@ -223,7 +249,7 @@ findBridge("__Tactile Info"); positionViserLogo(); boldChoiceLabels(); - freezeMainMenu(); + layoutSelectedNames(); preservePartialNumbers(); layoutTaxelBindings(); for (const label of document.querySelectorAll("label")) { diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 24d6a277c8..778dc70760 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -40,7 +40,7 @@ SELECTION_GIZMO_LINE_WIDTH = 2.5 ACTIVE_DATASET_OPACITY = 1 BINDING_OVERLAY_OPACITY = 0.4 -SAVED_BINDING_OPACITY = 0.2 +SAVED_BINDING_OPACITY = 0.3 class Editor: @@ -70,8 +70,12 @@ def __init__(self, state: State, args): self.preview_segment_folder = None self.selection_folder = None self.segment_widgets, self.taxel_widgets = [], [] + self.segment_item_folders = [] + self.segment_error_widgets, self.taxel_error_widgets = [], [] self.edit_joint_widgets, self.preview_joint_widgets = {}, {} self.preview_segment_widgets = [] + self.new_taxel_binding = Taxel() + self.new_taxel_binding_widgets = [] self.selected_kind: str | None = None self.selected_rows: list[int] = [] self.binding_phase: str | None = None @@ -108,6 +112,8 @@ def __init__(self, state: State, args): ) self.selection_gizmo.on_update(self._selection_gizmo_event) self._sync_side() + if self.state.side in self.state.paths: + self.info.value = f"{self.state.side.title()} metadata loaded." if args.urdf: self.urdf_path.value = str(Path(args.urdf).expanduser().resolve()) self._load_urdf() @@ -182,7 +188,7 @@ def _build_gui(self) -> None: ) self._action_button("Add with Input", self._add_segment_input) self.segment_click_button = self._action_button( - "Add with URDF Click", self._pick_segment + "Add to URDF Surface", self._pick_segment ) self.segment_row_folder = gui.add_folder( "All Segments", expand_by_default=False @@ -203,9 +209,35 @@ def _build_gui(self) -> None: "Center Position", (0.0, 0.0, 0.0), 0.001 ) self.grid_rotation = self._sliders("Rotation", (0.0, 0.0, 0.0)) + gui.add_html('Binding') + options = self._taxel_segment_options() + segment_fields = [ + self.server.gui.add_dropdown( + f"Segment {slot}", options, initial_value="None" + ) + for slot in range(4) + ] + for slot, segment in enumerate(segment_fields): + weight = self._weight_widget(0.0, True, False, slot) + self._watch( + segment, + lambda value, slot=slot: self._set_new_taxel_segment( + slot, value + ), + ) + self._watch( + weight, + lambda value, slot=slot: self._set_new_taxel_weight( + slot, value + ), + ) + self.new_taxel_binding_widgets.append((segment, weight)) self.taxel_capture_button = self._action_button( "Capture URDF Surface Pose", self._capture_taxel_pose ) + self.segment_capture_button = self._action_button( + "Capture Segment Pose", self._capture_segment_pose + ) self._action_button("Add Taxels", self._add_taxels_input) with gui.add_folder("Bind Taxels to Segments"): self.binding_start = gui.add_button("Start Binding to Segments") @@ -295,8 +327,6 @@ def _build_gui(self) -> None: self.selection_panel = gui.add_panel() self.selection_tab = self.selection_panel.add_tab("Selected Object") - with self.selection_tab: - self.selection_folder = gui.add_folder("Selection") self.selection_panel.float(x=10, y=140, width=320, height=520) def _numbers(self, prefix: str, values, step: float): @@ -344,8 +374,16 @@ def _run_action(self, callback, value=None) -> None: LOG.exception("Editor action failed") self.info.value = f'Error: {exc}' else: - if self.mode_value == "Preview" and self.preview_has_errors: + message = self.info.value + if self.mode_value == "Preview": self._update_preview_error() + errors = self.info.value + else: + errors = self._document_error_html() + if message and message != errors: + self.info.value = f"{message}
{errors}" if errors else message + elif errors: + self.info.value = errors self.redraw = True def _request_draw(self) -> None: @@ -365,10 +403,10 @@ def _set_mode(self, mode: str, checked: bool) -> None: other.value = False self.edit_tabs.visible = mode == "Edit" self.preview_panel.visible = mode == "Preview" - self.pick_mode = None + self._set_pick_mode(None) self.selected_kind, self.selected_rows = None, [] self._cancel_binding() - self._rebuild_selection() + self._sync_selection() if mode == "Preview": self._update_preview_error() @@ -379,9 +417,7 @@ def _toggle_side(self) -> None: def _sync_side(self) -> None: self.live_document, self.live_metadata_key = None, "" - self.pick_mode = None - self.segment_click_button.label = "Add with URDF Click" - self.taxel_capture_button.label = "Capture URDF Surface Pose" + self._set_pick_mode(None) self.binding_source_editor.value = True self.binding_source_live.value = False self.binding_source_live.disabled = True @@ -401,7 +437,6 @@ def _sync_side(self) -> None: self._rebuild_edit_lists() self._rebuild_joint_controls() self._rebuild_preview_segments() - self._rebuild_selection() self.redraw = True @staticmethod @@ -428,6 +463,8 @@ def _load(self) -> None: self.state.replace(document, path) self.preview[self.state.side] = self._preview_from(document) self._sync_side() + self.info.value = f"{self.state.side.title()} metadata loaded." + LOG.info("Metadata loaded from %s", path) def _save(self) -> None: path = self.metadata_path.value.strip() @@ -453,7 +490,11 @@ async def save(_): ) async def cancel(_): - self.events.put(modal.close) + def cancel_save(): + modal.close() + self.info.value = "Save canceled." + + self.events.put(lambda: self._run_action(cancel_save)) save_button.on_click(save) cancel_button.on_click(cancel) @@ -461,6 +502,7 @@ async def cancel(_): def _write_save(self, path: str) -> None: storage.save(self.state, path) self.metadata_path.value = self.state.paths[self.state.side] + self.info.value = f"{self.state.side.title()} metadata saved." self.redraw = True def _reset_camera(self) -> None: @@ -476,6 +518,9 @@ def _load_urdf(self) -> None: self.model.remove() self.model = geo.Model(path) self._rebuild_joint_controls() + self.info.value = ( + f"{self.info.value}
URDF loaded." if self.info.value else "URDF loaded." + ) def _update_urdf_alignment(self) -> None: self.urdf_alignment = geo.transform( @@ -590,18 +635,50 @@ def _add_joint_sliders(self, document: Document, output: dict) -> None: ) output[joint.name] = widget + @staticmethod + def _sync_joint_widget_values(document: Document, widgets: dict) -> None: + for name, widget in widgets.items(): + value = float(document.q.get(name, 0.0)) + widget.value = min(float(widget.max), max(float(widget.min), value)) + def _add_segment_input(self) -> None: pose = IDENTITY_POSE.copy() pose[:3] = [float(widget.value) for widget in self.new_segment_position] row = self.document.add_segment(self.new_segment_joint.value, pose) self.selected_kind, self.selected_rows = "segments", [row] self._rebuild_edit_lists() - self._rebuild_selection() + self.info.value = "Segment added." def _pick_segment(self) -> None: - self.pick_mode = None if self.pick_mode == "segment" else "segment" + self._toggle_pick_mode("segment") + + def _toggle_pick_mode(self, mode: str) -> None: + if self.pick_mode == mode: + self._set_pick_mode(None) + self.info.value = { + "segment": "Add Segment canceled.", + "taxel_pose": "Capture URDF Surface Pose canceled.", + "segment_pose": "Capture Segment Pose canceled.", + }[mode] + return + self._set_pick_mode(mode) + + def _set_pick_mode(self, mode: str | None) -> None: + self.pick_mode = None if self.pick_mode == mode else mode self.segment_click_button.label = ( - "● Add with URDF Click" if self.pick_mode else "Add with URDF Click" + "● Add to URDF Surface" + if self.pick_mode == "segment" + else "Add to URDF Surface" + ) + self.taxel_capture_button.label = ( + "● Capture URDF Surface Pose" + if self.pick_mode == "taxel_pose" + else "Capture URDF Surface Pose" + ) + self.segment_capture_button.label = ( + "● Capture Segment Pose" + if self.pick_mode == "segment_pose" + else "Capture Segment Pose" ) def _grid_values(self): @@ -615,27 +692,62 @@ def _grid_values(self): geo.transform(center, [widget.value for widget in self.grid_rotation]), ) + def _sync_new_taxel_binding_widgets(self) -> None: + options = self._taxel_segment_options() + for slot, (segment, weight) in enumerate(self.new_taxel_binding_widgets): + selected = self.new_taxel_binding.segments[slot] + if selected not in options: + selected = None + self.new_taxel_binding.segments[slot] = None + self.new_taxel_binding.weights[slot] = 0.0 + segment.options = options + segment.value = selected or "None" + weight.value = self.new_taxel_binding.weights[slot] + weight.disabled = selected is None + + def _set_new_taxel_segment(self, slot: int, value: str) -> None: + first = value != "None" and not any(self.new_taxel_binding.segments) + self.new_taxel_binding.segments[slot] = None if value == "None" else value + if self.new_taxel_binding.segments[slot] is None: + self.new_taxel_binding.weights[slot] = 0.0 + elif first: + self.new_taxel_binding.weights[slot] = 1.0 + self._sync_new_taxel_binding_widgets() + + def _set_new_taxel_weight(self, slot: int, value) -> None: + try: + Document.set_taxel_weight(self.new_taxel_binding, slot, float(value)) + finally: + self._sync_new_taxel_binding_widgets() + def _add_taxels_input(self) -> None: center, width, height, rows, columns, frame = self._grid_values() selected = self.document.add_taxels( geo.grid_poses(center, width, height, rows, columns, frame) ) + for row in selected: + self.document.taxels[row].segments = copy.deepcopy( + self.new_taxel_binding.segments + ) + self.document.taxels[row].weights = copy.deepcopy( + self.new_taxel_binding.weights + ) self.selected_kind, self.selected_rows = "taxels", selected self._rebuild_edit_lists() - self._rebuild_selection() + noun = "taxel" if len(selected) == 1 else "taxels" + self.info.value = f"{len(selected)} {noun} added." def _capture_taxel_pose(self) -> None: - self.pick_mode = None if self.pick_mode == "taxel_pose" else "taxel_pose" - self.taxel_capture_button.label = ( - "● Capture URDF Surface Pose" - if self.pick_mode - else "Capture URDF Surface Pose" - ) + self._toggle_pick_mode("taxel_pose") + + def _capture_segment_pose(self) -> None: + self._toggle_pick_mode("segment_pose") def _rebuild_edit_lists(self) -> None: self._rebuild_segment_list() self._rebuild_taxel_list() - self._rebuild_selection() + self._sync_new_taxel_binding_widgets() + self._sync_selection() self._update_preview_error() self.redraw = True @@ -643,6 +755,8 @@ def _rebuild_segment_list(self) -> None: if self.segment_row_folder is not None: self.segment_row_folder.remove() self.segment_widgets = [] + self.segment_item_folders = [] + self.segment_error_widgets = [] issues = self.document.validate() with self.segments_tab: self.segment_row_folder = self.server.gui.add_folder( @@ -655,9 +769,10 @@ def _rebuild_segment_list(self) -> None: for issue in issues if issue["kind"] == "segments" and issue["row"] == row ] - with self.server.gui.add_folder( + folder = self.server.gui.add_folder( f"{row}: {segment.joint}", expand_by_default=False - ): + ) + with folder: joint = self.server.gui.add_dropdown( "⚠ Name" if any(issue["field"] == "joint" for issue in row_issues) @@ -691,20 +806,30 @@ def _rebuild_segment_list(self) -> None: ), ) messages = [issue["message"] for issue in row_issues] - if messages: - self.server.gui.add_html( - '
' - + "
".join(messages) - + "
" - ) + error = self.server.gui.add_html( + '
' + + "
".join(messages) + + "
", + visible=bool(messages), + ) + self.segment_item_folders.append(folder) + self.segment_error_widgets.append(error) self.segment_widgets.append((joint, positions, rotations)) + def _taxel_segment_options(self) -> tuple[str, ...]: + # Dropdown values must stay unique even when the draft contains duplicate joints. + return ( + "None", + *dict.fromkeys(segment.joint for segment in self.document.segments), + ) + def _rebuild_taxel_list(self) -> None: if self.taxel_row_folder is not None: self.taxel_row_folder.remove() self.taxel_widgets = [] + self.taxel_error_widgets = [] issues = self.document.validate() - options = ("None", *[segment.joint for segment in self.document.segments]) + options = self._taxel_segment_options() with self.taxels_tab: self.taxel_row_folder = self.server.gui.add_folder( "All Taxels", expand_by_default=False @@ -736,6 +861,9 @@ def _rebuild_taxel_list(self) -> None: self._set_pose_rotation("taxels", row, axis, value) ), ) + self.server.gui.add_html( + 'Binding' + ) slots = [] segment_fields = [] for slot in range(4): @@ -779,12 +907,13 @@ def _rebuild_taxel_list(self) -> None: ) slots.append((segment, weight)) messages = [issue["message"] for issue in row_issues] - if messages: - self.server.gui.add_html( - '
' - + "
".join(dict.fromkeys(messages)) - + "
" - ) + error = self.server.gui.add_html( + '
' + + "
".join(dict.fromkeys(messages)) + + "
", + visible=bool(messages), + ) + self.taxel_error_widgets.append(error) self.taxel_widgets.append((positions, rotations, slots)) def _weight_widget(self, value, disabled: bool, invalid: bool, slot: int): @@ -796,9 +925,87 @@ def _weight_widget(self, value, disabled: bool, invalid: bool, slot: int): ) return self.server.gui.add_text(label, str(value), disabled=disabled) + @staticmethod + def _set_error_widget(widget, messages) -> None: + widget.content = ( + '
' + + "
".join(dict.fromkeys(messages)) + + "
" + ) + widget.visible = bool(messages) + + def _sync_segment_validation_widgets(self) -> None: + issues = self.document.validate() + for row, segment in enumerate(self.document.segments): + row_issues = [ + issue + for issue in issues + if issue["kind"] == "segments" and issue["row"] == row + ] + self.segment_item_folders[row].label = f"{row}: {segment.joint}" + self.segment_widgets[row][0].label = ( + "⚠ Name" + if any(issue["field"] == "joint" for issue in row_issues) + else "Name" + ) + self._set_error_widget( + self.segment_error_widgets[row], + [issue["message"] for issue in row_issues], + ) + + def _sync_taxel_binding_widgets(self, rows) -> None: + issues = self.document.validate() + options = self._taxel_segment_options() + for row in dict.fromkeys(rows): + if not 0 <= row < len(self.document.taxels): + continue + taxel = self.document.taxels[row] + row_issues = [ + issue + for issue in issues + if issue["kind"] == "taxels" and issue["row"] == row + ] + for slot, (segment, weight) in enumerate(self.taxel_widgets[row][2]): + selected = ( + taxel.segments[slot] if taxel.segments[slot] in options else "None" + ) + segment.options = options + segment.value = selected + segment.label = ( + f"⚠ Segment {slot}" + if any( + issue["field"] in (f"segment{slot}", "binding") + for issue in row_issues + ) + else f"Segment {slot}" + ) + weight.value = ( + str(taxel.weights[slot]) + if isinstance(weight.value, str) + else taxel.weights[slot] + ) + weight.disabled = selected == "None" + weight.label = ( + f"⚠ Weight {slot}" + if any( + issue["field"] in (f"weight{slot}", "weights") + for issue in row_issues + ) + else f"Weight {slot}" + ) + self._set_error_widget( + self.taxel_error_widgets[row], + [issue["message"] for issue in row_issues], + ) + if self.mode_value == "Preview": + self._update_preview_error() + self.redraw = True + def _set_segment_joint(self, row: int, value: str) -> None: self.document.segments[row].joint = value - self._rebuild_edit_lists() + self._sync_segment_validation_widgets() + self._sync_taxel_binding_widgets(range(len(self.document.taxels))) + self._sync_new_taxel_binding_widgets() def _values(self, kind: str): return self.document.segments if kind == "segments" else self.document.taxels @@ -817,21 +1024,22 @@ def _set_pose_rotation(self, kind: str, row: int, axis: int, value) -> None: def _set_taxel_segment(self, row: int, slot: int, value: str) -> None: taxel = self.document.taxels[row] + first = value != "None" and not any(taxel.segments) taxel.segments[slot] = None if value == "None" else value if taxel.segments[slot] is None: taxel.weights[slot] = 0.0 - self._rebuild_taxel_list() + elif first: + taxel.weights[slot] = 1.0 + self._sync_taxel_binding_widgets([row]) def _set_taxel_weight(self, row: int, slot: int, value) -> None: try: self.document.set_weight(row, slot, float(value)) finally: - self._rebuild_taxel_list() + self._sync_taxel_binding_widgets([row]) def _start_binding(self) -> None: - self.pick_mode = None - self.segment_click_button.label = "Add with URDF Click" - self.taxel_capture_button.label = "Capture URDF Surface Pose" + self._set_pick_mode(None) self.binding_phase = "taxels" self.binding_taxels, self.binding_segments = [], [] self.selected_kind, self.selected_rows = None, [] @@ -840,6 +1048,7 @@ def _start_binding(self) -> None: "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. " 'Click "Select Segments" after selecting taxels. Click another area to cancel.' ) + self.info.value = "Binding started." def _select_binding_segments(self) -> None: if not self.binding_taxels: @@ -850,6 +1059,7 @@ def _select_binding_segments(self) -> None: 'Select 1-4 segment(s). Hold Shift to multi-select. Click "Complete" ' "after selecting segments. Click another area to cancel." ) + self.info.value = "Taxels selected. Select segments." def _complete_binding(self) -> None: if not 1 <= len(self.binding_segments) <= 4: @@ -860,21 +1070,26 @@ def _complete_binding(self) -> None: "to start over." ) return - self.document.assign(self.binding_taxels, self.binding_segments) + rows = list(self.binding_taxels) + self.document.assign(rows, self.binding_segments) self._cancel_binding() - self._rebuild_taxel_list() + self._sync_taxel_binding_widgets(rows) + noun = "taxel" if len(rows) == 1 else "taxels" + self.info.value = f"Binding completed for {len(rows)} {noun}." def _set_binding_button(self, active: str | None) -> None: self.binding_start.disabled = active != "start" self.binding_next.disabled = active != "next" self.binding_complete.disabled = active != "complete" - def _cancel_binding(self) -> None: + def _cancel_binding(self, report: bool = False) -> None: self.binding_phase = None self.binding_taxels, self.binding_segments = [], [] if hasattr(self, "binding_next"): self._set_binding_button("start") self.binding_info.content = "" + if report: + self.info.value = "Binding canceled." def _joint_enabled(self, enabled: bool) -> None: self.joint_collection.disabled = not enabled @@ -882,7 +1097,7 @@ def _joint_enabled(self, enabled: bool) -> None: def _connect(self) -> None: if self.receiver is not None: - self._disconnect() + self._disconnect(report=True) return config = { "side": self.state.side, @@ -894,14 +1109,17 @@ def _connect(self) -> None: self.receiver = sdk.Receiver(config) self.connect_button.label = "Disconnect Live" self.live_status.content = "Current Status: Connecting" + self.info.value = "Connecting Live." - def _disconnect(self) -> None: + def _disconnect(self, report: bool = False) -> None: if self.receiver is not None: self.receiver.close() self.receiver = None self.connect_button.label = "Connect Live" self.live_status.content = "Current Status: Disconnected" self._update_preview_error() + if report: + self.info.value = "Live disconnected." self.redraw = True @property @@ -963,35 +1181,58 @@ def _update_preview_error(self) -> None: self.apply_segments_button.disabled = False self.apply_urdf_button.disabled = False + def _document_error_html(self) -> str: + messages = [] + labels = {"segments": "Segment", "taxels": "Taxel"} + for issue in self.document.validate(): + label = labels.get(issue["kind"], issue["kind"].title()) + location = f"{label} {issue['row']}" if issue["row"] is not None else label + messages.append(f"{location}: {issue['message']}") + return ( + '' + "
".join(messages) + "" + if messages + else "" + ) + def _apply_preview_segments(self) -> None: self.document.segments = copy.deepcopy(self.preview_document.segments) self._rebuild_edit_lists() + self.info.value = "Preview segments applied to Edit data." def _apply_preview_taxels(self) -> None: rows, _, _ = self._rendered_preview_taxels() self.document.taxels = copy.deepcopy([taxel for _, taxel in rows]) self._rebuild_edit_lists() + self.info.value = "Preview taxels applied to Edit data." def _apply_preview_urdf(self) -> None: self.document.q = copy.deepcopy(self.preview_document.q) - self._rebuild_joint_controls() + self._sync_joint_widget_values(self.document, self.edit_joint_widgets) + self.info.value = "Preview URDF joints applied to Edit data." def _apply_preview_all(self) -> None: - replacement = Document(self.state.side) - replacement.alignment = copy.deepcopy(self.document.alignment) - replacement.segments = copy.deepcopy(self.preview_document.segments) + document = self.document + document.segments = copy.deepcopy(self.preview_document.segments) rows, _, _ = self._rendered_preview_taxels() - replacement.taxels = copy.deepcopy([taxel for _, taxel in rows]) - replacement.q = copy.deepcopy(self.preview_document.q) - self.state.documents[self.state.side] = replacement + document.taxels = copy.deepcopy([taxel for _, taxel in rows]) + document.q = copy.deepcopy(self.preview_document.q) self._rebuild_edit_lists() - self._rebuild_joint_controls() + self._sync_joint_widget_values(document, self.edit_joint_widgets) + self.info.value = "Preview data applied to Edit data." def _restore_preview(self) -> None: - self.preview[self.state.side] = self._preview_from(self.document) - self._rebuild_joint_controls() - self._rebuild_preview_segments() + preview = self.preview_document + replacement = self._preview_from(self.document) + old_joints = [segment.joint for segment in preview.segments] + preview.alignment = replacement.alignment + preview.segments = replacement.segments + preview.taxels = replacement.taxels + preview.q = replacement.q + if old_joints != [segment.joint for segment in preview.segments]: + self._rebuild_preview_segments() + self._sync_preview_values() self._update_preview_error() + self.info.value = "Preview data reset from Edit data." def _rebuild_preview_segments(self) -> None: if self.preview_segment_folder is not None: @@ -1184,10 +1425,10 @@ def _handle_scene_event(self, event, box: bool) -> None: chosen = {kind: rows for kind, rows in chosen.items() if rows} if not chosen: if self.binding_phase is not None: - self._cancel_binding() + self._cancel_binding(report=True) else: self.selected_kind, self.selected_rows = None, [] - self._rebuild_selection() + self._sync_selection() self.redraw = True return kind = ( @@ -1217,10 +1458,10 @@ def _handle_scene_event(self, event, box: bool) -> None: return self.selected_kind = kind self.selected_rows = sorted(set(self.selected_rows) | set(rows)) - self._rebuild_selection() + self._sync_selection() elif self.selected_kind != kind or self.selected_rows != rows: self.selected_kind, self.selected_rows = kind, rows - self._rebuild_selection() + self._sync_selection() self.redraw = True @staticmethod @@ -1289,9 +1530,22 @@ def _box_rows(event, values) -> list[int]: return rows def _handle_pick(self, event) -> None: - mode, self.pick_mode = self.pick_mode, None - self.segment_click_button.label = "Add with URDF Click" - self.taxel_capture_button.label = "Capture URDF Surface Pose" + mode = self.pick_mode + self._set_pick_mode(None) + if mode == "segment_pose": + rows = self._ray_rows( + event, "segments", list(enumerate(self.document.segments)) + ) + if not rows: + self.info.value = "Capture Segment Pose canceled." + return + position, rotation = geo.components( + geo.matrix(self.document.segments[rows[0]].pose) + ) + self._set_values(self.grid_center, position) + self._set_values(self.grid_rotation, rotation) + self.info.value = "Segment pose captured." + return if self.model is None: raise ValueError("Load a URDF before using URDF Click") hit = self.model.hit( @@ -1302,6 +1556,11 @@ def _handle_pick(self, event) -> None: self.urdf_scale, ) if hit is None: + self.info.value = ( + "Add Segment canceled." + if mode == "segment" + else "Capture URDF Surface Pose canceled." + ) return if mode == "segment": value = IDENTITY_POSE.copy() @@ -1309,11 +1568,13 @@ def _handle_pick(self, event) -> None: row = self.document.add_segment(self.new_segment_joint.value, value) self.selected_kind, self.selected_rows = "segments", [row] self._rebuild_edit_lists() + self.info.value = "Segment added." else: frame = geo.surface_frame(hit[1], hit[2]) position, rotation = geo.components(frame) self._set_values(self.grid_center, position) self._set_values(self.grid_rotation, rotation) + self.info.value = "URDF surface pose captured." def _selection_values(self): if self.selected_kind is None: @@ -1322,65 +1583,75 @@ def _selection_values(self): return self.preview_document.segments return self._values(self.selected_kind) - def _rebuild_selection(self) -> None: - if self.selection_folder is not None: - self.selection_folder.remove() + def _sync_selection(self) -> None: values = self._selection_values() rows = [row for row in self.selected_rows if 0 <= row < len(values)] self.selected_rows = rows - with self.selection_tab: - self.selection_folder = self.server.gui.add_folder("Selection") - with self.selection_folder: - self.server.gui.add_markdown( - f"Type: **{self.selected_kind if rows else 'None'}** \n" - f"Count: **{len(rows)}**" - ) - names = [ - values[row].joint - if self.selected_kind == "segments" - else f"Taxel {row}" - for row in rows - ] - self.server.gui.add_text( - "Name", - names[0] if len(names) == 1 else json.dumps(names, indent=2), - multiline=True, - disabled=True, - ) - center = ( - np.asarray([values[row].pose[:3] for row in rows]).mean(axis=0) - if rows - else np.zeros(3) - ) - if len(rows) == 1: - _, rotation = geo.components(geo.matrix(values[rows[0]].pose)) - else: - rotation = (0.0, 0.0, 0.0) - self.selection_position_widgets = self._numbers("Position", center, 0.001) - self.selection_rotation_widgets = self._sliders("Rotation", rotation) - self.selection_rotation = np.asarray(rotation, dtype=float) - for widget in ( - *self.selection_position_widgets, - *self.selection_rotation_widgets, - ): - widget.disabled = not rows - for axis, widget in enumerate(self.selection_position_widgets): - self._watch( - widget, - lambda value, axis=axis: self._set_selection_position(axis, value), + if self.selection_folder is None: + with self.selection_tab: + self.selection_folder = self.server.gui.add_folder("Selection") + with self.selection_folder: + self.selection_summary = self.server.gui.add_markdown("") + self.selection_names = self.server.gui.add_text( + "__Selected Object Names", "", multiline=True, disabled=True ) - for axis, widget in enumerate(self.selection_rotation_widgets): - self._watch( - widget, - lambda value, axis=axis: self._set_selection_rotation(axis, value), + self.selection_position_widgets = self._numbers( + "Position", np.zeros(3), 0.001 + ) + self.selection_rotation_widgets = self._sliders("Rotation", np.zeros(3)) + for axis, widget in enumerate(self.selection_position_widgets): + self._watch( + widget, + lambda value, axis=axis: self._set_selection_position( + axis, value + ), + ) + for axis, widget in enumerate(self.selection_rotation_widgets): + self._watch( + widget, + lambda value, axis=axis: self._set_selection_rotation( + axis, value + ), + ) + self.selection_scale_widget = self.server.gui.add_number( + "Scale", 1.0, min=1e-6, step=0.01 + ) + self._watch(self.selection_scale_widget, self._set_selection_scale) + self.selection_delete = self._action_button( + "Delete", self._delete_selection ) - self.selection_scale = 1.0 - self.selection_scale_widget = self.server.gui.add_number( - "Scale", 1.0, min=1e-6, step=0.01, disabled=len(rows) < 2 - ) - self._watch(self.selection_scale_widget, self._set_selection_scale) - delete = self._action_button("Delete", self._delete_selection) - delete.disabled = not rows + + self.selection_summary.content = ( + f"Type: **{self.selected_kind if rows else 'None'}** \n" + f"Count: **{len(rows)}**" + ) + names = [ + values[row].joint if self.selected_kind == "segments" else f"Taxel {row}" + for row in rows + ] + self.selection_names.value = "\n".join(names) + center = ( + np.asarray([values[row].pose[:3] for row in rows]).mean(axis=0) + if rows + else np.zeros(3) + ) + rotation = ( + geo.components(geo.matrix(values[rows[0]].pose))[1] + if len(rows) == 1 + else np.zeros(3) + ) + self._set_values(self.selection_position_widgets, center) + self._set_values(self.selection_rotation_widgets, rotation) + self.selection_rotation = np.asarray(rotation, dtype=float) + for widget in ( + *self.selection_position_widgets, + *self.selection_rotation_widgets, + ): + widget.disabled = not rows + self.selection_scale = 1.0 + self.selection_scale_widget.value = 1.0 + self.selection_scale_widget.disabled = len(rows) < 2 + self.selection_delete.disabled = not rows def _set_selection_position(self, axis: int, value: float) -> None: values = self._selection_values() @@ -1447,9 +1718,11 @@ def _delete_selection(self) -> None: self._update_preview_error() else: self.document.delete(self.selected_kind, self.selected_rows) + self.selected_kind, self.selected_rows = None, [] self._rebuild_edit_lists() + return self.selected_kind, self.selected_rows = None, [] - self._rebuild_selection() + self._sync_selection() def _draw_instances( self, @@ -1578,7 +1851,11 @@ def _draw_dataset( color = SELECTED_COLOR if self.binding_phase == "taxels" and active: color = INACTIVE_COLOR - if self.pick_mode is not None and active: + if ( + self.pick_mode is not None + and self.pick_mode != "segment_pose" + and active + ): color = INACTIVE_COLOR segment_colors.append(color) taxel_colors = [TAXEL_COLOR] * len(taxels) @@ -1599,10 +1876,6 @@ def _draw_dataset( for row in range(len(taxels)): if row in side_taxel_rows: taxel_colors[row] = SIDE_HIGHLIGHT_COLOR - if self.selected_kind == "taxels" and row in self.selected_rows and active: - taxel_colors[row] = SELECTED_COLOR - if self.binding_phase == "taxels" and row in self.binding_taxels: - taxel_colors[row] = SELECTED_COLOR if self.binding_phase == "segments" and active: taxel_colors[row] = INACTIVE_COLOR if self.pick_mode is not None and active: @@ -1616,6 +1889,16 @@ def _draw_dataset( for row in error_rows: if 0 <= row < len(taxel_colors): taxel_colors[row] = ERROR_COLOR + # Selection is the final visual state so invalid selected taxels stay visible. + if active and self.pick_mode is None: + if self.selected_kind == "taxels": + for row in self.selected_rows: + if 0 <= row < len(taxel_colors): + taxel_colors[row] = SELECTED_COLOR + if self.binding_phase == "taxels": + for row in self.binding_taxels: + if 0 <= row < len(taxel_colors): + taxel_colors[row] = SELECTED_COLOR self._draw_instances( prefix + "/segments", "segments", segments, segment_colors, opacity ) @@ -1656,7 +1939,7 @@ def _draw(self) -> None: q, self.urdf_alignment, self.urdf_scale, - highlighted=self.pick_mode is not None, + highlighted=self.pick_mode in ("segment", "taxel_pose"), opacity=URDF_OPACITY * active_opacity, ) if self.mode_value == "Edit": From 208dd85bd7accafe87012bac8b67814cf08b0aab Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 19:16:40 -0700 Subject: [PATCH 22/36] Paginate tactile taxels and add selection editors Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 1 + scripts/tactile_metadata/README.md | 14 +- scripts/tactile_metadata/UI&Feature Design.md | 6 +- .../tactile_metadata_editor/ui.py | 174 +++++++++++++++--- 4 files changed, 165 insertions(+), 30 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 0abdfb2e21..ddeb7f4725 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -21,6 +21,7 @@ SPDX-License-Identifier: Apache-2.0 - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. - New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. - Preserve slot order and repeated segment indices. The first selected Segment receives weight 1 and later selections receive 0; manual weight edits change only that slot. Reject values outside 0~1, and let validation report totals other than 1 without normalization. +- All Taxels renders 50 global rows per page; never instantiate controls for the full Taxel array. The Selected Object panel reuses one Segment or Taxel editor and shows it only for a matching single selection. - SDK process isolation specifies where SDK classes are called; document the actual tracker/session calls required for live reception as well as export encoding. - Keep tool implementation inside this subtree; add missing dependencies to Isaac Capture's existing requirements and matching package metadata. Run checks only from this directory or its descendants; never run repository-wide fixing hooks that can change other directories. - Explain this standalone tool's setup from the user's prerequisites: define external libraries before packaging terms, and distinguish requirement files, generated version records, and installation commands. Identify inspected environments by their Python executable path; distinguish declared requirements from packages actually installed there. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index e570b75519..b5ad66ab1e 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -127,11 +127,12 @@ The three buttons are vertical with down arrows between adjacent steps, and at most one is enabled at a time. Between steps, only the required 3D selection is available. -Expand **All Taxels** to edit pose, four segment slots, and four weights. `None` -is an editor-only inactive slot with weight zero. Selecting the first active -segment sets its weight to one; later segment selections start at zero. Editing -a weight changes only that slot. Validation reports rows whose weights do not -sum to one. +Expand **All Taxels** to edit 50 rows at a time; Previous/Next switches pages +without changing global Taxel indices. Each visible row contains pose, four +segment slots, and four weights. `None` is an editor-only inactive slot with +weight zero. Selecting the first active segment sets its weight to one; later +segment selections start at zero. Editing a weight changes only that slot. +Validation reports rows whose weights do not sum to one. ## Scene selection @@ -152,6 +153,9 @@ objects, and scales multi-selection positions about their center. Edit mode changes Editing data; Preview mode changes Preview segments. Preview taxels are derived and cannot be selected. With no selection, the panel remains visible, shows Type `None` and Count `0`, and disables its input fields. +Exactly one selected Segment reveals its HandJoint editor; exactly one selected +Taxel reveals its four Segment/Weight binding slots. These editors reuse one +control set rather than creating controls for every Taxel. The selected-object 3-axis control supports axis-only translation. Dragging an axis moves every selected object by the same delta. Single selections use the diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index 14d8da802a..51ec378efd 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -54,7 +54,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), rotation (3 float sliders), and a left-labeled four-slot Segment/Weight binding grid matching each Taxel row. Each binding grid has no gaps between cells. Leave standard vertical spacing between this grid and "Capture URDF Surface Pose" without a visible spacer element. "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. "Capture Segment Pose" does the same using the clicked Editing segment's complete pose. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields and copies the binding grid into every new Taxel; newly created taxels are selected and previous objects are deselected. c) Vertically align "Start Binding to Segments", "Select Segments", and "Complete", with a centered down arrow between each adjacent pair. At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The first selected Segment receives weight 1 and later selected Segments receive weight 0. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. - d) (spannable) A full list of all taxels, with an id number (read-only, starting at 0, it's the index of the taxel in the list), their position (3 float input), editable rotation (3 float slider), editable binded segments (4 * dropdown list from the segments list, displayed as segment name (HandJoint enum name). "None" if not selected), binding weights (4 * float input. Grey out for non-selected segments.). This is to modify the binding of the taxels, display this info above this section. Selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. Editing a weight changes only that slot and never modifies the other weights. Weight fields reject numbers out of 0~1, and row validation reports when the total is not 1. + d) (spannable) A paginated list of all taxels, showing 50 rows per page with Previous/Next controls and global zero-based Taxel indices. Each visible row contains position (3 float input), editable rotation (3 float slider), editable bound segments (4 dropdowns by HandJoint name, with "None" for inactive slots), and four binding weights. Selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. Editing a weight changes only that slot and never modifies the other weights. Weight fields reject numbers out of 0~1, and row validation reports when the total is not 1. * "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. * Taxels data can be invalid for multiple reasons. When invalid, line the invalid field in red. Reasons including: a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. @@ -68,7 +68,9 @@ a) (read only) selected elements type and count (multi selection is stated in fo b) (editable) current position (defined above) c) (editable) current rotation (defined above) d) (editable single float) scaling (only for multi-selection, scale up/down the selected objects' position in space, based on the area center. For example, if one object was at (1, 2, 3) and the area center is at (1, 1, 1), scaling the area up to 2x will move the object to (1, 3, 5).) -e) Delete +e) When exactly one Segment is selected, show a Segment Editor for its HandJoint name. +f) When exactly one Taxel is selected, show a Taxel Editor for its four Segment/Weight binding slots. +g) Delete 10. Preview menu: a) A named "Binding Metadata Source" section containing bold Editor and Live checkbox labels. They are mutually exclusive and cannot both be unchecked; Editor is selected by default. Live remains disabled/grey until the first Live metadata is received. This selection affects rendered taxel poses because they are calculated from segment poses and the selected binding metadata. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 778dc70760..4d519a36c0 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -41,6 +41,7 @@ ACTIVE_DATASET_OPACITY = 1 BINDING_OVERLAY_OPACITY = 0.4 SAVED_BINDING_OPACITY = 0.3 +TAXEL_PAGE_SIZE = 50 class Editor: @@ -65,17 +66,21 @@ def __init__(self, state: State, args): self.scene_handles: dict[str, object] = {} self.segment_row_folder = None self.taxel_row_folder = None + self.taxel_page_container = None self.edit_joint_folder = None self.preview_joint_folder = None self.preview_segment_folder = None self.selection_folder = None - self.segment_widgets, self.taxel_widgets = [], [] + self.segment_widgets = [] + self.taxel_widgets = {} self.segment_item_folders = [] - self.segment_error_widgets, self.taxel_error_widgets = [], [] + self.segment_error_widgets = [] + self.taxel_error_widgets = {} self.edit_joint_widgets, self.preview_joint_widgets = {}, {} self.preview_segment_widgets = [] self.new_taxel_binding = Taxel() self.new_taxel_binding_widgets = [] + self.taxel_page = 0 self.selected_kind: str | None = None self.selected_rows: list[int] = [] self.binding_phase: str | None = None @@ -141,10 +146,10 @@ def _build_gui(self) -> None: self._action_button("New", self._new) self._action_button("Save", self._save) self._action_button("Load", self._load) - self._action_button("Reset Camera", self._reset_camera) with gui.add_folder("Mode"): self.mode_edit = gui.add_checkbox("Edit", True) self.mode_preview = gui.add_checkbox("Preview", False) + self._action_button("Reset Camera", self._reset_camera) self.show_saved = gui.add_checkbox("Show Saved Binding Poses", False) self._watch(self.mode_edit, lambda checked: self._set_mode("Edit", checked)) self._watch( @@ -564,9 +569,8 @@ def _update_segment_alignment(self) -> None: position, rotation = geo.components(geo.matrix(segment.pose)) self._set_values(positions, position) self._set_values(rotations, rotation) - for taxel, (positions, rotations, _) in zip( - self.document.taxels, self.taxel_widgets - ): + for row, (positions, rotations, _) in self.taxel_widgets.items(): + taxel = self.document.taxels[row] position, rotation = geo.components(geo.matrix(taxel.pose)) self._set_values(positions, position) self._set_values(rotations, rotation) @@ -733,6 +737,8 @@ def _add_taxels_input(self) -> None: self.new_taxel_binding.weights ) self.selected_kind, self.selected_rows = "taxels", selected + if selected: + self.taxel_page = selected[0] // TAXEL_PAGE_SIZE self._rebuild_edit_lists() noun = "taxel" if len(selected) == 1 else "taxels" self.info.value = f"{len(selected)} {noun} added." @@ -823,19 +829,48 @@ def _taxel_segment_options(self) -> tuple[str, ...]: *dict.fromkeys(segment.joint for segment in self.document.segments), ) + def _taxel_page_count(self) -> int: + return max(1, math.ceil(len(self.document.taxels) / TAXEL_PAGE_SIZE)) + + def _change_taxel_page(self, delta: int) -> None: + page = min( + self._taxel_page_count() - 1, + max(0, self.taxel_page + delta), + ) + if page == self.taxel_page: + return + self.taxel_page = page + self._rebuild_taxel_list() + def _rebuild_taxel_list(self) -> None: - if self.taxel_row_folder is not None: - self.taxel_row_folder.remove() - self.taxel_widgets = [] - self.taxel_error_widgets = [] + if self.taxel_page_container is not None: + self.taxel_page_container.remove() + self.taxel_widgets = {} + self.taxel_error_widgets = {} issues = self.document.validate() options = self._taxel_segment_options() - with self.taxels_tab: - self.taxel_row_folder = self.server.gui.add_folder( - "All Taxels", expand_by_default=False - ) - with self.taxel_row_folder: - for row, taxel in enumerate(self.document.taxels): + page_count = self._taxel_page_count() + self.taxel_page = min(self.taxel_page, page_count - 1) + first = self.taxel_page * TAXEL_PAGE_SIZE + last = min(first + TAXEL_PAGE_SIZE, len(self.document.taxels)) + with self.taxel_row_folder: + self.taxel_page_container = self.server.gui.add_folder(None) + with self.taxel_page_container: + previous = self._action_button( + "Previous Page", lambda: self._change_taxel_page(-1) + ) + previous.disabled = self.taxel_page == 0 + shown = f"{first}-{last - 1}" if last > first else "None" + self.server.gui.add_markdown( + f"Page **{self.taxel_page + 1} / {page_count}** \n" + f"Taxels: **{shown}** of **{len(self.document.taxels)}**" + ) + next_page = self._action_button( + "Next Page", lambda: self._change_taxel_page(1) + ) + next_page.disabled = self.taxel_page >= page_count - 1 + for row in range(first, last): + taxel = self.document.taxels[row] row_issues = [ issue for issue in issues @@ -913,8 +948,8 @@ def _rebuild_taxel_list(self) -> None: + "", visible=bool(messages), ) - self.taxel_error_widgets.append(error) - self.taxel_widgets.append((positions, rotations, slots)) + self.taxel_error_widgets[row] = error + self.taxel_widgets[row] = (positions, rotations, slots) def _weight_widget(self, value, disabled: bool, invalid: bool, slot: int): label = f"{'⚠ ' if invalid else ''}Weight {slot}" @@ -960,12 +995,15 @@ def _sync_taxel_binding_widgets(self, rows) -> None: if not 0 <= row < len(self.document.taxels): continue taxel = self.document.taxels[row] + widgets = self.taxel_widgets.get(row) + if widgets is None: + continue row_issues = [ issue for issue in issues if issue["kind"] == "taxels" and issue["row"] == row ] - for slot, (segment, weight) in enumerate(self.taxel_widgets[row][2]): + for slot, (segment, weight) in enumerate(widgets[2]): selected = ( taxel.segments[slot] if taxel.segments[slot] in options else "None" ) @@ -1006,6 +1044,7 @@ def _set_segment_joint(self, row: int, value: str) -> None: self._sync_segment_validation_widgets() self._sync_taxel_binding_widgets(range(len(self.document.taxels))) self._sync_new_taxel_binding_widgets() + self._sync_selection() def _values(self, kind: str): return self.document.segments if kind == "segments" else self.document.taxels @@ -1031,12 +1070,14 @@ def _set_taxel_segment(self, row: int, slot: int, value: str) -> None: elif first: taxel.weights[slot] = 1.0 self._sync_taxel_binding_widgets([row]) + self._sync_selection() def _set_taxel_weight(self, row: int, slot: int, value) -> None: try: self.document.set_weight(row, slot, float(value)) finally: self._sync_taxel_binding_widgets([row]) + self._sync_selection() def _start_binding(self) -> None: self._set_pick_mode(None) @@ -1613,6 +1654,47 @@ def _sync_selection(self) -> None: axis, value ), ) + self.selection_segment_editor = self.server.gui.add_folder( + "Segment Editor", visible=False + ) + with self.selection_segment_editor: + self.selection_segment_joint = self.server.gui.add_dropdown( + "Name", tuple(JOINTS) + ) + self._watch( + self.selection_segment_joint, + self._set_selected_segment_joint, + ) + self.selection_taxel_editor = self.server.gui.add_folder( + "Taxel Editor", visible=False + ) + self.selection_taxel_binding_widgets = [] + with self.selection_taxel_editor: + self.server.gui.add_html( + 'Binding' + ) + options = self._taxel_segment_options() + segment_fields = [ + self.server.gui.add_dropdown( + f"Segment {slot}", options, initial_value="None" + ) + for slot in range(4) + ] + for slot, segment in enumerate(segment_fields): + weight = self._weight_widget(0.0, True, False, slot) + self._watch( + segment, + lambda value, slot=slot: self._set_selected_taxel_segment( + slot, value + ), + ) + self._watch( + weight, + lambda value, slot=slot: self._set_selected_taxel_weight( + slot, value + ), + ) + self.selection_taxel_binding_widgets.append((segment, weight)) self.selection_scale_widget = self.server.gui.add_number( "Scale", 1.0, min=1e-6, step=0.01 ) @@ -1648,11 +1730,51 @@ def _sync_selection(self) -> None: *self.selection_rotation_widgets, ): widget.disabled = not rows + single_segment = self.selected_kind == "segments" and len(rows) == 1 + single_taxel = self.selected_kind == "taxels" and len(rows) == 1 + self.selection_segment_editor.visible = single_segment + self.selection_taxel_editor.visible = single_taxel + if single_segment: + self.selection_segment_joint.value = values[rows[0]].joint + if single_taxel: + taxel = values[rows[0]] + options = self._taxel_segment_options() + for slot, (segment, weight) in enumerate( + self.selection_taxel_binding_widgets + ): + selected = ( + taxel.segments[slot] if taxel.segments[slot] in options else "None" + ) + segment.options = options + segment.value = selected + weight.value = taxel.weights[slot] + weight.disabled = selected == "None" self.selection_scale = 1.0 self.selection_scale_widget.value = 1.0 self.selection_scale_widget.disabled = len(rows) < 2 self.selection_delete.disabled = not rows + def _set_selected_segment_joint(self, value: str) -> None: + if self.selected_kind != "segments" or len(self.selected_rows) != 1: + return + row = self.selected_rows[0] + if self.mode_value == "Edit": + self._set_segment_joint(row, value) + return + self.preview_document.segments[row].joint = value + self._rebuild_preview_segments() + self._sync_selection() + + def _set_selected_taxel_segment(self, slot: int, value: str) -> None: + if self.selected_kind != "taxels" or len(self.selected_rows) != 1: + return + self._set_taxel_segment(self.selected_rows[0], slot, value) + + def _set_selected_taxel_weight(self, slot: int, value) -> None: + if self.selected_kind != "taxels" or len(self.selected_rows) != 1: + return + self._set_taxel_weight(self.selected_rows[0], slot, value) + def _set_selection_position(self, axis: int, value: float) -> None: values = self._selection_values() center = np.asarray([values[row].pose[:3] for row in self.selected_rows]).mean( @@ -1696,13 +1818,19 @@ def _sync_selected_list_values(self) -> None: ) values = self._selection_values() for row in self.selected_rows: - if row >= len(widgets): - continue + if self.selected_kind == "segments": + if row >= len(widgets): + continue + row_widgets = widgets[row] + else: + row_widgets = widgets.get(row) + if row_widgets is None: + continue position, rotation = geo.components(geo.matrix(values[row].pose)) position_widgets, rotation_widgets = ( - (widgets[row][1], widgets[row][2]) + (row_widgets[1], row_widgets[2]) if self.selected_kind == "segments" - else (widgets[row][0], widgets[row][1]) + else (row_widgets[0], row_widgets[1]) ) self._set_values(position_widgets, position) self._set_values(rotation_widgets, rotation) From 09d3933c9ec02521ea58b1c543714cfe6a7a7277 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 19:37:10 -0700 Subject: [PATCH 23/36] Separate tactile editor menus and preserve UI state Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 2 + scripts/tactile_metadata/README.md | 6 +- scripts/tactile_metadata/UI&Feature Design.md | 3 +- .../editor_interactions.js | 132 +++++++++++++++++- .../tactile_metadata_editor/ui.py | 8 +- 5 files changed, 142 insertions(+), 9 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index ddeb7f4725..651f1d4476 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -32,8 +32,10 @@ SPDX-License-Identifier: Apache-2.0 - Selection side-highlighting resolves bindings by HandJoint name and includes only slots with positive weights. Keep `SIDE_HIGHLIGHT_COLOR` distinct from `SELECTED_COLOR`; selection color overrides validation error color on selected taxels. - When product placement says top/before/below and any sibling uses explicit Viser `order`, assign explicit order to every constrained sibling; source creation order does not override mixed default and explicit ordering. - Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. All Segments/Taxels lists, their entity folders, and Preview Segments start collapsed. +- General is a separate right-docked panel above the right-docked Editor/Preview main panel. - Client-side helpers may hide only their exact bridge input and label; never hide an ancestor container that can contain the entire Viser control panel. - Preserve focused formatting-sensitive numeric drafts such as `0.0` and `0.` client-side because Viser canonicalizes them to floats during controlled rerenders. Restore through the native input value setter and dispatch an `input` event; a raw DOM assignment leaves React/Mantine state stale and breaks the next digit. +- Preserve each Viser ScrollArea position across menu operations, with separate positions for Edit and Preview. Only user scrolling changes the stored position. - Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. - Call `/edit`, `/preview`, and `/saved` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. - During live redraw, update the active scene-node group in place. Hide only groups that should be inactive; never hide and immediately re-show the active group each frame. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index b5ad66ab1e..6fb46d4015 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -46,12 +46,10 @@ available HMD system in the same Python environment. The editor maintains independent left- and right-hand documents. **Toggle Side** selects the active document; each side has its own metadata path and data. -General controls are at the top of the right menu. The Selected Object panel is -always present as a floating window on the left. +General uses a separate right-docked menu above the Editor/Preview menu. The +Selected Object panel is always present as a floating window on the left. The **Mode** section uses bold, mutually exclusive **Edit** and **Preview** checkbox labels; one mode is always selected. -General and the URDF/Segments/Taxels tab selector stay fixed at the top of the -right menu. Only the active tab contents scroll. Focused numeric fields preserve partial decimal input such as `1.` until the next digit is entered or the field loses focus. diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index 51ec378efd..1a9041795f 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -20,8 +20,7 @@ About data cooridanates: all UI editor use visualization space. When save to met About LBS taxel rotation: determine by Polar Decomposition algorithm. At singularity, use taxel's bind rotation. Make sure to optimize for single-segment binding case, where the rotation can be directly determined without running Polar Decomposition. -1. At the top of the right menu, always display a General section containing: Toggle side, new/save/load metadata file, reset camera, a named "Mode" section containing bold Edit and Preview checkbox labels that are mutually exclusive and cannot both be unchecked, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". -* Freeze General and the URDF/Segments/Taxels tab selector at the top of the right menu. Scrolling affects only the active tab contents; the General section and tab selector do not scroll. +1. Use two separate menus docked on the right: General on top and Editor/Preview below. General contains Toggle side, new/save/load metadata file, reset camera, a named "Mode" section containing bold Edit and Preview checkbox labels that are mutually exclusive and cannot both be unchecked, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". 2. This tool actually manages 3 copies of pose data (pose of segments and taxels), with different data sources. a) Editing dataset: the data that the user is editing in the tool. It's the binding poses of the segments and taxels under editing. Plus the URDF joint values that's under Editing mode. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index b968ef2dcc..7523e4fbb2 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -9,8 +9,11 @@ let observer = null; let infoInput = null; let infoElement = null; + let selectedNamesTextarea = null; let raf = 0; const numberDrafts = new Map(); + const menuScrollStates = new WeakMap(); + const menuScrollListeners = new Map(); const nativeInputValueSetter = Object.getOwnPropertyDescriptor( HTMLInputElement.prototype, "value", @@ -85,6 +88,15 @@ input.setSelectionRange(draft.length, draft.length); } } + if ( + selectedNamesTextarea?.isConnected && + selectedNamesTextarea.dataset.tactileResizing !== "true" + ) { + const height = selectedNamesTextarea.dataset.tactileHeight; + if (height && selectedNamesTextarea.style.height !== height) { + selectedNamesTextarea.style.height = height; + } + } syncInfo(); raf = requestAnimationFrame(frame); }; @@ -105,6 +117,96 @@ return child?.parentElement === parent ? child : null; }; + const currentMode = () => { + for (const name of ["Edit", "Preview"]) { + const label = [...document.querySelectorAll("label")].find( + (candidate) => candidate.textContent?.trim() === name, + ); + const input = label?.htmlFor ? document.getElementById(label.htmlFor) : null; + if (input instanceof HTMLInputElement && input.checked) return name; + } + return "default"; + }; + + const targetMode = (target) => { + const label = + target.closest("label") ?? + (target.id ? document.querySelector(`label[for="${CSS.escape(target.id)}"]`) : null); + const name = label?.textContent?.trim(); + return name === "Edit" || name === "Preview" ? name : currentMode(); + }; + + const setupMenuScroll = () => { + for (const viewport of document.querySelectorAll( + "[data-radix-scroll-area-viewport]", + )) { + if (!(viewport instanceof HTMLElement) || menuScrollListeners.has(viewport)) { + continue; + } + const state = { + positions: new Map([[currentMode(), viewport.scrollTop]]), + restoring: false, + token: null, + }; + menuScrollStates.set(viewport, state); + const onScroll = () => { + if (!state.restoring) { + state.positions.set(currentMode(), viewport.scrollTop); + } + }; + menuScrollListeners.set(viewport, onScroll); + viewport.addEventListener("scroll", onScroll, { passive: true }); + } + }; + + const preserveMenuScroll = (event) => { + const target = event.target; + if (!(target instanceof HTMLElement)) return; + if ( + !target.closest( + "button, input, textarea, label, [role='button'], [role='tab'], [role='option']", + ) + ) { + return; + } + const current = currentMode(); + const next = targetMode(target); + const sourceViewport = target.closest("[data-radix-scroll-area-viewport]"); + const viewports = + next !== current + ? [...menuScrollListeners.keys()] + : sourceViewport instanceof HTMLElement + ? [sourceViewport] + : []; + for (const viewport of viewports) { + const state = menuScrollStates.get(viewport); + if (!state) continue; + state.positions.set(current, viewport.scrollTop); + const top = state.positions.get(next) ?? viewport.scrollTop; + const left = viewport.scrollLeft; + const token = {}; + state.token = token; + state.restoring = true; + let frames = 4; + const restore = () => { + if (state.token !== token || !viewport.isConnected) return; + viewport.scrollTop = top; + viewport.scrollLeft = left; + if (--frames > 0) { + requestAnimationFrame(restore); + } else { + state.restoring = false; + state.positions.set(next, viewport.scrollTop); + } + }; + requestAnimationFrame(restore); + } + }; + + const preserveMenuScrollKey = (event) => { + if (event.key === "Enter" || event.key === " ") preserveMenuScroll(event); + }; + const layoutSelectedNames = () => { for (const label of document.querySelectorAll("label")) { if (label.textContent?.trim() !== "__Selected Object Names") continue; @@ -119,6 +221,7 @@ valueColumn.style.flex = "1 1 100%"; valueColumn.style.width = "100%"; row.style.width = "100%"; + selectedNamesTextarea = textarea; if (textarea.dataset.tactileFixedRows !== "true") { const style = getComputedStyle(textarea); const fontSize = Number.parseFloat(style.fontSize); @@ -128,9 +231,26 @@ .map((name) => Number.parseFloat(style[name]) || 0) .reduce((sum, value) => sum + value, 0); textarea.rows = 2; - textarea.style.height = `${Math.ceil(2 * lineHeight + chrome)}px`; + const height = `${Math.ceil(2 * lineHeight + chrome)}px`; + textarea.style.height = height; textarea.style.overflowY = "auto"; + textarea.dataset.tactileHeight = height; textarea.dataset.tactileFixedRows = "true"; + textarea.addEventListener("pointerdown", (event) => { + const rect = textarea.getBoundingClientRect(); + if (event.clientX < rect.right - 20 || event.clientY < rect.bottom - 20) { + return; + } + textarea.dataset.tactileResizing = "true"; + const finish = () => { + textarea.dataset.tactileHeight = `${textarea.getBoundingClientRect().height}px`; + delete textarea.dataset.tactileResizing; + window.removeEventListener("pointerup", finish); + window.removeEventListener("pointercancel", finish); + }; + window.addEventListener("pointerup", finish, { once: true }); + window.addEventListener("pointercancel", finish, { once: true }); + }); } } }; @@ -249,6 +369,7 @@ findBridge("__Tactile Info"); positionViserLogo(); boldChoiceLabels(); + setupMenuScroll(); layoutSelectedNames(); preservePartialNumbers(); layoutTaxelBindings(); @@ -262,6 +383,8 @@ } }; + document.addEventListener("pointerdown", preserveMenuScroll, true); + document.addEventListener("keydown", preserveMenuScrollKey, true); observer = new MutationObserver(bind); observer.observe(document.body, { childList: true, subtree: true }); bind(); @@ -270,10 +393,17 @@ stop() { observer?.disconnect(); observer = null; + document.removeEventListener("pointerdown", preserveMenuScroll, true); + document.removeEventListener("keydown", preserveMenuScrollKey, true); + for (const [viewport, listener] of menuScrollListeners) { + viewport.removeEventListener("scroll", listener); + } + menuScrollListeners.clear(); cancelAnimationFrame(raf); infoElement?.remove(); infoElement = null; infoInput = null; + selectedNamesTextarea = null; numberDrafts.clear(); }, }; diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 4d519a36c0..9665284205 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -103,7 +103,8 @@ def __init__(self, state: State, args): self.server.initial_camera.position = self.CAMERA_POSITION self.server.initial_camera.look_at = self.CAMERA_LOOK_AT self.server.gui.configure_theme(control_width="medium", show_share_button=False) - self.server.gui.main_panel.dock_right() + self.main_panel = self.server.gui.main_panel + self.main_panel.dock_right() self._build_gui() self._register_scene_events() self.selection_gizmo = self.server.scene.add_transform_controls( @@ -139,7 +140,9 @@ def _preview_from(document: Document) -> Document: def _build_gui(self) -> None: gui = self.server.gui - with gui.add_folder("General"): + self.general_panel = gui.add_panel() + self.general_tab = self.general_panel.add_tab("General") + with self.general_tab: self.side_text = gui.add_markdown("") self._action_button("Toggle Side", self._toggle_side) self.metadata_path = gui.add_text("Metadata Path", "") @@ -156,6 +159,7 @@ def _build_gui(self) -> None: self.mode_preview, lambda checked: self._set_mode("Preview", checked) ) self._watch(self.show_saved, lambda _: self._request_draw()) + self.general_panel.dock_above(self.main_panel) self.edit_tabs = gui.add_tab_group() self.urdf_tab = self.edit_tabs.add_tab("URDF") From 78e4abeb20b4e83f572a1a0faab80e94d85554df Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 19:38:49 -0700 Subject: [PATCH 24/36] Clarify tactile metadata editor instructions Signed-off-by: Lotus Li --- scripts/tactile_metadata/README.md | 68 +++++++++++------------------- 1 file changed, 24 insertions(+), 44 deletions(-) diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 6fb46d4015..bed6b5701a 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -12,8 +12,8 @@ All edits remain local; live connections only receive Isaac Capture data. ## Set up and launch -Use an existing Isaac Capture environment with Python 3.11–3.13. Install the -shared UI requirements into that environment: +Use an existing Isaac Capture environment with Python 3.11 or newer. Install +the shared UI requirements into that environment: ```bash cd scripts/tactile_metadata @@ -58,7 +58,7 @@ Three pose datasets are displayed: - **Editing** contains editable segment and taxel binding poses plus Edit-mode URDF joint values. - **Saved** is the last loaded or saved metadata snapshot. Enable **Show Saved - Binding Poses** to render it at opacity 0.2. + Binding Poses** to render it at opacity 0.3. - **Preview** contains current segment poses and Preview-mode URDF joint values. Preview taxel poses are derived from the selected binding metadata. @@ -68,9 +68,8 @@ Loading applies that transform; saving applies its inverse. Changing the alignment transforms the visible editor values without changing the underlying metadata coordinates. -A fixed three-axis indicator in the top-right of the 3D view shows visualization -space relative to the render camera. The Viser icon remains at the bottom-left, -positioned above the fixed Info area. +A three-axis indicator in the top-right shows visualization space relative to +the camera. ## Edit metadata @@ -82,19 +81,17 @@ The **Edit** mode contains three tabs. 2. Use **URDF Alignment** to translate, rotate, and scale the model. 3. Expand **URDF Joints** to adjust each movable joint. -URDF Joints and the full Segment/Taxel lists are recursively collapsed by -default. Each joint/entity item and each nested Taxel Binding folder also starts -collapsed. Their displayed indices start at 0. +URDF joints are direct sliders. Segment and Taxel lists start collapsed, and +their displayed indices start at 0. -URDF configuration is session-only and is not saved in tactile metadata. Its -base opacity is 0.4, multiplied by the opacity of the dataset layer being -rendered. +URDF configuration is session-only and is not saved in tactile metadata. The +model renders at opacity 0.2. ### Segments - Use **Segments Alignment** to align all metadata poses. -- Add a segment from a HandJoint and XYZ position, or select **Add with URDF - Click** and click the model surface. +- Add a segment from a HandJoint and XYZ position, or select **Add to URDF + Surface** and click the model. - Segments are flat-shaded cubes with side length 0.004 m. Smooth color gradients and cast/received shadows are disabled. - Expand **All Segments** to edit HandJoint, position, and XYZ rotation. @@ -121,16 +118,14 @@ To bind taxels: 3. Select **Complete**. The first Segment receives weight one; later Segments receive weight zero. -The three buttons are vertical with down arrows between adjacent steps, and at -most one is enabled at a time. Between steps, only the required 3D selection is -available. +At most one binding-step button is enabled at a time. -Expand **All Taxels** to edit 50 rows at a time; Previous/Next switches pages -without changing global Taxel indices. Each visible row contains pose, four -segment slots, and four weights. `None` is an editor-only inactive slot with -weight zero. Selecting the first active segment sets its weight to one; later -segment selections start at zero. Editing a weight changes only that slot. -Validation reports rows whose weights do not sum to one. +Expand **All Taxels** to edit 50 rows at a time; **Previous Page** and **Next +Page** switch pages without changing global Taxel indices. Each visible row +contains pose, four segment slots, and four weights. `None` is an editor-only +inactive slot with weight zero. Selecting the first active segment sets its +weight to one; later segment selections start at zero. Editing a weight changes +only that slot. Validation reports rows whose weights do not sum to one. ## Scene selection @@ -146,7 +141,7 @@ Selected objects keep `SELECTED_COLOR`. Related objects use the separate weights to those HandJoints, while selecting taxels highlights their positive-weight segments. Zero-weight slots are ignored. -The **Selected Objects** panel edits position and rotation, deletes selected +The **Selected Object** panel edits position and rotation, deletes selected objects, and scales multi-selection positions about their center. Edit mode changes Editing data; Preview mode changes Preview segments. Preview taxels are derived and cannot be selected. With no selection, the panel remains visible, @@ -231,26 +226,11 @@ binding metadata affects only taxel deformation; it never seeds Preview segments. Enabled JointState mappings update Preview URDF joints. Live frames overwrite manual Static values while connected. Tactile force magnitude maps from blue at 0 N to red at 30 N, with values above -30 N clamped. Missing or inactive force data is light grey. - -For multi-segment LBS, taxel orientation uses polar decomposition of the blended -rotation. Single-segment bindings use their direct rigid transform; singular -blends fall back to the taxel bind rotation. - -## Implementation - -- `document.py` defines editor-native segments, taxels, validation, and per-side - state. -- `storage.py` converts between editor data and schema JSON and performs atomic - saves. -- `geometry.py` handles URDF loading, scene transforms, picking, grid creation, - HandPose conversion, and LBS. -- `sdk.py` receives HandPose, TactileMetadata/TactileDataArray, and optional - JointState in a child process started with the launching interpreter. -- `ui.py` owns Viser controls, scene state, selection, and Edit/Preview workflow. -- `editor_interactions.js` supplies list drag-and-drop and invalid-field styling. - -Local source checks: +30 N clamped. Missing or inactive force data uses the default taxel color. + +## Verify changes + +From `scripts/tactile_metadata`: ```bash ruff format --check tactile_metadata_editor From 508e922530bc1d441982dab2968dd43bd0c676fe Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 20:04:45 -0700 Subject: [PATCH 25/36] page enhancement Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 2 +- scripts/tactile_metadata/README.md | 15 +- scripts/tactile_metadata/UI&Feature Design.md | 2 +- .../editor_interactions.js | 56 +++++ .../tactile_metadata_editor/ui.py | 200 ++++++++++++++---- 5 files changed, 227 insertions(+), 48 deletions(-) diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md index 651f1d4476..cb6122be8d 100644 --- a/scripts/tactile_metadata/AGENTS.md +++ b/scripts/tactile_metadata/AGENTS.md @@ -21,7 +21,7 @@ SPDX-License-Identifier: Apache-2.0 - Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. - New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. - Preserve slot order and repeated segment indices. The first selected Segment receives weight 1 and later selections receive 0; manual weight edits change only that slot. Reject values outside 0~1, and let validation report totals other than 1 without normalization. -- All Taxels renders 50 global rows per page; never instantiate controls for the full Taxel array. The Selected Object panel reuses one Segment or Taxel editor and shows it only for a matching single selection. +- All Taxels renders 25 global rows per page. At both ends, an unlabeled page input and Jump share one line and Last/Next share the next; remove the input's label cell entirely. Page changes retarget a fixed pool of 25 row controls without removing or rebuilding the page container. The Selected Object panel reuses one Segment or Taxel editor and shows it only for a matching single selection. - SDK process isolation specifies where SDK classes are called; document the actual tracker/session calls required for live reception as well as export encoding. - Keep tool implementation inside this subtree; add missing dependencies to Isaac Capture's existing requirements and matching package metadata. Run checks only from this directory or its descendants; never run repository-wide fixing hooks that can change other directories. - Explain this standalone tool's setup from the user's prerequisites: define external libraries before packaging terms, and distinguish requirement files, generated version records, and installation commands. Identify inspected environments by their Python executable path; distinguish declared requirements from packages actually installed there. diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index bed6b5701a..5f3ac433d0 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -31,9 +31,9 @@ Metadata, side, and URDF inputs are optional: ```bash python -m tactile_metadata_editor.cli \ - --metadata ~/Downloads/right.metadata.json \ - --side right \ - --urdf "$(pwd)/model/right/linkerhand_FFG_right.urdf" + --metadata \ + --side \ + --urdf ``` `--side` defaults to `left`. Paths beginning with `~` are expanded. @@ -120,12 +120,15 @@ To bind taxels: At most one binding-step button is enabled at a time. -Expand **All Taxels** to edit 50 rows at a time; **Previous Page** and **Next -Page** switch pages without changing global Taxel indices. Each visible row +Expand **All Taxels** to edit 25 rows at a time. Page status shows the visible +global Taxel range. Use **Jump to page**, **Last Page** (previous page), or +**Next Page** from the controls above or below the rows. The unlabeled input and +Jump button share one line; Last/Next share the following line. Each visible row contains pose, four segment slots, and four weights. `None` is an editor-only inactive slot with weight zero. Selecting the first active segment sets its weight to one; later segment selections start at zero. Editing a weight changes -only that slot. Validation reports rows whose weights do not sum to one. +only that slot. Page changes reuse the same 25 row controls instead of rebuilding +the menu. Validation reports rows whose weights do not sum to one. ## Scene selection diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md index 1a9041795f..5fdf3be90b 100644 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ b/scripts/tactile_metadata/UI&Feature Design.md @@ -53,7 +53,7 @@ About LBS taxel rotation: determine by Polar Decomposition algorithm. At singula a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), rotation (3 float sliders), and a left-labeled four-slot Segment/Weight binding grid matching each Taxel row. Each binding grid has no gaps between cells. Leave standard vertical spacing between this grid and "Capture URDF Surface Pose" without a visible spacer element. "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. "Capture Segment Pose" does the same using the clicked Editing segment's complete pose. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields and copies the binding grid into every new Taxel; newly created taxels are selected and previous objects are deselected. c) Vertically align "Start Binding to Segments", "Select Segments", and "Complete", with a centered down arrow between each adjacent pair. At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The first selected Segment receives weight 1 and later selected Segments receive weight 0. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. * Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. - d) (spannable) A paginated list of all taxels, showing 50 rows per page with Previous/Next controls and global zero-based Taxel indices. Each visible row contains position (3 float input), editable rotation (3 float slider), editable bound segments (4 dropdowns by HandJoint name, with "None" for inactive slots), and four binding weights. Selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. Editing a weight changes only that slot and never modifies the other weights. Weight fields reject numbers out of 0~1, and row validation reports when the total is not 1. + d) (spannable) A paginated list of all taxels, showing 25 rows per page with global zero-based Taxel indices. Above the rows, show page/total and range/total status. The next line contains an unlabeled integer input and "Jump"; the following line contains Last Page (previous) and Next Page. Remove the NumberInput label cell entirely. Repeat the same two control lines below the rows. Build one fixed pool of 25 row controls and retarget their labels/values on page changes; never remove or rebuild the page container. Each visible row contains position (3 float input), editable rotation (3 float slider), editable bound segments (4 dropdowns by HandJoint name, with "None" for inactive slots), and four binding weights. Selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. Editing a weight changes only that slot and never modifies the other weights. Weight fields reject numbers out of 0~1, and row validation reports when the total is not 1. * "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. * Taxels data can be invalid for multiple reasons. When invalid, line the invalid field in red. Reasons including: a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. diff --git a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js index 7523e4fbb2..5bca834006 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js +++ b/scripts/tactile_metadata/tactile_metadata_editor/editor_interactions.js @@ -341,6 +341,61 @@ } }; + const layoutTaxelPagination = () => { + const containers = new Set(); + for (const label of document.querySelectorAll("label")) { + if (!label.textContent?.trim().startsWith("__Taxel Page Jump ")) continue; + let container = label.parentElement; + let controls = []; + while (container) { + const jumpChild = directChild(container, label); + if (jumpChild) { + controls = [jumpChild]; + for (let index = 0; index < 3; ++index) { + controls.push( + controls[controls.length - 1]?.nextElementSibling ?? null, + ); + } + if (controls.every(Boolean)) break; + } + container = container.parentElement; + } + if (!container || controls.some((control) => !control)) continue; + containers.add(container); + const columns = ["1", "2", "1", "2"]; + controls.forEach((control, index) => { + control.dataset.tactilePaginationControl = "true"; + control.style.gridColumn = columns[index]; + control.style.minWidth = "0"; + }); + const input = controls[0].querySelector("input"); + if (label && input) { + const row = directChild(controls[0], input); + const labelColumn = row ? directChild(row, label) : null; + const valueColumn = row ? directChild(row, input) : null; + if (labelColumn && valueColumn) { + labelColumn.style.display = "none"; + valueColumn.style.flex = "1 1 100%"; + valueColumn.style.width = "100%"; + } + } + } + for (const container of containers) { + Object.assign(container.style, { + display: "grid", + gridTemplateColumns: "minmax(0, 1fr) auto", + columnGap: "0.5em", + rowGap: "0.5em", + alignItems: "center", + }); + for (const child of container.children) { + if (child.dataset.tactilePaginationControl !== "true") { + child.style.gridColumn = "1 / -1"; + } + } + } + }; + const preservePartialNumbers = () => { for (const input of document.querySelectorAll("input")) { if (input.dataset.tactileNumberDraft === "true") continue; @@ -373,6 +428,7 @@ layoutSelectedNames(); preservePartialNumbers(); layoutTaxelBindings(); + layoutTaxelPagination(); for (const label of document.querySelectorAll("label")) { if (!label.textContent?.trim().startsWith("⚠")) continue; const field = label.parentElement?.querySelector("input, button"); diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 9665284205..30bc8f5e34 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -41,7 +41,7 @@ ACTIVE_DATASET_OPACITY = 1 BINDING_OVERLAY_OPACITY = 0.4 SAVED_BINDING_OPACITY = 0.3 -TAXEL_PAGE_SIZE = 50 +TAXEL_PAGE_SIZE = 25 class Editor: @@ -76,6 +76,11 @@ def __init__(self, state: State, args): self.segment_item_folders = [] self.segment_error_widgets = [] self.taxel_error_widgets = {} + self.taxel_page_rows = [] + self.taxel_page_jumps = [] + self.taxel_previous_buttons = [] + self.taxel_next_buttons = [] + self.taxel_page_info = None self.edit_joint_widgets, self.preview_joint_widgets = {}, {} self.preview_segment_widgets = [] self.new_taxel_binding = Taxel() @@ -755,7 +760,7 @@ def _capture_segment_pose(self) -> None: def _rebuild_edit_lists(self) -> None: self._rebuild_segment_list() - self._rebuild_taxel_list() + self._sync_taxel_page() self._sync_new_taxel_binding_widgets() self._sync_selection() self._update_preview_error() @@ -844,13 +849,43 @@ def _change_taxel_page(self, delta: int) -> None: if page == self.taxel_page: return self.taxel_page = page - self._rebuild_taxel_list() + self._sync_taxel_page() - def _rebuild_taxel_list(self) -> None: - if self.taxel_page_container is not None: - self.taxel_page_container.remove() + def _jump_taxel_page(self, value) -> None: + page = min( + self._taxel_page_count() - 1, + max(0, int(value) - 1), + ) + if page == self.taxel_page: + return + self.taxel_page = page + self._sync_taxel_page() + + def _add_taxel_page_controls(self, page_count: int, location: str) -> None: + jump_page = self.server.gui.add_number( + f"__Taxel Page Jump {location}", + self.taxel_page + 1, + min=1, + max=page_count, + step=1, + ) + self._action_button( + "Jump", + lambda jump_page=jump_page: self._jump_taxel_page(jump_page.value), + ) + previous = self._action_button("Last Page", lambda: self._change_taxel_page(-1)) + next_page = self._action_button("Next Page", lambda: self._change_taxel_page(1)) + self.taxel_page_jumps.append(jump_page) + self.taxel_previous_buttons.append(previous) + self.taxel_next_buttons.append(next_page) + + def _build_taxel_page(self) -> None: self.taxel_widgets = {} self.taxel_error_widgets = {} + self.taxel_page_rows = [] + self.taxel_page_jumps = [] + self.taxel_previous_buttons = [] + self.taxel_next_buttons = [] issues = self.document.validate() options = self._taxel_segment_options() page_count = self._taxel_page_count() @@ -860,44 +895,49 @@ def _rebuild_taxel_list(self) -> None: with self.taxel_row_folder: self.taxel_page_container = self.server.gui.add_folder(None) with self.taxel_page_container: - previous = self._action_button( - "Previous Page", lambda: self._change_taxel_page(-1) - ) - previous.disabled = self.taxel_page == 0 shown = f"{first}-{last - 1}" if last > first else "None" - self.server.gui.add_markdown( - f"Page **{self.taxel_page + 1} / {page_count}** \n" + self.taxel_page_info = self.server.gui.add_markdown( + f"Page **{self.taxel_page + 1} / {page_count}**" + f"  **{TAXEL_PAGE_SIZE} per Page** \n" f"Taxels: **{shown}** of **{len(self.document.taxels)}**" ) - next_page = self._action_button( - "Next Page", lambda: self._change_taxel_page(1) - ) - next_page.disabled = self.taxel_page >= page_count - 1 - for row in range(first, last): - taxel = self.document.taxels[row] - row_issues = [ - issue - for issue in issues - if issue["kind"] == "taxels" and issue["row"] == row - ] - with self.server.gui.add_folder( - f"Taxel {row}", expand_by_default=False - ): + self._add_taxel_page_controls(page_count, "top") + for page_slot in range(TAXEL_PAGE_SIZE): + row = first + page_slot + visible = row < last + taxel = self.document.taxels[row] if visible else Taxel() + row_issues = ( + [ + issue + for issue in issues + if issue["kind"] == "taxels" and issue["row"] == row + ] + if visible + else [] + ) + folder = self.server.gui.add_folder( + f"Taxel {row}", expand_by_default=False, visible=visible + ) + with folder: position, rotation = geo.components(geo.matrix(taxel.pose)) positions = self._numbers("Position", position, 0.001) rotations = self._sliders("Rotation", rotation) for axis, widget in enumerate(positions): self._watch( widget, - lambda value, row=row, axis=axis: ( - self._set_pose_component("taxels", row, axis, value) + lambda value, page_slot=page_slot, axis=axis: ( + self._set_paged_taxel_pose_component( + page_slot, axis, value + ) ), ) for axis, widget in enumerate(rotations): self._watch( widget, - lambda value, row=row, axis=axis: ( - self._set_pose_rotation("taxels", row, axis, value) + lambda value, page_slot=page_slot, axis=axis: ( + self._set_paged_taxel_pose_rotation( + page_slot, axis, value + ) ), ) self.server.gui.add_html( @@ -923,8 +963,15 @@ def _rebuild_taxel_list(self) -> None: ) segment_fields.append((segment, selected)) for slot, (segment, selected) in enumerate(segment_fields): + raw_weight = taxel.weights[slot] + display_weight = ( + raw_weight + if type(raw_weight) in (int, float) + and math.isfinite(raw_weight) + else 0.0 + ) weight = self._weight_widget( - taxel.weights[slot], + display_weight, selected == "None", any( issue["field"] in (f"weight{slot}", "weights") @@ -934,14 +981,16 @@ def _rebuild_taxel_list(self) -> None: ) self._watch( segment, - lambda value, row=row, slot=slot: ( - self._set_taxel_segment(row, slot, value) + lambda value, page_slot=page_slot, slot=slot: ( + self._set_paged_taxel_segment( + page_slot, slot, value + ) ), ) self._watch( weight, - lambda value, row=row, slot=slot: ( - self._set_taxel_weight(row, slot, value) + lambda value, page_slot=page_slot, slot=slot: ( + self._set_paged_taxel_weight(page_slot, slot, value) ), ) slots.append((segment, weight)) @@ -952,8 +1001,78 @@ def _rebuild_taxel_list(self) -> None: + "", visible=bool(messages), ) - self.taxel_error_widgets[row] = error - self.taxel_widgets[row] = (positions, rotations, slots) + self.taxel_page_rows.append( + (folder, positions, rotations, slots, error) + ) + if visible: + self.taxel_error_widgets[row] = error + self.taxel_widgets[row] = (positions, rotations, slots) + self._add_taxel_page_controls(page_count, "bottom") + + def _paged_taxel_row(self, page_slot: int) -> int | None: + row = self.taxel_page * TAXEL_PAGE_SIZE + page_slot + return row if 0 <= row < len(self.document.taxels) else None + + def _set_paged_taxel_pose_component(self, page_slot: int, axis: int, value) -> None: + row = self._paged_taxel_row(page_slot) + if row is not None: + self._set_pose_component("taxels", row, axis, value) + + def _set_paged_taxel_pose_rotation(self, page_slot: int, axis: int, value) -> None: + row = self._paged_taxel_row(page_slot) + if row is not None: + self._set_pose_rotation("taxels", row, axis, value) + + def _set_paged_taxel_segment(self, page_slot: int, slot: int, value: str) -> None: + row = self._paged_taxel_row(page_slot) + if row is not None: + self._set_taxel_segment(row, slot, value) + + def _set_paged_taxel_weight(self, page_slot: int, slot: int, value) -> None: + row = self._paged_taxel_row(page_slot) + if row is not None: + self._set_taxel_weight(row, slot, value) + + def _sync_taxel_page(self) -> None: + page_count = self._taxel_page_count() + self.taxel_page = min(self.taxel_page, page_count - 1) + if self.taxel_page_container is None: + self._build_taxel_page() + first = self.taxel_page * TAXEL_PAGE_SIZE + last = min(first + TAXEL_PAGE_SIZE, len(self.document.taxels)) + shown = f"{first}-{last - 1}" if last > first else "None" + self.taxel_page_info.content = ( + f"Page **{self.taxel_page + 1} / {page_count}**" + f"  **{TAXEL_PAGE_SIZE} per Page** \n" + f"Taxels: **{shown}** of **{len(self.document.taxels)}**" + ) + for jump in self.taxel_page_jumps: + jump.min = 1 + jump.max = page_count + jump.value = self.taxel_page + 1 + for button in self.taxel_previous_buttons: + button.disabled = self.taxel_page == 0 + for button in self.taxel_next_buttons: + button.disabled = self.taxel_page >= page_count - 1 + + self.taxel_widgets = {} + self.taxel_error_widgets = {} + for page_slot, (folder, positions, rotations, slots, error) in enumerate( + self.taxel_page_rows + ): + row = first + page_slot + visible = row < last + folder.visible = visible + if not visible: + continue + folder.label = f"Taxel {row}" + taxel = self.document.taxels[row] + position, rotation = geo.components(geo.matrix(taxel.pose)) + self._set_values(positions, position) + self._set_values(rotations, rotation) + self.taxel_widgets[row] = (positions, rotations, slots) + self.taxel_error_widgets[row] = error + self._sync_taxel_binding_widgets(range(first, last)) def _weight_widget(self, value, disabled: bool, invalid: bool, slot: int): label = f"{'⚠ ' if invalid else ''}Weight {slot}" @@ -1021,10 +1140,11 @@ def _sync_taxel_binding_widgets(self, rows) -> None: ) else f"Segment {slot}" ) + raw_weight = taxel.weights[slot] weight.value = ( - str(taxel.weights[slot]) - if isinstance(weight.value, str) - else taxel.weights[slot] + float(raw_weight) + if type(raw_weight) in (int, float) and math.isfinite(raw_weight) + else 0.0 ) weight.disabled = selected == "None" weight.label = ( From fa06ba7a20fb366b39ed24f1d6b0fc005cb645e7 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 20:07:00 -0700 Subject: [PATCH 26/36] remove tmp files Signed-off-by: Lotus Li --- scripts/tactile_metadata/AGENTS.md | 55 - scripts/tactile_metadata/DESIGN.md | 226 --- scripts/tactile_metadata/DESIGN.md.license | 3 - .../tactile_metadata/IMPLEMENTATION_PROMPT.md | 38 - scripts/tactile_metadata/UI&Feature Design.md | 99 -- .../model/left/linkerhand_FFG_left.urdf | 1265 ----------------- .../model/left/meshes/base_link.STL | 3 - .../model/left/meshes/index_distal.STL | 3 - .../model/left/meshes/index_metacarpals.STL | 3 - .../model/left/meshes/index_middle.STL | 3 - .../model/left/meshes/index_proximal.STL | 3 - .../model/left/meshes/middle_distal.STL | 3 - .../model/left/meshes/middle_metacarpals.STL | 3 - .../model/left/meshes/middle_middle.STL | 3 - .../model/left/meshes/middle_proximal.STL | 3 - .../model/left/meshes/pinky_distal.STL | 3 - .../model/left/meshes/pinky_metacarpals.STL | 3 - .../model/left/meshes/pinky_middle.STL | 3 - .../model/left/meshes/pinky_proximal.STL | 3 - .../model/left/meshes/ring_distal.STL | 3 - .../model/left/meshes/ring_metacarpals.STL | 3 - .../model/left/meshes/ring_middle.STL | 3 - .../model/left/meshes/ring_proximal.STL | 3 - .../model/left/meshes/thumb_distal.STL | 3 - .../model/left/meshes/thumb_metacarpals.STL | 3 - .../left/meshes/thumb_metacarpals_base1.STL | 3 - .../left/meshes/thumb_metacarpals_base2.STL | 3 - .../model/left/meshes/thumb_proximal.STL | 3 - .../model/right/linkerhand_FFG_right.urdf | 1265 ----------------- .../model/right/meshes/hand_base_link.STL | 3 - .../model/right/meshes/index_distal.STL | 3 - .../model/right/meshes/index_metacarpals.STL | 3 - .../model/right/meshes/index_middle.STL | 3 - .../model/right/meshes/index_proximal.STL | 3 - .../right/meshes/linkerhand_FFG_right.urdf | 1265 ----------------- .../model/right/meshes/middle_distal.STL | 3 - .../model/right/meshes/middle_metacarpals.STL | 3 - .../model/right/meshes/middle_middle.STL | 3 - .../model/right/meshes/middle_proximal.STL | 3 - .../model/right/meshes/pinky_distal.STL | 3 - .../model/right/meshes/pinky_metacarpals.STL | 3 - .../model/right/meshes/pinky_middle.STL | 3 - .../model/right/meshes/pinky_proximal.STL | 3 - .../model/right/meshes/ring_distal.STL | 3 - .../model/right/meshes/ring_metacarpals.STL | 3 - .../model/right/meshes/ring_middle.STL | 3 - .../model/right/meshes/ring_proximal.STL | 3 - .../model/right/meshes/thumb_distal.STL | 3 - .../model/right/meshes/thumb_metacarpals.STL | 3 - .../right/meshes/thumb_metacarpals_base1.STL | 3 - .../right/meshes/thumb_metacarpals_base2.STL | 3 - .../model/right/meshes/thumb_proximal.STL | 3 - 52 files changed, 4348 deletions(-) delete mode 100644 scripts/tactile_metadata/AGENTS.md delete mode 100644 scripts/tactile_metadata/DESIGN.md delete mode 100644 scripts/tactile_metadata/DESIGN.md.license delete mode 100644 scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md delete mode 100644 scripts/tactile_metadata/UI&Feature Design.md delete mode 100644 scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf delete mode 100644 scripts/tactile_metadata/model/left/meshes/base_link.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/index_distal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/index_middle.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/index_proximal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/middle_distal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/middle_middle.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/middle_proximal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_distal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_middle.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/ring_distal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/ring_middle.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/ring_proximal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_distal.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL delete mode 100644 scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL delete mode 100644 scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf delete mode 100644 scripts/tactile_metadata/model/right/meshes/hand_base_link.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/index_distal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/index_middle.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/index_proximal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf delete mode 100644 scripts/tactile_metadata/model/right/meshes/middle_distal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/middle_middle.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/middle_proximal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_distal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_middle.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/ring_distal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/ring_middle.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/ring_proximal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_distal.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL delete mode 100644 scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL diff --git a/scripts/tactile_metadata/AGENTS.md b/scripts/tactile_metadata/AGENTS.md deleted file mode 100644 index cb6122be8d..0000000000 --- a/scripts/tactile_metadata/AGENTS.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# Tactile metadata editor agent notes - -- 工具设计必须明确界面操作、持久化字段、坐标转换、绑定生成和 SDK 接入，不能把需求清单作为完整设计交给实现代理。 -- 设计按用户界面与功能、依赖、代码架构、模块内部实现及约定、代理执行规范分类；正确性、性能和扩展性归入相应类别，不与设计层级并列。代理执行规范与工具设计分开组织。 -- 用户要求完全重写时，从用户操作与数据流重新建立规格和实现提示词，不以搬运原段落或更换标题代替重写。 -- 多语言文档以 REUSE 实际识别的授权信息为准；钩子生成 `.license` 附属文件时，核对并随文档一起暂存，再执行完整检查。 -- 实现以本目录的 `DESIGN.md` 为依据；协议事实以仓库 schema、绑定和共享校验器为准。发现冲突时提供具体代码证据，并同步修正设计与实现提示词。 -- 首版的手套规则留在手部适配层；左右配置独立，通用编辑状态不使用 `HandJoint` 枚举作为内部标识。 -- 评审改变输入范围、术语或编辑实体时，同时更新界面、持久化、算法、验收与实现提示词；已取消的产品概念不能留作隐藏的数据模型。 -- 评审替代性 UI 规格时，以当前稿为产品范围，只把 schema/API 约束视为阻塞；不要把旧功能、状态增强或可接受的瞬态覆盖升级为要求，并逐字核对交互手势。 -- 对设计文档的结构与表达要求应直接落实到目标文档；页面布局按区块列出具体控件，不以聊天中的补充说明代替文档修改。 -- Keep URDF driving separate from taxel deformation: HandPose supplies skeleton and LBS poses, while live URDF motion uses JointState only. Remove obsolete driver controls, persisted options, algorithms, and acceptance cases together. -- Live Preview segments come only from `is_valid=True` HandPose joints in HandJoint order; invalid joints are absent, and an invalid WRIST yields an empty segment list because wrist-relative conversion is unavailable. The selected metadata source controls only taxel bindings and never seeds or overwrites Preview segments. -- Preview deformation is per taxel: omit invalid taxels and report their row errors without suppressing valid taxels. Preserve original row indices when matching live force values. Apply Taxels and Apply ALL copy exactly the valid subset returned for 3D rendering and never require every taxel to be valid. -- Preview apply operations remain independent and available despite taxel errors. Error reasons persist in the fixed Info area after disconnects and unrelated Preview actions. -- Draft forms always read the editable Document. Validation reports errors without blocking draft edits or replacements; received metadata replaces only explicitly selected sections and never becomes a form source by changing the visualization. -- New documents initialize every OpenXR HandJoint segment in the neutral hand pose, mirroring the left hand before visualization-space conversion; loading replaces these defaults instead of merging with them. -- Preserve slot order and repeated segment indices. The first selected Segment receives weight 1 and later selections receive 0; manual weight edits change only that slot. Reject values outside 0~1, and let validation report totals other than 1 without normalization. -- All Taxels renders 25 global rows per page. At both ends, an unlabeled page input and Jump share one line and Last/Next share the next; remove the input's label cell entirely. Page changes retarget a fixed pool of 25 row controls without removing or rebuilding the page container. The Selected Object panel reuses one Segment or Taxel editor and shows it only for a matching single selection. -- SDK process isolation specifies where SDK classes are called; document the actual tracker/session calls required for live reception as well as export encoding. -- Keep tool implementation inside this subtree; add missing dependencies to Isaac Capture's existing requirements and matching package metadata. Run checks only from this directory or its descendants; never run repository-wide fixing hooks that can change other directories. -- Explain this standalone tool's setup from the user's prerequisites: define external libraries before packaging terms, and distinguish requirement files, generated version records, and installation commands. Identify inspected environments by their Python executable path; distinguish declared requirements from packages actually installed there. -- Reuse the launching Python environment and its SDK through sys.executable. Do not create tool-specific project metadata, lockfiles, environments or SDK installations, and do not infer runtime requirements from old local build artifacts. -- Missing packages in the agent's inspection environment limit verification; report the unavailable checks. Do not turn verification requirements into proposals to install dependencies, provision environments, or substitute interpreters; maintain Isaac Capture's shared dependency declarations instead. -- Use Viser's standard controls and public callbacks, with document changes serialized through State. Segment and weight fields in Add Taxels and Taxel rows share the same independent-slot behavior. -- The selected-object transform control enables only XYZ axis translation. Serialize drag phases through the event queue; single selections use object axes, multi-selections use visualization-space axes, and the same translation delta applies to every selected object. -- Selection side-highlighting resolves bindings by HandJoint name and includes only slots with positive weights. Keep `SIDE_HIGHLIGHT_COLOR` distinct from `SELECTED_COLOR`; selection color overrides validation error color on selected taxels. -- When product placement says top/before/below and any sibling uses explicit Viser `order`, assign explicit order to every constrained sibling; source creation order does not override mixed default and explicit ordering. -- Route action and validation errors to the fixed, non-movable client-side Info area at the bottom of the 3D viewport, not a Viser panel, the right menu, per-button, or top-menu markdown. The Selected Object panel is a floating window on the left, not a dock. All Segments/Taxels lists, their entity folders, and Preview Segments start collapsed. -- General is a separate right-docked panel above the right-docked Editor/Preview main panel. -- Client-side helpers may hide only their exact bridge input and label; never hide an ancestor container that can contain the entire Viser control panel. -- Preserve focused formatting-sensitive numeric drafts such as `0.0` and `0.` client-side because Viser canonicalizes them to floats during controlled rerenders. Restore through the native input value setter and dispatch an `input` event; a raw DOM assignment leaves React/Mantine state stale and breaks the next digit. -- Preserve each Viser ScrollArea position across menu operations, with separate positions for Edit and Preview. Only user scrolling changes the stored position. -- Drain cross-thread UI events with nonblocking reads and `queue.Empty`; `Queue.empty()` is not a synchronization boundary. Give no-op-looking actions such as creating an empty document visible completion status. -- Call `/edit`, `/preview`, and `/saved` Viser scene-node groups, not image layers; this tool has no image-layer abstraction. -- During live redraw, update the active scene-node group in place. Hide only groups that should be inactive; never hide and immediately re-show the active group each frame. -- Keep metadata/force alignment with the producer and user; do not add source-row mappings, alignment baselines, force permutations, or list-order controls to the editor. -- JSON follows tactile_metadata.fbs and the existing bindings. Treat array and enum conversion for encoding as routine implementation rather than a separate product-format decision. -- Persist only schema-defined tactile metadata fields. Keep model configuration, calibration, source mappings and internal IDs in session state; audit actual save/export output rather than inferring persistence requirements from editing needs. -- Simplify persistence by deleting unsupported branches in place; do not relocate serialization into Document or introduce new state abstractions for a narrower file format. -- For local editor checks using a cached Ruff executable, match the version in the repository hook configuration; older binaries may not understand the shared lint rules. Keep every check and formatter target inside this subtree. -- Keep the GUI a thin adapter over shared document, geometry, storage and SDK operations. Prefer explicit native Viser widgets and one reusable property inspector; avoid widget-type inference, duplicate field editors in dialogs, and rebuilding scene nodes on every frame. Moving logic between files alone does not reduce implementation size. -- Segment cubes use flat shading with cast and receive shadows disabled; do not restore smooth vertex-normal gradients to semantic markers. -- Use URDF base opacity 0.4 multiplied by the rendered dataset group's opacity, with color-based link highlighting; do not add per-link visibility or opacity controls or state. -- Under a hard code budget, finish and count a complete executable slice before expanding the UI. Refactoring dispatch or moving functions is not evidence that the implementation fits; verify the formatted total and retain budget for integration fixes. -- Treat metadata wrist coordinates, Viser scene coordinates and camera screen orientation as distinct; diagnose an upside-down display from the scene/camera mapping, not URDF alignment. -- Separate OpenXR schema/reference-space contracts from provider basis conversion; verify the plugin conversion before claiming injected HandPose follows OpenXR joint axes. -- Rebase both the raw live skeleton and metadata deformation by the same full inverse WRIST pose; never reattach the world-space wrist transform when the live view is wrist-relative. -- Render viewport direction indicators in client-side screen space from the render camera; perspective scene nodes and server camera callbacks introduce distortion, delay and jumps. -- When reporting a captured protocol frame, include every row in protocol order; a summary may follow but must not replace the requested raw frame. diff --git a/scripts/tactile_metadata/DESIGN.md b/scripts/tactile_metadata/DESIGN.md deleted file mode 100644 index 2b912dc933..0000000000 --- a/scripts/tactile_metadata/DESIGN.md +++ /dev/null @@ -1,226 +0,0 @@ - - -# Tactile Metadata Editor 设计 - -Tactile Metadata Editor 是供触觉手套厂商使用的本地 GUI 工具，用于生成、修改、验证 TactileMetadata，并结合 URDF 和 Isaac Capture 数据检查手部运动与触觉分布。首版支持手套；设备来源和几何映射分别通过适配接口接入，以便跟随 schema 扩展。 - -本文按用户界面和功能、依赖、代码架构、模块内部实现及约定、实现代理规范组织。以下是实施规格，尚不是已实现功能。 - -## 1 用户界面和用户功能设计 - -### 1.1 输入与结果 - -文件输入只支持 URDF 和 TactileMetadata JSON。STL/OBJ 可以是 URDF 引用的网格资源，不作为独立设备输入；不提供 CSV 或裸二进制导入。载入 URDF 后，可新建 metadata，或把已有 metadata JSON 分配到明确的手侧并继续编辑。 - -典型操作是：载入 URDF 并确定手侧，在 Segments 页配置关节与绑定姿态，在指腹标记范围并输入 4×4，批量生成 16 个 taxel，设置输出顺序及最多四个关节的权重，再通过模型运动或实时数据检查。不要求厂商预先持有逐 taxel 位姿、四元数或 skinning weights。 - -保存、备份与导出只写触觉 metadata 数值，使用同一种 schema JSON 表示；可选 SDK 二进制仅编码同一组 metadata 数值。JSON 本身可重新打开编辑，不区分“可编辑版”和“纯数值版”。不生成编辑器工程、`.editor.json`、内部 ID 对应表或检查报告文件。 - -JSON 仅采用 [`tactile_metadata.fbs`](../../src/core/schema/fbs/tactile_metadata.fbs) 定义的字段、嵌套结构及 union 类型标识：顶层为 `geometry_mapping_type`、`geometry_mapping`、`segment_source_type`、`segment_source`；内容为 segment 绑定位姿、taxel 绑定位姿、四槽 segment 索引与权重、`joint_ids`。不添加 `editor` 扩展或其他工具专用字段。Python 使用 `json.load()` 读取；需要 SDK 编码时，在隔离子进程中进行现有绑定所需的数组和枚举转换，调用 `LBS`、`HandSkeletonRef`、`TactileMetadata` 构造器及 `to_bytes()` 生成符合该 schema 的 FlatBuffer。转换复用[现有绑定](../../src/core/schema/python/tactile_metadata_bindings.h)，属于常规实现工作，不构成待定产品规格。 - -URDF 路径、package 映射、q0、腕部校准、segment 与链接关联、手侧分配、实时数据源与关节映射、内部标识和显示状态只存在于当前会话，不写入 metadata 或另存配置文件。检查状态、草稿标记和 revision 只显示在界面或 CLI 输出中。 - -实时输入首版接入 TactileDataArray/TactileMetadata、HandPose 和可选 JointStateOutput。真实设备协议接入、原始 ADC/压力到牛顿的转换、设备控制及物理标定不在工具内完成。工具只接收现有 Isaac Capture 数据；编辑、保存和导出均不向 runtime 或生产设备发送 metadata、控制命令或其他数据。 - -### 1.2 窗口布局与显示 - -一个浏览器窗口包含三维画布和右侧标准 medium 宽度面板。面板顶部显示文件、已固定的 URDF 手侧及当前模式；下方是模型、Segments、布局、绑定、实时、导出六个页签。长表单滚动，复用 Viser 客户端及标准控件，不维护独立前端。 - -```text -┌──────────────────────────────────┬──────────────────────────────┐ -│ │ URDF 名称；手侧：双手（只读） │ -│ 三维模型与手骨架 │ Metadata 文件 * 保存 │ -│ │ 编辑对象：[左手] [右手] │ -│ taxel、选中关节、法线和切线 │ 模式：编辑 / 预览 / 实时 / 暂停│ -│ ├──────────────────────────────┤ -│ 相机：旋转 平移 缩放 重置 │ 模型 Segments 布局 绑定 实时 导出│ -│ 选择：单选 / 框选 / 穿透选择 │ 当前页控件与选择属性 │ -│ ├──────────────────────────────┤ -│ Info：草稿错误状态及具体原因 │ 连接和数据状态 │ -└──────────────────────────────────┴──────────────────────────────┘ -``` - -三维场景采用右手坐标约定：+X 向左、+Y 向上、+Z 向前，长度单位为米。模型为中性色，所有 URDF 链接使用固定不透明度（opacity）0.6，高亮只改变颜色。编辑时 taxel 按主影响关节着色，无绑定为灰色，选中轮廓为黄色，错误轮廓为红色。实时触觉用力向量模长映射蓝至红，默认范围 0–30 N，可调整上限，超限饱和；无有效数据为灰色，不能显示成零力。属性面板显示 fx、fy、fz 和模长，单位 N。 - -球形标记大小只影响显示，不改变 sensing location。选中 taxel 显示 +Z=外向法线 N、+X=传感器切线 T、+Y=N×T；X×Y=Z。双手使用同一右手坐标约定。多关节 LBS 动画只保证位置，不把混合旋转解释为协议定义的唯一方向。 - -六页表单始终显示并编辑 Document 中的草稿值，包括 PREVIEW、LIVE、PAUSED 及连接中断时。三维视图的 metadata 来源切换不改变表单数据；接收值只有通过“记录 metadata”显式替换所选草稿分组后才成为表单值。力值和连接状态是单独的只读实时信息。 - -允许草稿包含错误。出错字段及所属分组显示错误状态和原因，三维视图下方的紧凑 Info 区汇总当前手侧、分组、实体和原因，并可定位对应字段。保留能显示的有效几何；缺失或无效的位姿不以假位置绘制。错误不阻止继续编辑或替换数据，只阻止正式导出。 - -### 1.3 文件、URDF 与左右手 - -启动页提供“载入 URDF”和“打开 metadata JSON”。载入 URDF 时选择左手、右手或双手，完成后只读；替换 URDF 或改变手侧需要创建新文档，先处理未保存内容。URDF 不携带标准手侧字段，该选择是用户对模型的明确声明。 - -一个会话使用一个 URDF。双手选择要求模型实际包含左右手结构，随后分别指定腕部链接和关联链接；不能把单手模型自动镜像成双手。独立左右 URDF 分别打开文档。双手文档中的左右 metadata 独立，但共享同一 URDF 和绑定关节配置；共享关节的修改会明确列出受影响手侧。 - -顶部“编辑对象”只选择当前要编辑的 metadata，不修改模型手侧。单手不显示切侧控件；双手可以显示当前侧或双侧，另一侧 taxel 只读。任何显示排布偏移均不参与元数据计算。两侧数量与输出排列可以不同，各自的 taxel 索引从 0 开始。 - -新建 metadata 不要求输入预期数量，界面从当前 taxel 记录计算 N。检查与实时数据比较时才核对数量。全工具使用“taxel 数量”“taxel 索引”和“输出顺序”，不以 channel 表示 taxel。 - -打开 metadata JSON 时，用户通过界面或 `--sides` 明确分配左侧或右侧，不能从关节名称推断手侧。重新打开只恢复 metadata 数值及 segment、taxel 数组行序；需要表面操作和模型动画时，在当前会话重新载入 URDF、关联链接并设置腕部参考。仅有 JSON 时仍可查看和修改数值。 - -模型、JSON 和 package 资源均使用本地路径，不要求逐个上传网格。未保存标记只比较 metadata 数值；仅改变模型、校准、数据源或显示配置不构成 metadata 未保存。打开新文档、替换一侧 metadata 或退出时，有未保存数值则提供保存、放弃、取消。双手分别保存两个 metadata JSON；“保存全部”处理两侧，失败不得将任一侧误标为已保存。 - -### 1.4 模型页 - -页面提供 URDF 路径、统一正比例缩放、package:// 的本地目录映射和资源状态。缩放同时作用于网格与关节平移，不改变关节角或 metadata 数值。URDF 初始映射将其 +Z、+Y、+X 分别映射到场景 +Y、+X、+Z。可检查模型树、关节类型与范围；缺失资源列出具体路径，不从网络寻找模型。 - -用户为每侧选择腕部链接，并拾取腕中心 O、朝前臂的参考 F、朝手背的参考 D，建立来源根坐标系。双手分别校准，界面显示两套参考轴，不按左/右自动变号。关联链接限制该侧拾取范围；共享掌臂链接可以作为参考，但另一侧表面不能成为默认拾取对象。 - -### 1.5 Segments 页 - -页面位于布局之前，包含“绑定姿态”和“Segments”两组控件。 - -绑定姿态组提供 URDF 自由关节滑块、数值输入和“设为绑定姿态”。fixed 不可编辑，mimic 跟随主关节，限位始终可见。此姿态 q0 是静态布局和 bind poses 的共同参考；实时角度及运动预览角度不会自动写入 q0。 - -Segments 组从 hand.fbs 的 HandJoint 枚举选择需要的关节，排除 NUM_JOINTS。新草稿以腕部为根，在中性张开姿态中初始化全部 26 个 OpenXR 手部关节；左右手使用镜像姿态。初始显示映射将手部 -Z、+X、+Y 分别映射到场景 +Y、-X、-Z。表格显示语义名称、输出行号、关联 URDF 链接和参考系状态。合法 metadata 每侧同名语义只出现一次，不要求完整骨架，WRIST 使用根恒等位姿；草稿违反这些要求时显示错误并保留值。选择一行会高亮关节球、坐标轴和关联链接。 - -新建 segment 时选择语义和链接，在模型上定位关节中心及参考方向，通过控制器调整位置与扭转。已有 metadata 的 B_j 已给出坐标：选择目标链接后计算固定局部修正，保持原 B_j，而不是把数值吸附到 URDF 原点。 - -允许删除仍被引用的 segment；未解决的引用留在草稿中，绑定页和 Info 区显示原因，不隐式重绑。 - -设置新 q0 或修改腕部参考系时，先预览模型和附件变化；现有 B_j、taxel 的绑定根位姿和权重保持数值不变，确认后重算关节局部附件并使检查过期。实际 B_j 的修改通过关节参考系控制器完成。工具不隐式重投影、蒙皮重绑定或修改生产端 metadata。偏离模型的点保留并提示用户调整。 - -### 1.6 布局页 - -页面提供“批量添加”“放置单点”和所选 taxel 的属性。没有区域名、区域列表或持久化批次；批量添加只是一次生成操作，导入 metadata 与新建结果拥有相同编辑能力。 - -批量添加只有一种方式：输入范围和行列数量。操作依次为： - -1. 选择允许投影的链接和表面，输入 rows、cols。 -2. 拾取首点中心 O、首行末点中心 C、首列末点中心 R。显示列跨度、行跨度及推导间距；可直接修改跨度数值，仍属于同一个生成模式。 -3. 选择 row-major、column-major、row-snake 或 column-snake、行列反向和起始 taxel 索引，预览后应用或取消。 - -范围以首末 taxel 中心定义，不是传感片外边缘。单行只需 O/C，单列只需 O/R，单点只需 O 和切线方向，避免除以零。可在预览中排除实际不存在的单元；被排除单元不产生 taxel，投影失败单元则保留为有问题的候选，不能混为一谈。 - -预览显示位置、数量和输出索引，错误标红。应用时原子创建本批独立 taxel，随后丢弃范围、行列和扫描参数。没有基于隐藏区域对象的“重新生成”；需要调整时多选并移动、重新投影或删除后重新批量添加。 - -当前 taxel 数组行序就是输出顺序，不维护另一套输出索引。批量复制到另一侧只复用尚在界面的生成参数，必须重新拾取表面、绑定关节和分配索引，不镜像几何。 - -文件中的输出顺序由 taxel 数组行序表达；内部标识和临时编号不写入文件，重新打开后按文件行序建立编号。 - -选择提供三种入口：单击并用 Ctrl/Cmd 增减；框选画面范围；输入 taxel 索引范围或列表。框选模式暂时接管拖动，退出后恢复相机操作；默认只选择当前侧、可见且未被网格遮挡的点，“穿透选择”显式包含背面。多选属性中的不同值显示“多个值”，批量修改只作用于填写字段。 - -选中后可编辑绑定根中的位置、法线翻转、切线旋转和 taxel 索引。表面移动在指定表面重新拾取，自由移动标记为手工来源。高级折叠项提供 XYZW 四元数，不要求正常用户填写。列表每页 25 条，不创建逐 taxel 常驻控件。 - -### 1.7 绑定页 - -只有一套操作：编辑所选 taxel 的四个索引和权重槽，通常选择一至四个 segment。没有独立的刚性、两段过渡或手工模式。原始槽顺序始终保留，不按 segment 合并重复槽；权重在明确编辑时按下述规则更新。 - -右侧显示四槽，每槽包含 segment 下拉框、原始索引、颜色标记、0–1 权重滑块和独立的权重数值框；上方显示选中 taxel 数量和权重和。缺失、越界及重复索引仍显示原值。新选关节默认等分。通过滑块或数值框调整一个权重时，先替换该槽的值，再对该 taxel 的全部四槽执行 `w_k ← w_k / Σ_i w_i`，使权重和为 1；被调整槽的最终值也参与归一化。拒绝负权重输入；四槽含非有限值、负值或总和为零时，拒绝本次编辑、保留原行并显示原因。载入、显式记录接收值、检查、保存和导出保留原始权重，不自动归一化、累加或修复；其中的非法值仍须报告为错误。 - -选择或调整某槽权重时，通过颜色高亮对应关节、关联链接和当前 taxel，显示关节到 taxel 的连线，线宽或亮度表达权重。多选时保留黄色选择轮廓，以当前关节的权重给所选点着色；混合值显示为混合，不伪造统一值。批量调整对每个 taxel 独立执行四槽归一化。 - -“运动预览”使用草稿数组和 URDF 滑块检查绑定，草稿仍可编辑。“返回绑定姿态”结束预览。单段可显示运动坐标轴；多段只显示位置与力模长，不显示协议未定义的世界空间力箭头。偏离网格的点保持可见，不重新吸附来掩盖误差。 - -### 1.8 实时页 - -页面从上到下包含以下区块与控件： - -**数据源（每侧分别配置）** - -- 触觉 collection_id 输入框。By default "tactile". -- HandTracker 左手／右手输出选择. -- JointState 输入框 (optional, by default disabled). - -**URDF 驱动** - -实时 URDF 只能由启用的 JointState 输入驱动；未启用时隐藏 URDF，仅显示手骨架和 taxel。HandPose 不驱动 URDF，只提供实际手骨架和实时 taxel LBS 当前段位姿。EDIT/PREVIEW 中的 q0 编辑和手动运动预览不受此实时规则影响。 - -- JointState 映射 JSON 文本框：`{源关节名: [URDF 自由关节, 比例, 偏移]}`，整表应用。 -- 各关节的原始输入值、换算值、限位与映射状态。 - -**连接与播放** - -- 连接／断开按钮。 -- 播放／暂停及返回 q0 显示按钮；这些操作不改变草稿的可编辑性。 -- 记录 metadata 按钮，提供分组勾选项，默认全部选中。每组字段一并选择： - - Skeleton binding poses：`joint_ids`、`segment_bind_poses`。 - - Taxel binding poses：`taxel_bind_poses`。 - - Skinning：`taxel_segment_indices`、`taxel_segment_weights`。 -- 显式记录前先按第 1.1 节的相同 schema JSON 格式将当前 metadata 数值备份到临时位置，再替换所选分组；日志显示备份路径。 -- 三维可视化 metadata 来源切换：接收的 metadata／用户草稿，默认接收来源。所有页签中的 metadata 数值始终来自草稿。 -- 接收 metadata 缺失或无效时，HandPose 与 JointState 继续更新，三维 metadata 临时使用草稿并显示原因。 -- 连接状态。各数据源的接入类型、配置标识、有效性和本地读取时间。 -- 工具操作事件、警告和错误日志，包括替换前的草稿备份路径。 - -实时接收和三维播放不自动替换草稿。每次显式记录仅以接收值替换所选分组，不合并旧值；替换后即使产生引用或数组长度错误，也保留结果并在对应区域显示原因。 - -metadata 与力值的实际对应关系由生产端和使用者负责。工具按当前 metadata 与力数组的协议行序显示，执行协议结构、数量及有效性检查；不维护独立的源行映射或对齐基线，不为 metadata 编辑重新排列力值。切换三维 metadata 来源不建立或修复它与力流的对应关系。 - -### 1.9 导出、检查与状态跳转 - -导出页选择当前侧或全部，提供“本地检查”“SDK 检查”和“导出”。问题按 error、warning、info 显示，点击后定位页面和实体。error 阻止正式导出；当前会话中的近似来源产生 warning，显示在检查结果中，不引入逐项确认门槛。 - -本地与 SDK 状态分开。SDK 探测成功只表示环境可用；未完成元数据构造不能显示官方通过。JSON 导出只要求本地无 error；SDK 导出还要求官方构造与大小检查通过。导出目录仅包含各侧 metadata JSON 及用户选择的 SDK 二进制；检查结果输出到界面或 CLI，不作为附属文件保存。 - -| 状态 | 显示与可用操作 | 跳转 | -| --- | --- | --- | -| EDIT | q0 下的 URDF、segments 和 taxel；允许全部静态编辑 | 运动预览进入 PREVIEW；连接并播放进入 LIVE | -| PREVIEW | 滑块驱动模型与 LBS；表单始终编辑草稿 | 返回绑定姿态恢复 EDIT | -| LIVE | 连续更新三维模型、骨架和力值；三维 metadata 来源可切换，所有表单仍显示并编辑草稿 | 暂停进入 PAUSED | -| PAUSED | 冻结实时三维层；接收继续；表单和草稿操纵器仍可编辑 | 恢复取最新帧；返回 q0 显示进入 EDIT | -| EDIT 且已连接 | q0 编辑；后台只保留最新数据，不覆盖文档 | 播放重新进入 LIVE | -| 数据中断 | 保留最后画面并标记失效；草稿继续可编辑，不能把旧力视为当前值 | 可回 EDIT；重连后重新检查数据有效性 | - -所有模式下，数值表单及草稿操纵器都编辑 q0 下的手根坐标，不反解实时混合蒙皮，也不把实时位姿写回草稿。三维操纵器针对明确选中的草稿实体；接收 metadata 的三维图形只读，不能因拾取它而替换表单。PAUSED 只冻结实时三维层，草稿操纵器随编辑更新；恢复后使用最新帧及当前草稿重新计算。播放、暂停、选择、相机和实时帧不修改 Document。 - -本地保存允许未完成的 metadata 数值，仍使用第 1.1 节的相同字段结构；“编辑草稿，尚非有效 metadata”仅为界面提示，不写入文件。正式导出必须完整，校验作为独立步骤，不引入另一种序列化格式。导出使用启动时的 metadata 数值快照，继续编辑不改变导出内容，界面或 CLI 结果显示对应 revision。目标目录非空时要求新目录；全部手侧成功后才发布数值文件。 - -### 1.10 命令行 - -GUI 和 CLI 共用计算、检查与导出，通过 Python 模块入口启动。以下命令使用调用者选定的 Python 环境，从工具目录执行；`python` 指该环境的解释器，工具不替换它： - -```bash -cd scripts/tactile_metadata -python -m tactile_metadata_editor.cli gui --urdf glove.urdf --sides left -python -m tactile_metadata_editor.cli gui --urdf hands.urdf --sides both --left-metadata left.metadata.json --right-metadata right.metadata.json -python -m tactile_metadata_editor.cli gui --metadata left.metadata.json --sides left -python -m tactile_metadata_editor.cli check left.metadata.json -python -m tactile_metadata_editor.cli check left.metadata.json --sdk -python -m tactile_metadata_editor.cli export left.metadata.json --sides left --format sdk --out package -``` - -GUI 不带参数时进入启动页，默认监听 0.0.0.0:8080，可用 --port 修改。JSON 不携带手侧或 URDF 配置：GUI 通过界面或 `--sides` 明确手侧，CLI export 通过 `--sides left/right/both` 分配输入文件，手侧重复明确拒绝；纯数值 CLI 检查不需要手侧。检查结果输出到界面或 CLI 标准输出，不提供 `--report` 文件输出。实时连接在当前 GUI 会话中配置；`--demo` 使用内置合成数据验证实时交互，不是额外文件导入。 - -## 2 依赖 - -| 项目 | 决策 | 原因与用途 | -| --- | --- | --- | -| 语言 | Python >=3.11,<3.14 | 与 SDK 生态衔接，工具本身无需原生编译。 | -| 浏览器 GUI | viser==1.1.1 | 复用三维客户端、表单、框选、射线事件和控制器。 | -| URDF | yourdfpy==0.0.60 | 资源解析、链接图、关节配置和 FK。 | -| 网格处理 | trimesh==5.1.1 | URDF 网格资源、法线、连通分量和射线。 | -| 数值运算 | numpy>=2,<3 | 批量变换、数组校验和 LBS 位置计算。 | -| 查询加速 | rtree>=1.3,<2 | 三角形空间查询，不要求 Embree。 | -| SDK 集成 | 启动工具的 Python 环境中可用的 Isaac Capture SDK | 官方编码以及实时 tracker/session；复用该环境中的 SDK，离线流程不导入 SDK。 | -| 本地通信 | Python 标准库 subprocess、socket、json | SDK 隔离与实时数据传输，不增加消息中间件。 | -| 依赖维护与运行 | Isaac Capture 现有环境依赖声明及 Python 模块入口 | 补齐缺失库，复用启动工具的环境；不建立工具专用项目、虚拟环境或锁文件。 | -| 开发检查 | 仓库已有检查，按用户指定范围执行 | 不新增测试文件或测试依赖，不运行越出工作目录或可能改写其他目录的检查。 | - -将缺失库及所需版本约束加入 Isaac Capture 现有的 [`src/core/python/requirements*.txt`](../../src/core/python/) 对应依赖分组，并同步[根 pyproject.toml](../../pyproject.toml) 等对应包依赖声明。已有依赖按所需版本复用或补充约束，不另建工具专用 `pyproject.toml`、`uv.lock` 或 `.venv`。依赖准备使用调用者选定的同一 Python 环境；工具启动时不自动安装库、创建环境或构建 SDK。 - -版本依据为 [Viser](https://pypi.org/project/viser/1.1.1/)、[yourdfpy](https://pypi.org/project/yourdfpy/0.0.60/) 和 [trimesh](https://pypi.org/project/trimesh/5.1.1/) 发布页。实施时核对共享依赖声明和实际安装版本，并验证模型加载。Viser 框选接口以 [v1.1.1 源码](https://github.com/nerfstudio-project/viser/blob/v1.1.1/src/viser/_scene_api.py) 为准，不把 main 文档新增接口当成指定版本保证。 - -离线编辑不依赖 ROS、CUDA、OpenXR 会话或模拟器。官方检查、SDK 二进制导出和实时接收使用启动工具的 Python 环境中可导入的 SDK；实时接收还需要对应 runtime/设备，当前 live OpenXR 路径需要可用 HMD system，依据[设备文档](../../docs/source/device/tactile/index.rst)。SDK 仅在以 `sys.executable` 启动、继承启动环境的本地子进程中导入。不可用时显示原因，离线编辑仍可运行；不查找旧构建目录、不切换解释器、不提供 SDK 路径配置，也不要求实现机器预先具备未来用户的运行环境。 - - -## 3 给 agent 的范围规定风格规范和 restriction - -### 3.1 范围与工作规范 - -实现第一节定义的完整工具，工具代码及模型示例位于 scripts/tactile_metadata/。允许为本工具补齐 Isaac Capture 现有环境依赖声明及对应包元数据；不建立工具专用环境。不改 schema、golden 或无关子系统；真实冲突提供代码证据并同步规格。设计工具架构以确保well structured and functionally correct，并且concise without redundant codes, strictly limited to the explicitly described scope。制定的实现计划行数预估必须小于2000行（不包括文档）。 - -修改前列出目录并读取全部适用 AGENTS.md，保护已有改动。本地检查，不连接远程计算机，不自动提交或推送。不把本机 SDK/构建/资源绝对路径硬编码到工具或可移植文件。 - -领域类型集中，GUI/CLI 共享数组准备、检查和导出；按模块契约实现，不另造通用框架。遵守仓库日志、SPDX、简短注释、格式化和 Mandatory learning loop。文件输入范围、术语和批量操作生命周期是产品约束，不能为实现便利恢复被移除入口。 - -### 3.2 实施顺序与交付 - -制定实施计划后，按合理的逻辑顺序执行实施。 - -交付复用启动 Python 环境的工具、README中包括使用工具的step to step instruction，以本目录现有model/文件夹内urdf为示例、面向开发者的技术说明（解释模块和数据流等）。不交付工具专用环境或锁文件，不要求新增 URDF、网格或测试文件，也不新增测试依赖。合成实时源可在无 SDK/硬件时运行，实际 live 接收仍须实现，不能用合成数据代替。README 逐步说明在已有 Isaac Capture 环境中准备依赖并启动工具，以及离线编辑、schema JSON 继续编辑、四槽权重归一化、taxel 输出顺序和实时播放/暂停/编辑/恢复。 diff --git a/scripts/tactile_metadata/DESIGN.md.license b/scripts/tactile_metadata/DESIGN.md.license deleted file mode 100644 index 8cf194ccd1..0000000000 --- a/scripts/tactile_metadata/DESIGN.md.license +++ /dev/null @@ -1,3 +0,0 @@ -SPDX-FileCopyrightText: 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md b/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md deleted file mode 100644 index 45bc6065ff..0000000000 --- a/scripts/tactile_metadata/IMPLEMENTATION_PROMPT.md +++ /dev/null @@ -1,38 +0,0 @@ - - -# Tactile Metadata Editor 实现提示词 - -将以下提示词交给能够读取本仓库的实现代理。完整规格见同目录 [DESIGN.md](DESIGN.md)。 - -```text - -请实现 /Users/lotusl/Codes/IsaacTeleop/scripts/tactile_metadata/DESIGN.md -定义的 Tactile Metadata Editor。All implementation should use English instead of Chinese. - -When implementation, make sure to only modify folder /Users/lotusl/Codes/IsaacTeleop/scripts/tactile_metadata and its decendent files. It's where your implementation should fully locate. Don't write to any other paths, except for /tmp and your own workspace, and the IsaacTeleop folder's library config only to add the library dependencies as guided. Don't connect to any remote computer or write to any remote end. You're allowed to read codes in the whole ~/Codes/IsaacTeleop. Don't run pre-commit or other whole-repo formatter/tests. It's OK to run local formatter/test within tactile_metadata. - -先完整读取设计、仓库根及所有修改路径适用的 AGENTS.md，检查已有改动。 -这是实现任务，按已确定的规格工作。编码前完整了解领域、数据流及已有 -能力，以现有 schema、公开绑定、共享校验函数和 tracker/session 核实 -协议。 - -The following product decisions are settled and must be reflected throughout the implementation: -1. Weight editing: selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. A slider or numeric-field edit replaces only that slot and never modifies other weights. Reject non-finite values and values outside 0~1. Preserve slot order and repeated indices. Loading, explicitly recording received metadata, checking, saving, and exporting preserve raw weights; validation reports rows whose weights do not sum to 1. -2. Metadata persistence: saving, backups, and export write only tactile metadata values using the fields, nested structure, and union type discriminators defined by src/core/schema/fbs/tactile_metadata.fbs. The only top-level JSON fields are geometry_mapping_type, geometry_mapping, segment_source_type, and segment_source. Use one schema JSON representation for saving, reopening, and JSON export; optional SDK binaries encode the same values. Do not add an editor extension, tool-specific fields, project/configuration files, .editor.json, internal-ID correspondence tables, or saved check reports. Read with json.load(), perform the array and enum conversion required by the existing SDK constructors, and encode through LBS, HandSkeletonRef, TactileMetadata, and to_bytes() in the isolated SDK subprocess. Do not invent another metadata format or treat this conversion as an unresolved product decision. -3. Scope: metadata/force alignment is the producer's and user's responsibility. Display the current arrays in protocol row order and retain structural, count, and validity checks. Do not maintain source-row mappings, alignment baselines, or force permutations to compensate for metadata edits. Switching visualization sources does not establish or repair force alignment. Persist taxel output order through array row order, without saving internal IDs or temporary numbering. -4. URDF display: all links use fixed opacity 0.6. Do not add per-link visibility or opacity controls or state. Highlight associated links through color without changing opacity; retain mode-level model visibility. -5. Session state: URDF/package paths, q0, wrist calibration, segment/link associations, hand assignments, live-source and joint mappings, internal IDs, and display settings exist only in the current session. Reopening restores only metadata values and array order; the user assigns the hand through the GUI or --sides and sets up the URDF associations and calibration again when needed. Unsaved state tracks metadata values only. Incomplete saves use the same schema fields; show draft status, validation results, and revisions in the GUI or CLI output, never in metadata or companion files. Validate formal exports separately from serialization. Do not implement separate editable/numerical formats, session restoration, or a --report file output. - -Before implementation, do three analysis: -1. whether there's any unresolved 规格矛盾、错误假设、风险、缺失功能或 SDK 能力缺口. -2. whether the implementation can be scoped to 2000 lines of codes or fewer (this impentation shouldn't delete or modify any existing lines, only will add lines). -3. whether any operation would require my manual permission. -If finding any of the gaps above, stop and clarify the gaps. - -Over-engineering is absolutely forbidden. 2000 lines is the hard limit. If ever you exceed this number, it's considered to be a failure, and you need to remove all your changes and analyze what you did wrong. -Now redo the task with the new 2000 line limit and strictly obey the minimal implementation rule. - -``` diff --git a/scripts/tactile_metadata/UI&Feature Design.md b/scripts/tactile_metadata/UI&Feature Design.md deleted file mode 100644 index 5fdf3be90b..0000000000 --- a/scripts/tactile_metadata/UI&Feature Design.md +++ /dev/null @@ -1,99 +0,0 @@ - - -UI&Feature Design - -Global standard: unless improper for specific reason, for all float or int input fields, enable up/down arrow to adjust the value. For float input field, the step should be determined by the estimated range of the value. For int input field, the step should be 1. Only when a field has a clear specific limit range, the input field should reject illegal values. For sliders, when there's a clear specific limit range, use the limit range. -* Preserve partial focused float input such as `1.` while the user edits; controlled UI refreshes must not erase the decimal point before the next digit is entered. -* When "Save", check the validation of metadata. If has error, pop up warning window "Metadata contain errors, are you sure to save?" for user to select continue or cancel. -* Every field marked `(spannable)` is fully collapsed recursively by default: the list container, each entity/joint item, and nested Taxel Binding folders all start collapsed. -* Segment, taxel, and displayed URDF-joint indices start at 0. - -For field names, use title case for multi-word field names. For example, "URDF Path" instead of "urdf path". - -About data cooridanates: all UI editor use visualization space. When save to metadata file, do the coordinate transformation with Segments Alignment value. When load from metadata file, use the default VISUALIZATION_FROM_HAND value as Segments Alignement (and do transformation). Modifying Segments Alignment will cause values in editors also update. -* A fixed three-axis visualization-space direction indicator is displayed in the top-right of the 3D view and updates from the render camera. -* Place the Viser icon at the bottom-left of the view, above the fixed bottom Info area. - -About LBS taxel rotation: determine by Polar Decomposition algorithm. At singularity, use taxel's bind rotation. Make sure to optimize for single-segment binding case, where the rotation can be directly determined without running Polar Decomposition. - -1. Use two separate menus docked on the right: General on top and Editor/Preview below. General contains Toggle side, new/save/load metadata file, reset camera, a named "Mode" section containing bold Edit and Preview checkbox labels that are mutually exclusive and cannot both be unchecked, and a toggle to enable/disable the saved data visualization "Show Saved Binding Poses". - -2. This tool actually manages 3 copies of pose data (pose of segments and taxels), with different data sources. - a) Editing dataset: the data that the user is editing in the tool. It's the binding poses of the segments and taxels under editing. Plus the URDF joint values that's under Editing mode. - b) Saved dataset: the binding data that the metadata file stores. This doesn't contain URDF joint values. - c) Preview dataset: the segments poses, and a set of URDF joint values. It's not a binding data and can't be saved to the metadata file. - -3. When toggle the "Show Saved Binding Poses" on, the 3D scene will display the saved binding poses of the segments and taxels, at a opacity of 0.2. - -4. Two modes use different source of pose data: When switch to Edit mode, the Editing dataset will be used to render in the 3D scene, at the opacity of 1.0. When switch to Preview mode, the Preview dataset will be used to render in the 3D scene, at the opacity of 1.0. - -6. Two modes use different UI menu: When switch to Edit mode, enable "URDF", "Segments", "Taxels" 3 menu tabs. When switch to Preview mode, there won't be tabs. These tabs will be defined explicitly at following. - -7. URDF tab: contains following fields: - a) URDF Path: a text field to input the absolute path of the URDF file. - b) "Load" button, to load the URDF file at URDF Path in scene. - c) URDF Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation, and one float input field to scale the URDF model. - c) A field to modify each joint value in the loaded URDF file. Current UI field "URDF free joint" and "Preview joint" provide this function, but I want you to use a different UI that shows a spannable full list of URDF joints and their input sliders for the user to slide. - -8. Segments tab: - a) Segments Alignment: 3 float input fields to shift the position, 3 float slider fields to rotate the orientation. - b) Add new segments: input fields of segment name (HandJoint enum, dropdown selection), position (3 float input fields). An "Add with Input" button. - c) Add new segments by 3D scene interaction: a button to "Add with URDF Click" button. After clicking this button, the button shows a "pressed" shape, the 3D scene will grey out, with only the URDF (of the Editing dataset) highlighted. The user can click on the URDF to add a new segment on the URDF surface position. Clicking on any other area will cancel the operation. - d) (spannable) A full list of all segments, with an id number (read-only, starting at 0, it's the index of the segment in the list), their name (dropdown list of HandJoint, display in enum name), editable position (3 float input), editable rotation (3 float slider). This is to modify the binding poses of the segments, display this info above this section. -* It's invalid to have more than one segments to select the same HandJoint. At error, outline the repeated fields in red. -* Some fields in Taxels tab refer to segments in this Segments tab, and modifying segments may result in invalid taxel binding that's revealed in Taxels tab. -* Segments are displayed as flat-shaded cubes with side length 0.004m, without smooth color gradients or cast/received shadows. The three axes of each segment are not displayed by default. Clicking a segment on the 3D scene while in Edit mode will select the segment, highlight it, and reveal its 3-axis translation control. When a segment is newly added, it is selected and the previous selected objects are deselected. Clicking any other area in the 3D scene deselects the segment; dragging the 3D scene or clicking menu/UI areas does not. Clicking the same selected segment does nothing. - -9. Taxels tab: -* Taxels don't have their onw alignment, it uses the alignment of the segments. - a) Add new taxels: input fields of columns (int input), rows (int input), area width and height (float inputs in m), center position (3 float inputs), rotation (3 float sliders), and a left-labeled four-slot Segment/Weight binding grid matching each Taxel row. Each binding grid has no gaps between cells. Leave standard vertical spacing between this grid and "Capture URDF Surface Pose" without a visible spacer element. "Capture URDF Surface Pose" enters a pressed state and lets the user click the URDF to replace these Position and Rotation fields with the clicked surface pose without creating taxels. "Capture Segment Pose" does the same using the clicked Editing segment's complete pose. Clicking elsewhere cancels capture. The following "Add Taxels" button creates a row-major rows*columns planar grid from the current fields and copies the binding grid into every new Taxel; newly created taxels are selected and previous objects are deselected. - c) Vertically align "Start Binding to Segments", "Select Segments", and "Complete", with a centered down arrow between each adjacent pair. At most one is clickable: initially Start; after Start, no button until taxels are selected; then Select Segments; after it is clicked, no button until segments are selected; then Complete. The first selected Segment receives weight 1 and later selected Segments receive weight 0. Clicking elsewhere cancels and re-enables Start. More than four selected segments cancels the operation, reports the error in the Info panel at the bottom of the 3D display, and re-enables Start. -* Same logic as 3D scene segments editing: dragging in 3D scene will not cancel the action. - d) (spannable) A paginated list of all taxels, showing 25 rows per page with global zero-based Taxel indices. Above the rows, show page/total and range/total status. The next line contains an unlabeled integer input and "Jump"; the following line contains Last Page (previous) and Next Page. Remove the NumberInput label cell entirely. Repeat the same two control lines below the rows. Build one fixed pool of 25 row controls and retarget their labels/values on page changes; never remove or rebuild the page container. Each visible row contains position (3 float input), editable rotation (3 float slider), editable bound segments (4 dropdowns by HandJoint name, with "None" for inactive slots), and four binding weights. Selecting the first non-None Segment sets that slot's weight to 1; later Segment selections start at 0. Editing a weight changes only that slot and never modifies the other weights. Weight fields reject numbers out of 0~1, and row validation reports when the total is not 1. -* "None" is only for the UI display. When saving to metadata file, it's saved as any of the valid segment in the list, and its weight is 0. -* Taxels data can be invalid for multiple reasons. When invalid, line the invalid field in red. Reasons including: - a) the taxel is not bound to any segment or segment doesn't exist (caused by Segments tab modification). - segment selection fields. - b) the weights don't sum to 1. (Can happen when a segment is removed from Segments tab or by Preview apply to editor.) - weights input fields. -* Taxels are displayed as flat squares of size 0.002m*0.002m. A taxel with any validation error uses ERROR_COLOR unless selected; selection uses SELECTED_COLOR with higher priority. The three axes of each taxel are not displayed by default. Clicking a taxel on the 3D scene while in Edit mode will select the taxel, highlight it, and reveal its 3-axis translation control. The 3D scene modification rules are the same as segments. -* Multi-selection in 3D scene: While holding Shift and clicking on an object in 3D scene, the object will be added to the selection without de-selecting the previous selected object(s). It's not allowed to select both segments and taxels at the same time. So when taxels are already selected, Shift-clicking on a segment will not do anything, vise versa. When holding Shift, it's also allowed to left-button drag on the 3D scene to select a rectangular area of objects. If taxels are previously already selected, only taxels in the selected area will be selected, vise versa. When Shift is holding, dragging 3D scene won't change camera as it does otherwise. When nothing is selected, if a Shift dragging covers both segment and taxels, select the taxels but not segments. -* Selection also highlights related objects with the separate configured `SIDE_HIGHLIGHT_COLOR`: selecting segment(s) highlights taxels with a positive binding weight to any selected HandJoint; selecting taxel(s) highlights segments referenced by a positive binding weight. Zero-weight slots never produce this highlight, and selected objects retain `SELECTED_COLOR`. -* The 3-axis translation control is draggable only along its X/Y/Z axes and moves every selected object by the same translation. For one object it follows that object's orientation; for multiple objects it is centered on the selection with axes aligned to visualization-space coordinates. Rotation, plane translation, and scaling handles are disabled. -* An additional floating menu on the left side of the view is always displayed for selected object(s)' modification; it is not docked to the viewport edge. It applies also to single-object selection. With no selection it shows Type: None and Count: 0 and disables every input field: -a) (read only) selected elements type and count (multi selection is stated in following part) -b) (editable) current position (defined above) -c) (editable) current rotation (defined above) -d) (editable single float) scaling (only for multi-selection, scale up/down the selected objects' position in space, based on the area center. For example, if one object was at (1, 2, 3) and the area center is at (1, 1, 1), scaling the area up to 2x will move the object to (1, 3, 5).) -e) When exactly one Segment is selected, show a Segment Editor for its HandJoint name. -f) When exactly one Taxel is selected, show a Taxel Editor for its four Segment/Weight binding slots. -g) Delete - -10. Preview menu: - a) A named "Binding Metadata Source" section containing bold Editor and Live checkbox labels. They are mutually exclusive and cannot both be unchecked; Editor is selected by default. Live remains disabled/grey until the first Live metadata is received. This selection affects rendered taxel poses because they are calculated from segment poses and the selected binding metadata. - b) A checkbox toggling "Show Binding Poses". When checked, show the segments and taxels (without URDF) from the Editing dataset in the 3D scene at a opacity of 0.5. - c) A text input field "Tactile Collection ID", default to "tactile". - d) Four buttons: - - "Apply Segments" replaces only Editing HandJoint/segment poses from Preview. - - "Apply Taxels" replaces Editing taxels with exactly the valid derived Preview taxels currently rendered in the 3D scene; invalid taxels are omitted and do not block this operation. - - "Apply URDF" replaces only Editing URDF joint values. - - "Apply ALL" applies Preview Segments and URDF values plus exactly the same valid rendered taxel subset used by Apply Taxels. - Invalid Preview taxels do not disable any Apply operation; they are reported in Info and omitted from taxel application. -* When deffering taxel poses or applying binding relationships to the Edit dataset, segments should be referred to by their HandJoint enum name instead of indice. It means received Metadata requires one layer of translation to turn the binding indice into HandJoint. -* If the segment list is incompatible with the metadata's taxel bindings, show the missing HandJoint reason in the fixed Info area. Apply Taxels and Apply ALL remain available and use only the valid rendered subset. -* Preview deformation is independent per taxel. Invalid taxels are omitted and reported by row without preventing valid taxels from rendering. -Live section: - a) Checkbox "Use JointState to Drive URDF". When checked, enable the "JointState collection id" input field, and a Joint Mappings text field to receive a long json string to map the joint names in JointState to the URDF. The current field "Joint mappings JSON: source → [target, scale, offset]" is reusable. - b) A button "Connect Live" button. When clicked, replace the Preview segment list and poses with only the `is_valid=True` HandJoint entries from received HandPose, kept in HandJoint order. The selected binding metadata never seeds, filters, or overwrites Preview segments; it is used only to derive taxel poses from the valid HandPose segments. Manage a field to clearly state the connection state: Live or Disconnected. After clicking "Connect Live", the button changes text to "Disconnect Live". When disconnected, stop updating segment poses or URDF joint values. (If JointState is used, Live also updates URDF joint values.) Show warning: Live will override values in Static section. -* Invalid HandPose joints are absent from Preview. If WRIST is invalid, Preview has no live segments because wrist-relative conversion is unavailable. -* If the metadata -* While Live is connected, if TactileDataArray available, render the taxel force (norm) with corresponding color: 0 is blue, 30 (clamp) is red, in the middle do color interpolation. When force missing, show as light grey (default color). When disconnect, colors go back to light grey. -Static section: - a) A button "Reset Values" that resets Preview segments and URDF joint values from the Editing dataset binding poses and URDF values. - b) (spannable) A full list of segments to adjust values, fields are same as in Segments tab, but the data is for the Preview dataset. Make joint_name read-only. Above the section, show the information that this is for preview only. This field is in spannable style. This value reflects the preview segment poses, so will be updated with the Live mode streaming if hand data is received. - c) (spannable) A full list of URDF joints to adjust values, fields are same as in URDF tab, but the data is for the Preview dataset. Above the section, show the information that this is for preview only. This field is in spannable style. This values reflects real-time URDF joint values of Preview dataset, so will be updated with the Live mode streaming if JointState is applied. -* Segments are selectable and editable from interactive 3D scene under Preview mode, it's technicaly equal to modifying in the Static section. Taxels are not selectable or editable in Preview mode. Preview mode's taxel poses are always derived from its segments poses and the binding metadata it uses. - -Button info display: show error messages in a fixed, non-movable Info area occupying the bottom of the 3D display when an action fails. diff --git a/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf b/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf deleted file mode 100644 index bf2ef60d8b..0000000000 --- a/scripts/tactile_metadata/model/left/linkerhand_FFG_left.urdf +++ /dev/null @@ -1,1265 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/scripts/tactile_metadata/model/left/meshes/base_link.STL b/scripts/tactile_metadata/model/left/meshes/base_link.STL deleted file mode 100644 index caa668eb53..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/base_link.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7c8168b89ad9f225f29bf409719dabab78885fefefffdf2820845937fa9bc542 -size 2470434 diff --git a/scripts/tactile_metadata/model/left/meshes/index_distal.STL b/scripts/tactile_metadata/model/left/meshes/index_distal.STL deleted file mode 100644 index b50fbfb6fc..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/index_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:719d02f83aa1b69693f1b478ca9de4afb804a02630cd6920977c3f75b9a45a99 -size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL deleted file mode 100644 index 3596feb0e1..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/index_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:37c0c7070aa833f085bcf7120d903977abdafe3cbf95173eecd684be794afdd7 -size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/index_middle.STL b/scripts/tactile_metadata/model/left/meshes/index_middle.STL deleted file mode 100644 index c6066e326a..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/index_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:60c01971d5558a913db3431b3186245bccec6cd296800685b5324faa5833c725 -size 188084 diff --git a/scripts/tactile_metadata/model/left/meshes/index_proximal.STL b/scripts/tactile_metadata/model/left/meshes/index_proximal.STL deleted file mode 100644 index b1dbb3e657..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/index_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:87ddd901598bab2e64a11ec61193c3056ebfdd259d32fc4ed0da208e844b83a6 -size 652384 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_distal.STL b/scripts/tactile_metadata/model/left/meshes/middle_distal.STL deleted file mode 100644 index 2a01de4d6c..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/middle_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a1bf0b18f6a6e77089cc8f392e78bb832415570e388f9c61e98f956d5604a61f -size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL deleted file mode 100644 index 2d5e64465b..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/middle_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f74d86cc7e7290ff7ed873ce038c7b33718b0d0d157deaaa4ff964ece0abc828 -size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_middle.STL b/scripts/tactile_metadata/model/left/meshes/middle_middle.STL deleted file mode 100644 index fe4baad770..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/middle_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:abd61a575ea84c33362107e90c8964b4bd6f544cbda8ac2098651be8482947db -size 188084 diff --git a/scripts/tactile_metadata/model/left/meshes/middle_proximal.STL b/scripts/tactile_metadata/model/left/meshes/middle_proximal.STL deleted file mode 100644 index b4c4e99e78..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/middle_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:da4562a6e80b940e89ce26c9a12828d2eadba1cd4d4987995c2d484de1b326ba -size 652384 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_distal.STL b/scripts/tactile_metadata/model/left/meshes/pinky_distal.STL deleted file mode 100644 index e9f861bf6b..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/pinky_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:502aa6ff2d997975546a0b244af2969046c3e5b286d13ae482f4498e0deeb638 -size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL deleted file mode 100644 index e98beab8ac..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/pinky_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:0d00911724b3d736d22c79d2cc3b52cf4b7449e3635a4b328b90370452b9cb67 -size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_middle.STL b/scripts/tactile_metadata/model/left/meshes/pinky_middle.STL deleted file mode 100644 index 5529fa5991..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/pinky_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e72ffa11712ece3a77441d11876dc8dd0c881570e49b6c32f909b6d6e3d0d7da -size 226484 diff --git a/scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL b/scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL deleted file mode 100644 index a688ef8a46..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/pinky_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9fc1bf89257a7a0b6d735d10178c33ea9e3347c7419ce9ac45830b7b2e042486 -size 654284 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_distal.STL b/scripts/tactile_metadata/model/left/meshes/ring_distal.STL deleted file mode 100644 index 557d08cfa2..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/ring_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:9e4a4001c3797a4294c43a5902c5efc495136eaf3d6e8555552ef8ed8c850b59 -size 282484 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL deleted file mode 100644 index 98f858c212..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/ring_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:819a47007b09596026a4b9454a8fe5c101a3e88c4c83eb3d8b448ea3bcf6a161 -size 238984 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_middle.STL b/scripts/tactile_metadata/model/left/meshes/ring_middle.STL deleted file mode 100644 index f0e71a8f3a..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/ring_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:888c84e49b57a86626cc797bf71c5a7a7cfc867d80ee0f5e8591fdad3164bcd1 -size 188084 diff --git a/scripts/tactile_metadata/model/left/meshes/ring_proximal.STL b/scripts/tactile_metadata/model/left/meshes/ring_proximal.STL deleted file mode 100644 index 0c95de5c7c..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/ring_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8df7a9d6a15fd254ef66307f2044457abad5d6b439617718ff03209d29584d8b -size 652384 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_distal.STL b/scripts/tactile_metadata/model/left/meshes/thumb_distal.STL deleted file mode 100644 index 01ede11968..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/thumb_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:833a95573cf8de2e33b40896fc6de36ab9af8bccad9b8f6c8a9ac5c2699b5a62 -size 276184 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL deleted file mode 100644 index 399e701edc..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:7857b0bf11381e343c66ee63ec8c5289e294dd20adc04c78125e26ec45f8c083 -size 161184 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL deleted file mode 100644 index c0b399f3f6..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base1.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:90414281d61e1bad2073836dd754d5b0e0ce8b9179f3c23aefe95f0e0e3624dd -size 485784 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL b/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL deleted file mode 100644 index 63b4666ea9..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/thumb_metacarpals_base2.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b225b009ac9803e6f59476316e037710f0a86fa4d920272c2a4e865126630229 -size 628484 diff --git a/scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL b/scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL deleted file mode 100644 index d6310ac610..0000000000 --- a/scripts/tactile_metadata/model/left/meshes/thumb_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:6fbba678d35ba609a3abc62c4dececab1e3d0104ae5a6bdee918e89905eb65dc -size 210584 diff --git a/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf b/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf deleted file mode 100644 index 341ce5cbb4..0000000000 --- a/scripts/tactile_metadata/model/right/linkerhand_FFG_right.urdf +++ /dev/null @@ -1,1265 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/scripts/tactile_metadata/model/right/meshes/hand_base_link.STL b/scripts/tactile_metadata/model/right/meshes/hand_base_link.STL deleted file mode 100644 index c2a62e2e7e..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/hand_base_link.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:94c50cfcc29be7179f9e6d8352ab5a3213c21e4795cb8d58a346ebfff74ee8bc -size 1220684 diff --git a/scripts/tactile_metadata/model/right/meshes/index_distal.STL b/scripts/tactile_metadata/model/right/meshes/index_distal.STL deleted file mode 100644 index eb1de0fb2d..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/index_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b806badc8765acb6ed9401a0fb23d90e7ae6ff2f3a77ff40b014e063140b8e06 -size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL deleted file mode 100644 index bc629465fb..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/index_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:df03f980f92a0c66662081fb4c147294c60b9acdf8cc7905e21a9790d6c7720d -size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/index_middle.STL b/scripts/tactile_metadata/model/right/meshes/index_middle.STL deleted file mode 100644 index 4846357f98..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/index_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:17f81c7ad8297dac033dded0f0c89a521f766208a8efadb1b0f7c57d3e7c374a -size 188084 diff --git a/scripts/tactile_metadata/model/right/meshes/index_proximal.STL b/scripts/tactile_metadata/model/right/meshes/index_proximal.STL deleted file mode 100644 index 5b902ded0d..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/index_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:57b1aa6fc7a303a6b18417eb078092c332fa577f548d5747b500253a9daecae1 -size 651984 diff --git a/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf b/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf deleted file mode 100644 index c7e4515b77..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/linkerhand_FFG_right.urdf +++ /dev/null @@ -1,1265 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/scripts/tactile_metadata/model/right/meshes/middle_distal.STL b/scripts/tactile_metadata/model/right/meshes/middle_distal.STL deleted file mode 100644 index 148ba11db8..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/middle_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:459a2b44a893cb42eb2b4671f17985f1dabaa9bce6371f190e69a60badd1299f -size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL deleted file mode 100644 index 8a49231528..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/middle_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a2583b7237b2a33a7ad2ddc17f1ff61c6ec5d38335e8ba1340d43e6efecc1256 -size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/middle_middle.STL b/scripts/tactile_metadata/model/right/meshes/middle_middle.STL deleted file mode 100644 index d31133bc28..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/middle_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2e533f80d298bf783b5a20c30bae383fe53fe1d3b4430268070655f363ace2ba -size 188084 diff --git a/scripts/tactile_metadata/model/right/meshes/middle_proximal.STL b/scripts/tactile_metadata/model/right/meshes/middle_proximal.STL deleted file mode 100644 index 349a109f75..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/middle_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:8cff8d8a4a6b719b77fc525351ad0fbb0c961a73b859b0b0a4c07190a0aa186a -size 651984 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_distal.STL b/scripts/tactile_metadata/model/right/meshes/pinky_distal.STL deleted file mode 100644 index 9e86b13165..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/pinky_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:ca40e877cf35deaea1ccdb6880ded2f1d85460d47d8c1e31cdd36f0ef1c1d05c -size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL deleted file mode 100644 index 54b5a30e9d..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/pinky_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:a9d130c833b07a827a6ea194fcf3844dcc0947c7aad9f6c954d7ec7c3c9aefaa -size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_middle.STL b/scripts/tactile_metadata/model/right/meshes/pinky_middle.STL deleted file mode 100644 index 4cca9d63fe..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/pinky_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:45ce34033924e8a5392bbacbd9f7af752110606c68ff33f24daad8f3941fc8ab -size 226484 diff --git a/scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL b/scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL deleted file mode 100644 index 2e71e87d48..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/pinky_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:201bd33cc07460751846b8ca8e355c30d55c82d986874e8db73f8cdd40e33609 -size 654084 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_distal.STL b/scripts/tactile_metadata/model/right/meshes/ring_distal.STL deleted file mode 100644 index 7ab5495a38..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/ring_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:1953f24ce5d1b99658c64ef589c4ad5ae51f4538833d5dfa4b18514d3cc39fc7 -size 282484 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL deleted file mode 100644 index 2b8bcc7138..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/ring_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:b4b7d52ccc032e6c3c3061bdfe2478ebf6d68bc0ddcbb37723f77e18e2059532 -size 237984 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_middle.STL b/scripts/tactile_metadata/model/right/meshes/ring_middle.STL deleted file mode 100644 index e68bcbb27e..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/ring_middle.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5e079aa92ed3362b0e0bdbf6d2467927804df65c2d2a333a70f57dc7c6a60ff8 -size 188084 diff --git a/scripts/tactile_metadata/model/right/meshes/ring_proximal.STL b/scripts/tactile_metadata/model/right/meshes/ring_proximal.STL deleted file mode 100644 index 169b959e80..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/ring_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:5f726495555c728c61759f710d864f4688c0d9a8c3df6fb07bca02840b9498c8 -size 651984 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_distal.STL b/scripts/tactile_metadata/model/right/meshes/thumb_distal.STL deleted file mode 100644 index 3a24c5e2be..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/thumb_distal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:4f857160f2c186bd105f29ac90a630d6918612cbd97a010654e12d873c0546cf -size 276184 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL deleted file mode 100644 index 1c155faaed..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:f5b61b2e29e6eaa5623980d00465fb3a4f93c3b6d978fc0f50a703827cb2642c -size 161384 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL deleted file mode 100644 index 20919e64e3..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base1.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:dfdaebb2b720a4949986740bbadc3da519fe46c09bff8e145ead0640977256bc -size 484884 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL b/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL deleted file mode 100644 index eac307de4f..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/thumb_metacarpals_base2.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:2d4a436027f4b5ade7c50d15a334a109a3a4562073118ef5382784748ef55600 -size 628484 diff --git a/scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL b/scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL deleted file mode 100644 index bb116eb9e1..0000000000 --- a/scripts/tactile_metadata/model/right/meshes/thumb_proximal.STL +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:45237da8babd57d1b32b250d8bd6d313679e8fad313e1f9a793fba82c5e35a60 -size 210584 From 4147447dee935b7c7fdfdfc7cfefa1b42633c810 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 29 Oct 2026 22:44:22 -0700 Subject: [PATCH 27/36] Fix Apply ALL to preserve preview taxel geometry Signed-off-by: lotusl --- .../tactile_metadata_editor/ui.py | 2 +- scripts/tactile_metadata/test_editor.py | 47 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 scripts/tactile_metadata/test_editor.py diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 30bc8f5e34..9018f48741 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -1377,8 +1377,8 @@ def _apply_preview_urdf(self) -> None: def _apply_preview_all(self) -> None: document = self.document - document.segments = copy.deepcopy(self.preview_document.segments) rows, _, _ = self._rendered_preview_taxels() + document.segments = copy.deepcopy(self.preview_document.segments) document.taxels = copy.deepcopy([taxel for _, taxel in rows]) document.q = copy.deepcopy(self.preview_document.q) self._rebuild_edit_lists() diff --git a/scripts/tactile_metadata/test_editor.py b/scripts/tactile_metadata/test_editor.py new file mode 100644 index 0000000000..44525ab2b3 --- /dev/null +++ b/scripts/tactile_metadata/test_editor.py @@ -0,0 +1,47 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from types import SimpleNamespace +from unittest.mock import Mock + +import numpy as np + +from tactile_metadata_editor.document import Segment, State, Taxel +from tactile_metadata_editor.ui import Editor + + +class EditorTests(unittest.TestCase): + def make_editor(self): + editor = Editor.__new__(Editor) + editor.state = State() + editor.document.segments = [Segment("WRIST")] + editor.document.taxels = [ + Taxel(segments=["WRIST", None, None, None], weights=[1.0, 0.0, 0.0, 0.0]) + ] + editor.preview = {"left": editor._preview_from(editor.document)} + editor.binding_source_live = SimpleNamespace(value=False) + editor.info = SimpleNamespace(value="") + editor.edit_joint_widgets = {} + editor._rebuild_edit_lists = Mock() + editor._sync_joint_widget_values = Mock() + return editor + + def test_apply_all_preserves_rendered_taxels(self): + editor = self.make_editor() + editor.preview_document.segments[0].pose[0] = 0.01 + editor.preview_document.q = {"joint": 0.5} + rows, errors, _ = editor._rendered_preview_taxels() + self.assertFalse(errors) + expected = rows[0][1].pose + self.assertAlmostEqual(expected[0], 0.01) + + editor._apply_preview_all() + + np.testing.assert_allclose(editor.document.taxels[0].pose, expected) + self.assertEqual(editor.document.segments, editor.preview_document.segments) + self.assertEqual(editor.document.q, editor.preview_document.q) + + +if __name__ == "__main__": + unittest.main() From 0b268bd5d942caad398da4fa2534fe53fd26fbf7 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 29 Oct 2026 22:45:07 -0700 Subject: [PATCH 28/36] Reuse selected taxels when starting segment binding Signed-off-by: lotusl --- .../tactile_metadata_editor/ui.py | 10 +++++-- scripts/tactile_metadata/test_editor.py | 29 +++++++++++++++++++ 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 9018f48741..aae4a5b94b 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -1206,9 +1206,13 @@ def _set_taxel_weight(self, row: int, slot: int, value) -> None: def _start_binding(self) -> None: self._set_pick_mode(None) self.binding_phase = "taxels" - self.binding_taxels, self.binding_segments = [], [] - self.selected_kind, self.selected_rows = None, [] - self._set_binding_button(None) + self.binding_taxels = ( + self.selected_rows.copy() if self.selected_kind == "taxels" else [] + ) + self.binding_segments = [] + if not self.binding_taxels: + self.selected_kind, self.selected_rows = None, [] + self._set_binding_button("next" if self.binding_taxels else None) self.binding_info.content = ( "Select taxel(s) to bind in 3D scene. Hold Shift to multi-select. " 'Click "Select Segments" after selecting taxels. Click another area to cancel.' diff --git a/scripts/tactile_metadata/test_editor.py b/scripts/tactile_metadata/test_editor.py index 44525ab2b3..bfdb0ca658 100644 --- a/scripts/tactile_metadata/test_editor.py +++ b/scripts/tactile_metadata/test_editor.py @@ -42,6 +42,35 @@ def test_apply_all_preserves_rendered_taxels(self): self.assertEqual(editor.document.segments, editor.preview_document.segments) self.assertEqual(editor.document.q, editor.preview_document.q) + def test_start_binding_reuses_only_selected_taxels(self): + for kind, rows in ( + ("taxels", [0]), + ("taxels", [0, 1]), + ("segments", [0]), + (None, []), + ): + with self.subTest(kind=kind, rows=rows): + editor = self.make_editor() + editor.document.taxels.append(Taxel()) + editor.selected_kind, editor.selected_rows = kind, rows.copy() + editor._set_pick_mode = Mock() + editor.binding_info = SimpleNamespace(content="") + editor.binding_start = SimpleNamespace(disabled=False) + editor.binding_next = SimpleNamespace(disabled=True) + editor.binding_complete = SimpleNamespace(disabled=True) + + editor._start_binding() + + expected = rows if kind == "taxels" else [] + self.assertEqual(editor.binding_taxels, expected) + self.assertEqual(editor.binding_segments, []) + self.assertEqual(editor.selected_rows, expected) + self.assertEqual(editor.selected_kind, "taxels" if expected else None) + self.assertEqual(editor.binding_phase, "taxels") + self.assertTrue(editor.binding_start.disabled) + self.assertEqual(editor.binding_next.disabled, not expected) + self.assertTrue(editor.binding_complete.disabled) + if __name__ == "__main__": unittest.main() From 37c21ab30ec931c4ae4b78f9aac40e35e60e4bb9 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 29 Oct 2026 22:45:54 -0700 Subject: [PATCH 29/36] Preserve missing tactile bindings as invalid uint32 indices Signed-off-by: lotusl --- .../tactile_metadata_editor/storage.py | 8 ++++++-- scripts/tactile_metadata/test_editor.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/storage.py b/scripts/tactile_metadata/tactile_metadata_editor/storage.py index 975407e3ec..0552b0284a 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/storage.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/storage.py @@ -108,7 +108,6 @@ def metadata(document: Document) -> dict: inverse = np.linalg.inv(np.asarray(document.alignment, dtype=float)) joints = [segment.joint for segment in document.segments] indices = {joint: row for row, joint in enumerate(joints)} - fallback = 0 segment_poses = [ _json_pose(pose(inverse @ matrix(segment.pose))) for segment in document.segments @@ -117,7 +116,12 @@ def metadata(document: Document) -> dict: _json_pose(pose(inverse @ matrix(taxel.pose))) for taxel in document.taxels ] binding_indices = [ - {"values": [indices.get(joint, fallback) for joint in taxel.segments]} + { + "values": [ + indices.get(joint, 0 if joint is None else 0xFFFFFFFF) + for joint in taxel.segments + ] + } for taxel in document.taxels ] binding_weights = [{"values": list(taxel.weights)} for taxel in document.taxels] diff --git a/scripts/tactile_metadata/test_editor.py b/scripts/tactile_metadata/test_editor.py index bfdb0ca658..db4f55ac73 100644 --- a/scripts/tactile_metadata/test_editor.py +++ b/scripts/tactile_metadata/test_editor.py @@ -7,6 +7,7 @@ import numpy as np +from tactile_metadata_editor import storage from tactile_metadata_editor.document import Segment, State, Taxel from tactile_metadata_editor.ui import Editor @@ -71,6 +72,25 @@ def test_start_binding_reuses_only_selected_taxels(self): self.assertEqual(editor.binding_next.disabled, not expected) self.assertTrue(editor.binding_complete.disabled) + def test_deleted_segment_remains_invalid_after_round_trip(self): + document = self.make_editor().document + document.segments.append(Segment("INDEX_TIP")) + document.taxels[0].segments[0] = "INDEX_TIP" + document.delete("segments", [1]) + + encoded = storage.metadata(document) + expected = [{"values": [0xFFFFFFFF, 0, 0, 0]}] + self.assertEqual(encoded["geometry_mapping"]["taxel_segment_indices"], expected) + + loaded = storage.parse(encoded, document.side) + self.assertTrue( + any(issue["field"] == "segment0" for issue in loaded.validate()) + ) + self.assertEqual( + storage.metadata(loaded)["geometry_mapping"]["taxel_segment_indices"], + expected, + ) + if __name__ == "__main__": unittest.main() From 99d7713cf00370fb93de015877027a527721c2cb Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 22:45:03 -0700 Subject: [PATCH 30/36] Revert "Fix Avatar hand coordinate basis" This reverts commit 645550d20435d3030337ce7470942407609587e8. Signed-off-by: lotusl --- .../sharpa_avatar/core/avatar_hand_tracking_plugin.cpp | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp b/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp index 32fd6ade2d..9a7df1454d 100644 --- a/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp +++ b/src/plugins/sharpa_avatar/core/avatar_hand_tracking_plugin.cpp @@ -203,12 +203,6 @@ constexpr std::array, XR_HAND_JOINT_COUNT_EXT> kOpenXRSlot 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, std::nullopt, std::nullopt, 24, }; -// Avatar HUMAN uses +Z toward the fingertips and -X toward the back of the hand. -// OpenXR uses -Z toward the fingertips and +Y toward the back of the hand. -constexpr float kInvSqrt2 = 0.70710678f; -constexpr XrPosef kAvatarToOpenXR = { { kInvSqrt2, -kInvSqrt2, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }; -constexpr XrPosef kOpenXRToAvatar = { { -kInvSqrt2, kInvSqrt2, 0.0f, 0.0f }, { 0.0f, 0.0f, 0.0f } }; - } // anonymous namespace // Wrist offsets for XR controllers: align the controller aim pose to a natural @@ -691,9 +685,7 @@ void AvatarTracker::map_landmarks_to_openxr(const std::vector<::avatar::Pose>& l local_pose.orientation.z = lp.orientation.z; local_pose.orientation.w = lp.orientation.w; - const XrPosef openxr_pose = - oxr_utils::multiply_poses(kAvatarToOpenXR, oxr_utils::multiply_poses(local_pose, kOpenXRToAvatar)); - out_joints[j].pose = oxr_utils::multiply_poses(root_pose, openxr_pose); + out_joints[j].pose = oxr_utils::multiply_poses(root_pose, local_pose); out_joints[j].radius = 0.01f; out_joints[j].locationFlags = XR_SPACE_LOCATION_POSITION_VALID_BIT | XR_SPACE_LOCATION_ORIENTATION_VALID_BIT; From c8b86ecbe54bc8d1c308c3806577cbc610bc37c5 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 29 Oct 2026 22:47:40 -0700 Subject: [PATCH 31/36] Remove redundant tactile preview state and error handling Signed-off-by: lotusl --- .../tactile_metadata_editor/ui.py | 37 +++---------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index aae4a5b94b..5d679b4b6d 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -92,7 +92,6 @@ def __init__(self, state: State, args): self.binding_taxels: list[int] = [] self.binding_segments: list[int] = [] self.pick_mode: str | None = None - self.preview_has_errors = False self.selection_rotation = np.zeros(3) self.selection_scale = 1.0 self.selection_gizmo_dragging = False @@ -280,18 +279,10 @@ def _build_gui(self) -> None: ) self.show_binding = gui.add_checkbox("Show Binding Poses", False) self.collection = gui.add_text("Tactile Collection ID", "tactile") - self.apply_segments_button = self._action_button( - "Apply Segments", self._apply_preview_segments - ) - self.apply_taxels_button = self._action_button( - "Apply Taxels", self._apply_preview_taxels - ) - self.apply_urdf_button = self._action_button( - "Apply URDF", self._apply_preview_urdf - ) - self.apply_all_button = self._action_button( - "Apply ALL", self._apply_preview_all - ) + self._action_button("Apply Segments", self._apply_preview_segments) + self._action_button("Apply Taxels", self._apply_preview_taxels) + self._action_button("Apply URDF", self._apply_preview_urdf) + self._action_button("Apply ALL", self._apply_preview_all) gui.add_markdown( "Apply ALL fully overrides the editor's segments, taxels, bindings, and URDF joint values." ) @@ -1339,16 +1330,11 @@ def _rendered_preview_taxels( def _update_preview_error(self) -> None: _, errors, _ = self._rendered_preview_taxels() - self.preview_has_errors = bool(errors) self.preview_error.value = ( '' + "
".join(errors) + "" if errors else "" ) - self.apply_taxels_button.disabled = False - self.apply_all_button.disabled = False - self.apply_segments_button.disabled = False - self.apply_urdf_button.disabled = False def _document_error_html(self) -> str: messages = [] @@ -1488,17 +1474,10 @@ def _update_live(self) -> None: self.preview_document.q[target] = raw * scale + offset self._sync_preview_values() self._update_preview_error() - except ( - ValueError, - TypeError, - KeyError, - IndexError, - json.JSONDecodeError, - ) as exc: + except (ValueError, TypeError, KeyError, IndexError) as exc: self.preview_error.value = ( f'Live error: {exc}' ) - self.preview_has_errors = True self.redraw = True def _sync_preview_values(self) -> None: @@ -2208,11 +2187,7 @@ def _draw(self) -> None: True, ) else: - try: - valid_rows, _, source_count = self._rendered_preview_taxels() - except (ValueError, TypeError, KeyError, IndexError): - valid_rows = [] - source_count = 0 + valid_rows, _, source_count = self._rendered_preview_taxels() preview_taxels = [taxel for _, taxel in valid_rows] raw_forces = ( self.receiver.latest.get("forces") From 27308862d303fed1020e09c7fe1058cc8a5c3546 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 29 Oct 2026 23:18:48 -0700 Subject: [PATCH 32/36] Keep tactile metadata independent of hand side Signed-off-by: lotusl --- scripts/tactile_metadata/README.md | 15 ++-- .../tactile_metadata_editor/cli.py | 3 +- .../tactile_metadata_editor/document.py | 24 ++---- .../tactile_metadata_editor/storage.py | 10 +-- .../tactile_metadata_editor/ui.py | 79 +++++++++---------- scripts/tactile_metadata/test_document.py | 31 ++++++++ scripts/tactile_metadata/test_editor.py | 33 +++++++- 7 files changed, 120 insertions(+), 75 deletions(-) create mode 100644 scripts/tactile_metadata/test_document.py diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index 5f3ac433d0..bf40b7ea98 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -36,7 +36,8 @@ python -m tactile_metadata_editor.cli \ --urdf ``` -`--side` defaults to `left`. Paths beginning with `~` are expanded. +`--side` defaults to `left` and initializes **Hand Side**. Paths beginning with +`~` are expanded. Offline editing uses Viser, NumPy, yourdfpy, trimesh, and rtree. Live reception also requires the Isaac Capture SDK, a compatible OpenXR runtime, and an @@ -44,8 +45,10 @@ available HMD system in the same Python environment. ## Data and coordinates -The editor maintains independent left- and right-hand documents. **Toggle Side** -selects the active document; each side has its own metadata path and data. +The editor maintains one metadata document and file path. **Hand Side** selects +the default poses used by **New** and the hand stream used by **Connect Live**. +Changing it disconnects Live and clears received metadata without switching, +mirroring, or replacing the current document or its Saved and Preview datasets. General uses a separate right-docked menu above the Editor/Preview menu. The Selected Object panel is always present as a floating window on the left. The **Mode** section uses bold, mutually exclusive **Edit** and **Preview** @@ -167,9 +170,9 @@ Enter **Metadata Path**, then use **Load** or **Save**. Saving validates the current document. If errors remain, the editor asks whether to continue; invalid drafts can still be written. -**New** resets the active side and initializes all OpenXR HandJoint segments in -the neutral hand pose. The left-hand defaults are mirrored before conversion to -visualization space. +**New** resets the document and initializes all OpenXR HandJoint segments in the +neutral pose for **Hand Side**. The left-hand defaults are mirrored before +conversion to visualization space. The JSON contains only schema fields: diff --git a/scripts/tactile_metadata/tactile_metadata_editor/cli.py b/scripts/tactile_metadata/tactile_metadata_editor/cli.py index b3d65b7b04..4b83a3981b 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/cli.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/cli.py @@ -21,10 +21,9 @@ def main() -> int: from .ui import Editor state = State() - state.side = args.side if args.metadata: path = Path(args.metadata).expanduser().resolve() - state.replace(storage.load(path, args.side), str(path)) + state.replace(storage.load(path), str(path)) Editor(state, args).run() return 0 except (ImportError, OSError, RuntimeError, TypeError, ValueError) as exc: diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index 418c9366b8..79ce84991a 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -80,10 +80,7 @@ def pose_error(values: object) -> str | None: class Document: """Editor-native data. Poses and semantic bindings never use FlatBuffer row indices.""" - def __init__(self, side: str): - if side not in ("left", "right"): - raise ValueError("Side must be left or right") - self.side = side + def __init__(self): self.segments: list[Segment] = [] self.taxels: list[Taxel] = [] self.q: dict[str, float] = {} @@ -191,18 +188,13 @@ def issue(kind: str, row: int | None, field: str, message: str) -> None: class State: def __init__(self): - self.documents = {side: Document(side) for side in ("left", "right")} - self.saved: dict[str, Document | None] = {"left": None, "right": None} - self.paths: dict[str, str] = {} - self.side = "left" - - @property - def document(self) -> Document: - return self.documents[self.side] + self.document = Document() + self.saved: Document | None = None + self.path = "" def replace(self, document: Document, path: str | None = None) -> None: - self.documents[document.side] = document - self.saved[document.side] = document.clone() - self.saved[document.side].q = {} + self.document = document + self.saved = document.clone() + self.saved.q = {} if path is not None: - self.paths[document.side] = path + self.path = path diff --git a/scripts/tactile_metadata/tactile_metadata_editor/storage.py b/scripts/tactile_metadata/tactile_metadata_editor/storage.py index 0552b0284a..03edf26e84 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/storage.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/storage.py @@ -42,7 +42,6 @@ def _joint_name(value: object) -> str: def parse( value: object, - side: str, alignment: np.ndarray | list[list[float]] | None = None, ) -> Document: if not isinstance(value, dict): @@ -65,7 +64,7 @@ def parse( transform = np.asarray( VISUALIZATION_FROM_HAND if alignment is None else alignment, dtype=float ) - document = Document(side) + document = Document() document.alignment = transform.tolist() names = [_joint_name(joint) for joint in joints] document.segments = [ @@ -140,11 +139,10 @@ def metadata(document: Document) -> dict: def load( path: str | Path, - side: str, alignment: np.ndarray | list[list[float]] | None = None, ) -> Document: with Path(path).expanduser().resolve().open() as stream: - return parse(json.load(stream), side, alignment) + return parse(json.load(stream), alignment) def write(path: str | Path, value: object) -> None: @@ -162,6 +160,4 @@ def write(path: str | Path, value: object) -> None: def save(state: State, path: str | Path) -> None: write(path, metadata(state.document)) - state.paths[state.side] = str(Path(path).expanduser().resolve()) - state.saved[state.side] = state.document.clone() - state.saved[state.side].q = {} + state.replace(state.document, str(Path(path).expanduser().resolve())) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 5d679b4b6d..2fdb004304 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -50,12 +50,8 @@ class Editor: def __init__(self, state: State, args): self.state = state - for document in state.documents.values(): - document.alignment = geo.VISUALIZATION_FROM_HAND.tolist() - self.preview = { - side: self._preview_from(document) - for side, document in state.documents.items() - } + self.document.alignment = geo.VISUALIZATION_FROM_HAND.tolist() + self.preview_document = self._preview_from(self.document) self.live_document: Document | None = None self.live_metadata_key = "" self.receiver: sdk.Receiver | None = None @@ -109,7 +105,7 @@ def __init__(self, state: State, args): self.server.gui.configure_theme(control_width="medium", show_share_button=False) self.main_panel = self.server.gui.main_panel self.main_panel.dock_right() - self._build_gui() + self._build_gui(args.side) self._register_scene_events() self.selection_gizmo = self.server.scene.add_transform_controls( "/selection/translation", @@ -121,9 +117,9 @@ def __init__(self, state: State, args): visible=False, ) self.selection_gizmo.on_update(self._selection_gizmo_event) - self._sync_side() - if self.state.side in self.state.paths: - self.info.value = f"{self.state.side.title()} metadata loaded." + self._sync_document() + if self.state.path: + self.info.value = "Metadata loaded." if args.urdf: self.urdf_path.value = str(Path(args.urdf).expanduser().resolve()) self._load_urdf() @@ -132,23 +128,21 @@ def __init__(self, state: State, args): def document(self) -> Document: return self.state.document - @property - def preview_document(self) -> Document: - return self.preview[self.state.side] - @staticmethod def _preview_from(document: Document) -> Document: preview = document.clone() preview.taxels = [] return preview - def _build_gui(self) -> None: + def _build_gui(self, hand_side: str) -> None: gui = self.server.gui self.general_panel = gui.add_panel() self.general_tab = self.general_panel.add_tab("General") with self.general_tab: - self.side_text = gui.add_markdown("") - self._action_button("Toggle Side", self._toggle_side) + self.hand_side = gui.add_dropdown( + "Hand Side", ("left", "right"), initial_value=hand_side + ) + self._watch(self.hand_side, self._hand_side_changed) self.metadata_path = gui.add_text("Metadata Path", "") self._action_button("New", self._new) self._action_button("Save", self._save) @@ -415,19 +409,20 @@ def _set_mode(self, mode: str, checked: bool) -> None: if mode == "Preview": self._update_preview_error() - def _toggle_side(self) -> None: + def _hand_side_changed(self, _: str) -> None: + self._reset_live_metadata() self._disconnect() - self.state.side = "right" if self.state.side == "left" else "left" - self._sync_side() - def _sync_side(self) -> None: + def _reset_live_metadata(self) -> None: self.live_document, self.live_metadata_key = None, "" - self._set_pick_mode(None) self.binding_source_editor.value = True self.binding_source_live.value = False self.binding_source_live.disabled = True - self.side_text.content = f"**Side: {self.state.side.title()}**" - self.metadata_path.value = self.state.paths.get(self.state.side, "") + + def _sync_document(self) -> None: + self._reset_live_metadata() + self._set_pick_mode(None) + self.metadata_path.value = self.state.path position, rotation = geo.components( np.asarray(self.document.alignment, dtype=float) ) @@ -450,25 +445,25 @@ def _set_values(widgets, values) -> None: widget.value = float(value) def _new(self) -> None: - document = Document(self.state.side) + document = Document() document.alignment = geo.VISUALIZATION_FROM_HAND.tolist() - document.segments = geo.default_segments(self.state.side) - self.state.documents[self.state.side] = document - self.state.saved[self.state.side] = None - self.state.paths.pop(self.state.side, None) - self.preview[self.state.side] = self._preview_from(document) - self._sync_side() - self.info.value = f"New {self.state.side} metadata created." + document.segments = geo.default_segments(self.hand_side.value) + self.state.document = document + self.state.saved = None + self.state.path = "" + self.preview_document = self._preview_from(document) + self._sync_document() + self.info.value = "New metadata created." def _load(self) -> None: path = self.metadata_path.value.strip() if not path: raise ValueError("Enter Metadata Path") - document = storage.load(path, self.state.side) + document = storage.load(path) self.state.replace(document, path) - self.preview[self.state.side] = self._preview_from(document) - self._sync_side() - self.info.value = f"{self.state.side.title()} metadata loaded." + self.preview_document = self._preview_from(document) + self._sync_document() + self.info.value = "Metadata loaded." LOG.info("Metadata loaded from %s", path) def _save(self) -> None: @@ -506,8 +501,8 @@ def cancel_save(): def _write_save(self, path: str) -> None: storage.save(self.state, path) - self.metadata_path.value = self.state.paths[self.state.side] - self.info.value = f"{self.state.side.title()} metadata saved." + self.metadata_path.value = self.state.path + self.info.value = "Metadata saved." self.redraw = True def _reset_camera(self) -> None: @@ -543,7 +538,7 @@ def _update_segment_alignment(self) -> None: [widget.value for widget in self.segment_rotation], ) delta = geo.realign(self.document, alignment) - saved = self.state.saved[self.state.side] + saved = self.state.saved if saved is not None: geo.apply_delta(saved.segments, list(range(len(saved.segments))), delta) geo.apply_delta(saved.taxels, list(range(len(saved.taxels))), delta) @@ -1260,7 +1255,7 @@ def _connect(self) -> None: self._disconnect(report=True) return config = { - "side": self.state.side, + "side": self.hand_side.value, "collection": self.collection.value, "joint_state": self.joint_collection.value if self.joint_enabled.value @@ -1448,7 +1443,7 @@ def _update_live(self) -> None: key = json.dumps(raw_metadata, sort_keys=True) if key != self.live_metadata_key: self.live_document = storage.parse( - raw_metadata, self.state.side, self.document.alignment + raw_metadata, self.document.alignment ) self.live_metadata_key = key self.binding_source_live.disabled = False @@ -2221,7 +2216,7 @@ def _draw(self) -> None: ) else: self._hide_prefix("/edit") - saved = self.state.saved[self.state.side] + saved = self.state.saved if self.show_saved.value and saved is not None: self._draw_dataset( "/saved", diff --git a/scripts/tactile_metadata/test_document.py b/scripts/tactile_metadata/test_document.py new file mode 100644 index 0000000000..18a3ee8f02 --- /dev/null +++ b/scripts/tactile_metadata/test_document.py @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import unittest + +from tactile_metadata_editor.document import Document, Segment, State + + +class StateTests(unittest.TestCase): + def test_replace_keeps_one_document_and_an_independent_saved_snapshot(self): + state = State() + first = Document() + first.segments = [Segment("WRIST")] + first.q = {"joint": 0.5} + state.replace(first, "first.json") + + first.segments[0].pose[0] = 0.01 + self.assertEqual(state.saved.segments[0].pose[0], 0.0) + self.assertEqual(state.saved.q, {}) + self.assertEqual(first.q, {"joint": 0.5}) + + second = Document() + state.replace(second, "second.json") + self.assertIs(state.document, second) + self.assertEqual(state.path, "second.json") + self.assertIsNot(state.saved, second) + self.assertEqual(state.saved.segments, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/tactile_metadata/test_editor.py b/scripts/tactile_metadata/test_editor.py index db4f55ac73..2827a6ee4a 100644 --- a/scripts/tactile_metadata/test_editor.py +++ b/scripts/tactile_metadata/test_editor.py @@ -20,7 +20,7 @@ def make_editor(self): editor.document.taxels = [ Taxel(segments=["WRIST", None, None, None], weights=[1.0, 0.0, 0.0, 0.0]) ] - editor.preview = {"left": editor._preview_from(editor.document)} + editor.preview_document = editor._preview_from(editor.document) editor.binding_source_live = SimpleNamespace(value=False) editor.info = SimpleNamespace(value="") editor.edit_joint_widgets = {} @@ -82,7 +82,7 @@ def test_deleted_segment_remains_invalid_after_round_trip(self): expected = [{"values": [0xFFFFFFFF, 0, 0, 0]}] self.assertEqual(encoded["geometry_mapping"]["taxel_segment_indices"], expected) - loaded = storage.parse(encoded, document.side) + loaded = storage.parse(encoded) self.assertTrue( any(issue["field"] == "segment0" for issue in loaded.validate()) ) @@ -91,6 +91,35 @@ def test_deleted_segment_remains_invalid_after_round_trip(self): expected, ) + def test_hand_side_change_preserves_document_and_snapshots(self): + editor = self.make_editor() + editor.state.replace(editor.document, "metadata.json") + document, saved = editor.document, editor.state.saved + preview = editor.preview_document + receiver = SimpleNamespace(close=Mock()) + editor.receiver = receiver + editor.connect_button = SimpleNamespace(label="Disconnect Live") + editor.live_status = SimpleNamespace(content="Live") + editor._update_preview_error = Mock() + editor.live_document, editor.live_metadata_key = document.clone(), "left" + editor.binding_source_editor = SimpleNamespace(value=False) + editor.binding_source_live = SimpleNamespace(value=True, disabled=False) + + editor._hand_side_changed("right") + + receiver.close.assert_called_once() + editor._update_preview_error.assert_called_once() + self.assertIsNone(editor.receiver) + self.assertIs(editor.document, document) + self.assertIs(editor.state.saved, saved) + self.assertIs(editor.preview_document, preview) + self.assertEqual(editor.state.path, "metadata.json") + self.assertIsNone(editor.live_document) + self.assertEqual(editor.live_metadata_key, "") + self.assertTrue(editor.binding_source_editor.value) + self.assertFalse(editor.binding_source_live.value) + self.assertTrue(editor.binding_source_live.disabled) + if __name__ == "__main__": unittest.main() From 4c22774012601e9d644eea48e641a9bbc7831f31 Mon Sep 17 00:00:00 2001 From: lotusl Date: Thu, 29 Oct 2026 23:36:15 -0700 Subject: [PATCH 33/36] Address tactile editor typing, dependency and transform reviews Use consistent float defaults, an empty string for successful pose validation, explicit matrix copies and float64 transform annotations. Apply transforms once per distinct row. Relax UI dependency pins and document the Viser minimum for dockable panels and Trimesh 5 compatibility. Signed-off-by: lotusl --- pyproject.toml | 12 +++++---- .../tactile_metadata_editor/cli.py | 2 -- .../tactile_metadata_editor/document.py | 26 +++++++++---------- .../tactile_metadata_editor/geometry.py | 25 +++++++++++++----- .../tactile_metadata_editor/ui.py | 2 +- scripts/tactile_metadata/test_document.py | 3 +++ scripts/tactile_metadata/test_editor.py | 12 ++++++++- src/core/python/requirements-ui.txt | 12 +++++---- 8 files changed, 61 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 15319553f4..d4dabec57f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -49,11 +49,13 @@ dev = [ ] ui = [ "imgui[glfw]>=2.0.0", - "numpy>=2,<3", - "viser==1.1.1", - "yourdfpy==0.0.60", - "trimesh==5.1.1", - "rtree>=1.3,<2", + "numpy>=1.23.0", + # Dockable panels and Trimesh 5 compatibility require Viser >=1.1.1. + # https://github.com/viser-project/viser/pull/769 + "viser>=1.1.1", + "yourdfpy", + "trimesh", + "rtree", ] retargeters = [ "dex-retargeting>=0.4.6,<0.6.0", diff --git a/scripts/tactile_metadata/tactile_metadata_editor/cli.py b/scripts/tactile_metadata/tactile_metadata_editor/cli.py index 4b83a3981b..d50cf4defa 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/cli.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/cli.py @@ -1,7 +1,5 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -from __future__ import annotations - import argparse import sys from pathlib import Path diff --git a/scripts/tactile_metadata/tactile_metadata_editor/document.py b/scripts/tactile_metadata/tactile_metadata_editor/document.py index 79ce84991a..1ec978080d 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/document.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/document.py @@ -23,23 +23,23 @@ # fmt: off DEFAULT_OPENXR_SEGMENT_POSES = ( (-0.00244, 0.00122, -0.05917, 0.010296699, 0.02060433, -0.000212213, 0.999734662), - (0, 0, 0, 0, 0, 0, 1), + (0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0), (-0.022, -0.012, -0.028, -0.00692186, 0.30239121, 0.275661445, 0.912426661), (-0.047208158, -0.020249943, -0.065583071, -0.00692186, 0.30239121, 0.275661445, 0.912426661), (-0.065292271, -0.02616838, -0.09254484, -0.00692186, 0.30239121, 0.275661445, 0.912426661), (-0.080088363, -0.031010737, -0.114604468, -0.00692186, 0.30239121, 0.275661445, 0.912426661), - (-0.00528, 0, -0.02068, 0, 0.124664128, 0, 0.992199), - (-0.024, 0, -0.094, -0.017438446, 0.039983244, 0.00069791, 0.999047924), + (-0.00528, 0.0, -0.02068, 0.0, 0.124664128, 0.0, 0.992199), + (-0.024, 0.0, -0.094, -0.017438446, 0.039983244, 0.00069791, 0.999047924), (-0.027194641, -0.00139598, -0.133847779, -0.043584497, 0.039951273, 0.00174431, 0.99824909), (-0.029105295, -0.003487718, -0.157679985, -0.060999707, 0.039914746, 0.00244129, 0.997336397), (-0.030691675, -0.005925105, -0.177467419, -0.060999707, 0.039914746, 0.00244129, 0.997336397), (-0.00088, 0.00044, -0.02134, 0.010296699, 0.02060433, -0.000212213, 0.999734662), - (-0.004, 0.002, -0.097, -0.017452406, 0, 0, 0.999847695), - (-0.004, 0.000429523, -0.141972587, -0.043619387, 0, 0, 0.999048222), - (-0.004, -0.002010838, -0.169866039, -0.06104854, 0, 0, 0.998134798), - (-0.004, -0.004691964, -0.191702054, -0.06104854, 0, 0, 0.998134798), - (0.0033, 0, -0.02002, 0, -0.081592271, 0, 0.996665792), - (0.015, 0, -0.091, -0.017444553, -0.029990932, -0.000523494, 0.999397797), + (-0.004, 0.002, -0.097, -0.017452406, 0.0, 0.0, 0.999847695), + (-0.004, 0.000429523, -0.141972587, -0.043619387, 0.0, 0.0, 0.999048222), + (-0.004, -0.002010838, -0.169866039, -0.06104854, 0.0, 0.0, 0.998134798), + (-0.004, -0.004691964, -0.191702054, -0.06104854, 0.0, 0.0, 0.998134798), + (0.0033, 0.0, -0.02002, 0.0, -0.081592271, 0.0, 0.996665792), + (0.015, 0.0, -0.091, -0.017444553, -0.029990932, -0.000523494, 0.999397797), (0.017516954, -0.001465779, -0.132898883, -0.04359976, -0.029966951, -0.001308385, 0.998598684), (0.019070086, -0.003731828, -0.158753338, -0.06102107, -0.029939553, -0.001831181, 0.997685671), (0.020319944, -0.006291084, -0.1795593, -0.06102107, -0.029939553, -0.001831181, 0.997685671), @@ -65,7 +65,7 @@ class Taxel: weights: list[float] = field(default_factory=lambda: [0.0] * 4) -def pose_error(values: object) -> str | None: +def pose_error(values: object) -> str: if not isinstance(values, list) or len(values) != 7: return "Expected XYZ and XYZW" if any( @@ -74,7 +74,7 @@ def pose_error(values: object) -> str | None: return "Pose components must be finite numbers" if abs(sum(value * value for value in values[3:]) - 1.0) > 1e-4: return "Orientation must be a unit quaternion" - return None + return "" class Document: @@ -84,7 +84,7 @@ def __init__(self): self.segments: list[Segment] = [] self.taxels: list[Taxel] = [] self.q: dict[str, float] = {} - self.alignment = copy.deepcopy(IDENTITY_MATRIX) + self.alignment: list[list[float]] = [row.copy() for row in IDENTITY_MATRIX] def clone(self) -> Document: return copy.deepcopy(self) @@ -109,7 +109,7 @@ def delete(self, kind: str, rows: list[int]) -> None: def assign(self, taxels: list[int], segment_rows: list[int]) -> None: if not 1 <= len(segment_rows) <= 4: raise ValueError("Select one to four segments") - joints = [] + joints: list[str] = [] for row in segment_rows: if not 0 <= row < len(self.segments): raise ValueError("Selected segment no longer exists") diff --git a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py index e982562d75..cb958ab4bf 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/geometry.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/geometry.py @@ -3,10 +3,12 @@ from __future__ import annotations from collections import OrderedDict +from collections.abc import Iterable from pathlib import Path import numpy as np import trimesh +from numpy.typing import NDArray from trimesh import transformations as tf from yourdfpy import URDF @@ -19,17 +21,27 @@ ) VISUALIZATION_FROM_HAND = np.array( - [[-1, 0, 0, 0], [0, 0, -1, 0], [0, -1, 0, 0], [0, 0, 0, 1]], + [ + [-1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, -1.0, 0.0], + [0.0, -1.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], dtype=float, ) VISUALIZATION_FROM_URDF = np.array( - [[0, 1, 0, 0], [0, 0, 1, 0], [1, 0, 0, 0], [0, 0, 0, 1]], + [ + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [1.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 1.0], + ], dtype=float, ) -def matrix(values: list[float]) -> np.ndarray: - array = np.asarray(values, dtype=float) +def matrix(values: list[float]) -> NDArray[np.float64]: + array = np.asarray(values, dtype=np.float64) if ( array.shape != (7,) or not np.isfinite(array).all() @@ -104,8 +116,9 @@ def realign(document: Document, alignment: np.ndarray) -> np.ndarray: return delta -def apply_delta(values, rows: list[int], delta: np.ndarray) -> None: - for row in rows: +def apply_delta(values, rows: Iterable[int], delta: np.ndarray) -> None: + """Apply the transform once per distinct row.""" + for row in set(rows): if 0 <= row < len(values): values[row].pose = pose(delta @ matrix(values[row].pose)) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 2fdb004304..f6df49c309 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -38,7 +38,7 @@ FORCE_MAX_N = 30.0 SELECTION_GIZMO_SCALE_M = 0.005 SELECTION_GIZMO_LINE_WIDTH = 2.5 -ACTIVE_DATASET_OPACITY = 1 +ACTIVE_DATASET_OPACITY = 1.0 BINDING_OVERLAY_OPACITY = 0.4 SAVED_BINDING_OPACITY = 0.3 TAXEL_PAGE_SIZE = 25 diff --git a/scripts/tactile_metadata/test_document.py b/scripts/tactile_metadata/test_document.py index 18a3ee8f02..81943216ac 100644 --- a/scripts/tactile_metadata/test_document.py +++ b/scripts/tactile_metadata/test_document.py @@ -15,11 +15,14 @@ def test_replace_keeps_one_document_and_an_independent_saved_snapshot(self): state.replace(first, "first.json") first.segments[0].pose[0] = 0.01 + first.alignment[0][3] = 0.01 self.assertEqual(state.saved.segments[0].pose[0], 0.0) + self.assertEqual(state.saved.alignment[0][3], 0.0) self.assertEqual(state.saved.q, {}) self.assertEqual(first.q, {"joint": 0.5}) second = Document() + self.assertEqual(second.alignment[0][3], 0.0) state.replace(second, "second.json") self.assertIs(state.document, second) self.assertEqual(state.path, "second.json") diff --git a/scripts/tactile_metadata/test_editor.py b/scripts/tactile_metadata/test_editor.py index 2827a6ee4a..e240a22ef6 100644 --- a/scripts/tactile_metadata/test_editor.py +++ b/scripts/tactile_metadata/test_editor.py @@ -7,12 +7,22 @@ import numpy as np -from tactile_metadata_editor import storage +from tactile_metadata_editor import geometry as geo, storage from tactile_metadata_editor.document import Segment, State, Taxel from tactile_metadata_editor.ui import Editor class EditorTests(unittest.TestCase): + def test_apply_delta_transforms_duplicate_rows_once(self): + segments = [Segment("WRIST"), Segment("INDEX_TIP")] + delta = np.eye(4) + delta[0, 3] = 0.01 + + geo.apply_delta(segments, [0, 0], delta) + + np.testing.assert_allclose(segments[0].pose[:3], [0.01, 0.0, 0.0]) + np.testing.assert_allclose(segments[1].pose[:3], [0.0, 0.0, 0.0]) + def make_editor(self): editor = Editor.__new__(Editor) editor.state = State() diff --git a/src/core/python/requirements-ui.txt b/src/core/python/requirements-ui.txt index 8da97507a2..a0c3a7521b 100644 --- a/src/core/python/requirements-ui.txt +++ b/src/core/python/requirements-ui.txt @@ -1,7 +1,9 @@ # Optional UI dependencies for parameter tuning imgui[glfw]>=2.0.0 -numpy>=2,<3 -viser==1.1.1 -yourdfpy==0.0.60 -trimesh==5.1.1 -rtree>=1.3,<2 +numpy>=1.23.0 +# Dockable panels and Trimesh 5 compatibility require Viser >=1.1.1. +# https://github.com/viser-project/viser/pull/769 +viser>=1.1.1 +yourdfpy +trimesh +rtree From 14b2eb13c8a527b975732658bb12f6e104cd4233 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 23:50:24 -0700 Subject: [PATCH 34/36] When apply taxels, don't delete invalid taxels; and enlarge 3-axis grabber Signed-off-by: Lotus Li --- .../tactile_metadata_editor/ui.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index f6df49c309..474a20cde2 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -36,7 +36,7 @@ FORCE_MIN_COLOR = (0, 0, 255) FORCE_MAX_COLOR = (255, 0, 0) FORCE_MAX_N = 30.0 -SELECTION_GIZMO_SCALE_M = 0.005 +SELECTION_GIZMO_SCALE_M = 0.01 SELECTION_GIZMO_LINE_WIDTH = 2.5 ACTIVE_DATASET_OPACITY = 1.0 BINDING_OVERLAY_OPACITY = 0.4 @@ -1349,9 +1349,16 @@ def _apply_preview_segments(self) -> None: self._rebuild_edit_lists() self.info.value = "Preview segments applied to Edit data." + def _applied_preview_taxels(self) -> list[Taxel]: + # Invalid rows keep their binding metadata so row order still matches forces. + rows, _ = self._preview_taxels() + sources = self._binding_document().taxels + return copy.deepcopy( + [source if row is None else row for source, row in zip(sources, rows)] + ) + def _apply_preview_taxels(self) -> None: - rows, _, _ = self._rendered_preview_taxels() - self.document.taxels = copy.deepcopy([taxel for _, taxel in rows]) + self.document.taxels = self._applied_preview_taxels() self._rebuild_edit_lists() self.info.value = "Preview taxels applied to Edit data." @@ -1362,9 +1369,9 @@ def _apply_preview_urdf(self) -> None: def _apply_preview_all(self) -> None: document = self.document - rows, _, _ = self._rendered_preview_taxels() + taxels = self._applied_preview_taxels() document.segments = copy.deepcopy(self.preview_document.segments) - document.taxels = copy.deepcopy([taxel for _, taxel in rows]) + document.taxels = taxels document.q = copy.deepcopy(self.preview_document.q) self._rebuild_edit_lists() self._sync_joint_widget_values(document, self.edit_joint_widgets) From f71b699cbfd4bdbe8f4ad03b97fcbad1e28f2438 Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Thu, 29 Oct 2026 23:55:46 -0700 Subject: [PATCH 35/36] Update Readme with warning Signed-off-by: Lotus Li --- scripts/tactile_metadata/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/scripts/tactile_metadata/README.md b/scripts/tactile_metadata/README.md index bf40b7ea98..208560ce6e 100644 --- a/scripts/tactile_metadata/README.md +++ b/scripts/tactile_metadata/README.md @@ -10,6 +10,13 @@ previewing glove tactile metadata. It loads URDF models and [`TactileMetadata`](../../src/core/schema/fbs/tactile_metadata.fbs) JSON files. All edits remain local; live connections only receive Isaac Capture data. +> [!WARNING] +> The editor is designed for completing all work in one run. Saved metadata +> contains only schema fields, so ending the session loses editor-specific +> configuration such as URDF paths, alignments, and URDF joint values. + +**Todo**: add cross-run project saving-loading. + ## Set up and launch Use an existing Isaac Capture environment with Python 3.11 or newer. Install From 222e7f1b82a7b758c76eb6090813fd8c82fe995e Mon Sep 17 00:00:00 2001 From: Lotus Li Date: Fri, 30 Oct 2026 00:11:03 -0700 Subject: [PATCH 36/36] Make taxel visualizer smaller Signed-off-by: Lotus Li --- scripts/tactile_metadata/tactile_metadata_editor/ui.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/tactile_metadata/tactile_metadata_editor/ui.py b/scripts/tactile_metadata/tactile_metadata_editor/ui.py index 474a20cde2..44e4527ab8 100644 --- a/scripts/tactile_metadata/tactile_metadata_editor/ui.py +++ b/scripts/tactile_metadata/tactile_metadata_editor/ui.py @@ -22,7 +22,7 @@ LOG = logging.getLogger("isaaccapture.tactile_metadata.editor") SEGMENT_CUBE_SIDE_M = 0.004 -TAXEL_SIDE_M = 0.002 +TAXEL_SIDE_M = 0.0015 URDF_OPACITY = 0.2 SEGMENT_FLAT_SHADING = True VISUALIZER_CAST_SHADOW = False