diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 87404b4..86e81c1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -160,6 +160,11 @@ jobs: PORT=3000 pnpm start > server.log 2>&1 & bash scripts/smoke.sh http://localhost:3000 || { cat server.log; exit 1; } + # The production build in Chromium: offline after one visit, shares and file launches. + - run: pnpm exec playwright install --with-deps chromium + - name: End-to-end tests + run: pnpm test:e2e + docker: name: docker image runs-on: ubuntu-latest diff --git a/CONTEXT.md b/CONTEXT.md index 08516fe..2c0df66 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -8,14 +8,19 @@ building branching pipelines. ### Surfaces **Quick tool**: -A single-purpose page (Convert, Compress, Resize, Crop, Rotate, Strip metadata) that runs a fixed -pipeline. +A single-purpose page (Convert, Compress, Resize, Crop, Rotate, Strip metadata, Remove +background) that runs a fixed pipeline. _Avoid_: converter, feature page **Pipeline tool**: A saved pipeline opened as a quick tool page, without the canvas. _Avoid_: custom tool, app +**Built-in tool**: +A pipeline tool that ships with Hexlode, on its own page at `/tools/`, for a job people +search for by name and repeat. +_Avoid_: preset, landing page, recipe + **Studio**: The desktop editor where users build pipelines on a canvas. _Avoid_: canvas (as the product name), workspace, editor @@ -56,6 +61,10 @@ _Avoid_: autosave, backup A pipeline exported as a `.hexlode` file. _Avoid_: recipe file, export +**Pipeline link**: +A link to Hexlode that carries a whole pipeline file, compressed, after its `#`. +_Avoid_: share URL, permalink + ### Running **Run**: @@ -104,6 +113,20 @@ _Avoid_: preview image, test image The predicted work and duration of a run, shown before it starts. _Avoid_: forecast, quote +**On-device model**: +A machine learning model Hexlode runs in the browser, such as the one Remove background uses. It +downloads once from Hexlode's own server, after the user agrees, and stays in the browser. +_Avoid_: AI, cloud model + +**Inference worker**: +The one worker per page that runs on-device models, one image at a time, for every engine and +preview worker, so the page holds a single copy of each model. +_Avoid_: model worker, AI worker + +**Cut-out**: +An image whose background Remove background made transparent, leaving its subject. +_Avoid_: matte, sticker, mask (the mask is what the model returns) + ### Delivery **Output node**: diff --git a/DEPLOY.md b/DEPLOY.md index 7cffdd8..e11e54b 100644 --- a/DEPLOY.md +++ b/DEPLOY.md @@ -47,11 +47,22 @@ runs with analytics and error reports off. | `VITE_SENTRY_DSN` | Sentry DSN. Turns on error reports, logs and tracing in the browser and on the server. | | `HEXLODE_ENVIRONMENT` | `staging` on the staging application. Leave it unset in production, which reports as `production`. | | `PORT` | The port the server listens on. Defaults to `3000`; change the domain's port to match. | +| `HEXLODE_MODEL_DIR` | Where the server keeps on-device models it serves under `/models/`. Defaults to a folder in the system's temporary directory. | The `VITE_*` values are public: they reach every visitor's browser. Never put a secret in a variable that starts with `VITE_`. A change takes effect on the next deploy or restart. The image sets `HEXLODE_VERSION` itself; Sentry and PostHog label reports and events with it. +The first request for the background removal model makes the server fetch it once (94 MB) from +its pinned Hugging Face revision, so the container needs outbound HTTPS to `huggingface.co`. The +server checks the file's SHA-256 before keeping and serving it, and browsers never contact Hugging +Face (ADR 0017). The file lives in `HEXLODE_MODEL_DIR`, which a new container starts without; to +skip the fetch after each deploy, mount a volume there in Dokploy (**Advanced → Volumes**), for +example at `/models-cache`, and set `HEXLODE_MODEL_DIR=/models-cache`. The folder must belong to +the user the container runs as (`nonroot`, uid 65532) and must not be writable by other users; the +server refuses any other folder and answers model requests with an error, since it sends what is +in the folder to every visitor. It also checks a kept file's SHA-256 once after each start. + The Sentry organisation, project and auth token are not runtime settings. They are only needed where the image is built, to upload source maps; see [Readable Sentry stack traces](#readable-sentry-stack-traces). diff --git a/docs/adr/0013-pipeline-links-carry-the-pipeline.md b/docs/adr/0013-pipeline-links-carry-the-pipeline.md new file mode 100644 index 0000000..66d4984 --- /dev/null +++ b/docs/adr/0013-pipeline-links-carry-the-pipeline.md @@ -0,0 +1,11 @@ +# Pipeline links carry the whole pipeline + +A pipeline link puts the pipeline file, deflated and written in base64url, after the `#` of a link +to `/tools/shared`. Browsers never send the part after `#` to a server, so sharing a pipeline needs +no storage, no accounts and no server code, and Hexlode never holds a copy of anyone's pipeline. +The cost is length: a link stays under 7,800 characters after the `#`, which chat apps, email and +every browser keep intact. Templates make links of a few hundred characters; a pipeline with a +large watermark logo does not fit and is shared as a `.hexlode` file instead. The part after `#` +starts with its own version, apart from the file's, so the packing can change while old links keep +opening, and a link unpacks to at most 2 MB, so a crafted link cannot fill the memory. Once a cloud +mode exists, short links stored on a server can sit beside these without replacing them. diff --git a/docs/adr/0014-built-in-tools-are-pipelines-in-code.md b/docs/adr/0014-built-in-tools-are-pipelines-in-code.md new file mode 100644 index 0000000..be7cfc3 --- /dev/null +++ b/docs/adr/0014-built-in-tools-are-pipelines-in-code.md @@ -0,0 +1,17 @@ +# Built-in tools are pipelines in code, chosen for real jobs + +Built-in tools are pipelines defined in `src/features/built-in-tools/tools.ts` and shown with the +same page as saved pipeline tools, at `/tools/`. Saved pipelines keep `/tools/` with a +random UUID, so the two never collide, the page renders on the server for built-in slugs only, and +saved pipeline pages ask not to be indexed because their address means nothing outside one +browser. A tool can show the settings of chosen nodes, such as the watermark text, and can name +what must be set before it runs, such as a logo; everything else stays as the pipeline defines it, +and Open in the Studio carries the changed settings over. + +The list is short on purpose. A job earns a page only when people search for it by name in large +numbers, repeat it, and Hexlode does all of it on the device better than one quick tool. That rules +out a page per format pair: Convert already does PNG to JPG. The two exceptions, HEIC to JPG and +WebP to JPG, exist because the source format is the problem people search for (phones save HEIC, +websites serve WebP, and many apps open neither), and WebP to JPG also fills transparency with +white instead of black. Tools that follow a platform's rules name the guidance and the date it was +checked on the page, so a reader can see when the numbers might be stale. diff --git a/docs/adr/0015-service-worker-for-offline-use.md b/docs/adr/0015-service-worker-for-offline-use.md new file mode 100644 index 0000000..f740402 --- /dev/null +++ b/docs/adr/0015-service-worker-for-offline-use.md @@ -0,0 +1,58 @@ +# A service worker keeps the app working offline + +After one online visit, Hexlode opens and runs without a connection. One service worker at +`/sw.js` does this, registered in production builds only. A Vite plugin +(`src/features/offline/build-plugin.ts`) lists every file of the client build and of `public/` once +the bundle is complete, leaves out source maps and share cards, and bundles the worker +(`src/features/offline/service-worker.ts`) into one classic script with that list and a version +written in. The version is a hash of every listed file's path and contents, so any changed file +changes the worker's bytes and the browser installs the new worker on the next online visit. + +On install the worker downloads every listed file into `hexlode-app-`, codecs the visitor +has not used included, and the main pages into `hexlode-pages-`: the home page, the quick +tools, the Studio, the privacy page, the open page and the app shell. It then takes over at once +and deletes the two caches of earlier versions. It never deletes another cache, such as the on-device model's. + +Page loads go to the network first, and the worker keeps a copy of each page that loads only this +build's scripts. Offline it returns that copy, the same page with another query, or the app shell: +`/app-shell`, a route with server rendering turned off, so the server renders the root route alone +and the router renders the requested page in the browser. Build files come from the cache. The +worker answers nothing else, so `/api/` (health and auth), `/models/`, server functions, the Sentry +tunnel, PostHog and every other origin reach the network untouched. + +The same worker receives photos shared to the installed app on Android at `/share-target`, keeps +them in `hexlode-share` until Strip metadata takes them, and deletes them then. + +## Considered options + +- **Workbox or vite-plugin-pwa.** Mature, but a new dependency for a worker of a hundred lines, + and both expect a static `index.html` to fall back to. Hexlode renders every page on the server + with its deployment's settings (ADR 0008). +- **TanStack Start's SPA mode for the shell**, which prerenders `/_shell.html` during the build. + Nitro lists its static files before the prerender runs, so the server answered 404 for the shell; + in the dev server the option renders every page as the shell; and the shell carried the build's + settings instead of the deployment's. A route without server rendering gives the same page, + rendered on each request. +- **Cache first for pages.** Faster, but a deploy would show only on the second load. Network first + shows a new deploy on the next online load. +- **Waiting for every tab to close before a new worker takes over**, the default. An installed app + left open for days would keep its old worker. Pages load from the network first either way, so + taking over at once strands no page on an old build. + +## Consequences + +- The first visit downloads about 20 MB in the background, 16 MB of it codec WebAssembly, including + multi-threaded and SIMD variants a browser may never use. +- `pnpm dev` and the Vitest suites never register a worker. `pnpm test:e2e` checks the production + build offline. +- A page served by a newer deploy than the active worker is not kept, because its scripts are not + in the worker's cache. Offline, the earlier copy opens instead. +- A tab left open across a deploy can fail to load a part of the app it has not loaded yet, as it + could without a worker, because the new worker deletes the old build's files. +- If the worker is missing, as after the site's data was cleared, a share from Android reaches the + server. The server route answers with Strip metadata and reads none of the photos. +- The privacy page says what the copy holds and how to remove it. +- ONNX Runtime's WebAssembly for Remove background (about 41 MB across its two builds) is not + downloaded on install, since it is only needed after the user agrees to the model download + ([ADR 0017](./0017-on-device-background-removal.md)). The worker keeps it the first time it is + used, in the same versioned cache, so background removal works offline from then on. diff --git a/docs/adr/0016-heic-decoder.md b/docs/adr/0016-heic-decoder.md new file mode 100644 index 0000000..e6145a7 --- /dev/null +++ b/docs/adr/0016-heic-decoder.md @@ -0,0 +1,57 @@ +# HEIC is decoded with libheif in WebAssembly and never written + +HEIC and HEIF photos, the iPhone default, open wherever other images do. We decode them with +libheif and its libde265 HEVC decoder, compiled to WebAssembly by `libheif-js`, loaded in the +workers the first time a HEIC image needs pixels, like the jSquash codecs +([ADR 0002](./0002-jsquash-wasm-codecs.md)). Hexlode serves the loader (88 KB) and the `.wasm` +(1.46 MB, 490 KB compressed) itself, as files of their own. It never encodes HEIC: a HEIC image +whose pixels changed and that reaches Output, or Convert or Responsive set kept at their own +format, is saved as JPEG, or as PNG when it has transparency, and the run warns. Compress to size +makes it JPEG, since a target size needs a lossy format. A HEIC image whose pixels did not change +keeps its original bytes, and Strip metadata and Set copyright edit its metadata in place. + +## Considered options + +- **`libheif-js` (chosen).** It follows libheif releases (1.23.5, October 2026), gives raw RGBA + pixels of the primary image, and ships the `.wasm` beside its loader, so the browser compiles it + while it downloads. Its build compiles synchronously unless handed a module, so we compile it + first and instantiate it inside the loader's call. +- **`heic-to`.** The same libheif, also maintained, but it returns canvas blobs or bitmaps and + inlines the WebAssembly as base64 in multi-megabyte bundles. +- **`heic2any`, `elheif`, `@saschazar/wasm-heif`.** Older libheif builds, unmaintained since 2021 + to 2024; `heic2any` also decodes on the main thread. +- **`heic-decode`.** A thin wrapper over the inlined `libheif-js` bundle; it adds nothing we need. +- **Safari's own decoder through `createImageBitmap`.** Only Safari has one, and it would make + pixels differ by browser. Left out; it could later skip the download there. +- **Writing HEIC.** It needs an HEVC encoder such as x265 (GPL) and carries the patent question + below for encoding as well. Out of scope. + +## Orientation + +HEIF stores rotation, mirroring and cropping as `irot`, `imir` and `clap` properties of the image, +and libheif applies them, so decoded pixels are upright and the header reader reports the size +after them. Phones also write the EXIF orientation tag, which HEIF readers ignore. Reading HEIC +metadata sets that tag to 1, changing only its value, so a HEIC item has orientation 1, Rotate's +auto mode and Crop leave the rotation alone, and other formats get an upright tag. + +## Metadata + +EXIF and XMP live in items of the `meta` box, as in AVIF, and the colour profile in the primary +image's `colr` property. One HEIF module reads both formats; AVIF now carries its colour profile +too. Edits happen in place: removing and shrinking work, adding does not, and the run reports what +could not be kept. The colour profile describes the coded image, so it stays in the file even when +Strip metadata is told to remove it; other formats get it, so colours do not shift. + +## Consequences + +- **Licence.** libheif and libde265 are LGPL-3.0. They stay separately loaded files, and + `/licenses/jsquash.txt` holds their licence and links to the source of the exact versions. That + source lives upstream on GitHub; if it ever disappears, Hexlode must host it. +- **Patents.** HEVC is covered by patents licensed through pools such as Access Advance and Via LA, + and their terms can cover software decoders a product distributes. Browsers avoid shipping their + own HEVC decoder for this reason. Hexlode sends libde265 to every visitor who opens a HEIC + photo, free and without encoding. This is a risk the maintainer accepts or rejects; removing + HEIC means dropping the decoder and the `heic` format, nothing else depends on them. +- Only the primary image is decoded. Thumbnails, bursts, depth maps and HDR gain maps are not, + and images with more than 8 bits per channel come out at 8. +- Decoding is slow next to other formats, about 70 ms per megapixel, which the estimate counts. diff --git a/docs/adr/0017-on-device-background-removal.md b/docs/adr/0017-on-device-background-removal.md new file mode 100644 index 0000000..b38194e --- /dev/null +++ b/docs/adr/0017-on-device-background-removal.md @@ -0,0 +1,81 @@ +# On-device background removal with BiRefNet_lite, served by Hexlode + +Remove background runs BiRefNet_lite (MIT) on the device, through ONNX Runtime Web (MIT), on +WebGPU where the browser has a usable adapter and on WebAssembly otherwise. The model is the +512 × 512 ONNX export `studioludens/birefnet-lite-512`, revision +`962e67969ec6acec9c19fab76acee4f5e22e990c`, file `onnx/model_fp16.onnx`: 98,484,532 bytes, +SHA-256 `eff9216bb2f9d3f023d9c2b7196845a7485739ab1f231593633e4d2344ffc516`. It is a conversion of +`ZhengPeng7/BiRefNet_lite`, and both carry the MIT licence, which allows commercial use and serving +the weights to browsers as long as the notice travels with them; the app ships it at +`/licenses/background-removal.txt` with ONNX Runtime's licence and third-party notices. + +We chose it by running the candidates on composited photos with exact reference masks, one thread +of WebAssembly in Node: + +| Model | Input | Size | Mean IoU | Worst IoU | Time per photo | +|---|---|---|---|---|---| +| BiRefNet_lite, fp16 | 512 | 94 MB | 0.955 | 0.854 | 12 to 17 s | +| ISNet general use, fp32 | 1024 | 170 MB | 0.859 | 0.456 | 10 to 13 s | +| U2-NetP | 320 | 4.4 MB | 0.841 | 0.425 | 1.4 s | + +BiRefNet_lite also had the sharpest edges (the lowest mean error of the soft mask). ISNet and +U2-NetP took the church spire behind the cat as part of the subject; BiRefNet did not. The machine +was shared with other builds while measuring, so the times are high, and it had no GPU, so WebGPU +was not measured. Until a device has run the model, the estimate assumes 9 seconds a photo on +WebAssembly and 0.9 seconds on WebGPU. + +## Delivery + +Browsers download the model only from Hexlode's own server, at +`/models/birefnet-lite-512-fp16-eff9216b.onnx`, and only after the person agreed in a prompt that +states the size. The server route fetches the pinned file from its upstream revision on the first +request, checks its size and SHA-256 while writing it to `HEXLODE_MODEL_DIR` (the system's +temporary folder by default), and serves it from disk from then on with a year of caching, since +the name changes with the file. Because the server sends that file to every visitor, the folder must +belong to the server's user and not be writable by others, the download is created for that user +only, and a file already in the folder is served only after its SHA-256 matches, checked once after +each start; a file planted in a shared temporary folder is never sent. The browser checks the hash again before keeping the model in Cache +Storage, where the page and every worker can read it. ONNX Runtime's WebAssembly is served with the +app: the build with WebGPU (27 MB) where WebGPU works, the smaller build (14 MB) elsewhere. + +## Engine fit + +Each item travels through the whole pipeline in one engine worker, and the pool has one worker per +core. A model copy per worker would multiply about 2.5 GB of WebAssembly memory by the pool size, +so the page runs one shared inference worker, started when a run or a preview first needs a model. +Every engine worker and the preview worker get their own MessagePort to it. A node resizes the +upright image to 512 × 512 in its own worker and sends only that; the inference worker runs one +image at a time and returns the 512 × 512 mask; the node scales the mask to the full image, cuts +the subject out, and estimates the colour of soft edges (approximate fast foreground estimation, +the method BiRefNet's own code uses), so no fringe of the old background shows on white. Full-size +pixels never leave the worker that owns the item. + +The estimate times model runs one after another rather than sharing them across workers, adds the +start of the model once per page, and adds the download while the model is not in the browser, +unscaled by this device's speed. Before a run on this page it uses typical times for the backend +the browser offers (WebAssembly runs use one thread, because more need cross-origin isolation, +which Hexlode does not turn on); afterwards it uses what it measured. + +## Considered options + +- **Downloading the model while building the Docker image.** Robust at runtime, but every image, + published for two architectures on each merge, grows by 94 MB, and `pnpm dev` would still need + its own download. The route gives development and production one path; its cost is that the + server needs outbound HTTPS to Hugging Face once per model directory. A volume at + `HEXLODE_MODEL_DIR` keeps the file across deploys. +- **A model per engine worker, with a node memory cost that shrinks the pool.** Simpler messages, + but either memory grows with cores or every other node in the pipeline runs on one worker. +- **BiRefNet_lite at 1024.** ONNX Runtime's WebAssembly backend runs out of its 4 GB memory with it. +- **BRIA RMBG** models are licensed for non-commercial use only, and `@imgly/background-removal` is + AGPL, so neither fits Hexlode's Apache 2.0 code. + +## Consequences + +- The node pair matrix and node tests use a stand-in segmenter; the inference worker, the pool + and previews are tested with `threshold.onnx`, a 305-byte model with the real model's input and + output. The real model runs only in the opt-in `pnpm test:model` suite. +- Changing the model means a new file name, size, hash and upstream in + `src/features/background-removal/constants.ts`, a new version of the Remove background node so + step caches are invalidated, a new row of measurements here, and new minimums in the quality + suite. Bumping ONNX Runtime means updating `RUNTIME_BYTES` (a test + checks it) and the notices in `src/features/background-removal/licences/`. diff --git a/idea.md b/idea.md index de6e82f..2e5f343 100644 --- a/idea.md +++ b/idea.md @@ -1,6 +1,6 @@ # Hexlode product -> Updated: 2026-11-04 (node batches 2 and 3) +> Updated: 2026-11-10 (version 2: pipeline links, built-in tools, offline, HEIC, Remove background) > Delivery plan: [implementation.md](./implementation.md). Vocabulary: [CONTEXT.md](./CONTEXT.md). > Decisions and their reasons: [docs/adr/](./docs/adr/). @@ -32,10 +32,10 @@ retake them when the Studio's look changes. Below the hero, in one bordered column: -- The formats Hexlode opens and saves. +- The formats Hexlode opens and saves, then HEIC, which it opens. - The Studio: moving pictures of chaining steps, previews that follow a setting, work spread over workers, the step cache and saving a pipeline as a tool, then the templates. -- The six quick tools, each with a moving picture of its job, and the user's saved pipeline tools. +- The seven quick tools, each with a moving picture of its job, and the user's saved pipeline tools. - How Hexlode works on the device: images flow into the browser tab and out as a ZIP or a folder, with three notes: no upload needed, modern formats and open source. - A closing call to open the Studio. @@ -68,31 +68,69 @@ is kept in browser storage and applied before the page paints, so a light page n | Tool | Job | |---|---| -| Convert | Change format: JPEG, PNG, WebP, AVIF, JPEG XL or QOI. | +| Convert | Change format: JPEG, PNG, WebP, AVIF, JPEG XL or QOI, from those or HEIC. | | Compress | Reduce file size by quality setting or by target size. | | Resize | Change dimensions by width, height, percent or longest edge. | | Crop | Cut to an aspect ratio such as 1:1, 4:5 or 16:9, from the centre or an edge. | | Rotate | Turn photos upright, rotate by quarter turns or flip them. | | Strip metadata | Remove all metadata, only location data, or everything except copyright. | +| Remove background | Cut the subject out onto transparency with a model that runs on the device, saved as PNG or WebP. | + +Every quick tool opens JPEG, PNG, WebP, AVIF, JPEG XL, QOI and HEIC images. HEIC, the iPhone +default, is opened but never written: a HEIC photo whose pixels change comes out as JPEG, or PNG +when it has transparency, and the results say so. Strip metadata keeps HEIC photos HEIC. Each quick tool is a fixed pipeline that runs on the same engine as the Studio. A quick tool page -opens with a row of all six tools for switching, then the tool's name and one line, then two -numbered panels, side by side on wide screens and stacked on phones. The first holds the -drop area and the list of added images, with a thumbnail, a full-size preview and a remove button -for each. The second holds the settings for that job and, under them, the run button, progress and -the download button. The two panels are always the same height, and the drop area grows to fill -its panel. A results list with before and after sizes follows the panels. Crop shows the chosen -shape and position on a landscape and a portrait photo. The page renders on the server with the -tool in place, so it never waits for scripts before showing the drop area and settings. +opens with a row of all seven tools for switching, then the tool's name and one line, then two +numbered panels, side by side on wide screens and stacked on phones. The first holds the drop area +and the list of added images, with a thumbnail, a full-size preview and a remove button for each. +Files the browser cannot show itself, such as HEIC, JPEG XL and QOI, get a preview drawn on the +device. The second holds the settings for that job and, under them, the run button, progress and +the download button. The two panels are always the same height, and the drop area grows to fill its +panel. A results list with before and after sizes follows the panels. Crop shows the chosen shape +and position on a landscape and a portrait photo. The page renders on the server with the tool in +place, so it never waits for scripts before showing the drop area and settings. Settings keep their place when options change, and the panels keep their height: a setting that does not apply is shown disabled with the reason, not hidden, and text or fields that change with an option keep the room of the longest version. A single result downloads as the file itself; several download as a ZIP. Compress by quality never returns a file larger than the original. +Remove background runs a segmentation model on the device. The first time a pipeline needs it, +the page asks before downloading it, says its size and that it comes from Hexlode's own server, +and shows the progress; the browser keeps it for later runs and live previews. Its estimate counts +the download and how slowly the model runs without WebGPU. + A pipeline tool is a saved Studio pipeline opened as a quick tool page. The user builds the pipeline once, then drops new images into the simple page whenever they need it. +## Built-in tools + +A built-in tool is a pipeline tool that ships with Hexlode, on its own page at `/tools/`. +It looks like a quick tool page: the drop area and images first, then the pipeline's steps and, +for a tool that needs them, the settings of one or two of its nodes, such as the watermark text. +Each page renders on the server, has its own title, description and share card, and offers Open in +the Studio to change the pipeline. + +The list stays short on purpose. A job earns a page only when people search for it by name in large +numbers, repeat it, and Hexlode does all of it on the device better than a single quick tool. No +page exists only to catch a search. A tool that follows a platform's rules, such as a marketplace's +photo requirements, names the guidance its numbers come from and the date they were checked. The +list and the reason for each entry live in `src/features/built-in-tools/`. + +| Tool | Job | +|---|---| +| HEIC to JPG | Turns iPhone HEIC photos into JPEG, upright, keeping their date, camera and location. | +| WebP to JPG | Turns WebP images saved from the web into JPEG, with transparency made white. | +| JPG to PDF | Puts every image on its own page of one PDF, upright, on A4, Letter or the image size. | +| Instagram without cropping | Fits each photo into a 4:5 or square post 1080 pixels wide, with a blurred or plain border. | +| Product photos on white | Cuts each product out and centres it on a pure white square, filling about 85%, as a JPEG of up to 2000 pixels. | +| Favicon generator | Makes an ICO file, PNG icons, a web app manifest and the HTML tags from one image. | +| Watermark photos | Writes a text watermark on every photo, in a corner or tiled, keeping each format. | +| Add logo to photos | Places a logo on every photo at the same corner, size and opacity. Runs once a logo is chosen. | + +The Tools menu, the home page and the footer list the built-in tools after the quick tools. + ## Studio The Studio is for desktop screens. Phones and narrow windows get a message that points them to the @@ -179,14 +217,18 @@ browsers that allow it. ## Node catalogue -Items are images, data (JSON or text) or documents (PDF). Batch numbers match the phases in +Items are images, data (JSON or text) or documents (PDF). Images are JPEG, PNG, WebP, AVIF, +JPEG XL, QOI or HEIC, and every node that takes images takes all of them. HEIC (with HEIF) is +decoded but never encoded: where a node keeps an image's format, such as Convert set to keep each +format or Output with an image whose pixels changed, HEIC becomes JPEG, or PNG when it has +transparency, and the run warns. Batch numbers match the phases in [implementation.md](./implementation.md). ### Input and routing | Node | What it does | Batch | |---|---|---| -| Files | Takes dropped files or a folder. Starts every pipeline. | 1 | +| Files | Takes dropped files or a folder, including HEIC photos. Starts every pipeline. | 1 | | Filter | Routes items by rules on format, file size, dimensions, orientation or transparency. Each rule has its own output, plus an output for everything else. An item leaves by the first rule it matches. | 1 | | Inspect | Shows format, dimensions, size and metadata per item. Passes items through unchanged. | 1 | | Deduplicate | Drops exact duplicates (the same pixels), or near duplicates by image fingerprint. Waits for every image; duplicates leave by their own output. | 2 | @@ -199,7 +241,7 @@ Items are images, data (JSON or text) or documents (PDF). Batch numbers match th | Crop | Crops to an aspect preset (1:1, 4:5, 16:9 and others), from the centre or a chosen position. Turns the image upright first. | 1 | | Rotate / Flip | Rotates and flips, including automatic rotation from the camera orientation tag. | 1 | | Auto-trim | Removes plain-colour or transparent borders. | 2 | -| Pad / Extend | Adds space to reach an aspect ratio, filled with a colour or a blurred copy of the image. | 2 | +| Pad / Extend | Adds space to reach an aspect ratio, with an optional margin on every side, filled with a colour or a blurred copy of the image. | 2 | | Pixel-art upscale | Enlarges pixel art 2x to 4x with sharp edges. | 2 | | Split / Tile | Cuts one image into a grid of images. | 2 | @@ -211,6 +253,7 @@ Items are images, data (JSON or text) or documents (PDF). Batch numbers match th | Filters | Applies grayscale, sepia, duotone or invert. | 2 | | Sharpen / Blur | Sharpens, or blurs the whole image or a region. | 2 | | Background | Replaces transparency with a colour. | 2 | +| Remove background | Cuts the subject out onto transparency with a model that runs on the device, on WebGPU or WebAssembly. Turns the image upright first; JPEG becomes PNG, since JPEG has no transparency. The model downloads once from Hexlode's server after the user agrees. | 7 | ### Overlays @@ -224,15 +267,15 @@ Items are images, data (JSON or text) or documents (PDF). Batch numbers match th | Node | What it does | Batch | |---|---|---| -| Strip metadata | Removes all metadata, only location data, or everything except copyright, without re-encoding. Keeps the colour profile unless told otherwise, and keeps a camera orientation tag so photos stay upright. | 1 | +| Strip metadata | Removes all metadata, only location data, or everything except copyright, without re-encoding. Keeps the colour profile unless told otherwise, and keeps a camera orientation tag so photos stay upright. A HEIC photo keeps its colour profile, which belongs to the coded image. | 1 | | Set copyright / author | Writes author and copyright fields without re-encoding. Empty fields stay as they are. | 3 | ### Output and encoding | Node | What it does | Batch | |---|---|---| -| Convert | Encodes to WebP, AVIF, JPEG, JPEG XL, PNG or QOI with the encoder's real settings, or keeps each item's format. | 1 | -| Compress to size | Finds the highest quality that fits a target size such as 200 KB. | 1 | +| Convert | Encodes to WebP, AVIF, JPEG, JPEG XL, PNG or QOI with the encoder's real settings, or keeps each item's format (HEIC becomes JPEG, or PNG with transparency). | 1 | +| Compress to size | Finds the highest quality that fits a target size such as 200 KB. Kept at each format, it makes HEIC into JPEG. | 1 | | Optimize PNG | Makes PNG files smaller without changing pixels. | 1 | | Rename | Names files from a template such as `{name}-{width}w`. The extension follows the format. Output numbers duplicate names. | 1 | | Output | Saves items, passes them on, and delivers a ZIP or folder. | 1 | @@ -252,6 +295,34 @@ Items are images, data (JSON or text) or documents (PDF). Batch numbers match th - Nothing is stored in the browser until the user clicks Save or starts a run. - A pipeline exports to and imports from a `.hexlode` file. The file is versioned JSON, and older versions are migrated on import. +- Share copies a pipeline link: the pipeline file, compressed, after the `#` of a link to Hexlode. + The part after `#` never reaches the server, so the pipeline is not stored anywhere but the link. + Opening the link shows the pipeline as a pipeline tool page, with Save and Open in the Studio. + A pipeline too large for a link, such as one with a big watermark logo, offers its `.hexlode` + file instead. A damaged or unreadable link says so and opens nothing. +- Each saved pipeline counts its runs in the browser, and the run event carries that count, so + Hexlode can see whether people come back to their pipelines. + +## Offline and installing + +- After one online visit, Hexlode works without a connection: the home page, the quick tools, + the built-in tools, pipeline tools and the Studio open and run, with every codec, including formats not used yet. + Pages load from the network when there is one, so a new version shows on the next online visit; + without one they open from the copy kept in the browser. A small Offline badge shows in the top + bar while there is no connection. +- The copy holds Hexlode's own files, never images. Clearing the site's data removes it, as the + privacy page says. +- Hexlode can be installed from the browser as an app. +- Installed on desktop Chromium, Hexlode opens images and `.hexlode` files from the file manager. + A `.hexlode` file opens in the Studio. Images open on a page that asks what to do with them: a + quick tool, a built-in tool or a saved pipeline tool. The chosen page opens with the images already + added; they travel in the tab's memory. +- Installed on Android, Hexlode is offered in the share sheet for photos. Shared photos open in + Strip metadata with the photos added and the setting on removing only the location, since shared + photos are usually about to be posted. The photos wait in browser storage only until the page + takes them. +- Browsers on iPhone and iPad support neither, so there Hexlode is the offline app only. The + interface does not mention this. ## Analytics @@ -267,9 +338,10 @@ A signed-in cloud mode may later run large jobs on a server, keep running after and use a credit system. Local processing stays free and unlimited. The database and account code in the repository waits for this work. -## Out of scope for version 1 +## Out of scope for now - Accounts, sign-in, cloud processing and billing. -- HEIC input, animated images and colour-profile conversion. +- A command-line runner and a GitHub Action for pipelines. +- HEIC output, animated images and colour-profile conversion. - Dither and retro effects. - The Studio on phones. diff --git a/implementation.md b/implementation.md index 3f1c200..ec2f7f6 100644 --- a/implementation.md +++ b/implementation.md @@ -1,16 +1,20 @@ # Hexlode implementation plan -> Updated: 2026-11-04 +> Updated: 2026-11-10 > Product: [idea.md](./idea.md). Vocabulary: [CONTEXT.md](./CONTEXT.md). Decisions: [docs/adr/](./docs/adr/). ## Goal -Version 1 is the complete local product: the home page, six quick tools, pipeline tools, and the +Version 1 was the complete local product: the home page, six quick tools, pipeline tools, and the Studio with all 33 nodes from the catalogue in [idea.md](./idea.md). It ships in three phases. Phase 1 builds the whole application and node batch 1. Phases 2 and 3 only add node batches 2 and 3. Each phase is deployable and passes its exit gate before the next phase starts. -Active phase: **Phase 3**. Phases 2 and 3 are built and pass their exit gates. +Version 2 adds phases 4 to 7 on top of version 1. They make the pipeline easy to share and find, +keep the app working offline, and open the photos people actually have. + +Active phase: **Version 2, phases 4 to 7**, built together on one branch. Phases 1 to 3 are built +and pass their exit gates. ## Phase 1: the application and node batch 1 @@ -106,6 +110,93 @@ an ICO file, which is a document ([ADR 0012](./docs/adr/0012-ico-files-are-docum each image's icons in a folder named after it. Live previews show the text of data items, such as a palette or a snippet, in the inspector. +## Phase 4: pipelines people can share + +1. Pipeline links. The Studio and every pipeline tool page copy a link that carries the whole + pipeline after `#`, compressed. The part after `#` never reaches the server. Opening the link + shows the pipeline tool page with Save and Open in the Studio. A pipeline too large for a link + offers its `.hexlode` file instead. +2. Built-in tools: pipeline tools that ship with Hexlode at `/tools/`, rendered on the + server, each with an entry in `SEO_PAGES`, a share card and a place in the sitemap. Only jobs + people search for by name and repeat; the list and the reason for each entry live in + `src/features/built-in-tools/`. A built-in tool can show the settings of a few chosen nodes. +3. The run event carries how many times a saved pipeline has run. + +Exit gate: every template survives a pipeline link unchanged, a damaged link shows a clear message, +every built-in tool runs end to end on real fixtures and is in the sitemap, and the run count rises +by one per run of a saved pipeline. + +## Phase 5: offline and installable + +1. A service worker caches the built scripts, styles, fonts and codec WASM after the first visit, + with the main pages. A Vite plugin writes `/sw.js` from the client build, in production builds + only. Pages load from the network first and fall back to their cached copy offline, or to the + app shell, `/app-shell`, a route the server renders without its page. A badge shows when the app + is offline ([ADR 0015](./docs/adr/0015-service-worker-for-offline-use.md)). + ONNX Runtime's WebAssembly is kept the first time Remove background runs instead, so the first + visit does not download it. +2. Installed on desktop Chromium, Hexlode opens images and `.hexlode` files from the file manager: + the manifest's file handlers open `/open`, which reads `window.launchQueue`, sends a `.hexlode` + file to the Studio and asks which tool takes the images. Installed on Android, photos shared to + Hexlode go to the manifest's share target, which the service worker answers; they open in Strip + metadata, set to remove location. Quick tool and pipeline tool pages take files handed to them + in memory (`src/features/handover/`). + +Exit gate: a browser test loads a quick tool and the Studio offline after one online visit, files +from a share or a file launch land in the right page, and the privacy page explains the cache. + +## Phase 6: HEIC input + +HEIC and HEIF photos open wherever other images do, through a WASM decoder loaded on first use in +the worker. Hexlode decodes HEIC but does not write it +([ADR 0016](./docs/adr/0016-heic-decoder.md)). + +1. libheif with libde265, compiled to WebAssembly by `libheif-js`, decodes the primary image. Its + loader and `.wasm` are separate files Hexlode serves, credited on the codec licences page. +2. libheif applies `irot`, `imir` and `clap`, so pixels come out upright and the header reader + reports the size after them. Reading HEIC metadata sets the EXIF orientation tag to 1, so + nothing turns the photo a second time. +3. EXIF, XMP and the colour profile are read with the same HEIF code as AVIF, and edited in place. +4. A HEIC image that must be written in its own format becomes JPEG, or PNG when it has + transparency, with a run warning. +5. HEIC joins the formats in item types, the Files node, file intake, the file picker, the drop + area, the Filter node's format rule, the home page and the Convert copy. +6. The list of added files draws previews of HEIC, JPEG XL and QOI files in a worker, since + browsers other than Safari cannot show them. + +Exit gate: iPhone-style photos (grid tiles, `irot` rotation with an EXIF orientation, GPS, a +Display P3 profile, transparency) decode with the right dimensions, orientation and metadata, and +the node pair matrix passes with HEIC fixtures. + +## Phase 7: remove background + +A Remove background node and quick tool cut the subject out onto transparency, on the device +([ADR 0017](./docs/adr/0017-on-device-background-removal.md)). + +1. The model is BiRefNet_lite exported to ONNX at 512 × 512 in half precision (MIT, 94 MB), pinned + by revision and SHA-256 in `src/features/background-removal/constants.ts`. ONNX Runtime Web runs + it on WebGPU where the browser has an adapter and on one thread of WebAssembly otherwise; its + WebAssembly is served with the app, the smaller build without WebGPU where WebGPU is missing. +2. The server serves the model at `/models/`: the first request fetches the pinned file, + checks its size and hash, and keeps it in `HEXLODE_MODEL_DIR`. Browsers never contact a third + party. A page asks before the one-time download, says its size, shows its progress, checks the + hash again and keeps the model in Cache Storage. +3. One inference worker per page holds the model. The run controller gives each engine worker, + and the Studio gives the preview worker, a MessagePort to it when the pipeline needs a model. + The node resizes the upright image to the model's input in its own worker, sends only that, and + scales the returned mask back to cut the subject out, estimating the colour of soft edges so no + fringe of the old background shows on white. JPEG cut-outs become PNG. +4. The estimate times model runs one after another on the inference worker, adds the model's + start once and the download while it is missing, and uses measured times once the model ran. + Analytics record the download (size, time, error code) and each run's backend and timings. +5. The quick tool at `/remove-background` saves PNG or WebP. The node has no settings; its + inspector shows whether the model is in the browser and offers the download for previews. + +Exit gate: masks on licensed fixtures match their reference within the minimums in the quality +suite, the node pair matrix passes with the new node, the quick tool's output decodes with +transparency where the background was, and the estimate covers the model download and devices +without WebGPU. + ## Engine design - A pipeline is a directed graph without cycles. Items flow through it one by one, so an item can @@ -118,9 +209,11 @@ a palette or a snippet, in the inspector. pixels between them. Encoding happens at Convert, Compress to size and Optimize PNG, and at the nodes that make finished files: Best format, Responsive set, Favicon / App icons and Contact sheet. An image that reaches Output without an encoding node keeps its source format, and its - original bytes when nothing changed its pixels. + original bytes when nothing changed its pixels. HEIC is decoded but never encoded, so a HEIC + image whose pixels changed is saved as JPEG, or PNG when it has transparency, with a warning + ([ADR 0016](./docs/adr/0016-heic-decoder.md)). - Metadata is rewritten in the container without re-encoding for JPEG, PNG, WebP and JPEG XL. - AVIF metadata is read and can be removed in place but not added; QOI holds none. The run reports + AVIF and HEIC metadata is read and can be removed in place but not added; QOI holds none. The run reports a warning when a format cannot keep metadata. - Each source item travels through the whole pipeline inside one worker, so pixels never cross threads. The pool admits as many items as it has workers, which bounds memory by pool size, not @@ -163,14 +256,24 @@ removed. `vitest.config.ts` sit a few points under current coverage; raise them as coverage grows. - Test-only node types with data and document items, one-to-many and many-to-one behaviour prove what batches 2 and 3 need. They join the node pair matrix but never the product registry. +- `pnpm test:e2e` drives the production build in Chromium with Playwright, so run `pnpm build` + first. It loads the app offline after one visit, runs an image through a quick tool, and checks + shares and file launches. CI runs it after the build. - The batch of 500 images of 12 megapixels runs with `pnpm test:scale`. It takes minutes, so it is outside `pnpm validate`; run it before closing a phase. +- Nodes that run an on-device model use a stand-in segmenter in node tests and the node pair + matrix. The inference worker, the worker pool and previews run ONNX Runtime on `threshold.onnx`, + a tiny model with the real model's input and output (`scripts/generate-model-fixture.mjs`). + `pnpm test:model` runs the real background removal model in Node on licensed photos with + reference masks and checks intersection over union and edge colour. It downloads 94 MB once and + needs about 2.5 GB of memory, so it is outside `pnpm validate`; run it when the model, ONNX + Runtime or the cut-out code changes. - `pnpm validate` passes before every commit. - CI (`.github/workflows/`) runs on every pull request: Biome, types, commit messages, the generated theme and route tree, actionlint and hadolint; unit tests on Linux, macOS and Windows; - browser tests in three engines; coverage; the build with a smoke test of the server; and the - Docker image with a smoke test of the container, its health check, runtime settings and an ARM - build. CodeQL, `pnpm audit`, dependency review and PR titles run in their own workflows. A push + browser tests in three engines; coverage; the build with a smoke test of the server and the + end-to-end tests; and the Docker image with a smoke test of the container, its health check, + runtime settings and an ARM build. CodeQL, `pnpm audit`, dependency review and PR titles run in their own workflows. A push to `main` only publishes the `main` image and deploys staging, since the pull request already ran the checks; CodeQL, the audit and the scale test also run on a schedule. diff --git a/package.json b/package.json index a7606e1..ac116c8 100644 --- a/package.json +++ b/package.json @@ -36,7 +36,9 @@ "test:coverage": "vitest run --project unit --project browser --coverage", "test:browser": "vitest run --project browser", "test:watch": "vitest", - "test:scale": "vitest run --project scale" + "test:scale": "vitest run --project scale", + "test:e2e": "vitest run --project e2e", + "test:model": "vitest run --project model" }, "dependencies": { "@astryxdesign/core": "^0.2.0", @@ -70,9 +72,11 @@ "drizzle-kit": "^0.31.9", "drizzle-orm": "^0.45.1", "gsap": "^3.15.0", + "libheif-js": "1.23.5", "lucide-react": "^1.28.0", "motion": "^13.4.4", "nitro": "3.0.260610-beta", + "onnxruntime-web": "1.31.1", "pg": "^8.16.3", "posthog-js": "^1.444.4", "react": "^19.2.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6b2c53f..7916588 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -105,6 +105,9 @@ importers: gsap: specifier: ^3.15.0 version: 3.15.0 + libheif-js: + specifier: 1.23.5 + version: 1.23.5 lucide-react: specifier: ^1.28.0 version: 1.28.0(react@19.2.8) @@ -114,6 +117,9 @@ importers: nitro: specifier: 3.0.260610-beta version: 3.0.260610-beta(chokidar@5.0.0)(dotenv@17.4.2)(drizzle-orm@0.45.2(@opentelemetry/api@1.9.1)(@types/pg@8.20.3)(kysely@0.29.4)(pg@8.22.0))(jiti@2.7.0)(lru-cache@11.5.3)(vite@8.3.5(@types/node@24.13.3)(esbuild@0.28.1)(jiti@2.7.0)(tsx@4.23.4)(yaml@2.9.0)) + onnxruntime-web: + specifier: 1.31.1 + version: 1.31.1 pg: specifier: ^8.16.3 version: 8.22.0 @@ -1453,6 +1459,33 @@ packages: '@posthog/types@1.419.0': resolution: {integrity: sha512-XX7zRYt/vK8AzaD6V+oDquAzacZf5exJv971dja9En8Qf6g84Qvc3rpT0w6hDYYLWrjaSSkdjFfvvoXI0ouJDg==} + '@protobufjs/aspromise@1.1.2': + resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==} + + '@protobufjs/base64@1.1.2': + resolution: {integrity: sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==} + + '@protobufjs/codegen@2.0.5': + resolution: {integrity: sha512-zgXFLzW3Ap33e6d0Wlj4MGIm6Ce8O89n/apUaGNB/jx+hw+ruWEp7EwGUshdLKVRCxZW12fp9r40E1mQrf/34g==} + + '@protobufjs/eventemitter@1.1.1': + resolution: {integrity: sha512-vW1GmwMZNnL+gMRaovlh9yZX74kc+TTU3FObkkurpMaRtBfLP3ldjS9KQWlwZgraRE0+dheEEoAxdzcJQ8eXZg==} + + '@protobufjs/fetch@1.1.1': + resolution: {integrity: sha512-GpptLrs57adMSuHi3VNj0mAF8dwh36LMaYF6XyJ6JMWlVsc+t42tm1HSEDmOs3A8fC9yyeisgLhsTVQokOZ0zw==} + + '@protobufjs/float@1.0.2': + resolution: {integrity: sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==} + + '@protobufjs/path@1.1.2': + resolution: {integrity: sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==} + + '@protobufjs/pool@1.1.0': + resolution: {integrity: sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==} + + '@protobufjs/utf8@1.1.2': + resolution: {integrity: sha512-b1UQwcEZ4yCnMCD8DAL1VlbvBJE9/IX4FTIp7BG1xYpf29SLazLSrqUkj4w7Y5y7cCVP6E5tcqqcI0xemPkHug==} + '@rolldown/binding-android-arm-eabi@1.2.16': resolution: {integrity: sha512-ia3DwAs0OsMGS7+IConptSiyk9pEXkOx/AbJ2n/lAGxH3QkWQus1t8d7gnAT38FCU2sQu8lUbeR6AiZFv4JxaA==} engines: {node: ^20.19.0 || >=22.12.0} @@ -2986,6 +3019,9 @@ packages: resolution: {integrity: sha512-78/PXT1wlLLDgTzDs7sjq9hzz0vXD+zn+7wypEe4fXQxCmdmqfGsEPQxmiCSQI3ajFV91bVSsvNtrJRiW6nGng==} engines: {node: '>=10'} + flatbuffers@25.9.23: + resolution: {integrity: sha512-MI1qs7Lo4Syw0EOzUl0xjs2lsoeqFku44KpngfIduHBYvzm8h2+7K8YMQh1JtVVVrUvhLpNwqVi4DERegUJhPQ==} + flatted@3.4.4: resolution: {integrity: sha512-5+ybhBZANEJxaH3X5evAFatUxLfEHSr7n6kYJ+1Qd0mUqr4eu9gIf6GDbWHf8RJijHrjjO8G+la14SlL2SeS1Q==} @@ -3050,6 +3086,9 @@ packages: gsap@3.15.0: resolution: {integrity: sha512-dMW4CWBTUK1AEEDeZc1g4xpPGIrSf9fJF960qbTZmN/QwZIWY5wgliS6JWl9/25fpTGJrMRtSjGtOmPnfjZB+A==} + guid-typescript@1.0.9: + resolution: {integrity: sha512-Y8T4vYhEfwJOTbouREvG+3XDsjr8E3kIr7uf+JZ0BYloFsttiHU0WfvANVsR7TxNUJa/WpCnw/Ino/p+DeBhBQ==} + h3@2.0.1-rc.22: resolution: {integrity: sha512-Esv0DMIuPkCTSWCA0vO73vcTqwzH1wjSrAO1TXNu/K3up1sZHa9EKMapbmxCDYBeymC3fVTk4qxp7ogQWQ+KgA==} engines: {node: '>=20.11.1'} @@ -3177,6 +3216,10 @@ packages: launch-editor@2.14.2: resolution: {integrity: sha512-NHJEb30MGBCkmZXLZwz25t37QR4aY03ZzPYUJXOJaOa92uWwExI9SBQUuIznBYc1xkD7cJNGX7oc3Nl3sq2kQQ==} + libheif-js@1.23.5: + resolution: {integrity: sha512-umXZPthnWZtF3iG/5mQG+AZ2V6i0UeYpo3z2CkggTEgL8dKmIIwJT6TDuAMuabXa1yTW8uJ1ZAgR+CxNTklYYw==} + engines: {node: '>=8.0.0'} + lightningcss-android-arm64@1.32.0: resolution: {integrity: sha512-YK7/ClTt4kAK0vo6w3X+Pnm0D2cf2vPHbhOXdoNti1Ga0al1P4TBZhwjATvjNwLEBCnKvjJc2jQgHXH0NEwlAg==} engines: {node: '>= 12.0.0'} @@ -3341,6 +3384,9 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} + long@5.3.2: + resolution: {integrity: sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA==} + loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} hasBin: true @@ -3484,6 +3530,12 @@ packages: ohash@2.0.11: resolution: {integrity: sha512-RdR9FQrFwNBNXAr4GixM8YaRZRJ5PUWbKYbE5eOsrwAjJW0q2REGcf79oYPsLyskQCZG1PLN+S/K1V00joZAoQ==} + onnxruntime-common@1.31.1: + resolution: {integrity: sha512-F+nnYNiQtmNJ95V6R/isBJqiVpdANCrsMEg1OaF+Bkbl3GAtzWLR7Ng17Ld0QtYtYJyotOHPJzOQn1Y6PLZjpg==} + + onnxruntime-web@1.31.1: + resolution: {integrity: sha512-5GeH+bWQ+HxM10z7erGlsWqryrwMbSvm/1tkIsKTT48g0fI19VXVVC435t4t7VsPdSQeS7NmuZPXrQ73k82URw==} + oxc-parser@0.120.0: resolution: {integrity: sha512-WyPWZlcIm+Fkte63FGfgFB8mAAk33aH9h5N9lphXVOHSXEBFFsmYdOBedVKly363aWABjZdaj/m9lBfEY4wt+w==} engines: {node: ^20.19.0 || >=22.12.0} @@ -3592,6 +3644,9 @@ packages: resolution: {integrity: sha512-/E57AYkoeQ25qkxMj5PBOVgF8Kiu/h7cYS30Z5+R7WaiCCBfLq58ZI/dSeaEKb9WVJV5n/03QwrN3IeWIFllvw==} engines: {node: '>=6'} + platform@1.3.6: + resolution: {integrity: sha512-fnWVljUchTro6RiCFvCXBbNhJc2NijN7oIQxbwsyL0buWJPG85v81ehlHI9fXrJsMNgTofEoWIQeClKpgxFLrg==} + playwright-core@1.63.0: resolution: {integrity: sha512-rYCsBF/M5HjUch52bbtVONEFjv6Xu8sm8h72dNlR5bzIE1fvC/bxgspzkjSfU+MweEMmPM8KJebG6nnyxo5mCg==} engines: {node: '>=20'} @@ -3654,6 +3709,10 @@ packages: resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} engines: {node: '>=0.4.0'} + protobufjs@7.6.6: + resolution: {integrity: sha512-dYDWdjSl5RNb7SgPxGQcRU+GtvP7s2fpkrY0r432PcOIaZ0/rBcxEZnQN67iJhFuQiVw754JDoPruPCNdGsbjg==} + engines: {node: '>=12.0.0'} + proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} @@ -5285,6 +5344,26 @@ snapshots: '@posthog/types@1.419.0': {} + '@protobufjs/aspromise@1.1.2': {} + + '@protobufjs/base64@1.1.2': {} + + '@protobufjs/codegen@2.0.5': {} + + '@protobufjs/eventemitter@1.1.1': {} + + '@protobufjs/fetch@1.1.1': + dependencies: + '@protobufjs/aspromise': 1.1.2 + + '@protobufjs/float@1.0.2': {} + + '@protobufjs/path@1.1.2': {} + + '@protobufjs/pool@1.1.0': {} + + '@protobufjs/utf8@1.1.2': {} + '@rolldown/binding-android-arm-eabi@1.2.16': optional: true @@ -6742,6 +6821,8 @@ snapshots: locate-path: 6.0.0 path-exists: 4.0.0 + flatbuffers@25.9.23: {} + flatted@3.4.4: {} flow-estree@0.325.0: {} @@ -6792,6 +6873,8 @@ snapshots: gsap@3.15.0: {} + guid-typescript@1.0.9: {} + h3@2.0.1-rc.22(crossws@0.4.10(srvx@0.11.22)): dependencies: rou3: 0.8.1 @@ -6904,6 +6987,8 @@ snapshots: picocolors: 1.1.1 shell-quote: 1.12.0 + libheif-js@1.23.5: {} + lightningcss-android-arm64@1.32.0: optional: true @@ -7021,6 +7106,8 @@ snapshots: dependencies: p-locate: 5.0.0 + long@5.3.2: {} + loose-envify@1.4.0: dependencies: js-tokens: 4.0.0 @@ -7160,6 +7247,17 @@ snapshots: ohash@2.0.11: {} + onnxruntime-common@1.31.1: {} + + onnxruntime-web@1.31.1: + dependencies: + flatbuffers: 25.9.23 + guid-typescript: 1.0.9 + long: 5.3.2 + onnxruntime-common: 1.31.1 + platform: 1.3.6 + protobufjs: 7.6.6 + oxc-parser@0.120.0(@emnapi/core@2.0.0-alpha.3)(@emnapi/runtime@2.0.0-alpha.3): dependencies: '@oxc-project/types': 0.120.0 @@ -7279,6 +7377,8 @@ snapshots: dependencies: find-up: 3.0.0 + platform@1.3.6: {} + playwright-core@1.63.0: {} playwright@1.63.0: @@ -7327,6 +7427,20 @@ snapshots: progress@2.0.3: {} + protobufjs@7.6.6: + dependencies: + '@protobufjs/aspromise': 1.1.2 + '@protobufjs/base64': 1.1.2 + '@protobufjs/codegen': 2.0.5 + '@protobufjs/eventemitter': 1.1.1 + '@protobufjs/fetch': 1.1.1 + '@protobufjs/float': 1.0.2 + '@protobufjs/path': 1.1.2 + '@protobufjs/pool': 1.1.0 + '@protobufjs/utf8': 1.1.2 + '@types/node': 24.13.3 + long: 5.3.2 + proxy-from-env@1.1.0: {} query-selector-shadow-dom@1.0.1: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index d8d5dd3..328439c 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -4,6 +4,7 @@ allowBuilds: '@sentry/cli': true core-js: true esbuild: true + protobufjs: false overrides: # GHSA-qhr7-859c-m2p7 and GHSA-6j4f-fj2g-mc7p, through @sentry/vite-plugin > glob > minimatch. diff --git a/public/home/subject-daruma.webp b/public/home/subject-daruma.webp new file mode 100644 index 0000000..5bb1fef Binary files /dev/null and b/public/home/subject-daruma.webp differ diff --git a/public/og/add-logo-to-photos.png b/public/og/add-logo-to-photos.png new file mode 100644 index 0000000..19875ed Binary files /dev/null and b/public/og/add-logo-to-photos.png differ diff --git a/public/og/favicon-generator.png b/public/og/favicon-generator.png new file mode 100644 index 0000000..6220e52 Binary files /dev/null and b/public/og/favicon-generator.png differ diff --git a/public/og/heic-to-jpg.png b/public/og/heic-to-jpg.png new file mode 100644 index 0000000..6306ac6 Binary files /dev/null and b/public/og/heic-to-jpg.png differ diff --git a/public/og/instagram-no-crop.png b/public/og/instagram-no-crop.png new file mode 100644 index 0000000..bb5e3d5 Binary files /dev/null and b/public/og/instagram-no-crop.png differ diff --git a/public/og/jpg-to-pdf.png b/public/og/jpg-to-pdf.png new file mode 100644 index 0000000..2f9a18e Binary files /dev/null and b/public/og/jpg-to-pdf.png differ diff --git a/public/og/product-photos-white-background.png b/public/og/product-photos-white-background.png new file mode 100644 index 0000000..34a614f Binary files /dev/null and b/public/og/product-photos-white-background.png differ diff --git a/public/og/remove-background.png b/public/og/remove-background.png new file mode 100644 index 0000000..3267483 Binary files /dev/null and b/public/og/remove-background.png differ diff --git a/public/og/watermark-photos.png b/public/og/watermark-photos.png new file mode 100644 index 0000000..c5fe242 Binary files /dev/null and b/public/og/watermark-photos.png differ diff --git a/public/og/webp-to-jpg.png b/public/og/webp-to-jpg.png new file mode 100644 index 0000000..84763a5 Binary files /dev/null and b/public/og/webp-to-jpg.png differ diff --git a/scripts/generate-background-fixtures.mjs b/scripts/generate-background-fixtures.mjs new file mode 100644 index 0000000..d4c7244 --- /dev/null +++ b/scripts/generate-background-fixtures.mjs @@ -0,0 +1,146 @@ +/** + * Makes the fixtures of the background removal quality suite in + * src/features/background-removal/__tests__/fixtures: cut-out photos with a real transparent + * background, composited onto photos of rooms and streets. Because the cut-out's own transparency + * says exactly where the subject is, each composite comes with a reference mask, and with the + * subject on white for checking edges. Every source is public domain or CC0 on Wikimedia Commons; + * SOURCES.md lists them. The script checks each download against the SHA-1 Commons publishes. + * + * Run with `node scripts/generate-background-fixtures.mjs`. It needs the network. + */ +import { createHash } from 'node:crypto' +import { readFile, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = join(dirname(fileURLToPath(import.meta.url)), '..') +const out = join(root, 'src/features/background-removal/__tests__/fixtures') +const wasm = async (path) => WebAssembly.compile(await readFile(join(root, 'node_modules', path))) +const COMMONS = 'https://upload.wikimedia.org/wikipedia/commons/' +const USER_AGENT = 'hexlode-fixtures/1.0 (https://github.com/pixelactstudio/hexlode)' + +globalThis.ImageData ??= class ImageData { + constructor(data, width, height) { + this.data = data + this.width = width + this.height = height + } +} + +/** Originals on Wikimedia Commons, with the SHA-1 Commons publishes for each. */ +const SOURCES = { + emac: ['c/c3/Apple-eMac-FL_%28transparent%29.png', '6ba6a6ba520df965c6a4194d4c1105cdebce8451'], + daruma: [ + '6/69/Daruma0791_with_transparent_background.png', + '234b59015aefb5772e207cd9a94151afc75331b9', + ], + luggage: ['8/8f/Luggage-2708806_960_720.png', 'abbddb5c534326a397fd1c03f6d635b6439a33a5'], + cat: ['e/e6/Cat_Stretch.png', '794338aacf9ba2730ac1bd4ae7e0636b3629743d'], + woman: [ + '0/0a/Woman_in_White_Sleeveless_Dress_Sitting_on_Gray_Concrete_Seat_during_Daytime_transparent.png', + '5000377aa4f089ae3de267438467009307358844', + ], + apartment: ['9/98/Apartment-1851201_960_720.jpg', '9831ac7c3c95c4f2c7518f1421a59d1e6563e9f4'], + street: ['2/2b/Oxford-182705_960_720.jpg', '6b93b96cf0c18aa607e0fe6d47e1fb15108d1360'], +} + +const WIDTH = 512 +const HEIGHT = 342 +const MARGIN = 5 + +/** Subject, background, subject height in percent of the picture, and offset from the centre. */ +const FIXTURES = [ + ['emac', 'apartment', 80, 40], + ['daruma', 'street', 85, -53], + ['luggage', 'apartment', 60, -67], + ['cat', 'street', 80, -140], + ['woman', 'apartment', 95, -40], +] + +const pngDecode = await import('@jsquash/png/decode.js') +await pngDecode.init(await wasm('@jsquash/png/codec/pkg/squoosh_png_bg.wasm')) +const pngEncode = await import('@jsquash/png/encode.js') +await pngEncode.init(await wasm('@jsquash/png/codec/pkg/squoosh_png_bg.wasm')) +const jpegDecode = await import('@jsquash/jpeg/decode.js') +await jpegDecode.init(await wasm('@jsquash/jpeg/codec/dec/mozjpeg_dec.wasm')) +const jpegEncode = await import('@jsquash/jpeg/encode.js') +await jpegEncode.init(await wasm('@jsquash/jpeg/codec/enc/mozjpeg_enc.wasm')) +const resizer = await import('@jsquash/resize') +await resizer.initResize(await wasm('@jsquash/resize/lib/resize/pkg/squoosh_resize_bg.wasm')) + +async function download(name) { + const [path, sha1] = SOURCES[name] + const response = await fetch(COMMONS + path, { headers: { 'user-agent': USER_AGENT } }) + if (!response.ok) throw new Error(`${name}: ${response.status}`) + const bytes = new Uint8Array(await response.arrayBuffer()) + const actual = createHash('sha1').update(bytes).digest('hex') + if (actual !== sha1) throw new Error(`${name}: SHA-1 ${actual}, expected ${sha1}`) + return path.endsWith('.png') ? pngDecode.default(bytes.buffer) : jpegDecode.default(bytes.buffer) +} + +const resize = (image, width, height) => + resizer.default(image, { width, height, method: 'lanczos3', premultiply: true, linearRGB: false }) + +/** Scales the background to cover the picture and keeps its middle. */ +async function cover(image) { + const scale = Math.max(WIDTH / image.width, HEIGHT / image.height) + const scaled = await resize( + image, + Math.ceil(image.width * scale), + Math.ceil(image.height * scale), + ) + const left = Math.floor((scaled.width - WIDTH) / 2) + const top = Math.floor((scaled.height - HEIGHT) / 2) + const data = new Uint8ClampedArray(WIDTH * HEIGHT * 4) + for (let y = 0; y < HEIGHT; y += 1) { + const start = ((y + top) * scaled.width + left) * 4 + data.set(scaled.data.subarray(start, start + WIDTH * 4), y * WIDTH * 4) + } + return new ImageData(data, WIDTH, HEIGHT) +} + +const jpeg = (image) => + jpegEncode.default(image, { quality: 88, progressive: false, chroma_subsample: 2 }) + +for (const [name, background, percent, offset] of FIXTURES) { + const subject = await download(name) + const scene = await cover(await download(background)) + const height = Math.round((HEIGHT * percent) / 100) + const width = Math.round((subject.width * height) / subject.height) + const cutOut = await resize(subject, width, height) + const left = Math.round((WIDTH - width) / 2 + offset) + const top = HEIGHT - height - MARGIN + const photo = new Uint8ClampedArray(scene.data) + const onWhite = new Uint8ClampedArray(WIDTH * HEIGHT * 4).fill(255) + const mask = new Uint8ClampedArray(WIDTH * HEIGHT * 4) + for (let index = 3; index < mask.length; index += 4) mask[index] = 255 + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const px = x + left + const py = y + top + if (px < 0 || py < 0 || px >= WIDTH || py >= HEIGHT) continue + const from = (y * width + x) * 4 + const to = (py * WIDTH + px) * 4 + const alpha = cutOut.data[from + 3] / 255 + for (let channel = 0; channel < 3; channel += 1) { + const colour = cutOut.data[from + channel] + photo[to + channel] = colour * alpha + photo[to + channel] * (1 - alpha) + onWhite[to + channel] = colour * alpha + 255 * (1 - alpha) + mask[to + channel] = cutOut.data[from + 3] + } + } + } + await writeFile( + join(out, `${name}.jpg`), + new Uint8Array(await jpeg(new ImageData(photo, WIDTH, HEIGHT))), + ) + await writeFile( + join(out, `${name}-white.jpg`), + new Uint8Array(await jpeg(new ImageData(onWhite, WIDTH, HEIGHT))), + ) + await writeFile( + join(out, `${name}-mask.png`), + new Uint8Array(await pngEncode.default(new ImageData(mask, WIDTH, HEIGHT))), + ) + console.log(`Wrote ${name}`) +} diff --git a/scripts/generate-fixtures.mjs b/scripts/generate-fixtures.mjs index 91e52f8..9064c2a 100644 --- a/scripts/generate-fixtures.mjs +++ b/scripts/generate-fixtures.mjs @@ -1,14 +1,18 @@ /** * Generates the image fixtures in src/features/images/__tests__/fixtures. * Run with `node scripts/generate-fixtures.mjs`. The EXIF blocks are assembled byte by byte here, - * independently of the app's metadata code, so tests can check that code against them. + * independently of the app's metadata code, so tests can check that code against them. The HEIC + * fixtures also need libheif's `heif-enc` on PATH, as described at the end. * * Every image is 48x32: the left half red, the right half blue. Images with alpha have a fully - * transparent top half. + * transparent top half. grid.heic is 96x64 in four coloured quarters. */ -import { mkdir, readFile, writeFile } from 'node:fs/promises' +import { execFile } from 'node:child_process' +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' import { dirname, join } from 'node:path' import { fileURLToPath } from 'node:url' +import { promisify } from 'node:util' const root = join(dirname(fileURLToPath(import.meta.url)), '..') const out = join(root, 'src/features/images/__tests__/fixtures') @@ -200,3 +204,80 @@ await save('photo.qoi', await qoiEncode.default(pixels())) // A JPEG whose header is valid but whose image data is cut off. await save('malformed.jpg', new Uint8Array(jpeg).slice(0, 180)) console.log('Fixtures written to', out) + +/* + * HEIC fixtures, laid out the way phone cameras write them: HEVC coded, with EXIF in its own item, + * a camera rotation stored as an `irot` property next to the EXIF orientation tag, and the large + * photo cut into a grid of tiles with a thumbnail and a Display P3 colour profile. They are + * encoded by libheif's `heif-enc` with x265, which must be on PATH (for example through + * `nix shell nixpkgs#libheif`). heif-enc copies EXIF and the colour profile from a JPEG input, + * and turns its EXIF orientation into `irot` and `imir` while keeping the tag, as phones do. + */ +const execute = promisify(execFile) +const heifEnc = await execute('heif-enc', ['--version']).catch(() => null) +if (!heifEnc) { + console.log('heif-enc is not on PATH, so the HEIC fixtures were left as they are.') +} else { + /** Display P3 (DisplayP3-v4.icc) from github.com/saucecontrol/Compact-ICC-Profiles, CC0 1.0. */ + const DISPLAY_P3 = Uint8Array.from( + atob( + 'AAAB4GxjbXMEIAAAbW50clJHQiBYWVogB+IAAwAUAAkADgAdYWNzcE1TRlQAAAAAc2F3c2N0cmwAAAAAAAAAAAAAAAAAAPbWAAEAAAAA0y1oYW5kwzc6zlf4VsuhS9h6V6sQYQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKZGVzYwAAAPwAAAAiY3BydAAAASAAAAAid3RwdAAAAUQAAAAUY2hhZAAAAVgAAAAsclhZWgAAAYQAAAAUZ1hZWgAAAZgAAAAUYlhZWgAAAawAAAAUclRSQwAAAcAAAAAgZ1RSQwAAAcAAAAAgYlRSQwAAAcAAAAAgbWx1YwAAAAAAAAABAAAADGVuVVMAAAAGAAAAHABzAFAAMwAAbWx1YwAAAAAAAAABAAAADGVuVVMAAAAGAAAAHABDAEMAMAAAWFlaIAAAAAAAAPbWAAEAAAAA0y1zZjMyAAAAAAABDEIAAAXe///zJQAAB5MAAP2Q///7of///aIAAAPcAADAblhZWiAAAAAAAACD3wAAPb////+7WFlaIAAAAAAAAEq/AACxNwAACrlYWVogAAAAAAAAKDgAABEKAADIuXBhcmEAAAAAAAMAAAACZmkAAPKnAAANWQAAE9AAAApb', + ), + (character) => character.charCodeAt(0), + ) + const jpegWithIcc = (jpegBytes, icc) => { + const payload = [...new TextEncoder().encode('ICC_PROFILE'), 0, 1, 1, ...icc] + const length = payload.length + 2 + const segment = [0xff, 0xe2, length >> 8, length & 0xff, ...payload] + return Uint8Array.from([...jpegBytes.slice(0, 2), ...segment, ...jpegBytes.slice(2)]) + } + /** 96x64 in quarters: red top left, green top right, blue bottom left, white bottom right. */ + const quarters = () => { + const width = 96 + const height = 64 + const data = new Uint8ClampedArray(width * height * 4) + for (let y = 0; y < height; y += 1) { + for (let x = 0; x < width; x += 1) { + const top = y < height / 2 + const left = x < width / 2 + const colour = top + ? left + ? [255, 0, 0] + : [0, 255, 0] + : left + ? [0, 0, 255] + : [255, 255, 255] + data.set([...colour, 255], (y * width + x) * 4) + } + } + return new ImageData(data, width, height) + } + + const work = await mkdtemp(join(tmpdir(), 'hexlode-heic-')) + const heic = async (name, input, inputName, options = []) => { + const source = join(work, inputName) + await writeFile(source, new Uint8Array(input)) + const target = join(work, name) + await execute('heif-enc', ['-q', '90', ...options, '-o', target, source]) + await save(name, await readFile(target)) + return readFile(target) + } + const photo = await heic('photo.heic', jpegWithExif(jpeg, exif()), 'photo.jpg') + await heic( + 'oriented.heic', + jpegWithExif(jpeg, exif({ orientation: 6, gps: false })), + 'oriented.jpg', + ) + const large = await jpegEncode.default(quarters(), { quality: 95 }) + await heic( + 'grid.heic', + jpegWithIcc(jpegWithExif(large, exif({ orientation: 6 })), DISPLAY_P3), + 'grid.jpg', + ['--cut-tiles', '32', '--thumb', '24'], + ) + await heic('alpha.heic', await pngEncode.default(pixels({ alpha: true })), 'alpha.png') + // A HEIC whose boxes are intact but whose coded image is cut off. + await save('malformed.heic', new Uint8Array(photo).slice(0, photo.length - 120)) + await rm(work, { recursive: true, force: true }) + console.log('HEIC fixtures written with', heifEnc.stdout.trim().split('\n')[0]) +} diff --git a/scripts/generate-model-fixture.mjs b/scripts/generate-model-fixture.mjs new file mode 100644 index 0000000..baea0ea --- /dev/null +++ b/scripts/generate-model-fixture.mjs @@ -0,0 +1,98 @@ +/** + * Writes src/features/background-removal/__tests__/fixtures/threshold.onnx, a tiny stand-in for + * the background removal model with the same input and output: `input_image`, 1 × 3 × 512 × 512, + * in; logits, 1 × 1 × 512 × 512, out. It averages the three normalised channels, so bright pixels + * come out as the subject and dark ones as background. Tests run the real inference worker and + * ONNX Runtime on it in milliseconds, offline. + * + * Run with `node scripts/generate-model-fixture.mjs`. The ONNX protobuf is written by hand here, + * so the script needs no packages. + */ +import { mkdir, writeFile } from 'node:fs/promises' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = join(dirname(fileURLToPath(import.meta.url)), '..') +const out = join(root, 'src/features/background-removal/__tests__/fixtures/threshold.onnx') +const SIZE = 512 +/** Logits are 10 × (mean of the normalised channels − 0.2): white is +22, black −22. */ +const THRESHOLD = 0.2 +const SCALE = 10 + +function varint(value) { + const bytes = [] + let rest = BigInt(value) + do { + let byte = Number(rest & 0x7fn) + rest >>= 7n + if (rest > 0n) byte |= 0x80 + bytes.push(byte) + } while (rest > 0n) + return bytes +} + +const key = (field, wireType) => varint((field << 3) | wireType) +const int = (field, value) => [...key(field, 0), ...varint(value)] +const bytes = (field, data) => [...key(field, 2), ...varint(data.length), ...data] +const text = (field, value) => bytes(field, [...new TextEncoder().encode(value)]) + +function floats(field, values) { + const data = new Uint8Array(values.length * 4) + const view = new DataView(data.buffer) + for (const [index, value] of values.entries()) view.setFloat32(index * 4, value, true) + return bytes(field, [...data]) +} + +/** TensorProto: a float scalar. */ +const scalar = (name, value) => [...int(2, 1), ...floats(4, [value]), ...text(8, name)] + +/** ValueInfoProto: a float tensor of a fixed shape. */ +function valueInfo(name, shape) { + const dims = shape.flatMap((size) => bytes(1, int(1, size))) + const tensorType = [...int(1, 1), ...bytes(2, dims)] + return [...text(1, name), ...bytes(2, bytes(1, tensorType))] +} + +function node(opType, inputs, outputs, attributes = []) { + return [ + ...inputs.flatMap((input) => text(1, input)), + ...outputs.flatMap((output) => text(2, output)), + ...text(3, `${opType}_0`), + ...text(4, opType), + ...attributes.flatMap((attribute) => bytes(5, attribute)), + ] +} + +/** AttributeProto: a list of integers (type INTS). */ +const ints = (name, values) => [ + ...text(1, name), + ...values.flatMap((v) => int(8, v)), + ...int(20, 7), +] +/** AttributeProto: an integer (type INT). */ +const integer = (name, value) => [...text(1, name), ...int(3, value), ...int(20, 2)] + +const graph = [ + ...bytes( + 1, + node('ReduceMean', ['input_image'], ['mean'], [ints('axes', [1]), integer('keepdims', 1)]), + ), + ...bytes(1, node('Sub', ['mean', 'threshold'], ['centred'])), + ...bytes(1, node('Mul', ['centred', 'scale'], ['logits'])), + ...text(2, 'threshold'), + ...bytes(5, scalar('threshold', THRESHOLD)), + ...bytes(5, scalar('scale', SCALE)), + ...bytes(11, valueInfo('input_image', [1, 3, SIZE, SIZE])), + ...bytes(12, valueInfo('logits', [1, 1, SIZE, SIZE])), +] + +const model = [ + ...int(1, 7), + ...text(2, 'hexlode-tests'), + ...bytes(7, graph), + ...bytes(8, int(2, 13)), +] + +await mkdir(dirname(out), { recursive: true }) +await writeFile(out, Uint8Array.from(model)) +console.log(`Wrote ${out} (${model.length} bytes)`) diff --git a/src/client.tsx b/src/client.tsx index 8de511d..da14c32 100644 --- a/src/client.tsx +++ b/src/client.tsx @@ -2,6 +2,7 @@ import { StartClient } from '@tanstack/react-start/client' import { StrictMode, startTransition } from 'react' import { hydrateRoot } from 'react-dom/client' +import { registerServiceWorker } from '#/features/offline/register' import { PUBLIC_CONFIG_META } from '#/features/usage/constants' import { startErrorReporting } from '#/features/usage/error-reports' import { startAnalytics } from '#/features/usage/usage' @@ -23,3 +24,5 @@ startTransition(() => { , ) }) + +registerServiceWorker() diff --git a/src/features/app-shell/site-footer.tsx b/src/features/app-shell/site-footer.tsx index 254ae54..1ba6f1d 100644 --- a/src/features/app-shell/site-footer.tsx +++ b/src/features/app-shell/site-footer.tsx @@ -11,6 +11,7 @@ import { } from '#/features/app-shell/constants' import { FooterWordmark } from '#/features/app-shell/footer-wordmark' import { HexlodeMark } from '#/features/app-shell/hexlode-mark' +import { BUILT_IN_TOOLS } from '#/features/built-in-tools/tools' import { QUICK_TOOL_GROUPS } from '#/features/quick-tools/tool-ui' import { QUICK_TOOL_DEFINITIONS } from '#/features/quick-tools/tools' import { RouterLink } from '#/lib/router-link' @@ -23,6 +24,10 @@ const COLUMNS: { title: string; links: { label: string; href: string }[] }[] = [ href: QUICK_TOOL_DEFINITIONS[tool].path, })), }, + { + title: 'Ready-made tools', + links: BUILT_IN_TOOLS.map((tool) => ({ label: tool.name, href: `/tools/${tool.slug}` })), + }, { title: 'Hexlode', links: [ @@ -35,6 +40,7 @@ const COLUMNS: { title: string; links: { label: string; href: string }[] }[] = [ links: [ { label: 'GitHub', href: REPOSITORY_URL }, { label: 'Codec licences', href: '/licenses/jsquash.txt' }, + { label: 'Model licences', href: '/licenses/background-removal.txt' }, ], }, { @@ -84,7 +90,7 @@ export function SiteFooter() {