diff --git a/.github/workflows/dep_build_wasm_examples.yml b/.github/workflows/dep_build_wasm_examples.yml index ee2fe4cf..99d4cbdc 100644 --- a/.github/workflows/dep_build_wasm_examples.yml +++ b/.github/workflows/dep_build_wasm_examples.yml @@ -54,6 +54,7 @@ jobs: build/*/pulley/*.aot src/tests/rust_guests/component_sample/wit/component-world.wasm src/tests/c_guests/wasmsamples/components/runcomponent-world.wasm + src/tests/rust_guests/component_sample/target/wasm32-unknown-unknown/*/component_sample.wasm key: wasm-modules-${{ matrix.arch.name }}-${{ matrix.wasmtime_version.name }}-${{ hashFiles('src/tests/**', 'src/hyperlight_wasm_aot/**', 'Cargo.lock', '.cargo/config.toml', 'rust-toolchain.toml', 'Justfile', 'c.just', '.github/workflows/dep_build_wasm_examples.yml') }} - name: Remove default clang if: steps.modules-cache.outputs.cache-hit != 'true' @@ -106,6 +107,7 @@ jobs: build/*/pulley/*.aot src/tests/rust_guests/component_sample/wit/component-world.wasm src/tests/c_guests/wasmsamples/components/runcomponent-world.wasm + src/tests/rust_guests/component_sample/target/wasm32-unknown-unknown/*/component_sample.wasm key: ${{ steps.modules-cache.outputs.cache-primary-key }} # Build jobs compile examples against the WIT worlds, so they ship with the modules. - name: Upload Wasm Modules (wasmtime ${{ matrix.wasmtime_version.name }}) @@ -118,3 +120,4 @@ jobs: build/*/pulley/*.aot src/tests/rust_guests/component_sample/wit/component-world.wasm src/tests/c_guests/wasmsamples/components/runcomponent-world.wasm + src/tests/rust_guests/component_sample/target/wasm32-unknown-unknown/*/component_sample.wasm diff --git a/.github/workflows/dep_rust.yml b/.github/workflows/dep_rust.yml index 29d7346b..5cb138f4 100644 --- a/.github/workflows/dep_rust.yml +++ b/.github/workflows/dep_rust.yml @@ -170,8 +170,6 @@ jobs: run: just test ${{ matrix.config }} ${{ env.FEATURES }} working-directory: ./src/hyperlight_wasm - # Runs before Test Examples, which replaces the native components in - # build/ with Pulley ones. - name: Set up .NET uses: actions/setup-dotnet@0f4a354dd5547a28c70b7853dbd95d5b67886e73 # v6.1.0 with: diff --git a/CHANGELOG.md b/CHANGELOG.md index e5b108b6..038393bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). * Experimental: add the `experimental_runtime_file` feature, with `SandboxBuilder::with_runtime_file` to load the guest runtime from a file. * Experimental: add the `hyperlight-wasm-capi` crate, a C API that calls component functions with arguments and results in Hyperlight's component wire format. It is not published. * Experimental: add the `Hyperlight.Wasm` and `Hyperlight.Wasm.Build` .NET packages, which run components from .NET with typed C# bindings generated from WIT at build time. See `dotnet/README.md`. They are not published. +* Experimental: add `hyperlight-wasm-aot experimental-build-runtime`, which builds a guest runtime for a WIT world. `Hyperlight.Wasm.Build` runs it for `HyperlightWit` items with `Runtime` set, and precompiles `HyperlightComponent` items. `HyperlightWasmtimeVersion` selects the Wasmtime version for both. +* Accept a `.wit` file or a directory of `.wit` files in `WIT_WORLD` when building the guest runtime. `host_bindgen!()` without arguments still needs a wasm-encoded package. ### Changed diff --git a/Cargo.lock b/Cargo.lock index e0f47fbc..68012af7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,9 +319,9 @@ dependencies = [ [[package]] name = "cargo-hyperlight" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c52f1c9d4b6173aefb66fdc2e3e112ddb359fe2fbdbaafe1b9ba25f7ac578b39" +checksum = "cca0d3b8a8a79c7d55aecb3f68afe40675e70871ec7056f30fecee82cff1b877" dependencies = [ "anyhow", "clap", @@ -329,7 +329,7 @@ dependencies = [ "const_format", "glob", "libc", - "object 0.39.1", + "object 0.40.0", "os_str_bytes", "proc-macro2", "quote", @@ -339,6 +339,7 @@ dependencies = [ "serde", "serde_json", "tempfile", + "toml", "which", ] @@ -2090,10 +2091,13 @@ dependencies = [ name = "hyperlight-wasm-aot" version = "0.16.0" dependencies = [ + "cargo-hyperlight", "cargo-util-schemas", "cargo_metadata", "clap", + "hyperlight-wasm-runtime", "object 0.39.1", + "tar", "wasmtime 36.0.18", "wasmtime 47.0.4", ] @@ -2839,6 +2843,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "object" +version = "0.40.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd229a0361b9d0d4396176e02d65897f487eebeab7caa6d443855ee152ca0b9c" +dependencies = [ + "memchr", +] + [[package]] name = "oci-spec" version = "0.10.0" diff --git a/Justfile b/Justfile index 0ddc546e..06d09414 100644 --- a/Justfile +++ b/Justfile @@ -152,8 +152,8 @@ examples-ci target=default-target features="": (examples-modules target features # Runs the examples-ci examples on the modules build-ci-examples builds. examples-ci-prebuilt target=default-target features="": (run-examples-modules target features) (run-examples-components target features) (use-pulley-examples target) (run-examples-pulley target features) -# Builds every module the CI build jobs load, for the given profiles. C modules build -# in release only. Pulley builds go to build//pulley. +# Builds every module the CI build jobs load for the given profiles. C modules build in release +# only. Pulley builds go to build//pulley. build-ci-examples features="" profiles="debug release": (build-wasm-examples "release" features) #!/usr/bin/env bash set -euo pipefail @@ -183,16 +183,20 @@ run-examples-components target=default-target features="": {{ wit-world }} {{ wit-world-name-greeter }} cargo run {{ if features =="" {''} else {"--no-default-features -F kvm -F " + features } }} --profile={{ if target == "debug" {"dev"} else { target } }} --example component_greeter_example {{ wit-world }} cargo test -p hyperlight-wasm-capi {{ if features =="" {''} else {"--no-default-features -F kvm -F " + features } }} --profile={{ if target == "debug" {"dev"} else { target } }} -- --include-ignored -# Builds the native libraries and a guest runtime for the component_sample example world, for the .NET tests -dotnet-test-assets target=default-target features="": ensure-cargo-hyperlight +# Builds the native libraries for the .NET tests +dotnet-test-assets target=default-target features="": cargo build -p hyperlight-wasm-capi -p hyperlight-wasm-csharp-bindgen {{ if features =="" {''} else {"--no-default-features -F kvm -F " + features } }} --profile={{ if target == "debug" {"dev"} else { target } }} - {{ wit-world }} cargo hyperlight build {{ if features =~ "wasmtime_latest" {"--no-default-features -F wasmtime_latest"} else {""} }} --profile={{ if target == "debug" {"dev"} else { target } }} --manifest-path src/hyperlight_wasm_runtime/Cargo.toml --target-dir target/component-sample-runtime --locked # Runs the .NET tests. Needs the .NET 10 SDK or later. dotnet-test target=default-target features="": (build-rust-component-examples target features) (dotnet-test-prebuilt target features) -# Runs the .NET tests on the component_sample component that build-ci-examples builds. -dotnet-test-prebuilt target=default-target features="": (dotnet-test-assets target features) (dotnet-run-tests target) +# Runs the .NET tests on the component_sample component that build-ci-examples builds. The test +# build creates the runtime and compiles the component with hyperlight-wasm-aot. Release runs +# build the runtime in the hyperlight-wasm build script's directory to reuse its Wasmtime +# build. Debug runs use the default directory under obj. +dotnet-test-prebuilt target=default-target features="": (dotnet-test-assets target features) + cargo build -p hyperlight-wasm-aot + just dotnet-run-tests {{ target }} -p:HyperlightWasmAot={{ justfile_directory() }}/target/debug/hyperlight-wasm-aot{{ if os() == "windows" {".exe"} else {""} }} {{ if target == "debug" {""} else {"-p:HyperlightRuntimeTargetDirectory=" + justfile_directory() + "/target/hyperlight-wasm-runtime/"} }} {{ if features =~ "wasmtime_latest" {"-p:HyperlightWasmtimeVersion=latest"} else {""} }} dotnet-tests := "dotnet/tests/Hyperlight.Wasm.Tests" diff --git a/README.md b/README.md index dc3e11a5..a4796c99 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,9 @@ a binary encoding of a component type (e.g. the result of running `wasm-tools component wit -w -o /path/to/output.wasm /path/to/input.wit`), which will ensure that the resultant library includes a guest binary that is specialised to load components with -that type. Then, use `hyperlight_component_macro::host_bindgen!()` to +that type. The guest binary also accepts a `.wit` file or a directory of +`.wit` files, but `host_bindgen!()` without arguments needs the binary +encoding. Then, use `hyperlight_component_macro::host_bindgen!()` to generate bindings from the same component type in the host. For a complete (albeit small) example of this, see [this example](https://aka.ms/hyperlight-wasm-sockets-sample). diff --git a/RustDev.md b/RustDev.md index f31b434f..d7e3b4c0 100644 --- a/RustDev.md +++ b/RustDev.md @@ -26,6 +26,10 @@ wasm-tools component wit -w -o world.wasm world.wit export WIT_WORLD=$(readlink -f world.wasm) ``` +The guest binary also accepts a `.wit` file or a directory of `.wit` +files in `WIT_WORLD`. `host_bindgen!()` without arguments reads +`WIT_WORLD` too and needs the binary encoding. + ## `ProtoWasmSandbox` vs `WasmSandbox` vs `LoadedWasmSandbox` The three primary APIs with which you'll be interacting are [`ProtoWasmSandbox`](./src/hyperlight_wasm/src/sandbox/proto_wasm_sandbox.rs), [`WasmSandbox`](./src/hyperlight_wasm/src/sandbox/wasm_sandbox.rs) and [`LoadedWasmSandbox`](./src/hyperlight_wasm/src/sandbox/loaded_wasm_sandbox.rs). These are three different Rust `struct`s that provide a type-safe way to ensure the current state of the system. diff --git a/dotnet/README.md b/dotnet/README.md index c110cde5..d3884025 100644 --- a/dotnet/README.md +++ b/dotnet/README.md @@ -6,7 +6,7 @@ Run WebAssembly components in Hyperlight micro VMs from .NET, with typed bindings generated from WIT. * `Hyperlight.Wasm` is the runtime library. It wraps the native `hyperlight-wasm-capi` library. -* `Hyperlight.Wasm.Build` generates C# from WIT at build time. It needs no Rust toolchain. Visual Studio builds need Visual Studio 2026 or later. +* `Hyperlight.Wasm.Build` generates C# from WIT at build time, which needs no Rust toolchain. It can also build the guest runtime and components. Visual Studio builds need Visual Studio 2026 or later. Both target .NET 10 or later. The packages hold native libraries for the runtime identifier they were packed on. @@ -42,15 +42,29 @@ uint sum = component.Adder.Add(1, 2); ## Guest runtime and components -A component needs a guest runtime built for its WIT world, because the runtime's guest bindings are generated when it is compiled. Building a runtime needs Rust and a hyperlight-wasm checkout of the same version as the packages: +A component needs a guest runtime built for its WIT world, because the runtime's guest bindings are generated when it is compiled. Components must be precompiled. The runtime, the precompiled component and the packages must come from the same hyperlight-wasm version. + +Both steps use `hyperlight-wasm-aot`, which needs rustup and clang. Its runtime build is experimental, and uses Rust 1.94.1 unless `RUSTUP_TOOLCHAIN` is set. Install the packages' version without the `-preview` suffix: ```sh -wasm-tools component wit example.wit -w -o example-world.wasm -WIT_WORLD=example-world.wasm cargo hyperlight build --release \ - --manifest-path src/hyperlight_wasm_runtime/Cargo.toml --target-dir target/example-runtime +cargo install --locked hyperlight-wasm-aot --version ``` -Precompile the component with `hyperlight-wasm-aot compile --component`. The runtime, the AOT tool and the packages must come from the same hyperlight-wasm version. +### Building them with the project + +Set `Runtime` on a `HyperlightWit` item and list components as `HyperlightComponent` items. The build then creates the runtime and `.aot` in the output directory. The first runtime build takes several minutes. `HyperlightWasmAot` sets the tool's path, and `HyperlightWasmtimeVersion` selects `lts`, the default, or `latest`. + +```xml + + +``` + +### Building them by hand + +```sh +hyperlight-wasm-aot experimental-build-runtime example.wit example-runtime --world example +hyperlight-wasm-aot compile --component example.wasm example.aot +``` ## Sandboxes @@ -78,8 +92,8 @@ Hyperlight runs a helper process, `hyperlight_surrogate_.exe`, for each sa ## Development ```sh -just dotnet-test # builds the native libraries, a test runtime and component, then runs the tests -just dotnet-pack # packs both packages into dotnet/artifacts +just dotnet-test # builds the native libraries and test component, then runs the tests +just dotnet-pack # packs both packages into dotnet/artifacts ``` `HyperlightNativeDir` overrides where the projects find the native libraries. diff --git a/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.props b/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.props index 8072c94d..0b07ff45 100644 --- a/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.props +++ b/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.props @@ -3,15 +3,22 @@ + diff --git a/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.targets b/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.targets index fefb067b..c006ca39 100644 --- a/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.targets +++ b/dotnet/src/Hyperlight.Wasm.Build/build/Hyperlight.Wasm.Build.targets @@ -34,4 +34,70 @@ + + + + + + + hyperlight-wasm-aot + $(BaseIntermediateOutputPath)hyperlight-wasm-runtime/ + lts + + + + + + <_HyperlightRuntime Include="@(HyperlightWit)" Condition="'%(HyperlightWit.Runtime)' != ''"> + + $([System.String]::Copy('%(FullPath)').TrimEnd('\/')) + --world "%(HyperlightWit.World)" + + + + + + + + + + + + + + <_HyperlightWasmAotTime Condition="Exists('$(HyperlightWasmAot)')">$([System.IO.File]::GetLastWriteTime('$(HyperlightWasmAot)').Ticks) + + + + + + + + + + + + + + + + + diff --git a/dotnet/tests/Hyperlight.Wasm.Tests/ComponentTests.cs b/dotnet/tests/Hyperlight.Wasm.Tests/ComponentTests.cs index 83bb8a4e..1366e737 100644 --- a/dotnet/tests/Hyperlight.Wasm.Tests/ComponentTests.cs +++ b/dotnet/tests/Hyperlight.Wasm.Tests/ComponentTests.cs @@ -10,7 +10,7 @@ namespace Hyperlight.Wasm.Tests; public sealed class ComponentTests { - private static readonly string Assets = Path.Combine(AppContext.BaseDirectory, "assets"); + private static readonly string Assets = AppContext.BaseDirectory; private static readonly string Runtime = Path.Combine(Assets, "component-sample-runtime"); private static readonly string Component = Path.Combine(Assets, "component_sample.aot"); @@ -334,7 +334,7 @@ public sealed class SignalTests [Fact] public async Task GcSuspendsSpinningThreadAfterSandboxLoads() { - var assets = Path.Combine(AppContext.BaseDirectory, "assets"); + var assets = AppContext.BaseDirectory; using var sandbox = new WasmSandbox( Path.Combine(assets, "component-sample-runtime"), new SandboxOptions { ScratchSize = 100 << 20 }); diff --git a/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj b/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj index 56397d13..d9675b86 100644 --- a/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj +++ b/dotnet/tests/Hyperlight.Wasm.Tests/Hyperlight.Wasm.Tests.csproj @@ -26,18 +26,10 @@ + - - - - - - aarch64 - x86_64 - - - - + + diff --git a/src/hyperlight_wasm_aot/Cargo.toml b/src/hyperlight_wasm_aot/Cargo.toml index 000b2749..cf4cbc7b 100644 --- a/src/hyperlight_wasm_aot/Cargo.toml +++ b/src/hyperlight_wasm_aot/Cargo.toml @@ -21,6 +21,14 @@ clap = { version = "4.6", features = ["derive"] } cargo_metadata = "0.23" cargo-util-schemas = "=0.14.1" object = { version = "0.39.1", default-features = false, features = ["read_core", "elf"] } +cargo-hyperlight = "0.1.15" +tar = "0.4.46" +# Makes Cargo fetch the runtime source that `experimental-build-runtime` builds. +hyperlight-wasm-runtime.workspace = true + +[build-dependencies] +cargo_metadata = "0.23" +tar = "0.4.46" [features] gdb = ["wasmtime/debug-builtins", "wasmtime_lts/debug-builtins"] diff --git a/src/hyperlight_wasm_aot/build.rs b/src/hyperlight_wasm_aot/build.rs new file mode 100644 index 00000000..2409858f --- /dev/null +++ b/src/hyperlight_wasm_aot/build.rs @@ -0,0 +1,51 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright 2026 The Hyperlight Authors. + +// Records where `experimental-build-runtime` finds the hyperlight-wasm-runtime +// source, in OUT_DIR/runtime_source.rs. + +use std::path::PathBuf; +use std::{env, fs}; + +use cargo_metadata::MetadataCommand; + +fn main() { + println!("cargo::rerun-if-changed=build.rs"); + + let metadata = MetadataCommand::new() + .exec() + .expect("could not run cargo metadata"); + let runtime = metadata + .packages + .iter() + .find(|p| *p.name == "hyperlight-wasm-runtime") + .expect("hyperlight-wasm-runtime not found in cargo metadata"); + let manifest = runtime.manifest_path.as_std_path(); + let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + + let registry = runtime + .source + .as_ref() + .is_some_and(|s| s.repr.starts_with("registry+") || s.repr.starts_with("sparse+")); + let source = if !registry { + // A checkout, including a git one, builds the runtime in place, against + // its workspace's other crates. A packaged copy would use their released + // versions. + format!("RuntimeSource::Path({manifest:?})") + } else { + // Cargo can garbage collect registry sources, so embed them. + let prefix = format!("hyperlight-wasm-runtime-{}", runtime.version); + let archive = out_dir.join("runtime-source.tar"); + let mut builder = tar::Builder::new(fs::File::create(&archive).unwrap()); + builder + .append_dir_all(&prefix, manifest.parent().unwrap()) + .unwrap(); + builder.finish().unwrap(); + format!("RuntimeSource::Archive({prefix:?}, include_bytes!({archive:?}))") + }; + fs::write( + out_dir.join("runtime_source.rs"), + format!("const RUNTIME_SOURCE: RuntimeSource = {source};\n"), + ) + .unwrap(); +} diff --git a/src/hyperlight_wasm_aot/src/main.rs b/src/hyperlight_wasm_aot/src/main.rs index 0b39d404..dd5b3bcd 100644 --- a/src/hyperlight_wasm_aot/src/main.rs +++ b/src/hyperlight_wasm_aot/src/main.rs @@ -15,7 +15,8 @@ limitations under the License. */ use std::fmt::Display; -use std::path::Path; +use std::fs; +use std::path::{Path, PathBuf}; use cargo_metadata::{MetadataCommand, Package}; use cargo_util_schemas::manifest::PackageName; @@ -100,6 +101,193 @@ enum Commands { #[arg(long, value_enum, default_value = "lts")] wasmtime_version: WasmtimeVersion, }, + + /// Experimental: build a guest runtime for a WIT world + /// + /// Needs rustup and clang. Builds with Rust 1.94.1 unless RUSTUP_TOOLCHAIN is + /// set. Use a runtime with hosts and precompiled components from this version + /// of hyperlight-wasm. + ExperimentalBuildRuntime { + /// The `.wit` file, directory of `.wit` files or wasm-encoded WIT package that defines the world + wit: PathBuf, + + /// The output file path + output: PathBuf, + + /// The world to build for. Defaults to the last world in the package. + #[arg(long)] + world: Option, + + /// Wasmtime version the runtime uses. Precompile components with the same version. + #[arg(long, value_enum, default_value = "lts")] + wasmtime_version: WasmtimeVersion, + + /// Directory for build artifacts, reused by later builds. Runs of this command that share it run one at a time. + #[arg(long, default_value = "target/hyperlight-wasm-runtime")] + target_dir: PathBuf, + }, +} + +// Only one variant is built, chosen by build.rs. +#[allow(dead_code)] +enum RuntimeSource { + /// The runtime's manifest in a checkout. + Path(&'static str), + /// The top-level directory name and a tar of the runtime crate. + Archive(&'static str, &'static [u8]), +} + +include!(concat!(env!("OUT_DIR"), "/runtime_source.rs")); + +type BoxError = Box; + +/// The toolchain that builds the runtime when `RUSTUP_TOOLCHAIN` is unset. +/// Keep it in sync with rust-toolchain.toml. +const RUNTIME_TOOLCHAIN: &str = "1.94.1"; + +/// Returns the runtime's manifest, extracting an embedded source into `target_dir`. +/// The caller holds the lock on `target_dir`. +fn runtime_manifest(target_dir: &Path) -> Result { + match RUNTIME_SOURCE { + RuntimeSource::Path(manifest) => Ok(PathBuf::from(manifest)), + RuntimeSource::Archive(prefix, archive) => { + const MARKER: &str = ".hyperlight-wasm-aot-extracted"; + let parent = target_dir.join("source"); + let source = parent.join(prefix); + // The marker shows that an earlier extraction finished. + let marker = source.join(MARKER); + if !marker.exists() { + if source.exists() { + fs::remove_dir_all(&source)?; + } + tar::Archive::new(archive).unpack(&parent)?; + // Keep Cargo from treating an enclosing project as the workspace. + let manifest = source.join("Cargo.toml"); + let mut contents = fs::read_to_string(&manifest)?; + contents.push_str("\n[workspace]\n"); + fs::write(&manifest, contents)?; + fs::write(&marker, "")?; + } + Ok(source.join("Cargo.toml")) + } + } +} + +/// Writes `bytes` to a sibling temporary file, then renames it to `path`, so +/// an interrupted write never leaves a partial file at `path`. +fn write_atomic(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + // The process ID keeps concurrent writers and existing files apart. + let mut tmp = path.as_os_str().to_owned(); + tmp.push(format!(".{}.tmp", std::process::id())); + let tmp = PathBuf::from(tmp); + fs::write(&tmp, bytes) + .and_then(|()| fs::rename(&tmp, path)) + .inspect_err(|_| { + let _ = fs::remove_file(&tmp); + }) +} + +/// Waits for exclusive use of `target_dir`, which lasts until the returned file is dropped. +fn lock_target_dir(target_dir: &Path) -> Result { + fs::create_dir_all(target_dir)?; + let lock = fs::File::options() + .create(true) + .write(true) + .truncate(false) + .open(target_dir.join(".hyperlight-wasm-aot.lock"))?; + match lock.try_lock() { + Ok(()) => {} + Err(fs::TryLockError::WouldBlock) => { + eprintln!( + "Waiting for another runtime build in {}", + target_dir.display() + ); + lock.lock()?; + } + Err(fs::TryLockError::Error(e)) => return Err(e.into()), + } + Ok(lock) +} + +fn build_runtime( + wit: &Path, + output: &Path, + world: Option<&str>, + wasmtime_version: WasmtimeVersion, + target_dir: &Path, +) -> Result<(), BoxError> { + // The runtime resolves a relative WIT_WORLD against its own manifest. Keep + // symlinked names, since the guest macro picks the parser by extension. + let wit = + std::path::absolute(wit).map_err(|e| format!("could not read {}: {e}", wit.display()))?; + // Cargo runs from the runtime source, so pass it the same directory this + // process locks and copies from. + let target_dir = &std::path::absolute(target_dir)?; + // Every build in `target_dir` writes the same binary path, so hold the + // lock until this build's binary is copied. + let _lock = lock_target_dir(target_dir)?; + let manifest = runtime_manifest(target_dir)?; + let target = format!("{}-hyperlight-none", std::env::consts::ARCH); + + let mut cmd = cargo_hyperlight::cargo()?; + // Cargo reads `.cargo/config.toml` from its working directory upward, so + // run it from the runtime source, not the caller's project. + cmd.env_clear_cargo() + .current_dir( + manifest + .parent() + .ok_or("runtime manifest has no directory")?, + ) + .arg("build") + .arg("--release") + .arg("--target") + .arg(&target) + .arg("--target-dir") + .arg(target_dir) + .arg("--manifest-path") + .arg(&manifest) + .arg("--locked") + .env("WIT_WORLD", &wit); + match world { + Some(world) => cmd.env("WIT_WORLD_NAME", world), + None => cmd.env_remove("WIT_WORLD_NAME"), + }; + // Stable Rust 1.95 and later reject the guest's custom target. + if std::env::var_os("RUSTUP_TOOLCHAIN").is_none() { + cmd.env("RUSTUP_TOOLCHAIN", RUNTIME_TOOLCHAIN); + } + if wasmtime_version == WasmtimeVersion::Latest { + cmd.args(["--no-default-features", "--features", "wasmtime_latest"]); + } + // `{:#}` keeps the causes, such as why sysroot preparation failed. + cmd.status().map_err(|e| format!("{e:#}"))?; + + let built = target_dir + .join(&target) + .join("release") + .join("hyperlight-wasm-runtime"); + let copy_error = |e: std::io::Error| { + format!( + "could not copy {} to {}: {e}", + built.display(), + output.display() + ) + }; + let built_bytes = fs::read(&built).map_err(copy_error)?; + // An unchanged runtime keeps its timestamp, so later build steps can skip it. + // A changed one gets a new timestamp, even when Cargo reused an older binary. + if fs::read(output).ok().as_ref() != Some(&built_bytes) { + if let Some(parent) = output.parent().filter(|p| !p.as_os_str().is_empty()) { + fs::create_dir_all(parent)?; + } + write_atomic(output, &built_bytes).map_err(copy_error)?; + } + println!( + "Built guest runtime for {}: {}", + wit.display(), + output.display() + ); + Ok(()) } /// Precompile bytes using the LTS wasmtime version @@ -222,7 +410,7 @@ fn main() { } else { engine.precompile_module(&bytes).unwrap() }; - std::fs::write(outfile, serialized).unwrap(); + write_atomic(Path::new(&outfile), &serialized).unwrap(); } WasmtimeVersion::Lts => { let target_name = if pulley { @@ -251,7 +439,7 @@ fn main() { let bytes = std::fs::read(&input).unwrap(); let serialized = precompile_bytes_lts(&bytes, debug, minimal, pulley, component); - std::fs::write(outfile, serialized).unwrap(); + write_atomic(Path::new(&outfile), &serialized).unwrap(); } } } @@ -356,6 +544,24 @@ fn main() { } } } + Commands::ExperimentalBuildRuntime { + wit, + output, + world, + wasmtime_version, + target_dir, + } => { + if let Err(e) = build_runtime( + &wit, + &output, + world.as_deref(), + wasmtime_version, + &target_dir, + ) { + eprintln!("Error - {e}"); + std::process::exit(1); + } + } } } diff --git a/src/hyperlight_wasm_macro/src/lib.rs b/src/hyperlight_wasm_macro/src/lib.rs index 3b4c0a37..acde0664 100644 --- a/src/hyperlight_wasm_macro/src/lib.rs +++ b/src/hyperlight_wasm_macro/src/lib.rs @@ -27,14 +27,28 @@ mod wasmguest; /// wasmtime linker for component imports (which are implemented by /// calling to the Hyperlight host). /// +/// `WIT_WORLD` is a `.wit` file, a directory of `.wit` files or a +/// wasm-encoded WIT package. +/// /// If the WIT file contains multiple worlds, set the `WIT_WORLD_NAME` /// environment variable to select a specific world by name. If not set, /// the last world in the file will be used. #[proc_macro] pub fn wasm_guest_bindgen(_: proc_macro::TokenStream) -> proc_macro::TokenStream { - let path = std::env::var_os("WIT_WORLD").unwrap(); + let path = std::path::PathBuf::from(std::env::var_os("WIT_WORLD").unwrap()); let world_name = std::env::var("WIT_WORLD_NAME").ok(); - util::read_wit_type_from_file(path, world_name, |kebab_name, ct| { + let source = if path + .extension() + .is_some_and(|e| e.eq_ignore_ascii_case("wit")) + || std::path::Path::new(&std::env::var_os("CARGO_MANIFEST_DIR").unwrap()) + .join(&path) + .is_dir() + { + util::WitSource::Wit(path) + } else { + util::WitSource::Wasm(path) + }; + util::read_wit_type(source, world_name, |kebab_name, ct| { let decls = emit::run_state(true, true, |s| { // Emit type/trait definitions for all instances in the world rtypes::emit_toplevel(s, &kebab_name, ct); diff --git a/src/hyperlight_wasm_runtime/build.rs b/src/hyperlight_wasm_runtime/build.rs index 2ac2086f..bc5faff9 100644 --- a/src/hyperlight_wasm_runtime/build.rs +++ b/src/hyperlight_wasm_runtime/build.rs @@ -15,7 +15,7 @@ limitations under the License. */ use std::io::Write; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::{env, fs}; use cargo_metadata::{CargoOpt, MetadataCommand}; @@ -126,11 +126,16 @@ fn main() { } /// Identifies the `WIT_WORLD` world, so hosts can check that a snapshot -/// matches their bindings. Hashes the file and `WIT_WORLD_NAME`. +/// matches their bindings. Hashes the file, or the files a directory's WIT +/// package is read from, and `WIT_WORLD_NAME`. fn component_interface(path: &Path) -> String { - let bytes = fs::read(path).expect("could not read WIT_WORLD"); let mut hash = blake3::Hasher::new(); - hash.update(blake3::hash(&bytes).as_bytes()); + if path.is_dir() { + hash.update(b"\x02"); + hash_package_dir(&mut hash, path, true); + } else { + hash_file(&mut hash, path); + } // One file can hold several worlds. Unset selects the last one. if let Ok(world) = env::var("WIT_WORLD_NAME") { hash.update(b"\x01"); @@ -138,3 +143,54 @@ fn component_interface(path: &Path) -> String { } hash.finalize().to_string() } + +fn hash_file(hash: &mut blake3::Hasher, path: &Path) { + let bytes = fs::read(path).expect("could not read WIT_WORLD"); + hash.update(blake3::hash(&bytes).as_bytes()); +} + +fn hash_name(hash: &mut blake3::Hasher, name: &str) { + hash.update(&(name.len() as u64).to_le_bytes()); + hash.update(name.as_bytes()); +} + +fn sorted_entries(dir: &Path) -> Vec { + let mut entries = fs::read_dir(dir) + .and_then(|d| { + d.map(|e| e.map(|e| e.path())) + .collect::, _>>() + }) + .expect("could not read WIT_WORLD"); + entries.sort(); + entries +} + +/// Hashes the files wit-parser reads for a package directory: its `*.wit` +/// files and, at the top level, the packages in `deps`. +fn hash_package_dir(hash: &mut blake3::Hasher, dir: &Path, top_level: bool) { + for entry in sorted_entries(dir) { + let name = entry.file_name().unwrap().to_string_lossy(); + if !entry.is_dir() && name.ends_with(".wit") { + hash_name(hash, &name); + hash_file(hash, &entry); + } + } + let deps = dir.join("deps"); + if !top_level || !deps.is_dir() { + return; + } + hash.update(b"\x03"); + for entry in sorted_entries(&deps) { + let name = entry.file_name().unwrap().to_string_lossy(); + if entry.is_dir() { + hash_name(hash, &format!("{name}/")); + hash_package_dir(hash, &entry, false); + } else if matches!( + entry.extension().and_then(|e| e.to_str()), + Some("wit" | "wat" | "wasm") + ) { + hash_name(hash, &name); + hash_file(hash, &entry); + } + } +}