Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).
* Fix restore on WHP not rewinding the guest TSC on hosts with `TSC_ADJUST` support.
* HVF sandbox initialization accepts seeds that match Apple hypercall identifiers.
* Host-written snapshot memory has instruction-cache coherence before guest execution.
* GDB memory reads and writes translate each guest page on hosts with larger pages.
* ARM64 GDB snapshot recovery handles cancellation and crash detach. Snapshot checkpointing rejects software breakpoints before replacing memory.

### Removed
* **Breaking:** `SandboxBuilder::init_data`, `GuestEnvironment`, `GuestBlob`,
Expand Down
2 changes: 1 addition & 1 deletion Justfile
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ test-no-surrogate target=default-target:
# runs tests that exercise gdb debugging
test-rust-gdb-debugging target=default-target features="":
{{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} --example guest-debugging {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }}
{{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- hypervisor::gdb::
{{ cargo-cmd }} test --profile={{ if target == "debug" { "dev" } else { target } }} {{ target-triple-flag }} -p hyperlight-host --lib {{ if features =="" {'--features gdb'} else { "--features gdb," + features } }} -- hypervisor::gdb:: hypervisor::hyperlight_vm::debug::tests:: gdb_recovery_tests::

# rust test for crashdump
test-rust-crashdump target=default-target features="":
Expand Down
3 changes: 3 additions & 0 deletions src/hyperlight_common/src/arch/aarch64/exn.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@

use crate::vmem::bits;

/// HVC immediate for forwarding guest exceptions to the Hyper-V debugger.
pub const DEBUG_EXCEPTION_HVC: u16 = 0x485a;

const ESR_EC_UNKNOWN: u64 = 0b000000; // udf generates this
const ESR_EC_DATA_ABORT_LOWER_EL: u64 = 0b100100;
const ESR_EC_DATA_ABORT_SAME_EL: u64 = 0b100101;
Expand Down
17 changes: 17 additions & 0 deletions src/hyperlight_guest_bin/src/arch/aarch64/exception/entry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,24 @@ context_restore:\n
q_off = const offset_of!(ExceptionContext, q),
);

// Only synchronous exceptions have a syndrome for debug forwarding.
// The host resumes debug events at ERET and other synchronous exceptions at the prologue.
macro_rules! vbar_entry {
($et:literal, $ef:literal) => {
concat!(
"
.if {ExceptionType_",
$et,
"} == {ExceptionType_Synchronous}\n
stp x0, x1, [sp, #-16]!\n
mrs x0, mdscr_el1\n
tbz x0, #13, 1f\n
ldp x0, x1, [sp], #16\n
hvc #{debug_hvc}\n
eret\n
1:\n
ldp x0, x1, [sp], #16\n
.endif\n
sub sp, sp, #{ctx_size}\n
str x30, [sp, #{x30_off}]\n
bl context_save\n
Expand All @@ -129,6 +143,8 @@ macro_rules! vbar_entry {
global_asm!("
.balign 0x800\n
.global vbar\n
.global hyperlight_debug_vectors\n
hyperlight_debug_vectors:\n
vbar:\n",
vbar_entry!("Synchronous", "CurrentSP0"),
vbar_entry!("IRQ", "CurrentSP0"),
Expand All @@ -148,6 +164,7 @@ vbar:\n",
vbar_entry!("SError", "LowerAArch32"),
ctx_size = const size_of::<ExceptionContext>(),
x30_off = const offset_of!(ExceptionContext, x) + 15 * 0x010,
debug_hvc = const hyperlight_common::arch::exn::DEBUG_EXCEPTION_HVC,
handler = sym super::handle::handle_exception,
ExceptionType_Synchronous = const ExceptionType::Synchronous as u64,
ExceptionType_IRQ = const ExceptionType::Irq as u64,
Expand Down
203 changes: 202 additions & 1 deletion src/hyperlight_host/examples/guest-debugging/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,12 @@ fn main() -> hyperlight_host::Result<()> {
.host_function("Sleep5Secs", sleep_5_secs)
.build()?;

// Call guest function
#[cfg(target_arch = "x86_64")]
multi_use_sandbox_dbg
.call::<()>("UseSSE2Registers", ())
.unwrap();
#[cfg(target_arch = "aarch64")]
multi_use_sandbox_dbg.call::<()>("NoOp", ()).unwrap();

let message =
"Hello, World! I am executing inside of a VM with debugger attached :)\n".to_string();
Expand Down Expand Up @@ -291,6 +293,205 @@ mod tests {
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_bootstrap_step_preserves_live_stack() {
let (out_file_path, cmd_file_path, manifest_dir) =
gdb_test_paths("gdb-aarch64-bootstrap-stack");
let cmd = format!(
"set osabi none
file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
set language c
set $entry = $pc
set $instruction = *(unsigned int *)$entry
set *(unsigned int *)$entry = 0xd503201f
set $bootstrap_sp = $sp
set $canary = $sp - 16
set $saved = *(unsigned long long *)$canary
set *(unsigned long long *)$canary = 0x123456789abcdef0
set $sp = $sp - 32
stepi
if *(unsigned long long *)$canary != 0x123456789abcdef0
echo Bootstrap debug step overwrote the live stack\\n
quit 1
end
set *(unsigned long long *)$canary = $saved
set $sp = $bootstrap_sp
set *(unsigned int *)$entry = $instruction
set $pc = $entry
echo Bootstrap debug step preserved the live stack\\n
set logging enabled off
detach
quit
"
);
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, |contents| {
contents.contains("Bootstrap debug step preserved the live stack")
});
cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_continue_over_sw_breakpoint() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-continue");
let cmd = format!(
"file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
break simpleguest::no_op
commands 1
continue
end
break simpleguest::print_output
commands 2
echo Continued over ARM64 software breakpoint\\n
set logging enabled off
detach
quit
end
continue
"
);
#[cfg(windows)]
let cmd = format!("set osabi none\n{cmd}");
let checker =
|contents: String| contents.contains("Continued over ARM64 software breakpoint");
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);

cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_hardware_breakpoint_and_step() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-hardware");
let cmd = format!(
"set osabi none
file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
set $entry_pc = $pc
stepi
if $pc != $entry_pc + 4
echo Entry step did not execute one instruction\\n
quit 1
end
hbreak simpleguest::no_op
continue
set $return_pc = $x30
stepi
if $pc != $return_pc
echo Hardware RET step missed its destination\\n
quit 1
end
echo Hardware breakpoint and entry/RET stepping passed\\n
set logging enabled off
detach
quit
"
);
let checker = |contents: String| {
contents.contains("Hardware breakpoint and entry/RET stepping passed")
};
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_code_writes_and_self_loop() {
let (out_file_path, cmd_file_path, manifest_dir) =
gdb_test_paths("gdb-aarch64-code-writes");
let cmd = format!(
"set osabi none
file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
hbreak simpleguest::no_op
continue
delete 1
set language c
set $loop_pc = $pc
set $instruction = *(unsigned int *)$pc
set *(unsigned int *)$pc = 0x14000000
stepi
if $pc != $loop_pc
echo Self-loop step missed its destination\\n
quit 1
end
set *(unsigned int *)$pc = $instruction
set $return_pc = $x30
stepi
if $pc != $return_pc
echo Restored RET missed its destination\\n
quit 1
end
echo Code writes and self-loop stepping passed\\n
set logging enabled off
detach
quit
"
);
let checker =
|contents: String| contents.contains("Code writes and self-loop stepping passed");
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);
cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "aarch64")]
#[serial]
fn test_gdb_single_step() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-aarch64-step");
let cmd = format!(
"file {manifest_dir}/../tests/rust_guests/bin/debug/simpleguest
target remote :8080
set pagination off
set logging file {out_file_path}
set logging enabled on
break simpleguest::no_op
stepi
echo Stepped over ARM64 linear instruction\\n
continue
stepi
echo Stepped over ARM64 RET\\n
set logging enabled off
detach
quit
"
);
#[cfg(windows)]
let cmd = format!("set osabi none\n{cmd}");
let checker = |contents: String| {
contents.contains("Stepped over ARM64 linear instruction")
&& contents.contains("Stepped over ARM64 RET")
};
let result = run_guest_and_gdb(&cmd_file_path, &out_file_path, &cmd, checker);

cleanup(&out_file_path, &cmd_file_path);
assert!(result.is_ok(), "{}", result.unwrap_err());
}

#[test]
#[cfg(target_arch = "x86_64")]
#[serial]
fn test_gdb_sse_check() {
let (out_file_path, cmd_file_path, manifest_dir) = gdb_test_paths("gdb-sse");
Expand Down
97 changes: 9 additions & 88 deletions src/hyperlight_host/src/hypervisor/gdb/arch.rs
Original file line number Diff line number Diff line change
@@ -1,91 +1,12 @@
// SPDX-License-Identifier: Apache-2.0
// Copyright 2025 The Hyperlight Authors.

//! This file contains architecture specific code for the x86_64

use super::{DebugError, DebuggableVm, VcpuStopReason};
use crate::hypervisor::regs::CommonRegisters;
use crate::hypervisor::virtual_machine::RegisterError;

/// Errors that can occur when determining the vCPU stop reason
#[derive(Debug, thiserror::Error)]
pub enum VcpuStopReasonError {
#[error("Failed to get registers: {0}")]
GetRegs(#[from] RegisterError),
#[error("Failed to remove hardware breakpoint: {0}")]
RemoveHwBreakpoint(#[from] DebugError),
}

// Described in Table 6-1. Exceptions and Interrupts at Page 6-13 Vol. 1
// of Intel 64 and IA-32 Architectures Software Developer's Manual
/// Exception id for #DB
pub(crate) const DB_EX_ID: u32 = 1;
/// Exception id for #BP - triggered by the INT3 instruction
pub(crate) const BP_EX_ID: u32 = 3;

/// Software Breakpoint size in memory
pub(crate) const SW_BP_SIZE: usize = 1;
/// Software Breakpoint opcode - INT3
/// Check page 7-28 Vol. 3A of Intel 64 and IA-32
/// Architectures Software Developer's Manual
pub(crate) const SW_BP_OP: u8 = 0xCC;
/// Software Breakpoint written to memory
pub(crate) const SW_BP: [u8; SW_BP_SIZE] = [SW_BP_OP];
/// Maximum number of supported hardware breakpoints
pub(crate) const MAX_NO_OF_HW_BP: usize = 4;

/// Check page 19-4 Vol. 3B of Intel 64 and IA-32
/// Architectures Software Developer's Manual
/// Bit position of BS flag in DR6 debug register
pub(crate) const DR6_BS_FLAG_POS: usize = 14;
/// Bit mask of BS flag in DR6 debug register
pub(crate) const DR6_BS_FLAG_MASK: u64 = 1 << DR6_BS_FLAG_POS;
/// Bit position of HW breakpoints status in DR6 debug register
pub(crate) const DR6_HW_BP_FLAGS_POS: usize = 0;
/// Bit mask of HW breakpoints status in DR6 debug register
pub(crate) const DR6_HW_BP_FLAGS_MASK: u64 = 0x0F << DR6_HW_BP_FLAGS_POS;

/// Determine the reason the vCPU stopped
/// This is done by checking the DR6 register and the exception id
pub(crate) fn vcpu_stop_reason(
vm: &dyn DebuggableVm,
dr6: u64,
exception: u32,
) -> std::result::Result<VcpuStopReason, VcpuStopReasonError> {
let CommonRegisters { rip, .. } = vm.regs()?;
if DB_EX_ID == exception {
// If the BS flag in DR6 register is set, it means a single step
// instruction triggered the exit
// Check page 19-4 Vol. 3B of Intel 64 and IA-32
// Architectures Software Developer's Manual
if dr6 & DR6_BS_FLAG_MASK != 0 {
return Ok(VcpuStopReason::DoneStep);
}

// If any of the B0-B3 flags in DR6 register is set, it means a
// hardware breakpoint triggered the exit
// Check page 19-4 Vol. 3B of Intel 64 and IA-32
// Architectures Software Developer's Manual
if DR6_HW_BP_FLAGS_MASK & dr6 != 0 {
return Ok(VcpuStopReason::HwBp);
}
}

if BP_EX_ID == exception {
return Ok(VcpuStopReason::SwBp);
}

// Log an error and provide internal debugging info
tracing::error!(
r"The vCPU exited because of an unknown reason:
rip: {:?}
dr6: {:?}
exception: {:?}
",
rip,
dr6,
exception,
);

Ok(VcpuStopReason::Unknown)
}
#[cfg(target_arch = "aarch64")]
mod aarch64;
#[cfg(target_arch = "x86_64")]
mod x86_64;

#[cfg(target_arch = "aarch64")]
pub(crate) use aarch64::*;
#[cfg(target_arch = "x86_64")]
pub(crate) use x86_64::*;
Loading
Loading