diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 23e26fa76..f0b582342 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -123,8 +123,20 @@ jobs: mkw_macos_context_abi_tests \ mkw_macos_host_context_tests \ mkw_macos_guest_flat_memory_tests \ - mkw_macos_external_audio_tests + mkw_macos_external_audio_tests \ + mkw_ax_mix_kernels_tests - name: Test execution substrate shell: bash run: ctest --test-dir build-macos --output-on-failure + + - name: Check ARM audio kernels with sanitizers + shell: bash + run: | + clang++ -std=c++17 -O2 -fno-fast-math \ + -fsanitize=address,undefined -fno-sanitize-recover=all \ + -DMKW_AX_MIX_REQUIRE_NEON=1 \ + -Iruntime/include -Iruntime/src/hle/audio \ + runtime/tests/ax_mix_kernels_tests.cpp -o build-macos/ax-mix-sanitized + build-macos/ax-mix-sanitized + build-macos/mkw_ax_mix_kernels_tests --benchmark 20000 diff --git a/.github/workflows/recomp-test.yml b/.github/workflows/recomp-test.yml index 6ae6eb8ee..3c3daf3c6 100644 --- a/.github/workflows/recomp-test.yml +++ b/.github/workflows/recomp-test.yml @@ -69,6 +69,18 @@ jobs: shell: pwsh run: ./Launcher/Test-Recompilation.ps1 -Parallel 4 + - name: Test GX performance paths + shell: pwsh + run: | + $tools = Join-Path $PWD 'Launcher/artifacts/portable-tools' + $build = Join-Path $PWD 'build/recomp-test/native-build' + $env:PATH = "$build;$tools/llvm-mingw/bin;$tools/CMake/bin;$tools/Ninja;$env:PATH" + cmake -S build/recomp-test/runtime -B $build -DMKW_BUILD_AURORA_TESTS=ON -DFETCHCONTENT_FULLY_DISCONNECTED=OFF + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + cmake --build $build --parallel 3 --target gx_fifo_tests shader_module_cache_tests mkw_gx_texture_binding_tests mkw_gx_sync_tests mkw_gx_dl_cache_tests + if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } + ctest --test-dir $build --output-on-failure --no-tests=error -R '^(GXFifoTest|RuntimeDirectDrawTest|ShaderModuleCacheTest|mkw_gx_(texture_binding|sync|dl_cache)_tests)' + # Print cache results (even if the build fails) - name: Show sccache stats if: always() diff --git a/Launcher/macos/macos-x86_64-toolchain.cmake b/Launcher/macos/macos-x86_64-toolchain.cmake index f36749827..7ce5f7737 100644 --- a/Launcher/macos/macos-x86_64-toolchain.cmake +++ b/Launcher/macos/macos-x86_64-toolchain.cmake @@ -9,3 +9,12 @@ set(CMAKE_SYSTEM_NAME Darwin) set(CMAKE_SYSTEM_PROCESSOR x86_64) set(CMAKE_OSX_ARCHITECTURES x86_64 CACHE STRING "Target macOS architectures" FORCE) + +if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin" AND NOT DEFINED CMAKE_CROSSCOMPILING_EMULATOR + AND NOT DEFINED ENV{CMAKE_CROSSCOMPILING_EMULATOR}) + execute_process(COMMAND /usr/bin/arch -x86_64 /usr/bin/true + RESULT_VARIABLE _mkw_can_run_x86_64 OUTPUT_QUIET ERROR_QUIET) + if(_mkw_can_run_x86_64 EQUAL 0) + set(CMAKE_CROSSCOMPILING_EMULATOR /usr/bin/arch -x86_64) + endif() +endif() diff --git a/aurora-main b/aurora-main index 1a67fa419..5ba138858 160000 --- a/aurora-main +++ b/aurora-main @@ -1 +1 @@ -Subproject commit 1a67fa419c7ced727a8f4523471d7df18ed610d1 +Subproject commit 5ba1388584fa9dc78f16208c7a9ae10802e14b1c diff --git a/runtime/cmake/PlatformTests.cmake b/runtime/cmake/PlatformTests.cmake index 6f0dc65eb..52fac38a4 100644 --- a/runtime/cmake/PlatformTests.cmake +++ b/runtime/cmake/PlatformTests.cmake @@ -155,6 +155,7 @@ endif() if(TARGET gx_fifo_tests) target_sources(gx_fifo_tests PRIVATE "${MKW_TEST_RUNTIME_DIR}/tests/gx_fifo_invalidation_tests.cpp" + "${MKW_TEST_RUNTIME_DIR}/tests/gx_direct_draw_tests.cpp" "${MKW_TEST_RUNTIME_DIR}/src/hle/gx/gx_fifo.cpp") target_include_directories(gx_fifo_tests PRIVATE "${MKW_AURORA_DIR}/tests" "${MKW_TEST_RUNTIME_DIR}/include" @@ -164,6 +165,70 @@ if(TARGET gx_fifo_tests) if(MKW_PLATFORM_WINDOWS) target_link_libraries(gx_fifo_tests PRIVATE advapi32) endif() + + get_target_property(_mkw_gx_sources gx_fifo_tests SOURCES) + get_target_property(_mkw_gx_source_dir gx_fifo_tests SOURCE_DIR) + get_target_property(_mkw_gx_includes gx_fifo_tests INCLUDE_DIRECTORIES) + get_target_property(_mkw_gx_links gx_fifo_tests LINK_LIBRARIES) + set(_mkw_gx_encoder_sources) + foreach(_mkw_gx_source IN LISTS _mkw_gx_sources) + if(_mkw_gx_source MATCHES "(^|/)lib/") + get_filename_component(_mkw_gx_absolute "${_mkw_gx_source}" ABSOLUTE + BASE_DIR "${_mkw_gx_source_dir}") + list(APPEND _mkw_gx_encoder_sources "${_mkw_gx_absolute}") + endif() + endforeach() + + add_executable(mkw_gx_texture_binding_tests + "${MKW_TEST_RUNTIME_DIR}/tests/gx_texture_binding_tests.cpp" + "${MKW_AURORA_DIR}/tests/gx_test_stubs.cpp" + ${_mkw_gx_encoder_sources}) + add_executable(mkw_gx_sync_tests + "${MKW_TEST_RUNTIME_DIR}/tests/gx_worker_tests.cpp" + "${MKW_TEST_RUNTIME_DIR}/tests/gx_copy_tests.cpp" + "${MKW_AURORA_DIR}/tests/gx_test_stubs.cpp" + ${_mkw_gx_encoder_sources}) + target_compile_definitions(mkw_gx_sync_tests PRIVATE MKW_GX_COPY_TESTING=1) + foreach(_mkw_gx_test IN ITEMS mkw_gx_texture_binding_tests mkw_gx_sync_tests) + target_include_directories(${_mkw_gx_test} PRIVATE + ${_mkw_gx_includes} "${MKW_TEST_RUNTIME_DIR}/tests") + target_compile_definitions(${_mkw_gx_test} PRIVATE AURORA TARGET_PC=1) + target_compile_features(${_mkw_gx_test} PRIVATE cxx_std_20) + target_link_libraries(${_mkw_gx_test} PRIVATE ${_mkw_gx_links} Threads::Threads) + set_target_properties(${_mkw_gx_test} PROPERTIES UNITY_BUILD OFF) + aurora_copy_runtime_dlls(${_mkw_gx_test}) + add_test(NAME ${_mkw_gx_test} COMMAND ${_mkw_gx_test}) + set_tests_properties(${_mkw_gx_test} PROPERTIES TIMEOUT 120) + endforeach() + + add_executable(mkw_gx_dl_cache_tests "${MKW_TEST_RUNTIME_DIR}/tests/gx_dl_cache_tests.cpp") + target_include_directories(mkw_gx_dl_cache_tests PRIVATE + "${MKW_TEST_RUNTIME_DIR}/include" + "${MKW_TEST_RUNTIME_DIR}/third_party/toml11" + "${MKW_AURORA_DIR}/include") + target_compile_features(mkw_gx_dl_cache_tests PRIVATE cxx_std_20) + target_compile_definitions(mkw_gx_dl_cache_tests PRIVATE AURORA TARGET_PC=1 NOMINMAX) + target_link_libraries(mkw_gx_dl_cache_tests PRIVATE xxHash::xxhash ${AURORA_SDL3_TARGET}) + set_target_properties(mkw_gx_dl_cache_tests PROPERTIES UNITY_BUILD OFF) + add_test(NAME mkw_gx_dl_cache_tests COMMAND mkw_gx_dl_cache_tests) +endif() + +function(mkw_add_ax_mix_test target) + add_executable(${target} "${MKW_TEST_RUNTIME_DIR}/tests/ax_mix_kernels_tests.cpp") + target_include_directories(${target} PRIVATE + "${MKW_TEST_RUNTIME_DIR}/include" "${MKW_TEST_RUNTIME_DIR}/src/hle/audio") + target_compile_features(${target} PRIVATE cxx_std_17) + set_target_properties(${target} PROPERTIES UNITY_BUILD OFF) + if(NOT CMAKE_CROSSCOMPILING OR CMAKE_CROSSCOMPILING_EMULATOR) + add_test(NAME ${target} COMMAND ${target}) + endif() +endfunction() + +mkw_add_ax_mix_test(mkw_ax_mix_kernels_tests) +if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + target_compile_definitions(mkw_ax_mix_kernels_tests PRIVATE MKW_AX_MIX_REQUIRE_NEON=1) +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") + target_compile_options(mkw_ax_mix_kernels_tests PRIVATE -march=x86-64-v2) endif() add_executable(mkw_fiber_manager_tests @@ -297,6 +362,11 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(AMD64|amd64|x86_64|X86_64)$") ]=] MKW_HOST_SUPPORTS_X86_V3) cmake_pop_check_state() endif() + if((NOT CMAKE_CROSSCOMPILING OR CMAKE_CROSSCOMPILING_EMULATOR) AND MKW_HOST_SUPPORTS_X86_V3) + mkw_add_ax_mix_test(mkw_ax_mix_kernels_v3_tests) + target_compile_options(mkw_ax_mix_kernels_v3_tests PRIVATE -march=x86-64-v3) + endif() + # Compile the same oracle cases for both paths. The v2 binary proves the # scalar fallback stays free of FMA; the v3 binary checks the existing # vector intrinsic path against the same strict result. diff --git a/runtime/src/hle/audio/ax_mix_kernels.h b/runtime/src/hle/audio/ax_mix_kernels.h index 687d6bab4..d51a68972 100644 --- a/runtime/src/hle/audio/ax_mix_kernels.h +++ b/runtime/src/hle/audio/ax_mix_kernels.h @@ -3,7 +3,7 @@ #include "isa/big_endian.h" // AX/DSP fixed-point mix kernels: the inner loops of the AXWii command-list mix in ax_mix.cpp. -// Each exists as a scalar reference (a literal transcription of the loop it replaced) and an AVX2 form. +// Each has a scalar reference and architecture-specific SIMD forms. #include #include @@ -16,8 +16,23 @@ #define MKW_AX_MIX_AVX2 0 #endif +#if !MKW_AX_MIX_AVX2 && defined(__aarch64__) && defined(__ARM_NEON) && !defined(__AARCH64EB__) +#include +#define MKW_AX_MIX_NEON 1 +#else +#define MKW_AX_MIX_NEON 0 +#endif + namespace AxMixKernels { +// Mix buses retain the low 32 bits, matching the SIMD adds. +inline int32_t AddWrap32(int32_t value, uint32_t addend) { + const uint32_t bits = static_cast(value) + addend; + int32_t result; + std::memcpy(&result, &bits, sizeof(result)); + return result; +} + inline int16_t ClampToS16(int32_t value) { if (value < -0x8000) { return static_cast(-0x8000); @@ -39,7 +54,7 @@ inline uint16_t MixAddRampScalar(int32_t* out, const int16_t* input, uint32_t co const int32_t scaled = (static_cast(input[i]) * static_cast(volume)) >> 15; const int16_t sample = ClampToS16(scaled); - out[i] += sample; + out[i] = AddWrap32(out[i], static_cast(sample)); volume = static_cast(volume + delta); last = sample; } @@ -90,7 +105,7 @@ inline uint16_t MixAddRampAvx2(int32_t* out, const int16_t* input, uint32_t coun const int32_t scaled = (static_cast(input[i]) * static_cast(volume)) >> 15; const int16_t sample = ClampToS16(scaled); - out[i] += sample; + out[i] = AddWrap32(out[i], static_cast(sample)); volume = static_cast(volume + delta); last = sample; } @@ -99,10 +114,46 @@ inline uint16_t MixAddRampAvx2(int32_t* out, const int16_t* input, uint32_t coun } #endif +#if MKW_AX_MIX_NEON +inline uint16x4_t Ramp4(uint16_t volume, uint16_t delta) { + static constexpr uint16_t lanes[] = {0, 1, 2, 3}; + return vmla_n_u16(vdup_n_u16(volume), vld1_u16(lanes), delta); +} + +inline int16x4_t ScaleBlock4(int16x4_t samples, uint16x4_t ramp) { + const int32x4_t gain = vreinterpretq_s32_u32(vmovl_u16(ramp)); + const int32x4_t product = vmulq_s32(vmovl_s16(samples), gain); + return vqmovn_s32(vshrq_n_s32(product, 15)); +} + +inline uint16_t MixAddRampNeon(int32_t* out, const int16_t* input, uint32_t count, + uint16_t volume, uint16_t delta, int16_t& dpop) { + uint32_t i = 0; + int16_t last = dpop; + if (count >= 4) { + uint16x4_t ramp = Ramp4(volume, delta); + const uint16x4_t step = vdup_n_u16(static_cast(delta * 4u)); + int16x4_t block = vdup_n_s16(0); + for (; count - i >= 4; i += 4) { + block = ScaleBlock4(vld1_s16(input + i), ramp); + vst1q_s32(out + i, vaddw_s16(vld1q_s32(out + i), block)); + ramp = vadd_u16(ramp, step); + } + last = vget_lane_s16(block, 3); + volume = vget_lane_u16(ramp, 0); + } + volume = MixAddRampScalar(out + i, input + i, count - i, volume, delta, last); + dpop = last; + return volume; +} +#endif + inline uint16_t MixAddRamp(int32_t* out, const int16_t* input, uint32_t count, uint16_t volume, uint16_t delta, int16_t& dpop) { #if MKW_AX_MIX_AVX2 return MixAddRampAvx2(out, input, count, volume, delta, dpop); +#elif MKW_AX_MIX_NEON + return MixAddRampNeon(out, input, count, volume, delta, dpop); #else return MixAddRampScalar(out, input, count, volume, delta, dpop); #endif @@ -162,10 +213,29 @@ inline uint16_t ScaleRampAvx2(int16_t* samples, uint32_t count, uint16_t volume, } #endif +#if MKW_AX_MIX_NEON +inline uint16_t ScaleRampNeon(int16_t* samples, uint32_t count, uint16_t volume, + uint16_t delta) { + uint32_t i = 0; + if (count >= 4) { + uint16x4_t ramp = Ramp4(volume, delta); + const uint16x4_t step = vdup_n_u16(static_cast(delta * 4u)); + for (; count - i >= 4; i += 4) { + vst1_s16(samples + i, ScaleBlock4(vld1_s16(samples + i), ramp)); + ramp = vadd_u16(ramp, step); + } + volume = vget_lane_u16(ramp, 0); + } + return ScaleRampScalar(samples + i, count - i, volume, delta); +} +#endif + inline uint16_t ScaleRamp(int16_t* samples, uint32_t count, uint16_t volume, uint16_t delta) { #if MKW_AX_MIX_AVX2 return ScaleRampAvx2(samples, count, volume, delta); +#elif MKW_AX_MIX_NEON + return ScaleRampNeon(samples, count, volume, delta); #else return ScaleRampScalar(samples, count, volume, delta); #endif @@ -179,8 +249,9 @@ inline uint16_t ScaleRamp(int16_t* samples, uint32_t count, uint16_t volume, inline void MixAccumRamp32Scalar(int32_t* dst, const int32_t* src, const uint16_t* ramp, uint32_t count) { for (uint32_t i = 0; i < count; ++i) { - dst[i] += static_cast( - (static_cast(src[i]) * static_cast(ramp[i])) >> 15); + const int64_t product = static_cast(src[i]) * ramp[i]; + const uint32_t scaled = static_cast(static_cast(product) >> 15); + dst[i] = AddWrap32(dst[i], scaled); } } @@ -208,16 +279,37 @@ inline void MixAccumRamp32Avx2(int32_t* dst, const int32_t* src, const uint16_t* _mm256_add_epi32(acc, result)); } for (; i < count; ++i) { - dst[i] += static_cast( - (static_cast(src[i]) * static_cast(ramp[i])) >> 15); + const int64_t product = static_cast(src[i]) * ramp[i]; + const uint32_t scaled = static_cast(static_cast(product) >> 15); + dst[i] = AddWrap32(dst[i], scaled); } } #endif +#if MKW_AX_MIX_NEON +inline void MixAccumRamp32Neon(int32_t* dst, const int32_t* src, const uint16_t* ramp, + uint32_t count) { + uint32_t i = 0; + for (; count - i >= 4; i += 4) { + const int32x4_t source = vld1q_s32(src + i); + const int32x4_t gain = vreinterpretq_s32_u32(vmovl_u16(vld1_u16(ramp + i))); + const int64x2_t lowProduct = vmull_s32(vget_low_s32(source), vget_low_s32(gain)); + const int64x2_t highProduct = vmull_s32(vget_high_s32(source), vget_high_s32(gain)); + // Keep bits 15..46, including products whose shifted value exceeds int32. + const int32x4_t scaled = vcombine_s32(vshrn_n_s64(lowProduct, 15), + vshrn_n_s64(highProduct, 15)); + vst1q_s32(dst + i, vaddq_s32(vld1q_s32(dst + i), scaled)); + } + MixAccumRamp32Scalar(dst + i, src + i, ramp + i, count - i); +} +#endif + inline void MixAccumRamp32(int32_t* dst, const int32_t* src, const uint16_t* ramp, uint32_t count) { #if MKW_AX_MIX_AVX2 MixAccumRamp32Avx2(dst, src, ramp, count); +#elif MKW_AX_MIX_NEON + MixAccumRamp32Neon(dst, src, ramp, count); #else MixAccumRamp32Scalar(dst, src, ramp, count); #endif @@ -270,9 +362,31 @@ inline void LoadBigEndian32Avx2(int32_t* dst, const uint8_t* src, size_t count) } #endif +#if MKW_AX_MIX_NEON +inline void StoreBigEndian32Neon(uint8_t* dst, const int32_t* src, size_t count) { + size_t i = 0; + for (; count - i >= 4; i += 4) { + const uint8x16_t bytes = vreinterpretq_u8_s32(vld1q_s32(src + i)); + vst1q_u8(dst + i * sizeof(uint32_t), vrev32q_u8(bytes)); + } + StoreBigEndian32Scalar(dst + i * sizeof(uint32_t), src + i, count - i); +} + +inline void LoadBigEndian32Neon(int32_t* dst, const uint8_t* src, size_t count) { + size_t i = 0; + for (; count - i >= 4; i += 4) { + const uint8x16_t bytes = vld1q_u8(src + i * sizeof(uint32_t)); + vst1q_s32(dst + i, vreinterpretq_s32_u8(vrev32q_u8(bytes))); + } + LoadBigEndian32Scalar(dst + i, src + i * sizeof(uint32_t), count - i); +} +#endif + inline void StoreBigEndian32(uint8_t* dst, const int32_t* src, size_t count) { #if MKW_AX_MIX_AVX2 StoreBigEndian32Avx2(dst, src, count); +#elif MKW_AX_MIX_NEON + StoreBigEndian32Neon(dst, src, count); #else StoreBigEndian32Scalar(dst, src, count); #endif @@ -281,6 +395,8 @@ inline void StoreBigEndian32(uint8_t* dst, const int32_t* src, size_t count) { inline void LoadBigEndian32(int32_t* dst, const uint8_t* src, size_t count) { #if MKW_AX_MIX_AVX2 LoadBigEndian32Avx2(dst, src, count); +#elif MKW_AX_MIX_NEON + LoadBigEndian32Neon(dst, src, count); #else LoadBigEndian32Scalar(dst, src, count); #endif diff --git a/runtime/src/hle/gx/gx_copy.cpp b/runtime/src/hle/gx/gx_copy.cpp index 2f13b3b50..56640605f 100644 --- a/runtime/src/hle/gx/gx_copy.cpp +++ b/runtime/src/hle/gx/gx_copy.cpp @@ -165,19 +165,13 @@ PPC_NATIVE_OVERRIDE_VOID(8016fc24, GX__SetDispCopyGamma_8016fc24, (uint32_t g), extern "C" void GX__CopyDisp_8016fc38(uint32_t da, uint32_t c) { EnsureAuroraFrameActive(); if (GxWorker::ShouldDefer()) { - // The worker drains like the inline path; EndAuroraFrame joins it before the next frame. + // The copy is ordered with draws in Aurora's FIFO; frame end drains it. GxWorker::Submit([da, c] { - GXDrawDone(); GXCopyDisp(GuestToHostPtr(da), (GXBool)c); }); } else { - // GX copies are FIFO-ordered on hardware. Drain submitted draws before - // resolving the EFB so high-level copies see the same contents. - GXDrawDone(); GXCopyDisp(GuestToHostPtr(da), (GXBool)c); - // Present immediately so post-copy draws don't leak into this frame. Join at the DONE phase - // (not the cheaper SEALED phase GXDrawDone waits for) because ImGui's draw lists, owned by - // Aurora's render worker, replay during encode; aurora_end_frame would join here anyway. + // ImGui draw lists remain owned by the render worker until encoding completes. aurora_wait_for_frame_worker(); } ++g_gxFrameCount; @@ -195,8 +189,8 @@ extern "C" void GX__CopyTex_8016fd74(uint32_t da, uint32_t c) { // Begin the frame before going inline so a deferred begin reaches the worker first. EnsureAuroraFrameActive(); GX_SYNC_INLINE(); - // Match GX FIFO ordering: texture copies observe all prior draws. - GXDrawDone(); + // Flush dirty state without inserting a completion token before the ordered copy. + GXFlush(); const uint16_t rawSrcLeft = g_texCopyState.srcLeft; const uint16_t rawSrcTop = g_texCopyState.srcTop; const uint16_t rawSrcWidth = g_texCopyState.srcWidth; diff --git a/runtime/src/hle/gx/gx_dl.cpp b/runtime/src/hle/gx/gx_dl.cpp index 71e18fbcd..dc747106c 100644 --- a/runtime/src/hle/gx/gx_dl.cpp +++ b/runtime/src/hle/gx/gx_dl.cpp @@ -1,14 +1,11 @@ #include "gx_internal.h" #include "gx_stream_common.h" #include "gx_cp_decode.h" +#include "gx_dl_cache.h" #include "isa/big_endian.h" #include -#include - -// The display-list scan cache validates a cached scan against the live guest -// bytes with a 64-bit XXH3 digest (see GxDisplayListScanCache::CanReuse). -#include +#include namespace aurora::gx::fifo { bool in_display_list(); @@ -25,18 +22,9 @@ using namespace GxStream; using GxCpDecode::ApplyCpRegWrite; using GxCpDecode::SameVtxAttrFmt; -// Small display lists dominate the in-race call count. Cache them as well, but -// cap both individual entries and aggregate copied command bytes so malformed -// guest input cannot turn this optimization into unbounded host allocation. -constexpr uint32_t kDlScanCacheMaxEntryBytes = 64u * 1024u; constexpr size_t kDlScanCacheMaxEntries = 8192; constexpr size_t kDlScanCacheMaxStoredBytes = 8u * 1024u * 1024u; -// Display-list write tracking (audit F6a): re-digesting every list every call is the -// costliest step of GX__CallDisplayList, and wasted on BRRES shape lists that are written -// once at load and never touched again. -constexpr uint64_t kDlWriteGenerationUntracked = GxGuestWrite::kUntracked; - // Byte footprint of one vertex of the currently configured layout. Indexed attributes are // sized before the matrix-attribute check here (unlike BuildScanVertexLayout), but a // GX_INDEX16 matrix attribute is impossible on real hardware so both are unreachable. @@ -470,9 +458,6 @@ static HleGxVertexStateSnapshot CaptureGxVertexStateForCpWrites(const std::vecto } struct DlScanCacheEntry { - uint32_t listAddr = 0; - uint32_t nbytes = 0; - uint64_t layoutHash = 0; std::array maxIdx{}; std::array sawIdx{}; std::array maxXfIdx{}; @@ -484,28 +469,25 @@ struct DlScanCacheEntry { bool dlHasArrayStateWrites = false; bool dlHasCpWrites = false; bool scanOk = false; - // Flatten results for lists that need one. `flattenVerbatim` means the - // flattened stream is byte-identical to the source list, so aurora can be - // handed the guest bytes directly and no copy has to be stored. bool needsFlatten = false; bool flattenOk = false; - bool flattenVerbatim = false; }; struct DlScanCacheRecord { DlScanCacheEntry result{}; + uint32_t listAddr = 0; + uint32_t nbytes = 0; + uint64_t layoutHash = 0; // Answer of DisplayListMayContainDraw for these bytes bool mayContainDraw = false; uint64_t contentDigest = 0; - uint64_t writeGeneration = kDlWriteGenerationUntracked; + uint64_t writeGeneration = GxGuestWrite::kUntracked; std::vector cpWrites{}; - std::vector flattened{}; + bool streamIsSource = false; + std::vector stream{}; }; -struct DlScanCacheState { - std::unordered_map entries{}; - size_t storedCommandBytes = 0; -}; +using DlScanCacheState = GxDisplayListScanCache::BoundedCache; static uint64_t HashScanLayoutState() { if (!g_hleGxState.vtxLayoutHashDirty) { @@ -532,106 +514,36 @@ static uint64_t HashScanLayoutState() { return g_hleGxState.vtxLayoutHash; } -static uint64_t DlScanCacheKey(uint32_t listAddr, uint32_t nbytes, uint64_t layoutHash) { - return GxDisplayListScanCache::MakeIdentityKey(listAddr, nbytes, layoutHash); -} - static DlScanCacheState& DlScanCache() { - static thread_local DlScanCacheState s_cache; + static thread_local DlScanCacheState s_cache{kDlScanCacheMaxEntries, kDlScanCacheMaxStoredBytes}; return s_cache; } -static uint64_t DlContentDigest(const uint8_t* list, uint32_t nbytes) { - return static_cast(XXH3_64bits(list, static_cast(nbytes))); -} - - -struct DlScanCacheProbe { - const DlScanCacheRecord* record = nullptr; - uint64_t contentDigest = 0; - bool digestValid = false; - uint64_t writeGeneration = kDlWriteGenerationUntracked; -}; - -static DlScanCacheProbe ProbeDlScanCache(const uint8_t* list, uint32_t listAddr, uint32_t nbytes, - uint64_t layoutHash) { - DlScanCacheProbe probe{}; - probe.writeGeneration = GxGuestWrite::GenerationForRange(listAddr, nbytes); - - auto& s_cache = DlScanCache(); - const uint64_t key = DlScanCacheKey(listAddr, nbytes, layoutHash); - const auto it = s_cache.entries.find(key); - if (it != s_cache.entries.end()) { - auto& record = it->second; - const auto& entry = record.result; - const bool identityMatches = entry.listAddr == CanonicalizeGxMainRamAddress(listAddr) && - entry.nbytes == nbytes && entry.layoutHash == layoutHash; - if (identityMatches && probe.writeGeneration != kDlWriteGenerationUntracked && - record.writeGeneration == probe.writeGeneration) { - // Nothing has flushed a write over these bytes since the digest was - // taken, so the stored digest still describes them. Skip XXH3. - probe.record = &record; - probe.contentDigest = record.contentDigest; - probe.digestValid = true; - } else { - probe.contentDigest = DlContentDigest(list, nbytes); - probe.digestValid = true; - if (identityMatches && - GxDisplayListScanCache::CanReuse(entry.listAddr, entry.nbytes, entry.layoutHash, - record.contentDigest, listAddr, nbytes, layoutHash, - probe.contentDigest)) { - // False alarm: the bump did not actually change this list - record.writeGeneration = probe.writeGeneration; - probe.record = &record; - } - } - } - return probe; -} - -static size_t DlScanCacheRecordBytes(const DlScanCacheRecord& record) { - return record.flattened.size() + record.cpWrites.size() * sizeof(DlCpWrite); -} - static void StoreDlScanCache(uint32_t listAddr, uint32_t nbytes, uint64_t layoutHash, uint64_t contentDigest, uint64_t writeGeneration, bool mayContainDraw, const DlScanCacheEntry& entry, - std::vector&& cpWrites, const uint8_t* flattened, - uint32_t flattenedBytes) { + std::vector&& cpWrites, const uint8_t* stream, + uint32_t streamBytes, bool streamIsSource) try { auto& s_cache = DlScanCache(); - const uint64_t key = DlScanCacheKey(listAddr, nbytes, layoutHash); - auto old = s_cache.entries.find(key); - bool replacing = old != s_cache.entries.end(); - size_t replacedBytes = replacing ? DlScanCacheRecordBytes(old->second) : 0; - const size_t incomingBytes = static_cast(flattenedBytes) + - cpWrites.size() * sizeof(DlCpWrite); - if ((!replacing && s_cache.entries.size() >= kDlScanCacheMaxEntries) || - s_cache.storedCommandBytes - replacedBytes + incomingBytes > kDlScanCacheMaxStoredBytes) { - s_cache.entries.clear(); - s_cache.storedCommandBytes = 0; - replacing = false; - replacedBytes = 0; - } + const uint64_t key = GxDisplayListScanCache::MakeIdentityKey(listAddr, nbytes, layoutHash); DlScanCacheRecord record{}; record.result = entry; record.mayContainDraw = mayContainDraw; - record.result.listAddr = CanonicalizeGxMainRamAddress(listAddr); - record.result.nbytes = nbytes; - record.result.layoutHash = layoutHash; + record.listAddr = CanonicalizeGxMainRamAddress(listAddr); + record.nbytes = nbytes; + record.layoutHash = layoutHash; record.contentDigest = contentDigest; - // Sampled before the digest was taken, so a write racing the digest can only - // make the next call re-digest, never make it trust a stale entry. + // Sampled before capture so later write notifications force revalidation. record.writeGeneration = writeGeneration; record.cpWrites = std::move(cpWrites); - if (flattened != nullptr && flattenedBytes != 0) { - record.flattened.assign(flattened, flattened + flattenedBytes); - } - auto [it, inserted] = s_cache.entries.insert_or_assign(key, std::move(record)); - (void)inserted; - s_cache.storedCommandBytes += DlScanCacheRecordBytes(it->second); - if (replacing) { - s_cache.storedCommandBytes -= replacedBytes; - } + record.streamIsSource = streamIsSource; + if (streamBytes != 0) { + record.stream.assign(stream, stream + streamBytes); + } + const size_t incomingBytes = record.stream.size() + record.cpWrites.size() * sizeof(DlCpWrite); + s_cache.Insert(key, std::move(record), incomingBytes); +} catch (const std::bad_alloc&) { + // The current draw still has its scan results and stream. } static void BuildScanVertexLayout(GXVtxFmt vtxfmt, std::array& indexedSteps, @@ -860,7 +772,6 @@ struct DlInterpretVisitor { uint32_t DrawVertexSize(GXVtxFmt vtxfmt) { return CalcDLVertexSize(vtxfmt); } bool OnDraw(const uint8_t*, uint8_t opcode, GXVtxFmt vtxfmt, uint16_t vtxCount, const uint8_t* vertices, uint32_t& payloadBytes) { - EnsureAuroraFrameActive(); const HleGxVertexStateSnapshot drawGuestState = CaptureGxVertexState(); ApplyAuroraVtxStateForDlBegin(vtxfmt); GXBegin(OpcodeToGXPrimitive(opcode), vtxfmt, vtxCount); @@ -1315,27 +1226,29 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) return; } try { - const uint8_t* list = static_cast(GuestToHostPtr(listAddr, nbytes)); - if (!list) return; + const auto* guestList = static_cast(GuestToHostPtr(listAddr, nbytes)); + if (!guestList) return; - const bool allowScanCache = nbytes <= kDlScanCacheMaxEntryBytes; + // Frame waits can dispatch guest code before the worker-side snapshot is taken. + EnsureAuroraFrameActive(); + GxDisplayListScanCache::DisplayListBytes bytes{guestList, nbytes}; + const bool allowScanCache = bytes.CanCache(); const uint64_t scanLayoutHash = allowScanCache ? HashScanLayoutState() : 0; - // The digest is computed lazily inside the probe: an entry whose covering - // write-tracking granules have not moved since it was stored keeps its - // recorded digest and skips the XXH3 pass over the whole list entirely. - const DlScanCacheProbe probe = - allowScanCache ? ProbeDlScanCache(list, listAddr, nbytes, scanLayoutHash) - : DlScanCacheProbe{}; - const DlScanCacheRecord* cached = probe.record; + const uint64_t writeGeneration = allowScanCache + ? GxGuestWrite::GenerationForRange(listAddr, nbytes) + : GxGuestWrite::kUntracked; + const DlScanCacheRecord* cached = + allowScanCache ? GxDisplayListScanCache::Probe(DlScanCache(), bytes, listAddr, nbytes, + scanLayoutHash, writeGeneration) + : nullptr; // The scan a cached record came from already walked the list for this, // so only a miss pays for DisplayListMayContainDraw. const bool mayContainDraw = - (cached != nullptr) ? cached->mayContainDraw : DisplayListMayContainDraw(list, nbytes); + (cached != nullptr) ? cached->mayContainDraw : DisplayListMayContainDraw(bytes.Data(), nbytes); if (!mayContainDraw) { - EnsureAuroraFrameActive(); GXMarkFrameWork(); - GXCallDisplayList(list, nbytes); + GXCallDisplayList(cached ? cached->stream.data() : bytes.Data(), nbytes); // The republish is required even for register-only lists (verified by // experiment: dropping it corrupts in-race geometry decode). It is // instead memoized inside SyncAppliedVtxStateFromHleReal, so a run of @@ -1385,15 +1298,11 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) ApplyCpRegWrite(write.reg, write.value); } if (needsFlatten && flattenOk) { - if (entry.flattenVerbatim) { - auroraStream = list; - auroraStreamBytes = nbytes; - } else { - auroraStream = cached->flattened.data(); - auroraStreamBytes = static_cast(cached->flattened.size()); - } + auroraStream = cached->stream.data(); + auroraStreamBytes = static_cast(cached->stream.size()); } } else { + const auto* list = bytes.Data(); std::vector cpWrites{}; // The scan discovers the list's CP writes as it walks, so this path // cannot narrow the snapshot the way the cached one does. @@ -1425,9 +1334,6 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) // scan/flatten are pure functions of the list bytes and the layout-hash cache key. if (allowScanCache && scanOk && !dlHasNestedDl && (!needsFlatten || flattenOk)) { DlScanCacheEntry entry{}; - entry.listAddr = CanonicalizeGxMainRamAddress(listAddr); - entry.nbytes = nbytes; - entry.layoutHash = scanLayoutHash; entry.maxIdx = maxIdx; entry.sawIdx = sawIdx; entry.maxXfIdx = maxXfIdx; @@ -1441,15 +1347,12 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) entry.scanOk = scanOk; entry.needsFlatten = needsFlatten; entry.flattenOk = flattenOk; - entry.flattenVerbatim = flattenVerbatim; const bool storeFlattened = needsFlatten && flattenOk && !flattenVerbatim; - const uint64_t contentDigest = - probe.digestValid ? probe.contentDigest : DlContentDigest(list, nbytes); - StoreDlScanCache(listAddr, nbytes, scanLayoutHash, contentDigest, - probe.writeGeneration, mayContainDraw, entry, std::move(cpWrites), - storeFlattened ? flattened.data() : nullptr, - storeFlattened ? static_cast(flattened.size()) : 0u); + StoreDlScanCache(listAddr, nbytes, scanLayoutHash, bytes.Digest(), + writeGeneration, mayContainDraw, entry, std::move(cpWrites), + storeFlattened ? flattened.data() : list, + storeFlattened ? static_cast(flattened.size()) : nbytes, !storeFlattened); } } @@ -1465,7 +1368,6 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) flattenApplyOk = flattenArraysOk && flattenXfOk; } if (flattenApplyOk) { - EnsureAuroraFrameActive(); GXMarkFrameWork(); GXCallDisplayList(auroraStream, auroraStreamBytes); // No flip back to GX_DIRECT here @@ -1485,9 +1387,8 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) applyOk = arraysOk && xfOk; } if (applyOk) { - EnsureAuroraFrameActive(); GXMarkFrameWork(); - GXCallDisplayList(list, nbytes); + GXCallDisplayList(cached ? cached->stream.data() : bytes.Data(), nbytes); SyncAppliedVtxStateFromHleReal(); return; @@ -1498,10 +1399,14 @@ extern "C" void GX__CallDisplayList_80172f64(uint32_t listAddr, uint32_t nbytes) if (haveStateBeforeScan) { RestoreGxVertexState(stateBeforeScan); } - EnsureAuroraFrameActive(); + if (!GxWorker::OnWorker() && !g_auroraFrameActive.load(std::memory_order_acquire)) { + EnsureAuroraFrameForFifo(); + // A frame wait may have run another display list on this thread. + bytes = GxDisplayListScanCache::DisplayListBytes{guestList, nbytes}; + } GXMarkFrameWork(); InvalidateAppliedAuroraState(); - ParseDisplayList(list, nbytes); + ParseDisplayList(bytes.Data(), nbytes); InvalidateAppliedAuroraState(); } catch (...) {} } diff --git a/runtime/src/hle/gx/gx_dl_cache.h b/runtime/src/hle/gx/gx_dl_cache.h new file mode 100644 index 000000000..6627bad53 --- /dev/null +++ b/runtime/src/hle/gx/gx_dl_cache.h @@ -0,0 +1,227 @@ +#pragma once + +#include "gx_internal.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace GxDisplayListScanCache { + +template +class BoundedCache { + struct Entry { + uint64_t key; + Value value; + size_t bytes; + Entry* newerBytes = nullptr; + Entry* olderBytes = nullptr; + }; + using Entries = std::list; + using Iterator = typename Entries::iterator; + +public: + static constexpr size_t kMaxEvictionsPerStore = 64; + + BoundedCache(size_t maxEntries, size_t maxBytes) + : m_maxEntries(maxEntries), m_maxBytes(maxBytes) {} + BoundedCache(const BoundedCache&) = delete; + BoundedCache& operator=(const BoundedCache&) = delete; + + Value* Find(uint64_t key) { + const auto it = m_index.find(key); + return it == m_index.end() ? nullptr : &it->second->value; + } + + // Only validated hits refresh recency. + void Touch(uint64_t key) { + const auto it = m_index.find(key); + if (it != m_index.end() && it->second != m_entries.begin()) { + const auto entry = it->second; + UnlinkByteEntry(*entry); + LinkByteEntry(*entry); + m_entries.splice(m_entries.begin(), m_entries, entry); + } + } + + bool Insert(uint64_t key, Value&& value, size_t bytes) { + static_assert(std::is_nothrow_move_assignable_v); + if (m_maxEntries == 0 || bytes > m_maxBytes) { + return false; + } + const auto found = m_index.find(key); + Entry* const old = found == m_index.end() ? nullptr : &*found->second; + size_t retainedBytes = m_storedBytes - (old ? old->bytes : 0u); + const size_t availableBytes = m_maxBytes - bytes; + + std::array victims{}; + size_t victimCount = 0; + for (auto* victim = m_oldestByteEntry; + victim && retainedBytes > availableBytes && victimCount < victims.size(); + victim = victim->newerBytes) { + if (victim == old) { + continue; + } + victims[victimCount++] = victim; + retainedBytes -= victim->bytes; + } + // Retire at most one batch so later attempts can make progress too. + if (retainedBytes > availableBytes) { + for (size_t i = 0; i < victimCount; ++i) { + Erase(*victims[i]); + } + return false; + } + // Any byte victim also frees the one slot a new entry can need. + if (victimCount == 0 && !old && m_entries.size() == m_maxEntries) { + victims[victimCount++] = &m_entries.back(); + } + + if (!old) { + m_entries.push_front({key, std::move(value), bytes}); + try { + m_index.emplace(key, m_entries.begin()); + } catch (...) { + m_entries.pop_front(); + throw; + } + } else { + UnlinkByteEntry(*old); + old->value = std::move(value); + m_storedBytes -= old->bytes; + old->bytes = bytes; + m_entries.splice(m_entries.begin(), m_entries, found->second); + } + LinkByteEntry(m_entries.front()); + for (size_t i = 0; i < victimCount; ++i) { + Erase(*victims[i]); + } + m_storedBytes += bytes; + return true; + } + + void Clear() { + m_index.clear(); + m_entries.clear(); + m_newestByteEntry = nullptr; + m_oldestByteEntry = nullptr; + m_storedBytes = 0; + } + + size_t Size() const { return m_entries.size(); } + size_t StoredBytes() const { return m_storedBytes; } + +private: + void LinkByteEntry(Entry& entry) { + if (entry.bytes == 0) return; + entry.newerBytes = nullptr; + entry.olderBytes = m_newestByteEntry; + if (m_newestByteEntry) { + m_newestByteEntry->newerBytes = &entry; + } else { + m_oldestByteEntry = &entry; + } + m_newestByteEntry = &entry; + } + + void UnlinkByteEntry(Entry& entry) { + if (entry.bytes == 0) return; + if (entry.newerBytes) { + entry.newerBytes->olderBytes = entry.olderBytes; + } else { + m_newestByteEntry = entry.olderBytes; + } + if (entry.olderBytes) { + entry.olderBytes->newerBytes = entry.newerBytes; + } else { + m_oldestByteEntry = entry.newerBytes; + } + entry.newerBytes = nullptr; + entry.olderBytes = nullptr; + } + + void Erase(Entry& entry) { + UnlinkByteEntry(entry); + m_storedBytes -= entry.bytes; + const auto found = m_index.find(entry.key); + const auto position = found->second; + m_index.erase(found); + m_entries.erase(position); + } + + size_t m_maxEntries; + size_t m_maxBytes; + Entries m_entries; + std::unordered_map m_index; + Entry* m_newestByteEntry = nullptr; + Entry* m_oldestByteEntry = nullptr; + size_t m_storedBytes = 0; +}; + +inline constexpr uint32_t kMaxSourceBytes = 64u * 1024u; + +class DisplayListBytes { +public: + DisplayListBytes(const uint8_t* source, uint32_t size) : m_data(source), m_size(size) {} + + bool CanCache() const { return m_size <= kMaxSourceBytes; } + + const uint8_t* Data() { + if (!m_captured && CanCache()) { + // Each thread consumes its list synchronously before reusing the scratch. + static thread_local std::array scratch; + std::memcpy(scratch.data(), m_data, m_size); + m_data = scratch.data(); + m_captured = true; + } + return m_data; + } + + uint64_t SourceDigest() const { return XXH3_64bits(m_data, m_size); } + + uint64_t Digest() { + if (!m_digest) { + m_digest = XXH3_64bits(Data(), m_size); + } + return *m_digest; + } + +private: + const uint8_t* m_data; + uint32_t m_size; + bool m_captured = false; + std::optional m_digest; +}; + +template +const Record* Probe(BoundedCache& cache, DisplayListBytes& list, uint32_t listAddr, + uint32_t nbytes, uint64_t layoutHash, uint64_t writeGeneration) { + const uint64_t key = MakeIdentityKey(listAddr, nbytes, layoutHash); + auto* record = cache.Find(key); + if (!record) return nullptr; + + if (record->listAddr != CanonicalizeGxMainRamAddress(listAddr) || record->nbytes != nbytes || + record->layoutHash != layoutHash) { + return nullptr; + } + if (record->streamIsSource) { + // Submit the owned copy after comparison, even if the guest rewrites the list again. + if (list.SourceDigest() != record->contentDigest) return nullptr; + } else if (!GxGuestWrite::CanSkipDigest(record->writeGeneration, writeGeneration) && + list.Digest() != record->contentDigest) { + return nullptr; + } + record->writeGeneration = writeGeneration; + cache.Touch(key); + return record; +} + +} // namespace GxDisplayListScanCache diff --git a/runtime/src/hle/gx/gx_fifo.cpp b/runtime/src/hle/gx/gx_fifo.cpp index 139b86dba..4bacfc726 100644 --- a/runtime/src/hle/gx/gx_fifo.cpp +++ b/runtime/src/hle/gx/gx_fifo.cpp @@ -13,6 +13,7 @@ HleGxState g_hleGxState; namespace aurora::gx::fifo { bool submit_raw_draw(GXPrimitive prim, GXVtxFmt fmt, const uint8_t* vertices, uint16_t vtxCount, uint32_t vertexBytes); +void write_raw_draw_data(const uint8_t* data, uint32_t length); } void HleGxState::ResetVertex() { @@ -83,6 +84,73 @@ static bool TryGetRawDirectFifoVertexSize(GXVtxFmt fmt, uint32_t& vertexSize) { return vertexSize != 0; } +void GxStream::PrepareImmediateDirectDraw() { + auto& g = g_hleGxState; + g.fifoDirectBytesRemaining = 0; + g.fifoDirectVertexSize = 0; + g.fifoPendingDrawGeneration = g.vtxStateGeneration; + if (!g.inBegin || g.vertsRemaining > 0xffffu || g.currentVtxFmt >= GX_MAX_VTXFMT) { + return; + } + uint32_t vertexSize = 0; + for (int attr = 0; attr < 26; ++attr) { + if (g.vtxDesc[attr] == GX_NONE) { + continue; + } + if (g.vtxDesc[attr] != GX_DIRECT || attr == GX_VA_NRM || attr > GX_VA_TEX7) { + return; + } + const auto& fmt = g.vtxAttrFmt[g.currentVtxFmt][attr]; + if (attr == GX_VA_CLR0 || attr == GX_VA_CLR1) { + // Packed colors and RGBX padding are re-encoded by the attribute writer. + if (!((fmt.type == GX_RGB8 && fmt.cnt == GX_CLR_RGB) || + (fmt.type == GX_RGBA8 && fmt.cnt == GX_CLR_RGBA))) { + return; + } + } else if (!IsMatrixIndexAttr(static_cast(attr))) { + if (static_cast(fmt.type) > GX_F32 || fmt.frac > 31 || + static_cast(fmt.cnt) > 1u) { + return; + } + } + vertexSize += GetDirectAttrByteSizeForFifo(static_cast(attr), fmt); + } + g.fifoDirectVertexSize = vertexSize; + g.fifoDirectBytesRemaining = vertexSize * g.vertsRemaining; +} + +static uint32_t WriteImmediateDirectBytes(const uint8_t* bytes, uint32_t size) { + auto& g = g_hleGxState; + if (!g.inBegin || g.fifoDirectBytesRemaining == 0) { + return 0; + } + if (!g.auroraBeginCalled) { + if (g.fifoPendingDrawGeneration != g.vtxStateGeneration) { + GxStream::PrepareImmediateDirectDraw(); + if (g.fifoDirectBytesRemaining == 0) { + return 0; + } + } + EnsureAuroraFrameActive(); + ApplyAuroraVtxStateForRawBegin(g.currentVtxFmt); + EnsureDefaultGxAlphaCompare(); + GXBegin(g.currentPrim, g.currentVtxFmt, static_cast(g.vertsRemaining)); + g.auroraBeginCalled = true; + GXMarkFrameWork(); + } + const uint32_t count = std::min(size, g.fifoDirectBytesRemaining); + aurora::gx::fifo::write_raw_draw_data(bytes, count); + g.fifoDirectBytesRemaining -= count; + g.vertsRemaining = (g.fifoDirectBytesRemaining + g.fifoDirectVertexSize - 1u) / g.fifoDirectVertexSize; + if (g.fifoDirectBytesRemaining == 0) { + g.inBegin = false; + g.auroraBeginCalled = false; + g.ResetVertex(); + GXEnd(); + } + return count; +} + static bool TrySubmitRawDirectFifoDraw(const uint8_t* packet, uint32_t packetBytes, GXPrimitive prim, GXVtxFmt vtxFmt, uint16_t vtxCount) { if (packet == nullptr || packetBytes == 0 || vtxCount == 0) { @@ -369,6 +437,19 @@ static MKW_PPC_NO_INLINE void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes); // `val` is a raw big-endian bit pattern: the FIFO stream is type-agnostic, and // the float entry point converts before it gets here. void HleFifoWrite(u32 val, uint32_t sizeBytes) { + if (!IsDisplayListActive() && g_hleGxState.fifoDirectBytesRemaining != 0) { + const uint32_t count = sizeBytes == 0 || sizeBytes > 4 ? 4u : sizeBytes; + uint8_t bytes[4]; + BigEndian::Write32(bytes, val); + const uint8_t* data = bytes + 4u - count; + const uint32_t consumed = WriteImmediateDirectBytes(data, count); + if (consumed != 0) { + for (uint32_t i = consumed; i < count; ++i) { + HleFifoWriteGeneral(data[i], 1); + } + return; + } + } if ((sizeBytes == 0 || sizeBytes == 4) && !IsDisplayListActive() && TryAppendIncompleteXfWord(val)) { return; } @@ -385,6 +466,7 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { auto resetFifoBuffer = [&]() { g_hleGxState.fifoReadOffset = 0; g_hleGxState.fifoByteCount = 0; + g_hleGxState.fifoPendingDrawBytes = 0; }; auto compactFifoBuffer = [&]() { @@ -440,6 +522,7 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { } g_hleGxState.fifoReadOffset += count; g_hleGxState.fifoByteCount -= count; + g_hleGxState.fifoPendingDrawBytes = 0; if (g_hleGxState.fifoByteCount == 0) { g_hleGxState.fifoReadOffset = 0; } @@ -448,9 +531,17 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { }; pushBytes(val, sizeBytes == 0 ? 4 : sizeBytes); + if (g_hleGxState.fifoPendingDrawBytes != 0) { + if (g_hleGxState.fifoPendingDrawGeneration == g_hleGxState.vtxStateGeneration && + g_hleGxState.fifoByteCount < g_hleGxState.fifoPendingDrawBytes) { + return; + } + g_hleGxState.fifoPendingDrawBytes = 0; + } // Parse raw FIFO command packets written directly to the gather pipe // (e.g. NW4R/G3D paths that do not call the GXBegin wrapper function). +parseBufferedPackets: while (!g_hleGxState.inBegin) { if (g_hleGxState.fifoByteCount < 1) { break; @@ -535,8 +626,10 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { ? 3u + static_cast(vtxCount) * rawVertexSize : 0u; // A packet that cannot fit the buffer is streamed per vertex instead. - if (packetBytes != 0 && packetBytes <= g_hleGxState.fifoBytes.size()) { + if (packetBytes != 0 && packetBytes + 3u <= g_hleGxState.fifoBytes.size()) { if (g_hleGxState.fifoByteCount < packetBytes) { + g_hleGxState.fifoPendingDrawBytes = packetBytes; + g_hleGxState.fifoPendingDrawGeneration = g_hleGxState.vtxStateGeneration; break; } if (TrySubmitRawDirectFifoDraw(data, packetBytes, prim, vtxFmt, vtxCount)) { @@ -695,6 +788,9 @@ static void HleFifoWriteGeneral(u32 val, uint32_t sizeBytes) { } break; } + if (!g_hleGxState.inBegin && g_hleGxState.fifoByteCount != 0) { + goto parseBufferedPackets; + } } // Must behave exactly like GX_HLE_FIFO_Write32/16/8 run byte by byte. The ring is not @@ -851,6 +947,15 @@ static void FifoWriteBurstImpl(const uint8_t* data, uint32_t sizeBytes) { return; } + if (!IsDisplayListActive()) { + const uint32_t consumed = WriteImmediateDirectBytes(data, sizeBytes); + data += consumed; + sizeBytes -= consumed; + if (sizeBytes == 0) { + return; + } + } + const uint32_t applied = ApplyFifoPacketsDirect(data, sizeBytes); if (applied < sizeBytes) { HleFifoWriteBurstChunked(data + applied, sizeBytes - applied); diff --git a/runtime/src/hle/gx/gx_init.cpp b/runtime/src/hle/gx/gx_init.cpp index edf6f9c12..8243f8bfc 100644 --- a/runtime/src/hle/gx/gx_init.cpp +++ b/runtime/src/hle/gx/gx_init.cpp @@ -30,6 +30,7 @@ extern "C" uint32_t GX__Init_8016b850(uint32_t fifoBase, uint32_t fifoSize) constexpr uint32_t kGXDataSize = 0x600u; GXInit(GuestToHostPtr(fifoBase, fifoSize), fifoSize); + InvalidateTexObjLoads(); // Initialize GXData structure in guest memory try { diff --git a/runtime/src/hle/gx/gx_internal.h b/runtime/src/hle/gx/gx_internal.h index b8270a9b9..36c4cdf40 100644 --- a/runtime/src/hle/gx/gx_internal.h +++ b/runtime/src/hle/gx/gx_internal.h @@ -154,6 +154,8 @@ struct TexObjSlot : TexObjMeta { uint64_t guestShadow[4] = {0, 0, 0, 0}; bool hasGuestShadow = false; + GxTextureBindingContract::ValidatedLoad validatedLoad; + // DCStoreRange interval-index bookkeeping: what this slot currently // contributes to g_texObjIntervals, so a pending update that turns out to // be a no-op costs a compare instead of an erase + insert. @@ -256,6 +258,10 @@ struct HleGxState { std::array fifoBytes{}; size_t fifoReadOffset = 0; size_t fifoByteCount = 0; + uint32_t fifoPendingDrawBytes = 0; + uint64_t fifoPendingDrawGeneration = 0; + uint32_t fifoDirectBytesRemaining = 0; + uint32_t fifoDirectVertexSize = 0; void ResetVertex(); void InvalidateVtxLayoutHash() noexcept { @@ -347,6 +353,11 @@ bool ValidateTlutData(uint32_t objAddr, const TlutObjMeta& meta); TexObjMeta& GetTexObjMeta(uint32_t addr); TexObjMeta ExtractTexObjMetaFromGuest(uint32_t addr); bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta); +// Lookup and publication require g_texObjMutex; reset acquires it itself. +void RefreshTexObjDecodeShadow(TexObjSlot& slot) noexcept; +TexObjSlot* TryGetValidatedTexObjSlot(uint32_t addr); +void MarkTexObjLoadValidated(uint32_t addr); +void InvalidateTexObjLoads(); // Guest GXTexObj bytes captured on the game thread; while a scope is active on a thread, texobj // decoding for that address reads the captured bytes instead of live guest memory. diff --git a/runtime/src/hle/gx/gx_objects.cpp b/runtime/src/hle/gx/gx_objects.cpp index 645658dfb..7aa1d78af 100644 --- a/runtime/src/hle/gx/gx_objects.cpp +++ b/runtime/src/hle/gx/gx_objects.cpp @@ -511,14 +511,45 @@ bool ShadowEquals(const uint64_t (&a)[4], const uint64_t (&b)[4]) noexcept { return a[0] == b[0] && a[1] == b[1] && a[2] == b[2] && a[3] == b[3]; } +} // namespace + // Captures the current guest bytes as the slot's shadow so the cache is marked in sync. HLE // GXInitTexObj* callers fetch the meta reference before rewriting guest memory, so their capture // is intentionally stale; the next lookup just re-decodes once, which is the safe direction. -void MarkTexObjSyncedLocked(TexObjSlot& slot) noexcept { +void RefreshTexObjDecodeShadow(TexObjSlot& slot) noexcept { slot.hasGuestShadow = ReadTexObjShadow(slot.objAddr, slot.guestShadow); } -} // namespace +TexObjSlot* TryGetValidatedTexObjSlot(uint32_t addr) { + TexObjSlot* slot = FindTexObjSlot(addr); + if (slot == nullptr) { + return nullptr; + } + if (slot->validatedLoad.CanReuse(TexObjSnapshotFor(addr), slot->needsUpload, + slot->host && slot->host->storageLive && slot->host->constructed, + IsPaletteTexFormat(slot->format))) { + return slot; + } + slot->validatedLoad.Invalidate(); + return nullptr; +} + +void MarkTexObjLoadValidated(uint32_t addr) { + TexObjSlot* slot = FindTexObjSlot(addr); + if (slot != nullptr && !slot->needsUpload && slot->host && slot->host->constructed && + !IsPaletteTexFormat(slot->format)) { + // Only call-time bytes may identify a completed load, never newer producer memory. + slot->validatedLoad.Record(TexObjSnapshotFor(addr)); + } +} + +void InvalidateTexObjLoads() { + std::lock_guard guard(g_texObjMutex); + for (auto& [addr, slot] : g_TexObjMeta) { + slot.validatedLoad.Invalidate(); + } + g_boundTexMaps = {}; +} bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { TexObjSlot* slot = FindTexObjSlot(addr); @@ -542,7 +573,7 @@ bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { outMeta = MergeGuestTexObjMeta(*slot, extracted); return true; } - MarkTexObjSyncedLocked(*slot); + RefreshTexObjDecodeShadow(*slot); outMeta = *slot; return true; } @@ -553,8 +584,9 @@ bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { const bool extractedUsable = IsUsableTexObjMeta(extracted); if (extractedUsable) { QueueTexObjIntervalUpdate(*slot); + slot->validatedLoad.Invalidate(); static_cast(*slot) = extracted; - MarkTexObjSyncedLocked(*slot); + RefreshTexObjDecodeShadow(*slot); outMeta = extracted; return true; } @@ -575,8 +607,9 @@ bool TryGetOrExtractTexObjMeta(uint32_t addr, TexObjMeta& outMeta) { // queues the interval-index patch and re-captures the guest shadow before that happens. TexObjMeta& GetTexObjMeta(uint32_t addr) { TexObjSlot& slot = FindOrCreateTexObjSlot(addr); + slot.validatedLoad.Invalidate(); QueueTexObjIntervalUpdate(slot); - MarkTexObjSyncedLocked(slot); + RefreshTexObjDecodeShadow(slot); return slot; } @@ -604,6 +637,7 @@ GXTexObj* GetHostTexObj(uint32_t addr) { GXTexObj* CreateHostTexObj(uint32_t addr) { TexObjSlot& slot = FindOrCreateTexObjSlot(addr); + slot.validatedLoad.Invalidate(); // GXInitTexObj/GXInitTexObjCI land here and rewrite the entire guest struct, // resetting the LOD/filter words the HLE does not mirror into this cache. // The shadow captured before that rewrite mismatches afterwards, so the diff --git a/runtime/src/hle/gx/gx_stream_common.h b/runtime/src/hle/gx/gx_stream_common.h index 006ab4711..8167f37de 100644 --- a/runtime/src/hle/gx/gx_stream_common.h +++ b/runtime/src/hle/gx/gx_stream_common.h @@ -36,6 +36,9 @@ constexpr uint8_t GX_DRAW_POINTS_CMD = 0xB8; namespace GxStream { +// Called after the immediate begin state has been initialized. +void PrepareImmediateDirectDraw(); + // --- Big-endian scalar reads out of a command stream --------------------- inline uint16_t ReadBE16(const uint8_t* ptr) { return BigEndian::Read16(ptr); } diff --git a/runtime/src/hle/gx/gx_texture.cpp b/runtime/src/hle/gx/gx_texture.cpp index 0f7953334..45dfdd6bd 100644 --- a/runtime/src/hle/gx/gx_texture.cpp +++ b/runtime/src/hle/gx/gx_texture.cpp @@ -2,6 +2,9 @@ #include "gx_internal.h" #include "runtime_log.h" +// Aurora's source layout supplies the identity carried by GXLoadTexObj. +#include <../lib/gfx/texture.hpp> + #include #include @@ -486,13 +489,25 @@ static void BindUnloadableTexturePlaceholder(uint32_t tid) { GXInitTexObj(&s_placeholder, nullptr, 1, 1, GX_TF_RGBA8, GX_CLAMP, GX_CLAMP, GX_FALSE); s_placeholderReady = true; } - // A zeroed binding record can never compare equal to a real load (those - // always carry a non-zero objAddr), so the CanSkipHostLoad fast path cannot - // mistake a later genuine bind for a no-op. - g_boundTexMaps[tid] = BoundTexInfo{}; + g_boundTexMaps[tid] = {}; GXLoadTexObj(&s_placeholder, (GXTexMapID)tid); } +static void BindHostTexObj(uint32_t oa, uint32_t tid, GXTexObj* obj, const TexObjMeta& meta, + bool forceBind = false) { + const auto& hostObj = *reinterpret_cast(obj); + BoundTexInfo binding{}; + static_cast(binding) = meta; + binding.objAddr = oa; + binding.dataAddr = CanonicalizeGxMainRamAddress(meta.dataAddr); + binding.texObjId = hostObj.texObjId; + binding.texDataVersion = hostObj.texDataVersion; + if (forceBind || !GxTextureBindingContract::SameBinding(g_boundTexMaps[tid], binding)) { + GXLoadTexObj(obj, (GXTexMapID)tid); + g_boundTexMaps[tid] = binding; + } +} + static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { static uint32_t s_invalidMetaLogCount = 0; static uint32_t s_invalidTidLogCount = 0; @@ -511,6 +526,27 @@ static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { return; } + if (TexObjSlot* slot = TryGetValidatedTexObjSlot(oa)) { + try { + BindHostTexObj(oa, tid, slot->host->PublicPtr(), *slot); + RefreshTexObjDecodeShadow(*slot); + } catch (const std::exception& ex) { + slot->validatedLoad.Invalidate(); + if (s_hostExceptionLogCount++ < 64) { + RT_LOGF(RT_TAG_GX, "GXLoadTexObj rejected reason=host-exception (%s) oa=0x%08X tid=%u\n", + ex.what(), oa, tid); + } + BindUnloadableTexturePlaceholder(tid); + } catch (...) { + slot->validatedLoad.Invalidate(); + if (s_hostExceptionLogCount++ < 64) { + RT_LOGF(RT_TAG_GX, "GXLoadTexObj rejected reason=host-exception-unknown oa=0x%08X tid=%u\n", oa, tid); + } + BindUnloadableTexturePlaceholder(tid); + } + return; + } + // Check if we have metadata for this texture, or try to extract it from guest memory TexObjMeta meta; if (!TryGetOrExtractTexObjMeta(oa, meta)) { @@ -645,22 +681,9 @@ static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { meta.needsUpload = false; textureDataUploaded = true; } - // Everything the binding contract compares apart from objAddr is the - // shared sampler state, so copy that base wholesale; only dataAddr needs - // to be canonicalized on the way in. - BoundTexInfo newBound{}; - static_cast(newBound) = meta; - newBound.objAddr = oa; - newBound.dataAddr = CanonicalizeGxMainRamAddress(meta.dataAddr); - const BoundTexInfo& oldBound = g_boundTexMaps[tid]; - const bool canSkipHostLoad = GxTextureBindingContract::CanSkipHostLoad( - oldBound, newBound, needsInit, tlutChanged, textureDataUploaded); - - g_boundTexMaps[tid] = newBound; GetTexObjMeta(oa) = meta; - if (!canSkipHostLoad) { - GXLoadTexObj(obj, (GXTexMapID)tid); - } + BindHostTexObj(oa, tid, obj, meta, needsInit || tlutChanged || textureDataUploaded); + MarkTexObjLoadValidated(oa); } catch (const std::exception& ex) { if (s_hostExceptionLogCount++ < 64) { RT_LOGF(RT_TAG_GX, @@ -681,24 +704,22 @@ static void LoadTexObjImpl(uint32_t oa, uint32_t tid) { } extern "C" void GX__LoadTexObj_80170f2c(uint32_t oa, uint32_t tid) { - if (GxWorker::ShouldDefer()) { - // The CPU-side SDK reads the GXTexObj at call time; the texture data itself is read later. - std::optional bytes; + // Inline and deferred loads use the same complete call-time snapshot. + std::optional bytes; + if (tid < g_boundTexMaps.size()) { try { bytes = CaptureTexObjBytes(oa); } catch (...) { } - GxWorker::Submit([oa, tid, bytes] { - if (bytes) { - TexObjSnapshotScope snapshot(oa, *bytes); - LoadTexObjImpl(oa, tid); - } else { - LoadTexObjImpl(oa, tid); - } - }); - } else { - LoadTexObjImpl(oa, tid); } + GxWorker::Run([oa, tid, bytes] { + if (bytes) { + TexObjSnapshotScope snapshot(oa, *bytes); + LoadTexObjImpl(oa, tid); + } else { + LoadTexObjImpl(oa, tid); + } + }); try { uint32_t gd = Memory::Read32(kGXDataPtrAddr); if (gd) { Memory::Write32(gd + 0x5FCu, Memory::Read32(gd + 0x5FCu) | 1u); Memory::Write16(gd + 2, 0); } } catch (...) {} } PPC_NATIVE_OVERRIDE_VOID(80170f2c, GX__LoadTexObj_80170f2c, (uint32_t oa, uint32_t tid), (oa, tid)); diff --git a/runtime/src/hle/gx/gx_texture_binding_contract.h b/runtime/src/hle/gx/gx_texture_binding_contract.h index f8ca7b233..31577e752 100644 --- a/runtime/src/hle/gx/gx_texture_binding_contract.h +++ b/runtime/src/hle/gx/gx_texture_binding_contract.h @@ -1,12 +1,32 @@ #pragma once #include +#include namespace GxTextureBindingContract { -// Every piece of GX texture state that can change the bound view/sampler. Shared by `State` -// (this plus the guest GXTexObj address) and HLE's TexObjMeta (this plus the `userData`/ -// `needsUpload` bookkeeping fields), so a new sampler field here reaches both. +// Independent of the metadata shadow, which getters and setters may refresh. +struct ValidatedLoad { + uint8_t bytes[32]{}; + bool valid = false; + + void Invalidate() noexcept { valid = false; } + + void Record(const uint8_t* snapshot) noexcept { + valid = snapshot != nullptr; + if (valid) { + std::memcpy(bytes, snapshot, sizeof(bytes)); + } + } + + bool CanReuse(const uint8_t* snapshot, bool needsUpload, bool constructed, + bool isPalette) const noexcept { + return valid && snapshot != nullptr && !needsUpload && constructed && !isPalette && + std::memcmp(bytes, snapshot, sizeof(bytes)) == 0; + } +}; + +// Sampler fields shared by the binding record and HLE's TexObjMeta. struct SamplerState { uint32_t dataAddr = 0; uint16_t width = 0; @@ -31,6 +51,8 @@ struct SamplerState { // be part of the snapshot. struct State : SamplerState { uint32_t objAddr = 0; + uint32_t texObjId = 0; + uint32_t texDataVersion = 0; }; inline bool SameSamplerState(const SamplerState& lhs, const SamplerState& rhs) noexcept { @@ -53,14 +75,8 @@ inline bool SameSamplerState(const SamplerState& lhs, const SamplerState& rhs) n } inline bool SameBinding(const State& lhs, const State& rhs) noexcept { - return lhs.objAddr == rhs.objAddr && SameSamplerState(lhs, rhs); -} - -inline bool CanSkipHostLoad(const State& oldBinding, const State& newBinding, - bool needsInit, bool tlutChanged, - bool textureDataUploaded) noexcept { - return SameBinding(oldBinding, newBinding) && - !needsInit && !tlutChanged && !textureDataUploaded; + return lhs.objAddr == rhs.objAddr && lhs.texObjId == rhs.texObjId && lhs.texDataVersion == rhs.texDataVersion && + SameSamplerState(lhs, rhs); } } // namespace GxTextureBindingContract diff --git a/runtime/src/hle/gx/gx_vertex.cpp b/runtime/src/hle/gx/gx_vertex.cpp index 3a8c43b89..e0e90bced 100644 --- a/runtime/src/hle/gx/gx_vertex.cpp +++ b/runtime/src/hle/gx/gx_vertex.cpp @@ -329,11 +329,15 @@ static void GxBeginImpl(uint32_t t, uint32_t vf, uint32_t nv) { g_hleGxState.vertsRemaining=nv; g_hleGxState.inBegin=nv!=0; g_hleGxState.auroraBeginCalled=false; g_hleGxState.fifoReadOffset = 0; g_hleGxState.fifoByteCount=0; g_hleGxState.ResetVertex(); + g_hleGxState.fifoPendingDrawBytes = 0; + GxStream::PrepareImmediateDirectDraw(); } PPC_NATIVE_OVERRIDE_VOID(8016f0f0, GX__Begin_8016f0f0, (uint32_t t, uint32_t vf, uint32_t nv), (t, vf, nv)); extern "C" void GX__End_80044b30() { - GX_DEFER_CALL0(GX__End_80044b30); g_hleGxState.inBegin=false; GXEnd(); } + GX_DEFER_CALL0(GX__End_80044b30); + g_hleGxState.fifoDirectBytesRemaining = 0; + g_hleGxState.inBegin=false; GXEnd(); } PPC_NATIVE_OVERRIDE_VOID(80044b30, GX__End_80044b30, (), ()); extern "C" void GX__End_80048c30() { GX__End_80044b30(); } diff --git a/runtime/src/hle/gx/gx_worker.cpp b/runtime/src/hle/gx/gx_worker.cpp index ac8983e05..15f91ded5 100644 --- a/runtime/src/hle/gx/gx_worker.cpp +++ b/runtime/src/hle/gx/gx_worker.cpp @@ -54,7 +54,9 @@ uint64_t g_write = 0; std::atomic g_head{0}; std::atomic g_tail{0}; std::atomic g_consumerWaiting{false}; -std::atomic g_producerWaiting{false}; +constexpr uint64_t kNotWaiting = ~uint64_t{0}; +// Publish the required tail before rechecking it under g_tailMutex. +std::atomic g_waitTarget{kNotWaiting}; std::mutex g_headMutex; std::condition_variable g_headCv; std::mutex g_tailMutex; @@ -66,6 +68,16 @@ std::atomic g_exited{false}; std::thread* g_thread = nullptr; std::vector g_fifoBytes; +#if defined(MKW_GX_WORKER_TESTING) +bool g_testEnabled = true; +bool g_testWakeEachRecord = false; +std::atomic g_testNotifications{0}; +std::atomic g_testWaits{0}; +bool g_testAcknowledgeRecord = false; // Worker only. +std::atomic g_testAccountedTail{0}; +void (*g_testBeforeTailRecheck)() = nullptr; +#endif + constexpr size_t AlignUp(size_t value) { return (value + detail::kAlign - 1) & ~(detail::kAlign - 1); } inline void CpuRelax() { @@ -117,11 +129,22 @@ void WaitForTail(uint64_t minTail, bool service) { { PerfStats::WaitScope wait; std::unique_lock lock{g_tailMutex}; - g_producerWaiting.store(true, std::memory_order_seq_cst); + g_waitTarget.store(minTail, std::memory_order_seq_cst); +#if defined(MKW_GX_WORKER_TESTING) + if (g_testWakeEachRecord) { + g_waitTarget.store(0, std::memory_order_seq_cst); + } + if (g_testBeforeTailRecheck != nullptr) { + g_testBeforeTailRecheck(); + } +#endif if (g_tail.load(std::memory_order_seq_cst) < minTail) { +#if defined(MKW_GX_WORKER_TESTING) + g_testWaits.fetch_add(1, std::memory_order_relaxed); +#endif WaitTailBriefly(lock); } - g_producerWaiting.store(false, std::memory_order_relaxed); + g_waitTarget.store(kNotWaiting, std::memory_order_relaxed); } if (service && g_waitCallback != nullptr && g_tail.load(std::memory_order_acquire) < minTail) { g_waitCallback(); @@ -234,10 +257,19 @@ void WorkerMain() { } tail += header->size; g_tail.store(tail, std::memory_order_seq_cst); - if (g_producerWaiting.load(std::memory_order_seq_cst)) { + if (tail >= g_waitTarget.load(std::memory_order_seq_cst)) { std::lock_guard lock{g_tailMutex}; +#if defined(MKW_GX_WORKER_TESTING) + g_testNotifications.fetch_add(1, std::memory_order_relaxed); +#endif g_tailCv.notify_all(); } +#if defined(MKW_GX_WORKER_TESTING) + if (g_testAcknowledgeRecord) { + g_testAcknowledgeRecord = false; + g_testAccountedTail.store(tail, std::memory_order_release); + } +#endif } } } @@ -342,7 +374,12 @@ void Start() { if (Active()) { return; } - if (!RuntimeConfigFile::GxWorkerEnabled(true)) { +#if defined(MKW_GX_WORKER_TESTING) + const bool enabled = g_testEnabled; +#else + const bool enabled = RuntimeConfigFile::GxWorkerEnabled(true); +#endif + if (!enabled) { RT_LOGF(RT_TAG_GX, "GX worker disabled\n"); return; } @@ -350,6 +387,7 @@ void Start() { g_ring = static_cast(::operator new(kCapacity, std::align_val_t{64})); } g_fifoBytes.reserve(kFifoFlushBytes + 4096); + g_waitTarget.store(kNotWaiting, std::memory_order_relaxed); g_stop.store(false, std::memory_order_relaxed); g_abandon.store(false, std::memory_order_relaxed); g_exited.store(false, std::memory_order_relaxed); diff --git a/runtime/tests/ax_mix_kernels_tests.cpp b/runtime/tests/ax_mix_kernels_tests.cpp new file mode 100644 index 000000000..8909eb5a0 --- /dev/null +++ b/runtime/tests/ax_mix_kernels_tests.cpp @@ -0,0 +1,332 @@ +#include "ax_mix_kernels.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(MKW_AX_MIX_REQUIRE_NEON) && !MKW_AX_MIX_NEON +#error The ARM kernel fixture must select NEON +#endif + +namespace { + +struct Backend { + const char* name; + decltype(&AxMixKernels::MixAddRampScalar) mix; + decltype(&AxMixKernels::ScaleRampScalar) scale; + decltype(&AxMixKernels::MixAccumRamp32Scalar) accum; + decltype(&AxMixKernels::StoreBigEndian32Scalar) store; + decltype(&AxMixKernels::LoadBigEndian32Scalar) load; +}; + +constexpr Backend kScalar = {"scalar", AxMixKernels::MixAddRampScalar, + AxMixKernels::ScaleRampScalar, AxMixKernels::MixAccumRamp32Scalar, + AxMixKernels::StoreBigEndian32Scalar, AxMixKernels::LoadBigEndian32Scalar}; +#if MKW_AX_MIX_NEON +constexpr const char* kSelectedName = "NEON"; +#elif MKW_AX_MIX_AVX2 +constexpr const char* kSelectedName = "AVX2"; +#else +constexpr const char* kSelectedName = "selected scalar"; +#endif +constexpr Backend kSelected = {kSelectedName, AxMixKernels::MixAddRamp, + AxMixKernels::ScaleRamp, AxMixKernels::MixAccumRamp32, + AxMixKernels::StoreBigEndian32, AxMixKernels::LoadBigEndian32}; + +uint32_t g_random = 0x12345678u; + +char g_case[192]{}; + +uint32_t NextRandom() { + g_random ^= g_random << 13; + g_random ^= g_random >> 17; + g_random ^= g_random << 5; + return g_random; +} + +void Require(bool condition, const char* message) { + if (condition) return; + if (g_case[0]) std::fprintf(stderr, "%s: ", g_case); + throw std::runtime_error(message); +} + +int32_t Wrap32(int64_t value) { + const uint32_t bits = static_cast(value); + return bits <= 0x7fffffffu ? static_cast(bits) + : static_cast(static_cast(bits) - 0x100000000ll); +} + +int64_t FloorQ15(int64_t product) { + return product >= 0 ? product / 32768 : -((-product + 32767) / 32768); +} + +int16_t ScaleSample(int16_t sample, uint16_t volume) { + const int64_t scaled = FloorQ15(static_cast(sample) * volume); + return static_cast(std::clamp(scaled, -32768, 32767)); +} + +constexpr int16_t kSamples[] = {-32768, 32767, -32767, 16384, -16385, 1, -1, 0}; +constexpr int32_t kWords[] = {std::numeric_limits::min(), + std::numeric_limits::max(), -2147483647, 2147450880, + -65537, 65537, -1, 0, 1}; +constexpr uint16_t kGains[] = {0, 1, 0x3fff, 0x4000, 0x7fff, 0x8000, 0xfffe, 0xffff}; + +std::vector Counts() { + std::vector counts; + for (uint32_t count = 0; count <= 40; ++count) counts.push_back(count); + for (uint32_t count : {63u, 64u, 65u, 95u, 96u, 97u, 159u, 160u, 161u}) { + counts.push_back(count); + } + return counts; +} + +void TestRamps(const Backend& backend, uint32_t count, size_t offset, + uint16_t volume, uint16_t delta, unsigned pattern) { + std::snprintf(g_case, sizeof(g_case), "%s count=%u offset=%zu volume=%u delta=%u pattern=%u seed=%u", + backend.name, count, offset, unsigned(volume), unsigned(delta), pattern, g_random); + // Source ends at the allocation boundary so ASan also catches tail overreads. + std::vector input(offset + count, 12345); + std::vector initial(offset + count + 1, 0x13579bdf); + for (uint32_t i = 0; i < count; ++i) { + input[offset + i] = pattern == 0 ? kSamples[i % std::size(kSamples)] + : static_cast(static_cast(NextRandom() & 0xffffu) - 32768); + initial[offset + i] = pattern == 0 ? kWords[i % std::size(kWords)] + : Wrap32(NextRandom()); + } + const auto original = input; + auto expectedMix = initial; + auto expectedScale = input; + int16_t expectedDpop = -23456; + uint16_t expectedVolume = volume; + for (uint32_t i = 0; i < count; ++i) { + const int16_t scaled = ScaleSample(input[offset + i], expectedVolume); + expectedMix[offset + i] = Wrap32(static_cast(initial[offset + i]) + scaled); + expectedScale[offset + i] = scaled; + expectedDpop = scaled; + expectedVolume = static_cast(expectedVolume + delta); + } + auto mixed = initial; + int16_t dpop = -23456; + Require(backend.mix(mixed.data() + offset, input.data() + offset, count, + volume, delta, dpop) == expectedVolume, "mix final volume"); + Require(mixed == expectedMix, "mix output or sentinel"); + Require(dpop == expectedDpop, "mix de-pop sample"); + Require(input == original, "mix modified its source"); + + auto scaled = input; + Require(backend.scale(scaled.data() + offset, count, volume, delta) == expectedVolume, + "scale final volume"); + Require(scaled == expectedScale, "scale output or prefix"); +} + +void TestAccum(const Backend& backend, uint32_t count, size_t offset, unsigned pattern) { + std::snprintf(g_case, sizeof(g_case), "%s count=%u offset=%zu pattern=%u seed=%u", + backend.name, count, offset, pattern, g_random); + std::vector source(offset + count, 0x2468ace); + std::vector ramp(offset + count, 12345); + std::vector initial(offset + count + 1, 0x13579bdf); + for (uint32_t i = 0; i < count; ++i) { + source[offset + i] = pattern == 0 ? kWords[i % std::size(kWords)] : Wrap32(NextRandom()); + initial[offset + i] = pattern == 0 ? kWords[(i + 1) % std::size(kWords)] + : Wrap32(NextRandom()); + ramp[offset + i] = pattern == 0 ? kGains[i % std::size(kGains)] + : static_cast(NextRandom()); + } + const auto originalSource = source; + const auto originalRamp = ramp; + auto expected = initial; + for (uint32_t i = 0; i < count; ++i) { + expected[offset + i] = Wrap32(static_cast(initial[offset + i]) + + FloorQ15(static_cast(source[offset + i]) * ramp[offset + i])); + } + auto actual = initial; + backend.accum(actual.data() + offset, source.data() + offset, ramp.data() + offset, count); + Require(actual == expected, "aux accumulation or sentinel"); + Require(source == originalSource && ramp == originalRamp, "aux modified its source"); + + auto aliased = initial; + expected = initial; + for (uint32_t i = 0; i < count; ++i) { + expected[offset + i] = Wrap32(static_cast(initial[offset + i]) + + FloorQ15(static_cast(initial[offset + i]) * ramp[offset + i])); + } + backend.accum(aliased.data() + offset, aliased.data() + offset, ramp.data() + offset, count); + Require(aliased == expected, "aux in-place accumulation"); +} + +void TestAccumEdges(const Backend& backend) { + for (int32_t source : kWords) for (int32_t initial : kWords) for (uint16_t gain : kGains) { + std::snprintf(g_case, sizeof(g_case), "%s source=%d initial=%d gain=%u", + backend.name, source, initial, unsigned(gain)); + std::array src, dst; + std::array ramp; + src.fill(source); + dst.fill(initial); + ramp.fill(gain); + const int32_t expected = Wrap32(static_cast(initial) + + FloorQ15(static_cast(source) * gain)); + backend.accum(dst.data(), src.data(), ramp.data(), 17); + Require(std::all_of(dst.begin(), dst.end(), [expected](int32_t v) { return v == expected; }), + "aux full-width edge"); + } +} + +void TestEndian(const Backend& backend, size_t count, size_t offset) { + std::snprintf(g_case, sizeof(g_case), "%s count=%zu offset=%zu seed=%u", + backend.name, count, offset, g_random); + std::vector source(offset + count, 0x13579bdf); + for (size_t i = 0; i < count; ++i) { + source[offset + i] = i % 2 == 0 ? kWords[(i / 2) % std::size(kWords)] : Wrap32(NextRandom()); + } + const auto original = source; + std::vector expected(offset + count * 4 + 1, 0xa5); + for (size_t i = 0; i < count; ++i) { + const uint32_t word = static_cast(source[offset + i]); + for (unsigned byte = 0; byte < 4; ++byte) { + expected[offset + i * 4 + byte] = static_cast(word >> (24 - byte * 8)); + } + } + auto bytes = std::vector(expected.size(), 0xa5); + backend.store(bytes.data() + offset, source.data() + offset, count); + Require(bytes == expected, "big-endian store or sentinel"); + Require(source == original, "store modified its source"); + + // Drop the trailing sentinel before loading to expose overreads to ASan. + std::vector loadBytes(expected.begin(), expected.end() - 1); + std::vector loaded(offset + count + 1, 0x13579bdf); + auto expectedWords = loaded; + std::copy(source.begin() + offset, source.end(), expectedWords.begin() + offset); + backend.load(loaded.data() + offset, loadBytes.data() + offset, count); + Require(loaded == expectedWords, "big-endian load or sentinel"); + Require(std::equal(loadBytes.begin(), loadBytes.end(), expected.begin()), "load modified source"); +} + +void TestBackend(const Backend& backend) { + for (uint32_t count : Counts()) for (size_t offset : {1u, 2u, 3u}) { + for (uint16_t volume : kGains) for (uint16_t delta : kGains) { + TestRamps(backend, count, offset, volume, delta, 0); + } + for (unsigned pattern = 0; pattern < 8; ++pattern) TestAccum(backend, count, offset, pattern); + } + for (unsigned trial = 0; trial < 512; ++trial) { + const size_t count = NextRandom() % 162; + const size_t offset = 1 + NextRandom() % 3; + const auto volume = static_cast(NextRandom()); + const auto delta = static_cast(NextRandom()); + TestRamps(backend, count, offset, volume, delta, 1); + } + TestRamps(backend, 65539, 1, 0xffff, 0xffff, 0); + TestAccumEdges(backend); + for (size_t count = 0; count <= 161; ++count) for (size_t offset = 1; offset <= 15; ++offset) { + TestEndian(backend, count, offset); + } +} + +uint64_t Checksum(uint64_t hash, uint32_t value) { + return (hash ^ value) * 1099511628211ull; +} + +#if defined(_MSC_VER) +#define AX_TEST_NOINLINE __declspec(noinline) +#else +#define AX_TEST_NOINLINE __attribute__((noinline)) +#endif + +AX_TEST_NOINLINE uint64_t Benchmark(const Backend& backend, const std::string& kernel, + uint32_t count, uint32_t iterations) { + std::vector samples(count); + std::vector source(count), bus(count); + std::vector ramp(count); + std::vector bytes(count * 4 + 1); + for (uint32_t i = 0; i < count; ++i) { + samples[i] = kSamples[i % std::size(kSamples)]; + source[i] = kWords[i % std::size(kWords)]; + bus[i] = source[i]; + ramp[i] = kGains[i % std::size(kGains)]; + } + AxMixKernels::StoreBigEndian32Scalar(bytes.data() + 1, source.data(), count); + uint16_t volume = 0xfffe; + int16_t dpop = -12345; + const auto start = std::chrono::steady_clock::now(); + if (kernel == "mix") { + for (uint32_t i = 0; i < iterations; ++i) { + volume = backend.mix(bus.data(), samples.data(), count, volume, 0xffff, dpop); + } + } else if (kernel == "scale") { + for (uint32_t i = 0; i < iterations; ++i) { + samples[i % count] = kSamples[i % std::size(kSamples)]; + volume = backend.scale(samples.data(), count, volume, 0xffff); + } + } else if (kernel == "accum") { + for (uint32_t i = 0; i < iterations; ++i) { + backend.accum(bus.data(), source.data(), ramp.data(), count); + } + } else if (kernel == "store") { + for (uint32_t i = 0; i < iterations; ++i) { + source[i % count] = Wrap32(i); + backend.store(bytes.data() + 1, source.data(), count); + } + } else if (kernel == "load") { + for (uint32_t i = 0; i < iterations; ++i) { + bytes[1 + i % (count * 4)] = static_cast(i); + backend.load(bus.data(), bytes.data() + 1, count); + } + } + const auto elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - start).count(); + uint64_t hash = Checksum(Checksum(14695981039346656037ull, volume), static_cast(dpop)); + for (int32_t value : bus) hash = Checksum(hash, static_cast(value)); + for (int16_t value : samples) hash = Checksum(hash, static_cast(value)); + for (uint8_t value : bytes) hash = Checksum(hash, value); + std::printf("%s,%s,%u,%u,%.3f,%.3f,%016llx\n", backend.name, kernel.c_str(), count, + iterations, elapsed / iterations, elapsed / (static_cast(iterations) * count), + static_cast(hash)); + return hash; +} + +void RunBenchmarks(uint32_t iterations) { + std::puts("backend,kernel,samples,iterations,ns_per_block,ns_per_sample,checksum"); + for (uint32_t count : {6u, 18u, 32u, 96u, 160u}) { + for (const char* kernel : {"mix", "scale", "accum", "store", "load"}) { + const uint64_t scalar = Benchmark(kScalar, kernel, count, iterations); + const uint64_t selected = Benchmark(kSelected, kernel, count, iterations); + Require(scalar == selected, "benchmark checksum differs"); + } + } +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc >= 2 && std::string(argv[1]) == "--benchmark") { + uint32_t iterations = 200000; + if (argc == 3) { + const std::string argument = argv[2]; + const auto result = std::from_chars(argument.data(), argument.data() + argument.size(), iterations); + Require(result.ec == std::errc{} && result.ptr == argument.data() + argument.size() && + iterations > 0 && iterations <= 100000000, "invalid benchmark iterations"); + } + Require(argc <= 3, "usage: ax_mix_kernels_tests [--benchmark [iterations]]"); + RunBenchmarks(iterations); + } else { + Require(argc == 1, "usage: ax_mix_kernels_tests [--benchmark [iterations]]"); + g_random = 0x12345678u; + TestBackend(kScalar); + g_random = 0x12345678u; + TestBackend(kSelected); + std::printf("AX mix kernels (%s): passed\n", kSelected.name); + } + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/runtime/tests/gx_copy_tests.cpp b/runtime/tests/gx_copy_tests.cpp new file mode 100644 index 000000000..3b18cb063 --- /dev/null +++ b/runtime/tests/gx_copy_tests.cpp @@ -0,0 +1,271 @@ +#include "gx_test_common.hpp" +#include "../src/hle/gx/gx_internal.h" +#include "gx_sync_support/copy_observation.h" +#include "gx/readback.hpp" +#include "gx/texture.hpp" +#include "__gx.h" + +// Exercise the wrappers without registering guest ABI entry points in the host fixture. +#undef REGISTER_NATIVE_FUNCTION +#define REGISTER_NATIVE_FUNCTION(address, fn) +#include "../src/hle/gx/gx_copy.cpp" + +extern "C" void AuroraResetCopyProbesForTesting(); + +namespace aurora::gfx { +extern uint32_t g_testDrawCount; +namespace testing { +extern std::atomic resolvePassCount; +extern std::vector> readbackJobs; +extern std::vector readbackJobCopies; +std::vector copyObservations; + +void observe_copy(TextureHandle target, ClipRect rect, bool clearColor, bool clearAlpha, bool clearDepth, + GXTexFmt format, const ResolveOptions& options) { + copyObservations.push_back({std::move(target), rect, clearColor, clearAlpha, clearDepth, format, options, + g_testDrawCount}); +} +} // namespace testing +} // namespace aurora::gfx + +extern "C" { +const uint8_t kMkwGuestRegion = 0; +int g_gxFrameCount = 0; +} +TexCopyState g_texCopyState; + +namespace { +constexpr uint32_t kGuestBase = 0x80001000; +std::array g_guestBytes{}; +std::atomic g_drawDoneCallbacks{0}; +uint32_t g_frameJoins = 0; +uint32_t g_readyXfb = 0; +uint32_t g_presents = 0; +uint32_t g_overlayDraws = 0; + +void DrawDoneCallback() { ++g_drawDoneCallbacks; } +} // namespace + +void* GuestToHostPtr(uint32_t address, size_t length) { + const uint32_t canonical = CanonicalizeGxMainRamAddress(address); + const uint32_t base = CanonicalizeGxMainRamAddress(kGuestBase); + if (canonical < base || canonical - base >= g_guestBytes.size() || + length > g_guestBytes.size() - (canonical - base)) { + return nullptr; + } + return g_guestBytes.data() + (canonical - base); +} +void EnsureAuroraFrameActive() {} +void VI_HLE_SetXfbReady(uint32_t address) { g_readyXfb = address; } +void VI_HLE_PresentFrame(bool presentedXfb, bool paceToRetrace) { + EXPECT_TRUE(presentedXfb); + EXPECT_TRUE(paceToRetrace); + GxWorker::Sync(); + aurora::gx::fifo::drain(); + ++g_presents; +} +extern "C" void aurora_wait_for_frame_worker() { ++g_frameJoins; } +namespace settings_overlay { +void Draw() noexcept { ++g_overlayDraws; } +} // namespace settings_overlay + +namespace { +namespace stub = aurora::gfx::testing; +constexpr u32 kCopyWarmupFrames = 8; + +class GxCopyTest : public GXFifoTest, public ::testing::WithParamInterface { +protected: + void SetUp() override { + aurora::gx::fifo::init(); + GXFifoTest::SetUp(); + AuroraResetCopyProbesForTesting(); + AuroraSetGxProducerThread(); + g_texCopyState = {}; + g_gxFrameCount = 0; + g_drawDoneCallbacks = 0; + g_frameJoins = g_readyXfb = g_presents = g_overlayDraws = 0; + g_efbCopyDestinations.clear(); + g_efbCopyBuckets.clear(); + g_largestEfbCopyDestination = 0; + stub::copyObservations.clear(); + stub::resolvePassCount = 0; + stub::readbackJobs.clear(); + stub::readbackJobCopies.clear(); + aurora::gfx::g_testDrawCount = 0; + aurora::gfx::readback::set_supported(true); + GXSetDrawDoneCallback(DrawDoneCallback); + aurora::gx::fifo::begin_frame(); + if (GetParam()) { + GxWorker::Start(); + } + GxWorker::Run([] { + GXSetColorUpdate(GX_TRUE); + GXSetAlphaUpdate(GX_TRUE); + GXSetZMode(GX_TRUE, GX_ALWAYS, GX_TRUE); + GXSetPixelFmt(GX_PF_RGBA6_Z24, GX_ZC_LINEAR); + }); + } + void TearDown() override { + GxWorker::Stop(); + aurora::gx::fifo::drain(); + aurora::gx::fifo::end_frame(); + GXSetDrawDoneCallback(nullptr); + AuroraResetCopyProbesForTesting(); + aurora::gx::texture::shutdown(); + aurora::gfx::readback::shutdown(); + aurora::gfx::readback::set_supported(false); + GXFifoTest::TearDown(); + } + void Drain() { + GxWorker::Sync(); + AuroraGXSync(); + } + void Draw() { + GxWorker::Run([] { + GXClearVtxDesc(); + GXSetVtxDesc(GX_VA_POS, GX_DIRECT); + GXSetVtxAttrFmt(GX_VTXFMT0, GX_VA_POS, GX_POS_XYZ, GX_U8, 0); + GXBegin(GX_TRIANGLES, GX_VTXFMT0, 3); + GXPosition3u8(0, 1, 2); + GXPosition3u8(3, 4, 5); + GXPosition3u8(6, 7, 8); + GXEnd(); + }); + } + void Copy(uint32_t address, GXBool clear, GXTexFmt format = GX_TF_RGBA8, u16 width = 8, u16 height = 8) { + GX__SetTexCopySrc_8016f478(2, 4, width, height); + GX__SetTexCopyDst_8016f4dc(width, height, format, GX_FALSE); + GX__CopyTex_8016fd74(address, clear); + } + void WarmCopy(u16 width, u16 height) { + for (u32 frame = 0; frame <= kCopyWarmupFrames; ++frame) { + Copy(kGuestBase, GX_FALSE, GX_TF_RGBA8, width, height); + Drain(); + aurora::gx::texture::end_frame(); + } + } +}; + +TEST_P(GxCopyTest, InterleavedDrawsClearsAndReusedDestinationsNeedNoCompletionTokens) { + Draw(); + Copy(kGuestBase, GX_TRUE); + Draw(); + Copy(kGuestBase + 512, GX_FALSE); + Copy(kGuestBase, GX_FALSE); + Drain(); + ASSERT_EQ(stub::copyObservations.size(), 3u); + EXPECT_EQ(stub::copyObservations[0].draws, 1u); + EXPECT_EQ(stub::copyObservations[1].draws, 2u); + EXPECT_EQ(stub::copyObservations[2].draws, 2u); + const auto& first = stub::copyObservations[0]; + EXPECT_TRUE(first.clearColor); + EXPECT_TRUE(first.clearAlpha); + EXPECT_TRUE(first.clearDepth); + EXPECT_EQ(first.rect.x, 2); + EXPECT_EQ(first.rect.y, 4); + EXPECT_EQ(first.rect.width, 8); + EXPECT_EQ(first.rect.height, 8); + EXPECT_FALSE(stub::copyObservations[1].clearColor); + EXPECT_EQ(first.target, stub::copyObservations[2].target); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); + EXPECT_EQ(g_frameJoins, 0u); + EXPECT_EQ(g_efbCopyDestinations.size(), 2u); + GXDrawDone(); + EXPECT_EQ(g_drawDoneCallbacks.load(), 1u); +} + +TEST_P(GxCopyTest, ReadbackIsRecordedAfterItsCopyWithoutAnImplicitFence) { + Draw(); + Copy(kGuestBase, GX_FALSE); + const auto stream = AuroraReadbackCreateStream(); + ASSERT_NE(stream, 0u); + const AuroraReadbackCopySrc source{.key = 7, .src = GuestToHostPtr(kGuestBase)}; + ASSERT_TRUE(AuroraReadbackCaptureCopies(stream, &source, 1)); + Drain(); + ASSERT_EQ(stub::readbackJobs.size(), 1u); + ASSERT_EQ(stub::readbackJobCopies.size(), 1u); + ASSERT_EQ(stub::copyObservations.size(), 1u); + EXPECT_EQ(stub::readbackJobCopies[0], 1u); + ASSERT_EQ(stub::readbackJobs[0]->copies.size(), 1u); + EXPECT_EQ(stub::readbackJobs[0]->copies[0].texture, stub::copyObservations[0].target); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); + AuroraReadbackDestroyStream(stream); +} + +TEST_P(GxCopyTest, CopyStillFlushesUnpublishedDirtyState) { + GxWorker::Run([] { + GXClearVtxDesc(); + GXSetVtxDesc(GX_VA_POS, GX_DIRECT); + }); + GxWorker::Sync(); + ASSERT_NE(__gx->dirtyState, 0u); + Copy(kGuestBase, GX_FALSE); + EXPECT_EQ(__gx->dirtyState, 0u); + Drain(); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, InvalidationRetiresTheQueuedCopyBeforeLaterConsumers) { + Copy(kGuestBase, GX_FALSE); + Copy(kGuestBase + 512, GX_FALSE); + InvalidateEfbCopyDestinationsForRange(kGuestBase, 256); + Drain(); + EXPECT_FALSE(aurora::gx::g_gxState.copyTextures.contains(GuestToHostPtr(kGuestBase))); + EXPECT_TRUE(aurora::gx::g_gxState.copyTextures.contains(GuestToHostPtr(kGuestBase + 512))); + EXPECT_EQ(g_efbCopyDestinations.size(), 1u); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, LargeCopyBecomesNonpersistentAfterWarmup) { + WarmCopy(640, 480); + ASSERT_EQ(stub::copyObservations.size(), kCopyWarmupFrames + 1); + EXPECT_TRUE(stub::copyObservations[kCopyWarmupFrames - 1].options.persistent); + EXPECT_FALSE(stub::copyObservations.back().options.persistent); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, ChangingCopySizeStartsWithColdKey) { + WarmCopy(8, 8); + ASSERT_EQ(stub::copyObservations.size(), kCopyWarmupFrames + 1); + const auto& copies = aurora::gx::g_gxState.copyTextures; + const auto oldCopy = copies.find(GuestToHostPtr(kGuestBase)); + ASSERT_NE(oldCopy, copies.end()); + ASSERT_EQ(oldCopy->second.consecutiveFrames, kCopyWarmupFrames); + EXPECT_TRUE(stub::copyObservations.back().options.persistent); + Copy(kGuestBase, GX_FALSE, GX_TF_RGBA8, 640, 480); + Drain(); + ASSERT_EQ(stub::copyObservations.size(), kCopyWarmupFrames + 2); + EXPECT_TRUE(stub::copyObservations.back().options.persistent); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, DepthProbeStaysNonpersistentWithoutTheWait) { + const auto probe = kGuestBase + (GetParam() ? 2048u : 3072u); + Copy(probe, GX_FALSE, GX_TF_Z8, 4, 4); + Drain(); + ASSERT_EQ(stub::copyObservations.size(), 1u); + EXPECT_FALSE(stub::copyObservations.back().options.persistent); + EXPECT_FALSE(stub::readbackJobs.empty()); + EXPECT_EQ(g_guestBytes[probe - kGuestBase], 0xff); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +TEST_P(GxCopyTest, DisplayCopyRetainsPresentationAndInlineRenderWorkerJoin) { + Draw(); + GxWorker::Run([] { + GXSetDispCopySrc(0, 0, 640, 480); + GXSetDispCopyDst(640, 480); + }); + GX__CopyDisp_8016fc38(kGuestBase, GX_TRUE); + ASSERT_EQ(stub::copyObservations.size(), 1u); + EXPECT_EQ(stub::copyObservations.front().draws, 1u); + EXPECT_EQ(g_readyXfb, kGuestBase); + EXPECT_EQ(g_presents, 1u); + EXPECT_EQ(g_overlayDraws, 1u); + EXPECT_EQ(g_gxFrameCount, 1); + EXPECT_EQ(g_frameJoins, GetParam() ? 0u : 1u); + EXPECT_EQ(g_drawDoneCallbacks.load(), 0u); +} + +INSTANTIATE_TEST_SUITE_P(WorkerDisabledAndEnabled, GxCopyTest, ::testing::Bool()); +} // namespace diff --git a/runtime/tests/gx_direct_draw_tests.cpp b/runtime/tests/gx_direct_draw_tests.cpp new file mode 100644 index 000000000..259a64993 --- /dev/null +++ b/runtime/tests/gx_direct_draw_tests.cpp @@ -0,0 +1,304 @@ +#include "gx_test_common.hpp" +#include "../src/hle/gx/gx_internal.h" +#include "../src/hle/gx/gx_stream_common.h" +#include "isa/big_endian.h" + +#include +#include +#include + +namespace { +class RuntimeDirectDrawTest : public GXFifoTest { +protected: + void SetUp() override { + GXFifoTest::SetUp(); + reset_runtime(); + } + + void reset_runtime() { + GXInit(nullptr, 0); + aurora::gx::fifo::clear_buffer(); + g_hleGxState = {}; + g_dlRecordState = {}; + g_alphaCompareValid = false; + GxWorker::detail::g_active = false; + GxWorker::detail::t_onWorker = false; + } + + void position(GXCompCnt cnt = GX_POS_XYZ, GXCompType type = GX_U8, u8 frac = 0) { + g_hleGxState.vtxDesc[GX_VA_POS] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[GX_VTXFMT0][GX_VA_POS] = {cnt, type, frac}; + g_hleGxState.InvalidateVtxLayoutHash(); + } + + void begin(u16 count, bool direct = true) { + auto& g = g_hleGxState; + GxStream::PublishAuroraVtxState(GX_VTXFMT0, {false, GX_VA_POS, GX_VA_TEX7}); + g.currentVtxFmt = GX_VTXFMT0; + g.currentPrim = GX_TRIANGLES; + g.vertsRemaining = count; + g.inBegin = count != 0; + g.auroraBeginCalled = false; + g.fifoReadOffset = 0; + g.fifoByteCount = 0; + g.fifoPendingDrawBytes = 0; + g.ResetVertex(); + GxStream::PrepareImmediateDirectDraw(); + if (!direct) { + g.fifoDirectBytesRemaining = 0; + } + } + + void write(const std::vector& bytes, u32 width) { + size_t pos = 0; + for (; pos + width <= bytes.size(); pos += width) { + const u32 value = width == 4 ? BigEndian::Read32(bytes.data() + pos) + : width == 2 ? BigEndian::Read16(bytes.data() + pos) : bytes[pos]; + const auto remaining = g_hleGxState.fifoDirectBytesRemaining; + HleFifoWrite(value, width); + if (pos == 0 && remaining >= width) { + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, remaining - width); + } + } + for (; pos < bytes.size(); ++pos) { + HleFifoWrite(bytes[pos], 1); + } + } +}; + +TEST_F(RuntimeDirectDrawTest, DirectNumericPayloadMatchesIncrementalForEveryWriteWidth) { + for (GXCompType type : {GX_U8, GX_S8, GX_U16, GX_S16, GX_F32}) { + for (GXCompCnt cnt : {GX_POS_XY, GX_POS_XYZ}) { + for (u32 width : {1u, 2u, 4u}) { + const u32 bytesPerVertex = (cnt == GX_POS_XY ? 2u : 3u) * GetCompSizeBytes(type); + std::vector payload(bytesPerVertex * 3); + if (type == GX_F32) { + for (size_t i = 0; i < payload.size() / sizeof(float); ++i) { + const float value = static_cast(i + 1) / 7.0f; + BigEndian::WriteFloat32(payload.data(), i * sizeof(float), value); + } + } else { + for (size_t i = 0; i < payload.size(); ++i) { + payload[i] = static_cast(i * 17u + 11u); + } + } + reset_runtime(); + position(cnt, type, 7); + begin(3, false); + write(payload, width); + const auto reference = capture_fifo(); + reset_runtime(); + position(cnt, type, 7); + begin(3); + ASSERT_EQ(g_hleGxState.fifoDirectBytesRemaining, payload.size()); + write(payload, width); + EXPECT_EQ(capture_fifo(), reference) << "type=" << type << " count=" << cnt << " width=" << width; + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_EQ(g_hleGxState.vertsRemaining, 0u); + } + } + } +} + +TEST_F(RuntimeDirectDrawTest, MixedMatrixColorsAndAllTextureCoordinatesMatchIncremental) { + constexpr u32 bytesPerVertex = 45; + std::vector payload(bytesPerVertex * 3); + std::iota(payload.begin(), payload.end(), u8{64}); + for (u8 vertex = 0; vertex < 3; ++vertex) { + payload[vertex * bytesPerVertex] = vertex * 6; + payload[vertex * bytesPerVertex + 1] = vertex * 6 + 3; + } + std::vector reference; + for (bool direct : {false, true}) { + reset_runtime(); + position(GX_POS_XY, GX_S16, 11); + auto& g = g_hleGxState; + g.vtxDesc[GX_VA_PNMTXIDX] = GX_DIRECT; + g.vtxDesc[GX_VA_TEX7MTXIDX] = GX_DIRECT; + g.vtxDesc[GX_VA_CLR0] = GX_DIRECT; + g.vtxAttrFmt[0][GX_VA_CLR0] = {GX_CLR_RGB, GX_RGB8, 0}; + g.vtxDesc[GX_VA_CLR1] = GX_DIRECT; + g.vtxAttrFmt[0][GX_VA_CLR1] = {GX_CLR_RGBA, GX_RGBA8, 0}; + for (int tex = GX_VA_TEX0; tex <= GX_VA_TEX7; ++tex) { + g.vtxDesc[tex] = GX_DIRECT; + g.vtxAttrFmt[0][tex] = {GX_TEX_ST, GX_S16, static_cast(tex)}; + } + begin(3, direct); + ASSERT_EQ(g.fifoDirectBytesRemaining, direct ? payload.size() : 0u); + GX_HLE_FIFO_WriteBurst(payload.data(), static_cast(payload.size())); + if (!direct) { + reference = capture_fifo(); + } else { + EXPECT_EQ(capture_fifo(), reference); + } + } +} + +TEST_F(RuntimeDirectDrawTest, NormalAliasesIndexedAndReencodedColorsKeepFallback) { + for (GXCompCnt cnt : {GX_NRM_XYZ, GX_NRM_NBT, GX_NRM_NBT3}) { + reset_runtime(); + position(); + g_hleGxState.vtxDesc[GX_VA_NRM] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[0][GX_VA_NRM] = {cnt, GX_S16, 14}; + begin(3); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } + for (GXCompType type : {GX_RGB565, GX_RGBX8, GX_RGBA4, GX_RGBA6}) { + reset_runtime(); + position(); + g_hleGxState.vtxDesc[GX_VA_CLR0] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[0][GX_VA_CLR0] = {GX_CLR_RGBA, type, 0}; + begin(3); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } + for (GXAttrType type : {GX_INDEX8, GX_INDEX16}) { + reset_runtime(); + position(); + g_hleGxState.vtxDesc[GX_VA_POS] = type; + begin(3); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } +} + +TEST_F(RuntimeDirectDrawTest, CompletingWordAndBurstTailAreParsedImmediately) { + for (bool burst : {false, true}) { + reset_runtime(); + position(); + begin(1); + const std::vector stream{0x11, 0x22, 0x33, 0x48}; + if (burst) { + GX_HLE_FIFO_WriteBurst(stream.data(), static_cast(stream.size())); + } else { + HleFifoWrite(0x11223348, 4); + } + const auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + EXPECT_EQ(bytes.back(), 0x48); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + } +} + +TEST_F(RuntimeDirectDrawTest, RawPacketsReserveCompletingWordTailAtBufferBoundary) { + for (u16 packetBytes : {4093, 4094, 4095, 4096}) { + reset_runtime(); + g_hleGxState.vtxDesc[GX_VA_PNMTXIDX] = GX_DIRECT; + const u16 count = packetBytes - 3; + HleFifoWrite(0x90, 1); + HleFifoWrite(count, 2); + EXPECT_EQ(g_hleGxState.fifoPendingDrawBytes, packetBytes == 4093 ? packetBytes : 0u); + for (u32 i = 0; i < count - 1u; ++i) { + HleFifoWrite(0x12, 1); + } + HleFifoWrite(0x12480000, 4); + const auto bytes = capture_fifo(); + ASSERT_GE(bytes.size(), count + 4u); + const auto drawStart = bytes.size() - count - 4u; + EXPECT_EQ(bytes[drawStart], 0x90); + EXPECT_EQ(BigEndian::Read16(bytes.data() + drawStart + 1u), count); + for (size_t i = drawStart + 3u; i < drawStart + count + 3u; ++i) { + ASSERT_EQ(bytes[i], 0x12) << "packet=" << packetBytes; + } + EXPECT_EQ(bytes.back(), 0x48); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + EXPECT_EQ(g_hleGxState.fifoPendingDrawBytes, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); + } +} + +TEST_F(RuntimeDirectDrawTest, PendingRawLengthIsRecomputedAfterLayoutInvalidation) { + position(); + HleFifoWrite(0x90, 1); + HleFifoWrite(1, 2); + ASSERT_EQ(g_hleGxState.fifoPendingDrawBytes, 6u); + position(GX_POS_XY); + HleFifoWrite(0x1122, 2); + EXPECT_EQ(g_hleGxState.fifoPendingDrawBytes, 0u); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); +} + +TEST_F(RuntimeDirectDrawTest, CompletingWordCanStartPartialCpPacket) { + position(GX_POS_XY); + begin(1); + HleFifoWrite(0x11220850, 4); + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_EQ(g_hleGxState.fifoByteCount, 2u); + HleFifoWrite(3u << 9, 4); + EXPECT_EQ(g_hleGxState.vtxDesc[GX_VA_POS], GX_INDEX16); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); +} + +TEST_F(RuntimeDirectDrawTest, LayoutChangeBeforeFirstPayloadRechecksDirectEligibility) { + position(); + begin(1); + g_hleGxState.vtxDesc[GX_VA_NRM] = GX_DIRECT; + g_hleGxState.vtxAttrFmt[0][GX_VA_NRM] = {GX_NRM_XYZ, GX_S16, 14}; + g_hleGxState.InvalidateVtxLayoutHash(); + HleFifoWrite(0x11, 1); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + EXPECT_TRUE(g_hleGxState.auroraBeginCalled); + HleFifoWrite(0x2233, 2); + HleFifoWrite(0x44556677, 4); + HleFifoWrite(0x8899, 2); + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_FALSE(g_hleGxState.auroraBeginCalled); +} + +TEST_F(RuntimeDirectDrawTest, DirectPayloadLargerThanQueueStillMatchesIncremental) { + std::vector reference; + const std::vector payload(4096u * 3u, 0x32); + for (bool direct : {false, true}) { + reset_runtime(); + position(); + begin(4096, direct); + GX_HLE_FIFO_WriteBurst(payload.data(), static_cast(payload.size())); + EXPECT_FALSE(g_hleGxState.inBegin); + EXPECT_EQ(g_hleGxState.fifoByteCount, 0u); + if (!direct) { + reference = capture_fifo(); + } else { + EXPECT_EQ(capture_fifo(), reference); + } + } +} + +TEST_F(RuntimeDirectDrawTest, RecordingBypassesPreparedDirectPayload) { + position(); + begin(1); + g_dlRecordState = {0x80001000, 32, 0x80001000, 0, true}; + HleFifoWrite(0x11223344, 4); + EXPECT_EQ(g_dlRecordState.count, 4u); + EXPECT_EQ(Memory::Read32(0x80001000), 0x11223344u); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 3u); + EXPECT_TRUE(capture_fifo().empty()); + g_dlRecordState.active = false; +} + +TEST_F(RuntimeDirectDrawTest, ZeroCountDoesNotPrepareDirectPayload) { + position(); + begin(0); + EXPECT_EQ(g_hleGxState.fifoDirectBytesRemaining, 0u); + EXPECT_FALSE(g_hleGxState.inBegin); +} + +TEST_F(RuntimeDirectDrawTest, DISABLED_SubmissionBenchmark) { + constexpr u32 draws = 20000; + const std::vector payload(3u * 4u, 0x32); + for (bool direct : {false, true, true, false}) { + reset_runtime(); + position(); + const auto start = std::chrono::steady_clock::now(); + for (u32 i = 0; i < draws; ++i) { + begin(4, direct); + write(payload, 4); + } + const auto end = std::chrono::steady_clock::now(); + const auto bytes = capture_fifo(); + std::printf("direct=%d draws=%u elapsed_us=%lld fifo_bytes=%zu\n", direct, draws, + static_cast(std::chrono::duration_cast(end - start).count()), + bytes.size()); + } +} +} // namespace diff --git a/runtime/tests/gx_dl_cache_tests.cpp b/runtime/tests/gx_dl_cache_tests.cpp new file mode 100644 index 000000000..13def52b6 --- /dev/null +++ b/runtime/tests/gx_dl_cache_tests.cpp @@ -0,0 +1,401 @@ +#include "../src/hle/gx/gx_dl_cache.h" + +#include +#include +#include +#include +#include +#include +#include + +extern "C" const uint8_t kMkwGuestRegion = 0; + +namespace { +thread_local int g_allocationsBeforeFailure = -1; +} + +void* operator new(size_t size) { + if (g_allocationsBeforeFailure == 0) { + g_allocationsBeforeFailure = -1; + throw std::bad_alloc{}; + } + if (g_allocationsBeforeFailure > 0) --g_allocationsBeforeFailure; + if (auto* memory = std::malloc(size == 0 ? 1 : size)) return memory; + throw std::bad_alloc{}; +} + +void operator delete(void* memory) noexcept { std::free(memory); } +void operator delete(void* memory, size_t) noexcept { std::free(memory); } + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +struct Record { + uint32_t listAddr = 0; + uint32_t nbytes = 0; + uint64_t layoutHash = 0; + uint64_t contentDigest = 0; + uint64_t writeGeneration = GxGuestWrite::kUntracked; + std::vector cpWrites; + bool streamIsSource = false; + std::vector stream; + + size_t Bytes() const { return stream.size() + cpWrites.size() * sizeof(uint64_t); } +}; + +using Cache = GxDisplayListScanCache::BoundedCache; +using GxDisplayListScanCache::DisplayListBytes; +using GxDisplayListScanCache::MakeIdentityKey; +using GxDisplayListScanCache::Probe; + +bool Store(Cache& cache, uint64_t key, size_t flattenedBytes, size_t cpWrites = 0) { + Record record{}; + record.cpWrites.resize(cpWrites, key); + record.stream.resize(flattenedBytes, static_cast(key)); + const size_t bytes = record.Bytes(); + return cache.Insert(key, std::move(record), bytes); +} + +void EntryBudgetAndRecency() { + Cache cache{3, 1024}; + for (uint64_t key = 1; key <= 3; ++key) Require(Store(cache, key, 0), "initial insert failed"); + auto* hot = cache.Find(1); + cache.Touch(1); + Require(Store(cache, 4, 0), "entry-budget insert failed"); + Require(cache.Size() == 3 && cache.StoredBytes() == 0, "entry budget exceeded"); + Require(cache.Find(1) == hot && cache.Find(2) == nullptr && cache.Find(3) && cache.Find(4), + "entry-budget eviction discarded the hot working set"); + cache.Find(3); + cache.Touch(999); + Require(Store(cache, 5, 0), "second entry-budget insert failed"); + Require(!cache.Find(3) && cache.Find(1), "unvalidated probe refreshed recency"); +} + +void ByteBudgetAndReplacement() { + Cache cache{3, 12}; + Require(Store(cache, 1, 4) && Store(cache, 2, 4) && Store(cache, 3, 4), "byte-budget setup failed"); + auto* retained = cache.Find(3); + Require(Store(cache, 1, 8), "protected replacement failed"); + Require(cache.Size() == 2 && cache.StoredBytes() == 12, "replacement byte accounting failed"); + Require(cache.Find(1) && !cache.Find(2) && cache.Find(3) == retained, + "oldest replacement was evicted or unrelated record moved"); + Require(cache.Find(1)->stream == std::vector(8, 1), "replacement payload was lost"); + Require(Store(cache, 4, 4), "byte-budget eviction failed"); + Require(cache.Size() == 2 && cache.StoredBytes() == 12 && !cache.Find(3) && cache.Find(1), + "replacement was not promoted to most recent"); + Require(Store(cache, 1, 0), "shrinking replacement failed"); + Require(cache.Size() == 2 && cache.StoredBytes() == 4, "shrinking replacement leaked bytes"); + Require(Store(cache, 5, 0, 1), "CP-write insertion failed"); + Require(cache.Size() == 3 && cache.StoredBytes() == 12, "CP-write bytes were not counted"); + Require(Store(cache, 5, 4), "CP-write replacement failed"); + Require(cache.StoredBytes() == 8, "CP-write replacement counted old bytes twice"); +} + +void BytePressureSkipsEmptyRecords() { + Cache cache{514, 8}; + for (uint64_t key = 1; key <= 512; ++key) Require(Store(cache, key, 0), "empty record setup failed"); + Require(Store(cache, 513, 4) && Store(cache, 514, 4), "byte record setup failed"); + cache.Touch(513); + Require(Store(cache, 515, 4), "byte pressure starved behind empty records"); + Require(cache.Size() == 514 && cache.StoredBytes() == 8 && cache.Find(513) && !cache.Find(514), + "combined pressure failed to pick the oldest byte-bearing entry"); + for (uint64_t key = 1; key <= 512; ++key) Require(cache.Find(key), "byte pressure evicted an empty record"); + + const auto* promoted = cache.Find(512); + Require(Store(cache, 512, 4) && cache.Find(512) == promoted && !cache.Find(513), + "zero-to-positive replacement failed to join the byte order"); + Require(Store(cache, 515, 0) && Store(cache, 516, 8) && !cache.Find(512) && cache.Find(515), + "byte pressure evicted a positive-to-zero replacement"); + Require(Store(cache, 517, 0) && Store(cache, 518, 0) && !cache.Find(1) && cache.Find(516), + "entry pressure failed to pick the overall oldest entry"); +} + +void AllocationFailurePreservesVictims() { + for (const int failAfter : {0, 1}) { + Cache cache{2, 8}; + Require(Store(cache, 1, 4) && Store(cache, 2, 4), "allocation failure setup failed"); + auto* first = cache.Find(1); + auto* second = cache.Find(2); + Record record{}; + record.stream.resize(4); + bool failed = false; + g_allocationsBeforeFailure = failAfter; + try { + cache.Insert(3, std::move(record), 4); + } catch (const std::bad_alloc&) { + failed = true; + } + g_allocationsBeforeFailure = -1; + Require(failed, "cache insertion did not hit the allocation failure"); + Require(cache.Size() == 2 && cache.StoredBytes() == 8 && cache.Find(1) == first && + cache.Find(2) == second && !cache.Find(3), "allocation failure changed the working set"); + Require(Store(cache, 4, 4) && !cache.Find(1) && cache.Find(2) == second, + "allocation failure changed byte recency"); + } +} + +void RejectedInsertsPreserveRecords() { + Cache cache{2, 12}; + Require(Store(cache, 1, 8) && Store(cache, 2, 4), "oversize setup failed"); + const auto* old = cache.Find(1); + Require(!Store(cache, 3, 13) && !Store(cache, 1, 13), "oversized record was admitted"); + Require(!cache.Insert(1, Record{}, std::numeric_limits::max()), "overflowing size was admitted"); + Require(cache.Size() == 2 && cache.StoredBytes() == 12 && cache.Find(1) == old && cache.Find(2), + "rejected insertion changed the working set"); + Require(Store(cache, 3, 4) && !cache.Find(1) && cache.Find(2), "rejected insertion changed recency"); + + Cache disabled{0, 12}; + Require(!Store(disabled, 1, 0) && disabled.Size() == 0, "disabled entry budget accepted a record"); + Cache noBytes{1, 0}; + Require(Store(noBytes, 1, 0) && !Store(noBytes, 1, 1) && noBytes.Find(1), + "zero byte budget mishandled an empty record"); +} + +void EvictionLimitAndProtection() { + Cache cache{256, 128}; + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "bounded setup failed"); + const auto* protectedRecord = cache.Find(1); + Require(Store(cache, 1, 65), "replacement needing exactly 64 victims failed"); + Require(cache.Size() == 64 && cache.StoredBytes() == 128 && cache.Find(1) == protectedRecord, + "protected replacement was counted as a victim"); + for (uint64_t key = 2; key <= 65; ++key) Require(!cache.Find(key), "wrong LRU victim selected"); + for (uint64_t key = 66; key <= 128; ++key) Require(cache.Find(key), "too many records evicted"); + + cache.Clear(); + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "bounded refill failed"); + Require(Store(cache, 129, 64), "new insertion needing exactly 64 victims failed"); + Require(cache.Size() == 65 && cache.StoredBytes() == 128 && !cache.Find(64) && cache.Find(65), + "new insertion evicted the wrong count"); +} + +void BoundedRejectionsMakeProgress() { + Cache cache{256, 128}; + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "rejection setup failed"); + Require(!Store(cache, 129, 65), "new insertion exceeded 64 victims"); + Require(cache.Size() == 64 && cache.StoredBytes() == 64 && !cache.Find(64) && cache.Find(65), + "rejected insertion did not stop at 64 victims"); + Require(Store(cache, 129, 65) && cache.Size() == 64 && cache.StoredBytes() == 128, + "rejected insertion did not make progress for the next attempt"); + + cache.Clear(); + for (uint64_t key = 1; key <= 128; ++key) Require(Store(cache, key, 1), "protected rejection setup failed"); + const auto* protectedRecord = cache.Find(1); + Require(!Store(cache, 1, 66), "replacement exceeded 64 victims"); + Require(cache.Size() == 64 && cache.StoredBytes() == 64 && cache.Find(1) == protectedRecord && + protectedRecord->stream.size() == 1, "bounded rejection lost the protected replacement"); + Require(Store(cache, 1, 66) && cache.Size() == 63 && cache.StoredBytes() == 128, + "replacement did not make progress for the next attempt"); +} + +void ClearAndPointerLifetime() { + Cache cache{8192, 8192}; + Require(Store(cache, 0, 1), "pointer setup failed"); + const auto* first = cache.Find(0); + const auto* payload = first->stream.data(); + for (uint64_t key = 1; key < 8192; ++key) Require(Store(cache, key, 1), "growth insert failed"); + cache.Touch(0); + for (uint64_t key = 8192; key < 16384; ++key) { + Require(Store(cache, key, 1), "churn insert failed"); + Require(cache.Find(0) == first && first->stream.data() == payload, + "rehash or unrelated eviction invalidated a retained stream"); + cache.Touch(0); + } + Require(cache.Size() == 8192 && cache.StoredBytes() == 8192, "churn exceeded a budget"); + cache.Clear(); + cache.Clear(); + Require(cache.Size() == 0 && cache.StoredBytes() == 0 && !cache.Find(0), "clear left stale accounting"); + Require(Store(cache, 0, 0), "insertion after clear failed"); +} + +void LiveBytesStillInvalidateRetainedRecords() { + constexpr uint32_t address = 0x80010000; + constexpr uint32_t alias = 0xC0010000; + constexpr uint64_t layout = 0x12345678; + std::array bytes{0x08, 0x50, 0, 0, 0, 1, 0, 0}; + const auto key = MakeIdentityKey(address, bytes.size(), layout); + Cache cache{2, 32}; + Record record{}; + record.listAddr = CanonicalizeGxMainRamAddress(address); + record.nbytes = bytes.size(); + record.layoutHash = layout; + record.contentDigest = XXH3_64bits(bytes.data(), bytes.size()); + record.writeGeneration = GxGuestWrite::GenerationForRange(address, bytes.size()); + record.stream = {0}; + Require(cache.Insert(key, std::move(record), 1), "validation setup failed"); + auto* retained = cache.Find(key); + const auto probe = [&](uint32_t addr, uint32_t count, uint64_t currentLayout) { + const auto generation = GxGuestWrite::GenerationForRange(addr, count); + DisplayListBytes current{bytes.data(), count}; + return Probe(cache, current, addr, count, currentLayout, generation); + }; + Require(probe(alias, bytes.size(), layout) == retained, "unchanged alias was rejected"); + Require(!probe(address, bytes.size() - 1, layout) && !probe(address, bytes.size(), layout + 1) && + !probe(address + 32, bytes.size(), layout), "identity mismatch was reused"); + GxGuestWrite::NotifyWrite(alias, bytes.size()); + Require(Store(cache, 1, 16) && probe(address, bytes.size(), layout) == retained, + "unchanged bytes after a write were rejected"); + const auto validatedGeneration = GxGuestWrite::GenerationForRange(address, bytes.size()); + Require(retained->writeGeneration == validatedGeneration, "validated hit did not refresh the generation"); + Require(Store(cache, 2, 16) && cache.Find(key) == retained && !cache.Find(1), + "unrelated eviction discarded the validated record"); + bytes[5] ^= 0x80; + GxGuestWrite::NotifyWrite(alias, bytes.size()); + Require(!probe(address, bytes.size(), layout) && retained->writeGeneration == validatedGeneration, + "changed live commands reused a retained record"); + + constexpr uint32_t untrackedAddress = 0xF0000000; + Record untracked = *retained; + untracked.listAddr = untrackedAddress; + untracked.writeGeneration = GxGuestWrite::kUntracked; + Require(cache.Insert(MakeIdentityKey(untrackedAddress, bytes.size(), layout), std::move(untracked), 1), + "untracked validation setup failed"); + Require(!probe(untrackedAddress, bytes.size(), layout), "untracked writes bypassed digest validation"); +} + +void RawProbeAndScanUseTheSameCapture() { + constexpr uint32_t address = 0x80020000; + constexpr uint64_t layout = 0x12345678; + constexpr std::array original{0x90, 0, 1, 7}; + constexpr uint32_t size = original.size(); + const auto key = MakeIdentityKey(address, size, layout); + const auto generation = GxGuestWrite::GenerationForRange(address, size); + auto guest = original; + Cache cache{2, 32}; + { + DisplayListBytes bytes{guest.data(), size}; + Require(!Probe(cache, bytes, address, size, layout, generation), "empty cache hit"); + const auto* scanned = bytes.Data(); + guest.back() = 42; + Record record{}; + record.listAddr = CanonicalizeGxMainRamAddress(address); + record.nbytes = size; + record.layoutHash = layout; + record.writeGeneration = generation; + record.contentDigest = bytes.Digest(); + Require(scanned[3] == 7 && record.contentDigest == XXH3_64bits(original.data(), size), + "miss digest did not describe the captured bytes"); + record.streamIsSource = true; + record.stream.assign(scanned, scanned + size); + Require(cache.Insert(key, std::move(record), size), "raw validation setup failed"); + } + { + DisplayListBytes bytes{guest.data(), size}; + Require(!Probe(cache, bytes, address, size, layout, generation), + "unchanged generation bypassed raw-byte validation"); + const auto* rescanned = bytes.Data(); + guest.back() = 7; + Require(rescanned[3] == 42 && bytes.Digest() != cache.Find(key)->contentDigest, + "stale-hit scan did not retain the probed bytes"); + } + { + DisplayListBytes bytes{guest.data(), size}; + const auto* hit = Probe(cache, bytes, address, size, layout, generation); + Require(hit != nullptr, "unchanged raw bytes missed"); + guest.back() = 42; + Require(hit->stream[3] == 7, "raw hit submitted bytes different from the probe"); + } +} + +struct LegacyCache { + std::unordered_map entries; + size_t storedBytes = 0; + size_t maxEntries; + size_t maxBytes; + + Record* Find(uint64_t key) { + const auto it = entries.find(key); + return it == entries.end() ? nullptr : &it->second; + } + void Touch(uint64_t) {} + size_t Size() const { return entries.size(); } + bool Insert(uint64_t key, Record&& value, size_t bytes) { + const auto it = entries.find(key); + const bool replacing = it != entries.end(); + size_t replacedBytes = replacing ? it->second.Bytes() : 0; + if ((!replacing && entries.size() >= maxEntries) || storedBytes - replacedBytes + bytes > maxBytes) { + entries.clear(); + storedBytes = 0; + replacedBytes = 0; + } + entries.insert_or_assign(key, std::move(value)); + storedBytes = storedBytes - replacedBytes + bytes; + return true; + } +}; + +template +void Benchmark(Storage& cache, std::string_view policy, std::string_view scenario, + size_t hotCount, size_t coldCount, size_t bytesPerEntry) { + using Clock = std::chrono::steady_clock; + uint64_t hits = 0, misses = 0, evictions = 0, skips = 0; + std::vector batches; + const auto access = [&](uint64_t key) { + if (cache.Find(key)) { + ++hits; + cache.Touch(key); + } else { + ++misses; + Record record{}; + record.stream.resize(bytesPerEntry); + const auto before = cache.Size(); + const bool inserted = cache.Insert(key, std::move(record), bytesPerEntry); + skips += !inserted; + evictions += before + (inserted ? 1u : 0u) - cache.Size(); + } + }; + for (uint64_t key = 0; key < hotCount; ++key) access(key); + for (uint64_t round = 0; round < 256; ++round) { + const auto begin = Clock::now(); + for (uint64_t key = 0; key < hotCount; ++key) access(key); + for (uint64_t key = 0; key < coldCount; ++key) access(hotCount + round * coldCount + key); + batches.push_back(std::chrono::duration(Clock::now() - begin).count()); + } + std::sort(batches.begin(), batches.end()); + const auto percentile = [&](size_t percent) { return batches[(batches.size() - 1) * percent / 100]; }; + std::cout << policy << ',' << scenario << ',' << hits << ',' << misses << ',' << evictions << ',' << skips + << ',' << percentile(50) << ',' << percentile(95) << ',' << percentile(99) << '\n'; +} + +void Benchmarks() { + constexpr size_t maxBytes = 8u * 1024u * 1024u; + std::cout << "policy,scenario,hits,misses,evictions,skips,batch_p50_us,batch_p95_us,batch_p99_us\n"; + for (const bool bytePressure : {false, true}) { + const size_t hotCount = bytePressure ? 256 : 4096; + const size_t coldCount = bytePressure ? 64 : 1024; + const size_t bytesPerEntry = bytePressure ? 16384 : 64; + const auto scenario = bytePressure ? "bytes" : "entries"; + Cache cache{8192, maxBytes}; + Benchmark(cache, "incremental", scenario, hotCount, coldCount, bytesPerEntry); + LegacyCache legacy{{}, 0, 8192, maxBytes}; + Benchmark(legacy, "whole-cache", scenario, hotCount, coldCount, bytesPerEntry); + } +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string_view(argv[1]) == "--benchmark") { + Benchmarks(); + } else { + EntryBudgetAndRecency(); + ByteBudgetAndReplacement(); + BytePressureSkipsEmptyRecords(); + AllocationFailurePreservesVictims(); + RejectedInsertsPreserveRecords(); + EvictionLimitAndProtection(); + BoundedRejectionsMakeProgress(); + ClearAndPointerLifetime(); + LiveBytesStillInvalidateRetainedRecords(); + RawProbeAndScanUseTheSameCapture(); + std::cout << "display-list scan cache tests passed\n"; + } + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/runtime/tests/gx_sync_support/copy_observation.h b/runtime/tests/gx_sync_support/copy_observation.h new file mode 100644 index 000000000..c54d56531 --- /dev/null +++ b/runtime/tests/gx_sync_support/copy_observation.h @@ -0,0 +1,22 @@ +#pragma once + +#include "gfx/recording.hpp" + +#include + +namespace aurora::gfx::testing { +struct CopyObservation { + TextureHandle target; + ClipRect rect; + bool clearColor; + bool clearAlpha; + bool clearDepth; + GXTexFmt format; + ResolveOptions options; + uint32_t draws; +}; + +extern std::vector copyObservations; +void observe_copy(TextureHandle target, ClipRect rect, bool clearColor, bool clearAlpha, bool clearDepth, + GXTexFmt format, const ResolveOptions& options); +} // namespace aurora::gfx::testing diff --git a/runtime/tests/gx_texture_binding_support/runtime_seams.h b/runtime/tests/gx_texture_binding_support/runtime_seams.h new file mode 100644 index 000000000..694e73751 --- /dev/null +++ b/runtime/tests/gx_texture_binding_support/runtime_seams.h @@ -0,0 +1,104 @@ +#pragma once + +#include "../memory_slow_test_support.h" + +#include + +extern "C" const uint8_t kMkwGuestRegion = 0; + +namespace TextureBindingTest { +std::array memory{}; +uint32_t unreadableObject = 0; +uint32_t unreadableObjectAccesses = 0; +uint64_t rangeChecks = 0; +std::vector> retiredRanges; + +uint8_t* Pointer(uint32_t address, size_t length) { + const uint32_t offset = CanonicalizeGxMainRamAddress(address); + if (unreadableObject != 0 && offset >= unreadableObject && offset < unreadableObject + 32) { + ++unreadableObjectAccesses; + throw Memory::AccessViolation(address, length, "test unmapped range"); + } + if (offset > memory.size() || length > memory.size() - offset) { + throw Memory::AccessViolation(address, length, "test unmapped range"); + } + return memory.data() + offset; +} + +struct Command { + std::unique_ptr storage; + void (*run)(void*); +}; +std::deque commands; +} // namespace TextureBindingTest + +Memory::AccessViolation::AccessViolation(uint32_t address, size_t length, std::string_view reason) + : std::runtime_error(std::string(reason)), address_(address), length_(length), reason_(reason) {} + +bool Memory::Contains(uint32_t address, size_t length) { + ++TextureBindingTest::rangeChecks; + try { + TextureBindingTest::Pointer(address, length); + return true; + } catch (const Memory::AccessViolation&) { + return false; + } +} +uint8_t* Memory::GetPointer(uint32_t address) { return TextureBindingTest::Pointer(address, 1); } +uint8_t* Memory::GetPointer(uint32_t address, size_t length) { return TextureBindingTest::Pointer(address, length); } + +uint8_t MemoryInline::Read8Slow(uint32_t address) { return *TextureBindingTest::Pointer(address, 1); } +uint16_t MemoryInline::Read16Slow(uint32_t address) { return BigEndian::Read16(TextureBindingTest::Pointer(address, 2)); } +uint32_t MemoryInline::Read32Slow(uint32_t address) { return BigEndian::Read32(TextureBindingTest::Pointer(address, 4)); } +uint64_t MemoryInline::Read64Slow(uint32_t address) { + return (static_cast(Read32Slow(address)) << 32) | Read32Slow(address + 4); +} +void MemoryInline::Write8Slow(uint32_t address, uint8_t value) { *TextureBindingTest::Pointer(address, 1) = value; } +void MemoryInline::Write16Slow(uint32_t address, uint16_t value) { + BigEndian::Write16(TextureBindingTest::Pointer(address, 2), value); +} +void MemoryInline::Write32Slow(uint32_t address, uint32_t value) { + BigEndian::Write32(TextureBindingTest::Pointer(address, 4), value); +} +void MemoryInline::WriteFloat32Slow(uint32_t address, double value) { + const float narrowed = static_cast(value); + uint32_t bits; + std::memcpy(&bits, &narrowed, sizeof(bits)); + Write32Slow(address, bits); +} + +namespace GxWorker { +namespace detail { +std::atomic g_active{false}; +constinit thread_local bool t_onWorker = false; +constinit thread_local int t_inlineDepth = 0; +bool g_fifoPending = false; + +void* Reserve(size_t size, void (*run)(void*)) { + auto storage = std::make_unique(size); + void* result = storage.get(); + TextureBindingTest::commands.push_back({std::move(storage), run}); + return result; +} +void Commit() {} +void FlushFifo() { g_fifoPending = false; } +} // namespace detail + +void Sync() { + const bool wasWorker = detail::t_onWorker; + detail::t_onWorker = true; + while (!TextureBindingTest::commands.empty()) { + auto command = std::move(TextureBindingTest::commands.front()); + TextureBindingTest::commands.pop_front(); + command.run(command.storage.get()); + } + detail::t_onWorker = wasWorker; +} +void BeginInline() { Sync(); ++detail::t_inlineDepth; } +void EndInline() { --detail::t_inlineDepth; } +} // namespace GxWorker + +void InvalidateEfbCopyDestinationsForRange(uint32_t address, uint32_t size) { + TextureBindingTest::retiredRanges.emplace_back(address, size); +} +extern "C" void GxNotifyDisplayListMemoryWrite(uint32_t, uint32_t) {} diff --git a/runtime/tests/gx_texture_binding_tests.cpp b/runtime/tests/gx_texture_binding_tests.cpp new file mode 100644 index 000000000..c5d2d2872 --- /dev/null +++ b/runtime/tests/gx_texture_binding_tests.cpp @@ -0,0 +1,488 @@ +#include "gx_test_common.hpp" +#include "../src/hle/gx/gx_internal.h" +#include "isa/big_endian.h" + +#include + +// Keep the production loaders and metadata writers, without a guest dispatch registry. +#undef PPC_NATIVE_OVERRIDE +#undef PPC_NATIVE_OVERRIDE_VOID +#define PPC_NATIVE_OVERRIDE(...) +#define PPC_NATIVE_OVERRIDE_VOID(...) +#include "../src/hle/gx/gx_utils.cpp" +#include "../src/hle/gx/gx_objects.cpp" +#include "../src/hle/gx/gx_texture.cpp" +#include "gx_texture_binding_support/runtime_seams.h" + +namespace { +constexpr uint32_t kObject = 0x80001000; +constexpr uint32_t kOtherObject = kObject + 32; +constexpr uint32_t kPixels = 0x80010000; +constexpr uint32_t kOtherPixels = kPixels + 4096; +constexpr uint32_t kGuestGxData = 0x80003000; + +class RuntimeTextureBindingTest : public GXFifoTest { +protected: + void SetUp() override { + GXFifoTest::SetUp(); + AuroraSetGxProducerThread(); + GxWorker::detail::g_active = false; + GxWorker::detail::t_inlineDepth = 0; + GxWorker::detail::t_onWorker = false; + TextureBindingTest::memory.fill(0); + TextureBindingTest::unreadableObject = 0; + TextureBindingTest::unreadableObjectAccesses = 0; + TextureBindingTest::rangeChecks = 0; + TextureBindingTest::retiredRanges.clear(); + InvalidateTexObjLoads(); + // The production table never erases slots; reset its memo before fixture teardown. + g_texObjSlotMemo = nullptr; + g_texObjSlotMemoAddr = 0; + g_texObjIntervals.clear(); + g_texObjIntervalsPending.clear(); + g_texObjIntervalsRebuildAll = true; + g_TexObjMeta.clear(); + g_TlutObjMeta.clear(); + g_HostTlutObjMap.clear(); + g_tlutObjBoundsDirty = true; + Memory::Write32(kGXDataPtrAddr, kGuestGxData); + } + + void TearDown() override { + GxWorker::Sync(); + GxWorker::detail::g_active = false; + g_texObjSlotMemo = nullptr; + g_texObjSlotMemoAddr = 0; + g_texObjIntervals.clear(); + g_texObjIntervalsPending.clear(); + g_TexObjMeta.clear(); + g_HostTlutObjMap.clear(); + GXFifoTest::TearDown(); + } + + void Init(uint32_t object = kObject, uint32_t pixels = kPixels, uint32_t format = GX_TF_RGBA8) { + if (IsPaletteTexFormat(format)) { + GX__InitTexObjCI_80170a04(object, pixels, 8, 8, format, GX_REPEAT, GX_CLAMP, 0, 0); + } else { + GX__InitTexObj_801707f8(object, pixels, 8, 8, format, GX_REPEAT, GX_CLAMP, 0); + } + } + + void Load(uint32_t object = kObject, uint32_t map = 0) { GX__LoadTexObj_80170f2c(object, map); } + + void Warm(uint32_t object = kObject, uint32_t map = 0) { + Load(object, map); + GxWorker::Sync(); + EnsureTexObjIntervalsLocked(); + capture_fifo(); + } + + void ExpectFast(uint32_t object = kObject, uint32_t map = 0) { + const auto checks = TextureBindingTest::rangeChecks; + Load(object, map); + GxWorker::Sync(); + EXPECT_EQ(TextureBindingTest::rangeChecks, checks); + EXPECT_TRUE(g_texObjIntervalsPending.empty()); + } + + void ExpectFull(uint32_t object = kObject, uint32_t map = 0) { + const auto checks = TextureBindingTest::rangeChecks; + Load(object, map); + GxWorker::Sync(); + EXPECT_GT(TextureBindingTest::rangeChecks, checks); + } +}; + +TEST_F(RuntimeTextureBindingTest, RepeatedBindingSkipsMetadataAndExistingHostLoad) { + Init(); + Warm(); + Memory::Write32(kGuestGxData + 0x5FC, 0x20); + Memory::Write16(kGuestGxData + 2, 0x1234); + ExpectFast(); + EXPECT_TRUE(capture_fifo().empty()); + EXPECT_EQ(Memory::Read32(kGuestGxData + 0x5FC), 0x21u); + EXPECT_EQ(Memory::Read16(kGuestGxData + 2), 0); +} + +TEST_F(RuntimeTextureBindingTest, AlternatingObjectsAndMapsStillEmitRequiredBindings) { + Init(); + Init(kOtherObject, kOtherPixels); + Warm(); + Warm(kOtherObject, 1); + ExpectFast(kObject, 1); + auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[1].data, GuestToHostPtr(kPixels)); + ExpectFast(kOtherObject, 0); + bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[0].data, GuestToHostPtr(kOtherPixels)); +} + +TEST_F(RuntimeTextureBindingTest, BindOnlyAndFullPathsEmitIdenticalPackets) { + for (uint32_t format : {GX_TF_I4, GX_TF_RGBA8, GX_TF_CMPR, GX_TF_Z8, GX_TF_Z16, GX_TF_Z24X8}) { + Init(kObject, kPixels, format); + // Include values the full path has to sanitize before memoizing them. + Memory::Write32(kObject, Memory::Read32(kObject) | 3u | (3u << 19)); + Memory::Write32(kObject + 4, 0xFFFF); + Warm(); + g_boundTexMaps[2] = {}; + ExpectFast(kObject, 2); + const auto fast = capture_fifo(); + ASSERT_FALSE(fast.empty()); + g_boundTexMaps[2] = {}; + g_TexObjMeta.at(kObject).validatedLoad.Invalidate(); + ExpectFull(kObject, 2); + EXPECT_EQ(capture_fifo(), fast) << format; + } +} + +TEST_F(RuntimeTextureBindingTest, EverySnapshotByteParticipatesInValidation) { + Init(); + Warm(); + const auto original = CaptureTexObjBytes(kObject); + for (uint32_t offset = 0; offset < original.size(); ++offset) { + { + TexObjSnapshotScope before(kObject, original); + ASSERT_NE(TryGetValidatedTexObjSlot(kObject), nullptr) << offset; + } + const uint8_t changed = original[offset] ^ 0x80; + Memory::Write8(kObject + offset, changed); + const auto snapshot = CaptureTexObjBytes(kObject); + TexObjSnapshotScope scope(kObject, snapshot); + EXPECT_EQ(TryGetValidatedTexObjSlot(kObject), nullptr) << offset; + Memory::Write8(kObject + offset, original[offset]); + // Re-arm through a real successful load, using the original bytes. + { + TexObjSnapshotScope restored(kObject, original); + LoadTexObjImpl(kObject, 0); + } + } +} + +TEST_F(RuntimeTextureBindingTest, GetterShadowCannotValidateAnotherSnapshot) { + Init(); + Warm(); + auto newer = CaptureTexObjBytes(kObject); + newer[0x1C] ^= 1; + Memory::Write8(kObject + 0x1C, newer[0x1C]); + GX__GetTexObjAll_80170bf8(kObject, 0, 0, 0, 0, 0, 0, 0); + // This getter refreshes the decode shadow without doing a full texture load. + ExpectFull(); + Warm(); + ExpectFast(); +} + +TEST_F(RuntimeTextureBindingTest, ValidatedLoadRestoresClampedLodAfterGetterRefresh) { + constexpr uint32_t lodOutput = kGuestGxData + 0x600; + const auto getMaxLod = [&] { + GX__GetTexObjLODAll_80170cbc(kObject, 0, 0, 0, lodOutput, 0, 0, 0, 0); + return std::bit_cast(Memory::Read32(lodOutput)); + }; + + Init(); + Memory::Write32(kObject + 4, 0x5000); + Warm(); + ASSERT_EQ(getMaxLod(), 3.0f); + Memory::Write32(kObject + 4, 0x3000); + ASSERT_EQ(getMaxLod(), 3.0f); + Memory::Write32(kObject + 4, 0x5000); + + ExpectFast(); + EXPECT_TRUE(capture_fifo().empty()); + EXPECT_EQ(getMaxLod(), 3.0f); +} + +TEST_F(RuntimeTextureBindingTest, ProducerBacklogUsesExecutedBytes) { + Init(); + Warm(); + const auto oldBytes = CaptureTexObjBytes(kObject); + GxWorker::detail::g_active = true; + Load(kObject, 1); + Memory::Write32(kObject + 0x0C, CanonicalizeGxMainRamAddress(kOtherPixels) >> 5); + Load(kObject, 2); + // Producer has already moved on again before either queued load executes. + Memory::Write32(kObject + 0x0C, BigEndian::Read32(oldBytes.data() + 0x0C)); + GxWorker::Sync(); + auto bytes = capture_fifo(); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[1].data, GuestToHostPtr(kPixels)); + EXPECT_EQ(gxState().loadedTextures[2].data, GuestToHostPtr(kOtherPixels)); + EXPECT_EQ(g_TexObjMeta.at(kObject).dataAddr, CanonicalizeGxMainRamAddress(kOtherPixels)); + ExpectFull(kObject, 2); + Warm(kObject, 2); + ExpectFast(kObject, 2); +} + +TEST_F(RuntimeTextureBindingTest, MetadataAndHostWritersRequireFullLoad) { + Init(); + Warm(); + GX__InitTexObjLOD_80170a4c(kObject, GX_NEAR, GX_LINEAR, 0, 0, 0.5f, 1, 0, GX_ANISO_2); + ExpectFull(); + Warm(); + GX__InitTexObjWrapMode_80170b50(kObject, GX_MIRROR, GX_REPEAT); + ExpectFull(); + Warm(); + GX__InitTexObjTlut_80170b64(kObject, 3); + ExpectFull(); + Warm(); + GX__InitTexObjFilter_80170b6c(kObject, GX_LINEAR, GX_NEAR); + ExpectFull(); + Warm(); + GX__InitTexObjLODBias_80170b94(kObject, -0.5f); + ExpectFull(); + Warm(); + GX__InitTexObjUserData_80170be8(kObject, kOtherPixels); + ExpectFull(); + Warm(); + // Even an identical descriptor receives a new host texture identity. + Init(); + ExpectFull(); + EXPECT_FALSE(capture_fifo().empty()); + Warm(); + auto& meta = GetTexObjMeta(kObject); + meta.wrapT = GX_MIRROR; + ExpectFull(); +} + +TEST_F(RuntimeTextureBindingTest, PixelDmaBetweenQueuedLoadsCannotBeSkipped) { + Init(); + Warm(); + const auto* host = TryGetHostTexObj(kObject); + const auto oldVersion = reinterpret_cast(host)->texDataVersion; + GxWorker::detail::g_active = true; + Load(kObject, 1); + GxNotifyGuestRamDmaWrite(kPixels, 256); + Load(kObject, 1); + GxWorker::Sync(); + EXPECT_FALSE(g_TexObjMeta.at(kObject).needsUpload); + EXPECT_GT(reinterpret_cast(host)->texDataVersion, oldVersion); + EXPECT_EQ(TextureBindingTest::retiredRanges.size(), 1u); + auto bytes = capture_fifo(); + decode_fifo(bytes); + EXPECT_GT(gxState().loadedTextures[1].texDataVersion, oldVersion); +} + +TEST_F(RuntimeTextureBindingTest, PixelDmaRequiresRebindingEachLoadedMap) { + Init(); + for (uint32_t map : {0u, 1u, 2u}) { + Load(kObject, map); + } + decode_fifo(capture_fifo()); + const auto oldVersion = gxState().loadedTextures[0].texDataVersion; + ASSERT_NE(oldVersion, 0u); + ASSERT_EQ(gxState().loadedTextures[1].texDataVersion, oldVersion); + ASSERT_EQ(gxState().loadedTextures[2].texDataVersion, oldVersion); + + GxWorker::detail::g_active = true; + GxNotifyGuestRamDmaWrite(kPixels, 256); + ExpectFull(kObject, 0); + decode_fifo(capture_fifo()); + const auto newVersion = gxState().loadedTextures[0].texDataVersion; + ASSERT_GT(newVersion, oldVersion); + ASSERT_FALSE(g_TexObjMeta.at(kObject).needsUpload); + ASSERT_EQ(TextureBindingTest::retiredRanges.size(), 1u); + EnsureTexObjIntervalsLocked(); + + ExpectFast(kObject, 1); + auto bytes = capture_fifo(); + EXPECT_FALSE(bytes.empty()); + if (!bytes.empty()) { + decode_fifo(bytes); + } + EXPECT_EQ(gxState().loadedTextures[1].texDataVersion, newVersion); + ExpectFast(kObject, 1); + EXPECT_TRUE(capture_fifo().empty()); + + g_TexObjMeta.at(kObject).validatedLoad.Invalidate(); + ExpectFull(kObject, 2); + bytes = capture_fifo(); + EXPECT_FALSE(bytes.empty()); + if (!bytes.empty()) { + decode_fifo(bytes); + } + EXPECT_EQ(gxState().loadedTextures[2].texDataVersion, newVersion); +} + +TEST_F(RuntimeTextureBindingTest, HostReinitializationRequiresRebindingOtherMaps) { + Init(); + Load(); + Load(kObject, 1); + decode_fifo(capture_fifo()); + const auto oldId = gxState().loadedTextures[1].texObjId; + const auto oldVersion = gxState().loadedTextures[1].texDataVersion; + + Init(); + Load(); + decode_fifo(capture_fifo()); + const auto newId = gxState().loadedTextures[0].texObjId; + ASSERT_NE(newId, oldId); + ASSERT_EQ(gxState().loadedTextures[0].texDataVersion, oldVersion); + EnsureTexObjIntervalsLocked(); + + ExpectFast(kObject, 1); + const auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[1].texObjId, newId); +} + +TEST_F(RuntimeTextureBindingTest, DirtyAliasesAndAddressReuseUploadBeforeRebinding) { + Init(); + Warm(); + const auto oldVersion = reinterpret_cast(TryGetHostTexObj(kObject))->texDataVersion; + MarkTexObjsDirtyForRange(0xC0010000, 256); + EXPECT_TRUE(g_TexObjMeta.at(kObject).needsUpload); + ExpectFull(); + EXPECT_GT(reinterpret_cast(TryGetHostTexObj(kObject))->texDataVersion, oldVersion); + Warm(); + Memory::Write32(kObject + 0x0C, CanonicalizeGxMainRamAddress(kOtherPixels) >> 5); + ExpectFull(); + EXPECT_EQ(g_TexObjMeta.at(kObject).dataAddr, CanonicalizeGxMainRamAddress(kOtherPixels)); + auto bytes = capture_fifo(); + decode_fifo(bytes); + EXPECT_EQ(gxState().loadedTextures[0].data, GuestToHostPtr(kOtherPixels)); +} + +TEST_F(RuntimeTextureBindingTest, PaletteFormatsAlwaysKeepFullPathAcrossTlutChanges) { + for (uint32_t format : {GX_TF_C4, GX_TF_C8, GX_TF_C14X2}) { + Init(kObject, kPixels, format); + Warm(); + ExpectFull(); + GX__InitTlutObj_80170f80(kOtherObject, kOtherPixels, GX_TL_RGB565, 16); + GX__LoadTlut_80170fa8(kOtherObject, 0); + MarkTlutObjsDirtyForRange(kOtherPixels, 32); + GX__LoadTlut_80170fa8(kOtherObject, 0); + ExpectFull(); + } +} + +TEST_F(RuntimeTextureBindingTest, InvalidationAndCopyEvictionPreserveMetadataValidation) { + Init(); + Load(); + EnsureTexObjIntervalsLocked(); + auto bind = capture_fifo(); + ASSERT_FALSE(bind.empty()); + decode_fifo(bind); + ASSERT_EQ(gxState().loadedTextures[0].data, GuestToHostPtr(kPixels)); + GX__InvalidateTexAll_80171110(); + auto invalidate = capture_fifo(); + ASSERT_FALSE(invalidate.empty()); + decode_fifo(invalidate); + ExpectFast(); + EXPECT_TRUE(capture_fifo().empty()); + void* copyDest = GuestToHostPtr(kPixels); + ASSERT_NE(copyDest, nullptr); + gxState().copyTextures[copyDest] = {.revision = 1}; + GXDestroyCopyTex(copyDest); + decode_fifo(capture_fifo()); + EXPECT_FALSE(gxState().copyTextures.contains(copyDest)); + ExpectFast(); + EXPECT_TRUE(capture_fifo().empty()); +} + +TEST_F(RuntimeTextureBindingTest, ResetAndHostDestructionCannotReuseCompletedLoad) { + Init(); + Warm(); + InvalidateTexObjLoads(); + ExpectFull(); + EXPECT_FALSE(capture_fifo().empty()); + Warm(); + g_TexObjMeta.at(kObject).host->Destroy(); + ExpectFull(); + EXPECT_TRUE(g_TexObjMeta.at(kObject).host->constructed); +} + +TEST_F(RuntimeTextureBindingTest, UnreadableSnapshotDoesNotBecomeValidated) { + Init(); + Warm(); + TextureBindingTest::unreadableObject = CanonicalizeGxMainRamAddress(kObject); + ExpectFull(); + EXPECT_FALSE(g_TexObjMeta.at(kObject).validatedLoad.valid); + TextureBindingTest::unreadableObject = 0; + ExpectFull(); + EXPECT_TRUE(g_TexObjMeta.at(kObject).validatedLoad.valid); +} + +TEST_F(RuntimeTextureBindingTest, InvalidMapDoesNotCaptureOrValidateDescriptor) { + Init(); + for (bool defer : {false, true}) { + SCOPED_TRACE(defer); + GxWorker::detail::g_active = defer; + auto& validation = g_TexObjMeta.at(kObject).validatedLoad; + validation.Invalidate(); + Load(kObject, 8); + GxWorker::Sync(); + EXPECT_FALSE(validation.valid); + EXPECT_TRUE(capture_fifo().empty()); + + TextureBindingTest::unreadableObject = CanonicalizeGxMainRamAddress(kObject); + const auto accesses = TextureBindingTest::unreadableObjectAccesses; + Memory::Write32(kGuestGxData + 0x5FC, 0x20); + Memory::Write16(kGuestGxData + 2, 0x1234); + + Load(kObject, 8); + EXPECT_EQ(Memory::Read32(kGuestGxData + 0x5FC), 0x21u); + EXPECT_EQ(Memory::Read16(kGuestGxData + 2), 0u); + GxWorker::Sync(); + EXPECT_EQ(TextureBindingTest::unreadableObjectAccesses, accesses); + EXPECT_FALSE(validation.valid); + EXPECT_TRUE(capture_fifo().empty()); + + TextureBindingTest::unreadableObject = 0; + ExpectFull(); + EXPECT_TRUE(validation.valid); + capture_fifo(); + } +} + +TEST_F(RuntimeTextureBindingTest, InvalidGuestDescriptorKeepsPlaceholderFallback) { + Init(); + Load(); + EnsureTexObjIntervalsLocked(); + decode_fifo(capture_fifo()); + auto& loaded = gxState().loadedTextures[0]; + ASSERT_EQ(loaded.data, GuestToHostPtr(kPixels)); + const auto validId = loaded.texObjId; + + WriteGuestTexObj(kOtherObject, kOtherPixels, 8, 8, GX_CTF_R4, GX_CLAMP, GX_CLAMP, false, false, 0); + Load(kOtherObject); + auto bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(loaded.width(), 1u); + EXPECT_EQ(loaded.height(), 1u); + EXPECT_NE(loaded.texObjId, validId); + EXPECT_EQ(g_boundTexMaps[0].objAddr, 0u); + + EnsureTexObjIntervalsLocked(); + ExpectFast(); + bytes = capture_fifo(); + ASSERT_FALSE(bytes.empty()); + decode_fifo(bytes); + EXPECT_EQ(loaded.texObjId, validId); + EXPECT_EQ(loaded.data, GuestToHostPtr(kPixels)); +} + +TEST_F(RuntimeTextureBindingTest, DISABLED_BenchmarkRepeatedLoads) { + Init(); + Warm(); + constexpr uint32_t iterations = 250000; + auto& validation = g_TexObjMeta.at(kObject).validatedLoad; + for (bool forceFull : {true, false}) { + const auto checks = TextureBindingTest::rangeChecks; + const auto begin = std::chrono::steady_clock::now(); + for (uint32_t i = 0; i < iterations; ++i) { + if (forceFull) validation.Invalidate(); + Load(); + } + const auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - begin).count(); + std::printf("texture-load full=%d iterations=%u ns=%lld range-checks=%llu\n", forceFull, iterations, + static_cast(elapsed), + static_cast(TextureBindingTest::rangeChecks - checks)); + } +} +} // namespace diff --git a/runtime/tests/gx_worker_tests.cpp b/runtime/tests/gx_worker_tests.cpp new file mode 100644 index 000000000..4c7d4cdbd --- /dev/null +++ b/runtime/tests/gx_worker_tests.cpp @@ -0,0 +1,345 @@ +#define MKW_GX_WORKER_TESTING 1 +#include "../src/hle/gx/gx_worker.cpp" + +#include + +#include +#include +#include + +namespace PerfStats { +bool Enabled() noexcept { return false; } +void AddWait(std::chrono::steady_clock::duration) noexcept {} +} // namespace PerfStats + +namespace { +std::vector> g_fifoReplay; +std::function g_service; +std::function g_beforeRecheck; + +void ServiceWait() { g_service(); } +void BeforeRecheck() { g_beforeRecheck(); } + +template +bool WaitUntil(F&& ready) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{2}; + while (!ready()) { + if (std::chrono::steady_clock::now() >= deadline) { + return false; + } + std::this_thread::yield(); + } + return true; +} + +class Gate { +public: + ~Gate() { Open(); } + void Block() { + std::unique_lock lock{m_mutex}; + m_entered = true; + m_cv.notify_all(); + m_cv.wait(lock, [&] { return m_open; }); + } + bool WaitEntered() { + std::unique_lock lock{m_mutex}; + return m_cv.wait_for(lock, std::chrono::seconds{2}, [&] { return m_entered; }); + } + bool WaitOpened() { + std::unique_lock lock{m_mutex}; + return m_cv.wait_for(lock, std::chrono::seconds{2}, [&] { return m_open; }); + } + void Open() { + std::lock_guard lock{m_mutex}; + m_open = true; + m_cv.notify_all(); + } + +private: + std::mutex m_mutex; + std::condition_variable m_cv; + bool m_entered = false; + bool m_open = false; +}; + +class GxWorkerTest : public ::testing::Test { +protected: + void SetUp() override { + GxWorker::g_testEnabled = true; + GxWorker::g_testWakeEachRecord = false; + GxWorker::g_testNotifications = 0; + GxWorker::g_testWaits = 0; + g_fifoReplay.clear(); + GxWorker::Start(); + } + void TearDown() override { + GxWorker::g_testBeforeTailRecheck = nullptr; + GxWorker::SetWaitCallback(nullptr); + GxWorker::Stop(); + g_service = {}; + g_beforeRecheck = {}; + } +}; + +TEST_F(GxWorkerTest, DoesNotNotifyBeforeTheRequestedTail) { + Gate first; + Gate last; + uint32_t completed = 0; + GxWorker::Submit([&] { first.Block(); }); + for (uint32_t i = 0; i < 512; ++i) { + GxWorker::Submit([&, i] { EXPECT_EQ(completed++, i); }); + } + GxWorker::Submit([&] { last.Block(); }); + std::thread release{[&] { + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_testWaits.load() != 0; })); + first.Open(); + EXPECT_TRUE(last.WaitEntered()); + EXPECT_EQ(GxWorker::g_testNotifications.load(), 0u); + last.Open(); + }}; + GxWorker::Sync(); + release.join(); + EXPECT_EQ(completed, 512u); + EXPECT_GE(GxWorker::g_testWaits.load(), 1u); + EXPECT_EQ(GxWorker::g_waitTarget.load(), GxWorker::kNotWaiting); +} + +TEST_F(GxWorkerTest, ReachingTargetNotifiesTheBlockedProducer) { + Gate gate; + GxWorker::Submit([&] { gate.Block(); }); + GxWorker::Publish(); + if (!gate.WaitEntered()) { + gate.Open(); + GxWorker::Sync(); + FAIL() << "The worker did not enter the gate"; + } + std::unique_lock lock{GxWorker::g_tailMutex}; + GxWorker::g_waitTarget.store(GxWorker::g_write); + gate.Open(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds{2}; + while (GxWorker::g_testNotifications.load() == 0) { + if (GxWorker::g_tailCv.wait_until(lock, deadline) == std::cv_status::timeout) { + ADD_FAILURE() << "The worker did not signal the target"; + break; + } + } + EXPECT_EQ(GxWorker::g_tail.load(), GxWorker::g_write); + GxWorker::g_waitTarget.store(GxWorker::kNotWaiting); +} + +TEST_F(GxWorkerTest, RechecksCompletionAfterPublishingTheWaitTarget) { + Gate gate; + GxWorker::Submit([&] { gate.Block(); }); + g_beforeRecheck = [&] { + GxWorker::g_testBeforeTailRecheck = nullptr; + gate.Open(); + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_tail.load() >= GxWorker::g_waitTarget.load(); })); + }; + GxWorker::g_testBeforeTailRecheck = BeforeRecheck; + GxWorker::Sync(); + EXPECT_EQ(GxWorker::g_testWaits.load(), 0u); + EXPECT_EQ(GxWorker::g_waitTarget.load(), GxWorker::kNotWaiting); +} + +void CheckWorkSubmittedByCallback(bool syncInsideCallback) { + Gate gate; + uint32_t callbacks = 0; + std::atomic_uint32_t completed{0}; + GxWorker::Submit([&] { gate.Block(); ++completed; }); + g_service = [&] { + if (++callbacks == 1) { + GxWorker::Submit([&] { ++completed; }); + gate.Open(); + if (syncInsideCallback) { + GxWorker::Sync(); + EXPECT_EQ(completed.load(), 2u); + } + } + }; + GxWorker::SetWaitCallback(ServiceWait); + std::thread watchdog{[&] { + if (!gate.WaitOpened()) { + ADD_FAILURE() << "The wait callback did not release the worker gate"; + gate.Open(); + } + }}; + GxWorker::Sync(); + watchdog.join(); + EXPECT_GE(callbacks, 1u); + EXPECT_EQ(completed.load(), 2u); + GxWorker::SetWaitCallback(nullptr); + GxWorker::Publish(); + GxWorker::Sync(); + EXPECT_EQ(GxWorker::g_waitTarget.load(), GxWorker::kNotWaiting); +} + +TEST_F(GxWorkerTest, NestedSyncDrainsWorkBeforeReturningToCallback) { + CheckWorkSubmittedByCallback(true); +} + +TEST_F(GxWorkerTest, OuterSyncDrainsWorkSubmittedByCallback) { + CheckWorkSubmittedByCallback(false); +} + +TEST_F(GxWorkerTest, RingBackpressureAndWrapPreserveOrderWithoutServicingGuestCallbacks) { + Gate gate; + uint32_t callbacks = 0; + uint32_t completed = 0; + GxWorker::Submit([&] { gate.Block(); }); + std::thread release{[&] { + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_waitTarget.load() != GxWorker::kNotWaiting; })); + gate.Open(); + }}; + g_service = [&] { ++callbacks; }; + GxWorker::SetWaitCallback(ServiceWait); + for (uint32_t i = 0; i < 160; ++i) { + std::array bytes{}; + bytes.front() = static_cast(i); + bytes.back() = static_cast(i ^ 0xff); + GxWorker::Submit([&, bytes, i] { + EXPECT_EQ(completed++, i); + EXPECT_EQ(bytes.front(), static_cast(i)); + EXPECT_EQ(bytes.back(), static_cast(i ^ 0xff)); + }); + } + release.join(); + EXPECT_EQ(callbacks, 0u); + GxWorker::SetWaitCallback(nullptr); + GxWorker::Sync(); + EXPECT_EQ(completed, 160u); +} + +TEST_F(GxWorkerTest, FifoFlushInlineAndRestartRetainTheirBoundaries) { + GxWorker::FifoWrite(0x61, 1); + GxWorker::FifoWrite(0x12345678, 4); + const std::array burst{1, 2, 3}; + GxWorker::FifoBurst(burst.data(), burst.size()); + GxWorker::Submit([] { EXPECT_EQ(g_fifoReplay.size(), 5u); }); + { + GxWorker::InlineScope scope; + EXPECT_FALSE(GxWorker::ShouldDefer()); + EXPECT_EQ(g_fifoReplay, (std::vector>{ + {0x61, 1}, {0x12345678, 4}, {1, 1}, {2, 1}, {3, 1}})); + } + EXPECT_TRUE(GxWorker::ShouldDefer()); + auto lifetime = std::make_shared(0); + std::weak_ptr weak = lifetime; + GxWorker::Submit([owned = std::move(lifetime)] {}); + GxWorker::Stop(); + EXPECT_TRUE(weak.expired()); + GxWorker::g_testEnabled = false; + GxWorker::Start(); + EXPECT_FALSE(GxWorker::Active()); + bool ran = false; + GxWorker::Run([&] { ran = true; }); + EXPECT_TRUE(ran); + GxWorker::g_testEnabled = true; + GxWorker::Start(); + GxWorker::Submit([&] { ran = false; }); + GxWorker::Sync(); + EXPECT_FALSE(ran); +} + +TEST_F(GxWorkerTest, ThrowingCommandsStillCompleteTheirRecordAndReleaseTheirPayload) { + auto lifetime = std::make_shared(0); + std::weak_ptr weak = lifetime; + bool completed = false; + GxWorker::Submit([owned = std::move(lifetime)] { throw std::runtime_error("injected failure"); }); + GxWorker::Submit([&] { completed = true; }); + GxWorker::Sync(); + EXPECT_TRUE(completed); + EXPECT_TRUE(weak.expired()); +} + +#if GTEST_HAS_DEATH_TEST +TEST(GxWorkerDeathTest, AbandonWakesTheIdleConsumerAndSignalsExit) { + ASSERT_EXIT({ + GxWorker::Start(); + if (!WaitUntil([] { return GxWorker::g_consumerWaiting.load(); })) { + std::_Exit(2); + } + { + std::lock_guard lock{GxWorker::g_headMutex}; + } + GxWorker::Abandon(); + std::_Exit(!GxWorker::Active() && GxWorker::g_thread == nullptr && GxWorker::g_exited.load() ? 0 : 1); + }, ::testing::ExitedWithCode(0), ""); +} + +TEST(GxWorkerDeathTest, AbandonDoesNotExecuteTheFollowingRecord) { + ASSERT_EXIT({ + Gate gate; + std::atomic ran{false}; + GxWorker::Start(); + GxWorker::Submit([&] { gate.Block(); }); + GxWorker::Submit([&] { ran = true; }); + GxWorker::Publish(); + if (!gate.WaitEntered()) { + std::_Exit(2); + } + std::thread release{[&] { + WaitUntil([] { return GxWorker::g_abandon.load(); }); + gate.Open(); + }}; + GxWorker::Abandon(); + release.join(); + std::_Exit(!ran.load() && GxWorker::g_exited.load() ? 0 : 1); + }, ::testing::ExitedWithCode(0), ""); +} +#endif + +TEST_F(GxWorkerTest, DISABLED_WakeBenchmark) { + constexpr uint32_t recordsPerBatch = 20000; + std::array, 2> times; + std::array notifications{}; + std::array waits{}; + for (uint32_t batch = 0; batch < 40; ++batch) { + for (uint32_t order = 0; order < 2; ++order) { + const uint32_t mode = (batch + order) % 2; + GxWorker::g_testWakeEachRecord = mode != 0; + GxWorker::g_testNotifications = 0; + GxWorker::g_testWaits = 0; + uint32_t completed = 0; + Gate gate; + const auto start = std::chrono::steady_clock::now(); + GxWorker::Submit([&] { gate.Block(); }); + for (uint32_t i = 1; i < recordsPerBatch; ++i) { + GxWorker::Submit([&] { ++completed; }); + } + GxWorker::Submit([&] { + ++completed; + GxWorker::g_testAcknowledgeRecord = true; + }); + std::thread release{[&] { + EXPECT_TRUE(WaitUntil([] { return GxWorker::g_testWaits.load() != 0; })); + gate.Open(); + }}; + GxWorker::Sync(); + const auto end = std::chrono::steady_clock::now(); + release.join(); + times[mode].push_back(std::chrono::duration_cast(end - start).count()); + + // The worker accounts for notifications after publishing the tail. + ASSERT_TRUE(WaitUntil([] { return GxWorker::g_testAccountedTail.load() == GxWorker::g_write; })); + notifications[mode] += GxWorker::g_testNotifications.load(); + waits[mode] += GxWorker::g_testWaits.load(); + EXPECT_EQ(completed, recordsPerBatch); + } + } + for (uint32_t mode = 0; mode < 2; ++mode) { + std::sort(times[mode].begin(), times[mode].end()); + std::printf("wake_each=%u records=800000 notifications=%llu waits=%llu median_us=%lld p95_us=%lld\n", mode, + static_cast(notifications[mode]), static_cast(waits[mode]), + static_cast((times[mode][19] + times[mode][20]) / 2), + static_cast(times[mode][37])); + } + GxWorker::g_testWakeEachRecord = false; +} +} // namespace + +void GxWorkerReplayFifoWrite(uint32_t value, uint32_t sizeBytes) { g_fifoReplay.emplace_back(value, sizeBytes); } +void GxWorkerReplayFifoBurst(const uint8_t* data, uint32_t sizeBytes) { + for (uint32_t i = 0; i < sizeBytes; ++i) { + GxWorkerReplayFifoWrite(data[i], 1); + } +} diff --git a/translator/src/Translator.Cli/Program.cs b/translator/src/Translator.Cli/Program.cs index e1164dd2a..ddb719c89 100644 --- a/translator/src/Translator.Cli/Program.cs +++ b/translator/src/Translator.Cli/Program.cs @@ -37,8 +37,10 @@ CultureInfo.DefaultThreadCurrentUICulture = CultureInfo.InvariantCulture; var command = args.Length > 0 ? args[0] : "info"; -var (projectOption, tailWithoutProject) = ExtractOption(args.Skip(1).ToArray(), "--project"); -var (profileOption, tail) = ExtractOption(tailWithoutProject, "--profile"); +var (optionValues, enabledFlags, firstCommandArgument, helpRequested, unrecognizedOption) = + ParseCommandLineOptions(command, args); +var projectOption = optionValues.GetValueOrDefault("--project"); +var profileOption = optionValues.GetValueOrDefault("--profile"); var project = !string.IsNullOrWhiteSpace(projectOption) ? TranslationProjectConfig.Load(projectOption) : null; @@ -59,7 +61,15 @@ GuestSaveRestoreThunks.Current = GuestSaveRestoreThunks.FromFunctionMap(functionMap); } var root = project?.WorkspaceRoot ?? Directory.GetCurrentDirectory(); -var preferCachedInputs = HasFlag(tail, "--prefer-cached-inputs"); +if (helpRequested) + return ShowCommandHelp(command); +if (unrecognizedOption is not null) +{ + Console.Error.WriteLine($"Unrecognized option '{unrecognizedOption}' for command '{command}'."); + WriteUsage(Console.Error, new[] { command }); + return 2; +} +var preferCachedInputs = enabledFlags.Contains("--prefer-cached-inputs"); var dolFile = new Lazy(LoadDol); var relFile = new Lazy(LoadRel); var image = new Lazy(LoadImage); @@ -156,27 +166,18 @@ BaseTranslationOutputMetadata ReadBaseMetadataCached(string path) // Argument registers checked for callback function pointers at every call site var argumentRegisters = new[] { "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10" }; -// Argument validation runs before any command can execute. `--help` used to -// fall straight through to the command's parser, which ignored every argument -// it did not recognise, so `emit-build-shards --help` ran the command with its -// defaults and rewrote generated/build_shards. -if (HandleCommandLineOptions(command, tail) is { } commandLineExitCode) -{ - return commandLineExitCode; -} - return command switch { "help" or "--help" or "-h" or "-?" => ShowGlobalHelp(), "info" or "--info" or "--version" => RunInfo(), "generate-data-init" => RunGenerateDataInit(), - "translate-recursive" => RunTranslateRecursive(tail), - "translate-mod" => RunTranslateMod(tail), - "emit-build-shards" => RunEmitBuildShards(tail), - "emit-native-index" => RunEmitNativeIndex(tail), - "emit-base-manifest" => RunEmitBaseManifest(tail), - "check-base-mod-awareness" => RunCheckBaseModAwareness(tail), - "validate-retro-wfc-payload" => RunValidateRetroWfcPayload(tail), + "translate-recursive" => RunTranslateRecursive(firstCommandArgument), + "translate-mod" => RunTranslateMod(), + "emit-build-shards" => RunEmitBuildShards(), + "emit-native-index" => RunEmitNativeIndex(), + "emit-base-manifest" => RunEmitBaseManifest(), + "check-base-mod-awareness" => RunCheckBaseModAwareness(), + "validate-retro-wfc-payload" => RunValidateRetroWfcPayload(), _ => ShowHelp(command) }; @@ -185,11 +186,11 @@ BaseTranslationOutputMetadata ReadBaseMetadataCached(string path) /// Exit 0 = yes, 3 = no, anything else = error (also treat as no). Answering wrong in the permissive /// direction bakes vanilla code into a modded product, so every uncertainty exits non-zero. /// -int RunCheckBaseModAwareness(string[] argsTail) +int RunCheckBaseModAwareness() { - var metadataPath = OptionValue(argsTail, "--translation-output-metadata") + var metadataPath = optionValues.GetValueOrDefault("--translation-output-metadata") ?? Path.Combine(root, "generated", "base_translation_output.json"); - var codePul = OptionValue(argsTail, "--code-pul") ?? profile?.CodePul; + var codePul = optionValues.GetValueOrDefault("--code-pul") ?? profile?.CodePul; if (string.IsNullOrWhiteSpace(codePul)) { Console.Error.WriteLine("[translator] check-base-mod-awareness needs --code-pul or a profile that has one."); @@ -238,7 +239,7 @@ RuntimeNativeIndex ResolveRegionalNativeIndex(string nativeSourceDirectory, stri return index.ForRegion(GuestAddressTable.Load(tablePath), region); } -int RunEmitBuildShards(string[] argsTail) +int RunEmitBuildShards() { // The project decides the guest region the shards are built for. if (project is null) @@ -246,13 +247,13 @@ int RunEmitBuildShards(string[] argsTail) Console.Error.WriteLine("emit-build-shards requires --project ."); return 1; } - var baseMetadata = OptionValue(argsTail, "--base-metadata") ?? Path.Combine(root, "generated", "base_translation_output.json"); - var baseFunctions = OptionValue(argsTail, "--base-functions-dir") ?? Path.Combine(root, "generated", "functions"); - var output = OptionValue(argsTail, "--out") ?? Path.Combine(root, "generated", "build_shards"); - var nativeSources = OptionValue(argsTail, "--native-source-dir") ?? Path.Combine(root, "runtime", "src"); - var nativeIndexOption = OptionValue(argsTail, "--native-index"); - var resolvedProfile = OptionValue(argsTail, "--resolved-profile"); - var retroCpp = OptionValue(argsTail, "--retro-cpp-dir"); + var baseMetadata = optionValues.GetValueOrDefault("--base-metadata") ?? Path.Combine(root, "generated", "base_translation_output.json"); + var baseFunctions = optionValues.GetValueOrDefault("--base-functions-dir") ?? Path.Combine(root, "generated", "functions"); + var output = optionValues.GetValueOrDefault("--out") ?? Path.Combine(root, "generated", "build_shards"); + var nativeSources = optionValues.GetValueOrDefault("--native-source-dir") ?? Path.Combine(root, "runtime", "src"); + var nativeIndexOption = optionValues.GetValueOrDefault("--native-index"); + var resolvedProfile = optionValues.GetValueOrDefault("--resolved-profile"); + var retroCpp = optionValues.GetValueOrDefault("--retro-cpp-dir"); try { @@ -306,16 +307,16 @@ int RunEmitBuildShards(string[] argsTail) } } -int RunEmitNativeIndex(string[] argsTail) +int RunEmitNativeIndex() { - var output = OptionValue(argsTail, "--out"); + var output = optionValues.GetValueOrDefault("--out"); if (string.IsNullOrWhiteSpace(output)) { Console.Error.WriteLine("[translator] emit-native-index needs --out."); return 1; } var nativeSourcePath = Path.GetFullPath( - OptionValue(argsTail, "--native-source-dir") ?? RequireProject().Runtime.NativeRegistrationRoot); + optionValues.GetValueOrDefault("--native-source-dir") ?? RequireProject().Runtime.NativeRegistrationRoot); if (!Directory.Exists(nativeSourcePath)) { Console.Error.WriteLine($"[translator] native source directory not found: {nativeSourcePath}"); @@ -419,9 +420,9 @@ int RunInfo() return 0; } -int RunValidateRetroWfcPayload(string[] argsTail) +int RunValidateRetroWfcPayload() { - var directory = OptionValue(argsTail, "--directory"); + var directory = optionValues.GetValueOrDefault("--directory"); if (string.IsNullOrWhiteSpace(directory)) { Console.Error.WriteLine("--directory is required."); @@ -431,7 +432,7 @@ int RunValidateRetroWfcPayload(string[] argsTail) try { WiiCompiled.Setup.Common.RetroWfcPayload.ValidateStagedRetroWfcPayloadDirectory(directory, - gameId: OptionValue(argsTail, "--game-id")!); + gameId: optionValues.GetValueOrDefault("--game-id")!); Console.WriteLine("[translator] Retro WFC payload signature validated."); return 0; } @@ -447,24 +448,26 @@ int RunValidateRetroWfcPayload(string[] argsTail) } } -int RunTranslateRecursive(string[] argsTail) +int RunTranslateRecursive(string? firstCommandArgument) { - if (argsTail.Length == 0 || argsTail[0].StartsWith("--", StringComparison.Ordinal)) + if (firstCommandArgument is null || firstCommandArgument.StartsWith("--", StringComparison.Ordinal)) { Console.Error.WriteLine("Usage: translator translate-recursive [--outdir path] [--output-metadata path] [--threads N] [--prune-stale]"); return 1; } - var startAddr = ParseAddress(argsTail[0]); + var startAddr = ParseAddress(firstCommandArgument); var allowUnsupportedInstructions = RequireProject().Translation.AllowUnsupportedInstructions; - var outDir = OptionValue(argsTail, "--outdir") ?? RequireProject().Output.Functions; - var outputMetadataPath = OptionValue(argsTail, "--output-metadata"); - var productionSourceBundlePath = OptionValue(argsTail, "--production-source-bundle"); - var emitFunctionFiles = !HasFlag(argsTail, "--no-function-files"); + var outDir = optionValues.GetValueOrDefault("--outdir") ?? RequireProject().Output.Functions; + var outputMetadataPath = optionValues.GetValueOrDefault("--output-metadata"); + var productionSourceBundlePath = optionValues.GetValueOrDefault("--production-source-bundle"); + var emitFunctionFiles = !enabledFlags.Contains("--no-function-files"); + var boundedIr = enabledFlags.Contains("--bounded-ir"); if (!emitFunctionFiles && string.IsNullOrWhiteSpace(productionSourceBundlePath)) throw new ArgumentException("--no-function-files requires --production-source-bundle."); BaseTranslationOutputMetadata? previousOutputMetadata = null; - if (HasFlag(argsTail, "--prune-stale") && + var pruneStale = enabledFlags.Contains("--prune-stale"); + if (pruneStale && outputMetadataPath is not null && File.Exists(outputMetadataPath)) { @@ -478,8 +481,7 @@ outputMetadataPath is not null && $"[translator] Warning: existing output metadata could not be read; stale pruning will use one-time tree migration: {ex.Message}"); } } - var pruneStale = HasFlag(argsTail, "--prune-stale"); - var threadsOption = OptionValue(argsTail, "--threads"); + var threadsOption = optionValues.GetValueOrDefault("--threads"); var maxThreads = threadsOption != null ? Math.Max(1, ParseInt(threadsOption)) : Math.Max(1, Environment.ProcessorCount); Console.WriteLine($"[translator] Using up to {maxThreads} worker thread(s)."); @@ -954,14 +956,18 @@ void LogPhase(string label) var emissionAddresses = baseSummaries.Keys.OrderBy(address => address).ToArray(); var localGuestAbiContracts = new Dictionary(emissionAddresses.Length); - var canonicalGuestFunctions = new Dictionary(emissionAddresses.Length); + var bufferedGuestFunctions = boundedIr ? null : new Dictionary(emissionAddresses.Length); + IReadOnlyDictionary canonicalGuestFunctions = bufferedGuestFunctions is not null + ? bufferedGuestFunctions + : new CanonicalIrDictionary(canonicalIrStore, emissionAddresses); + Console.WriteLine($"[translator] Canonical IR mode: {(bufferedGuestFunctions is null ? "bounded" : "buffered")}."); var irInstructionCounts = new Dictionary(emissionAddresses.Length); // Each slot is an independent pure decode plus contract analysis, so // compute them in parallel into position-indexed arrays and fill the // dictionaries afterwards in the original ascending order. Dictionary // insertion order is preserved, so every downstream enumeration of // these maps is byte-for-byte the same as the serial version. - var canonicalByIndex = new IrFunction[emissionAddresses.Length]; + var canonicalByIndex = boundedIr ? null : new IrFunction[emissionAddresses.Length]; var localContractByIndex = new GuestAbiContract[emissionAddresses.Length]; var irInstructionCountByIndex = new int[emissionAddresses.Length]; IndexedParallel.For(emissionAddresses.Length, parallelOptions, i => @@ -969,14 +975,14 @@ void LogPhase(string label) var address = emissionAddresses[i]; if (!canonicalIrStore.TryGet(address, out var canonicalSsa)) throw new InvalidOperationException($"Missing canonical SSA while building ABI contract for 0x{address:X8}."); - canonicalByIndex[i] = canonicalSsa; + if (canonicalByIndex is not null) canonicalByIndex[i] = canonicalSsa; localContractByIndex[i] = GuestAbiContractAnalyzer.Analyze(canonicalSsa); irInstructionCountByIndex[i] = canonicalSsa.Blocks.Sum(block => block.Instructions.Count); }); for (var i = 0; i < emissionAddresses.Length; i++) { var address = emissionAddresses[i]; - canonicalGuestFunctions[address] = canonicalByIndex[i]; + if (bufferedGuestFunctions is not null) bufferedGuestFunctions[address] = canonicalByIndex![i]; localGuestAbiContracts[address] = localContractByIndex[i]; irInstructionCounts[address] = irInstructionCountByIndex[i]; } @@ -988,7 +994,10 @@ void LogPhase(string label) // rather than retaining every descendant dependency forever. var interproceduralGuestAbi = GuestAbiInterproceduralAnalyzer.Analyze( canonicalGuestFunctions, - nativeGuestEffects.Contracts); + nativeGuestEffects.Contracts, + retainBodies: !boundedIr, + maxDegreeOfParallelism: boundedIr ? maxThreads : null, + localContracts: boundedIr ? localGuestAbiContracts : null); var guestAbiContracts = interproceduralGuestAbi.Contracts .ToDictionary(static pair => pair.Key, static pair => pair.Value); var recursiveComponents = interproceduralGuestAbi.StronglyConnectedComponents @@ -1341,10 +1350,10 @@ static string StateMaskSortKey(GuestStateMask mask) => .Take(4) .ToArray(); var variantIndex = 0; + var targetFunction = canonicalGuestFunctions[target]; foreach (var outputGroup in rankedOutputs) { var output = outputGroup.Key; - var targetFunction = canonicalGuestFunctions[target]; var entryLive = GuestStateLivenessAnalyzer.Analyze( targetFunction, stateFreeLivenessContracts, output).BlockLiveIn[targetFunction.EntryLabel]; var publicContract = guestAbiContracts[target]; @@ -1624,12 +1633,12 @@ static bool ContainsInteriorBoundary(uint start, uint end, uint[] boundaries) return 0; } -int RunTranslateMod(string[] argsTail) +int RunTranslateMod() { - var outDir = OptionValue(argsTail, "--out") ?? profile?.Output; + var outDir = optionValues.GetValueOrDefault("--out") ?? profile?.Output; if (string.IsNullOrWhiteSpace(outDir)) { - return RunTranslateModCore(argsTail, null); + return RunTranslateModCore(null); } var publishedOutput = Path.GetFullPath(outDir); @@ -1641,7 +1650,7 @@ int RunTranslateMod(string[] argsTail) { var result = TransactionalDirectoryOutput.GenerateAndPublish( publishedOutput, - stagingDirectory => RunTranslateModCore(argsTail, stagingDirectory), + RunTranslateModCore, (stagingDirectory, exitCode) => PreserveTranslateModFailureDiagnostics( stagingDirectory, publishedOutput, @@ -1689,7 +1698,7 @@ void PreserveTranslateModFailureDiagnostics(string stagingDirectory, string publ Console.Error.WriteLine($"[translator] Preserved failure diagnostics: {diagnosticsDirectory}"); } -int RunTranslateModCore(string[] argsTail, string? outputDirectoryOverride) +int RunTranslateModCore(string? outputDirectoryOverride) { // Wall-clock accounting mirroring RunTranslateRecursive's LogPhase output. var modPhaseStart = DateTime.Now; @@ -1701,9 +1710,9 @@ void LogModPhase(string label) modLastPhaseSeconds = now; } - var codePul = OptionValue(argsTail, "--code-pul") ?? profile?.CodePul; - var baseManifestPath = OptionValue(argsTail, "--base-manifest"); - var outDir = outputDirectoryOverride ?? OptionValue(argsTail, "--out") ?? profile?.Output; + var codePul = optionValues.GetValueOrDefault("--code-pul") ?? profile?.CodePul; + var baseManifestPath = optionValues.GetValueOrDefault("--base-manifest"); + var outDir = outputDirectoryOverride ?? optionValues.GetValueOrDefault("--out") ?? profile?.Output; if (string.IsNullOrWhiteSpace(codePul) || string.IsNullOrWhiteSpace(baseManifestPath) || string.IsNullOrWhiteSpace(outDir)) @@ -1712,8 +1721,8 @@ void LogModPhase(string label) return 1; } - var modRoot = OptionValue(argsTail, "--mod-root") ?? profile?.ModRoot; - var baseTranslationOutputMetadataPath = OptionValue(argsTail, "--base-translation-output-metadata") + var modRoot = optionValues.GetValueOrDefault("--mod-root") ?? profile?.ModRoot; + var baseTranslationOutputMetadataPath = optionValues.GetValueOrDefault("--base-translation-output-metadata") ?? Path.Combine(root, "generated", "base_translation_output.json"); // The base translation bakes mod-safety decisions (leaf-inlining blocks, residency fences) // around the patch set it knew about; a base tree that never saw this Code.pul can splice @@ -1768,8 +1777,8 @@ void LogModPhase(string label) $"[translator] Warning: no base translation output metadata at {baseTranslationOutputMetadataPath}; " + "cannot verify the base translation is aware of this mod's patch set."); } - var skipRetroWfc = HasFlag(argsTail, "--skip-retro-wfc"); - var explicitRetroWfcPayloadSpec = OptionValue(argsTail, "--retro-wfc-payload"); + var skipRetroWfc = enabledFlags.Contains("--skip-retro-wfc"); + var explicitRetroWfcPayloadSpec = optionValues.GetValueOrDefault("--retro-wfc-payload"); var retroWfcPayloadSpec = skipRetroWfc ? null : explicitRetroWfcPayloadSpec ?? @@ -1780,19 +1789,19 @@ void LogModPhase(string label) Console.Error.WriteLine("[translator] Retro WFC lowering is project-specific and requires a profile with enable_retro_wfc: true."); return 1; } - var modName = OptionValue(argsTail, "--mod-name") + var modName = optionValues.GetValueOrDefault("--mod-name") ?? (!string.IsNullOrWhiteSpace(modRoot) ? Path.GetFileName(Path.GetFullPath(modRoot)) : "Pulsar Mod"); - var region = OptionValue(argsTail, "--region") ?? profile?.Region ?? "P"; + var region = optionValues.GetValueOrDefault("--region") ?? profile?.Region ?? "P"; if (project?.Identity.Region is { } projectRegion && !string.Equals(region, projectRegion, StringComparison.OrdinalIgnoreCase)) { Console.Error.WriteLine($"[translator] --region {region} does not match the project's region {projectRegion}."); return 1; } - var moduleGuestBase = OptionValue(argsTail, "--module-guest-base") is { } moduleBaseText + var moduleGuestBase = optionValues.GetValueOrDefault("--module-guest-base") is { } moduleBaseText ? ParseAddress(moduleBaseText) : profile?.ModuleGuestBase ?? 0x81700000u; - var moduleLinkBaseText = OptionValue(argsTail, "--module-link-base"); + var moduleLinkBaseText = optionValues.GetValueOrDefault("--module-link-base"); var moduleLinkBase = moduleLinkBaseText is not null ? ParseAddress(moduleLinkBaseText) : profile?.ModuleLinkBase ?? 0x803992E0u; @@ -1969,8 +1978,8 @@ void LogModPhase(string label) .OrderBy(s => s.Address) .ToList(); var cppFailureCount = 0; - var emitCpp = HasFlag(argsTail, "--emit-cpp"); - var maxThreads = OptionValue(argsTail, "--threads") is { } threadsText + var emitCpp = enabledFlags.Contains("--emit-cpp"); + var maxThreads = optionValues.GetValueOrDefault("--threads") is { } threadsText ? Math.Max(1, ParseInt(threadsText)) : Math.Max(1, Environment.ProcessorCount); LogModPhase("setup + patch planning"); @@ -1992,8 +2001,7 @@ void LogModPhase(string label) moduleLinkBase, selected.CodeSize, nonReturningModuleCallTargets, - maxThreads, - argsTail); + maxThreads); } LogModPhase("mod C++ emission"); @@ -2342,11 +2350,7 @@ int EmitModCpp( uint moduleLinkBase, uint moduleLinkedCodeSize, IReadOnlySet nonReturningCallTargets, - int maxThreads, - // The translate-mod command line, so mod emission honours the same codegen - // policy overrides (residency, boundary narrowing) as base emission instead - // of quietly diverging from it. - string[] argsTail) + int maxThreads) { var modImage = BuildSyntheticModProgramImage(outDir, overlayBuild); var inferredAbi = new InferredGuestFunctionAbiProvider(modImage); @@ -2880,17 +2884,17 @@ static bool RetroWfcHookSetsLinkRegister(RetroWfcExecutableHookPlan hook) => hook.TypeName is "call" or "branchCtrLink" || hook.Intent.Contains("Call", StringComparison.Ordinal); -int RunEmitBaseManifest(string[] argsTail) +int RunEmitBaseManifest() { var loadedProject = RequireProject(); - var outDir = OptionValue(argsTail, "--out") ?? Path.GetDirectoryName(loadedProject.Output.BaseManifest)!; - var functionsDir = OptionValue(argsTail, "--functions-dir") ?? loadedProject.Output.Functions; - var outputMetadataPath = OptionValue(argsTail, "--translation-output-metadata"); + var outDir = optionValues.GetValueOrDefault("--out") ?? Path.GetDirectoryName(loadedProject.Output.BaseManifest)!; + var functionsDir = optionValues.GetValueOrDefault("--functions-dir") ?? loadedProject.Output.Functions; + var outputMetadataPath = optionValues.GetValueOrDefault("--translation-output-metadata"); var outputMetadata = outputMetadataPath is null ? null : BaseTranslationOutputMetadataFile.Read(outputMetadataPath); outputMetadata?.RequireReleaseEligible(outputMetadataPath!); - var region = OptionValue(argsTail, "--region") ?? loadedProject.Identity.Region ?? "unknown"; + var region = optionValues.GetValueOrDefault("--region") ?? loadedProject.Identity.Region ?? "unknown"; var rel = relFile.Value ?? throw new InvalidOperationException("Base mod manifest generation currently requires the project's single configured REL."); var relImage = rel.BuildImage(loadedProject.Inputs.Rel!.LoadAddress); @@ -3697,6 +3701,7 @@ static string[] KnownCommands() => new[] new("--production-source-bundle", "path"), new("--threads", "N"), new("--no-function-files"), + new("--bounded-ir"), new("--prune-stale") }), "translate-mod" => (null, new CommandOption[] @@ -3770,60 +3775,60 @@ static string CommandUsage(string command) return text.ToString(); } -// Options every command tolerates. --project and --profile are consumed before -// dispatch, so they only reach a command's tail when they were repeated. -static string[] GlobalValueOptions() => new[] { "--project", "--profile" }; - -static string[] GlobalFlagOptions() => new[] { "--prefer-cached-inputs" }; - -/// -/// Returns an exit code when the command line was fully handled here (help, or an option the -/// command's parser would discard), null otherwise. Only inspects option-like tokens; positional -/// addresses, paths and passthrough values are never rejected. -/// -static int? HandleCommandLineOptions(string command, string[] argsTail) +static (Dictionary Values, HashSet Flags, string? FirstCommandArgument, + bool HelpRequested, string? UnrecognizedOption) ParseCommandLineOptions(string command, string[] args) { - if (CommandSpec(command) is not { } spec) + var spec = CommandSpec(command); + var valueNames = new HashSet(StringComparer.Ordinal) { "--project", "--profile" }; + var flagNames = new HashSet(StringComparer.Ordinal) { "--prefer-cached-inputs" }; + foreach (var option in spec?.Options ?? Array.Empty()) { - return null; + (option.Value is null ? flagNames : valueNames).Add(option.Name); } - if (argsTail.Any(static token => - string.Equals(token, "--help", StringComparison.Ordinal) || - string.Equals(token, "-h", StringComparison.Ordinal) || - string.Equals(token, "-?", StringComparison.Ordinal))) - { - return ShowCommandHelp(command); - } - - var values = new HashSet(StringComparer.Ordinal); - values.UnionWith(spec.Options.Where(static option => option.Value is not null).Select(static option => option.Name)); - values.UnionWith(GlobalValueOptions()); + var values = new Dictionary(StringComparer.Ordinal); var flags = new HashSet(StringComparer.Ordinal); - flags.UnionWith(spec.Options.Where(static option => option.Value is null).Select(static option => option.Name)); - flags.UnionWith(GlobalFlagOptions()); + string? firstCommandArgument = null; + string? unrecognizedOption = null; + var helpRequested = false; - for (var index = 0; index < argsTail.Length; index++) + for (var index = 1; index < args.Length; index++) { - var token = argsTail[index]; - if (values.Contains(token)) + var token = args[index]; + // The first --project and --profile pairs can precede the command's positional argument. + var firstGlobalValue = (token is "--project" or "--profile") && !values.ContainsKey(token); + if (!firstGlobalValue) + firstCommandArgument ??= token; + + if (valueNames.Contains(token)) { - // Step over the value so an option argument that happens to look - // like an option of its own is never rejected. - index++; + if (index + 1 == args.Length) + { + if (firstGlobalValue) throw new ArgumentException($"{token} requires a value."); + continue; + } + + var value = args[++index]; + values.TryAdd(token, value); + // Help historically takes precedence over command arguments, including value operands. + helpRequested |= !firstGlobalValue && IsHelpOption(value); continue; } - if (flags.Contains(token) || !token.StartsWith('-')) + if (flagNames.Contains(token)) { + flags.Add(token); continue; } - - Console.Error.WriteLine($"Unrecognized option '{token}' for command '{command}'."); - WriteUsage(Console.Error, new[] { command }); - return 2; + if (IsHelpOption(token)) + helpRequested = true; + else if (token.StartsWith('-')) + unrecognizedOption ??= token; } - return null; + return (values, flags, firstCommandArgument, spec is not null && helpRequested, + spec is not null ? unrecognizedOption : null); + + static bool IsHelpOption(string token) => token is "--help" or "-h" or "-?"; } ProgramImage LoadImage() @@ -3934,38 +3939,6 @@ static int ParseInt(string text) => ? int.Parse(text[2..], NumberStyles.HexNumber, CultureInfo.InvariantCulture) : int.Parse(text, CultureInfo.InvariantCulture); -static string? OptionValue(string[] argsTail, string name) -{ - var index = Array.IndexOf(argsTail, name); - if (index >= 0 && index + 1 < argsTail.Length) - { - return argsTail[index + 1]; - } - return null; -} - -static (string? Value, string[] Remaining) ExtractOption(string[] values, string name) -{ - for (var index = 0; index < values.Length; index++) - { - if (!string.Equals(values[index], name, StringComparison.Ordinal)) - { - continue; - } - if (index + 1 >= values.Length) - { - throw new ArgumentException($"{name} requires a value."); - } - - var remaining = values.Where((_, itemIndex) => itemIndex != index && itemIndex != index + 1).ToArray(); - return (values[index + 1], remaining); - } - - return (null, values); -} - -static bool HasFlag(string[] argsTail, string name) => argsTail.Any(a => string.Equals(a, name, StringComparison.Ordinal)); - /// /// Addresses a leaf splice must never cover: the decoded bytes aren't what the runtime /// executes there (native registration/fatal stub, exclusion, or a mod overlay rewrite). diff --git a/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs b/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs index f6817d3a1..aa3f529b6 100644 --- a/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs +++ b/translator/src/Translator.Core/Analysis/GuestAbiInterproceduralAnalyzer.cs @@ -16,9 +16,16 @@ public sealed record GuestAbiInterproceduralResult( /// public static class GuestAbiInterproceduralAnalyzer { + /// + /// If supplied, must come from the same bodies analyzed + /// without callee contracts and cover exactly the supplied function addresses. + /// public static GuestAbiInterproceduralResult Analyze( IReadOnlyDictionary functions, - IReadOnlyDictionary? externalContracts = null) + IReadOnlyDictionary? externalContracts = null, + bool retainBodies = true, + int? maxDegreeOfParallelism = null, + IReadOnlyDictionary? localContracts = null) { externalContracts ??= new Dictionary(); var addresses = functions.Keys.OrderBy(static address => address).ToArray(); @@ -26,21 +33,42 @@ public static GuestAbiInterproceduralResult Analyze( // GuestAbiContractAnalyzer.Analyze is pure and the round's contract snapshot is frozen, so // per-function analyses within a round can run on any thread. Determinism comes from // applying results serially in ascending-ordinal order, matching the old single-threaded loop. - var parallelOptions = new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }; - - // Resolving the IR by ordinal once keeps the caller-supplied dictionary - // off every parallel path: only this loop and BuildComponents touch it. - var bodies = new IrFunction[addresses.Length]; - for (var index = 0; index < addresses.Length; ++index) bodies[index] = functions[addresses[index]]; + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = maxDegreeOfParallelism ?? Environment.ProcessorCount + }; // Mirror of `contracts` keyed by ordinal. The fixpoint compares against // it from the parallel phase, where the dictionary must stay untouched. var current = new GuestAbiContract[addresses.Length]; - Parallel.For( - 0, - addresses.Length, - parallelOptions, - index => current[index] = GuestAbiContractAnalyzer.Analyze(bodies[index])); + if (localContracts is not null) + { + if (localContracts.Count != addresses.Length) + throw new ArgumentException("Local contracts must match the function addresses.", nameof(localContracts)); + for (var index = 0; index < addresses.Length; ++index) + { + if (!localContracts.TryGetValue(addresses[index], out var contract)) + throw new ArgumentException($"Missing local contract for 0x{addresses[index]:X8}.", nameof(localContracts)); + current[index] = contract; + } + } + + // Lazy callers must supply stable bodies and support concurrent reads. + var bodies = retainBodies ? new IrFunction[addresses.Length] : null; + if (bodies is not null) + { + for (var index = 0; index < addresses.Length; ++index) bodies[index] = functions[addresses[index]]; + } + IrFunction Body(int index) => bodies is not null ? bodies[index] : functions[addresses[index]]; + + if (localContracts is null) + { + Parallel.For( + 0, + addresses.Length, + parallelOptions, + index => current[index] = GuestAbiContractAnalyzer.Analyze(Body(index))); + } var contracts = new Dictionary(addresses.Length); for (var index = 0; index < addresses.Length; ++index) contracts.Add(addresses[index], current[index]); @@ -62,7 +90,7 @@ public static GuestAbiInterproceduralResult Analyze( (index, _, scratch) => { scratch.Clear(); - CollectDirectCallees(bodies[index], ordinals, scratch); + CollectDirectCallees(Body(index), ordinals, scratch); directCallees[index] = scratch.ToArray(); return scratch; }, @@ -95,7 +123,7 @@ public static GuestAbiInterproceduralResult Analyze( Parallel.For(0, worklistCount, parallelOptions, position => { var ordinal = worklist[position]; - var next = GuestAbiContractAnalyzer.Analyze(bodies[ordinal], visible); + var next = GuestAbiContractAnalyzer.Analyze(Body(ordinal), visible); roundResults[position] = Equivalent(current[ordinal], next) ? null : next; }); diff --git a/translator/src/Translator.Core/Translation/CanonicalIrDictionary.cs b/translator/src/Translator.Core/Translation/CanonicalIrDictionary.cs new file mode 100644 index 000000000..4a7acdb08 --- /dev/null +++ b/translator/src/Translator.Core/Translation/CanonicalIrDictionary.cs @@ -0,0 +1,67 @@ +using System.Collections; +using System.Diagnostics.CodeAnalysis; +using Translator.Core.Ir; + +namespace Translator.Core.Translation; + +/// +/// Fixed keys over canonical SSA decoded on demand without caching bodies. +/// The provider must supply a body for every address in the snapshot for as long +/// as the dictionary is used. +/// Concurrent readers require a thread-safe provider, such as CanonicalIrStore. +/// +public sealed class CanonicalIrDictionary : IReadOnlyDictionary +{ + private readonly ICanonicalIrProvider _provider; + private readonly IReadOnlyList _addresses; + private readonly HashSet _membership; + + public CanonicalIrDictionary(ICanonicalIrProvider provider, IEnumerable addresses) + { + ArgumentNullException.ThrowIfNull(provider); + ArgumentNullException.ThrowIfNull(addresses); + _provider = provider; + var snapshot = addresses.ToArray(); + _membership = new HashSet(snapshot); + if (_membership.Count != snapshot.Length) + throw new ArgumentException("Canonical IR addresses must be unique.", nameof(addresses)); + _addresses = Array.AsReadOnly(snapshot); + } + + public int Count => _addresses.Count; + public IEnumerable Keys => _addresses; + public IEnumerable Values + { + get + { + foreach (var address in _addresses) yield return this[address]; + } + } + + public IrFunction this[uint key] => TryGetValue(key, out var value) + ? value + : throw new KeyNotFoundException($"Address 0x{key:X8} is outside the canonical SSA snapshot."); + + public bool ContainsKey(uint key) => _membership.Contains(key); + + public bool TryGetValue(uint key, [MaybeNullWhen(false)] out IrFunction value) + { + if (!_membership.Contains(key)) + { + value = null; + return false; + } + + if (!_provider.TryGet(key, out value)) + throw new InvalidOperationException($"Missing canonical SSA for snapshot address 0x{key:X8}."); + return true; + } + + public IEnumerator> GetEnumerator() + { + foreach (var address in _addresses) + yield return new KeyValuePair(address, this[address]); + } + + IEnumerator IEnumerable.GetEnumerator() => GetEnumerator(); +} diff --git a/translator/tests/Translator.Tests/CanonicalIrDictionaryTests.cs b/translator/tests/Translator.Tests/CanonicalIrDictionaryTests.cs new file mode 100644 index 000000000..3411c0328 --- /dev/null +++ b/translator/tests/Translator.Tests/CanonicalIrDictionaryTests.cs @@ -0,0 +1,311 @@ +using System.Buffers.Binary; +using System.Collections; +using System.Text.Json; +using System.Threading; +using System.Threading.Tasks; +using Translator.Core.Analysis; +using Translator.Core.Ir; +using Translator.Core.Loading; +using Translator.Core.Translation; +using Xunit; + +namespace Translator.Tests; + +public sealed class CanonicalIrDictionaryTests +{ + [Fact] + public void KeysAndMembershipUseAnIndependentSnapshotWithoutDecoding() + { + var store = new CanonicalIrStore(); + store.Put(20, Function("second", new IrReturn(null))); + store.Put(10, Function("first", new IrReturn(null))); + store.Put(30, Function("outside", new IrReturn(null))); + var provider = new CountingProvider(store); + var addresses = new uint[] { 20, 10 }; + var view = new CanonicalIrDictionary(provider, addresses); + addresses[0] = 0; + + Assert.Equal(2, view.Count); + Assert.Equal(new uint[] { 20, 10 }, view.Keys); + Assert.True(view.ContainsKey(10)); + Assert.False(view.ContainsKey(0)); + Assert.False(view.ContainsKey(30)); + Assert.False(view.TryGetValue(30, out var missing)); + Assert.Null(missing); + Assert.Throws(() => view[30]); + Assert.Equal(0, provider.Reads); + + var first = view[10]; + Assert.NotSame(first, view[10]); + Assert.Equal(2, provider.Reads); + Assert.Equal("second", Assert.Single(view.Values.Take(1)).Name); + Assert.Equal(3, provider.Reads); + using var entries = view.GetEnumerator(); + Assert.Equal(3, provider.Reads); + Assert.True(entries.MoveNext()); + Assert.Equal(20u, entries.Current.Key); + Assert.Equal(4, provider.Reads); + Assert.Equal(view.Keys, view.Select(static pair => pair.Key)); + Assert.Equal(new[] { "second", "first" }, view.Values.Select(static body => body.Name)); + Assert.Equal(view.Keys, ((IEnumerable)view).Cast>().Select(static pair => pair.Key)); + } + + [Fact] + public void ReplacedBodiesAreReadAndMissingSnapshotBodiesViolateTheInvariant() + { + var store = new CanonicalIrStore(); + store.Put(10, Function("original", new IrReturn(null))); + var view = new CanonicalIrDictionary(store, new uint[] { 10 }); + Assert.Equal("original", view[10].Name); + store.Put(10, Function("replacement", new IrReturn(null))); + Assert.Equal("replacement", view[10].Name); + store.Remove(10); + + Assert.True(view.ContainsKey(10)); + Assert.Equal(new uint[] { 10 }, view.Keys); + Assert.Equal(view.Keys.Count(), view.Count); + var error = Assert.Throws(() => view.TryGetValue(10, out _)); + Assert.Contains("0x0000000A", error.Message, StringComparison.Ordinal); + Assert.Throws(() => view[10]); + Assert.Throws(() => view.Values.ToArray()); + Assert.Throws(() => view.ToArray()); + } + + [Fact] + public void ConstructorRejectsNullInputsAndDuplicateKeys() + { + var store = new CanonicalIrStore(); + Assert.Throws(() => new CanonicalIrDictionary(null!, Array.Empty())); + Assert.Throws(() => new CanonicalIrDictionary(store, null!)); + Assert.Throws(() => new CanonicalIrDictionary(store, new uint[] { 10, 20, 10 })); + var empty = new CanonicalIrDictionary(store, Array.Empty()); + Assert.Empty(empty); + Assert.Empty(empty.Keys); + Assert.Empty(empty.Values); + } + + [Fact] + public void ConcurrentReadersEnumerateInSnapshotOrderAndDecodeIndependentBodies() + { + var store = new CanonicalIrStore(); + var addresses = Enumerable.Range(1, 16).Reverse().Select(static index => (uint)index).ToArray(); + foreach (var address in addresses) store.Put(address, Function($"f{address}", new IrReturn(null))); + var provider = new CountingProvider(store); + var view = new CanonicalIrDictionary(provider, addresses); + var bodies = new IrFunction[32]; + + Parallel.For(0, bodies.Length, new ParallelOptions { MaxDegreeOfParallelism = 4 }, index => + { + Assert.Equal(addresses, view.Keys); + Assert.All(addresses, address => Assert.True(view.ContainsKey(address))); + Assert.False(view.TryGetValue(0, out _)); + Assert.Equal(addresses, view.Select(static pair => pair.Key)); + Assert.Equal(addresses.Select(static address => $"f{address}"), view.Values.Select(static body => body.Name)); + bodies[index] = view[1]; + }); + + Assert.Equal(bodies.Length * (2 * addresses.Length + 1), provider.Reads); + for (var index = 1; index < bodies.Length; ++index) Assert.NotSame(bodies[0], bodies[index]); + } + + [Fact] + public void LazyCanonicalBodyPreservesSerializedIrAndLoweredSource() + { + var memory = new byte[8]; + BinaryPrimitives.WriteUInt32BigEndian(memory.AsSpan(0, 4), 0x38630001u); // addi r3,r3,1 + BinaryPrimitives.WriteUInt32BigEndian(memory.AsSpan(4, 4), 0x4E800020u); // blr + var image = new ProgramImage(memory, + AddressRange.FromStartAndSize(MemoryLayout.RamBase, (uint)memory.Length), + AddressRange.FromStartAndSize(MemoryLayout.RamBase, (uint)memory.Length), default, "bounded-ir-test"); + var translator = new FunctionTranslator(image); + var options = TranslationOptions.Default with { PreferredName = "bounded_ir_test" }; + var canonical = translator.Discover(MemoryLayout.RamBase, options).Ssa.Function; + var store = new CanonicalIrStore(); + store.Put(MemoryLayout.RamBase, canonical); + var view = new CanonicalIrDictionary(store, new[] { MemoryLayout.RamBase }); + var buffered = translator.LowerCanonical(MemoryLayout.RamBase, canonical, options); + var lazy = translator.LowerCanonical(MemoryLayout.RamBase, view[MemoryLayout.RamBase], options); + + Assert.Equal(buffered.CxxCode, lazy.CxxCode); + Assert.Equal(JsonSerializer.Serialize(canonical), JsonSerializer.Serialize(view[MemoryLayout.RamBase])); + } + + [Theory] + [InlineData(1)] + [InlineData(2)] + [InlineData(-1)] + public void LazyAnalysisMatchesBufferedContractsAndComponents(int parallelism) + { + var functions = new Dictionary + { + [0x80003000] = Function("leaf", + new IrAssign("r7", IrValue.Register("r6")), + new IrCall("lr", "0x80004000", Array.Empty()), new IrReturn(null)), + [0x80001000] = Function("first", + new IrAssign("r5", IrValue.Register("r3")), + new IrCall("lr", "0x80002000", Array.Empty()), new IrReturn(null)), + [0x80002000] = Function("second", + new IrAssign("r6", IrValue.Register("r4")), + new IrCall("lr", "0x80001000", Array.Empty()), + new IrCall("lr", "0x80003000", Array.Empty()), new IrReturn(null)), + [0x80005000] = Function("self", + new IrCall("lr", "0x80005000", Array.Empty()), new IrReturn(null)), + [0x80006000] = Function("unknown", + new IrCall("lr", "0x80007000", Array.Empty()), + new IrIndirectCall("lr", IrValue.Register("ctr"), Array.Empty()), new IrReturn(null)) + }; + var external = new Dictionary + { + [0x80004000] = GuestAbiContractAnalyzer.Analyze(Function("external", + new IrAssign("r8", IrValue.Register("r9")), new IrReturn(null))) + }; + var store = new CanonicalIrStore(); + foreach (var pair in functions) store.Put(pair.Key, pair.Value); + var view = new CanonicalIrDictionary(store, functions.Keys.Reverse()); + var buffered = GuestAbiInterproceduralAnalyzer.Analyze(functions, external); + var lazy = GuestAbiInterproceduralAnalyzer.Analyze(view, external, + retainBodies: false, maxDegreeOfParallelism: parallelism); + + Assert.Equal(JsonSerializer.Serialize(buffered), JsonSerializer.Serialize(lazy)); + Assert.Equal(functions.Keys.OrderBy(static address => address), lazy.Contracts.Keys); + Assert.Contains(lazy.StronglyConnectedComponents, + component => component.SequenceEqual(new uint[] { 0x80001000, 0x80002000 })); + Assert.Contains(lazy.StronglyConnectedComponents, + component => component.SequenceEqual(new uint[] { 0x80005000 })); + Assert.True(lazy.Contracts[0x80006000].HasFullSynchronizationFence); + Assert.NotEqual(0u, lazy.Contracts[0x80003000].GprReadBeforeWriteMask & (1u << 9)); + } + + [Fact] + public void LocalContractsSaveOneLazyBodyReadWithoutChangingTheResult() + { + var store = new CanonicalIrStore(); + var body = Function("leaf", new IrReturn(null)); + store.Put(10, body); + var provider = new CountingProvider(store); + var view = new CanonicalIrDictionary(provider, new uint[] { 10 }); + + var buffered = GuestAbiInterproceduralAnalyzer.Analyze(view); + Assert.Equal(1, provider.Reads); + var lazy = GuestAbiInterproceduralAnalyzer.Analyze(view, retainBodies: false, maxDegreeOfParallelism: 1); + Assert.Equal(4, provider.Reads); + Assert.Equal(JsonSerializer.Serialize(buffered), JsonSerializer.Serialize(lazy)); + + var localContracts = new Dictionary { [10] = GuestAbiContractAnalyzer.Analyze(body) }; + var seeded = GuestAbiInterproceduralAnalyzer.Analyze(view, retainBodies: false, + maxDegreeOfParallelism: 1, localContracts: localContracts); + Assert.Equal(6, provider.Reads); + Assert.Equal(JsonSerializer.Serialize(lazy), JsonSerializer.Serialize(seeded)); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void LocalContractsMustMatchFunctionAddressesBeforeDecoding(bool includeUnrelatedAddress) + { + var provider = new CountingProvider(new CanonicalIrStore()); + var view = new CanonicalIrDictionary(provider, new uint[] { 10 }); + var localContracts = new Dictionary(); + if (includeUnrelatedAddress) + localContracts[20] = GuestAbiContractAnalyzer.Analyze(Function("other", new IrReturn(null))); + + var error = Assert.Throws(() => GuestAbiInterproceduralAnalyzer.Analyze( + view, retainBodies: false, maxDegreeOfParallelism: 1, localContracts: localContracts)); + Assert.Equal("localContracts", error.ParamName); + Assert.Equal(0, provider.Reads); + } + + [Fact] + public void LazyAnalysisRevisitsCallersOfInlineThunksAfterTheirCalleesChange() + { + var range = GuestSaveRestoreThunks.Current.SaveGpr; + Assert.NotNull(range); + var thunk = range!.BaseAddress; + var functions = new Dictionary + { + [0x80001000] = Function("caller", + new IrCall("lr", $"0x{thunk:X8}", Array.Empty()), new IrReturn(null)), + [thunk] = Function("thunk", + new IrCall("lr", "0x80002000", Array.Empty()), new IrReturn(null)), + [0x80002000] = Function("middle", + new IrCall("lr", "0x80003000", Array.Empty()), new IrReturn(null)), + [0x80003000] = Function("leaf", + new IrAssign("r8", IrValue.Register("r9")), new IrReturn(null)) + }; + var store = new CanonicalIrStore(); + foreach (var pair in functions) store.Put(pair.Key, pair.Value); + var buffered = GuestAbiInterproceduralAnalyzer.Analyze(functions); + var lazy = GuestAbiInterproceduralAnalyzer.Analyze(new CanonicalIrDictionary(store, functions.Keys), + retainBodies: false, maxDegreeOfParallelism: 2, + localContracts: functions.ToDictionary(static pair => pair.Key, + static pair => GuestAbiContractAnalyzer.Analyze(pair.Value))); + + Assert.Equal(JsonSerializer.Serialize(buffered), JsonSerializer.Serialize(lazy)); + Assert.DoesNotContain(thunk, lazy.Contracts[0x80001000].DirectCallTargets); + Assert.Equal(1u << 9, lazy.Contracts[0x80001000].GprReadBeforeWriteMask); + Assert.NotEqual(0u, lazy.Contracts[0x80001000].GprPossibleWriteMask & (1u << 8)); + } + + [Theory] + [InlineData(true, 0)] + [InlineData(false, 0)] + [InlineData(true, -2)] + [InlineData(false, -2)] + public void InvalidParallelismFailsBeforeDecoding(bool retainBodies, int parallelism) + { + var provider = new CountingProvider(new CanonicalIrStore()); + var view = new CanonicalIrDictionary(provider, new uint[] { 10 }); + Assert.Throws(() => GuestAbiInterproceduralAnalyzer.Analyze( + view, retainBodies: retainBodies, maxDegreeOfParallelism: parallelism)); + Assert.Equal(0, provider.Reads); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public void EmptyAnalysisPreservesEmptyResults(bool retainBodies) + { + var view = new CanonicalIrDictionary(new CanonicalIrStore(), Array.Empty()); + var result = GuestAbiInterproceduralAnalyzer.Analyze(view, retainBodies: retainBodies, maxDegreeOfParallelism: 1); + Assert.Empty(result.Contracts); + Assert.Empty(result.StronglyConnectedComponents); + } + + [Fact] + public void SingleWorkerLimitAppliesToLazyProviderReadsInEveryAnalysisPass() + { + var store = new CanonicalIrStore(); + var addresses = Enumerable.Range(1, 32).Select(static index => (uint)index).ToArray(); + foreach (var address in addresses) store.Put(address, Function($"f{address}", new IrReturn(null))); + var provider = new CountingProvider(store, requireSerial: true); + var result = GuestAbiInterproceduralAnalyzer.Analyze(new CanonicalIrDictionary(provider, addresses), + retainBodies: false, maxDegreeOfParallelism: 1); + Assert.Equal(addresses.Length, result.Contracts.Count); + Assert.Equal(3 * addresses.Length, provider.Reads); + } + + private static IrFunction Function(string name, params IrInstruction[] instructions) => + new(name, "entry", new[] { new IrBasicBlock("entry", instructions) }); + + private sealed class CountingProvider(ICanonicalIrProvider inner, bool requireSerial = false) : ICanonicalIrProvider + { + private int _reads; + private int _active; + public int Reads => Volatile.Read(ref _reads); + + public bool TryGet(uint entryPoint, out IrFunction function) + { + var active = Interlocked.Increment(ref _active); + try + { + if (requireSerial) Assert.Equal(1, active); + Interlocked.Increment(ref _reads); + return inner.TryGet(entryPoint, out function); + } + finally + { + Interlocked.Decrement(ref _active); + } + } + } +} diff --git a/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs b/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs index 7f9254f25..7461f3be0 100644 --- a/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs +++ b/translator/tests/Translator.Tests/TranslateRecursiveRuntimeConfigTests.cs @@ -1,4 +1,5 @@ using System.Reflection; +using System.Diagnostics; using System.Threading.Tasks; using Translator.Cli.Configuration; using Translator.Core.Mods; @@ -8,6 +9,130 @@ namespace Translator.Tests; public sealed class TranslateRecursiveRuntimeConfigTests { + [Fact] + public void BoundedIrMatchesBufferedFilesMetadataAndBundleAcrossThreadCounts() + { + var root = Path.Combine(Path.GetTempPath(), $"translator-bounded-ir-{Guid.NewGuid():N}"); + Directory.CreateDirectory(root); + try + { + const uint entry = 0x80001000u; + File.WriteAllBytes(Path.Combine(root, "main.dol"), SyntheticDolFactory.CreateBytes(entry, + sections: + [ + SyntheticDolFactory.Text(0, entry, + 0x38A30001u, // addi r5,r3,1 + 0x4800001Du, // bl 0x80001020 + 0x38650001u, // addi r3,r5,1 + 0x4E800020u), + SyntheticDolFactory.Text(1, entry + 0x20, + 0x38C40001u, // addi r6,r4,1 + 0x4BFFFFDDu, // bl 0x80001000 + 0x38660001u, // addi r3,r6,1 + 0x4E800020u) + ])); + var projectPath = Path.Combine(root, "recomp.yml"); + File.WriteAllText(projectPath, + """ + schema_version: 1 + project: + id: bounded-ir-test + memory: + sda_base: 0x80002000 + sda2_base: 0x80003000 + inputs: + dol: + path: main.dol + translation: + entry_points: [0x80001000, 0x80001020] + output: + root: generated + """); + + (Dictionary Sources, byte[] Metadata, byte[] Bundle) Translate( + string name, int threads, bool bounded, bool emitFiles = true) + { + var output = Path.Combine(root, name); + var functions = Path.Combine(output, "functions"); + var metadata = Path.Combine(output, "metadata.json"); + var bundle = Path.Combine(output, "sources.bundle"); + var args = new List + { + "translate-recursive", $"0x{entry:X8}", "--project", projectPath, + "--outdir", functions, "--output-metadata", metadata, + "--production-source-bundle", bundle, "--threads", threads.ToString() + }; + if (bounded) args.Add("--bounded-ir"); + if (!emitFiles) args.Add("--no-function-files"); + var run = InvokeCliProcess(root, args.ToArray()); + Assert.Equal(0, run.ExitCode); + Assert.Contains( + $"[translator] Canonical IR mode: {(bounded ? "bounded" : "buffered")}.", + run.Output, StringComparison.Ordinal); + var sources = Directory.Exists(functions) + ? Directory.GetFiles(functions, "*.cpp").ToDictionary( + static path => Path.GetFileName(path), static path => File.ReadAllText(path)) + : new Dictionary(); + return (sources, File.ReadAllBytes(metadata), File.ReadAllBytes(bundle)); + } + + var buffered = Translate("buffered", 1, bounded: false); + Assert.Equal(2, buffered.Sources.Count); + + var metadataCollision = InvokeCliProcess(root, "translate-recursive", "--project", projectPath, + $"0x{entry:X8}", "--outdir", "--output-metadata", "--bounded-ir", + "--output-metadata", "result.json", "--output-metadata", "ignored.json", "--threads", "0x1"); + Assert.Equal(0, metadataCollision.ExitCode); + Assert.Contains("[translator] Canonical IR mode: bounded.", metadataCollision.Output, StringComparison.Ordinal); + Assert.Contains("[translator] Using up to 1 worker thread(s).", metadataCollision.Output, StringComparison.Ordinal); + Assert.Equal(2, BaseTranslationOutputMetadataFile.Read(Path.Combine(root, "result.json")).Functions.Count); + Assert.Equal(2, Directory.GetFiles(Path.Combine(root, "--output-metadata"), "*.cpp").Length); + Assert.False(File.Exists(Path.Combine(root, "ignored.json"))); + Assert.False(File.Exists(Path.Combine(root, "--bounded-ir"))); + + var flagOperands = InvokeCliProcess(root, "translate-recursive", $"0x{entry:X8}", + "--outdir", "--bounded-ir", "--output-metadata", "--project", + "--production-source-bundle", "--no-function-files", "--threads", "-1", "--project", projectPath); + Assert.Equal(0, flagOperands.ExitCode); + Assert.Contains("[translator] Canonical IR mode: buffered.", flagOperands.Output, StringComparison.Ordinal); + Assert.Contains("[translator] Using up to 1 worker thread(s).", flagOperands.Output, StringComparison.Ordinal); + Assert.Equal(2, Directory.GetFiles(Path.Combine(root, "--bounded-ir"), "*.cpp").Length); + Assert.Equal(2, BaseTranslationOutputMetadataFile.Read(Path.Combine(root, "--project")).Functions.Count); + Assert.True(File.Exists(Path.Combine(root, "--no-function-files"))); + + foreach (var threads in new[] { 1, 3 }) + { + var lazy = Translate($"lazy-{threads}", threads, bounded: true); + Assert.Equal(buffered.Sources.Keys.OrderBy(static name => name), + lazy.Sources.Keys.OrderBy(static name => name)); + foreach (var pair in buffered.Sources) Assert.Equal(pair.Value, lazy.Sources[pair.Key]); + Assert.Equal(buffered.Metadata, lazy.Metadata); + Assert.Equal(buffered.Bundle, lazy.Bundle); + } + var bundleOnly = Translate("bundle-only", 3, bounded: true, emitFiles: false); + Assert.Empty(bundleOnly.Sources); + Assert.Equal(buffered.Metadata, bundleOnly.Metadata); + Assert.Equal(buffered.Bundle, bundleOnly.Bundle); + } + finally + { + if (Directory.Exists(root)) Directory.Delete(root, recursive: true); + } + } + + [Theory] + [InlineData("--unknown", "--help")] + [InlineData("--unknown", "-h")] + [InlineData("--outdir", "-?")] + public void CommandHelpTakesPrecedenceOverOtherArguments(string option, string help) + { + var run = InvokeCliProcess(Path.GetTempPath(), "translate-recursive", option, help); + Assert.Equal(0, run.ExitCode); + Assert.Contains("translator translate-recursive ", run.Output, StringComparison.Ordinal); + Assert.Contains("[--bounded-ir]", run.Output, StringComparison.Ordinal); + Assert.Empty(run.Error); + } + [Fact] public void UnsupportedOpcodeFailsStrictTranslation() { @@ -187,6 +312,29 @@ public void RepeatedRunsReuseUnchangedOutputsAndPruneStaleFiles() } } + private static (int ExitCode, string Output, string Error) InvokeCliProcess(string workingDirectory, params string[] args) + { + var startInfo = new ProcessStartInfo("dotnet") + { + WorkingDirectory = workingDirectory, + RedirectStandardOutput = true, + RedirectStandardError = true, + ArgumentList = { typeof(TranslationProjectConfig).Assembly.Location } + }; + foreach (var arg in args) startInfo.ArgumentList.Add(arg); + + using var process = Process.Start(startInfo)!; + var output = process.StandardOutput.ReadToEndAsync(); + var error = process.StandardError.ReadToEndAsync(); + if (!process.WaitForExit(30_000)) + { + process.Kill(entireProcessTree: true); + process.WaitForExit(); + throw new TimeoutException("Translator subprocess timed out."); + } + return (process.ExitCode, output.GetAwaiter().GetResult(), error.GetAwaiter().GetResult()); + } + private static int InvokeCli(params string[] args) { var entryPoint = typeof(TranslationProjectConfig).Assembly.EntryPoint;