diff --git a/source/runtime/core/private/threading/sync/Mutex.cpp b/source/runtime/core/private/threading/sync/Mutex.cpp index 39ea0ba5..529dd07d 100644 --- a/source/runtime/core/private/threading/sync/Mutex.cpp +++ b/source/runtime/core/private/threading/sync/Mutex.cpp @@ -39,19 +39,34 @@ GP_COLD void Mutex::lockSlow() noexcept continue; } - const gp::UInt32 prev = m_state.fetchOr(kMutexLockedBit | kMutexMayHaveWaitersBit, gp::MemoryOrder::Acquire); - if ((prev & kMutexLockedBit) == 0u) + gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed); + for (;;) { - // If we stole the lock while setting kMutexMayHaveWaitersBit and no prior waiters were flagged, - // clear the redundant waiter bit so unlock() stays on the 1-instruction fast path. - if ((prev & kMutexMayHaveWaitersBit) == 0u) + if ((expected & kMutexLockedBit) == 0u) { - m_state.store(kMutexLockedBit, gp::MemoryOrder::Relaxed); - } + if (m_state.compareExchangeWeak( + expected, expected | kMutexLockedBit, gp::MemoryOrder::Acquire, gp::MemoryOrder::Relaxed + )) + { #if GP_DO_CHECK - Fiber::adjustActiveNonYieldingLockCount(1); + Fiber::adjustActiveNonYieldingLockCount(1); #endif - return; + return; + } + continue; + } + + if ((expected & kMutexMayHaveWaitersBit) == 0u) + { + if (!m_state.compareExchangeWeak( + expected, expected | kMutexMayHaveWaitersBit, gp::MemoryOrder::Relaxed, gp::MemoryOrder::Relaxed + )) + { + continue; + } + } + + break; } (void)gp::platform::Futex::waitOnAddress(m_state, kMutexLockedBit | kMutexMayHaveWaitersBit); diff --git a/source/runtime/core/private/threading/sync/RecursiveMutex.cpp b/source/runtime/core/private/threading/sync/RecursiveMutex.cpp index b443b664..432f2507 100644 --- a/source/runtime/core/private/threading/sync/RecursiveMutex.cpp +++ b/source/runtime/core/private/threading/sync/RecursiveMutex.cpp @@ -149,19 +149,36 @@ GP_COLD void RecursiveMutex::lockContended(const gp::UInt64 callerToken) noexcep continue; } - const gp::UInt32 prev = m_state.fetchOr(kMutexLockedBit | kMutexMayHaveWaitersBit, gp::MemoryOrder::Acquire); - if ((prev & kMutexLockedBit) == 0u) + gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed); + for (;;) { - if ((prev & kMutexMayHaveWaitersBit) == 0u) + if ((expected & kMutexLockedBit) == 0u) { - m_state.store(kMutexLockedBit, gp::MemoryOrder::Relaxed); - } - m_ownerToken.store(callerToken, gp::MemoryOrder::Relaxed); - m_recursionDepth = 1u; + if (m_state.compareExchangeWeak( + expected, expected | kMutexLockedBit, gp::MemoryOrder::Acquire, gp::MemoryOrder::Relaxed + )) + { + m_ownerToken.store(callerToken, gp::MemoryOrder::Relaxed); + m_recursionDepth = 1u; #if GP_DO_CHECK - Fiber::adjustActiveNonYieldingLockCount(1); + Fiber::adjustActiveNonYieldingLockCount(1); #endif - return; + return; + } + continue; + } + + if ((expected & kMutexMayHaveWaitersBit) == 0u) + { + if (!m_state.compareExchangeWeak( + expected, expected | kMutexMayHaveWaitersBit, gp::MemoryOrder::Relaxed, gp::MemoryOrder::Relaxed + )) + { + continue; + } + } + + break; } (void)gp::platform::Futex::waitOnAddress(m_state, kMutexLockedBit | kMutexMayHaveWaitersBit); diff --git a/source/runtime/core/private/threading/sync/SharedMutex.cpp b/source/runtime/core/private/threading/sync/SharedMutex.cpp index f324e02d..bd32033a 100644 --- a/source/runtime/core/private/threading/sync/SharedMutex.cpp +++ b/source/runtime/core/private/threading/sync/SharedMutex.cpp @@ -82,42 +82,6 @@ GP_COLD void SharedMutex::lockSlow() noexcept else { (void)gp::platform::Futex::waitOnAddress(m_state, expected); - - // After waking from futex, preserve kSharedMutexWriterWaitingBit when claiming exclusive ownership so - // any additional parked writers are woken when we subsequently unlock(). - for (;;) - { - gp::UInt32 cur = m_state.load(gp::MemoryOrder::Relaxed); - if ((cur & (kSharedMutexWriterLockedBit | kSharedMutexReaderCountMask)) == 0u) - { - if (m_state.compareExchangeWeak( - cur, - cur | kSharedMutexWriterLockedBit | kSharedMutexWriterWaitingBit, - gp::MemoryOrder::Acquire, - gp::MemoryOrder::Relaxed - )) - { -#if GP_DO_CHECK - Fiber::adjustActiveNonYieldingLockCount(1); -#endif - return; - } - continue; - } - - if ((cur & kSharedMutexWriterWaitingBit) == 0u) - { - if (!m_state.compareExchangeWeak( - cur, cur | kSharedMutexWriterWaitingBit, gp::MemoryOrder::Relaxed, gp::MemoryOrder::Relaxed - )) - { - continue; - } - cur |= kSharedMutexWriterWaitingBit; - } - - (void)gp::platform::Futex::waitOnAddress(m_state, cur); - } } } } @@ -127,30 +91,26 @@ GP_COLD void SharedMutex::unlockSlow() noexcept gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed); for (;;) { - if ((expected & kSharedMutexWriterWaitingBit) != 0u) + const gp::UInt32 waitBits = expected & (kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit); + if (waitBits == 0u) { - const gp::UInt32 desired = expected & ~kSharedMutexWriterWaitingBit; - if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed)) - { - gp::platform::Futex::wakeOne(m_state); - return; - } - continue; + return; } - if ((expected & kSharedMutexReaderWaitingBit) != 0u) + const gp::UInt32 desired = expected & ~(kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit); + if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed)) { - const gp::UInt32 desired = expected & ~kSharedMutexReaderWaitingBit; - if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed)) + if ((waitBits & kSharedMutexWriterWaitingBit) != 0u) + { + gp::platform::Futex::wakeAll(m_state); + } + if ((waitBits & kSharedMutexReaderWaitingBit) != 0u) { m_readerWakeEpoch.fetchAdd(1u, gp::MemoryOrder::Release); gp::platform::Futex::wakeAll(m_readerWakeEpoch); - return; } - continue; + return; } - - return; } } @@ -235,13 +195,26 @@ GP_COLD void SharedMutex::lockSharedSlow() noexcept GP_COLD void SharedMutex::unlockSharedSlow() noexcept { gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed); - while ((expected & kSharedMutexReaderCountMask) == 0u && (expected & kSharedMutexWriterWaitingBit) != 0u) + while ((expected & (kSharedMutexWriterLockedBit | kSharedMutexReaderCountMask)) == 0u) { - if (m_state.compareExchangeWeak( - expected, expected & ~kSharedMutexWriterWaitingBit, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed - )) + const gp::UInt32 waitBits = expected & (kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit); + if (waitBits == 0u) { - gp::platform::Futex::wakeOne(m_state); + return; + } + + const gp::UInt32 desired = expected & ~(kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit); + if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed)) + { + if ((waitBits & kSharedMutexWriterWaitingBit) != 0u) + { + gp::platform::Futex::wakeAll(m_state); + } + if ((waitBits & kSharedMutexReaderWaitingBit) != 0u) + { + m_readerWakeEpoch.fetchAdd(1u, gp::MemoryOrder::Release); + gp::platform::Futex::wakeAll(m_readerWakeEpoch); + } return; } } diff --git a/source/runtime/core/tests/diagnostics/Assert.tests.cpp b/source/runtime/core/tests/diagnostics/Assert.tests.cpp index af192324..dd2205ae 100644 --- a/source/runtime/core/tests/diagnostics/Assert.tests.cpp +++ b/source/runtime/core/tests/diagnostics/Assert.tests.cpp @@ -286,6 +286,8 @@ TEST(AssertTest, ConcurrentEnsureCallSiteFiresOnlyOnceAcrossThreads) #if GP_DO_CHECK && defined(GTEST_HAS_DEATH_TEST) && GTEST_HAS_DEATH_TEST TEST(AssertDeathTest, DefaultHandlerLogsAndAbortsProcessOnFatalCheck) { + GTEST_FLAG_SET(death_test_style, "threadsafe"); + EXPECT_DEATH( { setAssertHandler(nullptr); diff --git a/source/runtime/core/tests/logging/Logger.tests.cpp b/source/runtime/core/tests/logging/Logger.tests.cpp index ac6770d3..5e7759a2 100644 --- a/source/runtime/core/tests/logging/Logger.tests.cpp +++ b/source/runtime/core/tests/logging/Logger.tests.cpp @@ -499,6 +499,8 @@ TEST(LoggerTest, FatalHandlerRegistrationAndQuery) #if defined(GTEST_HAS_DEATH_TEST) && GTEST_HAS_DEATH_TEST TEST(LoggerDeathTest, FatalMacroAndFunctionFlushAndAbortProcess) { + GTEST_FLAG_SET(death_test_style, "threadsafe"); + EXPECT_DEATH( { Logger& globalLogger = Logger::instance(); diff --git a/source/runtime/core/tests/memory/pointers/UniquePtr.tests.cpp b/source/runtime/core/tests/memory/pointers/UniquePtr.tests.cpp index f6f8f41e..a7e2e247 100644 --- a/source/runtime/core/tests/memory/pointers/UniquePtr.tests.cpp +++ b/source/runtime/core/tests/memory/pointers/UniquePtr.tests.cpp @@ -1416,6 +1416,8 @@ TEST_F(UniquePtrTest, CheckedBuildAssertsOnNullDereference) #if GP_DO_CHECK && defined(GTEST_HAS_DEATH_TEST) && GTEST_HAS_DEATH_TEST TEST_F(UniquePtrTest, CheckedBuildDeathOnNullDereferenceAndIndexing) { + GTEST_FLAG_SET(death_test_style, "threadsafe"); + EXPECT_DEATH( { gp::diagnostics::setAssertHandler(nullptr); diff --git a/source/runtime/core/tests/threading/sync/Mutex.tests.cpp b/source/runtime/core/tests/threading/sync/Mutex.tests.cpp index e9fb8ea2..fdafb82c 100644 --- a/source/runtime/core/tests/threading/sync/Mutex.tests.cpp +++ b/source/runtime/core/tests/threading/sync/Mutex.tests.cpp @@ -1057,6 +1057,11 @@ TEST(MutexTest, MultiWaiterFutexParkAndCascadeWakeupWithoutLostWakeups) gp::platform::Thread::spinWait(); } + while (!ctx.mutex.mayHaveWaiters()) + { + gp::platform::Thread::spinWait(); + } + // Sleep 2 ms so all 4 waiter threads exhaust their 32-iteration spin budget and park inside Futex::waitOnAddress gp::platform::Thread::sleep(2u); EXPECT_TRUE(ctx.mutex.mayHaveWaiters());