Skip to content
33 changes: 24 additions & 9 deletions source/runtime/core/private/threading/sync/Mutex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -39,19 +39,34 @@ GP_COLD void Mutex::lockSlow() noexcept
continue;
}

const gp::UInt32 prev = m_state.fetchOr(kMutexLockedBit | kMutexMayHaveWaitersBit, gp::MemoryOrder::Acquire);
if ((prev & kMutexLockedBit) == 0u)
gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed);
for (;;)
{
// If we stole the lock while setting kMutexMayHaveWaitersBit and no prior waiters were flagged,
// clear the redundant waiter bit so unlock() stays on the 1-instruction fast path.
if ((prev & kMutexMayHaveWaitersBit) == 0u)
if ((expected & kMutexLockedBit) == 0u)
{
m_state.store(kMutexLockedBit, gp::MemoryOrder::Relaxed);
}
if (m_state.compareExchangeWeak(
expected, expected | kMutexLockedBit, gp::MemoryOrder::Acquire, gp::MemoryOrder::Relaxed
))
{
#if GP_DO_CHECK
Fiber::adjustActiveNonYieldingLockCount(1);
Fiber::adjustActiveNonYieldingLockCount(1);
#endif
return;
return;
}
continue;
}

if ((expected & kMutexMayHaveWaitersBit) == 0u)
{
if (!m_state.compareExchangeWeak(
expected, expected | kMutexMayHaveWaitersBit, gp::MemoryOrder::Relaxed, gp::MemoryOrder::Relaxed
))
{
continue;
}
}

break;
}

(void)gp::platform::Futex::waitOnAddress(m_state, kMutexLockedBit | kMutexMayHaveWaitersBit);
Expand Down
35 changes: 26 additions & 9 deletions source/runtime/core/private/threading/sync/RecursiveMutex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -149,19 +149,36 @@ GP_COLD void RecursiveMutex::lockContended(const gp::UInt64 callerToken) noexcep
continue;
}

const gp::UInt32 prev = m_state.fetchOr(kMutexLockedBit | kMutexMayHaveWaitersBit, gp::MemoryOrder::Acquire);
if ((prev & kMutexLockedBit) == 0u)
gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed);
for (;;)
{
if ((prev & kMutexMayHaveWaitersBit) == 0u)
if ((expected & kMutexLockedBit) == 0u)
{
m_state.store(kMutexLockedBit, gp::MemoryOrder::Relaxed);
}
m_ownerToken.store(callerToken, gp::MemoryOrder::Relaxed);
m_recursionDepth = 1u;
if (m_state.compareExchangeWeak(
expected, expected | kMutexLockedBit, gp::MemoryOrder::Acquire, gp::MemoryOrder::Relaxed
))
{
m_ownerToken.store(callerToken, gp::MemoryOrder::Relaxed);
m_recursionDepth = 1u;
#if GP_DO_CHECK
Fiber::adjustActiveNonYieldingLockCount(1);
Fiber::adjustActiveNonYieldingLockCount(1);
#endif
return;
return;
}
continue;
}

if ((expected & kMutexMayHaveWaitersBit) == 0u)
{
if (!m_state.compareExchangeWeak(
expected, expected | kMutexMayHaveWaitersBit, gp::MemoryOrder::Relaxed, gp::MemoryOrder::Relaxed
))
{
continue;
}
}

break;
}

(void)gp::platform::Futex::waitOnAddress(m_state, kMutexLockedBit | kMutexMayHaveWaitersBit);
Expand Down
85 changes: 29 additions & 56 deletions source/runtime/core/private/threading/sync/SharedMutex.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -82,42 +82,6 @@ GP_COLD void SharedMutex::lockSlow() noexcept
else
{
(void)gp::platform::Futex::waitOnAddress(m_state, expected);

// After waking from futex, preserve kSharedMutexWriterWaitingBit when claiming exclusive ownership so
// any additional parked writers are woken when we subsequently unlock().
for (;;)
{
gp::UInt32 cur = m_state.load(gp::MemoryOrder::Relaxed);
if ((cur & (kSharedMutexWriterLockedBit | kSharedMutexReaderCountMask)) == 0u)
{
if (m_state.compareExchangeWeak(
cur,
cur | kSharedMutexWriterLockedBit | kSharedMutexWriterWaitingBit,
gp::MemoryOrder::Acquire,
gp::MemoryOrder::Relaxed
))
{
#if GP_DO_CHECK
Fiber::adjustActiveNonYieldingLockCount(1);
#endif
return;
}
continue;
}

if ((cur & kSharedMutexWriterWaitingBit) == 0u)
{
if (!m_state.compareExchangeWeak(
cur, cur | kSharedMutexWriterWaitingBit, gp::MemoryOrder::Relaxed, gp::MemoryOrder::Relaxed
))
{
continue;
}
cur |= kSharedMutexWriterWaitingBit;
}

(void)gp::platform::Futex::waitOnAddress(m_state, cur);
}
}
}
}
Expand All @@ -127,30 +91,26 @@ GP_COLD void SharedMutex::unlockSlow() noexcept
gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed);
for (;;)
{
if ((expected & kSharedMutexWriterWaitingBit) != 0u)
const gp::UInt32 waitBits = expected & (kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit);
if (waitBits == 0u)
{
const gp::UInt32 desired = expected & ~kSharedMutexWriterWaitingBit;
if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed))
{
gp::platform::Futex::wakeOne(m_state);
return;
}
continue;
return;
}

if ((expected & kSharedMutexReaderWaitingBit) != 0u)
const gp::UInt32 desired = expected & ~(kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit);
if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed))
{
const gp::UInt32 desired = expected & ~kSharedMutexReaderWaitingBit;
if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed))
if ((waitBits & kSharedMutexWriterWaitingBit) != 0u)
{
gp::platform::Futex::wakeAll(m_state);
}
if ((waitBits & kSharedMutexReaderWaitingBit) != 0u)
{
m_readerWakeEpoch.fetchAdd(1u, gp::MemoryOrder::Release);
gp::platform::Futex::wakeAll(m_readerWakeEpoch);
return;
}
continue;
return;
}

return;
}
}

Expand Down Expand Up @@ -235,13 +195,26 @@ GP_COLD void SharedMutex::lockSharedSlow() noexcept
GP_COLD void SharedMutex::unlockSharedSlow() noexcept
{
gp::UInt32 expected = m_state.load(gp::MemoryOrder::Relaxed);
while ((expected & kSharedMutexReaderCountMask) == 0u && (expected & kSharedMutexWriterWaitingBit) != 0u)
while ((expected & (kSharedMutexWriterLockedBit | kSharedMutexReaderCountMask)) == 0u)
{
if (m_state.compareExchangeWeak(
expected, expected & ~kSharedMutexWriterWaitingBit, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed
))
const gp::UInt32 waitBits = expected & (kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit);
if (waitBits == 0u)
{
gp::platform::Futex::wakeOne(m_state);
return;
}

const gp::UInt32 desired = expected & ~(kSharedMutexWriterWaitingBit | kSharedMutexReaderWaitingBit);
if (m_state.compareExchangeWeak(expected, desired, gp::MemoryOrder::Release, gp::MemoryOrder::Relaxed))
{
if ((waitBits & kSharedMutexWriterWaitingBit) != 0u)
{
gp::platform::Futex::wakeAll(m_state);
}
if ((waitBits & kSharedMutexReaderWaitingBit) != 0u)
{
m_readerWakeEpoch.fetchAdd(1u, gp::MemoryOrder::Release);
gp::platform::Futex::wakeAll(m_readerWakeEpoch);
}
return;
}
}
Expand Down
2 changes: 2 additions & 0 deletions source/runtime/core/tests/diagnostics/Assert.tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -286,6 +286,8 @@ TEST(AssertTest, ConcurrentEnsureCallSiteFiresOnlyOnceAcrossThreads)
#if GP_DO_CHECK && defined(GTEST_HAS_DEATH_TEST) && GTEST_HAS_DEATH_TEST
TEST(AssertDeathTest, DefaultHandlerLogsAndAbortsProcessOnFatalCheck)
{
GTEST_FLAG_SET(death_test_style, "threadsafe");

EXPECT_DEATH(
{
setAssertHandler(nullptr);
Expand Down
2 changes: 2 additions & 0 deletions source/runtime/core/tests/logging/Logger.tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -499,6 +499,8 @@ TEST(LoggerTest, FatalHandlerRegistrationAndQuery)
#if defined(GTEST_HAS_DEATH_TEST) && GTEST_HAS_DEATH_TEST
TEST(LoggerDeathTest, FatalMacroAndFunctionFlushAndAbortProcess)
{
GTEST_FLAG_SET(death_test_style, "threadsafe");

EXPECT_DEATH(
{
Logger& globalLogger = Logger::instance();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1416,6 +1416,8 @@ TEST_F(UniquePtrTest, CheckedBuildAssertsOnNullDereference)
#if GP_DO_CHECK && defined(GTEST_HAS_DEATH_TEST) && GTEST_HAS_DEATH_TEST
TEST_F(UniquePtrTest, CheckedBuildDeathOnNullDereferenceAndIndexing)
{
GTEST_FLAG_SET(death_test_style, "threadsafe");

EXPECT_DEATH(
{
gp::diagnostics::setAssertHandler(nullptr);
Expand Down
5 changes: 5 additions & 0 deletions source/runtime/core/tests/threading/sync/Mutex.tests.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1057,6 +1057,11 @@ TEST(MutexTest, MultiWaiterFutexParkAndCascadeWakeupWithoutLostWakeups)
gp::platform::Thread::spinWait();
}

while (!ctx.mutex.mayHaveWaiters())
{
gp::platform::Thread::spinWait();
}

// Sleep 2 ms so all 4 waiter threads exhaust their 32-iteration spin budget and park inside Futex::waitOnAddress
gp::platform::Thread::sleep(2u);
EXPECT_TRUE(ctx.mutex.mayHaveWaiters());
Expand Down
Loading