avoid accessing mutex memory after atomic unlock

this change is needed to fix a race condition and ensure that it's
possible to unlock and destroy or unmap the mutex as soon as
pthread_mutex_lock succeeds. POSIX explicitly gives such an example in
the rationale and requires an implementation to allow such usage.

1 files changed, 7 insertions, 7 deletions

diff --git a/src/thread/pthread_mutex_trylock.c b/src/thread/pthread_mutex_trylock.c
index 1fb6d0f2..4a424bc9 100644
--- a/src/thread/pthread_mutex_trylock.c
+++ b/src/thread/pthread_mutex_trylock.c
@@ -2,15 +2,14 @@
 
 int pthread_mutex_trylock(pthread_mutex_t *m)
 {
-	int tid;
-	int own;
+	int tid, old, own;
 	pthread_t self;
 
 	if (m->_m_type == PTHREAD_MUTEX_NORMAL)
-		return a_swap(&m->_m_lock, EBUSY);
+		return a_cas(&m->_m_lock, 0, EBUSY) & EBUSY;
 
 	self = pthread_self();
-	tid = self->tid | 0x80000000;
+	tid = self->tid;
 
 	if (m->_m_type >= 4) {
 		if (!self->robust_list.off)
@@ -20,14 +19,15 @@ int pthread_mutex_trylock(pthread_mutex_t *m)
 		self->robust_list.pending = &m->_m_next;
 	}
 
-	if (m->_m_lock == tid && (m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE) {
+	old = m->_m_lock;
+	own = old & 0x7fffffff;
+	if (own == tid && (m->_m_type&3) == PTHREAD_MUTEX_RECURSIVE) {
 		if ((unsigned)m->_m_count >= INT_MAX) return EAGAIN;
 		m->_m_count++;
 		return 0;
 	}
 
-	own = m->_m_lock;
-	if ((own && !(own & 0x40000000)) || a_cas(&m->_m_lock, own, tid)!=own)
+	if ((own && !(own & 0x40000000)) || a_cas(&m->_m_lock, old, tid)!=old)
 		return EBUSY;
 
 	m->_m_count = 1;