fix robust mutex unrecoverable status, and related clean-up

a robust mutex should not enter the unrecoverable status until it's
unlocked without marking it consistent. previously, flag 8 in the type
was used as an indication of unrecoverable, but only honored after
successful locking; this resulted in a race window where the
unrecoverable mutex could appear to a second thread as locked/busy
again while the first thread was in the process of observing it as
unrecoverable.

now, flag 8 is used to mean that the mutex is in the process of being
recovered, but not yet marked consistent. the flag only takes effect
in pthread_mutex_unlock, where it causes the value 0x40000000 (owner
dead flag, with old owner tid 0, an otherwise impossible state) to be
stored in the lock. subsequent lock attempts will interpret this state
as unrecoverable.

3 files changed, 4 insertions, 12 deletions

diff --git a/src/thread/pthread_mutex_consistent.c b/src/thread/pthread_mutex_consistent.c
index 0db976ab..96b83b52 100644
--- a/src/thread/pthread_mutex_consistent.c
+++ b/src/thread/pthread_mutex_consistent.c
@@ -5,6 +5,6 @@ int pthread_mutex_consistent(pthread_mutex_t *m)
 	if (!(m->_m_type & 8)) return EINVAL;
 	if ((m->_m_lock & 0x7fffffff) != __pthread_self()->tid)
 		return EPERM;
-	m->_m_type -= 8;
+	m->_m_type &= ~8U;
 	return 0;
 }
diff --git a/src/thread/pthread_mutex_trylock.c b/src/thread/pthread_mutex_trylock.c
index f871e9e0..8d256614 100644
--- a/src/thread/pthread_mutex_trylock.c
+++ b/src/thread/pthread_mutex_trylock.c
@@ -19,6 +19,7 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 		m->_m_count++;
 		return 0;
 	}
+	if (own == 0x40000000) return ENOTRECOVERABLE;
 
 	self->robust_list.pending = &m->_m_next;
 
@@ -35,16 +36,9 @@ int __pthread_mutex_trylock_owner(pthread_mutex_t *m)
 	self->robust_list.head = &m->_m_next;
 	self->robust_list.pending = 0;
 
-	if (type < 4) return 0;
-
-	if (type >= 8) {
-		m->_m_lock = 0;
-		return ENOTRECOVERABLE;
-	}
-
 	if (own) {
 		m->_m_count = 0;
-		m->_m_type += 8;
+		m->_m_type |= 8;
 		return EOWNERDEAD;
 	}
 
diff --git a/src/thread/pthread_mutex_unlock.c b/src/thread/pthread_mutex_unlock.c
index b4ed3f87..42837516 100644
--- a/src/thread/pthread_mutex_unlock.c
+++ b/src/thread/pthread_mutex_unlock.c
@@ -12,8 +12,6 @@ int pthread_mutex_unlock(pthread_mutex_t *m)
 	int priv = (m->_m_type & 128) ^ 128;
 
 	if (type != PTHREAD_MUTEX_NORMAL) {
-		if (!m->_m_lock)
-			return EPERM;
 		self = __pthread_self();
 		if ((m->_m_lock&0x7fffffff) != self->tid)
 			return EPERM;
@@ -26,7 +24,7 @@ int pthread_mutex_unlock(pthread_mutex_t *m)
 		*(void **)m->_m_prev = m->_m_next;
 		if (m->_m_next) ((void **)m->_m_next)[-1] = m->_m_prev;
 	}
-	cont = a_swap(&m->_m_lock, 0);
+	cont = a_swap(&m->_m_lock, (type & 8) ? 0x40000000 : 0);
 	if (type != PTHREAD_MUTEX_NORMAL && !priv) {
 		self->robust_list.pending = 0;
 		__vm_unlock_impl();