add new masked cancellation mode

this is a new extension which is presently intended only for
experimental and internal libc use. interface and behavior details may
change subject to feedback and experience from using it internally.

the basic concept for the new PTHREAD_CANCEL_MASKED state is that the
first cancellation point to observe the cancellation request fails
with an errno value of ECANCELED rather than acting on cancellation,
allowing the caller to process the status and choose whether/how to
act upon it.

2 files changed, 16 insertions, 10 deletions

diff --git a/src/thread/pthread_cancel.c b/src/thread/pthread_cancel.c
index 66e0817c..1b71aa42 100644
--- a/src/thread/pthread_cancel.c
+++ b/src/thread/pthread_cancel.c
@@ -2,9 +2,13 @@
 #include "syscall.h"
 #include "libc.h"
 
-void __cancel()
+long __cancel()
 {
-	pthread_exit(PTHREAD_CANCELED);
+	pthread_t self = __pthread_self();
+	if (self->canceldisable == PTHREAD_CANCEL_ENABLE || self->cancelasync)
+		pthread_exit(PTHREAD_CANCELED);
+	self->canceldisable = PTHREAD_CANCEL_DISABLE;
+	return -ECANCELED;
 }
 
 /* If __syscall_cp_asm has adjusted the stack pointer, it must provide a
@@ -23,13 +27,16 @@ long __syscall_cp_c(syscall_arg_t nr,
 {
 	pthread_t self;
 	long r;
+	int st;
 
-	if (!libc.has_thread_pointer || (self = __pthread_self())->canceldisable)
+	if (!libc.has_thread_pointer || (st=(self=__pthread_self())->canceldisable)
+	    && (st==PTHREAD_CANCEL_DISABLE || nr==SYS_close))
 		return __syscall(nr, u, v, w, x, y, z);
 
 	r = __syscall_cp_asm(&self->cancel, nr, u, v, w, x, y, z);
-	if (r==-EINTR && nr!=SYS_close && self->cancel && !self->canceldisable)
-		__cancel();
+	if (r==-EINTR && nr!=SYS_close && self->cancel &&
+	    self->canceldisable != PTHREAD_CANCEL_DISABLE)
+		r = __cancel();
 	return r;
 }
 
@@ -47,14 +54,13 @@ static void cancel_handler(int sig, siginfo_t *si, void *ctx)
 	extern const char __cp_begin[1], __cp_end[1];
 
 	a_barrier();
-	if (!self->cancel || self->canceldisable) return;
+	if (!self->cancel || self->canceldisable == PTHREAD_CANCEL_DISABLE) return;
 
 	_sigaddset(&uc->uc_sigmask, SIGCANCEL);
 
 	if (self->cancelasync || ip >= __cp_begin && ip < __cp_end) {
-		self->canceldisable = 1;
-		pthread_sigmask(SIG_SETMASK, &uc->uc_sigmask, 0);
-		__cancel();
+		((char **)&uc->uc_mcontext)[CANCEL_REG_IP] = (char *)__cp_cancel;
+		return;
 	}
 
 	__syscall(SYS_tkill, self->tid, SIGCANCEL);
diff --git a/src/thread/pthread_setcancelstate.c b/src/thread/pthread_setcancelstate.c
index 2268217d..822a1398 100644
--- a/src/thread/pthread_setcancelstate.c
+++ b/src/thread/pthread_setcancelstate.c
@@ -2,7 +2,7 @@
 
 int __pthread_setcancelstate(int new, int *old)
 {
-	if (new > 1U) return EINVAL;
+	if (new > 2U) return EINVAL;
 	if (!libc.has_thread_pointer) return ENOSYS;
 	struct pthread *self = __pthread_self();
 	if (old) *old = self->canceldisable;