overhaul system() and popen() to use vfork; fix various related bugs

since we target systems without overcommit, special care should be
taken that system() and popen(), like posix_spawn(), do not fail in
processes whose commit charges are too high to allow ordinary forking.

this in turn requires special precautions to ensure that the parent
process's signal handlers do not end up running in the shared-memory
child, where they could corrupt the state of the parent process.

popen has also been updated to use pipe2, so it does not have a
fd-leak race in multi-threaded programs. since pipe2 is missing on
older kernels, (non-atomic) emulation has been added.

some silly bugs in the old code should be gone too.

1 files changed, 17 insertions, 3 deletions

diff --git a/src/unistd/pipe2.c b/src/unistd/pipe2.c
index 83282bb9..04e0c128 100644
--- a/src/unistd/pipe2.c
+++ b/src/unistd/pipe2.c
@@ -1,8 +1,22 @@
-#define _GNU_SOURCE
 #include <unistd.h>
+#include <errno.h>
+#include <fcntl.h>
 #include "syscall.h"
 
-int pipe2(int fd[2], int flg)
+int pipe2(int fd[2], int flag)
 {
-	return syscall(SYS_pipe2, fd, flg);
+	if (!flag) return syscall(SYS_pipe, fd);
+	int ret = __syscall(SYS_pipe2, fd, flag);
+	if (ret != -ENOSYS) return __syscall_ret(ret);
+	ret = syscall(SYS_pipe, fd);
+	if (ret) return __syscall_ret(ret);
+	if (flag & O_CLOEXEC) {
+		fcntl(fd[0], F_SETFD, FD_CLOEXEC);
+		fcntl(fd[1], F_SETFD, FD_CLOEXEC);
+	}
+	if (flag & O_NONBLOCK) {
+		fcntl(fd[0], F_SETFL, fcntl(fd[0], F_GETFL) | O_NONBLOCK);
+		fcntl(fd[1], F_SETFL, fcntl(fd[1], F_GETFL) | O_NONBLOCK);
+	}
+	return 0;
 }