check for fd exhaustion in forkpty

we cannot report failure after forking, so the idea is to ensure prior
to fork that fd 0,1,2 exist. this will prevent dup2 from possibly
hitting a resource limit and failing in the child process. fcntl
rather than dup2 is used prior to forking to avoid race conditions.

1 files changed, 15 insertions, 2 deletions

diff --git a/src/misc/forkpty.c b/src/misc/forkpty.c
index 0bbf2de2..07f8d01e 100644
--- a/src/misc/forkpty.c
+++ b/src/misc/forkpty.c
@@ -1,14 +1,25 @@
 #include <pty.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <fcntl.h>
 
 int forkpty(int *m, char *name, const struct termios *tio, const struct winsize *ws)
 {
-	int s;
+	int s, t, i, istmp[3]={0};
 	pid_t pid;
 
 	if (openpty(m, &s, name, tio, ws) < 0) return -1;
-	pid = fork();
+
+	/* Ensure before forking that we don't exceed fd limit */
+	for (i=0; i<3; i++) {
+		if (fcntl(i, F_GETFL) < 0) {
+			t = fcntl(s, F_DUPFD, i);
+			if (t<0) break;
+			else if (t!=i) close(t);
+			else istmp[i] = 1;
+		}
+	}
+	pid = i==3 ? fork() : -1;
 	if (!pid) {
 		close(*m);
 		setsid();
@@ -19,6 +30,8 @@ int forkpty(int *m, char *name, const struct termios *tio, const struct winsize
 		if (s>2) close(s);
 		return 0;
 	}
+	for (i=0; i<3; i++)
+		if (istmp[i]) close(i);
 	close(s);
 	if (pid < 0) close(*m);
 	return pid;