DragonFly kernel List (threaded) for 2004-12
[
Date Prev][
Date Next]
[
Thread Prev][
Thread Next]
[
Date Index][
Thread Index]
usr.bin/make phk's FIFO code for limiting instances of make
Matt,
You said that you wanted to do the FIFO code import. After the last
set of patches are commited, the following patches are left.
I also think we should just import the MAKE_SHELL, keeping them
out just complicates the patch modifications that are needed to
import things from FreeBSD. Harti Brandt and I are cleaning up
the code. So hopefully, I can eventually isolate it later on.
Max
PatchSet 371
Date: 2004/11/12 08:58:06
Author: phk
Log:
Add code to enforce the paralleism count (-j N) for the entire tree
of submakes spawned during processing.
We create a fifo and stuff one character into it for each job we are
allowed to run. The name of the fifo is passed to child processes
in the MAKE_JOBS_FIFO environment variable.
A make which finds this variable on startup will open the fifo and
only spawn jobs when it managed to read a token from the fifo.
When the job completes a token is writen back to the fifo.
Slave make processes get one token for free: the one their parent
make got in order to run them. This makes the make processes
themselves invisible in the process counts.
The net effect is that "make -j 12 -s buildworld" will start at
most 12 jobs at the same time, instead of as previously up to
65 jobs would get started.
Members:
job.c:1.56->1.57
job.h:1.25->1.26
main.c:1.95->1.96
make.c:1.24->1.25
Index: job.c
===================================================================
RCS file: /usr/home/okumoto/Work/make/fbsd-cvs/src/usr.bin/make/job.c,v
retrieving revision 1.56
retrieving revision 1.57
diff -u -r1.56 -r1.57
--- job.c 12 Nov 2004 07:57:17 -0000 1.56
+++ job.c 12 Nov 2004 08:58:06 -0000 1.57
@@ -258,6 +258,9 @@
* jobs that were stopped due to concurrency
* limits or externally */
+STATIC int fifoFd; /* Fd of our job fifo */
+STATIC char fifoName[] = "/tmp/make_fifo_XXXXXXXXX";
+STATIC int fifoMaster;
static sig_atomic_t interrupted;
@@ -1109,6 +1112,8 @@
Punt("Cannot fork");
} else if (cpid == 0) {
+ if (fifoFd >= 0)
+ close(fifoFd);
/*
* Must duplicate the input stream down to the child's input and
* reset it to the beginning (again). Since the stream was marked
@@ -1921,9 +1926,10 @@
return;
}
- while ((pid = waitpid((pid_t) -1, &status,
- (block?0:WNOHANG)|WUNTRACED)) > 0)
- {
+ for (;;) {
+ pid = waitpid((pid_t) -1, &status, (block?0:WNOHANG)|WUNTRACED);
+ if (pid <= 0)
+ break;
DEBUGF(JOB, ("Process %d exited or stopped.\n", pid));
jnode = Lst_Find(jobs, (void *)&pid, JobCmpPid);
@@ -1945,8 +1951,17 @@
job = (Job *) Lst_Datum(jnode);
(void) Lst_Remove(jobs, jnode);
nJobs -= 1;
- DEBUGF(JOB, ("Job queue is no longer full.\n"));
- jobFull = FALSE;
+ if (fifoFd >= 0 && maxJobs > 1) {
+ write(fifoFd, "+", 1);
+ maxJobs--;
+ if (nJobs >= maxJobs)
+ jobFull = TRUE;
+ else
+ jobFull = FALSE;
+ } else {
+ DEBUGF(JOB, ("Job queue is no longer full.\n"));
+ jobFull = FALSE;
+ }
}
JobFinish(job, &status);
@@ -1972,7 +1987,7 @@
* -----------------------------------------------------------------------
*/
void
-Job_CatchOutput(void)
+Job_CatchOutput(int flag)
{
int nfds;
#ifdef USE_KQUEUE
@@ -2016,25 +2031,31 @@
readfds = outputs;
timeout.tv_sec = SEL_SEC;
timeout.tv_usec = SEL_USEC;
+ if (flag && jobFull && fifoFd >= 0)
+ FD_SET(fifoFd, &readfds);
- if ((nfds = select(FD_SETSIZE, &readfds, (fd_set *) 0,
- (fd_set *) 0, &timeout)) <= 0) {
- if (interrupted)
+ nfds = select(FD_SETSIZE, &readfds, (fd_set *) 0,
+ (fd_set *) 0, &timeout);
+ if (nfds <= 0) {
+ if (interrupted)
JobPassSig(interrupted);
return;
- } else {
- if (Lst_Open(jobs) == FAILURE) {
- Punt("Cannot open job table");
- }
- while (nfds && (ln = Lst_Next(jobs)) != NULL) {
- job = (Job *) Lst_Datum(ln);
- if (FD_ISSET(job->inPipe, &readfds)) {
- JobDoOutput(job, FALSE);
- nfds -= 1;
- }
+ }
+ if (fifoFd >= 0 && FD_ISSET(fifoFd, &readfds)) {
+ if (--nfds <= 0)
+ return;
+ }
+ if (Lst_Open(jobs) == FAILURE) {
+ Punt("Cannot open job table");
+ }
+ while (nfds && (ln = Lst_Next(jobs)) != NULL) {
+ job = (Job *) Lst_Datum(ln);
+ if (FD_ISSET(job->inPipe, &readfds)) {
+ JobDoOutput(job, FALSE);
+ nfds -= 1;
}
- Lst_Close(jobs);
}
+ Lst_Close(jobs);
#endif /* !USE_KQUEUE */
}
}
@@ -2091,20 +2112,67 @@
Job_Init(int maxproc)
{
GNode *begin; /* node for commands to do at the very start */
+ const char *env;
struct sigaction sa;
+ fifoFd = -1;
jobs = Lst_Init(FALSE);
stoppedJobs = Lst_Init(FALSE);
- maxJobs = maxproc;
+ env = getenv("MAKE_JOBS_FIFO");
+
+ if (env == NULL && maxproc > 1) {
+ /*
+ * We did not find the environment variable so we are the leader.
+ * Create the fifo, open it, write one char per allowed job into
+ * the pipe.
+ */
+ mktemp(fifoName);
+ if (!mkfifo(fifoName, 0600)) {
+ fifoFd = open(fifoName, O_RDWR | O_NONBLOCK, 0);
+ if (fifoFd >= 0) {
+ fifoMaster = 1;
+ fcntl(fifoFd, F_SETFL, O_NONBLOCK);
+ env = fifoName;
+ setenv("MAKE_JOBS_FIFO", env, 1);
+ while (maxproc-- > 0) {
+ write(fifoFd, "+", 1);
+ }
+ /*The master make does not get a magic token */
+ jobFull = TRUE;
+ maxJobs = 0;
+ } else {
+ unlink(fifoName);
+ env = NULL;
+ }
+ }
+ } else if (env != NULL) {
+ /*
+ * We had the environment variable so we are a slave.
+ * Open fifo and give ourselves a magic token which represents
+ * the token our parent make has grabbed to start his make process.
+ * Otherwise the sub-makes would gobble up tokens and the proper
+ * number of tokens to specify to -j would depend on the depth of * the tree and the order of execution.
+ */
+ fifoFd = open(env, O_RDWR, 0);
+ if (fifoFd >= 0) {
+ fcntl(fifoFd, F_SETFL, O_NONBLOCK);
+ maxJobs = 1;
+ jobFull = FALSE;
+ }
+ }
+ if (fifoFd <= 0) {
+ maxJobs = maxproc;
+ jobFull = FALSE;
+ } else {
+ }
nJobs = 0;
- jobFull = FALSE;
aborting = 0;
errors = 0;
lastNode = NULL;
- if (maxJobs == 1 || beVerbose == 0) {
+ if ((maxJobs == 1 && fifoFd < 0) || beVerbose == 0) {
/*
* If only one job can run at a time, there's no need for a banner,
* no is there?
@@ -2175,7 +2243,7 @@
if (begin != NULL) {
JobStart(begin, JOB_SPECIAL, (Job *)0);
while (nJobs) {
- Job_CatchOutput();
+ Job_CatchOutput(0);
Job_CatchChildren(!usePipes);
}
}
@@ -2199,7 +2267,19 @@
Boolean
Job_Full(void)
{
- return(aborting || jobFull);
+ char c;
+ int i;
+
+ if (aborting)
+ return(aborting);
+ if (fifoFd >= 0 && jobFull) {
+ i = read(fifoFd, &c, 1);
+ if (i > 0) {
+ maxJobs++;
+ jobFull = FALSE;
+ }
+ }
+ return(jobFull);
}
/*-
@@ -2499,7 +2579,7 @@
JobStart(interrupt, JOB_IGNDOTS, (Job *)0);
while (nJobs) {
- Job_CatchOutput();
+ Job_CatchOutput(0);
Job_CatchChildren(!usePipes);
}
}
@@ -2526,11 +2606,17 @@
JobStart(postCommands, JOB_SPECIAL | JOB_IGNDOTS, NULL);
while (nJobs) {
- Job_CatchOutput();
+ Job_CatchOutput(0);
Job_CatchChildren(!usePipes);
}
}
}
+ if (fifoFd >= 0) {
+ close(fifoFd);
+ fifoFd = -1;
+ if (fifoMaster)
+ unlink(fifoName);
+ }
return(errors);
}
@@ -2553,7 +2639,7 @@
{
aborting = ABORT_WAIT;
while (nJobs != 0) {
- Job_CatchOutput();
+ Job_CatchOutput(0);
Job_CatchChildren(!usePipes);
}
aborting = 0;
Index: job.h
===================================================================
RCS file: /usr/home/okumoto/Work/make/fbsd-cvs/src/usr.bin/make/job.h,v
retrieving revision 1.25
retrieving revision 1.26
diff -u -r1.25 -r1.26
--- job.h 11 Nov 2004 12:52:16 -0000 1.25
+++ job.h 12 Nov 2004 08:58:07 -0000 1.26
@@ -209,7 +209,7 @@
void Job_Touch(GNode *, Boolean);
Boolean Job_CheckCommands(GNode *, void (*abortProc)(const char *, ...));
void Job_CatchChildren(Boolean);
-void Job_CatchOutput(void);
+void Job_CatchOutput(int flag);
void Job_Make(GNode *);
void Job_Init(int);
Boolean Job_Full(void);
Index: main.c
===================================================================
RCS file: /usr/home/okumoto/Work/make/fbsd-cvs/src/usr.bin/make/main.c,v
retrieving revision 1.95
retrieving revision 1.96
diff -u -r1.95 -r1.96
--- main.c 11 Nov 2004 12:52:16 -0000 1.95
+++ main.c 12 Nov 2004 08:58:07 -0000 1.96
@@ -678,6 +678,8 @@
Var_Set(".CURDIR", curdir, VAR_GLOBAL);
Var_Set(".OBJDIR", objdir, VAR_GLOBAL);
+ if (getenv("MAKE_JOBS_FIFO") != NULL)
+ forceJobs = TRUE;
/*
* Be compatible if user did not specify -j and did not explicitly
* turned compatibility on
Index: make.c
===================================================================
RCS file: /usr/home/okumoto/Work/make/fbsd-cvs/src/usr.bin/make/make.c,v
retrieving revision 1.24
retrieving revision 1.25
diff -u -r1.24 -r1.25
--- make.c 23 Oct 2002 23:16:43 -0000 1.24
+++ make.c 12 Nov 2004 08:58:07 -0000 1.25
@@ -637,7 +637,7 @@
{
GNode *gn;
- while (!Job_Full() && !Lst_IsEmpty (toBeMade)) {
+ while (!Lst_IsEmpty (toBeMade) && !Job_Full()) {
gn = (GNode *) Lst_DeQueue (toBeMade);
DEBUGF(MAKE, ("Examining %s...", gn->name));
/*
@@ -840,7 +840,7 @@
* keepgoing flag was given.
*/
while (!Job_Empty ()) {
- Job_CatchOutput ();
+ Job_CatchOutput (!Lst_IsEmpty (toBeMade));
Job_CatchChildren (!usePipes);
(void)MakeStartJobs();
}
PatchSet 372
Date: 2004/11/12 13:14:56
Author: ceri
Log:
Wrap a comment properly.
Members:
job.c:1.57->1.58
Index: job.c
===================================================================
RCS file: /usr/home/okumoto/Work/make/fbsd-cvs/src/usr.bin/make/job.c,v
retrieving revision 1.57
retrieving revision 1.58
diff -u -r1.57 -r1.58
--- job.c 12 Nov 2004 08:58:06 -0000 1.57
+++ job.c 12 Nov 2004 13:14:56 -0000 1.58
@@ -2151,7 +2151,8 @@
* Open fifo and give ourselves a magic token which represents
* the token our parent make has grabbed to start his make process.
* Otherwise the sub-makes would gobble up tokens and the proper
- * number of tokens to specify to -j would depend on the depth of * the tree and the order of execution.
+ * number of tokens to specify to -j would depend on the depth of
+ * the tree and the order of execution.
*/
fifoFd = open(env, O_RDWR, 0);
if (fifoFd >= 0) {
PatchSet 373
Date: 2004/11/12 20:37:27
Author: phk
Log:
If -B is specified to get compat mode (as opposed to just not giving
a -j arg which does the same thing), remove the MAKE_JOBS_FIFO
environment variable so we decouple any resulting sub-makes from
the token pool.
Members:
main.c:1.96->1.97
Index: main.c
===================================================================
RCS file: /usr/home/okumoto/Work/make/fbsd-cvs/src/usr.bin/make/main.c,v
retrieving revision 1.96
retrieving revision 1.97
diff -u -r1.96 -r1.97
--- main.c 12 Nov 2004 08:58:07 -0000 1.96
+++ main.c 12 Nov 2004 20:37:27 -0000 1.97
@@ -188,6 +188,7 @@
case 'B':
compatMake = TRUE;
MFLAGS_append("-B", NULL);
+ unsetenv("MAKE_JOBS_FIFO");
break;
case 'P':
usePipes = FALSE;
[
Date Prev][
Date Next]
[
Thread Prev][
Thread Next]
[
Date Index][
Thread Index]
