/* * Implement 'meta' mode. * Adapted from John Birrell's patches to FreeBSD make. * --sjg */ /* * Copyright (c) 2009-2010, Juniper Networks, Inc. * Portions Copyright (c) 2009, John Birrell. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #if defined(USE_META) #ifdef HAVE_CONFIG_H # include "config.h" #endif #include #include #include #include #include #if !defined(HAVE_CONFIG_H) || defined(HAVE_ERR_H) #include #endif #include "make.h" #include "job.h" #ifdef HAVE_FILEMON_H # include #endif #if !defined(USE_FILEMON) && defined(FILEMON_SET_FD) # define USE_FILEMON #endif static BuildMon Mybm; /* for compat */ Boolean useMeta = FALSE; static Boolean useFilemon = FALSE; static Boolean writeMeta = FALSE; static Boolean metaEnv = FALSE; /* don't save env unless asked */ static Boolean metaVerbose = FALSE; static Boolean metaIgnoreCMDs = FALSE; /* ignore CMDs in .meta files */ extern Boolean forceJobs; extern Boolean comatMake; #define MAKE_META_PREFIX ".MAKE.META.PREFIX" #ifndef N2U # define N2U(n, u) (((n) + ((u) - 1)) / (u)) #endif #ifndef ROUNDUP # define ROUNDUP(n, u) (N2U((n), (u)) * (u)) #endif #if !defined(HAVE_STRSEP) # define strsep(s, d) stresep((s), (d), 0) #endif /* * Filemon is a kernel module which snoops certain syscalls. * * C chdir * E exec * F [v]fork * L [sym]link * M rename * R read * W write * S stat * * See meta_oodate below - we mainly care about 'E' and 'R'. * * We can still use meta mode without filemon, but * the benefits are more limited. */ #ifdef USE_FILEMON # ifndef _PATH_FILEMON # define _PATH_FILEMON "/dev/filemon" # endif /* * Open the filemon device. */ static void filemon_open(BuildMon *pbm) { int retry; pbm->mon_fd = pbm->filemon_fd = -1; if (!useFilemon) return; for (retry = 5; retry >= 0; retry--) { if ((pbm->filemon_fd = open(_PATH_FILEMON, O_RDWR)) >= 0) break; } if (pbm->filemon_fd < 0) { useFilemon = FALSE; warn("Could not open %s", _PATH_FILEMON); return; } /* * We use a file outside of '.' * to avoid a FreeBSD kernel bug where unlink invalidates * cwd causing getcwd to do a lot more work. * We only care about the descriptor. */ pbm->mon_fd = mkTempFile("filemon.XXXXXX", NULL); if (ioctl(pbm->filemon_fd, FILEMON_SET_FD, &pbm->mon_fd) < 0) { err(1, "Could not set filemon file descriptor!"); } /* we don't need these once we exec */ (void)fcntl(pbm->mon_fd, F_SETFD, 1); (void)fcntl(pbm->filemon_fd, F_SETFD, 1); } /* * Read the build monitor output file and write records to the target's * metadata file. */ static void filemon_read(FILE *mfp, int fd) { FILE *fp; char buf[BUFSIZ]; /* Check if we're not writing to a meta data file.*/ if (mfp == NULL) { if (fd >= 0) close(fd); /* not interested */ return; } /* rewind */ lseek(fd, SEEK_SET, 0); if ((fp = fdopen(fd, "r")) == NULL) err(1, "Could not read build monitor file '%d'", fd); fprintf(mfp, "-- filemon acquired metadata --\n"); while (fgets(buf, sizeof(buf), fp)) { fprintf(mfp, "%s", buf); } fflush(mfp); clearerr(fp); fclose(fp); } #endif static char * meta_name(struct GNode *gn, char *mname, size_t mnamelen, const char *dname, const char *tname) { char buf[MAXPATHLEN]; char cwd[MAXPATHLEN]; char *rp; char *cp; char *tp; char *p[4]; /* >= number of possible uses */ int i; i = 0; if (!dname) dname = Var_Value(".OBJDIR", gn, &p[i++]); if (!tname) tname = Var_Value(TARGET, gn, &p[i++]); /* * Weed out relative paths from the target file name. * We have to be careful though since if target is a * symlink, the result will be unstable. * So we use realpath() just to get the dirname, and leave the * basename as given to us. */ if ((cp = strrchr(tname, '/'))) { if (realpath(tname, buf)) { if ((rp = strrchr(buf, '/'))) { rp++; cp++; if (strcmp(cp, rp) != 0) strlcpy(rp, cp, sizeof(buf) - (rp - buf)); } tname = buf; } } if (realpath(dname, cwd)) dname = cwd; /* on some systems dirname may modify its arg */ tp = bmake_strdup(tname); if (strcmp(dname, dirname(tp)) == 0) snprintf(mname, mnamelen, "%s.meta", tname); else { snprintf(mname, mnamelen, "%s/%s.meta", dname, tname); /* * Replace path separators in the file name after the * current object directory path. */ cp = mname + strlen(dname) + 1; while (*cp != '\0') { if (*cp == '/') *cp = '_'; cp++; } } free(tp); for (i--; i >= 0; i--) { if (p[i]) free(p[i]); } return (mname); } /* * Return true if running ${.MAKE} * Bypassed if target is flagged .MAKE */ static int is_submake(void *cmdp, void *gnp) { static char *p_make = NULL; static int p_len; char *cmd = cmdp; GNode *gn = gnp; char *mp = NULL; char *cp; char *cp2; int rc = 0; /* keep looking */ if (!p_make) { p_make = Var_Value(".MAKE", gn, &cp); p_len = strlen(p_make); } cp = strchr(cmd, '$'); if ((cp)) { mp = Var_Subst(NULL, cmd, gn, FALSE); cmd = mp; } cp2 = strstr(cmd, p_make); if ((cp2)) { switch (cp2[p_len]) { case '\0': case ' ': case '\t': case '\n': rc = 1; break; } if (cp2 > cmd && rc > 0) { switch (cp2[-1]) { case ' ': case '\t': case '\n': break; default: rc = 0; /* no match */ break; } } } if (mp) free(mp); return (rc); } typedef struct meta_file_s { FILE *fp; GNode *gn; } meta_file_t; static int printCMD(void *cmdp, void *mfpp) { meta_file_t *mfp = mfpp; char *cmd = cmdp; char *cp = NULL; if (strchr(cmd, '$')) { cmd = cp = Var_Subst(NULL, cmd, mfp->gn, FALSE); } fprintf(mfp->fp, "CMD %s\n", cmd); if (cp) free(cp); return 0; } /* * Certain node types never get a .meta file */ #define SKIP_META_TYPE(_type) do { \ if ((gn->type & __CONCAT(OP_, _type))) { \ if (DEBUG(META)) { \ fprintf(debug_file, "Skipping meta for %s: .%s\n", \ gn->name, __STRING(_type)); \ } \ return (NULL); \ } \ } while (0) static FILE * meta_create(BuildMon *pbm, GNode *gn) { extern char **environ; meta_file_t mf; char buf[MAXPATHLEN]; char curdir[MAXPATHLEN]; char objdir[MAXPATHLEN]; char **ptr; const char *cname; const char *dname; const char *tname; char *fname; const char *cp; char *p[4]; /* >= possible uses */ int i; struct stat fs; /* This may be a phony node which we don't want meta data for... */ /* Skip .meta for .BEGIN, .END, .ERROR etc as well. */ /* Or it may be explicitly flagged as .NOMETA */ SKIP_META_TYPE(NOMETA); /* Unless it is explicitly flagged as .META */ if (!(gn->type & OP_META)) { SKIP_META_TYPE(PHONY); SKIP_META_TYPE(SPECIAL); SKIP_META_TYPE(MAKE); } mf.fp = NULL; i = 0; dname = Var_Value(".OBJDIR", gn, &p[i++]); cname = Var_Value(".CURDIR", gn, &p[i++]); tname = Var_Value(TARGET, gn, &p[i++]); /* The object directory may not exist. Check it.. */ if (stat(dname, &fs) != 0) { if (DEBUG(META)) fprintf(debug_file, "Skipping meta for %s: no .OBJDIR\n", gn->name); goto out; } /* Check if there are no commands to execute. */ if (Lst_IsEmpty(gn->commands)) { if (DEBUG(META)) fprintf(debug_file, "Skipping meta for %s: no commands\n", gn->name); goto out; } /* make sure these are canonical */ if (realpath(dname, objdir)) dname = objdir; if (realpath(cname, curdir)) cname = curdir; /* If we aren't in the object directory, don't create a meta file. */ if (strcmp(cname, dname) == 0) { if (DEBUG(META)) fprintf(debug_file, "Skipping meta for %s: .OBJDIR == .CURDIR\n", gn->name); goto out; } if (!(gn->type & OP_META)) { /* We do not generate .meta files for sub-makes */ if (Lst_ForEach(gn->commands, is_submake, gn)) { if (DEBUG(META)) fprintf(debug_file, "Skipping meta for %s: .MAKE\n", gn->name); goto out; } } if (metaVerbose) { char *mp; /* Describe the target we are building */ mp = Var_Subst(NULL, "${" MAKE_META_PREFIX "}", gn, 0); if (*mp) fprintf(stdout, "%s\n", mp); free(mp); } /* Get the basename of the target */ if ((cp = strrchr(tname, '/')) == NULL) { cp = tname; } else { cp++; } fflush(stdout); if (strcmp(cp, makeDependfile) == 0) goto out; if (!writeMeta) /* Don't create meta data. */ goto out; fname = meta_name(gn, pbm->meta_fname, sizeof(pbm->meta_fname), dname, tname); if ((mf.fp = fopen(fname, "w")) == NULL) err(1, "Could not open meta file '%s'", fname); fprintf(mf.fp, "# Meta data file %s\n", fname); mf.gn = gn; Lst_ForEach(gn->commands, printCMD, &mf); fprintf(mf.fp, "CWD %s\n", getcwd(buf, sizeof(buf))); fprintf(mf.fp, "TARGET %s\n", tname); if (metaEnv) { for (ptr = environ; *ptr != NULL; ptr++) fprintf(mf.fp, "ENV %s\n", *ptr); } fprintf(mf.fp, "-- command output --\n"); fflush(mf.fp); Var_Append(".MAKE.META.FILES", fname, VAR_GLOBAL); Var_Append(".MAKE.META.CREATED", fname, VAR_GLOBAL); out: for (i--; i >= 0; i--) { if (p[i]) free(p[i]); } return (mf.fp); } void meta_init(const char *make_mode) { static int once = 0; useMeta = TRUE; useFilemon = TRUE; writeMeta = TRUE; if (make_mode) { if (strstr(make_mode, "env")) metaEnv = TRUE; if (strstr(make_mode, "verb")) metaVerbose = TRUE; if (strstr(make_mode, "read")) writeMeta = FALSE; if (strstr(make_mode, "nofilemon")) useFilemon = FALSE; if (strstr(make_mode, "ignore-cmd")) metaIgnoreCMDs = TRUE; /* for backwards compatability */ Var_Set(".MAKE.META_CREATED", "${.MAKE.META.CREATED}", VAR_GLOBAL, 0); Var_Set(".MAKE.META_FILES", "${.MAKE.META.FILES}", VAR_GLOBAL, 0); } if (metaVerbose && !Var_Exists(MAKE_META_PREFIX, VAR_GLOBAL)) { /* * The default value for MAKE_META_PREFIX * prints the absolute path of the target. * This works be cause :H will generate '.' if there is no / * and :tA will resolve that to cwd. */ Var_Set(MAKE_META_PREFIX, "Building ${.TARGET:H:tA}/${.TARGET:T}", VAR_GLOBAL, 0); } if (once) return; once = 1; memset(&Mybm, 0, sizeof(Mybm)); } /* * In each case below we allow for job==NULL */ void meta_job_start(Job *job, GNode *gn) { BuildMon *pbm; if (job != NULL) { pbm = &job->bm; } else { pbm = &Mybm; } pbm->mfp = meta_create(pbm, gn); #ifdef USE_FILEMON_ONCE /* compat mode we open the filemon dev once per command */ if (job == NULL) return; #endif #ifdef USE_FILEMON if (pbm->mfp != NULL && useFilemon) { filemon_open(pbm); } else { pbm->mon_fd = pbm->filemon_fd = -1; } #endif } /* * The child calls this before doing anything. * It does not disturb our state. */ void meta_job_child(Job *job) { #ifdef USE_FILEMON BuildMon *pbm; pid_t pid; if (job != NULL) { pbm = &job->bm; } else { pbm = &Mybm; } pid = getpid(); if (pbm->mfp != NULL && useFilemon) { if (ioctl(pbm->filemon_fd, FILEMON_SET_PID, &pid) < 0) { err(1, "Could not set filemon pid!"); } } #endif } void meta_job_error(Job *job, GNode *gn, int flags, int status) { char cwd[MAXPATHLEN]; BuildMon *pbm; if (job != NULL) { pbm = &job->bm; } else { if (!gn) gn = job->node; pbm = &Mybm; } if (pbm->mfp != NULL) { fprintf(pbm->mfp, "*** Error code %d%s\n", status, (flags & JOB_IGNERR) ? "(ignored)" : ""); } if (gn) { Var_Set(".ERROR_TARGET", gn->path ? gn->path : gn->name, VAR_GLOBAL, 0); } getcwd(cwd, sizeof(cwd)); Var_Set(".ERROR_CWD", cwd, VAR_GLOBAL, 0); if (pbm) { Var_Set(".ERROR_META_FILE", pbm->meta_fname, VAR_GLOBAL, 0); } } void meta_job_output(Job *job, char *cp, const char *nl) { BuildMon *pbm; if (job != NULL) { pbm = &job->bm; } else { pbm = &Mybm; } if (pbm->mfp != NULL) { if (metaVerbose) { static char *meta_prefix = NULL; static int meta_prefix_len; if (!meta_prefix) { char *cp2; meta_prefix = Var_Subst(NULL, "${" MAKE_META_PREFIX "}", VAR_GLOBAL, 0); if ((cp2 = strchr(meta_prefix, '$'))) meta_prefix_len = cp2 - meta_prefix; else meta_prefix_len = strlen(meta_prefix); } if (strncmp(cp, meta_prefix, meta_prefix_len) == 0) { cp = strchr(cp+1, '\n'); if (!cp++) return; } } fprintf(pbm->mfp, "%s%s", cp, nl); } } void meta_cmd_finish(void *pbmp) { #ifdef USE_FILEMON BuildMon *pbm = pbmp; if (!pbm) pbm = &Mybm; if (pbm->filemon_fd >= 0) { close(pbm->filemon_fd); filemon_read(pbm->mfp, pbm->mon_fd); pbm->filemon_fd = pbm->mon_fd = -1; } #endif } void meta_job_finish(Job *job) { BuildMon *pbm; if (job != NULL) { pbm = &job->bm; } else { pbm = &Mybm; } if (pbm->mfp != NULL) { meta_cmd_finish(pbm); fclose(pbm->mfp); pbm->mfp = NULL; } } /* * Fetch a full line from fp - growing bufp if needed * Return length in bufp. */ static int fgetLine(char **bufp, size_t *szp, int o, FILE *fp) { char *buf = *bufp; size_t bufsz = *szp; struct stat fs; int x; if (fgets(&buf[o], bufsz - o, fp) != NULL) { check_newline: x = o + strlen(&buf[o]); if (buf[x - 1] == '\n') return x; /* * We need to grow the buffer. * The meta file can give us a clue. */ if (fstat(fileno(fp), &fs) == 0) { size_t newsz; char *p; newsz = ROUNDUP((fs.st_size / 2), BUFSIZ); if (newsz <= bufsz) newsz = ROUNDUP(fs.st_size, BUFSIZ); if (DEBUG(META)) fprintf(debug_file, "growing buffer %u -> %u\n", bufsz, newsz); p = bmake_realloc(buf, newsz); if (p) { *bufp = buf = p; *szp = bufsz = newsz; /* fetch the rest */ if (!fgets(&buf[x], bufsz - x, fp)) return x; /* truncated! */ goto check_newline; } } } return 0; } /* * When running with 'meta' functionality, a target can be out-of-date * if any of the references in it's meta data file is more recent. */ Boolean meta_oodate(GNode *gn, Boolean oodate) { char latestdir[MAXPATHLEN]; char fname[MAXPATHLEN]; char fname1[MAXPATHLEN]; char *p; char *cp; FILE *fp; Boolean ignoreOODATE = FALSE; /* * We need to check if the target is out-of-date. This includes * checking if the expanded command has changed. This in turn * requires that all variables are set in the same way that they * would be if the target needs to be re-built. */ Make_DoAllVar(gn); if (oodate) return oodate; /* we're done */ if (getcwd(latestdir, sizeof(latestdir)) == NULL) err(1, "Could not get current working directory"); meta_name(gn, fname, sizeof(fname), NULL, NULL); if ((fp = fopen(fname, "r")) != NULL) { static char *buf = NULL; static size_t bufsz; int lineno = 0; int f = 0; int x; LstNode ln; struct stat fs; if (!buf) { bufsz = 8 * BUFSIZ; buf = bmake_malloc(bufsz); } /* we want to track all the .meta we read */ Var_Append(".MAKE.META.FILES", fname, VAR_GLOBAL); ln = Lst_First(gn->commands); while (!oodate && (x = fgetLine(&buf, &bufsz, 0, fp)) > 0) { lineno++; if (buf[x - 1] == '\n') buf[x - 1] = '\0'; else warnx("%s: %d: line truncated at %u", fname, lineno, x); /* Find the start of the build monitor section. */ if (!f) { if (strncmp(buf, "-- filemon", 10) == 0) { f = 1; continue; } if (strncmp(buf, "# buildmon", 10) == 0) { f = 1; continue; } } /* Delimit the record type. */ p = buf; strsep(&p, " "); if (f) { /* Process according to record type. */ switch (buf[0]) { case 'C': /* Skip the pid. */ if (strsep(&p, " ") == NULL) break; /* Update the latest directory. */ strlcpy(latestdir, p, sizeof(latestdir)); break; case 'R': case 'E': /* Skip the pid. */ if (strsep(&p, " ") == NULL) break; /* * Check for runtime files that can't * be part of the dependencies because * they are _expected_ to change. */ if (strncmp(p, "/var/", 5) == 0) break; /* Ignore device files. */ if (strncmp(p, "/dev/", 5) == 0) break; /* Ignore /etc/ files. */ if (strncmp(p, "/etc/", 5) == 0) break; /* * The rest of the record is the * file name. * Check if it's not an absolute * path. */ if (*p != '/') { /* Use the latest path seen. */ snprintf(fname1, sizeof(fname1), "%s/%s", latestdir, p); p = fname1; } if (stat(p, &fs) == 0 && !S_ISDIR(fs.st_mode) && fs.st_mtime > gn->mtime) { if (DEBUG(META)) fprintf(debug_file, "%s: %d: file '%s' is newer than the target...\n", fname, lineno, p); oodate = TRUE; } break; default: break; } /* * Compare the current command with the one in the * meta data file. */ } else if (strcmp(buf, "CMD") == 0) { if (ln == NULL) { if (DEBUG(META)) fprintf(debug_file, "%s: %d: there were more build commands in the meta data file than there are now...\n", fname, lineno); oodate = TRUE; } else { char *cmd = (char *)Lst_Datum(ln); if (!ignoreOODATE) { if (strstr(cmd, "$?")) ignoreOODATE = TRUE; else if ((cp = strstr(cmd, ".OODATE"))) { /* check for $[{(].OODATE[)}] */ if (cp > cmd + 2 && cp[-2] == '$') ignoreOODATE = TRUE; } if (ignoreOODATE && DEBUG(META)) fprintf(debug_file, "%s: %d: cannot compare commands using .OODATE\n", fname, lineno); } cmd = Var_Subst(NULL, cmd, gn, TRUE); if ((cp = strchr(cmd, '\n'))) { int n; /* * This command contains newlines, we need to * fetch more from the .meta file before we * attempt a comparison. */ /* first put the newline back at buf[x - 1] */ buf[x - 1] = '\n'; do { /* now fetch the next line */ if ((n = fgetLine(&buf, &bufsz, x, fp)) <= 0) break; x = n; lineno++; if (buf[x - 1] != '\n') { warnx("%s: %d: line truncated at %u", fname, lineno, x); break; } cp = strchr(++cp, '\n'); } while (cp); if (buf[x - 1] == '\n') buf[x - 1] = '\0'; } if (!ignoreOODATE && !(gn->type & OP_NOMETA_CMP) && strcmp(p, cmd) != 0) { if (DEBUG(META)) fprintf(debug_file, "%s: %d: a build command has changed\n%s\nvs\n%s\n", fname, lineno, p, cmd); if (!metaIgnoreCMDs) oodate = TRUE; } free(cmd); ln = Lst_Succ(ln); } } else if (strcmp(buf, "CWD") == 0) { char curdir[MAXPATHLEN]; if (strcmp(p, getcwd(curdir, sizeof(curdir))) != 0) { if (DEBUG(META)) fprintf(debug_file, "%s: %d: the current working directory has changed from '%s' to '%s'\n", fname, lineno, p, curdir); oodate = TRUE; } } } /* * Check if there are extra commands now * that weren't in the meta data file. */ if (!oodate && ln != NULL) { if (DEBUG(META)) fprintf(debug_file, "%s: %d: there are extra build commands now that weren't in the meta data file\n", fname, lineno); oodate = TRUE; } fclose(fp); } return oodate; } /* support for compat mode */ static int childPipe[2]; void meta_compat_start(void) { #ifdef USE_FILEMON_ONCE /* * We need to re-open filemon for each cmd. */ BuildMon *pbm = &Mybm; if (pbm->mfp != NULL && useFilemon) { filemon_open(pbm); } else { pbm->mon_fd = pbm->filemon_fd = -1; } #endif if (pipe(childPipe) < 0) Punt("Cannot create pipe: %s", strerror(errno)); /* Set close-on-exec flag for both */ (void)fcntl(childPipe[0], F_SETFD, 1); (void)fcntl(childPipe[1], F_SETFD, 1); } void meta_compat_child(void) { meta_job_child(NULL); if (dup2(childPipe[1], 1) < 0 || dup2(1, 2) < 0) { execError("dup2", "pipe"); _exit(1); } } void meta_compat_parent(void) { FILE *fp; char buf[BUFSIZ]; close(childPipe[1]); /* child side */ fp = fdopen(childPipe[0], "r"); while (fgets(buf, sizeof(buf), fp)) { meta_job_output(NULL, buf, ""); printf("%s", buf); } fclose(fp); } #endif /* USE_META */