3 from __future__ import print_function
6 This script parses each "meta" file and extracts the
7 information needed to deduce build and src dependencies.
9 It works much the same as the original shell script, but is
10 *much* more efficient.
12 The parsing work is handled by the class MetaFile.
13 We only pay attention to a subset of the information in the
14 "meta" files. Specifically:
16 'CWD' to initialize our notion.
18 'C' to track chdir(2) on a per process basis
20 'R' files read are what we really care about.
21 directories read, provide a clue to resolving
22 subsequent relative paths. That is if we cannot find
23 them relative to 'cwd', we check relative to the last
26 'W' files opened for write or read-write,
27 for filemon V3 and earlier.
33 'V' the filemon version, this record is used as a clue
34 that we have reached the interesting bit.
40 $Id: meta2deps.py,v 1.17 2014/04/05 22:56:54 sjg Exp $
42 Copyright (c) 2011-2013, Juniper Networks, Inc.
45 Redistribution and use in source and binary forms, with or without
46 modification, are permitted provided that the following conditions
48 1. Redistributions of source code must retain the above copyright
49 notice, this list of conditions and the following disclaimer.
50 2. Redistributions in binary form must reproduce the above copyright
51 notice, this list of conditions and the following disclaimer in the
52 documentation and/or other materials provided with the distribution.
54 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
55 "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
56 LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
57 A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
58 OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
59 SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
60 LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
61 DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
62 THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
63 (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
64 OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
70 def getv(dict, key, d=None):
71 """Lookup key in dict and return value or the supplied default."""
76 def resolve(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr):
78 Return an absolute path, resolving via cwd or last_dir if needed.
80 if path.endswith('/.'):
82 if len(path) > 0 and path[0] == '/':
86 if path.startswith('./'):
90 for d in [last_dir, cwd]:
93 p = '/'.join([d,path])
95 print("looking for:", p, end=' ', file=debug_out)
96 if not os.path.exists(p):
98 print("nope", file=debug_out)
102 print("found:", p, file=debug_out)
106 def abspath(path, cwd, last_dir=None, debug=0, debug_out=sys.stderr):
108 Return an absolute path, resolving via cwd or last_dir if needed.
109 this gets called a lot, so we try to avoid calling realpath
110 until we know we have something.
112 rpath = resolve(path, cwd, last_dir, debug, debug_out)
115 if (path.find('./') > 0 or
116 path.endswith('/..') or
117 os.path.islink(path)):
118 return os.path.realpath(path)
121 def sort_unique(list, cmp=None, key=None, reverse=False):
122 list.sort(cmp, key, reverse)
132 return ['/' + x + '/',
138 """class to parse meta files generated by bmake."""
151 def __init__(self, name, conf={}):
152 """if name is set we will parse it now.
153 conf can have the follwing keys:
155 SRCTOPS list of tops of the src tree(s).
157 CURDIR the src directory 'bmake' was run from.
159 RELDIR the relative path from SRCTOP to CURDIR
161 MACHINE the machine we built for.
162 set to 'none' if we are not cross-building.
163 More specifically if machine cannot be deduced from objdirs.
166 Sometimes MACHINE isn't enough.
169 when we build for the pseudo machine 'host'
170 the object tree uses HOST_TARGET rather than MACHINE.
172 OBJROOTS a list of the common prefix for all obj dirs it might
175 DPDEPS names an optional file to which per file dependencies
177 For example if 'some/path/foo.h' is read from SRCTOP
178 then 'DPDEPS_some/path/foo.h +=' "RELDIR" is output.
179 This can allow 'bmake' to learn all the dirs within
180 the tree that depend on 'foo.h'
182 debug desired debug level
184 debug_out open file to send debug output to (sys.stderr)
189 self.debug = getv(conf, 'debug', 0)
190 self.debug_out = getv(conf, 'debug_out', sys.stderr)
192 self.machine = getv(conf, 'MACHINE', '')
193 self.machine_arch = getv(conf, 'MACHINE_ARCH', '')
194 self.target_spec = getv(conf, 'TARGET_SPEC', '')
195 self.curdir = getv(conf, 'CURDIR')
196 self.reldir = getv(conf, 'RELDIR')
197 self.dpdeps = getv(conf, 'DPDEPS')
201 # some of the steps below we want to do only once
203 self.host_target = getv(conf, 'HOST_TARGET')
204 for srctop in getv(conf, 'SRCTOPS', []):
205 if srctop[-1] != '/':
207 if not srctop in self.srctops:
208 self.srctops.append(srctop)
209 _srctop = os.path.realpath(srctop)
210 if _srctop[-1] != '/':
212 if not _srctop in self.srctops:
213 self.srctops.append(_srctop)
215 trim_list = add_trims(self.machine)
216 if self.machine == 'host':
217 trim_list += add_trims(self.host_target)
219 trim_list += add_trims(self.target_spec)
221 for objroot in getv(conf, 'OBJROOTS', []):
223 if objroot.endswith(e):
224 # this is not what we want - fix it
225 objroot = objroot[0:-len(e)]
228 if not objroot in self.objroots:
229 self.objroots.append(objroot)
230 _objroot = os.path.realpath(objroot)
231 if objroot[-1] == '/':
233 if not _objroot in self.objroots:
234 self.objroots.append(_objroot)
236 # we want the longest match
237 self.srctops.sort(reverse=True)
238 self.objroots.sort(reverse=True)
241 print("host_target=", self.host_target, file=self.debug_out)
242 print("srctops=", self.srctops, file=self.debug_out)
243 print("objroots=", self.objroots, file=self.debug_out)
245 self.dirdep_re = re.compile(r'([^/]+)/(.+)')
247 if self.dpdeps and not self.reldir:
249 print("need reldir:", end=' ', file=self.debug_out)
251 srctop = self.find_top(self.curdir, self.srctops)
253 self.reldir = self.curdir.replace(srctop,'')
255 print(self.reldir, file=self.debug_out)
257 self.dpdeps = None # we cannot do it?
259 self.cwd = os.getcwd() # make sure this is initialized
265 """reset state if we are being passed meta files from multiple directories."""
271 def dirdeps(self, sep='\n'):
273 return sep.strip() + sep.join(self.obj_deps)
275 def src_dirdeps(self, sep='\n'):
276 """return SRC_DIRDEPS"""
277 return sep.strip() + sep.join(self.src_deps)
279 def file_depends(self, out=None):
280 """Append DPDEPS_${file} += ${RELDIR}
281 for each file we saw, to the output file."""
284 for f in sort_unique(self.file_deps):
285 print('DPDEPS_%s += %s' % (f, self.reldir), file=out)
287 def seenit(self, dir):
288 """rememer that we have seen dir."""
291 def add(self, list, data, clue=''):
292 """add data to list if it isn't already there."""
296 print("%s: %sAdd: %s" % (self.name, clue, data), file=self.debug_out)
298 def find_top(self, path, list):
299 """the logical tree may be split across multiple trees"""
301 if path.startswith(top):
303 print("found in", top, file=self.debug_out)
307 def find_obj(self, objroot, dir, path, input):
308 """return path within objroot, taking care of .dirdep files"""
310 for ddepf in [path + '.dirdep', dir + '/.dirdep']:
311 if not ddep and os.path.exists(ddepf):
312 ddep = open(ddepf, 'r').readline().strip('# \n')
314 print("found %s: %s\n" % (ddepf, ddep), file=self.debug_out)
315 if ddep.endswith(self.machine):
316 ddep = ddep[0:-(1+len(self.machine))]
317 elif self.target_spec and ddep.endswith(self.target_spec):
318 ddep = ddep[0:-(1+len(self.target_spec))]
321 # no .dirdeps, so remember that we've seen the raw input
324 if self.machine == 'none':
325 if dir.startswith(objroot):
326 return dir.replace(objroot,'')
328 m = self.dirdep_re.match(dir.replace(objroot,''))
331 dmachine = m.group(1)
332 if dmachine != self.machine:
333 if not (self.machine == 'host' and
334 dmachine == self.host_target):
336 print("adding .%s to %s" % (dmachine, ddep), file=self.debug_out)
337 ddep += '.' + dmachine
341 def try_parse(self, name=None, file=None):
342 """give file and line number causing exception"""
344 self.parse(name, file)
347 print('{}:{}: '.format(self.name, self.line), end=' ', file=sys.stderr)
350 def parse(self, name=None, file=None):
351 """A meta file looks like:
353 # Meta data file "path"
358 -- filemon acquired metadata --
368 L "pid" "src" "target"
373 We go to some effort to avoid processing a dependency more than once.
374 Of the above record types only C,E,F,L,R,V and W are of interest.
377 version = 0 # unknown
382 cwd = last_dir = self.cwd
384 f = open(self.name, 'r')
392 self.seenit(self.curdir) # we ignore this
394 interesting = 'CEFLRV'
397 # ignore anything we don't care about
398 if not line[0] in interesting:
401 print("input:", line, end=' ', file=self.debug_out)
410 # we cannot ignore 'W' records
411 # as they may be 'rw'
415 self.cwd = cwd = last_dir = w[1]
416 self.seenit(cwd) # ignore this
418 print("%s: CWD=%s" % (self.name, cwd), file=self.debug_out)
424 pid_cwd[last_pid] = cwd
425 pid_last_dir[last_pid] = last_dir
426 cwd = getv(pid_cwd, pid, self.cwd)
427 last_dir = getv(pid_last_dir, pid, self.cwd)
434 pid_last_dir[npid] = cwd
438 cwd = abspath(w[2], cwd, None, self.debug, self.debug_out)
439 if cwd.endswith('/.'):
443 print("cwd=", cwd, file=self.debug_out)
446 if w[2] in self.seen:
448 print("seen:", w[2], file=self.debug_out)
452 path = w[2].strip("'")
455 # we are never interested in .dirdep files as dependencies
456 if path.endswith('.dirdep'):
458 # we don't want to resolve the last component if it is
460 path = resolve(path, cwd, last_dir, self.debug, self.debug_out)
463 dir,base = os.path.split(path)
466 print("seen:", dir, file=self.debug_out)
468 # we can have a path in an objdir which is a link
469 # to the src dir, we may need to add dependencies for each
471 dir = abspath(dir, cwd, last_dir, self.debug, self.debug_out)
472 if rdir == dir or rdir.find('./') > 0:
474 # now put path back together
475 path = '/'.join([dir,base])
477 print("raw=%s rdir=%s dir=%s path=%s" % (w[2], rdir, dir, path), file=self.debug_out)
479 if w[0] == 'W' and path.endswith('.dirdep'):
481 if path in [last_dir, cwd, self.cwd, self.curdir]:
483 print("skipping:", path, file=self.debug_out)
485 if os.path.isdir(path):
489 print("ldir=", last_dir, file=self.debug_out)
493 # finally, we get down to it
494 if dir == self.cwd or dir == self.curdir:
496 srctop = self.find_top(path, self.srctops)
499 self.add(self.file_deps, path.replace(srctop,''), 'file')
500 self.add(self.src_deps, dir.replace(srctop,''), 'src')
503 if rdir and not rdir.startswith(srctop):
504 dir = rdir # for below
510 for dir in [dir,rdir]:
513 objroot = self.find_top(dir, self.objroots)
517 ddep = self.find_obj(objroot, dir, path, w[2])
519 self.add(self.obj_deps, ddep, 'obj')
521 # don't waste time looking again
528 def main(argv, klass=MetaFile, xopts='', xoptf=None):
529 """Simple driver for class MetaFile.
532 script [options] [key=value ...] "meta" ...
534 Options and key=value pairs contribute to the
535 dictionary passed to MetaFile.
538 add "SRCTOP" to the "SRCTOPS" list.
543 add "OBJROOT" to the "OBJROOTS" list.
558 # import Psyco if we can
559 # it can speed things up quite a bit
574 machine = os.environ['MACHINE']
576 conf['MACHINE'] = machine
577 machine_arch = os.environ['MACHINE_ARCH']
579 conf['MACHINE_ARCH'] = machine_arch
580 srctop = os.environ['SB_SRC']
582 conf['SRCTOPS'].append(srctop)
583 objroot = os.environ['SB_OBJROOT']
585 conf['OBJROOTS'].append(objroot)
592 opts, args = getopt.getopt(argv[1:], 'a:dS:C:O:R:m:D:H:qT:' + xopts)
595 conf['MACHINE_ARCH'] = a
601 conf['HOST_TARGET'] = a
603 if a not in conf['SRCTOPS']:
604 conf['SRCTOPS'].append(a)
608 if a not in conf['OBJROOTS']:
609 conf['OBJROOTS'].append(a)
617 conf['TARGET_SPEC'] = a
621 conf['debug'] = debug
623 # get any var=val assignments
628 if k in ['SRCTOP','OBJROOT','SRCTOPS','OBJROOTS']:
644 debug_out = getv(conf, 'debug_out', sys.stderr)
647 print("config:", file=debug_out)
648 print("psyco=", have_psyco, file=debug_out)
649 for k,v in list(conf.items()):
650 print("%s=%s" % (k,v), file=debug_out)
653 if a.endswith('.meta'):
655 elif a.startswith('@'):
656 # there can actually multiple files per line
657 for line in open(a[1:]):
658 for f in line.strip().split():
664 print(m.src_dirdeps('\nsrc:'))
666 dpdeps = getv(conf, 'DPDEPS')
668 m.file_depends(open(dpdeps, 'wb'))
672 if __name__ == '__main__':
676 # yes, this goes to stdout
677 print("ERROR: ", sys.exc_info()[1])