count-headers: Initial file.

2015-11-11  Andrew MacLeod  <amacleod@redhat.com>

	* count-headers: Initial file.
	* gcc-order-headers: Initial file.
	* graph-header-logs: Initial file.
	* graph-include-web: Initial file.
	* headerutils.py: Initial file.
	* included-by: Initial file.
	* README: Initial file.
	* reduce-headers: Initial file.
	* replace-header: Initial file.
	* show-headers: Initial file.

From-SVN: r230171

contrib/ChangeLog

+2015-11-11  Andrew MacLeod  <amacleod@redhat.com>
+
+	* header-tools: New.  Directory containing a set of tools for
+	manipulating header files.
+
 2015-10-30  Nathan Sidwell  <nathan@acm.org>
 
 	* config-list.mk (nvptx-none): Add it.

contrib/header-tools/ChangeLog

+2015-11-11  Andrew MacLeod  <amacleod@redhat.com>
+
+	* count-headers: Initial file.
+	* gcc-order-headers: Initial file.
+	* graph-header-logs: Initial file.
+	* graph-include-web: Initial file.
+	* headerutils.py: Initial file.
+	* included-by: Initial file.
+	* README: Initial file.
+	* reduce-headers: Initial file.
+	* replace-header: Initial file.
+	* show-headers: Initial file.

contrib/header-tools/count-headers

+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+
+from headerutils import *
+
+usage = False
+src = list ()
+flist = { }
+process_h = True
+process_c = True
+verbose = False
+all_inc = True
+level = 0
+
+only_use_list = list ()
+
+for x in sys.argv[1:]:
+  if x[0:2] == "-h":
+    usage = True
+  else:
+    src.append (x)
+
+
+if not usage and len (src) > 0:
+  incl = { }
+  for fn in src:
+    src = readwholefile (fn)
+    dup = { }
+    for line in src:
+      d = find_pound_include (line, True, True)
+      if d != "" and d[-2:] ==".h":
+        if dup.get (d) == None:
+          if incl.get (d) == None:
+            incl[d] = 1
+          else:
+            incl[d] = incl[d]+ 1
+          dup[d] = 1
+
+  l = list ()
+  for i in incl:
+    l.append ((incl[i], i))
+  l.sort (key=lambda tup:tup[0], reverse=True)
+
+  for f in l:
+    print str (f[0]) + " : " + f[1]
+
+else:
+  print "count-headers file1 [filen]"
+  print "Count the number of occurrences of all includes across all listed files"
+
+ 
+
+
+
+

contrib/header-tools/graph-header-logs

+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+
+from headerutils import *
+
+header_roots = { }
+extra_edges = list()
+verbose = False
+verbosity = 0
+nodes = list()
+
+def unpretty (name):
+  if name[-2:] == "_h":
+    name = name[:-2] + ".h"
+  return name.replace("_", "-")
+
+def pretty_name (name):
+  name = os.path.basename (name)
+  return name.replace(".","_").replace("-","_").replace("/","_").replace("+","_");
+
+depstring = ("In file included from", "                 from")
+
+# indentation indicates nesting levels of included files
+ignore = [ "coretypes_h",
+             "machmode_h",
+             "signop_h",
+             "wide_int_h",
+             "double_int_h",
+             "real_h",
+             "fixed_value_h",
+             "hash_table_h",
+               "statistics_h",
+               "ggc_h",
+               "vec_h",
+               "hashtab_h",
+               "inchash_h",
+               "mem_stats_traits_h",
+               "hash_map_traits_h",
+               "mem_stats_h",
+               "hash_map_h",
+             "hash_set_h",
+             "input_h",
+               "line_map_h",
+             "is_a_h",
+           "system_h",
+           "config_h" ]
+
+def process_log_file (header, logfile):
+  if header_roots.get (header) != None:
+    print "Error: already processed log file: " + header + ".log"
+    return
+  hname = pretty_name (header)
+  header_roots[hname] = { }
+  
+  sline = list();
+  incfrom = list()
+  newinc = True
+  for line in logfile:
+    if len (line) > 21 and line[:21] in depstring:
+      if newinc:
+        incfrom = list()
+        newinc = False
+      fn = re.findall(ur".*/(.*?):", line)
+      if len(fn) != 1:
+        continue
+      if fn[0][-2:] != ".h":
+        continue
+      n = pretty_name (fn[0])
+      if n not in ignore:
+        incfrom.append (n)
+      continue
+    newinc = True
+    note = re.findall (ur"^.*note: (.*)", line)
+    if len(note) > 0:
+      sline.append (("note", note[0]))
+    else:
+      err_msg = re.findall (ur"^.*: error: (.*)", line)
+      if len(err_msg) == 1:
+        msg = err_msg[0]
+        if (len (re.findall("error: forward declaration", line))) != 0:
+          continue
+        path = re.findall (ur"^(.*?):.*error: ", line)
+        if len(path) != 1:
+          continue
+        if path[0][-2:] != ".h":
+          continue
+        fname = pretty_name (path[0])
+        if fname in ignore or fname[0:3] == "gt_":
+          continue
+        sline.append (("error", msg, fname, incfrom))
+
+  print str(len(sline)) + " lines to process"
+  lastline = "note"
+  for line in sline:
+    if line[0] != "note" and lastline[0] == "error":
+      fname = lastline[2]
+      msg = lastline[1]
+      incfrom = lastline[3]
+      string = ""
+      ofname = fname
+      if len(incfrom) != 0:
+        for t in incfrom:
+          string = string + t + " : "
+          ee = (fname, t)
+          if ee not in extra_edges:
+            extra_edges.append (ee)
+          fname = t
+          print string
+
+      if hname not in nodes:
+        nodes.append(hname)
+      if fname not in nodes:
+        nodes.append (ofname)
+      for y in incfrom:
+        if y not in nodes:
+          nodes.append (y)
+
+
+      if header_roots[hname].get(fname) == None:
+        header_roots[hname][fname] = list()
+      if msg not in header_roots[hname][fname]:
+        print string + ofname + " : " +msg
+        header_roots[hname][fname].append (msg)
+    lastline = line;
+
+
+dotname = "graph.dot"
+graphname = "graph.png"
+
+
+def build_dot_file (file_list):
+  output = open(dotname, "w")
+  output.write ("digraph incweb {\n");
+  for x in file_list:
+    if os.path.exists (x) and x[-4:] == ".log":
+      header =  x[:-4]
+      logfile = open(x).read().splitlines()
+      process_log_file (header, logfile)
+    elif os.path.exists (x + ".log"):
+      logfile = open(x + ".log").read().splitlines()
+      process_log_file (x, logfile)
+
+  for n in nodes:
+    fn = unpretty(n)
+    label = n + " [ label = \"" + fn  + "\" ];"
+    output.write (label + "\n")
+    if os.path.exists (fn):
+      h = open(fn).read().splitlines()
+      for l in h:
+        t = find_pound_include (l, True, False)
+        if t != "":
+          t = pretty_name (t)
+          if t in ignore or t[-2:] != "_h":
+            continue
+          if t not in nodes:
+            nodes.append (t)
+          ee = (t, n)
+          if ee not in extra_edges:
+            extra_edges.append (ee)
+
+  depcount = list()
+  for h in header_roots:
+    for dep in header_roots[h]:
+      label = " [ label = "+ str(len(header_roots[h][dep])) + " ];"
+      string = h + " -> " + dep + label
+      output.write (string + "\n");
+      if verbose:
+        depcount.append ((h, dep, len(header_roots[h][dep])))
+
+  for ee in extra_edges:
+    string = ee[0] + " -> " + ee[1] + "[ color=red ];"
+    output.write (string + "\n");
+
+  
+  if verbose:
+    depcount.sort(key=lambda tup:tup[2])
+    for x in depcount:
+      print " ("+str(x[2])+ ") : " + x[0] + " -> " + x[1]
+      if (x[2] <= verbosity):
+        for l in header_roots[x[0]][x[1]]:
+          print "            " + l
+
+  output.write ("}\n");
+
+
+files = list()
+dohelp = False
+edge_thresh = 0
+for arg in sys.argv[1:]:
+  if arg[0:2] == "-o":
+    dotname = arg[2:]+".dot"
+    graphname = arg[2:]+".png"
+  elif arg[0:2] == "-h":
+    dohelp = True
+  elif arg[0:2] == "-v":
+    verbose = True
+    if len(arg) > 2:
+      verbosity = int (arg[2:])
+      if (verbosity == 9):
+        verbosity = 9999
+  elif arg[0:1] == "-":
+    print "Unrecognized option " + arg
+    dohelp = True
+  else:
+    files.append (arg)
+    
+if len(sys.argv) == 1:
+  dohelp = True
+
+if dohelp:
+  print "Parses the log files from the reduce-headers tool to generate"
+  print "dependency graphs for the include web for specified files."
+  print "Usage:  [-nnum] [-h] [-v[n]] [-ooutput] file1 [[file2] ... [filen]]"
+  print "       -ooutput : Specifies output to output.dot and output.png"
+  print "                  Defaults to 'graph.dot and graph.png"
+  print "       -vn : verbose mode, shows the number of connections, and if n"
+  print "             is specified, show the messages if # < n. 9 is infinity"
+  print "       -h : help"
+else:
+  print files
+  build_dot_file (files)
+  os.system ("dot -Tpng " + dotname + " -o" + graphname)
+
+

contrib/header-tools/graph-include-web

+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+
+from headerutils import *
+
+def pretty_name (name):
+  return name.replace(".","_").replace("-","_").replace("/","_").replace("+","_");
+
+
+include_files = list()
+edges = 0
+one_c = False
+clink = list()
+noterm = False
+
+def build_inclist (output, filen):
+  global edges
+  global one_c
+  global clink
+  global noterm
+  inc = build_include_list (filen)
+  if one_c and filen[-2:] == ".c":
+    pn = "all_c"
+  else:
+    pn = pretty_name(filen)
+  for nm in inc:
+    if pn == "all_c":
+      if nm not in clink:
+        if len(build_include_list(nm)) != 0 or not noterm:
+          output.write (pretty_name(nm) + " -> " + pn + ";\n")
+          edges = edges + 1
+          if nm not in include_files:
+            include_files.append(nm)
+        clink.append (nm)
+    else:
+      output.write (pretty_name(nm) + " -> " + pn + ";\n")
+      edges = edges + 1
+      if nm not in include_files:
+        include_files.append(nm)
+  return len(inc) == 0
+
+dotname = "graph.dot"
+graphname = "graph.png"
+
+def build_dot_file (file_list):
+  global one_c
+  output = open(dotname, "w")
+  output.write ("digraph incweb {\n");
+  if one_c:
+    output.write ("all_c [shape=box];\n");
+  for x in file_list:
+    if x[-2:] == ".h":
+      include_files.append (x)
+    elif os.path.exists (x):
+      build_inclist (output, x)
+      if not one_c:
+        output.write (pretty_name (x) + "[shape=box];\n")
+
+  for x in include_files:
+    term = build_inclist (output, x)
+    if term:
+      output.write (pretty_name(x) + " [style=filled];\n")
+
+  output.write ("}\n");
+
+
+files = list()
+dohelp = False
+edge_thresh = 0
+for arg in sys.argv[1:]:
+  if arg[0:2] == "-o":
+    dotname = arg[2:]+".dot"
+    graphname = arg[2:]+".png"
+  elif arg[0:2] == "-h":
+    dohelp = True
+  elif arg[0:2] == "-a":
+    one_c = True
+    if arg[0:3] == "-at":
+      noterm = True
+  elif arg[0:2] == "-f":
+    if not os.path.exists (arg[2:]):
+      print "Option " + arg +" doesn't specify a proper file"
+      dohelp = True
+    else:
+      sfile = open (arg[2:], "r")
+      srcdata = sfile.readlines()
+      sfile.close()
+      for x in srcdata:
+        files.append(x.rstrip())
+  elif arg[0:2] == "-n":
+    edge_thresh = int (arg[2:])
+  elif arg[0:1] == "-":
+    print "Unrecognized option " + arg
+    dohelp = True
+  else:
+    files.append (arg)
+    
+if len(sys.argv) == 1:
+  dohelp = True
+
+if dohelp:
+  print "Generates a graph of the include web for specified files."
+  print "Usage:  [-finput_file] [-h] [-ooutput] [file1 ... [filen]]"
+  print "  -finput_file : Input file containing a list of files to process."
+  print "  -ooutput : Specifies output to output.dot and output.png."
+  print "             defaults to graph.dot and graph.png."
+  print "  -nnum : Specifies the # of edges beyond which sfdp is invoked. def=0."
+  print "  -a : Aggregate all .c files to 1 file.  Shows only include web."
+  print "  -at : Aggregate, but don't include terminal.h to .c links."
+  print "  -h : Print this help."
+else:
+  print files
+  build_dot_file (files)
+  if edges > edge_thresh:
+    os.system ("sfdp -Tpng " + dotname + " -o" + graphname)
+  else:
+    os.system ("dot -Tpng " + dotname + " -o" + graphname)
+
+

contrib/header-tools/included-by

+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+
+from headerutils import *
+
+
+
+usage = False
+src = list()
+flist = { }
+process_h = False
+process_c = False
+verbose = False
+level = 0
+match_all = False
+num_match = 1
+
+file_list = list()
+current = True
+deeper = True
+scanfiles = True
+for x in sys.argv[1:]:
+  if x[0:2] == "-h":
+    usage = True
+  elif x[0:2] == "-i":
+    process_h = True
+  elif x[0:2] == "-s" or x[0:2] == "-c":
+    process_c = True
+  elif x[0:2] == "-v":
+    verbose = True
+  elif x[0:2] == "-a":
+    match_all = True
+  elif x[0:2] == "-n":
+    num_match = int(x[2:])
+  elif x[0:2] == "-1":
+    deeper = False
+  elif x[0:2] == "-2":
+    current = False
+  elif x[0:2] == "-f":
+    file_list = open (x[2:]).read().splitlines()
+    scanfiles = False
+  elif x[0] == "-":
+    print "Error: Unknown option " + x
+    usage = True
+  else:
+    src.append (x)
+
+if match_all:
+  num_match = len (src)
+
+if not process_h and not process_c:
+  process_h = True
+  process_c = True
+
+if len(src) == 0:
+  usage = True
+
+if not usage:
+  if scanfiles:
+    if process_h:
+      file_list = find_gcc_files ("\*.h", current, deeper)
+    if process_c:
+      file_list = file_list + find_gcc_files ("\*.c", current, deeper)
+      file_list = file_list + find_gcc_files ("\*.cc", current, deeper)
+  else:
+    newlist = list()
+    for x in file_list:
+      if process_h and x[-2:] == ".h":
+        newlist.append (x)
+      elif process_c and (x[-2:] == ".c" or x[-3:] == ".cc"):
+        newlist.append (x)
+    file_list = newlist;
+     
+  file_list.sort()
+  for fn in file_list:
+    found = find_unique_include_list (fn)
+    careabout = list()
+    output = ""
+    for inc in found:
+      if inc in src:
+        careabout.append (inc)
+        if output == "":
+          output = fn
+        if verbose:
+          output = output + " [" + inc +"]"
+    if len (careabout) < num_match:
+        output = ""
+    if output != "":
+      print output
+else:
+  print "included-by [-h] [-i] [-c] [-v] [-a] [-nx] file1 [file2] ... [filen]"
+  print "find the list of all files in subdirectories that include any of "
+  print "the listed files. processed to a depth of 3 subdirs"
+  print " -h  : Show this message"
+  print " -i  : process only header files (*.h) for #include"
+  print " -c  : process only source files (*.c *.cc) for #include"
+  print "       If nothing is specified, defaults to -i -c"
+  print " -s  : Same as -c."
+  print " -v  : Show which include(s) were found"
+  print " -nx : Only list files which have at least x different matches. Default = 1"
+  print " -a  : Show only files which all listed files are included"
+  print "       This is equivilent to -nT where T == # of items in list"
+  print " -flistfile  : Show only files contained in the list of files"
+
+ 
+
+
+
+

contrib/header-tools/replace-header

+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+
+from headerutils import *
+
+
+files = list()
+replace = list()
+find = ""
+usage = False
+
+for x in sys.argv[1:]:
+  if x[0:2] == "-h":
+    usage = True
+  elif x[0:2] == "-f" and find == "":
+    find = x[2:]
+  elif x[0:2] == "-r":
+    replace.append (x[2:])
+  elif x[0:1] == "-":
+    print "Error: unrecognized option " + x
+    usage = True
+  else:
+    files.append (x)
+
+if find == "":
+  usage = True
+
+if usage:
+  print "replace-header -fheader -rheader [-rheader] file1 [filen.]"
+  sys.exit(0)
+
+string = ""
+for x in replace:
+  string = string + " '"+x+"'"
+print "Replacing '"+find+"'  with"+string
+
+for x in files:
+  src = readwholefile (x)
+  src = find_replace_include (find, replace, src)
+  if (len(src) > 0):
+    print x + ": Changed"
+    out = open(x, "w")
+    for line in src:
+      out.write (line);
+    out.close ()
+  else:
+    print x
+
+
+

contrib/header-tools/show-headers

+#! /usr/bin/python2
+import os.path
+import sys
+import shlex
+import re
+
+from headerutils import *
+
+
+tabstop = 2
+padding = "                                                                  "
+seen = { }
+output = list()
+summary = list()
+sawcore = False
+
+# list of headers to emphasize
+highlight = list ()
+
+bld_dir = ""
+# search path for headers
+incl_dirs = ["../include", "../libcpp/include", "common", "c-family", "c", "cp", "config" ]
+# extra search paths to look in *after* the directory the source file is in. 
+
+# append (1) to the end of the first line which includes INC in list INC.
+def append_1 (output, inc):
+  for n,t in enumerate (output):
+    idx = t.find(inc)
+    if idx != -1:
+      eos = idx + len (inc)
+      t = t[:eos] + "  (1)" + t[eos+1:]
+      output[n] = t
+      return
+
+# These headers show up as duplicates in rtl.h due to conditional code arund the includes
+rtl_core = [ "machmode.h" , "signop.h" , "wide-int.h" , "double-int.h" , "real.h" , "fixed-value.h" , "statistics.h" , "vec.h" , "hash-table.h" , "hash-set.h" , "input.h" , "is-a.h" ]
+
+def find_include_data (inc):
+  global sawcore
+  for x in incl_dirs:
+    nm = x+"/"+inc
+    if os.path.exists (nm):
+      info = find_unique_include_list (nm)
+      # rtl.h mimics coretypes for GENERATOR FILES, remove if coretypes.h seen.
+      if inc == "coretypes.h":
+        sawcore = True
+      elif inc  == "rtl.h" and sawcore:
+        for i in rtl_core:
+          if i in info:
+            info.remove (i)
+      return info
+  return list()
+
+def process_include (inc, indent):
+  if inc[-2:] != ".h":
+    return
+  bname  = os.path.basename (inc)
+  if bname in highlight:
+    arrow = "                <<-------"
+    if bname not in summary:
+      summary.append (bname)
+  else:
+    arrow = ""
+  if seen.get(inc) == None:
+    seen[inc] = 1
+    output.append (padding[:indent*tabstop] + bname + arrow)
+    info = find_include_data (inc)
+    for y in info:
+      process_include (y, indent+1)
+  else:
+    seen[inc] += 1
+    if (seen[inc] == 2):
+      append_1(output, inc)
+    output.append (padding[:indent*tabstop] + bname + "  ("+str(seen[inc])+")" + arrow)
+
+    
+
+extradir = list()
+usage = False
+src = list()
+
+for x in sys.argv[1:]:
+  if x[0:2] == "-i":
+    bld = x[2:]
+    extradir.append (bld)
+  elif x[0:2] == "-s":
+    highlight.append (os.path.basename (x[2:]))
+  elif x[0:2] == "-h":
+    usage = True
+  else:
+    src.append (x)
+
+if len(src) != 1:
+  usage = True
+elif not os.path.exists (src[0]):
+  print src[0] + ": Requested source file does not exist.\n"
+  usage = True
+
+if usage:
+  print "show-headers [-idir] [-sfilen] file1 "
+  print " "
+  print " Show a hierarchical visual format how many times each header file"
+  print " is included in a source file.  Should be run from the source directory"
+  print " files from find-include-depends"
+  print "      -s : search for a header, and point it out."
+  print "      -i : Specifies additonal directories to search for includes."
+  sys.exit(0)
+
+
+
+if extradir:
+  incl_dirs = extradir + incl_dirs;
+
+blddir = find_gcc_bld_dir ("../..")
+
+if blddir:
+  print "Using build directory: " + blddir
+  incl_dirs.insert (0, blddir)
+else:
+  print "Could not find a build directory, better results if you specify one with -i"
+
+# search path is now ".", blddir, extradirs_from_-i, built_in_incl_dirs
+incl_dirs.insert (0, ".")
+
+# if source is in a subdirectory, prepend the subdirectory to the search list
+x = src[0]
+srcpath = os.path.dirname(x)
+if srcpath:
+  incl_dirs.insert (0, srcpath)
+
+output = list()
+sawcore = False
+
+data = open (x).read().splitlines()
+for line in data:
+  d = find_pound_include (line, True, True)
+  if d and d[-2:] == ".h":
+    process_include (d, 1)
+
+print "\n" + x
+for line in output:
+  print line
+
+if highlight:
+  print " "
+  for h in summary:
+    print h + " is included by source file."
+  for h in highlight:
+    if h not in summary:
+      print h + " is not included by source file."
+