Refactor the results directory reading into a separate module.

This change is made in anticipation of needing more than one kind
of reader in future. Most notably, a reader to extract data from
the MySQL results database would be a nice addition.

BUG=None
TEST=run showbootdata on any pre-existing data set

Change-Id: I5811aa1427a2714ce6ba392cdec584b4236162fd
Reviewed-on: http://gerrit.chromium.org/gerrit/2644
Reviewed-by: Mike Truty <truty@chromium.org>
Tested-by: Richard Barnette <jrbarnette@chromium.org>

bootperf-bin/perfprinter.py

@@ -9,72 +9,18 @@ import os
 import os.path
 import re
 
-import resultset
 
-
+def PrintRawData(reader, dirlist, use_timestats, keylist):
-_PERF_KEYVAL_PATTERN = re.compile("(.*){perf}=(.*)\n")
-
-
-def ReadKeyvalFile(results, file_):
-  """Read an autotest keyval file, and process the results.
-
-  The `file_` parameter is a file object with contents in autotest
-  perf keyval format:
-      <keyname>{perf}=<value>
-
-  Each iteration of the test is terminated with a single blank line,
-  including the last iteration.  Each iteration's results are added
-  to the `results` parameter, which should be an instance of
-  TestResultSet.
-
-  """
-  kvd = {}
-  for line in iter(file_):
-    if line == "\n":
-      results.AddIterationResults(kvd)
-      kvd = {}
-      continue
-    m = _PERF_KEYVAL_PATTERN.match(line)
-    if m is None:
-      continue
-    kvd[m.group(1)] = m.group(2)
-
-
-_RESULTS_PATH = (
-    "summary/platform_BootPerfServer/platform_BootPerfServer/results/keyval")
-
-
-def ReadResultsDirectory(dir_):
-  """Process results from a 'bootperf' output directory.
-
-  The accumulated results are returned in a newly created
-  TestResultSet object.
-
-  """
-  res_set = resultset.TestResultSet(dir_)
-  dirlist = fnmatch.filter(os.listdir(dir_), "run.???")
-  dirlist.sort()
-  for run in dirlist:
-    keyval_path = os.path.join(dir_, run, _RESULTS_PATH)
-    try:
-      kvf = open(keyval_path)
-    except IOError:
-      continue
-    ReadKeyvalFile(res_set, kvf)
-  res_set.FinalizeResults()
-  return res_set
-
-
-def PrintRawData(dirlist, use_timestats, keylist):
   """Print 'bootperf' results in "raw data" format."""
   for dir_ in dirlist:
+    results = reader(dir_)
     if use_timestats:
-      keyset = ReadResultsDirectory(dir_).TimeKeySet()
+      keyset = results.TimeKeySet()
     else:
-      keyset = ReadResultsDirectory(dir_).DiskKeySet()
+      keyset = results.DiskKeySet()
     for i in range(0, keyset.num_iterations):
       if len(dirlist) > 1:
-        line = "%s %3d" % (dir_, i)
+        line = "%s %3d" % (results.name, i)
       else:
         line = "%3d" % i
       if keylist is not None:
@@ -87,7 +33,7 @@ def PrintRawData(dirlist, use_timestats, keylist):
       print line
 
 
-def PrintStatisticsSummary(dirlist, use_timestats, keylist):
+def PrintStatisticsSummary(reader, dirlist, use_timestats, keylist):
   """Print 'bootperf' results in "summary of averages" format."""
   if use_timestats:
     header = "%5s %3s  %5s %3s  %s" % (
@@ -99,10 +45,11 @@ def PrintStatisticsSummary(dirlist, use_timestats, keylist):
     format = "%6s %2d%%  %6s %2d%%  %s"
   havedata = False
   for dir_ in dirlist:
+    results = reader(dir_)
     if use_timestats:
-      keyset = ReadResultsDirectory(dir_).TimeKeySet()
+      keyset = results.TimeKeySet()
     else:
-      keyset = ReadResultsDirectory(dir_).DiskKeySet()
+      keyset = results.DiskKeySet()
     if keylist is not None:
       markers = keylist
     else:
@@ -110,7 +57,7 @@ def PrintStatisticsSummary(dirlist, use_timestats, keylist):
     if havedata:
       print
     if len(dirlist) > 1:
-      print "%s" % dir_,
+      print "%s" % results.name,
     print "(on %d cycles):" % keyset.num_iterations
     print header
     prevvalue = 0

bootperf-bin/resultsdir.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Routines for reading performance results from a directory.
+
+The directory should match the format created by the 'bootperf'
+script; see comments in that script for a summary of the layout.
+
+"""
+
+import fnmatch
+import os
+import re
+
+import resultset
+
+
+_PERF_KEYVAL_PATTERN = re.compile("(.*){perf}=(.*)\n")
+
+
+def _ReadKeyvalFile(results, file_):
+  """Read an autotest keyval file, and process the results.
+
+  The `file_` parameter is a file object with contents in autotest
+  perf keyval format:
+      <keyname>{perf}=<value>
+
+  Each iteration of the test is terminated with a single blank line,
+  including the last iteration.  Each iteration's results are added
+  to the `results` parameter, which should be an instance of
+  TestResultSet.
+
+  """
+  kvd = {}
+  for line in iter(file_):
+    if line == "\n":
+      results.AddIterationResults(kvd)
+      kvd = {}
+      continue
+    m = _PERF_KEYVAL_PATTERN.match(line)
+    if m is None:
+      continue
+    kvd[m.group(1)] = m.group(2)
+
+
+_RESULTS_PATH = (
+    "summary/platform_BootPerfServer/platform_BootPerfServer/results/keyval")
+
+
+def ReadResultsDirectory(dir_):
+  """Process results from a 'bootperf' output directory.
+
+  The accumulated results are returned in a newly created
+  TestResultSet object.
+
+  """
+  res_set = resultset.TestResultSet(dir_)
+  dirlist = fnmatch.filter(os.listdir(dir_), "run.???")
+  dirlist.sort()
+  for rundir in dirlist:
+    keyval_path = os.path.join(dir_, rundir, _RESULTS_PATH)
+    try:
+      kvf = open(keyval_path)
+    except IOError:
+      continue
+    _ReadKeyvalFile(res_set, kvf)
+  res_set.FinalizeResults()
+  return res_set

bootperf-bin/showbootdata

@@ -36,80 +36,104 @@ for one boot cycle.
 import sys
 import optparse
 
+import resultsdir
 import perfprinter
 
 
-_usage = "%prog [options] [results-directory ...]"
+_USAGE = "%prog [options] [results-directory ...]"
-_description = """\
+_DESCRIPTION = """\
 Summarize boot time performance results.  The result directory
 arguments are directories previously specified as output for the
 'bootperf' script.
 """
-optparser = optparse.OptionParser(usage=_usage, description=_description)
 
-optgroup = optparse.OptionGroup(
-    optparser, "Selecting boot time or disk statistics (choose one)")
-optgroup.add_option(
-    "-d", "--diskstats", action="store_true",
-    dest="use_diskstats",
-    help="use statistics for bytes read since kernel startup")
-optgroup.add_option(
-    "-t", "--timestats", action="store_true",
-    dest="use_timestats",
-    help="use statistics for time since kernel startup (default)")
-optparser.add_option_group(optgroup)
-optparser.set_defaults(use_diskstats=False)
-optparser.set_defaults(use_timestats=False)
 
-optgroup = optparse.OptionGroup(optparser, "Event selection")
+def _SetupOptions():
-optgroup.add_option(
+  optparser = optparse.OptionParser(usage=_USAGE, description=_DESCRIPTION)
-    "-e", "--event", action="append",
-    dest="eventnames",
-    help="restrict statistics to the comma-separated list of events")
-optparser.add_option_group(optgroup)
 
-optgroup = optparse.OptionGroup(
+  optgroup = optparse.OptionGroup(
-    optparser, "Display mode selection (choose one)")
+      optparser, "Selecting boot time or disk statistics (choose one)")
-optgroup.add_option(
+  optgroup.add_option(
-    "-a", "--averages", action="store_true",
+      "-t", "--timestats", action="store_true",
-    dest="print_averages",
+      dest="use_timestats",
-    help="display a summary of the averages of chosen statistics (default)")
+      help="use statistics for time since kernel startup (default)")
-optgroup.add_option(
+  optgroup.add_option(
-    "-r", "--rawdata", action="store_true",
+      "-d", "--diskstats", action="store_true",
-    dest="print_raw",
+      dest="use_diskstats",
-    help="display raw data from all boot iterations")
+      help="use statistics for bytes read since kernel startup")
-optparser.add_option_group(optgroup)
+  optparser.add_option_group(optgroup)
-optparser.set_defaults(print_averages=False)
+  optparser.set_defaults(use_diskstats=False)
-optparser.set_defaults(print_raw=False)
+  optparser.set_defaults(use_timestats=False)
+
+  optgroup = optparse.OptionGroup(optparser, "Event selection")
+  optgroup.add_option(
+      "-e", "--event", action="append",
+      dest="eventnames",
+      help="restrict statistics to the comma-separated list of events")
+  optparser.add_option_group(optgroup)
+
+  optgroup = optparse.OptionGroup(
+      optparser, "Display mode selection (choose one)")
+  optgroup.add_option(
+      "-a", "--averages", action="store_true",
+      dest="print_averages",
+      help="display a summary of the averages of chosen statistics (default)")
+  optgroup.add_option(
+      "-r", "--rawdata", action="store_true",
+      dest="print_raw",
+      help="display raw data from all boot iterations")
+  optparser.add_option_group(optgroup)
+  optparser.set_defaults(print_averages=False)
+  optparser.set_defaults(print_raw=False)
+  return optparser
+
+
+def _ProcessDisplayOptions(options):
+  display_count = 0
+  if options.print_averages:
+    display_count += 1
+    printfunc = perfprinter.PrintStatisticsSummary
+  if options.print_raw:
+    display_count += 1
+    printfunc = perfprinter.PrintRawData
+  if display_count == 0:
+    printfunc = perfprinter.PrintStatisticsSummary
+  elif display_count > 1:
+    print >>sys.stderr, "Can't use -a and -r together.\n"
+    return None
+  return printfunc
+
+
+def _ProcessStatsOptions(options):
+  if options.use_timestats and options.use_diskstats:
+    print >>sys.stderr, "Can't use -t and -d together.\n"
+    return None
+  return not options.use_diskstats
+
+
+def _ProcessKeylistOptions(options):
+  if not options.eventnames:
+    return None
+  keylist = []
+  for kl in options.eventnames:
+    keylist.extend(kl.split(','))
+  return keylist
 
 
 def main(argv):
+  optparser = _SetupOptions()
   (options, args) = optparser.parse_args(argv)
-  if options.print_averages and options.print_raw:
+  printfunc = _ProcessDisplayOptions(options)
-    print >>sys.stderr, "Can't use -a and -r together.\n"
+  use_timestats = _ProcessStatsOptions(options)
+  keylist = _ProcessKeylistOptions(options)
+  if printfunc is None or use_timestats is None:
     optparser.print_help()
     sys.exit(1)
-  elif options.print_raw:
-    printfunc = perfprinter.PrintRawData
-  else:
-    printfunc = perfprinter.PrintStatisticsSummary
-  if options.use_timestats and options.use_diskstats:
-    print >>sys.stderr, "Can't use -t and -d together.\n"
-    optparser.print_help()
-    sys.exit(1)
-  elif options.use_diskstats:
-    use_timestats = False
-  else:
-    use_timestats = True
-  if options.eventnames:
-    keylist = []
-    for kl in options.eventnames:
-      keylist.extend(kl.split(','))
-  else:
-    keylist = None
   if not args:
     args = ["."]
-  printfunc(args, use_timestats, keylist)
+  printfunc(resultsdir.ReadResultsDirectory,
+            args, use_timestats, keylist)
+
 
 if __name__ == "__main__":
   main(sys.argv[1:])