Add more cbuild options, revise configs

BUG=chromium-os:11229
TEST=Local, unittests

Review URL: http://codereview.chromium.org/6245013

Change-Id: I103d693a2e6ce033c1f495f9f48450156ad72e94

bin/cbuildbot.py

@@ -323,10 +323,16 @@ def _IncrementalCheckout(buildroot, retries=_DEFAULT_RETRIES):
   RepoSync(buildroot, retries)
 
 
-def _MakeChroot(buildroot):
+def _MakeChroot(buildroot, replace=False):
   """Wrapper around make_chroot."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
-  RunCommand(['./make_chroot', '--fast'], cwd=cwd)
+  
+  cmd = ['./make_chroot', '--fast']
+  
+  if replace:
+    cmd.append('--replace')
+  
+  RunCommand(cmd, cwd=cwd)
 
 
 def _GetPortageEnvVar(buildroot, board, envvar):
@@ -358,7 +364,7 @@ def _SetupBoard(buildroot, board='x86-generic'):
              cwd=cwd, enter_chroot=True)
 
 
-def _Build(buildroot, emptytree):
+def _Build(buildroot, emptytree, build_autotest=True, usepkg=True):
   """Wrapper around build_packages."""
   cwd = os.path.join(buildroot, 'src', 'scripts')
   if emptytree:
@@ -366,6 +372,12 @@ def _Build(buildroot, emptytree):
   else:
     cmd = ['./build_packages']
 
+  if not build_autotest:
+    cmd.append('--nowithautotest')
+  
+  if not usepkg:
+    cmd.append('--nousepkg')
+
   RunCommand(cmd, cwd=cwd, enter_chroot=True)
 
 
@@ -559,7 +571,6 @@ def _ArchiveTestResults(buildroot, board, test_results_dir,
 
 def _GetConfig(config_name):
   """Gets the configuration for the build"""
-  default = config['default']
   buildconfig = {}
   if not config.has_key(config_name):
     Warning('Non-existent configuration specified.')
@@ -570,13 +581,7 @@ def _GetConfig(config_name):
       Warning('  %s' % name)
     sys.exit(1)
 
-  buildconfig = config[config_name]
+  return config[config_name]
-
-  for key in default.iterkeys():
-    if not buildconfig.has_key(key):
-      buildconfig[key] = default[key]
-
-  return buildconfig
 
 
 def _ResolveOverlays(buildroot, overlays):
@@ -644,8 +649,6 @@ def main():
   parser = optparse.OptionParser(usage=usage)
   parser.add_option('-a', '--acl', default='private',
                     help='ACL to set on GSD archives')
-  parser.add_option('--archive_build', action='store_true', default=False,
-                    help='Run the archive_build script.')
   parser.add_option('-r', '--buildroot',
                     help='root directory where build occurs', default=".")
   parser.add_option('-n', '--buildnumber',
@@ -730,7 +733,7 @@ def main():
         Die('Missing overlay: %s' % path)
 
     if not os.path.isdir(chroot_path):
-      _MakeChroot(buildroot)
+      _MakeChroot(buildroot, buildconfig['chroot_replace'])
 
     if not os.path.isdir(boardpath):
       _SetupBoard(buildroot, board=buildconfig['board'])
@@ -750,14 +753,17 @@ def main():
     _EnableLocalAccount(buildroot)
 
     if options.build:
-      _Build(buildroot, emptytree)
+      _Build(buildroot,
+             emptytree,
+             build_autotest=(buildconfig['vm_tests'] and options.tests),
+             usepkg=buildconfig['usepkg'])
 
       if buildconfig['unittests'] and options.tests:
         _RunUnitTests(buildroot)
 
       _BuildImage(buildroot)
 
-    if buildconfig['tests'] and options.tests:
+    if buildconfig['vm_tests'] and options.tests:
       _BuildVMImageForTesting(buildroot)
       test_results_dir = '/tmp/run_remote_tests.%s' % options.buildnumber
       try:
@@ -791,7 +797,7 @@ def main():
         if buildconfig['important'] and not options.debug:
           cbuildbot_comm.PublishStatus(cbuildbot_comm.STATUS_BUILD_COMPLETE)
 
-    if options.archive_build:
+    if buildconfig['archive_build']:
       _LegacyArchiveBuild(bot_id,
                           buildconfig,
                           options.buildnumber,

bin/cbuildbot_config.py

@@ -8,121 +8,144 @@ Each dictionary entry is in turn a dictionary of config_param->value.
 
 config_param's:
 board -- The board of the image to build.
-uprev -- Uprevs the local ebuilds to build new changes since last stable.
+
-         build.  If master then also pushes these changes on success.
 master -- This bot pushes changes to the overlays.
 important -- Master bot uses important bots to determine overall status.
              i.e. if master bot succeeds and other important slaves succeed
              then the master will uprev packages.  This should align
-             with info vs. closer except for the master.
+             with info vs. closer except for the master.and options.tests
 hostname -- Needed for 'important' slaves.  The hostname of the bot.  Should
             match hostname in slaves.cfg in buildbot checkout.
-unittests -- Runs unittests for packages.
+
-tests -- Runs the smoke suite and au test harness in a qemu-based VM using KVM.
+uprev -- Uprevs the local ebuilds to build new changes since last stable.
+         build.  If master then also pushes these changes on success.
 rev_overlays -- Select what overlays to look at for revving. This can be
                 'public', 'private' or 'both'.
 push_overlays -- Select what overlays to push at. This should be a subset of
                  rev_overlays for the particular builder.  Must be None if
                  not a master.  There should only be one master bot pushing
                  changes to each overlay per branch.
-test_mod -- Create a test mod image. (default True)
+                 
-factory_install_mod -- Create a factory install image. (default True)
+unittests -- Runs unittests for packages.
-factory_test_mod -- Create a factory test image. (default True)
+vm_tests -- Runs the smoke suite and au test harness in a qemu-based VM
+            using KVM.
+
+usepkg -- Use binary packages to bootstrap, when possible. (emerge --usepkg)
+chroot_replace -- wipe and replace chroot, but not source.
+
+archive_build -- Do we run archive_build.sh
+test_mod -- Create a test mod image for archival.
+factory_install_mod -- Create a factory install image for archival.
+factory_test_mod -- Create a factory test image for archival.
 """
 
+# TODO(dgarrett) Make test_mod, factory_install_mod, factory_test_mod options
+# go away when these options work for arm.
+
+default = {
+  # 'board' No default value
+
+  'master' : False,
+  'important' : False,
+  # 'hostname' No default value
+
+  'uprev' : False,
+  'rev_overlays': 'public',
+  'push_overlays': None,
+
+  'unittests' : True,
+  'vm_tests' : True,
+
+  'usepkg' : True,
+  'chroot_replace' : False,
+  
+  'archive_build' : False,
+  'test_mod' : True,
+  'factory_install_mod' : True,
+  'factory_test_mod' : True,
+}
 
 config = {}
-config['default'] = {
+
-  'board' : 'x86-generic',
+config['x86-generic-pre-flight-queue'] = default.copy()
-  'uprev' : False,
+config['x86-generic-pre-flight-queue'].update({
-  'master' : False,
-  'important' : False,
-  'unittests' : False,
-  'tests' : False,
-  'rev_overlays': 'public',
-  'push_overlays': None,
-}
-config['x86-generic-pre-flight-queue'] = {
   'board' : 'x86-generic',
   'uprev' : True,
   'master' : True,
-  'important' : False,
   'hostname' : 'chromeosbuild2',
-  'unittests' : True,
-  'tests' : True,
   'rev_overlays': 'public',
   'push_overlays': 'public',
-}
+})
+
 config['x86-generic-chrome-pre-flight-queue'] = \
     config['x86-generic-pre-flight-queue']
-config['x86-mario-pre-flight-queue'] = {
+
+config['x86-mario-pre-flight-queue'] = default.copy()
+config['x86-mario-pre-flight-queue'].update({
   'board' : 'x86-mario',
   'uprev' : True,
   'master' : True,
-  'important' : False,
-  'unittests' : True,
-  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': 'private',
-}
+})
-config['x86-mario-pre-flight-branch'] = {
+
+config['x86-mario-pre-flight-branch'] = default.copy()
+config['x86-mario-pre-flight-branch'].update({
   'board' : 'x86-mario',
   'uprev' : True,
   'master' : True,
-  'important' : False,
-  'unittests' : True,
-  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': 'both',
-}
+})
-config['x86_agz_bin'] = {
+
+config['x86-agz-bin'] = default.copy()
+config['x86-agz-bin'].update({
   'board' : 'x86-agz',
   'uprev' : True,
-  'master' : False,
-  'important' : False,
-  'unittests' : True,
-  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': None,
-}
+})
-config['x86_dogfood_bin'] = {
+
+config['x86-dogfood-bin'] = default.copy()
+config['x86-dogfood-bin'].update({
   'board' : 'x86-dogfood',
   'uprev' : True,
-  'master' : False,
-  'important' : False,
-  'unittests' : True,
-  'tests' : True,
   'rev_overlays': 'both',
   'push_overlays': None,
-}
+})
-config['x86_pineview_bin'] = {
+
+config['x86-pineview-bin'] = default.copy()
+config['x86-pineview-bin'].update({
   'board' : 'x86-pineview',
   'uprev' : True,
-  'master' : False,
-  'important' : False,
-  'unittests': True,
   'rev_overlays': 'public',
   'push_overlays': None,
-}
+})
-config['arm_tegra2_bin'] = {
+
+config['arm-tegra2-bin'] = default.copy()
+config['arm-tegra2-bin'].update({
   'board' : 'tegra2_dev-board',
   'uprev' : True,
-  'master' : False,
-  'important' : False,
   'unittests' : False,
+  'vm_tests' : False,
   'rev_overlays': 'public',
-  'push_overlays': None,
   'factory_install_mod' : False,
   'factory_test_mod' : False,
-}
+})
-config['arm_generic_bin'] = {
+
+config['arm-generic-bin'] = default.copy()
+config['arm-generic-bin'].update({
   'board' : 'arm-generic',
   'uprev' : True,
-  'master' : False,
-  'important' : False,
   'unittests' : False,
-  'rev_overlays': 'public',
+  'vm_tests' : False,
-  'push_overlays': None,
   'factory_install_mod' : False,
   'factory_test_mod' : False,
-}
+})
+
+# TODO(dgarrett) delete when buildbot updated to use new names
+config['x86_agz_bin'] = config['x86-agz-bin']
+config['x86_dogfood_bin'] = config['x86-dogfood-bin']
+config['x86_pineview_bin'] = config['x86-pineview-bin'] 
+config['arm_tegra2_bin'] = config['arm-tegra2-bin'] 
+config['arm_generic_bin'] = config['arm-generic-bin']