Merge branch 'master' of ssh://gitrw.chromium.org:9222/crosutils

common.sh

@@ -77,6 +77,8 @@ fi
 # Load shflags
 if [[ -f /usr/lib/shflags ]]; then
   . /usr/lib/shflags
+elif [ -f ./lib/shflags/shflags ]; then
+  . "./lib/shflags/shflags"
 else
   . "${SRC_ROOT}/scripts/lib/shflags/shflags"
 fi

cros_generate_update_payload

@@ -119,6 +119,7 @@ DEFINE_string src_image "" "Optional: a source image. If specified, this makes\
  a delta update."
 DEFINE_boolean old_style "$FLAGS_TRUE" "Generate an old-style .gz full update."
 DEFINE_string output "" "Output file"
+DEFINE_boolean outside_chroot "$FLAGS_FALSE" "Running outside of chroot."
 DEFINE_boolean patch_kernel "$FLAGS_FALSE" "Whether or not to patch the kernel \
 with the patch from the stateful partition (default: false)"
 DEFINE_string private_key "" "Path to private key in .pem format."
@@ -133,8 +134,13 @@ eval set -- "${FLAGS_ARGV}"
 
 set -e
 
-if [ -n "$FLAGS_src_image" ]; then
-  # We need to be in the chroot for generating delta images
+if [ -n "$FLAGS_src_image" ] && \
+   [ "$FLAGS_outside_chroot" -eq "$FLAGS_FALSE" ]; then
+  # We need to be in the chroot for generating delta images.
+  # by specifying --outside_chroot you can choose not to assert
+  # this will allow us to run this script outside chroot.
+  # Running this script outside chroot requires copying delta_generator binary
+  # and also copying few shared libraries with it.
   assert_inside_chroot
 fi
 
@@ -202,7 +208,7 @@ if [ "$DELTA" -eq "$FLAGS_TRUE" -o "$FLAGS_old_style" -eq "$FLAGS_FALSE" ]; then
     DST_MNT=$(mktemp -d /tmp/src_root.XXXXXX)
     sudo mount -o loop,ro "$DST_ROOT" "$DST_MNT"
 
-    sudo "$GENERATOR" \
+    sudo LD_LIBRARY_PATH=${LD_LIBRARY_PATH}  PATH=${PATH} "$GENERATOR" \
         -new_dir "$DST_MNT" -new_image "$DST_ROOT" -new_kernel "$DST_KERNEL" \
         -old_dir "$SRC_MNT" -old_image "$SRC_ROOT" -old_kernel "$SRC_KERNEL" \
         -out_file "$FLAGS_output" -private_key "$FLAGS_private_key"

cros_workon

@@ -35,6 +35,7 @@ commands:
   start:  Moves an ebuild to live (intended to support development)
   stop:  Moves an ebuild to stable (use last known good)
   list:  List of live ebuilds (workon ebuilds if --all)
+  list-all:  List all of the live ebuilds for all setup boards
   iterate:  For each ebuild, cd to the source dir and run a commond"
 FLAGS "$@" || exit 1
 eval set -- "${FLAGS_ARGV}"
@@ -53,6 +54,7 @@ shift
 [ -n "${FLAGS_board}" ] && [ "${FLAGS_host}" = ${FLAGS_TRUE} ] && \
   FLAGS_board="" # kill board
 [ -z "${FLAGS_board}" ] && [ "${FLAGS_host}" = ${FLAGS_FALSE} ] && \
+[ "${WORKON_CMD}" != "list-all" ] && \
   die "You must specify either --host or --board="
 
 if [ -n "${FLAGS_board}" ]; then
@@ -136,6 +138,18 @@ show_live_ebuilds () {
   sed -n 's/^[~=]\(.*\)-9999$/\1/p' "${WORKON_FILE}"
 }
 
+# Display ebuilds currently part of the live branch and open for development
+# for any board that currently has live ebuilds.
+show_all_live_ebuilds () {
+  for workon_file in ${WORKON_DIR}/*; do
+    if [ -s "${workon_file}" ]; then
+      echo -e "${V_BOLD_GREEN}$(basename ${workon_file}):${V_VIDOFF}"
+      sed -n 's/^[~=]\(.*\)-9999$/    \1/p' "${workon_file}"
+      echo ""
+    fi
+  done
+}
+
 # This is called only for "cros-workon start". We dont handle the "stop" case since the local changes are ignored anyway since the 9999.ebuild is masked and we dont want to deal with what to do with the user's local changes.
 regen_manifest_and_sync() {
   # Nothing to do unless you are working on the minilayout
@@ -208,7 +222,7 @@ if [ ${FLAGS_all} = "${FLAGS_TRUE}" ]; then
   case ${WORKON_CMD} in
     start) ATOM_LIST=$(show_workon_ebuilds ${BOARD_KEYWORD});;
     stop|iterate) ATOM_LIST=$(show_live_ebuilds);;
-    list) ;;
+    list|list-all) ;;
     *) die "--all is invalid for the given command";;
   esac
 else # not selected --all
@@ -228,6 +242,7 @@ case ${WORKON_CMD} in
   start)  ebuild_to_live "${ATOM_LIST}" ;;
   stop)   ebuild_to_stable "${ATOM_LIST}" ;;
   list)   [ ${FLAGS_all} = "${FLAGS_FALSE}" ] && show_live_ebuilds || show_workon_ebuilds ${BOARD_KEYWORD} ;;
+  list-all) show_all_live_ebuilds ;;
   iterate)ebuild_iterate "${ATOM_LIST}" ;;
   *)      die "$(basename $0): command '${WORKON_CMD}' not recognized" ;;
 esac

parallel_emerge

@@ -638,7 +638,8 @@ class DepGraphGenerator(object):
     # are blacklisting them from automatic rebuilds because one of their
     # dependencies needs to be recompiled.
     rebuild_blacklist = set()
-    for pkg in ("media-plugins/o3d", "dev-java/icedtea"):
+    for pkg in ("chromeos-base/chromeos-chrome", "media-plugins/o3d",
+                "dev-java/icedtea"):
       for match in final_db.match_pkgs(pkg):
         rebuild_blacklist.add(str(match.cpv))
 
@@ -1234,8 +1235,9 @@ def PrintDepsMap(deps_map):
 
 
 class EmergeJobState(object):
-  __slots__ = ["done", "filename", "last_output_seek", "last_output_timestamp",
-               "pkgname", "retcode", "start_timestamp", "target"]
+  __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek",
+               "last_output_timestamp", "pkgname", "retcode", "start_timestamp",
+               "target"]
 
   def __init__(self, target, pkgname, done, filename, start_timestamp,
                retcode=None):
@@ -1253,6 +1255,11 @@ class EmergeJobState(object):
     # The filename where output is currently stored.
     self.filename = filename
 
+    # The timestamp of the last time we printed the name of the log file. We
+    # print this at the beginning of the job, so this starts at
+    # start_timestamp.
+    self.last_notify_timestamp = start_timestamp
+
     # The location (in bytes) of the end of the last complete line we printed.
     # This starts off at zero. We use this to jump to the right place when we
     # print output from the same ebuild multiple times.
@@ -1548,6 +1555,36 @@ class EmergeQueue(object):
   def _Status(self):
     """Print status."""
     current_time = time.time()
+    no_output = True
+
+    # Print interim output every minute if --show-output is used. Otherwise,
+    # print notifications about running packages every 2 minutes, and print
+    # full output for jobs that have been running for 60 minutes or more.
+    if self._show_output:
+      interval = 60
+      notify_interval = 0
+    else:
+      interval = 60 * 60
+      notify_interval = 60 * 2
+    for target, job in self._jobs.iteritems():
+      if job:
+        last_timestamp = max(job.start_timestamp, job.last_output_timestamp)
+        if last_timestamp + interval < current_time:
+          self._print_queue.put(JobPrinter(job))
+          job.last_output_timestamp = current_time
+          no_output = False
+        elif (notify_interval and
+              job.last_notify_timestamp + notify_interval < current_time):
+          job_seconds = current_time - job.start_timestamp
+          args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename)
+          info = "Still building %s (%dm%.1fs). Logs in %s" % args
+          job.last_notify_timestamp = current_time
+          self._Print(info)
+          no_output = False
+
+    # If we haven't printed any messages yet, print a general status message
+    # here.
+    if no_output:
       seconds = current_time - GLOBAL_START
       line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s "
               "[Time %dm%.1fs Load %s]")
@@ -1556,19 +1593,6 @@ class EmergeQueue(object):
                           len(self._retry_queue), self._total_jobs,
                           seconds / 60, seconds % 60, self._LoadAvg()))
 
-    # Print interim output every minute if --show-output is used. Otherwise,
-    # only print output if a job has been running for 60 minutes or more.
-    if self._show_output:
-      interval = 60
-    else:
-      interval = 60 * 60
-    for target, job in self._jobs.iteritems():
-      if job:
-        last_timestamp = max(job.start_timestamp, job.last_output_timestamp)
-        if last_timestamp + interval < current_time:
-          self._print_queue.put(JobPrinter(job))
-          job.last_output_timestamp = current_time
-
   def _Finish(self, target):
     """Mark a target as completed and unblock dependecies."""
     for dep in self._deps_map[target]["provides"]:

serve_factory_packages.py

@@ -122,11 +122,23 @@ def sign_build(image, output):
 
 
 def build_factory_packages(signed_image, base_image, fw_updater, folder, board):
-  """Build image and modify mini omaha config.
+  """Build factory packages and modify mini omaha config.
+
+  Args:
+    signed_image: signed image
+    base_image: base image
+    fw_updater: firmware updater
+    folder: destination folder to write packages
+    board: platform to build
   """
   cmd = ('./make_factory_package.sh --release %s --factory %s'
-         ' --firmware_updater %s --subfolder %s --board %s'
-         % (signed_image, base_image, fw_updater, folder, board))
+         ' --subfolder %s --board %s'
+         % (signed_image, base_image, folder, board))
+  if fw_updater:
+    cmd = '%s --firmware_updater %s' % (cmd, fw_updater)
+  else:
+    print ('No --firmware_updater specified. Not including firmware shellball.')
+
   print 'Building factory packages: %s' % cmd
   build_packages_process = KillableProcess(cmd, cwd=SCRIPTS_DIR)
   build_packages_process.start()
@@ -146,11 +158,12 @@ def main(argv):
 
   if not FLAGS.base_image:
     exit('No --base_image specified.')
-  if not FLAGS.firmware_updater:
-    exit('No --firmware_updater specified.')
+
+  if FLAGS.firmware_updater:
+    assert_is_file(FLAGS.firmware_updater,
+                   'Invalid or missing firmware updater.')
 
   assert_is_file(FLAGS.base_image, 'Invalid or missing base image.')
-  assert_is_file(FLAGS.firmware_updater, 'Invalid or missing firmware updater.')
 
   signed_image = os.path.join(os.path.dirname(FLAGS.base_image),
                               '%s_ssd_signed.bin' % FLAGS.board)