From 27dbab06f26ca4e23a3f39e58f4c280b98129214 Mon Sep 17 00:00:00 2001 From: Scott Zawalski Date: Tue, 2 Nov 2010 00:46:37 -0700 Subject: [PATCH 1/6] Update prebuilt.py call to reference proper external path BUG=NA TEST=Ran on a full buildbot run to verify paths Review URL: http://codereview.chromium.org/4282002 --- archive_build.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/archive_build.sh b/archive_build.sh index c9d7790952..84f1d51d98 100755 --- a/archive_build.sh +++ b/archive_build.sh @@ -273,7 +273,8 @@ then # This will upload prebuilt packages to Google Storage. prebuilt_cmd="${SCRIPTS_DIR}/prebuilt.py" prebuilt_cmd="$prebuilt_cmd -u gs://chromeos-prebuilt --git-sync -V master" - prebuilt_cmd="$prebuilt_cmd -p $(readlink -f ../../../..) -b ${FLAGS_board}" + prebuilt_cmd="$prebuilt_cmd -p $(readlink -f ../../../../build/chromiumos)" + prebuilt_cmd="$prebuilt_cmd -b ${FLAGS_board}" if [ "${FLAGS_BOARD}" == "x86-generic" ] then From c3874d110652c9f06d9ab4303a8e4767672a61f8 Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 2 Nov 2010 09:01:47 -0700 Subject: [PATCH 2/6] Add chromeos-chrome to rebuild blacklist for faster builds. chromeos-chrome has mostly self-contained dependencies, so there usually isn't need to rebuild it just because a dependency changed. This allows for us to use the binaries for chromeos-chrome more often. BUG=chromium-os:8394 TEST=Check that chrome doesn't rebuild with ./parallel_emerge -gp --workon=libcros --board=x86-generic chromeos-chrome Change-Id: Ifa14c890917991a8d11f1f0e757f28686d611a72 Review URL: http://codereview.chromium.org/4243003 --- parallel_emerge | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/parallel_emerge b/parallel_emerge index 37214241e1..acf17fb2d6 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -638,7 +638,8 @@ class DepGraphGenerator(object): # are blacklisting them from automatic rebuilds because one of their # dependencies needs to be recompiled. rebuild_blacklist = set() - for pkg in ("media-plugins/o3d", "dev-java/icedtea"): + for pkg in ("chromeos-base/chromeos-chrome", "media-plugins/o3d", + "dev-java/icedtea"): for match in final_db.match_pkgs(pkg): rebuild_blacklist.add(str(match.cpv)) From abb9144d97ee35386e1fbce74aefb5c990d26bc7 Mon Sep 17 00:00:00 2001 From: Anton Staaf Date: Tue, 2 Nov 2010 11:23:00 -0700 Subject: [PATCH 3/6] Change "cros_workon list" to show packages for all boards. Change-Id: Id4880b423eaabbdefc91060a04629a9018295b4d BUG=None TEST="cros_workon list" works, so does "cros_workon --board tegra2_seaboard start chromeos-u-boot" Review URL: http://codereview.chromium.org/4131008 --- cros_workon | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/cros_workon b/cros_workon index 6892e7a669..c48b6431e5 100755 --- a/cros_workon +++ b/cros_workon @@ -35,6 +35,7 @@ commands: start: Moves an ebuild to live (intended to support development) stop: Moves an ebuild to stable (use last known good) list: List of live ebuilds (workon ebuilds if --all) + list-all: List all of the live ebuilds for all setup boards iterate: For each ebuild, cd to the source dir and run a commond" FLAGS "$@" || exit 1 eval set -- "${FLAGS_ARGV}" @@ -53,6 +54,7 @@ shift [ -n "${FLAGS_board}" ] && [ "${FLAGS_host}" = ${FLAGS_TRUE} ] && \ FLAGS_board="" # kill board [ -z "${FLAGS_board}" ] && [ "${FLAGS_host}" = ${FLAGS_FALSE} ] && \ +[ "${WORKON_CMD}" != "list-all" ] && \ die "You must specify either --host or --board=" if [ -n "${FLAGS_board}" ]; then @@ -136,6 +138,18 @@ show_live_ebuilds () { sed -n 's/^[~=]\(.*\)-9999$/\1/p' "${WORKON_FILE}" } +# Display ebuilds currently part of the live branch and open for development +# for any board that currently has live ebuilds. +show_all_live_ebuilds () { + for workon_file in ${WORKON_DIR}/*; do + if [ -s "${workon_file}" ]; then + echo -e "${V_BOLD_GREEN}$(basename ${workon_file}):${V_VIDOFF}" + sed -n 's/^[~=]\(.*\)-9999$/ \1/p' "${workon_file}" + echo "" + fi + done +} + # This is called only for "cros-workon start". We dont handle the "stop" case since the local changes are ignored anyway since the 9999.ebuild is masked and we dont want to deal with what to do with the user's local changes. regen_manifest_and_sync() { # Nothing to do unless you are working on the minilayout @@ -208,7 +222,7 @@ if [ ${FLAGS_all} = "${FLAGS_TRUE}" ]; then case ${WORKON_CMD} in start) ATOM_LIST=$(show_workon_ebuilds ${BOARD_KEYWORD});; stop|iterate) ATOM_LIST=$(show_live_ebuilds);; - list) ;; + list|list-all) ;; *) die "--all is invalid for the given command";; esac else # not selected --all @@ -228,6 +242,7 @@ case ${WORKON_CMD} in start) ebuild_to_live "${ATOM_LIST}" ;; stop) ebuild_to_stable "${ATOM_LIST}" ;; list) [ ${FLAGS_all} = "${FLAGS_FALSE}" ] && show_live_ebuilds || show_workon_ebuilds ${BOARD_KEYWORD} ;; + list-all) show_all_live_ebuilds ;; iterate)ebuild_iterate "${ATOM_LIST}" ;; *) die "$(basename $0): command '${WORKON_CMD}' not recognized" ;; esac From 0addea30fd87819a90b21b72a2f91b887cadf92c Mon Sep 17 00:00:00 2001 From: David James Date: Tue, 2 Nov 2010 14:52:23 -0700 Subject: [PATCH 4/6] Print status messages in parallel_emerge about jobs that are longer than 2 mins. Currently, if a build is slow, you only get debug output after an hour. This is to allow for uncluttered output. If output is too cluttered, it's hard to distinguish regular output from errors. The problem with this approach is that it's often hard to debug why the build is slow. Now that Chrome builds by default, it takes over an hour to build, and people see little indication as to why. You can show the output with build_packages --showoutput, but that is often too verbose and clutters the output too much. Here's an example log snippet that is hard to debug: Pending 2, Ready 0, Running 1, Retrying 0, Total 22 [Time 5m20.5s Load 3.69 3.04 2.66] Pending 2, Ready 0, Running 1, Retrying 0, Total 22 [Time 5m25.5s Load 3.40 2.99 2.65] ... yada yada yada ... Pending 2, Ready 0, Running 1, Retrying 0, Total 22 [Time 45m32.9s Load 1.00 1.18 2.95] From the above output, we see that a package is building for a long time, but we don't know what package. We should output the package name every two minutes at least so people know what package is taking so long. That's what this change implements. BUG=chromium-os:8575 TEST=Confirmed new status appear for regular build_packages. Confirmed build_packages --showoutput is unchanged. Change-Id: Ie18b23ac7a8a6e2c24b43ec3691606c7da5e43cb Review URL: http://codereview.chromium.org/4318003 --- parallel_emerge | 43 +++++++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 10 deletions(-) diff --git a/parallel_emerge b/parallel_emerge index acf17fb2d6..66b2334da6 100755 --- a/parallel_emerge +++ b/parallel_emerge @@ -1235,8 +1235,9 @@ def PrintDepsMap(deps_map): class EmergeJobState(object): - __slots__ = ["done", "filename", "last_output_seek", "last_output_timestamp", - "pkgname", "retcode", "start_timestamp", "target"] + __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek", + "last_output_timestamp", "pkgname", "retcode", "start_timestamp", + "target"] def __init__(self, target, pkgname, done, filename, start_timestamp, retcode=None): @@ -1254,6 +1255,11 @@ class EmergeJobState(object): # The filename where output is currently stored. self.filename = filename + # The timestamp of the last time we printed the name of the log file. We + # print this at the beginning of the job, so this starts at + # start_timestamp. + self.last_notify_timestamp = start_timestamp + # The location (in bytes) of the end of the last complete line we printed. # This starts off at zero. We use this to jump to the right place when we # print output from the same ebuild multiple times. @@ -1549,26 +1555,43 @@ class EmergeQueue(object): def _Status(self): """Print status.""" current_time = time.time() - seconds = current_time - GLOBAL_START - line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " - "[Time %dm%.1fs Load %s]") - qsize = self._emerge_queue.qsize() - self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, - len(self._retry_queue), self._total_jobs, - seconds / 60, seconds % 60, self._LoadAvg())) + no_output = True # Print interim output every minute if --show-output is used. Otherwise, - # only print output if a job has been running for 60 minutes or more. + # print notifications about running packages every 2 minutes, and print + # full output for jobs that have been running for 60 minutes or more. if self._show_output: interval = 60 + notify_interval = 0 else: interval = 60 * 60 + notify_interval = 60 * 2 for target, job in self._jobs.iteritems(): if job: last_timestamp = max(job.start_timestamp, job.last_output_timestamp) if last_timestamp + interval < current_time: self._print_queue.put(JobPrinter(job)) job.last_output_timestamp = current_time + no_output = False + elif (notify_interval and + job.last_notify_timestamp + notify_interval < current_time): + job_seconds = current_time - job.start_timestamp + args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename) + info = "Still building %s (%dm%.1fs). Logs in %s" % args + job.last_notify_timestamp = current_time + self._Print(info) + no_output = False + + # If we haven't printed any messages yet, print a general status message + # here. + if no_output: + seconds = current_time - GLOBAL_START + line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " + "[Time %dm%.1fs Load %s]") + qsize = self._emerge_queue.qsize() + self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, + len(self._retry_queue), self._total_jobs, + seconds / 60, seconds % 60, self._LoadAvg())) def _Finish(self, target): """Mark a target as completed and unblock dependecies.""" From 7f55e42020bc9ab5da5fb0db577804351bc7eca2 Mon Sep 17 00:00:00 2001 From: Huyen Nguyen Date: Tue, 2 Nov 2010 15:24:50 -0700 Subject: [PATCH 5/6] Make --firmware_updater an optional flag. Make --firmware_updater an optional flag. Change-Id: If7ec165a3ca123f1e0e5782f203d63e45a82582e BUG=7474 TEST=run locally Review URL: http://codereview.chromium.org/4237002 --- serve_factory_packages.py | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/serve_factory_packages.py b/serve_factory_packages.py index a3a08bd237..41c35176e9 100644 --- a/serve_factory_packages.py +++ b/serve_factory_packages.py @@ -122,11 +122,23 @@ def sign_build(image, output): def build_factory_packages(signed_image, base_image, fw_updater, folder, board): - """Build image and modify mini omaha config. + """Build factory packages and modify mini omaha config. + + Args: + signed_image: signed image + base_image: base image + fw_updater: firmware updater + folder: destination folder to write packages + board: platform to build """ cmd = ('./make_factory_package.sh --release %s --factory %s' - ' --firmware_updater %s --subfolder %s --board %s' - % (signed_image, base_image, fw_updater, folder, board)) + ' --subfolder %s --board %s' + % (signed_image, base_image, folder, board)) + if fw_updater: + cmd = '%s --firmware_updater %s' % (cmd, fw_updater) + else: + print ('No --firmware_updater specified. Not including firmware shellball.') + print 'Building factory packages: %s' % cmd build_packages_process = KillableProcess(cmd, cwd=SCRIPTS_DIR) build_packages_process.start() @@ -146,11 +158,12 @@ def main(argv): if not FLAGS.base_image: exit('No --base_image specified.') - if not FLAGS.firmware_updater: - exit('No --firmware_updater specified.') + + if FLAGS.firmware_updater: + assert_is_file(FLAGS.firmware_updater, + 'Invalid or missing firmware updater.') assert_is_file(FLAGS.base_image, 'Invalid or missing base image.') - assert_is_file(FLAGS.firmware_updater, 'Invalid or missing firmware updater.') signed_image = os.path.join(os.path.dirname(FLAGS.base_image), '%s_ssd_signed.bin' % FLAGS.board) From c6e23cdb38d4646e25caff98dc1debd92d2bab3d Mon Sep 17 00:00:00 2001 From: Raja Aluri Date: Tue, 2 Nov 2010 15:55:57 -0700 Subject: [PATCH 6/6] 1. Not to assert inside chroot, if so desired. 2. Source shflags, if they are in the current directory Change-Id: Id22e597a6fb71905f2d0ca5c4a1d94ce5a8a931f Review URL: http://codereview.chromium.org/4177005 --- common.sh | 2 ++ cros_generate_update_payload | 12 +++++++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/common.sh b/common.sh index 9a9699ede7..25abd9b4c3 100644 --- a/common.sh +++ b/common.sh @@ -77,6 +77,8 @@ fi # Load shflags if [[ -f /usr/lib/shflags ]]; then . /usr/lib/shflags +elif [ -f ./lib/shflags/shflags ]; then + . "./lib/shflags/shflags" else . "${SRC_ROOT}/scripts/lib/shflags/shflags" fi diff --git a/cros_generate_update_payload b/cros_generate_update_payload index 829f7a3a45..dd0a67c06e 100755 --- a/cros_generate_update_payload +++ b/cros_generate_update_payload @@ -119,6 +119,7 @@ DEFINE_string src_image "" "Optional: a source image. If specified, this makes\ a delta update." DEFINE_boolean old_style "$FLAGS_TRUE" "Generate an old-style .gz full update." DEFINE_string output "" "Output file" +DEFINE_boolean outside_chroot "$FLAGS_FALSE" "Running outside of chroot." DEFINE_boolean patch_kernel "$FLAGS_FALSE" "Whether or not to patch the kernel \ with the patch from the stateful partition (default: false)" DEFINE_string private_key "" "Path to private key in .pem format." @@ -133,8 +134,13 @@ eval set -- "${FLAGS_ARGV}" set -e -if [ -n "$FLAGS_src_image" ]; then - # We need to be in the chroot for generating delta images +if [ -n "$FLAGS_src_image" ] && \ + [ "$FLAGS_outside_chroot" -eq "$FLAGS_FALSE" ]; then + # We need to be in the chroot for generating delta images. + # by specifying --outside_chroot you can choose not to assert + # this will allow us to run this script outside chroot. + # Running this script outside chroot requires copying delta_generator binary + # and also copying few shared libraries with it. assert_inside_chroot fi @@ -202,7 +208,7 @@ if [ "$DELTA" -eq "$FLAGS_TRUE" -o "$FLAGS_old_style" -eq "$FLAGS_FALSE" ]; then DST_MNT=$(mktemp -d /tmp/src_root.XXXXXX) sudo mount -o loop,ro "$DST_ROOT" "$DST_MNT" - sudo "$GENERATOR" \ + sudo LD_LIBRARY_PATH=${LD_LIBRARY_PATH} PATH=${PATH} "$GENERATOR" \ -new_dir "$DST_MNT" -new_image "$DST_ROOT" -new_kernel "$DST_KERNEL" \ -old_dir "$SRC_MNT" -old_image "$SRC_ROOT" -old_kernel "$SRC_KERNEL" \ -out_file "$FLAGS_output" -private_key "$FLAGS_private_key"