From 615fe57ff0320d2c36b03d177af1e1b6c49c0005 Mon Sep 17 00:00:00 2001 From: Matt Tennant Date: Mon, 26 Mar 2012 11:30:33 -0700 Subject: [PATCH] build_packages: Upload command stats after each run. This changelist adds code in common.sh to support collecting command statistics before calling upload_command_stats to upload those stats to an appspot instance. The presence of a file at ~/.disable_build_stats_upload will disable all uploading of build command stats. BUG=chromium-os:27355 TEST=`build_packages --board=x86-generic` with missing appspot instance shows upload error but still goes through with build. TEST=`build_packages --board=x86-generic` with real appspot instance completes upload with all expected stats and does not affect build. TEST=From outside chroot: `touch ~/.disable_build_stats_upload` `cros_sdk` 'build_packages --board=x86-generic` Nothing uploaded due to .disable file Change-Id: Iac071d1cc55a44335fc7c846960c7ae45fc93ed8 Reviewed-on: https://gerrit.chromium.org/gerrit/19401 Tested-by: Matt Tennant Reviewed-by: Matt Tennant Commit-Ready: Matt Tennant --- build_packages | 19 +++++++++-- common.sh | 76 +++++++++++++++++++++++++++++++++++++++-- sdk_lib/enter_chroot.sh | 12 +++++-- 3 files changed, 99 insertions(+), 8 deletions(-) diff --git a/build_packages b/build_packages index bbe4864f99..a1212f7337 100755 --- a/build_packages +++ b/build_packages @@ -186,9 +186,24 @@ if [[ "${FLAGS_showoutput}" -eq "${FLAGS_TRUE}" && \ # Only parallel_emerge supports --show-output. EMERGE_FLAGS+=" --show-output" fi + +# Prepare tmp file to capture emerge output from tee. +tmpfile=$(mktemp -t tmp.build_packages-emerge.XXXXXX) +trap "rm -f '${tmpfile}'" EXIT + info "Merging board packages ${PACKAGES}" -sudo -E ${EMERGE_BOARD_CMD} -uDNv ${EMERGE_FLAGS} ${PACKAGES} +( + set -o pipefail + sudo -E ${EMERGE_BOARD_CMD} -uDNv ${EMERGE_FLAGS} ${PACKAGES} | \ + tee "${tmpfile}" +) + +# Extract total package count from emerge output. +package_count=$(awk '$0 ~ /^Total: [0-9]+ packages/ { print $2 }' "${tmpfile}") +rm "${tmpfile}" +trap - EXIT echo "Builds complete" -print_time_elapsed +EXTRA_COMMAND_STATS[package_count]=${package_count} +command_completed echo "Done" diff --git a/common.sh b/common.sh index 59c3b8d8cd..24ff0e50a0 100644 --- a/common.sh +++ b/common.sh @@ -557,6 +557,10 @@ safe_umount_tree() { fi } +get_git_id() { + git var GIT_COMMITTER_IDENT | sed -e 's/^.*<\(\S\+\)>.*$/\1/' +} + # Fixes symlinks that are incorrectly prefixed with the build root ${1} # rather than the real running root '/'. # TODO(sosa) - Merge setup - cleanup below with this method. @@ -664,13 +668,79 @@ enable_rw_mount() { # Get current timestamp. Assumes common.sh runs at startup. start_time=$(date +%s) -# Print time elsapsed since start_time. -print_time_elapsed() { +# Get time elapsed since start_time in seconds. +get_elapsed_seconds() { local end_time=$(date +%s) local elapsed_seconds=$(($end_time - $start_time)) + echo ${elapsed_seconds} +} + +# Print time elapsed since start_time. +print_time_elapsed() { + # Optional first arg to specify elapsed_seconds. If not given, will + # recalculate elapsed time to now. Optional second arg to specify + # command name associated with elapsed time. + local elapsed_seconds=${1:-$(get_elapsed_seconds)} + local cmd_base=$2 + local minutes=$(($elapsed_seconds / 60)) local seconds=$(($elapsed_seconds % 60)) - echo "Elapsed time: ${minutes}m${seconds}s" + + if [ -n "${cmd_base}" ]; then + info "Elapsed time (${cmd_base}): ${minutes}m${seconds}s" + else + info "Elapsed time: ${minutes}m${seconds}s" + fi +} + +# Associative array for filling in extra command-specific stats before +# calling command_completed. +declare -A EXTRA_COMMAND_STATS + +# Save original command line. +command_line_arr=( "$0" "$@" ) + +command_completed() { + # Call print_elapsed_time regardless. + local run_time=$(get_elapsed_seconds) + local cmd_base=$(basename "${command_line_arr[0]}") + print_time_elapsed ${run_time} ${cmd_base} + + # Prepare command stats in an associative array. Additonal command-specific + # stats can be added through EXTRA_COMMAND_STATS associative array. + declare -A stats + stats=( + [cmd_line]=${command_line_arr[*]} + [cmd_base]=${cmd_base} + [cmd_args]=${command_line_arr[*]:1} + [run_time]=${run_time} + [username]=$(get_git_id) + [board]=${FLAGS_board} + [host]=$(hostname -f) + [cpu_count]=$(grep -c processor /proc/cpuinfo) + [cpu_type]=$(uname -p) + ) + + local attr + for attr in "${!EXTRA_COMMAND_STATS[@]}"; do + stats[${attr}]=${EXTRA_COMMAND_STATS[${attr}]} + done + + # Prepare temporary file for stats. + local tmpfile=$(mktemp -t tmp.stats.XXXXXX) + trap "rm -f '${tmpfile}'" EXIT + + # Write stats out to temporary file. + echo "Chromium OS Build Command Stats - Version 1" > "${tmpfile}" + for attr in "${!stats[@]}"; do + echo "${attr} ${stats[${attr}]}" + done >> "${tmpfile}" + + # Call upload_command_stats on the stats file. If it fails do not stop. + "${GCLIENT_ROOT}"/chromite/bin/upload_command_stats "${tmpfile}" || true + + rm "${tmpfile}" + trap - EXIT } # The board and variant command line options can be used in a number of ways diff --git a/sdk_lib/enter_chroot.sh b/sdk_lib/enter_chroot.sh index a6c4a38978..b75dce1262 100755 --- a/sdk_lib/enter_chroot.sh +++ b/sdk_lib/enter_chroot.sh @@ -79,6 +79,13 @@ fi # TODO: replace shflags with something less error-prone, or contribute a fix. switch_to_strict_mode +# These config files are to be copied into chroot if they exist in home dir. +FILES_TO_COPY_TO_CHROOT=( + .gdata_cred.txt # User/password for Google Docs on chromium.org + .gdata_token # Auth token for Google Docs on chromium.org + .disable_build_stats_upload # Presence of file disables command stats upload +) + INNER_CHROME_ROOT=$FLAGS_chrome_root_mount # inside chroot CHROME_ROOT_CONFIG="/var/cache/chrome_root" # inside chroot INNER_DEPOT_TOOLS_ROOT="/home/$USER/depot_tools" # inside chroot @@ -397,9 +404,8 @@ setup_env() { git config -f ${FLAGS_chroot}/home/${USER}/.gitconfig --replace-all \ user.email "${ident_email}" || true - # Copy ~/.gdata_cred.txt and ~/.gdata_token to chroot if they exist. These - # files contain credentials for reading/writing Google Docs on chromium.org. - for fn in ".gdata_cred.txt" ".gdata_token"; do + # Certain files get copied into the chroot when entering. + for fn in "${FILES_TO_COPY_TO_CHROOT[@]}"; do copy_into_chroot_if_exists "${HOME}/${fn}" "/home/${USER}/${fn}" done