From 11d783b95c10ea6e6c06b59d4ab09e397815cd49 Mon Sep 17 00:00:00 2001 From: Michael Marineau Date: Thu, 15 May 2014 15:47:13 -0700 Subject: [PATCH] remove(symbols): Kill off scripts for managing ChromeOS debug symbols --- cros_generate_breakpad_symbols | 290 --------------------------------- cros_generate_stacks_bvt | 78 --------- upload_symbols | 240 --------------------------- 3 files changed, 608 deletions(-) delete mode 100755 cros_generate_breakpad_symbols delete mode 100755 cros_generate_stacks_bvt delete mode 100755 upload_symbols diff --git a/cros_generate_breakpad_symbols b/cros_generate_breakpad_symbols deleted file mode 100755 index 444adff014..0000000000 --- a/cros_generate_breakpad_symbols +++ /dev/null @@ -1,290 +0,0 @@ -#!/bin/bash -# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Script to generate minidump symbols in the format required by -# minidump_stackwalk to dump stack information. -# -# NOTE: This script must be run from the chromeos build chroot environment. -# - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -# Script must be run inside the chroot -restart_in_chroot_if_needed "$@" - -# Flags -DEFINE_string board "$DEFAULT_BOARD" "The board to build packages for." -DEFINE_string minidump_symbol_root "" \ - "Symbol root (defaults to /usr/lib/debug/breakpad for board)" -DEFINE_boolean verbose ${FLAGS_FALSE} "Be verbose." - -DUMP_SYMS="dump_syms" -DUMP_SYMS32="dump_syms.32" - -ERROR_COUNT=0 - -debug() { - if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then - info "$@" - fi -} - -# Each job sets this on their own; we declare it -# globally so the exit trap can always see the last -# setting of it w/in a job worker. -SYM_FILE= -JOB_FILE= -NOTIFIED= - -# The master process sets these up, which each worker -# than uses for communicating once they've finished. -CONTROL_PIPE= -CONTROL_PIPE_FD= - -_worker_finished() { - if [ -z "${NOTIFIED}" ]; then - debug "Sending notification of $BASHPID ${1-1}" - echo "$BASHPID ${1-1}" > /dev/fd/${CONTROL_PIPE_FD} - NOTIFIED=1 - fi -} - -_cleanup_worker() { - rm -f "${SYM_FILE}" "${ERR_FILE}" - _worker_finished 1 -} - -_cleanup_master() { - set +eu - rm -f "${CONTROL_PIPE}" - if [ ${#JOBS_ARRAY[@]} != 0 ]; then - kill -s SIGINT "${!JOBS_ARRAY[@]}" &> /dev/null - wait - # Clear the array. - JOBS_ARRAY=( ) - fi -} - -declare -A JOBS_ARRAY - -finish_job() { - local finished result - read -r -u ${CONTROL_PIPE_FD} finished result - # Bash doesn't allow for zombies, but tell it to clean up its intenral - # bookkeeping. Note bash can be buggy here- if a new process has slipped - # into that pid, bash doesn't use its internal accounting first, and - # can throw an error; doesn't matter, thus this form. - ! wait ${finished} &> /dev/null - if [ "${result-1}" -ne "0" ]; then - : $(( ++ERROR_COUNT )) - fi - # Bit of a hack, but it works well enough. - debug "finished ${finished} with result ${result-1}" - unset JOBS_ARRAY[${finished}] -} - -run_job() { - local debug_file=${1} text_file=${2} newpid - - if [ ${#JOBS_ARRAY[@]} -ge ${NUM_JOBS} ]; then - # Reclaim a spot. - finish_job - fi - - dump_file "${debug_file}" "${text_file}" & - newpid=$! - debug "Started ${debug_file} ${text_file} at ${newpid}" - JOBS_ARRAY[$newpid]=1 -} - -# Given path to a debug file, return its text file -get_text_for_debug() { - local debug_file=$1 - local text_dir=$(dirname "${debug_file#$DEBUG_ROOT}") - local text_path=${SYSROOT}${text_dir}/$(basename "${debug_file}" .debug) - echo ${text_path} -} - -# Given path to a text file, return its debug file -get_debug_for_text() { - local text_file=$1 - local text_path=${text_file#${SYSROOT}} - local debug_path=${DEBUG_ROOT}${text_path}.debug - echo ${debug_path} -} - -# Returns true if the file given is a 32-bit ELF file. -is_32b_elf() { - local elf="$1" - sudo file "${elf}" | grep -q "ELF 32-bit" -} - -# Dump given debug and text file. Returns 1 if any errors, even -# if they can be ignored, but only sets ERROR_COUNT if the error should not -# be ignored (and we should not proceed to upload). -dump_file() { - trap '_cleanup_worker; exit 1' INT TERM - trap _cleanup_worker EXIT - local debug_file="$1" - local text_file="$2" - local debug_directory="$(dirname "${debug_file}")" - local dump_syms_prog="${DUMP_SYMS}" - # 32-bit dump_syms must be used to dump a 32-bit ELF file - if is_32b_elf "${text_file}"; then - dump_syms_prog="${DUMP_SYMS32}" - debug "Using ${dump_syms_prog} for 32-bit file ${text_file}" - fi - SYM_FILE=$(mktemp -t "breakpad.sym.XXXXXX") - # Dump symbols as root in order to read all files. - if ! sudo "${dump_syms_prog}" "${text_file}" "${debug_directory}" \ - > "${SYM_FILE}" 2> /dev/null; then - # Try dumping just the main file to get public-only symbols. - ERR_FILE=$(mktemp -t "breakpad.err.XXXXXX") - if ! sudo "${dump_syms_prog}" "${text_file}" > "${SYM_FILE}" \ - 2> "${ERR_FILE}"; then - # A lot of files (like kernel files) contain no debug information, do - # not consider such occurrences as errors. - if grep -q "file contains no debugging information" "${ERR_FILE}"; then - warn "No symbols found for ${text_file}" - _worker_finished 0 - exit 0 - fi - error "Unable to dump symbols for ${text_file}:" - error "$(<"${ERR_FILE}")" - exit 1 - else - warn "File ${text_file} did not have debug info, using linkage symbols" - fi - fi - local file_id=$(head -1 ${SYM_FILE} | cut -d' ' -f4) - local module_name=$(head -1 ${SYM_FILE} | cut -d' ' -f5) - # Show file upload success and symbol info for easier lookup - debug "Dumped symbols from ${text_file} for ${module_name}|${file_id}." - # Sanity check: if we've created the same named file in the /usr/lib/debug - # directory during the src_compile stage of an ebuild, verify our sym file - # is the same. - local installed_sym="${DEBUG_ROOT}"/$(basename "${text_file}").sym - if [ -e "${installed_sym}" ]; then - if ! cmp --quiet "${installed_sym}" "${SYM_FILE}"; then - error "${installed_sym} differ from current sym file:" - error "$(diff "${installed_sym}" "${SYM_FILE}")" - : $(( ++ERROR_COUNT )) - exit 1 - fi - fi - - local container_dir="${FLAGS_minidump_symbol_root}/${module_name}/${file_id}" - sudo mkdir -p "${container_dir}" - sudo mv "${SYM_FILE}" "${container_dir}/${module_name}.sym" - _worker_finished 0 - exit 0 -} - -# Convert the given debug file. No return value. -process_file() { - local debug_file="$1" - local text_file="$(get_text_for_debug ${debug_file})" - if [ -h "${debug_file}" ]; then - # Don't follow symbolic links. In particular, we don't want to bother - # with the *.debug links in the "debug/.build-id/" directory. - debug "Skipping symbolic link: ${debug_file}" - return 0 - fi - if [ "${text_file##*.}" == "ko" ]; then - # Skip kernel objects. We can't use their symbols and they sometimes - # have objects with empty text sections which trigger errors in dump_sym. - debug "Skipping kernel object: ${text_file}" - return 0 - fi - if [ ! -f "${text_file}" ]; then - # Allow files to not exist, for instance if they are in the INSTALL_MASK. - warn "Binary does not exist: ${text_file}" - return 0 - fi - - run_job "${debug_file}" "${text_file}" -} - -main() { - - # Parse command line - FLAGS_HELP="usage: $0 [flags] []" - FLAGS "$@" || exit 1 - eval set -- "${FLAGS_ARGV}" - - switch_to_strict_mode - - [ -n "$FLAGS_board" ] || die_notrace "--board is required." - - SYSROOT="/build/${FLAGS_board}" - - if [[ -z "${FLAGS_minidump_symbol_root}" ]]; then - FLAGS_minidump_symbol_root="${SYSROOT}/usr/lib/debug/breakpad" - fi - - info "Writing minidump symbols to ${FLAGS_minidump_symbol_root}" - - DEBUG_ROOT="${SYSROOT}/usr/lib/debug" - sudo rm -rf "${FLAGS_minidump_symbol_root}" - - # Open our control pipe. - trap '_cleanup_master; exit 1' INT TERM - trap _cleanup_master EXIT - CONTROL_PIPE=$(mktemp -t "breakpad.fifo.XXXXXX") - rm "${CONTROL_PIPE}" - mkfifo "${CONTROL_PIPE}" - exec {CONTROL_PIPE_FD}<>${CONTROL_PIPE} - - # We require our stderr (which error/info/warn go through) to be a - # pipe for atomic write reasons; thus if it isn't, abuse cat to make it - # so. - if [ ! -p /dev/stderr ]; then - debug "Replacing stderr with a cat process for pipe requirements..." - exec 2> >(cat 1>&2) - fi - - if [ -z "${FLAGS_ARGV}" ]; then - # Sort on size; we want to start the largest first since it's likely going - # to be the chrome binary (which can take 98% of the runtime when we're - # running with parallelization for 6 or higher). - for debug_file in $(find "${DEBUG_ROOT}" -name \*.debug \ - -type f -exec stat -c '%s %n' {} + | sort -gr | cut -d' ' -f2-); do - process_file "${debug_file}" - done - else - for either_file in ${FLAGS_ARGV}; do - either_file=${either_file#\'} - either_file=${either_file%\'} - if [ ! -h "${either_file}" -a ! -f "${either_file}" ]; then - error "Specified file ${either_file} does not exist" - : $(( ++ERROR_COUNT )) - continue - fi - if [ "${either_file##*.}" == "debug" ]; then - debug_file="${either_file}" - else - debug_file="$(get_debug_for_text ${either_file})" - fi - process_file "${debug_file}" - done - fi - - while [[ ${#JOBS_ARRAY[@]} != 0 ]]; do - finish_job - done - - local size=$(sudo find "${FLAGS_minidump_symbol_root}" \ - -type f -name '*.sym' -exec du -b {} + | \ - awk '{t += $1} END {print t}') - info "Generated ${size:-0}B of unique debug information" - - if [[ ${ERROR_COUNT} == 0 ]]; then - return 0 - fi - die_notrace "Encountered ${ERROR_COUNT} problems" -} - -main "$@" diff --git a/cros_generate_stacks_bvt b/cros_generate_stacks_bvt deleted file mode 100755 index 5e6f112ab8..0000000000 --- a/cros_generate_stacks_bvt +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Script to generate stackdumps from BVT failures. - -# This can only run inside the chroot since we need minidump_stackwalk. -. "$(dirname $0)/common.sh" || exit 1 -assert_inside_chroot "$@" - -usage() { - echo "Usage: $0 url_or_path_to_debug_tgz url_to_bvt_test_results" -} - -if [ -z "$1" ] ; then - usage - die_notrace "The URL or path to symbols tarball (debug.tgz) is required" -fi - -if [ -z "$2" ] ; then - usage - die_notrace "The URL to BVT test results is required" -fi - -# Die on any errors. -switch_to_strict_mode - -BREAKPAD_DIR="debug/breakpad" -STACKS_GENERATED="" -OUTPUT_DIR="$(mktemp -d)" - -extract_tarball() { - info "Extracting breakpad symbols from $1..." - tar zxf "$1" -C "${OUTPUT_DIR}" "${BREAKPAD_DIR}" -} - -generate_stacktrace() { - echo "$1.txt" - minidump_stackwalk "$1" "${OUTPUT_DIR}/${BREAKPAD_DIR}" \ - >"$1.txt" 2>/dev/null -} - -find_and_generate_stacktraces() { - find "${OUTPUT_DIR}" -name "*.dmp" | - while read filename ; do - generate_stacktrace "${filename}" - done -} - -cleanup() { - if [ -n "${OUTPUT_DIR}" -a -z "${STACKS_GENERATED}" ] ; then - rm -rf "${OUTPUT_DIR}" - fi -} - -trap cleanup INT TERM EXIT - -info "Downloading minidumps from $2..." -wget -q -nv -r -l 10 -np -A "*.dmp" -P "${OUTPUT_DIR}" $2 -if [[ -z "$(find "${OUTPUT_DIR}" -name "*.dmp")" ]] ; then - die "No minidumps found" -fi - -if [[ -f "$1" ]] ; then - extract_tarball "$1" -else - info "Downloading symbols tarball from $1..." - wget -P "${OUTPUT_DIR}" "$1" - TARBALL="${OUTPUT_DIR}/$(basename $1)" - extract_tarball "$TARBALL" - rm -f "$TARBALL" -fi - -info "Generating stack traces..." -STACKS_GENERATED=$(find_and_generate_stacktraces) -echo $STACKS_GENERATED diff --git a/upload_symbols b/upload_symbols deleted file mode 100755 index 3c9035239c..0000000000 --- a/upload_symbols +++ /dev/null @@ -1,240 +0,0 @@ -#!/bin/bash -# Copyright (c) 2011 The Chromium OS Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# Script to upload all debug symbols required for crash reporting -# purposes. This script need only be used to upload release builds -# symbols or to debug crashes on non-release builds (in which case try -# to only upload the symbols for those executables involved). - -SCRIPT_ROOT=$(dirname $(readlink -f "$0")) -. "${SCRIPT_ROOT}/common.sh" || exit 1 - -# Script must be run inside the chroot if not in "testing" mode. -if [[ "$1" != "--testing" ]]; then - restart_in_chroot_if_needed "$@" -fi - -# Flags -DEFINE_string board "$DEFAULT_BOARD" "The board to build packages for." -DEFINE_string breakpad_root "" "Root directory for breakpad symbols." -DEFINE_boolean official_build ${FLAGS_FALSE} "Point to official symbol server." -DEFINE_boolean regenerate ${FLAGS_FALSE} "Regenerate all symbols." -# Default of 290M is relative to current 300M limit the Crash server enforces. -DEFINE_integer strip_cfi 290000000 "Strip CFI data for files above this size." -DEFINE_boolean testing ${FLAGS_FALSE} \ - "Run in testing mode (should be first argument)." -DEFINE_boolean verbose ${FLAGS_FALSE} "Be verbose." -DEFINE_boolean yes ${FLAGS_FALSE} "Answer yes to all prompts." - -# Number of seconds to wait before retrying an upload. The delay will double -# for each subsequent retry of the same symbol file. -INITIAL_RETRY_DELAY=1 -# Allow up to 7 attempts to upload a symbol file (total delay may be -# 1+2+4+8+16+32=63 seconds). -MAX_RETRIES=6 -# Number of total errors, ${TOTAL_ERROR_COUNT}, before retries are no longer -# attempted. This is used to avoid lots of errors causing unreasonable delays. -MAX_TOTAL_ERRORS_FOR_RETRY=3 # don't bother retrying after 3 errors - -# Testing parameters. These are only relevant if the "--testing" command-line -# option is passed. -# Specifies how many attempts should pretend to give an error before -# succeeding. NOTE: If this number is greater than ${TEST_MAX_RETRIES}, then -# it will never succeed. -TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS=3 -# Overrides ${MAX_RETRIES} in "testing" mode. -TEST_MAX_RETRIES=2 -# Overrides ${MAX_TOTAL_ERRORS_FOR_RETRY} in "testing" mode. -TEST_MAX_TOTAL_ERRORS_FOR_RETRY=2 - -SYM_UPLOAD="sym_upload" - -TOTAL_ERROR_COUNT=0 - -OUT_DIR=$(mktemp -d "/tmp/err.XXXX") - -cleanup() { - rm -rf "${OUT_DIR}" -} - -really_upload() { - if [ ${FLAGS_yes} -eq ${FLAGS_TRUE} ]; then - return 0 - fi - echo "Uploading symbols for an entire Chromium OS build is really only " - echo "necessary for release builds and in a few cases for developers " - echo "to debug problems. It will take considerable time to run. For " - echo "developer debugging purposes, consider instead passing specific files " - echo "to upload." - read -p "Are you sure you want to upload all build symbols (y/N)? " SURE - SURE="${SURE:0:1}" # Get just the first character - if [ "${SURE}" != "y" ]; then - echo "Ok, better safe than sorry." - return 1 - fi - return 0 -} - -# Upload the given symbol file to given URL. -upload_file() { - local symbol_file="$1" - local upload_url="$2" - local upload_file="${symbol_file}" - # If the symbols size is too big, strip out the call frame info. The CFI is - # unnecessary for 32b x86 targets where the frame pointer is used (as all of - # ours have) and it accounts for over half the size of the symbols uploaded. - if [ ${FLAGS_strip_cfi} -ne 0 ]; then - local symbol_size="$(stat -c%s ${symbol_file})" - if [ ${symbol_size} -gt ${FLAGS_strip_cfi} ]; then - if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then - warn "Stripping CFI for ${symbol_file} due to size ${symbol_size} > \ -${FLAGS_strip_cfi}." - fi - upload_file="${OUT_DIR}/stripped.sym" - sed '/^STACK CFI/d' < "${symbol_file}" > "${upload_file}" - fi - fi - if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then - info "Uploading ${symbol_file}" - fi - local upload_size="$(stat -c%s ${upload_file})" - if [ ${upload_size} -gt ${FLAGS_strip_cfi} ]; then - # Emit an annotation in order to flag the current step in buildbots. - # NOTE: Must be on a line by itself. - echo "@@@STEP_WARNINGS@@@" >&2 - error "Upload file ${upload_file} is awfully large, risking rejection by \ -symbol server (${upload_size} > ${FLAGS_strip_cfi})" - let ++TOTAL_ERROR_COUNT - fi - - # Upload the symbol file, allowing for ${MAX_RETRIES} number of retries - # before giving an error. However, don't retry if the total errors have - # reached ${MAX_TOTAL_ERRORS_FOR_RETRY}. - local success=0 - local attempts=0 - local retry_delay=${INITIAL_RETRY_DELAY} - while [ ${attempts} -le ${MAX_RETRIES} ]; do - if [ ${attempts} -gt 0 ]; then - if [ ${TOTAL_ERROR_COUNT} -ge ${MAX_TOTAL_ERRORS_FOR_RETRY} ]; then - warn "Not retrying to upload symbols in ${symbol_file} \ -due to too many total errors" - break - fi - warn "Retry #${attempts} to upload symbols in ${symbol_file} \ -(sleeping ${retry_delay} seconds)" - sleep "${retry_delay}" - let retry_delay=retry_delay*2 - fi - # In testing mode show command that would be run. - if [ ${FLAGS_testing} -eq ${FLAGS_TRUE} ]; then - echo "TEST: ${SYM_UPLOAD}" "${upload_file}" "${upload_url}" - fi - # Run the sym_upload command, redirecting its output to files so we can - # check them. - { - if [ ${FLAGS_testing} -eq ${FLAGS_FALSE} ]; then - "${SYM_UPLOAD}" "${upload_file}" "${upload_url}" - elif [ ${attempts} -lt ${TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS} ]; then - # Pretend to fail with an error message. - (echo "INFO: Testing info message"; - echo "ERROR: Testing error message" >&2; - exit 1) - else - # Pretend to succeed. - (echo "Successfully sent the symbol file.") - fi - } > "${OUT_DIR}/stdout" 2> "${OUT_DIR}/stderr" - # Check if sym_upload command succeeded. - if grep -q "Successfully sent the symbol file." "${OUT_DIR}/stdout"; then - success=1 - break - fi - let ++attempts - done - if [ ${success} -ne 1 ]; then - error "Unable to upload symbols in ${symbol_file}:" - cat "${OUT_DIR}/stderr" >&2 - let ++TOTAL_ERROR_COUNT - return 0 - fi - - if [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ]; then - size=$(wc -c "${upload_file}" | cut -d' ' -f1) - info "Successfully uploaded ${size}B." - fi - return 0 -} - -main() { - trap cleanup EXIT - - # Parse command line - FLAGS_HELP="usage: $0 [flags] []" - FLAGS "$@" || exit 1 - eval set -- "${FLAGS_ARGV}" - - if [ ${FLAGS_testing} -eq ${FLAGS_TRUE} ]; then - info "Running in testing mode:" - info " MAX_RETRIES=${TEST_ERRORS_FOR_THIS_MANY_ATTEMPTS}" - MAX_RETRIES=${TEST_MAX_RETRIES} - info " MAX_TOTAL_ERRORS_FOR_RETRY=${TEST_MAX_TOTAL_ERRORS_FOR_RETRY}" - MAX_TOTAL_ERRORS_FOR_RETRY=${TEST_MAX_TOTAL_ERRORS_FOR_RETRY} - fi - - [ -n "$FLAGS_board" ] || die_notrace "--board is required." - - SYSROOT="/build/${FLAGS_board}" - - local upload_url="" - if [ $FLAGS_official_build -eq $FLAGS_TRUE ]; then - upload_url="http://clients2.google.com/cr/symbol" - else - upload_url="http://clients2.google.com/cr/staging_symbol" - warn "This is an unofficial build, uploading to staging server." - fi - info "Uploading symbols to ${upload_url} from ${SYSROOT}." - - DEFAULT_BREAKPAD_ROOT="${SYSROOT}/usr/lib/debug/breakpad" - if [ -z "${FLAGS_breakpad_root}" ]; then - FLAGS_breakpad_root="${DEFAULT_BREAKPAD_ROOT}" - else - if [ ${FLAGS_regenerate} -eq ${FLAGS_TRUE} ]; then - warn "Assuming --noregenerate when --breakpad_root is specified" - FLAGS_regenerate=${FLAGS_FALSE} - fi - fi - - if [ -z "${FLAGS_ARGV}" ]; then - if [ ${FLAGS_regenerate} -eq ${FLAGS_TRUE} ]; then - really_upload || exit 1 - info "Clearing ${DEFAULT_BREAKPAD_ROOT}" - sudo rm -rf "${DEFAULT_BREAKPAD_ROOT}" - info "Generating all breakpad symbol files." - local verbosity="" - local generate_script="${SCRIPTS_DIR}/cros_generate_breakpad_symbols" - [ ${FLAGS_verbose} -eq ${FLAGS_TRUE} ] && verbosity="--verbose" - if ! "${generate_script}" --board=${FLAGS_board} ${verbosity}; then - error "Some errors while generating symbols; uploading anyway" - let ++TOTAL_ERROR_COUNT - fi - fi - - info "Uploading all breakpad symbol files." - for sym_file in $(find "${FLAGS_breakpad_root}" -name \*.sym); do - # sleep for 200ms to avoid DoS'ing symbol server (crosbug.com/26596) - sleep .2 - upload_file "${sym_file}" "${upload_url}" - done - else - error "Unexpected args ${FLAGS_ARGV}" - fi - - if [ ${TOTAL_ERROR_COUNT} -ne 0 ]; then - die "Encountered ${TOTAL_ERROR_COUNT} problem(s)" - fi - return 0 -} - -main "$@"