mirror of
https://github.com/flatcar/scripts.git
synced 2025-09-23 06:31:18 +02:00
Simple way to run client/server autotest(s) from server.
This is a script to run client or server autotests on a live Chromium OS instance, collect results, and optionally upload the to an autotest database. This includes functional and performance tests. We assume the remote instance is running an appropriate image installed (one created using mod_image_for_test.sh and possibly installed using image_to_live.sh). An example run might be run_remote_tests.sh --remote=192.168.1.5 BootPerfServer -o results.txt This example will run src/platform/testing/server_tests/system_BootPerfServer 5 times on instance at 192.168.1.5 and collect results in result.txt. Also refactors and improves readability in image_to_live.sh. Review URL: http://codereview.chromium.org/519041
This commit is contained in:
Ken Mixter
parent
8e0816ece0
commit
689b9ee48e
153
image_to_live.sh
153
image_to_live.sh
@ -4,39 +4,20 @@
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# Script to convert the output of build_image.sh to a usb image.
|
||||
# Script to update an image onto a live running ChromiumOS instance.
|
||||
|
||||
# Load common constants. This should be the first executable line.
|
||||
# The path to common.sh should be relative to your script's location.
|
||||
. "$(dirname "$0")/common.sh"
|
||||
|
||||
assert_outside_chroot
|
||||
. "$(dirname $0)/common.sh"
|
||||
. "$(dirname $0)/remote_access.sh"
|
||||
|
||||
cd $(dirname "$0")
|
||||
|
||||
DEFAULT_PRIVATE_KEY="$SRC_ROOT/platform/testing/testing_rsa"
|
||||
|
||||
DEFINE_string ip "" "IP address of running Chromium OS instance"
|
||||
DEFINE_boolean ignore_version $FLAGS_TRUE \
|
||||
DEFINE_boolean ignore_version ${FLAGS_TRUE} \
|
||||
"Ignore existing version on running instance and always update"
|
||||
DEFINE_boolean ignore_hostname $FLAGS_TRUE \
|
||||
DEFINE_boolean ignore_hostname ${FLAGS_TRUE} \
|
||||
"Ignore existing AU hostname on running instance use this hostname"
|
||||
DEFINE_string private_key "$DEFAULT_PRIVATE_KEY" \
|
||||
"Private key of root account on instance"
|
||||
|
||||
FLAGS "$@" || exit 1
|
||||
eval set -- "${FLAGS_ARGV}"
|
||||
|
||||
set -e
|
||||
|
||||
if [ -z "$FLAGS_ip" ]; then
|
||||
echo "Please specify the IP of the Chromium OS instance"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
TMP=$(mktemp -d /tmp/image_to_live.XXXX)
|
||||
TMP_PRIVATE_KEY=$TMP/private_key
|
||||
TMP_KNOWN_HOSTS=$TMP/known_hosts
|
||||
DEFINE_boolean update_known_hosts ${FLAGS_FALSE} \
|
||||
"Update your known_hosts with the new remote instance's key"
|
||||
|
||||
function kill_all_devservers {
|
||||
! pkill -f 'python devserver.py'
|
||||
@ -44,36 +25,14 @@ function kill_all_devservers {
|
||||
|
||||
function cleanup {
|
||||
kill_all_devservers
|
||||
rm -rf $TMP
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
function remote_sh {
|
||||
# Disable strict host checking so that we don't prompt the user when
|
||||
# the host key file is removed and just go ahead and add it.
|
||||
REMOTE_OUT=$(ssh -o StrictHostKeyChecking=no -o \
|
||||
UserKnownHostsFile=$TMP_KNOWN_HOSTS root@$FLAGS_ip "$@")
|
||||
return ${PIPESTATUS[0]}
|
||||
rm -rf "${TMP}"
|
||||
}
|
||||
|
||||
function remote_reboot_sh {
|
||||
rm -f $TMP_KNOWN_HOSTS
|
||||
rm -f "${TMP_KNOWN_HOSTS}"
|
||||
remote_sh "$@"
|
||||
}
|
||||
|
||||
function set_up_remote_access {
|
||||
if [ -z "$SSH_AGENT_PID" ]; then
|
||||
eval `ssh-agent`
|
||||
fi
|
||||
cp $FLAGS_private_key $TMP_PRIVATE_KEY
|
||||
chmod 0400 $TMP_PRIVATE_KEY
|
||||
ssh-add $TMP_PRIVATE_KEY
|
||||
|
||||
# Verify the client is reachable before continuing
|
||||
remote_sh "true"
|
||||
}
|
||||
|
||||
function start_dev_server {
|
||||
kill_all_devservers
|
||||
sudo -v
|
||||
@ -89,7 +48,7 @@ function start_dev_server {
|
||||
function prepare_update_metadata {
|
||||
remote_sh "mount -norw,remount /"
|
||||
|
||||
if [ $FLAGS_ignore_version -eq $FLAGS_TRUE ]; then
|
||||
if [[ ${FLAGS_ignore_version} -eq ${FLAGS_TRUE} ]]; then
|
||||
echo "Forcing update independent of the current version"
|
||||
remote_sh "cat /etc/lsb-release |\
|
||||
grep -v CHROMEOS_RELEASE_VERSION > /etc/lsb-release~;\
|
||||
@ -97,8 +56,8 @@ function prepare_update_metadata {
|
||||
echo 'CHROMEOS_RELEASE_VERSION=0.0.0.0' >> /etc/lsb-release"
|
||||
fi
|
||||
|
||||
if [ $FLAGS_ignore_hostname -eq $FLAGS_TRUE ]; then
|
||||
echo "Forcing update from $HOSTNAME"
|
||||
if [ ${FLAGS_ignore_hostname} -eq ${FLAGS_TRUE} ]; then
|
||||
echo "Forcing update from ${HOSTNAME}"
|
||||
remote_sh "cat /etc/lsb-release |\
|
||||
grep -v '^CHROMEOS_AUSERVER=' |\
|
||||
grep -v '^CHROMEOS_DEVSERVER=' > /etc/lsb-release~;\
|
||||
@ -112,12 +71,12 @@ function prepare_update_metadata {
|
||||
}
|
||||
|
||||
function run_auto_update {
|
||||
echo "Starting update and clear away prior"
|
||||
UPDATE_FILE=/var/log/softwareupdate.log
|
||||
echo "Starting update"
|
||||
local update_file=/var/log/softwareupdate.log
|
||||
# Clear it out so we don't see a prior run and make sure it
|
||||
# exists so the first tail below can't fail if it races the
|
||||
# memento updater first write and wins.
|
||||
remote_sh "rm -f $UPDATE_FILE; touch $UPDATE_FILE; \
|
||||
remote_sh "rm -f ${update_file}; touch ${update_file}; \
|
||||
/opt/google/memento_updater/memento_updater.sh</dev/null>&/dev/null&"
|
||||
|
||||
local update_error
|
||||
@ -125,50 +84,52 @@ function run_auto_update {
|
||||
local progress
|
||||
|
||||
update_error=1
|
||||
output_file=$TMP/output
|
||||
output_file="${TMP}/output"
|
||||
|
||||
while true; do
|
||||
# The softwareupdate.log gets pretty bit with download progress
|
||||
# lines so only look in the last 100 lines for status.
|
||||
remote_sh "tail -100 $UPDATE_FILE"
|
||||
echo "$REMOTE_OUT" > $output_file
|
||||
progress=$(tail -4 $output_file | grep 0K | head -1)
|
||||
if [ -n "$progress" ]; then
|
||||
echo "Image fetching progress: $progress"
|
||||
remote_sh "tail -100 ${update_file}"
|
||||
echo "${REMOTE_OUT}" > "${output_file}"
|
||||
progress=$(tail -4 "${output_file}" | grep 0K | head -1)
|
||||
if [ -n "${progress}" ]; then
|
||||
echo "Image fetching progress: ${progress}"
|
||||
fi
|
||||
if grep -q 'updatecheck status="noupdate"' $output_file; then
|
||||
if grep -q 'updatecheck status="noupdate"' "${output_file}"; then
|
||||
echo "devserver is claiming there is no update available."
|
||||
echo "Consider setting --ignore_version."
|
||||
break
|
||||
fi
|
||||
if grep -q 'Autoupdate applied. You should now reboot' $output_file; then
|
||||
if grep -q 'Autoupdate applied. You should now reboot' "${output_file}"
|
||||
then
|
||||
echo "Autoupdate was successful."
|
||||
update_error=0
|
||||
fi
|
||||
if grep -q 'Memento AutoUpdate terminating' $output_file; then
|
||||
if grep -q 'Memento AutoUpdate terminating' "${output_file}"; then
|
||||
break
|
||||
fi
|
||||
# Sleep for a while so that ssh handling doesn't slow down the install
|
||||
sleep 2
|
||||
done
|
||||
|
||||
return $update_error
|
||||
return ${update_error}
|
||||
}
|
||||
|
||||
function remote_reboot {
|
||||
echo "Rebooting."
|
||||
remote_sh "touch /tmp/awaiting_reboot; reboot"
|
||||
local output_file
|
||||
output_file=$TMP/output
|
||||
output_file="${TMP}/output"
|
||||
|
||||
while true; do
|
||||
REMOTE_OUT=""
|
||||
# This may fail while the machine is done so generate output and a
|
||||
# boolean result to distinguish between down/timeout and real failure
|
||||
! remote_reboot_sh "echo 0; [ -e /tmp/awaiting_reboot ] && echo '1'; true"
|
||||
echo "$REMOTE_OUT" > $output_file
|
||||
if grep -q "0" $output_file; then
|
||||
if grep -q "1" $output_file; then
|
||||
! remote_sh_allow_changed_host_key \
|
||||
"echo 0; [ -e /tmp/awaiting_reboot ] && echo '1'; true"
|
||||
echo "${REMOTE_OUT}" > "${output_file}"
|
||||
if grep -q "0" "${output_file}"; then
|
||||
if grep -q "1" "${output_file}"; then
|
||||
echo "Not yet rebooted"
|
||||
else
|
||||
echo "Rebooted and responding"
|
||||
@ -179,26 +140,54 @@ function remote_reboot {
|
||||
done
|
||||
}
|
||||
|
||||
set_up_remote_access
|
||||
function main() {
|
||||
assert_outside_chroot
|
||||
|
||||
if remote_sh [ -e /tmp/memento_autoupdate_completed ]; then
|
||||
cd $(dirname "$0")
|
||||
|
||||
FLAGS "$@" || exit 1
|
||||
eval set -- "${FLAGS_ARGV}"
|
||||
|
||||
set -e
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
TMP=$(mktemp -d /tmp/image_to_live.XXXX)
|
||||
|
||||
remote_access_init
|
||||
|
||||
if remote_sh [ -e /tmp/memento_autoupdate_completed ]; then
|
||||
echo "Machine has been updated but not yet rebooted. Rebooting it now."
|
||||
echo "Rerun this script if you still wish to update it."
|
||||
remote_reboot
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
start_dev_server
|
||||
start_dev_server
|
||||
|
||||
prepare_update_metadata
|
||||
prepare_update_metadata
|
||||
|
||||
if ! run_auto_update; then
|
||||
if ! run_auto_update; then
|
||||
echo "Update was not successful."
|
||||
exit
|
||||
fi
|
||||
fi
|
||||
|
||||
remote_reboot
|
||||
remote_reboot
|
||||
|
||||
remote_sh "grep ^CHROMEOS_RELEASE_DESCRIPTION= /etc/lsb-release"
|
||||
RELEASE_DESCRIPTION=$(echo $REMOTE_OUT | cut -d '=' -f 2)
|
||||
echo "Update was successful and rebooted to $RELEASE_DESCRIPTION"
|
||||
if [[ ${FLAGS_update_hostkey} -eq ${FLAGS_TRUE} ]]; then
|
||||
local known_hosts="${HOME}/.ssh/known_hosts"
|
||||
cp "${known_hosts}" "${known_hosts}~"
|
||||
grep -v "^${FLAGS_remote} " "${known_hosts}" > "${TMP}/new_known_hosts"
|
||||
cat "${TMP}/new_known_hosts" "${TMP_KNOWN_HOSTS}" > "${known_hosts}"
|
||||
chmod 0640 "${known_hosts}"
|
||||
echo "New updated in ${known_hosts}, backup made."
|
||||
fi
|
||||
|
||||
remote_sh "grep ^CHROMEOS_RELEASE_DESCRIPTION= /etc/lsb-release"
|
||||
local release_description=$(echo $REMOTE_OUT | cut -d '=' -f 2)
|
||||
echo "Update was successful and rebooted to $release_description"
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
main $@
|
||||
|
||||
48
remote_access.sh
Normal file
48
remote_access.sh
Normal file
@ -0,0 +1,48 @@
|
||||
# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# Library for setting up remote access and running remote commands.
|
||||
|
||||
DEFAULT_PRIVATE_KEY="$SRC_ROOT/platform/testing/testing_rsa"
|
||||
|
||||
DEFINE_string remote "" "remote hostname/IP of running Chromium OS instance"
|
||||
DEFINE_string private_key "$DEFAULT_PRIVATE_KEY" \
|
||||
"Private key of root account on remote host"
|
||||
|
||||
function remote_sh() {
|
||||
# Disable strict host checking so that we don't prompt the user when
|
||||
# the host key file is removed and just go ahead and add it.
|
||||
REMOTE_OUT=$(ssh -o StrictHostKeyChecking=no -o \
|
||||
UserKnownHostsFile=$TMP_KNOWN_HOSTS root@$FLAGS_remote "$@")
|
||||
return ${PIPESTATUS[0]}
|
||||
}
|
||||
|
||||
function remote_sh_allow_changed_host_key() {
|
||||
rm -f $TMP_KNOWN_HOSTS
|
||||
remote_sh "$@"
|
||||
}
|
||||
|
||||
function set_up_remote_access() {
|
||||
if [ -z "$SSH_AGENT_PID" ]; then
|
||||
eval `ssh-agent`
|
||||
fi
|
||||
cp $FLAGS_private_key $TMP_PRIVATE_KEY
|
||||
chmod 0400 $TMP_PRIVATE_KEY
|
||||
ssh-add $TMP_PRIVATE_KEY
|
||||
|
||||
# Verify the client is reachable before continuing
|
||||
echo "Initiating first contact with remote host"
|
||||
remote_sh "true"
|
||||
echo "Connection OK"
|
||||
}
|
||||
|
||||
function remote_access_init() {
|
||||
TMP_PRIVATE_KEY=$TMP/private_key
|
||||
TMP_KNOWN_HOSTS=$TMP/known_hosts
|
||||
if [ -z "$FLAGS_remote" ]; then
|
||||
echo "Please specify --remote=<IP-or-hostname> of the Chromium OS instance"
|
||||
exit 1
|
||||
fi
|
||||
set_up_remote_access
|
||||
}
|
||||
178
run_remote_tests.sh
Executable file
178
run_remote_tests.sh
Executable file
@ -0,0 +1,178 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2009 The Chromium OS Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
# Script to run client or server tests on a live remote image.
|
||||
|
||||
# Load common constants. This should be the first executable line.
|
||||
# The path to common.sh should be relative to your script's location.
|
||||
|
||||
. "$(dirname $0)/common.sh"
|
||||
. "$(dirname $0)/remote_access.sh"
|
||||
|
||||
DEFAULT_OUTPUT_FILE=test-output-$(date '+%Y%m%d.%H%M%S')
|
||||
|
||||
DEFINE_boolean cleanup ${FLAGS_TRUE} "Clean up temp directory"
|
||||
DEFINE_integer iterations 1 "Iterations to run every top level test" i
|
||||
DEFINE_string output_file "${DEFAULT_OUTPUT_FILE}" "Test run output" o
|
||||
DEFINE_boolean verbose ${FLAGS_FALSE} "Show verbose autoserv output" v
|
||||
DEFINE_boolean update_db ${FLAGS_FALSE} "Put results in autotest database" u
|
||||
DEFINE_string machine_desc "" "Machine description used in database"
|
||||
|
||||
function cleanup() {
|
||||
if [[ $FLAGS_cleanup -eq ${FLAGS_TRUE} ]]; then
|
||||
rm -rf "${TMP}"
|
||||
else
|
||||
echo "Left temporary files at ${TMP}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Returns an error if the test_result_file has text which indicates
|
||||
# the test was not run successfully.
|
||||
# Arguments:
|
||||
# $1 - file name of autotest status for to check for success
|
||||
# Returns:
|
||||
# None
|
||||
function is_successful_test() {
|
||||
local file="$1"
|
||||
# To be successful, must not have FAIL or BAD in the file.
|
||||
if egrep -q "(BAD|FAIL)" "${file}"; then
|
||||
return 1
|
||||
fi
|
||||
# To be successful, must have GOOD in the file.
|
||||
if ! grep -q GOOD "${file}"; then
|
||||
return 1
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
# Removes single quotes around parameter
|
||||
# Arguments:
|
||||
# $1 - string which optionally has surrounding quotes
|
||||
# Returns:
|
||||
# None, but prints the string without quotes.
|
||||
function remove_quotes() {
|
||||
echo "$1" | sed -e "s/^'//; s/'$//"
|
||||
}
|
||||
|
||||
function main() {
|
||||
assert_outside_chroot
|
||||
|
||||
cd $(dirname "$0")
|
||||
|
||||
FLAGS "$@" || exit 1
|
||||
|
||||
if [[ -z "${FLAGS_ARGV}" ]]; then
|
||||
echo "Please specify tests to run, like:"
|
||||
echo " $0 --remote=MyMachine SystemBootPerf"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local parse_cmd="$(dirname $0)/../third_party/autotest/files/tko/parse.py"
|
||||
|
||||
if [[ ${FLAGS_update_db} -eq ${FLAGS_TRUE} && ! -x "${parse_cmd}" ]]; then
|
||||
echo "Cannot find parser ${parse_cmd}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set -e
|
||||
|
||||
# Set global TMP for remote_access.sh's sake
|
||||
TMP=$(mktemp -d /tmp/run_remote_tests.XXXX)
|
||||
|
||||
rm -f "${FLAGS_output_file}"
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
cp -r "${SRC_ROOT}/third_party/autotest/files" "${TMP}/autotest"
|
||||
|
||||
local control_files_to_run=""
|
||||
|
||||
# Now search for tests which unambiguously include the given identifier
|
||||
local search_path=$(echo ${TMP}/autotest/{client,server}/{tests,site_tests})
|
||||
for test_request in $FLAGS_ARGV; do
|
||||
test_request=$(remove_quotes "${test_request}")
|
||||
! finds=$(find ${search_path} -type f -name control | \
|
||||
egrep "${test_request}")
|
||||
if [[ -z "${finds}" ]]; then
|
||||
echo "Can not find match for ${test_request}"
|
||||
exit 1
|
||||
fi
|
||||
local matches=$(echo "${finds}" | wc -l)
|
||||
if [[ ${matches} -gt 1 ]]; then
|
||||
echo "${test_request} is ambiguous:"
|
||||
echo "${finds}"
|
||||
exit 1
|
||||
fi
|
||||
for i in $(seq 1 $FLAGS_iterations); do
|
||||
control_files_to_run="${control_files_to_run} '${finds}'"
|
||||
done
|
||||
done
|
||||
|
||||
echo "Running the following control files: ${control_files_to_run}"
|
||||
|
||||
remote_access_init
|
||||
|
||||
# Set the default machine description to the machine's IP
|
||||
if [[ -z "${FLAGS_machine_desc}" ]]; then
|
||||
FLAGS_machine_desc="${FLAGS_remote}"
|
||||
fi
|
||||
|
||||
local autoserv="${TMP}/autotest/server/autoserv"
|
||||
|
||||
for control_file in ${control_files_to_run}; do
|
||||
# Assume a line starts with TEST_TYPE =
|
||||
control_file=$(remove_quotes "${control_file}")
|
||||
local type=$(egrep '^\s*TEST_TYPE\s*=' "${control_file}" | head -1)
|
||||
type=$(python -c "${type}; print TEST_TYPE.lower()")
|
||||
local option
|
||||
if [ "${type}" == "client" ]; then
|
||||
option="-c"
|
||||
elif [ "${type}" == "server" ]; then
|
||||
option="-s"
|
||||
else
|
||||
echo "Unknown type of test (${type}) in ${control_file}"
|
||||
exit 1
|
||||
fi
|
||||
echo "Running ${type} test ${control_file}"
|
||||
local short_name=$(basename $(dirname "${control_file}"))
|
||||
local timestamp=$(date '+%s')
|
||||
local results_dir="${TMP}/${short_name},${FLAGS_machine_desc},${timestamp}"
|
||||
rm -rf "${results_dir}"
|
||||
local verbose=""
|
||||
if [[ ${FLAGS_verbose} -eq $FLAGS_TRUE ]]; then
|
||||
verbose="--verbose"
|
||||
fi
|
||||
${autoserv} -m "${FLAGS_remote}" "${option}" "${control_file}" \
|
||||
-r "${results_dir}" ${verbose}
|
||||
local test_status="${results_dir}/status"
|
||||
local test_result_dir="${results_dir}/${short_name}"
|
||||
local keyval_file="${test_result_dir}/results/keyval"
|
||||
if is_successful_test "${test_status}"; then
|
||||
echo "${control_file} succeeded." | tee -a "${FLAGS_output_file}"
|
||||
if [[ -f "${keyval_file}" ]]; then
|
||||
echo "Keyval was:" | tee -a "${FLAGS_output_file}"
|
||||
cat "${keyval_file}" | tee -a "${FLAGS_output_file}"
|
||||
fi
|
||||
else
|
||||
echo "${control_file} failed:" | tee -a "${FLAGS_output_file}"
|
||||
cat "${test_status}" | tee -a "${FLAGS_output_file}"
|
||||
# Leave around output directory if the test failed.
|
||||
FLAGS_cleanup=${FLAGS_FALSE}
|
||||
fi
|
||||
|
||||
# Update the database with results.
|
||||
if [[ ${FLAGS_update_db} -eq ${FLAGS_TRUE} ]]; then
|
||||
if ! "${parse_cmd}" -o "${results_dir}"; then
|
||||
echo "Parse failed." | tee -a "${FLAGS_output_file}"
|
||||
FLAGS_cleanup=${FLAGS_FALSE}
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Output stored to ${FLAGS_output_file}"
|
||||
}
|
||||
|
||||
main $@
|
||||
Loading…
x
Reference in New Issue
Block a user