Add ability to vm library to retry until we can ssh into it.

Change-Id: I4de388bb1af6cd4568cdc154971927941eb73800

BUG=7503
TEST=Ran test by closing vm's as they start up to see retry work.

Review URL: http://codereview.chromium.org/3598019

bin/cros_run_vm_test

@@ -10,6 +10,8 @@
 . "$(dirname $0)/../lib/cros_vm_lib.sh"
 . "$(dirname "$0")/../lib/cros_vm_constants.sh"
 
+MAX_RETRIES=3
+
 DEFINE_string image_path "" "Full path of the VM image"
 DEFINE_string test_case "" "Name of the test case to run"
 
@@ -32,6 +34,8 @@ fi
 
 trap stop_kvm EXIT
 start_kvm "${FLAGS_image_path}"
+info "Checking for ssh access to virtual machine."
+retry_until_ssh ${MAX_RETRIES}
 "$(dirname $0)"/../run_remote_tests.sh \
   --ssh_port=${FLAGS_ssh_port} \
   --remote=127.0.0.1 \

lib/cros_vm_lib.sh

@@ -12,6 +12,7 @@ DEFINE_boolean snapshot ${FLAGS_FALSE} "Don't commit changes to image."
 DEFINE_integer ssh_port 9222 "Port to tunnel ssh traffic over."
 
 KVM_PID_FILE=/tmp/kvm.$$.pid
+LIVE_VM_IMAGE=
 
 function get_pid() {
   sudo cat "${KVM_PID_FILE}"
@@ -55,18 +56,49 @@ function start_kvm() {
       ${snapshot} \
       -net user,hostfwd=tcp::${FLAGS_ssh_port}-:22 \
       -hda "${1}"
+
+    LIVE_VM_IMAGE="${1}"
   fi
 }
 
+# Checks to see if we can access the target virtual machine with ssh.
+function ssh_ping() {
+  "$(dirname $0)"/../ssh_test.sh \
+    --ssh_port=${FLAGS_ssh_port} \
+    --remote=127.0.0.1
+}
+
+# Tries to ssh into live image $1 times.  After first failure, a try involves
+# shutting down and restarting kvm.
+function retry_until_ssh() {
+  local can_ssh_into=1
+  local retries=0
+  ssh_ping && can_ssh_into=0
+
+  while [ ${can_ssh_into} -eq 1 ] && [ ${retries} -lt ${1} ]; do
+    echo "Failed to connect to virtual machine, retrying ... " >&2
+    stop_kvm || echo "Could not stop kvm.  Retrying anyway." >&2
+    start_kvm "${LIVE_VM_IMAGE}"
+    ssh_ping && can_ssh_into=0
+    retries=$((retries + 1))
+  done
+  return ${can_ssh_into}
+}
+
 function stop_kvm() {
   if [ "${FLAGS_persist}" -eq "${FLAGS_TRUE}" ]; then
     echo "Persist requested.  Use --ssh_port ${FLAGS_ssh_port} " \
       "--kvm_pid ${KVM_PID_FILE} to re-connect to it."
   else
-    echo "Stopping the KVM instance"
+    echo "Stopping the KVM instance" >&2
     local pid=$(get_pid)
-    echo "Killing ${pid}"
+    if [ -n "${pid}" ]; then
-    sudo kill ${pid}
+      echo "Killing ${pid}"
-    sudo rm "${KVM_PID_FILE}"
+      sudo kill ${pid}
+      sudo rm "${KVM_PID_FILE}"
+    else
+      echo "No kvm pid found to stop." >&2
+      return 1
+    fi
   fi
 }