From f5850903d901db2ffe66dee7244465f845a0b2c2 Mon Sep 17 00:00:00 2001
From: David James <davidjames@chromium.org>
Date: Fri, 30 Sep 2011 10:51:48 -0700
Subject: [PATCH] Tweak ssh settings to detect when remote server dies.

Right now, it seems that we don't consistently detect when the kvm is rebooted.
To improve this, I've added ServerKeepAlive messages, such that our connection
will be dropped if the target host is unreachable for 15 seconds. Further, I've
also tightened the ConnectTimeout from 120 seconds to 30 seconds so that we
detect timeouts earlier, but added 4 retries, so that all connections are
robust to temporary unresponsiveness (e.g. ssh server restarting).

BUG=chromium-os:21082, chromium-os:20305
TEST=alex-binary trybot run.

Change-Id: Ibd000bc3158138ee9c617e56bd7b11d4bd8bcb61
Reviewed-on: http://gerrit.chromium.org/gerrit/8574
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: David James <davidjames@chromium.org>
Commit-Ready: David James <davidjames@chromium.org>
---
 remote_access.sh | 33 ++++++++++++++++++---------------
 1 file changed, 18 insertions(+), 15 deletions(-)

diff --git a/remote_access.sh b/remote_access.sh
index 77a418d94f..b363856983 100644
--- a/remote_access.sh
+++ b/remote_access.sh
@@ -13,34 +13,37 @@ DEFINE_string private_key "$DEFAULT_PRIVATE_KEY" \
 DEFINE_integer ssh_port 22 \
   "SSH port of the remote machine running Chromium OS instance"
 
+SSH_CONNECT_SETTINGS="-o Protocol=2 -o ConnectTimeout=30 \
+  -o ConnectionAttempts=4 -o ServerAliveInterval=10 \
+  -o ServerAliveCountMax=3 -o StrictHostKeyChecking=no"
+
 # Copies $1 to $2 on remote host
 function remote_cp_to() {
-  REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -o ConnectTimeout=120 \
-    -i $TMP_PRIVATE_KEY $1 root@$FLAGS_remote:$2)
+  REMOTE_OUT=$(scp -P ${FLAGS_ssh_port} $SSH_CONNECT_SETTINGS \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY $1 \
+    root@$FLAGS_remote:$2)
   return ${PIPESTATUS[0]}
 }
 
 # Copies a list of remote files specified in file $1 to local location
 # $2.  Directory paths in $1 are collapsed into $2.
 function remote_rsync_from() {
-  rsync -e "ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-             -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -o ConnectTimeout=120 \
-             -i $TMP_PRIVATE_KEY" \
+  rsync -e "ssh -p ${FLAGS_ssh_port} $SSH_CONNECT_SETTINGS \
+             -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY" \
     --no-R --files-from=$1 root@${FLAGS_remote}:/ $2
 }
 
 function _verbose_remote_sh() {
-  REMOTE_OUT=$(ssh -vp ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -o ConnectTimeout=120 \
-    -i $TMP_PRIVATE_KEY root@$FLAGS_remote "$@")
+  REMOTE_OUT=$(ssh -vp ${FLAGS_ssh_port} $SSH_CONNECT_SETTINGS \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY \
+    root@$FLAGS_remote "$@")
   return ${PIPESTATUS[0]}
 }
 
 function _non_verbose_remote_sh() {
-  REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -o ConnectTimeout=120 \
-    -i $TMP_PRIVATE_KEY root@$FLAGS_remote "$@")
+  REMOTE_OUT=$(ssh -p ${FLAGS_ssh_port} $SSH_CONNECT_SETTINGS \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY \
+    root@$FLAGS_remote "$@")
   return ${PIPESTATUS[0]}
 }
 
@@ -58,9 +61,9 @@ function remote_sh() {
 }
 
 function remote_sh_raw() {
-  ssh -p ${FLAGS_ssh_port} -o StrictHostKeyChecking=no \
-    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -o ConnectTimeout=120 \
-    -i $TMP_PRIVATE_KEY $EXTRA_REMOTE_SH_ARGS root@$FLAGS_remote "$@"
+  ssh -p ${FLAGS_ssh_port} $SSH_CONNECT_SETTINGS \
+    -o UserKnownHostsFile=$TMP_KNOWN_HOSTS -i $TMP_PRIVATE_KEY \
+    $EXTRA_REMOTE_SH_ARGS root@$FLAGS_remote "$@"
   return $?
 }