Retry on build_image failures and make sbt more aggressive by default

As with build_packages, only retries by default if you passed --jobs
even though build_image will indeed spuriously fail with only one job.
sync_build_test now uses jobs=#cores by default for both package and
image building.  Eventually will also use this for gclient sync'ing
once that is better vetted.

Review URL: http://codereview.chromium.org/1564035

build_image

@@ -35,6 +35,8 @@ DEFINE_boolean installmask $FLAGS_TRUE \
   "Use INSTALL_MASK to shrink the resulting image."
 DEFINE_integer jobs -1 \
   "How many packages to build in parallel at maximum."
+DEFINE_integer retries -1 \
+  "On image mastering failure, the number of times to retry"
 DEFINE_boolean statefuldev $FLAGS_FALSE \
   "Install development packages on stateful partition rather than the rootfs"
 DEFINE_string to "" \
@@ -283,6 +285,11 @@ fi
 
 if [[ $FLAGS_jobs -ne -1 ]]; then
   EMERGE_JOBS="--jobs=$FLAGS_jobs"
+  if [[ $FLAGS_retries -eq -1 ]]; then
+    # The jobs flag can be flaky. Retry once by default,
+    # without the jobs flag.
+    FLAGS_retries=1
+  fi
 fi
 
 # Prepare stateful partition with some pre-created directories
@@ -305,7 +312,7 @@ sudo mount --bind "${STATEFUL_DIR}/var" "${ROOT_FS_DIR}/var"
 # runtime packages for chrome os. This builds up a chrome os image from binary
 # packages with runtime dependencies only.  We use INSTALL_MASK to trim the
 # image size as much as possible.
-sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
+eretry sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
   --root="$ROOT_FS_DIR" --root-deps=rdeps \
   --usepkgonly chromeos $EMERGE_JOBS
 
@@ -315,7 +322,7 @@ ROOT_DEV_DIR="$ROOT_FS_DIR"
 
 # Install development packages.
 if [[ $FLAGS_withdev -eq $FLAGS_TRUE ]] ; then
-  sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
+  eretry sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
       --root="$ROOT_DEV_DIR" --root-deps=rdeps \
       --usepkgonly chromeos-dev $EMERGE_JOBS
 
@@ -329,14 +336,14 @@ if [[ $FLAGS_withdev -eq $FLAGS_TRUE ]] ; then
 fi
 
 if [ -n "$FLAGS_factory_server" ]; then
-  sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
+  eretry sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
       --root="$ROOT_DEV_DIR" --root-deps=rdeps \
       --usepkgonly chromeos-factoryinstall $EMERGE_JOBS
 fi
 
 # Install packages required for testing.
 if [[ $FLAGS_withtest -eq $FLAGS_TRUE ]] ; then
-  sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
+  eretry sudo INSTALL_MASK="$INSTALL_MASK" emerge-${BOARD} \
       --root="$ROOT_DEV_DIR" --root-deps=rdeps \
       --usepkgonly chromeos-test $EMERGE_JOBS
 fi

sync_build_test.sh

@@ -63,8 +63,6 @@ DEFINE_boolean image_to_live ${FLAGS_FALSE} \
     "Put the resulting image on live instance (requires --remote)"
 DEFINE_string image_to_usb "" \
     "Treat this device as USB and put the image on it after build"
-# You can set jobs > 1 but then your build may break and you may need
-# to retry.  Setting it to 1 is best for non-interactive sessions.
 DEFINE_boolean interactive ${FLAGS_FALSE} \
     "Tell user what we plan to do and wait for input to proceed" i
 DEFINE_integer jobs -1 "Concurrent build jobs"
@@ -199,9 +197,21 @@ function validate_and_set_param_defaults() {
       exit 1
     fi
   fi
+
+  if [[ ${FLAGS_jobs} -eq -1 ]]; then
+    FLAGS_jobs=$(cat /proc/cpuinfo | grep -c processor)
+  fi
 }
 
 
+function describe_jobs() {
+  local jobs=" single job (slow but no retries)"
+  if [[ ${FLAGS_jobs} -gt 1 ]]; then
+    jobs=" ${FLAGS_jobs} jobs"
+  fi
+  echo ${jobs}
+}
+
 # Prints a description of what we are doing or did
 function describe_steps() {
   if [[ ${FLAGS_sync} -eq ${FLAGS_TRUE} ]]; then
@@ -216,21 +226,19 @@ function describe_steps() {
   local set_passwd=${FLAGS_FALSE}
   if [[ ${FLAGS_build} -eq ${FLAGS_TRUE} ]]; then
     local withdev=""
-    local jobs=" single job (slow but safe)"
+    local jobs=$(describe_jobs)
-    if [[ ${FLAGS_jobs} -gt 1 ]]; then
-      jobs=" ${FLAGS_jobs} jobs (may cause build failure)"
-    fi
     if [[ ${FLAGS_withdev} -eq ${FLAGS_TRUE} ]]; then
       withdev=" with dev packages"
     fi
-    echo " * Build image${withdev}${jobs}"
+    echo " * Build image${withdev} with ${jobs}"
     set_passwd=${FLAGS_TRUE}
     if [[ ${FLAGS_build_autotest} -eq ${FLAGS_TRUE} ]]; then
       echo " * Cross-build autotest client tests (build_autotest)"
     fi
   fi
   if [[ ${FLAGS_master} -eq ${FLAGS_TRUE} ]]; then
-    echo " * Master image (build_image)"
+    local jobs=$(describe_jobs)
+    echo " * Master image (build_image) with ${jobs}"
   fi
   if [[ -n "${FLAGS_grab_buildbot}" ]]; then
     if [[ "${FLAGS_grab_buildbot}" == "LATEST" ]]; then