diff --git a/build_image b/build_image
index 7b9abc7f18..9b94d99c56 100755
--- a/build_image
+++ b/build_image
@@ -92,11 +92,6 @@ if [ "${FLAGS_rootfs_size}" -gt "${FLAGS_rootfs_partition_size}" ] ; then
   exit 1
 fi
 
-if [ ${FLAGS_jobs} -ne -1 ]; then
-  EMERGE_JOBS="--jobs=${FLAGS_jobs}"
-  FLAGS_fast=${FLAGS_TRUE}
-fi
-
 EMERGE_BOARD_CMD="emerge-${FLAGS_board}"
 if [ "${FLAGS_fast}" -eq "${FLAGS_TRUE}" ]; then
   echo "Using alternate emerge"
@@ -166,6 +161,10 @@ if [[ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ]] ; then
   INSTALL_MASK="${INSTALL_MASK} ${FACTORY_INSTALL_MASK}"
 fi
 
+if [[ ${FLAGS_jobs} -ne -1 ]]; then
+  EMERGE_JOBS="--jobs=${FLAGS_jobs}"
+fi
+
 # Figure out ARCH from the given toolchain.
 # TODO: Move to common.sh as a function after scripts are switched over.
 TC_ARCH=$(echo "${CHOST}" | awk -F'-' '{ print $1 }')
diff --git a/common.sh b/common.sh
index 446be805ed..262895be69 100644
--- a/common.sh
+++ b/common.sh
@@ -325,13 +325,14 @@ function die {
 }
 
 # Retry an emerge command according to $FLAGS_retries
+# The $EMERGE_JOBS flags will only be added the first time the command is run
 function eretry () {
-  $* && return 0
   local i=
   for i in $(seq $FLAGS_retries); do
     echo Retrying $*
-    $* && return 0
+    $* $EMERGE_JOBS && return 0
   done
+  $* && return 0
   return 1
 }