diff --git a/build_image b/build_image
index ad3c54156f..273d0a4342 100755
--- a/build_image
+++ b/build_image
@@ -240,13 +240,6 @@ emerge_to_image() {
 # Check that the build root is sane.
 "${BUILD_LIBRARY_DIR}/test_build_root" --root="${BOARD_ROOT}"
 
-# Freshen kernel with correct USE flags. This is a noop if we have
-# the right kernel prebuilt. Factory install uses USE="initramfs".
-# We don't allow building from source with the image as a target,
-# and it's not possible to store prebuilts for the same package
-# with different use flags.
-sudo -E ${EMERGE_BOARD_CMD} -uDNv -g virtual/kernel
-
 # Use canonical path since some tools (e.g. mount) do not like symlinks.
 # Append build attempt to output directory.
 IMAGE_SUBDIR="${CHROMEOS_VERSION_STRING}-a${FLAGS_build_attempt}"
@@ -638,6 +631,12 @@ create_base_image() {
     copy_gcc_libs "${ROOT_FS_DIR}" $atom
   done
 
+  if [ ${FLAGS_factory_install} -eq ${FLAGS_TRUE} ]; then
+    # Install our custom factory install kernel with the appropriate use flags
+    # to the image.
+    emerge_custom_kernel "${ROOT_FS_DIR}"
+  fi
+
   # We "emerge --root=${ROOT_FS_DIR} --root-deps=rdeps --usepkgonly" all of the
   # runtime packages for chrome os. This builds up a chrome os image from
   # binary packages with runtime dependencies only.  We use INSTALL_MASK to
diff --git a/common.sh b/common.sh
index da2eb84b64..476043e133 100644
--- a/common.sh
+++ b/common.sh
@@ -687,3 +687,39 @@ function find_coreboot_component () {
   local override=$2
   find_fw_component "firmware coreboot" "${component}" "${override}"
 }
+
+function emerge_custom_kernel() {
+  local install_root="$1"
+  local root=${FLAGS_build_root}/${FLAGS_board}
+  local tmp_pkgdir=${root}/custom-packages
+
+  # Clean up any leftover state in custom directories.
+  sudo rm -rf ${tmp_pkgdir}
+
+  # Update chromeos-initramfs to contain the latest binaries from the build
+  # tree. This is basically just packaging up already-built binaries from
+  # $root. We are careful not to muck with the existing prebuilts so that
+  # prebuilts can be uploaded in parallel.
+  # TODO(davidjames): Implement ABI deps so that chromeos-initramfs will be
+  # rebuilt automatically when its dependencies change.
+  sudo -E PKGDIR=${tmp_pkgdir} $EMERGE_BOARD_CMD -1 \
+    chromeos-base/chromeos-initramfs || die "Cannot emerge chromeos-initramfs"
+
+  # Verify all dependencies of the kernel are installed. This should be a
+  # no-op, but it's good to check in case a developer didn't run
+  # build_packages.
+  local kernel=$(portageq-${FLAGS_board} expand_virtual ${root} virtual/kernel)
+  sudo -E PKGDIR=${tmp_pkgdir} $EMERGE_BOARD_CMD --onlydeps \
+    ${kernel} || die "Cannot emerge kernel dependencies"
+
+  # Build the kernel. This uses the standard root so that we can pick up the
+  # initramfs from there. But we don't actually install the kernel to the
+  # standard root, because that'll muck up the kernel debug symbols there,
+  # which we want to upload in parallel.
+  sudo -E PKGDIR=${tmp_pkgdir} $EMERGE_BOARD_CMD --buildpkgonly \
+    ${kernel} || die "Cannot emerge kernel"
+
+  # Install the custom kernel to the provided install root.
+  sudo -E PKGDIR=${tmp_pkgdir} $EMERGE_BOARD_CMD --usepkgonly \
+    --root=${install_root} ${kernel} || die "Cannot emerge kernel to root"
+}
diff --git a/mod_image_for_recovery.sh b/mod_image_for_recovery.sh
index 9d8c0e774c..d1d35d3436 100755
--- a/mod_image_for_recovery.sh
+++ b/mod_image_for_recovery.sh
@@ -148,20 +148,8 @@ BOAT
   die "$* failed"
 }
 
-emerge_recovery_kernel() {
-  echo "Emerging custom recovery initramfs and kernel"
-  local emerge_flags="-uDNv1 --usepkg=n --selective=n"
-
-  $EMERGE_BOARD_CMD \
-    $emerge_flags --binpkg-respect-use=y \
-    chromeos-initramfs || failboat "emerge initramfs"
-  USE="fbconsole initramfs" $EMERGE_BOARD_CMD \
-                    $emerge_flags --binpkg-respect-use=y \
-                    virtual/kernel || failboat "emerge kernel"
-}
-
 create_recovery_kernel_image() {
-  local sysroot="${FLAGS_build_root}/${FLAGS_board}"
+  local sysroot="$FACTORY_ROOT"
   local vmlinuz="$sysroot/boot/vmlinuz"
   local root_offset=$(partoffset "$FLAGS_image" 3)
   local root_size=$(partsize "$FLAGS_image" 3)
@@ -288,8 +276,9 @@ install_recovery_kernel() {
   sudo $GPT add -i 2 -S 1 "$RECOVERY_IMAGE"
 
   # Repeat for the legacy bioses.
-  # Replace vmlinuz.A with the recovery version
-  local sysroot="${FLAGS_build_root}/${FLAGS_board}"
+  # Replace vmlinuz.A with the recovery version we built.
+  # TODO(wad): Extract the $RECOVERY_KERNEL_IMAGE and grab vmlinuz from there.
+  local sysroot="$FACTORY_ROOT"
   local vmlinuz="$sysroot/boot/vmlinuz"
   local failed=0
 
@@ -441,8 +430,12 @@ if [ -z "$INSTALL_VBLOCK" ]; then
   die "Could not copy the vblock from stateful."
 fi
 
+# Build the recovery kernel.
+FACTORY_ROOT="${FLAGS_build_root}/${FLAGS_board}/factory-root"
+USE="fbconsole initramfs" emerge_custom_kernel "$FACTORY_ROOT" \
+  || failboat "Cannot emerge custom kernel"
+
 if [ -z "$FLAGS_kernel_image" ]; then
-  emerge_recovery_kernel
   create_recovery_kernel_image
   echo "Recovery kernel created at $RECOVERY_KERNEL_IMAGE"
 else