coreos-kernel: convert to a two stage kernel build

The new coreos-modules ebuild does the bulk of the work, building and
installing all modules to /usr/lib/modules/... and is now the home of
the kernel config. This in turn allows the coreos-kernel ebuild to
depend on coreos-firmware and allowing dracut to handle kernel modules
and firmware instead of appending them to the cpio ourselves.

sdk_container/src/third_party/coreos-overlay/eclass/coreos-kernel.eclass

@@ -12,7 +12,7 @@ COREOS_SOURCE_NAME="linux-${PV}-coreos${COREOS_SOURCE_REVISION}"
 
 [[ ${EAPI} != "5" ]] && die "Only EAPI=5 is supported"
 
-inherit linux-info savedconfig toolchain-funcs
+inherit linux-info toolchain-funcs
 
 HOMEPAGE="http://www.kernel.org"
 LICENSE="GPL-2 freedist"
@@ -20,28 +20,7 @@ SLOT="0/${PVR}"
 SRC_URI=""
 IUSE=""
 
-DEPEND="
+DEPEND="=sys-kernel/coreos-sources-${COREOS_SOURCE_VERSION}"
-	app-arch/gzip
-	app-shells/bash
-	sys-apps/coreutils
-	sys-apps/findutils
-	sys-apps/grep
-	sys-apps/ignition:=
-	sys-apps/less
-	sys-apps/sed
-	sys-apps/shadow
-	sys-apps/systemd
-	sys-apps/seismograph
-	sys-apps/util-linux
-	sys-fs/btrfs-progs
-	sys-fs/e2fsprogs
-	sys-fs/mdadm
-	sys-fs/xfsprogs
-	=sys-kernel/coreos-sources-${COREOS_SOURCE_VERSION}
-	sys-kernel/bootengine:=
-	sys-kernel/dracut
-	virtual/udev
-"
 
 # Do not analyze or strip installed files
 RESTRICT="binchecks strip"
@@ -73,39 +52,26 @@ find_defconfig() {
 	die "No defconfig found for ${ARCH} and ${PVR} in ${FILESDIR}"
 }
 
-# @FUNCTION: get_bootengine_lib
+config_update() {
-# @DESCRIPTION:
+	key="${1%%=*}"
-# Check if /lib is a symlink in the current cpio. If so we need to use
+	sed -i -e "/^${key}=/d" build/.config || die
-# the target path (usually lib64) instead when adding new things.
+	echo "$1" >> build/.config || die
-# When extracting with GNU cpio the first entry (the symlink) wins but
-# in the kernel the second entry (as a directory) definition wins.
-# As if using cpio isn't bad enough already.
-# If lib doesn't exist or isn't a symlink then nothing is returned.
-get_bootengine_lib() {
-	cpio -itv --quiet < build/bootengine.cpio | \
-		awk '$1 ~ /^l/ && $9 == "lib" { print $11 }'
-	assert
 }
 
-# @FUNCTION: update_bootengine_cpio
+# Get the path to the architecture's kernel image.
-# @DESCRIPTION:
+kernel_path() {
-# Append files in the given directory to the bootengine cpio.
+	local kernel_arch=$(tc-arch-kernel)
-# Allows us to stick kernel modules into the initramfs built into the image.
+	case "${kernel_arch}" in
-update_bootengine_cpio() {
+		arm64)	echo build/arch/arm64/boot/Image;;
-	local extra_root="$1"
+		x86)	echo build/arch/x86/boot/bzImage;;
-	local cpio_args=(
+		*)		die "Unsupported kernel arch '${kernel_arch}'";;
-		--create --append --null
+	esac
-		# dracut uses the 'newc' cpio format
+}
-		--format=newc
-		# squash file ownership to root for new files.
-		--owner=root:root
-		# append to our local copy of bootengine
-		-F "${S}/build/bootengine.cpio"
-	)
 
-	echo "Updating bootengine.cpio"
+# Get the make target to build the kernel image.
-	(cd "${extra_root}" && find . -print0 | cpio "${cpio_args[@]}") || \
+kernel_target() {
-		die "cpio update failed!"
+	local path=$(kernel_path)
+	echo "${path##*/}"
 }
 
 kmake() {
@@ -181,11 +147,11 @@ shred_keys() {
 }
 
 # Populate /lib/modules/$(uname -r)/{build,source}
-prepare-lib-modules-release-dirs() {
+install_build_source() {
 	local kernel_arch=$(tc-arch-kernel)
 
-	# build and source must cleaned up to avoid referencing $ROOT
+	# remove the broken symlinks referencing $ROOT
-	rm "${D}/usr/lib/modules/${version}"/{build,source} || die
+	rm "${D}/usr/lib/modules/${KV_FULL}"/{build,source} || die
 
 	# Install a stripped source for out-of-tree module builds (Debian-derived)
 	{
@@ -194,29 +160,12 @@ prepare-lib-modules-release-dirs() {
 		find source/arch/${kernel_arch} -follow \( -name 'module.lds' -o -name 'Kbuild.platforms' -o -name 'Platform' \) -print
 		find $(find source/arch/${kernel_arch} -follow \( -name include -o -name scripts \) -follow -type d -print) -print
 		find source/include source/scripts -follow -print
+		find build/ -print
 	} | cpio -pd \
 		--preserve-modification-time \
 		--owner=root:root \
 		--dereference \
-		"${D}/usr/lib/modules/${version}" || die
+		"${D}/usr/lib/modules/${KV_FULL}" || die
-
-	# Clean up the build tree and install for out-of-tree module builds
-	find "build/" -follow -maxdepth 1 -name 'System.map' -print \
-		| cpio -pd \
-		--preserve-modification-time \
-		--owner=root:root \
-		"${D}/usr/lib/modules/${version}" || die
-	kmake clean
-	find "build/" -type d -empty -delete || die
-	rm --recursive \
-		"build/bootengine.cpio" \
-		"build/.config.old" \
-		|| die
-
-	find "build/" -print | cpio -pd \
-		--preserve-modification-time \
-		--owner=root:root \
-		"${D}/usr/lib/modules/${version}" || die
 }
 
 coreos-kernel_pkg_pretend() {
@@ -228,43 +177,20 @@ coreos-kernel_pkg_pretend() {
 	fi
 }
 
-# We are bad, we want to get around the sandbox.  So do the creation of the
-# cpio image in pkg_setup() where we are free to mount filesystems, chroot,
-# and other fun stuff.
 coreos-kernel_pkg_setup() {
 	[[ "${MERGE_TYPE}" == binary ]] && return
 
-	if [[ "${ROOT:-/}" != / ]]; then
+	# tc-arch-kernel requires a call to get_version from linux-info.eclass
-		${ROOT}/usr/sbin/update-bootengine -m -c ${ROOT} || die
+	get_version || die "Failed to detect kernel version in ${KERNEL_DIR}"
-	else
-		update-bootengine || die
-	fi
 }
 
 coreos-kernel_src_unpack() {
-	# tc-arch-kernel requires a call to get_version from linux-info.eclass
-	get_version || die "Failed to detect kernel version in ${KERNEL_DIR}"
-
 	# we more or less reproduce the layout in /lib/modules/$(uname -r)/
 	mkdir -p "${S}/build" || die
 	mkdir -p "${S}/source" || die
 	ln -s "${KERNEL_DIR}"/* "${S}/source/" || die
 }
 
-coreos-kernel_src_prepare() {
-	restore_config build/.config
-	if [[ ! -f build/.config ]]; then
-		local config="$(find_defconfig)"
-		elog "Building using default config ${config}"
-		cp "${config}" build/.config || die
-	fi
-
-	# copy the cpio initrd to the output build directory so we can tack it
-	# onto the kernel image itself.
-	cp "${ROOT}"/usr/share/bootengine/bootengine.cpio build/bootengine.cpio \
-		|| die "cp bootengine.cpio failed, try emerge-\$BOARD bootengine"
-}
-
 coreos-kernel_src_configure() {
 	# Use default for any options not explitly set in defconfig
 	kmake olddefconfig
@@ -273,50 +199,4 @@ coreos-kernel_src_configure() {
 	kmake savedefconfig
 }
 
-coreos-kernel_src_compile() {
+EXPORT_FUNCTIONS pkg_pretend pkg_setup src_unpack src_configure
-	setup_keys
-
-	# Build both vmlinux and modules (moddep checks symbols in vmlinux)
-	kmake vmlinux modules
-
-	# Install modules and add them to the initramfs image
-	local bootengine_root="${T}/bootengine"
-	kmake INSTALL_MOD_PATH="${bootengine_root}" \
-		  INSTALL_MOD_STRIP="--strip-unneeded" \
-		  modules_install
-
-	local bootengine_lib=$(get_bootengine_lib)
-	if [[ -n "${bootengine_lib}" ]]; then
-		mkdir -p "$(dirname "${bootengine_root}/${bootengine_lib}")" || die
-		mv "${bootengine_root}/lib" \
-			"${bootengine_root}/${bootengine_lib}" || die
-	fi
-	update_bootengine_cpio "${bootengine_root}"
-
-	# Build the final kernel image
-	kmake
-}
-
-coreos-kernel_src_install() {
-	dodir /usr/boot
-	kmake INSTALL_PATH="${D}/usr/boot" install
-	# Install modules to /usr, assuming USE=symlink-usr
-	# Install firmware to a temporary (bogus) location.
-	# The linux-firmware package will be used instead.
-	# Stripping must be done here, not portage, to preserve sigs.
-	# Uncomment vdso_install for easy access to debug symbols in gdb:
-	#   set debug-file-directory /lib/modules/4.0.7-coreos-r2/vdso/
-	kmake INSTALL_MOD_PATH="${D}/usr" \
-		  INSTALL_MOD_STRIP="--strip-unneeded" \
-		  INSTALL_FW_PATH="${T}/fw" \
-		  modules_install # vdso_install
-
-	local version=$(kmake -s --no-print-directory kernelrelease)
-	dosym "vmlinuz-${version}" /usr/boot/vmlinuz
-	dosym "config-${version}" /usr/boot/config
-
-	shred_keys
-	prepare-lib-modules-release-dirs
-}
-
-EXPORT_FUNCTIONS pkg_pretend pkg_setup src_unpack src_prepare src_configure src_compile src_install

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-firmware/coreos-firmware-99999999.ebuild

@@ -23,10 +23,10 @@ DESCRIPTION="Linux firmware files"
 HOMEPAGE="http://git.kernel.org/?p=linux/kernel/git/firmware/linux-firmware.git"
 
 LICENSE="GPL-1 GPL-2 GPL-3 BSD freedist"
-SLOT="0"
+SLOT="0/${PVR}"
 IUSE=""
 
-CDEPEND=">=sys-kernel/coreos-kernel-3.17.6-r1:="
+CDEPEND=">=sys-kernel/coreos-modules-4.6.3-r1:="
 DEPEND="${CDEPEND}
 	sys-kernel/coreos-sources"
 RDEPEND="${CDEPEND}

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/coreos-kernel-4.6.3-r1.ebuild

@@ -0,0 +1,85 @@
+# Copyright 2014-2016 CoreOS, Inc.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=5
+COREOS_SOURCE_REVISION=""
+inherit coreos-kernel
+
+DESCRIPTION="CoreOS Linux kernel"
+KEYWORDS="amd64 arm64"
+
+RDEPEND="=sys-kernel/coreos-modules-${PVR}"
+DEPEND="${RDEPEND}
+	app-arch/gzip
+	app-shells/bash
+	sys-apps/coreutils
+	sys-apps/findutils
+	sys-apps/grep
+	sys-apps/ignition:=
+	sys-apps/less
+	sys-apps/sed
+	sys-apps/shadow
+	sys-apps/systemd
+	sys-apps/seismograph
+	sys-apps/util-linux
+	sys-fs/btrfs-progs
+	sys-fs/e2fsprogs
+	sys-fs/mdadm
+	sys-fs/xfsprogs
+	>=sys-kernel/coreos-firmware-20160331-r1:=
+	>=sys-kernel/bootengine-0.0.4:=
+	sys-kernel/dracut
+	virtual/udev"
+
+# We are bad, we want to get around the sandbox.  So do the creation of the
+# cpio image in pkg_setup() where we are free to mount filesystems, chroot,
+# and other fun stuff.
+pkg_setup() {
+	coreos-kernel_pkg_setup
+
+	[[ "${MERGE_TYPE}" == binary ]] && return
+
+	# Fail early if we didn't detect the build installed by coreos-modules
+	[[ -n "${KV_OUT_DIR}" ]] || die "Failed to detect modules build tree"
+
+	if [[ "${ROOT:-/}" != / ]]; then
+		${ROOT}/usr/sbin/update-bootengine -m -c ${ROOT} -k "${KV_FULL}" || die
+	else
+		update-bootengine -k "${KV_FULL}" || die
+	fi
+}
+
+src_prepare() {
+	# KV_OUT_DIR points to the minimal build tree installed by coreos-modules
+	# Pull in the config and public module signing key
+	cp -v "${KV_OUT_DIR}/.config" build/ || die
+	local sig_key="$(getconfig MODULE_SIG_KEY)"
+	mkdir -p "build/${sig_key%/*}" || die
+	cp -v "${KV_OUT_DIR}/${sig_key}" "build/${sig_key}" || die
+
+	# Symlink to bootengine.cpio so we can stick with relative paths in .config
+	ln -sv "${ROOT}"/usr/share/bootengine/bootengine.cpio build/ || die
+	config_update 'CONFIG_INITRAMFS_SOURCE="bootengine.cpio"'
+}
+
+src_compile() {
+	kmake "$(kernel_target)"
+
+	# sanity check :)
+	[[ -e build/certs/signing_key.pem ]] && die "created a new key!"
+}
+
+src_install() {
+	# coreos-postinst expects to find the kernel in /usr/boot
+	insinto "/usr/boot"
+	newins "$(kernel_path)" "vmlinuz-${KV_FULL}"
+	dosym "vmlinuz-${KV_FULL}" "/usr/boot/vmlinuz"
+
+	insinto "/usr/lib/modules/${KV_FULL}/build"
+	doins build/System.map
+	# TODO: vmlinux would be useful for debugging but it is >300MB
+
+	# Uncomment vdso_install for easy access to debug symbols in gdb:
+	#   set debug-file-directory /lib/modules/4.0.7-coreos-r2/vdso/
+	#kmake INSTALL_MOD_PATH="${D}/usr" vdso_install
+}

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-kernel/coreos-kernel-4.6.3.ebuild

@@ -1,9 +0,0 @@
-# Copyright 2014 CoreOS, Inc.
-# Distributed under the terms of the GNU General Public License v2
-
-EAPI=5
-COREOS_SOURCE_REVISION=""
-inherit coreos-kernel
-
-DESCRIPTION="CoreOS Linux kernel"
-KEYWORDS="amd64 arm64"

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/coreos-modules-4.6.3-r1.ebuild

@@ -0,0 +1,57 @@
+# Copyright 2014-2016 CoreOS, Inc.
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=5
+COREOS_SOURCE_REVISION=""
+inherit coreos-kernel savedconfig
+
+DESCRIPTION="CoreOS Linux kernel modules"
+KEYWORDS="amd64 arm64"
+RDEPEND="!<sys-kernel/coreos-kernel-4.6.3-r1"
+
+src_prepare() {
+	restore_config build/.config
+	if [[ ! -f build/.config ]]; then
+		local config="$(find_defconfig)"
+		elog "Building using default config ${config}"
+		cp "${config}" build/.config || die
+	fi
+
+	# Check that an old pre-ebuild-split config didn't leak in.
+	grep -q "^CONFIG_INITRAMFS_SOURCE=" build/.config && \
+		die "CONFIG_INITRAMFS_SOURCE must be removed from kernel config"
+}
+
+src_compile() {
+	# Generate module signing key
+	setup_keys
+
+	# Build both vmlinux and modules (moddep checks symbols in vmlinux)
+	kmake vmlinux modules
+}
+
+src_install() {
+	# Install modules to /usr, assuming USE=symlink-usr
+	# Install firmware to a temporary (bogus) location.
+	# The linux-firmware package will be used instead.
+	# Stripping must be done here, not portage, to preserve sigs.
+	kmake INSTALL_MOD_PATH="${D}/usr" \
+		  INSTALL_MOD_STRIP="--strip-unneeded" \
+		  INSTALL_FW_PATH="${T}/fw" \
+		  modules_install
+
+	# Clean up the build tree
+	shred_keys
+	kmake clean
+	find "build/" -type d -empty -delete || die
+	rm "build/.config.old" || die
+
+	# Install /lib/modules/${KV_FULL}/{build,source}
+	install_build_source
+
+	# Not strictly required but this is where we used to install the config.
+	dodir "/usr/boot"
+	local build="lib/modules/${KV_FULL}/build"
+	dosym "../${build}/.config" "/usr/boot/config-${KV_FULL}"
+	dosym "../${build}/.config" "/usr/boot/config"
+}

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/amd64_defconfig-4.6

@@ -31,7 +31,6 @@ CONFIG_NAMESPACES=y
 CONFIG_USER_NS=y
 CONFIG_SCHED_AUTOGROUP=y
 CONFIG_BLK_DEV_INITRD=y
-CONFIG_INITRAMFS_SOURCE="bootengine.cpio"
 CONFIG_EXPERT=y
 # CONFIG_COMPAT_BRK is not set
 CONFIG_PROFILING=y

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/arm64_defconfig-4.6

@@ -30,7 +30,6 @@ CONFIG_NAMESPACES=y
 CONFIG_USER_NS=y
 CONFIG_SCHED_AUTOGROUP=y
 CONFIG_BLK_DEV_INITRD=y
-CONFIG_INITRAMFS_SOURCE="bootengine.cpio"
 CONFIG_EXPERT=y
 CONFIG_KALLSYMS_ALL=y
 # CONFIG_COMPAT_BRK is not set

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/metadata.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE pkgmetadata SYSTEM "http://www.gentoo.org/dtd/metadata.dtd">
+<pkgmetadata>
+</pkgmetadata>