sysext: Move OEM sysext build to image phase

Move OEM sysext building from the vms phase to the image phase. This
ensures OEM sysexts are signed with the same ephemeral key as other
sysexts, which is generated during image build and discarded afterward.

- Add create_oem_sysexts() to build all OEM sysexts during image build
- Add oem_sysexts.sh with OEM sysext definitions
- Update install_oem_sysext() to use prebuilt sysexts
- Add OEM sysext download to vms.sh for CI builds

Signed-off-by: Daniel Zatovic <daniel.zatovic@gmail.com>

build_image

@@ -60,10 +60,12 @@ different forms.  This scripts can be used to build the following:
 prod - Production image for CoreOS. This image is for booting (default if no argument is given).
 prodtar - Production container tar ball (implies prod). This can e.g. be used to run the Flatcar production image as a container (run machinectl import-tar or docker import).
 container - Developer image with single filesystem, bootable by nspawn.
+sysext - Build extra sysexts (podman, python, zfs, etc.).
+oem_sysext - Build OEM sysexts for all supported platforms.
 
 Examples:
 
-build_image --board=<board> [prod] [prodtar] [container] - builds developer and production images/tars.
+build_image --board=<board> [prod] [prodtar] [container] [sysext] [oem_sysext] - builds developer and production images/tars.
 ...
 "
 show_help_if_requested "$@"
@@ -81,7 +83,7 @@ DEFINE_string version "" \
 # Parse command line.
 FLAGS "$@" || exit 1
 
-eval set -- "${FLAGS_ARGV:-prod}"
+eval set -- "${FLAGS_ARGV:-prod oem_sysext}"
 
 # Only now can we die on error.  shflags functions leak non-zero error codes,
 # so will die prematurely if 'switch_to_strict_mode' is specified before now.
@@ -103,17 +105,20 @@ fi
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
 
 PROD_IMAGE=0
 PROD_TAR=0
 CONTAINER=0
 SYSEXT=0
+OEM_SYSEXT=0
 for arg in "$@"; do
   case "${arg}" in
     prod) PROD_IMAGE=1 ;;
     prodtar) PROD_IMAGE=1 PROD_TAR=1 ;;
     container) CONTAINER=1 ;;
     sysext) SYSEXT=1 ;;
+    oem_sysext) OEM_SYSEXT=1 ;;
     *)    die_notrace "Unknown image type ${arg}" ;;
   esac
 done
@@ -187,6 +192,9 @@ fi
 if [[ "${SYSEXT}" -eq 1 ]]; then
   create_prod_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}"
 fi
+if [[ "${OEM_SYSEXT}" -eq 1 ]]; then
+  create_oem_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}"
+fi
 
 if [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then
   zip_update_tools

build_library/oem_sysexts.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+# OEM sysext helpers.
+
+# Auto-detect scripts repo root from this file's location.
+# oem_sysexts.sh is at: <scripts_repo>/build_library/oem_sysexts.sh
+_OEM_SYSEXTS_SCRIPTS_ROOT="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/..")"
+
+get_oem_overlay_root() {
+  local overlay_root="/mnt/host/source/src/third_party/coreos-overlay"
+
+  if [[ ! -d "${overlay_root}" ]]; then
+    overlay_root="${_OEM_SYSEXTS_SCRIPTS_ROOT}/sdk_container/src/third_party/coreos-overlay"
+  fi
+
+  if [[ ! -d "${overlay_root}" ]]; then
+    echo "No coreos-overlay repo found (tried SDK and ${_OEM_SYSEXTS_SCRIPTS_ROOT})" >&2
+    exit 1
+  fi
+
+  printf '%s' "${overlay_root}"
+}
+
+_get_oem_ids() {
+  local arch list_var_name
+  arch=${1}; shift
+  list_var_name=${1}; shift
+
+  local overlay_root
+  overlay_root=$(get_oem_overlay_root)
+
+  local -a ebuilds=("${overlay_root}/coreos-base/common-oem-files/common-oem-files-"*'.ebuild')
+  if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then
+    echo "No coreos-base/common-oem-files ebuilds?!" >&2
+    exit 1
+  fi
+
+  # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS,
+  # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last
+  # one. Also defines global-by-default EAPI, which we make local
+  # here to avoid making it global.
+  local EAPI
+  source "${ebuilds[0]}" flatcar-local-variables
+
+  local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
+  local all_oemids=(
+    "${COMMON_OEMIDS[@]}"
+    "${arch_oemids_ref[@]}"
+  )
+
+  mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort)
+}
+
+# Gets a list of OEMs that are using sysexts.
+#
+# 1 - arch
+# 2 - name of an array variable to store the result in
+get_oem_id_list() {
+  _get_oem_ids "$@"
+}
+
+# Gets a list of OEM sysext descriptors.
+#
+# 1 - arch
+# 2 - name of an array variable to store the result in
+#
+# Format: "name|metapackage|useflags"
+get_oem_sysext_matrix() {
+  local arch list_var_name
+  arch=${1}; shift
+  list_var_name=${1}; shift
+
+  local -a oem_ids
+  _get_oem_ids "${arch}" oem_ids
+
+  local -a matrix=()
+  local oem_id
+  for oem_id in "${oem_ids[@]}"; do
+    matrix+=("oem-${oem_id}|coreos-base/oem-${oem_id}|${oem_id}")
+  done
+
+  local -n matrix_ref="${list_var_name}"
+  matrix_ref=("${matrix[@]}")
+}

build_library/prod_image_util.sh

@@ -3,6 +3,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+source "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
+
 # Lookup the current version of a binary package, downloading it if needed.
 # Usage: get_binary_pkg some-pkg/name
 # Prints: some-pkg/name-1.2.3
@@ -273,6 +275,52 @@ create_prod_sysexts() {
   done
 }
 
+create_oem_sysexts() {
+  local image_name="$1"
+  local image_sysext_base="${image_name%.bin}_sysext.squashfs"
+  local overlay_path
+  overlay_path=$(portageq get_repo_path / coreos-overlay)
+
+  local -a oem_sysexts
+  get_oem_sysext_matrix "${ARCH}" oem_sysexts
+
+  local sysext name metapkg useflags
+  for sysext in "${oem_sysexts[@]}"; do
+    IFS="|" read -r name metapkg useflags <<< "${sysext}"
+
+    # Check for manglefs script in the package's files directory
+    local mangle_script="${overlay_path}/${metapkg}/files/manglefs.sh"
+    if [[ ! -x "${mangle_script}" ]]; then
+      mangle_script=
+    fi
+
+    sudo rm -f "${BUILD_DIR}/${name}.raw" \
+        "${BUILD_DIR}/flatcar_test_update-${name}.gz" \
+        "${BUILD_DIR}/${name}_"*
+
+    info "Building OEM sysext ${name} with USE=${useflags}"
+    # The --install_root_basename="${name}-oem-sysext-rootfs" flag is
+    # important - it sets the name of a rootfs directory, which is
+    # used to determine the package target in
+    # coreos/base/profile.bashrc
+    #
+    # OEM sysexts use no compression here since they will be stored
+    # in a compressed OEM partition.
+    USE="${useflags}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \
+        --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \
+        --image_builddir="${BUILD_DIR}" \
+        --metapkgs="${metapkg}" \
+        --install_root_basename="${name}-oem-sysext-rootfs" \
+        --compression=none \
+        ${mangle_script:+--manglefs_script="${mangle_script}"} \
+        "${name}"
+    delta_generator \
+      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
+      -new_image "${BUILD_DIR}/${name}.raw" \
+      -out_file "${BUILD_DIR}/flatcar_test_update-${name}.gz"
+  done
+}
+
 sbsign_prod_image() {
   local image_name="$1"
   local disk_layout="$2"

build_library/vm_image_util.sh

@@ -568,7 +568,8 @@ install_oem_package() {
     sudo rm -rf "${oem_tmp}"
 }
 
-# Write the OEM sysext file into the OEM partition.
+# Install the prebuilt OEM sysext file into the OEM partition.
+# The sysext should have been built by 'build_image oem_sysext'.
 install_oem_sysext() {
     local oem_sysext=$(_get_vm_opt OEM_SYSEXT)
 
@@ -576,63 +577,24 @@ install_oem_sysext() {
         return 0
     fi
 
-    local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext"
-    local built_sysext_filename="${oem_sysext}.raw"
-    local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}"
+    local prebuilt_sysext_filename="${oem_sysext}.raw"
+    local prebuilt_sysext_path="${FLAGS_from}/${prebuilt_sysext_filename}"
     local version="${FLATCAR_VERSION}"
-    local metapkg="coreos-base/${oem_sysext}"
-    # The --install_root_basename="${name}-oem-sysext-rootfs" flag is
-    # important - it sets the name of a rootfs directory, which is
-    # used to determine the package target in
-    # coreos/base/profile.bashrc
-    #
-    # OEM sysexts are stored in the compressed partition, so we disable
-    # compression to avoid double-compression.
-    local build_sysext_flags=(
-        --board="${BOARD}"
-        --squashfs_base="${VM_SRC_SYSEXT_IMG}"
-        --image_builddir="${built_sysext_dir}"
-        --metapkgs="${metapkg}"
-        --compression=none
-        --install_root_basename="${VM_IMG_TYPE}-oem-sysext-rootfs"
-    )
-    local overlay_path mangle_fs
-    overlay_path=$(portageq get_repo_path / coreos-overlay)
-    mangle_fs="${overlay_path}/${metapkg}/files/manglefs.sh"
-    if [[ -x "${mangle_fs}" ]]; then
-        build_sysext_flags+=(
-            --manglefs_script="${mangle_fs}"
-        )
-    fi
 
-    mkdir -p "${built_sysext_dir}"
-    sudo -E "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}"
+    if [[ ! -f "${prebuilt_sysext_path}" ]]; then
+        die "Prebuilt OEM sysext not found at ${prebuilt_sysext_path}. Run 'build_image oem_sysext' first."
+    fi
 
     local installed_sysext_oem_dir='/oem/sysext'
     local installed_sysext_file_prefix="${oem_sysext}-${version}"
     local installed_sysext_filename="${installed_sysext_file_prefix}.raw"
     local installed_sysext_abspath="${installed_sysext_oem_dir}/${installed_sysext_filename}"
-    info "Installing ${oem_sysext} sysext"
+
+    info "Installing ${oem_sysext} sysext from prebuilt image"
     sudo install -Dpm 0644 \
-         "${built_sysext_path}" \
+         "${prebuilt_sysext_path}" \
          "${VM_TMP_ROOT}${installed_sysext_abspath}" ||
         die "Could not install ${oem_sysext} sysext"
-    # Move sysext image and reports to a destination directory to
-    # upload them, thus making them available as separate artifacts to
-    # download.
-    local upload_dir to_move
-    upload_dir="$(_dst_dir)"
-    for to_move in "${built_sysext_dir}/${oem_sysext}"*; do
-        mv "${to_move}" "${upload_dir}/${to_move##*/}"
-    done
-    # Generate dev-key-signed update payload for testing
-    delta_generator \
-      -private_key "/usr/share/update_engine/update-payload-key.key.pem" \
-      -new_image "${upload_dir}/${built_sysext_filename}" \
-      -out_file "${upload_dir}/flatcar_test_update-${oem_sysext}.gz"
-    # Remove sysext_dir if building sysext and installing it
-    # succeeded.
-    rm -rf "${built_sysext_dir}"
 
     # Mark the installed sysext as active.
     sudo touch "${VM_TMP_ROOT}${installed_sysext_oem_dir}/active-${oem_sysext}"

build_packages

@@ -118,6 +118,7 @@ fi
 . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1
 . "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1
+. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1
 
 # Setup all the emerge command/flags.
 EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select )
@@ -288,50 +289,55 @@ fi
 export KBUILD_BUILD_USER="${BUILD_USER:-build}"
 export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}"
 
+# Build sysext packages from an array of sysext definitions.
+# Usage: build_sysext_packages "description" "${SYSEXT_ARRAY[@]}"
+# Array format: "name|packages|useflags|arches"
+build_sysext_packages() {
+  local description="$1"
+  shift
+  local sysexts=("$@")
+
+  info "Merging ${description} packages now"
+  for sysext in "${sysexts[@]}"; do
+    local sysext_name package_atoms useflags arches
+    IFS="|" read -r sysext_name package_atoms useflags arches <<< "$sysext"
+    [[ -z ${arches} || ,${arches}, == *,"${ARCH}",* ]] || continue
+
+    info "Building packages for $sysext_name sysext with USE=$useflags"
+    IFS=,
+    for package in $package_atoms; do
+       # --buildpkgonly does not install dependencies, so we install them
+       # separately before building the binary package
+       sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
+         env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
+         "${EMERGE_FLAGS[@]}" \
+         --quiet \
+         --onlydeps \
+         --binpkg-respect-use=y \
+         "${package}"
+
+       sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
+         env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
+         "${EMERGE_FLAGS[@]}" \
+         --quiet \
+         --buildpkgonly \
+         --binpkg-respect-use=y \
+         "${package}"
+    done
+    unset IFS
+  done
+}
+
 info "Merging board packages now"
 sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@"
 
-info "Merging sysext packages now"
-for sysext in "${EXTRA_SYSEXTS[@]}"; do
-  IFS="|" read -r SYSEXT_NAME PACKAGE_ATOMS USEFLAGS ARCHES <<< "$sysext"
+build_sysext_packages "extra sysexts" "${EXTRA_SYSEXTS[@]}"
 
-  arch_array=("${ARCHES//,/ }")
-  if [[ -n $ARCHES ]]; then
-    should_skip=1
-    for arch in "${arch_array[@]}"; do
-      if [[ $arch == "$ARCH" ]]; then
-        should_skip=0
-      fi
-    done
-    if [[ $should_skip -eq 1 ]]; then
-      continue
-    fi
-  fi
-
-
-  info "Building packages for $SYSEXT_NAME sysext with USE=$USEFLAGS"
-  IFS=,
-  for package in $PACKAGE_ATOMS; do
-     # --buildpkgonly does not install dependencies, so we install them
-     # separately before building the binary package
-     sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
-       env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
-       "${EMERGE_FLAGS[@]}" \
-       --quiet \
-       --onlydeps \
-       --binpkg-respect-use=y \
-       "${package}"
-
-     sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \
-       env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \
-       "${EMERGE_FLAGS[@]}" \
-       --quiet \
-       --buildpkgonly \
-       --binpkg-respect-use=y \
-       "${package}"
-  done
-  unset IFS
-done
+declare -a oem_sysexts
+get_oem_sysext_matrix "${ARCH}" oem_sysexts
+if [[ ${#oem_sysexts[@]} -gt 0 ]]; then
+  build_sysext_packages "OEM sysexts" "${oem_sysexts[@]}"
+fi
 
 info "Removing obsolete packages"
 # The return value of emerge is not clearly reliable. It may fail with

build_sysext

@@ -216,7 +216,7 @@ if [[ ${#} -lt 1 ]]; then
    show_help_if_requested -h
 fi
 
-info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression".
+info "Building '${SYSEXTNAME}' sysext with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression".
 
 for package; do
   echo "Installing package into sysext image: $package"

ci-automation/image.sh

@@ -103,7 +103,7 @@ function _image_build_impl() {
                           --base_sysexts="${base_sysexts_param}" \
                           --output_root="${CONTAINER_IMAGE_ROOT}" \
                           --only_store_compressed \
-                          prodtar container sysext
+                          prodtar container sysext oem_sysext
 
     # copy resulting images + push to buildcache
     local images_out="images/"

ci-automation/image_changes.sh

@@ -149,7 +149,7 @@ function run_image_changes_job() {
     "${cb}"
 
     local -a oemids base_sysexts extra_sysexts
-    get_oem_id_list . "${arch}" oemids
+    get_oem_id_list "${arch}" oemids
     get_base_sysext_list . base_sysexts
     get_extra_sysext_list . "${arch}" extra_sysexts
     generate_image_changes_report \
@@ -241,38 +241,7 @@ function git_tag_for_nightly() {
     git_tag_ref=$(git -C "${scripts_repo}" describe --tags --abbrev=0 --match='*-nightly-*' --exclude='*-INTERMEDIATE' "${search_object}")
 }
 
-# Gets a list of OEMs that are using sysexts.
-#
-# 1 - scripts repo
-# 2 - arch
-# 3 - name of an array variable to store the result in
-function get_oem_id_list() {
-    local scripts_repo arch list_var_name
-    scripts_repo=${1}; shift
-    arch=${1}; shift
-    list_var_name=${1}; shift
-
-    local -a ebuilds=("${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild')
-    if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then
-        echo "No coreos-base/common-oem-files ebuilds?!" >&2
-        exit 1
-    fi
-
-    # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS,
-    # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last
-    # one. Also defines global-by-default EAPI, which we make local
-    # here to avoid making it global.
-    local EAPI
-    source "${ebuilds[0]}" flatcar-local-variables
-
-    local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS"
-    local all_oemids=(
-        "${COMMON_OEMIDS[@]}"
-        "${arch_oemids_ref[@]}"
-    )
-
-    mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort)
-}
+source build_library/oem_sysexts.sh
 
 function get_base_sysext_list() {
     local scripts_repo=${1}; shift

ci-automation/vms.sh

@@ -116,6 +116,17 @@ function _vm_build_impl() {
     for file in flatcar_production_image.bin.bz2 flatcar_production_image_sysext.squashfs flatcar_production_image.vmlinuz version.txt; do
         copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}"
     done
+
+    # Download prebuilt OEM sysexts
+    source build_library/oem_sysexts.sh
+    local -a oem_ids
+    get_oem_id_list "${arch}" oem_ids
+    local oem_id name
+    for oem_id in "${oem_ids[@]}"; do
+        name="oem-${oem_id}"
+        copy_from_buildcache "images/${arch}/${vernum}/${name}.raw" "${images_in}"
+    done
+
     lbunzip2 "${images_in}/flatcar_production_image.bin.bz2"
     ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \
             -v "${vernum}" \