build_sysext: bugfixes, QoL changes, help updated

This change improves build_sysext by sourcing a missing lib dependency,
adding a number of comfort / quality-of-life options, and updating the
output of '--help' accordingly.

The OEM sysext finction in build_library/vm_image_util.sh is also
updated to use new command line format.

1. Include missing dependency toolchain_util.sh to fix an error in
   board_options.sh (get_board_arch undefined).
2. Use positional parameters for mandatory arguments.
   build_dir and sysext_name are mandatory and are now positional
   arguments instead of options.
   binary_package is the third positional argument but can be omitted
   if --metapkgs was specified.
3. --squashfs_base is now guessed better and will use the most recent
   build by default.
4. A new boolean flag --ignore_version_mismatch for the more daring
   developer was added. The flag will cause the script to continue if a
   version mismatch between SDK board packages and squashfs base is
   detected.
5. Error messages were improved for when mandatory parameters were not
   provided.
6. The '--help' message was improved and adjusted to the new parameters.

Signed-off-by: Thilo Fromm <thilofromm@microsoft.com>

build_library/vm_image_util.sh

@@ -540,7 +540,6 @@ install_oem_sysext() {
     local metapkg="coreos-base/${oem_sysext}"
     local build_sysext_flags=(
         --board="${BOARD}"
-        --build_dir="${built_sysext_dir}"
         --squashfs_base="${VM_SRC_SYSEXT_IMG}"
         --metapkgs="${metapkg}"
     )
@@ -554,7 +553,7 @@ install_oem_sysext() {
     fi
 
     mkdir -p "${built_sysext_dir}"
-    sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}"
+    sudo "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${built_sysext_dir}" "${oem_sysext}"
 
     local installed_sysext_oem_dir='/oem/sysext'
     local installed_sysext_file_prefix="${oem_sysext}-${version}"

build_sysext

@@ -14,42 +14,55 @@ SCRIPT_ROOT=$(dirname "$(readlink -f "$0")")
 assert_inside_chroot
 assert_root_user
 
+default_squashfs_base="$(readlink -f "${SCRIPT_ROOT}/../build/images/amd64-usr/latest/${FLATCAR_PRODUCTION_IMAGE_SYSEXT_BASE}")"
+
 # All these are used to set up the 'BUILD_DIR' variable
 DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build a sysext for."
-DEFINE_string build_dir '' \
-  "Directory used for building the sysext image, must exist, but should be empty."
 DEFINE_string metapkgs '' \
   "Comma-separated list of meta-packages to build from source and install into sysext image."
+DEFINE_string squashfs_base "${default_squashfs_base}" \
+  "The path to the squashfs base image. Defaults to the most current image built."
 DEFINE_string manglefs_script '' \
   "A path to executable that will customize the rootfs of the sysext image."
-DEFINE_string squashfs_base '' \
+DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \
-  "The path to the squashfs base image."
+  "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS."
 
-FLAGS_HELP="USAGE: build_sysext [flags] [sysext name] [binary packages to install into image].
+FLAGS_HELP="USAGE: build_sysext [flags] <build_dir> <sysext_name> <binary_package> [<binary_package> ...]
 
 This script is used to build a Flatcar sysext image.
+The sysext will be based on an OS image build's sysext base squashfs, i.e. it is specific to a Flatcar build or release.
+The base squashfs can either come from a local build or a downloaded from an official release.
 
 Examples:
 
-Builds a sysext image named interpreters in the images
+Builds a sysext image named 'interpreters' in the 'images'
-directory with dev-lang/python and dev-lang/perl packages for amd64:
+sub-directory with 'dev-lang/python' and 'dev-lang/perl' packages for the
+most recent amd64 production image:
 
-sudo build_sysext \
+sudo build_sysext \\
-     --board=amd64-usr \
+     --board=amd64-usr \\
-     --build_dir=images \
+     images \\
      interpreters dev-lang/python dev-lang/perl
 
-Builds a sysext image named oem-azure in the oem-images directory with
-metapackage coreos-base/oem-azure for arm64:
 
-sudo build_sysext \
+Builds a sysext image named 'oem-azure' in the 'oem-images' sub-directory with
-     --board=arm64-usr \
+metapackage 'coreos-base/oem-azure' for the arm64 squashfs base at
-     --build_dir=oem-images \
+'build/artifacts/flatcar_production_image_sysext.squashfs':
-     --metapkgs=coreos-base/oem-azure \
+
-     --mangle_fs=…/coreos-base/oem-azure/files/manglefs.sh \
+sudo build_sysext \\
+     --board=arm64-usr \\
+     --metapkgs=coreos-base/oem-azure \\
+     --mangle_fs=sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh \\
+     --squashfs_base=build/artifacts/flatcar_production_image_sysext.squashfs \\
+     oem-images \\
      oem-azure
 
+
+Mandatory command line parameters:
+  <build_dir>       - build directory to build the sysext in. Must exist but should be empty.
+  <sysext_name>     - name of the sysext output file.
+  <binary_package>  - List of existing binary packages to install. Can be omitted if --metapkgs was specified.
 "
 
 show_help_if_requested "$@"
@@ -59,14 +72,28 @@ FLAGS "$@" || exit 1
 
 eval set -- "${FLAGS_ARGV}"
 
-source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+# Validate command line parameters
-source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
 
-if [[ -z "${FLAGS_build_dir}" ]]; then
+build_dir="${1:-}"
-  die "Need a build directory to be specified with a --build_dir option"
+if [[ ! -d "${build_dir}" ]]; then
+  die "Build directory not provided or directory does not exist."
+fi
+BUILD_DIR=$(realpath "${build_dir}")
+shift
+
+SYSEXTNAME="${1:-}"
+if [[ -z "${SYSEXTNAME}" ]]; then
+  die "No sysext name provided."
+fi
+shift
+
+if [[ ! -f "${FLAGS_squashfs_base}" ]] ; then
+    die "Squashfs base '${FLAGS_squashfs_base}' not found."
 fi
 
-BUILD_DIR=$(realpath "${FLAGS_build_dir}")
+source "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1
+source "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1
+source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1
 
 # Architecture values are taken from systemd.unit(5).
 declare -A SYSEXT_ARCHES
@@ -85,7 +112,6 @@ _get_sysext_arch() {
     fi
 }
 
-
 set -euo pipefail
 
 cleanup() {
@@ -99,21 +125,10 @@ cleanup() {
   rm -rf "${dirs[@]}" || true
 }
 
-if [[ ${#} -lt 1 ]]; then
-   show_help_if_requested -h
-  die 'Expected at least one parameter for sysext image name'
-fi
-
-SYSEXTNAME="${1}"
-shift
-
 # Set up trap to execute cleanup() on script exit
 trap cleanup EXIT
 
 ARCH=$(_get_sysext_arch "${FLAGS_board}")
-if [[ -z "${FLAGS_squashfs_base}" ]]; then
-  FLAGS_squashfs_base="${BUILD_DIR}/flatcar_production_image_sysext.squashfs"
-fi
 cleanup
 
 mkdir "${BUILD_DIR}/fs-root"
@@ -123,10 +138,13 @@ mkdir "${BUILD_DIR}/workdir"
 mount -t overlay overlay -o lowerdir="${BUILD_DIR}/fs-root",upperdir="${BUILD_DIR}/install-root",workdir="${BUILD_DIR}/workdir" "${BUILD_DIR}/install-root"
 VERSION_BOARD=$(grep "^VERSION=" ${BUILD_DIR}/fs-root/usr/lib/os-release | cut -d = -f 2-)
 if [ "$VERSION_BOARD" != "$FLATCAR_VERSION" ]; then
-  echo "$VERSION_BOARD"
+  warn "Base squashfs version: $VERSION_BOARD"
-  echo "$FLATCAR_VERSION"
+  warn "SDK board packages version: $FLATCAR_VERSION"
-  echo "Version mismatch between board flatcar release and SDK container flatcar release"
+  if [[ "${FLAGS_ignore_version_mismatch}" = "${FLAGS_TRUE}" ]] ; then
-  exit 1
+    warn "Ignoring version mismatch as requested."
+  else
+    die "Version mismatch between board flatcar release and SDK container flatcar release."
+  fi
 fi
 
 if [[ -n "${FLAGS_metapkgs}" ]]; then
@@ -135,6 +153,11 @@ if [[ -n "${FLAGS_metapkgs}" ]]; then
   set -- "${metapkgs[@]}" "${@}"
 fi
 
+if [[ ${#} -lt 1 ]]; then
+   error 'No packages or meta packages to install.'
+   show_help_if_requested -h
+fi
+
 for package; do
   echo "Installing package into sysext image: $package"
   FEATURES="-ebuild-locks" emerge \