diff --git a/build_image b/build_image index c446b7e658..aa0e57f158 100755 --- a/build_image +++ b/build_image @@ -60,10 +60,12 @@ different forms. This scripts can be used to build the following: prod - Production image for CoreOS. This image is for booting (default if no argument is given). prodtar - Production container tar ball (implies prod). This can e.g. be used to run the Flatcar production image as a container (run machinectl import-tar or docker import). container - Developer image with single filesystem, bootable by nspawn. +sysext - Build extra sysexts (podman, python, zfs, etc.). +oem_sysext - Build OEM sysexts for all supported platforms. Examples: -build_image --board= [prod] [prodtar] [container] - builds developer and production images/tars. +build_image --board= [prod] [prodtar] [container] [sysext] [oem_sysext] - builds developer and production images/tars. ... " show_help_if_requested "$@" @@ -81,7 +83,7 @@ DEFINE_string version "" \ # Parse command line. FLAGS "$@" || exit 1 -eval set -- "${FLAGS_ARGV:-prod}" +eval set -- "${FLAGS_ARGV:-prod oem_sysext}" # Only now can we die on error. shflags functions leak non-zero error codes, # so will die prematurely if 'switch_to_strict_mode' is specified before now. @@ -103,17 +105,20 @@ fi . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/vm_image_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 PROD_IMAGE=0 PROD_TAR=0 CONTAINER=0 SYSEXT=0 +OEM_SYSEXT=0 for arg in "$@"; do case "${arg}" in prod) PROD_IMAGE=1 ;; prodtar) PROD_IMAGE=1 PROD_TAR=1 ;; container) CONTAINER=1 ;; sysext) SYSEXT=1 ;; + oem_sysext) OEM_SYSEXT=1 ;; *) die_notrace "Unknown image type ${arg}" ;; esac done @@ -187,6 +192,9 @@ fi if [[ "${SYSEXT}" -eq 1 ]]; then create_prod_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}" fi +if [[ "${OEM_SYSEXT}" -eq 1 ]]; then + create_oem_sysexts "${FLATCAR_PRODUCTION_IMAGE_NAME}" +fi if [[ ${FLAGS_extract_update} -eq ${FLAGS_TRUE} ]]; then zip_update_tools diff --git a/build_library/oem_sysexts.sh b/build_library/oem_sysexts.sh new file mode 100644 index 0000000000..7d50efc4be --- /dev/null +++ b/build_library/oem_sysexts.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# OEM sysext helpers. + +# Auto-detect scripts repo root from this file's location. +# oem_sysexts.sh is at: /build_library/oem_sysexts.sh +_OEM_SYSEXTS_SCRIPTS_ROOT="$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/..")" + +get_oem_overlay_root() { + local overlay_root="/mnt/host/source/src/third_party/coreos-overlay" + + if [[ ! -d "${overlay_root}" ]]; then + overlay_root="${_OEM_SYSEXTS_SCRIPTS_ROOT}/sdk_container/src/third_party/coreos-overlay" + fi + + if [[ ! -d "${overlay_root}" ]]; then + echo "No coreos-overlay repo found (tried SDK and ${_OEM_SYSEXTS_SCRIPTS_ROOT})" >&2 + exit 1 + fi + + printf '%s' "${overlay_root}" +} + +_get_oem_ids() { + local arch list_var_name + arch=${1}; shift + list_var_name=${1}; shift + + local overlay_root + overlay_root=$(get_oem_overlay_root) + + local -a ebuilds=("${overlay_root}/coreos-base/common-oem-files/common-oem-files-"*'.ebuild') + if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then + echo "No coreos-base/common-oem-files ebuilds?!" >&2 + exit 1 + fi + + # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS, + # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last + # one. Also defines global-by-default EAPI, which we make local + # here to avoid making it global. + local EAPI + source "${ebuilds[0]}" flatcar-local-variables + + local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS" + local all_oemids=( + "${COMMON_OEMIDS[@]}" + "${arch_oemids_ref[@]}" + ) + + mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort) +} + +# Gets a list of OEMs that are using sysexts. +# +# 1 - arch +# 2 - name of an array variable to store the result in +get_oem_id_list() { + _get_oem_ids "$@" +} + +# Gets a list of OEM sysext descriptors. +# +# 1 - arch +# 2 - name of an array variable to store the result in +# +# Format: "name|metapackage|useflags" +get_oem_sysext_matrix() { + local arch list_var_name + arch=${1}; shift + list_var_name=${1}; shift + + local -a oem_ids + _get_oem_ids "${arch}" oem_ids + + local -a matrix=() + local oem_id + for oem_id in "${oem_ids[@]}"; do + matrix+=("oem-${oem_id}|coreos-base/oem-${oem_id}|${oem_id}") + done + + local -n matrix_ref="${list_var_name}" + matrix_ref=("${matrix[@]}") +} diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 7a0c34b50c..b6d760fba2 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -3,6 +3,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +source "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 + # Lookup the current version of a binary package, downloading it if needed. # Usage: get_binary_pkg some-pkg/name # Prints: some-pkg/name-1.2.3 @@ -273,6 +275,52 @@ create_prod_sysexts() { done } +create_oem_sysexts() { + local image_name="$1" + local image_sysext_base="${image_name%.bin}_sysext.squashfs" + local overlay_path + overlay_path=$(portageq get_repo_path / coreos-overlay) + + local -a oem_sysexts + get_oem_sysext_matrix "${ARCH}" oem_sysexts + + local sysext name metapkg useflags + for sysext in "${oem_sysexts[@]}"; do + IFS="|" read -r name metapkg useflags <<< "${sysext}" + + # Check for manglefs script in the package's files directory + local mangle_script="${overlay_path}/${metapkg}/files/manglefs.sh" + if [[ ! -x "${mangle_script}" ]]; then + mangle_script= + fi + + sudo rm -f "${BUILD_DIR}/${name}.raw" \ + "${BUILD_DIR}/flatcar_test_update-${name}.gz" \ + "${BUILD_DIR}/${name}_"* + + info "Building OEM sysext ${name} with USE=${useflags}" + # The --install_root_basename="${name}-oem-sysext-rootfs" flag is + # important - it sets the name of a rootfs directory, which is + # used to determine the package target in + # coreos/base/profile.bashrc + # + # OEM sysexts use no compression here since they will be stored + # in a compressed OEM partition. + USE="${useflags}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \ + --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ + --image_builddir="${BUILD_DIR}" \ + --metapkgs="${metapkg}" \ + --install_root_basename="${name}-oem-sysext-rootfs" \ + --compression=none \ + ${mangle_script:+--manglefs_script="${mangle_script}"} \ + "${name}" + delta_generator \ + -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ + -new_image "${BUILD_DIR}/${name}.raw" \ + -out_file "${BUILD_DIR}/flatcar_test_update-${name}.gz" + done +} + sbsign_prod_image() { local image_name="$1" local disk_layout="$2" diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 73eb43b812..ba71581219 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -568,7 +568,8 @@ install_oem_package() { sudo rm -rf "${oem_tmp}" } -# Write the OEM sysext file into the OEM partition. +# Install the prebuilt OEM sysext file into the OEM partition. +# The sysext should have been built by 'build_image oem_sysext'. install_oem_sysext() { local oem_sysext=$(_get_vm_opt OEM_SYSEXT) @@ -576,63 +577,24 @@ install_oem_sysext() { return 0 fi - local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext" - local built_sysext_filename="${oem_sysext}.raw" - local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}" + local prebuilt_sysext_filename="${oem_sysext}.raw" + local prebuilt_sysext_path="${FLAGS_from}/${prebuilt_sysext_filename}" local version="${FLATCAR_VERSION}" - local metapkg="coreos-base/${oem_sysext}" - # The --install_root_basename="${name}-oem-sysext-rootfs" flag is - # important - it sets the name of a rootfs directory, which is - # used to determine the package target in - # coreos/base/profile.bashrc - # - # OEM sysexts are stored in the compressed partition, so we disable - # compression to avoid double-compression. - local build_sysext_flags=( - --board="${BOARD}" - --squashfs_base="${VM_SRC_SYSEXT_IMG}" - --image_builddir="${built_sysext_dir}" - --metapkgs="${metapkg}" - --compression=none - --install_root_basename="${VM_IMG_TYPE}-oem-sysext-rootfs" - ) - local overlay_path mangle_fs - overlay_path=$(portageq get_repo_path / coreos-overlay) - mangle_fs="${overlay_path}/${metapkg}/files/manglefs.sh" - if [[ -x "${mangle_fs}" ]]; then - build_sysext_flags+=( - --manglefs_script="${mangle_fs}" - ) - fi - mkdir -p "${built_sysext_dir}" - sudo -E "${build_sysext_env[@]}" "${SCRIPT_ROOT}/build_sysext" "${build_sysext_flags[@]}" "${oem_sysext}" + if [[ ! -f "${prebuilt_sysext_path}" ]]; then + die "Prebuilt OEM sysext not found at ${prebuilt_sysext_path}. Run 'build_image oem_sysext' first." + fi local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${version}" local installed_sysext_filename="${installed_sysext_file_prefix}.raw" local installed_sysext_abspath="${installed_sysext_oem_dir}/${installed_sysext_filename}" - info "Installing ${oem_sysext} sysext" + + info "Installing ${oem_sysext} sysext from prebuilt image" sudo install -Dpm 0644 \ - "${built_sysext_path}" \ + "${prebuilt_sysext_path}" \ "${VM_TMP_ROOT}${installed_sysext_abspath}" || die "Could not install ${oem_sysext} sysext" - # Move sysext image and reports to a destination directory to - # upload them, thus making them available as separate artifacts to - # download. - local upload_dir to_move - upload_dir="$(_dst_dir)" - for to_move in "${built_sysext_dir}/${oem_sysext}"*; do - mv "${to_move}" "${upload_dir}/${to_move##*/}" - done - # Generate dev-key-signed update payload for testing - delta_generator \ - -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ - -new_image "${upload_dir}/${built_sysext_filename}" \ - -out_file "${upload_dir}/flatcar_test_update-${oem_sysext}.gz" - # Remove sysext_dir if building sysext and installing it - # succeeded. - rm -rf "${built_sysext_dir}" # Mark the installed sysext as active. sudo touch "${VM_TMP_ROOT}${installed_sysext_oem_dir}/active-${oem_sysext}" diff --git a/build_packages b/build_packages index d300edff82..da1d1e9b91 100755 --- a/build_packages +++ b/build_packages @@ -118,6 +118,7 @@ fi . "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/test_image_content.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/extra_sysexts.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 # Setup all the emerge command/flags. EMERGE_FLAGS=( --update --deep --newuse --verbose --backtrack=30 --select ) @@ -288,50 +289,55 @@ fi export KBUILD_BUILD_USER="${BUILD_USER:-build}" export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}" +# Build sysext packages from an array of sysext definitions. +# Usage: build_sysext_packages "description" "${SYSEXT_ARRAY[@]}" +# Array format: "name|packages|useflags|arches" +build_sysext_packages() { + local description="$1" + shift + local sysexts=("$@") + + info "Merging ${description} packages now" + for sysext in "${sysexts[@]}"; do + local sysext_name package_atoms useflags arches + IFS="|" read -r sysext_name package_atoms useflags arches <<< "$sysext" + [[ -z ${arches} || ,${arches}, == *,"${ARCH}",* ]] || continue + + info "Building packages for $sysext_name sysext with USE=$useflags" + IFS=, + for package in $package_atoms; do + # --buildpkgonly does not install dependencies, so we install them + # separately before building the binary package + sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ + env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ + "${EMERGE_FLAGS[@]}" \ + --quiet \ + --onlydeps \ + --binpkg-respect-use=y \ + "${package}" + + sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ + env USE="$useflags" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ + "${EMERGE_FLAGS[@]}" \ + --quiet \ + --buildpkgonly \ + --binpkg-respect-use=y \ + "${package}" + done + unset IFS + done +} + info "Merging board packages now" sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@" -info "Merging sysext packages now" -for sysext in "${EXTRA_SYSEXTS[@]}"; do - IFS="|" read -r SYSEXT_NAME PACKAGE_ATOMS USEFLAGS ARCHES <<< "$sysext" +build_sysext_packages "extra sysexts" "${EXTRA_SYSEXTS[@]}" - arch_array=("${ARCHES//,/ }") - if [[ -n $ARCHES ]]; then - should_skip=1 - for arch in "${arch_array[@]}"; do - if [[ $arch == "$ARCH" ]]; then - should_skip=0 - fi - done - if [[ $should_skip -eq 1 ]]; then - continue - fi - fi - - - info "Building packages for $SYSEXT_NAME sysext with USE=$USEFLAGS" - IFS=, - for package in $PACKAGE_ATOMS; do - # --buildpkgonly does not install dependencies, so we install them - # separately before building the binary package - sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ - env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ - "${EMERGE_FLAGS[@]}" \ - --quiet \ - --onlydeps \ - --binpkg-respect-use=y \ - "${package}" - - sudo --preserve-env=MODULES_SIGN_KEY,MODULES_SIGN_CERT \ - env USE="$USEFLAGS" FEATURES="-ebuild-locks binpkg-multi-instance" "${EMERGE_CMD[@]}" \ - "${EMERGE_FLAGS[@]}" \ - --quiet \ - --buildpkgonly \ - --binpkg-respect-use=y \ - "${package}" - done - unset IFS -done +declare -a oem_sysexts +get_oem_sysext_matrix "${ARCH}" oem_sysexts +if [[ ${#oem_sysexts[@]} -gt 0 ]]; then + build_sysext_packages "OEM sysexts" "${oem_sysexts[@]}" +fi info "Removing obsolete packages" # The return value of emerge is not clearly reliable. It may fail with diff --git a/build_sysext b/build_sysext index 0b7ea4d5f3..4a0d450e0b 100755 --- a/build_sysext +++ b/build_sysext @@ -216,7 +216,7 @@ if [[ ${#} -lt 1 ]]; then show_help_if_requested -h fi -info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression". +info "Building '${SYSEXTNAME}' sysext with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression". for package; do echo "Installing package into sysext image: $package" diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 09ca5e904f..552c382e04 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -103,7 +103,7 @@ function _image_build_impl() { --base_sysexts="${base_sysexts_param}" \ --output_root="${CONTAINER_IMAGE_ROOT}" \ --only_store_compressed \ - prodtar container sysext + prodtar container sysext oem_sysext # copy resulting images + push to buildcache local images_out="images/" diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh index 6de6c5cee3..9279cdf930 100644 --- a/ci-automation/image_changes.sh +++ b/ci-automation/image_changes.sh @@ -149,7 +149,7 @@ function run_image_changes_job() { "${cb}" local -a oemids base_sysexts extra_sysexts - get_oem_id_list . "${arch}" oemids + get_oem_id_list "${arch}" oemids get_base_sysext_list . base_sysexts get_extra_sysext_list . "${arch}" extra_sysexts generate_image_changes_report \ @@ -241,38 +241,7 @@ function git_tag_for_nightly() { git_tag_ref=$(git -C "${scripts_repo}" describe --tags --abbrev=0 --match='*-nightly-*' --exclude='*-INTERMEDIATE' "${search_object}") } -# Gets a list of OEMs that are using sysexts. -# -# 1 - scripts repo -# 2 - arch -# 3 - name of an array variable to store the result in -function get_oem_id_list() { - local scripts_repo arch list_var_name - scripts_repo=${1}; shift - arch=${1}; shift - list_var_name=${1}; shift - - local -a ebuilds=("${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild') - if [[ ${#ebuilds[@]} -eq 0 ]] || [[ ! -e ${ebuilds[0]} ]]; then - echo "No coreos-base/common-oem-files ebuilds?!" >&2 - exit 1 - fi - - # This defines local COMMON_OEMIDS, AMD64_ONLY_OEMIDS, - # ARM64_ONLY_OEMIDS and OEMIDS variable. We don't use the last - # one. Also defines global-by-default EAPI, which we make local - # here to avoid making it global. - local EAPI - source "${ebuilds[0]}" flatcar-local-variables - - local -n arch_oemids_ref="${arch^^}_ONLY_OEMIDS" - local all_oemids=( - "${COMMON_OEMIDS[@]}" - "${arch_oemids_ref[@]}" - ) - - mapfile -t "${list_var_name}" < <(printf '%s\n' "${all_oemids[@]}" | sort) -} +source build_library/oem_sysexts.sh function get_base_sysext_list() { local scripts_repo=${1}; shift diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index b882d19a04..26f98ee4e1 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -116,6 +116,17 @@ function _vm_build_impl() { for file in flatcar_production_image.bin.bz2 flatcar_production_image_sysext.squashfs flatcar_production_image.vmlinuz version.txt; do copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}" done + + # Download prebuilt OEM sysexts + source build_library/oem_sysexts.sh + local -a oem_ids + get_oem_id_list "${arch}" oem_ids + local oem_id name + for oem_id in "${oem_ids[@]}"; do + name="oem-${oem_id}" + copy_from_buildcache "images/${arch}/${vernum}/${name}.raw" "${images_in}" + done + lbunzip2 "${images_in}/flatcar_production_image.bin.bz2" ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \ -v "${vernum}" \