From 706356c459a913460bb8201754b5879f5a7ee154 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 14 Mar 2023 17:27:50 +0100 Subject: [PATCH 01/17] build_library/disk_util: Add btrfs UUID randomizing This is necessary if we want to mount a copy of the production image. --- build_library/disk_util | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/build_library/disk_util b/build_library/disk_util index f5fdd6d6fd..e47f0932a2 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -743,18 +743,29 @@ def Tune(options): config, partitions = LoadPartitionConfig(options) GetPartitionTableFromImage(options, config, partitions) part = GetPartition(partitions, options.partition) + action_done = False if not part['image_compat']: raise InvalidLayout("Disk layout is incompatible with existing image") if options.disable2fs_rw is not None: + action_done = True if part.get('fs_type', None) in ('ext2', 'ext4'): Tune2fsReadWrite(options, part, options.disable2fs_rw) elif part.get('fs_type', None) == 'btrfs': ReadWriteSubvol(options, part, options.disable2fs_rw) else: raise Exception("Partition %s is not a ext2 or ext4 or btrfs" % options.partition) - else: + + if options.randomize_uuid is not None: + action_done = True + if part.get('fs_type', None) == 'btrfs': + with PartitionLoop(options, part) as loop_dev: + Sudo(['btrfstune', '-m', loop_dev]) + else: + raise Exception("Partition %s is not btrfs" % options.partition) + + if not action_done: raise Exception("No options specified!") @@ -1059,6 +1070,8 @@ def main(argv): help='disable mounting ext2 filesystems read-write') a.add_argument('--enable2fs_rw', action='store_false', dest='disable2fs_rw', help='re-enable mounting ext2 filesystems read-write') + a.add_argument('--randomize_uuid', action='store_true', default=None, + help='randomize btrfs UUIDs in the partition') a.add_argument('disk_image', help='path to disk image file') a.add_argument('partition', help='number or label of partition to edit') a.set_defaults(func=Tune) From 5105ed5677ef41493c58a3eafa252cc181179c42 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 15 Mar 2023 16:13:46 +0100 Subject: [PATCH 02/17] build_library: Move report function to a separate file I'd like to use them in other places, and I don't need all the other stuff build_image_util.sh provides. --- build_library/build_image_util.sh | 61 +-------------- build_library/reports_util.sh | 119 ++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 59 deletions(-) create mode 100644 build_library/reports_util.sh diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 214c0f46fc..25ac63be6c 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -19,6 +19,8 @@ fi BUILD_DIR="${FLAGS_output_root}/${BOARD}/${IMAGE_SUBDIR}" OUTSIDE_OUTPUT_DIR="../build/images/${BOARD}/${IMAGE_SUBDIR}" +source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 + set_build_symlinks() { local build=$(basename ${BUILD_DIR}) local link @@ -239,65 +241,6 @@ systemd_enable() { sudo ln -sf "../${unit_file}" "${wants_dir}/${unit_alias}" } -# Generate a ls-like listing of a directory tree. -# The ugly printf is used to predictable time format and size in bytes. -write_contents() { - info "Writing ${2##*/}" - pushd "$1" >/dev/null - # %M - file permissions - # %n - number of hard links to file - # %u - file's user name - # %g - file's group name - # %s - size in bytes - # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes) - # %P - file's path - # %l - symlink target (empty if not a symlink) - sudo TZ=UTC find -printf \ - '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ - | sed -e 's/ -> $//' > "$2" - popd >/dev/null -} - -# Generate a listing that can be used by other tools to analyze -# image/file size changes. -write_contents_with_technical_details() { - info "Writing ${2##*/}" - pushd "$1" >/dev/null - # %M - file permissions - # %D - ID of a device where file resides - # %i - inode number - # %n - number of hard links to file - # %s - size in bytes - # %P - file's path - sudo find -printf \ - '%M %D %i %n %s ./%P\n' > "$2" - popd >/dev/null -} - -# Generate a report like the following: -# -# File Size Used Avail Use% Type -# /boot 127M 62M 65M 50% vfat -# /usr 983M 721M 212M 78% ext2 -# / 6,0G 13M 5,6G 1% ext4 -# SUM 7,0G 796M 5,9G 12% - -write_disk_space_usage() { - info "Writing ${2##*/}" - pushd "${1}" >/dev/null - # The sed's first command turns './' into '/ ', second - # command replaces '- ' with 'SUM' for the total row. All this to - # keep the numbers neatly aligned in columns. - sudo df \ - --human-readable \ - --total \ - --output='file,size,used,avail,pcent,fstype' \ - ./boot ./usr ./ | \ - sed \ - -e 's#^\.\(/[^ ]*\)#\1 #' \ - -e 's/^- /SUM/' >"${2}" - popd >/dev/null -} - # "equery list" a potentially uninstalled board package query_available_package() { local pkg="$1" diff --git a/build_library/reports_util.sh b/build_library/reports_util.sh new file mode 100644 index 0000000000..0873d9e960 --- /dev/null +++ b/build_library/reports_util.sh @@ -0,0 +1,119 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +if [[ -n "${FLATCAR_REPORTS_UTIL_SH_INCLUDED:-}" ]]; then + return 0 +fi + +FLATCAR_REPORTS_UTIL_SH_INCLUDED=1 + +# Generate a ls-like listing of a directory tree. +# The ugly printf is used to predictable time format and size in bytes. +# +# Usage: +# write_contents "${rootfs}" ${contents_file}" +write_contents() { + local rootfs="${1}"; shift + local output="${1}"; shift + info "Writing ${output##*/}" + # Ensure output is an absolute path before we change the working + # directory. + output=$(realpath "${output}") + pushd "${rootfs}" >/dev/null + # %M - file permissions + # %n - number of hard links to file + # %u - file's user name + # %g - file's group name + # %s - size in bytes + # %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes) + # %P - file's path + # %l - symlink target (empty if not a symlink) + sudo TZ=UTC find -printf \ + '%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \ + | sed -e 's/ -> $//' >"${output}" + popd >/dev/null +} + +# Generate a listing that can be used by other tools to analyze +# image/file size changes. +# +# Usage: +# write_contents_with_technical_details "${rootfs}" ${output_file}" +write_contents_with_technical_details() { + local rootfs="${1}"; shift + local output="${1}"; shift + info "Writing ${output##*/}" + # Ensure output is an absolute path before we change the working + # directory. + output=$(realpath "${output}") + pushd "${rootfs}" >/dev/null + # %M - file permissions + # %D - ID of a device where file resides + # %i - inode number + # %n - number of hard links to file + # %s - size in bytes + # %P - file's path + sudo find -printf \ + '%M %D %i %n %s ./%P\n' >"${output}" + popd >/dev/null +} + +# Generate a report like the following if more than one relative path +# in rootfs was passed: +# +# File Size Used Avail Use% Type +# /boot 127M 62M 65M 50% vfat +# /usr 983M 721M 212M 78% ext2 +# / 6,0G 13M 5,6G 1% ext4 +# SUM 7,0G 796M 5,9G 12% - +# +# or, in case of 0 or 1 relative path: +# +# File Size Used Avail Use% Type +# / 27M 27M 0 100% squashfs +# +# Usage: +# write_disk_space_usage_in_paths "${rootfs}" "${output_file}" ./boot ./usr ./ +write_disk_space_usage_in_paths() { + local rootfs="${1}"; shift + local output="${1}"; shift + info "Writing ${output##*/}" + # Ensure output is an absolute path before we change the working + # directory. + output=$(realpath "${output}") + pushd "${rootfs}" >/dev/null + local extra_flags + extra_flags=() + if [[ ${#} -eq 0 ]]; then + set -- ./ + fi + if [[ ${#} -gt 1 ]]; then + extra_flags+=('--total') + fi + # The sed's first command turns './' into '/ ', second + # command replaces '- ' with 'SUM' for the total row. All this to + # keep the numbers neatly aligned in columns. + sudo df \ + --human-readable \ + "${extra_flags[@]}" \ + --output='file,size,used,avail,pcent,fstype' \ + "${@}" | \ + sed \ + -e 's#^\.\(/[^ ]*\)#\1 #' \ + -e 's/^- /SUM/' >"${output}" + popd >/dev/null +} + +# Generate a report like the following: +# +# File Size Used Avail Use% Type +# /boot 127M 62M 65M 50% vfat +# /usr 983M 721M 212M 78% ext2 +# / 6,0G 13M 5,6G 1% ext4 +# SUM 7,0G 796M 5,9G 12% - +write_disk_space_usage() { + write_disk_space_usage_in_paths "${1}" "${2}" ./boot ./usr ./ +} From 86d7eb589474392263fab26cc118da0290944933 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 23 Feb 2023 14:02:10 +0100 Subject: [PATCH 03/17] *: Add OEM sysext building utilities --- build_library/oem_sysext_util.sh | 201 +++++++++++++++++++++++++++++++ build_oem_sysext | 77 ++++++++++++ 2 files changed, 278 insertions(+) create mode 100755 build_library/oem_sysext_util.sh create mode 100755 build_oem_sysext diff --git a/build_library/oem_sysext_util.sh b/build_library/oem_sysext_util.sh new file mode 100755 index 0000000000..e83effa01d --- /dev/null +++ b/build_library/oem_sysext_util.sh @@ -0,0 +1,201 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +source "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 + +_generate_listing() { + local rootfs="${1%/}"; shift + local listing="${1}"; shift + + local slashes="${rootfs//[^\/]}" + local slash_count="${#slashes}" + + # Invoking find with sudo as it's used for traversing root-owned + # rootfs, which means that some places may be unreachable by the + # sdk user. + sudo find "${rootfs}//" | cut -d/ -f$((slash_count + 2))- | sort >"${listing}" +} + +_prepend_action () { + local -n prepend_array="${1}"; shift + + prepend_array=( "${#}" "${@}" "${prepend_array[@]}" ) +} + +_invoke_actions () { + local arg_count + local command + while [[ "${#}" -gt 0 ]]; do + arg_count="${1}" + shift + command=( "${@:1:${arg_count}}" ) + shift "${arg_count}" + "${command[@]}" || : + done +} + +# Architecture values are taken from systemd.unit(5). +declare -A SYSEXT_ARCHES +SYSEXT_ARCHES['amd64-usr']='x86-64' +SYSEXT_ARCHES['arm64-usr']='arm64' + +declare -r SYSEXT_ARCHES + +# Usage: _get_sysext_arch board [board...] +_get_sysext_arch() { + local board + for board in "$@"; do + if [[ ${#SYSEXT_ARCHES["${board}"]} -ne 0 ]]; then + echo "${SYSEXT_ARCHES["${board}"]}" + else + die "Unknown board '${board}'" + fi + done +} + +oem_sysext_create() { + local oem="${1}"; shift + local board="${1}"; shift + local version_id="${1}"; shift + local prod_image="${1}"; shift + local prod_pkgdb="${1}"; shift + local work_dir="${1}"; shift + + local base_pkg="coreos-base/${oem}" + local sysext_work_dir="${work_dir}/sysext-${oem}" + local prod_rw_image="${sysext_work_dir}/prod_for_sysext.bin" + local prod_rw_rootfs="${sysext_work_dir}/prod_rw_rootfs" + + local cleanup_actions=() + trap '_invoke_actions "${cleanup_actions[@]}"' EXIT + + _prepend_action cleanup_actions rmdir "${sysext_work_dir}" + mkdir -p "${sysext_work_dir}" + + info 'Creating a production image copy for work rootfs' + _prepend_action cleanup_actions rm -f "${prod_rw_image}" + cp --sparse=always "${prod_image}" "${prod_rw_image}" + + info 'Preparing work image for mounting' + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + tune --randomize_uuid "${prod_rw_image}" OEM + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + tune --enable2fs_rw "${prod_rw_image}" USR-A + + info "Mounting work image to ${prod_rw_rootfs}" + _prepend_action cleanup_actions rmdir "${prod_rw_rootfs}" + _prepend_action cleanup_actions "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + umount "${prod_rw_rootfs}" + "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ + mount --writable_verity "${prod_rw_image}" "${prod_rw_rootfs}" + + local initial_files="${sysext_work_dir}/initial_files" + info "Generating list of initial files in work image" + _prepend_action cleanup_actions rm -f "${initial_files}" + _generate_listing "${prod_rw_rootfs}" "${initial_files}" + + info "Stuffing package database into into ${prod_rw_rootfs}" + sudo tar -xf "${prod_pkgdb}" -C "${prod_rw_rootfs}" + + # Split into two steps because we want to always install + # $${base_pkg} from the ebuild (build_packages doesn't handle it) + # *but* we never want to build anything else from source + # here. emerge doesn't have a way to enforce this in a single + # command. + info "Building ${base_pkg}" + "emerge-${board}" --nodeps --buildpkgonly --usepkg n --verbose "${base_pkg}" + + info "Installing ${base_pkg} to ${prod_rw_rootfs}" + sudo emerge \ + --config-root="/build/${board}" \ + --root="${prod_rw_rootfs}" \ + --sysroot="${prod_rw_rootfs}" \ + --root-deps=rdeps \ + --usepkgonly \ + --verbose \ + "${base_pkg}" + + info "Removing portage db from ${prod_rw_rootfs}" + sudo rm -rf \ + "${prod_rw_rootfs}/var/cache/edb" \ + "${prod_rw_rootfs}/var/db/pkg" + + local all_files="${sysext_work_dir}/all_files" + local sysext_files="${sysext_work_dir}/sysext_files" + + info "Generating list of files in work image after installing OEM package" + _prepend_action cleanup_actions rm -f "${all_files}" + _generate_listing "${prod_rw_rootfs}" "${all_files}" + + info "Generating list of files for sysext image" + _prepend_action cleanup_actions rm -f "${sysext_files}" + comm -1 -3 "${initial_files}" "${all_files}" >"${sysext_files}" + + info "Copying files for sysext image" + local sysext_rootfs="${sysext_work_dir}/sysext_rootfs" + _prepend_action cleanup_actions rm -rf "${sysext_rootfs}" + rsync --links --files-from="${sysext_files}" "${prod_rw_rootfs}" "${sysext_rootfs}" + + info "Mangling files for sysext image" + local overlay_path mangle_fs + overlay_path=$(portageq get_repo_path / coreos) + mangle_fs="${overlay_path}/${base_pkg}/files/manglefs.sh" + if [[ -x "${mangle_fs}" ]]; then + "${mangle_fs}" "${sysext_rootfs}" + fi + + local entry + info "Removing non-/usr directories from sysext image" + for entry in "${sysext_rootfs}"/*; do + if [[ "${entry}" = */usr ]]; then + continue + fi + info " Removing ${entry##*/}" + rm -rf "${entry}" + done + + local metadata metadata_file + info "Adding sysext metadata" + mkdir -p "${sysext_rootfs}/usr/lib/extension-release.d" + metadata=( + 'ID=flatcar' + "VERSION_ID=${version_id}" + "ARCHITECTURE=$(_get_sysext_arch "${board}")" + ) + metadata_file="${sysext_rootfs}/usr/lib/extension-release.d/extension-release.${oem}" + printf '%s\n' "${metadata[@]}" >"${metadata_file}" + + info "Generating a squashfs image" + local sysext_raw_image_filename="${oem}.raw" + local output_raw_image="${sysext_work_dir}/${sysext_raw_image_filename}" + _prepend_action cleanup_actions rm -f "${output_raw_image}" + mksquashfs "${sysext_rootfs}" "${output_raw_image}" -all-root + + info "Generating image reports" + local sysext_mounted="${sysext_work_dir}/squashfs_mounted" + _prepend_action cleanup_actions rmdir "${sysext_mounted}" + mkdir "${sysext_mounted}" + _prepend_action cleanup_actions sudo umount "${sysext_mounted}" + sudo mount -t squashfs -o loop "${output_raw_image}" "${sysext_mounted}" + local contents="${sysext_raw_image_filename%.raw}_contents.txt" + local contents_wtd="${sysext_raw_image_filename%.raw}_contents_wtd.txt" + local disk_usage="${sysext_raw_image_filename%.raw}_disk_usage.txt" + _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents}" + write_contents "${sysext_mounted}" "${sysext_work_dir}/${contents}" + _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${contents_wtd}" + write_contents_with_technical_details "${sysext_mounted}" "${sysext_work_dir}/${contents_wtd}" + _prepend_action cleanup_actions rm -f "${sysext_work_dir}/${disk_usage}" + write_disk_space_usage_in_paths "${sysext_mounted}" "${sysext_work_dir}/${disk_usage}" + + local to_move + for to_move in "${sysext_raw_image_filename}" "${contents}" "${contents_wtd}" "${disk_usage}"; do + mv "${sysext_work_dir}/${to_move}" "${work_dir}/${to_move}" + done + + info "Alles jut, cleaning up" + trap - EXIT + _invoke_actions "${cleanup_actions[@]}" +} diff --git a/build_oem_sysext b/build_oem_sysext new file mode 100755 index 0000000000..ece37ebd8f --- /dev/null +++ b/build_oem_sysext @@ -0,0 +1,77 @@ +#!/bin/bash +# +# Copyright (c) 2023 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +SCRIPT_ROOT=$(dirname $(readlink -f "$0")) +. "${SCRIPT_ROOT}/common.sh" || exit 1 + +# Script must run inside the chroot +assert_inside_chroot + +assert_not_root_user + +# Developer-visible flags. +DEFINE_string board "${DEFAULT_BOARD}" \ + "The board to build an image for." +DEFINE_string build_dir "" \ + "Directory in which to place image result directories (named by version)" +DEFINE_string prod_image_path "" \ + "Path to the generic production image" +DEFINE_string prod_pkgdb_path "" \ + "Path to the tarball with portage package database from generic image production image" +DEFINE_string version_id "${FLATCAR_VERSION_ID}" \ + "Version ID stored inside the sysext extension" + +FLAGS_HELP="USAGE: build_oem_sysext [flags] [oem name]. +This script is used to build a Flatcar OEM sysext images. +The built image is in /oem-.raw. + +Examples: + +build_oem_sysext \ + --board=amd64-usr \ + --build_dir= \ + --prod_image_path= \ + --prod_pkgdb_path= \ + --version_id=\"\${FLATCAR_VERSION_ID}\" \ + oem-azure +... +" +show_help_if_requested "$@" + +# Parse command line. +FLAGS "$@" || exit 1 +if [[ -z "${FLAGS_ARGV}" ]]; then + echo 'No OEM given' + exit 0 +fi + +eval set -- "${FLAGS_ARGV}" + +# Only now can we die on error. shflags functions leak non-zero error codes, +# so will die prematurely if 'switch_to_strict_mode' is specified before now. +switch_to_strict_mode + +# N.B. Ordering matters for some of the libraries below, because +# some of the files contain initialization used by later files. +. "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/board_options.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/oem_sysext_util.sh" || exit 1 + +BUILD_DIR=${FLAGS_build_dir:-"${BUILD_DIR}"} + +if [[ -z "${FLAGS_prod_image_path}" ]]; then + error "--prod_image_path is required." + exit 1 +fi + +if [[ -z "${FLAGS_prod_pkgdb_path}" ]]; then + error "--prod_pkgdb_path is required." + exit 1 +fi + +for oem; do + oem_sysext_create "${oem}" "${BOARD}" "${FLAGS_version_id}" "${FLAGS_prod_image_path}" "${FLAGS_prod_pkgdb_path}" "${BUILD_DIR}" +done From a5ecf0d79f1841e17ad511e280ad42ba9450d391 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 23 Feb 2023 14:03:21 +0100 Subject: [PATCH 04/17] build_library/prod_image_util.sh: Generate a tarball with portage database Will come in handy when generating OEM sysexts. We can mount the generic image, put the image database back into the image and emerge extra packages without the need to drop all DEPENDS and BDEPENDS from the ebuilds. --- build_library/prod_image_util.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index bc2a39e182..8145712705 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -77,6 +77,7 @@ create_prod_image() { local image_initrd_contents="${image_name%.bin}_initrd_contents.txt" local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt" local image_disk_usage="${image_name%.bin}_disk_usage.txt" + local image_pkgdb="${image_name%.bin}_pkgdb.tar.xz" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -100,6 +101,8 @@ create_prod_image() { || die_notrace "coreos-au-key is missing the 'official' use flag" fi + tar -cf "${BUILD_DIR}/${image_pkgdb}" -C "${root_fs_dir}" var/cache/edb var/db/pkg + # clean-ups of things we do not need sudo rm ${root_fs_dir}/etc/csh.env sudo rm -rf ${root_fs_dir}/etc/env.d From df6e2aa5058766b8b7542ff45aab2af8d86347a7 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 23 Feb 2023 14:05:45 +0100 Subject: [PATCH 05/17] ci-automation: Download portage database from bincache for vms --- ci-automation/vms.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 8dcd831e9a..a27cba0175 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -103,9 +103,11 @@ function _vm_build_impl() { formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g') local images_in="images-in/" + local file rm -rf "${images_in}" - copy_from_buildcache "images/${arch}/${vernum}/flatcar_production_image.bin.bz2" "${images_in}" - copy_from_buildcache "images/${arch}/${vernum}/version.txt" "${images_in}" + for file in flatcar_production_image.bin.bz2 flatcar_production_image_pkgdb.tar.xz version.txt; do + copy_from_buildcache "images/${arch}/${vernum}/${file}" "${images_in}" + done lbunzip2 "${images_in}/flatcar_production_image.bin.bz2" ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \ -v "${vernum}" \ From 46b98ba0d25d95256d3394bdaa52471bffa671ed Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 23 Feb 2023 14:07:43 +0100 Subject: [PATCH 06/17] image_to_vm: Require pkgdb tarball and handle sysext building --- build_library/vm_image_util.sh | 64 +++++++++++++++++++++++++++++++--- common.sh | 1 + image_to_vm.sh | 3 +- 3 files changed, 63 insertions(+), 5 deletions(-) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 9738565aa5..4cd7f455a7 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -71,6 +71,7 @@ VM_IMG_TYPE=DEFAULT # Set at runtime to the source and destination image paths VM_SRC_IMG= +VM_SRC_PKGDB= VM_TMP_IMG= VM_TMP_DIR= VM_TMP_ROOT= @@ -98,6 +99,12 @@ IMG_DEFAULT_OEM_USE= # Forced USE flags for the OEM package IMG_FORCE_OEM_USE= +# If set install the given package name to the OEM sysext image +IMG_DEFAULT_OEM_SYSEXT= + +# Forced OEM package name overriding what may be in the format +IMG_FORCE_OEM_SYSEXT= + # Hook to do any final tweaks or grab data while fs is mounted. IMG_DEFAULT_FS_HOOK= @@ -345,13 +352,18 @@ set_vm_oem_pkg() { # Validate and set source vm image path set_vm_paths() { - local src_dir="$1" - local dst_dir="$2" - local src_name="$3" + local src_dir="${1}"; shift + local dst_dir="${1}"; shift + local src_name="${1}"; shift + local pkgdb_name="${1}"; shift VM_SRC_IMG="${src_dir}/${src_name}" if [[ ! -f "${VM_SRC_IMG}" ]]; then - die "Source image does not exist: $VM_SRC_IMG" + die "Source image does not exist: ${VM_SRC_IMG}" + fi + VM_SRC_PKGDB="${src_dir}/${pkgdb_name}" + if [[ ! -f "${VM_SRC_PKGDB}" ]]; then + die "Source package database does not exist: ${VM_SRC_PKGDB}" fi local dst_name="$(_src_to_dst_name "${src_name}" "_image.$(_disk_ext)")" @@ -517,6 +529,50 @@ install_oem_aci() { rm -rf "${aci_dir}" } +# Write the OEM sysext file into the OEM partition. +install_oem_sysext() { + local oem_sysext=$(_get_vm_opt OEM_SYSEXT) + + if [[ -z "${oem_sysext}" ]]; then + return 0 + fi + + local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext" + local built_sysext_filename="${oem_sysext}.raw" + local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}" + + "${SCRIPT_ROOT}/build_oem_sysext" \ + --board="${BOARD}" \ + --build_dir="${built_sysext_dir}" \ + --prod_image_path="${VM_SRC_IMG}" \ + --prod_pkgdb_path="${VM_SRC_PKGDB}" \ + "${oem_sysext}" + + local installed_sysext_oem_dir='/oem/sysext' + local installed_sysext_file_prefix="${oem_sysext}-${FLATCAR_VERSION}" + local installed_sysext_filename="${installed_sysext_file_prefix}.raw" + local installed_sysext_abspath="${installed_sysext_oem_dir}/${installed_sysext_filename}" + info "Installing ${oem_sysext} sysext" + sudo install -Dpm 0644 \ + "${built_sysext_path}" \ + "${VM_TMP_ROOT}${installed_sysext_abspath}" || + die "Could not install ${oem_sysext} sysext" + # Move sysext image and reports to a destination directory to + # upload them, thus making them available as separate artifacts to + # download. + local upload_dir to_move + upload_dir="$(_dst_dir)" + for to_move in "${built_sysext_dir}/${oem_sysext}"*; do + mv "${to_move}" "${upload_dir}/${to_move##*/}" + done + # Remove sysext_dir if building sysext and installing it + # succeeded. + rm -rf "${built_sysext_dir}" + + # Mark the installed sysext as active. + sudo touch "${VM_TMP_ROOT}${installed_sysext_oem_dir}/active-${oem_sysext}" +} + # Any other tweaks required? run_fs_hook() { local fs_hook=$(_get_vm_opt FS_HOOK) diff --git a/common.sh b/common.sh index c479295794..eafbc928c1 100644 --- a/common.sh +++ b/common.sh @@ -425,6 +425,7 @@ BUILD_DIR= # Standard filenames FLATCAR_DEVELOPER_CONTAINER_NAME="flatcar_developer_container.bin" FLATCAR_PRODUCTION_IMAGE_NAME="flatcar_production_image.bin" +FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME="flatcar_production_image_pkgdb.tar.xz" # ----------------------------------------------------------------------------- # Functions diff --git a/image_to_vm.sh b/image_to_vm.sh index c32d9d1d43..525490e313 100755 --- a/image_to_vm.sh +++ b/image_to_vm.sh @@ -105,7 +105,7 @@ if [ -f "${FLAGS_from}/version.txt" ]; then FLATCAR_VERSION_STRING="${FLATCAR_VERSION}" fi -set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" +set_vm_paths "${FLAGS_from}" "${FLAGS_to}" "${FLATCAR_PRODUCTION_IMAGE_NAME}" "${FLATCAR_PRODUCTION_IMAGE_PKGDB_NAME}" # Make sure things are cleaned up on failure trap vm_cleanup EXIT @@ -118,6 +118,7 @@ setup_disk_image "${FLAGS_disk_layout}" # Optionally install any OEM packages install_oem_package install_oem_aci +install_oem_sysext run_fs_hook # Changes done, glue it together From 5ef38b98aa1eca0df2467ae026cccca9530b6f46 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 22 Mar 2023 12:00:02 +0100 Subject: [PATCH 07/17] coreos-base/coreos-init: Pull in updated azure udev rules --- ...eos-init-0.0.1-r180.ebuild => coreos-init-0.0.1-r181.ebuild} | 0 .../coreos-base/coreos-init/coreos-init-9999.ebuild | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/{coreos-init-0.0.1-r180.ebuild => coreos-init-0.0.1-r181.ebuild} (100%) diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r180.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r181.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r180.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-0.0.1-r181.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild index 8fe3b1899d..41c47517fa 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos-init/coreos-init-9999.ebuild @@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com" if [[ "${PV}" == 9999 ]]; then KEYWORDS="~amd64 ~arm ~arm64 ~x86" else - CROS_WORKON_COMMIT="93b80ace22806bae4ab521f16fa9f4d1d3172e77" # flatcar-master + CROS_WORKON_COMMIT="658eb0ea0fb8e89f8aa56ccf57867eb88b53fc27" # flatcar-master KEYWORDS="amd64 arm arm64 x86" fi From 86eada1cc97827b10bfc2799433e5f466e88b3b8 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 13:03:19 +0200 Subject: [PATCH 08/17] overlay app-emulation/wa-linux-agent: Redo the package for sysext Since the contents of this package will be now a part of the sysext image, we don't need any special OEM-specific hacks. We don't need to install the package in /usr/share/oem directory any more, so update the ebuild to use the Gentoo python machinery to install files in the usual locations. This can also use a normal python package, so replace dependencies on dev-lang/python-oem and dev-python/distro-oem with dev-lang/python and dev-python/distro, respectively. The waagent.conf file we used to provide is updated (to disable auto updates, for example, and dropped obsolete options) and now is a part of the patch, so it is installed by the python machinery. --- .../files/0001-flatcar-changes.patch | 418 ++++++++++++++++++ .../wa-linux-agent/files/waagent.conf | 127 ------ .../wa-linux-agent-2.6.0.2-r2.ebuild | 36 -- .../wa-linux-agent-2.6.0.2-r3.ebuild | 33 ++ 4 files changed, 451 insertions(+), 163 deletions(-) create mode 100644 sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf delete mode 100644 sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild create mode 100644 sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch new file mode 100644 index 0000000000..6953cdea85 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/0001-flatcar-changes.patch @@ -0,0 +1,418 @@ +From 90b28746c0d8698a080eb7082e0e14054aee0a02 Mon Sep 17 00:00:00 2001 +From: Krzesimir Nowak +Date: Mon, 27 Feb 2023 15:59:21 +0100 +Subject: [PATCH] flatcar changes + +--- + azurelinuxagent/common/osutil/coreos.py | 39 +----- + azurelinuxagent/common/osutil/coreoscommon.py | 57 ++++++++ + azurelinuxagent/common/osutil/factory.py | 3 + + azurelinuxagent/common/osutil/flatcar.py | 41 ++++++ + config/flatcar/waagent.conf | 122 ++++++++++++++++++ + init/flatcar/10-waagent-sysext.conf | 2 + + init/flatcar/waagent.service | 30 +++++ + setup.py | 20 ++- + 8 files changed, 272 insertions(+), 42 deletions(-) + create mode 100644 azurelinuxagent/common/osutil/coreoscommon.py + create mode 100644 azurelinuxagent/common/osutil/flatcar.py + create mode 100644 config/flatcar/waagent.conf + create mode 100644 init/flatcar/10-waagent-sysext.conf + create mode 100644 init/flatcar/waagent.service + +diff --git a/azurelinuxagent/common/osutil/coreos.py b/azurelinuxagent/common/osutil/coreos.py +index fc0a6604..314008f0 100644 +--- a/azurelinuxagent/common/osutil/coreos.py ++++ b/azurelinuxagent/common/osutil/coreos.py +@@ -17,11 +17,10 @@ + # + + import os +-import azurelinuxagent.common.utils.shellutil as shellutil +-from azurelinuxagent.common.osutil.default import DefaultOSUtil ++from azurelinuxagent.common.osutil.coreoscommon import CoreosCommonUtil + + +-class CoreOSUtil(DefaultOSUtil): ++class CoreOSUtil(CoreosCommonUtil): + + def __init__(self): + super(CoreOSUtil, self).__init__() +@@ -46,40 +45,6 @@ class CoreOSUtil(DefaultOSUtil): + def get_agent_bin_path(): + return "/usr/share/oem/bin" + +- def is_sys_user(self, username): +- # User 'core' is not a sysuser. +- if username == 'core': +- return False +- return super(CoreOSUtil, self).is_sys_user(username) +- +- def is_dhcp_enabled(self): +- return True +- +- def start_network(self): +- return shellutil.run("systemctl start systemd-networkd", chk_err=False) +- +- def restart_if(self, ifname=None, retries=None, wait=None): +- shellutil.run("systemctl restart systemd-networkd") +- +- def restart_ssh_service(self): +- # SSH is socket activated on CoreOS. No need to restart it. +- pass +- +- def stop_dhcp_service(self): +- return shellutil.run("systemctl stop systemd-networkd", chk_err=False) +- +- def start_dhcp_service(self): +- return shellutil.run("systemctl start systemd-networkd", chk_err=False) +- +- def start_agent_service(self): +- return shellutil.run("systemctl start {0}".format(self.service_name), chk_err=False) +- +- def stop_agent_service(self): +- return shellutil.run("systemctl stop {0}".format(self.service_name), chk_err=False) +- +- def get_dhcp_pid(self): +- return self._get_dhcp_pid(["systemctl", "show", "-p", "MainPID", "systemd-networkd"]) +- + def conf_sshd(self, disable_password): + # In CoreOS, /etc/sshd_config is mount readonly. Skip the setting. + pass +diff --git a/azurelinuxagent/common/osutil/coreoscommon.py b/azurelinuxagent/common/osutil/coreoscommon.py +new file mode 100644 +index 00000000..fde9a456 +--- /dev/null ++++ b/azurelinuxagent/common/osutil/coreoscommon.py +@@ -0,0 +1,57 @@ ++# ++# Copyright 2023 Microsoft Corporation ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# ++# Requires Python 2.6+ and Openssl 1.0+ ++# ++ ++import azurelinuxagent.common.utils.shellutil as shellutil ++from azurelinuxagent.common.osutil.default import DefaultOSUtil ++ ++ ++class CoreosCommonUtil(DefaultOSUtil): ++ ++ def is_sys_user(self, username): ++ # User 'core' is not a sysuser. ++ if username == 'core': ++ return False ++ return super(CoreOSUtil, self).is_sys_user(username) ++ ++ def is_dhcp_enabled(self): ++ return True ++ ++ def start_network(self): ++ return shellutil.run("systemctl start systemd-networkd", chk_err=False) ++ ++ def restart_if(self, ifname=None, retries=None, wait=None): ++ shellutil.run("systemctl restart systemd-networkd") ++ ++ def restart_ssh_service(self): ++ # SSH is socket activated on CoreOS. No need to restart it. ++ pass ++ ++ def stop_dhcp_service(self): ++ return shellutil.run("systemctl stop systemd-networkd", chk_err=False) ++ ++ def start_dhcp_service(self): ++ return shellutil.run("systemctl start systemd-networkd", chk_err=False) ++ ++ def start_agent_service(self): ++ return shellutil.run("systemctl start {0}".format(self.service_name), chk_err=False) ++ ++ def stop_agent_service(self): ++ return shellutil.run("systemctl stop {0}".format(self.service_name), chk_err=False) ++ ++ def get_dhcp_pid(self): ++ return self._get_dhcp_pid(["systemctl", "show", "-p", "MainPID", "systemd-networkd"]) +diff --git a/azurelinuxagent/common/osutil/factory.py b/azurelinuxagent/common/osutil/factory.py +index b5ee0b09..9280c645 100644 +--- a/azurelinuxagent/common/osutil/factory.py ++++ b/azurelinuxagent/common/osutil/factory.py +@@ -27,6 +27,7 @@ from .clearlinux import ClearLinuxUtil + from .coreos import CoreOSUtil + from .debian import DebianOSBaseUtil, DebianOSModernUtil + from .default import DefaultOSUtil ++from .flatcar import FlatcarUtil + from .freebsd import FreeBSDOSUtil + from .gaia import GaiaOSUtil + from .iosxe import IosxeOSUtil +@@ -82,6 +83,8 @@ def _get_osutil(distro_name, distro_code_name, distro_version, distro_full_name) + return DebianOSBaseUtil() + + if distro_name in ("flatcar", "coreos") or distro_code_name in ("flatcar", "coreos"): ++ if Version(distro_version) >= Version("3550"): ++ return FlatcarUtil() + return CoreOSUtil() + + if distro_name in ("suse", "sle_hpc", "sles", "opensuse"): +diff --git a/azurelinuxagent/common/osutil/flatcar.py b/azurelinuxagent/common/osutil/flatcar.py +new file mode 100644 +index 00000000..3d1bf535 +--- /dev/null ++++ b/azurelinuxagent/common/osutil/flatcar.py +@@ -0,0 +1,41 @@ ++# ++# Copyright 2023 Microsoft Corporation ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); ++# you may not use this file except in compliance with the License. ++# You may obtain a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, ++# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ++# See the License for the specific language governing permissions and ++# limitations under the License. ++# ++# Requires Python 2.6+ and Openssl 1.0+ ++# ++ ++import os ++import shutil ++ ++import azurelinuxagent.common.conf as conf ++ ++from azurelinuxagent.common.osutil.coreoscommon import CoreosCommonUtil ++ ++ ++class FlatcarUtil(CoreosCommonUtil): ++ ++ @staticmethod ++ def get_systemd_unit_file_install_path(): ++ return "/usr/lib/systemd/system" ++ ++ def conf_sshd(self, disable_password): ++ # make sure that the config file stops being a symlink ++ conf_file_path = conf.get_sshd_conf_file_path() ++ conf_file_path2 = f"{conf_file_path}.wal.tmp" ++ shutil.copy(conf_file_path, conf_file_path2) ++ os.remove(conf_file_path) ++ os.rename(conf_file_path2, conf_file_path) ++ super(CoreosCommonUtil, self).conf_sshd(disable_password) ++ pass +diff --git a/config/flatcar/waagent.conf b/config/flatcar/waagent.conf +new file mode 100644 +index 00000000..b453c634 +--- /dev/null ++++ b/config/flatcar/waagent.conf +@@ -0,0 +1,122 @@ ++# ++# Microsoft Azure Linux Agent Configuration ++# ++ ++# Enable extension handling. Do not disable this unless you do not need password reset, ++# backup, monitoring, or any extension handling whatsoever. ++Extensions.Enabled=y ++ ++# Which provisioning agent to use. Supported values are "auto" (default), "waagent", ++# "cloud-init", or "disabled". ++Provisioning.Agent=waagent ++ ++# Password authentication for root account will be unavailable. ++Provisioning.DeleteRootPassword=n ++ ++# Generate fresh host key pair. ++Provisioning.RegenerateSshHostKeyPair=n ++ ++# Supported values are "rsa", "dsa", "ecdsa", "ed25519", and "auto". ++# The "auto" option is supported on OpenSSH 5.9 (2011) and later. ++Provisioning.SshHostKeyPairType=auto ++ ++# Monitor host name changes and publish changes via DHCP requests. ++Provisioning.MonitorHostName=y ++ ++# Decode CustomData from Base64. ++Provisioning.DecodeCustomData=y ++ ++# Execute CustomData after provisioning. ++Provisioning.ExecuteCustomData=n ++ ++# Algorithm used by crypt when generating password hash. ++#Provisioning.PasswordCryptId=6 ++ ++# Length of random salt used when generating password hash. ++#Provisioning.PasswordCryptSaltLength=10 ++ ++# Allow reset password of sys user ++Provisioning.AllowResetSysUser=n ++ ++# Format if unformatted. If 'n', resource disk will not be mounted. ++ResourceDisk.Format=y ++ ++# File system on the resource disk ++# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. ++ResourceDisk.Filesystem=ext4 ++ ++# Mount point for the resource disk ++ResourceDisk.MountPoint=/mnt/resource ++ ++# Create and use swapfile on resource disk. ++ResourceDisk.EnableSwap=n ++ ++# Size of the swapfile. ++ResourceDisk.SwapSizeMB=0 ++ ++# Comma-seperated list of mount options. See mount(8) for valid options. ++ResourceDisk.MountOptions=None ++ ++# Respond to load balancer probes if requested by Windows Azure. ++LBProbeResponder=y ++ ++# Enable verbose logging (y|n) ++Logs.Verbose=n ++ ++# Enable Console logging, default is y ++# Logs.Console=y ++ ++# Is FIPS enabled ++OS.EnableFIPS=n ++ ++# Set the path to SSH keys and configuration files ++OS.SshDir=/etc/ssh ++ ++# Root device timeout in seconds. ++OS.RootDeviceScsiTimeout=300 ++ ++# If "None", the system default version is used. ++OS.OpensslPath=None ++ ++# If set, agent will use proxy server to access internet ++#HttpProxy.Host=None ++#HttpProxy.Port=None ++ ++# Detect Scvmm environment, default is n ++# DetectScvmmEnv=n ++ ++# ++# Lib.Dir=/var/lib/waagent ++ ++# ++# DVD.MountPoint=/mnt/cdrom/secure ++ ++# ++# Pid.File=/var/run/waagent.pid ++ ++# ++# Extension.LogDir=/var/log/azure ++ ++# ++# Home.Dir=/home ++ ++# Enable RDMA management and set up, should only be used in HPC images ++# OS.EnableRDMA=y ++ ++# Enable or disable goal state processing auto-update, default is enabled ++AutoUpdate.Enabled=n ++ ++# Determine the update family, this should not be changed ++# AutoUpdate.GAFamily=Prod ++ ++# Determine if the overprovisioning feature is enabled. If yes, hold extension ++# handling until inVMArtifactsProfile.OnHold is false. ++# Default is enabled ++# EnableOverProvisioning=y ++ ++# Allow fallback to HTTP if HTTPS is unavailable ++# Note: Allowing HTTP (vs. HTTPS) may cause security risks ++# OS.AllowHTTP=n ++ ++# Add firewall rules to protect access to Azure host node services ++OS.EnableFirewall=y +diff --git a/init/flatcar/10-waagent-sysext.conf b/init/flatcar/10-waagent-sysext.conf +new file mode 100644 +index 00000000..f756dbc9 +--- /dev/null ++++ b/init/flatcar/10-waagent-sysext.conf +@@ -0,0 +1,2 @@ ++[Unit] ++Upholds=waagent.service +diff --git a/init/flatcar/waagent.service b/init/flatcar/waagent.service +new file mode 100644 +index 00000000..d0d6f7c8 +--- /dev/null ++++ b/init/flatcar/waagent.service +@@ -0,0 +1,30 @@ ++[Unit] ++Description=Microsoft Azure Linux Agent ++Wants=network-online.target sshd.service sshd-keygen.service ++After=network-online.target sshd-keygen.service ++ ++[Service] ++Type=simple ++ ++# Symlink the config if it's missing in /etc. This is a workaround for ++# the fact that this software comes to Flatcar as a sysext and as such ++# can't use the /etc overlay solution by putting the config into ++# /usr/share/flatcar/etc. ++# ++ExecStartPre=/bin/bash -c 'if [[ ! -e /etc/waagent.conf ]]; then ln -sf ../usr/share/waagent/waagent.conf /etc/waagent.conf; fi' ++ ++# This could be done also with: ++# ++# ExecStart=/usr/bin/python -u /usr/sbin/waagent -daemon ++# ++# But this would mean that logs from waagent in journal will be ++# denoted as coming from python instead. ++# ++Environment=PYTHONUNBUFFERED=x ++ExecStart=/usr/sbin/waagent -daemon ++ ++Restart=always ++RestartSec=5s ++ ++[Install] ++WantedBy=multi-user.target +diff --git a/setup.py b/setup.py +index d38d74d6..57b0edb9 100755 +--- a/setup.py ++++ b/setup.py +@@ -125,12 +125,22 @@ def get_data_files(name, version, fullname): # pylint: disable=R0912 + src=["init/arch/waagent.service"]) + elif name in ('coreos', 'flatcar'): + set_bin_files(data_files, dest=agent_bin_path) +- set_conf_files(data_files, dest="/usr/share/oem", +- src=["config/coreos/waagent.conf"]) + set_logrotate_files(data_files) +- set_udev_files(data_files) +- set_files(data_files, dest="/usr/share/oem", +- src=["init/coreos/cloud-config.yml"]) ++ if int(version.split('.')[0]) >= 3550: ++ # Not installing udev rules, Flatcar already has those ++ set_conf_files(data_files, dest="/usr/share/waagent", ++ src=["config/flatcar/waagent.conf"]) ++ set_systemd_files(data_files, dest=systemd_dir_path, ++ src=["init/flatcar/waagent.service"]) ++ multi_user_target_drop_in_dir=f"{systemd_dir_path}/multi-user.target.d" ++ set_systemd_files(data_files, dest=multi_user_target_drop_in_dir, ++ src=["init/flatcar/10-waagent-sysext.conf"]) ++ else: ++ set_udev_files(data_files) ++ set_conf_files(data_files, dest="/usr/share/oem", ++ src=["config/coreos/waagent.conf"]) ++ set_files(data_files, dest="/usr/share/oem", ++ src=["init/coreos/cloud-config.yml"]) + elif "Clear Linux" in fullname: + set_bin_files(data_files, dest=agent_bin_path) + set_conf_files(data_files, dest="/usr/share/defaults/waagent", +-- +2.25.1 + diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf deleted file mode 100644 index 3d65d06b50..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/files/waagent.conf +++ /dev/null @@ -1,127 +0,0 @@ -# -# Microsoft Azure Linux Agent Configuration -# - -# Enable instance creation -Provisioning.Enabled=y - -# Enable extension handling. Do not disable this unless you do not need password reset, -# backup, monitoring, or any extension handling whatsoever. -Extensions.Enabled=y - -# Rely on cloud-init to provision -Provisioning.UseCloudInit=n - -# Password authentication for root account will be unavailable. -Provisioning.DeleteRootPassword=n - -# Generate fresh host key pair. -Provisioning.RegenerateSshHostKeyPair=n - -# Supported values are "rsa", "dsa", "ecdsa", "ed25519", and "auto". -# The "auto" option is supported on OpenSSH 5.9 (2011) and later. -Provisioning.SshHostKeyPairType=auto - -# Monitor host name changes and publish changes via DHCP requests. -Provisioning.MonitorHostName=y - -# Decode CustomData from Base64. -Provisioning.DecodeCustomData=y - -# Execute CustomData after provisioning. -Provisioning.ExecuteCustomData=n - -# Algorithm used by crypt when generating password hash. -#Provisioning.PasswordCryptId=6 - -# Length of random salt used when generating password hash. -#Provisioning.PasswordCryptSaltLength=10 - -# Allow reset password of sys user -Provisioning.AllowResetSysUser=n - -# Format if unformatted. If 'n', resource disk will not be mounted. -ResourceDisk.Format=y - -# File system on the resource disk -# Typically ext3 or ext4. FreeBSD images should use 'ufs2' here. -ResourceDisk.Filesystem=ext4 - -# Mount point for the resource disk -ResourceDisk.MountPoint=/mnt/resource - -# Create and use swapfile on resource disk. -ResourceDisk.EnableSwap=n - -# Size of the swapfile. -ResourceDisk.SwapSizeMB=0 - -# Comma-seperated list of mount options. See man(8) for valid options. -ResourceDisk.MountOptions=None - -# Enable verbose logging (y|n) -Logs.Verbose=n - -# Is FIPS enabled -OS.EnableFIPS=n - -# Root device timeout in seconds. -OS.RootDeviceScsiTimeout=300 - -# If "None", the system default version is used. -OS.OpensslPath=None - -# Set the SSH ClientAliveInterval -# OS.SshClientAliveInterval=180 - -# Set the path to SSH keys and configuration files -OS.SshDir=/etc/ssh - -# If set, agent will use proxy server to access internet -#HttpProxy.Host=None -#HttpProxy.Port=None - -# Detect Scvmm environment, default is n -# DetectScvmmEnv=n - -# -# Lib.Dir=/var/lib/waagent - -# -# DVD.MountPoint=/mnt/cdrom/secure - -# -# Pid.File=/var/run/waagent.pid - -# -# Extension.LogDir=/var/log/azure - -# -# Home.Dir=/home - -# Enable RDMA management and set up, should only be used in HPC images -# OS.EnableRDMA=y - -# Enable or disable goal state processing auto-update, default is enabled -# AutoUpdate.Enabled=y - -# Determine the update family, this should not be changed -# AutoUpdate.GAFamily=Prod - -# Determine if the overprovisioning feature is enabled. If yes, hold extension -# handling until inVMArtifactsProfile.OnHold is false. -# Default is enabled -# EnableOverProvisioning=y - -# Allow fallback to HTTP if HTTPS is unavailable -# Note: Allowing HTTP (vs. HTTPS) may cause security risks -# OS.AllowHTTP=n - -# Add firewall rules to protect access to Azure host node services -OS.EnableFirewall=y - -# Enforce control groups limits on the agent and extensions -CGroups.EnforceLimits=n - -# CGroups which are excluded from limits, comma separated -CGroups.Excluded=customscript,runcommand diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild deleted file mode 100644 index 685a5a66e5..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r2.ebuild +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2014 CoreOS, Inc.. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="Windows Azure Linux Agent" -HOMEPAGE="https://github.com/Azure/WALinuxAgent" -KEYWORDS="amd64 arm64" -SRC_URI="${HOMEPAGE}/archive/v${PV}.tar.gz -> ${P}.tar.gz" - -LICENSE="Apache-2.0" -SLOT="0" -IUSE="" - -# Depending on specific version of python-oem allows us to notice when -# we update the major version of python and then to make sure that we -# install the package in correctly versioned site-packages directory. -DEP_PYVER="3.10" - -RDEPEND=" -dev-lang/python-oem:${DEP_PYVER} -dev-python/distro-oem -" - -S="${WORKDIR}/WALinuxAgent-${PV}" - -src_install() { - into "/oem" - dobin "${S}/bin/waagent" - - insinto "/oem/python/$(get_libdir)/python${DEP_PYVER}/site-packages" - doins -r "${S}/azurelinuxagent/" - - insinto "/oem" - doins "${FILESDIR}/waagent.conf" -} diff --git a/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild new file mode 100644 index 0000000000..ee70a04763 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/app-emulation/wa-linux-agent/wa-linux-agent-2.6.0.2-r3.ebuild @@ -0,0 +1,33 @@ +# Copyright (c) 2014 CoreOS, Inc.. All rights reserved. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +# Don't use DISTUTILS_USE_PEP517=setuptools because this installs +# everything inside /usr/lib/pythonX_Y/site-packages, even files that +# ought to be put into /etc or /sbin. +PYTHON_COMPAT=( python3_{9..11} ) + +inherit distutils-r1 + +DESCRIPTION="Windows Azure Linux Agent" +HOMEPAGE="https://github.com/Azure/WALinuxAgent" +SRC_URI="${HOMEPAGE}/archive/v${PV}.tar.gz -> ${P}.tar.gz" + +LICENSE="Apache-2.0" +KEYWORDS="amd64 arm64" +SLOT="0" +IUSE="" +RESTRICT="" + +BDEPEND=" + dev-python/distro +" +RDEPEND="${BDEPEND} +" + +S="${WORKDIR}/WALinuxAgent-${PV}" + +PATCHES=( + "${FILESDIR}/0001-flatcar-changes.patch" +) From 14a55a27b56f4bff7de07470897f5752984bded6 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 13:06:06 +0200 Subject: [PATCH 09/17] overlay dev-python/distro-oem: Drop unnecessary package --- .../dev-python/distro-oem/Manifest | 1 - .../dev-python/distro-oem/README.md | 4 -- .../distro-oem/distro-oem-1.7.0-r2.ebuild | 41 ------------------- .../dev-python/distro-oem/metadata.xml | 4 -- 4 files changed, 50 deletions(-) delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest deleted file mode 100644 index fbb1ae14b6..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/Manifest +++ /dev/null @@ -1 +0,0 @@ -DIST distro-1.7.0.tar.gz 58164 BLAKE2B 22bbd2daf9cac589530eac9a58767db6b9e389b77719516f7386a9377b49ba4c9b696165701acc42366b760b9a632c70a2243a58c12a367fef2a0a770a4aea44 SHA512 14516ecab33ee8c57c35a8279eb515fd699031fabac7d8886092ea98696797d55503179870aeb513a85e1a66c7e69f2f60bb6ea9fc935be975cb5135e1917ecc diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md deleted file mode 100644 index 7fcb570b32..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/README.md +++ /dev/null @@ -1,4 +0,0 @@ -This package is a hacked-up way to install a distro module for oem -packages to use. It's meant to be used by dev-lang/python-oem, thus -not using any python-specific eclasses and whatnot, to avoid pulling -python dependency into the production image. diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild deleted file mode 100644 index 330acec882..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/distro-oem-1.7.0-r2.ebuild +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021-2022 Microsoft Corporation -# Distributed under the terms of GNU General Public License v2 - -EAPI=8 - -MY_PN='distro' -MY_P="${MY_PN}-${PV}" - -DESCRIPTION="Reliable machine-readable Linux distribution information for Python" -HOMEPAGE=" - https://distro.readthedocs.io/en/latest/ - https://pypi.org/project/distro/ - https://github.com/python-distro/distro/" -SRC_URI="mirror://pypi/${MY_PN:0:1}/${MY_PN}/${MY_P}.tar.gz" - -LICENSE="Apache-2.0" -KEYWORDS="amd64 arm64" - -# Depending on specific version of python-oem allows us to notice when -# we update the major version of python and then to make sure that we -# install the package in correctly versioned site-packages directory. -DEP_PYVER="3.10" - -SLOT="0" -RDEPEND="dev-lang/python-oem:${DEP_PYVER}" - -S="${WORKDIR}/${MY_P}" - -src_compile() { - # nothing to do - : -} - -src_install() { - insinto "/oem/python/$(get_libdir)/python${DEP_PYVER}/site-packages" - local ssd="${S}/src/distro" - doins "${ssd}/distro.py" - doins "${ssd}/__init__.py" - doins "${ssd}/__main__.py" - doins "${ssd}/py.typed" -} diff --git a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml b/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml deleted file mode 100644 index 097975e3ad..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-python/distro-oem/metadata.xml +++ /dev/null @@ -1,4 +0,0 @@ - - - - From ee45899915f5e61f9d1924365bf4b4c06a3ab879 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 13:05:34 +0200 Subject: [PATCH 10/17] overlay dev-lang/python-oem: Drop unnecessary package --- ...ebuild => board-packages-0.0.1-r11.ebuild} | 0 .../board-packages-0.0.1.ebuild | 1 - .../dev-lang/python-oem/Manifest | 3 - .../dev-lang/python-oem/README.md | 45 -- .../dev-lang/python-oem/metadata.xml | 43 -- .../python-oem-3.10.10_p2-r1.ebuild | 461 ------------------ .../profiles/coreos/base/package.use.mask | 1 - .../coreos/targets/generic/package.use | 4 - 8 files changed, 558 deletions(-) rename sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/{board-packages-0.0.1-r10.ebuild => board-packages-0.0.1-r11.ebuild} (100%) delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml delete mode 100644 sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r10.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r11.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r10.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1-r11.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild index 0be0428a6b..e95bd1644d 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-devel/board-packages/board-packages-0.0.1.ebuild @@ -34,6 +34,5 @@ RDEPEND=" coreos-base/coreos coreos-base/coreos-dev coreos-base/flatcar-eks - dev-lang/python-oem x11-drivers/nvidia-drivers " diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest deleted file mode 100644 index a60f8dcf2a..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/Manifest +++ /dev/null @@ -1,3 +0,0 @@ -DIST Python-3.10.10.tar.xz 19627028 BLAKE2B 57fc6869fa05586158a170c1892d93a3036823bfafb9484b9d70bca6cdc3e76f75357622eace4bde9a4c0ca62a1bb79665e5751b41655f9f4d7e345547013ad8 SHA512 f0aee65970a68287b34c4eafcf35c6fa09c81ba234ac356db16fbbc6c36417e4ac67071e616d118f5e192d541d7f177dcab5585b9780e842f656c09e01c37ced -DIST Python-3.10.10.tar.xz.asc 833 BLAKE2B fd60e6268f7dd6676ea58bd7e80c513506ac9810c1a62ff060134207b0fd8e7b096d5f11f3cc536a1578144ff54c00bcb076d3c3f5889a69a898660dd280312b SHA512 591746d74c6123bf36c763b6e8e1de1554f02eeff30c855623ef0f12d3864d5573eb5efe96d6e142f24627c77b90738ada3456df4ad59bddcb008658f2ca8af9 -DIST python-gentoo-patches-3.10.10_p2.tar.xz 13992 BLAKE2B e18e708888dd28c8f238d4897aff79483a679396a168d8b5ff4f5e8c7f09cec5f1b13aeb327d3dc3e2149c2117c25da050987f1f1c3322b56c87245ba2d0b54d SHA512 14bc218a2f3c64ef9f42682fd1364208bcaa74f787dee39bd9566e40764c260a65fd42961be47a6e6c6227091cb2fef83e1d689302448647560689e20e07efe0 diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md deleted file mode 100644 index 9ebd462b49..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/README.md +++ /dev/null @@ -1,45 +0,0 @@ -Modifications made: - -- Keep using internal expat and libffi, thus dropping dev-libs/libffi - and dev-libs/expat from the dependencies. - -- Drop dev-python/gentoo-common dependency, it provides the - EXTERNALLY-MANAGED file, but we will provide our own. - -- Since this package is installed only for OEM partition as a binary - package, and the installation there happens after the packages - database is removed, we unset the RDEPEND variable. The RDEPEND - variable needs to be empty as it's also used during the binary - package installation. The contents of RDEPEND are already inside the - DEPEND variable, so we are safe. - -- We modify the configure flags: - - - Add `--prefix=/oem/python` as `/oem` is where the OEM partition is - mounted. - - - Add `--with-platlibdir="$(get_libdir)"`, this is to make sure that - consistent library directory gets picked. In our case for both - amd64 and arm64, it's lib64. - - - Change `--enable-shared` to `--disable-shared`. This will skip - building dynamic libraries, as we don't need them. - - - Add `--includedir=/discard/include` and change `--mandir` and - `--infodir` to also use `/discard` to install files there. Makes - it easy to remove the unnecessary files. - - - We disable loadable sqlite extensions. - - - As we want to use the internal versions of expat and libffi, we - change `--with-system-{expat,ffi}` to - `--without-system-{expat,ffi}`. - - - Comment out the `--with-wheel-pkg-dir` as it's some ensurepip - stuff we are disabling anyway. - -- Essentially drop `src_install` and write our own variant, where we - run `make altinstall`, remove unnecessary files (the original - `src_install` could be read to find out which files to remove), - creates a versionless python symlink, adds an EXTERNALLY-MANAGED - file, and removes the `/discard` directory. diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml deleted file mode 100644 index 66d5aec84c..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/metadata.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - python@gentoo.org - Python - - - - Build Bluetooth protocol support in socket module - - - Install the ensurepip module that uses bundled wheels - to bootstrap pip and setuptools (if disabled, it will - be only possible to use venv `--without-pip`) - - - Link readline extension against dev-libs/libedit - instead of sys-libs/readline - - - Optimize the build using Profile Guided Optimization (PGO) - by running Python's test suite and collecting statistics - based on its performance. This will take longer to build. - - - Optimize the build using Link Time Optimization (LTO) - - - Disable pymalloc when running under - dev-util/valgrind is detected (may incur minor - performance penalty even when valgrind is not used) - - - Install Windows executables required to create an executable - installer for MS Windows - - - - cpe:/a:python:python - python/cpython - - diff --git a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild b/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild deleted file mode 100644 index ef732a7132..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/dev-lang/python-oem/python-oem-3.10.10_p2-r1.ebuild +++ /dev/null @@ -1,461 +0,0 @@ -# Copyright 1999-2023 Gentoo Authors -# Distributed under the terms of the GNU General Public License v2 - -EAPI="7" -WANT_LIBTOOL="none" - -inherit autotools check-reqs flag-o-matic multiprocessing pax-utils -inherit prefix python-utils-r1 toolchain-funcs verify-sig - -MY_PV=${PV/_rc/rc} -MY_P="Python-${MY_PV%_p*}" -PYVER=$(ver_cut 1-2) -PATCHSET="python-gentoo-patches-${MY_PV}" - -DESCRIPTION="An interpreted, interactive, object-oriented programming language" -HOMEPAGE=" - https://www.python.org/ - https://github.com/python/cpython/ -" -SRC_URI=" - https://www.python.org/ftp/python/${PV%%_*}/${MY_P}.tar.xz - https://dev.gentoo.org/~mgorny/dist/python/${PATCHSET}.tar.xz - verify-sig? ( - https://www.python.org/ftp/python/${PV%%_*}/${MY_P}.tar.xz.asc - ) -" -S="${WORKDIR}/${MY_P}" - -LICENSE="PSF-2" -SLOT="${PYVER}" -KEYWORDS="~alpha amd64 arm arm64 hppa ~ia64 ~loong ~m68k ~mips ppc ppc64 ~riscv ~s390 sparc x86" -IUSE=" - bluetooth build +ensurepip examples gdbm hardened libedit lto - +ncurses pgo +readline +sqlite +ssl test tk valgrind +xml -" -RESTRICT="!test? ( test )" - -# Do not add a dependency on dev-lang/python to this ebuild. -# If you need to apply a patch which requires python for bootstrapping, please -# run the bootstrap code on your dev box and include the results in the -# patchset. See bug 447752. - -# Flatcar: Drop a dependency on dev-libs/expat, we will use the internal one. -# Flatcar: Drop a dependency on dev-libs/libffi, we will use the internal one. -# Flatcar: Drop a dependency on dev-python/gentoo-common, we will install our own EXTERNALLY-MANAGED file -RDEPEND=" - app-arch/bzip2:= - app-arch/xz-utils:= - dev-lang/python-exec[python_targets_python3_10(-)] - dev-python/gentoo-common - sys-apps/util-linux:= - >=sys-libs/zlib-1.1.3:= - virtual/libcrypt:= - virtual/libintl - ensurepip? ( dev-python/ensurepip-wheels ) - gdbm? ( sys-libs/gdbm:=[berkdb] ) - ncurses? ( >=sys-libs/ncurses-5.2:= ) - readline? ( - !libedit? ( >=sys-libs/readline-4.1:= ) - libedit? ( dev-libs/libedit:= ) - ) - sqlite? ( >=dev-db/sqlite-3.3.8:3= ) - ssl? ( >=dev-libs/openssl-1.1.1:= ) - tk? ( - >=dev-lang/tcl-8.0:= - >=dev-lang/tk-8.0:= - dev-tcltk/blt:= - dev-tcltk/tix - ) - !! /dev/null || die - # We disable _ctypes and _crypt for CBUILD because Python's setup.py can't handle locating - # libdir correctly for cross. - PYTHON_DISABLE_MODULES="${PYTHON_DISABLE_MODULES} _ctypes _crypt" \ - ECONF_SOURCE="${S}" econf_build "${myeconfargs_cbuild[@]}" - - # Avoid as many dependencies as possible for the cross build. - cat >> Makefile <<-EOF || die - MODULE_NIS=disabled - MODULE__DBM=disabled - MODULE__GDBM=disabled - MODULE__DBM=disabled - MODULE__SQLITE3=disabled - MODULE__HASHLIB=disabled - MODULE__SSL=disabled - MODULE__CURSES=disabled - MODULE__CURSES_PANEL=disabled - MODULE_READLINE=disabled - MODULE__TKINTER=disabled - MODULE_PYEXPAT=disabled - MODULE_ZLIB=disabled - EOF - - # Unfortunately, we do have to build this immediately, and - # not in src_compile, because CHOST configure for Python - # will check the existence of the Python it was pointed to - # immediately. - PYTHON_DISABLE_MODULES="${PYTHON_DISABLE_MODULES} _ctypes _crypt" emake - popd &> /dev/null || die - fi - - # pass system CFLAGS & LDFLAGS as _NODIST, otherwise they'll get - # propagated to sysconfig for built extensions - local -x CFLAGS_NODIST=${CFLAGS} - local -x LDFLAGS_NODIST=${LDFLAGS} - local -x CFLAGS= LDFLAGS= - - # Fix implicit declarations on cross and prefix builds. Bug #674070. - if use ncurses; then - append-cppflags -I"${ESYSROOT}"/usr/include/ncursesw - fi - - hprefixify setup.py - econf "${myeconfargs[@]}" - - if grep -q "#define POSIX_SEMAPHORES_NOT_ENABLED 1" pyconfig.h; then - eerror "configure has detected that the sem_open function is broken." - eerror "Please ensure that /dev/shm is mounted as a tmpfs with mode 1777." - die "Broken sem_open function (bug 496328)" - fi - - # install epython.py as part of stdlib - echo "EPYTHON='python${PYVER}'" > Lib/epython.py || die -} - -src_compile() { - # Ensure sed works as expected - # https://bugs.gentoo.org/594768 - local -x LC_ALL=C - # Prevent using distutils bundled by setuptools. - # https://bugs.gentoo.org/823728 - export SETUPTOOLS_USE_DISTUTILS=stdlib - - # Save PYTHONDONTWRITEBYTECODE so that 'has_version' doesn't - # end up writing bytecode & violating sandbox. - # bug #831897 - local -x _PYTHONDONTWRITEBYTECODE=${PYTHONDONTWRITEBYTECODE} - - if use pgo ; then - # bug 660358 - local -x COLUMNS=80 - local -x PYTHONDONTWRITEBYTECODE= - - addpredict "/usr/lib/python${PYVER}/site-packages" - fi - - # also need to clear the flags explicitly here or they end up - # in _sysconfigdata* - emake CPPFLAGS= CFLAGS= LDFLAGS= - - # Restore saved value from above. - local -x PYTHONDONTWRITEBYTECODE=${_PYTHONDONTWRITEBYTECODE} - - # Work around bug 329499. See also bug 413751 and 457194. - if has_version dev-libs/libffi[pax-kernel]; then - pax-mark E python - else - pax-mark m python - fi -} - -src_test() { - # Tests will not work when cross compiling. - if tc-is-cross-compiler; then - elog "Disabling tests due to crosscompiling." - return - fi - - local test_opts=( - -u-network - -j "$(makeopts_jobs)" - - # fails - -x test_gdb - ) - - if use sparc ; then - # bug #788022 - test_opts+=( - -x test_multiprocessing_fork - -x test_multiprocessing_forkserver - ) - fi - - # workaround docutils breaking tests - cat > Lib/docutils.py <<-EOF || die - raise ImportError("Thou shalt not import!") - EOF - - # bug 660358 - local -x COLUMNS=80 - local -x PYTHONDONTWRITEBYTECODE= - # workaround https://bugs.gentoo.org/775416 - addwrite "/usr/lib/python${PYVER}/site-packages" - - nonfatal emake test EXTRATESTOPTS="${test_opts[*]}" \ - CPPFLAGS= CFLAGS= LDFLAGS= < /dev/tty - local ret=${?} - - rm Lib/docutils.py || die - - [[ ${ret} -eq 0 ]] || die "emake test failed" -} - -# Flatcar: Rewrite src_install to just run make altinstall, remove -# some installed files (refer to the original src_install to see which -# files to drop), adding symlinks and the EXTERNALLY-MANAGED file, and -# removing the /discard directory. -src_install() { - local prefix=/oem/python - local eprefix="${ED}${prefix}" - local libdir="${prefix}/$(get_libdir)" - local elibdir="${eprefix}/$(get_libdir)" - local pythonplatlibdir="${libdir}/python${PYVER}" - local epythonplatlibdir="${elibdir}/python${PYVER}" - local bindir="${prefix}/bin" - local ebindir="${eprefix}/bin" - - emake DESTDIR="${D}" altinstall - - rm -r "${epythonplatlibdir}"/ensurepip || die - rm -r "${epythonplatlibdir}/"{sqlite3,test/test_sqlite*} || die - rm -r "${ebindir}/idle${PYVER}" || die - rm -r "${epythonplatlibdir}/"{idlelib,tkinter,test/test_tk*} || die - - # create a simple versionless 'python' symlink - dosym "python${PYVER}" "${bindir}/python" - dosym "python${PYVER}" "${bindir}/python3" - - insinto "${pythonplatlibdir}" - # https://peps.python.org/pep-0668/ - newins - EXTERNALLY-MANAGED <<-EOF - [externally-managed] - Error= - Please contact Flatcar maintainers if some python package - is necessary for this OEM image. - EOF - - rm -r "${ED}/discard" || die -} diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask index c02b3aba0d..ab7f1c9a9e 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/package.use.mask @@ -26,7 +26,6 @@ sys-libs/glibc -crypt # We don't use pip. dev-lang/python ensurepip -dev-lang/python-oem ensurepip # Pulls dev-python/sphinx, which in turn pulls a lot of other python stuff. sys-fs/btrfs-progs man diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use index 62ab683076..269d4bdc21 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/package.use @@ -7,10 +7,6 @@ app-editors/vim minimal -crypt # minimal: Don't pull app-vim/gentoo-syntax app-editors/vim-core minimal dev-lang/python gdbm - -# Disable everything for python-oem except of build and xml -dev-lang/python-oem -bluetooth build -ensurepip -examples -gdbm -hardened -libedit -lto -ncurses -pgo -readline -sqlite -ssl -test -tk -valgrind xml - dev-libs/dbus-glib tools dev-libs/elfutils -utils dev-libs/openssl pkcs11 From 70d33ebabd85496e591f0ce04d53c30c8834a448 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 13:12:30 +0200 Subject: [PATCH 11/17] overlay coreos-base/common-oem-files: New package Th package will generate and install both grub.cfg and oem-release files into /usr/share/oem. Each platform can customize the process by providing their own fragments for each of the two files if necessary. --- .../common-oem-files-0.ebuild | 86 +++++++++++++++++++ .../coreos-base/common-oem-files/metadata.xml | 4 + 2 files changed, 90 insertions(+) create mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild create mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild new file mode 100644 index 0000000000..2b4d25670b --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild @@ -0,0 +1,86 @@ +# Copyright (c) 2023 The Flatcar Maintainers. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +OEMIDS=( +) + +DESCRIPTION='Common OEM files' +HOMEPAGE='https://www.flatcar.org/' + +LICENSE='Apache-2.0' +SLOT='0' +KEYWORDS='amd64 arm64' +IUSE="${OEMIDS[*]}" +REQUIRED_USE="^^ ( ${OEMIDS[*]} )" + +# No source directory. +S="${WORKDIR}" + +DEPEND="" +RDEPEND="${DEPEND}" +BDEPEND=" + app-portage/gentoolkit +" + +src_compile() { + local oemid package ebuild version name homepage lines + + for oemid in "${OEMIDS[@]}"; do + if use "${oemid}"; then break; fi + done + + package="coreos-base/oem-${oemid}" + ebuild=$(equery which "${package}") + version=${ebuild##*"oem-${oemid}-"} + version=${version%%'.ebuild'} + if [[ -z "${version}" ]]; then + die "Could not deduce a version from ebuild ${ebuild##*/} (${ebuild})" + fi + name=$(source <(grep -F 'OEM_NAME=' "${ebuild}"); echo "${OEM_NAME}") + if [[ -z "${name}" ]]; then + die "Missing OEM_NAME variable in ${ebuild##*/}" + fi + # We need to prefix the HOMEPAGE variable with SYSEXT_, because + # portage marks HOMEPAGE as readonly and this gets propagated to + # subshells, so sourcing a snippet with HOMEPAGE=foo won't + # overwrite the readonly variable. + homepage=$(source <(grep -F 'HOMEPAGE=' "${ebuild}" | sed -e 's/^/SYSEXT_/'); echo "${SYSEXT_HOMEPAGE}") + lines=( + "ID=${oemid}" + "VERSION_ID=${version}" + "NAME=\"${name}\"" + ) + if [[ -n "${homepage}" ]]; then + lines+=( "HOME_URL=\"${homepage}\"" ) + fi + lines+=( + 'BUG_REPORT_URL="https://issues.flatcar.org"' + ) + + { + printf '%s\n' "${lines[@]}" + if [[ -e "${FILESDIR}/${oemid}/oem-release.frag" ]]; then + cat "${FILESDIR}/${oemid}/oem-release.frag" + fi + } >"${T}/oem-release" + + lines=( + '# Flatcar GRUB settings' + '' + "set oem_id=\"${oemid}\"" + ) + { + printf '%s\n' "${lines[@]}" + if [[ -e "${FILESDIR}/${oemid}/grub.cfg.frag" ]]; then + cat "${FILESDIR}/${oemid}/grub.cfg.frag" + fi + } >"${T}/grub.cfg" +} + +src_install() { + insinto "/oem" + doins "${T}/grub.cfg" + doins "${T}/oem-release" +} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml new file mode 100644 index 0000000000..7c900b19e8 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/metadata.xml @@ -0,0 +1,4 @@ + + + + From 995910cd783f5e2c8e88d0e791ac769e64dc76fb Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 13:14:55 +0200 Subject: [PATCH 12/17] overlay coreos-base/oem-azure: Make it a sysext package This package will be used for the sysext image, instead of for installing files into /usr/share/oem. This means that we can drop some files or move them elsewhere. The systemd service file is not needed, because it is installed by the app-emulation/wa-linux-agent package now. This also means that the ignition file as lost its purpose. The grub.cfg and oem-release must be installed in /usr/share/oem, next to the sysext raw image file, so handling of these files is moved to the newly added coreos-base/common-oem-files package. `eject` symlink to `/usr/bin/true` is installed in the newly added manglefs.sh script. With this done, we also opt into building an OEM sysext image for Azure platform. --- build_library/vm_image_util.sh | 4 +- .../common-oem-files-0.ebuild | 1 + .../files/azure/grub.cfg.frag} | 4 -- .../coreos-base/oem-azure/files/base/README | 4 -- .../coreos-base/oem-azure/files/base/base.ign | 37 ------------------- .../coreos-base/oem-azure/files/manglefs.sh | 16 ++++++++ .../coreos-base/oem-azure/files/oem-release | 5 --- .../oem-azure/files/units/waagent.service | 15 -------- .../oem-azure/oem-azure-2.6.0.2-r2.ebuild | 36 ------------------ .../oem-azure/oem-azure-2.6.0.2-r3.ebuild | 21 +++++++++++ 10 files changed, 41 insertions(+), 102 deletions(-) rename sdk_container/src/third_party/coreos-overlay/coreos-base/{oem-azure/files/grub.cfg => common-oem-files/files/azure/grub.cfg.frag} (89%) delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign create mode 100755 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild create mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 4cd7f455a7..c7ab823929 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -279,7 +279,9 @@ IMG_exoscale_OEM_PACKAGE=oem-exoscale ## azure IMG_azure_DISK_FORMAT=vhd_fixed IMG_azure_DISK_LAYOUT=azure -IMG_azure_OEM_PACKAGE=oem-azure +IMG_azure_OEM_USE=azure +IMG_azure_OEM_PACKAGE=common-oem-files +IMG_azure_OEM_SYSEXT=oem-azure ## hyper-v IMG_hyperv_DISK_FORMAT=vhd diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild index 2b4d25670b..be541bc00c 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild @@ -4,6 +4,7 @@ EAPI=8 OEMIDS=( + azure ) DESCRIPTION='Common OEM files' diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/grub.cfg b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag similarity index 89% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/grub.cfg rename to sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag index 15e7e50555..48d22ee886 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/grub.cfg +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/azure/grub.cfg.frag @@ -1,7 +1,3 @@ -# Flatcar GRUB settings - -set oem_id="azure" - set linux_append="flatcar.autologin" # Azure only has a serial console. diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README deleted file mode 100644 index d128309fef..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/README +++ /dev/null @@ -1,4 +0,0 @@ -These Ignition configs are part of the OEM configuration. Do not modify -them. If you want to write an Ignition config directly to disk, put it in -../config.ign and it will be applied at first boot instead of a config -in userdata. diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign deleted file mode 100644 index 5c359a9410..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/base/base.ign +++ /dev/null @@ -1,37 +0,0 @@ -{ - "ignition": { - "version": "2.1.0" - }, - "storage": { - "files": [ - { - "filesystem": "root", - "path": "/etc/systemd/system/waagent.service", - "contents": { - "source": "oem:///units/waagent.service" - }, - "mode": 292 - }, - { - "filesystem": "root", - "path": "/etc/systemd/system/nvidia.service", - "contents": { - "source": "oem:///units/nvidia.service" - }, - "mode": 292 - } - ] - }, - "systemd": { - "units": [ - { - "name": "waagent.service", - "enabled": true - }, - { - "name": "nvidia.service", - "enabled": true - } - ] - } -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh new file mode 100755 index 0000000000..28637a0eeb --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/manglefs.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +rootfs="${1}" + +to_delete=( + /usr/include + /usr/lib/debug + /usr/share/gdb + /usr/lib64/pkgconfig +) + +rm -rf "${to_delete[@]/#/${rootfs}}" + +ln -sf /usr/bin/true "${rootfs}/usr/bin/eject" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release deleted file mode 100644 index fa11b4c3e0..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/oem-release +++ /dev/null @@ -1,5 +0,0 @@ -ID=azure -VERSION_ID=@@OEM_VERSION_ID@@ -NAME="Microsoft Azure" -HOME_URL="https://azure.microsoft.com/" -BUG_REPORT_URL="https://issues.flatcar.org" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service deleted file mode 100644 index d8c6e71ad2..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/files/units/waagent.service +++ /dev/null @@ -1,15 +0,0 @@ -[Unit] -Description=Microsoft Azure Agent -Wants=network-online.target sshd-keygen.service -After=network-online.target sshd-keygen.service - -[Service] -Type=simple -Restart=always -RestartSec=5s -Environment=PATH=/oem/python/bin:/oem/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin -Environment=PYTHONUNBUFFERED=x -ExecStart=/oem/bin/waagent -daemon - -[Install] -WantedBy=multi-user.target diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild deleted file mode 100644 index 91a72143d4..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r2.ebuild +++ /dev/null @@ -1,36 +0,0 @@ -# Copyright (c) 2013 CoreOS, Inc.. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="OEM suite for Azure" -HOMEPAGE="" -SRC_URI="" - -LICENSE="Apache-2.0" -SLOT="0" -KEYWORDS="amd64 arm64" -IUSE="" - -# no source directory -S="${WORKDIR}" - -RDEPEND=" - ~app-emulation/wa-linux-agent-${PV} - x11-drivers/nvidia-drivers -" - -src_prepare() { - default - sed -e "s\\@@OEM_VERSION_ID@@\\${PVR}\\g" \ - "${FILESDIR}/oem-release" > "${T}/oem-release" || die -} - -src_install() { - insinto "/oem" - doins "${FILESDIR}/grub.cfg" - doins "${T}/oem-release" - doins -r "${FILESDIR}/base" - doins -r "${FILESDIR}/units" - dosym "/usr/bin/true" "/oem/bin/eject" -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild new file mode 100644 index 0000000000..24fa7fbe41 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-azure/oem-azure-2.6.0.2-r3.ebuild @@ -0,0 +1,21 @@ +# Copyright (c) 2013 CoreOS, Inc.. All rights reserved. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +DESCRIPTION="OEM suite for Azure" +HOMEPAGE="https://azure.microsoft.com/" +SRC_URI="" + +LICENSE="Apache-2.0" +SLOT="0" +KEYWORDS="amd64 arm64" +IUSE="" + +RDEPEND=" + ~app-emulation/wa-linux-agent-${PV} + x11-drivers/nvidia-drivers +" + +# for coreos-base/common-oem-files +OEM_NAME="Microsoft Azure" From 8b17ca2b0213cadff92acf7d265a013cf03bf8d3 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 31 Mar 2023 15:37:43 +0200 Subject: [PATCH 13/17] overlay coreos-base/oem-qemu: Make it a sysext package It isn't doing much as nothing QEMU-specific was being installed into the OEM partition. With that done, we opt into building an OEM sysext image for QEMU platform. --- build_library/vm_image_util.sh | 12 ++++++-- .../common-oem-files-0.ebuild | 1 + .../common-oem-files/files/qemu/grub.cfg.frag | 1 + .../coreos-base/oem-qemu/files/grub.cfg | 4 --- .../coreos-base/oem-qemu/files/oem-release | 5 ---- .../oem-qemu/oem-qemu-0.0.1-r1.ebuild | 28 ------------------- .../oem-qemu/oem-qemu-0.0.2.ebuild | 15 ++++++++++ 7 files changed, 26 insertions(+), 40 deletions(-) create mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release delete mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild create mode 100644 sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index c7ab823929..b3c636f078 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -131,17 +131,23 @@ IMG_DEFAULT_CPUS=2 IMG_qemu_DISK_FORMAT=qcow2 IMG_qemu_DISK_LAYOUT=vm IMG_qemu_CONF_FORMAT=qemu -IMG_qemu_OEM_PACKAGE=oem-qemu +IMG_qemu_OEM_USE=qemu +IMG_qemu_OEM_PACKAGE=common-oem-files +IMG_qemu_OEM_SYSEXT=oem-qemu IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi -IMG_qemu_uefi_OEM_PACKAGE=oem-qemu +IMG_qemu_uefi_OEM_USE=qemu +IMG_qemu_uefi_OEM_PACKAGE=common-oem-files +IMG_qemu_uefi_OEM_SYSEXT=oem-qemu IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 IMG_qemu_uefi_secure_DISK_LAYOUT=vm IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure -IMG_qemu_uefi_secure_OEM_PACKAGE=oem-qemu +IMG_qemu_uefi_secure_OEM_USE=qemu +IMG_qemu_uefi_secure_OEM_PACKAGE=common-oem-files +IMG_qemu_uefi_secure_OEM_SYSEXT=oem-qemu ## xen IMG_xen_CONF_FORMAT=xl diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild index be541bc00c..e3e58924f9 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-0.ebuild @@ -4,6 +4,7 @@ EAPI=8 OEMIDS=( + qemu azure ) diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag new file mode 100644 index 0000000000..4f9e06c2c8 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/files/qemu/grub.cfg.frag @@ -0,0 +1 @@ +set linux_append="flatcar.autologin" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg deleted file mode 100644 index 2cd3a0a310..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/grub.cfg +++ /dev/null @@ -1,4 +0,0 @@ -# Flatcar GRUB settings - -set oem_id="qemu" -set linux_append="flatcar.autologin" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release deleted file mode 100644 index 280e43175d..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/files/oem-release +++ /dev/null @@ -1,5 +0,0 @@ -ID=qemu -VERSION_ID=@@OEM_VERSION_ID@@ -NAME="QEMU" -HOME_URL="https://www.qemu.org/" -BUG_REPORT_URL="https://issues.flatcar.org" diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild deleted file mode 100644 index be8761ea9e..0000000000 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.1-r1.ebuild +++ /dev/null @@ -1,28 +0,0 @@ -# Copyright (c) 2020 Kinvolk GmbH. All rights reserved. -# Distributed under the terms of the GNU General Public License v2 - -EAPI=7 - -DESCRIPTION="OEM suite for QEMU" -HOMEPAGE="" -SRC_URI="" - -LICENSE="GPL-2" -SLOT="0" -KEYWORDS="amd64 arm64" -IUSE="" - -# no source directory -S="${WORKDIR}" - -src_prepare() { - default - sed -e "s\\@@OEM_VERSION_ID@@\\${PVR}\\g" \ - "${FILESDIR}/oem-release" > "${T}/oem-release" || die -} - -src_install() { - insinto "/oem" - doins "${FILESDIR}/grub.cfg" - doins "${T}/oem-release" -} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild new file mode 100644 index 0000000000..fb19d212a0 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/oem-qemu/oem-qemu-0.0.2.ebuild @@ -0,0 +1,15 @@ +# Copyright (c) 2020 Kinvolk GmbH. All rights reserved. +# Distributed under the terms of the GNU General Public License v2 + +EAPI=8 + +DESCRIPTION="OEM suite for QEMU" +HOMEPAGE="https://www.qemu.org/" +SRC_URI="" + +LICENSE="GPL-2" +SLOT="0" +KEYWORDS="amd64 arm64" +IUSE="" + +OEM_NAME="QEMU" From 23df3170ff7f061ff83f78d271ee66610d2ba32c Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 2 Jun 2023 13:39:51 +0200 Subject: [PATCH 14/17] build_library: Force initial version of OEM sysexts for now We don't have an update process of the OEM sysexts implemented yet, so use a fake "initial" version for them and make them independent from OS version. --- build_library/oem_sysext_util.sh | 9 +++++++-- build_library/vm_image_util.sh | 15 +++++++++------ 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/build_library/oem_sysext_util.sh b/build_library/oem_sysext_util.sh index e83effa01d..21e014db48 100755 --- a/build_library/oem_sysext_util.sh +++ b/build_library/oem_sysext_util.sh @@ -157,12 +157,17 @@ oem_sysext_create() { rm -rf "${entry}" done - local metadata metadata_file + local metadata metadata_file metadata_version_entry info "Adding sysext metadata" mkdir -p "${sysext_rootfs}/usr/lib/extension-release.d" + if [[ "${version_id}" = 'initial' ]]; then + metadata_version_entry="SYSEXT_LEVEL=1.0" + else + metadata_version_entry="VERSION_ID=${version_id}" + fi metadata=( 'ID=flatcar' - "VERSION_ID=${version_id}" + "${metadata_version_entry}" "ARCHITECTURE=$(_get_sysext_arch "${board}")" ) metadata_file="${sysext_rootfs}/usr/lib/extension-release.d/extension-release.${oem}" diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index b3c636f078..802f0aa137 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -548,13 +548,16 @@ install_oem_sysext() { local built_sysext_dir="${FLAGS_to}/${oem_sysext}-sysext" local built_sysext_filename="${oem_sysext}.raw" local built_sysext_path="${built_sysext_dir}/${built_sysext_filename}" + local build_oem_sysext_flags=( + --board="${BOARD}" + --build_dir="${built_sysext_dir}" + --prod_image_path="${VM_SRC_IMG}" + --prod_pkgdb_path="${VM_SRC_PKGDB}" + # TODO: Drop this when we implement updating OEM sysexts. + --version_id=initial + ) - "${SCRIPT_ROOT}/build_oem_sysext" \ - --board="${BOARD}" \ - --build_dir="${built_sysext_dir}" \ - --prod_image_path="${VM_SRC_IMG}" \ - --prod_pkgdb_path="${VM_SRC_PKGDB}" \ - "${oem_sysext}" + "${SCRIPT_ROOT}/build_oem_sysext" "${build_oem_sysext_flags[@]}" "${oem_sysext}" local installed_sysext_oem_dir='/oem/sysext' local installed_sysext_file_prefix="${oem_sysext}-${FLATCAR_VERSION}" From 8a52f237118048f12a2a5794491371c0ae4a4ba5 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 2 Jun 2023 13:56:50 +0200 Subject: [PATCH 15/17] overlay sys-kernel/bootengine: Pull in initial sysext MVP --- ...ootengine-0.0.38-r14.ebuild => bootengine-0.0.38-r15.ebuild} | 0 .../coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/{bootengine-0.0.38-r14.ebuild => bootengine-0.0.38-r15.ebuild} (100%) diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r14.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r15.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r14.ebuild rename to sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-0.0.38-r15.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild index 4a747a5175..a8a4a65545 100644 --- a/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/sys-kernel/bootengine/bootengine-9999.ebuild @@ -10,7 +10,7 @@ CROS_WORKON_REPO="https://github.com" if [[ "${PV}" == 9999 ]]; then KEYWORDS="~amd64 ~arm ~arm64 ~x86" else - CROS_WORKON_COMMIT="130003986dfdab46a21c7f34054239e59583e0f6" # flatcar-master + CROS_WORKON_COMMIT="2c85973e01da92c60ad3c8cdcab702b4b508d10f" # flatcar-master KEYWORDS="amd64 arm arm64 x86" fi From a94b389c9ae026f160ebb78148eb82c915fe0e21 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 14:05:20 +0200 Subject: [PATCH 16/17] changelog: Add an entry --- changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md diff --git a/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md b/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md new file mode 100644 index 0000000000..a1cc3afffe --- /dev/null +++ b/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md @@ -0,0 +1 @@ +- Azure and QEMU images currently use sysext images for additional platform-specific software. For Azure images this also means that the image will have a normal python installation available through the sysext image. From 1e19586631a2fba15b3e90f62b1d055a806489d7 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 6 Jun 2023 15:12:15 +0200 Subject: [PATCH 17/17] changelog: Improve wording on sysext changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Kai Lüke --- changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md b/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md index a1cc3afffe..5be23d3a69 100644 --- a/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md +++ b/changelog/changes/2023-06-06-sysext-for-azure-and-qemu-oem.md @@ -1 +1 @@ -- Azure and QEMU images currently use sysext images for additional platform-specific software. For Azure images this also means that the image will have a normal python installation available through the sysext image. +- Azure and QEMU OEM images now use systemd-sysext images for layering additional platform-specific software on top of `/usr`. For Azure images this also means that the image has a normal Python installation available through the sysext image. The OEM software is still not updated but this will be added soon.