torcx removal: address PR comments

- updated github actions for runc, containerd, and docker to not handle
  nonexistent ebuilds in app-torcx/ anymore
- removed spurious package_run_dependencies from build_image_util.sh
- build_sysext: generate pkginfo before mangle script runs
  use zstd for compression; add cli flag to select compression
- ci_automation_common.sh: remove spurious `/` from match string
- coreos, board-packages, bootengine: bump ebuild revisions
- kernel commonconfig: add squashfs zstd support

Signed-off-by: Thilo Fromm <thilofromm@microsoft.com>

.github/workflows/containerd-apply-patch.sh

@@ -28,21 +28,13 @@ git mv "${containerdEbuildOldSymlink}" "${containerdEbuildNewSymlink}"
 sed -i "s/CONTAINERD_COMMIT=\"\(.*\)\"/CONTAINERD_COMMIT=\"${COMMIT_HASH}\"/g" "${containerdEbuildMain}"
 sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${containerdEbuildMain}"
 
-
-DOCKER_VERSION=$(sed -n "s/^DIST docker-\([0-9]*\.[0-9]*\.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
-# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
-versionTorcx=${DOCKER_VERSION%.*}
-torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
-sed -i "s/containerd-${VERSION_OLD}/containerd-${VERSION_NEW}/g" "${torcxEbuildFile}"
-
 popd
 
 URL="https://github.com/containerd/containerd/releases/tag/v${VERSION_NEW}"
 
 generate_update_changelog 'containerd' "${VERSION_NEW}" "${URL}" 'containerd'
 
-commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}" \
+commit_changes app-containers/containerd "${VERSION_OLD}" "${VERSION_NEW}"
-               app-torcx/docker
 
 cleanup_repo
 

.github/workflows/docker-apply-patch.sh

@@ -33,12 +33,6 @@ git mv "${cliEbuildOld}" "${cliEbuildNew}"
 sed -i "s/GIT_COMMIT=\(.*\)/GIT_COMMIT=${COMMIT_HASH_CLI}/g" "${cliEbuildNew}"
 sed -i "s/v${VERSION_OLD}/v${VERSION_NEW}/g" "${cliEbuildNew}"
 
-# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
-versionTorcx=${VERSION_OLD%.*}
-torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
-sed -i "s/docker-${VERSION_OLD}/docker-${VERSION_NEW}/g" "${torcxEbuildFile}"
-sed -i "s/docker-cli-${VERSION_OLD}/docker-cli-${VERSION_NEW}/g" "${torcxEbuildFile}"
-
 # update also docker versions used by the current runc ebuild file.
 versionRunc=$(sed -n "s/^DIST runc-\([0-9]*.[0-9]*.*\)\.tar.*/\1/p" app-containers/runc/Manifest | sort -ruV | head -n1)
 runcEbuildFile=$(get_ebuild_filename app-containers/runc "${versionRunc}")
@@ -63,7 +57,6 @@ generate_update_changelog 'Docker' "${VERSION_NEW}" "${URL}" 'docker'
 regenerate_manifest app-containers/docker-cli "${VERSION_NEW}"
 commit_changes app-containers/docker "${VERSION_OLD}" "${VERSION_NEW}" \
                app-containers/docker-cli \
-               app-torcx/docker \
                app-containers/runc
 
 cleanup_repo

.github/workflows/runc-apply-patch.sh

@@ -36,11 +36,6 @@ sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" app-containers/containerd/c
 
 dockerVersion=$(sed -n "s/^DIST docker-\([0-9]*.[0-9]*.[0-9]*\).*/\1/p" app-containers/docker/Manifest | sort -ruV | head -n1)
 
-# torcx ebuild file has a docker version with only major and minor versions, like 19.03.
-versionTorcx=${dockerVersion%.*}
-torcxEbuildFile=$(get_ebuild_filename app-torcx/docker "${versionTorcx}")
-sed -i "s/runc-${VERSION_OLD}/runc-${VERSION_NEW}/g" "${torcxEbuildFile}"
-
 popd
 
 URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}"
@@ -48,8 +43,7 @@ URL="https://github.com/opencontainers/runc/releases/tag/v${VERSION_NEW}"
 generate_update_changelog 'runc' "${VERSION_NEW}" "${URL}" 'runc'
 
 commit_changes app-containers/runc "${VERSION_OLD}" "${VERSION_NEW}" \
-               app-containers/containerd \
+               app-containers/containerd
-               app-torcx/docker
 
 cleanup_repo
 

build_library/build_image_util.sh

@@ -261,17 +261,6 @@ image_packages_portage() {
         equery --no-color list --format '$cpv::$repo' '*'
 }
 
-# List dependencies for a package runtime dependencies
-
-function package_run_dependencies() (
-  pkg=${1:?}
-  ebuild=$(equery-${BOARD} w "${pkg}")
-  function inherit() { : ; }
-  . "${ebuild}"
-  echo ${RDEPEND}
-)
-
-
 # List packages implicitly contained in rootfs, such as in initramfs.
 image_packages_implicit() {
     local profile="${BUILD_DIR}/configroot/etc/portage/profile"

build_sysext

@@ -31,6 +31,8 @@ DEFINE_boolean generate_pkginfo "${FLAGS_FALSE}" \
   "Generate an additional squashfs '<sysext_name>_pkginfo.raw' with portage package meta-information (/var/db ...). Useful for creating sysext dependencies; see 'base_pkginfo' below."
 DEFINE_string base_pkginfo "" \
   "Colon-separated list of pkginfo squashfs paths / files generated via 'generate_pkginfo' to base this sysext on. The corresponding base sysexts are expected to be merged with the sysext generated."
+DEFINE_string compression "zstd" \
+  "Compression to use for sysext squashfs. One of 'gzip', 'lzo', 'lz4', 'xz', or 'zstd'. Must be supported by the Flatcar squashfs kernel module in order for the sysext to work."
 DEFINE_boolean ignore_version_mismatch "${FLAGS_FALSE}" \
   "Ignore version mismatch between SDK board packages and base squashfs. DANGEROUS."
 
@@ -193,7 +195,7 @@ if [[ ${#} -lt 1 ]]; then
    show_help_if_requested -h
 fi
 
-info "Building '${SYSEXTNAME}' with (meta-)packages '${@}' in '${BUILD_DIR}'".
+info "Building '${SYSEXTNAME}' squashfs with (meta-)packages '${@}' in '${BUILD_DIR}' using '${FLAGS_compression}' compression".
 
 for package; do
   echo "Installing package into sysext image: $package"
@@ -211,6 +213,13 @@ done
 umount "${BUILD_DIR}/install-root"
 umount "${BUILD_DIR}/fs-root"
 
+if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
+  info "  Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'"
+  mkdir -p "${BUILD_DIR}/img-pkginfo/var/db"
+  cp -R "${BUILD_DIR}/install-root/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/"
+  mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend -comp "${FLAGS_compression}"
+fi
+
 if [[ -n "${FLAGS_manglefs_script}" ]]; then
   if [[ ! -x "${FLAGS_manglefs_script}" ]]; then
     die "${FLAGS_manglefs_script} is not executable"
@@ -218,13 +227,6 @@ if [[ -n "${FLAGS_manglefs_script}" ]]; then
   "${FLAGS_manglefs_script}" "${BUILD_DIR}/install-root"
 fi
 
-if [[ "$FLAGS_generate_pkginfo" = "${FLAGS_TRUE}" ]] ; then
-  info "  Creating pkginfo squashfs '${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw'"
-  mkdir -p "${BUILD_DIR}/img-pkginfo/var/db"
-  cp -R "${BUILD_DIR}/install-root/var/db/pkg" "${BUILD_DIR}/img-pkginfo/var/db/"
-  mksquashfs "${BUILD_DIR}/img-pkginfo" "${BUILD_DIR}/${SYSEXTNAME}_pkginfo.raw" -noappend
-fi
-
 info "Writing ${SYSEXTNAME}_packages.txt"
 ROOT="${BUILD_DIR}/install-root" PORTAGE_CONFIGROOT="${BUILD_DIR}/install-root"\
       equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXTNAME}_packages.txt"
@@ -245,7 +247,7 @@ all_fields=(
   "ARCHITECTURE=${ARCH}"
 )
 printf '%s\n' "${all_fields[@]}" >"${BUILD_DIR}/install-root/usr/lib/extension-release.d/extension-release.${SYSEXTNAME}"
-mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend
+mksquashfs "${BUILD_DIR}/install-root" "${BUILD_DIR}/${SYSEXTNAME}.raw" -noappend -comp "${FLAGS_compression}"
 rm -rf "${BUILD_DIR}"/{fs-root,install-root,workdir}
 
 # Generate reports

ci-automation/ci_automation_common.sh

@@ -193,7 +193,7 @@ function docker_image_from_buildcache() {
     local id_file_url_release="https://mirror.release.flatcar-linux.net/containers/${version}/${id_file}"
 
     local local_image=""
-    if image_exists_locally "/${name}" "${version}" ; then
+    if image_exists_locally "${name}" "${version}" ; then
         local_image="${name}:${version}"
     elif image_exists_locally "${CONTAINER_REGISTRY}/${name}" "${version}" ; then
         local_image="${CONTAINER_REGISTRY}/${name}:${version}"

sdk_container/src/third_party/coreos-overlay/sys-kernel/coreos-modules/files/commonconfig-6.1

@@ -854,6 +854,7 @@ CONFIG_SQUASHFS_LZ4=y
 CONFIG_SQUASHFS_LZO=y
 CONFIG_SQUASHFS_XATTR=y
 CONFIG_SQUASHFS_XZ=y
+CONFIG_SQUASHFS_ZSTD=y
 CONFIG_STACKPROTECTOR_STRONG=y
 CONFIG_STRICT_DEVMEM=y
 CONFIG_STRIP_ASM_SYMS=y