From b2a4a5f0d767ac49ec684d7a838732ff6f515e1c Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Fri, 23 Jun 2023 11:56:30 +0200 Subject: [PATCH 1/3] oem_sysext_util: Mount overlay on top of /usr before installing sysext packages After changes to the inode size, the sysext installation runs out of space because the installation happens on a mounted production image. This is problematic because the /usr partition is only 1024MB in size and gets full. Mount a temporary overlay so that we can use that for installation, and discard it afterwards. This also means we no longer need to disable verity and in fact could live without copying the prod image. I won't make that change since we're working on a new script to automate building of sysexts using the overlay approach. Signed-off-by: Jeremi Piotrowski --- build_library/oem_sysext_util.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/build_library/oem_sysext_util.sh b/build_library/oem_sysext_util.sh index 21e014db48..e5e057bdcd 100755 --- a/build_library/oem_sysext_util.sh +++ b/build_library/oem_sysext_util.sh @@ -68,6 +68,8 @@ oem_sysext_create() { local sysext_work_dir="${work_dir}/sysext-${oem}" local prod_rw_image="${sysext_work_dir}/prod_for_sysext.bin" local prod_rw_rootfs="${sysext_work_dir}/prod_rw_rootfs" + local sysext_overlay_work="${sysext_work_dir}/overlay.work" + local sysext_overlay_upper="${sysext_work_dir}/overlay.upper" local cleanup_actions=() trap '_invoke_actions "${cleanup_actions[@]}"' EXIT @@ -82,21 +84,29 @@ oem_sysext_create() { info 'Preparing work image for mounting' "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ tune --randomize_uuid "${prod_rw_image}" OEM - "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ - tune --enable2fs_rw "${prod_rw_image}" USR-A info "Mounting work image to ${prod_rw_rootfs}" _prepend_action cleanup_actions rmdir "${prod_rw_rootfs}" _prepend_action cleanup_actions "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ umount "${prod_rw_rootfs}" "${BUILD_LIBRARY_DIR}/disk_util" --disk_layout=base \ - mount --writable_verity "${prod_rw_image}" "${prod_rw_rootfs}" + mount "${prod_rw_image}" "${prod_rw_rootfs}" local initial_files="${sysext_work_dir}/initial_files" info "Generating list of initial files in work image" _prepend_action cleanup_actions rm -f "${initial_files}" _generate_listing "${prod_rw_rootfs}" "${initial_files}" + # /usr partition may be too small to fit the sysext files, so mount + # an overlay temporarily. + _prepend_action cleanup_actions sudo rm -rf "${sysext_overlay_work}" "${sysext_overlay_upper}" + mkdir -p "${sysext_overlay_work}" "${sysext_overlay_upper}" + + _prepend_action cleanup_actions sudo umount "${prod_rw_rootfs}/usr" + sudo mount -t overlay usr-overlay \ + -o lowerdir="${prod_rw_rootfs}/usr",upperdir="${sysext_overlay_upper}",workdir="${sysext_overlay_work}" \ + "${prod_rw_rootfs}/usr" + info "Stuffing package database into into ${prod_rw_rootfs}" sudo tar -xf "${prod_pkgdb}" -C "${prod_rw_rootfs}" From 6ebbe5f2b9923b59f571100d7df58492ce205359 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Mon, 26 Jun 2023 12:12:29 +0200 Subject: [PATCH 2/3] ci-automation: Align CONTAINER_*_ROOT with usual location The SDK container bind mounts __build__/images to the containers image directory, but the CI uses a different path for images. This causes issues when building the oem sysext, because it requires mounting an overlayfs. The current path (~/build/...) is an overlayfs and an ovlerayfs can't be an upper directory. Align the CONTAINER_IMAGE_ROOT and CONTAINER_TORCX_ROOT values with standard practices to that oem sysext building in jenkins ci works. Signed-off-by: Jeremi Piotrowski --- ci-automation/ci-config.env | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci-automation/ci-config.env b/ci-automation/ci-config.env index 5ddc05accc..5773a4c1c6 100644 --- a/ci-automation/ci-config.env +++ b/ci-automation/ci-config.env @@ -28,8 +28,8 @@ CI_GIT_AUTHOR="flatcar-ci" CI_GIT_EMAIL="infra+ci@flatcar-linux.org" # build artifacts go here (in container) -CONTAINER_TORCX_ROOT="/home/sdk/build/torcx" -CONTAINER_IMAGE_ROOT="/home/sdk/build/images" +CONTAINER_TORCX_ROOT="/home/sdk/trunk/src/build/torcx" +CONTAINER_IMAGE_ROOT="/home/sdk/trunk/src/build/images" # Set it to "1" or "true" or "t" or "y" or "yes" to always run a full # nightly build. Any other value will allow build shortcuts. From 1fc599f33c18dc7b71525635d9ee4a5d9e32ddd7 Mon Sep 17 00:00:00 2001 From: Jeremi Piotrowski Date: Tue, 27 Jun 2023 11:19:29 +0200 Subject: [PATCH 3/3] ci-automation: Publish torcx_output_root to bincache `build_image` depends on accesss to the torcx manifest and the "content addressable nature" of the directory. We currently rely on the torcx output root structure being preserved in the container image. While we're moving the torcx output root out of the container image, preserve its contents so that they can be restored from bincache. --- ci-automation/image.sh | 8 +++++++- ci-automation/packages.sh | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ci-automation/image.sh b/ci-automation/image.sh index f45b78eaac..92e9a22cd5 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -84,11 +84,17 @@ function _image_build_impl() { official_arg="--noofficial" fi + local torcx_root_tar="torcx_root.tar.zst" apply_local_patches + copy_from_buildcache "images/${arch}/${vernum}/torcx/${torcx_root_tar}" . + # build image and related artifacts ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ - mkdir -p "${CONTAINER_IMAGE_ROOT}" + mkdir -p "${CONTAINER_IMAGE_ROOT}" "${CONTAINER_TORCX_ROOT}" + ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ + -v "${vernum}" \ + tar --zstd -xf "${torcx_root_tar}" -C "${CONTAINER_TORCX_ROOT}" ./run_sdk_container -n "${image_container}" -C "${packages_image}" \ -v "${vernum}" \ ./set_official --board="${arch}-usr" "${official_arg}" diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index add6fc3c61..1d4faf42c8 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -123,6 +123,11 @@ function _packages_build_impl() { # generate image + push to build cache docker_commit_to_buildcache "${packages_container}" "${packages_image}" "${docker_vernum}" + # publish torcx output root for consumption by build_image + local torcx_root_tar="torcx_root.tar.zst" + tar --zstd -cpf "${torcx_root_tar}" -C "${torcx_tmp}/torcx" . + copy_to_buildcache "images/${arch}/${vernum}/torcx" "${torcx_root_tar}" + # Publish torcx manifest and docker tarball to "images" cache so tests can pull it later. create_digests "${SIGNER}" \ "${torcx_tmp}/torcx/${arch}-usr/latest/torcx_manifest.json" \