mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-10 06:26:57 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #355 from flatcar-linux/kai/image-package-diff

Browse Source 
ci-automation: Use the package container for VM image building
This commit is contained in:
Kai Lüke 2022-06-29 15:37:02 +02:00 committed by GitHub
commit eb02ef6788
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 87 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
ci-automation/ci_automation_common.sh
View File

@ -122,6 +122,18 @@ function gen_sshcmd() {
}
# --
function copy_dir_from_buildcache() {
local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
local local_path="$2"
local sshcmd="$(gen_sshcmd)"
mkdir -p "${local_path}"
rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \
"${local_path}"
}
# --
function copy_to_buildcache() {
local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
shift

54
ci-automation/image.sh
View File

@ -35,10 +35,7 @@
#
# OUTPUT:
#
# 1. Exported container image with OS image, dev container, and related artifacts at
# /home/sdk/image/[ARCH], torcx packages at /home/sdk/torcx
# named "flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
# pushed to buildcache.
# 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache.
# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
# to be run after this step finishes / when this step is aborted.
# 3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
@ -85,7 +82,7 @@ function _image_build_impl() {
official_arg="--noofficial"
fi
# build image and store it in the container
# build image and related artifacts
./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
-v "${vernum}" \
mkdir -p "${CONTAINER_IMAGE_ROOT}"
@ -98,7 +95,50 @@ function _image_build_impl() {
--output_root="${CONTAINER_IMAGE_ROOT}" \
--torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
# rename container and push to build cache
docker_commit_to_buildcache "${image_container}" "${image}" "${docker_vernum}"
# copy resulting images + push to buildcache
local images_out="images/"
rm -rf "${images_out}"
./run_sdk_container -n "${image_container}" -C "${packages_image}" \
-v "${vernum}" \
mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
sign_artifacts "${SIGNER}" "images/latest/"*
copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
(
set +x
# Don't fail the whole job
set +e
echo "==================================================================="
export BOARD_A="${arch}-usr"
export FROM_A="release"
export VERSION_A="current"
if [ "${channel}" = "developer" ]; then
export CHANNEL_A="alpha"
else
export CHANNEL_A="${channel}"
fi
export FROM_B="file://${PWD}/images/latest"
# Use the directory directly (and BOARD_B and CHANNEL_B are unused)
export VERSION_B="."
echo "== Image differences compared to ${CHANNEL_A} ${VERSION_A} =="
rm -f package-diff
curl -fsSLO --retry-delay 1 --retry 60 --retry-connrefused --retry-max-time 60 --connect-timeout 20 "https://raw.githubusercontent.com/flatcar-linux/flatcar-build-scripts/master/package-diff"
chmod +x package-diff
echo "Package updates, compared to ${CHANNEL_A} ${VERSION_A}:"
FILE=flatcar_production_image_packages.txt ./package-diff "${VERSION_A}" "${VERSION_B}"
echo
echo "Image file changes, compared to ${CHANNEL_A} ${VERSION_A}:"
FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 ./package-diff "${VERSION_A}" "${VERSION_B}"
echo
echo "Image kernel config changes, compared to ${CHANNEL_A} ${VERSION_A}:"
FILE=flatcar_production_image_kernel_config.txt ./package-diff "${VERSION_A}" "${VERSION_B}"
echo
echo "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${CHANNEL_A} ${VERSION_A}:"
FILE=flatcar_production_image_contents.txt CALCSIZE=1 ./package-diff "${VERSION_A}" "${VERSION_B}"
echo
BASE_URL="http://${BUILDCACHE_SERVER}/images/${arch}/${vernum}"
echo "Image URL: ${BASE_URL}/flatcar_production_image.bin.bz2"
)
}
# --

49
ci-automation/vms.sh
View File

@ -9,15 +9,16 @@
# vm_build() should be called w/ the positional INPUT parameters below.
# Vendor images build automation stub.
# This script will build one or more vendor images ("vm") using a pre-built image container.
# This script will build one or more vendor images ("vm") using a pre-built packages container.
#
# PREREQUISITES:
#
# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
# 2. Scripts repo version tag of OS image version to be built is available and checked out.
# 3. Flatcar image container is available via build cache server
# 3. Flatcar packages container is available via build cache server
# from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
# or present locally. Must contain packages and image.
# or present locally. Must contain packages.
# 4. The generic Flatcar image must be present in build cache server.
#
# INPUT:
#
@ -66,19 +67,13 @@ function _vm_build_impl() {
local vernum="${FLATCAR_VERSION}"
local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
local image="flatcar-images-${arch}"
local image_image="${image}:${docker_vernum}"
local vms_container="flatcar-vms-${docker_vernum}"
local packages="flatcar-packages-${arch}"
local packages_image="${packages}:${docker_vernum}"
docker_image_from_buildcache "${image}" "${docker_vernum}"
docker_image_from_buildcache "${packages}" "${docker_vernum}"
# clean up dangling containers from previous builds
docker container rm -f "${vms_container}" || true
local images_out="images/"
rm -rf "${images_out}"
echo "docker container rm -f '${vms_container}'" >> ci-cleanup.sh
local vms="flatcar-vms-${arch}"
local vms_container="${vms}-${docker_vernum}"
# automatically add PXE to formats if we build for Equinix Metal (packet).
local has_packet=0
@ -103,26 +98,38 @@ function _vm_build_impl() {
# Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
local images_in="images-in/"
rm -rf "${images_in}"
copy_dir_from_buildcache "images/${arch}/${vernum}/" "${images_in}"
./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \
-v "${vernum}" \
mkdir -p "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest"
./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
-v "${vernum}" \
mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input"
for format in ${formats}; do
echo " ################### VENDOR '${format}' ################### "
COMPRESSION_FORMAT="bz2"
if [[ "${format}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then
COMPRESSION_FORMAT="gz,bz2"
fi
./run_sdk_container -n "${vms_container}" -C "${image_image}" \
./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
-v "${vernum}" \
./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
--from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
--from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" \
--to "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
--image_compression_formats="${COMPRESSION_FORMAT}"
done
# copy resulting images + push to buildcache
./run_sdk_container -n "${vms_container}" \
local images_out="images/"
rm -rf "${images_out}"
./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
-v "${vernum}" \
cp --reflink=auto -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
cd "images/latest"
sign_artifacts "${SIGNER}" *
copy_to_buildcache "images/${arch}/${vernum}/" *
sign_artifacts "${SIGNER}" "images/latest/"*
copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
}
# --