ci-automation: Use the package container for VM image building

The image job builds an image container that is multiple GBs big and
takes >10 mins to be loaded in the vms job. The vms job can also do its
work by running from the packages container from the packages job when
it fetchs the built image from bincache first and assuming the images
job copies it there.
Skip generating the image container and instead use the packages
container for VM image building by copying the image folder first to
bincache and then retrieving it from there. While reworking this we
also address the issue that the VMs container had used the same name
for both architectures, causing a race when both run in parallel on
the same worker.

ci-automation/ci_automation_common.sh

@@ -122,6 +122,18 @@ function gen_sshcmd() {
 }
 # --
 
+function copy_dir_from_buildcache() {
+    local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
+    local local_path="$2"
+
+    local sshcmd="$(gen_sshcmd)"
+    mkdir -p "${local_path}"
+    rsync --partial -a -e "${sshcmd}" "${BUILDCACHE_USER}@${BUILDCACHE_SERVER}:${remote_path}" \
+        "${local_path}"
+}
+
+# --
+
 function copy_to_buildcache() {
     local remote_path="${BUILDCACHE_PATH_PREFIX}/$1"
     shift

ci-automation/image.sh

@@ -35,10 +35,7 @@
 #
 # OUTPUT:
 #
-#   1. Exported container image with OS image, dev container, and related artifacts at
-#        /home/sdk/image/[ARCH], torcx packages at /home/sdk/torcx
-#        named "flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
-#        pushed to buildcache.
+#   1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache.
 #   2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
 #        to be run after this step finishes / when this step is aborted.
 #   3. If signer key was passed, signatures of artifacts from point 1, pushed along to buildcache.
@@ -85,7 +82,7 @@ function _image_build_impl() {
             official_arg="--noofficial"
     fi
 
-    # build image and store it in the container
+    # build image and related artifacts
     ./run_sdk_container -x ./ci-cleanup.sh -n "${image_container}" -C "${packages_image}" \
             -v "${vernum}" \
             mkdir -p "${CONTAINER_IMAGE_ROOT}"
@@ -98,7 +95,14 @@ function _image_build_impl() {
                           --output_root="${CONTAINER_IMAGE_ROOT}" \
                           --torcx_root="${CONTAINER_TORCX_ROOT}" prodtar container
 
-    # rename container and push to build cache
-    docker_commit_to_buildcache "${image_container}" "${image}" "${docker_vernum}"
+    # copy resulting images + push to buildcache
+    local images_out="images/"
+    rm -rf "${images_out}"
+    ./run_sdk_container -n "${image_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
+
+    sign_artifacts "${SIGNER}" "images/latest/"*
+    copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
 }
 # --

ci-automation/vms.sh

@@ -9,15 +9,16 @@
 # vm_build() should be called w/ the positional INPUT parameters below.
 
 # Vendor images build automation stub.
-#   This script will build one or more vendor images ("vm") using a pre-built image container.
+#   This script will build one or more vendor images ("vm") using a pre-built packages container.
 #
 # PREREQUISITES:
 #
 #   1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
 #   2. Scripts repo version tag of OS image version to be built is available and checked out.
-#   3. Flatcar image container is available via build cache server
+#   3. Flatcar packages container is available via build cache server
 #       from "/containers/[VERSION]/flatcar-images-[ARCH]-[FLATCAR_VERSION].tar.gz"
-#       or present locally. Must contain packages and image.
+#       or present locally. Must contain packages.
+#   4. The generic Flatcar image must be present in build cache server.
 #
 # INPUT:
 #
@@ -66,19 +67,13 @@ function _vm_build_impl() {
     local vernum="${FLATCAR_VERSION}"
     local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
 
-    local image="flatcar-images-${arch}"
-    local image_image="${image}:${docker_vernum}"
-    local vms_container="flatcar-vms-${docker_vernum}"
+    local packages="flatcar-packages-${arch}"
+    local packages_image="${packages}:${docker_vernum}"
 
-    docker_image_from_buildcache "${image}" "${docker_vernum}"
+    docker_image_from_buildcache "${packages}" "${docker_vernum}"
 
-    # clean up dangling containers from previous builds
-    docker container rm -f "${vms_container}" || true
-
-    local images_out="images/"
-    rm -rf "${images_out}"
-
-    echo "docker container rm -f '${vms_container}'" >> ci-cleanup.sh
+    local vms="flatcar-vms-${arch}"
+    local vms_container="${vms}-${docker_vernum}"
 
     # automatically add PXE to formats if we build for Equinix Metal (packet).
     local has_packet=0
@@ -103,26 +98,38 @@ function _vm_build_impl() {
     # Keep compatibility with SDK scripts where "equinix_metal" remains unknown.
     formats=$(echo "$formats" | tr ' ' '\n' | sed 's/equinix_metal/packet/g')
 
+    local images_in="images-in/"
+    rm -rf "${images_in}"
+    copy_dir_from_buildcache "images/${arch}/${vernum}/" "${images_in}"
+    ./run_sdk_container -x ./ci-cleanup.sh -n "${vms_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mkdir -p "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest"
+    ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
+            -v "${vernum}" \
+            mv "${images_in}" "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input"
+
     for format in ${formats}; do
         echo " ###################  VENDOR '${format}' ################### "
         COMPRESSION_FORMAT="bz2"
         if [[ "${format}" =~ ^(openstack|openstack_mini|digitalocean)$ ]];then
             COMPRESSION_FORMAT="gz,bz2"
         fi
-        ./run_sdk_container -n "${vms_container}" -C "${image_image}" \
+        ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
             -v "${vernum}" \
             ./image_to_vm.sh --format "${format}" --board="${arch}-usr" \
-                --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
+                --from "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest-input" \
+                --to "${CONTAINER_IMAGE_ROOT}/${arch}-usr/latest" \
                 --image_compression_formats="${COMPRESSION_FORMAT}"
     done
 
     # copy resulting images + push to buildcache
-    ./run_sdk_container -n "${vms_container}" \
+    local images_out="images/"
+    rm -rf "${images_out}"
+    ./run_sdk_container -n "${vms_container}" -C "${packages_image}" \
         -v "${vernum}" \
-        cp --reflink=auto -R "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
+        mv "${CONTAINER_IMAGE_ROOT}/${arch}-usr/" "./${images_out}/"
 
-    cd "images/latest"
-    sign_artifacts "${SIGNER}" *
-    copy_to_buildcache "images/${arch}/${vernum}/" *
+    sign_artifacts "${SIGNER}" "images/latest/"*
+    copy_to_buildcache "images/${arch}/${vernum}/" "images/latest/"*
 }
 # --