From 19ae51910b84042dd88fd3bf3e170c15eb0be4df Mon Sep 17 00:00:00 2001 From: Benjamin Gilbert Date: Fri, 8 Nov 2019 17:59:16 -0500 Subject: [PATCH 01/10] test_image_content: whitelist OpenSSH GLSA It only affects a default-disabled USE flag, and our overlay ebuild is too old to even support it. --- build_library/test_image_content.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/build_library/test_image_content.sh b/build_library/test_image_content.sh index 260acc0c61..3b1075c8ef 100644 --- a/build_library/test_image_content.sh +++ b/build_library/test_image_content.sh @@ -7,6 +7,7 @@ GLSA_WHITELIST=( 201908-14 # backported both CVE fixes 201909-01 # Perl, SDK only 201909-08 # backported fix + 201911-01 # package too old to even have the affected USE flag ) glsa_image() { From 0ac4e27840f7153e1b43df1cc16e19930c3c65a9 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Fri, 15 Nov 2019 13:46:29 +0100 Subject: [PATCH 02/10] build_library: replace coreos with flatcar in disk layout To make GPT partition type strings include flatcar- strings instead of coreos-, we need to change the disk layout file. --- build_library/disk_layout.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build_library/disk_layout.json b/build_library/disk_layout.json index 6c91ae736a..84b452c12d 100644 --- a/build_library/disk_layout.json +++ b/build_library/disk_layout.json @@ -26,7 +26,7 @@ "3":{ "label":"USR-A", "uuid":"7130c94a-213a-4e5a-8e26-6cce9662f132", - "type":"coreos-rootfs", + "type":"flatcar-rootfs", "blocks":"2097152", "fs_blocks":"260094", "fs_type":"ext2", @@ -36,7 +36,7 @@ "4":{ "label":"USR-B", "uuid":"e03dd35c-7c2d-4a47-b3fe-27f15780a57c", - "type":"coreos-rootfs", + "type":"flatcar-rootfs", "blocks":"2097152", "fs_blocks":"262144" }, @@ -56,18 +56,18 @@ }, "7":{ "label":"OEM-CONFIG", - "type":"coreos-reserved", + "type":"flatcar-reserved", "blocks":"131072" }, "8":{ "type":"blank", - "label":"coreos-reserved", + "label":"flatcar-reserved", "blocks":"0" }, "9":{ "label":"ROOT", "fs_label":"ROOT", - "type":"coreos-resize", + "type":"flatcar-resize", "blocks":"4427776", "fs_type":"ext4", "mount":"/" From 45bb2c039bfcfdcb57c4d88e727004443894e707 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Mon, 18 Nov 2019 13:32:29 +0100 Subject: [PATCH 03/10] build_library: replace coreos-usr with flatcar-usr in disk_util The disk_util also needs an update for the GPT type string. --- build_library/disk_util | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build_library/disk_util b/build_library/disk_util index 0e79976860..42b2880c53 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -400,7 +400,7 @@ def FormatExt(part, device): if 'fs_label' in part: tune_cmd += ['-L', part['fs_label']] - if part['type'] == 'coreos-usr': + if part['type'] == 'flatcar-usr': tune_cmd += ['-U', 'clear', '-T', '20091119110000', '-c', '0', '-i', '0', # Disable auto fsck From da050d7ed1696f8896f7ecb3204280b94c4d1cc5 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Fri, 29 Nov 2019 08:07:11 +0100 Subject: [PATCH 04/10] build_library: make toolchain build rust for cross-builds To make packages-matrix build rust sources for arm64-usr, we need to make toolchain include the rust target for arm. Unfortunately, catalyst does not support the cross-build rust target by default. We also cannot add `dev-lang/rust` to `$TOOLCHAIN_PKGS`, because then catalyst would tries to build a `cross-*/rust` package, which does not exist. So we need to manually run emerge for rust, without `--newuse -u` options. If the `--newuse -u` options are given, the rust will not be installed at all. --- build_library/catalyst_toolchains.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build_library/catalyst_toolchains.sh b/build_library/catalyst_toolchains.sh index 2ab4c46dae..f5ef591e5f 100644 --- a/build_library/catalyst_toolchains.sh +++ b/build_library/catalyst_toolchains.sh @@ -35,6 +35,11 @@ build_target_toolchain() { # --root is required because run_merge overrides ROOT= PORTAGE_CONFIGROOT="$ROOT" \ run_merge -u --root="$ROOT" --sysroot="$ROOT" "${TOOLCHAIN_PKGS[@]}" + + export clst_myemergeopts="$( echo "$clst_myemergeopts" | sed -e 's/--newuse//' )" + + PORTAGE_CONFIGROOT="$ROOT" \ + run_merge --root="$ROOT" --sysroot="$ROOT" dev-lang/rust } configure_crossdev_overlay / /tmp/crossdev From 3e979ed7f1f2e3fb6bec42d36a75d37fe89e6508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Mon, 2 Dec 2019 15:28:06 +0100 Subject: [PATCH 05/10] build_library/toolchain_util.sh: Install Rust from cross-compilation toolchain --- build_library/toolchain_util.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 5ab2c6e470..0951d7d971 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -324,6 +324,15 @@ install_cross_toolchain() { else $sudo emerge "${emerge_flags[@]}" \ "cross-${cross_chost}/gdb" "${cross_pkgs[@]}" + if [ "${cross_chost}" = aarch64-cros-linux-gnu ]; then + # Here we need to take only the binary packages from the toolchain builds + # because the standard Rust packages don't include the arm64 cross target. + # Building from source is ok because the cross-compiler got installed. + FILTERED="$(echo $PORTAGE_BINHOST | tr ' ' '\n' | grep toolchain | xargs echo)" + # If no aarch64 folder exists, try to remove any existing Rust packages. + [ ! -d /usr/lib/rust-*/rustlib/aarch64-unknown-linux-gnu ] && ($sudo emerge -C dev-lang/rust || true) + $sudo PORTAGE_BINHOST="$FILTERED" emerge "${emerge_flags[@]}" dev-lang/rust + fi fi # Setup environment and wrappers for our shiny new toolchain From eb1decdb9ec4e25fea40d2ccf66d7996a1fb96f0 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Thu, 12 Dec 2019 12:48:56 +0100 Subject: [PATCH 06/10] jenkins: add ami_vmdk to format list for ARM The `image-matrix.groovy` script reads the list of image formats for ARM, from `formats-arm64-usr.txt` So we need to also add ami_vmdk to the list to publish AWS ARM images. --- jenkins/formats-arm64-usr.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/formats-arm64-usr.txt b/jenkins/formats-arm64-usr.txt index fe6f2e2598..12e131e462 100644 --- a/jenkins/formats-arm64-usr.txt +++ b/jenkins/formats-arm64-usr.txt @@ -1,3 +1,4 @@ +ami_vmdk openstack openstack_mini packet From 37da2494a2a580c7a023e59b1cb275add0e2d25f Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Fri, 13 Dec 2019 00:15:49 +0000 Subject: [PATCH 07/10] build_library/disk_util: work around losetup bug Retry losetup if it fails, up to 5 times with 5 seconds between retries. --- build_library/disk_util | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 0e79976860..2930885941 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -11,6 +11,7 @@ import re import subprocess import sys import tempfile +import time import uuid # First sector we can use. @@ -433,11 +434,22 @@ def FormatFat(part, device): def PartitionLoop(options, partition): """Allocate (and automatically free) loop devices for a partition.""" - loop_dev = subprocess.check_output(['sudo', 'losetup', - '--offset', str(partition['first_byte']), - '--sizelimit', str(partition['bytes']), - '--find', '--show', options.disk_image]) - loop_dev = loop_dev.strip() + for i in range(0,5): + try: + loop_dev = subprocess.check_output(['sudo', 'losetup', + '--offset', str(partition['first_byte']), + '--sizelimit', str(partition['bytes']), + '--find', '--show', options.disk_image]) + loop_dev = loop_dev.strip() + err = None + break + except subprocess.CalledProcessError as error: + print("Failed to set up loopback, attempt %d" % i) + err = error + time.sleep(5) + + if err is not None: + raise err try: yield loop_dev From c0ed59f90204e6e732644852875f888d1f540f74 Mon Sep 17 00:00:00 2001 From: Andrew Jeddeloh Date: Mon, 16 Dec 2019 23:08:33 +0000 Subject: [PATCH 08/10] disk_util: retry loopback mounts if they fail This fails frequently but should succeed if retried. This should reduce failed builds. --- build_library/disk_util | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/build_library/disk_util b/build_library/disk_util index 2930885941..21c7c6b77d 100755 --- a/build_library/disk_util +++ b/build_library/disk_util @@ -598,9 +598,22 @@ def Mount(options): mount_opts.append('subvol=%s' % mount['fs_subvolume']) Sudo(['mkdir', '-p', full_path]) - Sudo(['mount', '-t', mount.get('fs_type', 'auto'), - '-o', ','.join(mount_opts), - options.disk_image, full_path]) + # This tends to fail, retry if it does + err = None + for i in range(0,5): + try: + Sudo(['mount', '-t', mount.get('fs_type', 'auto'), + '-o', ','.join(mount_opts), + options.disk_image, full_path]) + err = None + break + except subprocess.CalledProcessError as e: + print("Error mounting %s, attempt %d" % (full_path, i)) + err = e + time.sleep(5) + + if err is not None: + raise err for src, dst in mount.get('binds', {}).iteritems(): # src may be relative or absolute, os.path.join handles this. From 4f9514139c199d123454c8bd42f59e69ab45b202 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kai=20L=C3=BCke?= Date: Wed, 22 Jan 2020 10:42:12 +0100 Subject: [PATCH 09/10] Use same APPID for arm64 and amd64 Nebraska supports an additional arch variable to distinguish both boards. --- build_library/set_lsb_release | 5 ----- signing/transfer.sh | 16 +++++++--------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release index c6a680135d..97b278c8e9 100755 --- a/build_library/set_lsb_release +++ b/build_library/set_lsb_release @@ -31,12 +31,7 @@ OS_ID="flatcar" OS_ID_LIKE="coreos" OS_PRETTY_NAME="$OS_NAME $FLATCAR_VERSION (${OS_CODENAME})" -# temporarily point arm at a different appid until update support -# is completed in update_engine and the CoreUpdate service. FLATCAR_APPID="{e96281a6-d1af-4bde-9a0a-97b76e56dc57}" -if [[ "${FLAGS_board}" == arm64-* ]]; then - FLATCAR_APPID="{103867da-e3a2-4c92-b0b3-7fbd7f7d8b71}" -fi # DISTRIB_* are the standard lsb-release names sudo mkdir -p "${ROOT_FS_DIR}/usr/share/flatcar" "${ROOT_FS_DIR}/etc/flatcar" diff --git a/signing/transfer.sh b/signing/transfer.sh index 361146c773..c308fc48d4 100755 --- a/signing/transfer.sh +++ b/signing/transfer.sh @@ -2,9 +2,7 @@ set -eux -declare -A APPID -APPID[amd64-usr]=e96281a6-d1af-4bde-9a0a-97b76e56dc57 -APPID[arm64-usr]=103867da-e3a2-4c92-b0b3-7fbd7f7d8b71 +APPID=e96281a6-d1af-4bde-9a0a-97b76e56dc57 declare -A RELEASE_CHANNEL RELEASE_CHANNEL[alpha]=Alpha @@ -92,7 +90,7 @@ upload() { "$(dirname $0)/../core_roller_upload" \ --user="${ROLLER_USERNAME}" \ --api_key="${ROLLER_API_KEY}" \ - --app_id="${APPID[${board}]}" \ + --app_id="${APPID}" \ --board="${board}" \ --version="${version}" \ --payload="${payload}" @@ -111,7 +109,7 @@ upload() { --user="${ROLLER_USERNAME}" \ --key="${ROLLER_API_KEY}" \ channel update \ - --app-id="${APPID[${board}]}" \ + --app-id="${APPID}" \ --channel="${canary_channel}" \ --version="${version}" fi @@ -130,7 +128,7 @@ ready() { --user="${ROLLER_USERNAME}" \ --key="${ROLLER_API_KEY}" \ group percent \ - --app-id="${APPID[${board}]}" \ + --app-id="${APPID}" \ --group-id="${channel}" \ --update-percent=0 @@ -141,7 +139,7 @@ ready() { --user="${ROLLER_USERNAME}" \ --key="${ROLLER_API_KEY}" \ channel update \ - --app-id="${APPID[${board}]}" \ + --app-id="${APPID}" \ --channel="${RELEASE_CHANNEL[${channel}]}" \ --publish=true \ --version="${version}" @@ -162,7 +160,7 @@ roll() { --user="${ROLLER_USERNAME}" \ --key="${ROLLER_API_KEY}" \ group percent \ - --app-id="${APPID[${board}]}" \ + --app-id="${APPID}" \ --group-id="${channel}" \ --update-percent=100 else @@ -174,7 +172,7 @@ roll() { --user="${ROLLER_USERNAME}" \ --key="${ROLLER_API_KEY}" \ rollout create linear \ - --app-id="${APPID[${board}]}" \ + --app-id="${APPID}" \ --group-id="${channel}" \ --duration="${seconds}" \ --frame-size="60" From fc835de3aa638bda9fe23283f540a10550b3d255 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Tue, 21 Jan 2020 16:09:46 +0100 Subject: [PATCH 10/10] build_library: fix sync URI under Gentoo env for developer containers When running `emerge-gitclone` in a developer container, it still tries to fetch from coreos repos. We should make it fetch from flatcar-linux repos. --- build_library/dev_container_util.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 7c228cae60..6257befde9 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -39,12 +39,12 @@ disabled = true [coreos] location = /var/lib/portage/coreos-overlay sync-type = git -sync-uri = https://github.com/coreos/coreos-overlay.git +sync-uri = https://github.com/flatcar-linux/coreos-overlay.git [portage-stable] location = /var/lib/portage/portage-stable sync-type = git -sync-uri = https://github.com/coreos/portage-stable.git +sync-uri = https://github.com/flatcar-linux/portage-stable.git EOF # Now set the correct profile