From f104a53aeb325d6672114afe6ee791753fe65d62 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Fri, 28 Aug 2020 13:51:22 -0400 Subject: [PATCH 01/13] build_packages: export build host information Signed-off-by: Vincent Batts --- build_packages | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build_packages b/build_packages index 2aeea80033..7933ec8c9f 100755 --- a/build_packages +++ b/build_packages @@ -235,6 +235,9 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then break_dep_loop sys-apps/systemd cryptsetup fi +export KBUILD_BUILD_USER="${BUILD_USER:-build}" +export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}" + info "Merging board packages now" sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@" From a43fcbb5341f8b88619d40027b7155f4fad5940e Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 1 Sep 2020 16:58:54 -0400 Subject: [PATCH 02/13] contrib: allow user to provide cloud-config template Respecting that substitutions will still be made, the user may want to also install their own unit files or similar Signed-off-by: Vincent Batts --- contrib/create-basic-configdrive | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/contrib/create-basic-configdrive b/contrib/create-basic-configdrive index ece7849c6c..13b904ecc5 100755 --- a/contrib/create-basic-configdrive +++ b/contrib/create-basic-configdrive @@ -15,6 +15,7 @@ Options: -l http://IP:PORT Listen URL for client communication. -u http://IP:PORT Listen URL for server communication. -n NAME etcd node name. + -c FILE cloud-config template to use, instead of the default. -p DEST Create config-drive ISO image to the given path. -S FILE SSH keys file. -t TOKEN Token ID from https://discovery.etcd.io. @@ -44,11 +45,12 @@ hostname: " REGEX_SSH_FILE="^ssh-(rsa|dss|ed25519) [-A-Za-z0-9+\/]+[=]{0,2} .+" -while getopts "d:e:H:i:n:p:S:t:l:u:h" OPTION +while getopts "d:e:c:H:i:n:p:S:t:l:u:h" OPTION do case $OPTION in d) ETCD_DISCOVERY="$OPTARG" ;; e) ETCD_ADDR="$OPTARG" ;; + c) CLOUD_CONFIG_FILE="${OPTARG}" ;; H) HNAME="$OPTARG" ;; i) ETCD_PEER_URLS="$OPTARG" ;; n) ETCD_NAME="$OPTARG" ;; @@ -145,6 +147,10 @@ if [ -z "$ETCD_LISTEN_CLIENT_URLS" ]; then ETCD_LISTEN_CLIENT_URLS=$DEFAULT_ETCD_LISTEN_CLIENT_URLS fi +if [ -n "${CLOUD_CONFIG_FILE}" ]; then + CLOUD_CONFIG="$(cat ${CLOUD_CONFIG_FILE})" +fi + WORKDIR="${DEST}/tmp.${RANDOM}" mkdir "$WORKDIR" trap "rm -rf '${WORKDIR}'" EXIT From b8360e2c20cb1d3b980f522bab4025d7b9d59e26 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 1 Sep 2020 17:00:29 -0400 Subject: [PATCH 03/13] contrib: allow newer ssh keys Signed-off-by: Vincent Batts --- contrib/create-basic-configdrive | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/create-basic-configdrive b/contrib/create-basic-configdrive index 13b904ecc5..5d7c6448b5 100755 --- a/contrib/create-basic-configdrive +++ b/contrib/create-basic-configdrive @@ -43,7 +43,7 @@ ssh_authorized_keys: - hostname: " -REGEX_SSH_FILE="^ssh-(rsa|dss|ed25519) [-A-Za-z0-9+\/]+[=]{0,2} .+" +REGEX_SSH_FILE="^(ssh-(rsa|dss|ed25519)|ecdsa-sha2-nistp521) [-A-Za-z0-9+\/]+[=]{0,2} .+" while getopts "d:e:c:H:i:n:p:S:t:l:u:h" OPTION do From acac817ea11fff96504d1a3542e4edf401af84b4 Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 1 Sep 2020 17:14:10 -0400 Subject: [PATCH 04/13] contrib: shellcheck lint Signed-off-by: Vincent Batts --- contrib/create-basic-configdrive | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/contrib/create-basic-configdrive b/contrib/create-basic-configdrive index 5d7c6448b5..a7872498c1 100755 --- a/contrib/create-basic-configdrive +++ b/contrib/create-basic-configdrive @@ -65,16 +65,16 @@ do done # root user forbidden -if [ $(id -u) -eq 0 ]; then +if [ "$(id -u)" -eq 0 ]; then echo "$0: This script should not be run as root." >&2 exit 1 fi # mkisofs/genisoimage tool check -if which mkisofs &>/dev/null; then - mkisofs=$(which mkisofs) -elif which genisoimage &>/dev/null; then - mkisofs=$(which genisoimage) +if command -v mkisofs &>/dev/null; then + mkisofs=$(command -v mkisofs) +elif command -v genisoimage &>/dev/null; then + mkisofs=$(command -v genisoimage) else echo "$0: mkisofs or genisoimage tool is required to create image." >&2 exit 1 @@ -95,12 +95,12 @@ if [[ ! -r "$SSH_FILE" ]]; then exit 1 fi -if [ $(cat "$SSH_FILE" | wc -l) -eq 0 ]; then +if [ "$(wc -l < "$SSH_FILE")" -eq 0 ]; then echo "$0: The SSH file (${SSH_FILE}) is empty." >&2 exit 1 fi -if [ $(grep -v -E "$REGEX_SSH_FILE" "$SSH_FILE" | wc -l) -gt 0 ]; then +if [ "$(grep -c -v -E "$REGEX_SSH_FILE" "$SSH_FILE")" -gt 0 ]; then echo "$0: The SSH file (${SSH_FILE}) content is invalid." >&2 exit 1 fi @@ -114,12 +114,12 @@ if [[ ! -d "$DEST" ]]; then exit 1 fi -if [ ! -z "$ETCD_DISCOVERY" ] && [ ! -z "$TOKEN" ]; then +if [ -n "$ETCD_DISCOVERY" ] && [ -n "$TOKEN" ]; then echo "$0: You cannot specify both discovery token and discovery URL." >&2 exit 1 fi -if [ ! -z "$TOKEN" ]; then +if [ -n "$TOKEN" ]; then ETCD_DISCOVERY="${DEFAULT_ETCD_DISCOVERY//TOKEN/$TOKEN}" fi @@ -148,12 +148,12 @@ if [ -z "$ETCD_LISTEN_CLIENT_URLS" ]; then fi if [ -n "${CLOUD_CONFIG_FILE}" ]; then - CLOUD_CONFIG="$(cat ${CLOUD_CONFIG_FILE})" + CLOUD_CONFIG="$(cat "${CLOUD_CONFIG_FILE}")" fi WORKDIR="${DEST}/tmp.${RANDOM}" mkdir "$WORKDIR" -trap "rm -rf '${WORKDIR}'" EXIT +trap 'rm -rf "${WORKDIR}"' EXIT CONFIG_DIR="${WORKDIR}/openstack/latest" CONFIG_FILE="${CONFIG_DIR}/user_data" @@ -161,7 +161,7 @@ CONFIGDRIVE_FILE="${DEST}/${HNAME}.iso" mkdir -p "$CONFIG_DIR" -while read l; do +while read -r l; do if [ -z "$SSH_KEY" ]; then SSH_KEY="$l" else @@ -182,8 +182,8 @@ CLOUD_CONFIG="${CLOUD_CONFIG//${HNAME}}" echo "$CLOUD_CONFIG" > "$CONFIG_FILE" $mkisofs -R -V config-2 -o "$CONFIGDRIVE_FILE" "$WORKDIR" - -if [ "$?" -eq 0 ] ; then +ret="$?" +if [ "${ret}" -eq 0 ] ; then echo echo echo "Success! The config-drive image was created on ${CONFIGDRIVE_FILE}" From ed7de96c1f121c91078c1a9200ac389a95c5d07c Mon Sep 17 00:00:00 2001 From: Vincent Batts Date: Tue, 15 Sep 2020 17:49:33 -0400 Subject: [PATCH 05/13] qemu_template: shell lint and update Signed-off-by: Vincent Batts --- build_library/qemu_template.sh | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/build_library/qemu_template.sh b/build_library/qemu_template.sh index bf840a0ea5..7f28cf5a93 100755 --- a/build_library/qemu_template.sh +++ b/build_library/qemu_template.sh @@ -1,6 +1,6 @@ #!/bin/sh -SCRIPT_DIR="`dirname "$0"`" +SCRIPT_DIR="$(dirname "$0")" VM_BOARD= VM_NAME= VM_UUID= @@ -11,7 +11,7 @@ VM_MEMORY= VM_CDROM= VM_PFLASH_RO= VM_PFLASH_RW= -VM_NCPUS="`grep -c ^processor /proc/cpuinfo`" +VM_NCPUS="$(nproc)" SSH_PORT=2222 SSH_KEYS="" CLOUD_CONFIG_FILE="" @@ -112,10 +112,12 @@ write_ssh_keys() { if [ -z "${CONFIG_IMAGE}" ]; then CONFIG_DRIVE=$(mktemp -t -d flatcar-configdrive.XXXXXXXXXX) - if [ $? -ne 0 ] || [ ! -d "$CONFIG_DRIVE" ]; then + ret=$? + if [ "$ret" -ne 0 ] || [ ! -d "$CONFIG_DRIVE" ]; then echo "$0: mktemp -d failed!" >&2 exit 1 fi + # shellcheck disable=SC2064 trap "rm -rf '$CONFIG_DRIVE'" EXIT mkdir -p "${CONFIG_DRIVE}/openstack/latest" @@ -126,7 +128,8 @@ if [ -z "${CONFIG_IMAGE}" ]; then exit 1 fi SSH_KEYS_TEXT=$(cat "$SSH_KEYS") - if [ $? -ne 0 ] || [ -z "$SSH_KEYS_TEXT" ]; then + ret=$? + if [ "$ret" -ne 0 ] || [ -z "$SSH_KEYS_TEXT" ]; then echo "$0: Failed to read SSH keys from $SSH_KEYS" >&2 exit 1 fi @@ -134,7 +137,8 @@ if [ -z "${CONFIG_IMAGE}" ]; then "${CONFIG_DRIVE}/openstack/latest/user_data" elif [ -n "${CLOUD_CONFIG_FILE}" ]; then cp "${CLOUD_CONFIG_FILE}" "${CONFIG_DRIVE}/openstack/latest/user_data" - if [ $? -ne 0 ]; then + ret=$? + if [ "$ret" -ne 0 ]; then echo "$0: Failed to copy cloudinit file from $CLOUD_CONFIG_FILE" >&2 exit 1 fi From cc147a1255baab38a47900ca67bcd64c78088f60 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Mon, 14 Sep 2020 17:27:03 +0530 Subject: [PATCH 06/13] Add the new OEM type, and selective upload azure as private Signed-off-by: Sayan Chowdhury --- build_library/release_util.sh | 2 ++ build_library/vm_image_util.sh | 6 ++++++ jenkins/vm.sh | 10 +++++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/build_library/release_util.sh b/build_library/release_util.sh index 12d8a82807..7b20d383df 100644 --- a/build_library/release_util.sh +++ b/build_library/release_util.sh @@ -22,6 +22,8 @@ DEFINE_boolean parallel ${FLAGS_TRUE} \ "Enable parallelism in gsutil." DEFINE_boolean upload ${UPLOAD_DEFAULT} \ "Upload all packages/images via gsutil." +DEFINE_boolean private ${FLAGS_TRUE} \ + "Upload the image as a private object." DEFINE_string upload_root "${FLATCAR_UPLOAD_ROOT}" \ "Upload prefix, board/version/etc will be appended. Must be a gs:// URL." DEFINE_string upload_path "" \ diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index eb6b1fee02..d530204602 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -36,6 +36,7 @@ VALID_IMG_TYPES=( digitalocean exoscale azure + azure_premium hyperv niftycloud cloudsigma @@ -270,6 +271,11 @@ IMG_azure_DISK_FORMAT=vhd IMG_azure_DISK_LAYOUT=azure IMG_azure_OEM_PACKAGE=oem-azure +## azure premium +IMG_azure_premium_DISK_FORMAT=vhd +IMG_azure_premium_DISK_LAYOUT=azure +IMG_azure_premium_OEM_PACKAGE=oem-azure + ## hyper-v IMG_hyperv_DISK_FORMAT=vhd IMG_hyperv_OEM_PACKAGE=oem-hyperv diff --git a/jenkins/vm.sh b/jenkins/vm.sh index 2526fc69bd..be2913a26c 100644 --- a/jenkins/vm.sh +++ b/jenkins/vm.sh @@ -43,6 +43,13 @@ img=src/flatcar_production_image.bin [[ "${img}.bz2" -nt "${img}" ]] && enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2" +PRIVATE_UPLOAD_OPT="" +if [[ "${FORMAT}" == 'azure_premium' ]] +then + PRIVATE_UPLOAD_OPT="--private" + UPLOAD_ROOT=${UPLOAD_PRIVATE_ROOT} +fi + script image_to_vm.sh \ --board="${BOARD}" \ --format="${FORMAT}" \ @@ -54,4 +61,5 @@ script image_to_vm.sh \ --sign_digests="${SIGNING_USER}" \ --download_root="${DOWNLOAD_ROOT}" \ --upload_root="${UPLOAD_ROOT}" \ - --upload + --upload \ + ${PRIVATE_UPLOAD_OPT} From 9953cc8c8fd72146315577b23e5f7ae3b0262091 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Mon, 21 Sep 2020 20:31:41 +0530 Subject: [PATCH 07/13] build_library: Rename the images to use pro instead of premium Signed-off-by: Sayan Chowdhury --- build_library/vm_image_util.sh | 10 +++++----- jenkins/vm.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index d530204602..5a47135ade 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -36,7 +36,7 @@ VALID_IMG_TYPES=( digitalocean exoscale azure - azure_premium + azure_pro hyperv niftycloud cloudsigma @@ -271,10 +271,10 @@ IMG_azure_DISK_FORMAT=vhd IMG_azure_DISK_LAYOUT=azure IMG_azure_OEM_PACKAGE=oem-azure -## azure premium -IMG_azure_premium_DISK_FORMAT=vhd -IMG_azure_premium_DISK_LAYOUT=azure -IMG_azure_premium_OEM_PACKAGE=oem-azure +## azure pro +IMG_azure_pro_DISK_FORMAT=vhd +IMG_azure_pro_DISK_LAYOUT=azure +IMG_azure_pro_OEM_PACKAGE=oem-azure ## hyper-v IMG_hyperv_DISK_FORMAT=vhd diff --git a/jenkins/vm.sh b/jenkins/vm.sh index be2913a26c..5c11f9242e 100644 --- a/jenkins/vm.sh +++ b/jenkins/vm.sh @@ -44,7 +44,7 @@ img=src/flatcar_production_image.bin enter lbunzip2 -k -f "/mnt/host/source/${img}.bz2" PRIVATE_UPLOAD_OPT="" -if [[ "${FORMAT}" == 'azure_premium' ]] +if [[ "${FORMAT}" == 'azure_pro' ]] then PRIVATE_UPLOAD_OPT="--private" UPLOAD_ROOT=${UPLOAD_PRIVATE_ROOT} From 23c72c0a5cbb632c738e1e8ca3b140f8bfbb1a7c Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Mon, 28 Sep 2020 23:09:36 +0530 Subject: [PATCH 08/13] Add Azure Pro to the list of amd64 format list Signed-off-by: Sayan Chowdhury --- jenkins/formats-amd64-usr.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/jenkins/formats-amd64-usr.txt b/jenkins/formats-amd64-usr.txt index 9f98a63209..33d48e4123 100644 --- a/jenkins/formats-amd64-usr.txt +++ b/jenkins/formats-amd64-usr.txt @@ -1,6 +1,7 @@ ami ami_vmdk azure +azure_pro gce iso pxe From 8799028007d6647dd8d63acfd8d1c66ce1be95c7 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Thu, 8 Oct 2020 11:51:24 +0530 Subject: [PATCH 09/13] Add OEM package for QEMU Signed-off-by: Sayan Chowdhury --- build_library/vm_image_util.sh | 44 ++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 5a47135ade..3b55668c07 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -8,10 +8,24 @@ VALID_IMG_TYPES=( ami ami_vmdk - pxe + azure + azure_pro + brightbox + cloudsigma + cloudstack + cloudstack_vhd + digitalocean + exoscale + gce + hyperv + interoute iso + niftycloud openstack openstack_mini + packet + parallels + pxe qemu qemu_uefi qemu_uefi_secure @@ -24,35 +38,26 @@ VALID_IMG_TYPES=( vagrant_vmware_fusion virtualbox vmware + vmware_insecure vmware_ova vmware_raw - vmware_insecure - parallels xen - gce - brightbox - cloudstack - cloudstack_vhd - digitalocean - exoscale - azure - azure_pro - hyperv - niftycloud - cloudsigma - packet - interoute ) #list of oem package names, minus the oem- prefix VALID_OEM_PACKAGES=( azure + cloudsigma cloudstack digitalocean ec2-compat exoscale gce hyperv + interoute + niftycloud + packet + qemu rackspace rackspace-onmetal vagrant @@ -60,10 +65,6 @@ VALID_OEM_PACKAGES=( vagrant-virtualbox virtualbox vmware - niftycloud - cloudsigma - packet - interoute ) # Set at runtime to one of the above types @@ -124,14 +125,17 @@ IMG_DEFAULT_CPUS=2 IMG_qemu_DISK_FORMAT=qcow2 IMG_qemu_DISK_LAYOUT=vm IMG_qemu_CONF_FORMAT=qemu +IMG_qemu_OEM_PACKAGE=oem-qemu IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi +IMG_qemu_uefi_OEM_PACKAGE=oem-qemu IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 IMG_qemu_uefi_secure_DISK_LAYOUT=vm IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure +IMG_qemu_uefi_secure_OEM_PACKAGE=oem-qemu ## xen IMG_xen_CONF_FORMAT=xl From 829cec45e89af8b1ad33a40464f88f83dc97ed68 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Mon, 26 Oct 2020 14:08:06 +0100 Subject: [PATCH 10/13] jenkins: do not configure ccache variables Setting the invalid CCACHE_ variables resulted in strange failure in projects depending on meson, newer version like 0.55.3. For example systemd build fails like the following errors: ``` * ACCESS DENIED: utimes: /mnt/host/source/ccache * ACCESS DENIED: utimes: /mnt/host/source/ccache F: utimes S: deny P: /mnt/host/source/ccache A: /mnt/host/source/ccache R: /mnt/host/source/ccache C: ccache cc /build/amd64-usr/var/tmp/portage/sys-apps/systemd-246/work/systemd-246-abi_x86_64.amd64/meson-private/sanitycheckc.c -o /build/amd64-usr/var/tmp/portage/sys-apps/systemd-246/work/systemd-246-abi_x86_64.amd64/meson-private/sanitycheckc.exe -O1 -pipe -pipe -D_FILE_OFFSET_BITS=64 ``` We should not set up ccache at all, as it has been already disabled in coreos-overlay repo. --- build_library/catalyst.sh | 7 ++----- build_library/toolchain_util.sh | 1 - jenkins/packages.sh | 11 ----------- update_chroot | 4 +--- 4 files changed, 3 insertions(+), 20 deletions(-) diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 196057aeff..7e32419a81 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -236,12 +236,9 @@ catalyst_init() { } write_configs() { - # No catalyst config option, so defined via environment - export CCACHE_DIR="$TEMPDIR/ccache" - info "Creating output directories..." - mkdir -m 775 -p "$TEMPDIR/portage/repos.conf" "$DISTDIR" "$CCACHE_DIR" - chown portage:portage "$DISTDIR" "$CCACHE_DIR" + mkdir -m 775 -p "$TEMPDIR/portage/repos.conf" "$DISTDIR" + chown portage:portage "$DISTDIR" info "Writing out catalyst configs..." info " catalyst.conf" catalyst_conf > "$TEMPDIR/catalyst.conf" diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 95e6b4595f..d48f1c4f6c 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -344,7 +344,6 @@ install_cross_toolchain() { # Setup environment and wrappers for our shiny new toolchain gcc_set_latest_profile "${cross_chost}" - $sudo CC_QUIET=1 ccache-config --install-links "${cross_chost}" $sudo CC_QUIET=1 sysroot-config --install-links "${cross_chost}" } diff --git a/jenkins/packages.sh b/jenkins/packages.sh index 352f74f598..d467945197 100644 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -1,9 +1,5 @@ #!/bin/bash -ex -# Use a ccache dir that persists across SDK recreations. -# XXX: alternatively use a ccache dir that is usable by all jobs on a given node. -mkdir -p ccache - enter() { local verify_key= trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN @@ -13,8 +9,6 @@ enter() { sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ chroot/etc/portage/gangue.json bin/cork enter --bind-gpg-agent=false -- env \ - CCACHE_DIR=/mnt/host/source/ccache \ - CCACHE_MAXSIZE=5G \ FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ @@ -33,9 +27,6 @@ export FLATCAR_BUILD_ID # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" -# Figure out if ccache is doing us any good in this scheme. -enter ccache --zero-stats - script setup_board \ --board="${BOARD}" \ --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \ @@ -61,5 +52,3 @@ script build_torcx_store \ --torcx_upload_root="${TORCX_PKG_DOWNLOAD_ROOT}" \ --tectonic_torcx_download_root="${TECTONIC_TORCX_DOWNLOAD_ROOT}" \ --upload - -enter ccache --show-stats diff --git a/update_chroot b/update_chroot index 2f0cb492d7..f590b92142 100755 --- a/update_chroot +++ b/update_chroot @@ -71,7 +71,7 @@ PORTAGE_BINHOST="$FLAGS_binhost $(get_sdk_binhost)" : ${PORTAGE_USERNAME:=${USER}} # Clean up old distfiles cache. It used to be split for 'host' and -# 'target' but that just duplicates files. Also a ccache was in there. +# 'target' but that just duplicates files. if [[ -d "${REPO_CACHE_DIR}/distfiles/host" ]]; then info "Cleaning up old distfiles cache..." sudo mv "${REPO_CACHE_DIR}"/{distfiles/host,distfiles.host} @@ -95,7 +95,6 @@ PORT_LOGDIR="/var/log/portage" PORTAGE_BINHOST="$PORTAGE_BINHOST" PORTAGE_USERNAME="${PORTAGE_USERNAME}" MAKEOPTS="--jobs=${NUM_JOBS} --load-average=$((NUM_JOBS * 2))" -CCACHE_UMASK="002" # Generally there isn't any need to add packages to @world by default. # You can use --select to override this. @@ -214,7 +213,6 @@ EMERGE_CMD="emerge" # can cause serious issues later. info "Updating basic system packages" sudo -E ${EMERGE_CMD} --quiet "${EMERGE_FLAGS[@]}" \ - dev-util/ccache \ sys-apps/portage \ sys-devel/crossdev \ sys-devel/sysroot-wrappers \ From 6eae505f16e5c90dcebb882d2302f62fe5ca31b7 Mon Sep 17 00:00:00 2001 From: Dongsu Park Date: Fri, 13 Nov 2020 17:04:59 +0100 Subject: [PATCH 11/13] build_library: Ignore broken symlink in Kernel source tree Kernel source tree started to have a broken link `tools/testing/selftests/powerpc/copyloops/memcpy_mcsafe_64.S`. Especially in case of Kernel 5.8.18, like: ``` broken link: /usr/src/linux-5.8.18-coreos/tools/testing/selftests/powerpc/copyloops/memcpy_mcsafe_64.S ERROR build_packages: test_image_content: Failed symlink check ``` Ignore the symlink when checking broken symlinks. --- build_library/check_root | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build_library/check_root b/build_library/check_root index 2efb15e3c4..fb181a5756 100755 --- a/build_library/check_root +++ b/build_library/check_root @@ -128,7 +128,8 @@ IGNORE_SYMLINK = ( # Other b"/etc/lsb-release", # set later in the build process b"/usr/share/coreos", # set later in the build process - b"/etc/coreos" # set later in the build process + b"/etc/coreos", # set later in the build process + b"/usr/src/linux-*/tools/testing/selftests/powerpc/copyloops/memcpy_mcsafe_64.S" # broken symlink in Kernel source tree ) From 22b08b0ae417bd48cd7ebb7531eea889ffb39b33 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Fri, 13 Nov 2020 13:37:20 +0100 Subject: [PATCH 12/13] build_packages, build_image_util.sh: fix up liblto symlink Signed-off-by: Thilo Fromm --- build_library/build_image_util.sh | 2 ++ build_packages | 2 ++ common.sh | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+) diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 45e82494a8..af6ace6787 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -153,6 +153,8 @@ emerge_to_image() { # Make sure profile.env has been generated sudo -E ROOT="${root_fs_dir}" env-update --no-ldconfig + fixup_liblto_softlinks "$root_fs_dir" + # TODO(marineam): just call ${BUILD_LIBRARY_DIR}/check_root directly once # all tests are fatal, for now let the old function skip soname errors. ROOT="${root_fs_dir}" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ diff --git a/build_packages b/build_packages index 7933ec8c9f..df822efbd3 100755 --- a/build_packages +++ b/build_packages @@ -252,6 +252,8 @@ fi eclean-$BOARD -d packages +fixup_liblto_softlinks "${BOARD_ROOT}" + info "Checking build root" test_image_content "${BOARD_ROOT}" diff --git a/common.sh b/common.sh index 23509f7945..014b257307 100644 --- a/common.sh +++ b/common.sh @@ -966,3 +966,23 @@ clean_qemu_static() { *) die "Unsupported arch" ;; esac } + +# Fix up liblto softlink created by gcc-config +fixup_liblto_softlinks() { + local root="$1" + + info "fixup_liblto_softlinks: Looking for broken softlinks in '${root}'" + + local link + local target + # check both native (/usr/CHOST/) as well as cross compile (/usr/CHOST/CTARGET) paths + { ls -l "${root}/"usr/*/binutils-bin/lib/bfd-plugins/liblto_plugin.so 2>/dev/null; + ls -l "${root}/"usr/*/*/binutils-bin/lib/bfd-plugins/liblto_plugin.so 2>/dev/null; } \ + | sed 's:.* \([^[:space:]]\+\) -> \([^[:space:]]\+\):\1 \2:' \ + | while read link target; do + local newtarget=$(echo "$target" | sed "s:${root}:/:") + info " Fixing up broken $link -> $target" + info " sudo ln -sf $newtarget $link" + sudo ln -sf $newtarget $link + done +} From 9e2098bfff34ce7411d97bb5098980efa85b68a8 Mon Sep 17 00:00:00 2001 From: Sayan Chowdhury Date: Tue, 17 Nov 2020 00:08:17 +0530 Subject: [PATCH 13/13] Update the azure pro package reference to the oem-azure-pro This commit also fixes the BINHOST URL for the for the developer container portage Signed-off-by: Sayan Chowdhury --- build_library/dev_container_util.sh | 12 ++++++++++-- build_library/vm_image_util.sh | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 201cf7d0a7..b4bba72d3d 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -2,6 +2,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +get_binhost_url() { + if [ "${DEFAULT_GROUP}" == "developer" ]; then + echo "https://storage.googleapis.com/flatcar-jenkins/${DEFAULT_GROUP}/boards/${BOARD}/${FLATCAR_VERSION}/$1" + else + echo "https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/$1" + fi +} + configure_dev_portage() { # Need profiles at the bare minimum local repo @@ -25,8 +33,8 @@ PKGDIR="/var/lib/portage/pkgs" PORT_LOGDIR="/var/log/portage" PORTDIR="/var/lib/portage/portage-stable" PORTDIR_OVERLAY="/var/lib/portage/coreos-overlay" -PORTAGE_BINHOST="https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/pkgs/ -https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/toolchain/" +PORTAGE_BINHOST="$(get_binhost_url 'pkgs') +$(get_binhost_url 'toolchain')" EOF sudo_clobber "$1/etc/portage/repos.conf/coreos.conf" <