diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 45e82494a8..af6ace6787 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -153,6 +153,8 @@ emerge_to_image() { # Make sure profile.env has been generated sudo -E ROOT="${root_fs_dir}" env-update --no-ldconfig + fixup_liblto_softlinks "$root_fs_dir" + # TODO(marineam): just call ${BUILD_LIBRARY_DIR}/check_root directly once # all tests are fatal, for now let the old function skip soname errors. ROOT="${root_fs_dir}" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 196057aeff..7e32419a81 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -236,12 +236,9 @@ catalyst_init() { } write_configs() { - # No catalyst config option, so defined via environment - export CCACHE_DIR="$TEMPDIR/ccache" - info "Creating output directories..." - mkdir -m 775 -p "$TEMPDIR/portage/repos.conf" "$DISTDIR" "$CCACHE_DIR" - chown portage:portage "$DISTDIR" "$CCACHE_DIR" + mkdir -m 775 -p "$TEMPDIR/portage/repos.conf" "$DISTDIR" + chown portage:portage "$DISTDIR" info "Writing out catalyst configs..." info " catalyst.conf" catalyst_conf > "$TEMPDIR/catalyst.conf" diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 201cf7d0a7..b4bba72d3d 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -2,6 +2,14 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +get_binhost_url() { + if [ "${DEFAULT_GROUP}" == "developer" ]; then + echo "https://storage.googleapis.com/flatcar-jenkins/${DEFAULT_GROUP}/boards/${BOARD}/${FLATCAR_VERSION}/$1" + else + echo "https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/$1" + fi +} + configure_dev_portage() { # Need profiles at the bare minimum local repo @@ -25,8 +33,8 @@ PKGDIR="/var/lib/portage/pkgs" PORT_LOGDIR="/var/log/portage" PORTDIR="/var/lib/portage/portage-stable" PORTDIR_OVERLAY="/var/lib/portage/coreos-overlay" -PORTAGE_BINHOST="https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/pkgs/ -https://storage.googleapis.com/flatcar-jenkins/boards/${BOARD}/${FLATCAR_VERSION_ID}/toolchain/" +PORTAGE_BINHOST="$(get_binhost_url 'pkgs') +$(get_binhost_url 'toolchain')" EOF sudo_clobber "$1/etc/portage/repos.conf/coreos.conf" <&2 exit 1 fi + # shellcheck disable=SC2064 trap "rm -rf '$CONFIG_DRIVE'" EXIT mkdir -p "${CONFIG_DRIVE}/openstack/latest" @@ -126,7 +128,8 @@ if [ -z "${CONFIG_IMAGE}" ]; then exit 1 fi SSH_KEYS_TEXT=$(cat "$SSH_KEYS") - if [ $? -ne 0 ] || [ -z "$SSH_KEYS_TEXT" ]; then + ret=$? + if [ "$ret" -ne 0 ] || [ -z "$SSH_KEYS_TEXT" ]; then echo "$0: Failed to read SSH keys from $SSH_KEYS" >&2 exit 1 fi @@ -134,7 +137,8 @@ if [ -z "${CONFIG_IMAGE}" ]; then "${CONFIG_DRIVE}/openstack/latest/user_data" elif [ -n "${CLOUD_CONFIG_FILE}" ]; then cp "${CLOUD_CONFIG_FILE}" "${CONFIG_DRIVE}/openstack/latest/user_data" - if [ $? -ne 0 ]; then + ret=$? + if [ "$ret" -ne 0 ]; then echo "$0: Failed to copy cloudinit file from $CLOUD_CONFIG_FILE" >&2 exit 1 fi diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 95e6b4595f..d48f1c4f6c 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -344,7 +344,6 @@ install_cross_toolchain() { # Setup environment and wrappers for our shiny new toolchain gcc_set_latest_profile "${cross_chost}" - $sudo CC_QUIET=1 ccache-config --install-links "${cross_chost}" $sudo CC_QUIET=1 sysroot-config --install-links "${cross_chost}" } diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 5a47135ade..3834a2e4b7 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -8,10 +8,24 @@ VALID_IMG_TYPES=( ami ami_vmdk - pxe + azure + azure_pro + brightbox + cloudsigma + cloudstack + cloudstack_vhd + digitalocean + exoscale + gce + hyperv + interoute iso + niftycloud openstack openstack_mini + packet + parallels + pxe qemu qemu_uefi qemu_uefi_secure @@ -24,35 +38,26 @@ VALID_IMG_TYPES=( vagrant_vmware_fusion virtualbox vmware + vmware_insecure vmware_ova vmware_raw - vmware_insecure - parallels xen - gce - brightbox - cloudstack - cloudstack_vhd - digitalocean - exoscale - azure - azure_pro - hyperv - niftycloud - cloudsigma - packet - interoute ) #list of oem package names, minus the oem- prefix VALID_OEM_PACKAGES=( azure + cloudsigma cloudstack digitalocean ec2-compat exoscale gce hyperv + interoute + niftycloud + packet + qemu rackspace rackspace-onmetal vagrant @@ -60,10 +65,6 @@ VALID_OEM_PACKAGES=( vagrant-virtualbox virtualbox vmware - niftycloud - cloudsigma - packet - interoute ) # Set at runtime to one of the above types @@ -124,14 +125,17 @@ IMG_DEFAULT_CPUS=2 IMG_qemu_DISK_FORMAT=qcow2 IMG_qemu_DISK_LAYOUT=vm IMG_qemu_CONF_FORMAT=qemu +IMG_qemu_OEM_PACKAGE=oem-qemu IMG_qemu_uefi_DISK_FORMAT=qcow2 IMG_qemu_uefi_DISK_LAYOUT=vm IMG_qemu_uefi_CONF_FORMAT=qemu_uefi +IMG_qemu_uefi_OEM_PACKAGE=oem-qemu IMG_qemu_uefi_secure_DISK_FORMAT=qcow2 IMG_qemu_uefi_secure_DISK_LAYOUT=vm IMG_qemu_uefi_secure_CONF_FORMAT=qemu_uefi_secure +IMG_qemu_uefi_secure_OEM_PACKAGE=oem-qemu ## xen IMG_xen_CONF_FORMAT=xl @@ -274,7 +278,7 @@ IMG_azure_OEM_PACKAGE=oem-azure ## azure pro IMG_azure_pro_DISK_FORMAT=vhd IMG_azure_pro_DISK_LAYOUT=azure -IMG_azure_pro_OEM_PACKAGE=oem-azure +IMG_azure_pro_OEM_PACKAGE=oem-azure-pro ## hyper-v IMG_hyperv_DISK_FORMAT=vhd diff --git a/build_packages b/build_packages index 2aeea80033..df822efbd3 100755 --- a/build_packages +++ b/build_packages @@ -235,6 +235,9 @@ if [[ "${FLAGS_usepkgonly}" -eq "${FLAGS_FALSE}" ]]; then break_dep_loop sys-apps/systemd cryptsetup fi +export KBUILD_BUILD_USER="${BUILD_USER:-build}" +export KBUILD_BUILD_HOST="${BUILD_HOST:-pony-truck.infra.kinvolk.io}" + info "Merging board packages now" sudo -E "${EMERGE_CMD[@]}" "${EMERGE_FLAGS[@]}" "$@" @@ -249,6 +252,8 @@ fi eclean-$BOARD -d packages +fixup_liblto_softlinks "${BOARD_ROOT}" + info "Checking build root" test_image_content "${BOARD_ROOT}" diff --git a/common.sh b/common.sh index 23509f7945..014b257307 100644 --- a/common.sh +++ b/common.sh @@ -966,3 +966,23 @@ clean_qemu_static() { *) die "Unsupported arch" ;; esac } + +# Fix up liblto softlink created by gcc-config +fixup_liblto_softlinks() { + local root="$1" + + info "fixup_liblto_softlinks: Looking for broken softlinks in '${root}'" + + local link + local target + # check both native (/usr/CHOST/) as well as cross compile (/usr/CHOST/CTARGET) paths + { ls -l "${root}/"usr/*/binutils-bin/lib/bfd-plugins/liblto_plugin.so 2>/dev/null; + ls -l "${root}/"usr/*/*/binutils-bin/lib/bfd-plugins/liblto_plugin.so 2>/dev/null; } \ + | sed 's:.* \([^[:space:]]\+\) -> \([^[:space:]]\+\):\1 \2:' \ + | while read link target; do + local newtarget=$(echo "$target" | sed "s:${root}:/:") + info " Fixing up broken $link -> $target" + info " sudo ln -sf $newtarget $link" + sudo ln -sf $newtarget $link + done +} diff --git a/contrib/create-basic-configdrive b/contrib/create-basic-configdrive index ece7849c6c..a7872498c1 100755 --- a/contrib/create-basic-configdrive +++ b/contrib/create-basic-configdrive @@ -15,6 +15,7 @@ Options: -l http://IP:PORT Listen URL for client communication. -u http://IP:PORT Listen URL for server communication. -n NAME etcd node name. + -c FILE cloud-config template to use, instead of the default. -p DEST Create config-drive ISO image to the given path. -S FILE SSH keys file. -t TOKEN Token ID from https://discovery.etcd.io. @@ -42,13 +43,14 @@ ssh_authorized_keys: - hostname: " -REGEX_SSH_FILE="^ssh-(rsa|dss|ed25519) [-A-Za-z0-9+\/]+[=]{0,2} .+" +REGEX_SSH_FILE="^(ssh-(rsa|dss|ed25519)|ecdsa-sha2-nistp521) [-A-Za-z0-9+\/]+[=]{0,2} .+" -while getopts "d:e:H:i:n:p:S:t:l:u:h" OPTION +while getopts "d:e:c:H:i:n:p:S:t:l:u:h" OPTION do case $OPTION in d) ETCD_DISCOVERY="$OPTARG" ;; e) ETCD_ADDR="$OPTARG" ;; + c) CLOUD_CONFIG_FILE="${OPTARG}" ;; H) HNAME="$OPTARG" ;; i) ETCD_PEER_URLS="$OPTARG" ;; n) ETCD_NAME="$OPTARG" ;; @@ -63,16 +65,16 @@ do done # root user forbidden -if [ $(id -u) -eq 0 ]; then +if [ "$(id -u)" -eq 0 ]; then echo "$0: This script should not be run as root." >&2 exit 1 fi # mkisofs/genisoimage tool check -if which mkisofs &>/dev/null; then - mkisofs=$(which mkisofs) -elif which genisoimage &>/dev/null; then - mkisofs=$(which genisoimage) +if command -v mkisofs &>/dev/null; then + mkisofs=$(command -v mkisofs) +elif command -v genisoimage &>/dev/null; then + mkisofs=$(command -v genisoimage) else echo "$0: mkisofs or genisoimage tool is required to create image." >&2 exit 1 @@ -93,12 +95,12 @@ if [[ ! -r "$SSH_FILE" ]]; then exit 1 fi -if [ $(cat "$SSH_FILE" | wc -l) -eq 0 ]; then +if [ "$(wc -l < "$SSH_FILE")" -eq 0 ]; then echo "$0: The SSH file (${SSH_FILE}) is empty." >&2 exit 1 fi -if [ $(grep -v -E "$REGEX_SSH_FILE" "$SSH_FILE" | wc -l) -gt 0 ]; then +if [ "$(grep -c -v -E "$REGEX_SSH_FILE" "$SSH_FILE")" -gt 0 ]; then echo "$0: The SSH file (${SSH_FILE}) content is invalid." >&2 exit 1 fi @@ -112,12 +114,12 @@ if [[ ! -d "$DEST" ]]; then exit 1 fi -if [ ! -z "$ETCD_DISCOVERY" ] && [ ! -z "$TOKEN" ]; then +if [ -n "$ETCD_DISCOVERY" ] && [ -n "$TOKEN" ]; then echo "$0: You cannot specify both discovery token and discovery URL." >&2 exit 1 fi -if [ ! -z "$TOKEN" ]; then +if [ -n "$TOKEN" ]; then ETCD_DISCOVERY="${DEFAULT_ETCD_DISCOVERY//TOKEN/$TOKEN}" fi @@ -145,9 +147,13 @@ if [ -z "$ETCD_LISTEN_CLIENT_URLS" ]; then ETCD_LISTEN_CLIENT_URLS=$DEFAULT_ETCD_LISTEN_CLIENT_URLS fi +if [ -n "${CLOUD_CONFIG_FILE}" ]; then + CLOUD_CONFIG="$(cat "${CLOUD_CONFIG_FILE}")" +fi + WORKDIR="${DEST}/tmp.${RANDOM}" mkdir "$WORKDIR" -trap "rm -rf '${WORKDIR}'" EXIT +trap 'rm -rf "${WORKDIR}"' EXIT CONFIG_DIR="${WORKDIR}/openstack/latest" CONFIG_FILE="${CONFIG_DIR}/user_data" @@ -155,7 +161,7 @@ CONFIGDRIVE_FILE="${DEST}/${HNAME}.iso" mkdir -p "$CONFIG_DIR" -while read l; do +while read -r l; do if [ -z "$SSH_KEY" ]; then SSH_KEY="$l" else @@ -176,8 +182,8 @@ CLOUD_CONFIG="${CLOUD_CONFIG//${HNAME}}" echo "$CLOUD_CONFIG" > "$CONFIG_FILE" $mkisofs -R -V config-2 -o "$CONFIGDRIVE_FILE" "$WORKDIR" - -if [ "$?" -eq 0 ] ; then +ret="$?" +if [ "${ret}" -eq 0 ] ; then echo echo echo "Success! The config-drive image was created on ${CONFIGDRIVE_FILE}" diff --git a/jenkins/packages.sh b/jenkins/packages.sh index 352f74f598..d467945197 100644 --- a/jenkins/packages.sh +++ b/jenkins/packages.sh @@ -1,9 +1,5 @@ #!/bin/bash -ex -# Use a ccache dir that persists across SDK recreations. -# XXX: alternatively use a ccache dir that is usable by all jobs on a given node. -mkdir -p ccache - enter() { local verify_key= trap 'sudo rm -f chroot/etc/portage/gangue.*' RETURN @@ -13,8 +9,6 @@ enter() { sudo ln -f "${GOOGLE_APPLICATION_CREDENTIALS}" \ chroot/etc/portage/gangue.json bin/cork enter --bind-gpg-agent=false -- env \ - CCACHE_DIR=/mnt/host/source/ccache \ - CCACHE_MAXSIZE=5G \ FLATCAR_DEV_BUILDS="${DOWNLOAD_ROOT}" \ FLATCAR_DEV_BUILDS_SDK="${DOWNLOAD_ROOT_SDK}" \ {FETCH,RESUME}COMMAND_GS="/usr/bin/gangue get \ @@ -33,9 +27,6 @@ export FLATCAR_BUILD_ID # Set up GPG for signing uploads. gpg --import "${GPG_SECRET_KEY_FILE}" -# Figure out if ccache is doing us any good in this scheme. -enter ccache --zero-stats - script setup_board \ --board="${BOARD}" \ --getbinpkgver=${RELEASE_BASE:-"${FLATCAR_VERSION}" --toolchainpkgonly} \ @@ -61,5 +52,3 @@ script build_torcx_store \ --torcx_upload_root="${TORCX_PKG_DOWNLOAD_ROOT}" \ --tectonic_torcx_download_root="${TECTONIC_TORCX_DOWNLOAD_ROOT}" \ --upload - -enter ccache --show-stats diff --git a/update_chroot b/update_chroot index 2f0cb492d7..f590b92142 100755 --- a/update_chroot +++ b/update_chroot @@ -71,7 +71,7 @@ PORTAGE_BINHOST="$FLAGS_binhost $(get_sdk_binhost)" : ${PORTAGE_USERNAME:=${USER}} # Clean up old distfiles cache. It used to be split for 'host' and -# 'target' but that just duplicates files. Also a ccache was in there. +# 'target' but that just duplicates files. if [[ -d "${REPO_CACHE_DIR}/distfiles/host" ]]; then info "Cleaning up old distfiles cache..." sudo mv "${REPO_CACHE_DIR}"/{distfiles/host,distfiles.host} @@ -95,7 +95,6 @@ PORT_LOGDIR="/var/log/portage" PORTAGE_BINHOST="$PORTAGE_BINHOST" PORTAGE_USERNAME="${PORTAGE_USERNAME}" MAKEOPTS="--jobs=${NUM_JOBS} --load-average=$((NUM_JOBS * 2))" -CCACHE_UMASK="002" # Generally there isn't any need to add packages to @world by default. # You can use --select to override this. @@ -214,7 +213,6 @@ EMERGE_CMD="emerge" # can cause serious issues later. info "Updating basic system packages" sudo -E ${EMERGE_CMD} --quiet "${EMERGE_FLAGS[@]}" \ - dev-util/ccache \ sys-apps/portage \ sys-devel/crossdev \ sys-devel/sysroot-wrappers \