diff --git a/build_image b/build_image index 8fe2e9e26a..6a2377d69f 100755 --- a/build_image +++ b/build_image @@ -18,7 +18,7 @@ assert_inside_chroot assert_not_root_user DEFAULT_GROUP=developer -DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net" +DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}" # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index b48ca19936..f64fd3bd6f 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -142,15 +142,25 @@ get_board_binhost() { board="$1" shift + local no_toolchain=0 if [[ $# -eq 0 ]]; then - set -- "${FLATCAR_VERSION_ID}" + if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then + # containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url + # and toolchain packages are at the same location as OS image ones + set -- "${FLATCAR_VERSION_ID}-${FLATCAR_BUILD_ID}" + no_toolchain=1 + else + set -- "${FLATCAR_VERSION_ID}" + fi fi for ver in "$@"; do if [[ $toolchain_only -eq 0 ]]; then echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/" fi - echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" + if [[ $no_toolchain -eq 0 ]]; then + echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" + fi done } diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index 97aabf4dcb..f1ce917d1f 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -89,6 +89,7 @@ function garbage_collect() { else echo "## ${version} is an OS image version. ##" rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*" + rmpat="${BUILDCACHE_PATH_PREFIX}/binpkgs/boards/*/${os_docker_vernum}/*" rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*" rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/" fi diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh new file mode 100644 index 0000000000..f7591b5a77 --- /dev/null +++ b/ci-automation/push_pkgs.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# push_packages() should be called w/ the positional INPUT parameters below. + +# OS image binary packages publisher automation stub. +# This script will publish the packages from a pre-built packages container to +# the buildcache server, effectively turning the build cache into a +# binary packages server for the SDK. +# +# PREREQUISITES: +# +# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# 3. Flatcar packages container is available via build cache server +# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" +# or present locally. Container must contain binary packages and torcx artefacts. +# +# INPUT: +# +# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). +# +# OUTPUT: +# +# 1. Binary packages published to buildcache at "binpkgs/boards/[ARCH]-usr/[VERSION]/pkgs". +# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + +set -eu + +# This function is run _inside_ the SDK container +function image_build__copy_to_bincache() { + local arch="$1" + local version="$2" + + source ci-automation/ci_automation_common.sh + + cd /build/$arch-usr/var/lib/portage/pkgs/ + copy_to_buildcache "binpkgs/boards/$arch-usr/$version/pkgs" * +} +# -- + +function push_packages() { + local arch="$1" + + source ci-automation/ci_automation_common.sh + init_submodules + + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_VERSION}" + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + + local packages="flatcar-packages-${arch}" + local packages_image="${packages}:${docker_vernum}" + + docker_image_from_buildcache "${packages}" "${docker_vernum}" + + local cmd="source ci-automation/push_pkgs.sh" + cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'" + + local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}" + ./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \ + bash -c "$cmd" +} +# -- diff --git a/common.sh b/common.sh index 1f155004e4..2fb98cc071 100644 --- a/common.sh +++ b/common.sh @@ -3,6 +3,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1 + # All scripts should die on error unless commands are specifically excepted # by prefixing with '!' or surrounded by 'set +e' / 'set -e'. @@ -337,7 +339,11 @@ readonly COREOS_EPOCH=1372636800 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) # Download URL prefix for SDK and board binary packages -: ${FLATCAR_DEV_BUILDS:=https://mirror.release.flatcar-linux.net} +if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then + : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}} +else + : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_PROD}} +fi # Load developer's custom settings. Default location is in scripts dir, # since that's available both inside and outside the chroot. By convention, diff --git a/set_version b/set_version index abf89b498d..240002a14d 100755 --- a/set_version +++ b/set_version @@ -2,7 +2,9 @@ set -euo pipefail -DEFAULT_BASE_URL="https://mirror.release.flatcar-linux.net" +. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1 + +DEFAULT_BASE_URL="${SETTING_BINPKG_SERVER_PROD}" DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer" DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk" DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk" @@ -42,7 +44,7 @@ Usage: $0 FLAGS... outside of the SDK chroot. If /dev/stdout or /dev/stderr is used, only new values are printed. --binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'), - e.g., 'https://bucket.release.flatcar-linux.net/flatcar-jenkins'. + e.g., '${SETTING_BINPKG_SERVER_DEV}'. This will update BOARD and SDK URLs accordingly. " exit 1 diff --git a/settings.env b/settings.env new file mode 100644 index 0000000000..3999fd0098 --- /dev/null +++ b/settings.env @@ -0,0 +1,8 @@ +# Flatcar SDK settings + +# Binary package caches, for releases and for development (nightlies etc.) +SETTING_BINPKG_SERVER_PROD="https://mirror.release.flatcar-linux.net" + +# development servers / bin caches. +SETTING_BINPKG_SERVER_DEV="https://bucket.release.flatcar-linux.net/flatcar-jenkins" +SETTING_BINPKG_SERVER_DEV_CONTAINERISED="https://bincache.flatcar-linux.net/binpkgs"