diff --git a/build_image b/build_image index 8fe2e9e26a..6a2377d69f 100755 --- a/build_image +++ b/build_image @@ -18,7 +18,7 @@ assert_inside_chroot assert_not_root_user DEFAULT_GROUP=developer -DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net" +DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}" # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index b48ca19936..e88981a479 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -142,15 +142,25 @@ get_board_binhost() { board="$1" shift + local pkgs_include_toolchain=0 if [[ $# -eq 0 ]]; then - set -- "${FLATCAR_VERSION_ID}" + if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then + # containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url + # and toolchain packages are at the same location as OS image ones + set -- "${FLATCAR_VERSION_ID}-${FLATCAR_BUILD_ID}" + pkgs_include_toolchain=1 + else + set -- "${FLATCAR_VERSION_ID}" + fi fi for ver in "$@"; do if [[ $toolchain_only -eq 0 ]]; then echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/" fi - echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" + if [[ $pkgs_include_toolchain -eq 0 ]]; then + echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" + fi done } diff --git a/ci-automation/README.md b/ci-automation/README.md index 4197d9bd04..f7c61d551e 100644 --- a/ci-automation/README.md +++ b/ci-automation/README.md @@ -23,23 +23,23 @@ The build pipeline can be used to build everything from scratch, including the S The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux. ``` - .---------. .------------. - | scripts | | CI | - | repo | | automation | - `---------´ `------------´ - | | - | "alpha-3449.0.0-dev23" - | | - | _______v_______ - +------- clone -----> ( SDK bootstrap ) - | `-------------´ - |<- tag: sdk-3499.0.0-dev23 -´| - | | - | _______v_______ - +-- clone -> ( SDK container ) - | sdk-3499.0.0-dev23 `-------------´ - | | - v v + .---------. .------------. .--------. + | scripts | | CI | | Build | + | repo | | automation | | cache | + `---------´ `------------´ `--------´ + | | | + | "alpha-3449.0.0-dev23" | + | | | + | _______v_______ | + +------- clone -----> ( SDK bootstrap ) | + | `-------------´ | + |<- tag: sdk-3499.0.0-dev23 -´|`--- sdk tarball --->| + | | | + | _______v_______ | + +-- clone -> ( SDK container ) | + | sdk-3499.0.0-dev23 `-------------´ | + | |`- sdk container --->| + v v image continue to OS image build | @@ -51,27 +51,34 @@ The build pipeline can be used to build everything from scratch, including the S 3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages). This step updates the versionfile, recording the Flatcar OS image version just built. It will generate and push a new version tag to the scripts repo. -4. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. +4. Packages are published and the generic OS image is built. + 1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch. + 2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. 5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images. ``` - .---------. .------------. - | scripts | | CI | - | repo | | automation | - `---------´ `------------´ - | | - | "alpha-3449.0.0-dev23" - | | - | ____v_____ - +---------- clone --------> ( packages ) - | `--------´ - |<-- tag: alpha-3499.0.0-dev23 --´| - | | - | ___v___ - +----- clone ---> ( image ) - | alpha-3499.0.0-dev23 `-----´ - | | - | __v__ - +----- clone ---> ( vms ) - alpha-3499.0.0-dev23 `---´ + .---------. .------------. .--------. + | scripts | | CI | | Build | + | repo | | automation | | cache | + `---------´ `------------´ `--------´ + | | | + | "alpha-3449.0.0-dev23" | + | | | + | ____v_____ | + +---------- clone --------> ( packages ) | + | `--------´ | + |<-- tag: alpha-3499.0.0-dev23 --´|`- sdk + OS packages -->| + | | container image | + | ______v_______ | + | ( publish pkgs ) | + | `------------´ | + | |`-- binary packages --->| + | ___v__ | + +----- clone ---> ( image ) | + | alpha-3499.0.0-dev23 `-----´ | + | |`-- sdk + packages + -->| + | __v__ OS image cnt img | + +----- clone ---> ( vms ) | + alpha-3499.0.0-dev23 `---´ | + `- vendor OS images ---->| ``` diff --git a/ci-automation/garbage_collect.sh b/ci-automation/garbage_collect.sh index 97aabf4dcb..02003ac6b9 100644 --- a/ci-automation/garbage_collect.sh +++ b/ci-automation/garbage_collect.sh @@ -89,6 +89,7 @@ function garbage_collect() { else echo "## ${version} is an OS image version. ##" rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*" + rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/binpkgs/boards/*/${os_docker_vernum}/*" rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*" rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/" fi diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh new file mode 100644 index 0000000000..f9d764e71c --- /dev/null +++ b/ci-automation/push_pkgs.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# +# Copyright (c) 2021 The Flatcar Maintainers. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# >>> This file is supposed to be SOURCED from the repository ROOT. <<< +# +# push_packages() should be called w/ the positional INPUT parameters below. + +# OS image binary packages publisher automation stub. +# This script will publish the packages from a pre-built packages container to +# the buildcache server, effectively turning the build cache into a +# binary packages server for the SDK. +# +# PREREQUISITES: +# +# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt +# 2. Scripts repo version tag of OS image version to be built is available and checked out. +# 3. Flatcar packages container is available via build cache server +# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz" +# or present locally. Container must contain binary packages and torcx artifacts. +# +# INPUT: +# +# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). +# +# OUTPUT: +# +# 1. Binary packages published to buildcache at "binpkgs/boards/[ARCH]-usr/[VERSION]/pkgs". +# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources, +# to be run after this step finishes / when this step is aborted. + +set -eu + +# This function is run _inside_ the SDK container +function image_build__copy_to_bincache() { + local arch="$1" + local version="$2" + + source ci-automation/ci_automation_common.sh + + cd /build/$arch-usr/var/lib/portage/pkgs/ + copy_to_buildcache "binpkgs/boards/$arch-usr/$version/pkgs" * +} +# -- + +function push_packages() { + local arch="$1" + + source ci-automation/ci_automation_common.sh + init_submodules + + source sdk_container/.repo/manifests/version.txt + local vernum="${FLATCAR_VERSION}" + local docker_vernum="$(vernum_to_docker_image_version "${vernum}")" + + local packages="flatcar-packages-${arch}" + local packages_image="${packages}:${docker_vernum}" + + docker_image_from_buildcache "${packages}" "${docker_vernum}" + + local cmd="source ci-automation/push_pkgs.sh" + cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'" + + local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}" + ./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \ + bash -c "$cmd" +} +# -- diff --git a/common.sh b/common.sh index 1f155004e4..2fb98cc071 100644 --- a/common.sh +++ b/common.sh @@ -3,6 +3,8 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. +. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1 + # All scripts should die on error unless commands are specifically excepted # by prefixing with '!' or surrounded by 'set +e' / 'set -e'. @@ -337,7 +339,11 @@ readonly COREOS_EPOCH=1372636800 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) # Download URL prefix for SDK and board binary packages -: ${FLATCAR_DEV_BUILDS:=https://mirror.release.flatcar-linux.net} +if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then + : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}} +else + : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_PROD}} +fi # Load developer's custom settings. Default location is in scripts dir, # since that's available both inside and outside the chroot. By convention, diff --git a/set_version b/set_version index abf89b498d..240002a14d 100755 --- a/set_version +++ b/set_version @@ -2,7 +2,9 @@ set -euo pipefail -DEFAULT_BASE_URL="https://mirror.release.flatcar-linux.net" +. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1 + +DEFAULT_BASE_URL="${SETTING_BINPKG_SERVER_PROD}" DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer" DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk" DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk" @@ -42,7 +44,7 @@ Usage: $0 FLAGS... outside of the SDK chroot. If /dev/stdout or /dev/stderr is used, only new values are printed. --binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'), - e.g., 'https://bucket.release.flatcar-linux.net/flatcar-jenkins'. + e.g., '${SETTING_BINPKG_SERVER_DEV}'. This will update BOARD and SDK URLs accordingly. " exit 1 diff --git a/settings.env b/settings.env new file mode 100644 index 0000000000..3999fd0098 --- /dev/null +++ b/settings.env @@ -0,0 +1,8 @@ +# Flatcar SDK settings + +# Binary package caches, for releases and for development (nightlies etc.) +SETTING_BINPKG_SERVER_PROD="https://mirror.release.flatcar-linux.net" + +# development servers / bin caches. +SETTING_BINPKG_SERVER_DEV="https://bucket.release.flatcar-linux.net/flatcar-jenkins" +SETTING_BINPKG_SERVER_DEV_CONTAINERISED="https://bincache.flatcar-linux.net/binpkgs"