Merge pull request #209 from flatcar-linux/t-lo/container-ci-builds-publish-nightly-packages

ci-automation + setup_board: publish and use binpkgs
This commit is contained in:
Thilo Fromm 2022-01-10 12:09:21 +01:00 committed by GitHub
commit 567a526a5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 147 additions and 43 deletions

View File

@ -18,7 +18,7 @@ assert_inside_chroot
assert_not_root_user assert_not_root_user
DEFAULT_GROUP=developer DEFAULT_GROUP=developer
DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net" DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}"
# Developer-visible flags. # Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \ DEFINE_string board "${DEFAULT_BOARD}" \

View File

@ -142,15 +142,25 @@ get_board_binhost() {
board="$1" board="$1"
shift shift
local pkgs_include_toolchain=0
if [[ $# -eq 0 ]]; then if [[ $# -eq 0 ]]; then
set -- "${FLATCAR_VERSION_ID}" if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
# containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url
# and toolchain packages are at the same location as OS image ones
set -- "${FLATCAR_VERSION_ID}-${FLATCAR_BUILD_ID}"
pkgs_include_toolchain=1
else
set -- "${FLATCAR_VERSION_ID}"
fi
fi fi
for ver in "$@"; do for ver in "$@"; do
if [[ $toolchain_only -eq 0 ]]; then if [[ $toolchain_only -eq 0 ]]; then
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/" echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
fi fi
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" if [[ $pkgs_include_toolchain -eq 0 ]]; then
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
fi
done done
} }

View File

@ -23,23 +23,23 @@ The build pipeline can be used to build everything from scratch, including the S
The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux. The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux.
``` ```
.---------. .------------. .---------. .------------. .--------.
| scripts | | CI | | scripts | | CI | | Build |
| repo | | automation | | repo | | automation | | cache |
`---------´ `------------´ `---------´ `------------´ `--------´
| | | | |
| "alpha-3449.0.0-dev23" | "alpha-3449.0.0-dev23" |
| | | | |
| _______v_______ | _______v_______ |
+------- clone -----> ( SDK bootstrap ) +------- clone -----> ( SDK bootstrap ) |
| `-------------´ | `-------------´ |
|<- tag: sdk-3499.0.0-dev23 -´| |<- tag: sdk-3499.0.0-dev23 -´|`--- sdk tarball --->|
| | | | |
| _______v_______ | _______v_______ |
+-- clone -> ( SDK container ) +-- clone -> ( SDK container ) |
| sdk-3499.0.0-dev23 `-------------´ | sdk-3499.0.0-dev23 `-------------´ |
| | | |`- sdk container --->|
v v v v image
continue to OS continue to OS
image build image build
| |
@ -51,27 +51,34 @@ The build pipeline can be used to build everything from scratch, including the S
3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages). 3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages).
This step updates the versionfile, recording the Flatcar OS image version just built. This step updates the versionfile, recording the Flatcar OS image version just built.
It will generate and push a new version tag to the scripts repo. It will generate and push a new version tag to the scripts repo.
4. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it. 4. Packages are published and the generic OS image is built.
1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images. 5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
``` ```
.---------. .------------. .---------. .------------. .--------.
| scripts | | CI | | scripts | | CI | | Build |
| repo | | automation | | repo | | automation | | cache |
`---------´ `------------´ `---------´ `------------´ `--------´
| | | | |
| "alpha-3449.0.0-dev23" | "alpha-3449.0.0-dev23" |
| | | | |
| ____v_____ | ____v_____ |
+---------- clone --------> ( packages ) +---------- clone --------> ( packages ) |
| `--------´ | `--------´ |
|<-- tag: alpha-3499.0.0-dev23 --´| |<-- tag: alpha-3499.0.0-dev23 --´|`- sdk + OS packages -->|
| | | | container image |
| ___v___ | ______v_______ |
+----- clone ---> ( image ) | ( publish pkgs ) |
| alpha-3499.0.0-dev23 `-----´ | `------------´ |
| | | |`-- binary packages --->|
| __v__ | ___v__ |
+----- clone ---> ( vms ) +----- clone ---> ( image ) |
alpha-3499.0.0-dev23 `---´ | alpha-3499.0.0-dev23 `-----´ |
| |`-- sdk + packages + -->|
| __v__ OS image cnt img |
+----- clone ---> ( vms ) |
alpha-3499.0.0-dev23 `---´ |
`- vendor OS images ---->|
``` ```

View File

@ -89,6 +89,7 @@ function garbage_collect() {
else else
echo "## ${version} is an OS image version. ##" echo "## ${version} is an OS image version. ##"
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*" rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/binpkgs/boards/*/${os_docker_vernum}/*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*" rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/" rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
fi fi

View File

@ -0,0 +1,70 @@
#!/bin/bash
#
# Copyright (c) 2021 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
#
# push_packages() should be called w/ the positional INPUT parameters below.
# OS image binary packages publisher automation stub.
# This script will publish the packages from a pre-built packages container to
# the buildcache server, effectively turning the build cache into a
# binary packages server for the SDK.
#
# PREREQUISITES:
#
# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
# 2. Scripts repo version tag of OS image version to be built is available and checked out.
# 3. Flatcar packages container is available via build cache server
# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
# or present locally. Container must contain binary packages and torcx artifacts.
#
# INPUT:
#
# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
#
# OUTPUT:
#
# 1. Binary packages published to buildcache at "binpkgs/boards/[ARCH]-usr/[VERSION]/pkgs".
# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
# to be run after this step finishes / when this step is aborted.
set -eu
# This function is run _inside_ the SDK container
function image_build__copy_to_bincache() {
local arch="$1"
local version="$2"
source ci-automation/ci_automation_common.sh
cd /build/$arch-usr/var/lib/portage/pkgs/
copy_to_buildcache "binpkgs/boards/$arch-usr/$version/pkgs" *
}
# --
function push_packages() {
local arch="$1"
source ci-automation/ci_automation_common.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"
local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
local packages="flatcar-packages-${arch}"
local packages_image="${packages}:${docker_vernum}"
docker_image_from_buildcache "${packages}" "${docker_vernum}"
local cmd="source ci-automation/push_pkgs.sh"
cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'"
local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}"
./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \
bash -c "$cmd"
}
# --

View File

@ -3,6 +3,8 @@
# Use of this source code is governed by a BSD-style license that can be # Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file. # found in the LICENSE file.
. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
# All scripts should die on error unless commands are specifically excepted # All scripts should die on error unless commands are specifically excepted
# by prefixing with '!' or surrounded by 'set +e' / 'set -e'. # by prefixing with '!' or surrounded by 'set +e' / 'set -e'.
@ -337,7 +339,11 @@ readonly COREOS_EPOCH=1372636800
TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 )) TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
# Download URL prefix for SDK and board binary packages # Download URL prefix for SDK and board binary packages
: ${FLATCAR_DEV_BUILDS:=https://mirror.release.flatcar-linux.net} if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
: ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}}
else
: ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_PROD}}
fi
# Load developer's custom settings. Default location is in scripts dir, # Load developer's custom settings. Default location is in scripts dir,
# since that's available both inside and outside the chroot. By convention, # since that's available both inside and outside the chroot. By convention,

View File

@ -2,7 +2,9 @@
set -euo pipefail set -euo pipefail
DEFAULT_BASE_URL="https://mirror.release.flatcar-linux.net" . "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
DEFAULT_BASE_URL="${SETTING_BINPKG_SERVER_PROD}"
DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer" DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer"
DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk" DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk"
DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk" DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk"
@ -42,7 +44,7 @@ Usage: $0 FLAGS...
outside of the SDK chroot. If /dev/stdout or outside of the SDK chroot. If /dev/stdout or
/dev/stderr is used, only new values are printed. /dev/stderr is used, only new values are printed.
--binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'), --binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'),
e.g., 'https://bucket.release.flatcar-linux.net/flatcar-jenkins'. e.g., '${SETTING_BINPKG_SERVER_DEV}'.
This will update BOARD and SDK URLs accordingly. This will update BOARD and SDK URLs accordingly.
" "
exit 1 exit 1

8
settings.env Normal file
View File

@ -0,0 +1,8 @@
# Flatcar SDK settings
# Binary package caches, for releases and for development (nightlies etc.)
SETTING_BINPKG_SERVER_PROD="https://mirror.release.flatcar-linux.net"
# development servers / bin caches.
SETTING_BINPKG_SERVER_DEV="https://bucket.release.flatcar-linux.net/flatcar-jenkins"
SETTING_BINPKG_SERVER_DEV_CONTAINERISED="https://bincache.flatcar-linux.net/binpkgs"