mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-06 20:47:00 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #209 from flatcar-linux/t-lo/container-ci-builds-publish-nightly-packages

Browse Source 
ci-automation + setup_board: publish and use binpkgs
This commit is contained in:
Thilo Fromm 2022-01-10 12:09:21 +01:00 committed by GitHub
commit 567a526a5d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 147 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
build_image
View File

@ -18,7 +18,7 @@ assert_inside_chroot
assert_not_root_user
DEFAULT_GROUP=developer
DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net"
DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}"
# Developer-visible flags.
DEFINE_string board "${DEFAULT_BOARD}" \

14
build_library/toolchain_util.sh
View File

@ -142,15 +142,25 @@ get_board_binhost() {
board="$1"
shift
local pkgs_include_toolchain=0
if [[ $# -eq 0 ]]; then
set -- "${FLATCAR_VERSION_ID}"
if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
# containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url
# and toolchain packages are at the same location as OS image ones
set -- "${FLATCAR_VERSION_ID}-${FLATCAR_BUILD_ID}"
pkgs_include_toolchain=1
else
set -- "${FLATCAR_VERSION_ID}"
fi
fi
for ver in "$@"; do
if [[ $toolchain_only -eq 0 ]]; then
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
fi
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
if [[ $pkgs_include_toolchain -eq 0 ]]; then
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
fi
done
}

81
ci-automation/README.md
View File

@ -23,23 +23,23 @@ The build pipeline can be used to build everything from scratch, including the S
The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux.
```
.---------. .------------.
| scripts | | CI |
| repo | | automation |
`---------´ `------------´
| |
| "alpha-3449.0.0-dev23"
| |
| _______v_______
+------- clone -----> ( SDK bootstrap )
| `-------------´
|<- tag: sdk-3499.0.0-dev23 -´|
| |
| _______v_______
+-- clone -> ( SDK container )
| sdk-3499.0.0-dev23 `-------------´
| |
v v
.---------. .------------. .--------.
| scripts | | CI | | Build |
| repo | | automation | | cache |
`---------´ `------------´ `--------´
| | |
| "alpha-3449.0.0-dev23" |
| | |
| _______v_______ |
+------- clone -----> ( SDK bootstrap ) |
| `-------------´ |
|<- tag: sdk-3499.0.0-dev23 -´|`--- sdk tarball --->|
| | |
| _______v_______ |
+-- clone -> ( SDK container ) |
| sdk-3499.0.0-dev23 `-------------´ |
| |`- sdk container --->|
v v image
continue to OS
image build
|
@ -51,27 +51,34 @@ The build pipeline can be used to build everything from scratch, including the S
3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages).
This step updates the versionfile, recording the Flatcar OS image version just built.
It will generate and push a new version tag to the scripts repo.
4. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
4. Packages are published and the generic OS image is built.
1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
```
.---------. .------------.
| scripts | | CI |
| repo | | automation |
`---------´ `------------´
| |
| "alpha-3449.0.0-dev23"
| |
| ____v_____
+---------- clone --------> ( packages )
| `--------´
|<-- tag: alpha-3499.0.0-dev23 --´|
| |
| ___v___
+----- clone ---> ( image )
| alpha-3499.0.0-dev23 `-----´
| |
| __v__
+----- clone ---> ( vms )
alpha-3499.0.0-dev23 `---´
.---------. .------------. .--------.
| scripts | | CI | | Build |
| repo | | automation | | cache |
`---------´ `------------´ `--------´
| | |
| "alpha-3449.0.0-dev23" |
| | |
| ____v_____ |
+---------- clone --------> ( packages ) |
| `--------´ |
|<-- tag: alpha-3499.0.0-dev23 --´|`- sdk + OS packages -->|
| | container image |
| ______v_______ |
| ( publish pkgs ) |
| `------------´ |
| |`-- binary packages --->|
| ___v__ |
+----- clone ---> ( image ) |
| alpha-3499.0.0-dev23 `-----´ |
| |`-- sdk + packages + -->|
| __v__ OS image cnt img |
+----- clone ---> ( vms ) |
alpha-3499.0.0-dev23 `---´ |
`- vendor OS images ---->|
```

1
ci-automation/garbage_collect.sh
View File

@ -89,6 +89,7 @@ function garbage_collect() {
else
echo "## ${version} is an OS image version. ##"
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/binpkgs/boards/*/${os_docker_vernum}/*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
fi

70
ci-automation/push_pkgs.sh Normal file
View File

@ -0,0 +1,70 @@
#!/bin/bash
#
# Copyright (c) 2021 The Flatcar Maintainers.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
#
# push_packages() should be called w/ the positional INPUT parameters below.
# OS image binary packages publisher automation stub.
# This script will publish the packages from a pre-built packages container to
# the buildcache server, effectively turning the build cache into a
# binary packages server for the SDK.
#
# PREREQUISITES:
#
# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
# 2. Scripts repo version tag of OS image version to be built is available and checked out.
# 3. Flatcar packages container is available via build cache server
# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
# or present locally. Container must contain binary packages and torcx artifacts.
#
# INPUT:
#
# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
#
# OUTPUT:
#
# 1. Binary packages published to buildcache at "binpkgs/boards/[ARCH]-usr/[VERSION]/pkgs".
# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
# to be run after this step finishes / when this step is aborted.
set -eu
# This function is run _inside_ the SDK container
function image_build__copy_to_bincache() {
local arch="$1"
local version="$2"
source ci-automation/ci_automation_common.sh
cd /build/$arch-usr/var/lib/portage/pkgs/
copy_to_buildcache "binpkgs/boards/$arch-usr/$version/pkgs" *
}
# --
function push_packages() {
local arch="$1"
source ci-automation/ci_automation_common.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"
local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
local packages="flatcar-packages-${arch}"
local packages_image="${packages}:${docker_vernum}"
docker_image_from_buildcache "${packages}" "${docker_vernum}"
local cmd="source ci-automation/push_pkgs.sh"
cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'"
local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}"
./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \
bash -c "$cmd"
}
# --

8
common.sh
View File

@ -3,6 +3,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
# All scripts should die on error unless commands are specifically excepted
# by prefixing with '!' or surrounded by 'set +e' / 'set -e'.
@ -337,7 +339,11 @@ readonly COREOS_EPOCH=1372636800
TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
# Download URL prefix for SDK and board binary packages
: ${FLATCAR_DEV_BUILDS:=https://mirror.release.flatcar-linux.net}
if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
: ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}}
else
: ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_PROD}}
fi
# Load developer's custom settings. Default location is in scripts dir,
# since that's available both inside and outside the chroot. By convention,

6
set_version
View File

@ -2,7 +2,9 @@
set -euo pipefail
DEFAULT_BASE_URL="https://mirror.release.flatcar-linux.net"
. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
DEFAULT_BASE_URL="${SETTING_BINPKG_SERVER_PROD}"
DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer"
DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk"
DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk"
@ -42,7 +44,7 @@ Usage: $0 FLAGS...
outside of the SDK chroot. If /dev/stdout or
/dev/stderr is used, only new values are printed.
--binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'),
e.g., 'https://bucket.release.flatcar-linux.net/flatcar-jenkins'.
e.g., '${SETTING_BINPKG_SERVER_DEV}'.
This will update BOARD and SDK URLs accordingly.
"
exit 1

8
settings.env Normal file
View File

@ -0,0 +1,8 @@
# Flatcar SDK settings
# Binary package caches, for releases and for development (nightlies etc.)
SETTING_BINPKG_SERVER_PROD="https://mirror.release.flatcar-linux.net"
# development servers / bin caches.
SETTING_BINPKG_SERVER_DEV="https://bucket.release.flatcar-linux.net/flatcar-jenkins"
SETTING_BINPKG_SERVER_DEV_CONTAINERISED="https://bincache.flatcar-linux.net/binpkgs"