mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-07 13:06:59 +02:00
Merge pull request #209 from flatcar-linux/t-lo/container-ci-builds-publish-nightly-packages
ci-automation + setup_board: publish and use binpkgs
This commit is contained in:
commit
567a526a5d
@ -18,7 +18,7 @@ assert_inside_chroot
|
|||||||
assert_not_root_user
|
assert_not_root_user
|
||||||
|
|
||||||
DEFAULT_GROUP=developer
|
DEFAULT_GROUP=developer
|
||||||
DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net"
|
DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}"
|
||||||
|
|
||||||
# Developer-visible flags.
|
# Developer-visible flags.
|
||||||
DEFINE_string board "${DEFAULT_BOARD}" \
|
DEFINE_string board "${DEFAULT_BOARD}" \
|
||||||
|
@ -142,15 +142,25 @@ get_board_binhost() {
|
|||||||
board="$1"
|
board="$1"
|
||||||
shift
|
shift
|
||||||
|
|
||||||
|
local pkgs_include_toolchain=0
|
||||||
if [[ $# -eq 0 ]]; then
|
if [[ $# -eq 0 ]]; then
|
||||||
set -- "${FLATCAR_VERSION_ID}"
|
if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
|
||||||
|
# containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url
|
||||||
|
# and toolchain packages are at the same location as OS image ones
|
||||||
|
set -- "${FLATCAR_VERSION_ID}-${FLATCAR_BUILD_ID}"
|
||||||
|
pkgs_include_toolchain=1
|
||||||
|
else
|
||||||
|
set -- "${FLATCAR_VERSION_ID}"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
for ver in "$@"; do
|
for ver in "$@"; do
|
||||||
if [[ $toolchain_only -eq 0 ]]; then
|
if [[ $toolchain_only -eq 0 ]]; then
|
||||||
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
|
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
|
||||||
fi
|
fi
|
||||||
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
|
if [[ $pkgs_include_toolchain -eq 0 ]]; then
|
||||||
|
echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -23,23 +23,23 @@ The build pipeline can be used to build everything from scratch, including the S
|
|||||||
The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux.
|
The resulting image will come in "amd64", "arm64", and "all" flavours, with support for respective OS target architectures. This step builds the Flatcar SDK container images published at ghcr.io/flatcar-linux.
|
||||||
|
|
||||||
```
|
```
|
||||||
.---------. .------------.
|
.---------. .------------. .--------.
|
||||||
| scripts | | CI |
|
| scripts | | CI | | Build |
|
||||||
| repo | | automation |
|
| repo | | automation | | cache |
|
||||||
`---------´ `------------´
|
`---------´ `------------´ `--------´
|
||||||
| |
|
| | |
|
||||||
| "alpha-3449.0.0-dev23"
|
| "alpha-3449.0.0-dev23" |
|
||||||
| |
|
| | |
|
||||||
| _______v_______
|
| _______v_______ |
|
||||||
+------- clone -----> ( SDK bootstrap )
|
+------- clone -----> ( SDK bootstrap ) |
|
||||||
| `-------------´
|
| `-------------´ |
|
||||||
|<- tag: sdk-3499.0.0-dev23 -´|
|
|<- tag: sdk-3499.0.0-dev23 -´|`--- sdk tarball --->|
|
||||||
| |
|
| | |
|
||||||
| _______v_______
|
| _______v_______ |
|
||||||
+-- clone -> ( SDK container )
|
+-- clone -> ( SDK container ) |
|
||||||
| sdk-3499.0.0-dev23 `-------------´
|
| sdk-3499.0.0-dev23 `-------------´ |
|
||||||
| |
|
| |`- sdk container --->|
|
||||||
v v
|
v v image
|
||||||
continue to OS
|
continue to OS
|
||||||
image build
|
image build
|
||||||
|
|
|
|
||||||
@ -51,27 +51,34 @@ The build pipeline can be used to build everything from scratch, including the S
|
|||||||
3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages).
|
3. Packages build (`packages.sh`): using the SDK container version recorded in the versionfile, build OS image packages and generate a new container image (containing both SDK and packages).
|
||||||
This step updates the versionfile, recording the Flatcar OS image version just built.
|
This step updates the versionfile, recording the Flatcar OS image version just built.
|
||||||
It will generate and push a new version tag to the scripts repo.
|
It will generate and push a new version tag to the scripts repo.
|
||||||
4. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
|
4. Packages are published and the generic OS image is built.
|
||||||
|
1. Binary packages are published (`push_pkgs.sh`) to the build cache, making them available to developers who base their work on the main branch.
|
||||||
|
2. Image build (`image.sh`): Using the container from 3., build an OS image and torcx store, and generate a new container image with everything in it.
|
||||||
5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
|
5. VMs build (`vms.sh`). Using the packages+torcx+image container from 4., build vendor images. Results are vendor-specific OS images.
|
||||||
|
|
||||||
```
|
```
|
||||||
.---------. .------------.
|
.---------. .------------. .--------.
|
||||||
| scripts | | CI |
|
| scripts | | CI | | Build |
|
||||||
| repo | | automation |
|
| repo | | automation | | cache |
|
||||||
`---------´ `------------´
|
`---------´ `------------´ `--------´
|
||||||
| |
|
| | |
|
||||||
| "alpha-3449.0.0-dev23"
|
| "alpha-3449.0.0-dev23" |
|
||||||
| |
|
| | |
|
||||||
| ____v_____
|
| ____v_____ |
|
||||||
+---------- clone --------> ( packages )
|
+---------- clone --------> ( packages ) |
|
||||||
| `--------´
|
| `--------´ |
|
||||||
|<-- tag: alpha-3499.0.0-dev23 --´|
|
|<-- tag: alpha-3499.0.0-dev23 --´|`- sdk + OS packages -->|
|
||||||
| |
|
| | container image |
|
||||||
| ___v___
|
| ______v_______ |
|
||||||
+----- clone ---> ( image )
|
| ( publish pkgs ) |
|
||||||
| alpha-3499.0.0-dev23 `-----´
|
| `------------´ |
|
||||||
| |
|
| |`-- binary packages --->|
|
||||||
| __v__
|
| ___v__ |
|
||||||
+----- clone ---> ( vms )
|
+----- clone ---> ( image ) |
|
||||||
alpha-3499.0.0-dev23 `---´
|
| alpha-3499.0.0-dev23 `-----´ |
|
||||||
|
| |`-- sdk + packages + -->|
|
||||||
|
| __v__ OS image cnt img |
|
||||||
|
+----- clone ---> ( vms ) |
|
||||||
|
alpha-3499.0.0-dev23 `---´ |
|
||||||
|
`- vendor OS images ---->|
|
||||||
```
|
```
|
||||||
|
@ -89,6 +89,7 @@ function garbage_collect() {
|
|||||||
else
|
else
|
||||||
echo "## ${version} is an OS image version. ##"
|
echo "## ${version} is an OS image version. ##"
|
||||||
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
|
rmpat="${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-packages-*"
|
||||||
|
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/binpkgs/boards/*/${os_docker_vernum}/*"
|
||||||
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
|
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/containers/${os_docker_vernum}/flatcar-images-*"
|
||||||
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
|
rmpat="${rmpat} ${BUILDCACHE_PATH_PREFIX}/images/*/${os_vernum}/"
|
||||||
fi
|
fi
|
||||||
|
70
ci-automation/push_pkgs.sh
Normal file
70
ci-automation/push_pkgs.sh
Normal file
@ -0,0 +1,70 @@
|
|||||||
|
#!/bin/bash
|
||||||
|
#
|
||||||
|
# Copyright (c) 2021 The Flatcar Maintainers.
|
||||||
|
# Use of this source code is governed by a BSD-style license that can be
|
||||||
|
# found in the LICENSE file.
|
||||||
|
|
||||||
|
# >>> This file is supposed to be SOURCED from the repository ROOT. <<<
|
||||||
|
#
|
||||||
|
# push_packages() should be called w/ the positional INPUT parameters below.
|
||||||
|
|
||||||
|
# OS image binary packages publisher automation stub.
|
||||||
|
# This script will publish the packages from a pre-built packages container to
|
||||||
|
# the buildcache server, effectively turning the build cache into a
|
||||||
|
# binary packages server for the SDK.
|
||||||
|
#
|
||||||
|
# PREREQUISITES:
|
||||||
|
#
|
||||||
|
# 1. SDK version and OS image version are recorded in sdk_container/.repo/manifests/version.txt
|
||||||
|
# 2. Scripts repo version tag of OS image version to be built is available and checked out.
|
||||||
|
# 3. Flatcar packages container is available via build cache server
|
||||||
|
# from "/containers/[VERSION]/flatcar-packages-[ARCH]-[FLATCAR_VERSION].tar.gz"
|
||||||
|
# or present locally. Container must contain binary packages and torcx artifacts.
|
||||||
|
#
|
||||||
|
# INPUT:
|
||||||
|
#
|
||||||
|
# 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64").
|
||||||
|
#
|
||||||
|
# OUTPUT:
|
||||||
|
#
|
||||||
|
# 1. Binary packages published to buildcache at "binpkgs/boards/[ARCH]-usr/[VERSION]/pkgs".
|
||||||
|
# 2. "./ci-cleanup.sh" with commands to clean up temporary build resources,
|
||||||
|
# to be run after this step finishes / when this step is aborted.
|
||||||
|
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
# This function is run _inside_ the SDK container
|
||||||
|
function image_build__copy_to_bincache() {
|
||||||
|
local arch="$1"
|
||||||
|
local version="$2"
|
||||||
|
|
||||||
|
source ci-automation/ci_automation_common.sh
|
||||||
|
|
||||||
|
cd /build/$arch-usr/var/lib/portage/pkgs/
|
||||||
|
copy_to_buildcache "binpkgs/boards/$arch-usr/$version/pkgs" *
|
||||||
|
}
|
||||||
|
# --
|
||||||
|
|
||||||
|
function push_packages() {
|
||||||
|
local arch="$1"
|
||||||
|
|
||||||
|
source ci-automation/ci_automation_common.sh
|
||||||
|
init_submodules
|
||||||
|
|
||||||
|
source sdk_container/.repo/manifests/version.txt
|
||||||
|
local vernum="${FLATCAR_VERSION}"
|
||||||
|
local docker_vernum="$(vernum_to_docker_image_version "${vernum}")"
|
||||||
|
|
||||||
|
local packages="flatcar-packages-${arch}"
|
||||||
|
local packages_image="${packages}:${docker_vernum}"
|
||||||
|
|
||||||
|
docker_image_from_buildcache "${packages}" "${docker_vernum}"
|
||||||
|
|
||||||
|
local cmd="source ci-automation/push_pkgs.sh"
|
||||||
|
cmd="$cmd; image_build__copy_to_bincache '$arch' '$vernum'"
|
||||||
|
|
||||||
|
local my_name="flatcar-packages-publisher-${arch}-${docker_vernum}"
|
||||||
|
./run_sdk_container -x ./ci-cleanup.sh -n "${my_name}" -C "${packages_image}" \
|
||||||
|
bash -c "$cmd"
|
||||||
|
}
|
||||||
|
# --
|
@ -3,6 +3,8 @@
|
|||||||
# Use of this source code is governed by a BSD-style license that can be
|
# Use of this source code is governed by a BSD-style license that can be
|
||||||
# found in the LICENSE file.
|
# found in the LICENSE file.
|
||||||
|
|
||||||
|
. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
|
||||||
|
|
||||||
# All scripts should die on error unless commands are specifically excepted
|
# All scripts should die on error unless commands are specifically excepted
|
||||||
# by prefixing with '!' or surrounded by 'set +e' / 'set -e'.
|
# by prefixing with '!' or surrounded by 'set +e' / 'set -e'.
|
||||||
|
|
||||||
@ -337,7 +339,11 @@ readonly COREOS_EPOCH=1372636800
|
|||||||
TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
|
TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
|
||||||
|
|
||||||
# Download URL prefix for SDK and board binary packages
|
# Download URL prefix for SDK and board binary packages
|
||||||
: ${FLATCAR_DEV_BUILDS:=https://mirror.release.flatcar-linux.net}
|
if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
|
||||||
|
: ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}}
|
||||||
|
else
|
||||||
|
: ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_PROD}}
|
||||||
|
fi
|
||||||
|
|
||||||
# Load developer's custom settings. Default location is in scripts dir,
|
# Load developer's custom settings. Default location is in scripts dir,
|
||||||
# since that's available both inside and outside the chroot. By convention,
|
# since that's available both inside and outside the chroot. By convention,
|
||||||
|
@ -2,7 +2,9 @@
|
|||||||
|
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
DEFAULT_BASE_URL="https://mirror.release.flatcar-linux.net"
|
. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
|
||||||
|
|
||||||
|
DEFAULT_BASE_URL="${SETTING_BINPKG_SERVER_PROD}"
|
||||||
DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer"
|
DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer"
|
||||||
DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk"
|
DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk"
|
||||||
DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk"
|
DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk"
|
||||||
@ -42,7 +44,7 @@ Usage: $0 FLAGS...
|
|||||||
outside of the SDK chroot. If /dev/stdout or
|
outside of the SDK chroot. If /dev/stdout or
|
||||||
/dev/stderr is used, only new values are printed.
|
/dev/stderr is used, only new values are printed.
|
||||||
--binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'),
|
--binhost Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'),
|
||||||
e.g., 'https://bucket.release.flatcar-linux.net/flatcar-jenkins'.
|
e.g., '${SETTING_BINPKG_SERVER_DEV}'.
|
||||||
This will update BOARD and SDK URLs accordingly.
|
This will update BOARD and SDK URLs accordingly.
|
||||||
"
|
"
|
||||||
exit 1
|
exit 1
|
||||||
|
8
settings.env
Normal file
8
settings.env
Normal file
@ -0,0 +1,8 @@
|
|||||||
|
# Flatcar SDK settings
|
||||||
|
|
||||||
|
# Binary package caches, for releases and for development (nightlies etc.)
|
||||||
|
SETTING_BINPKG_SERVER_PROD="https://mirror.release.flatcar-linux.net"
|
||||||
|
|
||||||
|
# development servers / bin caches.
|
||||||
|
SETTING_BINPKG_SERVER_DEV="https://bucket.release.flatcar-linux.net/flatcar-jenkins"
|
||||||
|
SETTING_BINPKG_SERVER_DEV_CONTAINERISED="https://bincache.flatcar-linux.net/binpkgs"
|
Loading…
Reference in New Issue
Block a user