mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-06 04:26:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

scripts, CI, workflows: remove submodule handling (main)

Browse Source
This commit is contained in:
Thilo Fromm 2023-04-13 12:26:36 +02:00
parent 15b3beb5a0
commit 401af830d1
25 changed files with 68 additions and 264 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
.github/workflows/ci.yaml vendored
View File

@ -11,24 +11,6 @@ on:
Space-separated vendor formats to build.
required: true
default: qemu_uefi
portage_remote:
description: |
The remote we should pull portage-stable from. This defaults to whatever the submodule is set to in this repo.
If triggered by a change in the portage repo, please set this to the remote which is proposing a change.
required: false
portage_ref:
description: |
This is the ref we will use to pull the changes from the portage_remote.
required: false
coreos_remote:
description: |
The remote we should pull coreos-overlay from. This defaults to whatever the submodule is set to in this repo.
If triggered by a change in the portage repo, please set this to the remote which is proposing a change.
required: false
coreos_ref:
description: |
This is the ref we will use to pull the changes from the coreos_remote.
required: false
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }}
@ -75,7 +57,6 @@ jobs:
with:
path: scripts
fetch-depth: 0
submodules: true
# Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
# instead of its head commit. That commit is not connected to any branch.
@ -96,23 +77,10 @@ jobs:
shell: bash
run: |
arch="${{ matrix.arch }}"
COREOS_REMOTE=""
COREOS_REF=""
PORTAGE_REMOTE=""
PORTAGE_REF=""
IMAGE_FORMATS="qemu_uefi"
[ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}"
[ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}"
[ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}"
[ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}"
[ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}"
echo "arch=${arch}" >> $GITHUB_ENV
echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV
echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV
echo "PORTAGE_REMOTE=${PORTAGE_REMOTE}" >> $GITHUB_ENV
echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV
IMAGE_FORMATS="qemu_uefi"
[ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}"
echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV
# Artifact root for images and torcx tarball as seen from within the container
@ -124,29 +92,6 @@ jobs:
# this with its IP address.
echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV
- name: Checkout submodules
shell: bash
run: |
if [ "${COREOS_REMOTE}" != "" -a "${COREOS_REF}" != "" ]
then
REMOTE="${COREOS_REMOTE}"
REPO_PATH="sdk_container/src/third_party/coreos-overlay"
[[ "$REMOTE" == "https:*" ]] || REMOTE="https://github.com/${COREOS_REMOTE}"
git -C "$REPO_PATH" remote add test "$REMOTE"
git -C "$REPO_PATH" fetch test
git -C "$REPO_PATH" checkout "test/${COREOS_REF}"
fi
if [ "${PORTAGE_REMOTE}" != "" -a "${PORTAGE_REF}" != "" ]
then
REMOTE="${PORTAGE_REMOTE}"
REPO_PATH="sdk_container/src/third_party/portage-stable"
[[ "$REMOTE" == "https:*" ]] || REMOTE="https://github.com/${PORTAGE_REMOTE}"
git -C "$REPO_PATH" remote add test "$REMOTE"
git -C "$REPO_PATH" fetch test
git -C "$REPO_PATH" checkout "test/${PORTAGE_REF}"
fi
- name: Build packages
shell: bash
run: |

1
.github/workflows/run-kola-tests.yaml vendored
View File

@ -58,7 +58,6 @@ jobs:
with:
path: scripts
fetch-depth: 0
submodules: true
# Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
# instead of its head commit. That commit is not connected to any branch.

22
README.md
View File

@ -11,12 +11,10 @@ The SDK can be used to
[flatcar-docs]: https://www.flatcar.org/docs/latest/reference/developer-guides/sdk-modifying-flatcar/
# Using the scripts repository: submodules and tags
# Using the scripts repository
The repository is meant to be the entry point for Flatcar builds and development.
For building packages, there are 2 additional repositories, [coreos-overlay](https://github.com/flatcar/coreos-overlay) and [portage-stable](https://github.com/flatcar/portage-stable), which contain all packages' `ebuild` (build configuration) files.
These repositories are included in `scripts` via git submodules and are used by the SDK container wrapper scripts detailed on further below.
The submodules reside in:
Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/):
```
scripts
+--sdk_container
@ -26,10 +24,10 @@ scripts
+------portage-stable
```
When working with the scripts repo always make sure to initialise and to update these submodules; otherwise builds will break because build configuration is missing:
```bash
$ git clone --recurse-submodules https://github.com/flatcar/scripts.git
```
`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
Consider it a small sub-set of Gentoo.
`coreos-overlay` contains significantly modified or even entirely self-written ebuilds.
The `scripts` repository makes ample use of tags to mark releases.
Sometimes, local and origin tags can diverge (e.g. when re-tagging something locally to test a build).
@ -54,7 +52,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready
The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo:
* Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container.
* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay gitmodules into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
## Standalone mode
@ -67,7 +65,7 @@ In standalone mode, the SDK is just another Docker container. Interaction with t
* Start the image in interactive (tty) mode: `docker run -ti ghcr.io/flatcar/flatcar-sdk-all:3033.0.0`
You are now inside the SDK container (the hostname will likely differ):
`sdk@f236fda982a4 ~/trunk/src/scripts $`
* Initialise the SDK in self-contained mode. This needs to be done once per container and will check out the scripts, coreos-overlay, and portage-stable repositories into the container.
* Initialise the SDK in self-contained mode. This needs to be done once per container and will check out the scripts repository into the container.
`sdk@f236fda982a4 ../sdk_init_selfcontained.sh`
You can now work with the SDK container.
@ -84,7 +82,7 @@ To start a container in privileged mode with `/dev` available use:
This is the preferred mode of working with the SDK.
Interaction with the container happens via wrapper scripts from the scripts repository.
Both the host's scripts repo as well as its submodules (portage-stable and coreos-overlay) are made available in the container, allowing for work on these repos directly.
Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly.
The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency.
To clone the scripts repo and pick a version:
@ -92,8 +90,6 @@ To clone the scripts repo and pick a version:
* Optionally, check out a release tag to base your work on
* list releases (e.g. all Alpha releases): `git tag -l alpha-*`
* check out the release version, e.g. `3033.0.0`: `git checkout 3033.0.0`
* Make sure to initialise and fetch git submodules - Flatcar's ebuilds are in 2 separate repositories, connected to `scripts` via submodules.
* `git submodule init; git submodule update`
To use the SDK container:
* Fetch image and start the SDK container: `./run_sdk_container -t`

54
bootstrap_sdk
View File

@ -23,9 +23,9 @@
# built in stage 1. See "stage_repo()" documentation further below for more.
# This stage uses:
# - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo
# or a git ref via --stage1_portage_ref command line option
# or a custom path via --stage1_portage_path command line option
# - coreos-overlay from the SDK's /var/lib/gentoo/repos/coreos-overlay
# or a git ref via --stage1_overlay_ref command line option
# or a custom path via --stage1_overlay_path command line option
# Command line option refs need caution though, since
# stage1 must not contain updated ebuilds (see build_stage1 below).
#
@ -59,10 +59,10 @@ TYPE="flatcar-sdk"
. "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
DEFINE_string stage1_portage_ref "" \
"Custom portage repo git ref to use in stage 1 (USE WITH CAUTION)"
DEFINE_string stage1_overlay_ref "" \
"Custom overlay repo git ref to use in stage 1 (USE WITH CAUTION)"
DEFINE_string stage1_portage_path "" \
"Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
DEFINE_string stage1_overlay_path "" \
"Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)"
## Define the stage4 config template
@ -113,11 +113,11 @@ chmod 1777 "${ROOT_OVERLAY}/tmp"
cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
# Stage 1 uses "known-good" ebuild repos (both coreos-overlay and portage-stable)
# Stage 1 uses "known-good" ebuilds (from both coreos-overlay and portage-stable)
# to build a minimal toolchain (USE="-*") for stage 2.
#
# No package updates must happen in stage 1, so we use the portage-stable and
# coreos-overlay repos included with the current SDK (from the SDK chroot's
# coreos-overlay paths included with the current SDK (from the SDK chroot's
# /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with
# 'cork enter', i.e. the SDK we run ./bootstrap_sdk in.
#
@ -125,21 +125,27 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"
# a minimal stage 2 from known-good ebuild versions - the same ebuild versions
# that were used to build the very SDK we run ./bootstrap_sdk in.
#
# This is needed because stage 1 lacks proper isolation and will link all packages built for
# DANGER ZONE
#
# Stage 1 lacks proper isolation and will link all packages built for
# stage 2 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries
# installed into the FS root of the stage 2 seed ("/tmp/stage1root" in the catalyst chroot).
# This is why we must prevent any updated package ebuilds to "leak" into stage 1, hence we use
# "known good" ebuild repo versions outlined above.
#
# In special circumstances it may be required to circumvent this and use git
# refs of either (or both) portage and overlay. The command line options
# --stage1-portage-ref and --stage1-overlay-ref may be used to specify
# a git ref known to work for stage1. In that case the stage1 seed (i.e. the seed SDK)
# In special circumstances it may be required to circumvent this and use custom paths
# for either (or both) portage and overlay. The command line options
# --stage1-portage-path and --stage1-overlay-path may be used to specify
# a repo path known to work for stage1. In that case the stage1 seed (i.e. the seed SDK)
# will be updated prior to starting to build stage 2.
# NOTE that this should never be used to introduce library updates in stage 1. All binaries
# produced in stage 1 are linked against libraries in the seed tarball, NOT libraries produced
# by stage one. Therefore, these binaries will cease to work in stage 2 when linked against
# outdated "seed tarball" libraries which have been updated to newer versions in stage 1.
stage_repo() {
local repo="$1"
local gitref="$2"
local path="$2"
local dest="$3"
local gitname="$repo"
@ -147,16 +153,14 @@ stage_repo() {
gitname="portage-stable"
fi
if [ -z "$gitref" ]; then
if [ -z "$path" ]; then
cp -R "/var/gentoo/repos/${repo}" "$dest"
info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1."
else
info "Using git ref '$gitref' for ebuild repo '$repo' ('$gitname') in stage 1."
mkdir "$dest/$repo"
( cd "$dest/$repo" \
&& git clone "https://github.com/flatcar/$gitname.git" . \
&& git fetch --all \
&& git checkout "$gitref" )
cp -R "${path}/"* "$dest/${repo}/"
info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1."
info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it."
fi
(
set -euo pipefail
@ -186,9 +190,9 @@ build_stage1() {
mkdir "$stage1_repos"
# prepare ebuild repos for stage 1, either from the local SDK (default)
# or from git refs specified via command line flags
stage_repo "gentoo" "${FLAGS_stage1_portage_ref}" "$stage1_repos"
stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_ref}" "$stage1_repos"
# or from custom paths specified via command line flags
stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos"
stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos"
# Create a snapshot of "known-good" portage-stable repo copy for use in stage 1
# This requires us to create a custom catalyst config to point it to the
@ -209,8 +213,8 @@ build_stage1() {
-e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \
"$TEMPDIR/stage1.spec"
# If we are to use a git ref for either ebuild repo we want to update the stage1 seed SDK
if [ -n "${FLAGS_stage1_portage_ref}" -o -n "${FLAGS_stage1_overlay_ref}" ] ; then
# If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK
if [ -n "${FLAGS_stage1_portage_path}" -o -n "${FLAGS_stage1_overlay_path}" ] ; then
sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec"
echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \
>>"$TEMPDIR/stage1.spec"

5
build_library/build_image_util.sh
View File

@ -400,8 +400,7 @@ get_metadata() {
# SRC_URI is empty for the special github.com/flatcar projects
if [ -z "${val}" ]; then
# The grep invocation gives errors when the ebuild file is not present.
# This can happen if a "scripts" branch does not match the "coreos-overlay" branch
# or when the binary packages from ./build_packages are outdated.
# This can happen when the binary packages from ./build_packages are outdated.
val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)"
if [ -n "${val}" ]; then
val="https://github.com/${val}"
@ -541,8 +540,6 @@ You can read it with "less licenses.json.bz2" or convert it to a text format wit
bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"'
The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz".
Build system files and patches used to build these projects are located at:
https://github.com/flatcar/coreos-overlay/
https://github.com/flatcar/portage-stable/
https://github.com/flatcar/scripts/
Information on how to build Flatcar Container Linux can be found under:
https://www.flatcar.org/docs/latest/reference/developer-guides/sdk-modifying-flatcar/

4
build_library/dev_container_util.sh
View File

@ -48,13 +48,9 @@ main-repo = portage-stable
[coreos]
location = /var/lib/portage/coreos-overlay
sync-type = git
sync-uri = https://github.com/flatcar/coreos-overlay.git
[portage-stable]
location = /var/lib/portage/portage-stable
sync-type = git
sync-uri = https://github.com/flatcar/portage-stable.git
EOF
# Now set the correct profile, we do not use the eselect tool - it

2
ci-automation/README.md
View File

@ -39,7 +39,7 @@ image_build amd64
### SDK bootstrap build
1. SDK Bootstrap (`sdk.sh`): Use a seed SDK tarball and seed SDK container image to build a new SDK tarball.
The resulting SDK tarball will use packages and versions pinned in the coreos-overlay and portage-stable submodules.
The resulting SDK tarball will ship packages and versions from the updated coreos-overlay and portage-stable ebuild directories.
This step updates the versionfile, recording the SDK container version just built.
It will generate and push a new version tag to the scripts repo.
2. SDK container build (`sdk_container.sh`) : use SDK tarball to build an SDK container image.

66
ci-automation/ci_automation_common.sh
View File

@ -12,23 +12,6 @@ source ci-automation/ci-config.env
: ${TEST_WORK_DIR:='__TESTS__'}
function init_submodules() {
git submodule init
git submodule update
}
# --
function update_submodule() {
local submodule="$1"
local commit_ish="$2"
cd "sdk_container/src/third_party/${submodule}"
git fetch --all --tags
git checkout "${commit_ish}"
cd -
}
# --
function check_version_string() {
local version="$1"
@ -39,16 +22,6 @@ function check_version_string() {
}
# --
function update_submodules() {
local coreos_git="$1"
local portage_git="$2"
init_submodules
update_submodule "coreos-overlay" "${coreos_git}"
update_submodule "portage-stable" "${portage_git}"
}
# --
function update_and_push_version() {
local version="$1"
local push_to_branch="${2:-false}"
@ -62,8 +35,6 @@ function update_and_push_version() {
fi
# Add and commit local changes
git add "sdk_container/src/third_party/coreos-overlay"
git add "sdk_container/src/third_party/portage-stable"
git add "sdk_container/.repo/manifests/version.txt"
git commit --allow-empty -m "New version: ${version}"
@ -77,7 +48,7 @@ function update_and_push_version() {
# - the remote tag has changes compared to the local tree (rc: 1)
if [ "$ret" = "0" ]; then
echo "Reusing existing tag" >&2
git checkout -f --recurse-submodules "${version}"
git checkout -f "${version}"
return
elif [ "$ret" = "1" ]; then
echo "Remote tag exists already and is not equal" >&2
@ -476,35 +447,20 @@ function list_files() {
}
# --
# Looks for ../scripts.patch, ../overlay.patch, ../portage.patch and
# applies them to the current repo or the respective sub-module checkout.
# Applies ../scripts.patch to the current repo.
function apply_local_patches() {
local patch_files=(../scripts.patch ../overlay.patch ../portage.patch)
local patch_file
local patch_file="../scripts.patch"
local patch_id
local dirarg
echo "Looking for local patches ${patch_files[*]}"
for patch_file in "${patch_files[@]}"; do
if [ "${patch_file}" = "../scripts.patch" ]; then
dirarg=()
elif [ "${patch_file}" = "../overlay.patch" ]; then
dirarg=("-C" "sdk_container/src/third_party/coreos-overlay/")
elif [ "${patch_file}" = "../portage.patch" ]; then
dirarg=("-C" "sdk_container/src/third_party/portage-stable/")
echo "Looking for local patches ${patch_file}"
patch_id=$(test -e "${patch_file}" && { cat "${patch_file}" | git patch-id | cut -d ' ' -f 1 ; } || true)
if [ "${patch_id}" != "" ]; then
if git "${dirarg[@]}" log --no-merges -p HEAD | git patch-id | cut -d ' ' -f 1 | grep -q "${patch_id}"; then
echo "Skipping already applied ${patch_file}"
else
echo "wrong case: unexpected ${patch_file}"
exit 1
echo "Applying ${patch_file}"
GIT_COMMITTER_NAME="Flatcar Buildbot" GIT_COMMITTER_EMAIL="buildbot@flatcar-linux.org" git am -3 "$PWD/${patch_file}"
fi
patch_id=$(test -e "${patch_file}" && { cat "${patch_file}" | git patch-id | cut -d ' ' -f 1 ; } || true)
if [ "${patch_id}" != "" ]; then
if git "${dirarg[@]}" log --no-merges -p HEAD | git patch-id | cut -d ' ' -f 1 | grep -q "${patch_id}"; then
echo "Skipping already applied ${patch_file}"
else
echo "Applying ${patch_file}"
GIT_COMMITTER_NAME="Flatcar Buildbot" GIT_COMMITTER_EMAIL="buildbot@flatcar-linux.org" git "${dirarg[@]}" am -3 "$PWD/${patch_file}"
fi
fi
done
fi
}
# --

5
ci-automation/image.sh
View File

@ -35,10 +35,6 @@
#
# 3. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
#
# 4. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module.
#
# 5. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module.
#
# OUTPUT:
#
# 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache.
@ -66,7 +62,6 @@ function _image_build_impl() {
channel="$(get_git_channel)"
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"

1
ci-automation/image_changes.sh
View File

@ -47,7 +47,6 @@ function _image_changes_impl() {
channel="$(get_git_channel)"
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"

37
ci-automation/packages-tag.sh
View File

@ -32,29 +32,15 @@
#
# OPTIONAL INPUT:
#
# 2. coreos-overlay repository tag to use (commit-ish).
# Optional - use scripts repo sub-modules as-is if not set.
# This version will be checked out / pulled from remote in the coreos-overlay git submodule.
# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
# 1. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
#
# 3. portage-stable repository tag to use (commit-ish).
# Optional - use scripts repo sub-modules as-is if not set.
# This version will be checked out / pulled from remote in the portage-stable git submodule.
# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
#
# 4. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
#
# 5. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module.
#
# 6. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module.
#
# 7. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build.
# 2. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build.
# See the description in ci-config.env.
#
# OUTPUT:
#
# 1. Updated scripts repository
# - version tag w/ submodules
# - version tag
# - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version
# 2. "./skip-build" as flag file to signal that the build should stop
@ -71,34 +57,21 @@ function packages_tag() {
function _packages_tag_impl() {
local version="$1"
local coreos_git="${2:-}"
local portage_git="${3:-}"
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
check_version_string "${version}"
source sdk_container/.repo/manifests/version.txt
local sdk_version="${FLATCAR_SDK_VERSION}"
if [ -n "${coreos_git}" ] ; then
update_submodule "coreos-overlay" "${coreos_git}"
fi
if [ -n "${portage_git}" ] ; then
update_submodule "portage-stable" "${portage_git}"
fi
# Create new tag in scripts repo w/ updated versionfile + submodules.
# Create new tag in scripts repo w/ updated versionfile
# Also push the changes to the branch ONLY IF we're doing a nightly
# build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch
# (`scripts` and submodules).
local push_branch="false"
if [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \
&& [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
&& [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \
&& [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
&& [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then
push_branch="true"
local existing_tag=""
# Check for the existing tag only when we allow shortcutting

5
ci-automation/packages.sh
View File

@ -38,10 +38,6 @@
#
# 4. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
#
# 5. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module.
#
# 6. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module.
#
# OUTPUT:
#
# 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages
@ -68,7 +64,6 @@ function _packages_build_impl() {
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local sdk_version="${FLATCAR_SDK_VERSION}"

1
ci-automation/push_pkgs.sh
View File

@ -78,7 +78,6 @@ function _push_packages_impl() {
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"

1
ci-automation/release.sh
View File

@ -188,7 +188,6 @@ function _release_build_impl() {
source sdk_lib/sdk_container_common.sh
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
# Needed because we are not the SDK container here

43
ci-automation/sdk_bootstrap.sh
View File

@ -26,41 +26,27 @@
#
# OPTIONAL INPUT:
#
# 3. coreos-overlay repository tag to use (commit-ish).
# This version will be checked out / pulled from remote in the coreos-overlay git submodule.
# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
# Leave empty to use coreos-overlay as-is.
#
# 4. portage-stable repository tag to use (commit-ish).
# This version will be checked out / pulled from remote in the portage-stable git submodule.
# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed.
# Leave empty to use portage-stable as-is.
#
# 5. ARCH. Environment variable. Target architecture for the SDK to run on.
# 3. ARCH. Environment variable. Target architecture for the SDK to run on.
# Either "amd64" or "arm64"; defaults to "amd64" if not set.
#
# 6. SIGNER. Environment variable. Name of the owner of the artifact signing key.
# 4. SIGNER. Environment variable. Name of the owner of the artifact signing key.
# Defaults to nothing if not set - in such case, artifacts will not be signed.
# If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored.
#
# 7. SIGNING_KEY. Environment variable. The artifact signing key.
# 5. SIGNING_KEY. Environment variable. The artifact signing key.
# Defaults to nothing if not set - in such case, artifacts will not be signed.
# If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored.
#
# 8. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
# 6. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
#
# 9. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module.
#
# 10. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module.
#
# 11. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build.
# 7. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build.
# See the description in ci-config.env.
#
# OUTPUT:
#
# 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache.
# 2. Updated scripts repository
# - version tag w/ submodules
# - version tag
# - sdk_container/.repo/manifests/version.txt denotes new SDK version
# 3. "./ci-cleanup.sh" with commands to clean up temporary build resources,
# to be run after this step finishes / when this step is aborted.
@ -81,32 +67,19 @@ function sdk_bootstrap() {
function _sdk_bootstrap_impl() {
local seed_version="$1"
local version="$2"
local coreos_git="${3-}"
local portage_git="${4-}"
: ${ARCH:="amd64"}
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
check_version_string "${version}"
if [ -n "${coreos_git}" ] ; then
update_submodule "coreos-overlay" "${coreos_git}"
fi
if [ -n "${portage_git}" ] ; then
update_submodule "portage-stable" "${portage_git}"
fi
# Create new tag in scripts repo w/ updated versionfile + submodules.
# Create new tag in scripts repo w/ updated versionfile.
# Also push the changes to the branch ONLY IF we're doing a nightly
# build of the 'main' branch AND we're definitely ON the main branch
# (`scripts` and submodules).
local push_branch="false"
if [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \
&& [ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] \
&& [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main" ] \
&& [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main" ] ; then
&& [ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] ; then
push_branch="true"
local existing_tag=""
# Check for the existing tag only when we allow shortcutting

2
ci-automation/sdk_container.sh
View File

@ -56,8 +56,6 @@ function _sdk_container_build_impl() {
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_SDK_VERSION}"
local sdk_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2"

1
ci-automation/test.sh
View File

@ -140,7 +140,6 @@ function _test_run_impl() {
source ci-automation/tapfile_helper_lib.sh
source ci-automation/ci_automation_common.sh
source sdk_lib/sdk_container_common.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"

5
ci-automation/vms.sh
View File

@ -38,10 +38,6 @@
#
# 3. A file ../scripts.patch to apply with "git am -3" for the scripts repo.
#
# 4. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module.
#
# 5. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module.
#
# OUTPUT:
#
# 1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ )
@ -68,7 +64,6 @@ function _vm_build_impl() {
source ci-automation/ci_automation_common.sh
source ci-automation/gpg_setup.sh
init_submodules
source sdk_container/.repo/manifests/version.txt
local vernum="${FLATCAR_VERSION}"

2
retag-for-jenkins
View File

@ -16,8 +16,6 @@ if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then
echo "restart the leaf job after retagging. Note: Just starting a leaf job with your"
echo "branch as reference is not valid because it would overwrite the nightly build"
echo "artifacts!"
echo
echo "TODO: Add feature to update coreos-overlay/portage-stable submodule refs"
exit 1
fi

2
run_sdk_container
View File

@ -124,8 +124,6 @@ if [ -z "$stat" ] ; then
$docker create $tty -i \
-v /dev:/dev \
-v "$(pwd)/sdk_container:/mnt/host/source/" \
-v "$(pwd)/sdk_container/git-override/.git-coreos-overlay:/mnt/host/source/src/third_party/coreos-overlay/.git" \
-v "$(pwd)/sdk_container/git-override/.git-portage-stable:/mnt/host/source/src/third_party/portage-stable/.git" \
-v "$(pwd)/__build__/images:/mnt/host/source/src/build" \
-v "$(pwd):/mnt/host/source/src/scripts" \
$gpg_volumes \

1
sdk_container/git-override/.git-coreos-overlay
View File

@ -1 +0,0 @@
gitdir: ../../scripts/.git/modules/sdk_container/src/third_party/coreos-overlay

1
sdk_container/git-override/.git-portage-stable
View File

@ -1 +0,0 @@
gitdir: ../../scripts/.git/modules/sdk_container/src/third_party/portage-stable

5
sdk_container/git-override/README.md
View File

@ -1,5 +0,0 @@
# GIT overrides for submodules
In the SDK container, the scripts root is at a different relative path to the submodules.
The overrides in this directory are into `coreos-overlay/.git` and `portage-stable/.git` so the submodules can find their parents.

2
sdk_container/src/third_party/coreos-overlay/coreos-base/emerge-gitclone/emerge-gitclone-9999.ebuild vendored
View File

@ -10,7 +10,7 @@ CROS_WORKON_LOCALDIR="src/platform"
if [[ "${PV}" == 9999 ]]; then
KEYWORDS="~amd64 ~arm ~arm64 ~x86"
else
CROS_WORKON_COMMIT="401e84df7b12b0f385879583003a7cc0bd9bbc60" # flatcar-master
CROS_WORKON_COMMIT="9240efb80504da934b2315cc89bddb81b739b214" # flatcar-master
KEYWORDS="amd64 arm arm64 x86"
fi

3
sdk_lib/sdk_init_selfcontained.sh
View File

@ -31,6 +31,3 @@ version="$(source /mnt/host/source/.repo/manifests/version.txt; echo $FLATCAR_VE
mkdir -p /home/sdk/trunk/src/third_party/
clone_version scripts /home/sdk/trunk/src/scripts "$version"
clone_version portage-stable /home/sdk/trunk/src/third_party/portage-stable "$version"
clone_version coreos-overlay /home/sdk/trunk/src/third_party/coreos-overlay "$version"