From 401af830d11162fdbab19311655e3b6d435fd2d3 Mon Sep 17 00:00:00 2001 From: Thilo Fromm Date: Thu, 13 Apr 2023 12:26:36 +0200 Subject: [PATCH] scripts, CI, workflows: remove submodule handling (main) --- .github/workflows/ci.yaml | 61 +---------------- .github/workflows/run-kola-tests.yaml | 1 - README.md | 22 +++---- bootstrap_sdk | 54 ++++++++------- build_library/build_image_util.sh | 5 +- build_library/dev_container_util.sh | 4 -- ci-automation/README.md | 2 +- ci-automation/ci_automation_common.sh | 66 ++++--------------- ci-automation/image.sh | 5 -- ci-automation/image_changes.sh | 1 - ci-automation/packages-tag.sh | 37 ++--------- ci-automation/packages.sh | 5 -- ci-automation/push_pkgs.sh | 1 - ci-automation/release.sh | 1 - ci-automation/sdk_bootstrap.sh | 43 +++--------- ci-automation/sdk_container.sh | 2 - ci-automation/test.sh | 1 - ci-automation/vms.sh | 5 -- retag-for-jenkins | 2 - run_sdk_container | 2 - .../git-override/.git-coreos-overlay | 1 - .../git-override/.git-portage-stable | 1 - sdk_container/git-override/README.md | 5 -- .../emerge-gitclone-9999.ebuild | 2 +- sdk_lib/sdk_init_selfcontained.sh | 3 - 25 files changed, 68 insertions(+), 264 deletions(-) delete mode 100644 sdk_container/git-override/.git-coreos-overlay delete mode 100644 sdk_container/git-override/.git-portage-stable delete mode 100644 sdk_container/git-override/README.md diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2892c66970..349c1bb06e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -11,24 +11,6 @@ on: Space-separated vendor formats to build. required: true default: qemu_uefi - portage_remote: - description: | - The remote we should pull portage-stable from. This defaults to whatever the submodule is set to in this repo. - If triggered by a change in the portage repo, please set this to the remote which is proposing a change. - required: false - portage_ref: - description: | - This is the ref we will use to pull the changes from the portage_remote. - required: false - coreos_remote: - description: | - The remote we should pull coreos-overlay from. This defaults to whatever the submodule is set to in this repo. - If triggered by a change in the portage repo, please set this to the remote which is proposing a change. - required: false - coreos_ref: - description: | - This is the ref we will use to pull the changes from the coreos_remote. - required: false concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} @@ -75,7 +57,6 @@ jobs: with: path: scripts fetch-depth: 0 - submodules: true # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR # instead of its head commit. That commit is not connected to any branch. @@ -96,23 +77,10 @@ jobs: shell: bash run: | arch="${{ matrix.arch }}" - COREOS_REMOTE="" - COREOS_REF="" - PORTAGE_REMOTE="" - PORTAGE_REF="" - IMAGE_FORMATS="qemu_uefi" - - [ -z "${{ github.event.inputs.coreos_remote }}" ] || COREOS_REMOTE="${{ github.event.inputs.coreos_remote }}" - [ -z "${{ github.event.inputs.coreos_ref }}" ] || COREOS_REF="${{ github.event.inputs.coreos_ref }}" - [ -z "${{ github.event.inputs.portage_remote }}" ] || PORTAGE_REMOTE="${{ github.event.inputs.portage_remote }}" - [ -z "${{ github.event.inputs.portage_ref }}" ] || PORTAGE_REF="${{ github.event.inputs.portage_ref }}" - [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" - echo "arch=${arch}" >> $GITHUB_ENV - echo "COREOS_REMOTE=${COREOS_REMOTE}" >> $GITHUB_ENV - echo "COREOS_REF=${COREOS_REF}" >> $GITHUB_ENV - echo "PORTAGE_REMOTE=${PORTAGE_REMOTE}" >> $GITHUB_ENV - echo "PORTAGE_REF=${PORTAGE_REF}" >> $GITHUB_ENV + + IMAGE_FORMATS="qemu_uefi" + [ -z "${{ github.event.inputs.image_formats }}" ] || IMAGE_FORMATS="${{ github.event.inputs.image_formats }}" echo "IMAGE_FORMATS=${IMAGE_FORMATS}" >> $GITHUB_ENV # Artifact root for images and torcx tarball as seen from within the container @@ -124,29 +92,6 @@ jobs: # this with its IP address. echo "TORCX_TESTS_PACKAGE_URL=http://localhost:12345" >> $GITHUB_ENV - - name: Checkout submodules - shell: bash - run: | - if [ "${COREOS_REMOTE}" != "" -a "${COREOS_REF}" != "" ] - then - REMOTE="${COREOS_REMOTE}" - REPO_PATH="sdk_container/src/third_party/coreos-overlay" - [[ "$REMOTE" == "https:*" ]] || REMOTE="https://github.com/${COREOS_REMOTE}" - git -C "$REPO_PATH" remote add test "$REMOTE" - git -C "$REPO_PATH" fetch test - git -C "$REPO_PATH" checkout "test/${COREOS_REF}" - fi - - if [ "${PORTAGE_REMOTE}" != "" -a "${PORTAGE_REF}" != "" ] - then - REMOTE="${PORTAGE_REMOTE}" - REPO_PATH="sdk_container/src/third_party/portage-stable" - [[ "$REMOTE" == "https:*" ]] || REMOTE="https://github.com/${PORTAGE_REMOTE}" - git -C "$REPO_PATH" remote add test "$REMOTE" - git -C "$REPO_PATH" fetch test - git -C "$REPO_PATH" checkout "test/${PORTAGE_REF}" - fi - - name: Build packages shell: bash run: | diff --git a/.github/workflows/run-kola-tests.yaml b/.github/workflows/run-kola-tests.yaml index 82332b50c0..53846d852a 100644 --- a/.github/workflows/run-kola-tests.yaml +++ b/.github/workflows/run-kola-tests.yaml @@ -58,7 +58,6 @@ jobs: with: path: scripts fetch-depth: 0 - submodules: true # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR # instead of its head commit. That commit is not connected to any branch. diff --git a/README.md b/README.md index 8d2c28140a..262180cec5 100644 --- a/README.md +++ b/README.md @@ -11,12 +11,10 @@ The SDK can be used to [flatcar-docs]: https://www.flatcar.org/docs/latest/reference/developer-guides/sdk-modifying-flatcar/ -# Using the scripts repository: submodules and tags +# Using the scripts repository The repository is meant to be the entry point for Flatcar builds and development. -For building packages, there are 2 additional repositories, [coreos-overlay](https://github.com/flatcar/coreos-overlay) and [portage-stable](https://github.com/flatcar/portage-stable), which contain all packages' `ebuild` (build configuration) files. -These repositories are included in `scripts` via git submodules and are used by the SDK container wrapper scripts detailed on further below. -The submodules reside in: +Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/): ``` scripts +--sdk_container @@ -26,10 +24,10 @@ scripts +------portage-stable ``` -When working with the scripts repo always make sure to initialise and to update these submodules; otherwise builds will break because build configuration is missing: -```bash -$ git clone --recurse-submodules https://github.com/flatcar/scripts.git -``` +`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions). +Consider it a small sub-set of Gentoo. + +`coreos-overlay` contains significantly modified or even entirely self-written ebuilds. The `scripts` repository makes ample use of tags to mark releases. Sometimes, local and origin tags can diverge (e.g. when re-tagging something locally to test a build). @@ -54,7 +52,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo: * Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container. -* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay gitmodules into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar. +* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar. ## Standalone mode @@ -67,7 +65,7 @@ In standalone mode, the SDK is just another Docker container. Interaction with t * Start the image in interactive (tty) mode: `docker run -ti ghcr.io/flatcar/flatcar-sdk-all:3033.0.0` You are now inside the SDK container (the hostname will likely differ): `sdk@f236fda982a4 ~/trunk/src/scripts $` -* Initialise the SDK in self-contained mode. This needs to be done once per container and will check out the scripts, coreos-overlay, and portage-stable repositories into the container. +* Initialise the SDK in self-contained mode. This needs to be done once per container and will check out the scripts repository into the container. `sdk@f236fda982a4 ../sdk_init_selfcontained.sh` You can now work with the SDK container. @@ -84,7 +82,7 @@ To start a container in privileged mode with `/dev` available use: This is the preferred mode of working with the SDK. Interaction with the container happens via wrapper scripts from the scripts repository. -Both the host's scripts repo as well as its submodules (portage-stable and coreos-overlay) are made available in the container, allowing for work on these repos directly. +Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly. The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency. To clone the scripts repo and pick a version: @@ -92,8 +90,6 @@ To clone the scripts repo and pick a version: * Optionally, check out a release tag to base your work on * list releases (e.g. all Alpha releases): `git tag -l alpha-*` * check out the release version, e.g. `3033.0.0`: `git checkout 3033.0.0` -* Make sure to initialise and fetch git submodules - Flatcar's ebuilds are in 2 separate repositories, connected to `scripts` via submodules. - * `git submodule init; git submodule update` To use the SDK container: * Fetch image and start the SDK container: `./run_sdk_container -t` diff --git a/bootstrap_sdk b/bootstrap_sdk index d1e7a3d369..54ab9e8979 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -23,9 +23,9 @@ # built in stage 1. See "stage_repo()" documentation further below for more. # This stage uses: # - portage-stable from the SDK's /var/lib/gentoo/repos/gentoo -# or a git ref via --stage1_portage_ref command line option +# or a custom path via --stage1_portage_path command line option # - coreos-overlay from the SDK's /var/lib/gentoo/repos/coreos-overlay -# or a git ref via --stage1_overlay_ref command line option +# or a custom path via --stage1_overlay_path command line option # Command line option refs need caution though, since # stage1 must not contain updated ebuilds (see build_stage1 below). # @@ -59,10 +59,10 @@ TYPE="flatcar-sdk" . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 -DEFINE_string stage1_portage_ref "" \ - "Custom portage repo git ref to use in stage 1 (USE WITH CAUTION)" -DEFINE_string stage1_overlay_ref "" \ - "Custom overlay repo git ref to use in stage 1 (USE WITH CAUTION)" +DEFINE_string stage1_portage_path "" \ + "Path to custom portage ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)" +DEFINE_string stage1_overlay_path "" \ + "Path to custom overlay ebuilds tree to use in stage 1 (DANGEROUS; USE WITH CAUTION)" ## Define the stage4 config template @@ -113,11 +113,11 @@ chmod 1777 "${ROOT_OVERLAY}/tmp" cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" -# Stage 1 uses "known-good" ebuild repos (both coreos-overlay and portage-stable) +# Stage 1 uses "known-good" ebuilds (from both coreos-overlay and portage-stable) # to build a minimal toolchain (USE="-*") for stage 2. # # No package updates must happen in stage 1, so we use the portage-stable and -# coreos-overlay repos included with the current SDK (from the SDK chroot's +# coreos-overlay paths included with the current SDK (from the SDK chroot's # /var/lib/gentoo/repos/). "Current SDK" refers to the SDK we entered with # 'cork enter', i.e. the SDK we run ./bootstrap_sdk in. # @@ -125,21 +125,27 @@ cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp" # a minimal stage 2 from known-good ebuild versions - the same ebuild versions # that were used to build the very SDK we run ./bootstrap_sdk in. # -# This is needed because stage 1 lacks proper isolation and will link all packages built for +# DANGER ZONE +# +# Stage 1 lacks proper isolation and will link all packages built for # stage 2 against its own seed libraries ("/" in the catalyst chroot) instead of against libraries # installed into the FS root of the stage 2 seed ("/tmp/stage1root" in the catalyst chroot). # This is why we must prevent any updated package ebuilds to "leak" into stage 1, hence we use # "known good" ebuild repo versions outlined above. # -# In special circumstances it may be required to circumvent this and use git -# refs of either (or both) portage and overlay. The command line options -# --stage1-portage-ref and --stage1-overlay-ref may be used to specify -# a git ref known to work for stage1. In that case the stage1 seed (i.e. the seed SDK) +# In special circumstances it may be required to circumvent this and use custom paths +# for either (or both) portage and overlay. The command line options +# --stage1-portage-path and --stage1-overlay-path may be used to specify +# a repo path known to work for stage1. In that case the stage1 seed (i.e. the seed SDK) # will be updated prior to starting to build stage 2. +# NOTE that this should never be used to introduce library updates in stage 1. All binaries +# produced in stage 1 are linked against libraries in the seed tarball, NOT libraries produced +# by stage one. Therefore, these binaries will cease to work in stage 2 when linked against +# outdated "seed tarball" libraries which have been updated to newer versions in stage 1. stage_repo() { local repo="$1" - local gitref="$2" + local path="$2" local dest="$3" local gitname="$repo" @@ -147,16 +153,14 @@ stage_repo() { gitname="portage-stable" fi - if [ -z "$gitref" ]; then + if [ -z "$path" ]; then cp -R "/var/gentoo/repos/${repo}" "$dest" info "Using local SDK's ebuild repo '$repo' ('$gitname') in stage 1." else - info "Using git ref '$gitref' for ebuild repo '$repo' ('$gitname') in stage 1." mkdir "$dest/$repo" - ( cd "$dest/$repo" \ - && git clone "https://github.com/flatcar/$gitname.git" . \ - && git fetch --all \ - && git checkout "$gitref" ) + cp -R "${path}/"* "$dest/${repo}/" + info "Using custom path '$path' for ebuild repo '$repo' ('$gitname') in stage 1." + info "This may break stage 2. YOU HAVE BEEN WARNED. You break it, you keep it." fi ( set -euo pipefail @@ -186,9 +190,9 @@ build_stage1() { mkdir "$stage1_repos" # prepare ebuild repos for stage 1, either from the local SDK (default) - # or from git refs specified via command line flags - stage_repo "gentoo" "${FLAGS_stage1_portage_ref}" "$stage1_repos" - stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_ref}" "$stage1_repos" + # or from custom paths specified via command line flags + stage_repo "gentoo" "${FLAGS_stage1_portage_path}" "$stage1_repos" + stage_repo "coreos-overlay" "${FLAGS_stage1_overlay_path}" "$stage1_repos" # Create a snapshot of "known-good" portage-stable repo copy for use in stage 1 # This requires us to create a custom catalyst config to point it to the @@ -209,8 +213,8 @@ build_stage1() { -e "s,^portage_overlay:.*,portage_overlay: $stage1_repos/coreos-overlay," \ "$TEMPDIR/stage1.spec" - # If we are to use a git ref for either ebuild repo we want to update the stage1 seed SDK - if [ -n "${FLAGS_stage1_portage_ref}" -o -n "${FLAGS_stage1_overlay_ref}" ] ; then + # If we are to use a custom path for either ebuild repo we want to update the stage1 seed SDK + if [ -n "${FLAGS_stage1_portage_path}" -o -n "${FLAGS_stage1_overlay_path}" ] ; then sed -i 's/^update_seed: no/update_seed: yes/' "$TEMPDIR/stage1.spec" echo "update_seed_command: --update --deep --newuse --complete-graph --rebuild-if-new-ver --rebuild-exclude cross-*-cros-linux-gnu/* sys-devel/gcc " \ >>"$TEMPDIR/stage1.spec" diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index b717d80ee9..d0ed76f318 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -400,8 +400,7 @@ get_metadata() { # SRC_URI is empty for the special github.com/flatcar projects if [ -z "${val}" ]; then # The grep invocation gives errors when the ebuild file is not present. - # This can happen if a "scripts" branch does not match the "coreos-overlay" branch - # or when the binary packages from ./build_packages are outdated. + # This can happen when the binary packages from ./build_packages are outdated. val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)" if [ -n "${val}" ]; then val="https://github.com/${val}" @@ -541,8 +540,6 @@ You can read it with "less licenses.json.bz2" or convert it to a text format wit bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"' The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz". Build system files and patches used to build these projects are located at: -https://github.com/flatcar/coreos-overlay/ -https://github.com/flatcar/portage-stable/ https://github.com/flatcar/scripts/ Information on how to build Flatcar Container Linux can be found under: https://www.flatcar.org/docs/latest/reference/developer-guides/sdk-modifying-flatcar/ diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index a9e834bb3a..a0c07477cd 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -48,13 +48,9 @@ main-repo = portage-stable [coreos] location = /var/lib/portage/coreos-overlay -sync-type = git -sync-uri = https://github.com/flatcar/coreos-overlay.git [portage-stable] location = /var/lib/portage/portage-stable -sync-type = git -sync-uri = https://github.com/flatcar/portage-stable.git EOF # Now set the correct profile, we do not use the eselect tool - it diff --git a/ci-automation/README.md b/ci-automation/README.md index 9e84e35ef1..9899580c6a 100644 --- a/ci-automation/README.md +++ b/ci-automation/README.md @@ -39,7 +39,7 @@ image_build amd64 ### SDK bootstrap build 1. SDK Bootstrap (`sdk.sh`): Use a seed SDK tarball and seed SDK container image to build a new SDK tarball. - The resulting SDK tarball will use packages and versions pinned in the coreos-overlay and portage-stable submodules. + The resulting SDK tarball will ship packages and versions from the updated coreos-overlay and portage-stable ebuild directories. This step updates the versionfile, recording the SDK container version just built. It will generate and push a new version tag to the scripts repo. 2. SDK container build (`sdk_container.sh`) : use SDK tarball to build an SDK container image. diff --git a/ci-automation/ci_automation_common.sh b/ci-automation/ci_automation_common.sh index 355a52ce27..e6bfa05edb 100644 --- a/ci-automation/ci_automation_common.sh +++ b/ci-automation/ci_automation_common.sh @@ -12,23 +12,6 @@ source ci-automation/ci-config.env : ${TEST_WORK_DIR:='__TESTS__'} -function init_submodules() { - git submodule init - git submodule update -} -# -- - -function update_submodule() { - local submodule="$1" - local commit_ish="$2" - - cd "sdk_container/src/third_party/${submodule}" - git fetch --all --tags - git checkout "${commit_ish}" - cd - -} -# -- - function check_version_string() { local version="$1" @@ -39,16 +22,6 @@ function check_version_string() { } # -- -function update_submodules() { - local coreos_git="$1" - local portage_git="$2" - - init_submodules - update_submodule "coreos-overlay" "${coreos_git}" - update_submodule "portage-stable" "${portage_git}" -} -# -- - function update_and_push_version() { local version="$1" local push_to_branch="${2:-false}" @@ -62,8 +35,6 @@ function update_and_push_version() { fi # Add and commit local changes - git add "sdk_container/src/third_party/coreos-overlay" - git add "sdk_container/src/third_party/portage-stable" git add "sdk_container/.repo/manifests/version.txt" git commit --allow-empty -m "New version: ${version}" @@ -77,7 +48,7 @@ function update_and_push_version() { # - the remote tag has changes compared to the local tree (rc: 1) if [ "$ret" = "0" ]; then echo "Reusing existing tag" >&2 - git checkout -f --recurse-submodules "${version}" + git checkout -f "${version}" return elif [ "$ret" = "1" ]; then echo "Remote tag exists already and is not equal" >&2 @@ -476,35 +447,20 @@ function list_files() { } # -- -# Looks for ../scripts.patch, ../overlay.patch, ../portage.patch and -# applies them to the current repo or the respective sub-module checkout. +# Applies ../scripts.patch to the current repo. function apply_local_patches() { - local patch_files=(../scripts.patch ../overlay.patch ../portage.patch) - local patch_file + local patch_file="../scripts.patch" local patch_id - local dirarg - echo "Looking for local patches ${patch_files[*]}" - for patch_file in "${patch_files[@]}"; do - if [ "${patch_file}" = "../scripts.patch" ]; then - dirarg=() - elif [ "${patch_file}" = "../overlay.patch" ]; then - dirarg=("-C" "sdk_container/src/third_party/coreos-overlay/") - elif [ "${patch_file}" = "../portage.patch" ]; then - dirarg=("-C" "sdk_container/src/third_party/portage-stable/") + echo "Looking for local patches ${patch_file}" + patch_id=$(test -e "${patch_file}" && { cat "${patch_file}" | git patch-id | cut -d ' ' -f 1 ; } || true) + if [ "${patch_id}" != "" ]; then + if git "${dirarg[@]}" log --no-merges -p HEAD | git patch-id | cut -d ' ' -f 1 | grep -q "${patch_id}"; then + echo "Skipping already applied ${patch_file}" else - echo "wrong case: unexpected ${patch_file}" - exit 1 + echo "Applying ${patch_file}" + GIT_COMMITTER_NAME="Flatcar Buildbot" GIT_COMMITTER_EMAIL="buildbot@flatcar-linux.org" git am -3 "$PWD/${patch_file}" fi - patch_id=$(test -e "${patch_file}" && { cat "${patch_file}" | git patch-id | cut -d ' ' -f 1 ; } || true) - if [ "${patch_id}" != "" ]; then - if git "${dirarg[@]}" log --no-merges -p HEAD | git patch-id | cut -d ' ' -f 1 | grep -q "${patch_id}"; then - echo "Skipping already applied ${patch_file}" - else - echo "Applying ${patch_file}" - GIT_COMMITTER_NAME="Flatcar Buildbot" GIT_COMMITTER_EMAIL="buildbot@flatcar-linux.org" git "${dirarg[@]}" am -3 "$PWD/${patch_file}" - fi - fi - done + fi } # -- diff --git a/ci-automation/image.sh b/ci-automation/image.sh index 12328e5301..f45b78eaac 100644 --- a/ci-automation/image.sh +++ b/ci-automation/image.sh @@ -35,10 +35,6 @@ # # 3. A file ../scripts.patch to apply with "git am -3" for the scripts repo. # -# 4. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. -# -# 5. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. -# # OUTPUT: # # 1. OS image, dev container, related artifacts, and torcx packages pushed to buildcache. @@ -66,7 +62,6 @@ function _image_build_impl() { channel="$(get_git_channel)" source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_VERSION}" diff --git a/ci-automation/image_changes.sh b/ci-automation/image_changes.sh index 7712e73ce3..85a1eb3a00 100644 --- a/ci-automation/image_changes.sh +++ b/ci-automation/image_changes.sh @@ -47,7 +47,6 @@ function _image_changes_impl() { channel="$(get_git_channel)" source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_VERSION}" diff --git a/ci-automation/packages-tag.sh b/ci-automation/packages-tag.sh index 1aedb530e5..2faa0dc8f8 100644 --- a/ci-automation/packages-tag.sh +++ b/ci-automation/packages-tag.sh @@ -32,29 +32,15 @@ # # OPTIONAL INPUT: # -# 2. coreos-overlay repository tag to use (commit-ish). -# Optional - use scripts repo sub-modules as-is if not set. -# This version will be checked out / pulled from remote in the coreos-overlay git submodule. -# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. +# 1. A file ../scripts.patch to apply with "git am -3" for the scripts repo. # -# 3. portage-stable repository tag to use (commit-ish). -# Optional - use scripts repo sub-modules as-is if not set. -# This version will be checked out / pulled from remote in the portage-stable git submodule. -# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. -# -# 4. A file ../scripts.patch to apply with "git am -3" for the scripts repo. -# -# 5. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. -# -# 6. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. -# -# 7. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build. +# 2. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build. # See the description in ci-config.env. # # OUTPUT: # # 1. Updated scripts repository -# - version tag w/ submodules +# - version tag # - sdk_container/.repo/manifests/version.txt denotes new FLATCAR OS version # 2. "./skip-build" as flag file to signal that the build should stop @@ -71,34 +57,21 @@ function packages_tag() { function _packages_tag_impl() { local version="$1" - local coreos_git="${2:-}" - local portage_git="${3:-}" source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules check_version_string "${version}" source sdk_container/.repo/manifests/version.txt local sdk_version="${FLATCAR_SDK_VERSION}" - if [ -n "${coreos_git}" ] ; then - update_submodule "coreos-overlay" "${coreos_git}" - fi - if [ -n "${portage_git}" ] ; then - update_submodule "portage-stable" "${portage_git}" - fi - - # Create new tag in scripts repo w/ updated versionfile + submodules. + # Create new tag in scripts repo w/ updated versionfile # Also push the changes to the branch ONLY IF we're doing a nightly # build of the 'main'/'flatcar-MAJOR' branch AND we're definitely ON the respective branch - # (`scripts` and submodules). local push_branch="false" if [[ "${version}" =~ ^(stable|alpha|beta|lts)-[0-9.]+-nightly-[-0-9]+$ ]] \ - && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \ - && [[ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] \ - && [[ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then + && [[ "$(git rev-parse --abbrev-ref HEAD)" =~ ^flatcar-[0-9]+$ ]] ; then push_branch="true" local existing_tag="" # Check for the existing tag only when we allow shortcutting diff --git a/ci-automation/packages.sh b/ci-automation/packages.sh index 871475b7e1..add6fc3c61 100644 --- a/ci-automation/packages.sh +++ b/ci-automation/packages.sh @@ -38,10 +38,6 @@ # # 4. A file ../scripts.patch to apply with "git am -3" for the scripts repo. # -# 5. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. -# -# 6. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. -# # OUTPUT: # # 1. Exported container image "flatcar-packages-[ARCH]-[VERSION].tar.gz" with binary packages @@ -68,7 +64,6 @@ function _packages_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules source sdk_container/.repo/manifests/version.txt local sdk_version="${FLATCAR_SDK_VERSION}" diff --git a/ci-automation/push_pkgs.sh b/ci-automation/push_pkgs.sh index 4b0c153148..86439c599b 100644 --- a/ci-automation/push_pkgs.sh +++ b/ci-automation/push_pkgs.sh @@ -78,7 +78,6 @@ function _push_packages_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_VERSION}" diff --git a/ci-automation/release.sh b/ci-automation/release.sh index f0e8efc27b..930fd9d904 100644 --- a/ci-automation/release.sh +++ b/ci-automation/release.sh @@ -188,7 +188,6 @@ function _release_build_impl() { source sdk_lib/sdk_container_common.sh source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules source sdk_container/.repo/manifests/version.txt # Needed because we are not the SDK container here diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 38199f24a0..bb18f94eb9 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -26,41 +26,27 @@ # # OPTIONAL INPUT: # -# 3. coreos-overlay repository tag to use (commit-ish). -# This version will be checked out / pulled from remote in the coreos-overlay git submodule. -# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. -# Leave empty to use coreos-overlay as-is. -# -# 4. portage-stable repository tag to use (commit-ish). -# This version will be checked out / pulled from remote in the portage-stable git submodule. -# The submodule config will be updated to point to this version before the TARGET SDK tag is created and pushed. -# Leave empty to use portage-stable as-is. -# -# 5. ARCH. Environment variable. Target architecture for the SDK to run on. +# 3. ARCH. Environment variable. Target architecture for the SDK to run on. # Either "amd64" or "arm64"; defaults to "amd64" if not set. # -# 6. SIGNER. Environment variable. Name of the owner of the artifact signing key. +# 4. SIGNER. Environment variable. Name of the owner of the artifact signing key. # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNING_KEY environment variable should also be provided, otherwise this environment variable will be ignored. # -# 7. SIGNING_KEY. Environment variable. The artifact signing key. +# 5. SIGNING_KEY. Environment variable. The artifact signing key. # Defaults to nothing if not set - in such case, artifacts will not be signed. # If provided, SIGNER environment variable should also be provided, otherwise this environment variable will be ignored. # -# 8. A file ../scripts.patch to apply with "git am -3" for the scripts repo. +# 6. A file ../scripts.patch to apply with "git am -3" for the scripts repo. # -# 9. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. -# -# 10. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. -# -# 11. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build. +# 7. AVOID_NIGHTLY_BUILD_SHORTCUTS. Environment variable. Tells the script to build the SDK even if nothing has changed since last nightly build. # See the description in ci-config.env. # # OUTPUT: # # 1. SDK tarball (gentoo catalyst output) of the new SDK, pushed to buildcache. # 2. Updated scripts repository -# - version tag w/ submodules +# - version tag # - sdk_container/.repo/manifests/version.txt denotes new SDK version # 3. "./ci-cleanup.sh" with commands to clean up temporary build resources, # to be run after this step finishes / when this step is aborted. @@ -81,32 +67,19 @@ function sdk_bootstrap() { function _sdk_bootstrap_impl() { local seed_version="$1" local version="$2" - local coreos_git="${3-}" - local portage_git="${4-}" : ${ARCH:="amd64"} source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules check_version_string "${version}" - if [ -n "${coreos_git}" ] ; then - update_submodule "coreos-overlay" "${coreos_git}" - fi - if [ -n "${portage_git}" ] ; then - update_submodule "portage-stable" "${portage_git}" - fi - - # Create new tag in scripts repo w/ updated versionfile + submodules. + # Create new tag in scripts repo w/ updated versionfile. # Also push the changes to the branch ONLY IF we're doing a nightly # build of the 'main' branch AND we're definitely ON the main branch - # (`scripts` and submodules). local push_branch="false" if [[ "${version}" =~ ^main-[0-9.]+-nightly-[-0-9]+$ ]] \ - && [ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] \ - && [ "$(git -C sdk_container/src/third_party/coreos-overlay/ rev-parse --abbrev-ref HEAD)" = "main" ] \ - && [ "$(git -C sdk_container/src/third_party/portage-stable/ rev-parse --abbrev-ref HEAD)" = "main" ] ; then + && [ "$(git rev-parse --abbrev-ref HEAD)" = "main" ] ; then push_branch="true" local existing_tag="" # Check for the existing tag only when we allow shortcutting diff --git a/ci-automation/sdk_container.sh b/ci-automation/sdk_container.sh index fa77faa356..e387558191 100644 --- a/ci-automation/sdk_container.sh +++ b/ci-automation/sdk_container.sh @@ -56,8 +56,6 @@ function _sdk_container_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules - source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_SDK_VERSION}" local sdk_tarball="flatcar-sdk-${ARCH}-${vernum}.tar.bz2" diff --git a/ci-automation/test.sh b/ci-automation/test.sh index 2082aa83dd..9010183cee 100644 --- a/ci-automation/test.sh +++ b/ci-automation/test.sh @@ -140,7 +140,6 @@ function _test_run_impl() { source ci-automation/tapfile_helper_lib.sh source ci-automation/ci_automation_common.sh source sdk_lib/sdk_container_common.sh - init_submodules source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_VERSION}" diff --git a/ci-automation/vms.sh b/ci-automation/vms.sh index 6969c7318a..8dcd831e9a 100644 --- a/ci-automation/vms.sh +++ b/ci-automation/vms.sh @@ -38,10 +38,6 @@ # # 3. A file ../scripts.patch to apply with "git am -3" for the scripts repo. # -# 4. A file ../overlay.patch to apply with "git am -3" for the coreos-overlay sub-module. -# -# 5. A file ../portage.patch to apply with "git am -3" for the portage-stable sub-module. -# # OUTPUT: # # 1. Exported VM image(s), pushed to buildcache ( images/[ARCH]/[FLATCAR_VERSION]/ ) @@ -68,7 +64,6 @@ function _vm_build_impl() { source ci-automation/ci_automation_common.sh source ci-automation/gpg_setup.sh - init_submodules source sdk_container/.repo/manifests/version.txt local vernum="${FLATCAR_VERSION}" diff --git a/retag-for-jenkins b/retag-for-jenkins index 469efbc131..52f97371f6 100755 --- a/retag-for-jenkins +++ b/retag-for-jenkins @@ -16,8 +16,6 @@ if [ $# -lt 1 ] || [ "$1" = "-h" ] || [ "$1" = "--help" ]; then echo "restart the leaf job after retagging. Note: Just starting a leaf job with your" echo "branch as reference is not valid because it would overwrite the nightly build" echo "artifacts!" - echo - echo "TODO: Add feature to update coreos-overlay/portage-stable submodule refs" exit 1 fi diff --git a/run_sdk_container b/run_sdk_container index be445cff13..b34b7958c1 100755 --- a/run_sdk_container +++ b/run_sdk_container @@ -124,8 +124,6 @@ if [ -z "$stat" ] ; then $docker create $tty -i \ -v /dev:/dev \ -v "$(pwd)/sdk_container:/mnt/host/source/" \ - -v "$(pwd)/sdk_container/git-override/.git-coreos-overlay:/mnt/host/source/src/third_party/coreos-overlay/.git" \ - -v "$(pwd)/sdk_container/git-override/.git-portage-stable:/mnt/host/source/src/third_party/portage-stable/.git" \ -v "$(pwd)/__build__/images:/mnt/host/source/src/build" \ -v "$(pwd):/mnt/host/source/src/scripts" \ $gpg_volumes \ diff --git a/sdk_container/git-override/.git-coreos-overlay b/sdk_container/git-override/.git-coreos-overlay deleted file mode 100644 index 507e22fe6e..0000000000 --- a/sdk_container/git-override/.git-coreos-overlay +++ /dev/null @@ -1 +0,0 @@ -gitdir: ../../scripts/.git/modules/sdk_container/src/third_party/coreos-overlay diff --git a/sdk_container/git-override/.git-portage-stable b/sdk_container/git-override/.git-portage-stable deleted file mode 100644 index b3af445704..0000000000 --- a/sdk_container/git-override/.git-portage-stable +++ /dev/null @@ -1 +0,0 @@ -gitdir: ../../scripts/.git/modules/sdk_container/src/third_party/portage-stable diff --git a/sdk_container/git-override/README.md b/sdk_container/git-override/README.md deleted file mode 100644 index fca5ac22ad..0000000000 --- a/sdk_container/git-override/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# GIT overrides for submodules - -In the SDK container, the scripts root is at a different relative path to the submodules. -The overrides in this directory are into `coreos-overlay/.git` and `portage-stable/.git` so the submodules can find their parents. - diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/emerge-gitclone/emerge-gitclone-9999.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/emerge-gitclone/emerge-gitclone-9999.ebuild index 88b22bb742..25fe078a25 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/emerge-gitclone/emerge-gitclone-9999.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/emerge-gitclone/emerge-gitclone-9999.ebuild @@ -10,7 +10,7 @@ CROS_WORKON_LOCALDIR="src/platform" if [[ "${PV}" == 9999 ]]; then KEYWORDS="~amd64 ~arm ~arm64 ~x86" else - CROS_WORKON_COMMIT="401e84df7b12b0f385879583003a7cc0bd9bbc60" # flatcar-master + CROS_WORKON_COMMIT="9240efb80504da934b2315cc89bddb81b739b214" # flatcar-master KEYWORDS="amd64 arm arm64 x86" fi diff --git a/sdk_lib/sdk_init_selfcontained.sh b/sdk_lib/sdk_init_selfcontained.sh index 3ddda3c5e6..76b08b3c21 100755 --- a/sdk_lib/sdk_init_selfcontained.sh +++ b/sdk_lib/sdk_init_selfcontained.sh @@ -31,6 +31,3 @@ version="$(source /mnt/host/source/.repo/manifests/version.txt; echo $FLATCAR_VE mkdir -p /home/sdk/trunk/src/third_party/ clone_version scripts /home/sdk/trunk/src/scripts "$version" -clone_version portage-stable /home/sdk/trunk/src/third_party/portage-stable "$version" -clone_version coreos-overlay /home/sdk/trunk/src/third_party/coreos-overlay "$version" -