*: changes for new pipeline picked from the main branch

.gitignore

@@ -8,3 +8,15 @@
 *.pyc
 *~
 /cbuildbot_package.list
+
+# Flatcar SDK tarballs
+*.tar.bz2
+
+# SDK container env passing helpers
+sdk_container/.env
+sdk_container/.sdkenv
+
+# build cache / artefacts directories
+__build__/
+sdk_container/.cache
+sdk_container/.config

bootstrap_sdk

@@ -40,7 +40,6 @@ TYPE="flatcar-sdk"
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
-ROOT_OVERLAY=${TEMPDIR}/stage4_overlay
 
 ## Define the stage4 config template
 catalyst_stage4() {
@@ -63,6 +62,7 @@ export GENTOO_MIRRORS
 
 catalyst_init "$@"
 check_gsutil_opts
+ROOT_OVERLAY=${TEMPDIR}/stage4_overlay
 
 if [[ "$STAGES" =~ stage4 ]]; then
     info "Setting release to ${FLATCAR_VERSION}"
@@ -107,9 +107,11 @@ if [[ "$STAGES" =~ stage4 ]]; then
     sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \
         "${BINPKGS}"/*
 
-    # Upload the SDK toolchain packages
-    sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
-        "toolchain/" "${BINPKGS}/crossdev"/*
+    if [ -d "${BINPKGS}/crossdev" ]; then
+        # Upload the SDK toolchain packages
+        sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \
+            "toolchain/" "${BINPKGS}/crossdev"/*
+    fi
 fi
 
 command_completed

build_image

@@ -18,7 +18,7 @@ assert_inside_chroot
 assert_not_root_user
 
 DEFAULT_GROUP=developer
-DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net"
+DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}"
 
 # Developer-visible flags.
 DEFINE_string board "${DEFAULT_BOARD}" \
@@ -36,7 +36,7 @@ DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \
 DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \
   "The torcx manifest describing torcx packages for this image (or blank for none)"
 DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \
-  "Directory in which torcx packages can be found"
+  "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \
   "Directory in which to place image result directories (named by version)"
 DEFINE_string disk_layout "" \
@@ -95,7 +95,7 @@ check_gsutil_opts
 
 # Patch around default values not being able to depend on other flags.
 if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then
-  FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json"
+  FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json"
 fi
 
 # If downloading packages is enabled ensure the board is configured properly.

build_library/build_image_util.sh

@@ -31,6 +31,7 @@ set_build_symlinks() {
 cleanup_mounts() {
   info "Cleaning up mounts"
   "${BUILD_LIBRARY_DIR}/disk_util" umount "$1" || true
+  rmdir "${1}" || true
 }
 
 delete_prompt() {
@@ -333,7 +334,7 @@ get_metadata() {
     if [ "${key}" = "SRC_URI" ]; then
         local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)"
         local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild"
-        # SRC_URI is empty for the special github.com/flatcar-linux projects
+        # SRC_URI is empty for the special github.com/flatcar projects
         if [ -z "${val}" ]; then
             # The grep invocation gives errors when the ebuild file is not present.
             # This can happen if a "scripts" branch does not match the "coreos-overlay" branch
@@ -341,7 +342,7 @@ get_metadata() {
             val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)"
             if [ -n "${val}" ]; then
                 val="https://github.com/${val}"
-                # All github.com/flatcar-linux projects specify their commit
+                # All github.com/flatcar projects specify their commit
                 local commit=""
                 commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)"
                 if [ -n "${commit}" ]; then
@@ -381,7 +382,7 @@ get_metadata() {
 #       "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"],
 #       "description": "System and service manager for Linux",
 #       "homepage": "https://www.freedesktop.org/wiki/Software/systemd",
-#       "source": "https://github.com/flatcar-linux/systemd ",
+#       "source": "https://github.com/systemd/systemd ",
 #       "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n"
 #     }
 #   ]
@@ -475,11 +476,11 @@ You can read it with "less licenses.json.bz2" or convert it to a text format wit
 bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"'
 The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz".
 Build system files and patches used to build these projects are located at:
-https://github.com/flatcar-linux/coreos-overlay/
-https://github.com/flatcar-linux/portage-stable/
-https://github.com/flatcar-linux/scripts/
+https://github.com/flatcar/coreos-overlay/
+https://github.com/flatcar/portage-stable/
+https://github.com/flatcar/scripts/
 Information on how to build Flatcar Container Linux can be found under:
-https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/
+https://www.flatcar.org/docs/latest/reference/developer-guides/sdk-modifying-flatcar/
 EOF
     sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json
     # Compress the file from 2.1 MB to 0.39 MB
@@ -743,5 +744,6 @@ EOF
     pushd "${BUILD_DIR}" >/dev/null
     zip --quiet -r -9 "${BUILD_DIR}/${pcr_policy}" pcrs
     popd >/dev/null
+    rm -rf "${BUILD_DIR}/pcrs"
   fi
 }

build_library/dev_container_util.sh

@@ -50,12 +50,12 @@ disabled = true
 [coreos]
 location = /var/lib/portage/coreos-overlay
 sync-type = git
-sync-uri = https://github.com/flatcar-linux/coreos-overlay.git
+sync-uri = https://github.com/flatcar/coreos-overlay.git
 
 [portage-stable]
 location = /var/lib/portage/portage-stable
 sync-type = git
-sync-uri = https://github.com/flatcar-linux/portage-stable.git
+sync-uri = https://github.com/flatcar/portage-stable.git
 EOF
 
     # Now set the correct profile

build_library/grub.cfg

@@ -130,7 +130,7 @@ function gptprio {
         echo
         echo "Reading or updating the GPT failed!"
         echo "Please file a bug with any messages above to Flatcar:"
-        echo " https://issues.flatcar-linux.org"
+        echo " https://issues.flatcar.org"
         abort
     fi
 

build_library/set_lsb_release

@@ -54,8 +54,8 @@ VERSION_ID=$FLATCAR_VERSION_ID
 BUILD_ID=$FLATCAR_BUILD_ID
 PRETTY_NAME="$OS_PRETTY_NAME"
 ANSI_COLOR="38;5;75"
-HOME_URL="https://flatcar-linux.org/"
-BUG_REPORT_URL="https://issues.flatcar-linux.org"
+HOME_URL="https://flatcar.org/"
+BUG_REPORT_URL="https://issues.flatcar.org"
 FLATCAR_BOARD="$FLAGS_board"
 EOF
 sudo ln -sf "../usr/lib/os-release" "${ROOT_FS_DIR}/etc/os-release"

build_library/toolchain_util.sh

@@ -142,15 +142,25 @@ get_board_binhost() {
     board="$1"
     shift
 
+    local pkgs_include_toolchain=0
     if [[ $# -eq 0 ]]; then
-        set -- "${FLATCAR_VERSION_ID}"
+        if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
+            # containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url
+            #  and toolchain packages are at the same location as OS image ones
+            set -- "${FLATCAR_VERSION_ID}+${FLATCAR_BUILD_ID}"
+            pkgs_include_toolchain=1
+        else
+            set -- "${FLATCAR_VERSION_ID}"
+        fi
     fi
 
     for ver in "$@"; do
         if [[ $toolchain_only -eq 0 ]]; then
             echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/"
         fi
-        echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
+        if [[ $pkgs_include_toolchain -eq 0 ]]; then
+            echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/"
+        fi
     done
 }
 
@@ -179,7 +189,12 @@ get_sdk_binhost() {
         set -- "${FLATCAR_SDK_VERSION}"
     fi
 
-    FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-$FLATCAR_DEV_BUILDS/sdk}"
+    if [ "${FLATCAR_DEV_BUILDS}" != "${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}" ] ; then
+        FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-${FLATCAR_DEV_BUILDS}/sdk}"
+    else
+        # ALWAYS use a released SDK version, never a nightly, for SDK binpkgs
+        FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-${SETTING_BINPKG_SERVER_PROD}/sdk}"
+    fi
     for ver in "$@"; do
         # Usually only crossdev needs to be fetched from /toolchain/ in the setup_board step.
         # The entry for /pkgs/ is there if something needs to be reinstalled in the SDK

build_library/vm_image_util.sh

@@ -530,6 +530,8 @@ install_oem_aci() {
         "${aci_path}" \
         "${VM_TMP_ROOT}/usr/share/oem/flatcar-oem-${oem_aci}.aci" ||
     die "Could not install ${oem_aci} OEM ACI"
+    # Remove aci_dir if building ACI and installing it succeeded
+    rm -rf "${aci_dir}"
 }
 
 # Any other tweaks required?

build_packages

@@ -38,8 +38,12 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
   "Don't update toolchain automatically."
 DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \
   "Don't run the chroot upgrade automatically; use with care."
+DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \
+  "Directory in which to place torcx stores and manifests (named by board/version)"
 DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \
   "Don't build a new torcx store from the updated sysroot."
+DEFINE_string torcx_extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -262,7 +266,10 @@ upload_packages
 
 # Build a new torcx store with the updated packages, passing flags through.
 if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then
-  "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}"
+  "${SCRIPTS_DIR}"/build_torcx_store \
+                  --board="${BOARD}" \
+                  --output_root="${FLAGS_torcx_output_root}" \
+                  --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}"
 fi
 
 info "Builds complete"

build_toolchains

@@ -15,8 +15,6 @@ FORCE_STAGES="stage4"
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
 
-ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
-
 ## Define the stage4 config template
 catalyst_stage4() {
 cat <<EOF
@@ -32,6 +30,8 @@ catalyst_stage_default
 catalyst_init "$@"
 check_gsutil_opts
 
+ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay"
+
 # toolchain_util.sh is required by catalyst_toolchains.sh
 mkdir -p "${ROOT_OVERLAY}/tmp"
 cp "${BUILD_LIBRARY_DIR}/toolchain_util.sh" "${ROOT_OVERLAY}/tmp"

build_torcx_store

@@ -16,6 +16,8 @@ DEFINE_string board "${DEFAULT_BOARD}" \
   "The board to build packages for."
 DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/torcx" \
   "Directory in which to place torcx stores and manifests (named by board/version)"
+DEFINE_string extra_pkg_url "" \
+  "URL to directory where the torcx packages will be available for downloading"
 
 # include upload options
 . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1
@@ -108,7 +110,8 @@ function torcx_package() {
         local version=${pkg:${#name}+1}
         local manifest_path="${2}"
         local type="${3}"
-        local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default
+        local extra_pkg_url="${4}"
+        local deppkg digest file rpath sha512sum source_pkg rdepends tmproot tmppkgroot update_default tmpfile
         local pkg_cas_file pkg_cas_root
         local pkg_locations=()
         local name=${name##*/}
@@ -220,6 +223,9 @@ function torcx_package() {
         if [[ "${FLAGS_upload}" -eq ${FLAGS_TRUE} ]]; then
                 pkg_locations+=("$(download_tectonic_torcx_url "pkgs/${BOARD}/${name}/${digest}/${name}:${version}.torcx.tgz")")
         fi
+        if [[ -n "${extra_pkg_url}" ]]; then
+            pkg_locations+=("${extra_pkg_url}/${name}:${version}.torcx.tgz")
+        fi
         torcx_manifest::add_pkg "${manifest_path}" \
             "${name}" \
             "${version}" \
@@ -250,8 +256,12 @@ EXTRA_IMAGES=(
 mkdir -p "${BUILD_DIR}"
 manifest_path="${BUILD_DIR}/torcx_manifest.json"
 torcx_manifest::create_empty "${manifest_path}"
-for pkg in "${@:-${DEFAULT_IMAGES[@]}}" ; do torcx_package "${pkg#=}" "${manifest_path}" "default" ; done
-for pkg in "${EXTRA_IMAGES[@]}" ; do torcx_package "${pkg#=}" "${manifest_path}" "extra" ; done
+for pkg in "${@:-${DEFAULT_IMAGES[@]}}"; do
+        torcx_package "${pkg#=}" "${manifest_path}" "default" "${FLAGS_extra_pkg_url}"
+done
+for pkg in "${EXTRA_IMAGES[@]}"; do
+        torcx_package "${pkg#=}" "${manifest_path}" "extra" "${FLAGS_extra_pkg_url}"
+done
 
 set_build_symlinks latest "${FLAGS_group}-latest"
 

common.sh

@@ -3,6 +3,8 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
+. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
+
 # All scripts should die on error unless commands are specifically excepted
 # by prefixing with '!' or surrounded by 'set +e' / 'set -e'.
 
@@ -334,7 +336,11 @@ readonly COREOS_EPOCH=1372636800
 TODAYS_VERSION=$(( (`date +%s` - ${COREOS_EPOCH}) / 86400 ))
 
 # Download URL prefix for SDK and board binary packages
-: ${FLATCAR_DEV_BUILDS:=https://mirror.release.flatcar-linux.net}
+if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then
+    : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}}
+else
+    : ${FLATCAR_DEV_BUILDS:=${SETTING_BINPKG_SERVER_PROD}}
+fi
 
 # Load developer's custom settings.  Default location is in scripts dir,
 # since that's available both inside and outside the chroot.  By convention,

contrib/create-coreos-vdi

@@ -12,7 +12,7 @@ This tool creates a Flatcar VDI image to be used with VirtualBox.
 "
 
 # Image signing key: buildbot@flatcar-linux.org
-GPG_KEY_URL="https://flatcar-linux.org/security/image-signing-key/Flatcar_Image_Signing_Key.pem"
+GPG_KEY_URL="https://www.flatcar.org/security/image-signing-key/Flatcar_Image_Signing_Key.pem"
 GPG_LONG_ID="E25D9AED0593B34A"
 GPG_KEY="$(wget -qO- $GPG_KEY_URL)"
 

jenkins/systemd-run-wrap.sh

@@ -15,7 +15,7 @@ for VARNAME in $(sh -c 'compgen -v'); do
   ARGS+=("--setenv" "${VARNAME}=${VAL}")
 done
 
-UNITNAME="run-$(date '+%s')"
+UNITNAME="run-$(date '+%s')-${RANDOM}"
 
 # The --pipe option does not stop the unit when the systemd-run process is killed, we have to do this through a trap
 # (and --pty as alternative doesn't behave well because it leads to processes expecting stdin when there is none)

set_version

@@ -2,7 +2,9 @@
 
 set -euo pipefail
 
-DEFAULT_BASE_URL="https://mirror.release.flatcar-linux.net"
+. "$(dirname ${BASH_SOURCE[0]})/settings.env" || exit 1
+
+DEFAULT_BASE_URL="${SETTING_BINPKG_SERVER_PROD}"
 DEV_BOARD_URL="${DEFAULT_BASE_URL}/developer"
 DEFAULT_SDK_URL="${DEFAULT_BASE_URL}/sdk"
 DEV_SDK_URL="${DEFAULT_BASE_URL}/developer/sdk"
@@ -41,7 +43,8 @@ Usage: $0 FLAGS...
   --file FILE:                                     Modify another file than ${FILE}, useful if run
                                                    outside of the SDK chroot. If /dev/stdout or
                                                    /dev/stderr is used, only new values are printed.
-  --binhost                                        Use a custom binhost (defaults to '${DEFAULT_BASE_URL}').
+  --binhost                                        Use a custom binhost (defaults to '${DEFAULT_BASE_URL}'),
+                                                   e.g., '${SETTING_BINPKG_SERVER_DEV}'.
                                                    This will update BOARD and SDK URLs accordingly.
 "
   exit 1

setup_board

@@ -27,6 +27,8 @@ DEFINE_boolean getbinpkg "${FLAGS_TRUE}" \
   "Download binary packages from remote repository."
 DEFINE_string getbinpkgver "" \
   "Use binary packages from a specific version."
+DEFINE_string pkgdir "" \
+  "Use binary packages from a custom directory instead of /build/[ARCH]/var/lib/portage/pkgs/."
 DEFINE_string binhost "" \
   "Use binary packages from a specific location instead of $FLATCAR_DEV_BUILDS/... "
 DEFINE_boolean toolchainpkgonly "${FLAGS_FALSE}" \
@@ -100,7 +102,7 @@ generate_all_wrappers() {
   wrapper="/usr/local/bin/cros_workon-${BOARD_VARIANT}"
   sudo_clobber "${wrapper}" <<EOF
 #!/bin/bash
-exec cros_workon --board ${BOARD_VARIANT} "\$@"
+exec "${SRC_ROOT}/scripts/cros_workon" --board ${BOARD_VARIANT} "\$@"
 EOF
   wrappers+=( "${wrapper}" )
 
@@ -162,6 +164,7 @@ BOARD_ARCH=$(get_board_arch "$BOARD")
 BOARD_CHOST=$(get_board_chost ${BOARD})
 PORTAGE_PROFILE=$(get_board_profile "$BOARD")
 BOARD_BINHOST="$FLAGS_binhost $(generate_binhost_list)"
+BOARD_PKGDIR="${FLAGS_pkgdir:-${BOARD_ROOT}/var/lib/portage/pkgs}"
 
 if [[ ${FLAGS_regen_configs_only} -eq ${FLAGS_TRUE} ]]; then
   FLAGS_regen_configs=${FLAGS_TRUE}
@@ -261,7 +264,7 @@ PORTAGE_USERNAME="$(portageq envvar PORTAGE_USERNAME)"
 # Board specific settings
 CHOST="${BOARD_CHOST}"
 ROOT="${BOARD_ROOT}/"
-PKGDIR="${BOARD_ROOT}/var/lib/portage/pkgs"
+PKGDIR="${BOARD_PKGDIR}"
 PORT_LOGDIR="${BOARD_ROOT}/var/log/portage"
 PORTAGE_TMPDIR="${BOARD_ROOT}/var/tmp"
 PORTAGE_BINHOST="${BOARD_BINHOST}"
@@ -354,7 +357,7 @@ info "The SYSROOT is: ${BOARD_ROOT}"
 # NOTE: Printing the working-on ebuilds does not only serve the informative
 # purpose. It also causes the ${BOARD_ROOT}/etc/portage/package.* files to be
 # regenerated.
-WORKING_ON=$(cros_workon --board=${FLAGS_board} list)
+WORKING_ON=$("${SRC_ROOT}/scripts/cros_workon" --board=${FLAGS_board} list)
 if [ -n "${WORKING_ON}" ]; then
   info
   info "Currently working on the following ebuilds for this board:"

update_chroot

@@ -26,6 +26,8 @@ DEFINE_integer jobs "${NUM_JOBS}" \
   "How many packages to build in parallel at maximum."
 DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \
   "Don't update the toolchains."
+DEFINE_boolean setuponly "${FLAGS_FALSE}" \
+  "Only configure portage, without updating packages. Useful when only boostrap_sdk/build_toolchains will be called"
 DEFINE_string toolchain_boards "" \
   "Extra toolchains to setup for the specified boards."
 DEFINE_string dev_builds_sdk "" \
@@ -220,6 +222,11 @@ sudo -E ${EMERGE_CMD} --quiet "${EMERGE_FLAGS[@]}" \
 
 gcc_set_latest_profile "$(portageq envvar CHOST)"
 
+if [[ "${FLAGS_setuponly}" -eq "${FLAGS_TRUE}" ]]; then
+  command_completed
+  exit 0
+fi
+
 if [[ "${FLAGS_skip_toolchain_update}" -eq "${FLAGS_FALSE}" && \
       -n "${FLAGS_toolchain_boards}" ]]; then
   CROSS_CHOSTS=( $(get_board_chost ${FLAGS_toolchain_boards} | sort -u) )