diff --git a/.gitignore b/.gitignore index cf694dcf24..a62ac41ee1 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,15 @@ *.pyc *~ /cbuildbot_package.list + +# Flatcar SDK tarballs +*.tar.bz2 + +# SDK container env passing helpers +sdk_container/.env +sdk_container/.sdkenv + +# build cache / artefacts directories +__build__/ +sdk_container/.cache +sdk_container/.config diff --git a/bootstrap_sdk b/bootstrap_sdk index 9bfbe92c77..8646267867 100755 --- a/bootstrap_sdk +++ b/bootstrap_sdk @@ -40,7 +40,6 @@ TYPE="flatcar-sdk" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 -ROOT_OVERLAY=${TEMPDIR}/stage4_overlay ## Define the stage4 config template catalyst_stage4() { @@ -63,6 +62,7 @@ export GENTOO_MIRRORS catalyst_init "$@" check_gsutil_opts +ROOT_OVERLAY=${TEMPDIR}/stage4_overlay if [[ "$STAGES" =~ stage4 ]]; then info "Setting release to ${FLATCAR_VERSION}" @@ -107,9 +107,11 @@ if [[ "$STAGES" =~ stage4 ]]; then sign_and_upload_files "packages" "${def_upload_path}" "pkgs/" \ "${BINPKGS}"/* - # Upload the SDK toolchain packages - sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ - "toolchain/" "${BINPKGS}/crossdev"/* + if [ -d "${BINPKGS}/crossdev" ]; then + # Upload the SDK toolchain packages + sign_and_upload_files "cross toolchain packages" "${def_upload_path}" \ + "toolchain/" "${BINPKGS}/crossdev"/* + fi fi command_completed diff --git a/build_image b/build_image index eb54e4d571..49b3b4c92f 100755 --- a/build_image +++ b/build_image @@ -18,7 +18,7 @@ assert_inside_chroot assert_not_root_user DEFAULT_GROUP=developer -DEFAULT_DEVCONTAINER_BINHOST="https://mirror.release.flatcar-linux.net" +DEFAULT_DEVCONTAINER_BINHOST="${SETTING_BINPKG_SERVER_PROD}" # Developer-visible flags. DEFINE_string board "${DEFAULT_BOARD}" \ @@ -36,7 +36,7 @@ DEFINE_string base_dev_pkg "coreos-base/coreos-dev" \ DEFINE_string torcx_manifest "${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" \ "The torcx manifest describing torcx packages for this image (or blank for none)" DEFINE_string torcx_root "${DEFAULT_BUILD_ROOT}/torcx" \ - "Directory in which torcx packages can be found" + "Directory in which torcx packages can be found. Will update the default --torcx_manifest if set." DEFINE_string output_root "${DEFAULT_BUILD_ROOT}/images" \ "Directory in which to place image result directories (named by version)" DEFINE_string disk_layout "" \ @@ -95,7 +95,7 @@ check_gsutil_opts # Patch around default values not being able to depend on other flags. if [ "x${FLAGS_torcx_manifest}" = "x${DEFAULT_BUILD_ROOT}/torcx/${DEFAULT_BOARD}/latest/torcx_manifest.json" ]; then - FLAGS_torcx_manifest="${DEFAULT_BUILD_ROOT}/torcx/${FLAGS_board}/latest/torcx_manifest.json" + FLAGS_torcx_manifest="${FLAGS_torcx_root}/${FLAGS_board}/latest/torcx_manifest.json" fi # If downloading packages is enabled ensure the board is configured properly. diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index 6d7cd4836d..13f7c5f804 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -31,6 +31,7 @@ set_build_symlinks() { cleanup_mounts() { info "Cleaning up mounts" "${BUILD_LIBRARY_DIR}/disk_util" umount "$1" || true + rmdir "${1}" || true } delete_prompt() { @@ -333,7 +334,7 @@ get_metadata() { if [ "${key}" = "SRC_URI" ]; then local package_name="$(echo "${pkg%%:*}" | cut -d / -f 2)" local ebuild_path="${prefix}/var/db/pkg/${pkg%%:*}/${package_name}.ebuild" - # SRC_URI is empty for the special github.com/flatcar-linux projects + # SRC_URI is empty for the special github.com/flatcar projects if [ -z "${val}" ]; then # The grep invocation gives errors when the ebuild file is not present. # This can happen if a "scripts" branch does not match the "coreos-overlay" branch @@ -341,7 +342,7 @@ get_metadata() { val="$(grep "CROS_WORKON_PROJECT=" "${ebuild_path}" | cut -d '"' -f 2)" if [ -n "${val}" ]; then val="https://github.com/${val}" - # All github.com/flatcar-linux projects specify their commit + # All github.com/flatcar projects specify their commit local commit="" commit="$(grep "CROS_WORKON_COMMIT=" "${ebuild_path}" | cut -d '"' -f 2)" if [ -n "${commit}" ]; then @@ -381,7 +382,7 @@ get_metadata() { # "licenses": ["GPL-2", "LGPL-2.1", "MIT", "public-domain"], # "description": "System and service manager for Linux", # "homepage": "https://www.freedesktop.org/wiki/Software/systemd", -# "source": "https://github.com/flatcar-linux/systemd ", +# "source": "https://github.com/systemd/systemd ", # "files": "somefile 63a5736879fa647ac5a8d5317e7cb8b0\nsome -> link\n" # } # ] @@ -475,11 +476,11 @@ You can read it with "less licenses.json.bz2" or convert it to a text format wit bzcat licenses.json.bz2 | jq -r '.[] | "\(.project):\nDescription: \(.description)\nLicenses: \(.licenses)\nHomepage: \(.homepage)\nSource code: \(.source)\nFiles:\n\(.files)\n"' The license texts are available under /usr/share/licenses/common/ and can be read with "less NAME.gz". Build system files and patches used to build these projects are located at: -https://github.com/flatcar-linux/coreos-overlay/ -https://github.com/flatcar-linux/portage-stable/ -https://github.com/flatcar-linux/scripts/ +https://github.com/flatcar/coreos-overlay/ +https://github.com/flatcar/portage-stable/ +https://github.com/flatcar/scripts/ Information on how to build Flatcar Container Linux can be found under: -https://docs.flatcar-linux.org/os/sdk-modifying-flatcar/ +https://www.flatcar.org/docs/latest/reference/developer-guides/sdk-modifying-flatcar/ EOF sudo cp "${json_input}" "${root_fs_dir}"/usr/share/licenses/licenses.json # Compress the file from 2.1 MB to 0.39 MB @@ -743,5 +744,6 @@ EOF pushd "${BUILD_DIR}" >/dev/null zip --quiet -r -9 "${BUILD_DIR}/${pcr_policy}" pcrs popd >/dev/null + rm -rf "${BUILD_DIR}/pcrs" fi } diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index 34a028baef..b4a6753bde 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -50,12 +50,12 @@ disabled = true [coreos] location = /var/lib/portage/coreos-overlay sync-type = git -sync-uri = https://github.com/flatcar-linux/coreos-overlay.git +sync-uri = https://github.com/flatcar/coreos-overlay.git [portage-stable] location = /var/lib/portage/portage-stable sync-type = git -sync-uri = https://github.com/flatcar-linux/portage-stable.git +sync-uri = https://github.com/flatcar/portage-stable.git EOF # Now set the correct profile diff --git a/build_library/grub.cfg b/build_library/grub.cfg index ca6c5f1824..d62b98ec3f 100644 --- a/build_library/grub.cfg +++ b/build_library/grub.cfg @@ -130,7 +130,7 @@ function gptprio { echo echo "Reading or updating the GPT failed!" echo "Please file a bug with any messages above to Flatcar:" - echo " https://issues.flatcar-linux.org" + echo " https://issues.flatcar.org" abort fi diff --git a/build_library/set_lsb_release b/build_library/set_lsb_release index 7cf81f2e47..5337303eb2 100755 --- a/build_library/set_lsb_release +++ b/build_library/set_lsb_release @@ -54,8 +54,8 @@ VERSION_ID=$FLATCAR_VERSION_ID BUILD_ID=$FLATCAR_BUILD_ID PRETTY_NAME="$OS_PRETTY_NAME" ANSI_COLOR="38;5;75" -HOME_URL="https://flatcar-linux.org/" -BUG_REPORT_URL="https://issues.flatcar-linux.org" +HOME_URL="https://flatcar.org/" +BUG_REPORT_URL="https://issues.flatcar.org" FLATCAR_BOARD="$FLAGS_board" EOF sudo ln -sf "../usr/lib/os-release" "${ROOT_FS_DIR}/etc/os-release" diff --git a/build_library/toolchain_util.sh b/build_library/toolchain_util.sh index 8023b844f2..b658c83c71 100644 --- a/build_library/toolchain_util.sh +++ b/build_library/toolchain_util.sh @@ -142,15 +142,25 @@ get_board_binhost() { board="$1" shift + local pkgs_include_toolchain=0 if [[ $# -eq 0 ]]; then - set -- "${FLATCAR_VERSION_ID}" + if [[ "${FLATCAR_BUILD_ID}" =~ ^nightly-.*$ ]] ; then + # containerised nightly build; this uses [VERSION]-[BUILD_ID] for binpkg url + # and toolchain packages are at the same location as OS image ones + set -- "${FLATCAR_VERSION_ID}+${FLATCAR_BUILD_ID}" + pkgs_include_toolchain=1 + else + set -- "${FLATCAR_VERSION_ID}" + fi fi for ver in "$@"; do if [[ $toolchain_only -eq 0 ]]; then echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/pkgs/" fi - echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" + if [[ $pkgs_include_toolchain -eq 0 ]]; then + echo "${FLATCAR_DEV_BUILDS}/boards/${board}/${ver}/toolchain/" + fi done } @@ -179,7 +189,12 @@ get_sdk_binhost() { set -- "${FLATCAR_SDK_VERSION}" fi - FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-$FLATCAR_DEV_BUILDS/sdk}" + if [ "${FLATCAR_DEV_BUILDS}" != "${SETTING_BINPKG_SERVER_DEV_CONTAINERISED}" ] ; then + FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-${FLATCAR_DEV_BUILDS}/sdk}" + else + # ALWAYS use a released SDK version, never a nightly, for SDK binpkgs + FLATCAR_DEV_BUILDS_SDK="${FLATCAR_DEV_BUILDS_SDK-${SETTING_BINPKG_SERVER_PROD}/sdk}" + fi for ver in "$@"; do # Usually only crossdev needs to be fetched from /toolchain/ in the setup_board step. # The entry for /pkgs/ is there if something needs to be reinstalled in the SDK diff --git a/build_library/vm_image_util.sh b/build_library/vm_image_util.sh index 91e4680d4b..ec92db0762 100644 --- a/build_library/vm_image_util.sh +++ b/build_library/vm_image_util.sh @@ -530,6 +530,8 @@ install_oem_aci() { "${aci_path}" \ "${VM_TMP_ROOT}/usr/share/oem/flatcar-oem-${oem_aci}.aci" || die "Could not install ${oem_aci} OEM ACI" + # Remove aci_dir if building ACI and installing it succeeded + rm -rf "${aci_dir}" } # Any other tweaks required? diff --git a/build_packages b/build_packages index aaeca62a64..752124ade1 100755 --- a/build_packages +++ b/build_packages @@ -38,8 +38,12 @@ DEFINE_boolean skip_toolchain_update "${FLAGS_FALSE}" \ "Don't update toolchain automatically." DEFINE_boolean skip_chroot_upgrade "${FLAGS_FALSE}" \ "Don't run the chroot upgrade automatically; use with care." +DEFINE_string torcx_output_root "${DEFAULT_BUILD_ROOT}/torcx" \ + "Directory in which to place torcx stores and manifests (named by board/version)" DEFINE_boolean skip_torcx_store "${FLAGS_FALSE}" \ "Don't build a new torcx store from the updated sysroot." +DEFINE_string torcx_extra_pkg_url "" \ + "URL to directory where the torcx packages will be available for downloading" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 @@ -262,7 +266,10 @@ upload_packages # Build a new torcx store with the updated packages, passing flags through. if [ "${FLAGS_skip_torcx_store}" -eq "${FLAGS_FALSE}" ]; then - "${SCRIPTS_DIR}"/build_torcx_store --board="${BOARD}" + "${SCRIPTS_DIR}"/build_torcx_store \ + --board="${BOARD}" \ + --output_root="${FLAGS_torcx_output_root}" \ + --extra_pkg_url="${FLAGS_torcx_extra_pkg_url}" fi info "Builds complete" diff --git a/build_toolchains b/build_toolchains index 85c09001a9..4168c01a10 100755 --- a/build_toolchains +++ b/build_toolchains @@ -15,8 +15,6 @@ FORCE_STAGES="stage4" # include upload options . "${BUILD_LIBRARY_DIR}/release_util.sh" || exit 1 -ROOT_OVERLAY="${TEMPDIR}/stage4-${ARCH}-$FLAGS_version-overlay" - ## Define the stage4 config template catalyst_stage4() { cat <