mirror of
https://github.com/armbian/build.git
synced 2025-09-08 07:11:21 +02:00
> tl-dr: > - maximize OCI cache hit ratio across nightlies/releases/PRs/etc; > - publish simple `Version:`'s that don't include a crazy hash in repo and images > - introduce `output/packages-hashed` directory > - radically change the `output/debs` directory structure - simplify artifact's `prepare_version()` method for `deb` and `deb-tar` artifacts: - `artifact_base_dir` and `artifact_final_file` will now be auto-calculated; thus removed from each artifact (except `rootfs`) - `artifact_deb_repo` ("global", "jammy", "bookworm") is now required; "global" means common across all RELEASES - `artifact_deb_arch` is now required, "all" is arch-independent, otherwise use `${ARCH}` - `artifact_map_debs` is now auto-calculated based on the above, and shouldn't be specified manually - `artifact_final_version_reversioned` is optional, and can force the final version of the artifact (specific for the `base-files` case) - artifacts that need special handling for reversioning can add function names to `artifact_debs_reversion_functions` array (`base-files` and `bsp-cli` cases) - artifacts `prepare_version()` should set `artifact_version`, but _never_ include it in other variables; `artifact_version` is now changed by framework after `prepare_version()` returns - no longer use/refer/mention `${REVISION}` when building packages. All packages should be `${REVISION}`-agnostic. - `${REVISION}` (actually, `artifact_final_version_reversioned`) will be automatically swapped in the `control` file during reversioning - `fakeroot_dpkg_deb_build()` now takes exactly two arguments: the directory to pack, and the deb ID (key of `artifact_map_packages` dict); add this change in all the artifact's code for this - `obtain_complete_artifact()`: - automatically adds `-Rxxxx` "revisioning-hash" to `artifact_version`, by hashing the revisioning functions and any `artifact_debs_reversion_functions` set - calculates more complex subdirectory paths for both the `output/packages-hashed` and `output/debs`/`output/debs-beta` directories - with the new subdirectories we can be sure a re-version is already done correctly and can skip it (eg, for partial `download-debs` re-runs) - in the future we can automatically clean/remove old versions that are no longer relevant based on the dir structure - exports a lot more information to JSON, including the new subdirectory paths - comment-out code that implemented `skip_unpack_if_found_in_caches`, I'm very unsure why we had this in the first place - `obtain_artifact_from_remote_cache()` - for `deb` type artifacts, OCI won't preserve the subdirectory structure, so move downloaded files to the correct subdirectory manually - this is not needed for `deb-tar`, since that can preserve the dir structure itself - introduce `artifacts-reversion.sh` and its main function `artifact_reversion_for_deployment()` - this has the logic for reversioning .deb's, by `ar`-unpacking them, changing `control.tar` (and possibly `data.tar`), handling `.xz` compression, etc. - also handles hashing those functions, for consistency. Any changes in reversioning code actually change the artifact itself so we're not caught by surprise - by default, it changes `control` file only: - replace `Version:` (which is the hash-version originally) with `artifact_final_version_reversioned` (which is mostly just `${REVISION}`) - add a custom field `Armbian-Original-Hash:` with the original hash-version - `artifact_reversion_for_deployment()` is called by - new CLI wrapper `cli_obtain_complete_artifact()`, used for CLI building of specific artifact, but also for `download-artifact` - `build_artifact_for_image()` used during image build - `armbian-bsp-cli-deb.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `artifact-armbian-base-files.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `kernel`: - add some custom fields to `DEBIAN/control`: - `Armbian-Kernel-Version:` / `Armbian-Kernel-Version-Family:` (for future use: cleanup of usage of `Source: ` field which should be removed) - declutter the `Description:` field, moving long description out of the first line - obtain `IMAGE_INSTALLED_KERNEL_VERSION` from the reversioned deb (this is still a hack and has not been fixed) - `uboot`: - declutter the `Description:` field, moving long description out of the first line - use the reversioned .deb when deploying u-boot to the image - `main_default_build_packages()` now stores reversioned values and complete paths to reversioned .deb's - `list_installed_packages()` now compares custom field `Armbian-Original-Hash: `, and not the `Version:` to make sure debs in the image are the ones we want - `install_artifact_deb_chroot()` is a new wrapper around `install_deb_chroot()` for easy handling of reversioned debs - use it everywhere `install_deb_chroot()` was used in `distro-agnostic.sh` and `distro-specific.sh`
216 lines
8.6 KiB
Bash
216 lines
8.6 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
|
#
|
|
# This file is a part of the Armbian Build Framework
|
|
# https://github.com/armbian/build/
|
|
|
|
function artifact_uboot_config_dump() {
|
|
artifact_input_variables[BOOTSOURCE]="${BOOTSOURCE}"
|
|
artifact_input_variables[BOOTBRANCH]="${BOOTBRANCH}"
|
|
artifact_input_variables[BOOTPATCHDIR]="${BOOTPATCHDIR}"
|
|
artifact_input_variables[BOARD]="${BOARD}"
|
|
artifact_input_variables[BRANCH]="${BRANCH}"
|
|
artifact_input_variables[ARCH]="${ARCH}"
|
|
}
|
|
|
|
function artifact_uboot_prepare_version() {
|
|
artifact_version="undetermined" # outer scope
|
|
artifact_version_reason="undetermined" # outer scope
|
|
|
|
# Prepare the version, "sans-repos": just the armbian/build repo contents are available.
|
|
# It is OK to reach out to the internet for a curl or ls-remote, but not for a git clone/fetch.
|
|
|
|
# - Given BOOTSOURCE and BOOTBRANCH, get:
|
|
# - SHA1 of the commit (this is generic... and used for other pkgs)
|
|
# - The first 10 lines of the root Makefile at that commit (cached lookup, same SHA1=same Makefile)
|
|
# - This gives us the full version plus codename.
|
|
# - Get the u-boot patches hash. (could just hash the BOOTPATCHDIR non-disabled contents, or use Python patching proper?)
|
|
# - Hash of the relevant lib/ bash sources involved, say compilation/uboot*.sh etc
|
|
# All those produce a version string like:
|
|
# 2023.11-<4-digit-SHA1>_<4_digit_patches>
|
|
|
|
debug_var BOOTSOURCE
|
|
debug_var BOOTBRANCH
|
|
debug_var BOOTPATCHDIR
|
|
debug_var BOARD
|
|
debug_var BRANCH
|
|
|
|
declare short_hash_size=4
|
|
|
|
declare -A GIT_INFO_UBOOT=([GIT_SOURCE]="${BOOTSOURCE}" [GIT_REF]="${BOOTBRANCH}")
|
|
run_memoized GIT_INFO_UBOOT "git2info" memoized_git_ref_to_info "include_makefile_body"
|
|
debug_dict GIT_INFO_UBOOT
|
|
|
|
# Sanity check, the SHA1 gotta be sane.
|
|
[[ "${GIT_INFO_UBOOT[SHA1]}" =~ ^[0-9a-f]{40}$ ]] || exit_with_error "SHA1 is not sane: '${GIT_INFO_UBOOT[SHA1]}'"
|
|
|
|
declare short_sha1="${GIT_INFO_UBOOT[SHA1]:0:${short_hash_size}}"
|
|
|
|
# get the uboot patches hash...
|
|
# @TODO: why not just delegate this to the python patching, with some "dry-run" / hash-only option?
|
|
# @TODO: this is even more grave in case of u-boot: v2022.10 has patches for many boards inside, gotta resolve.
|
|
declare patches_hash="undetermined"
|
|
declare hash_files="undetermined"
|
|
declare -a uboot_patch_dirs=()
|
|
for patch_dir in ${BOOTPATCHDIR}; do
|
|
uboot_patch_dirs+=("${SRC}/patch/u-boot/${patch_dir}" "${USERPATCHES_PATH}/u-boot/${patch_dir}")
|
|
done
|
|
|
|
if [[ -n "${ATFSOURCE}" && "${ATFSOURCE}" != "none" ]]; then
|
|
uboot_patch_dirs+=("${SRC}/patch/atf/${ATFPATCHDIR}" "${USERPATCHES_PATH}/atf/${ATFPATCHDIR}")
|
|
fi
|
|
|
|
if [[ -n "${CRUSTCONFIG}" ]]; then
|
|
uboot_patch_dirs+=("${SRC}/patch/crust/${CRUSTPATCHDIR}" "${USERPATCHES_PATH}/crust/${CRUSTPATCHDIR}")
|
|
fi
|
|
|
|
calculate_hash_for_all_files_in_dirs "${uboot_patch_dirs[@]}"
|
|
patches_hash="${hash_files}"
|
|
declare uboot_patches_hash_short="${patches_hash:0:${short_hash_size}}"
|
|
|
|
# Hash the extension hooks
|
|
declare -a extension_hooks_to_hash=(
|
|
"post_uboot_custom_postprocess" "fetch_custom_uboot" "build_custom_uboot"
|
|
"pre_config_uboot_target" "post_uboot_custom_postprocess" "post_uboot_custom_postprocess"
|
|
"post_config_uboot_target"
|
|
)
|
|
declare -a extension_hooks_hashed=("$(dump_extension_method_sources_functions "${extension_hooks_to_hash[@]}")")
|
|
declare hash_hooks="undetermined"
|
|
hash_hooks="$(echo "${extension_hooks_hashed[@]}" | sha256sum | cut -d' ' -f1)"
|
|
|
|
# Hash the old-timey hooks
|
|
declare hash_functions="undetermined"
|
|
calculate_hash_for_function_bodies "write_uboot_platform" "write_uboot_platform_mtd" "setup_write_uboot_platform"
|
|
declare hash_uboot_functions="${hash_functions}"
|
|
|
|
# Hash those two together
|
|
declare hash_hooks_and_functions="undetermined"
|
|
hash_hooks_and_functions="$(echo "${hash_hooks}" "${hash_uboot_functions}" | sha256sum | cut -d' ' -f1)"
|
|
declare hash_hooks_and_functions_short="${hash_hooks_and_functions:0:${short_hash_size}}"
|
|
|
|
# Hash variables that affect the build and package of u-boot
|
|
declare -a vars_to_hash=(
|
|
"${BOOTDELAY}" "${UBOOT_DEBUGGING}" "${UBOOT_TARGET_MAP}" # general for all families
|
|
"${BOOT_SCENARIO}" "${BOOT_SUPPORT_SPI}" "${BOOT_SOC}" # rockchip stuff, sorry.
|
|
"${DDR_BLOB}" "${BL31_BLOB}" "${MINILOADER_BLOB}" # More rockchip stuff, even more sorry.
|
|
"${ATF_COMPILE}" "${ATFBRANCH}" "${ATFPATCHDIR}" # arm-trusted-firmware stuff
|
|
"${CRUSTCONFIG}" "${CRUSTBRANCH}" "${CRUSTPATCHDIR}" # crust stuff
|
|
)
|
|
declare hash_variables="undetermined" # will be set by calculate_hash_for_variables(), which normalizes the input
|
|
calculate_hash_for_variables "${vars_to_hash[@]}"
|
|
declare var_config_hash_short="${hash_variables:0:${short_hash_size}}"
|
|
|
|
# get the hashes of the lib/ bash sources involved...
|
|
declare hash_files="undetermined"
|
|
calculate_hash_for_bash_deb_artifact "${SRC}"/lib/functions/compilation/uboot*.sh # expansion
|
|
declare bash_hash="${hash_files}"
|
|
declare bash_hash_short="${bash_hash:0:${short_hash_size}}"
|
|
|
|
# outer scope
|
|
artifact_version="${GIT_INFO_UBOOT[MAKEFILE_VERSION]}-S${short_sha1}-P${uboot_patches_hash_short}-H${hash_hooks_and_functions_short}-V${var_config_hash_short}-B${bash_hash_short}"
|
|
|
|
declare -a reasons=(
|
|
"version \"${GIT_INFO_UBOOT[MAKEFILE_FULL_VERSION]}\""
|
|
"git revision \"${GIT_INFO_UBOOT[SHA1]}\""
|
|
"patches hash \"${patches_hash}\""
|
|
"Extension hooks hash \"${hash_hooks}\""
|
|
"uboot functions hash \"${hash_uboot_functions}\""
|
|
"variables hash \"${vars_config_hash}\""
|
|
"framework bash hash \"${bash_hash}\""
|
|
)
|
|
|
|
artifact_deb_repo="global"
|
|
artifact_deb_arch="${ARCH}"
|
|
artifact_version_reason="${reasons[*]}" # outer scope
|
|
artifact_map_packages=(["uboot"]="linux-u-boot-${BOARD}-${BRANCH}")
|
|
artifact_name="uboot-${BOARD}-${BRANCH}"
|
|
artifact_type="deb"
|
|
|
|
return 0
|
|
}
|
|
|
|
function artifact_uboot_build_from_sources() {
|
|
LOG_SECTION="fetch_and_build_host_tools" do_with_logging fetch_and_build_host_tools
|
|
|
|
if [[ -n "${ATFSOURCE}" && "${ATFSOURCE}" != "none" ]]; then
|
|
if [[ "${ARTIFACT_BUILD_INTERACTIVE:-"no"}" == "yes" ]]; then
|
|
display_alert "Running ATF build in interactive mode" "log file will be incomplete" "info"
|
|
compile_atf
|
|
|
|
if [[ "${CREATE_PATCHES_ATF:-"no"}" == "yes" ]]; then
|
|
return 0 # stop here, otherwise it would build u-boot below...
|
|
fi
|
|
else
|
|
LOG_SECTION="compile_atf" do_with_logging compile_atf
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "${CRUSTCONFIG}" ]]; then
|
|
if [[ "${ARTIFACT_BUILD_INTERACTIVE:-"no"}" == "yes" ]]; then
|
|
display_alert "Running crust build in interactive mode" "log file will be incomplete" "info"
|
|
compile_crust
|
|
|
|
if [[ "${CREATE_PATCHES_CRUST:-"no"}" == "yes" ]]; then
|
|
return 0 # stop here, otherwise it would build u-boot below...
|
|
fi
|
|
else
|
|
LOG_SECTION="compile_crust" do_with_logging compile_crust
|
|
fi
|
|
fi
|
|
|
|
declare uboot_git_revision="not_determined_yet"
|
|
LOG_SECTION="uboot_prepare_git" do_with_logging_unless_user_terminal uboot_prepare_git
|
|
|
|
# Hack, if ARTIFACT_BUILD_INTERACTIVE=yes, don't run under logging manager. Emit a warning about it.
|
|
if [[ "${ARTIFACT_BUILD_INTERACTIVE:-"no"}" == "yes" ]]; then
|
|
display_alert "Running uboot build in interactive mode" "log file will be incomplete" "info"
|
|
compile_uboot
|
|
else
|
|
LOG_SECTION="compile_uboot" do_with_logging compile_uboot
|
|
fi
|
|
}
|
|
|
|
function artifact_uboot_cli_adapter_pre_run() {
|
|
declare -g ARMBIAN_COMMAND_REQUIRE_BASIC_DEPS="yes" # Require prepare_host_basic to run before the command.
|
|
|
|
# "gimme root on a Linux machine"
|
|
cli_standard_relaunch_docker_or_sudo
|
|
}
|
|
|
|
function artifact_uboot_cli_adapter_config_prep() {
|
|
# Sanity check / cattle guard
|
|
# If UBOOT_CONFIGURE=yes, or CREATE_PATCHES=yes, user must have used the correct CLI commands, and only add those params.
|
|
if [[ "${UBOOT_CONFIGURE}" == "yes" && ("${ARMBIAN_COMMAND}" != "uboot-config") ]]; then
|
|
exit_with_error "UBOOT_CONFIGURE=yes is not supported anymore. Please use the new 'uboot-config' CLI command. Current command: '${ARMBIAN_COMMAND}'"
|
|
fi
|
|
|
|
if [[ "${CREATE_PATCHES}" == "yes" && "${ARMBIAN_COMMAND}" != "uboot-patch" ]]; then
|
|
exit_with_error "CREATE_PATCHES=yes is not supported anymore. Please use the new 'uboot-patch' CLI command. Current command: '${ARMBIAN_COMMAND}'"
|
|
fi
|
|
|
|
use_board="yes" prep_conf_main_minimal_ni < /dev/null # no stdin for this, so it bombs if tries to be interactive.
|
|
}
|
|
|
|
function artifact_uboot_get_default_oci_target() {
|
|
artifact_oci_target_base="${GHCR_SOURCE}/armbian/os/"
|
|
}
|
|
|
|
function artifact_uboot_is_available_in_local_cache() {
|
|
is_artifact_available_in_local_cache
|
|
}
|
|
|
|
function artifact_uboot_is_available_in_remote_cache() {
|
|
is_artifact_available_in_remote_cache
|
|
}
|
|
|
|
function artifact_uboot_obtain_from_remote_cache() {
|
|
obtain_artifact_from_remote_cache
|
|
}
|
|
|
|
function artifact_uboot_deploy_to_remote_cache() {
|
|
upload_artifact_to_oci
|
|
}
|