mirror of
https://github.com/armbian/build.git
synced 2025-09-09 15:51:10 +02:00
> tl-dr: > - maximize OCI cache hit ratio across nightlies/releases/PRs/etc; > - publish simple `Version:`'s that don't include a crazy hash in repo and images > - introduce `output/packages-hashed` directory > - radically change the `output/debs` directory structure - simplify artifact's `prepare_version()` method for `deb` and `deb-tar` artifacts: - `artifact_base_dir` and `artifact_final_file` will now be auto-calculated; thus removed from each artifact (except `rootfs`) - `artifact_deb_repo` ("global", "jammy", "bookworm") is now required; "global" means common across all RELEASES - `artifact_deb_arch` is now required, "all" is arch-independent, otherwise use `${ARCH}` - `artifact_map_debs` is now auto-calculated based on the above, and shouldn't be specified manually - `artifact_final_version_reversioned` is optional, and can force the final version of the artifact (specific for the `base-files` case) - artifacts that need special handling for reversioning can add function names to `artifact_debs_reversion_functions` array (`base-files` and `bsp-cli` cases) - artifacts `prepare_version()` should set `artifact_version`, but _never_ include it in other variables; `artifact_version` is now changed by framework after `prepare_version()` returns - no longer use/refer/mention `${REVISION}` when building packages. All packages should be `${REVISION}`-agnostic. - `${REVISION}` (actually, `artifact_final_version_reversioned`) will be automatically swapped in the `control` file during reversioning - `fakeroot_dpkg_deb_build()` now takes exactly two arguments: the directory to pack, and the deb ID (key of `artifact_map_packages` dict); add this change in all the artifact's code for this - `obtain_complete_artifact()`: - automatically adds `-Rxxxx` "revisioning-hash" to `artifact_version`, by hashing the revisioning functions and any `artifact_debs_reversion_functions` set - calculates more complex subdirectory paths for both the `output/packages-hashed` and `output/debs`/`output/debs-beta` directories - with the new subdirectories we can be sure a re-version is already done correctly and can skip it (eg, for partial `download-debs` re-runs) - in the future we can automatically clean/remove old versions that are no longer relevant based on the dir structure - exports a lot more information to JSON, including the new subdirectory paths - comment-out code that implemented `skip_unpack_if_found_in_caches`, I'm very unsure why we had this in the first place - `obtain_artifact_from_remote_cache()` - for `deb` type artifacts, OCI won't preserve the subdirectory structure, so move downloaded files to the correct subdirectory manually - this is not needed for `deb-tar`, since that can preserve the dir structure itself - introduce `artifacts-reversion.sh` and its main function `artifact_reversion_for_deployment()` - this has the logic for reversioning .deb's, by `ar`-unpacking them, changing `control.tar` (and possibly `data.tar`), handling `.xz` compression, etc. - also handles hashing those functions, for consistency. Any changes in reversioning code actually change the artifact itself so we're not caught by surprise - by default, it changes `control` file only: - replace `Version:` (which is the hash-version originally) with `artifact_final_version_reversioned` (which is mostly just `${REVISION}`) - add a custom field `Armbian-Original-Hash:` with the original hash-version - `artifact_reversion_for_deployment()` is called by - new CLI wrapper `cli_obtain_complete_artifact()`, used for CLI building of specific artifact, but also for `download-artifact` - `build_artifact_for_image()` used during image build - `armbian-bsp-cli-deb.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `artifact-armbian-base-files.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `kernel`: - add some custom fields to `DEBIAN/control`: - `Armbian-Kernel-Version:` / `Armbian-Kernel-Version-Family:` (for future use: cleanup of usage of `Source: ` field which should be removed) - declutter the `Description:` field, moving long description out of the first line - obtain `IMAGE_INSTALLED_KERNEL_VERSION` from the reversioned deb (this is still a hack and has not been fixed) - `uboot`: - declutter the `Description:` field, moving long description out of the first line - use the reversioned .deb when deploying u-boot to the image - `main_default_build_packages()` now stores reversioned values and complete paths to reversioned .deb's - `list_installed_packages()` now compares custom field `Armbian-Original-Hash: `, and not the `Version:` to make sure debs in the image are the ones we want - `install_artifact_deb_chroot()` is a new wrapper around `install_deb_chroot()` for easy handling of reversioned debs - use it everywhere `install_deb_chroot()` was used in `distro-agnostic.sh` and `distro-specific.sh`
129 lines
4.6 KiB
Bash
129 lines
4.6 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
|
#
|
|
# This file is a part of the Armbian Build Framework
|
|
# https://github.com/armbian/build/
|
|
function determine_artifacts_to_build_for_image() {
|
|
# outer scope: declare -a artifacts_to_build=()
|
|
if [[ "${BOOTCONFIG}" != "none" ]]; then
|
|
artifacts_to_build+=("uboot")
|
|
fi
|
|
if [[ -n $KERNELSOURCE ]]; then
|
|
artifacts_to_build+=("kernel")
|
|
fi
|
|
|
|
if [[ "${INSTALL_ARMBIAN_FIRMWARE:-yes}" == "yes" ]]; then
|
|
if [[ ${BOARD_FIRMWARE_INSTALL:-""} == "-full" ]]; then
|
|
artifacts_to_build+=("full_firmware")
|
|
else
|
|
artifacts_to_build+=("firmware")
|
|
fi
|
|
fi
|
|
|
|
# Userspace, RELEASE+ARCH specific, replaces the original distro's base-files
|
|
# This is always built, but only installed if KEEP_ORIGINAL_OS_RELEASE!=yes.
|
|
artifacts_to_build+=("armbian-base-files")
|
|
|
|
if [[ "${DISTRIBUTION}" == "Ubuntu" ]]; then
|
|
artifacts_to_build+=("fake_ubuntu_advantage_tools")
|
|
fi
|
|
|
|
if [[ "${PACKAGE_LIST_RM}" != *armbian-config* ]]; then
|
|
if [[ $BUILD_MINIMAL != yes ]]; then
|
|
artifacts_to_build+=("armbian-config")
|
|
fi
|
|
fi
|
|
|
|
if [[ "${PACKAGE_LIST_RM}" != *armbian-zsh* ]]; then
|
|
if [[ $BUILD_MINIMAL != yes ]]; then
|
|
artifacts_to_build+=("armbian-zsh")
|
|
fi
|
|
fi
|
|
|
|
if [[ $PLYMOUTH == yes ]]; then
|
|
artifacts_to_build+=("armbian-plymouth-theme")
|
|
fi
|
|
|
|
# Userspace, BOARD+BRANCH specific (not RELEASE)
|
|
artifacts_to_build+=("armbian-bsp-cli")
|
|
|
|
# Userspace, RELEASE-specific artifacts.
|
|
if [[ -n "${RELEASE}" ]]; then
|
|
if [[ -n "${DESKTOP_ENVIRONMENT}" ]]; then
|
|
artifacts_to_build+=("armbian-desktop")
|
|
artifacts_to_build+=("armbian-bsp-desktop")
|
|
fi
|
|
fi
|
|
|
|
# If we're only dumping the config, include the rootfs artifact.
|
|
# In a "real" build, this artifact is built/consumed by get_or_create_rootfs_cache_chroot_sdcard(), not here.
|
|
if [[ "${CONFIG_DEFS_ONLY}" == "yes" ]]; then
|
|
artifacts_to_build+=("rootfs")
|
|
fi
|
|
|
|
}
|
|
|
|
function main_default_build_packages() {
|
|
# early cleaning for sources, since fetch_and_build_host_tools() uses it.
|
|
if [[ "${CLEAN_LEVEL}" == *sources* ]]; then
|
|
LOG_SECTION="cleaning_early_sources" do_with_logging general_cleaning "sources"
|
|
fi
|
|
|
|
# ignore updates help on building all images - for internal purposes
|
|
if [[ "${IGNORE_UPDATES}" != "yes" ]]; then
|
|
LOG_SECTION="clean_deprecated_mountpoints" do_with_logging clean_deprecated_mountpoints
|
|
|
|
for cleaning_fragment in $(tr ',' ' ' <<< "${CLEAN_LEVEL}"); do
|
|
if [[ $cleaning_fragment != sources ]] && [[ $cleaning_fragment != none ]] && [[ $cleaning_fragment != make* ]]; then
|
|
LOG_SECTION="cleaning_${cleaning_fragment}" do_with_logging general_cleaning "${cleaning_fragment}"
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# determine which artifacts to build.
|
|
declare -a artifacts_to_build=()
|
|
determine_artifacts_to_build_for_image
|
|
display_alert "Artifacts to build:" "${artifacts_to_build[*]}" "debug"
|
|
|
|
# For each artifact, try to obtain them from the local cache, remote cache, or build them.
|
|
# Store info about all artifacts in the process, for later use (eg during package installation in distro-agnostic).
|
|
declare -g -a image_artifacts_all=()
|
|
declare -g -A image_artifacts_packages=()
|
|
declare -g -A image_artifacts_packages_version=()
|
|
declare -g -A image_artifacts_packages_version_reversioned=()
|
|
declare -g -A image_artifacts_debs=()
|
|
declare -g -A image_artifacts_debs_reversioned=()
|
|
declare one_artifact one_artifact_package
|
|
for one_artifact in "${artifacts_to_build[@]}"; do
|
|
declare -A artifact_map_packages=()
|
|
declare -A artifact_map_debs=()
|
|
declare -A artifact_map_debs_reversioned=()
|
|
declare artifact_version
|
|
|
|
WHAT="${one_artifact}" build_artifact_for_image
|
|
|
|
# store info about this artifact's debs and packages
|
|
for one_artifact_package in "${!artifact_map_packages[@]}"; do
|
|
image_artifacts_all+=("${one_artifact_package}")
|
|
image_artifacts_packages["${one_artifact_package}"]="${artifact_map_packages[${one_artifact_package}]}"
|
|
image_artifacts_debs["${one_artifact_package}"]="${artifact_map_debs[${one_artifact_package}]}"
|
|
image_artifacts_debs_reversioned["${one_artifact_package}"]="${artifact_map_debs_reversioned[${one_artifact_package}]}"
|
|
image_artifacts_packages_version["${artifact_map_packages[${one_artifact_package}]}"]="${artifact_version}"
|
|
image_artifacts_packages_version_reversioned["${artifact_map_packages[${one_artifact_package}]}"]="${artifact_final_version_reversioned}"
|
|
done
|
|
done
|
|
|
|
debug_dict image_artifacts_packages
|
|
debug_dict image_artifacts_debs
|
|
debug_dict image_artifacts_packages_version
|
|
|
|
overlayfs_wrapper "cleanup"
|
|
reset_uid_owner "${DEB_STORAGE}"
|
|
|
|
# At this point, the WORKDIR should be clean. Add debug info.
|
|
debug_tmpfs_show_usage "AFTER ALL PKGS BUILT"
|
|
}
|