mirror of
https://github.com/armbian/build.git
synced 2025-09-10 16:21:37 +02:00
> tl-dr: > - maximize OCI cache hit ratio across nightlies/releases/PRs/etc; > - publish simple `Version:`'s that don't include a crazy hash in repo and images > - introduce `output/packages-hashed` directory > - radically change the `output/debs` directory structure - simplify artifact's `prepare_version()` method for `deb` and `deb-tar` artifacts: - `artifact_base_dir` and `artifact_final_file` will now be auto-calculated; thus removed from each artifact (except `rootfs`) - `artifact_deb_repo` ("global", "jammy", "bookworm") is now required; "global" means common across all RELEASES - `artifact_deb_arch` is now required, "all" is arch-independent, otherwise use `${ARCH}` - `artifact_map_debs` is now auto-calculated based on the above, and shouldn't be specified manually - `artifact_final_version_reversioned` is optional, and can force the final version of the artifact (specific for the `base-files` case) - artifacts that need special handling for reversioning can add function names to `artifact_debs_reversion_functions` array (`base-files` and `bsp-cli` cases) - artifacts `prepare_version()` should set `artifact_version`, but _never_ include it in other variables; `artifact_version` is now changed by framework after `prepare_version()` returns - no longer use/refer/mention `${REVISION}` when building packages. All packages should be `${REVISION}`-agnostic. - `${REVISION}` (actually, `artifact_final_version_reversioned`) will be automatically swapped in the `control` file during reversioning - `fakeroot_dpkg_deb_build()` now takes exactly two arguments: the directory to pack, and the deb ID (key of `artifact_map_packages` dict); add this change in all the artifact's code for this - `obtain_complete_artifact()`: - automatically adds `-Rxxxx` "revisioning-hash" to `artifact_version`, by hashing the revisioning functions and any `artifact_debs_reversion_functions` set - calculates more complex subdirectory paths for both the `output/packages-hashed` and `output/debs`/`output/debs-beta` directories - with the new subdirectories we can be sure a re-version is already done correctly and can skip it (eg, for partial `download-debs` re-runs) - in the future we can automatically clean/remove old versions that are no longer relevant based on the dir structure - exports a lot more information to JSON, including the new subdirectory paths - comment-out code that implemented `skip_unpack_if_found_in_caches`, I'm very unsure why we had this in the first place - `obtain_artifact_from_remote_cache()` - for `deb` type artifacts, OCI won't preserve the subdirectory structure, so move downloaded files to the correct subdirectory manually - this is not needed for `deb-tar`, since that can preserve the dir structure itself - introduce `artifacts-reversion.sh` and its main function `artifact_reversion_for_deployment()` - this has the logic for reversioning .deb's, by `ar`-unpacking them, changing `control.tar` (and possibly `data.tar`), handling `.xz` compression, etc. - also handles hashing those functions, for consistency. Any changes in reversioning code actually change the artifact itself so we're not caught by surprise - by default, it changes `control` file only: - replace `Version:` (which is the hash-version originally) with `artifact_final_version_reversioned` (which is mostly just `${REVISION}`) - add a custom field `Armbian-Original-Hash:` with the original hash-version - `artifact_reversion_for_deployment()` is called by - new CLI wrapper `cli_obtain_complete_artifact()`, used for CLI building of specific artifact, but also for `download-artifact` - `build_artifact_for_image()` used during image build - `armbian-bsp-cli-deb.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `artifact-armbian-base-files.sh`: move `${REVISION}` related stuff from the main package build to new reversioning functions. - `kernel`: - add some custom fields to `DEBIAN/control`: - `Armbian-Kernel-Version:` / `Armbian-Kernel-Version-Family:` (for future use: cleanup of usage of `Source: ` field which should be removed) - declutter the `Description:` field, moving long description out of the first line - obtain `IMAGE_INSTALLED_KERNEL_VERSION` from the reversioned deb (this is still a hack and has not been fixed) - `uboot`: - declutter the `Description:` field, moving long description out of the first line - use the reversioned .deb when deploying u-boot to the image - `main_default_build_packages()` now stores reversioned values and complete paths to reversioned .deb's - `list_installed_packages()` now compares custom field `Armbian-Original-Hash: `, and not the `Version:` to make sure debs in the image are the ones we want - `install_artifact_deb_chroot()` is a new wrapper around `install_deb_chroot()` for easy handling of reversioned debs - use it everywhere `install_deb_chroot()` was used in `distro-agnostic.sh` and `distro-specific.sh`
185 lines
8.6 KiB
Bash
185 lines
8.6 KiB
Bash
#!/usr/bin/env bash
|
|
#
|
|
# SPDX-License-Identifier: GPL-2.0
|
|
#
|
|
# Copyright (c) 2013-2023 Igor Pecovnik, igor@armbian.com
|
|
#
|
|
# This file is a part of the Armbian Build Framework
|
|
# https://github.com/armbian/build/
|
|
|
|
# for RAW deb building. does a bunch of magic to "DEBIAN" directory. Arguments are the open package directory and the artifact_deb_id
|
|
function fakeroot_dpkg_deb_build() {
|
|
# check artifact_name and artifact_version is set otherwise exit_with_error
|
|
[[ -z "${artifact_name}" ]] && exit_with_error "fakeroot_dpkg_deb_build: artifact_name is not set"
|
|
[[ -z "${artifact_version}" ]] && exit_with_error "fakeroot_dpkg_deb_build: artifact_version is not set"
|
|
[[ -z "${artifact_deb_repo}" ]] && exit_with_error "fakeroot_dpkg_deb_build: artifact_deb_repo is not set"
|
|
|
|
display_alert "Building .deb package" "${artifact_name}: $*" "debug"
|
|
|
|
declare package_directory="${1}"
|
|
declare artifact_deb_id="${2}"
|
|
|
|
if [[ ! -d "${package_directory}" ]]; then
|
|
exit_with_error "fakeroot_dpkg_deb_build: can't find source package directory: ${package_directory}"
|
|
fi
|
|
|
|
# Check artifact_deb_id is set and not empty
|
|
if [[ -z "${artifact_deb_id}" ]]; then
|
|
exit_with_error "fakeroot_dpkg_deb_build: artifact_deb_id (2nd parameter) is not set, called with package_directory: '${package_directory}'"
|
|
fi
|
|
|
|
# Obtain from the globals
|
|
declare -A -g artifact_map_packages
|
|
declare -A -g artifact_map_debs
|
|
debug_dict artifact_map_packages
|
|
debug_dict artifact_map_debs
|
|
declare artifact_deb_package="${artifact_map_packages[${artifact_deb_id}]}"
|
|
declare artifact_deb_rel_path="${artifact_map_debs[${artifact_deb_id}]}"
|
|
|
|
# If either is empty, bomb
|
|
if [[ -z "${artifact_deb_package}" ]]; then
|
|
exit_with_error "fakeroot_dpkg_deb_build: artifact_deb_package (artifact_map_packages) is not set or found for '${artifact_deb_id}'"
|
|
fi
|
|
if [[ -z "${artifact_deb_rel_path}" ]]; then
|
|
exit_with_error "fakeroot_dpkg_deb_build: artifact_deb_rel_path (artifact_map_debs) is not set or found for '${artifact_deb_id}'"
|
|
fi
|
|
|
|
# Show the total human size of the source package directory.
|
|
display_alert "Source package size" "${package_directory}: $(du -sh "${package_directory}" | cut -f1)" "debug"
|
|
|
|
# Lets fix all packages with Installed-Size:
|
|
# get the size of the package in bytes
|
|
declare -i pkg_size_bytes
|
|
pkg_size_bytes=$(du -s -b "${package_directory}" | cut -f1)
|
|
# edit DEBIAN/control, removed any Installed-Size: line
|
|
sed -i '/^Installed-Size:/d' "${package_directory}/DEBIAN/control"
|
|
# add the new Installed-Size: line. The disk space is given as the integer value of the estimated installed size in bytes, divided by 1024 and rounded up.
|
|
declare -i installed_size
|
|
installed_size=$(((pkg_size_bytes + 1023) / 1024))
|
|
echo "Installed-Size: ${installed_size}" >> "${package_directory}/DEBIAN/control"
|
|
|
|
# Lets create DEBIAN/md5sums, for all the files in ${package_directory}. Do not include the paths in the md5sums file. Don't include the DEBIAN/* files.
|
|
find "${package_directory}" -type f -not -path "${package_directory}/DEBIAN/*" -print0 | xargs -0 md5sum | sed "s|${package_directory}/||g" > "${package_directory}/DEBIAN/md5sums"
|
|
|
|
# Parse the DEBIAN/control and get the real package name...
|
|
declare control_package_name
|
|
control_package_name=$(grep -E "^Package:" "${package_directory}/DEBIAN/control" | cut -d' ' -f2)
|
|
|
|
# generate minimal DEBIAN/changelog
|
|
cat <<- EOF > "${package_directory}"/DEBIAN/changelog
|
|
${control_package_name} (${artifact_version}) ${artifact_deb_repo}; urgency=low
|
|
|
|
* Initial changelog entry for ${control_package_name} package hash ${artifact_version}
|
|
|
|
-- $MAINTAINER <$MAINTAINERMAIL> $(date -R)
|
|
EOF
|
|
|
|
# Also a usr/share/doc/${control_package_name}/changelog.gz
|
|
mkdir -p "${package_directory}/usr/share/doc/${control_package_name}"
|
|
gzip -9 -c "${package_directory}/DEBIAN/changelog" > "${package_directory}/usr/share/doc/${control_package_name}/changelog.gz"
|
|
|
|
# find the DEBIAN scripts (postinst, prerm, etc) and run shellcheck on them.
|
|
dpkg_deb_run_shellcheck_on_scripts "${package_directory}"
|
|
|
|
# Debug, dump the generated postrm/preinst/postinst
|
|
if [[ "${SHOW_DEBUG}" == "yes" || "${SHOW_DEBIAN}" == "yes" ]]; then
|
|
# Dump the CONTROL file to the log
|
|
run_tool_batcat --file-name "${artifact_name}/DEBIAN/control" "${package_directory}/DEBIAN/control"
|
|
|
|
if [[ -f "${package_directory}/DEBIAN/changelog" ]]; then
|
|
run_tool_batcat --file-name "${artifact_name}/DEBIAN/changelog" "${package_directory}/DEBIAN/changelog"
|
|
fi
|
|
|
|
if [[ -f "${package_directory}/DEBIAN/postrm" ]]; then
|
|
run_tool_batcat --file-name "${artifact_name}/DEBIAN/postrm.sh" "${package_directory}/DEBIAN/postrm"
|
|
fi
|
|
|
|
if [[ -f "${package_directory}/DEBIAN/preinst" ]]; then
|
|
run_tool_batcat --file-name "${artifact_name}/DEBIAN/preinst.sh" "${package_directory}/DEBIAN/preinst"
|
|
fi
|
|
|
|
if [[ -f "${package_directory}/DEBIAN/postinst" ]]; then
|
|
run_tool_batcat --file-name "${artifact_name}/DEBIAN/postinst.sh" "${package_directory}/DEBIAN/postinst"
|
|
fi
|
|
|
|
run_tool_batcat --file-name "${artifact_name}/DEBIAN/md5sums" "${package_directory}/DEBIAN/md5sums"
|
|
fi
|
|
|
|
declare deb_final_filename="${PACKAGES_HASHED_STORAGE}/${artifact_deb_rel_path}"
|
|
declare deb_final_dir
|
|
deb_final_dir=$(dirname "${deb_final_filename}")
|
|
|
|
mkdir -p "${deb_final_dir}"
|
|
run_host_command_logged_raw fakeroot dpkg-deb -b "-Z${DEB_COMPRESS}" "${package_directory}" "${deb_final_filename}"
|
|
}
|
|
|
|
function dpkg_deb_run_shellcheck_on_scripts() {
|
|
declare pkg_dir="$1"
|
|
[[ -z "${pkg_dir}" ]] && exit_with_error "dpkg_deb_run_shellcheck_on_scripts: no package directory specified"
|
|
[[ ! -d "${pkg_dir}" ]] && exit_with_error "dpkg_deb_run_shellcheck_on_scripts: pkg dir '${pkg_dir}' doesn't exist"
|
|
[[ ! -d "${pkg_dir}/DEBIAN" ]] && exit_with_error "dpkg_deb_run_shellcheck_on_scripts: pkg dir '${pkg_dir}'/DEBIAN doesn't exist"
|
|
|
|
# parse the DEBIAN/control script to find the name of the package
|
|
declare pkg_name
|
|
pkg_name=$(grep -E "^Package:" "${pkg_dir}/DEBIAN/control" | cut -d: -f2 | tr -d '[:space:]')
|
|
[[ -z "${pkg_name}" ]] && exit_with_error "dpkg_deb_run_shellcheck_on_scripts: can't find package name in ${pkg_dir}/DEBIAN/control"
|
|
|
|
# use "find" to find executable files in the package directory
|
|
declare -a executables=($(find "${pkg_dir}/DEBIAN" -type f -executable || true))
|
|
|
|
# @TODO: also include other, executable, shells scripts found in the dir; e.g. /usr/bin, /usr/sbin, etc.
|
|
|
|
# if more than zero items in array...
|
|
if [[ ${#executables[@]} -gt 0 ]]; then
|
|
display_alert "Running shellcheck on package scripts" "${executables[*]}" "debug"
|
|
if shellcheck_debian_control_scripts "${executables[@]}"; then
|
|
display_alert "shellcheck found no problems in package scripts" "shellchecked ${#executables[@]} scripts in '${pkg_name}'" "info"
|
|
else
|
|
display_alert "shellcheck found problems in package scripts; see above" "shellcheck failed for '${pkg_name}'" "wrn"
|
|
fi
|
|
else
|
|
display_alert "shellcheck found no problems in package scripts" "no scripts found for '${pkg_name}'" "info"
|
|
fi
|
|
}
|
|
|
|
function artifact_package_hook_helper_board_side_functions() {
|
|
declare script="${1}"
|
|
shift
|
|
# each remaining arg is a function name; for each, run 'declare -f', remove the first, second and last line (function declaration, open brace, close brac, and add it to contents.
|
|
declare -a functions=("$@")
|
|
declare contents=""
|
|
declare newline=$'\n'
|
|
for function in "${functions[@]}"; do
|
|
contents+="${newline}## begin contents of '${function}'${newline}"
|
|
contents+="$(declare -f "${function}" | sed -e '2d' -e '1d' -e '$d')"
|
|
contents+="${newline}## end contents of '${function}'${newline}"
|
|
done
|
|
generic_artifact_package_hook_helper "${script}" "${contents}"
|
|
}
|
|
|
|
function generic_artifact_package_hook_helper() {
|
|
# check '$destination' is set
|
|
[[ -z "${destination}" ]] && exit_with_error "generic_artifact_package_hook_helper: destination not set"
|
|
declare script="${1}"
|
|
declare contents="${2}"
|
|
declare package_DEBIAN_dir="${destination}"/DEBIAN
|
|
[[ ! -d "${package_DEBIAN_dir}" ]] && exit_with_error "generic_artifact_package_hook_helper: package DEBIAN dir '${package_DEBIAN_dir}' doesn't exist"
|
|
|
|
cat >> "${package_DEBIAN_dir}/${script}" <<- EOT
|
|
#!/bin/bash
|
|
echo "Armbian '${artifact_name:?}' for '${artifact_version:?}': '${script}' starting."
|
|
set +e # NO ERROR CONTROL, for compatibility with legacy Armbian scripts.
|
|
#set -e # Debugging
|
|
|
|
$(echo "${contents}")
|
|
|
|
set +x # Disable debugging
|
|
echo "Armbian '${artifact_name:?}' for '${artifact_version:?}': '${script}' finishing."
|
|
true
|
|
EOT
|
|
chmod 755 "${package_DEBIAN_dir}/${script}"
|
|
|
|
# produce log asset for script (@TODO: batcat?)
|
|
LOG_ASSET="deb-${artifact_name:?}-${script}.sh" do_with_log_asset run_host_command_logged cat "${package_DEBIAN_dir}/${script}"
|
|
}
|