diff --git a/pkg_auto/Makefile b/pkg_auto/Makefile index b2795d456a..ae9a4fa077 100644 --- a/pkg_auto/Makefile +++ b/pkg_auto/Makefile @@ -1,2 +1,12 @@ +# ignoring SC2034 (VARIABLE_NAME appears unused), too many false +# positives +# +# ignoring SC2178 (Variable was used as an array but is now assigned a +# string.) - buggy for local variables, happens very often for +# references +SHELLCHECK_OPTS := -e SC2034 -e SC2178 + +all: shellcheck + shellcheck: - docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh + docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --source-path=SCRIPTDIR/impl/for-shellcheck --external-sources --check-sourced $(SHELLCHECK_OPTS) *.sh impl/*.sh diff --git a/pkg_auto/download_sdk_and_listings.sh b/pkg_auto/download_sdk_and_listings.sh index 11b06f79ef..8e48039248 100755 --- a/pkg_auto/download_sdk_and_listings.sh +++ b/pkg_auto/download_sdk_and_listings.sh @@ -148,9 +148,9 @@ function download() { } if [[ -n ${SCRIPTS} ]]; then - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") fi @@ -159,19 +159,16 @@ ver_dash="${VERSION_ID}${BUILD_ID:+-}${BUILD_ID}" exts=(zst bz2 gz) -# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds zst_cmds=( zstd ) -# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds bz2_cmds=( lbunzip2 pbunzip2 bunzip2 ) -# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds gz_cmds=( unpigz gunzip diff --git a/pkg_auto/generate_config.sh b/pkg_auto/generate_config.sh index ce354be9c3..7d390197af 100755 --- a/pkg_auto/generate_config.sh +++ b/pkg_auto/generate_config.sh @@ -9,9 +9,8 @@ set -euo pipefail ## -a: aux directory ## -d: debug package - list many times ## -h: this help -## -i: SDK image override in form of ${arch}:${name}, the name part -## should be a valid docker image with an optional tag -## -ip: add SDK image overrides using flatcar-packages images +## -i: override SDK image, it should be a valid docker image with an +## optional tag ## -n: new base ## -o: old base ## -r: reports directory @@ -27,24 +26,19 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" -# shellcheck disable=SC2034 # used by name below gc_aux_directory='' -# shellcheck disable=SC2034 # used by name below gc_new_base='' -# shellcheck disable=SC2034 # used by name below gc_old_base='' -# shellcheck disable=SC2034 # used by name below gc_reports_directory='' -# shellcheck disable=SC2034 # used by name below gc_scripts_directory='' -# shellcheck disable=SC2034 # used by name below gc_cleanup_opts='' -# gc_${arch}_sdk_img are declared on demand +gc_image_override='' gc_debug_packages=() declare -A opt_map opt_map=( ['-a']=gc_aux_directory + ['-i']=gc_image_override ['-n']=gc_new_base ['-o']=gc_old_base ['-r']=gc_reports_directory @@ -68,36 +62,6 @@ while [[ ${#} -gt 0 ]]; do print_help exit 0 ;; - -i) - if [[ -z ${2:-} ]]; then - fail 'missing value for -i' - fi - arch=${2%%:*} - image_name=${2#*:} - var_name="gc_${arch}_sdk_img" - unset arch - # shellcheck disable=SC2178 # shellcheck does not grok refs - declare -n ref="${var_name}" - unset var_name - # shellcheck disable=SC2178 # shellcheck does not grok refs - ref=${image_name} - unset image_name - unset -n ref - shift 2 - ;; - -ip) - for arch in "${gc_arches[@]}"; do - var_name="gc_${arch}_sdk_img" - # shellcheck disable=SC2178 # shellcheck does not grok refs - declare -n ref="${var_name}" - unset var_name - # shellcheck disable=SC2178 # shellcheck does not grok refs - ref="flatcar-packages-${arch}" - unset -n ref - done - unset arch - shift - ;; --) shift break @@ -110,9 +74,7 @@ while [[ ${#} -gt 0 ]]; do if [[ -z ${2:-} ]]; then fail 'missing value for -w' fi - # shellcheck disable=SC2178 # shellcheck does not grok refs declare -n ref="${var_name}" - # shellcheck disable=SC2178 # shellcheck does not grok refs ref=${2} unset -n ref unset var_name @@ -134,15 +96,10 @@ pairs=( 'old-base' gc_old_base 'new-base' gc_new_base 'cleanups' gc_cleanup_opts + 'sdk-image-override' gc_image_override + 'debug-packages' gc_debug_packages_csv ) -for arch in "${gc_arches[@]}"; do - pairs+=( "${arch}-sdk-img" "gc_${arch}_sdk_img" ) -done - -pairs+=( 'debug-packages' gc_debug_packages_csv ) - - if [[ ${#} -ne 1 ]]; then fail 'expected one positional parameters: a path for the config' fi @@ -157,7 +114,6 @@ config=${1}; shift name=${pairs["${name_idx}"]} opt_idx=$((opt_idx + 2)) name_idx=$((name_idx + 2)) - # shellcheck disable=SC2178 # shellcheck does not grok refs declare -n ref="${name}" if [[ -n ${ref:-} ]]; then printf '%s: %s\n' "${opt}" "${ref}" diff --git a/pkg_auto/impl/debug.sh b/pkg_auto/impl/debug.sh new file mode 100644 index 0000000000..47dcfb9425 --- /dev/null +++ b/pkg_auto/impl/debug.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +if [[ -z ${__DEBUG_SH_INCLUDED__:-} ]]; then +__DEBUG_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Adds package names for which debugging output may be enabled. +function pkg_debug_add() { + if [[ ${#} -eq 0 ]]; then + return + fi + if ! declare -p __D_DEBUG_PACKAGES >/dev/null 2>&1; then + declare -gA __D_DEBUG_PACKAGES=() + fi + local pkg + for pkg; do + __D_DEBUG_PACKAGES["${pkg}"]=x + done +} + +# Resets debugging state to zero (no packages to debug, debug output +# disabled). +function pkg_debug_reset() { + unset __D_DEBUG_PACKAGES __D_DEBUG +} + +# Enables debugging output when specific packages are processed. +# +# Params: +# +# @ - package names to enable debugging for +function pkg_debug_enable() { + local -A pkg_set=() + local -a vals=() + local pkg + + if ! declare -p __D_DEBUG_PACKAGES >/dev/null 2>&1; then + return 0 + fi + + for pkg; do + if [[ -n ${pkg_set["${pkg}"]:-} ]]; then + continue + fi + pkg_set["${pkg}"]=x + if [[ -n ${__D_DEBUG_PACKAGES["${pkg}"]:-} ]]; then + vals+=( "${pkg}" ) + fi + done + if [[ ${#vals[@]} -gt 0 ]]; then + declare -g __D_DEBUG + join_by __D_DEBUG ',' "${vals[@]}" + fi +} + +# Get a list of package names for which debugging output may be +# enabled. +function pkg_debug_packages() { + local -n pkg_names_ref=${1}; shift + + if ! declare -p __D_DEBUG_PACKAGES >/dev/null 2>&1; then + pkg_names_ref=() + else + pkg_names_ref=( "${!__D_DEBUG_PACKAGES[@]}" ) + fi +} + +# Checks if debugging output is enabled. Returns true if yes, +# otherwise false. +function pkg_debug_enabled() { + local -i ret=0 + [[ -n ${__D_DEBUG:-} ]] || ret=1 + return ${ret} +} + +# Disables debugging output. +function pkg_debug_disable() { + unset __D_DEBUG +} + +# Prints passed parameters if debugging is enabled. +# +# Params: +# +# @ - parameters to print +function pkg_debug() { + if [[ -n ${__D_DEBUG:-} ]]; then + pkg_debug_print_c "${__D_DEBUG}" "${@}" + fi +} + +# Prints passed lines if debugging is enabled. +# +# Params: +# +# @ - lines to print +function pkg_debug_lines() { + if [[ -n ${__D_DEBUG:-} ]]; then + pkg_debug_print_lines_c "${__D_DEBUG}" "${@}" + fi +} + +# Prints passed parameters unconditionally with debug +# formatting. Useful for more complicated debugging logic using +# pkg_debug_enabled. +# +# if pkg_debug_enabled; then +# pkg_debug_print 'debug message' +# fi +# +# Params: +# +# @ - parameters to print +function pkg_debug_print() { + if [[ -z ${__D_DEBUG:+notempty} ]]; then + info "bad use of pkg_debug_print, __D_DEBUG is unset" + debug_stacktrace + fail '' + fi + pkg_debug_print_c "${__D_DEBUG}" "${@}" +} + +# Prints passed lines unconditionally with debug formatting. Useful +# for more complicated debugging logic using pkg_debug_enabled. +# +# if pkg_debug_enabled; then +# pkg_debug_print_lines 'debug' 'message' +# fi +# +# Params: +# +# @ - lines to print +function pkg_debug_print_lines() { + if [[ -z ${__D_DEBUG:+notempty} ]]; then + info "bad use of pkg_debug_print_lines, __D_DEBUG is unset" + debug_stacktrace + fail '' + fi + pkg_debug_print_lines_c "${__D_DEBUG}" "${@}" +} + +# Prints passed parameters unconditionally with custom debug +# formatting. Useful for either more complicated debugging logic using +# pkg_debug_packages or for non-package-specific debugging situations. +# +# local -a dpkgs=() +# pkg_debug_packages dpkgs +# local pkg +# for pkg in "${dpkgs[@]}"; do pkg_debug_print_c "${pkg}" 'debug message' +# +# Params: +# +# 1 - comma-separated package names +# @ - parameters to print +function pkg_debug_print_c() { + local d_debug=${1} + info "DEBUG(${d_debug}): ${*}" +} + +# Prints passed lines unconditionally with custom debug formatting. +# Useful for either more complicated debugging logic using +# pkg_debug_packages or for non-package-specific debugging situations. +# +# local -a dpkgs=() lines=( 'debug' 'message; ) +# pkg_debug_packages dpkgs +# local pkg +# for pkg in "${dpkgs[@]}"; do pkg_debug_print_lines_c "${pkg}" "${lines[@]}" +# +# Params: +# +# 1 - comma-separated package names +# @ - lines to print +function pkg_debug_print_lines_c() { + local d_debug=${1} + info_lines "${@/#/"DEBUG(${d_debug}): "}" +} + +# Prints current stacktrace. +function debug_stacktrace() { + local -i idx=0 last_idx=${#FUNCNAME[@]} + + ((last_idx--)) + while [[ idx -lt last_idx ]]; do + info "at ${FUNCNAME[idx]}, invoked by ${FUNCNAME[$((idx + 1))]} in ${BASH_SOURCE[idx + 1]}:${BASH_LINENO[idx]}" + ((++idx)) + done +} + +fi diff --git a/pkg_auto/impl/for-shellcheck/README.md b/pkg_auto/impl/for-shellcheck/README.md new file mode 100644 index 0000000000..223cd8c9cf --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/README.md @@ -0,0 +1,2 @@ +Shellcheck is being pointed here when processing source bash +directives for generated files. diff --git a/pkg_auto/impl/for-shellcheck/globals b/pkg_auto/impl/for-shellcheck/globals new file mode 100644 index 0000000000..f2060a0238 --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/globals @@ -0,0 +1,68 @@ +local -a GIT_ENV_VARS ARCHES WHICH REPORTS +local SDK_PKGS BOARD_PKGS +local SYNC_SCRIPT PKG_LIST_SORT_SCRIPT + +GIT_ENV_VARS=( + GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} +) + +SYNC_SCRIPT='/path/to/sync_with_gentoo.sh' +PKG_LIST_SORT_SCRIPT='/path/to/sort_packages_list.py' + +ARCHES=( 'amd64' 'arm64' ) +WHICH=('old' 'new') +SDK_PKGS='sdk-pkgs' +BOARD_PKGS='board-pkgs' +REPORTS=( "${SDK_PKGS}" "${BOARD_PKGS}" ) + +local SCRIPTS OLD_STATE NEW_STATE OLD_STATE_BRANCH NEW_STATE_BRANCH +local PORTAGE_STABLE_SUFFIX OLD_PORTAGE_STABLE NEW_PORTAGE_STABLE REPORTS_DIR +local NEW_STATE_PACKAGES_LIST AUX_DIR +local COREOS_OVERLAY_SUFFIX OLD_COREOS_OVERLAY NEW_COREOS_OVERLAY + +SCRIPTS='/path/to/scripts' +OLD_STATE='/path/to/old_state' +NEW_STATE='/path/to/new_state' +OLD_STATE_BRANCH='old-state' +NEW_STATE_BRANCH='new-state' +PORTAGE_STABLE_SUFFIX='sdk_container/src/third_party/portage-stable' +OLD_PORTAGE_STABLE='/path/to/old_state/portage-stable' +NEW_PORTAGE_STABLE='/path/to/new_state/portage-stable' +REPORTS_DIR='/path/to/reports' + +COREOS_OVERLAY_SUFFIX='sdk_container/src/third_party/coreos-overlay' +OLD_COREOS_OVERLAY='/path/to/old_state/coreos-overlay' +NEW_COREOS_OVERLAY='/path/to/new_state/coreos-overlay' + +NEW_STATE_PACKAGES_LIST="${NEW_STATE}/.github/workflows/portage-stable-packages-list" + +AUX_DIR='/path/to/aux' + +local 'SDK_IMAGE' + +SDK_IMAGE='ghcr.io/flatcar/flatcar-sdk-all:1234.5.6-build-tag' + +local -A LISTING_KINDS + +LISTING_KINDS=( + ['akamai']='oem-akamai_packages.txt' + ['ami']='oem-ami_packages.txt' + ['azure']='oem-azure_packages.txt' + ['containerd']='containerd-flatcar_packages.txt' + ['dev']='flatcar_developer_container_packages.txt' + ['digitalocean']='oem-digitalocean_packages.txt' + ['docker']='docker-flatcar_packages.txt' + ['gce']='oem-gce_packages.txt' + ['hetzner']='oem-hetzner_packages.txt' + ['kubevirt']='oem-kubevirt_packages.txt' + ['openstack']='oem-openstack_packages.txt' + ['packet']='oem-packet_packages.txt' + ['prod']='flatcar_production_image_packages.txt' + ['proxmoxve']='oem-proxmoxve_packages.txt' + ['qemu']='oem-qemu_packages.txt' + ['scaleway']='oem-scaleway_packages.txt' + ['sysext-podman']='flatcar-podman_packages.txt' + ['sysext-python']='flatcar-python_packages.txt' + ['sysext-zfs']='flatcar-zfs_packages.txt' + ['vmware']='oem-vmware_packages.txt' +) diff --git a/pkg_auto/impl/for-shellcheck/version.txt b/pkg_auto/impl/for-shellcheck/version.txt new file mode 100644 index 0000000000..2804576264 --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/version.txt @@ -0,0 +1,4 @@ +FLATCAR_VERSION=1234.5.6+build-tag +FLATCAR_VERSION_ID=1234.5.6 +FLATCAR_BUILD_ID="build-tag" +FLATCAR_SDK_VERSION=1234.5.6+build-tag diff --git a/pkg_auto/impl/gentoo_ver.sh b/pkg_auto/impl/gentoo_ver.sh index 1df280be27..19576dd1d7 100644 --- a/pkg_auto/impl/gentoo_ver.sh +++ b/pkg_auto/impl/gentoo_ver.sh @@ -12,7 +12,10 @@ __GENTOO_VER_SH_INCLUDED__=x source "$(dirname "${BASH_SOURCE[0]}")/util.sh" -VER_ERE="^([0-9]+(\.[0-9]+)*)([a-z]?)((_(alpha|beta|pre|rc|p)[0-9]*)*)(-r[0-9]+)?$" +VER_ERE_UNBOUNDED="([0-9]+(\.[0-9]+)*)([a-z]?)((_(alpha|beta|pre|rc|p)[0-9]*)*)(-r[0-9]+)?" +VER_ERE='^'"${VER_ERE_UNBOUNDED}"'$' + +PKG_ERE_UNBOUNDED="[A-Za-z0-9_][A-Za-z0-9+_.-]*/[A-Za-z0-9_][A-Za-z0-9+_-]*" # @FUNCTION: _ver_compare_int # @USAGE: diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index 13f4e456ea..ec7480dd74 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -97,77 +97,23 @@ function package_info_for_board() { # 1 - directory path function set_eo() { local dir=${1}; shift + # rest are architectures - SDK_EO="${dir}/sdk-emerge-output" - BOARD_EO="${dir}/board-emerge-output" - # shellcheck disable=SC2034 # used indirectly in cat_eo_f - SDK_EO_F="${SDK_EO}-filtered" - # shellcheck disable=SC2034 # used indirectly in cat_eo_f - BOARD_EO_F="${BOARD_EO}-filtered" - # shellcheck disable=SC2034 # used indirectly in cat_eo_w - SDK_EO_W="${SDK_EO}-warnings" - # shellcheck disable=SC2034 # used indirectly in cat_eo_w - BOARD_EO_W="${BOARD_EO}-warnings" -} + declare -g EGENCACHE_W="${dir}/egencache-warnings" + declare -g SDK_EO="${dir}/sdk-emerge-output" + declare -g SDK_EO_F="${SDK_EO}-filtered" + declare -g SDK_EO_W="${SDK_EO}-warnings" + declare -g SDK_EO_J="${SDK_EO}-junk" -# Print the contents of file, path of which is stored in a variable of -# a given name. -# -# Params: -# -# 1 - name of the variable -function cat_var() { - local var_name - var_name=${1}; shift - local -n ref="${var_name}" - - if [[ -z "${ref+isset}" ]]; then - fail "${var_name} unset" - fi - if [[ ! -e "${ref}" ]]; then - fail "${ref} does not exist" - fi - - cat "${ref}" -} - -# Print contents of the emerge output of a given kind. Kind can be -# either either sdk or board. -# -# Params: -# -# 1 - kind -function cat_eo() { - local kind - kind=${1}; shift - - cat_var "${kind^^}_EO" -} - -# Print contents of the filtered emerge output of a given kind. Kind -# can be either either sdk or board. The filtered emerge output -# contains only lines with package information. -# -# Params: -# -# 1 - kind -function cat_eo_f() { - local kind - kind=${1}; shift - cat_var "${kind^^}_EO_F" -} - -# Print contents of a file that stores warnings that were printed by -# emerge. The warnings are specific to a kind (sdk or board). -# -# Params: -# -# 1 - kind -function cat_eo_w() { - local kind - kind=${1}; shift - - cat_var "${kind^^}_EO_W" + local arch + local board_eo + for arch; do + board_eo=${dir}/${arch}-board-emerge-output + declare -g "${arch^^}_BOARD_EO=${board_eo}" + declare -g "${arch^^}_BOARD_EO_F=${board_eo}-filtered" + declare -g "${arch^^}_BOARD_EO_W=${board_eo}-warnings" + declare -g "${arch^^}_BOARD_EO_J=${board_eo}-junk" + done } # JSON output would be more verbose, but probably would not require @@ -193,7 +139,7 @@ FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${V # Filters sdk reports to get the package information. function filter_sdk_eo() { - cat_eo sdk | xgrep -e "${FULL_LINE_RE}" + cat "${SDK_EO}" | xgrep -e "${FULL_LINE_RE}" } # Filters board reports for a given arch to get the package @@ -203,11 +149,12 @@ function filter_sdk_eo() { # # 1 - architecture function filter_board_eo() { - local arch + local arch name arch=${1}; shift + name=${arch^^}_BOARD_EO # Replace ${arch}-usr in the output with a generic word BOARD. - cat_eo board | \ + cat "${!name}" | \ xgrep -e "${FULL_LINE_RE}" | \ sed -e "s#/build/${arch}-usr/#/build/BOARD/#" } @@ -215,13 +162,17 @@ function filter_board_eo() { # Filters sdk reports to get anything but the package information # (i.e. junk). function junk_sdk_eo() { - cat_eo sdk | xgrep -v -e "${FULL_LINE_RE}" + cat "${SDK_EO}" | xgrep -v -e "${FULL_LINE_RE}" } # Filters board reports to get anything but the package information # (i.e. junk). function junk_board_eo() { - cat_eo board | xgrep -v -e "${FULL_LINE_RE}" + local arch name + arch=${1}; shift + name=${arch^^}_BOARD_EO + + cat "${!name}" | xgrep -v -e "${FULL_LINE_RE}" } # More regexp-like abominations follow. @@ -274,21 +225,6 @@ PKG_REPO_SED_FILTERS=( -e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}${SPACES_RE}.*/\1 \3/" ) -# Applies some sed filter over the emerge output of a given kind. -# Results are printed. -# -# Params: -# -# 1 - kind (sdk or board) -# @ - parameters passed to sed -function sed_eo_and_sort() { - local kind - kind=${1}; shift - # rest goes to sed - - cat_eo_f "${kind}" | sed "${@}" | sort -} - # Applies some sed filter over the SDK emerge output. Results are # printed. # @@ -296,9 +232,7 @@ function sed_eo_and_sort() { # # @ - parameters passed to sed function packages_for_sdk() { - # args are passed to sed_eo_and_sort - - sed_eo_and_sort sdk "${@}" + cat "${SDK_EO_F}" | sed "${@}" | sort } # Applies some sed filter over the board emerge output. Results are @@ -308,9 +242,12 @@ function packages_for_sdk() { # # @ - parameters passed to sed function packages_for_board() { - # args are passed to sed_eo_and_sort + local arch=${1}; shift + # rest goes to sed - sed_eo_and_sort board "${@}" + local name=${arch^^}_BOARD_EO_F + + sed "${@}" "${!name}" | sort } # Prints package name, slot and version information for SDK. @@ -334,24 +271,26 @@ function versions_sdk_with_key_values() { # Prints package name, slot and version information for board. function versions_board() { + local arch=${1}; shift local -a sed_opts sed_opts=( -e '/to \/build\/BOARD\// ! d' "${PKG_VER_SLOT_SED_FILTERS[@]}" ) - packages_for_board "${sed_opts[@]}" + packages_for_board "${arch}" "${sed_opts[@]}" } # Prints package name, slot, version and key-values information for # build dependencies of board. Key-values may be something like # USE="foo bar -baz". function board_bdeps() { + local arch=${1}; shift local -a sed_opts sed_opts=( -e '/to \/build\/BOARD\// d' "${PKG_VER_SLOT_KV_SED_FILTERS[@]}" ) - packages_for_board "${sed_opts[@]}" + packages_for_board "${arch}" "${sed_opts[@]}" } # Print package name and source repository names information for SDK. @@ -366,24 +305,29 @@ function package_sources_sdk() { # Print package name and source repository names information for # board. function package_sources_board() { + local arch=${1}; shift local -a sed_opts sed_opts=( "${PKG_REPO_SED_FILTERS[@]}" ) - packages_for_board "${sed_opts[@]}" + packages_for_board "${arch}" "${sed_opts[@]}" } # Checks if no errors were produced by emerge when generating # reports. It is assumed that emerge will print a line with "ERROR" in # it to denote a failure. function ensure_no_errors() { - local kind + local -a files=( "${SDK_EO_W}" ) + local arch name - for kind in sdk board; do - if cat_eo_w "${kind}" | grep --quiet --fixed-strings 'ERROR'; then - fail "there are errors in emerge output warnings files" - fi + for arch; do + name=${arch^^}_BOARD_EO_W + files+=( "${!name}" ) done + + if grep --quiet --fixed-strings 'ERROR' "${files[@]}"; then + fail "there are errors in emerge output warnings files" + fi } # Stores a path to a package.provided file inside the given root @@ -400,7 +344,6 @@ function get_provided_file() { path_var_name=${1}; shift local -n path_ref="${path_var_name}" - # shellcheck disable=SC2034 # reference to external variable path_ref="${root}/etc/portage/profile/package.provided/ignore_cross_packages" } @@ -458,10 +401,17 @@ function revert_crossdev_stuff() { # Checks if the expected reports were generated by emerge. function ensure_valid_reports() { - local kind var_name - for kind in sdk board; do - var_name="${kind^^}_EO_F" - if [[ ! -s ${!var_name} ]]; then + local -a files=( "${SDK_EO_F}" ) + local arch name + + for arch; do + name=${arch^^}_BOARD_EO_F + files+=( "${!name}" ) + done + + local file + for file in "${files[@]}"; do + if [[ ! -s ${file} ]]; then fail "report files are missing or are empty" fi done @@ -484,4 +434,37 @@ function clean_empty_warning_files() { done } +function get_num_proc() { + local -n num_proc_ref=${1}; shift + local -i num_proc=1 rv=1 + + # stolen from portage + [[ rv -eq 0 ]] || { rv=0; num_proc=$(getconf _NPROCESSORS_ONLN 2>/dev/null) || rv=1; } + [[ rv -eq 0 ]] || { rv=0; num_proc=$(sysctl -n hw.ncpu 2>/dev/null) || rv=1; } + # stolen from common.sh + [[ rv -eq 0 ]] || { rv=0; num_proc=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null) || rv=1; } + [[ rv -eq 0 ]] || { rv=0; num_proc=1; } + + num_proc_ref=${num_proc} +} + +function generate_cache_for() { + local repo=${1}; shift + + local -i gcf_num_proc + local load_avg + get_num_proc gcf_num_proc + load_avg=$(bc <<< "${gcf_num_proc} * 0.75") + egencache --repo "${repo}" --jobs="${gcf_num_proc}" --load-average="${load_avg}" --update +} + +function copy_cache_to_reports() { + local repo=${1}; shift + local reports_dir=${1}; shift + + local repo_dir + repo_dir=$(portageq get_repo_path / "${repo}") + cp -a "${repo_dir}/metadata/md5-cache" "${reports_dir}/${repo}-cache" +} + fi diff --git a/pkg_auto/impl/lcs.sh b/pkg_auto/impl/lcs.sh new file mode 100644 index 0000000000..d7e4f5b23e --- /dev/null +++ b/pkg_auto/impl/lcs.sh @@ -0,0 +1,375 @@ +#!/bin/bash + +# Implementation of longest common subsequence (LCS) algorithm, with +# memoization and scoring. The algorithm is a base of utilities like +# diff - it finds the common items in two sequences. +# +# The memoization part is about remembering the results of LCS +# computation for shorter sequences, so they can be reused when doing +# computation for longer sequences. +# +# The scoring part is about generalization of the comparison function, +# which normally would return a boolean value describing whether two +# items are equal or not. With scoring, two things change. First is +# that the returned value from the comparison/scoring function is not +# a boolean, but rather an integer describing the degree of equality +# of the items. And second is that with booleans, the winning +# subsequence was the longest one, but with scores is not necessarily +# the longest one, but with the highest total score. This changes the +# algorithm a bit. While the typical LCS algorithm goes like as +# follows (s1 and s2 are sequences, i1 and i2 are indices in their +# respective sequences): +# +# +# +# LCS(s1, s2, i1, i2) -> sequence: +# if (i1 >= len(s1) || i2 >= len(s2)): return () +# if (s1[i1] == s2[i2]): +# l = LCS(s1, s2, i1 + 1, i2 + 1) +# return (s1[i1], l...) +# else: +# l1 = LCS(s1, s2, i1 + 1, i2) +# l2 = LCS(s1, s2, i1, i2 + 1) +# if (len(l1) > len(l2)): return l1 +# return l2 +# +# +# +# The score LCS goes more or less like this: +# +# +# +# SLCS(s1, s2, i1, i2) -> tuple(score, sequence): +# if (i1 >= len(s1) || i2 >= len(21)): return (score: 0, sequence: ()) +# score = score_func(s1[i1], s2[i2]) +# if (score == max_score): +# # matches the "equal" case in LCS above +# l = SLCS(s1, s2, i1 + 1, i2 + 1) +# return (score: score + l.score, sequence: (s1[i1], l.sequence...) +# else if (score == 0): +# # matches the "not equal" case in LCS above +# l1 = SLCS(s1, s2, i1 + 1, i2) +# l2 = SLCS(s1, s2, i1, i2 + 1) +# if (l1.score > l2.score): return l1 +# return l2 +# else: +# # new "equal, but not quite" case +# l = SLCS(s1, s2, i1 + 1, i2 + 1) +# l.score = score + l.score +# l.sequence = (s1[i1], l.sequence...) +# l1 = SLCS(s1, s2, i1 + 1, i2) +# l2 = SLCS(s1, s2, i1, i2 + 1) +# return tuple_with_max_score(l, l1, l2) +# +# +# +# The difference in the implementation below is that instead of +# starting at index 0 and go up for each sequence, we start at max +# index and go down. + +if [[ -z ${__LCS_SH_INCLUDED__:-} ]]; then +__LCS_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Constants to use for accessing fields in common items. +declare -gri LCS_X1_IDX=0 LCS_X2_IDX=1 LCS_IDX1_IDX=2 LCS_IDX2_IDX=3 + +# Computes the longest common subsequence of two sequences and stores +# it in the passed array. The common items stored in the array with +# longest common subsequence are actually names of arrays, where each +# array stores an item from the first sequence, an item from the +# second sequence, an index of the first item in first sequence and an +# index of the second item in the second sequence. To access those +# fields, use the LCS_*_IDX constants. +# +# The function optionally takes a name of the score function. If no +# such function is passed, the simple string score function is used +# (gives score 1 if strings are equal, otherwise gives score 0). The +# function should take either one or three parameters. The first +# parameter is always a name of a score integer variable. If only one +# parameter is passed, the function should write the maximum score to +# the score variable. When three parameters are passed, then the +# second and third are items from the sequences passed to lcs_run and +# they should be compared and rated. The function should rate the +# items with zero if they are completely different, with max score if +# they are the same, and with something in between zero and max score +# if they are somewhat equal but not really. +# +# Params: +# +# 1 - a name of an array variable containing the first sequence +# 2 - a name of an array variable containing the second sequence +# 3 - a name of an array variable where the longest common subsequence +# will be stored +# 4 - optional, a name of a score function +# +# Example: +# +# a=( A B C D E F G ) +# b=( B C D G K ) +# c=() +# +# lcs_run a b c +# +# echo "a: ${a[*]}" +# echo "b: ${b[*]}" +# +# cn=() +# for i in "${c[@]}"; do +# declare -n ref=${i} +# n=${ref[LCS_X1_IDX]} +# unset -n ref +# cn+=( "${n}" ) +# done +# echo "c: ${cn[*]}" +# unset "${c[@]}" +function lcs_run() { + local seq1_name=${1}; shift + local seq2_name=${1}; shift + local common_name=${1}; shift + local score_func=__lcs_str_score + if [[ ${#} -gt 0 ]]; then + score_func=${1}; shift + fi + + if ! declare -pF "${score_func}" >/dev/null 2>/dev/null; then + fail "${score_func@Q} is not a function" + fi + + local -i lr_max_score + "${score_func}" lr_max_score + + # lcs_memo_items_set is a set of common item names; the set owns + # the common items + # + # lcs_memo_map is a mapping from a memo key (which encodes indices + # from both sequences) to a pair, being a score and a name of an + # longest common subsequence for the encoded indices, separated + # with semicolon + local -A lcs_memo_map=() lcs_memo_items_set=() + local -a lr_lcs_state=( "${score_func}" lcs_memo_map lcs_memo_items_set "${seq1_name}" "${seq2_name}" "${lr_max_score}" ) + + local -n seq1=${seq1_name} seq2=${seq2_name} + local -i idx1=$(( ${#seq1[@]} - 1 )) idx2=$(( ${#seq2[@]} - 1 )) + unset -n seq1 seq2 + + __lcs_recurse lr_lcs_state "${idx1}" "${idx2}" + + local -n common_ref=${common_name} + local lr_memo_key='' + __lcs_make_memo_key "${idx1}" "${idx2}" lr_memo_key + local pair=${lcs_memo_map["${lr_memo_key}"]} + local -n memoized_items_ref=${pair#*:} + + common_ref=( "${memoized_items_ref[@]}" ) + + # steal the items from the items set that are a part of desired + # LCS, so they are not freed now, but later, when the LCS is freed + local item + for item in "${common_ref[@]}"; do + unset "lcs_memo_items_set[${item}]" + done + + # free the unneeded items + unset "${!lcs_memo_items_set[@]}" + # free all the LCSes, we already have a copy of the one we wanted + local items_var_name memo_key + for memo_key in "${!lcs_memo_map[@]}"; do + pair=${lcs_memo_map["${memo_key}"]} + items_var_name=${pair#*:} + if [[ ${items_var_name} != EMPTY_ARRAY ]]; then + unset "${items_var_name}" + fi + done +} + +## +## Details +## + +declare -gri __LCS_SCORE_IDX=0 __LCS_MEMO_MAP_IDX=1 __LCS_MEMO_ITEMS_SET_IDX=2 __LCS_SEQ1_IDX=3 __LCS_SEQ2_IDX=4 __LCS_MAX_SCORE_IDX=5 + +function __lcs_recurse() { + local lcs_state_name=${1}; shift + local -i i1=${1}; shift + local -i i2=${1}; shift + + if [[ i1 -lt 0 || i2 -lt 0 ]]; then + return 0 + fi + + local -n lcs_state=${lcs_state_name} + + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local memo_key='' + __lcs_make_memo_key ${i1} ${i2} memo_key + local pair=${memo_map_ref["${memo_key}"]:-} + if [[ -n ${pair} ]]; then + return 0 + fi + unset pair memo_key + unset -n memo_map_ref + + local -n seq1_ref=${lcs_state[__LCS_SEQ1_IDX]} + local -n seq2_ref=${lcs_state[__LCS_SEQ2_IDX]} + local x1=${seq1_ref[i1]} + local x2=${seq2_ref[i2]} + unset -n seq2_ref seq1_ref + + local -i lr_diff_score + local score_func=${lcs_state[__LCS_SCORE_IDX]} + "${score_func}" lr_diff_score "${x1}" "${x2}" + unset score_func + + local -i max_score=${lcs_state[__LCS_MAX_SCORE_IDX]} + if [[ lr_diff_score -gt max_score ]]; then + lr_diff_score=max_score + fi + if [[ lr_diff_score -lt 0 ]]; then + lr_diff_score=0 + fi + + if [[ lr_diff_score -eq max_score ]]; then + unset max_score + + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) $((i2 - 1)) + + local n + gen_varname __LCS_COMMON n + + declare -a -g "${n}=()" + + # retrieve memoized result for i1-1 and i2-1 (what we just + # called above), make a copy of it, add the prepared item and + # memoize that copy for i1 and i2; also add the prepared item + # to the set of prepared items + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local previous_memo_key + __lcs_make_memo_key $((i1 - 1)) $((i2 - 1)) previous_memo_key + local previous_pair=${memo_map_ref["${previous_memo_key}"]:-'0:EMPTY_ARRAY'} + local previous_score=${previous_pair%%:*} + local -n previous_memoized_prepared_items_ref=${previous_pair#*:} + local -n c_ref=${n} + local prepared_item_to_insert + + gen_varname __LCS_PREP prepared_item_to_insert + declare -g -a "${prepared_item_to_insert}=( ${x1@Q} ${x2@Q} ${i1@Q} ${i2@Q} )" + + c_ref=( "${previous_memoized_prepared_items_ref[@]}" "${prepared_item_to_insert}" ) + + local memo_key + __lcs_make_memo_key ${i1} ${i2} memo_key + memo_map_ref["${memo_key}"]="$((lr_diff_score + previous_score)):${n}" + + local -n memo_items_set=${lcs_state[__LCS_MEMO_ITEMS_SET_IDX]} + memo_items_set["${prepared_item_to_insert}"]=x + elif [[ lr_diff_score -eq 0 ]]; then + unset max_score + + __lcs_recurse "${lcs_state_name}" ${i1} $((i2 - 1)) + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) ${i2} + + # retrieve memoized results for i1 and i2-1 and for i1-1 and + # i2 (what we just called above), and memoize the longer + # result for i1 and i2 + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local lr_memo_key='' + + __lcs_make_memo_key ${i1} $((i2 - 1)) lr_memo_key + local previous_pair1=${memo_map_ref["${lr_memo_key}"]:-'0:EMPTY_ARRAY'} + local -i previous_score1=${previous_pair1%%:*} + + __lcs_make_memo_key $((i1 - 1)) ${i2} lr_memo_key + local previous_pair2=${memo_map_ref["${lr_memo_key}"]:-'0:EMPTY_ARRAY'} + local -i previous_score2=${previous_pair2%%:*} + + __lcs_make_memo_key ${i1} ${i2} lr_memo_key + if [[ previous_score1 -gt previous_score2 ]]; then + memo_map_ref["${lr_memo_key}"]=${previous_pair1} + else + memo_map_ref["${lr_memo_key}"]=${previous_pair2} + fi + else + unset max_score + + # 1 + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) $((i2 - 1)) + # 2 + __lcs_recurse "${lcs_state_name}" ${i1} $((i2 - 1)) + # 3 + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) ${i2} + + local lr_mk1 lr_mk2 lr_mk3 + __lcs_make_memo_key $((i1 - 1)) $((i2 - 1)) lr_mk1 + __lcs_make_memo_key ${i1} $((i2 - 1)) lr_mk2 + __lcs_make_memo_key $((i1 - 1)) ${i2} lr_mk3 + + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local pair1=${memo_map_ref["${lr_mk1}"]:-'0:EMPTY_ARRAY'} + local pair2=${memo_map_ref["${lr_mk2}"]:-'0:EMPTY_ARRAY'} + local pair3=${memo_map_ref["${lr_mk3}"]:-'0:EMPTY_ARRAY'} + local -i score1=${pair1%%:*} + local -i score2=${pair2%%:*} + local -i score3=${pair3%%:*} + local -i pick # either 1, 2 or 3 + score1=$((score1 + lr_diff_score)) + + if [[ score1 -gt score2 ]]; then + if [[ score1 -gt score3 ]]; then + pick=1 + else + pick=3 + fi + elif [[ score2 -gt score3 ]]; then + pick=2 + else + pick=3 + fi + + if [[ pick -eq 1 ]]; then + local lr_new_lcs + gen_varname __LCS_COMMON lr_new_lcs + + declare -a -g "${lr_new_lcs}=()" + + local -n previous_memoized_prepared_items_ref=${pair1#*:} + local -n c_ref=${lr_new_lcs} + local prepared_item_to_insert + + gen_varname __LCS_PREP prepared_item_to_insert + declare -g -a "${prepared_item_to_insert}=( ${x1@Q} ${x2@Q} ${i1@Q} ${i2@Q} )" + + c_ref=( "${previous_memoized_prepared_items_ref[@]}" "${prepared_item_to_insert}" ) + + local lr_memo_key + __lcs_make_memo_key ${i1} ${i2} lr_memo_key + memo_map_ref["${lr_memo_key}"]="${score1}:${lr_new_lcs}" + + local -n memo_items_set=${lcs_state[__LCS_MEMO_ITEMS_SET_IDX]} + memo_items_set["${prepared_item_to_insert}"]=x + else + local -n picked_pair="pair${pick}" + local lr_memo_key='' + __lcs_make_memo_key ${i1} ${i2} lr_memo_key + memo_map_ref["${lr_memo_key}"]=${picked_pair} + fi + fi +} + +function __lcs_make_memo_key() { + local i1=${1}; shift + local i2=${1}; shift + local -n ref=${1}; shift + local key="x${i1}x${i2}x" + key=${key//-/_} + ref=${key} +} + +function __lcs_str_score() { + local -n score_ref=${1}; shift + score_ref=1 + [[ ${#} -eq 0 || ${1} = "${2}" ]] || score_ref=0 +} + +fi diff --git a/pkg_auto/impl/md5_cache_diff_lib.sh b/pkg_auto/impl/md5_cache_diff_lib.sh new file mode 100644 index 0000000000..3dcab5ca63 --- /dev/null +++ b/pkg_auto/impl/md5_cache_diff_lib.sh @@ -0,0 +1,1636 @@ +#!/bin/bash + +# This file implements computing a diff between two cache files. +# +# The diff is done for EAPI, KEYWORDS, {B,R,I,P,}DEPEND and LICENSE +# fields. While doing a diff for the first two is rather trivial +# (these are just a string or an array of simple objects), then doing +# the diffs for the rest, that involves groups is a bit more +# involved. The group is first sorted, then flattened. Then both old +# and new sorted and flattened groups are fed into the LCS algorithm +# to get the common items. These can be used to figure out the +# differences between the old and new groups. +# +# Sorting a group merges all the similar subgroups into one. Say there +# is: +# +# !foo? ( sys-libs/bar ) +# !foo? ( app-admin/baz ) +# +# and they will get merged into: +# +# !foo? ( +# app-admin/baz +# sys-libs/bar +# ) +# +# There are exceptions - "any of" subgroups are not merged (only +# sorted), so: +# +# || ( +# sys-libs/foo +# sys-libs/bar +# ) +# || ( +# app-admin/foo +# app-admin/bar +# ) +# +# stay almost as they are - the order of the packages with the group +# will change to become ordered: +# +# || ( +# sys-libs/bar +# sys-libs/foo +# ) +# || ( +# app-admin/bar +# app-admin/foo +# ) +# +# Also unnamed "all-of" groups inside "any-of" groups are not merged +# too, so: +# +# || ( +# ( +# dev-lang/python:3.11 +# dev-python/setuptools[python_3_11] +# ) +# ( +# dev-lang/python:3.12 +# dev-python/setuptools[python_3_12] +# ) +# ) +# +# stay as they are. +# +# Flattening turns the group-as-a-recursive-structure into a list of +# tokens, so to speak. Each token can be than treated as a separate +# line for the LCS algorithm. This means that for the following +# groups: +# +# SUBGROUP { +# type: any-of, +# items: sys-libs/foo sys-libs/bar +# } +# +# MAINGROUP { +# type: all-off, +# use name: foo, +# use mode: disabled, +# items: SUBGROUP app-admin/bar +# } +# +# flattening of MAINGROUP will result in the following list: +# +# !foo? +# ( +# || +# ( +# sys-libs/foo +# sys-libs/bar +# ) +# app-admin/bar +# ) +# +# The LCS algorithm takes a score function. The one used here is a +# function that returns maximum score of 3 for equal tokens, score of +# 1 for package dependency specifications with the same name and 0 for +# the rest. This means that ">=sys-libs/foo-1.2.3" and +# ">=sys-libs/foo-1.2.3" will result in score 3 (same tokens), +# ">=sys-libs/foo-1.2.3" and ">=sys-libs/foo-3.2.1" will result in +# score 1 (same package name), and "sys-libs/foo" and "sys-libs/bar" +# will result in score 0 (the rest). The reason for giving a non-zero +# score on just package name equality is that we want to handle +# changes for a package (like a plain version bump) differently than +# dependency replacement (like replacing autotools with cmake, for +# instance). On the other hand, the reason for not giving max score on +# just package name equality was to increase the diff quality. Without +# scoring, a change from: +# +# dev-lang/python:3.11 +# +# to: +# +# dev-lang/python:3.10 +# dev-lang/python:3.11 +# +# was interpreted two actions: first, as changing slot of +# dev-lang/python from 3.11 to 3.10 and second, as adding +# dev-lang/python:3.11. With scoring, the change was interpreted a +# single action - adding dev-lang/python:3.10. + +if [[ -z ${__MD5_CACHE_DIFF_LIB_SH_INCLUDED__:-} ]]; then +__MD5_CACHE_DIFF_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +source "${PKG_AUTO_IMPL_DIR}/lcs.sh" +source "${PKG_AUTO_IMPL_DIR}/md5_cache_lib.sh" +source "${PKG_AUTO_IMPL_DIR}/mvm.sh" + +# +# Diff report. Contains indent-lines. Indent-line is a string of +# :. Indent level is a number describing +# indentation level. Line is an actual line, duh. +# + +# Indices to access fields of the diff report: +# DR_INDENT_IDX - a current indent level to use +# DR_LINES_IDX - a name of an array containing indent-lines +declare -gri DR_INDENT_IDX=0 DR_LINES_IDX=1 + +# Declares empty diff reports. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function diff_report_declare() { + struct_declare -ga "${@}" "( '0' 'EMPTY_ARRAY' )" +} + +# Unsets diff reports, can take several names, just like unset. +function diff_report_unset() { + local dr_var_name lines_var_name + for dr_var_name; do + local -n dr_ref=${dr_var_name} + + lines_var_name=${dr_ref[DR_LINES_IDX]} + if [[ ${lines_var_name} != EMPTY_ARRAY ]]; then + unset "${lines_var_name}" + fi + + unset -n dr_ref + done + unset "${@}" +} + +# Stores an non-empty string if diff report is empty (has no lines +# yet). +# +# Params: +# +# 1 - diff report +# 2 - name of a variable where the string will be stored +function diff_report_is_empty() { + local -n dr_ref=${1}; shift + local -n out_ref=${1}; shift + + local -n lines_ref=${dr_ref[DR_LINES_IDX]} + if [[ ${#lines_ref[@]} -eq 0 ]]; then + out_ref=x + else + out_ref= + fi +} + +# Appends a line with a current indentation to a diff report. +# +# Params: +# +# 1 - diff report +# 2 - a line +function diff_report_append() { + local dr_var_name=${1}; shift + local line=${1}; shift + + local -n dr_ref=${dr_var_name} + local dra_lines_name=${dr_ref[DR_LINES_IDX]} + if [[ ${dra_lines_name} == EMPTY_ARRAY ]]; then + gen_varname dra_lines_name + declare -ga "${dra_lines_name}=()" + dr_ref[DR_LINES_IDX]=${dra_lines_name} + fi + + local -i indent + indent=${dr_ref[DR_INDENT_IDX]} + + local indent_line=${indent}:${line} + pkg_debug " ${indent_line} (${dr_var_name@Q})" + + local -n lines_ref=${dra_lines_name} + lines_ref+=( "${indent_line}" ) +} + +# Appends a line with an increased-by-1 indentation to a diff report. +# +# Params: +# +# 1 - diff report +# 2 - a line +function diff_report_append_indented() { + local dr_var_name=${1}; shift + local line=${1}; shift + + diff_report_indent "${dr_var_name}" + diff_report_append "${dr_var_name}" "${line}" + diff_report_dedent "${dr_var_name}" +} + +# Appends all the lines from the source diff report to a diff report, +# increasing indentation level of each line by the current indentation +# level. +# +# Params: +# +# 1 - diff report to grow +# 2 - source diff report +function diff_report_append_diff_report() { + local dr_var_name=${1}; shift + local -n dr_ref2=${1}; shift + + local -n dr_ref=${dr_var_name} + local dra_lines_name=${dr_ref[DR_LINES_IDX]} + if [[ ${dra_lines_name} == EMPTY_ARRAY ]]; then + gen_varname dra_lines_name + declare -ga "${dra_lines_name}=()" + dr_ref[DR_LINES_IDX]=${dra_lines_name} + fi + + local -i indent + indent=${dr_ref[DR_INDENT_IDX]} + local -n lines_ref=${dra_lines_name} + + local -n lines2_ref=${dr_ref2[DR_LINES_IDX]} + local indent_line2 line2 + local -i indent2 + for indent_line2 in "${lines2_ref[@]}"; do + indent2=${indent_line2%%:*} + line2=${indent_line2#*:} + indent2=$((indent + indent2)) + indent_line2=${indent2}:${line2} + pkg_debug " ${indent_line2} (${dr_var_name@Q})" + lines_ref+=( "${indent_line2}" ) + done +} + +# Increases current indentation level of a diff report. +# +# Params: +# +# 1 - diff report to grow +function diff_report_indent() { + local -n dr_ref=${1}; shift + + ((++dr_ref[DR_INDENT_IDX])) +} + +# Decreases current indentation level of a diff report. +# +# Params: +# +# 1 - diff report to grow +function diff_report_dedent() { + local -n dr_ref=${1}; shift + + ((dr_ref[DR_INDENT_IDX]--)) +} + +# Diffs two cache files. +# +# Params: +# +# 1 - old cache file +# 2 - new cache file +# 3 - diff report where the diff will be written to +function diff_cache_data() { + local old_var_name=${1}; shift + local new_var_name=${1}; shift + local dr_var_name=${1}; shift + + __mcdl_diff_eapi "${old_var_name}" "${new_var_name}" "${dr_var_name}" + __mcdl_diff_keywords "${old_var_name}" "${new_var_name}" "${dr_var_name}" + __mcdl_diff_iuse "${old_var_name}" "${new_var_name}" "${dr_var_name}" + + local -i idx + for idx in PCF_BDEPEND_IDX PCF_DEPEND_IDX PCF_IDEPEND_IDX PCF_PDEPEND_IDX PCF_RDEPEND_IDX PCF_LICENSE_IDX; do + __mcdl_diff_deps "${old_var_name}" "${new_var_name}" ${idx} "${dr_var_name}" + done +} + +# +# Implemetation details. +# + +function __mcdl_diff_eapi() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local dr_var_name=${1}; shift + + local old_eapi=${old_ref[PCF_EAPI_IDX]} + local new_eapi=${new_ref[PCF_EAPI_IDX]} + + if [[ ${old_eapi} != "${new_eapi}" ]]; then + diff_report_append "${dr_var_name}" "EAPI changed from ${old_eapi@Q} to ${new_eapi@Q}" + fi +} + +function __mcdl_diff_iuse() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local dr_var_name=${1}; shift + + local old_iuses_var_name=${old_ref[PCF_IUSE_IDX]} + local new_iuses_var_name=${new_ref[PCF_IUSE_IDX]} + + local -A old_map=() new_map=() removed_iuses same_iuses added_iuses + + local -n old_iuses_ref=${old_iuses_var_name} new_iuses_ref=${new_iuses_var_name} + + local -i idx=0 + local iuse_var_name name + for iuse_var_name in "${old_iuses_ref[@]}"; do + local -n iuse_ref=${iuse_var_name} + name=${iuse_ref[IUSE_NAME_IDX]} + old_map["${name}"]=${idx} + unset -n iuse_ref + ((++idx)) + done + + idx=0 + for iuse_var_name in "${new_iuses_ref[@]}"; do + local -n iuse_ref=${iuse_var_name} + name=${iuse_ref[IUSE_NAME_IDX]} + new_map["${name}"]=${idx} + unset -n iuse_ref + ((++idx)) + done + + sets_split old_map new_map removed_iuses added_iuses same_iuses + + local iuse + for iuse in "${!removed_iuses[@]}"; do + diff_report_append "${dr_var_name}" "removed IUSE flag ${iuse@Q}" + diff_report_append_indented "${dr_var_name}" "TODO: describe removed IUSE flag" + done + for iuse in "${!added_iuses[@]}"; do + diff_report_append "${dr_var_name}" "added IUSE flag ${iuse@Q}" + diff_report_append_indented "${dr_var_name}" "TODO: describe added IUSE flag" + done + local old_mode new_mode mode_str + local -i old_idx new_idx + for iuse in "${!same_iuses[@]}"; do + old_idx=${old_map["${iuse}"]} + new_idx=${new_map["${iuse}"]} + local -n old_iuse_ref=${old_iuses_ref[${old_idx}]} new_iuse_ref=${new_iuses_ref[${new_idx}]} + old_mode=${old_iuse_ref[IUSE_MODE_IDX]} + new_mode=${new_iuse_ref[IUSE_MODE_IDX]} + unset -n new_iuse_ref old_iuse_ref + if [[ ${old_mode} -ne ${new_mode} ]]; then + case ${new_mode} in + "${IUSE_DISABLED}") + mode_str=disabled + ;; + "${IUSE_ENABLED}") + mode_str=enabled + ;; + esac + diff_report_append "${dr_var_name}" "IUSE flag ${iuse@Q} became ${mode_str} by default" + fi + done +} + +function __mcdl_diff_keywords() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local dr_var_name=${1}; shift + + local old_kws_var_name=${old_ref[PCF_KEYWORDS_IDX]} + local new_kws_var_name=${new_ref[PCF_KEYWORDS_IDX]} + + local -n old_kws_ref=${old_kws_var_name} + local -n new_kws_ref=${new_kws_var_name} + + if [[ ${#old_kws_ref[@]} -ne ${#new_kws_ref[@]} ]]; then + fail 'keywords number should always be the same' + fi + + local index old_name new_name old_level new_level level_str + for (( index=0; index < ${#old_kws_ref[@]}; ++index )); do + local -n old_kw_ref=${old_kws_ref["${index}"]} + local -n new_kw_ref=${new_kws_ref["${index}"]} + old_name=${old_kw_ref[KW_NAME_IDX]} + new_name=${new_kw_ref[KW_NAME_IDX]} + if [[ ${old_name} != "${new_name}" ]]; then + fail 'keywords of the same name should always have the same index' + fi + old_level=${old_kw_ref[KW_LEVEL_IDX]} + new_level=${new_kw_ref[KW_LEVEL_IDX]} + local extra_todo='' + if [[ ${old_level} -ne ${new_level} ]]; then + case ${new_level} in + "${KW_STABLE}") + level_str='stable' + extra_todo='TODO: update/drop accept keywords for this package in overlay profiles' + ;; + "${KW_UNSTABLE}") + level_str='unstable' + extra_todo='TODO: add/update accept keywords for this package in overlay profiles' + ;; + "${KW_BROKEN}") + level_str='broken' + ;; + "${KW_UNKNOWN}") + level_str='unknown' + ;; + esac + diff_report_append "${dr_var_name}" "package became ${level_str} on ${new_name@Q}" + if [[ -n ${extra_todo} ]]; then + diff_report_append_indented "${dr_var_name}" "${extra_todo}" + fi + fi + unset -n new_kw old_kw + done +} + +function __mcdl_flatten_group() { + local -n group_ref=${1}; shift + local flattened_items_var_name=${1}; shift + + local -n flattened_items_ref=${flattened_items_var_name} + local -n group_items_ref=${group_ref[GROUP_ITEMS_IDX]} + + case ${group_ref[GROUP_TYPE_IDX]} in + "${GROUP_ALL_OF}") + local name=${group_ref[GROUP_USE_IDX]} + if [[ -n ${name} ]]; then + case ${group_ref[GROUP_ENABLED_IDX]} in + "${GROUP_USE_ENABLED}") + : + ;; + "${GROUP_USE_DISABLED}") + name="!${name}" + ;; + esac + name+='?' + flattened_items_ref+=( "i:${name}" ) + fi + unset name + ;; + "${GROUP_ANY_OF}") + flattened_items_ref+=( "i:||" ) + ;; + esac + flattened_items_ref+=( "o:(" ) + + local item_var_name item_type + for item_var_name in "${group_items_ref[@]}"; do + local -n item_ref=${item_var_name} + item_type=${item_ref:0:1} + case ${item_type} in + 'e') + # do not add empty items to the list + : + ;; + 'g') + local subgroup_name=${item_ref:2} + __mcdl_flatten_group "${subgroup_name}" "${flattened_items_var_name}" + unset subgroup_name + ;; + 'l'|'p') + flattened_items_ref+=( "${item_ref}" ) + ;; + *) + fail "item ${item_ref} is bad" + ;; + esac + unset -n item_ref + done + flattened_items_ref+=( "c:)" ) +} + +function __mcdl_flattened_group_item_score() { + local -n score_ref=${1}; shift + + score_ref=3 + + if [[ ${#} -eq 0 ]]; then + return + fi + + local i1=${1}; shift + local i2=${1}; shift + + local t1=${i1:0:1} + local t2=${i2:0:1} + + if [[ ${t1} != "${t2}" ]]; then + score_ref=0 + return + fi + + local v1=${i1:2} + local v2=${i2:2} + case ${t1} in + 'l'|'i') + [[ "${v1}" = "${v2}" ]] || score_ref=0 + ;; + 'p') + local -n p1_ref=${v1} p2_ref=${v2} + local n1=${p1_ref[PDS_NAME_IDX]} n2=${p2_ref[PDS_NAME_IDX]} + if [[ "${n1}" = "${n2}" ]]; then + local p1_str p2_str + pds_to_string "${v1}" p1_str + pds_to_string "${v2}" p2_str + [[ "${p1_str}" = "${p2_str}" ]] || score_ref=1 + unset p2_str p1_str + else + score_ref=0 + fi + unset n2 n1 + unset -n p2_ref p1_ref + ;; + 'o'|'c') + # parens are the same everywhere + : + ;; + *) + fail "item ${i1} or ${i2} is bad" + ;; + esac +} + +function __mcdl_ur_mode_description() { + local mode=${1}; shift + local -n str_ref=${1}; shift + + case ${mode} in + '+') + str_ref='enabled' + ;; + '=') + str_ref='strict' + ;; + '!=') + str_ref='reversed strict' + ;; + '?') + str_ref='enabled if enabled' + ;; + '!?') + str_ref='disabled if disabled' + ;; + '-') + str_ref='disabled' + ;; + esac +} + +function __mcdl_ur_pretend_description() { + local pretend=${1}; shift + local -n str_ref=${1}; shift + + case ${pretend} in + '') + str_ref='must exist, no pretending if missing' + ;; + '+') + str_ref='will pretend as enabled if missing' + ;; + '-') + str_ref='will pretend as disabled if missing' + ;; + esac +} + +function __mcdl_iuse_stack_to_string() { + local -n stack_ref=${1}; shift + local -n str_ref=${1}; shift + + str_ref='' + # we always ignore the first element - it's a name of the toplevel + # unnamed group, so stack of length 1 means no groups were + # encountered + if [[ ${#stack_ref[@]} -le 1 ]]; then + return 0 + fi + + local iuse + # we always ignore the first element - it's a name of the toplevel + # unnamed group + for iuse in "${stack_ref[@]:1}"; do + str_ref+="${iuse@Q} -> " + done + str_ref=${str_ref:0:$((${#str_ref} - 4))} +} + +function __mcdl_pds_diff() { + local -n old_pds_ref=${1}; shift + local -n new_pds_ref=${1}; shift + local old_stack_name=${1}; shift + local new_stack_name=${1}; shift + local dr_var_name=${1}; shift + + local name=${new_pds_ref[PDS_NAME_IDX]} + + local old_iuses new_iuses + __mcdl_iuse_stack_to_string "${old_stack_name}" old_iuses + __mcdl_iuse_stack_to_string "${new_stack_name}" new_iuses + + diff_report_declare local_pds_dr + + if [[ ${old_iuses} != "${new_iuses}" ]]; then + if [[ -z ${new_iuses} ]]; then + diff_report_append local_pds_dr "dropped all USE conditionals" + else + diff_report_append local_pds_dr "USE conditionals changed to ${new_iuses}" + fi + fi + + local -i old_blocks=${old_pds_ref[PDS_BLOCKS_IDX]} new_blocks=${new_pds_ref[PDS_BLOCKS_IDX]} + if [[ old_blocks -ne new_blocks ]]; then + local block + case ${new_blocks} in + "${PDS_NO_BLOCK}") + block='no' + ;; + "${PDS_WEAK_BLOCK}") + block='weak' + ;; + "${PDS_STRONG_BLOCK}") + block='strong' + ;; + esac + diff_report_append local_pds_dr "changed to ${block} block" + unset block + fi + + local old_op=${old_pds_ref[PDS_OP_IDX]} old_ver=${old_pds_ref[PDS_VER_IDX]} new_op=${new_pds_ref[PDS_OP_IDX]} new_ver=${new_pds_ref[PDS_VER_IDX]} + if [[ ${old_op} = '=*' ]]; then + old_op='=' + old_ver+='*' + fi + if [[ ${new_op} = '=*' ]]; then + new_op='=' + new_ver+='*' + fi + if [[ ${old_op} != "${new_op}" || ${old_ver} != "${new_ver}" ]]; then + if [[ -z ${new_op} && -z ${new_ver} ]]; then + diff_report_append local_pds_dr "dropped version constraint" + elif [[ -z ${old_op} && -z ${old_ver} ]]; then + diff_report_append local_pds_dr "added version constraint ${new_op}${new_ver}" + else + diff_report_append local_pds_dr "changed version constraint from ${old_op}${old_ver} to ${new_op}${new_ver}" + fi + fi + + local old_slot=${old_pds_ref[PDS_SLOT_IDX]} new_slot=${new_pds_ref[PDS_SLOT_IDX]} + if [[ ${old_slot} != "${new_slot}" ]]; then + if [[ -z ${new_slot} ]]; then + diff_report_append local_pds_dr "dropped slot constraint" + elif [[ -z ${old_slot} ]]; then + diff_report_append local_pds_dr "added slot constraint ${new_slot}" + else + diff_report_append local_pds_dr "changed slot constraint from ${old_slot} to ${new_slot}" + fi + fi + + local -n old_urs_ref=${old_pds_ref[PDS_UR_IDX]} new_urs_ref=${new_pds_ref[PDS_UR_IDX]} + local -A old_name_index=() new_name_index=() + + local ur_name use_name + local -i idx=0 + for ur_name in "${old_urs_ref[@]}"; do + local -n ur_ref=${ur_name} + use_name=${ur_ref[UR_NAME_IDX]} + old_name_index["${use_name}"]=${idx} + ((++idx)) + unset -n ur_ref + done + + idx=0 + for ur_name in "${new_urs_ref[@]}"; do + local -n ur_ref=${ur_name} + use_name=${ur_ref[UR_NAME_IDX]} + new_name_index["${use_name}"]=${idx} + ((++idx)) + unset -n ur_ref + done + + local -A only_old_urs=() only_new_urs=() common_urs=() + sets_split old_name_index new_name_index only_old_urs only_new_urs common_urs + for use_name in "${!only_old_urs[@]}"; do + diff_report_append local_pds_dr "dropped ${use_name} use requirement" + done + local pd_mode_str pd_pretend_str + for use_name in "${!only_new_urs[@]}"; do + idx=${new_name_index["${use_name}"]} + local -n ur_ref=${new_urs_ref[${idx}]} + __mcdl_ur_mode_description "${ur_ref[UR_MODE_IDX]}" pd_mode_str + __mcdl_ur_pretend_description "${ur_ref[UR_PRETEND_IDX]}" pd_pretend_str + diff_report_append local_pds_dr "added ${use_name} use ${pd_mode_str} requirement (${pd_pretend_str})" + unset -n ur_ref + done + + local -i old_idx new_idx + local old_mode new_mode old_pretend new_pretend pd_old_mode_str pd_new_mode_str pd_old_pretend_str pd_new_pretend_str + for use_name in "${!common_urs[@]}"; do + old_idx=${old_name_index["${use_name}"]} + new_idx=${new_name_index["${use_name}"]} + + local -n old_ur_ref=${old_urs_ref[${old_idx}]} new_ur_ref=${new_urs_ref[${new_idx}]} + old_mode=${old_ur_ref[UR_MODE_IDX]} + new_mode=${new_ur_ref[UR_MODE_IDX]} + if [[ ${old_mode} != "${new_mode}" ]]; then + __mcdl_ur_mode_description "${old_mode}" pd_old_mode_str + __mcdl_ur_mode_description "${new_mode}" pd_new_mode_str + diff_report_append local_pds_dr "mode of use requirement on ${use_name} changed from ${pd_old_mode_str} to ${pd_new_mode_str}" + fi + + old_pretend=${old_ur_ref[UR_PRETEND_IDX]} + new_pretend=${new_ur_ref[UR_PRETEND_IDX]} + if [[ ${old_pretend} != "${new_pretend}" ]]; then + __mcdl_ur_pretend_description "${old_pretend}" pd_old_pretend_str + __mcdl_ur_pretend_description "${new_pretend}" pd_new_pretend_str + diff_report_append local_pds_dr "pretend mode of use requirement on ${use_name} changed from ${pd_old_pretend_str@Q} to ${pd_new_pretend_str@Q}" + fi + unset -n new_ur_ref old_ur_ref + done + + local local_pds_dr_empty + diff_report_is_empty local_pds_dr local_pds_dr_empty + if [[ -z ${local_pds_dr_empty} ]]; then + local iuse_str='' + if [[ -n ${old_iuses} ]]; then + iuse_str=" with USE conditionals ${old_iuses}" + fi + + local block_str='' + local -i new_blocks=${new_pds_ref[PDS_BLOCKS_IDX]} + case ${new_blocks} in + "${PDS_NO_BLOCK}") + block_str='' + ;; + "${PDS_WEAK_BLOCK}") + block_str=' weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + block_str=' strong blocker' + ;; + esac + + diff_report_append "${dr_var_name}" "changes for ${name}${block_str}${iuse_str}:" + diff_report_indent "${dr_var_name}" + diff_report_append_diff_report "${dr_var_name}" local_pds_dr + diff_report_dedent "${dr_var_name}" + fi + diff_report_unset local_pds_dr +} + +function __mcdl_merge_and_sort_tagged_subgroups() { + local -n empty_subgroup_item_names_ref=${1}; shift + local -n subgroup_tag_to_named_subgroup_item_names_map_ref=${1}; shift + local -n any_of_subgroup_item_names_ref=${1}; shift + local skip_subgroup_merging=${1}; shift + local subgroup_tag=${1}; shift + # skip the mvc name, the group names will be the rest of the + # args + shift + # rest are subgroup item names + + if [[ ${#} -eq 0 ]]; then return 0; fi + + local subgroup_item_name subgroup_name other_subgroup_item_name other_subgroup_name + if [[ ${subgroup_tag} = '@any-of@' ]]; then + for subgroup_item_name; do + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + __mcdl_sort_group "${subgroup_name}" + any_of_subgroup_item_names_ref+=( "${subgroup_item_name}" ) + done + elif [[ -n ${skip_subgroup_merging} ]]; then + for subgroup_item_name; do + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + __mcdl_sort_group "${subgroup_name}" + if [[ ${subgroup_tag} = '@empty@' ]]; then + empty_subgroup_item_names_ref+=( "${subgroup_item_name}" ) + else + subgroup_tag_to_named_subgroup_item_names_map_ref["${subgroup_tag}"]=${subgroup_item_name} + fi + done + else + subgroup_item_name=${1}; shift + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + local other_subgroup_items_name + for other_subgroup_item_name; do + local -n item_ref=${other_subgroup_item_name} + other_subgroup_name=${item_ref:2} + unset -n item_ref + local -n other_subgroup_ref=${other_subgroup_name} + other_subgroup_items_name=${other_subgroup_ref[GROUP_ITEMS_IDX]} + local -n other_subgroup_items_ref=${other_subgroup_items_name} + group_add_items "${subgroup_name}" "${other_subgroup_items_ref[@]}" + unset -n other_subgroup_items_ref + other_subgroup_ref[GROUP_ITEMS_IDX]='EMPTY_ARRAY' + unset -n other_subgroup_ref + unset "${other_subgroup_items_name}" + item_unset "${other_subgroup_item_name}" + done + __mcdl_sort_group "${subgroup_name}" + if [[ ${subgroup_tag} = '@empty@' ]]; then + empty_subgroup_item_names_ref+=( "${subgroup_item_name}" ) + else + subgroup_tag_to_named_subgroup_item_names_map_ref["${subgroup_tag}"]=${subgroup_item_name} + fi + fi +} + +function __mcdl_sort_group() { + local group_name=${1}; shift + + local -n group_ref=${group_name} + local -a subgroup_item_names license_item_names pds_item_names=() + local -n items_ref=${group_ref[GROUP_ITEMS_IDX]} + local is_any_of_group= + + subgroup_item_names=() + license_item_names=() + pds_item_names=() + + if [[ ${#items_ref[@]} -eq 0 ]]; then + return 0 + fi + + local -i group_type=${group_ref[GROUP_TYPE_IDX]} + if [[ group_type -eq GROUP_ANY_OF ]]; then + is_any_of_group=x + fi + unset group_type + + local item_name item_type + for item_name in "${items_ref[@]}"; do + local -n item=${item_name} + item_type=${item:0:1} + unset -n item + case ${item_type} in + e) + # these won't show up in sorted group + : + ;; + g) + subgroup_item_names+=( "${item_name}" ) + ;; + l) + license_item_names+=( "${item_name}" ) + ;; + p) + pds_item_names+=( "${item_name}" ) + ;; + esac + done + unset item_name item_type + items_ref=() + unset -n items_ref group_ref + + ## + ## add sorted pds into group + ## + + if [[ ${#pds_item_names[@]} -gt 0 ]]; then + # the package name may appear more than one time in the same + # group, this happens for ranged deps (e.g >=2.0, <3.0) + local pds_item_name + local pds_name pkg_name counted_pkg_name + local -i pkg_name_count + local -A pkg_name_to_count_map=() + local -a pkg_names=() + local -A pkg_name_to_item_name_map=() + for pds_item_name in "${pds_item_names[@]}"; do + local -n item_ref=${pds_item_name} + pds_name=${item_ref:2} + unset -n item_ref + local -n pds_ref=${pds_name} + pkg_name=${pds_ref[PDS_NAME_IDX]} + unset -n pds_ref + pkg_name_count=${pkg_name_to_count_map["${pkg_name}"]:-0} + counted_pkg_name="${pkg_name}^${pkg_name_count}" + ((++pkg_name_count)) + pkg_name_to_count_map["${pkg_name}"]=${pkg_name_count} + pkg_names+=( "${counted_pkg_name}" ) + pkg_name_to_item_name_map["${counted_pkg_name}"]=${pds_item_name} + done + local -a sorted_pkg_names sorted_items + mapfile -t sorted_pkg_names < <(printf '%s\n' "${pkg_names[@]}" | sort -t '^' -k 1,1 -k2n,2) + + for pkg_name in "${sorted_pkg_names[@]}"; do + pds_item_name=${pkg_name_to_item_name_map["${pkg_name}"]} + sorted_items+=( "${pds_item_name}" ) + done + group_add_items "${group_name}" "${sorted_items[@]}" + + unset sorted_items sorted_pkg_names pkg_name_to_item_name_map pkg_names pkg_name_to_count_map pkg_name_count pkg_name_count counted_pkg_name pkg_name pds_name pds_item_name + fi + + ## + ## add sorted licenses into group + ## + + if [[ ${#license_item_names[@]} -gt 0 ]]; then + local -A license_to_item_name_map=() + + local license_item_name license + for license_item_name in "${license_item_names[@]}"; do + local -n item_ref=${license_item_name} + license=${item_ref:2} + unset -n item_ref + license_to_item_name_map["${license}"]=${license_item_name} + done + local -a sorted_licenses sorted_items + mapfile -t sorted_licenses < <(printf '%s\n' "${!license_to_item_name_map[@]}" | sort) + for license in "${sorted_licenses[@]}"; do + license_item_name=${license_to_item_name_map["${license}"]} + sorted_items+=( "${license_item_name}" ) + done + group_add_items "${group_name}" "${sorted_items[@]}" + + unset sorted_items sorted_licenses license_item_name license license_to_item_name_map + fi + + ## + ## add sorted subgroups into group + ## + + if [[ ${#subgroup_item_names[@]} -gt 0 ]]; then + # - collect group names based on their type, use name and enabled use + # - unnamed all-of go first (tag: empty) + # - named all-of go next (tag: use name + enabled use) + # - any-of go each separately (tag: any-of) + # - merge all-of groups with the same tag + # - exception - unnamed all-of groups inside any-of group should not be merged + # - sort each group and add to main group + local subgroup_item_name subgroup_name subgroup_type subgroup_use subgroup_tag subgroup_use_mode + local dsg_tagged_subgroups_item_names_array_mvm_name + gen_varname dsg_tagged_subgroups_item_names_array_mvm_name + mvm_declare "${dsg_tagged_subgroups_item_names_array_mvm_name}" + for subgroup_item_name in "${subgroup_item_names[@]}"; do + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + local -n subgroup_ref=${subgroup_name} + subgroup_type=${subgroup_ref[GROUP_TYPE_IDX]} + case ${subgroup_type} in + "${GROUP_ALL_OF}") + subgroup_use=${subgroup_ref[GROUP_USE_IDX]} + if [[ -z ${subgroup_use} ]]; then + subgroup_tag='@empty@' + else + subgroup_use_mode=${subgroup_ref[GROUP_ENABLED_IDX]} + case ${subgroup_use_mode} in + "${GROUP_USE_ENABLED}") + subgroup_tag='+' + ;; + "${GROUP_USE_DISABLED}") + subgroup_tag='-' + ;; + esac + subgroup_tag+=${subgroup_use} + fi + ;; + "${GROUP_ANY_OF}") + subgroup_tag="@any-of@" + ;; + esac + unset -n subgroup_ref + mvm_add "${dsg_tagged_subgroups_item_names_array_mvm_name}" "${subgroup_tag}" "${subgroup_item_name}" + done + unset subgroup_use_mode subgroup_tag subgroup_use subgroup_type subgroup_name subgroup_item_name + + local -A subgroup_tag_to_named_subgroup_item_names_map=() + local -a any_of_subgroup_item_names=() empty_subgroup_item_names=() + mvm_iterate "${dsg_tagged_subgroups_item_names_array_mvm_name}" __mcdl_merge_and_sort_tagged_subgroups \ + empty_subgroup_item_names \ + subgroup_tag_to_named_subgroup_item_names_map \ + any_of_subgroup_item_names \ + "${is_any_of_group}" + mvm_unset "${dsg_tagged_subgroups_item_names_array_mvm_name}" + + group_add_items "${group_name}" "${empty_subgroup_item_names[@]}" + if [[ ${#subgroup_tag_to_named_subgroup_item_names_map[@]} -gt 0 ]]; then + local -a sorted_subgroup_tags sorted_items + local subgroup_tag subgroup_item_name + mapfile -t sorted_subgroup_tags < <(printf '%s\n' "${!subgroup_tag_to_named_subgroup_item_names_map[@]}" | sort) + for subgroup_tag in "${sorted_subgroup_tags[@]}"; do + if [[ ${subgroup_tag:0:1} = '@' && ${subgroup_tag: -1:1} = '@' ]]; then + continue + fi + subgroup_item_name=${subgroup_tag_to_named_subgroup_item_names_map["${subgroup_tag}"]} + sorted_items+=( "${subgroup_item_name}" ) + done + group_add_items "${group_name}" "${sorted_items[@]}" + unset sorted_items subgroup_item_name subgroup_tag sorted_subgroup_tags + fi + group_add_items "${group_name}" "${any_of_subgroup_item_names[@]}" + fi +} + +function __mcdl_iuse_stack_to_string_ps() { + local stack_name=${1}; shift + local -n str_ref=${1}; shift + local prefix=${1}; shift + local suffix=${1}; shift + + local tmp_str + __mcdl_iuse_stack_to_string "${stack_name}" tmp_str + + if [[ -n ${tmp_str} ]]; then + str_ref="${prefix}${tmp_str}${suffix}" + else + str_ref='' + fi +} + +function __mcdl_debug_group() { + local group_name=${1}; shift + local label=${1}; shift + + if pkg_debug_enabled; then + local dg_group_str + group_to_string "${group_name}" dg_group_str + pkg_debug_print "${label}: ${dg_group_str}" + fi +} + +function __mcdl_debug_flattened_list() { + local list_name=${1}; shift + local label=${1}; shift + + if pkg_debug_enabled; then + local item item_type + local -n list_ref=${list_name} + pkg_debug_print "${label}:" + for item in "${list_ref[@]}"; do + item_type=${item:0:1} + case ${item_type} in + 'i'|'o'|'c'|'l') + pkg_debug_print " ${item}" + ;; + 'p') + local dfl_pds_str + pds_to_string "${item:2}" dfl_pds_str + pkg_debug_print " ${item} (${dfl_pds_str})" + unset dfl_pds_str + ;; + esac + done + unset -n list_ref + fi +} + +function __mcdl_debug_diff() { + local -n dd_old_list_ref=${1}; shift + local -n dd_new_list_ref=${1}; shift + local -n dd_common_list_ref=${1}; shift + + if ! pkg_debug_enabled; then + return 0 + fi + + pkg_debug_print "diff between old and new flattened lists" + local -i last_idx1=0 last_idx2=0 idx1 idx2 + local ci_name + for ci_name in "${dd_common_list_ref[@]}"; do + local -n ci_ref=${ci_name} + idx1=${ci_ref[LCS_IDX1_IDX]} + idx2=${ci_ref[LCS_IDX2_IDX]} + unset -n ci_ref + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_list_ref["${last_idx1}"]} + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l'|'i'|'o'|'c') + pkg_debug_print " -${v1}" + ;; + 'p') + local p_str + pds_to_string "${v1}" p_str + pkg_debug_print " -${p_str}" + unset p_str + ;; + *) + fail "item ${item1} is bad" + ;; + esac + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_list_ref["${last_idx2}"]} + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l'|'i'|'o'|'c') + pkg_debug_print " +${v2}" + ;; + 'p') + local p_str + pds_to_string "${v2}" p_str + pkg_debug_print " +${p_str}" + unset p_str + ;; + *) + fail "item ${item2} is bad" + ;; + esac + unset v2 t2 item2 + ((++last_idx2)) + done + + local item1=${dd_old_list_ref["${idx1}"]} item2=${dd_new_list_ref["${idx2}"]} + local t1=${item1:0:1} v1=${item1:2} t2=${item2:0:1} v2=${item2:2} + + case ${t1} in + 'l'|'i'|'o'|'c') + pkg_debug_print " ${v2}" + ;; + 'p') + local p1_str p2_str + pds_to_string "${v1}" p1_str + pds_to_string "${v2}" p2_str + if [[ "${p1_str}" != "${p2_str}" ]]; then + pkg_debug_print " ${p1_str} -> ${p2_str}" + else + pkg_debug_print " ${p1_str}" + fi + unset p2_str p1_str + ;; + *) + fail "item ${item1} or ${item2} is bad" + ;; + esac + + unset v2 t2 item2 v1 t1 item1 + ((++last_idx1)) + ((++last_idx2)) + done + + idx1=${#dd_old_list_ref[@]} + idx2=${#dd_new_list_ref[@]} + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_list_ref["${last_idx1}"]} + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l'|'i'|'o'|'c') + pkg_debug_print " -${v1}" + ;; + 'p') + local p_str + pds_to_string "${v1}" p_str + pkg_debug_print " -${p_str}" + unset p_str + ;; + *) + fail "item ${item1} is bad" + ;; + esac + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_list_ref["${last_idx2}"]} + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l'|'i'|'o'|'c') + pkg_debug_print " +${v2}" + ;; + 'p') + local p_str + pds_to_string "${v2}" p_str + pkg_debug_print " +${p_str}" + unset p_str + ;; + *) + fail "item ${item2} is bad" + ;; + esac + unset v2 t2 item2 + ((++last_idx2)) + done +} + +function __mcdl_debug_iuse_stack() { + local iuse_stack_name=${1}; shift + local label=${1}; shift + + if pkg_debug_enabled; then + local dis_str + __mcdl_iuse_stack_to_string "${iuse_stack_name}" dis_str + pkg_debug_print "${label}: ${dis_str}" + fi +} + +function __mcdl_diff_deps() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local deps_idx=${1}; shift + local dr_var_name=${1}; shift + + local label + case ${deps_idx} in + "${PCF_BDEPEND_IDX}") + label='build dependencies' + ;; + "${PCF_DEPEND_IDX}") + label='dependencies' + ;; + "${PCF_IDEPEND_IDX}") + label='install dependencies' + ;; + "${PCF_PDEPEND_IDX}") + label='post dependencies' + ;; + "${PCF_RDEPEND_IDX}") + label='runtime dependencies' + ;; + "${PCF_LICENSE_IDX}") + label='licenses' + ;; + *) + fail "bad cache file index ${deps_idx@Q}" + ;; + esac + + local old_group_name=${old_ref[${deps_idx}]} new_group_name=${new_ref[${deps_idx}]} + local -a dd_old_flattened_list=() dd_new_flattened_list=() + + group_declare old_sorted_group new_sorted_group + + group_copy old_sorted_group "${old_group_name}" + __mcdl_debug_group old_sorted_group "copy of old ${label} group" + __mcdl_sort_group old_sorted_group + __mcdl_debug_group old_sorted_group "sorted old ${label} group" + + group_copy new_sorted_group "${new_group_name}" + __mcdl_debug_group new_sorted_group "copy of new ${label} group" + __mcdl_sort_group new_sorted_group + __mcdl_debug_group old_sorted_group "sorted new ${label} group" + + __mcdl_flatten_group old_sorted_group dd_old_flattened_list + __mcdl_flatten_group new_sorted_group dd_new_flattened_list + + __mcdl_debug_flattened_list dd_old_flattened_list "flattened old ${label} group" + __mcdl_debug_flattened_list dd_new_flattened_list "flattened new ${label} group" + + local -a dd_common_items=() + + lcs_run dd_old_flattened_list dd_new_flattened_list dd_common_items __mcdl_flattened_group_item_score + + diff_report_declare local_dr + + __mcdl_debug_diff dd_old_flattened_list dd_new_flattened_list dd_common_items + + local -a old_iuse_stack=() new_iuse_stack=() + # a stack of counters for naming unnamed groups like + # + # || ( ( a/b-1 c/d-1 ) ( a/b-2 c/d-2 ) ) + # + # The "( a/b-1 c/d-1 )" and "( a/b-2 c/d-2 )" groups are unnamed + # within the "||" group. + local old_group_unnamed_counter_stack=( 0 ) new_group_unnamed_counter_stack=( 0 ) + local -i last_idx1=0 last_idx2=0 idx1 idx2 + local ci_name + local prev_item1='e:' prev_item2='e:' + for ci_name in "${dd_common_items[@]}"; do + local -n ci_ref=${ci_name} + idx1=${ci_ref[LCS_IDX1_IDX]} + idx2=${ci_ref[LCS_IDX2_IDX]} + unset -n ci_ref + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_flattened_list["${last_idx1}"]} + pkg_debug "old item ${item1@Q}" + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "dropped license ${v1@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + pds_to_string "${v1}" p_str + local -n p_ref=${v1} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item1:0:1} = 'i' ]]; then + subgroup_name=${prev_item1:2} + else + local -i counter=${old_group_unnamed_counter_stack[-1]} + ((++counter)) + old_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + old_group_unnamed_counter_stack+=( 0 ) + old_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'old_iuse_stack[-1]' 'old_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after dropping last name" + ;; + *) + fail "item ${item1} is bad" + ;; + esac + prev_item1=${item1} + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_flattened_list["${last_idx2}"]} + pkg_debug "new item ${item2@Q}" + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "added license ${v2@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + pds_to_string "${v2}" p_str + local -n p_ref=${v2} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item2:0:1} = 'i' ]]; then + subgroup_name=${prev_item2:2} + else + local -i counter=${new_group_unnamed_counter_stack[-1]} + ((++counter)) + new_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + new_group_unnamed_counter_stack+=( 0 ) + new_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'new_iuse_stack[-1]' 'new_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping last name" + ;; + *) + fail "item ${item2} is bad" + ;; + esac + prev_item2=${item2} + unset v2 t2 item2 + ((++last_idx2)) + done + + local item1=${dd_old_flattened_list["${idx1}"]} item2=${dd_new_flattened_list["${idx2}"]} + pkg_debug "old item ${item1@Q} and new item ${item2@Q}" + local t1=${item1:0:1} v1=${item1:2} t2=${item2:0:1} v2=${item2:2} + + case ${t1} in + 'l') + # not interesting + : + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item1:0:1} = 'i' ]]; then + subgroup_name=${prev_item1:2} + else + local -i counter=${old_group_unnamed_counter_stack[-1]} + ((++counter)) + old_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + old_group_unnamed_counter_stack+=( 0 ) + old_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after adding common ${subgroup_name@Q}" + + if [[ ${prev_item2:0:1} = 'i' ]]; then + subgroup_name=${prev_item2:2} + else + local -i counter=${new_group_unnamed_counter_stack[-1]} + ((++counter)) + new_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + new_group_unnamed_counter_stack+=( 0 ) + new_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after adding common ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'old_iuse_stack[-1]' 'old_group_unnamed_counter_stack[-1]' 'new_iuse_stack[-1]' 'new_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after dropping common last name" + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping common last name" + ;; + 'p') + __mcdl_pds_diff "${item1:2}" "${item2:2}" old_iuse_stack new_iuse_stack local_dr + ;; + *) + fail "item ${item1} or ${item2} is bad" + ;; + esac + + prev_item1=${item1} + prev_item2=${item2} + unset v2 t2 item2 v1 t1 item1 + ((++last_idx1)) + ((++last_idx2)) + done + unset "${dd_common_items[@]}" + + idx1=${#dd_old_flattened_list[@]} + idx2=${#dd_new_flattened_list[@]} + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_flattened_list["${last_idx1}"]} + pkg_debug "old item ${item1@Q}" + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "dropped license ${v1@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + pds_to_string "${v1}" p_str + local -n p_ref=${v1} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item1:0:1} = 'i' ]]; then + subgroup_name=${prev_item1:2} + else + local -i counter=${old_group_unnamed_counter_stack[-1]} + ((++counter)) + old_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + old_group_unnamed_counter_stack+=( 0 ) + old_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'old_iuse_stack[-1]' 'old_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after dropping last name" + ;; + *) + fail "item ${item1} is bad" + ;; + esac + prev_item1=${item1} + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_flattened_list["${last_idx2}"]} + pkg_debug "new item ${item2@Q}" + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "added license ${v2@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + pds_to_string "${v2}" p_str + local -n p_ref=${v2} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item2:0:1} = 'i' ]]; then + subgroup_name=${prev_item2:2} + else + local -i counter=${new_group_unnamed_counter_stack[-1]} + ((++counter)) + new_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + new_group_unnamed_counter_stack+=( 0 ) + new_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'new_iuse_stack[-1]' 'new_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping last name" + ;; + *) + fail "item ${item2} is bad" + ;; + esac + prev_item2=${item2} + unset v2 t2 item2 + ((++last_idx2)) + done + local local_dr_empty + diff_report_is_empty local_dr local_dr_empty + if [[ -z ${local_dr_empty} ]]; then + diff_report_append "${dr_var_name}" "${label}:" + diff_report_indent "${dr_var_name}" + diff_report_append_diff_report "${dr_var_name}" local_dr + diff_report_dedent "${dr_var_name}" + fi + diff_report_unset local_dr + + group_unset new_sorted_group old_sorted_group +} + +fi diff --git a/pkg_auto/impl/md5_cache_lib.sh b/pkg_auto/impl/md5_cache_lib.sh new file mode 100644 index 0000000000..e3808f179f --- /dev/null +++ b/pkg_auto/impl/md5_cache_lib.sh @@ -0,0 +1,1177 @@ +#!/bin/bash + +# This file implements parsing the md5-metadata cache files and +# accessing the parsed results. Not the entirety of the cache file is +# parsed, only the parts that were needed at the time of writing +# it. So currently the exposed parts of parsed cache files are EAPI, +# IUSE, KEYWORDS, LICENSE, {B,R,P,I,}DEPEND and _eclasses_. The +# _eclasses_ part discards the checksums, though, so only names are +# available. + +if [[ -z ${__MD5_CACHE_LIB_SH_INCLUDED__:-} ]]; then +__MD5_CACHE_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +source "${PKG_AUTO_IMPL_DIR}/debug.sh" +source "${PKG_AUTO_IMPL_DIR}/gentoo_ver.sh" + +# +# Cache file +# + +# Indices to access fields of the cache file: +# PCF_EAPI_IDX - a string/number describing EAPI +# PCF_KEYWORDS_IDX - a name of an array containing keyword objects +# PCF_IUSE_IDX - a name of an array containing iuse objects +# PCF_BDEPEND_IDX - a name of a group object with build dependencies +# PCF_DEPEND_IDX - a name of a group object with dependencies +# PCF_IDEPEND_IDX - a name of a group object with install dependencies +# PCF_PDEPEND_IDX - a name of a group object with post dependencies +# PCF_RDEPEND_IDX - a name of a group object with runtime dependencies +# PCF_LICENSE_IDX - a name of a group object with licenses +# PCF_ECLASSES_IDX - a name of an array with used eclasses +declare -gri PCF_EAPI_IDX=0 PCF_KEYWORDS_IDX=1 PCF_IUSE_IDX=2 PCF_BDEPEND_IDX=3 PCF_DEPEND_IDX=4 PCF_IDEPEND_IDX=5 PCF_PDEPEND_IDX=6 PCF_RDEPEND_IDX=7 PCF_LICENSE_IDX=8 PCF_ECLASSES_IDX=9 + +# Declares empty cache files. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function cache_file_declare() { + struct_declare -ga "${@}" "( '0' 'EMPTY_ARRAY' 'EMPTY_ARRAY' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_ARRAY' )" +} + +# Unsets cache files, can take several names, just like unset. +function cache_file_unset() { + local name + for name; do + local -n cache_file_ref=${name} + + __mcl_unset_array "${cache_file_ref[PCF_KEYWORDS_IDX]}" kw_unset + __mcl_unset_array "${cache_file_ref[PCF_IUSE_IDX]}" iuse_unset + __mcl_unset_array "${cache_file_ref[PCF_ECLASSES_IDX]}" unset + + group_unset \ + "${cache_file_ref[PCF_BDEPEND_IDX]}" \ + "${cache_file_ref[PCF_DEPEND_IDX]}" \ + "${cache_file_ref[PCF_IDEPEND_IDX]}" \ + "${cache_file_ref[PCF_PDEPEND_IDX]}" \ + "${cache_file_ref[PCF_RDEPEND_IDX]}" \ + "${cache_file_ref[PCF_LICENSE_IDX]}" + + unset -n cache_file_ref + done + unset "${@}" +} + +# Parses a cache file under the passed path. +# +# Params: +# +# 1 - cache file +# 2 - path to the cache file +function parse_cache_file() { + local -n cache_file_ref=${1}; shift + local path=${1}; shift + # rest are architectures + + if pkg_debug_enabled; then + local -a file_lines + mapfile -t file_lines <"${path}" + pkg_debug_print_lines "parsing ${path@Q}" "${file_lines[@]}" + unset file_lines + fi + + local -n pkg_eapi_ref='cache_file_ref[PCF_EAPI_IDX]' + local -n pkg_keywords_ref='cache_file_ref[PCF_KEYWORDS_IDX]' + local -n pkg_iuse_ref='cache_file_ref[PCF_IUSE_IDX]' + local -n pkg_bdepend_group_name_ref='cache_file_ref[PCF_BDEPEND_IDX]' + local -n pkg_depend_group_name_ref='cache_file_ref[PCF_DEPEND_IDX]' + local -n pkg_idepend_group_name_ref='cache_file_ref[PCF_IDEPEND_IDX]' + local -n pkg_pdepend_group_name_ref='cache_file_ref[PCF_PDEPEND_IDX]' + local -n pkg_rdepend_group_name_ref='cache_file_ref[PCF_RDEPEND_IDX]' + local -n pkg_license_group_name_ref='cache_file_ref[PCF_LICENSE_IDX]' + local -n pkg_eclasses_ref='cache_file_ref[PCF_ECLASSES_IDX]' + + local l key + while read -r l; do + key=${l%%=*} + pkg_debug "parsing ${key@Q}" + case ${key} in + 'EAPI') + pkg_eapi_ref=${l#*=} + pkg_debug "EAPI: ${pkg_eapi_ref}" + ;; + 'KEYWORDS') + __mcl_parse_keywords "${l#*=}" pkg_keywords_ref "${@}" + ;; + 'IUSE') + __mcl_parse_iuse "${l#*=}" pkg_iuse_ref + ;; + 'BDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_bdepend_group_name_ref + ;; + 'DEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_depend_group_name_ref + ;; + 'IDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_idepend_group_name_ref + ;; + 'PDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_pdepend_group_name_ref + ;; + 'RDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_rdepend_group_name_ref + ;; + 'LICENSE') + __mcl_parse_dsf "${__MCL_DSF_LICENSE}" "${l#*=}" pkg_license_group_name_ref + ;; + '_eclasses_') + __mcl_parse_eclasses "${l#*=}" pkg_eclasses_ref + ;; + *) + pkg_debug "Not parsing ${key@Q}, ignoring" + ;; + esac + done <"${path}" +} + +# +# Use requirement (the part in the square brackets in the package +# dependency specification). +# + +# Indices to access fields of the use requirement: +# UR_NAME_IDX - a use name +# UR_MODE_IDX - a string describing the mode: "+" is enabled, "=" +# is strict relation, "!=" reversed strict relation, +# "?" is "enabled if enabled in the ebuild", "!?" is +# "disabled if disabled in the ebuild", and "-" is +# disabled) +# UR_PRETEND_IDX - a string describing pretend mode: it can be either +# empty (no pretending if use is missing in the +# package), "+" (pretend enabled if missing in the +# package), or "-" (pretend disabled if missing in +# the package) +declare -gri UR_NAME_IDX=0 UR_MODE_IDX=1 UR_PRETEND_IDX=2 + +# Declares empty use requirements. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function ur_declare() { + struct_declare -ga "${@}" "( 'ITS_UNSET' '+' '' )" +} + +# Unsets use requirements, can take several names, just like unset. +function ur_unset() { + unset "${@}" +} + +# Copies use requirement into another. +# +# Params: +# +# 1 - use requirement to be clobbered +# 2 - the source use requirement +function ur_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local -i idx + for idx in UR_NAME_IDX UR_MODE_IDX UR_PRETEND_IDX; do + to_clobber_ref[idx]=${to_copy_ref[idx]} + done +} + +# Stringifies use requirement. +# +# Params: +# +# 1 - use requirement +# 2 - name of a variable where the string form will be stored +function ur_to_string() { + local -n ur_ref=${1}; shift + local -n str_ref=${1}; shift + + case ${ur_ref[UR_MODE_IDX]} in + '!'*) + str_ref='!' + ;; + '-'*) + str_ref='-' + ;; + *) + str_ref='' + ;; + esac + str_ref+=${ur_ref[UR_NAME_IDX]} + + local p=${ur_ref[UR_PRETEND_IDX]} + if [[ -n ${p} ]]; then + str_ref+="(${p})" + fi + case ${ur_ref[UR_MODE_IDX]} in + *'=') + str_ref+='=' + ;; + *'?') + str_ref+='?' + ;; + esac +} + +# +# Package depedency specification (or PDS) +# + +# Enumeration describing blocker mode of a package. Self-describing. +declare -gri PDS_NO_BLOCK=0 PDS_WEAK_BLOCK=1 PDS_STRONG_BLOCK=2 + +# Indices to access fields of the package dependency specification: +# PDS_BLOCKS_IDX - a number describing blocker mode (use PDS_NO_BLOCK, +# PDS_WEAK_BLOCK and PDS_STRONG_BLOCK) +# PDS_OP_IDX - a string describing the relational operator, can be +# empty or one of "<", "<=", "=", "=*", "~", ">=", or +# ">" ("=*" is same as "=" with asterisk appended to +# version) +# if empty, the version will also be empty +# PDS_NAME_IDX - a qualified package name (so category/name) +# PDS_VER_IDX - a version of the package, may be empty (if so, +# operator is also empty) +# PDS_SLOT_IDX - a string describing the slot operator, without the +# preceding colon +# PDS_UR_IDX - a name of an array variable containing use +# requirements +declare -gri PDS_BLOCKS_IDX=0 PDS_OP_IDX=1 PDS_NAME_IDX=2 PDS_VER_IDX=3 PDS_SLOT_IDX=4 PDS_UR_IDX=5 + +# Declares empty package definition specifications. Can take flags +# that are passed to declare. Usually only -r or -t make sense to +# pass. Can take several names, just like declare. Takes no +# initializers - this is hardcoded. +function pds_declare() { + struct_declare -ga "${@}" "( ${PDS_NO_BLOCK@Q} '' 'ITS_UNSET' '' '' 'EMPTY_ARRAY' )" +} + +# Unsets package definition specifications, can take several names, +# just like unset. +function pds_unset() { + local use_reqs_name name + for name; do + local -n pds_ref=${name} + + use_reqs_name=${pds_ref[PDS_UR_IDX]} + local -n use_reqs_ref=${use_reqs_name} + ur_unset "${use_reqs_ref[@]}" + unset -n use_reqs_ref + if [[ ${use_reqs_name} != 'EMPTY_ARRAY' ]]; then + unset "${use_reqs_name}" + fi + + unset -n pds_ref + done + unset "${@}" +} + +# Copies package dependency specification into another. +# +# Params: +# +# 1 - package dependency specification to be clobbered +# 2 - the source package dependency specification +function pds_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local -i idx + for idx in PDS_BLOCKS_IDX PDS_OP_IDX PDS_NAME_IDX PDS_VER_IDX PDS_SLOT_IDX; do + to_clobber_ref[idx]=${to_copy_ref[idx]} + done + + if [[ ${to_copy_ref[PDS_UR_IDX]} = 'EMPTY_ARRAY' || ${#to_copy_ref[PDS_UR_IDX]} -eq 0 ]]; then + to_clobber_ref[PDS_UR_IDX]='EMPTY_ARRAY' + else + local pc_ur_array_name + gen_varname pc_ur_array_name + declare -ga "${pc_ur_array_name}=()" + + local -n urs_to_copy_ref=${to_copy_ref[PDS_UR_IDX]} urs_ref=${pc_ur_array_name} + local ur_name pc_ur_name + for ur_name in "${urs_to_copy_ref[@]}"; do + gen_varname pc_ur_name + ur_declare "${pc_ur_name}" + ur_copy "${pc_ur_name}" "${ur_name}" + urs_ref+=( "${pc_ur_name}" ) + done + + to_clobber_ref[PDS_UR_IDX]=${pc_ur_array_name} + fi +} + +# Adds use requirements to the package dependency specification. +# +# Params: +# +# 1 - package dependency specification +# @ - use requirements +function pds_add_urs() { + local -n pds_ref=${1}; shift + # rest are use requirements + + if [[ ${#} -eq 0 ]]; then + return 0 + fi + + local use_reqs_name=${pds_ref[PDS_UR_IDX]} + if [[ ${use_reqs_name} = 'EMPTY_ARRAY' ]]; then + local ura_name + gen_varname ura_name + declare -ga "${ura_name}=()" + pds_ref[PDS_UR_IDX]=${ura_name} + use_reqs_name=${ura_name} + unset ura_name + fi + + local -n use_reqs_ref=${use_reqs_name} + use_reqs_ref+=( "${@}" ) +} + +# Stringifies package dependency specification. +# +# Params: +# +# 1 - package dependency specification +# 2 - name of a variable where the string form will be stored +function pds_to_string() { + local -n pds_ref=${1}; shift + local -n str_ref=${1}; shift + + case ${pds_ref[PDS_BLOCKS_IDX]} in + "${PDS_NO_BLOCK}") + str_ref='' + ;; + "${PDS_WEAK_BLOCK}") + str_ref='!' + ;; + "${PDS_STRONG_BLOCK}") + str_ref='!!' + ;; + esac + local op=${pds_ref[PDS_OP_IDX]} + local v=${pds_ref[PDS_VER_IDX]} + if [[ ${op} = '=*' ]]; then + op='=' + # if there's an op, then we assume version is not empty - so + # version will never end up being just * + v+='*' + fi + str_ref+=${op}${pds_ref[PDS_NAME_IDX]} + if [[ -n ${v} ]]; then + str_ref+=-${v} + fi + local s=${pds_ref[PDS_SLOT_IDX]} + if [[ -n ${s} ]]; then + str_ref+=:${s} + fi + local -n urs_ref=${pds_ref[PDS_UR_IDX]} + if [[ ${#urs_ref[@]} -gt 0 ]]; then + str_ref+='[' + local u ur_str + for u in "${urs_ref[@]}"; do + ur_to_string "${u}" ur_str + str_ref+=${ur_str}, + done + unset ur_str u + str_ref=${str_ref:0:$(( ${#str_ref} - 1 ))}']' + fi +} + +# +# Group. A structure for describing {B,R,I,P,}DEPEND and LICENSE +# fields. Contains items, which can be either package dependency +# specifications or another groups. So it is a recursive structure. +# + +# Enumeration describing type of a group. Self-describing. +declare -gri GROUP_ALL_OF=0 GROUP_ANY_OF=1 + +# Enumeration describing whether items in group are for enabled or disabled USE. Self-describing. +declare -gri GROUP_USE_ENABLED=0 GROUP_USE_DISABLED=1 + +# Indices to access fields of the group: +# GROUP_TYPE_IDX - a number describing a type of the group (use +# GROUP_ALL_OF and GROUP_ANY_OF) +# GROUP_USE_IDX - a USE name of the group, may be empty, and must +# be empty for GROUP_ANY_OF groups +# GROUP_ENABLED_IDX - a number describing mode of the USE, should be +# ignored if USE name is empty (use +# GROUP_USE_ENABLED and GROUP_USE_DISABLED) +# GROUP_ITEMS_IDX - a name of an array containing items +declare -gri GROUP_TYPE_IDX=0 GROUP_USE_IDX=1 GROUP_ENABLED_IDX=2 GROUP_ITEMS_IDX=3 + +# Declares empty groups. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function group_declare() { + struct_declare -ga "${@}" "( ${GROUP_ALL_OF@Q} '' ${GROUP_USE_ENABLED@Q} 'EMPTY_ARRAY' )" +} + +# An empty readonly group. +group_declare -r EMPTY_GROUP + +# Unsets groups, can take several names, just like unset. +function group_unset() { + local -a to_unset=() + local name items_name + for name; do + if [[ ${name} == 'EMPTY_GROUP' ]]; then + continue + fi + + local -n group_ref=${name} + items_name=${group_ref[GROUP_ITEMS_IDX]} + unset -n group_ref + + to_unset+=( "${name}" ) + if [[ ${items_name} == 'EMPTY_ARRAY' ]]; then + continue + fi + to_unset+=( "${items_name}" ) + + local -n items_ref=${items_name} + item_unset "${items_ref[@]}" + unset -n items_ref + done + unset "${to_unset[@]}" +} + +# Copies group into another. +# +# Params: +# +# 1 - group to be clobbered +# 2 - the source group +function group_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local -i idx + for idx in GROUP_TYPE_IDX GROUP_USE_IDX GROUP_ENABLED_IDX; do + to_clobber_ref[idx]=${to_copy_ref[idx]} + done + + if [[ ${to_copy_ref[GROUP_ITEMS_IDX]} = 'EMPTY_ARRAY' || ${#to_copy_ref[GROUP_ITEMS_IDX]} -eq 0 ]]; then + to_clobber_ref[GROUP_ITEMS_IDX]='EMPTY_ARRAY' + else + local gc_items_name + gen_varname gc_items_name + declare -ga "${gc_items_name}=()" + + local -n items_to_copy_ref=${to_copy_ref[GROUP_ITEMS_IDX]} + local -n items_ref=${gc_items_name} + local item_name_to_copy gc_item_name + for item_name_to_copy in "${items_to_copy_ref[@]}"; do + gen_varname gc_item_name + item_declare "${gc_item_name}" + item_copy "${gc_item_name}" "${item_name_to_copy}" + items_ref+=( "${gc_item_name}" ) + done + unset -n items_ref items_to_copy_ref + to_clobber_ref[GROUP_ITEMS_IDX]="${gc_items_name}" + fi +} + +# Adds items to the group. +# +# Params: +# +# 1 - group +# @ - items +function group_add_items() { + local -n group_ref=${1}; shift + # rest are items to add + + if [[ ${#} -eq 0 ]]; then + return 0 + fi + + local items_name=${group_ref[GROUP_ITEMS_IDX]} + if [[ ${items_name} = 'EMPTY_ARRAY' ]]; then + local ia_name + gen_varname ia_name + declare -ga "${ia_name}=()" + group_ref[GROUP_ITEMS_IDX]=${ia_name} + items_name=${ia_name} + unset ia_name + fi + + local -n items_ref=${items_name} + items_ref+=( "${@}" ) +} + +# Stringifies group. +# +# Params: +# +# 1 - group +# 2 - name of a variable where the string form will be stored +function group_to_string() { + local -n group_ref=${1}; shift + local -n str_ref=${1}; shift + + local t=${group_ref[GROUP_TYPE_IDX]} + case ${t} in + "${GROUP_ALL_OF}") + local u=${group_ref[GROUP_USE_IDX]} + if [[ -n ${u} ]]; then + local e=${group_ref[GROUP_ENABLED_IDX]} + case ${e} in + "${GROUP_USE_ENABLED}") + str_ref='' + ;; + "${GROUP_USE_DISABLED}") + str_ref='!' + esac + unset e + str_ref+="${u}? " + else + str_ref='' + fi + unset u + ;; + "${GROUP_ANY_OF}") + str_ref='|| ' + ;; + esac + + str_ref+='( ' + local -n item_names_ref=${group_ref[GROUP_ITEMS_IDX]} + if [[ ${#item_names_ref[@]} -gt 0 ]]; then + local item_name item_str + for item_name in "${item_names_ref[@]}"; do + item_to_string "${item_name}" item_str + str_ref+="${item_str} " + done + unset item_str item_name + fi + str_ref+=')' +} + +# +# Item. A string of :. Mode is a single char and describes +# the data. +# +# Modes: +# "e" - empty, data is meaningless and should be just empty. +# "g" - group, data is a group (a name of a variable containing a +# group) +# "l" - license, data is a name of a license (plain string) +# "p" - package dependency specification, data is pds (a name of a +# variable containing a pds) + +# Declares empty items. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function item_declare() { + struct_declare -g "${@}" 'e:' +} + +# Unsets items, can take several names, just like unset. +function item_unset() { + local name + for name; do + local -n item_ref=${name} + + case ${item_ref} in + e:*) + # noop + : + ;; + g:*) + group_unset "${item_ref:2}" + ;; + l:*) + # noop, license is just a string + ;; + p:*) + pds_unset "${item_ref:2}" + ;; + esac + unset -n item_ref + done + + unset "${@}" +} + +# Copies item into another. +# +# Params: +# +# 1 - item to be clobbered +# 2 - the source item +function item_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local ic_name + local t=${to_copy_ref:0:1} v=${to_copy_ref:2} + case ${t} in + 'e'|'l') + to_clobber_ref=${to_copy_ref} + ;; + 'g') + gen_varname ic_name + group_declare "${ic_name}" + group_copy "${ic_name}" "${v}" + to_clobber_ref="g:${ic_name}" + ;; + 'p') + gen_varname ic_name + pds_declare "${ic_name}" + pds_copy "${ic_name}" "${v}" + to_clobber_ref="p:${ic_name}" + ;; + esac +} + +# Stringifies item. +# +# Params: +# +# 1 - item +# 2 - name of a variable where the string form will be stored +function item_to_string() { + local -n item_ref=${1}; shift + local -n str_ref=${1}; shift + + local t=${item_ref:0:1} + case ${t} in + e) + str_ref='' + ;; + g) + local group_name=${item_ref:2} + local group_str + group_to_string "${group_name}" group_str + str_ref=${group_str} + unset group_str group_name + ;; + l) + str_ref=${item_ref:2} + ;; + p) + local pds_name=${item_ref:2} + local pds_str='' + pds_to_string "${pds_name}" pds_str + str_ref=${pds_str} + unset pds_str pds_name + ;; + esac +} + +# +# Keyword +# +# n - name of keyword +# m - stable (amd64), unstable (~amd64), broken (-amd64), unknown (absent in KEYWORDS) + +# Enumeration describing mode of the keyword. +# KW_STABLE - like "amd64" +# KW_UNSTABLE - like "~amd64" +# KW_BROKEN - like "-amd64" +# KW_UNKNOWN - missing from KEYWORDS entirely +declare -gri KW_STABLE=0 KW_UNSTABLE=1 KW_BROKEN=2 KW_UNKNOWN=3 + +# Indices to access fields of the keyword: +# KW_NAME_IDX - name of the architecture +# KW_LEVEL_IDX - mode of the keyword (use KW_STABLE, KW_UNSTABLE, +# KW_BROKEN and KW_UNKNOWN) +declare -gri KW_NAME_IDX=0 KW_LEVEL_IDX=1 + +# Declares empty keywords. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function kw_declare() { + struct_declare -ga "${@}" "( 'ITS_UNSET' ${KW_UNSTABLE@Q} )" +} + +# Unsets keywords, can take several names, just like unset. +function kw_unset() { + unset "${@}" +} + +# Stringifies keyword. +# +# Params: +# +# 1 - keyword +# 2 - name of a variable where the string form will be stored +function kw_to_string() { + local -n kw_ref=${1}; shift + local -n str_ref=${1}; shift + + local n=${kw_ref[KW_NAME_IDX]} + case ${kw_ref[KW_LEVEL_IDX]} in + "${KW_STABLE}") + str_ref=${n} + ;; + "${KW_UNSTABLE}") + str_ref="~${n}" + ;; + "${KW_BROKEN}") + str_ref="-${n}" + ;; + "${KW_UNKNOWN}") + str_ref='' + ;; + esac +} + +# +# IUSE +# + +# n - name of IUSE flag +# m - 0 or 1 (0 disabled, 1 enabled) + +# Enumeration describing mode of the IUSE. Self-describing. +declare -gri IUSE_DISABLED=0 IUSE_ENABLED=1 + +# Indices to access fields of the IUSE: +# IUSE_NAME_IDX - IUSE name +# IUSE_MODE_IDX - IUSE mode (use IUSE_DISABLED and IUSE_ENABLED) +declare -gri IUSE_NAME_IDX=0 IUSE_MODE_IDX=1 + +# Declares empty IUSEs. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function iuse_declare() { + struct_declare -ga "${@}" "( 'ITS_UNSET' ${IUSE_DISABLED@Q} )" +} + +# Unsets IUSEs, can take several names, just like unset. +function iuse_unset() { + unset "${@}" +} + +# Stringifies IUSE. +# +# Params: +# +# 1 - IUSE +# 2 - name of a variable where the string form will be stored +function iuse_to_string() { + local -n iuse_ref=${1}; shift + local -n str_ref=${1}; shift + + case ${iuse_ref[IUSE_MODE_IDX]} in + "${IUSE_ENABLED}") + str_ref='+' + ;; + "${IUSE_DISABLED}") + str_ref='' + ;; + esac + str_ref+=${iuse_ref[IUSE_NAME_IDX]} +} + +# +# Implementation details +# + +# parse dependency specification format (DSF) + +declare -gri __MCL_DSF_DEPEND=0 __MCL_DSF_LICENSE=1 + +function __mcl_parse_dsf() { + local -i dsf_type=${1}; shift + local dep=${1}; shift + local -n top_group_out_var_name_ref=${1}; shift + + local -a group_stack + local pd_group pd_item pd_group_created='' pd_pds + + gen_varname pd_group + group_declare "${pd_group}" + group_stack+=( "${pd_group}" ) + + local -a tokens + mapfile -t tokens <<<"${dep// /$'\n'}" + local -i last_index + + local token + for token in "${tokens[@]}"; do + if [[ ${token} = '||' ]]; then + # "any of" group, so create the group, make it an item, add + # to current group and mark the new group as current + gen_varname pd_group + group_declare "${pd_group}" + local -n g_ref=${pd_group} + g_ref[GROUP_TYPE_IDX]=${GROUP_ANY_OF} + unset -n g_ref + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="g:${pd_group}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + + group_stack+=( "${pd_group}" ) + pd_group_created=x + elif [[ ${token} =~ ^!?[A-Za-z0-9][A-Za-z0-9+_-]*\?$ ]]; then + # "use" group, so create the group, make it an item, add + # to current group and mark the new group as current + local -i disabled=GROUP_USE_ENABLED + local use=${token%?} + + if [[ ${use} = '!'* ]]; then + disabled=GROUP_USE_DISABLED + use=${use:1} + fi + + gen_varname pd_group + group_declare "${pd_group}" + local -n g_ref=${pd_group} + g_ref[GROUP_TYPE_IDX]=${GROUP_ALL_OF} + g_ref[GROUP_USE_IDX]=${use} + g_ref[GROUP_ENABLED_IDX]=${disabled} + unset -n g_ref + + unset use disabled + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="g:${pd_group}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + + group_stack+=( "${pd_group}" ) + pd_group_created=x + elif [[ ${token} = '(' ]]; then + # beginning of a group; usually it is already created, + # because it is an "any of" or "use" group, but it is + # legal to specify just a "all of" group + if [[ -n ${pd_group_created} ]]; then + pd_group_created= + else + gen_varname pd_group + group_declare "${pd_group}" + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="g:${pd_group}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + + group_stack+=( "${pd_group}" ) + fi + elif [[ ${token} = ')' ]]; then + # end of a group, pop it from the stack + last_index=$(( ${#group_stack[@]} - 1 )) + unset "group_stack[${last_index}]" + elif [[ ${token} =~ ^[A-Za-z0-9_][A-Za-z0-9+_.-]*$ ]]; then + # license + if [[ dsf_type -ne __MCL_DSF_LICENSE ]]; then + fail "license tokens are only allowed for LICENSE keys (token: ${token@Q})" + fi + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="l:${token}" + unset -n i_ref + group_add_items "${group_stack[-1]}" "${pd_item}" + elif [[ ${token} =~ ^!?!?(<|<=|=|~|>=|>)?[A-Za-z0-9_][A-Za-z0-9+_.-]*/[A-Za-z0-9_] ]]; then + # pds + if [[ dsf_type -ne __MCL_DSF_DEPEND ]]; then + fail "package dependency specification is only allowed for DEPEND-like keys (token: ${token@Q})" + fi + + local -i blocks=PDS_NO_BLOCK + local operator='' name='' version='' slot='' + local -a use_requirements=() + + case ${token} in + '!!'*) + blocks=PDS_STRONG_BLOCK + token=${token:2} + ;; + '!'*) + blocks=PDS_WEAK_BLOCK + token=${token:1} + ;; + esac + + case ${token} in + '<='*|'>='*) + operator=${token:0:2} + token=${token:2} + ;; + '<'*|'='*|'~'*|'>'*) + operator=${token:0:1} + token=${token:1} + ;; + esac + + if [[ ${token} = *']' ]]; then + local use_reqs_string=${token#*'['} + use_reqs_string=${use_reqs_string%']'} + token=${token%"[${use_reqs_string}]"} + local -a use_reqs + mapfile -t use_reqs <<<"${use_reqs_string//,/$'\n'}" + + unset use_reqs_string + + local ur name mode pretend pd_ur + for ur in "${use_reqs[@]}"; do + name='' + mode='' + pretend='' + case ${ur} in + '-'*) + mode='-' + ur=${ur:1} + ;; + '!'*) + mode='!' + ur=${ur:1} + ;; + esac + if [[ ${mode} != '-' ]]; then + case ${ur} in + *'=') + mode+='=' + ur=${ur%'='} + ;; + *'?') + mode+='?' + ur=${ur%'?'} + ;; + esac + fi + if [[ -z ${mode} ]]; then + mode='+' + fi + if [[ ${ur} =~ \(([+-])\)$ ]]; then + pretend=${BASH_REMATCH[1]} + ur=${ur%"(${pretend})"} + fi + name=${ur} + gen_varname pd_ur + ur_declare "${pd_ur}" + local -n u_ref=${pd_ur} + u_ref[UR_NAME_IDX]=${name} + u_ref[UR_MODE_IDX]=${mode} + u_ref[UR_PRETEND_IDX]=${pretend} + unset -n u_ref + use_requirements+=( "${pd_ur}" ) + done + unset pd_ur pretend mode name ur use_reqs + fi + + if [[ ${token} = *:* ]]; then + slot=${token#*:} + token=${token%:"${slot}"} + fi + + if [[ ${token} = *'*' ]]; then + operator='=*' + token=${token%'*'} + fi + local ver_ere_with_dash="-(${VER_ERE_UNBOUNDED})$" + if [[ ${token} =~ ${ver_ere_with_dash} ]]; then + version=${BASH_REMATCH[1]} + token=${token%"-${version}"} + fi + name=${token} + + gen_varname pd_pds + pds_declare "${pd_pds}" + local -n p_ref=${pd_pds} + p_ref[PDS_BLOCKS_IDX]=${blocks} + p_ref[PDS_OP_IDX]=${operator} + p_ref[PDS_NAME_IDX]=${name} + p_ref[PDS_VER_IDX]=${version} + p_ref[PDS_SLOT_IDX]=${slot} + unset -n p_ref + pds_add_urs "${pd_pds}" "${use_requirements[@]}" + unset use_requirements slot version name operator blocks + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="p:${pd_pds}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + else + fail "unknown token ${token@Q}" + fi + done + + if [[ ${#group_stack[@]} -ne 1 ]]; then + fail "botched parsing, group stack has ${#group_stack[@]} groups instead of 1" + fi + + top_group_out_var_name_ref=${group_stack[0]} + + if pkg_debug_enabled; then + local pd_group_str + group_to_string "${group_stack[0]}" pd_group_str + pkg_debug_print "dsf: ${pd_group_str}" + fi +} + +function __mcl_parse_eclasses() { + local eclasses_string=${1}; shift + local -n eclasses_out_var_name_ref=${1}; shift + + local eclasses_var_name + gen_varname eclasses_var_name + declare -ga "${eclasses_var_name}=()" + local -n eclasses_ref=${eclasses_var_name} + + local -a tokens + mapfile -t tokens <<<"${eclasses_string//$'\t'/$'\n'}" + + local token + local -i eclass_name_now=1 + for token in "${tokens[@]}"; do + if [[ eclass_name_now -eq 1 ]]; then + eclasses_ref+=( "${token}" ) + fi + eclass_name_now=$((eclass_name_now ^ 1)) + done + eclasses_out_var_name_ref=${eclasses_var_name} + + if pkg_debug_enabled; then + local joined_eclasses_string + join_by joined_eclasses_string ' ' "${eclasses_ref[@]}" + pkg_debug_print "eclasses: ${joined_eclasses_string}" + fi +} + +function __mcl_parse_keywords() { + local keywords_string=${1}; shift + local -n keywords_out_var_name_ref=${1}; shift + # rest are architectures + + local keywords_var_name + gen_varname keywords_var_name + declare -ga "${keywords_var_name}=()" + local -n keywords_ref=${keywords_var_name} + + local -A keywords_set=() + + local -a tokens + mapfile -t tokens <<<"${keywords_string// /$'\n'}" + local token + for token in "${tokens[@]}"; do + keywords_set["${token}"]=x + done + + local has_hyphen_star=${keywords_set['-*']:-} + local arch mark kw_level_pair kw kw_name + local -i level + for arch; do + for kw_level_pair in "${arch}@${KW_STABLE}" "~${arch}@${KW_UNSTABLE}" "-${arch}@${KW_BROKEN}"; do + kw=${kw_level_pair%@*} + level=${kw_level_pair#*@} + mark=${keywords_set["${kw}"]:-} + if [[ -n ${mark} ]]; then + gen_varname kw_name + kw_declare "${kw_name}" + local -n k_ref=${kw_name} + k_ref[KW_NAME_IDX]=${arch} + k_ref[KW_LEVEL_IDX]=${level} + unset -n k_ref + keywords_ref+=( "${kw_name}" ) + break + fi + done + if [[ -z ${mark} ]]; then + gen_varname kw_name + kw_declare "${kw_name}" + local -n k_ref=${kw_name} + k_ref[KW_NAME_IDX]=${arch} + if [[ -n ${has_hyphen_star} ]]; then + k_ref[KW_LEVEL_IDX]=${KW_BROKEN} + else + k_ref[KW_LEVEL_IDX]=${KW_UNKNOWN} + fi + unset -n k_ref + keywords_ref+=( "${kw_name}" ) + fi + done + keywords_out_var_name_ref=${keywords_var_name} + + if pkg_debug_enabled; then + local -a all_kws_strings=() + local kw_name pk_kw_str + local joined_kws_string + for kw_name in "${keywords_ref[@]}"; do + kw_to_string "${kw_name}" pk_kw_str + all_kws_strings+=( "${pk_kw_str}" ) + done + join_by joined_kws_string ' ' "${all_kws_strings[@]}" + pkg_debug_print "keywords: ${joined_kws_string}" + fi +} + +function __mcl_parse_iuse() { + local iuse_string=${1}; shift + local -n iuse_out_var_name_ref=${1}; shift + + local iuse_var_name + gen_varname iuse_var_name + declare -ga "${iuse_var_name}=()" + local -n iuse_ref=${iuse_var_name} + + local -a tokens + mapfile -t tokens <<<"${iuse_string// /$'\n'}" + local token pi_iuse + for token in "${tokens[@]}"; do + gen_varname pi_iuse + iuse_declare "${pi_iuse}" + local -n i_ref=${pi_iuse} + if [[ ${token} = '+'* ]]; then + i_ref[IUSE_MODE_IDX]=${IUSE_ENABLED} + token=${token:1} + fi + i_ref[IUSE_NAME_IDX]=${token} + unset -n i_ref + iuse_ref+=( "${pi_iuse}" ) + done + + iuse_out_var_name_ref=${iuse_var_name} + + if pkg_debug_enabled; then + local -a all_iuse_strings=() + local iuse_name pi_iuse_str + local joined_iuse_string + for iuse_name in "${iuse_ref[@]}"; do + iuse_to_string "${iuse_name}" pi_iuse_str + all_iuse_strings+=( "${pi_iuse_str}" ) + done + join_by joined_iuse_string ' ' "${all_iuse_strings[@]}" + pkg_debug_print "IUSE: ${joined_iuse_string}" + fi +} + +function __mcl_unset_array() { + local array_name=${1}; shift + local item_unset_func=${1}; shift + + if [[ ${array_name} = EMPTY_ARRAY ]]; then + return 0 + fi + local -n array_ref=${array_name} + "${item_unset_func}" "${array_ref[@]}" + unset -n array_ref + unset "${array_name}" +} + +fi diff --git a/pkg_auto/impl/mvm.sh b/pkg_auto/impl/mvm.sh index 74a8756493..1a56b6a22a 100644 --- a/pkg_auto/impl/mvm.sh +++ b/pkg_auto/impl/mvm.sh @@ -133,7 +133,6 @@ function mvm_declare() { storage_map_ref=() local -n mvm_ref=${mvm_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable mvm_ref=( ['name']="${mvm_var_name}" ['constructor']="${constructor}" @@ -193,7 +192,6 @@ function __mvm_mvc_name() { mvc_name_var_name=${1}; shift local -n mvc_name_ref=${mvc_name_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable mvc_name_ref="mvm_${name}_mvc_${counter}" } @@ -246,10 +244,8 @@ function mvm_c_get_extra() { local extras_map_var_name extras_map_var_name=${mvm['extras']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n extras_map_ref=${extras_map_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable extra_ref=${extras_map_ref["${extra}"]:-} } @@ -273,10 +269,8 @@ function mvm_c_get() { local storage_map_var_name storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable value_ref=${storage_map_ref["${key}"]:-} } @@ -290,7 +284,6 @@ function __mvm_c_make_new_mvc() { name=${mvm['name']} counter=${mvm['counter']} storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} __mvm_mvc_name "${name}" "${counter}" "${mvc_name_var_name}" @@ -348,7 +341,6 @@ function mvm_c_remove() { local storage_map_var_name storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} if [[ -z ${storage_map_ref["${key}"]:-} ]]; then @@ -387,7 +379,6 @@ function mvm_c_iterate() { local storage_map_var_name helper storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} helper=${mvm['iteration_helper']} @@ -467,7 +458,6 @@ function mvm_mvc_array_destructor() { function mvm_mvc_array_adder() { local array_var_name array_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n array_ref=${array_var_name} array_ref+=( "${@}" ) @@ -481,7 +471,6 @@ function mvm_mvc_array_iteration_helper() { callback=${1}; shift # rest are extra args passed to cb - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n array_ref=${array_var_name} "${callback}" "${@}" "${key}" "${array_var_name}" "${array_ref[@]}" } @@ -512,11 +501,9 @@ function mvm_mvc_map_destructor() { function mvm_mvc_map_adder() { local map_var_name map_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n map_ref=${map_var_name} while [[ ${#} -gt 1 ]]; do - # shellcheck disable=SC2034 # it's a reference to external variable map_ref["${1}"]=${2} shift 2 done @@ -532,7 +519,6 @@ function mvm_mvc_set_constructor() { declare -g -A "${set_var_name}" - # shellcheck disable=SC2178 # shellcheck does not grok refs local -n set_ref=${set_var_name} set_ref=() } @@ -548,7 +534,6 @@ function mvm_mvc_set_adder() { local set_var_name set_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n set_ref=${set_var_name} while [[ ${#} -gt 0 ]]; do set_ref["${1}"]=x @@ -565,7 +550,6 @@ function mvm_mvc_set_iteration_helper() { callback=${1}; shift # rest are extra args passed to cb - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n set_ref=${set_var_name} "${callback}" "${@}" "${key}" "${set_var_name}" "${!set_ref[@]}" } diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index d96bc60765..dbab438027 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -51,7 +51,9 @@ __PKG_AUTO_LIB_SH_INCLUDED__=x source "$(dirname "${BASH_SOURCE[0]}")/util.sh" source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" +source "${PKG_AUTO_IMPL_DIR}/debug.sh" source "${PKG_AUTO_IMPL_DIR}/gentoo_ver.sh" +source "${PKG_AUTO_IMPL_DIR}/md5_cache_diff_lib.sh" # Sets up the workdir using the passed config. The config can be # created basing on the config_template file or using the @@ -72,15 +74,14 @@ function setup_workdir_with_config() { workdir=${1}; shift config_file=${1}; shift - local cfg_scripts cfg_aux cfg_reports cfg_old_base cfg_new_base + local cfg_scripts cfg_aux cfg_reports cfg_old_base cfg_new_base cfg_sdk_image_override local -a cfg_cleanups cfg_debug_packages - local -A cfg_overrides # some defaults cfg_old_base='origin/main' cfg_new_base='' cfg_cleanups=('ignore') - cfg_overrides=() + cfg_sdk_image_override='' cfg_debug_packages=() local line key value swwc_stripped var_name arch @@ -95,25 +96,20 @@ function setup_workdir_with_config() { case ${key} in scripts|aux|reports) var_name="cfg_${key//-/_}" - local -n var=${var_name} - var=$(realpath "${value}") - unset -n var + local -n var_ref=${var_name} + var_ref=$(realpath "${value}") + unset -n var_ref ;; - old-base|new-base) + old-base|new-base|sdk-image-override) var_name="cfg_${key//-/_}" - local -n var=${var_name} - var=${value} - unset -n var + local -n var_ref=${var_name} + var_ref=${value} + unset -n var_ref ;; cleanups|debug-packages) var_name="cfg_${key//-/_}" mapfile -t "${var_name}" <<<"${value//,/$'\n'}" ;; - *-sdk-img) - arch=${key%%-*} - # shellcheck disable=SC2034 # used by name below - cfg_overrides["${arch}"]=${value} - ;; esac done < <(cat_meaningful "${config_file}") if [[ -z "${cfg_new_base}" ]]; then @@ -131,8 +127,10 @@ function setup_workdir_with_config() { add_cleanup "rm -f ${WORKDIR@Q}/config" cp -a "${config_file}" "${WORKDIR}/config" setup_worktrees_in_workdir "${cfg_scripts}" "${cfg_old_base}" "${cfg_new_base}" "${cfg_reports}" "${cfg_aux}" - override_sdk_image_names cfg_overrides - add_debug_packages "${cfg_debug_packages[@]}" + if [[ -n ${cfg_sdk_image_override} ]]; then + override_sdk_image_name "${cfg_sdk_image_override}" + fi + pkg_debug_add "${cfg_debug_packages[@]}" } # Goes over the list of automatically updated packages and synces them @@ -177,10 +175,9 @@ function save_new_state() { local branch_name branch_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" info "saving new state to branch ${branch_name}" - # shellcheck disable=SC2153 # SCRIPTS is not a misspelling, it comes from globals file git -C "${SCRIPTS}" branch --force "${branch_name}" "${NEW_STATE_BRANCH}" } @@ -244,53 +241,15 @@ function setup_worktrees_in_workdir() { extend_globals_file "${scripts}" "${old_state}" "${new_state}" "${reports_dir}" "${aux_dir}" } -# Adds overridden SDK image names to the globals file. +# Adds an overridden SDK image name to the globals file. # # Params: # -# 1 - name of a map variable; should be a mapping of architecture to -# the image name -function override_sdk_image_names() { - local -n overrides_map_ref=${1} +# 1 - image name +function override_sdk_image_name() { + local image_name=${1}; shift - if [[ ${#overrides_map_ref[@]} -eq 0 ]]; then - return 0 - fi - - local arch image_name upcase_arch - local -a lines - lines=() - for arch in "${!overrides_map_ref[@]}"; do - image_name=${overrides_map_ref["${arch}"]} - upcase_arch=${arch^^} - if [[ ${#lines[@]} -eq 0 ]]; then - # separate overrides from initial values - lines+=( '' ) - fi - lines+=( "${upcase_arch}_SDK_IMAGE=${image_name@Q}" ) - done - append_to_globals "${lines[@]}" -} - -# Adds information about packages to be debugged to the globals file. -# -# Params: -# -# @ - a list of packages to be debugged -function add_debug_packages() { - local -a prepared lines - prepared=( "${@@Q}" ) - prepared=( "${prepared[@]/#/' ['}" ) - prepared=( "${prepared[@]/%/']=x'}" ) - lines=( - '' - 'local -A DEBUG_PACKAGES' - '' - 'DEBUG_PACKAGES=(' - "${prepared[@]}" - ')' - ) - append_to_globals "${lines[@]}" + append_to_globals "SDK_IMAGE=${image_name@Q}" } # Appends passed lines to the globals file. @@ -322,7 +281,7 @@ function process_profile_updates_directory() { local -a ppud_ordered_names get_ordered_update_filenames ppud_ordered_names - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local bf ps_f co_f pkg f line old new @@ -332,9 +291,7 @@ function process_profile_updates_directory() { for bf in "${ppud_ordered_names[@]}"; do # coreos-overlay updates may overwrite updates from # portage-stable, but only from the file of the same name - # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file ps_f=${NEW_PORTAGE_STABLE}/profiles/updates/${bf} - # shellcheck disable=SC2153 # NEW_COREOS_OVERLAY is not a misspelling, it comes from globals file co_f=${NEW_COREOS_OVERLAY}/profiles/updates/${bf} for f in "${ps_f}" "${co_f}"; do if [[ ! -f ${f} ]]; then @@ -375,7 +332,7 @@ function process_profile_updates_directory() { function get_ordered_update_filenames() { local ordered_names_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A names_set=() @@ -405,12 +362,12 @@ function get_ordered_update_filenames() { # 3 - old name # 4 - new name function update_rename_maps() { - local -n ft_map=${1}; shift + local -n ft_map_ref=${1}; shift local tf_set_mvm_var_name=${1}; shift local old_name=${1}; shift local new_name=${1}; shift - local prev_new_name=${ft_map["${old_name}"]:-} + local prev_new_name=${ft_map_ref["${old_name}"]:-} if [[ -n ${prev_new_name} ]] && [[ ${prev_new_name} != "${new_name}" ]]; then fail_lines \ @@ -423,18 +380,18 @@ function update_rename_maps() { local urm_set_var_name mvm_get "${tf_set_mvm_var_name}" "${old_name}" urm_set_var_name if [[ -n ${urm_set_var_name} ]]; then - local -n old_set=${urm_set_var_name} - new_set+=( "${!old_set[@]}" ) - unset -n old_set + local -n old_set_ref=${urm_set_var_name} + new_set+=( "${!old_set_ref[@]}" ) + unset -n old_set_ref fi new_set+=( "${old_name}" ) mvm_add "${tf_set_mvm_var_name}" "${new_name}" "${new_set[@]}" local old for old in "${new_set[@]}"; do - ft_map["${old}"]=${new_name} + ft_map_ref["${old}"]=${new_name} done - unset -n ft_map + unset -n ft_map_ref } # Sets up a worktree and necessary cleanups. @@ -563,26 +520,20 @@ NEW_STATE_PACKAGES_LIST="\${NEW_STATE}/.github/workflows/portage-stable-packages AUX_DIR=${aux_dir@Q} EOF - # shellcheck disable=SC1090 # generated file + # shellcheck source=for-shellcheck/globals source "${globals_file}" local last_nightly_version_id last_nightly_build_id - # shellcheck disable=SC1091,SC2153 # sourcing generated file, NEW_STATE is not misspelled + # shellcheck source=for-shellcheck/version.txt last_nightly_version_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") - local -a locals definitions - locals=() - definitions=() - local sdk_image_name + local -a locals=() definitions=() + local sdk_image_name sdk_image_var_name=SDK_IMAGE sdk_image_name="ghcr.io/flatcar/flatcar-sdk-all:${last_nightly_version_id}-${last_nightly_build_id}" - local arch sdk_image_var_name - for arch in "${ARCHES[@]}"; do - sdk_image_var_name="${arch^^}_SDK_IMAGE" - locals+=( "${sdk_image_var_name@Q}" ) - definitions+=( "${sdk_image_var_name}=${sdk_image_name@Q}" ) - done + locals+=( "${sdk_image_var_name@Q}" ) + definitions+=( "${sdk_image_var_name}=${sdk_image_name@Q}" ) append_to_globals \ '' \ @@ -594,7 +545,6 @@ EOF local packages_file tag filename stripped old for arch in "${ARCHES[@]}"; do - # shellcheck disable=SC2153 # AUX_DIR is not a misspelling, it comes from globals file for packages_file in "${AUX_DIR}/${arch}/"*_packages.txt; do filename=${packages_file##*/} stripped=${filename%_packages.txt} @@ -656,18 +606,15 @@ EOF function setup_git_env() { local bot_name bot_email role what - # shellcheck disable=SC2034 # used indirectly bot_name='Flatcar Buildbot' - # shellcheck disable=SC2034 # used indirectly bot_email='buildbot@flatcar-linux.org' for role in AUTHOR COMMITTER; do for what in name email; do - local -n var="GIT_${role}_${what^^}" - local -n value="bot_${what}" - # shellcheck disable=SC2034 # it's a reference to external variable - var=${value} - unset -n value - unset -n var + local -n var_ref="GIT_${role}_${what^^}" + local -n value_ref="bot_${what}" + var_ref=${value_ref} + unset -n value_ref + unset -n var_ref done done } @@ -686,7 +633,7 @@ function run_sync() { missing_in_scripts=() missing_in_gentoo=() - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -x "${GIT_ENV_VARS[@]}" @@ -697,7 +644,6 @@ function run_sync() { local package while read -r package; do - # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file if [[ ! -e "${NEW_PORTAGE_STABLE}/${package}" ]]; then # If this happens, it means that the package was moved to overlay # or dropped, the list ought to be updated. @@ -716,7 +662,6 @@ function run_sync() { fi packages_to_update+=( "${package}" ) done < <(cat_meaningful "${NEW_STATE_PACKAGES_LIST}") - # shellcheck disable=SC2153 # SYNC_SCRIPT is not a misspelling env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${packages_to_update[@]}" save_missing_in_scripts "${missing_in_scripts[@]}" @@ -854,7 +799,7 @@ function handle_missing_in_scripts() { hmis_missing_in_scripts=() load_missing_in_scripts hmis_missing_in_scripts - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" if [[ ${#hmis_missing_in_scripts[@]} -eq 0 ]]; then @@ -869,7 +814,6 @@ function handle_missing_in_scripts() { join_by missing_re '\|' "${missing_in_scripts[@]}" add_cleanup "rm -f ${dir@Q}/pkg_list" xgrep --invert-match --line-regexp --fixed-strings --regexp="${missing_re}" "${NEW_STATE_PACKAGES_LIST}" >"${dir}/pkg_list" - # shellcheck disable=SC2153 # PKG_LIST_SORT_SCRIPT is not a misspelling "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" local -x "${GIT_ENV_VARS[@]}" @@ -908,11 +852,10 @@ function lines_to_file() { # # @ - lines to add function manual() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'manual work needed:' "${@}" - # shellcheck disable=SC2153 # REPORTS_DIR is not a misspelling, it comes from globals file lines_to_file "${REPORTS_DIR}/manual-work-needed" "${@}" } @@ -923,7 +866,7 @@ function manual() { # # @ - lines to add function pkg_warn() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'pkg warn:' "${@}" @@ -937,7 +880,7 @@ function pkg_warn() { # # @ - lines to add function devel_warn() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'developer warn:' "${@}" @@ -959,7 +902,7 @@ function handle_missing_in_gentoo() { return 0; fi - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A hmig_rename_map=() @@ -1030,23 +973,14 @@ function process_listings() { local pkg_to_tags_mvm_var_name pkg_to_tags_mvm_var_name=${1} - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - local ver_ere pkg_ere - # VER_ERE comes from gentoo_ver.sh - ver_ere=${VER_ERE} - # regexp begins with ^ and ends with $, so strip them - ver_ere=${ver_ere#'^'} - ver_ere=${ver_ere%'$'} - pkg_ere='[a-z0-9]*-?[a-z0-9]*/[a-z0-9A-Z_+-]*' - #mvm_debug_enable pl_pkg_to_tags_set_mvm mvm_declare pl_pkg_to_tags_set_mvm mvm_mvc_set local arch kind file listing pkg for arch in "${ARCHES[@]}"; do - # shellcheck disable=SC2153 # LISTING_KINDS is not a misspelling, it comes from globals file for kind in "${!LISTING_KINDS[@]}"; do file=${LISTING_KINDS["${kind}"]} listing="${AUX_DIR}/${arch}/${file}" @@ -1060,38 +994,25 @@ function process_listings() { # acct-group/adm-0-r2::portage-stable while read -r pkg; do pkg_debug_enable "${pkg}" - pkg_debug "processing listings: adding tag ${kind^^}" + pkg_debug "processing listing ${arch}/${file}: adding tag ${kind^^}" pkg_debug_disable mvm_add pl_pkg_to_tags_set_mvm "${pkg}" "${kind^^}" - done < <(sed -E -e 's#^('"${pkg_ere}"')-'"${ver_ere}"'::.*#\1#' "${listing}") + # VER_ERE_UNBOUNDED and PKG_ERE_UNBOUNDED come from gentoo_ver.sh + done < <(sed -E -e 's#^('"${PKG_ERE_UNBOUNDED}"')-'"${VER_ERE_UNBOUNDED}"'::.*#\1#' "${listing}") done done mvm_iterate pl_pkg_to_tags_set_mvm set_mvm_to_array_mvm_cb "${pkg_to_tags_mvm_var_name}" mvm_unset pl_pkg_to_tags_set_mvm #mvm_debug_disable pl_pkg_to_tags_set_mvm - if pkg_debug_possible; then - mvm_iterate "${pkg_to_tags_mvm_var_name}" debug_dump_package_tags "${pkg_to_tags_mvm_var_name}" - fi -} - -# A debug function that prints the package tags. Used as a callback to -# mvm_iterate. -# -# Params: -# -# 1 - name of the array mvm variable (extra arg of the callback) -# 2 - name of the package -# 3 - name of the array variable holding tags (unused) -# @ - tags -function debug_dump_package_tags() { - local map_name=${1}; shift - local pkg=${1}; shift - shift # we don't care about array variable name - # rest are array elements, which are tags - pkg_debug_enable "${pkg}" - pkg_debug "tags for ${pkg} stored in ${map_name}: ${*}" - pkg_debug_disable + local -a pl_debug_pkgs pl_tags_array_name + pkg_debug_packages pl_debug_pkgs + for pkg in "${pl_debug_pkgs[@]}"; do + mvm_get "${pkg_to_tags_mvm_var_name}" "${pkg}" pl_tags_array_name + local -n tags_ref=${pl_tags_array_name:-EMPTY_ARRAY} + pkg_debug_print_c "${pkg}" "tags stored in ${pkg_to_tags_mvm_var_name}: ${tags_ref[*]}" + unset -n tags_ref + done } # A callback to mvm_iterate that turns a set mvm to an array mvm. It @@ -1135,84 +1056,101 @@ function set_mvm_to_array_mvm_cb() { # stored in salvaged-reports subdirectory of the reports directory. # Otherwise they will end up in reports-from-sdk subdirectory. function generate_sdk_reports() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" mkdir "${WORKDIR}/pkg-reports" - local arch sdk_image_var_name sdk_image_name + if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${SDK_IMAGE}"; then + fail "No SDK image named ${SDK_IMAGE@Q} available locally, pull it before running this script" + fi + local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch - local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy - local -a report_files run_sdk_container_args - for arch in "${ARCHES[@]}"; do - sdk_image_var_name="${arch^^}_SDK_IMAGE" - sdk_image_name=${!sdk_image_var_name} - if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${sdk_image_name}"; then - fail "No SDK image named ${sdk_image_name@Q} available locally, pull it before running this script" + local pkg_auto_copy rv + local sdk_reports_dir top_dir dir entry full_path + local -a dir_queue all_dirs all_files + + for sdk_run_kind in "${WHICH[@]}"; do + state_var_name="${sdk_run_kind^^}_STATE" + sdk_run_state="${!state_var_name}_sdk_run" + state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" + sdk_run_state_branch="${!state_branch_var_name}-sdk-run" + + add_cleanup \ + "git -C ${sdk_run_state@Q} reset --hard HEAD" \ + "git -C ${sdk_run_state@Q} clean -ffdx" \ + "git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ + "git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" + git -C "${SCRIPTS}" \ + worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}" + + pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX") + add_cleanup "rm -rf ${pkg_auto_copy@Q}" + cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}" + local -a run_sdk_container_args=( + -C "${SDK_IMAGE}" + -n "pkg-${sdk_run_kind}" + -U + -m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto" + --rm + ./pkg_auto/inside_sdk_container.sh pkg-reports "${ARCHES[@]}" + ) + rv=0 + env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?} + unset run_sdk_container_args + if [[ ${rv} -ne 0 ]]; then + local salvaged_dir + salvaged_dir="${REPORTS_DIR}/salvaged-reports" + { + info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue" + info + for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do + info "from ${file}:" + echo + cat "${file}" + echo + done + info + info 'whatever reports generated by the failed run are saved in' + info "${salvaged_dir@Q} directory" + info + } >&2 + rm -rf "${salvaged_dir}" + cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}" + unset salvaged_dir + fail "copying done, stopping now" fi - - # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file - for sdk_run_kind in "${WHICH[@]}"; do - state_var_name="${sdk_run_kind^^}_STATE" - sdk_run_state="${!state_var_name}_sdk_run_${arch}" - state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" - sdk_run_state_branch="${!state_branch_var_name}-sdk-run-${arch}" - - add_cleanup \ - "git -C ${sdk_run_state@Q} reset --hard HEAD" \ - "git -C ${sdk_run_state@Q} clean -ffdx" \ - "git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ - "git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" - git -C "${SCRIPTS}" \ - worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}" - - pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX") - add_cleanup "rm -rf ${pkg_auto_copy@Q}" - cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}" - local -a run_sdk_container_args=( - -C "${sdk_image_name}" - -n "pkg-${sdk_run_kind}-${arch}" - -a "${arch}" - -U - -m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto" - --rm - ./pkg_auto/inside_sdk_container.sh "${arch}" pkg-reports - ) - rv=0 - env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?} - if [[ ${rv} -ne 0 ]]; then - { - salvaged_dir="${REPORTS_DIR}/salvaged-reports" - info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue" - info - for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do - info "from ${file}:" - echo - cat "${file}" - echo - done - info - info 'whatever reports generated by the failed run are saved in' - info "${salvaged_dir@Q} directory" - info - } >&2 - rm -rf "${salvaged_dir}" - cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}" - fail "copying done, stopping now" + sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}" + top_dir="${sdk_run_state}/pkg-reports" + dir_queue=( "${top_dir}" ) + all_dirs=() + all_files=() + while [[ ${#dir_queue[@]} -gt 0 ]]; do + dir=${dir_queue[0]} + dir_queue=( "${dir_queue[@]:1}" ) + entry=${dir#"${top_dir}"} + if [[ -z ${entry} ]]; then + all_dirs=( "${sdk_reports_dir}" "${all_dirs[@]}" ) + else + entry=${entry#/} + all_dirs=( "${sdk_reports_dir}/${entry}" "${all_dirs[@]}" ) fi - sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}-${arch}" - report_files=() - for full_file in "${sdk_run_state}/pkg-reports/"*; do - file=${full_file##"${sdk_run_state}/pkg-reports/"} - report_files+=( "${sdk_reports_dir}/${file}" ) + for full_path in "${dir}/"*; do + if [[ -d ${full_path} ]]; then + dir_queue+=( "${full_path}" ) + else + entry=${full_path##"${top_dir}/"} + all_files+=( "${sdk_reports_dir}/${entry}" ) + fi done - add_cleanup \ - "rm -f ${report_files[*]@Q}" \ - "rmdir ${sdk_reports_dir@Q}" - mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}" done + add_cleanup \ + "rm -f ${all_files[*]@Q}" \ + "rmdir ${all_dirs[*]@Q}" + mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}" done + cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk" } @@ -1240,7 +1178,6 @@ function pkginfo_name() { report=${1}; shift local -n pi_name_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable pi_name_ref="pkginfo_${which}_${arch}_${report//-/_}_pimap_mvm" } @@ -1256,7 +1193,6 @@ function pkginfo_destructor() { # Adder callback used by mvm_declare for pkginfo mvms. function pkginfo_adder() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n map_ref=${1}; shift local mark @@ -1338,7 +1274,7 @@ function pkginfo_c_process_file() { local -n pkg_set_ref=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local which arch report @@ -1346,19 +1282,35 @@ function pkginfo_c_process_file() { mvm_c_get_extra 'arch' arch mvm_c_get_extra 'report' report + local report_file + case ${report}:${arch} in + "${SDK_PKGS}:arm64") + # short-circuit it, there's no arm64 sdk + return 0 + ;; + "${SDK_PKGS}:amd64") + report_file="${WORKDIR}/pkg-reports/${which}/${report}" + ;; + "${BOARD_PKGS}:"*) + report_file="${WORKDIR}/pkg-reports/${which}/${arch}-${report}" + ;; + *) + local c=${report}:${arch} + devel_warn "unknown report-architecture combination (${c@Q})" + return 0 + esac + local pkg version_slot throw_away v s - # shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else while read -r pkg version_slot throw_away; do pkg_debug_enable "${pkg}" pkg_debug "${which} ${arch} ${report}: ${version_slot}" v=${version_slot%%:*} s=${version_slot##*:} mvm_c_add "${pkg}" "${s}" "${v}" - # shellcheck disable=SC2034 # it's a reference to external variable pkg_set_ref["${pkg}"]='x' mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}" pkg_debug_disable - done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" + done <"${report_file}" } # Gets a profile of the pkginfo mvm. The "profile" is a confusing @@ -1397,7 +1349,7 @@ function read_reports() { all_pkgs_var_name=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A rr_all_packages_set @@ -1418,7 +1370,7 @@ function read_reports() { # Destroys the pkginfo maps for all the reports. function unset_report_mvms() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local arch which report @@ -1453,9 +1405,7 @@ function ver_min_max() { max=${v} fi done - # shellcheck disable=SC2034 # it's a reference to external variable min_ref=${min} - # shellcheck disable=SC2034 # it's a reference to external variable max_ref=${max} } @@ -1489,8 +1439,8 @@ function consistency_check_for_package() { local -A empty_map empty_map=() - local -n slot_version1_map=${ccfp_slot_version1_map_var_name:-empty_map} - local -n slot_version2_map=${ccfp_slot_version2_map_var_name:-empty_map} + local -n slot_version1_map_ref=${ccfp_slot_version1_map_var_name:-empty_map} + local -n slot_version2_map_ref=${ccfp_slot_version2_map_var_name:-empty_map} local ccfp_slots_set_var_name mvm_get "${pkg_slots_set_mvm_var_name}" "${pkg}" ccfp_slots_set_var_name @@ -1508,8 +1458,8 @@ function consistency_check_for_package() { local s v1 v2 ccfp_min ccfp_max mm pkg_debug "all slots iterated over: ${!slots_set_ref[*]}" for s in "${!slots_set_ref[@]}"; do - v1=${slot_version1_map["${s}"]:-} - v2=${slot_version2_map["${s}"]:-} + v1=${slot_version1_map_ref["${s}"]:-} + v2=${slot_version2_map_ref["${s}"]:-} pkg_debug "v1: ${v1}, v2: ${v2}" if [[ -n ${v1} ]] && [[ -n ${v2} ]]; then @@ -1563,8 +1513,8 @@ function consistency_check_for_package() { elif [[ ${#profile_1_slots[@]} -eq 1 ]] && [[ ${#profile_2_slots[@]} -eq 1 ]]; then s1=${profile_1_slots[0]} s2=${profile_2_slots[0]} - v1=${slot_version1_map["${s1}"]:-} - v2=${slot_version2_map["${s2}"]:-} + v1=${slot_version1_map_ref["${s1}"]:-} + v2=${slot_version2_map_ref["${s2}"]:-} if [[ ${v1} != "${v2}" ]]; then pkg_warn \ "- version mismatch:" \ @@ -1597,12 +1547,11 @@ function consistency_check_for_package() { function consistency_checks() { local which pkg_slots_set_mvm_var_name pkg_slot_verminmax_mvm_var_name which=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n all_pkgs_ref=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift pkg_slot_verminmax_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local cc_pimap_mvm_1_var_name cc_pimap_mvm_2_var_name pkg @@ -1675,7 +1624,6 @@ function consistency_checks() { done local cc_slots_set_var_name s cc_min cc_max verminmax - # shellcheck disable=SC2034 # used by name below local -A empty_map=() local -a verminmax_map_var_names verminmaxes local cc_slot_verminmax_map_var_name @@ -1687,11 +1635,11 @@ function consistency_checks() { mvm_get "${name}" "${pkg}" cc_slot_verminmax_map_var_name verminmax_map_var_names+=("${cc_slot_verminmax_map_var_name}") done - if pkg_debug_possible; then + if pkg_debug_enabled; then for name in "${verminmax_map_var_names[@]}"; do local -n slot_verminmax_map_ref=${name:-empty_map} - pkg_debug "all slots in ${name}: ${!slot_verminmax_map_ref[*]}" - pkg_debug "all vmms in ${name}: ${slot_verminmax_map_ref[*]}" + pkg_debug_print "all slots in ${name}: ${!slot_verminmax_map_ref[*]}" + pkg_debug_print "all vmms in ${name}: ${slot_verminmax_map_ref[*]}" unset -n slot_verminmax_map_ref done fi @@ -1737,28 +1685,36 @@ function consistency_checks() { function read_package_sources() { local -n package_sources_map_ref=${1}; shift - local arch which report pkg repo saved_repo - for arch in "${ARCHES[@]}"; do - for which in "${WHICH[@]}"; do - for report in sdk-package-repos board-package-repos; do - while read -r pkg repo; do - saved_repo=${package_sources_map_ref["${pkg}"]:-} - if [[ -n ${saved_repo} ]]; then - if [[ ${saved_repo} != "${repo}" ]]; then - pkg_warn \ - '- different repos used for the package:' \ - " - package: ${pkg}" \ - ' - repos:' \ - " - ${saved_repo}" \ - " - ${repo}" - fi - else - package_sources_map_ref["${pkg}"]=${repo} - fi - done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" - done + # shellcheck source=for-shellcheck/globals + source "${WORKDIR}/globals" + + local -a files=() + local which arch + for which in "${WHICH[@]}"; do + files+=( "${WORKDIR}/pkg-reports/${which}/sdk-package-repos" ) + for arch in "${ARCHES[@]}"; do + files+=( "${WORKDIR}/pkg-reports/${which}/${arch}-board-package-repos" ) done done + + local file pkg repo saved_repo + for file in "${files[@]}"; do + while read -r pkg repo; do + saved_repo=${package_sources_map_ref["${pkg}"]:-} + if [[ -n ${saved_repo} ]]; then + if [[ ${saved_repo} != "${repo}" ]]; then + pkg_warn \ + '- different repos used for the package:' \ + " - package: ${pkg}" \ + ' - repos:' \ + " - ${saved_repo}" \ + " - ${repo}" + fi + else + package_sources_map_ref["${pkg}"]=${repo} + fi + done <"${file}" + done } # This monstrosity takes renames map and package tags information, @@ -1772,11 +1728,10 @@ function read_package_sources() { # 2 - name of the package tags map mvm variable function handle_package_changes() { local pkg_to_tags_mvm_var_name - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n renamed_old_to_new_map_ref=${1}; shift pkg_to_tags_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a hpc_all_pkgs @@ -1913,7 +1868,6 @@ function handle_package_changes() { local hpc_changed hpc_slot_changed hpc_update_dir_non_slot hpc_category_dir local which slots_set_var_name_var_name slot_verminmax_map_var_name_var_name filtered_slots_set_var_name verminmax local -A hpc_old_filtered_slots_set hpc_new_filtered_slots_set - # shellcheck disable=SC2034 # used by name below, in a special case empty_map_or_set=() while [[ ${pkg_idx} -lt ${#old_pkgs[@]} ]]; do old_name=${old_pkgs["${pkg_idx}"]} @@ -2020,7 +1974,6 @@ function handle_package_changes() { update_dir_non_slot "${new_name}" hpc_update_dir_non_slot mkdir -p "${hpc_update_dir_non_slot}" - # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE comes from globals file generate_non_ebuild_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" generate_full_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" generate_package_mention_reports "${NEW_STATE}" "${old_name}" "${new_name}" @@ -2163,7 +2116,6 @@ function handle_package_changes() { # 1 - name of the set variable # 2 - name of the variable where the element will be stored function get_first_from_set() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n set_ref=${1}; shift local -n return_ref=${1}; shift @@ -2172,56 +2124,9 @@ function get_first_from_set() { return_ref=${item} return 0 done - # shellcheck disable=SC2034 # it's a reference to external variable return_ref='' } -# Does the set operation on two passed sets - both set differences and -# an intersection. -# -# Params: -# -# 1 - name of the first set variable -# 2 - name of the second set variable -# 3 - name of the set variable that will contain elements that exist -# in first set, but not the second -# 4 - name of the set variable that will contain elements that exist -# in second set, but not the first -# 5 - name of the set variable that will contain elements that exist -# in both first and second sets -function sets_split() { - local -n first_set_ref=${1}; shift - local -n second_set_ref=${1}; shift - local -n only_in_first_set_ref=${1}; shift - local -n only_in_second_set_ref=${1}; shift - local -n common_set_ref=${1}; shift - - only_in_first_set_ref=() - only_in_second_set_ref=() - common_set_ref=() - - local item mark - - for item in "${!first_set_ref[@]}"; do - mark=${second_set_ref["${item}"]:-} - if [[ -z "${mark}" ]]; then - # shellcheck disable=SC2034 # it's a reference to external variable - only_in_first_set_ref["${item}"]=x - else - # shellcheck disable=SC2034 # it's a reference to external variable - common_set_ref["${item}"]=x - fi - done - - for item in "${!second_set_ref[@]}"; do - mark=${first_set_ref["${item}"]:-} - if [[ -z "${mark}" ]]; then - # shellcheck disable=SC2034 # it's a reference to external variable - only_in_second_set_ref["${item}"]=x - fi - done -} - # Write information to reports directory about the package update # (meaning specifically that the new version is greater than the old # one). @@ -2245,7 +2150,7 @@ function handle_pkg_update() { old=${1}; shift new=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_no_r new_no_r @@ -2255,26 +2160,37 @@ function handle_pkg_update() { local pkg_name pkg_name=${new_pkg#*/} local -a lines - lines=( "from ${old} to ${new}") + lines=( "0:from ${old} to ${new}") if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( "renamed from ${old_pkg}" ) + lines+=( "0:renamed from ${old_pkg}" ) fi - # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" - # shellcheck disable=SC2034 # these variables are used by name local hpu_update_dir hpu_update_dir_non_slot update_dir_non_slot "${new_pkg}" hpu_update_dir_non_slot update_dir "${new_pkg}" "${old_s}" "${new_s}" hpu_update_dir + + local diff_report_name + gen_varname diff_report_name + diff_report_declare "${diff_report_name}" + generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" + + local -n diff_report_ref=${diff_report_name} + local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} + lines+=( "${diff_lines_ref[@]}" ) + unset -n diff_lines_ref + unset -n diff_report_ref + diff_report_unset "${diff_report_name}" + if [[ -s "${hpu_update_dir}/ebuild.diff" ]]; then - lines+=( 'TODO: review ebuild.diff' ) + lines+=( '0:TODO: review ebuild.diff' ) fi if [[ -s "${hpu_update_dir_non_slot}/other.diff" ]]; then - lines+=( 'TODO: review other.diff' ) + lines+=( '0:TODO: review other.diff' ) fi - lines+=( 'TODO: review occurences' ) + lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( 'TODO: review occurences-for-old-name' ) + lines+=( '0:TODO: review occurences-for-old-name' ) fi local -a hpu_tags @@ -2283,7 +2199,7 @@ function handle_pkg_update() { if ver_test "${new_no_r}" -gt "${old_no_r}"; then # version bump generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpu_tags[@]}" - lines+=( 'release notes: TODO' ) + lines+=( '0:release notes: TODO' ) fi generate_summary_stub "${new_pkg}" "${hpu_tags[@]}" -- "${lines[@]}" @@ -2314,7 +2230,7 @@ function handle_pkg_as_is() { v=${1}; shift local -n changed_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local hpai_update_dir @@ -2323,12 +2239,12 @@ function handle_pkg_as_is() { local pkg_name pkg_name=${new_pkg#/} local -a lines - lines=( "still at ${v}" ) + lines=( "0:still at ${v}" ) local renamed renamed= if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( "renamed from ${old_pkg}" ) + lines+=( "0:renamed from ${old_pkg}" ) renamed=x fi generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${v}" "${v}" @@ -2337,23 +2253,38 @@ function handle_pkg_as_is() { update_dir "${new_pkg}" "${old_s}" "${new_s}" hpai_update_dir local modified modified= + + local diff_report_name + gen_varname diff_report_name + diff_report_declare "${diff_report_name}" + generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${v}" "${v}" + + local -n diff_report_ref=${diff_report_name} + local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} + if [[ ${#diff_lines_ref[@]} -gt 0 ]]; then + lines+=( "${diff_lines_ref[@]}" ) + modified=x + fi + unset -n diff_lines_ref + unset -n diff_report_ref + diff_report_unset "${diff_report_name}" + if [[ -s "${hpai_update_dir}/ebuild.diff" ]]; then - lines+=( 'TODO: review ebuild.diff' ) + lines+=( '0:TODO: review ebuild.diff' ) modified=x fi if [[ -s "${hpai_update_dir_non_slot}/other.diff" ]]; then - lines+=( 'TODO: review other.diff' ) + lines+=( '0:TODO: review other.diff' ) modified=x fi if [[ -z ${renamed} ]] && [[ -z ${modified} ]]; then # Nothing relevant has changed, return early. return 0 fi - # shellcheck disable=SC2034 # ref to an external variable changed_ref=x - lines+=( 'TODO: review occurences' ) + lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( 'TODO: review occurences-for-old-name' ) + lines+=( '0:TODO: review occurences-for-old-name' ) fi local -a hpai_tags @@ -2384,7 +2315,7 @@ function handle_pkg_downgrade() { old=${1}; shift new=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_no_r new_no_r @@ -2394,24 +2325,37 @@ function handle_pkg_downgrade() { local pkg_name pkg_name=${new_pkg#*/} local -a lines - lines=( "downgraded from ${old} to ${new}" ) + lines=( "0:downgraded from ${old} to ${new}" ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( "renamed from ${old_pkg}" ) + lines+=( "0:renamed from ${old_pkg}" ) fi generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" local hpd_update_dir hpd_update_dir_non_slot update_dir_non_slot "${new_pkg}" hpd_update_dir_non_slot update_dir "${new_pkg}" "${old_s}" "${new_s}" hpd_update_dir + + local diff_report_name + gen_varname diff_report_name + diff_report_declare "${diff_report_name}" + generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" + + local -n diff_report_ref=${diff_report_name} + local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} + lines+=( "${diff_lines_ref[@]}" ) + unset -n diff_lines_ref + unset -n diff_report_ref + diff_report_unset "${diff_report_name}" + if [[ -s "${hpd_update_dir}/ebuild.diff" ]]; then - lines+=( 'TODO: review ebuild.diff' ) + lines+=( '0:TODO: review ebuild.diff' ) fi if [[ -s "${hpd_update_dir_non_slot}/other.diff" ]]; then - lines+=( 'TODO: review other.diff' ) + lines+=( '0:TODO: review other.diff' ) fi - lines+=( 'TODO: review occurences' ) + lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( 'TODO: review occurences-for-old-name' ) + lines+=( '0:TODO: review occurences-for-old-name' ) fi local -a hpd_tags @@ -2420,7 +2364,7 @@ function handle_pkg_downgrade() { if ver_test "${new_no_r}" -lt "${old_no_r}"; then # version bump generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpd_tags[@]}" - lines+=( "release notes: TODO" ) + lines+=( "0:release notes: TODO" ) fi generate_summary_stub "${new_pkg}" "${hpd_tags[@]}" -- "${lines[@]}" @@ -2448,10 +2392,9 @@ function tags_for_pkg() { pkg_debug "no tags available" tags_ref=() else - local -n tags_in_mvm=${tfp_tags_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable - tags_ref=( "${tags_in_mvm[@]}" ) - pkg_debug "tags available: ${tags_in_mvm[*]}" + local -n tags_in_mvm_ref=${tfp_tags_var_name} + tags_ref=( "${tags_in_mvm_ref[@]}" ) + pkg_debug "tags available: ${tags_in_mvm_ref[*]}" fi pkg_debug_disable } @@ -2488,7 +2431,7 @@ function generate_changelog_entry_stub() { gces_tags='SDK' fi - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" printf '%s %s: %s ([%s](TODO))\n' '-' "${gces_tags}" "${pkg_name}" "${v}" >>"${REPORTS_DIR}/updates/changelog_stubs" @@ -2505,7 +2448,7 @@ function generate_summary_stub() { pkg=${1}; shift # rest are tags separated followed by double dash followed by lines - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a tags @@ -2526,10 +2469,16 @@ function generate_summary_stub() { printf ' [%s]' "${tags[@]}" fi printf '\n' - if [[ ${#} -gt 0 ]]; then - printf ' - %s\n' "${@}" - printf '\n' - fi + local indent_line indent line + for indent_line; do + indent=${indent_line%%:*} + line=${indent_line#*:} + if [[ ${indent} -gt 0 ]]; then + printf -- ' %.0s' $(seq 1 "${indent}") + fi + printf ' - %s\n' "${line}" + done + printf '\n' } >>"${REPORTS_DIR}/updates/summary_stubs" } @@ -2635,6 +2584,33 @@ function generate_ebuild_diff() { xdiff --unified=3 "${old_path}" "${new_path}" >"${ged_update_dir}/ebuild.diff" } +function generate_cache_diff_report() { + local diff_report_var_name=${1}; shift + local old_cache_dir=${1}; shift + local new_cache_dir=${1}; shift + local old_pkg=${1}; shift + local new_pkg=${1}; shift + local old=${1}; shift + local new=${1}; shift + + # shellcheck source=for-shellcheck/globals + source "${WORKDIR}/globals" + + local old_entry=${old_cache_dir}/${old_pkg}-${old} + local new_entry=${new_cache_dir}/${new_pkg}-${new} + + local old_cache_name new_cache_name + gen_varname old_cache_name + gen_varname new_cache_name + cache_file_declare "${old_cache_name}" "${new_cache_name}" + parse_cache_file "${old_cache_name}" "${old_entry}" "${ARCHES[@]}" + parse_cache_file "${new_cache_name}" "${new_entry}" "${ARCHES[@]}" + + diff_cache_data "${old_cache_name}" "${new_cache_name}" "${diff_report_var_name}" + + cache_file_unset "${old_cache_name}" "${new_cache_name}" +} + # Generate a report with information where the old and new packages # are mentioned in entire scripts repository. May result in two # separate reports if the package got renamed. @@ -2717,10 +2693,9 @@ function update_dir_non_slot() { pkg=${1}; shift local -n dir_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - # shellcheck disable=SC2034 # it's a reference to external variable dir_ref="${REPORTS_DIR}/updates/${pkg}" } @@ -2750,7 +2725,6 @@ function update_dir() { local ud_non_slot_dir update_dir_non_slot "${pkg}" ud_non_slot_dir - # shellcheck disable=SC2034 # it's a reference to external variable dir_ref="${ud_non_slot_dir}/${slot_dir}" } @@ -2768,7 +2742,7 @@ function grep_pkg() { pkg=${1}; shift # rest are directories - GIT_PAGER= git -C "${scripts}" grep "${pkg}"'\(-[0-9]\|[^a-zA-Z0-9_-]\|$\)' -- "${@}" || : + GIT_PAGER='' git -C "${scripts}" grep "${pkg}"'\(-[0-9]\|[^a-zA-Z0-9_-]\|$\)' -- "${@}" || : } # Prints the passed files preceding and following with BEGIN ENTRY and @@ -2792,7 +2766,6 @@ function handle_gentoo_sync() { mvm_declare hgs_pkg_to_tags_mvm process_listings hgs_pkg_to_tags_mvm - # shellcheck disable=SC2034 # passed to other function through a name local -A hgs_renames_old_to_new_map=() process_profile_updates_directory hgs_renames_old_to_new_map @@ -2801,11 +2774,10 @@ function handle_gentoo_sync() { mvm_unset hgs_pkg_to_tags_mvm #mvm_debug_disable hgs_pkg_to_tags_mvm - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_head new_head - # shellcheck disable=SC2153 # OLD_STATE is not a misspelling old_head=$(git -C "${OLD_STATE}" rev-parse HEAD) new_head=$(git -C "${NEW_STATE}" rev-parse HEAD) @@ -2814,7 +2786,6 @@ function handle_gentoo_sync() { local path in_ps category if [[ "${old_head}" != "${new_head}" ]]; then while read -r path; do - # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then continue fi @@ -2998,7 +2969,7 @@ function handle_eclass() { local eclass eclass=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a lines @@ -3006,11 +2977,11 @@ function handle_eclass() { if [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]] && [[ -e "${NEW_PORTAGE_STABLE}/${eclass}" ]]; then mkdir -p "${REPORTS_DIR}/updates/${eclass}" xdiff --unified=3 "${OLD_PORTAGE_STABLE}/${eclass}" "${NEW_PORTAGE_STABLE}/${eclass}" >"${REPORTS_DIR}/updates/${eclass}/eclass.diff" - lines+=( 'TODO: review the diff' ) + lines+=( '0:TODO: review the diff' ) elif [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]]; then - lines+=( 'unused, dropped' ) + lines+=( '0:unused, dropped' ) else - lines+=( 'added from Gentoo' ) + lines+=( '0:added from Gentoo' ) fi generate_summary_stub "${eclass}" -- "${lines[@]}" } @@ -3020,17 +2991,15 @@ function handle_eclass() { # and SDK), a full diff between all the profiles, and a list of # possibly irrelevant files that has changed too. function handle_profiles() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - local -a files - files=() - local arch which report - for arch in "${ARCHES[@]}"; do - for which in "${WHICH[@]}"; do - for report in sdk-profiles board-profiles; do - files+=("${WORKDIR}/pkg-reports/${which}-${arch}/${report}") - done + local -a files=() + local which arch + for which in "${WHICH[@]}"; do + files+=("${WORKDIR}/pkg-reports/${which}/sdk-profiles") + for arch in "${ARCHES[@]}"; do + files+=("${WORKDIR}/pkg-reports/${which}/${arch}-board-profiles") done done local -A profile_dirs_set @@ -3086,20 +3055,16 @@ function handle_profiles() { done <"${out_dir}/full.diff" lines_to_file_truncate "${out_dir}/relevant.diff" "${relevant_lines[@]}" lines_to_file_truncate "${out_dir}/possibly-irrelevant-files" "${possibly_irrelevant_files[@]}" - generate_summary_stub profiles -- 'TODO: review the diffs' + generate_summary_stub profiles -- '0:TODO: review the diffs' } # Handles changes in license directory. Generates brief reports and # diffs about dropped, added or modified licenses. function handle_licenses() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - local -a dropped added changed - dropped=() - added=() - changed=() - + local -a dropped=() added=() changed=() local line hl_stripped # Lines are: # @@ -3110,7 +3075,6 @@ function handle_licenses() { # Files /licenses/BSL-1.1 and /licenses/BSL-1.1 differ while read -r line; do if [[ ${line} = 'Only in '* ]]; then - # shellcheck disable=SC2153 # OLD_STATE is not a misspelling, it comes from globals file strip_out "${line##*:}" hl_stripped if [[ ${line} = *"${OLD_STATE}"* ]]; then dropped+=( "${hl_stripped}" ) @@ -3155,22 +3119,22 @@ function handle_licenses() { local joined if [[ ${#dropped[@]} -gt 0 ]]; then join_by joined ', ' "${dropped[@]}" - lines+=( "dropped ${joined}" ) + lines+=( "0:dropped ${joined}" ) fi if [[ ${#added[@]} -gt 0 ]]; then join_by joined ', ' "${added[@]}" - lines+=( "added ${joined}" ) + lines+=( "0:added ${joined}" ) fi if [[ ${#changed[@]} -gt 0 ]]; then join_by joined ', ' "${changed[@]}" - lines+=( "updated ${joined}" ) + lines+=( "0:updated ${joined}" ) fi generate_summary_stub licenses -- "${lines[@]}" } # Generates reports about changes inside the scripts directory. function handle_scripts() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local out_dir @@ -3178,70 +3142,7 @@ function handle_scripts() { mkdir -p "${out_dir}" xdiff --unified=3 --recursive "${OLD_PORTAGE_STABLE}/scripts" "${NEW_PORTAGE_STABLE}/scripts" >"${out_dir}/scripts.diff" - generate_summary_stub scripts -- 'TODO: review the diffs' -} - -# Enables debug logs when specific packages are processed. -# -# It is expected that globals were already sourced, otherwise -# debugging won't be enabled at all. -# -# Params: -# -# @ - package names to enable debugging for -function pkg_debug_enable() { - local -A pkg_set - pkg_set=() - local -a vals - vals=() - local pkg - for pkg; do - if [[ -n ${pkg_set["${pkg}"]:-} ]]; then - continue - fi - pkg_set["${pkg}"]=x - if [[ -n ${DEBUG_PACKAGES["${pkg}"]:-} ]]; then - vals+=( "${pkg}" ) - fi - done - if [[ ${#vals[@]} -gt 0 ]]; then - declare -g PKG_AUTO_LIB_DEBUG - join_by PKG_AUTO_LIB_DEBUG ',' "${vals[@]}" - fi -} - -# Returns true or false whether any debugging has been enabled. -function pkg_debug_possible() { - local ret=0 - [[ ${#DEBUG_PACKAGES[@]} -gt 0 ]] || ret=1 - return ${ret} -} - -# Disables debug logs to be printed. -function pkg_debug_disable() { - unset PKG_AUTO_LIB_DEBUG -} - -# Prints passed parameters if debugging is enabled. -# -# Params: -# -# @ - parameters to print -function pkg_debug() { - if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then - info "DEBUG(${PKG_AUTO_LIB_DEBUG}): ${*}" - fi -} - -# Prints passed lines if debugging is enabled. -# -# Params: -# -# @ - lines to print -function pkg_debug_lines() { - if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then - info_lines "${@/#/"DEBUG(${PKG_AUTO_LIB_DEBUG}): "}" - fi + generate_summary_stub scripts -- '0:TODO: review the diffs' } fi diff --git a/pkg_auto/impl/print_profile_tree.sh b/pkg_auto/impl/print_profile_tree.sh index 96ce744840..757adb7c9c 100755 --- a/pkg_auto/impl/print_profile_tree.sh +++ b/pkg_auto/impl/print_profile_tree.sh @@ -69,7 +69,6 @@ function get_repo_from_profile_path() { path=${1}; shift local -n repo_dir_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable repo_dir_ref="${path%/profiles/*}" } @@ -78,7 +77,6 @@ function repo_path_to_name() { path=${1}; shift local -n name_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable name_ref=${repo_data_r["${path}"]:-''} } @@ -138,7 +136,6 @@ function process_profile() { done <"${parent_file}" fi - # shellcheck disable=SC2034 # it's a reference to external variable children_ref=( "${children[@]}" ) } @@ -152,7 +149,6 @@ function get_profile_name() { repo_path=${repo_data["${repo_name}"]} profile_name=${profile_path#"${repo_path}/profiles/"} - # shellcheck disable=SC2034 # it's a reference to external variable profile_name_ref="${profile_name}" } diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh index c2763ad3e8..267b550d11 100644 --- a/pkg_auto/impl/util.sh +++ b/pkg_auto/impl/util.sh @@ -3,6 +3,9 @@ if [[ -z ${__UTIL_SH_INCLUDED__:-} ]]; then __UTIL_SH_INCLUDED__=x +declare -gra EMPTY_ARRAY=() +declare -grA EMPTY_MAP=() + # Works like dirname, but without spawning new processes. # # Params: @@ -34,7 +37,6 @@ function dirname_out() { dir_ref='.' return 0 fi - # shellcheck disable=SC2034 # it's a reference to external variable dir_ref=${dn} } @@ -65,7 +67,6 @@ function basename_out() { cleaned_up=${cleaned_up//+(\/)/\/} # keep last component dn=${cleaned_up##*/} - # shellcheck disable=SC2034 # it's a reference to external variable base_ref=${dn} } @@ -83,7 +84,6 @@ THIS=$(realpath "${THIS}") THIS_DIR=$(realpath "${THIS_DIR}") dirname_out "${BASH_SOURCE[0]}" PKG_AUTO_IMPL_DIR PKG_AUTO_IMPL_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}") -# shellcheck disable=SC2034 # may be used by scripts sourcing this file PKG_AUTO_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}/..") # Prints an info line. @@ -162,7 +162,6 @@ function join_by() { printf -v "${output_var_name}" '%s' "${first}" "${@/#/${delimiter}}"; else local -n output_ref=${output_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable output_ref='' fi } @@ -204,7 +203,6 @@ function strip_out() { t=${l} t=${t/#+([[:space:]])} t=${t/%+([[:space:]])} - # shellcheck disable=SC2034 # it's a reference to external variable out_ref=${t} } @@ -214,10 +212,8 @@ function strip_out() { # # 1 - name of an array variable, where the architectures will be stored function get_valid_arches() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n arches_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable arches_ref=( 'amd64' 'arm64' ) } @@ -232,7 +228,6 @@ function get_valid_arches() { # 2 - separator string # @ - strings function all_pairs() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n pairs_ref=${1}; shift local sep=${1}; shift @@ -250,4 +245,92 @@ function all_pairs() { done } +# Does the set operation on two passed sets - both set differences and +# an intersection. +# +# Params: +# +# 1 - name of the first set variable +# 2 - name of the second set variable +# 3 - name of the set variable that will contain elements that exist +# in first set, but not the second +# 4 - name of the set variable that will contain elements that exist +# in second set, but not the first +# 5 - name of the set variable that will contain elements that exist +# in both first and second sets +function sets_split() { + local -n first_set_ref=${1}; shift + local -n second_set_ref=${1}; shift + local -n only_in_first_set_ref=${1}; shift + local -n only_in_second_set_ref=${1}; shift + local -n common_set_ref=${1}; shift + + only_in_first_set_ref=() + only_in_second_set_ref=() + common_set_ref=() + + local item mark + + for item in "${!first_set_ref[@]}"; do + mark=${second_set_ref["${item}"]:-} + if [[ -z ${mark} ]]; then + only_in_first_set_ref["${item}"]=x + else + common_set_ref["${item}"]=x + fi + done + + for item in "${!second_set_ref[@]}"; do + mark=${first_set_ref["${item}"]:-} + if [[ -z ${mark} ]]; then + only_in_second_set_ref["${item}"]=x + fi + done +} + +declare -gi __UTIL_SH_COUNTER=0 + +# Generates a globally unique name for a variable. Can be given a +# prefix to override the default __PA_VAR one. +# +# Params: +# +# (optional) a prefix +# 1 - name of a variable, where the generated name will be stored +function gen_varname() { + local prefix='__PA_VAR' # pa = pkg-auto + if [[ ${#} -gt 1 ]]; then + # we passed a prefix + prefix=${1}; shift + fi + local -n name_ref=${1}; shift + + name_ref="${prefix}_${__UTIL_SH_COUNTER}" + __UTIL_SH_COUNTER=$((__UTIL_SH_COUNTER + 1)) +} + +# Declares variables with a given initializer. +# +# Params: +# +# @: flags passed to declare, followed by variable names, followed by +# an initializer +function struct_declare() { + local -a args=() + while [[ $# -gt 0 ]]; do + if [[ ${1} != -* ]]; then + break + fi + args+=( "${1}" ) + shift + done + if [[ ${#} -lt 2 ]]; then + fail "bad use of struct_declare" + fi + local definition=${*: -1} + set -- "${@:1:$((${#} - 1))}" + set -- "${@/%/=${definition}}" + declare "${args[@]}" "${@}" +} + fi diff --git a/pkg_auto/inside_sdk_container.sh b/pkg_auto/inside_sdk_container.sh index 8651917916..1d53b9661b 100755 --- a/pkg_auto/inside_sdk_container.sh +++ b/pkg_auto/inside_sdk_container.sh @@ -1,33 +1,34 @@ #!/bin/bash ## -## Gathers information about SDK and board packages. Also collects -## info about actual build deps of board packages, which may be useful -## for verifying if SDK provides those. +## Gathers information about SDK packages and board packages for each +## passed architecture. Also collects info about actual build deps of +## board packages, which may be useful for verifying if SDK provides +## those. ## ## Reports generated: ## sdk-pkgs - contains package information for SDK ## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK -## board-pkgs - contains package information for board for chosen architecture -## board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies +## ${arch}-board-pkgs - contains package information for board for chosen architecture +## ${arch}-board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies ## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order -## board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order +## ${arch}-board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order ## sdk-package-repos - contains package information with their repos for SDK -## board-package-repos - contains package information with their repos for board +## ${arch}-board-package-repos - contains package information with their repos for board ## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports -## board-emerge-output - contains raw emerge output for board being a base for other reports +## ${arch}-board-emerge-output - contains raw emerge output for board being a base for other reports ## sdk-emerge-output-filtered - contains only lines with package information for SDK -## board-emerge-output-filtered - contains only lines with package information for board +## ${arch}-board-emerge-output-filtered - contains only lines with package information for board ## sdk-emerge-output-junk - contains only junk lines for SDK -## board-emerge-output-junk - contains only junk lines for board +## ${arch}-board-emerge-output-junk - contains only junk lines for board ## *-warnings - warnings printed by emerge or other tools ## ## Parameters: ## -h: this help ## ## Positional: -## 1 - architecture (amd64 or arm64) -## 2 - reports directory +## 1 - reports directory +## # - architectures (currently only amd64 or arm64 are valid) ## set -euo pipefail @@ -54,49 +55,71 @@ while [[ ${#} -gt 0 ]]; do esac done -if [[ ${#} -ne 2 ]]; then - fail 'Expected two parameters: board architecture and reports directory' +if [[ ${#} -lt 2 ]]; then + fail 'Expected at least two parameters: reports directory and one or more board architectures' fi -arch=${1}; shift reports_dir=${1}; shift +# rest are architectures mkdir -p "${reports_dir}" -set_eo "${reports_dir}" +set_eo "${reports_dir}" "${@}" + +echo 'Running egencache for portage-stable' +generate_cache_for 'portage-stable' 2>"${EGENCACHE_W}" +echo 'Running egencache for coreos-overlay' +generate_cache_for 'coreos-overlay' 2>>"${EGENCACHE_W}" + +echo 'Copying portage-stable cache to reports' +copy_cache_to_reports 'portage-stable' "${reports_dir}" 2>>"${EGENCACHE_W}" +echo 'Copying coreos-overlay cache to reports' +copy_cache_to_reports 'coreos-overlay' "${reports_dir}" 2>>"${EGENCACHE_W}" echo 'Running pretend-emerge to get complete report for SDK' package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}" -echo 'Running pretend-emerge to get complete report for board' -package_info_for_board "${arch}" >"${BOARD_EO}" 2>"${BOARD_EO_W}" +for arch; do + be=${arch^^}_BOARD_EO + bew=${arch^^}_BOARD_EO_W + echo "Running pretend-emerge to get complete report for ${arch} board" + package_info_for_board "${arch}" >"${!be}" 2>"${!bew}" +done -ensure_no_errors +ensure_no_errors "${@}" echo 'Separating emerge info from junk in SDK emerge output' filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}" -junk_sdk_eo >"${SDK_EO}-junk" 2>>"${SDK_EO_W}" -echo 'Separating emerge info from junk in board emerge output' -filter_board_eo "${arch}" >"${BOARD_EO_F}" 2>>"${BOARD_EO_W}" -junk_board_eo >"${BOARD_EO}-junk" 2>>"${BOARD_EO_W}" +junk_sdk_eo >"${SDK_EO_J}" 2>>"${SDK_EO_W}" +for arch; do + bej=${arch^^}_BOARD_EO_J + bef=${arch^^}_BOARD_EO_F + bew=${arch^^}_BOARD_EO_W + echo "Separating emerge info from junk in ${arch} board emerge output" + filter_board_eo "${arch}" >"${!bef}" 2>>"${!bew}" + junk_board_eo "${arch}" >"${!bej}" 2>>"${!bew}" +done -ensure_valid_reports +ensure_valid_reports "${@}" echo 'Generating SDK packages listing' versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings" echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)' versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings" -echo 'Generating board packages listing' -versions_board >"${reports_dir}/board-pkgs" 2>"${reports_dir}/board-pkgs-warnings" -echo 'Generating board packages bdeps listing' -board_bdeps >"${reports_dir}/board-bdeps" 2>"${reports_dir}/board-bdeps-warnings" echo 'Generating SDK profiles evaluation list' ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings" -echo 'Generating board profiles evaluation list' -ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/board-profiles" 2>"${reports_dir}/board-profiles-warnings" echo 'Generating SDK package source information' package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings" -echo 'Generating board package source information' -package_sources_board >"${reports_dir}/board-package-repos" 2>"${reports_dir}/board-package-repos-warnings" + +for arch; do + echo "Generating ${arch} board packages listing" + versions_board "${arch}" >"${reports_dir}/${arch}-board-pkgs" 2>"${reports_dir}/${arch}-board-pkgs-warnings" + echo "Generating ${arch} board packages bdeps listing" + board_bdeps "${arch}" >"${reports_dir}/${arch}-board-bdeps" 2>"${reports_dir}/${arch}-board-bdeps-warnings" + echo "Generating ${arch} board profiles evaluation list" + ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/${arch}-board-profiles" 2>"${reports_dir}/${arch}-board-profiles-warnings" + echo "Generating ${arch} board package source information" + package_sources_board "${arch}" >"${reports_dir}/${arch}-board-package-repos" 2>"${reports_dir}/${arch}-board-package-repos-warnings" +done echo "Cleaning empty warning files" clean_empty_warning_files "${reports_dir}"