From cfd6a4057bd08c4f4faa025b1eccef802e5e7def Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 17:53:16 +0200 Subject: [PATCH 01/18] pkg-auto: Move sets_split to util.sh This will be used in other places, so make it available without importing the big pkg_auto_lib.sh file. --- pkg_auto/impl/pkg_auto_lib.sh | 46 -------------------------------- pkg_auto/impl/util.sh | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 46 deletions(-) diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index d96bc60765..d53183b9c4 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -2176,52 +2176,6 @@ function get_first_from_set() { return_ref='' } -# Does the set operation on two passed sets - both set differences and -# an intersection. -# -# Params: -# -# 1 - name of the first set variable -# 2 - name of the second set variable -# 3 - name of the set variable that will contain elements that exist -# in first set, but not the second -# 4 - name of the set variable that will contain elements that exist -# in second set, but not the first -# 5 - name of the set variable that will contain elements that exist -# in both first and second sets -function sets_split() { - local -n first_set_ref=${1}; shift - local -n second_set_ref=${1}; shift - local -n only_in_first_set_ref=${1}; shift - local -n only_in_second_set_ref=${1}; shift - local -n common_set_ref=${1}; shift - - only_in_first_set_ref=() - only_in_second_set_ref=() - common_set_ref=() - - local item mark - - for item in "${!first_set_ref[@]}"; do - mark=${second_set_ref["${item}"]:-} - if [[ -z "${mark}" ]]; then - # shellcheck disable=SC2034 # it's a reference to external variable - only_in_first_set_ref["${item}"]=x - else - # shellcheck disable=SC2034 # it's a reference to external variable - common_set_ref["${item}"]=x - fi - done - - for item in "${!second_set_ref[@]}"; do - mark=${first_set_ref["${item}"]:-} - if [[ -z "${mark}" ]]; then - # shellcheck disable=SC2034 # it's a reference to external variable - only_in_second_set_ref["${item}"]=x - fi - done -} - # Write information to reports directory about the package update # (meaning specifically that the new version is greater than the old # one). diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh index c2763ad3e8..23e8de104e 100644 --- a/pkg_auto/impl/util.sh +++ b/pkg_auto/impl/util.sh @@ -250,4 +250,53 @@ function all_pairs() { done } +# Does the set operation on two passed sets - both set differences and +# an intersection. +# +# Params: +# +# 1 - name of the first set variable +# 2 - name of the second set variable +# 3 - name of the set variable that will contain elements that exist +# in first set, but not the second +# 4 - name of the set variable that will contain elements that exist +# in second set, but not the first +# 5 - name of the set variable that will contain elements that exist +# in both first and second sets +function sets_split() { + local -n first_set_ref=${1}; shift + local -n second_set_ref=${1}; shift + # shellcheck disable=SC2178 # shellcheck does not grok references + local -n only_in_first_set_ref=${1}; shift + # shellcheck disable=SC2178 # shellcheck does not grok references + local -n only_in_second_set_ref=${1}; shift + # shellcheck disable=SC2178 # shellcheck does not grok references + local -n common_set_ref=${1}; shift + + only_in_first_set_ref=() + only_in_second_set_ref=() + common_set_ref=() + + local item mark + + for item in "${!first_set_ref[@]}"; do + mark=${second_set_ref["${item}"]:-} + if [[ -z ${mark} ]]; then + # shellcheck disable=SC2034 # it's a reference to external variable + only_in_first_set_ref["${item}"]=x + else + # shellcheck disable=SC2034 # it's a reference to external variable + common_set_ref["${item}"]=x + fi + done + + for item in "${!second_set_ref[@]}"; do + mark=${first_set_ref["${item}"]:-} + if [[ -z ${mark} ]]; then + # shellcheck disable=SC2034 # it's a reference to external variable + only_in_second_set_ref["${item}"]=x + fi + done +} + fi From 3931cbff5f34a1660dbbdd1c3495a14b95b59977 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 17:56:04 +0200 Subject: [PATCH 02/18] pkg-auto: Add a global variable name generator function Some upcoming libraries will use this for their global variables. The function is using a single counter, which ensures that the generated names will be globally unique. --- pkg_auto/impl/util.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh index 23e8de104e..11afcdf8b5 100644 --- a/pkg_auto/impl/util.sh +++ b/pkg_auto/impl/util.sh @@ -3,6 +3,9 @@ if [[ -z ${__UTIL_SH_INCLUDED__:-} ]]; then __UTIL_SH_INCLUDED__=x +declare -gra EMPTY_ARRAY=() +declare -grA EMPTY_MAP=() + # Works like dirname, but without spawning new processes. # # Params: @@ -299,4 +302,26 @@ function sets_split() { done } +declare -gi __UTIL_SH_COUNTER=0 + +# Generates a globally unique name for a variable. Can be given a +# prefix to override the default __PA_VAR one. +# +# Params: +# +# (optional) a prefix +# 1 - name of a variable, where the generated name will be stored +function gen_varname() { + local prefix='__PA_VAR' # pa = pkg-auto + if [[ ${#} -gt 1 ]]; then + # we passed a prefix + prefix=${1}; shift + fi + local -n name_ref=${1}; shift + + # shellcheck disable=SC2034 # shellcheck does not grok references + name_ref="${prefix}_${__UTIL_SH_COUNTER}" + __UTIL_SH_COUNTER=$((__UTIL_SH_COUNTER + 1)) +} + fi From b52676a64b05fc251f326a6b3c5043f175ca75c6 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 11 Apr 2025 14:49:21 +0200 Subject: [PATCH 03/18] pkg-auto: Add function for declaring structs Declaring structs differs a bit from declaring typical variables in that it takes one initializer and applies it to all the declared variables. Will be used a lot by upcoming libraries. --- pkg_auto/impl/util.sh | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh index 11afcdf8b5..b220060574 100644 --- a/pkg_auto/impl/util.sh +++ b/pkg_auto/impl/util.sh @@ -324,4 +324,28 @@ function gen_varname() { __UTIL_SH_COUNTER=$((__UTIL_SH_COUNTER + 1)) } +# Declares variables with a given initializer. +# +# Params: +# +# @: flags passed to declare, followed by variable names, followed by +# an initializer +function struct_declare() { + local -a args=() + while [[ $# -gt 0 ]]; do + if [[ ${1} != -* ]]; then + break + fi + args+=( "${1}" ) + shift + done + if [[ ${#} -lt 2 ]]; then + fail "bad use of struct_declare" + fi + local definition=${*: -1} + set -- "${@:1:$((${#} - 1))}" + set -- "${@/%/=${definition}}" + declare "${args[@]}" "${@}" +} + fi From 7b5841c0397968a4e3e636384b78d19c93202815 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 17 Feb 2025 09:05:21 +0100 Subject: [PATCH 04/18] pkg-auto: Add md5-metadata cache to reports This adds an explicit generation of md5-metadata cache before any we do any emerge invocations. That way we can have a copy of reports even if emerge fails for some reason. But mostly the reason for this copying is to consume the data later, outside the SDK container. --- pkg_auto/impl/inside_sdk_container_lib.sh | 29 ++++++++++++++++++----- pkg_auto/inside_sdk_container.sh | 10 ++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index 13f4e456ea..f578162b1e 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -98,16 +98,18 @@ function package_info_for_board() { function set_eo() { local dir=${1}; shift - SDK_EO="${dir}/sdk-emerge-output" - BOARD_EO="${dir}/board-emerge-output" + # shellcheck disable=SC2034 # used externally + declare -g EGENCACHE_W="${dir}/egencache-warnings" + declare -g SDK_EO="${dir}/sdk-emerge-output" + declare -g BOARD_EO="${dir}/board-emerge-output" # shellcheck disable=SC2034 # used indirectly in cat_eo_f - SDK_EO_F="${SDK_EO}-filtered" + declare -g SDK_EO_F="${SDK_EO}-filtered" # shellcheck disable=SC2034 # used indirectly in cat_eo_f - BOARD_EO_F="${BOARD_EO}-filtered" + declare -g BOARD_EO_F="${BOARD_EO}-filtered" # shellcheck disable=SC2034 # used indirectly in cat_eo_w - SDK_EO_W="${SDK_EO}-warnings" + declare -g SDK_EO_W="${SDK_EO}-warnings" # shellcheck disable=SC2034 # used indirectly in cat_eo_w - BOARD_EO_W="${BOARD_EO}-warnings" + declare -g BOARD_EO_W="${BOARD_EO}-warnings" } # Print the contents of file, path of which is stored in a variable of @@ -484,4 +486,19 @@ function clean_empty_warning_files() { done } +function generate_cache_for() { + local repo=${1}; shift + + egencache --repo "${repo}" --update +} + +function copy_cache_to_reports() { + local repo=${1}; shift + local reports_dir=${1}; shift + + local repo_dir + repo_dir=$(portageq get_repo_path / "${repo}") + cp -a "${repo_dir}/metadata/md5-cache" "${reports_dir}/${repo}-cache" +} + fi diff --git a/pkg_auto/inside_sdk_container.sh b/pkg_auto/inside_sdk_container.sh index 8651917916..6e128a1992 100755 --- a/pkg_auto/inside_sdk_container.sh +++ b/pkg_auto/inside_sdk_container.sh @@ -65,6 +65,16 @@ mkdir -p "${reports_dir}" set_eo "${reports_dir}" +echo 'Running egencache for portage-stable' +generate_cache_for 'portage-stable' 2>"${EGENCACHE_W}" +echo 'Running egencache for coreos-overlay' +generate_cache_for 'coreos-overlay' 2>>"${EGENCACHE_W}" + +echo 'Copying portage-stable cache to reports' +copy_cache_to_reports 'portage-stable' "${reports_dir}" 2>>"${EGENCACHE_W}" +echo 'Copying coreos-overlay cache to reports' +copy_cache_to_reports 'coreos-overlay' "${reports_dir}" 2>>"${EGENCACHE_W}" + echo 'Running pretend-emerge to get complete report for SDK' package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}" echo 'Running pretend-emerge to get complete report for board' From 00572476c43b72268187e031a2a5dba1eac6e192 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 18:07:52 +0200 Subject: [PATCH 05/18] pkg-auto: Simplify SDK image override handling There used to be a possibility to override used SDK image per architecture, but the need for it disappeared once SDK images started to contain the initial form of board rootfs for both amd64 and arm64. This eliminated problems with cyclic dependencies errors popping up while gather the package reports. So with this change it is now only possible to specify just one SDK image to use for any arch. This feature is not used all that often anyway. --- pkg_auto/generate_config.sh | 48 +++++----------------------- pkg_auto/impl/pkg_auto_lib.sh | 59 ++++++++++------------------------- 2 files changed, 23 insertions(+), 84 deletions(-) diff --git a/pkg_auto/generate_config.sh b/pkg_auto/generate_config.sh index ce354be9c3..e6c31abdcd 100755 --- a/pkg_auto/generate_config.sh +++ b/pkg_auto/generate_config.sh @@ -9,9 +9,8 @@ set -euo pipefail ## -a: aux directory ## -d: debug package - list many times ## -h: this help -## -i: SDK image override in form of ${arch}:${name}, the name part -## should be a valid docker image with an optional tag -## -ip: add SDK image overrides using flatcar-packages images +## -i: override SDK image, it should be a valid docker image with an +## optional tag ## -n: new base ## -o: old base ## -r: reports directory @@ -39,12 +38,14 @@ gc_reports_directory='' gc_scripts_directory='' # shellcheck disable=SC2034 # used by name below gc_cleanup_opts='' -# gc_${arch}_sdk_img are declared on demand +# shellcheck disable=SC2034 # used by name below +gc_image_override='' gc_debug_packages=() declare -A opt_map opt_map=( ['-a']=gc_aux_directory + ['-i']=gc_image_override ['-n']=gc_new_base ['-o']=gc_old_base ['-r']=gc_reports_directory @@ -68,36 +69,6 @@ while [[ ${#} -gt 0 ]]; do print_help exit 0 ;; - -i) - if [[ -z ${2:-} ]]; then - fail 'missing value for -i' - fi - arch=${2%%:*} - image_name=${2#*:} - var_name="gc_${arch}_sdk_img" - unset arch - # shellcheck disable=SC2178 # shellcheck does not grok refs - declare -n ref="${var_name}" - unset var_name - # shellcheck disable=SC2178 # shellcheck does not grok refs - ref=${image_name} - unset image_name - unset -n ref - shift 2 - ;; - -ip) - for arch in "${gc_arches[@]}"; do - var_name="gc_${arch}_sdk_img" - # shellcheck disable=SC2178 # shellcheck does not grok refs - declare -n ref="${var_name}" - unset var_name - # shellcheck disable=SC2178 # shellcheck does not grok refs - ref="flatcar-packages-${arch}" - unset -n ref - done - unset arch - shift - ;; --) shift break @@ -134,15 +105,10 @@ pairs=( 'old-base' gc_old_base 'new-base' gc_new_base 'cleanups' gc_cleanup_opts + 'sdk-image-override' gc_image_override + 'debug-packages' gc_debug_packages_csv ) -for arch in "${gc_arches[@]}"; do - pairs+=( "${arch}-sdk-img" "gc_${arch}_sdk_img" ) -done - -pairs+=( 'debug-packages' gc_debug_packages_csv ) - - if [[ ${#} -ne 1 ]]; then fail 'expected one positional parameters: a path for the config' fi diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index d53183b9c4..60471f873d 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -72,15 +72,14 @@ function setup_workdir_with_config() { workdir=${1}; shift config_file=${1}; shift - local cfg_scripts cfg_aux cfg_reports cfg_old_base cfg_new_base + local cfg_scripts cfg_aux cfg_reports cfg_old_base cfg_new_base cfg_sdk_image_override local -a cfg_cleanups cfg_debug_packages - local -A cfg_overrides # some defaults cfg_old_base='origin/main' cfg_new_base='' cfg_cleanups=('ignore') - cfg_overrides=() + cfg_sdk_image_override='' cfg_debug_packages=() local line key value swwc_stripped var_name arch @@ -99,7 +98,7 @@ function setup_workdir_with_config() { var=$(realpath "${value}") unset -n var ;; - old-base|new-base) + old-base|new-base|sdk-image-override) var_name="cfg_${key//-/_}" local -n var=${var_name} var=${value} @@ -109,11 +108,6 @@ function setup_workdir_with_config() { var_name="cfg_${key//-/_}" mapfile -t "${var_name}" <<<"${value//,/$'\n'}" ;; - *-sdk-img) - arch=${key%%-*} - # shellcheck disable=SC2034 # used by name below - cfg_overrides["${arch}"]=${value} - ;; esac done < <(cat_meaningful "${config_file}") if [[ -z "${cfg_new_base}" ]]; then @@ -131,7 +125,9 @@ function setup_workdir_with_config() { add_cleanup "rm -f ${WORKDIR@Q}/config" cp -a "${config_file}" "${WORKDIR}/config" setup_worktrees_in_workdir "${cfg_scripts}" "${cfg_old_base}" "${cfg_new_base}" "${cfg_reports}" "${cfg_aux}" - override_sdk_image_names cfg_overrides + if [[ -n ${cfg_sdk_image_override} ]]; then + override_sdk_image_name "${cfg_sdk_image_override}" + fi add_debug_packages "${cfg_debug_packages[@]}" } @@ -244,32 +240,15 @@ function setup_worktrees_in_workdir() { extend_globals_file "${scripts}" "${old_state}" "${new_state}" "${reports_dir}" "${aux_dir}" } -# Adds overridden SDK image names to the globals file. +# Adds an overridden SDK image name to the globals file. # # Params: # -# 1 - name of a map variable; should be a mapping of architecture to -# the image name -function override_sdk_image_names() { - local -n overrides_map_ref=${1} +# 1 - image name +function override_sdk_image_name() { + local image_name=${1}; shift - if [[ ${#overrides_map_ref[@]} -eq 0 ]]; then - return 0 - fi - - local arch image_name upcase_arch - local -a lines - lines=() - for arch in "${!overrides_map_ref[@]}"; do - image_name=${overrides_map_ref["${arch}"]} - upcase_arch=${arch^^} - if [[ ${#lines[@]} -eq 0 ]]; then - # separate overrides from initial values - lines+=( '' ) - fi - lines+=( "${upcase_arch}_SDK_IMAGE=${image_name@Q}" ) - done - append_to_globals "${lines[@]}" + append_to_globals "SDK_IMAGE=${image_name@Q}" } # Adds information about packages to be debugged to the globals file. @@ -572,17 +551,11 @@ EOF # shellcheck disable=SC1091 # sourcing generated file last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") - local -a locals definitions - locals=() - definitions=() - local sdk_image_name + local -a locals=() definitions=() + local sdk_image_name sdk_image_var_name=SDK_IMAGE sdk_image_name="ghcr.io/flatcar/flatcar-sdk-all:${last_nightly_version_id}-${last_nightly_build_id}" - local arch sdk_image_var_name - for arch in "${ARCHES[@]}"; do - sdk_image_var_name="${arch^^}_SDK_IMAGE" - locals+=( "${sdk_image_var_name@Q}" ) - definitions+=( "${sdk_image_var_name}=${sdk_image_name@Q}" ) - done + locals+=( "${sdk_image_var_name@Q}" ) + definitions+=( "${sdk_image_var_name}=${sdk_image_name@Q}" ) append_to_globals \ '' \ @@ -1146,7 +1119,7 @@ function generate_sdk_reports() { local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy local -a report_files run_sdk_container_args for arch in "${ARCHES[@]}"; do - sdk_image_var_name="${arch^^}_SDK_IMAGE" + sdk_image_var_name="SDK_IMAGE" sdk_image_name=${!sdk_image_var_name} if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${sdk_image_name}"; then fail "No SDK image named ${sdk_image_name@Q} available locally, pull it before running this script" From 80d12ea75f592da6f23e99bd2d9926b6b3e76d15 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 18:09:52 +0200 Subject: [PATCH 06/18] pkg-auto: Rework reports generation Reports generation used to be executed four times. The number of runs was a result of cartesian product of two sets - old and new state, and of amd64 and arm64 architectures. It was pretty much a slow process because egencache was called implicitly four times, and it was running in a single-threaded fashion, and also SDK reports were duplicated (they were the same for old-amd64 and old-arm64, and the same for new-amd64 and new-arm64 runs). This changes the generation, so it is being run only two times - once for old state and once for new state. Every run generates SDK packages reports and per-architecture board packages reports. Egencache will now utilize more threads too. --- pkg_auto/impl/inside_sdk_container_lib.sh | 175 +++++++--------- pkg_auto/impl/pkg_auto_lib.sh | 232 +++++++++++++--------- pkg_auto/inside_sdk_container.sh | 77 ++++--- 3 files changed, 255 insertions(+), 229 deletions(-) diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index f578162b1e..9acec5f9c2 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -97,79 +97,24 @@ function package_info_for_board() { # 1 - directory path function set_eo() { local dir=${1}; shift + # rest are architectures # shellcheck disable=SC2034 # used externally declare -g EGENCACHE_W="${dir}/egencache-warnings" declare -g SDK_EO="${dir}/sdk-emerge-output" - declare -g BOARD_EO="${dir}/board-emerge-output" - # shellcheck disable=SC2034 # used indirectly in cat_eo_f declare -g SDK_EO_F="${SDK_EO}-filtered" - # shellcheck disable=SC2034 # used indirectly in cat_eo_f - declare -g BOARD_EO_F="${BOARD_EO}-filtered" - # shellcheck disable=SC2034 # used indirectly in cat_eo_w declare -g SDK_EO_W="${SDK_EO}-warnings" - # shellcheck disable=SC2034 # used indirectly in cat_eo_w - declare -g BOARD_EO_W="${BOARD_EO}-warnings" -} + declare -g SDK_EO_J="${SDK_EO}-junk" -# Print the contents of file, path of which is stored in a variable of -# a given name. -# -# Params: -# -# 1 - name of the variable -function cat_var() { - local var_name - var_name=${1}; shift - local -n ref="${var_name}" - - if [[ -z "${ref+isset}" ]]; then - fail "${var_name} unset" - fi - if [[ ! -e "${ref}" ]]; then - fail "${ref} does not exist" - fi - - cat "${ref}" -} - -# Print contents of the emerge output of a given kind. Kind can be -# either either sdk or board. -# -# Params: -# -# 1 - kind -function cat_eo() { - local kind - kind=${1}; shift - - cat_var "${kind^^}_EO" -} - -# Print contents of the filtered emerge output of a given kind. Kind -# can be either either sdk or board. The filtered emerge output -# contains only lines with package information. -# -# Params: -# -# 1 - kind -function cat_eo_f() { - local kind - kind=${1}; shift - cat_var "${kind^^}_EO_F" -} - -# Print contents of a file that stores warnings that were printed by -# emerge. The warnings are specific to a kind (sdk or board). -# -# Params: -# -# 1 - kind -function cat_eo_w() { - local kind - kind=${1}; shift - - cat_var "${kind^^}_EO_W" + local arch + local board_eo + for arch; do + board_eo=${dir}/${arch}-board-emerge-output + declare -g "${arch^^}_BOARD_EO=${board_eo}" + declare -g "${arch^^}_BOARD_EO_F=${board_eo}-filtered" + declare -g "${arch^^}_BOARD_EO_W=${board_eo}-warnings" + declare -g "${arch^^}_BOARD_EO_J=${board_eo}-junk" + done } # JSON output would be more verbose, but probably would not require @@ -195,7 +140,7 @@ FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${V # Filters sdk reports to get the package information. function filter_sdk_eo() { - cat_eo sdk | xgrep -e "${FULL_LINE_RE}" + cat "${SDK_EO}" | xgrep -e "${FULL_LINE_RE}" } # Filters board reports for a given arch to get the package @@ -205,11 +150,12 @@ function filter_sdk_eo() { # # 1 - architecture function filter_board_eo() { - local arch + local arch name arch=${1}; shift + name=${arch^^}_BOARD_EO # Replace ${arch}-usr in the output with a generic word BOARD. - cat_eo board | \ + cat "${!name}" | \ xgrep -e "${FULL_LINE_RE}" | \ sed -e "s#/build/${arch}-usr/#/build/BOARD/#" } @@ -217,13 +163,17 @@ function filter_board_eo() { # Filters sdk reports to get anything but the package information # (i.e. junk). function junk_sdk_eo() { - cat_eo sdk | xgrep -v -e "${FULL_LINE_RE}" + cat "${SDK_EO}" | xgrep -v -e "${FULL_LINE_RE}" } # Filters board reports to get anything but the package information # (i.e. junk). function junk_board_eo() { - cat_eo board | xgrep -v -e "${FULL_LINE_RE}" + local arch name + arch=${1}; shift + name=${arch^^}_BOARD_EO + + cat "${!name}" | xgrep -v -e "${FULL_LINE_RE}" } # More regexp-like abominations follow. @@ -276,21 +226,6 @@ PKG_REPO_SED_FILTERS=( -e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}${SPACES_RE}.*/\1 \3/" ) -# Applies some sed filter over the emerge output of a given kind. -# Results are printed. -# -# Params: -# -# 1 - kind (sdk or board) -# @ - parameters passed to sed -function sed_eo_and_sort() { - local kind - kind=${1}; shift - # rest goes to sed - - cat_eo_f "${kind}" | sed "${@}" | sort -} - # Applies some sed filter over the SDK emerge output. Results are # printed. # @@ -298,9 +233,7 @@ function sed_eo_and_sort() { # # @ - parameters passed to sed function packages_for_sdk() { - # args are passed to sed_eo_and_sort - - sed_eo_and_sort sdk "${@}" + cat "${SDK_EO_F}" | sed "${@}" | sort } # Applies some sed filter over the board emerge output. Results are @@ -310,9 +243,12 @@ function packages_for_sdk() { # # @ - parameters passed to sed function packages_for_board() { - # args are passed to sed_eo_and_sort + local arch=${1}; shift + # rest goes to sed - sed_eo_and_sort board "${@}" + local name=${arch^^}_BOARD_EO_F + + cat "${!name}" | sed "${@}" | sort } # Prints package name, slot and version information for SDK. @@ -336,24 +272,26 @@ function versions_sdk_with_key_values() { # Prints package name, slot and version information for board. function versions_board() { + local arch=${1}; shift local -a sed_opts sed_opts=( -e '/to \/build\/BOARD\// ! d' "${PKG_VER_SLOT_SED_FILTERS[@]}" ) - packages_for_board "${sed_opts[@]}" + packages_for_board "${arch}" "${sed_opts[@]}" } # Prints package name, slot, version and key-values information for # build dependencies of board. Key-values may be something like # USE="foo bar -baz". function board_bdeps() { + local arch=${1}; shift local -a sed_opts sed_opts=( -e '/to \/build\/BOARD\// d' "${PKG_VER_SLOT_KV_SED_FILTERS[@]}" ) - packages_for_board "${sed_opts[@]}" + packages_for_board "${arch}" "${sed_opts[@]}" } # Print package name and source repository names information for SDK. @@ -368,21 +306,29 @@ function package_sources_sdk() { # Print package name and source repository names information for # board. function package_sources_board() { + local arch=${1}; shift local -a sed_opts sed_opts=( "${PKG_REPO_SED_FILTERS[@]}" ) - packages_for_board "${sed_opts[@]}" + packages_for_board "${arch}" "${sed_opts[@]}" } # Checks if no errors were produced by emerge when generating # reports. It is assumed that emerge will print a line with "ERROR" in # it to denote a failure. function ensure_no_errors() { - local kind + local -a files=( "${SDK_EO_W}" ) + local arch name - for kind in sdk board; do - if cat_eo_w "${kind}" | grep --quiet --fixed-strings 'ERROR'; then + for arch; do + name=${arch^^}_BOARD_EO_W + files+=( "${!name}" ) + done + + local file + for file in "${files[@]}"; do + if cat "${file}" | grep --quiet --fixed-strings 'ERROR'; then fail "there are errors in emerge output warnings files" fi done @@ -460,10 +406,17 @@ function revert_crossdev_stuff() { # Checks if the expected reports were generated by emerge. function ensure_valid_reports() { - local kind var_name - for kind in sdk board; do - var_name="${kind^^}_EO_F" - if [[ ! -s ${!var_name} ]]; then + local -a files=( "${SDK_EO_F}" ) + local arch name + + for arch; do + name=${arch^^}_BOARD_EO_F + files+=( "${!name}" ) + done + + local file + for file in "${files[@]}"; do + if [[ ! -s ${file} ]]; then fail "report files are missing or are empty" fi done @@ -486,10 +439,28 @@ function clean_empty_warning_files() { done } +function get_num_proc() { + local -n num_proc_ref=${1}; shift + local -i num_proc=1 rv=1 + + # stolen from portage + [[ rv -eq 0 ]] || { rv=0; num_proc=$(getconf _NPROCESSORS_ONLN 2>/dev/null) || rv=1; } + [[ rv -eq 0 ]] || { rv=0; num_proc=$(sysctl -n hw.ncpu 2>/dev/null) || rv=1; } + # stolen from common.sh + [[ rv -eq 0 ]] || { rv=0; num_proc=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null) || rv=1; } + [[ rv -eq 0 ]] || { rv=0; num_proc=1; } + + num_proc_ref=${num_proc} +} + function generate_cache_for() { local repo=${1}; shift - egencache --repo "${repo}" --update + local -i gcf_num_proc + local load_avg + get_num_proc gcf_num_proc + load_avg=$(bc <<< "${gcf_num_proc} * 0.75") + egencache --repo "${repo}" --jobs="${gcf_num_proc}" --load-average="${load_avg}" --update } function copy_cache_to_reports() { diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 60471f873d..f2f0d3cc7a 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -1114,78 +1114,96 @@ function generate_sdk_reports() { add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" mkdir "${WORKDIR}/pkg-reports" - local arch sdk_image_var_name sdk_image_name + if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${SDK_IMAGE}"; then + fail "No SDK image named ${SDK_IMAGE@Q} available locally, pull it before running this script" + fi + local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch - local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy - local -a report_files run_sdk_container_args - for arch in "${ARCHES[@]}"; do - sdk_image_var_name="SDK_IMAGE" - sdk_image_name=${!sdk_image_var_name} - if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${sdk_image_name}"; then - fail "No SDK image named ${sdk_image_name@Q} available locally, pull it before running this script" + local pkg_auto_copy rv + local sdk_reports_dir top_dir dir entry full_path + local -a dir_queue all_dirs all_files + + # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file + for sdk_run_kind in "${WHICH[@]}"; do + state_var_name="${sdk_run_kind^^}_STATE" + sdk_run_state="${!state_var_name}_sdk_run" + state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" + sdk_run_state_branch="${!state_branch_var_name}-sdk-run" + + add_cleanup \ + "git -C ${sdk_run_state@Q} reset --hard HEAD" \ + "git -C ${sdk_run_state@Q} clean -ffdx" \ + "git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ + "git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" + git -C "${SCRIPTS}" \ + worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}" + + pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX") + add_cleanup "rm -rf ${pkg_auto_copy@Q}" + cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}" + local -a run_sdk_container_args=( + -C "${SDK_IMAGE}" + -n "pkg-${sdk_run_kind}" + -U + -m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto" + --rm + ./pkg_auto/inside_sdk_container.sh pkg-reports "${ARCHES[@]}" + ) + rv=0 + env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?} + unset run_sdk_container_args + if [[ ${rv} -ne 0 ]]; then + local salvaged_dir + salvaged_dir="${REPORTS_DIR}/salvaged-reports" + { + info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue" + info + for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do + info "from ${file}:" + echo + cat "${file}" + echo + done + info + info 'whatever reports generated by the failed run are saved in' + info "${salvaged_dir@Q} directory" + info + } >&2 + rm -rf "${salvaged_dir}" + cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}" + unset salvaged_dir + fail "copying done, stopping now" fi - - # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file - for sdk_run_kind in "${WHICH[@]}"; do - state_var_name="${sdk_run_kind^^}_STATE" - sdk_run_state="${!state_var_name}_sdk_run_${arch}" - state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" - sdk_run_state_branch="${!state_branch_var_name}-sdk-run-${arch}" - - add_cleanup \ - "git -C ${sdk_run_state@Q} reset --hard HEAD" \ - "git -C ${sdk_run_state@Q} clean -ffdx" \ - "git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ - "git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" - git -C "${SCRIPTS}" \ - worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}" - - pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX") - add_cleanup "rm -rf ${pkg_auto_copy@Q}" - cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}" - local -a run_sdk_container_args=( - -C "${sdk_image_name}" - -n "pkg-${sdk_run_kind}-${arch}" - -a "${arch}" - -U - -m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto" - --rm - ./pkg_auto/inside_sdk_container.sh "${arch}" pkg-reports - ) - rv=0 - env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?} - if [[ ${rv} -ne 0 ]]; then - { - salvaged_dir="${REPORTS_DIR}/salvaged-reports" - info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue" - info - for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do - info "from ${file}:" - echo - cat "${file}" - echo - done - info - info 'whatever reports generated by the failed run are saved in' - info "${salvaged_dir@Q} directory" - info - } >&2 - rm -rf "${salvaged_dir}" - cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}" - fail "copying done, stopping now" + sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}" + top_dir="${sdk_run_state}/pkg-reports" + dir_queue=( "${top_dir}" ) + all_dirs=() + all_files=() + while [[ ${#dir_queue[@]} -gt 0 ]]; do + dir=${dir_queue[0]} + dir_queue=( "${dir_queue[@]:1}" ) + entry=${dir#"${top_dir}"} + if [[ -z ${entry} ]]; then + all_dirs=( "${sdk_reports_dir}" "${all_dirs[@]}" ) + else + entry=${entry#/} + all_dirs=( "${sdk_reports_dir}/${entry}" "${all_dirs[@]}" ) fi - sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}-${arch}" - report_files=() - for full_file in "${sdk_run_state}/pkg-reports/"*; do - file=${full_file##"${sdk_run_state}/pkg-reports/"} - report_files+=( "${sdk_reports_dir}/${file}" ) + for full_path in "${dir}/"*; do + if [[ -d ${full_path} ]]; then + dir_queue+=( "${full_path}" ) + else + entry=${full_path##"${top_dir}/"} + all_files+=( "${sdk_reports_dir}/${entry}" ) + fi done - add_cleanup \ - "rm -f ${report_files[*]@Q}" \ - "rmdir ${sdk_reports_dir@Q}" - mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}" done + add_cleanup \ + "rm -f ${all_files[*]@Q}" \ + "rmdir ${all_dirs[*]@Q}" + mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}" done + cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk" } @@ -1319,6 +1337,24 @@ function pkginfo_c_process_file() { mvm_c_get_extra 'arch' arch mvm_c_get_extra 'report' report + local report_file + case ${report}:${arch} in + "${SDK_PKGS}:arm64") + # short-circuit it, there's no arm64 sdk + return 0 + ;; + "${SDK_PKGS}:amd64") + report_file="${WORKDIR}/pkg-reports/${which}/${report}" + ;; + "${BOARD_PKGS}:"*) + report_file="${WORKDIR}/pkg-reports/${which}/${arch}-${report}" + ;; + *) + local c=${report}:${arch} + devel_warn "unknown report-architecture combination (${c@Q})" + return 0 + esac + local pkg version_slot throw_away v s # shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else while read -r pkg version_slot throw_away; do @@ -1331,7 +1367,7 @@ function pkginfo_c_process_file() { pkg_set_ref["${pkg}"]='x' mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}" pkg_debug_disable - done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" + done <"${report_file}" } # Gets a profile of the pkginfo mvm. The "profile" is a confusing @@ -1710,28 +1746,36 @@ function consistency_checks() { function read_package_sources() { local -n package_sources_map_ref=${1}; shift - local arch which report pkg repo saved_repo - for arch in "${ARCHES[@]}"; do - for which in "${WHICH[@]}"; do - for report in sdk-package-repos board-package-repos; do - while read -r pkg repo; do - saved_repo=${package_sources_map_ref["${pkg}"]:-} - if [[ -n ${saved_repo} ]]; then - if [[ ${saved_repo} != "${repo}" ]]; then - pkg_warn \ - '- different repos used for the package:' \ - " - package: ${pkg}" \ - ' - repos:' \ - " - ${saved_repo}" \ - " - ${repo}" - fi - else - package_sources_map_ref["${pkg}"]=${repo} - fi - done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" - done + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a files=() + local which arch + for which in "${WHICH[@]}"; do + files+=( "${WORKDIR}/pkg-reports/${which}/sdk-package-repos" ) + for arch in "${ARCHES[@]}"; do + files+=( "${WORKDIR}/pkg-reports/${which}/${arch}-board-package-repos" ) done done + + local file pkg repo saved_repo + for file in "${files[@]}"; do + while read -r pkg repo; do + saved_repo=${package_sources_map_ref["${pkg}"]:-} + if [[ -n ${saved_repo} ]]; then + if [[ ${saved_repo} != "${repo}" ]]; then + pkg_warn \ + '- different repos used for the package:' \ + " - package: ${pkg}" \ + ' - repos:' \ + " - ${saved_repo}" \ + " - ${repo}" + fi + else + package_sources_map_ref["${pkg}"]=${repo} + fi + done <"${file}" + done } # This monstrosity takes renames map and package tags information, @@ -2950,14 +2994,12 @@ function handle_profiles() { # shellcheck disable=SC1091 # generated file source "${WORKDIR}/globals" - local -a files - files=() - local arch which report - for arch in "${ARCHES[@]}"; do - for which in "${WHICH[@]}"; do - for report in sdk-profiles board-profiles; do - files+=("${WORKDIR}/pkg-reports/${which}-${arch}/${report}") - done + local -a files=() + local which arch + for which in "${WHICH[@]}"; do + files+=("${WORKDIR}/pkg-reports/${which}/sdk-profiles") + for arch in "${ARCHES[@]}"; do + files+=("${WORKDIR}/pkg-reports/${which}/${arch}-board-profiles") done done local -A profile_dirs_set diff --git a/pkg_auto/inside_sdk_container.sh b/pkg_auto/inside_sdk_container.sh index 6e128a1992..1d53b9661b 100755 --- a/pkg_auto/inside_sdk_container.sh +++ b/pkg_auto/inside_sdk_container.sh @@ -1,33 +1,34 @@ #!/bin/bash ## -## Gathers information about SDK and board packages. Also collects -## info about actual build deps of board packages, which may be useful -## for verifying if SDK provides those. +## Gathers information about SDK packages and board packages for each +## passed architecture. Also collects info about actual build deps of +## board packages, which may be useful for verifying if SDK provides +## those. ## ## Reports generated: ## sdk-pkgs - contains package information for SDK ## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK -## board-pkgs - contains package information for board for chosen architecture -## board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies +## ${arch}-board-pkgs - contains package information for board for chosen architecture +## ${arch}-board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies ## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order -## board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order +## ${arch}-board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order ## sdk-package-repos - contains package information with their repos for SDK -## board-package-repos - contains package information with their repos for board +## ${arch}-board-package-repos - contains package information with their repos for board ## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports -## board-emerge-output - contains raw emerge output for board being a base for other reports +## ${arch}-board-emerge-output - contains raw emerge output for board being a base for other reports ## sdk-emerge-output-filtered - contains only lines with package information for SDK -## board-emerge-output-filtered - contains only lines with package information for board +## ${arch}-board-emerge-output-filtered - contains only lines with package information for board ## sdk-emerge-output-junk - contains only junk lines for SDK -## board-emerge-output-junk - contains only junk lines for board +## ${arch}-board-emerge-output-junk - contains only junk lines for board ## *-warnings - warnings printed by emerge or other tools ## ## Parameters: ## -h: this help ## ## Positional: -## 1 - architecture (amd64 or arm64) -## 2 - reports directory +## 1 - reports directory +## # - architectures (currently only amd64 or arm64 are valid) ## set -euo pipefail @@ -54,16 +55,16 @@ while [[ ${#} -gt 0 ]]; do esac done -if [[ ${#} -ne 2 ]]; then - fail 'Expected two parameters: board architecture and reports directory' +if [[ ${#} -lt 2 ]]; then + fail 'Expected at least two parameters: reports directory and one or more board architectures' fi -arch=${1}; shift reports_dir=${1}; shift +# rest are architectures mkdir -p "${reports_dir}" -set_eo "${reports_dir}" +set_eo "${reports_dir}" "${@}" echo 'Running egencache for portage-stable' generate_cache_for 'portage-stable' 2>"${EGENCACHE_W}" @@ -77,36 +78,48 @@ copy_cache_to_reports 'coreos-overlay' "${reports_dir}" 2>>"${EGENCACHE_W}" echo 'Running pretend-emerge to get complete report for SDK' package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}" -echo 'Running pretend-emerge to get complete report for board' -package_info_for_board "${arch}" >"${BOARD_EO}" 2>"${BOARD_EO_W}" +for arch; do + be=${arch^^}_BOARD_EO + bew=${arch^^}_BOARD_EO_W + echo "Running pretend-emerge to get complete report for ${arch} board" + package_info_for_board "${arch}" >"${!be}" 2>"${!bew}" +done -ensure_no_errors +ensure_no_errors "${@}" echo 'Separating emerge info from junk in SDK emerge output' filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}" -junk_sdk_eo >"${SDK_EO}-junk" 2>>"${SDK_EO_W}" -echo 'Separating emerge info from junk in board emerge output' -filter_board_eo "${arch}" >"${BOARD_EO_F}" 2>>"${BOARD_EO_W}" -junk_board_eo >"${BOARD_EO}-junk" 2>>"${BOARD_EO_W}" +junk_sdk_eo >"${SDK_EO_J}" 2>>"${SDK_EO_W}" +for arch; do + bej=${arch^^}_BOARD_EO_J + bef=${arch^^}_BOARD_EO_F + bew=${arch^^}_BOARD_EO_W + echo "Separating emerge info from junk in ${arch} board emerge output" + filter_board_eo "${arch}" >"${!bef}" 2>>"${!bew}" + junk_board_eo "${arch}" >"${!bej}" 2>>"${!bew}" +done -ensure_valid_reports +ensure_valid_reports "${@}" echo 'Generating SDK packages listing' versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings" echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)' versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings" -echo 'Generating board packages listing' -versions_board >"${reports_dir}/board-pkgs" 2>"${reports_dir}/board-pkgs-warnings" -echo 'Generating board packages bdeps listing' -board_bdeps >"${reports_dir}/board-bdeps" 2>"${reports_dir}/board-bdeps-warnings" echo 'Generating SDK profiles evaluation list' ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings" -echo 'Generating board profiles evaluation list' -ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/board-profiles" 2>"${reports_dir}/board-profiles-warnings" echo 'Generating SDK package source information' package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings" -echo 'Generating board package source information' -package_sources_board >"${reports_dir}/board-package-repos" 2>"${reports_dir}/board-package-repos-warnings" + +for arch; do + echo "Generating ${arch} board packages listing" + versions_board "${arch}" >"${reports_dir}/${arch}-board-pkgs" 2>"${reports_dir}/${arch}-board-pkgs-warnings" + echo "Generating ${arch} board packages bdeps listing" + board_bdeps "${arch}" >"${reports_dir}/${arch}-board-bdeps" 2>"${reports_dir}/${arch}-board-bdeps-warnings" + echo "Generating ${arch} board profiles evaluation list" + ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/${arch}-board-profiles" 2>"${reports_dir}/${arch}-board-profiles-warnings" + echo "Generating ${arch} board package source information" + package_sources_board "${arch}" >"${reports_dir}/${arch}-board-package-repos" 2>"${reports_dir}/${arch}-board-package-repos-warnings" +done echo "Cleaning empty warning files" clean_empty_warning_files "${reports_dir}" From fff6bd78b2f8cdb3cc658356309e5fcebd4b52d2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 18:18:03 +0200 Subject: [PATCH 07/18] pkg-auto: Move regular expression for package name to gentoo_ver I did it initially, because I think I'll use this expression in other place too. In the end I didn't but I still think it's a better place for it. --- pkg_auto/impl/gentoo_ver.sh | 5 ++++- pkg_auto/impl/pkg_auto_lib.sh | 11 ++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/pkg_auto/impl/gentoo_ver.sh b/pkg_auto/impl/gentoo_ver.sh index 1df280be27..19576dd1d7 100644 --- a/pkg_auto/impl/gentoo_ver.sh +++ b/pkg_auto/impl/gentoo_ver.sh @@ -12,7 +12,10 @@ __GENTOO_VER_SH_INCLUDED__=x source "$(dirname "${BASH_SOURCE[0]}")/util.sh" -VER_ERE="^([0-9]+(\.[0-9]+)*)([a-z]?)((_(alpha|beta|pre|rc|p)[0-9]*)*)(-r[0-9]+)?$" +VER_ERE_UNBOUNDED="([0-9]+(\.[0-9]+)*)([a-z]?)((_(alpha|beta|pre|rc|p)[0-9]*)*)(-r[0-9]+)?" +VER_ERE='^'"${VER_ERE_UNBOUNDED}"'$' + +PKG_ERE_UNBOUNDED="[A-Za-z0-9_][A-Za-z0-9+_.-]*/[A-Za-z0-9_][A-Za-z0-9+_-]*" # @FUNCTION: _ver_compare_int # @USAGE: diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index f2f0d3cc7a..57fa697325 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -1006,14 +1006,6 @@ function process_listings() { # shellcheck disable=SC1091 # generated file source "${WORKDIR}/globals" - local ver_ere pkg_ere - # VER_ERE comes from gentoo_ver.sh - ver_ere=${VER_ERE} - # regexp begins with ^ and ends with $, so strip them - ver_ere=${ver_ere#'^'} - ver_ere=${ver_ere%'$'} - pkg_ere='[a-z0-9]*-?[a-z0-9]*/[a-z0-9A-Z_+-]*' - #mvm_debug_enable pl_pkg_to_tags_set_mvm mvm_declare pl_pkg_to_tags_set_mvm mvm_mvc_set @@ -1036,7 +1028,8 @@ function process_listings() { pkg_debug "processing listings: adding tag ${kind^^}" pkg_debug_disable mvm_add pl_pkg_to_tags_set_mvm "${pkg}" "${kind^^}" - done < <(sed -E -e 's#^('"${pkg_ere}"')-'"${ver_ere}"'::.*#\1#' "${listing}") + # VER_ERE_UNBOUNDED and PKG_ERE_UNBOUNDED come from gentoo_ver.sh + done < <(sed -E -e 's#^('"${PKG_ERE_UNBOUNDED}"')-'"${VER_ERE_UNBOUNDED}"'::.*#\1#' "${listing}") done done From e1a4d8e5a92786ab2ac6b72859351a27cdddfbe2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 18:17:56 +0200 Subject: [PATCH 08/18] pkg-auto: Move debugging stuff to a separate file I'll use it also in new libraries. --- pkg_auto/impl/debug.sh | 190 ++++++++++++++++++++++++++++++++++ pkg_auto/impl/pkg_auto_lib.sh | 125 +++------------------- 2 files changed, 204 insertions(+), 111 deletions(-) create mode 100644 pkg_auto/impl/debug.sh diff --git a/pkg_auto/impl/debug.sh b/pkg_auto/impl/debug.sh new file mode 100644 index 0000000000..47dcfb9425 --- /dev/null +++ b/pkg_auto/impl/debug.sh @@ -0,0 +1,190 @@ +#!/bin/bash + +if [[ -z ${__DEBUG_SH_INCLUDED__:-} ]]; then +__DEBUG_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Adds package names for which debugging output may be enabled. +function pkg_debug_add() { + if [[ ${#} -eq 0 ]]; then + return + fi + if ! declare -p __D_DEBUG_PACKAGES >/dev/null 2>&1; then + declare -gA __D_DEBUG_PACKAGES=() + fi + local pkg + for pkg; do + __D_DEBUG_PACKAGES["${pkg}"]=x + done +} + +# Resets debugging state to zero (no packages to debug, debug output +# disabled). +function pkg_debug_reset() { + unset __D_DEBUG_PACKAGES __D_DEBUG +} + +# Enables debugging output when specific packages are processed. +# +# Params: +# +# @ - package names to enable debugging for +function pkg_debug_enable() { + local -A pkg_set=() + local -a vals=() + local pkg + + if ! declare -p __D_DEBUG_PACKAGES >/dev/null 2>&1; then + return 0 + fi + + for pkg; do + if [[ -n ${pkg_set["${pkg}"]:-} ]]; then + continue + fi + pkg_set["${pkg}"]=x + if [[ -n ${__D_DEBUG_PACKAGES["${pkg}"]:-} ]]; then + vals+=( "${pkg}" ) + fi + done + if [[ ${#vals[@]} -gt 0 ]]; then + declare -g __D_DEBUG + join_by __D_DEBUG ',' "${vals[@]}" + fi +} + +# Get a list of package names for which debugging output may be +# enabled. +function pkg_debug_packages() { + local -n pkg_names_ref=${1}; shift + + if ! declare -p __D_DEBUG_PACKAGES >/dev/null 2>&1; then + pkg_names_ref=() + else + pkg_names_ref=( "${!__D_DEBUG_PACKAGES[@]}" ) + fi +} + +# Checks if debugging output is enabled. Returns true if yes, +# otherwise false. +function pkg_debug_enabled() { + local -i ret=0 + [[ -n ${__D_DEBUG:-} ]] || ret=1 + return ${ret} +} + +# Disables debugging output. +function pkg_debug_disable() { + unset __D_DEBUG +} + +# Prints passed parameters if debugging is enabled. +# +# Params: +# +# @ - parameters to print +function pkg_debug() { + if [[ -n ${__D_DEBUG:-} ]]; then + pkg_debug_print_c "${__D_DEBUG}" "${@}" + fi +} + +# Prints passed lines if debugging is enabled. +# +# Params: +# +# @ - lines to print +function pkg_debug_lines() { + if [[ -n ${__D_DEBUG:-} ]]; then + pkg_debug_print_lines_c "${__D_DEBUG}" "${@}" + fi +} + +# Prints passed parameters unconditionally with debug +# formatting. Useful for more complicated debugging logic using +# pkg_debug_enabled. +# +# if pkg_debug_enabled; then +# pkg_debug_print 'debug message' +# fi +# +# Params: +# +# @ - parameters to print +function pkg_debug_print() { + if [[ -z ${__D_DEBUG:+notempty} ]]; then + info "bad use of pkg_debug_print, __D_DEBUG is unset" + debug_stacktrace + fail '' + fi + pkg_debug_print_c "${__D_DEBUG}" "${@}" +} + +# Prints passed lines unconditionally with debug formatting. Useful +# for more complicated debugging logic using pkg_debug_enabled. +# +# if pkg_debug_enabled; then +# pkg_debug_print_lines 'debug' 'message' +# fi +# +# Params: +# +# @ - lines to print +function pkg_debug_print_lines() { + if [[ -z ${__D_DEBUG:+notempty} ]]; then + info "bad use of pkg_debug_print_lines, __D_DEBUG is unset" + debug_stacktrace + fail '' + fi + pkg_debug_print_lines_c "${__D_DEBUG}" "${@}" +} + +# Prints passed parameters unconditionally with custom debug +# formatting. Useful for either more complicated debugging logic using +# pkg_debug_packages or for non-package-specific debugging situations. +# +# local -a dpkgs=() +# pkg_debug_packages dpkgs +# local pkg +# for pkg in "${dpkgs[@]}"; do pkg_debug_print_c "${pkg}" 'debug message' +# +# Params: +# +# 1 - comma-separated package names +# @ - parameters to print +function pkg_debug_print_c() { + local d_debug=${1} + info "DEBUG(${d_debug}): ${*}" +} + +# Prints passed lines unconditionally with custom debug formatting. +# Useful for either more complicated debugging logic using +# pkg_debug_packages or for non-package-specific debugging situations. +# +# local -a dpkgs=() lines=( 'debug' 'message; ) +# pkg_debug_packages dpkgs +# local pkg +# for pkg in "${dpkgs[@]}"; do pkg_debug_print_lines_c "${pkg}" "${lines[@]}" +# +# Params: +# +# 1 - comma-separated package names +# @ - lines to print +function pkg_debug_print_lines_c() { + local d_debug=${1} + info_lines "${@/#/"DEBUG(${d_debug}): "}" +} + +# Prints current stacktrace. +function debug_stacktrace() { + local -i idx=0 last_idx=${#FUNCNAME[@]} + + ((last_idx--)) + while [[ idx -lt last_idx ]]; do + info "at ${FUNCNAME[idx]}, invoked by ${FUNCNAME[$((idx + 1))]} in ${BASH_SOURCE[idx + 1]}:${BASH_LINENO[idx]}" + ((++idx)) + done +} + +fi diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 57fa697325..2a4eaa1a14 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -51,6 +51,7 @@ __PKG_AUTO_LIB_SH_INCLUDED__=x source "$(dirname "${BASH_SOURCE[0]}")/util.sh" source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" +source "${PKG_AUTO_IMPL_DIR}/debug.sh" source "${PKG_AUTO_IMPL_DIR}/gentoo_ver.sh" # Sets up the workdir using the passed config. The config can be @@ -128,7 +129,7 @@ function setup_workdir_with_config() { if [[ -n ${cfg_sdk_image_override} ]]; then override_sdk_image_name "${cfg_sdk_image_override}" fi - add_debug_packages "${cfg_debug_packages[@]}" + pkg_debug_add "${cfg_debug_packages[@]}" } # Goes over the list of automatically updated packages and synces them @@ -251,27 +252,6 @@ function override_sdk_image_name() { append_to_globals "SDK_IMAGE=${image_name@Q}" } -# Adds information about packages to be debugged to the globals file. -# -# Params: -# -# @ - a list of packages to be debugged -function add_debug_packages() { - local -a prepared lines - prepared=( "${@@Q}" ) - prepared=( "${prepared[@]/#/' ['}" ) - prepared=( "${prepared[@]/%/']=x'}" ) - lines=( - '' - 'local -A DEBUG_PACKAGES' - '' - 'DEBUG_PACKAGES=(' - "${prepared[@]}" - ')' - ) - append_to_globals "${lines[@]}" -} - # Appends passed lines to the globals file. # # Params: @@ -1025,7 +1005,7 @@ function process_listings() { # acct-group/adm-0-r2::portage-stable while read -r pkg; do pkg_debug_enable "${pkg}" - pkg_debug "processing listings: adding tag ${kind^^}" + pkg_debug "processing listing ${arch}/${file}: adding tag ${kind^^}" pkg_debug_disable mvm_add pl_pkg_to_tags_set_mvm "${pkg}" "${kind^^}" # VER_ERE_UNBOUNDED and PKG_ERE_UNBOUNDED come from gentoo_ver.sh @@ -1036,28 +1016,14 @@ function process_listings() { mvm_iterate pl_pkg_to_tags_set_mvm set_mvm_to_array_mvm_cb "${pkg_to_tags_mvm_var_name}" mvm_unset pl_pkg_to_tags_set_mvm #mvm_debug_disable pl_pkg_to_tags_set_mvm - if pkg_debug_possible; then - mvm_iterate "${pkg_to_tags_mvm_var_name}" debug_dump_package_tags "${pkg_to_tags_mvm_var_name}" - fi -} - -# A debug function that prints the package tags. Used as a callback to -# mvm_iterate. -# -# Params: -# -# 1 - name of the array mvm variable (extra arg of the callback) -# 2 - name of the package -# 3 - name of the array variable holding tags (unused) -# @ - tags -function debug_dump_package_tags() { - local map_name=${1}; shift - local pkg=${1}; shift - shift # we don't care about array variable name - # rest are array elements, which are tags - pkg_debug_enable "${pkg}" - pkg_debug "tags for ${pkg} stored in ${map_name}: ${*}" - pkg_debug_disable + local -a pl_debug_pkgs pl_tags_array_name + pkg_debug_packages pl_debug_pkgs + for pkg in "${pl_debug_pkgs[@]}"; do + mvm_get "${pkg_to_tags_mvm_var_name}" "${pkg}" pl_tags_array_name + local -n tags_ref=${pl_tags_array_name:-EMPTY_ARRAY} + pkg_debug_print_c "${pkg}" "tags stored in ${pkg_to_tags_mvm_var_name}: ${tags_ref[*]}" + unset -n tags_ref + done } # A callback to mvm_iterate that turns a set mvm to an array mvm. It @@ -1689,11 +1655,11 @@ function consistency_checks() { mvm_get "${name}" "${pkg}" cc_slot_verminmax_map_var_name verminmax_map_var_names+=("${cc_slot_verminmax_map_var_name}") done - if pkg_debug_possible; then + if pkg_debug_enabled; then for name in "${verminmax_map_var_names[@]}"; do local -n slot_verminmax_map_ref=${name:-empty_map} - pkg_debug "all slots in ${name}: ${!slot_verminmax_map_ref[*]}" - pkg_debug "all vmms in ${name}: ${slot_verminmax_map_ref[*]}" + pkg_debug_print "all slots in ${name}: ${!slot_verminmax_map_ref[*]}" + pkg_debug_print "all vmms in ${name}: ${slot_verminmax_map_ref[*]}" unset -n slot_verminmax_map_ref done fi @@ -3143,67 +3109,4 @@ function handle_scripts() { generate_summary_stub scripts -- 'TODO: review the diffs' } -# Enables debug logs when specific packages are processed. -# -# It is expected that globals were already sourced, otherwise -# debugging won't be enabled at all. -# -# Params: -# -# @ - package names to enable debugging for -function pkg_debug_enable() { - local -A pkg_set - pkg_set=() - local -a vals - vals=() - local pkg - for pkg; do - if [[ -n ${pkg_set["${pkg}"]:-} ]]; then - continue - fi - pkg_set["${pkg}"]=x - if [[ -n ${DEBUG_PACKAGES["${pkg}"]:-} ]]; then - vals+=( "${pkg}" ) - fi - done - if [[ ${#vals[@]} -gt 0 ]]; then - declare -g PKG_AUTO_LIB_DEBUG - join_by PKG_AUTO_LIB_DEBUG ',' "${vals[@]}" - fi -} - -# Returns true or false whether any debugging has been enabled. -function pkg_debug_possible() { - local ret=0 - [[ ${#DEBUG_PACKAGES[@]} -gt 0 ]] || ret=1 - return ${ret} -} - -# Disables debug logs to be printed. -function pkg_debug_disable() { - unset PKG_AUTO_LIB_DEBUG -} - -# Prints passed parameters if debugging is enabled. -# -# Params: -# -# @ - parameters to print -function pkg_debug() { - if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then - info "DEBUG(${PKG_AUTO_LIB_DEBUG}): ${*}" - fi -} - -# Prints passed lines if debugging is enabled. -# -# Params: -# -# @ - lines to print -function pkg_debug_lines() { - if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then - info_lines "${@/#/"DEBUG(${PKG_AUTO_LIB_DEBUG}): "}" - fi -} - fi From 7c9c2dfea4a975e08e5203fe412332040ba166c4 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 16 Apr 2025 15:11:17 +0200 Subject: [PATCH 09/18] pkg-auto: Fix some reference variable names I'm trying to follow a convention where reference variable names end with "_ref". --- pkg_auto/impl/pkg_auto_lib.sh | 54 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 2a4eaa1a14..4342e59254 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -95,15 +95,15 @@ function setup_workdir_with_config() { case ${key} in scripts|aux|reports) var_name="cfg_${key//-/_}" - local -n var=${var_name} - var=$(realpath "${value}") - unset -n var + local -n var_ref=${var_name} + var_ref=$(realpath "${value}") + unset -n var_ref ;; old-base|new-base|sdk-image-override) var_name="cfg_${key//-/_}" - local -n var=${var_name} - var=${value} - unset -n var + local -n var_ref=${var_name} + var_ref=${value} + unset -n var_ref ;; cleanups|debug-packages) var_name="cfg_${key//-/_}" @@ -364,12 +364,12 @@ function get_ordered_update_filenames() { # 3 - old name # 4 - new name function update_rename_maps() { - local -n ft_map=${1}; shift + local -n ft_map_ref=${1}; shift local tf_set_mvm_var_name=${1}; shift local old_name=${1}; shift local new_name=${1}; shift - local prev_new_name=${ft_map["${old_name}"]:-} + local prev_new_name=${ft_map_ref["${old_name}"]:-} if [[ -n ${prev_new_name} ]] && [[ ${prev_new_name} != "${new_name}" ]]; then fail_lines \ @@ -382,18 +382,18 @@ function update_rename_maps() { local urm_set_var_name mvm_get "${tf_set_mvm_var_name}" "${old_name}" urm_set_var_name if [[ -n ${urm_set_var_name} ]]; then - local -n old_set=${urm_set_var_name} - new_set+=( "${!old_set[@]}" ) - unset -n old_set + local -n old_set_ref=${urm_set_var_name} + new_set+=( "${!old_set_ref[@]}" ) + unset -n old_set_ref fi new_set+=( "${old_name}" ) mvm_add "${tf_set_mvm_var_name}" "${new_name}" "${new_set[@]}" local old for old in "${new_set[@]}"; do - ft_map["${old}"]=${new_name} + ft_map_ref["${old}"]=${new_name} done - unset -n ft_map + unset -n ft_map_ref } # Sets up a worktree and necessary cleanups. @@ -615,12 +615,12 @@ function setup_git_env() { bot_email='buildbot@flatcar-linux.org' for role in AUTHOR COMMITTER; do for what in name email; do - local -n var="GIT_${role}_${what^^}" - local -n value="bot_${what}" + local -n var_ref="GIT_${role}_${what^^}" + local -n value_ref="bot_${what}" # shellcheck disable=SC2034 # it's a reference to external variable - var=${value} - unset -n value - unset -n var + var_ref=${value_ref} + unset -n value_ref + unset -n var_ref done done } @@ -1457,8 +1457,8 @@ function consistency_check_for_package() { local -A empty_map empty_map=() - local -n slot_version1_map=${ccfp_slot_version1_map_var_name:-empty_map} - local -n slot_version2_map=${ccfp_slot_version2_map_var_name:-empty_map} + local -n slot_version1_map_ref=${ccfp_slot_version1_map_var_name:-empty_map} + local -n slot_version2_map_ref=${ccfp_slot_version2_map_var_name:-empty_map} local ccfp_slots_set_var_name mvm_get "${pkg_slots_set_mvm_var_name}" "${pkg}" ccfp_slots_set_var_name @@ -1476,8 +1476,8 @@ function consistency_check_for_package() { local s v1 v2 ccfp_min ccfp_max mm pkg_debug "all slots iterated over: ${!slots_set_ref[*]}" for s in "${!slots_set_ref[@]}"; do - v1=${slot_version1_map["${s}"]:-} - v2=${slot_version2_map["${s}"]:-} + v1=${slot_version1_map_ref["${s}"]:-} + v2=${slot_version2_map_ref["${s}"]:-} pkg_debug "v1: ${v1}, v2: ${v2}" if [[ -n ${v1} ]] && [[ -n ${v2} ]]; then @@ -1531,8 +1531,8 @@ function consistency_check_for_package() { elif [[ ${#profile_1_slots[@]} -eq 1 ]] && [[ ${#profile_2_slots[@]} -eq 1 ]]; then s1=${profile_1_slots[0]} s2=${profile_2_slots[0]} - v1=${slot_version1_map["${s1}"]:-} - v2=${slot_version2_map["${s2}"]:-} + v1=${slot_version1_map_ref["${s1}"]:-} + v2=${slot_version2_map_ref["${s2}"]:-} if [[ ${v1} != "${v2}" ]]; then pkg_warn \ "- version mismatch:" \ @@ -2378,10 +2378,10 @@ function tags_for_pkg() { pkg_debug "no tags available" tags_ref=() else - local -n tags_in_mvm=${tfp_tags_var_name} + local -n tags_in_mvm_ref=${tfp_tags_var_name} # shellcheck disable=SC2034 # it's a reference to external variable - tags_ref=( "${tags_in_mvm[@]}" ) - pkg_debug "tags available: ${tags_in_mvm[*]}" + tags_ref=( "${tags_in_mvm_ref[@]}" ) + pkg_debug "tags available: ${tags_in_mvm_ref[*]}" fi pkg_debug_disable } From 3233f6eafe936d96bddc5a00feab6345b359ad7b Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 17:49:40 +0200 Subject: [PATCH 10/18] pkg-auto: Add a library for parsing md5-metadata cache files --- pkg_auto/impl/md5_cache_lib.sh | 1177 ++++++++++++++++++++++++++++++++ 1 file changed, 1177 insertions(+) create mode 100644 pkg_auto/impl/md5_cache_lib.sh diff --git a/pkg_auto/impl/md5_cache_lib.sh b/pkg_auto/impl/md5_cache_lib.sh new file mode 100644 index 0000000000..e3808f179f --- /dev/null +++ b/pkg_auto/impl/md5_cache_lib.sh @@ -0,0 +1,1177 @@ +#!/bin/bash + +# This file implements parsing the md5-metadata cache files and +# accessing the parsed results. Not the entirety of the cache file is +# parsed, only the parts that were needed at the time of writing +# it. So currently the exposed parts of parsed cache files are EAPI, +# IUSE, KEYWORDS, LICENSE, {B,R,P,I,}DEPEND and _eclasses_. The +# _eclasses_ part discards the checksums, though, so only names are +# available. + +if [[ -z ${__MD5_CACHE_LIB_SH_INCLUDED__:-} ]]; then +__MD5_CACHE_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +source "${PKG_AUTO_IMPL_DIR}/debug.sh" +source "${PKG_AUTO_IMPL_DIR}/gentoo_ver.sh" + +# +# Cache file +# + +# Indices to access fields of the cache file: +# PCF_EAPI_IDX - a string/number describing EAPI +# PCF_KEYWORDS_IDX - a name of an array containing keyword objects +# PCF_IUSE_IDX - a name of an array containing iuse objects +# PCF_BDEPEND_IDX - a name of a group object with build dependencies +# PCF_DEPEND_IDX - a name of a group object with dependencies +# PCF_IDEPEND_IDX - a name of a group object with install dependencies +# PCF_PDEPEND_IDX - a name of a group object with post dependencies +# PCF_RDEPEND_IDX - a name of a group object with runtime dependencies +# PCF_LICENSE_IDX - a name of a group object with licenses +# PCF_ECLASSES_IDX - a name of an array with used eclasses +declare -gri PCF_EAPI_IDX=0 PCF_KEYWORDS_IDX=1 PCF_IUSE_IDX=2 PCF_BDEPEND_IDX=3 PCF_DEPEND_IDX=4 PCF_IDEPEND_IDX=5 PCF_PDEPEND_IDX=6 PCF_RDEPEND_IDX=7 PCF_LICENSE_IDX=8 PCF_ECLASSES_IDX=9 + +# Declares empty cache files. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function cache_file_declare() { + struct_declare -ga "${@}" "( '0' 'EMPTY_ARRAY' 'EMPTY_ARRAY' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_GROUP' 'EMPTY_ARRAY' )" +} + +# Unsets cache files, can take several names, just like unset. +function cache_file_unset() { + local name + for name; do + local -n cache_file_ref=${name} + + __mcl_unset_array "${cache_file_ref[PCF_KEYWORDS_IDX]}" kw_unset + __mcl_unset_array "${cache_file_ref[PCF_IUSE_IDX]}" iuse_unset + __mcl_unset_array "${cache_file_ref[PCF_ECLASSES_IDX]}" unset + + group_unset \ + "${cache_file_ref[PCF_BDEPEND_IDX]}" \ + "${cache_file_ref[PCF_DEPEND_IDX]}" \ + "${cache_file_ref[PCF_IDEPEND_IDX]}" \ + "${cache_file_ref[PCF_PDEPEND_IDX]}" \ + "${cache_file_ref[PCF_RDEPEND_IDX]}" \ + "${cache_file_ref[PCF_LICENSE_IDX]}" + + unset -n cache_file_ref + done + unset "${@}" +} + +# Parses a cache file under the passed path. +# +# Params: +# +# 1 - cache file +# 2 - path to the cache file +function parse_cache_file() { + local -n cache_file_ref=${1}; shift + local path=${1}; shift + # rest are architectures + + if pkg_debug_enabled; then + local -a file_lines + mapfile -t file_lines <"${path}" + pkg_debug_print_lines "parsing ${path@Q}" "${file_lines[@]}" + unset file_lines + fi + + local -n pkg_eapi_ref='cache_file_ref[PCF_EAPI_IDX]' + local -n pkg_keywords_ref='cache_file_ref[PCF_KEYWORDS_IDX]' + local -n pkg_iuse_ref='cache_file_ref[PCF_IUSE_IDX]' + local -n pkg_bdepend_group_name_ref='cache_file_ref[PCF_BDEPEND_IDX]' + local -n pkg_depend_group_name_ref='cache_file_ref[PCF_DEPEND_IDX]' + local -n pkg_idepend_group_name_ref='cache_file_ref[PCF_IDEPEND_IDX]' + local -n pkg_pdepend_group_name_ref='cache_file_ref[PCF_PDEPEND_IDX]' + local -n pkg_rdepend_group_name_ref='cache_file_ref[PCF_RDEPEND_IDX]' + local -n pkg_license_group_name_ref='cache_file_ref[PCF_LICENSE_IDX]' + local -n pkg_eclasses_ref='cache_file_ref[PCF_ECLASSES_IDX]' + + local l key + while read -r l; do + key=${l%%=*} + pkg_debug "parsing ${key@Q}" + case ${key} in + 'EAPI') + pkg_eapi_ref=${l#*=} + pkg_debug "EAPI: ${pkg_eapi_ref}" + ;; + 'KEYWORDS') + __mcl_parse_keywords "${l#*=}" pkg_keywords_ref "${@}" + ;; + 'IUSE') + __mcl_parse_iuse "${l#*=}" pkg_iuse_ref + ;; + 'BDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_bdepend_group_name_ref + ;; + 'DEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_depend_group_name_ref + ;; + 'IDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_idepend_group_name_ref + ;; + 'PDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_pdepend_group_name_ref + ;; + 'RDEPEND') + __mcl_parse_dsf "${__MCL_DSF_DEPEND}" "${l#*=}" pkg_rdepend_group_name_ref + ;; + 'LICENSE') + __mcl_parse_dsf "${__MCL_DSF_LICENSE}" "${l#*=}" pkg_license_group_name_ref + ;; + '_eclasses_') + __mcl_parse_eclasses "${l#*=}" pkg_eclasses_ref + ;; + *) + pkg_debug "Not parsing ${key@Q}, ignoring" + ;; + esac + done <"${path}" +} + +# +# Use requirement (the part in the square brackets in the package +# dependency specification). +# + +# Indices to access fields of the use requirement: +# UR_NAME_IDX - a use name +# UR_MODE_IDX - a string describing the mode: "+" is enabled, "=" +# is strict relation, "!=" reversed strict relation, +# "?" is "enabled if enabled in the ebuild", "!?" is +# "disabled if disabled in the ebuild", and "-" is +# disabled) +# UR_PRETEND_IDX - a string describing pretend mode: it can be either +# empty (no pretending if use is missing in the +# package), "+" (pretend enabled if missing in the +# package), or "-" (pretend disabled if missing in +# the package) +declare -gri UR_NAME_IDX=0 UR_MODE_IDX=1 UR_PRETEND_IDX=2 + +# Declares empty use requirements. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function ur_declare() { + struct_declare -ga "${@}" "( 'ITS_UNSET' '+' '' )" +} + +# Unsets use requirements, can take several names, just like unset. +function ur_unset() { + unset "${@}" +} + +# Copies use requirement into another. +# +# Params: +# +# 1 - use requirement to be clobbered +# 2 - the source use requirement +function ur_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local -i idx + for idx in UR_NAME_IDX UR_MODE_IDX UR_PRETEND_IDX; do + to_clobber_ref[idx]=${to_copy_ref[idx]} + done +} + +# Stringifies use requirement. +# +# Params: +# +# 1 - use requirement +# 2 - name of a variable where the string form will be stored +function ur_to_string() { + local -n ur_ref=${1}; shift + local -n str_ref=${1}; shift + + case ${ur_ref[UR_MODE_IDX]} in + '!'*) + str_ref='!' + ;; + '-'*) + str_ref='-' + ;; + *) + str_ref='' + ;; + esac + str_ref+=${ur_ref[UR_NAME_IDX]} + + local p=${ur_ref[UR_PRETEND_IDX]} + if [[ -n ${p} ]]; then + str_ref+="(${p})" + fi + case ${ur_ref[UR_MODE_IDX]} in + *'=') + str_ref+='=' + ;; + *'?') + str_ref+='?' + ;; + esac +} + +# +# Package depedency specification (or PDS) +# + +# Enumeration describing blocker mode of a package. Self-describing. +declare -gri PDS_NO_BLOCK=0 PDS_WEAK_BLOCK=1 PDS_STRONG_BLOCK=2 + +# Indices to access fields of the package dependency specification: +# PDS_BLOCKS_IDX - a number describing blocker mode (use PDS_NO_BLOCK, +# PDS_WEAK_BLOCK and PDS_STRONG_BLOCK) +# PDS_OP_IDX - a string describing the relational operator, can be +# empty or one of "<", "<=", "=", "=*", "~", ">=", or +# ">" ("=*" is same as "=" with asterisk appended to +# version) +# if empty, the version will also be empty +# PDS_NAME_IDX - a qualified package name (so category/name) +# PDS_VER_IDX - a version of the package, may be empty (if so, +# operator is also empty) +# PDS_SLOT_IDX - a string describing the slot operator, without the +# preceding colon +# PDS_UR_IDX - a name of an array variable containing use +# requirements +declare -gri PDS_BLOCKS_IDX=0 PDS_OP_IDX=1 PDS_NAME_IDX=2 PDS_VER_IDX=3 PDS_SLOT_IDX=4 PDS_UR_IDX=5 + +# Declares empty package definition specifications. Can take flags +# that are passed to declare. Usually only -r or -t make sense to +# pass. Can take several names, just like declare. Takes no +# initializers - this is hardcoded. +function pds_declare() { + struct_declare -ga "${@}" "( ${PDS_NO_BLOCK@Q} '' 'ITS_UNSET' '' '' 'EMPTY_ARRAY' )" +} + +# Unsets package definition specifications, can take several names, +# just like unset. +function pds_unset() { + local use_reqs_name name + for name; do + local -n pds_ref=${name} + + use_reqs_name=${pds_ref[PDS_UR_IDX]} + local -n use_reqs_ref=${use_reqs_name} + ur_unset "${use_reqs_ref[@]}" + unset -n use_reqs_ref + if [[ ${use_reqs_name} != 'EMPTY_ARRAY' ]]; then + unset "${use_reqs_name}" + fi + + unset -n pds_ref + done + unset "${@}" +} + +# Copies package dependency specification into another. +# +# Params: +# +# 1 - package dependency specification to be clobbered +# 2 - the source package dependency specification +function pds_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local -i idx + for idx in PDS_BLOCKS_IDX PDS_OP_IDX PDS_NAME_IDX PDS_VER_IDX PDS_SLOT_IDX; do + to_clobber_ref[idx]=${to_copy_ref[idx]} + done + + if [[ ${to_copy_ref[PDS_UR_IDX]} = 'EMPTY_ARRAY' || ${#to_copy_ref[PDS_UR_IDX]} -eq 0 ]]; then + to_clobber_ref[PDS_UR_IDX]='EMPTY_ARRAY' + else + local pc_ur_array_name + gen_varname pc_ur_array_name + declare -ga "${pc_ur_array_name}=()" + + local -n urs_to_copy_ref=${to_copy_ref[PDS_UR_IDX]} urs_ref=${pc_ur_array_name} + local ur_name pc_ur_name + for ur_name in "${urs_to_copy_ref[@]}"; do + gen_varname pc_ur_name + ur_declare "${pc_ur_name}" + ur_copy "${pc_ur_name}" "${ur_name}" + urs_ref+=( "${pc_ur_name}" ) + done + + to_clobber_ref[PDS_UR_IDX]=${pc_ur_array_name} + fi +} + +# Adds use requirements to the package dependency specification. +# +# Params: +# +# 1 - package dependency specification +# @ - use requirements +function pds_add_urs() { + local -n pds_ref=${1}; shift + # rest are use requirements + + if [[ ${#} -eq 0 ]]; then + return 0 + fi + + local use_reqs_name=${pds_ref[PDS_UR_IDX]} + if [[ ${use_reqs_name} = 'EMPTY_ARRAY' ]]; then + local ura_name + gen_varname ura_name + declare -ga "${ura_name}=()" + pds_ref[PDS_UR_IDX]=${ura_name} + use_reqs_name=${ura_name} + unset ura_name + fi + + local -n use_reqs_ref=${use_reqs_name} + use_reqs_ref+=( "${@}" ) +} + +# Stringifies package dependency specification. +# +# Params: +# +# 1 - package dependency specification +# 2 - name of a variable where the string form will be stored +function pds_to_string() { + local -n pds_ref=${1}; shift + local -n str_ref=${1}; shift + + case ${pds_ref[PDS_BLOCKS_IDX]} in + "${PDS_NO_BLOCK}") + str_ref='' + ;; + "${PDS_WEAK_BLOCK}") + str_ref='!' + ;; + "${PDS_STRONG_BLOCK}") + str_ref='!!' + ;; + esac + local op=${pds_ref[PDS_OP_IDX]} + local v=${pds_ref[PDS_VER_IDX]} + if [[ ${op} = '=*' ]]; then + op='=' + # if there's an op, then we assume version is not empty - so + # version will never end up being just * + v+='*' + fi + str_ref+=${op}${pds_ref[PDS_NAME_IDX]} + if [[ -n ${v} ]]; then + str_ref+=-${v} + fi + local s=${pds_ref[PDS_SLOT_IDX]} + if [[ -n ${s} ]]; then + str_ref+=:${s} + fi + local -n urs_ref=${pds_ref[PDS_UR_IDX]} + if [[ ${#urs_ref[@]} -gt 0 ]]; then + str_ref+='[' + local u ur_str + for u in "${urs_ref[@]}"; do + ur_to_string "${u}" ur_str + str_ref+=${ur_str}, + done + unset ur_str u + str_ref=${str_ref:0:$(( ${#str_ref} - 1 ))}']' + fi +} + +# +# Group. A structure for describing {B,R,I,P,}DEPEND and LICENSE +# fields. Contains items, which can be either package dependency +# specifications or another groups. So it is a recursive structure. +# + +# Enumeration describing type of a group. Self-describing. +declare -gri GROUP_ALL_OF=0 GROUP_ANY_OF=1 + +# Enumeration describing whether items in group are for enabled or disabled USE. Self-describing. +declare -gri GROUP_USE_ENABLED=0 GROUP_USE_DISABLED=1 + +# Indices to access fields of the group: +# GROUP_TYPE_IDX - a number describing a type of the group (use +# GROUP_ALL_OF and GROUP_ANY_OF) +# GROUP_USE_IDX - a USE name of the group, may be empty, and must +# be empty for GROUP_ANY_OF groups +# GROUP_ENABLED_IDX - a number describing mode of the USE, should be +# ignored if USE name is empty (use +# GROUP_USE_ENABLED and GROUP_USE_DISABLED) +# GROUP_ITEMS_IDX - a name of an array containing items +declare -gri GROUP_TYPE_IDX=0 GROUP_USE_IDX=1 GROUP_ENABLED_IDX=2 GROUP_ITEMS_IDX=3 + +# Declares empty groups. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function group_declare() { + struct_declare -ga "${@}" "( ${GROUP_ALL_OF@Q} '' ${GROUP_USE_ENABLED@Q} 'EMPTY_ARRAY' )" +} + +# An empty readonly group. +group_declare -r EMPTY_GROUP + +# Unsets groups, can take several names, just like unset. +function group_unset() { + local -a to_unset=() + local name items_name + for name; do + if [[ ${name} == 'EMPTY_GROUP' ]]; then + continue + fi + + local -n group_ref=${name} + items_name=${group_ref[GROUP_ITEMS_IDX]} + unset -n group_ref + + to_unset+=( "${name}" ) + if [[ ${items_name} == 'EMPTY_ARRAY' ]]; then + continue + fi + to_unset+=( "${items_name}" ) + + local -n items_ref=${items_name} + item_unset "${items_ref[@]}" + unset -n items_ref + done + unset "${to_unset[@]}" +} + +# Copies group into another. +# +# Params: +# +# 1 - group to be clobbered +# 2 - the source group +function group_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local -i idx + for idx in GROUP_TYPE_IDX GROUP_USE_IDX GROUP_ENABLED_IDX; do + to_clobber_ref[idx]=${to_copy_ref[idx]} + done + + if [[ ${to_copy_ref[GROUP_ITEMS_IDX]} = 'EMPTY_ARRAY' || ${#to_copy_ref[GROUP_ITEMS_IDX]} -eq 0 ]]; then + to_clobber_ref[GROUP_ITEMS_IDX]='EMPTY_ARRAY' + else + local gc_items_name + gen_varname gc_items_name + declare -ga "${gc_items_name}=()" + + local -n items_to_copy_ref=${to_copy_ref[GROUP_ITEMS_IDX]} + local -n items_ref=${gc_items_name} + local item_name_to_copy gc_item_name + for item_name_to_copy in "${items_to_copy_ref[@]}"; do + gen_varname gc_item_name + item_declare "${gc_item_name}" + item_copy "${gc_item_name}" "${item_name_to_copy}" + items_ref+=( "${gc_item_name}" ) + done + unset -n items_ref items_to_copy_ref + to_clobber_ref[GROUP_ITEMS_IDX]="${gc_items_name}" + fi +} + +# Adds items to the group. +# +# Params: +# +# 1 - group +# @ - items +function group_add_items() { + local -n group_ref=${1}; shift + # rest are items to add + + if [[ ${#} -eq 0 ]]; then + return 0 + fi + + local items_name=${group_ref[GROUP_ITEMS_IDX]} + if [[ ${items_name} = 'EMPTY_ARRAY' ]]; then + local ia_name + gen_varname ia_name + declare -ga "${ia_name}=()" + group_ref[GROUP_ITEMS_IDX]=${ia_name} + items_name=${ia_name} + unset ia_name + fi + + local -n items_ref=${items_name} + items_ref+=( "${@}" ) +} + +# Stringifies group. +# +# Params: +# +# 1 - group +# 2 - name of a variable where the string form will be stored +function group_to_string() { + local -n group_ref=${1}; shift + local -n str_ref=${1}; shift + + local t=${group_ref[GROUP_TYPE_IDX]} + case ${t} in + "${GROUP_ALL_OF}") + local u=${group_ref[GROUP_USE_IDX]} + if [[ -n ${u} ]]; then + local e=${group_ref[GROUP_ENABLED_IDX]} + case ${e} in + "${GROUP_USE_ENABLED}") + str_ref='' + ;; + "${GROUP_USE_DISABLED}") + str_ref='!' + esac + unset e + str_ref+="${u}? " + else + str_ref='' + fi + unset u + ;; + "${GROUP_ANY_OF}") + str_ref='|| ' + ;; + esac + + str_ref+='( ' + local -n item_names_ref=${group_ref[GROUP_ITEMS_IDX]} + if [[ ${#item_names_ref[@]} -gt 0 ]]; then + local item_name item_str + for item_name in "${item_names_ref[@]}"; do + item_to_string "${item_name}" item_str + str_ref+="${item_str} " + done + unset item_str item_name + fi + str_ref+=')' +} + +# +# Item. A string of :. Mode is a single char and describes +# the data. +# +# Modes: +# "e" - empty, data is meaningless and should be just empty. +# "g" - group, data is a group (a name of a variable containing a +# group) +# "l" - license, data is a name of a license (plain string) +# "p" - package dependency specification, data is pds (a name of a +# variable containing a pds) + +# Declares empty items. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function item_declare() { + struct_declare -g "${@}" 'e:' +} + +# Unsets items, can take several names, just like unset. +function item_unset() { + local name + for name; do + local -n item_ref=${name} + + case ${item_ref} in + e:*) + # noop + : + ;; + g:*) + group_unset "${item_ref:2}" + ;; + l:*) + # noop, license is just a string + ;; + p:*) + pds_unset "${item_ref:2}" + ;; + esac + unset -n item_ref + done + + unset "${@}" +} + +# Copies item into another. +# +# Params: +# +# 1 - item to be clobbered +# 2 - the source item +function item_copy() { + local -n to_clobber_ref=${1}; shift + local -n to_copy_ref=${1}; shift + + local ic_name + local t=${to_copy_ref:0:1} v=${to_copy_ref:2} + case ${t} in + 'e'|'l') + to_clobber_ref=${to_copy_ref} + ;; + 'g') + gen_varname ic_name + group_declare "${ic_name}" + group_copy "${ic_name}" "${v}" + to_clobber_ref="g:${ic_name}" + ;; + 'p') + gen_varname ic_name + pds_declare "${ic_name}" + pds_copy "${ic_name}" "${v}" + to_clobber_ref="p:${ic_name}" + ;; + esac +} + +# Stringifies item. +# +# Params: +# +# 1 - item +# 2 - name of a variable where the string form will be stored +function item_to_string() { + local -n item_ref=${1}; shift + local -n str_ref=${1}; shift + + local t=${item_ref:0:1} + case ${t} in + e) + str_ref='' + ;; + g) + local group_name=${item_ref:2} + local group_str + group_to_string "${group_name}" group_str + str_ref=${group_str} + unset group_str group_name + ;; + l) + str_ref=${item_ref:2} + ;; + p) + local pds_name=${item_ref:2} + local pds_str='' + pds_to_string "${pds_name}" pds_str + str_ref=${pds_str} + unset pds_str pds_name + ;; + esac +} + +# +# Keyword +# +# n - name of keyword +# m - stable (amd64), unstable (~amd64), broken (-amd64), unknown (absent in KEYWORDS) + +# Enumeration describing mode of the keyword. +# KW_STABLE - like "amd64" +# KW_UNSTABLE - like "~amd64" +# KW_BROKEN - like "-amd64" +# KW_UNKNOWN - missing from KEYWORDS entirely +declare -gri KW_STABLE=0 KW_UNSTABLE=1 KW_BROKEN=2 KW_UNKNOWN=3 + +# Indices to access fields of the keyword: +# KW_NAME_IDX - name of the architecture +# KW_LEVEL_IDX - mode of the keyword (use KW_STABLE, KW_UNSTABLE, +# KW_BROKEN and KW_UNKNOWN) +declare -gri KW_NAME_IDX=0 KW_LEVEL_IDX=1 + +# Declares empty keywords. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function kw_declare() { + struct_declare -ga "${@}" "( 'ITS_UNSET' ${KW_UNSTABLE@Q} )" +} + +# Unsets keywords, can take several names, just like unset. +function kw_unset() { + unset "${@}" +} + +# Stringifies keyword. +# +# Params: +# +# 1 - keyword +# 2 - name of a variable where the string form will be stored +function kw_to_string() { + local -n kw_ref=${1}; shift + local -n str_ref=${1}; shift + + local n=${kw_ref[KW_NAME_IDX]} + case ${kw_ref[KW_LEVEL_IDX]} in + "${KW_STABLE}") + str_ref=${n} + ;; + "${KW_UNSTABLE}") + str_ref="~${n}" + ;; + "${KW_BROKEN}") + str_ref="-${n}" + ;; + "${KW_UNKNOWN}") + str_ref='' + ;; + esac +} + +# +# IUSE +# + +# n - name of IUSE flag +# m - 0 or 1 (0 disabled, 1 enabled) + +# Enumeration describing mode of the IUSE. Self-describing. +declare -gri IUSE_DISABLED=0 IUSE_ENABLED=1 + +# Indices to access fields of the IUSE: +# IUSE_NAME_IDX - IUSE name +# IUSE_MODE_IDX - IUSE mode (use IUSE_DISABLED and IUSE_ENABLED) +declare -gri IUSE_NAME_IDX=0 IUSE_MODE_IDX=1 + +# Declares empty IUSEs. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function iuse_declare() { + struct_declare -ga "${@}" "( 'ITS_UNSET' ${IUSE_DISABLED@Q} )" +} + +# Unsets IUSEs, can take several names, just like unset. +function iuse_unset() { + unset "${@}" +} + +# Stringifies IUSE. +# +# Params: +# +# 1 - IUSE +# 2 - name of a variable where the string form will be stored +function iuse_to_string() { + local -n iuse_ref=${1}; shift + local -n str_ref=${1}; shift + + case ${iuse_ref[IUSE_MODE_IDX]} in + "${IUSE_ENABLED}") + str_ref='+' + ;; + "${IUSE_DISABLED}") + str_ref='' + ;; + esac + str_ref+=${iuse_ref[IUSE_NAME_IDX]} +} + +# +# Implementation details +# + +# parse dependency specification format (DSF) + +declare -gri __MCL_DSF_DEPEND=0 __MCL_DSF_LICENSE=1 + +function __mcl_parse_dsf() { + local -i dsf_type=${1}; shift + local dep=${1}; shift + local -n top_group_out_var_name_ref=${1}; shift + + local -a group_stack + local pd_group pd_item pd_group_created='' pd_pds + + gen_varname pd_group + group_declare "${pd_group}" + group_stack+=( "${pd_group}" ) + + local -a tokens + mapfile -t tokens <<<"${dep// /$'\n'}" + local -i last_index + + local token + for token in "${tokens[@]}"; do + if [[ ${token} = '||' ]]; then + # "any of" group, so create the group, make it an item, add + # to current group and mark the new group as current + gen_varname pd_group + group_declare "${pd_group}" + local -n g_ref=${pd_group} + g_ref[GROUP_TYPE_IDX]=${GROUP_ANY_OF} + unset -n g_ref + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="g:${pd_group}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + + group_stack+=( "${pd_group}" ) + pd_group_created=x + elif [[ ${token} =~ ^!?[A-Za-z0-9][A-Za-z0-9+_-]*\?$ ]]; then + # "use" group, so create the group, make it an item, add + # to current group and mark the new group as current + local -i disabled=GROUP_USE_ENABLED + local use=${token%?} + + if [[ ${use} = '!'* ]]; then + disabled=GROUP_USE_DISABLED + use=${use:1} + fi + + gen_varname pd_group + group_declare "${pd_group}" + local -n g_ref=${pd_group} + g_ref[GROUP_TYPE_IDX]=${GROUP_ALL_OF} + g_ref[GROUP_USE_IDX]=${use} + g_ref[GROUP_ENABLED_IDX]=${disabled} + unset -n g_ref + + unset use disabled + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="g:${pd_group}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + + group_stack+=( "${pd_group}" ) + pd_group_created=x + elif [[ ${token} = '(' ]]; then + # beginning of a group; usually it is already created, + # because it is an "any of" or "use" group, but it is + # legal to specify just a "all of" group + if [[ -n ${pd_group_created} ]]; then + pd_group_created= + else + gen_varname pd_group + group_declare "${pd_group}" + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="g:${pd_group}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + + group_stack+=( "${pd_group}" ) + fi + elif [[ ${token} = ')' ]]; then + # end of a group, pop it from the stack + last_index=$(( ${#group_stack[@]} - 1 )) + unset "group_stack[${last_index}]" + elif [[ ${token} =~ ^[A-Za-z0-9_][A-Za-z0-9+_.-]*$ ]]; then + # license + if [[ dsf_type -ne __MCL_DSF_LICENSE ]]; then + fail "license tokens are only allowed for LICENSE keys (token: ${token@Q})" + fi + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="l:${token}" + unset -n i_ref + group_add_items "${group_stack[-1]}" "${pd_item}" + elif [[ ${token} =~ ^!?!?(<|<=|=|~|>=|>)?[A-Za-z0-9_][A-Za-z0-9+_.-]*/[A-Za-z0-9_] ]]; then + # pds + if [[ dsf_type -ne __MCL_DSF_DEPEND ]]; then + fail "package dependency specification is only allowed for DEPEND-like keys (token: ${token@Q})" + fi + + local -i blocks=PDS_NO_BLOCK + local operator='' name='' version='' slot='' + local -a use_requirements=() + + case ${token} in + '!!'*) + blocks=PDS_STRONG_BLOCK + token=${token:2} + ;; + '!'*) + blocks=PDS_WEAK_BLOCK + token=${token:1} + ;; + esac + + case ${token} in + '<='*|'>='*) + operator=${token:0:2} + token=${token:2} + ;; + '<'*|'='*|'~'*|'>'*) + operator=${token:0:1} + token=${token:1} + ;; + esac + + if [[ ${token} = *']' ]]; then + local use_reqs_string=${token#*'['} + use_reqs_string=${use_reqs_string%']'} + token=${token%"[${use_reqs_string}]"} + local -a use_reqs + mapfile -t use_reqs <<<"${use_reqs_string//,/$'\n'}" + + unset use_reqs_string + + local ur name mode pretend pd_ur + for ur in "${use_reqs[@]}"; do + name='' + mode='' + pretend='' + case ${ur} in + '-'*) + mode='-' + ur=${ur:1} + ;; + '!'*) + mode='!' + ur=${ur:1} + ;; + esac + if [[ ${mode} != '-' ]]; then + case ${ur} in + *'=') + mode+='=' + ur=${ur%'='} + ;; + *'?') + mode+='?' + ur=${ur%'?'} + ;; + esac + fi + if [[ -z ${mode} ]]; then + mode='+' + fi + if [[ ${ur} =~ \(([+-])\)$ ]]; then + pretend=${BASH_REMATCH[1]} + ur=${ur%"(${pretend})"} + fi + name=${ur} + gen_varname pd_ur + ur_declare "${pd_ur}" + local -n u_ref=${pd_ur} + u_ref[UR_NAME_IDX]=${name} + u_ref[UR_MODE_IDX]=${mode} + u_ref[UR_PRETEND_IDX]=${pretend} + unset -n u_ref + use_requirements+=( "${pd_ur}" ) + done + unset pd_ur pretend mode name ur use_reqs + fi + + if [[ ${token} = *:* ]]; then + slot=${token#*:} + token=${token%:"${slot}"} + fi + + if [[ ${token} = *'*' ]]; then + operator='=*' + token=${token%'*'} + fi + local ver_ere_with_dash="-(${VER_ERE_UNBOUNDED})$" + if [[ ${token} =~ ${ver_ere_with_dash} ]]; then + version=${BASH_REMATCH[1]} + token=${token%"-${version}"} + fi + name=${token} + + gen_varname pd_pds + pds_declare "${pd_pds}" + local -n p_ref=${pd_pds} + p_ref[PDS_BLOCKS_IDX]=${blocks} + p_ref[PDS_OP_IDX]=${operator} + p_ref[PDS_NAME_IDX]=${name} + p_ref[PDS_VER_IDX]=${version} + p_ref[PDS_SLOT_IDX]=${slot} + unset -n p_ref + pds_add_urs "${pd_pds}" "${use_requirements[@]}" + unset use_requirements slot version name operator blocks + + gen_varname pd_item + item_declare "${pd_item}" + local -n i_ref=${pd_item} + i_ref="p:${pd_pds}" + unset -n i_ref + + group_add_items "${group_stack[-1]}" "${pd_item}" + else + fail "unknown token ${token@Q}" + fi + done + + if [[ ${#group_stack[@]} -ne 1 ]]; then + fail "botched parsing, group stack has ${#group_stack[@]} groups instead of 1" + fi + + top_group_out_var_name_ref=${group_stack[0]} + + if pkg_debug_enabled; then + local pd_group_str + group_to_string "${group_stack[0]}" pd_group_str + pkg_debug_print "dsf: ${pd_group_str}" + fi +} + +function __mcl_parse_eclasses() { + local eclasses_string=${1}; shift + local -n eclasses_out_var_name_ref=${1}; shift + + local eclasses_var_name + gen_varname eclasses_var_name + declare -ga "${eclasses_var_name}=()" + local -n eclasses_ref=${eclasses_var_name} + + local -a tokens + mapfile -t tokens <<<"${eclasses_string//$'\t'/$'\n'}" + + local token + local -i eclass_name_now=1 + for token in "${tokens[@]}"; do + if [[ eclass_name_now -eq 1 ]]; then + eclasses_ref+=( "${token}" ) + fi + eclass_name_now=$((eclass_name_now ^ 1)) + done + eclasses_out_var_name_ref=${eclasses_var_name} + + if pkg_debug_enabled; then + local joined_eclasses_string + join_by joined_eclasses_string ' ' "${eclasses_ref[@]}" + pkg_debug_print "eclasses: ${joined_eclasses_string}" + fi +} + +function __mcl_parse_keywords() { + local keywords_string=${1}; shift + local -n keywords_out_var_name_ref=${1}; shift + # rest are architectures + + local keywords_var_name + gen_varname keywords_var_name + declare -ga "${keywords_var_name}=()" + local -n keywords_ref=${keywords_var_name} + + local -A keywords_set=() + + local -a tokens + mapfile -t tokens <<<"${keywords_string// /$'\n'}" + local token + for token in "${tokens[@]}"; do + keywords_set["${token}"]=x + done + + local has_hyphen_star=${keywords_set['-*']:-} + local arch mark kw_level_pair kw kw_name + local -i level + for arch; do + for kw_level_pair in "${arch}@${KW_STABLE}" "~${arch}@${KW_UNSTABLE}" "-${arch}@${KW_BROKEN}"; do + kw=${kw_level_pair%@*} + level=${kw_level_pair#*@} + mark=${keywords_set["${kw}"]:-} + if [[ -n ${mark} ]]; then + gen_varname kw_name + kw_declare "${kw_name}" + local -n k_ref=${kw_name} + k_ref[KW_NAME_IDX]=${arch} + k_ref[KW_LEVEL_IDX]=${level} + unset -n k_ref + keywords_ref+=( "${kw_name}" ) + break + fi + done + if [[ -z ${mark} ]]; then + gen_varname kw_name + kw_declare "${kw_name}" + local -n k_ref=${kw_name} + k_ref[KW_NAME_IDX]=${arch} + if [[ -n ${has_hyphen_star} ]]; then + k_ref[KW_LEVEL_IDX]=${KW_BROKEN} + else + k_ref[KW_LEVEL_IDX]=${KW_UNKNOWN} + fi + unset -n k_ref + keywords_ref+=( "${kw_name}" ) + fi + done + keywords_out_var_name_ref=${keywords_var_name} + + if pkg_debug_enabled; then + local -a all_kws_strings=() + local kw_name pk_kw_str + local joined_kws_string + for kw_name in "${keywords_ref[@]}"; do + kw_to_string "${kw_name}" pk_kw_str + all_kws_strings+=( "${pk_kw_str}" ) + done + join_by joined_kws_string ' ' "${all_kws_strings[@]}" + pkg_debug_print "keywords: ${joined_kws_string}" + fi +} + +function __mcl_parse_iuse() { + local iuse_string=${1}; shift + local -n iuse_out_var_name_ref=${1}; shift + + local iuse_var_name + gen_varname iuse_var_name + declare -ga "${iuse_var_name}=()" + local -n iuse_ref=${iuse_var_name} + + local -a tokens + mapfile -t tokens <<<"${iuse_string// /$'\n'}" + local token pi_iuse + for token in "${tokens[@]}"; do + gen_varname pi_iuse + iuse_declare "${pi_iuse}" + local -n i_ref=${pi_iuse} + if [[ ${token} = '+'* ]]; then + i_ref[IUSE_MODE_IDX]=${IUSE_ENABLED} + token=${token:1} + fi + i_ref[IUSE_NAME_IDX]=${token} + unset -n i_ref + iuse_ref+=( "${pi_iuse}" ) + done + + iuse_out_var_name_ref=${iuse_var_name} + + if pkg_debug_enabled; then + local -a all_iuse_strings=() + local iuse_name pi_iuse_str + local joined_iuse_string + for iuse_name in "${iuse_ref[@]}"; do + iuse_to_string "${iuse_name}" pi_iuse_str + all_iuse_strings+=( "${pi_iuse_str}" ) + done + join_by joined_iuse_string ' ' "${all_iuse_strings[@]}" + pkg_debug_print "IUSE: ${joined_iuse_string}" + fi +} + +function __mcl_unset_array() { + local array_name=${1}; shift + local item_unset_func=${1}; shift + + if [[ ${array_name} = EMPTY_ARRAY ]]; then + return 0 + fi + local -n array_ref=${array_name} + "${item_unset_func}" "${array_ref[@]}" + unset -n array_ref + unset "${array_name}" +} + +fi From 3b11e2404a39df26722f956932340c5285e238dc Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 17:54:06 +0200 Subject: [PATCH 11/18] pkg-auto: Add the highest-score-common-subsequence algorithm library It's a more general variant of the longest-common-subsequence algorithm. --- pkg_auto/impl/lcs.sh | 375 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 375 insertions(+) create mode 100644 pkg_auto/impl/lcs.sh diff --git a/pkg_auto/impl/lcs.sh b/pkg_auto/impl/lcs.sh new file mode 100644 index 0000000000..d7e4f5b23e --- /dev/null +++ b/pkg_auto/impl/lcs.sh @@ -0,0 +1,375 @@ +#!/bin/bash + +# Implementation of longest common subsequence (LCS) algorithm, with +# memoization and scoring. The algorithm is a base of utilities like +# diff - it finds the common items in two sequences. +# +# The memoization part is about remembering the results of LCS +# computation for shorter sequences, so they can be reused when doing +# computation for longer sequences. +# +# The scoring part is about generalization of the comparison function, +# which normally would return a boolean value describing whether two +# items are equal or not. With scoring, two things change. First is +# that the returned value from the comparison/scoring function is not +# a boolean, but rather an integer describing the degree of equality +# of the items. And second is that with booleans, the winning +# subsequence was the longest one, but with scores is not necessarily +# the longest one, but with the highest total score. This changes the +# algorithm a bit. While the typical LCS algorithm goes like as +# follows (s1 and s2 are sequences, i1 and i2 are indices in their +# respective sequences): +# +# +# +# LCS(s1, s2, i1, i2) -> sequence: +# if (i1 >= len(s1) || i2 >= len(s2)): return () +# if (s1[i1] == s2[i2]): +# l = LCS(s1, s2, i1 + 1, i2 + 1) +# return (s1[i1], l...) +# else: +# l1 = LCS(s1, s2, i1 + 1, i2) +# l2 = LCS(s1, s2, i1, i2 + 1) +# if (len(l1) > len(l2)): return l1 +# return l2 +# +# +# +# The score LCS goes more or less like this: +# +# +# +# SLCS(s1, s2, i1, i2) -> tuple(score, sequence): +# if (i1 >= len(s1) || i2 >= len(21)): return (score: 0, sequence: ()) +# score = score_func(s1[i1], s2[i2]) +# if (score == max_score): +# # matches the "equal" case in LCS above +# l = SLCS(s1, s2, i1 + 1, i2 + 1) +# return (score: score + l.score, sequence: (s1[i1], l.sequence...) +# else if (score == 0): +# # matches the "not equal" case in LCS above +# l1 = SLCS(s1, s2, i1 + 1, i2) +# l2 = SLCS(s1, s2, i1, i2 + 1) +# if (l1.score > l2.score): return l1 +# return l2 +# else: +# # new "equal, but not quite" case +# l = SLCS(s1, s2, i1 + 1, i2 + 1) +# l.score = score + l.score +# l.sequence = (s1[i1], l.sequence...) +# l1 = SLCS(s1, s2, i1 + 1, i2) +# l2 = SLCS(s1, s2, i1, i2 + 1) +# return tuple_with_max_score(l, l1, l2) +# +# +# +# The difference in the implementation below is that instead of +# starting at index 0 and go up for each sequence, we start at max +# index and go down. + +if [[ -z ${__LCS_SH_INCLUDED__:-} ]]; then +__LCS_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Constants to use for accessing fields in common items. +declare -gri LCS_X1_IDX=0 LCS_X2_IDX=1 LCS_IDX1_IDX=2 LCS_IDX2_IDX=3 + +# Computes the longest common subsequence of two sequences and stores +# it in the passed array. The common items stored in the array with +# longest common subsequence are actually names of arrays, where each +# array stores an item from the first sequence, an item from the +# second sequence, an index of the first item in first sequence and an +# index of the second item in the second sequence. To access those +# fields, use the LCS_*_IDX constants. +# +# The function optionally takes a name of the score function. If no +# such function is passed, the simple string score function is used +# (gives score 1 if strings are equal, otherwise gives score 0). The +# function should take either one or three parameters. The first +# parameter is always a name of a score integer variable. If only one +# parameter is passed, the function should write the maximum score to +# the score variable. When three parameters are passed, then the +# second and third are items from the sequences passed to lcs_run and +# they should be compared and rated. The function should rate the +# items with zero if they are completely different, with max score if +# they are the same, and with something in between zero and max score +# if they are somewhat equal but not really. +# +# Params: +# +# 1 - a name of an array variable containing the first sequence +# 2 - a name of an array variable containing the second sequence +# 3 - a name of an array variable where the longest common subsequence +# will be stored +# 4 - optional, a name of a score function +# +# Example: +# +# a=( A B C D E F G ) +# b=( B C D G K ) +# c=() +# +# lcs_run a b c +# +# echo "a: ${a[*]}" +# echo "b: ${b[*]}" +# +# cn=() +# for i in "${c[@]}"; do +# declare -n ref=${i} +# n=${ref[LCS_X1_IDX]} +# unset -n ref +# cn+=( "${n}" ) +# done +# echo "c: ${cn[*]}" +# unset "${c[@]}" +function lcs_run() { + local seq1_name=${1}; shift + local seq2_name=${1}; shift + local common_name=${1}; shift + local score_func=__lcs_str_score + if [[ ${#} -gt 0 ]]; then + score_func=${1}; shift + fi + + if ! declare -pF "${score_func}" >/dev/null 2>/dev/null; then + fail "${score_func@Q} is not a function" + fi + + local -i lr_max_score + "${score_func}" lr_max_score + + # lcs_memo_items_set is a set of common item names; the set owns + # the common items + # + # lcs_memo_map is a mapping from a memo key (which encodes indices + # from both sequences) to a pair, being a score and a name of an + # longest common subsequence for the encoded indices, separated + # with semicolon + local -A lcs_memo_map=() lcs_memo_items_set=() + local -a lr_lcs_state=( "${score_func}" lcs_memo_map lcs_memo_items_set "${seq1_name}" "${seq2_name}" "${lr_max_score}" ) + + local -n seq1=${seq1_name} seq2=${seq2_name} + local -i idx1=$(( ${#seq1[@]} - 1 )) idx2=$(( ${#seq2[@]} - 1 )) + unset -n seq1 seq2 + + __lcs_recurse lr_lcs_state "${idx1}" "${idx2}" + + local -n common_ref=${common_name} + local lr_memo_key='' + __lcs_make_memo_key "${idx1}" "${idx2}" lr_memo_key + local pair=${lcs_memo_map["${lr_memo_key}"]} + local -n memoized_items_ref=${pair#*:} + + common_ref=( "${memoized_items_ref[@]}" ) + + # steal the items from the items set that are a part of desired + # LCS, so they are not freed now, but later, when the LCS is freed + local item + for item in "${common_ref[@]}"; do + unset "lcs_memo_items_set[${item}]" + done + + # free the unneeded items + unset "${!lcs_memo_items_set[@]}" + # free all the LCSes, we already have a copy of the one we wanted + local items_var_name memo_key + for memo_key in "${!lcs_memo_map[@]}"; do + pair=${lcs_memo_map["${memo_key}"]} + items_var_name=${pair#*:} + if [[ ${items_var_name} != EMPTY_ARRAY ]]; then + unset "${items_var_name}" + fi + done +} + +## +## Details +## + +declare -gri __LCS_SCORE_IDX=0 __LCS_MEMO_MAP_IDX=1 __LCS_MEMO_ITEMS_SET_IDX=2 __LCS_SEQ1_IDX=3 __LCS_SEQ2_IDX=4 __LCS_MAX_SCORE_IDX=5 + +function __lcs_recurse() { + local lcs_state_name=${1}; shift + local -i i1=${1}; shift + local -i i2=${1}; shift + + if [[ i1 -lt 0 || i2 -lt 0 ]]; then + return 0 + fi + + local -n lcs_state=${lcs_state_name} + + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local memo_key='' + __lcs_make_memo_key ${i1} ${i2} memo_key + local pair=${memo_map_ref["${memo_key}"]:-} + if [[ -n ${pair} ]]; then + return 0 + fi + unset pair memo_key + unset -n memo_map_ref + + local -n seq1_ref=${lcs_state[__LCS_SEQ1_IDX]} + local -n seq2_ref=${lcs_state[__LCS_SEQ2_IDX]} + local x1=${seq1_ref[i1]} + local x2=${seq2_ref[i2]} + unset -n seq2_ref seq1_ref + + local -i lr_diff_score + local score_func=${lcs_state[__LCS_SCORE_IDX]} + "${score_func}" lr_diff_score "${x1}" "${x2}" + unset score_func + + local -i max_score=${lcs_state[__LCS_MAX_SCORE_IDX]} + if [[ lr_diff_score -gt max_score ]]; then + lr_diff_score=max_score + fi + if [[ lr_diff_score -lt 0 ]]; then + lr_diff_score=0 + fi + + if [[ lr_diff_score -eq max_score ]]; then + unset max_score + + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) $((i2 - 1)) + + local n + gen_varname __LCS_COMMON n + + declare -a -g "${n}=()" + + # retrieve memoized result for i1-1 and i2-1 (what we just + # called above), make a copy of it, add the prepared item and + # memoize that copy for i1 and i2; also add the prepared item + # to the set of prepared items + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local previous_memo_key + __lcs_make_memo_key $((i1 - 1)) $((i2 - 1)) previous_memo_key + local previous_pair=${memo_map_ref["${previous_memo_key}"]:-'0:EMPTY_ARRAY'} + local previous_score=${previous_pair%%:*} + local -n previous_memoized_prepared_items_ref=${previous_pair#*:} + local -n c_ref=${n} + local prepared_item_to_insert + + gen_varname __LCS_PREP prepared_item_to_insert + declare -g -a "${prepared_item_to_insert}=( ${x1@Q} ${x2@Q} ${i1@Q} ${i2@Q} )" + + c_ref=( "${previous_memoized_prepared_items_ref[@]}" "${prepared_item_to_insert}" ) + + local memo_key + __lcs_make_memo_key ${i1} ${i2} memo_key + memo_map_ref["${memo_key}"]="$((lr_diff_score + previous_score)):${n}" + + local -n memo_items_set=${lcs_state[__LCS_MEMO_ITEMS_SET_IDX]} + memo_items_set["${prepared_item_to_insert}"]=x + elif [[ lr_diff_score -eq 0 ]]; then + unset max_score + + __lcs_recurse "${lcs_state_name}" ${i1} $((i2 - 1)) + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) ${i2} + + # retrieve memoized results for i1 and i2-1 and for i1-1 and + # i2 (what we just called above), and memoize the longer + # result for i1 and i2 + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local lr_memo_key='' + + __lcs_make_memo_key ${i1} $((i2 - 1)) lr_memo_key + local previous_pair1=${memo_map_ref["${lr_memo_key}"]:-'0:EMPTY_ARRAY'} + local -i previous_score1=${previous_pair1%%:*} + + __lcs_make_memo_key $((i1 - 1)) ${i2} lr_memo_key + local previous_pair2=${memo_map_ref["${lr_memo_key}"]:-'0:EMPTY_ARRAY'} + local -i previous_score2=${previous_pair2%%:*} + + __lcs_make_memo_key ${i1} ${i2} lr_memo_key + if [[ previous_score1 -gt previous_score2 ]]; then + memo_map_ref["${lr_memo_key}"]=${previous_pair1} + else + memo_map_ref["${lr_memo_key}"]=${previous_pair2} + fi + else + unset max_score + + # 1 + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) $((i2 - 1)) + # 2 + __lcs_recurse "${lcs_state_name}" ${i1} $((i2 - 1)) + # 3 + __lcs_recurse "${lcs_state_name}" $((i1 - 1)) ${i2} + + local lr_mk1 lr_mk2 lr_mk3 + __lcs_make_memo_key $((i1 - 1)) $((i2 - 1)) lr_mk1 + __lcs_make_memo_key ${i1} $((i2 - 1)) lr_mk2 + __lcs_make_memo_key $((i1 - 1)) ${i2} lr_mk3 + + local -n memo_map_ref=${lcs_state[__LCS_MEMO_MAP_IDX]} + local pair1=${memo_map_ref["${lr_mk1}"]:-'0:EMPTY_ARRAY'} + local pair2=${memo_map_ref["${lr_mk2}"]:-'0:EMPTY_ARRAY'} + local pair3=${memo_map_ref["${lr_mk3}"]:-'0:EMPTY_ARRAY'} + local -i score1=${pair1%%:*} + local -i score2=${pair2%%:*} + local -i score3=${pair3%%:*} + local -i pick # either 1, 2 or 3 + score1=$((score1 + lr_diff_score)) + + if [[ score1 -gt score2 ]]; then + if [[ score1 -gt score3 ]]; then + pick=1 + else + pick=3 + fi + elif [[ score2 -gt score3 ]]; then + pick=2 + else + pick=3 + fi + + if [[ pick -eq 1 ]]; then + local lr_new_lcs + gen_varname __LCS_COMMON lr_new_lcs + + declare -a -g "${lr_new_lcs}=()" + + local -n previous_memoized_prepared_items_ref=${pair1#*:} + local -n c_ref=${lr_new_lcs} + local prepared_item_to_insert + + gen_varname __LCS_PREP prepared_item_to_insert + declare -g -a "${prepared_item_to_insert}=( ${x1@Q} ${x2@Q} ${i1@Q} ${i2@Q} )" + + c_ref=( "${previous_memoized_prepared_items_ref[@]}" "${prepared_item_to_insert}" ) + + local lr_memo_key + __lcs_make_memo_key ${i1} ${i2} lr_memo_key + memo_map_ref["${lr_memo_key}"]="${score1}:${lr_new_lcs}" + + local -n memo_items_set=${lcs_state[__LCS_MEMO_ITEMS_SET_IDX]} + memo_items_set["${prepared_item_to_insert}"]=x + else + local -n picked_pair="pair${pick}" + local lr_memo_key='' + __lcs_make_memo_key ${i1} ${i2} lr_memo_key + memo_map_ref["${lr_memo_key}"]=${picked_pair} + fi + fi +} + +function __lcs_make_memo_key() { + local i1=${1}; shift + local i2=${1}; shift + local -n ref=${1}; shift + local key="x${i1}x${i2}x" + key=${key//-/_} + ref=${key} +} + +function __lcs_str_score() { + local -n score_ref=${1}; shift + score_ref=1 + [[ ${#} -eq 0 || ${1} = "${2}" ]] || score_ref=0 +} + +fi From 016f35d8f5c38a2e58f8e1b61cc847c850d516c5 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 17:57:30 +0200 Subject: [PATCH 12/18] pkg-auto: Add the md5-metadata cache diff library --- pkg_auto/impl/md5_cache_diff_lib.sh | 1636 +++++++++++++++++++++++++++ 1 file changed, 1636 insertions(+) create mode 100644 pkg_auto/impl/md5_cache_diff_lib.sh diff --git a/pkg_auto/impl/md5_cache_diff_lib.sh b/pkg_auto/impl/md5_cache_diff_lib.sh new file mode 100644 index 0000000000..3dcab5ca63 --- /dev/null +++ b/pkg_auto/impl/md5_cache_diff_lib.sh @@ -0,0 +1,1636 @@ +#!/bin/bash + +# This file implements computing a diff between two cache files. +# +# The diff is done for EAPI, KEYWORDS, {B,R,I,P,}DEPEND and LICENSE +# fields. While doing a diff for the first two is rather trivial +# (these are just a string or an array of simple objects), then doing +# the diffs for the rest, that involves groups is a bit more +# involved. The group is first sorted, then flattened. Then both old +# and new sorted and flattened groups are fed into the LCS algorithm +# to get the common items. These can be used to figure out the +# differences between the old and new groups. +# +# Sorting a group merges all the similar subgroups into one. Say there +# is: +# +# !foo? ( sys-libs/bar ) +# !foo? ( app-admin/baz ) +# +# and they will get merged into: +# +# !foo? ( +# app-admin/baz +# sys-libs/bar +# ) +# +# There are exceptions - "any of" subgroups are not merged (only +# sorted), so: +# +# || ( +# sys-libs/foo +# sys-libs/bar +# ) +# || ( +# app-admin/foo +# app-admin/bar +# ) +# +# stay almost as they are - the order of the packages with the group +# will change to become ordered: +# +# || ( +# sys-libs/bar +# sys-libs/foo +# ) +# || ( +# app-admin/bar +# app-admin/foo +# ) +# +# Also unnamed "all-of" groups inside "any-of" groups are not merged +# too, so: +# +# || ( +# ( +# dev-lang/python:3.11 +# dev-python/setuptools[python_3_11] +# ) +# ( +# dev-lang/python:3.12 +# dev-python/setuptools[python_3_12] +# ) +# ) +# +# stay as they are. +# +# Flattening turns the group-as-a-recursive-structure into a list of +# tokens, so to speak. Each token can be than treated as a separate +# line for the LCS algorithm. This means that for the following +# groups: +# +# SUBGROUP { +# type: any-of, +# items: sys-libs/foo sys-libs/bar +# } +# +# MAINGROUP { +# type: all-off, +# use name: foo, +# use mode: disabled, +# items: SUBGROUP app-admin/bar +# } +# +# flattening of MAINGROUP will result in the following list: +# +# !foo? +# ( +# || +# ( +# sys-libs/foo +# sys-libs/bar +# ) +# app-admin/bar +# ) +# +# The LCS algorithm takes a score function. The one used here is a +# function that returns maximum score of 3 for equal tokens, score of +# 1 for package dependency specifications with the same name and 0 for +# the rest. This means that ">=sys-libs/foo-1.2.3" and +# ">=sys-libs/foo-1.2.3" will result in score 3 (same tokens), +# ">=sys-libs/foo-1.2.3" and ">=sys-libs/foo-3.2.1" will result in +# score 1 (same package name), and "sys-libs/foo" and "sys-libs/bar" +# will result in score 0 (the rest). The reason for giving a non-zero +# score on just package name equality is that we want to handle +# changes for a package (like a plain version bump) differently than +# dependency replacement (like replacing autotools with cmake, for +# instance). On the other hand, the reason for not giving max score on +# just package name equality was to increase the diff quality. Without +# scoring, a change from: +# +# dev-lang/python:3.11 +# +# to: +# +# dev-lang/python:3.10 +# dev-lang/python:3.11 +# +# was interpreted two actions: first, as changing slot of +# dev-lang/python from 3.11 to 3.10 and second, as adding +# dev-lang/python:3.11. With scoring, the change was interpreted a +# single action - adding dev-lang/python:3.10. + +if [[ -z ${__MD5_CACHE_DIFF_LIB_SH_INCLUDED__:-} ]]; then +__MD5_CACHE_DIFF_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +source "${PKG_AUTO_IMPL_DIR}/lcs.sh" +source "${PKG_AUTO_IMPL_DIR}/md5_cache_lib.sh" +source "${PKG_AUTO_IMPL_DIR}/mvm.sh" + +# +# Diff report. Contains indent-lines. Indent-line is a string of +# :. Indent level is a number describing +# indentation level. Line is an actual line, duh. +# + +# Indices to access fields of the diff report: +# DR_INDENT_IDX - a current indent level to use +# DR_LINES_IDX - a name of an array containing indent-lines +declare -gri DR_INDENT_IDX=0 DR_LINES_IDX=1 + +# Declares empty diff reports. Can take flags that are passed to +# declare. Usually only -r or -t make sense to pass. Can take several +# names, just like declare. Takes no initializers - this is hardcoded. +function diff_report_declare() { + struct_declare -ga "${@}" "( '0' 'EMPTY_ARRAY' )" +} + +# Unsets diff reports, can take several names, just like unset. +function diff_report_unset() { + local dr_var_name lines_var_name + for dr_var_name; do + local -n dr_ref=${dr_var_name} + + lines_var_name=${dr_ref[DR_LINES_IDX]} + if [[ ${lines_var_name} != EMPTY_ARRAY ]]; then + unset "${lines_var_name}" + fi + + unset -n dr_ref + done + unset "${@}" +} + +# Stores an non-empty string if diff report is empty (has no lines +# yet). +# +# Params: +# +# 1 - diff report +# 2 - name of a variable where the string will be stored +function diff_report_is_empty() { + local -n dr_ref=${1}; shift + local -n out_ref=${1}; shift + + local -n lines_ref=${dr_ref[DR_LINES_IDX]} + if [[ ${#lines_ref[@]} -eq 0 ]]; then + out_ref=x + else + out_ref= + fi +} + +# Appends a line with a current indentation to a diff report. +# +# Params: +# +# 1 - diff report +# 2 - a line +function diff_report_append() { + local dr_var_name=${1}; shift + local line=${1}; shift + + local -n dr_ref=${dr_var_name} + local dra_lines_name=${dr_ref[DR_LINES_IDX]} + if [[ ${dra_lines_name} == EMPTY_ARRAY ]]; then + gen_varname dra_lines_name + declare -ga "${dra_lines_name}=()" + dr_ref[DR_LINES_IDX]=${dra_lines_name} + fi + + local -i indent + indent=${dr_ref[DR_INDENT_IDX]} + + local indent_line=${indent}:${line} + pkg_debug " ${indent_line} (${dr_var_name@Q})" + + local -n lines_ref=${dra_lines_name} + lines_ref+=( "${indent_line}" ) +} + +# Appends a line with an increased-by-1 indentation to a diff report. +# +# Params: +# +# 1 - diff report +# 2 - a line +function diff_report_append_indented() { + local dr_var_name=${1}; shift + local line=${1}; shift + + diff_report_indent "${dr_var_name}" + diff_report_append "${dr_var_name}" "${line}" + diff_report_dedent "${dr_var_name}" +} + +# Appends all the lines from the source diff report to a diff report, +# increasing indentation level of each line by the current indentation +# level. +# +# Params: +# +# 1 - diff report to grow +# 2 - source diff report +function diff_report_append_diff_report() { + local dr_var_name=${1}; shift + local -n dr_ref2=${1}; shift + + local -n dr_ref=${dr_var_name} + local dra_lines_name=${dr_ref[DR_LINES_IDX]} + if [[ ${dra_lines_name} == EMPTY_ARRAY ]]; then + gen_varname dra_lines_name + declare -ga "${dra_lines_name}=()" + dr_ref[DR_LINES_IDX]=${dra_lines_name} + fi + + local -i indent + indent=${dr_ref[DR_INDENT_IDX]} + local -n lines_ref=${dra_lines_name} + + local -n lines2_ref=${dr_ref2[DR_LINES_IDX]} + local indent_line2 line2 + local -i indent2 + for indent_line2 in "${lines2_ref[@]}"; do + indent2=${indent_line2%%:*} + line2=${indent_line2#*:} + indent2=$((indent + indent2)) + indent_line2=${indent2}:${line2} + pkg_debug " ${indent_line2} (${dr_var_name@Q})" + lines_ref+=( "${indent_line2}" ) + done +} + +# Increases current indentation level of a diff report. +# +# Params: +# +# 1 - diff report to grow +function diff_report_indent() { + local -n dr_ref=${1}; shift + + ((++dr_ref[DR_INDENT_IDX])) +} + +# Decreases current indentation level of a diff report. +# +# Params: +# +# 1 - diff report to grow +function diff_report_dedent() { + local -n dr_ref=${1}; shift + + ((dr_ref[DR_INDENT_IDX]--)) +} + +# Diffs two cache files. +# +# Params: +# +# 1 - old cache file +# 2 - new cache file +# 3 - diff report where the diff will be written to +function diff_cache_data() { + local old_var_name=${1}; shift + local new_var_name=${1}; shift + local dr_var_name=${1}; shift + + __mcdl_diff_eapi "${old_var_name}" "${new_var_name}" "${dr_var_name}" + __mcdl_diff_keywords "${old_var_name}" "${new_var_name}" "${dr_var_name}" + __mcdl_diff_iuse "${old_var_name}" "${new_var_name}" "${dr_var_name}" + + local -i idx + for idx in PCF_BDEPEND_IDX PCF_DEPEND_IDX PCF_IDEPEND_IDX PCF_PDEPEND_IDX PCF_RDEPEND_IDX PCF_LICENSE_IDX; do + __mcdl_diff_deps "${old_var_name}" "${new_var_name}" ${idx} "${dr_var_name}" + done +} + +# +# Implemetation details. +# + +function __mcdl_diff_eapi() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local dr_var_name=${1}; shift + + local old_eapi=${old_ref[PCF_EAPI_IDX]} + local new_eapi=${new_ref[PCF_EAPI_IDX]} + + if [[ ${old_eapi} != "${new_eapi}" ]]; then + diff_report_append "${dr_var_name}" "EAPI changed from ${old_eapi@Q} to ${new_eapi@Q}" + fi +} + +function __mcdl_diff_iuse() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local dr_var_name=${1}; shift + + local old_iuses_var_name=${old_ref[PCF_IUSE_IDX]} + local new_iuses_var_name=${new_ref[PCF_IUSE_IDX]} + + local -A old_map=() new_map=() removed_iuses same_iuses added_iuses + + local -n old_iuses_ref=${old_iuses_var_name} new_iuses_ref=${new_iuses_var_name} + + local -i idx=0 + local iuse_var_name name + for iuse_var_name in "${old_iuses_ref[@]}"; do + local -n iuse_ref=${iuse_var_name} + name=${iuse_ref[IUSE_NAME_IDX]} + old_map["${name}"]=${idx} + unset -n iuse_ref + ((++idx)) + done + + idx=0 + for iuse_var_name in "${new_iuses_ref[@]}"; do + local -n iuse_ref=${iuse_var_name} + name=${iuse_ref[IUSE_NAME_IDX]} + new_map["${name}"]=${idx} + unset -n iuse_ref + ((++idx)) + done + + sets_split old_map new_map removed_iuses added_iuses same_iuses + + local iuse + for iuse in "${!removed_iuses[@]}"; do + diff_report_append "${dr_var_name}" "removed IUSE flag ${iuse@Q}" + diff_report_append_indented "${dr_var_name}" "TODO: describe removed IUSE flag" + done + for iuse in "${!added_iuses[@]}"; do + diff_report_append "${dr_var_name}" "added IUSE flag ${iuse@Q}" + diff_report_append_indented "${dr_var_name}" "TODO: describe added IUSE flag" + done + local old_mode new_mode mode_str + local -i old_idx new_idx + for iuse in "${!same_iuses[@]}"; do + old_idx=${old_map["${iuse}"]} + new_idx=${new_map["${iuse}"]} + local -n old_iuse_ref=${old_iuses_ref[${old_idx}]} new_iuse_ref=${new_iuses_ref[${new_idx}]} + old_mode=${old_iuse_ref[IUSE_MODE_IDX]} + new_mode=${new_iuse_ref[IUSE_MODE_IDX]} + unset -n new_iuse_ref old_iuse_ref + if [[ ${old_mode} -ne ${new_mode} ]]; then + case ${new_mode} in + "${IUSE_DISABLED}") + mode_str=disabled + ;; + "${IUSE_ENABLED}") + mode_str=enabled + ;; + esac + diff_report_append "${dr_var_name}" "IUSE flag ${iuse@Q} became ${mode_str} by default" + fi + done +} + +function __mcdl_diff_keywords() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local dr_var_name=${1}; shift + + local old_kws_var_name=${old_ref[PCF_KEYWORDS_IDX]} + local new_kws_var_name=${new_ref[PCF_KEYWORDS_IDX]} + + local -n old_kws_ref=${old_kws_var_name} + local -n new_kws_ref=${new_kws_var_name} + + if [[ ${#old_kws_ref[@]} -ne ${#new_kws_ref[@]} ]]; then + fail 'keywords number should always be the same' + fi + + local index old_name new_name old_level new_level level_str + for (( index=0; index < ${#old_kws_ref[@]}; ++index )); do + local -n old_kw_ref=${old_kws_ref["${index}"]} + local -n new_kw_ref=${new_kws_ref["${index}"]} + old_name=${old_kw_ref[KW_NAME_IDX]} + new_name=${new_kw_ref[KW_NAME_IDX]} + if [[ ${old_name} != "${new_name}" ]]; then + fail 'keywords of the same name should always have the same index' + fi + old_level=${old_kw_ref[KW_LEVEL_IDX]} + new_level=${new_kw_ref[KW_LEVEL_IDX]} + local extra_todo='' + if [[ ${old_level} -ne ${new_level} ]]; then + case ${new_level} in + "${KW_STABLE}") + level_str='stable' + extra_todo='TODO: update/drop accept keywords for this package in overlay profiles' + ;; + "${KW_UNSTABLE}") + level_str='unstable' + extra_todo='TODO: add/update accept keywords for this package in overlay profiles' + ;; + "${KW_BROKEN}") + level_str='broken' + ;; + "${KW_UNKNOWN}") + level_str='unknown' + ;; + esac + diff_report_append "${dr_var_name}" "package became ${level_str} on ${new_name@Q}" + if [[ -n ${extra_todo} ]]; then + diff_report_append_indented "${dr_var_name}" "${extra_todo}" + fi + fi + unset -n new_kw old_kw + done +} + +function __mcdl_flatten_group() { + local -n group_ref=${1}; shift + local flattened_items_var_name=${1}; shift + + local -n flattened_items_ref=${flattened_items_var_name} + local -n group_items_ref=${group_ref[GROUP_ITEMS_IDX]} + + case ${group_ref[GROUP_TYPE_IDX]} in + "${GROUP_ALL_OF}") + local name=${group_ref[GROUP_USE_IDX]} + if [[ -n ${name} ]]; then + case ${group_ref[GROUP_ENABLED_IDX]} in + "${GROUP_USE_ENABLED}") + : + ;; + "${GROUP_USE_DISABLED}") + name="!${name}" + ;; + esac + name+='?' + flattened_items_ref+=( "i:${name}" ) + fi + unset name + ;; + "${GROUP_ANY_OF}") + flattened_items_ref+=( "i:||" ) + ;; + esac + flattened_items_ref+=( "o:(" ) + + local item_var_name item_type + for item_var_name in "${group_items_ref[@]}"; do + local -n item_ref=${item_var_name} + item_type=${item_ref:0:1} + case ${item_type} in + 'e') + # do not add empty items to the list + : + ;; + 'g') + local subgroup_name=${item_ref:2} + __mcdl_flatten_group "${subgroup_name}" "${flattened_items_var_name}" + unset subgroup_name + ;; + 'l'|'p') + flattened_items_ref+=( "${item_ref}" ) + ;; + *) + fail "item ${item_ref} is bad" + ;; + esac + unset -n item_ref + done + flattened_items_ref+=( "c:)" ) +} + +function __mcdl_flattened_group_item_score() { + local -n score_ref=${1}; shift + + score_ref=3 + + if [[ ${#} -eq 0 ]]; then + return + fi + + local i1=${1}; shift + local i2=${1}; shift + + local t1=${i1:0:1} + local t2=${i2:0:1} + + if [[ ${t1} != "${t2}" ]]; then + score_ref=0 + return + fi + + local v1=${i1:2} + local v2=${i2:2} + case ${t1} in + 'l'|'i') + [[ "${v1}" = "${v2}" ]] || score_ref=0 + ;; + 'p') + local -n p1_ref=${v1} p2_ref=${v2} + local n1=${p1_ref[PDS_NAME_IDX]} n2=${p2_ref[PDS_NAME_IDX]} + if [[ "${n1}" = "${n2}" ]]; then + local p1_str p2_str + pds_to_string "${v1}" p1_str + pds_to_string "${v2}" p2_str + [[ "${p1_str}" = "${p2_str}" ]] || score_ref=1 + unset p2_str p1_str + else + score_ref=0 + fi + unset n2 n1 + unset -n p2_ref p1_ref + ;; + 'o'|'c') + # parens are the same everywhere + : + ;; + *) + fail "item ${i1} or ${i2} is bad" + ;; + esac +} + +function __mcdl_ur_mode_description() { + local mode=${1}; shift + local -n str_ref=${1}; shift + + case ${mode} in + '+') + str_ref='enabled' + ;; + '=') + str_ref='strict' + ;; + '!=') + str_ref='reversed strict' + ;; + '?') + str_ref='enabled if enabled' + ;; + '!?') + str_ref='disabled if disabled' + ;; + '-') + str_ref='disabled' + ;; + esac +} + +function __mcdl_ur_pretend_description() { + local pretend=${1}; shift + local -n str_ref=${1}; shift + + case ${pretend} in + '') + str_ref='must exist, no pretending if missing' + ;; + '+') + str_ref='will pretend as enabled if missing' + ;; + '-') + str_ref='will pretend as disabled if missing' + ;; + esac +} + +function __mcdl_iuse_stack_to_string() { + local -n stack_ref=${1}; shift + local -n str_ref=${1}; shift + + str_ref='' + # we always ignore the first element - it's a name of the toplevel + # unnamed group, so stack of length 1 means no groups were + # encountered + if [[ ${#stack_ref[@]} -le 1 ]]; then + return 0 + fi + + local iuse + # we always ignore the first element - it's a name of the toplevel + # unnamed group + for iuse in "${stack_ref[@]:1}"; do + str_ref+="${iuse@Q} -> " + done + str_ref=${str_ref:0:$((${#str_ref} - 4))} +} + +function __mcdl_pds_diff() { + local -n old_pds_ref=${1}; shift + local -n new_pds_ref=${1}; shift + local old_stack_name=${1}; shift + local new_stack_name=${1}; shift + local dr_var_name=${1}; shift + + local name=${new_pds_ref[PDS_NAME_IDX]} + + local old_iuses new_iuses + __mcdl_iuse_stack_to_string "${old_stack_name}" old_iuses + __mcdl_iuse_stack_to_string "${new_stack_name}" new_iuses + + diff_report_declare local_pds_dr + + if [[ ${old_iuses} != "${new_iuses}" ]]; then + if [[ -z ${new_iuses} ]]; then + diff_report_append local_pds_dr "dropped all USE conditionals" + else + diff_report_append local_pds_dr "USE conditionals changed to ${new_iuses}" + fi + fi + + local -i old_blocks=${old_pds_ref[PDS_BLOCKS_IDX]} new_blocks=${new_pds_ref[PDS_BLOCKS_IDX]} + if [[ old_blocks -ne new_blocks ]]; then + local block + case ${new_blocks} in + "${PDS_NO_BLOCK}") + block='no' + ;; + "${PDS_WEAK_BLOCK}") + block='weak' + ;; + "${PDS_STRONG_BLOCK}") + block='strong' + ;; + esac + diff_report_append local_pds_dr "changed to ${block} block" + unset block + fi + + local old_op=${old_pds_ref[PDS_OP_IDX]} old_ver=${old_pds_ref[PDS_VER_IDX]} new_op=${new_pds_ref[PDS_OP_IDX]} new_ver=${new_pds_ref[PDS_VER_IDX]} + if [[ ${old_op} = '=*' ]]; then + old_op='=' + old_ver+='*' + fi + if [[ ${new_op} = '=*' ]]; then + new_op='=' + new_ver+='*' + fi + if [[ ${old_op} != "${new_op}" || ${old_ver} != "${new_ver}" ]]; then + if [[ -z ${new_op} && -z ${new_ver} ]]; then + diff_report_append local_pds_dr "dropped version constraint" + elif [[ -z ${old_op} && -z ${old_ver} ]]; then + diff_report_append local_pds_dr "added version constraint ${new_op}${new_ver}" + else + diff_report_append local_pds_dr "changed version constraint from ${old_op}${old_ver} to ${new_op}${new_ver}" + fi + fi + + local old_slot=${old_pds_ref[PDS_SLOT_IDX]} new_slot=${new_pds_ref[PDS_SLOT_IDX]} + if [[ ${old_slot} != "${new_slot}" ]]; then + if [[ -z ${new_slot} ]]; then + diff_report_append local_pds_dr "dropped slot constraint" + elif [[ -z ${old_slot} ]]; then + diff_report_append local_pds_dr "added slot constraint ${new_slot}" + else + diff_report_append local_pds_dr "changed slot constraint from ${old_slot} to ${new_slot}" + fi + fi + + local -n old_urs_ref=${old_pds_ref[PDS_UR_IDX]} new_urs_ref=${new_pds_ref[PDS_UR_IDX]} + local -A old_name_index=() new_name_index=() + + local ur_name use_name + local -i idx=0 + for ur_name in "${old_urs_ref[@]}"; do + local -n ur_ref=${ur_name} + use_name=${ur_ref[UR_NAME_IDX]} + old_name_index["${use_name}"]=${idx} + ((++idx)) + unset -n ur_ref + done + + idx=0 + for ur_name in "${new_urs_ref[@]}"; do + local -n ur_ref=${ur_name} + use_name=${ur_ref[UR_NAME_IDX]} + new_name_index["${use_name}"]=${idx} + ((++idx)) + unset -n ur_ref + done + + local -A only_old_urs=() only_new_urs=() common_urs=() + sets_split old_name_index new_name_index only_old_urs only_new_urs common_urs + for use_name in "${!only_old_urs[@]}"; do + diff_report_append local_pds_dr "dropped ${use_name} use requirement" + done + local pd_mode_str pd_pretend_str + for use_name in "${!only_new_urs[@]}"; do + idx=${new_name_index["${use_name}"]} + local -n ur_ref=${new_urs_ref[${idx}]} + __mcdl_ur_mode_description "${ur_ref[UR_MODE_IDX]}" pd_mode_str + __mcdl_ur_pretend_description "${ur_ref[UR_PRETEND_IDX]}" pd_pretend_str + diff_report_append local_pds_dr "added ${use_name} use ${pd_mode_str} requirement (${pd_pretend_str})" + unset -n ur_ref + done + + local -i old_idx new_idx + local old_mode new_mode old_pretend new_pretend pd_old_mode_str pd_new_mode_str pd_old_pretend_str pd_new_pretend_str + for use_name in "${!common_urs[@]}"; do + old_idx=${old_name_index["${use_name}"]} + new_idx=${new_name_index["${use_name}"]} + + local -n old_ur_ref=${old_urs_ref[${old_idx}]} new_ur_ref=${new_urs_ref[${new_idx}]} + old_mode=${old_ur_ref[UR_MODE_IDX]} + new_mode=${new_ur_ref[UR_MODE_IDX]} + if [[ ${old_mode} != "${new_mode}" ]]; then + __mcdl_ur_mode_description "${old_mode}" pd_old_mode_str + __mcdl_ur_mode_description "${new_mode}" pd_new_mode_str + diff_report_append local_pds_dr "mode of use requirement on ${use_name} changed from ${pd_old_mode_str} to ${pd_new_mode_str}" + fi + + old_pretend=${old_ur_ref[UR_PRETEND_IDX]} + new_pretend=${new_ur_ref[UR_PRETEND_IDX]} + if [[ ${old_pretend} != "${new_pretend}" ]]; then + __mcdl_ur_pretend_description "${old_pretend}" pd_old_pretend_str + __mcdl_ur_pretend_description "${new_pretend}" pd_new_pretend_str + diff_report_append local_pds_dr "pretend mode of use requirement on ${use_name} changed from ${pd_old_pretend_str@Q} to ${pd_new_pretend_str@Q}" + fi + unset -n new_ur_ref old_ur_ref + done + + local local_pds_dr_empty + diff_report_is_empty local_pds_dr local_pds_dr_empty + if [[ -z ${local_pds_dr_empty} ]]; then + local iuse_str='' + if [[ -n ${old_iuses} ]]; then + iuse_str=" with USE conditionals ${old_iuses}" + fi + + local block_str='' + local -i new_blocks=${new_pds_ref[PDS_BLOCKS_IDX]} + case ${new_blocks} in + "${PDS_NO_BLOCK}") + block_str='' + ;; + "${PDS_WEAK_BLOCK}") + block_str=' weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + block_str=' strong blocker' + ;; + esac + + diff_report_append "${dr_var_name}" "changes for ${name}${block_str}${iuse_str}:" + diff_report_indent "${dr_var_name}" + diff_report_append_diff_report "${dr_var_name}" local_pds_dr + diff_report_dedent "${dr_var_name}" + fi + diff_report_unset local_pds_dr +} + +function __mcdl_merge_and_sort_tagged_subgroups() { + local -n empty_subgroup_item_names_ref=${1}; shift + local -n subgroup_tag_to_named_subgroup_item_names_map_ref=${1}; shift + local -n any_of_subgroup_item_names_ref=${1}; shift + local skip_subgroup_merging=${1}; shift + local subgroup_tag=${1}; shift + # skip the mvc name, the group names will be the rest of the + # args + shift + # rest are subgroup item names + + if [[ ${#} -eq 0 ]]; then return 0; fi + + local subgroup_item_name subgroup_name other_subgroup_item_name other_subgroup_name + if [[ ${subgroup_tag} = '@any-of@' ]]; then + for subgroup_item_name; do + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + __mcdl_sort_group "${subgroup_name}" + any_of_subgroup_item_names_ref+=( "${subgroup_item_name}" ) + done + elif [[ -n ${skip_subgroup_merging} ]]; then + for subgroup_item_name; do + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + __mcdl_sort_group "${subgroup_name}" + if [[ ${subgroup_tag} = '@empty@' ]]; then + empty_subgroup_item_names_ref+=( "${subgroup_item_name}" ) + else + subgroup_tag_to_named_subgroup_item_names_map_ref["${subgroup_tag}"]=${subgroup_item_name} + fi + done + else + subgroup_item_name=${1}; shift + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + local other_subgroup_items_name + for other_subgroup_item_name; do + local -n item_ref=${other_subgroup_item_name} + other_subgroup_name=${item_ref:2} + unset -n item_ref + local -n other_subgroup_ref=${other_subgroup_name} + other_subgroup_items_name=${other_subgroup_ref[GROUP_ITEMS_IDX]} + local -n other_subgroup_items_ref=${other_subgroup_items_name} + group_add_items "${subgroup_name}" "${other_subgroup_items_ref[@]}" + unset -n other_subgroup_items_ref + other_subgroup_ref[GROUP_ITEMS_IDX]='EMPTY_ARRAY' + unset -n other_subgroup_ref + unset "${other_subgroup_items_name}" + item_unset "${other_subgroup_item_name}" + done + __mcdl_sort_group "${subgroup_name}" + if [[ ${subgroup_tag} = '@empty@' ]]; then + empty_subgroup_item_names_ref+=( "${subgroup_item_name}" ) + else + subgroup_tag_to_named_subgroup_item_names_map_ref["${subgroup_tag}"]=${subgroup_item_name} + fi + fi +} + +function __mcdl_sort_group() { + local group_name=${1}; shift + + local -n group_ref=${group_name} + local -a subgroup_item_names license_item_names pds_item_names=() + local -n items_ref=${group_ref[GROUP_ITEMS_IDX]} + local is_any_of_group= + + subgroup_item_names=() + license_item_names=() + pds_item_names=() + + if [[ ${#items_ref[@]} -eq 0 ]]; then + return 0 + fi + + local -i group_type=${group_ref[GROUP_TYPE_IDX]} + if [[ group_type -eq GROUP_ANY_OF ]]; then + is_any_of_group=x + fi + unset group_type + + local item_name item_type + for item_name in "${items_ref[@]}"; do + local -n item=${item_name} + item_type=${item:0:1} + unset -n item + case ${item_type} in + e) + # these won't show up in sorted group + : + ;; + g) + subgroup_item_names+=( "${item_name}" ) + ;; + l) + license_item_names+=( "${item_name}" ) + ;; + p) + pds_item_names+=( "${item_name}" ) + ;; + esac + done + unset item_name item_type + items_ref=() + unset -n items_ref group_ref + + ## + ## add sorted pds into group + ## + + if [[ ${#pds_item_names[@]} -gt 0 ]]; then + # the package name may appear more than one time in the same + # group, this happens for ranged deps (e.g >=2.0, <3.0) + local pds_item_name + local pds_name pkg_name counted_pkg_name + local -i pkg_name_count + local -A pkg_name_to_count_map=() + local -a pkg_names=() + local -A pkg_name_to_item_name_map=() + for pds_item_name in "${pds_item_names[@]}"; do + local -n item_ref=${pds_item_name} + pds_name=${item_ref:2} + unset -n item_ref + local -n pds_ref=${pds_name} + pkg_name=${pds_ref[PDS_NAME_IDX]} + unset -n pds_ref + pkg_name_count=${pkg_name_to_count_map["${pkg_name}"]:-0} + counted_pkg_name="${pkg_name}^${pkg_name_count}" + ((++pkg_name_count)) + pkg_name_to_count_map["${pkg_name}"]=${pkg_name_count} + pkg_names+=( "${counted_pkg_name}" ) + pkg_name_to_item_name_map["${counted_pkg_name}"]=${pds_item_name} + done + local -a sorted_pkg_names sorted_items + mapfile -t sorted_pkg_names < <(printf '%s\n' "${pkg_names[@]}" | sort -t '^' -k 1,1 -k2n,2) + + for pkg_name in "${sorted_pkg_names[@]}"; do + pds_item_name=${pkg_name_to_item_name_map["${pkg_name}"]} + sorted_items+=( "${pds_item_name}" ) + done + group_add_items "${group_name}" "${sorted_items[@]}" + + unset sorted_items sorted_pkg_names pkg_name_to_item_name_map pkg_names pkg_name_to_count_map pkg_name_count pkg_name_count counted_pkg_name pkg_name pds_name pds_item_name + fi + + ## + ## add sorted licenses into group + ## + + if [[ ${#license_item_names[@]} -gt 0 ]]; then + local -A license_to_item_name_map=() + + local license_item_name license + for license_item_name in "${license_item_names[@]}"; do + local -n item_ref=${license_item_name} + license=${item_ref:2} + unset -n item_ref + license_to_item_name_map["${license}"]=${license_item_name} + done + local -a sorted_licenses sorted_items + mapfile -t sorted_licenses < <(printf '%s\n' "${!license_to_item_name_map[@]}" | sort) + for license in "${sorted_licenses[@]}"; do + license_item_name=${license_to_item_name_map["${license}"]} + sorted_items+=( "${license_item_name}" ) + done + group_add_items "${group_name}" "${sorted_items[@]}" + + unset sorted_items sorted_licenses license_item_name license license_to_item_name_map + fi + + ## + ## add sorted subgroups into group + ## + + if [[ ${#subgroup_item_names[@]} -gt 0 ]]; then + # - collect group names based on their type, use name and enabled use + # - unnamed all-of go first (tag: empty) + # - named all-of go next (tag: use name + enabled use) + # - any-of go each separately (tag: any-of) + # - merge all-of groups with the same tag + # - exception - unnamed all-of groups inside any-of group should not be merged + # - sort each group and add to main group + local subgroup_item_name subgroup_name subgroup_type subgroup_use subgroup_tag subgroup_use_mode + local dsg_tagged_subgroups_item_names_array_mvm_name + gen_varname dsg_tagged_subgroups_item_names_array_mvm_name + mvm_declare "${dsg_tagged_subgroups_item_names_array_mvm_name}" + for subgroup_item_name in "${subgroup_item_names[@]}"; do + local -n item_ref=${subgroup_item_name} + subgroup_name=${item_ref:2} + unset -n item_ref + local -n subgroup_ref=${subgroup_name} + subgroup_type=${subgroup_ref[GROUP_TYPE_IDX]} + case ${subgroup_type} in + "${GROUP_ALL_OF}") + subgroup_use=${subgroup_ref[GROUP_USE_IDX]} + if [[ -z ${subgroup_use} ]]; then + subgroup_tag='@empty@' + else + subgroup_use_mode=${subgroup_ref[GROUP_ENABLED_IDX]} + case ${subgroup_use_mode} in + "${GROUP_USE_ENABLED}") + subgroup_tag='+' + ;; + "${GROUP_USE_DISABLED}") + subgroup_tag='-' + ;; + esac + subgroup_tag+=${subgroup_use} + fi + ;; + "${GROUP_ANY_OF}") + subgroup_tag="@any-of@" + ;; + esac + unset -n subgroup_ref + mvm_add "${dsg_tagged_subgroups_item_names_array_mvm_name}" "${subgroup_tag}" "${subgroup_item_name}" + done + unset subgroup_use_mode subgroup_tag subgroup_use subgroup_type subgroup_name subgroup_item_name + + local -A subgroup_tag_to_named_subgroup_item_names_map=() + local -a any_of_subgroup_item_names=() empty_subgroup_item_names=() + mvm_iterate "${dsg_tagged_subgroups_item_names_array_mvm_name}" __mcdl_merge_and_sort_tagged_subgroups \ + empty_subgroup_item_names \ + subgroup_tag_to_named_subgroup_item_names_map \ + any_of_subgroup_item_names \ + "${is_any_of_group}" + mvm_unset "${dsg_tagged_subgroups_item_names_array_mvm_name}" + + group_add_items "${group_name}" "${empty_subgroup_item_names[@]}" + if [[ ${#subgroup_tag_to_named_subgroup_item_names_map[@]} -gt 0 ]]; then + local -a sorted_subgroup_tags sorted_items + local subgroup_tag subgroup_item_name + mapfile -t sorted_subgroup_tags < <(printf '%s\n' "${!subgroup_tag_to_named_subgroup_item_names_map[@]}" | sort) + for subgroup_tag in "${sorted_subgroup_tags[@]}"; do + if [[ ${subgroup_tag:0:1} = '@' && ${subgroup_tag: -1:1} = '@' ]]; then + continue + fi + subgroup_item_name=${subgroup_tag_to_named_subgroup_item_names_map["${subgroup_tag}"]} + sorted_items+=( "${subgroup_item_name}" ) + done + group_add_items "${group_name}" "${sorted_items[@]}" + unset sorted_items subgroup_item_name subgroup_tag sorted_subgroup_tags + fi + group_add_items "${group_name}" "${any_of_subgroup_item_names[@]}" + fi +} + +function __mcdl_iuse_stack_to_string_ps() { + local stack_name=${1}; shift + local -n str_ref=${1}; shift + local prefix=${1}; shift + local suffix=${1}; shift + + local tmp_str + __mcdl_iuse_stack_to_string "${stack_name}" tmp_str + + if [[ -n ${tmp_str} ]]; then + str_ref="${prefix}${tmp_str}${suffix}" + else + str_ref='' + fi +} + +function __mcdl_debug_group() { + local group_name=${1}; shift + local label=${1}; shift + + if pkg_debug_enabled; then + local dg_group_str + group_to_string "${group_name}" dg_group_str + pkg_debug_print "${label}: ${dg_group_str}" + fi +} + +function __mcdl_debug_flattened_list() { + local list_name=${1}; shift + local label=${1}; shift + + if pkg_debug_enabled; then + local item item_type + local -n list_ref=${list_name} + pkg_debug_print "${label}:" + for item in "${list_ref[@]}"; do + item_type=${item:0:1} + case ${item_type} in + 'i'|'o'|'c'|'l') + pkg_debug_print " ${item}" + ;; + 'p') + local dfl_pds_str + pds_to_string "${item:2}" dfl_pds_str + pkg_debug_print " ${item} (${dfl_pds_str})" + unset dfl_pds_str + ;; + esac + done + unset -n list_ref + fi +} + +function __mcdl_debug_diff() { + local -n dd_old_list_ref=${1}; shift + local -n dd_new_list_ref=${1}; shift + local -n dd_common_list_ref=${1}; shift + + if ! pkg_debug_enabled; then + return 0 + fi + + pkg_debug_print "diff between old and new flattened lists" + local -i last_idx1=0 last_idx2=0 idx1 idx2 + local ci_name + for ci_name in "${dd_common_list_ref[@]}"; do + local -n ci_ref=${ci_name} + idx1=${ci_ref[LCS_IDX1_IDX]} + idx2=${ci_ref[LCS_IDX2_IDX]} + unset -n ci_ref + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_list_ref["${last_idx1}"]} + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l'|'i'|'o'|'c') + pkg_debug_print " -${v1}" + ;; + 'p') + local p_str + pds_to_string "${v1}" p_str + pkg_debug_print " -${p_str}" + unset p_str + ;; + *) + fail "item ${item1} is bad" + ;; + esac + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_list_ref["${last_idx2}"]} + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l'|'i'|'o'|'c') + pkg_debug_print " +${v2}" + ;; + 'p') + local p_str + pds_to_string "${v2}" p_str + pkg_debug_print " +${p_str}" + unset p_str + ;; + *) + fail "item ${item2} is bad" + ;; + esac + unset v2 t2 item2 + ((++last_idx2)) + done + + local item1=${dd_old_list_ref["${idx1}"]} item2=${dd_new_list_ref["${idx2}"]} + local t1=${item1:0:1} v1=${item1:2} t2=${item2:0:1} v2=${item2:2} + + case ${t1} in + 'l'|'i'|'o'|'c') + pkg_debug_print " ${v2}" + ;; + 'p') + local p1_str p2_str + pds_to_string "${v1}" p1_str + pds_to_string "${v2}" p2_str + if [[ "${p1_str}" != "${p2_str}" ]]; then + pkg_debug_print " ${p1_str} -> ${p2_str}" + else + pkg_debug_print " ${p1_str}" + fi + unset p2_str p1_str + ;; + *) + fail "item ${item1} or ${item2} is bad" + ;; + esac + + unset v2 t2 item2 v1 t1 item1 + ((++last_idx1)) + ((++last_idx2)) + done + + idx1=${#dd_old_list_ref[@]} + idx2=${#dd_new_list_ref[@]} + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_list_ref["${last_idx1}"]} + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l'|'i'|'o'|'c') + pkg_debug_print " -${v1}" + ;; + 'p') + local p_str + pds_to_string "${v1}" p_str + pkg_debug_print " -${p_str}" + unset p_str + ;; + *) + fail "item ${item1} is bad" + ;; + esac + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_list_ref["${last_idx2}"]} + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l'|'i'|'o'|'c') + pkg_debug_print " +${v2}" + ;; + 'p') + local p_str + pds_to_string "${v2}" p_str + pkg_debug_print " +${p_str}" + unset p_str + ;; + *) + fail "item ${item2} is bad" + ;; + esac + unset v2 t2 item2 + ((++last_idx2)) + done +} + +function __mcdl_debug_iuse_stack() { + local iuse_stack_name=${1}; shift + local label=${1}; shift + + if pkg_debug_enabled; then + local dis_str + __mcdl_iuse_stack_to_string "${iuse_stack_name}" dis_str + pkg_debug_print "${label}: ${dis_str}" + fi +} + +function __mcdl_diff_deps() { + local -n old_ref=${1}; shift + local -n new_ref=${1}; shift + local deps_idx=${1}; shift + local dr_var_name=${1}; shift + + local label + case ${deps_idx} in + "${PCF_BDEPEND_IDX}") + label='build dependencies' + ;; + "${PCF_DEPEND_IDX}") + label='dependencies' + ;; + "${PCF_IDEPEND_IDX}") + label='install dependencies' + ;; + "${PCF_PDEPEND_IDX}") + label='post dependencies' + ;; + "${PCF_RDEPEND_IDX}") + label='runtime dependencies' + ;; + "${PCF_LICENSE_IDX}") + label='licenses' + ;; + *) + fail "bad cache file index ${deps_idx@Q}" + ;; + esac + + local old_group_name=${old_ref[${deps_idx}]} new_group_name=${new_ref[${deps_idx}]} + local -a dd_old_flattened_list=() dd_new_flattened_list=() + + group_declare old_sorted_group new_sorted_group + + group_copy old_sorted_group "${old_group_name}" + __mcdl_debug_group old_sorted_group "copy of old ${label} group" + __mcdl_sort_group old_sorted_group + __mcdl_debug_group old_sorted_group "sorted old ${label} group" + + group_copy new_sorted_group "${new_group_name}" + __mcdl_debug_group new_sorted_group "copy of new ${label} group" + __mcdl_sort_group new_sorted_group + __mcdl_debug_group old_sorted_group "sorted new ${label} group" + + __mcdl_flatten_group old_sorted_group dd_old_flattened_list + __mcdl_flatten_group new_sorted_group dd_new_flattened_list + + __mcdl_debug_flattened_list dd_old_flattened_list "flattened old ${label} group" + __mcdl_debug_flattened_list dd_new_flattened_list "flattened new ${label} group" + + local -a dd_common_items=() + + lcs_run dd_old_flattened_list dd_new_flattened_list dd_common_items __mcdl_flattened_group_item_score + + diff_report_declare local_dr + + __mcdl_debug_diff dd_old_flattened_list dd_new_flattened_list dd_common_items + + local -a old_iuse_stack=() new_iuse_stack=() + # a stack of counters for naming unnamed groups like + # + # || ( ( a/b-1 c/d-1 ) ( a/b-2 c/d-2 ) ) + # + # The "( a/b-1 c/d-1 )" and "( a/b-2 c/d-2 )" groups are unnamed + # within the "||" group. + local old_group_unnamed_counter_stack=( 0 ) new_group_unnamed_counter_stack=( 0 ) + local -i last_idx1=0 last_idx2=0 idx1 idx2 + local ci_name + local prev_item1='e:' prev_item2='e:' + for ci_name in "${dd_common_items[@]}"; do + local -n ci_ref=${ci_name} + idx1=${ci_ref[LCS_IDX1_IDX]} + idx2=${ci_ref[LCS_IDX2_IDX]} + unset -n ci_ref + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_flattened_list["${last_idx1}"]} + pkg_debug "old item ${item1@Q}" + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "dropped license ${v1@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + pds_to_string "${v1}" p_str + local -n p_ref=${v1} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item1:0:1} = 'i' ]]; then + subgroup_name=${prev_item1:2} + else + local -i counter=${old_group_unnamed_counter_stack[-1]} + ((++counter)) + old_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + old_group_unnamed_counter_stack+=( 0 ) + old_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'old_iuse_stack[-1]' 'old_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after dropping last name" + ;; + *) + fail "item ${item1} is bad" + ;; + esac + prev_item1=${item1} + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_flattened_list["${last_idx2}"]} + pkg_debug "new item ${item2@Q}" + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "added license ${v2@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + pds_to_string "${v2}" p_str + local -n p_ref=${v2} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item2:0:1} = 'i' ]]; then + subgroup_name=${prev_item2:2} + else + local -i counter=${new_group_unnamed_counter_stack[-1]} + ((++counter)) + new_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + new_group_unnamed_counter_stack+=( 0 ) + new_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'new_iuse_stack[-1]' 'new_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping last name" + ;; + *) + fail "item ${item2} is bad" + ;; + esac + prev_item2=${item2} + unset v2 t2 item2 + ((++last_idx2)) + done + + local item1=${dd_old_flattened_list["${idx1}"]} item2=${dd_new_flattened_list["${idx2}"]} + pkg_debug "old item ${item1@Q} and new item ${item2@Q}" + local t1=${item1:0:1} v1=${item1:2} t2=${item2:0:1} v2=${item2:2} + + case ${t1} in + 'l') + # not interesting + : + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item1:0:1} = 'i' ]]; then + subgroup_name=${prev_item1:2} + else + local -i counter=${old_group_unnamed_counter_stack[-1]} + ((++counter)) + old_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + old_group_unnamed_counter_stack+=( 0 ) + old_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after adding common ${subgroup_name@Q}" + + if [[ ${prev_item2:0:1} = 'i' ]]; then + subgroup_name=${prev_item2:2} + else + local -i counter=${new_group_unnamed_counter_stack[-1]} + ((++counter)) + new_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + new_group_unnamed_counter_stack+=( 0 ) + new_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after adding common ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'old_iuse_stack[-1]' 'old_group_unnamed_counter_stack[-1]' 'new_iuse_stack[-1]' 'new_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after dropping common last name" + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping common last name" + ;; + 'p') + __mcdl_pds_diff "${item1:2}" "${item2:2}" old_iuse_stack new_iuse_stack local_dr + ;; + *) + fail "item ${item1} or ${item2} is bad" + ;; + esac + + prev_item1=${item1} + prev_item2=${item2} + unset v2 t2 item2 v1 t1 item1 + ((++last_idx1)) + ((++last_idx2)) + done + unset "${dd_common_items[@]}" + + idx1=${#dd_old_flattened_list[@]} + idx2=${#dd_new_flattened_list[@]} + while [[ ${last_idx1} -lt ${idx1} ]]; do + local item1=${dd_old_flattened_list["${last_idx1}"]} + pkg_debug "old item ${item1@Q}" + local t1=${item1:0:1} v1=${item1:2} + case ${t1} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "dropped license ${v1@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + pds_to_string "${v1}" p_str + local -n p_ref=${v1} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item1:0:1} = 'i' ]]; then + subgroup_name=${prev_item1:2} + else + local -i counter=${old_group_unnamed_counter_stack[-1]} + ((++counter)) + old_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + old_group_unnamed_counter_stack+=( 0 ) + old_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'old_iuse_stack[-1]' 'old_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack old_iuse_stack "old iuse stack after dropping last name" + ;; + *) + fail "item ${item1} is bad" + ;; + esac + prev_item1=${item1} + unset v1 t1 item1 + ((++last_idx1)) + done + while [[ ${last_idx2} -lt ${idx2} ]]; do + local item2=${dd_new_flattened_list["${last_idx2}"]} + pkg_debug "new item ${item2@Q}" + local t2=${item2:0:1} v2=${item2:2} + case ${t2} in + 'l') + local use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "added license ${v2@Q}${use_str}" + unset use_str + ;; + 'p') + local p_str use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + pds_to_string "${v2}" p_str + local -n p_ref=${v2} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + ;; + 'i') + # This will be stored in prev item and used in 'o' + # item handling. + : + ;; + 'o') + local subgroup_name + if [[ ${prev_item2:0:1} = 'i' ]]; then + subgroup_name=${prev_item2:2} + else + local -i counter=${new_group_unnamed_counter_stack[-1]} + ((++counter)) + new_group_unnamed_counter_stack[-1]=${counter} + subgroup_name="unnamed-all-of-${counter}" + fi + new_group_unnamed_counter_stack+=( 0 ) + new_iuse_stack+=( "${subgroup_name}" ) + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after adding ${subgroup_name@Q}" + unset subgroup_name + ;; + 'c') + unset 'new_iuse_stack[-1]' 'new_group_unnamed_counter_stack[-1]' + __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping last name" + ;; + *) + fail "item ${item2} is bad" + ;; + esac + prev_item2=${item2} + unset v2 t2 item2 + ((++last_idx2)) + done + local local_dr_empty + diff_report_is_empty local_dr local_dr_empty + if [[ -z ${local_dr_empty} ]]; then + diff_report_append "${dr_var_name}" "${label}:" + diff_report_indent "${dr_var_name}" + diff_report_append_diff_report "${dr_var_name}" local_dr + diff_report_dedent "${dr_var_name}" + fi + diff_report_unset local_dr + + group_unset new_sorted_group old_sorted_group +} + +fi From 13e84333b8ff8a3fe8f3191d0c49b4f7d94e278e Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 11 Apr 2025 17:07:12 +0200 Subject: [PATCH 13/18] pkg-auto: Add md5-metadata cache diff reports to package reports --- pkg_auto/impl/pkg_auto_lib.sh | 139 ++++++++++++++++++++++++++-------- 1 file changed, 107 insertions(+), 32 deletions(-) diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 4342e59254..1f7b33c7bd 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -53,6 +53,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/util.sh" source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" source "${PKG_AUTO_IMPL_DIR}/debug.sh" source "${PKG_AUTO_IMPL_DIR}/gentoo_ver.sh" +source "${PKG_AUTO_IMPL_DIR}/md5_cache_diff_lib.sh" # Sets up the workdir using the passed config. The config can be # created basing on the config_template file or using the @@ -2185,9 +2186,9 @@ function handle_pkg_update() { local pkg_name pkg_name=${new_pkg#*/} local -a lines - lines=( "from ${old} to ${new}") + lines=( "0:from ${old} to ${new}") if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( "renamed from ${old_pkg}" ) + lines+=( "0:renamed from ${old_pkg}" ) fi # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" @@ -2196,15 +2197,28 @@ function handle_pkg_update() { local hpu_update_dir hpu_update_dir_non_slot update_dir_non_slot "${new_pkg}" hpu_update_dir_non_slot update_dir "${new_pkg}" "${old_s}" "${new_s}" hpu_update_dir + + local diff_report_name + gen_varname diff_report_name + diff_report_declare "${diff_report_name}" + generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" + + local -n diff_report_ref=${diff_report_name} + local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} + lines+=( "${diff_lines_ref[@]}" ) + unset -n diff_lines_ref + unset -n diff_report_ref + diff_report_unset "${diff_report_name}" + if [[ -s "${hpu_update_dir}/ebuild.diff" ]]; then - lines+=( 'TODO: review ebuild.diff' ) + lines+=( '0:TODO: review ebuild.diff' ) fi if [[ -s "${hpu_update_dir_non_slot}/other.diff" ]]; then - lines+=( 'TODO: review other.diff' ) + lines+=( '0:TODO: review other.diff' ) fi - lines+=( 'TODO: review occurences' ) + lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( 'TODO: review occurences-for-old-name' ) + lines+=( '0:TODO: review occurences-for-old-name' ) fi local -a hpu_tags @@ -2213,7 +2227,7 @@ function handle_pkg_update() { if ver_test "${new_no_r}" -gt "${old_no_r}"; then # version bump generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpu_tags[@]}" - lines+=( 'release notes: TODO' ) + lines+=( '0:release notes: TODO' ) fi generate_summary_stub "${new_pkg}" "${hpu_tags[@]}" -- "${lines[@]}" @@ -2253,12 +2267,12 @@ function handle_pkg_as_is() { local pkg_name pkg_name=${new_pkg#/} local -a lines - lines=( "still at ${v}" ) + lines=( "0:still at ${v}" ) local renamed renamed= if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( "renamed from ${old_pkg}" ) + lines+=( "0:renamed from ${old_pkg}" ) renamed=x fi generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${v}" "${v}" @@ -2267,12 +2281,28 @@ function handle_pkg_as_is() { update_dir "${new_pkg}" "${old_s}" "${new_s}" hpai_update_dir local modified modified= + + local diff_report_name + gen_varname diff_report_name + diff_report_declare "${diff_report_name}" + generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${v}" "${v}" + + local -n diff_report_ref=${diff_report_name} + local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} + if [[ ${#diff_lines_ref[@]} -gt 0 ]]; then + lines+=( "${diff_lines_ref[@]}" ) + modified=x + fi + unset -n diff_lines_ref + unset -n diff_report_ref + diff_report_unset "${diff_report_name}" + if [[ -s "${hpai_update_dir}/ebuild.diff" ]]; then - lines+=( 'TODO: review ebuild.diff' ) + lines+=( '0:TODO: review ebuild.diff' ) modified=x fi if [[ -s "${hpai_update_dir_non_slot}/other.diff" ]]; then - lines+=( 'TODO: review other.diff' ) + lines+=( '0:TODO: review other.diff' ) modified=x fi if [[ -z ${renamed} ]] && [[ -z ${modified} ]]; then @@ -2281,9 +2311,9 @@ function handle_pkg_as_is() { fi # shellcheck disable=SC2034 # ref to an external variable changed_ref=x - lines+=( 'TODO: review occurences' ) + lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( 'TODO: review occurences-for-old-name' ) + lines+=( '0:TODO: review occurences-for-old-name' ) fi local -a hpai_tags @@ -2324,24 +2354,37 @@ function handle_pkg_downgrade() { local pkg_name pkg_name=${new_pkg#*/} local -a lines - lines=( "downgraded from ${old} to ${new}" ) + lines=( "0:downgraded from ${old} to ${new}" ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( "renamed from ${old_pkg}" ) + lines+=( "0:renamed from ${old_pkg}" ) fi generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" local hpd_update_dir hpd_update_dir_non_slot update_dir_non_slot "${new_pkg}" hpd_update_dir_non_slot update_dir "${new_pkg}" "${old_s}" "${new_s}" hpd_update_dir + + local diff_report_name + gen_varname diff_report_name + diff_report_declare "${diff_report_name}" + generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" + + local -n diff_report_ref=${diff_report_name} + local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} + lines+=( "${diff_lines_ref[@]}" ) + unset -n diff_lines_ref + unset -n diff_report_ref + diff_report_unset "${diff_report_name}" + if [[ -s "${hpd_update_dir}/ebuild.diff" ]]; then - lines+=( 'TODO: review ebuild.diff' ) + lines+=( '0:TODO: review ebuild.diff' ) fi if [[ -s "${hpd_update_dir_non_slot}/other.diff" ]]; then - lines+=( 'TODO: review other.diff' ) + lines+=( '0:TODO: review other.diff' ) fi - lines+=( 'TODO: review occurences' ) + lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then - lines+=( 'TODO: review occurences-for-old-name' ) + lines+=( '0:TODO: review occurences-for-old-name' ) fi local -a hpd_tags @@ -2350,7 +2393,7 @@ function handle_pkg_downgrade() { if ver_test "${new_no_r}" -lt "${old_no_r}"; then # version bump generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpd_tags[@]}" - lines+=( "release notes: TODO" ) + lines+=( "0:release notes: TODO" ) fi generate_summary_stub "${new_pkg}" "${hpd_tags[@]}" -- "${lines[@]}" @@ -2456,10 +2499,16 @@ function generate_summary_stub() { printf ' [%s]' "${tags[@]}" fi printf '\n' - if [[ ${#} -gt 0 ]]; then - printf ' - %s\n' "${@}" - printf '\n' - fi + local indent_line indent line + for indent_line; do + indent=${indent_line%%:*} + line=${indent_line#*:} + if [[ ${indent} -gt 0 ]]; then + printf -- ' %.0s' $(seq 1 "${indent}") + fi + printf ' - %s\n' "${line}" + done + printf '\n' } >>"${REPORTS_DIR}/updates/summary_stubs" } @@ -2565,6 +2614,32 @@ function generate_ebuild_diff() { xdiff --unified=3 "${old_path}" "${new_path}" >"${ged_update_dir}/ebuild.diff" } +function generate_cache_diff_report() { + local diff_report_var_name=${1}; shift + local old_cache_dir=${1}; shift + local new_cache_dir=${1}; shift + local old_pkg=${1}; shift + local new_pkg=${1}; shift + local old=${1}; shift + local new=${1}; shift + + source "${WORKDIR}/globals" + + local old_entry=${old_cache_dir}/${old_pkg}-${old} + local new_entry=${new_cache_dir}/${new_pkg}-${new} + + local old_cache_name new_cache_name + gen_varname old_cache_name + gen_varname new_cache_name + cache_file_declare "${old_cache_name}" "${new_cache_name}" + parse_cache_file "${old_cache_name}" "${old_entry}" "${ARCHES[@]}" + parse_cache_file "${new_cache_name}" "${new_entry}" "${ARCHES[@]}" + + diff_cache_data "${old_cache_name}" "${new_cache_name}" "${diff_report_var_name}" + + cache_file_unset "${old_cache_name}" "${new_cache_name}" +} + # Generate a report with information where the old and new packages # are mentioned in entire scripts repository. May result in two # separate reports if the package got renamed. @@ -2936,11 +3011,11 @@ function handle_eclass() { if [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]] && [[ -e "${NEW_PORTAGE_STABLE}/${eclass}" ]]; then mkdir -p "${REPORTS_DIR}/updates/${eclass}" xdiff --unified=3 "${OLD_PORTAGE_STABLE}/${eclass}" "${NEW_PORTAGE_STABLE}/${eclass}" >"${REPORTS_DIR}/updates/${eclass}/eclass.diff" - lines+=( 'TODO: review the diff' ) + lines+=( '0:TODO: review the diff' ) elif [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]]; then - lines+=( 'unused, dropped' ) + lines+=( '0:unused, dropped' ) else - lines+=( 'added from Gentoo' ) + lines+=( '0:added from Gentoo' ) fi generate_summary_stub "${eclass}" -- "${lines[@]}" } @@ -3014,7 +3089,7 @@ function handle_profiles() { done <"${out_dir}/full.diff" lines_to_file_truncate "${out_dir}/relevant.diff" "${relevant_lines[@]}" lines_to_file_truncate "${out_dir}/possibly-irrelevant-files" "${possibly_irrelevant_files[@]}" - generate_summary_stub profiles -- 'TODO: review the diffs' + generate_summary_stub profiles -- '0:TODO: review the diffs' } # Handles changes in license directory. Generates brief reports and @@ -3083,15 +3158,15 @@ function handle_licenses() { local joined if [[ ${#dropped[@]} -gt 0 ]]; then join_by joined ', ' "${dropped[@]}" - lines+=( "dropped ${joined}" ) + lines+=( "0:dropped ${joined}" ) fi if [[ ${#added[@]} -gt 0 ]]; then join_by joined ', ' "${added[@]}" - lines+=( "added ${joined}" ) + lines+=( "0:added ${joined}" ) fi if [[ ${#changed[@]} -gt 0 ]]; then join_by joined ', ' "${changed[@]}" - lines+=( "updated ${joined}" ) + lines+=( "0:updated ${joined}" ) fi generate_summary_stub licenses -- "${lines[@]}" } @@ -3106,7 +3181,7 @@ function handle_scripts() { mkdir -p "${out_dir}" xdiff --unified=3 --recursive "${OLD_PORTAGE_STABLE}/scripts" "${NEW_PORTAGE_STABLE}/scripts" >"${out_dir}/scripts.diff" - generate_summary_stub scripts -- 'TODO: review the diffs' + generate_summary_stub scripts -- '0:TODO: review the diffs' } fi From 296efcdb22a2cc82139794cc2d15c79c5bab40ff Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 11 Apr 2025 14:56:21 +0200 Subject: [PATCH 14/18] pkg-auto: Add a setup for shellcheck's "source" checking That way shellcheck sources some prepared files and learns about some variables the sourced files define. Thanks to that, we can remove some of the "shellcheck disable" clauses. --- pkg_auto/Makefile | 4 +- pkg_auto/download_sdk_and_listings.sh | 4 +- pkg_auto/impl/for-shellcheck/README.md | 2 + pkg_auto/impl/for-shellcheck/globals | 68 +++++++++++++++++++ pkg_auto/impl/for-shellcheck/version.txt | 4 ++ pkg_auto/impl/pkg_auto_lib.sh | 84 ++++++++++-------------- 6 files changed, 112 insertions(+), 54 deletions(-) create mode 100644 pkg_auto/impl/for-shellcheck/README.md create mode 100644 pkg_auto/impl/for-shellcheck/globals create mode 100644 pkg_auto/impl/for-shellcheck/version.txt diff --git a/pkg_auto/Makefile b/pkg_auto/Makefile index b2795d456a..f71092b3c7 100644 --- a/pkg_auto/Makefile +++ b/pkg_auto/Makefile @@ -1,2 +1,4 @@ +all: shellcheck + shellcheck: - docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh + docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --source-path=SCRIPTDIR/impl/for-shellcheck --external-sources --check-sourced *.sh impl/*.sh diff --git a/pkg_auto/download_sdk_and_listings.sh b/pkg_auto/download_sdk_and_listings.sh index 11b06f79ef..8cfafb7e6d 100755 --- a/pkg_auto/download_sdk_and_listings.sh +++ b/pkg_auto/download_sdk_and_listings.sh @@ -148,9 +148,9 @@ function download() { } if [[ -n ${SCRIPTS} ]]; then - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") fi diff --git a/pkg_auto/impl/for-shellcheck/README.md b/pkg_auto/impl/for-shellcheck/README.md new file mode 100644 index 0000000000..223cd8c9cf --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/README.md @@ -0,0 +1,2 @@ +Shellcheck is being pointed here when processing source bash +directives for generated files. diff --git a/pkg_auto/impl/for-shellcheck/globals b/pkg_auto/impl/for-shellcheck/globals new file mode 100644 index 0000000000..f2060a0238 --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/globals @@ -0,0 +1,68 @@ +local -a GIT_ENV_VARS ARCHES WHICH REPORTS +local SDK_PKGS BOARD_PKGS +local SYNC_SCRIPT PKG_LIST_SORT_SCRIPT + +GIT_ENV_VARS=( + GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} +) + +SYNC_SCRIPT='/path/to/sync_with_gentoo.sh' +PKG_LIST_SORT_SCRIPT='/path/to/sort_packages_list.py' + +ARCHES=( 'amd64' 'arm64' ) +WHICH=('old' 'new') +SDK_PKGS='sdk-pkgs' +BOARD_PKGS='board-pkgs' +REPORTS=( "${SDK_PKGS}" "${BOARD_PKGS}" ) + +local SCRIPTS OLD_STATE NEW_STATE OLD_STATE_BRANCH NEW_STATE_BRANCH +local PORTAGE_STABLE_SUFFIX OLD_PORTAGE_STABLE NEW_PORTAGE_STABLE REPORTS_DIR +local NEW_STATE_PACKAGES_LIST AUX_DIR +local COREOS_OVERLAY_SUFFIX OLD_COREOS_OVERLAY NEW_COREOS_OVERLAY + +SCRIPTS='/path/to/scripts' +OLD_STATE='/path/to/old_state' +NEW_STATE='/path/to/new_state' +OLD_STATE_BRANCH='old-state' +NEW_STATE_BRANCH='new-state' +PORTAGE_STABLE_SUFFIX='sdk_container/src/third_party/portage-stable' +OLD_PORTAGE_STABLE='/path/to/old_state/portage-stable' +NEW_PORTAGE_STABLE='/path/to/new_state/portage-stable' +REPORTS_DIR='/path/to/reports' + +COREOS_OVERLAY_SUFFIX='sdk_container/src/third_party/coreos-overlay' +OLD_COREOS_OVERLAY='/path/to/old_state/coreos-overlay' +NEW_COREOS_OVERLAY='/path/to/new_state/coreos-overlay' + +NEW_STATE_PACKAGES_LIST="${NEW_STATE}/.github/workflows/portage-stable-packages-list" + +AUX_DIR='/path/to/aux' + +local 'SDK_IMAGE' + +SDK_IMAGE='ghcr.io/flatcar/flatcar-sdk-all:1234.5.6-build-tag' + +local -A LISTING_KINDS + +LISTING_KINDS=( + ['akamai']='oem-akamai_packages.txt' + ['ami']='oem-ami_packages.txt' + ['azure']='oem-azure_packages.txt' + ['containerd']='containerd-flatcar_packages.txt' + ['dev']='flatcar_developer_container_packages.txt' + ['digitalocean']='oem-digitalocean_packages.txt' + ['docker']='docker-flatcar_packages.txt' + ['gce']='oem-gce_packages.txt' + ['hetzner']='oem-hetzner_packages.txt' + ['kubevirt']='oem-kubevirt_packages.txt' + ['openstack']='oem-openstack_packages.txt' + ['packet']='oem-packet_packages.txt' + ['prod']='flatcar_production_image_packages.txt' + ['proxmoxve']='oem-proxmoxve_packages.txt' + ['qemu']='oem-qemu_packages.txt' + ['scaleway']='oem-scaleway_packages.txt' + ['sysext-podman']='flatcar-podman_packages.txt' + ['sysext-python']='flatcar-python_packages.txt' + ['sysext-zfs']='flatcar-zfs_packages.txt' + ['vmware']='oem-vmware_packages.txt' +) diff --git a/pkg_auto/impl/for-shellcheck/version.txt b/pkg_auto/impl/for-shellcheck/version.txt new file mode 100644 index 0000000000..2804576264 --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/version.txt @@ -0,0 +1,4 @@ +FLATCAR_VERSION=1234.5.6+build-tag +FLATCAR_VERSION_ID=1234.5.6 +FLATCAR_BUILD_ID="build-tag" +FLATCAR_SDK_VERSION=1234.5.6+build-tag diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 1f7b33c7bd..7181c188d0 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -175,10 +175,9 @@ function save_new_state() { local branch_name branch_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" info "saving new state to branch ${branch_name}" - # shellcheck disable=SC2153 # SCRIPTS is not a misspelling, it comes from globals file git -C "${SCRIPTS}" branch --force "${branch_name}" "${NEW_STATE_BRANCH}" } @@ -282,7 +281,7 @@ function process_profile_updates_directory() { local -a ppud_ordered_names get_ordered_update_filenames ppud_ordered_names - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local bf ps_f co_f pkg f line old new @@ -292,9 +291,7 @@ function process_profile_updates_directory() { for bf in "${ppud_ordered_names[@]}"; do # coreos-overlay updates may overwrite updates from # portage-stable, but only from the file of the same name - # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file ps_f=${NEW_PORTAGE_STABLE}/profiles/updates/${bf} - # shellcheck disable=SC2153 # NEW_COREOS_OVERLAY is not a misspelling, it comes from globals file co_f=${NEW_COREOS_OVERLAY}/profiles/updates/${bf} for f in "${ps_f}" "${co_f}"; do if [[ ! -f ${f} ]]; then @@ -335,7 +332,7 @@ function process_profile_updates_directory() { function get_ordered_update_filenames() { local ordered_names_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A names_set=() @@ -523,13 +520,13 @@ NEW_STATE_PACKAGES_LIST="\${NEW_STATE}/.github/workflows/portage-stable-packages AUX_DIR=${aux_dir@Q} EOF - # shellcheck disable=SC1090 # generated file + # shellcheck source=for-shellcheck/globals source "${globals_file}" local last_nightly_version_id last_nightly_build_id - # shellcheck disable=SC1091,SC2153 # sourcing generated file, NEW_STATE is not misspelled + # shellcheck source=for-shellcheck/version.txt last_nightly_version_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") local -a locals=() definitions=() @@ -548,7 +545,6 @@ EOF local packages_file tag filename stripped old for arch in "${ARCHES[@]}"; do - # shellcheck disable=SC2153 # AUX_DIR is not a misspelling, it comes from globals file for packages_file in "${AUX_DIR}/${arch}/"*_packages.txt; do filename=${packages_file##*/} stripped=${filename%_packages.txt} @@ -640,7 +636,7 @@ function run_sync() { missing_in_scripts=() missing_in_gentoo=() - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -x "${GIT_ENV_VARS[@]}" @@ -651,7 +647,6 @@ function run_sync() { local package while read -r package; do - # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file if [[ ! -e "${NEW_PORTAGE_STABLE}/${package}" ]]; then # If this happens, it means that the package was moved to overlay # or dropped, the list ought to be updated. @@ -670,7 +665,6 @@ function run_sync() { fi packages_to_update+=( "${package}" ) done < <(cat_meaningful "${NEW_STATE_PACKAGES_LIST}") - # shellcheck disable=SC2153 # SYNC_SCRIPT is not a misspelling env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${packages_to_update[@]}" save_missing_in_scripts "${missing_in_scripts[@]}" @@ -808,7 +802,7 @@ function handle_missing_in_scripts() { hmis_missing_in_scripts=() load_missing_in_scripts hmis_missing_in_scripts - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" if [[ ${#hmis_missing_in_scripts[@]} -eq 0 ]]; then @@ -823,7 +817,6 @@ function handle_missing_in_scripts() { join_by missing_re '\|' "${missing_in_scripts[@]}" add_cleanup "rm -f ${dir@Q}/pkg_list" xgrep --invert-match --line-regexp --fixed-strings --regexp="${missing_re}" "${NEW_STATE_PACKAGES_LIST}" >"${dir}/pkg_list" - # shellcheck disable=SC2153 # PKG_LIST_SORT_SCRIPT is not a misspelling "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" local -x "${GIT_ENV_VARS[@]}" @@ -862,11 +855,10 @@ function lines_to_file() { # # @ - lines to add function manual() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'manual work needed:' "${@}" - # shellcheck disable=SC2153 # REPORTS_DIR is not a misspelling, it comes from globals file lines_to_file "${REPORTS_DIR}/manual-work-needed" "${@}" } @@ -877,7 +869,7 @@ function manual() { # # @ - lines to add function pkg_warn() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'pkg warn:' "${@}" @@ -891,7 +883,7 @@ function pkg_warn() { # # @ - lines to add function devel_warn() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'developer warn:' "${@}" @@ -913,7 +905,7 @@ function handle_missing_in_gentoo() { return 0; fi - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A hmig_rename_map=() @@ -984,7 +976,7 @@ function process_listings() { local pkg_to_tags_mvm_var_name pkg_to_tags_mvm_var_name=${1} - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" #mvm_debug_enable pl_pkg_to_tags_set_mvm @@ -992,7 +984,6 @@ function process_listings() { local arch kind file listing pkg for arch in "${ARCHES[@]}"; do - # shellcheck disable=SC2153 # LISTING_KINDS is not a misspelling, it comes from globals file for kind in "${!LISTING_KINDS[@]}"; do file=${LISTING_KINDS["${kind}"]} listing="${AUX_DIR}/${arch}/${file}" @@ -1068,7 +1059,7 @@ function set_mvm_to_array_mvm_cb() { # stored in salvaged-reports subdirectory of the reports directory. # Otherwise they will end up in reports-from-sdk subdirectory. function generate_sdk_reports() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" @@ -1083,7 +1074,6 @@ function generate_sdk_reports() { local sdk_reports_dir top_dir dir entry full_path local -a dir_queue all_dirs all_files - # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file for sdk_run_kind in "${WHICH[@]}"; do state_var_name="${sdk_run_kind^^}_STATE" sdk_run_state="${!state_var_name}_sdk_run" @@ -1289,7 +1279,7 @@ function pkginfo_c_process_file() { local -n pkg_set_ref=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local which arch report @@ -1366,7 +1356,7 @@ function read_reports() { all_pkgs_var_name=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A rr_all_packages_set @@ -1387,7 +1377,7 @@ function read_reports() { # Destroys the pkginfo maps for all the reports. function unset_report_mvms() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local arch which report @@ -1571,7 +1561,7 @@ function consistency_checks() { pkg_slots_set_mvm_var_name=${1}; shift pkg_slot_verminmax_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local cc_pimap_mvm_1_var_name cc_pimap_mvm_2_var_name pkg @@ -1706,7 +1696,7 @@ function consistency_checks() { function read_package_sources() { local -n package_sources_map_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a files=() @@ -1753,7 +1743,7 @@ function handle_package_changes() { local -n renamed_old_to_new_map_ref=${1}; shift pkg_to_tags_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a hpc_all_pkgs @@ -1997,7 +1987,6 @@ function handle_package_changes() { update_dir_non_slot "${new_name}" hpc_update_dir_non_slot mkdir -p "${hpc_update_dir_non_slot}" - # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE comes from globals file generate_non_ebuild_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" generate_full_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" generate_package_mention_reports "${NEW_STATE}" "${old_name}" "${new_name}" @@ -2176,7 +2165,7 @@ function handle_pkg_update() { old=${1}; shift new=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_no_r new_no_r @@ -2190,7 +2179,6 @@ function handle_pkg_update() { if [[ ${old_pkg} != "${new_pkg}" ]]; then lines+=( "0:renamed from ${old_pkg}" ) fi - # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" # shellcheck disable=SC2034 # these variables are used by name @@ -2258,7 +2246,7 @@ function handle_pkg_as_is() { v=${1}; shift local -n changed_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local hpai_update_dir @@ -2344,7 +2332,7 @@ function handle_pkg_downgrade() { old=${1}; shift new=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_no_r new_no_r @@ -2461,7 +2449,7 @@ function generate_changelog_entry_stub() { gces_tags='SDK' fi - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" printf '%s %s: %s ([%s](TODO))\n' '-' "${gces_tags}" "${pkg_name}" "${v}" >>"${REPORTS_DIR}/updates/changelog_stubs" @@ -2478,7 +2466,7 @@ function generate_summary_stub() { pkg=${1}; shift # rest are tags separated followed by double dash followed by lines - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a tags @@ -2623,6 +2611,7 @@ function generate_cache_diff_report() { local old=${1}; shift local new=${1}; shift + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_entry=${old_cache_dir}/${old_pkg}-${old} @@ -2722,7 +2711,7 @@ function update_dir_non_slot() { pkg=${1}; shift local -n dir_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" # shellcheck disable=SC2034 # it's a reference to external variable @@ -2806,11 +2795,10 @@ function handle_gentoo_sync() { mvm_unset hgs_pkg_to_tags_mvm #mvm_debug_disable hgs_pkg_to_tags_mvm - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_head new_head - # shellcheck disable=SC2153 # OLD_STATE is not a misspelling old_head=$(git -C "${OLD_STATE}" rev-parse HEAD) new_head=$(git -C "${NEW_STATE}" rev-parse HEAD) @@ -2819,7 +2807,6 @@ function handle_gentoo_sync() { local path in_ps category if [[ "${old_head}" != "${new_head}" ]]; then while read -r path; do - # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then continue fi @@ -3003,7 +2990,7 @@ function handle_eclass() { local eclass eclass=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a lines @@ -3025,7 +3012,7 @@ function handle_eclass() { # and SDK), a full diff between all the profiles, and a list of # possibly irrelevant files that has changed too. function handle_profiles() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a files=() @@ -3095,14 +3082,10 @@ function handle_profiles() { # Handles changes in license directory. Generates brief reports and # diffs about dropped, added or modified licenses. function handle_licenses() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - local -a dropped added changed - dropped=() - added=() - changed=() - + local -a dropped=() added=() changed=() local line hl_stripped # Lines are: # @@ -3113,7 +3096,6 @@ function handle_licenses() { # Files /licenses/BSL-1.1 and /licenses/BSL-1.1 differ while read -r line; do if [[ ${line} = 'Only in '* ]]; then - # shellcheck disable=SC2153 # OLD_STATE is not a misspelling, it comes from globals file strip_out "${line##*:}" hl_stripped if [[ ${line} = *"${OLD_STATE}"* ]]; then dropped+=( "${hl_stripped}" ) @@ -3173,7 +3155,7 @@ function handle_licenses() { # Generates reports about changes inside the scripts directory. function handle_scripts() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local out_dir From 0d06b737acd0cd2794c526a8c22a64047d82771d Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 11 Apr 2025 14:58:54 +0200 Subject: [PATCH 15/18] pkg-auto: Disable shellcheck reference warnings Two warnings, SC2034 and SC2178, pop up very often with the references - shellcheck handles them poorly and produces a ton of bogus warnings about them. Silence the warnings and drop most of the "shellcheck disable" clauses. --- pkg_auto/Makefile | 10 +++++++++- pkg_auto/download_sdk_and_listings.sh | 3 --- pkg_auto/generate_config.sh | 10 ---------- pkg_auto/impl/inside_sdk_container_lib.sh | 2 -- pkg_auto/impl/mvm.sh | 16 ---------------- pkg_auto/impl/pkg_auto_lib.sh | 21 --------------------- pkg_auto/impl/print_profile_tree.sh | 4 ---- pkg_auto/impl/util.sh | 15 --------------- 8 files changed, 9 insertions(+), 72 deletions(-) diff --git a/pkg_auto/Makefile b/pkg_auto/Makefile index f71092b3c7..ae9a4fa077 100644 --- a/pkg_auto/Makefile +++ b/pkg_auto/Makefile @@ -1,4 +1,12 @@ +# ignoring SC2034 (VARIABLE_NAME appears unused), too many false +# positives +# +# ignoring SC2178 (Variable was used as an array but is now assigned a +# string.) - buggy for local variables, happens very often for +# references +SHELLCHECK_OPTS := -e SC2034 -e SC2178 + all: shellcheck shellcheck: - docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --source-path=SCRIPTDIR/impl/for-shellcheck --external-sources --check-sourced *.sh impl/*.sh + docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --source-path=SCRIPTDIR/impl/for-shellcheck --external-sources --check-sourced $(SHELLCHECK_OPTS) *.sh impl/*.sh diff --git a/pkg_auto/download_sdk_and_listings.sh b/pkg_auto/download_sdk_and_listings.sh index 8cfafb7e6d..8e48039248 100755 --- a/pkg_auto/download_sdk_and_listings.sh +++ b/pkg_auto/download_sdk_and_listings.sh @@ -159,19 +159,16 @@ ver_dash="${VERSION_ID}${BUILD_ID:+-}${BUILD_ID}" exts=(zst bz2 gz) -# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds zst_cmds=( zstd ) -# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds bz2_cmds=( lbunzip2 pbunzip2 bunzip2 ) -# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds gz_cmds=( unpigz gunzip diff --git a/pkg_auto/generate_config.sh b/pkg_auto/generate_config.sh index e6c31abdcd..7d390197af 100755 --- a/pkg_auto/generate_config.sh +++ b/pkg_auto/generate_config.sh @@ -26,19 +26,12 @@ set -euo pipefail source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" -# shellcheck disable=SC2034 # used by name below gc_aux_directory='' -# shellcheck disable=SC2034 # used by name below gc_new_base='' -# shellcheck disable=SC2034 # used by name below gc_old_base='' -# shellcheck disable=SC2034 # used by name below gc_reports_directory='' -# shellcheck disable=SC2034 # used by name below gc_scripts_directory='' -# shellcheck disable=SC2034 # used by name below gc_cleanup_opts='' -# shellcheck disable=SC2034 # used by name below gc_image_override='' gc_debug_packages=() @@ -81,9 +74,7 @@ while [[ ${#} -gt 0 ]]; do if [[ -z ${2:-} ]]; then fail 'missing value for -w' fi - # shellcheck disable=SC2178 # shellcheck does not grok refs declare -n ref="${var_name}" - # shellcheck disable=SC2178 # shellcheck does not grok refs ref=${2} unset -n ref unset var_name @@ -123,7 +114,6 @@ config=${1}; shift name=${pairs["${name_idx}"]} opt_idx=$((opt_idx + 2)) name_idx=$((name_idx + 2)) - # shellcheck disable=SC2178 # shellcheck does not grok refs declare -n ref="${name}" if [[ -n ${ref:-} ]]; then printf '%s: %s\n' "${opt}" "${ref}" diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index 9acec5f9c2..acd2b25863 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -99,7 +99,6 @@ function set_eo() { local dir=${1}; shift # rest are architectures - # shellcheck disable=SC2034 # used externally declare -g EGENCACHE_W="${dir}/egencache-warnings" declare -g SDK_EO="${dir}/sdk-emerge-output" declare -g SDK_EO_F="${SDK_EO}-filtered" @@ -348,7 +347,6 @@ function get_provided_file() { path_var_name=${1}; shift local -n path_ref="${path_var_name}" - # shellcheck disable=SC2034 # reference to external variable path_ref="${root}/etc/portage/profile/package.provided/ignore_cross_packages" } diff --git a/pkg_auto/impl/mvm.sh b/pkg_auto/impl/mvm.sh index 74a8756493..1a56b6a22a 100644 --- a/pkg_auto/impl/mvm.sh +++ b/pkg_auto/impl/mvm.sh @@ -133,7 +133,6 @@ function mvm_declare() { storage_map_ref=() local -n mvm_ref=${mvm_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable mvm_ref=( ['name']="${mvm_var_name}" ['constructor']="${constructor}" @@ -193,7 +192,6 @@ function __mvm_mvc_name() { mvc_name_var_name=${1}; shift local -n mvc_name_ref=${mvc_name_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable mvc_name_ref="mvm_${name}_mvc_${counter}" } @@ -246,10 +244,8 @@ function mvm_c_get_extra() { local extras_map_var_name extras_map_var_name=${mvm['extras']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n extras_map_ref=${extras_map_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable extra_ref=${extras_map_ref["${extra}"]:-} } @@ -273,10 +269,8 @@ function mvm_c_get() { local storage_map_var_name storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable value_ref=${storage_map_ref["${key}"]:-} } @@ -290,7 +284,6 @@ function __mvm_c_make_new_mvc() { name=${mvm['name']} counter=${mvm['counter']} storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} __mvm_mvc_name "${name}" "${counter}" "${mvc_name_var_name}" @@ -348,7 +341,6 @@ function mvm_c_remove() { local storage_map_var_name storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} if [[ -z ${storage_map_ref["${key}"]:-} ]]; then @@ -387,7 +379,6 @@ function mvm_c_iterate() { local storage_map_var_name helper storage_map_var_name=${mvm['storage']} - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n storage_map_ref=${storage_map_var_name} helper=${mvm['iteration_helper']} @@ -467,7 +458,6 @@ function mvm_mvc_array_destructor() { function mvm_mvc_array_adder() { local array_var_name array_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n array_ref=${array_var_name} array_ref+=( "${@}" ) @@ -481,7 +471,6 @@ function mvm_mvc_array_iteration_helper() { callback=${1}; shift # rest are extra args passed to cb - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n array_ref=${array_var_name} "${callback}" "${@}" "${key}" "${array_var_name}" "${array_ref[@]}" } @@ -512,11 +501,9 @@ function mvm_mvc_map_destructor() { function mvm_mvc_map_adder() { local map_var_name map_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n map_ref=${map_var_name} while [[ ${#} -gt 1 ]]; do - # shellcheck disable=SC2034 # it's a reference to external variable map_ref["${1}"]=${2} shift 2 done @@ -532,7 +519,6 @@ function mvm_mvc_set_constructor() { declare -g -A "${set_var_name}" - # shellcheck disable=SC2178 # shellcheck does not grok refs local -n set_ref=${set_var_name} set_ref=() } @@ -548,7 +534,6 @@ function mvm_mvc_set_adder() { local set_var_name set_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n set_ref=${set_var_name} while [[ ${#} -gt 0 ]]; do set_ref["${1}"]=x @@ -565,7 +550,6 @@ function mvm_mvc_set_iteration_helper() { callback=${1}; shift # rest are extra args passed to cb - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n set_ref=${set_var_name} "${callback}" "${@}" "${key}" "${set_var_name}" "${!set_ref[@]}" } diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 7181c188d0..a2d7b8851a 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -606,15 +606,12 @@ EOF function setup_git_env() { local bot_name bot_email role what - # shellcheck disable=SC2034 # used indirectly bot_name='Flatcar Buildbot' - # shellcheck disable=SC2034 # used indirectly bot_email='buildbot@flatcar-linux.org' for role in AUTHOR COMMITTER; do for what in name email; do local -n var_ref="GIT_${role}_${what^^}" local -n value_ref="bot_${what}" - # shellcheck disable=SC2034 # it's a reference to external variable var_ref=${value_ref} unset -n value_ref unset -n var_ref @@ -1181,7 +1178,6 @@ function pkginfo_name() { report=${1}; shift local -n pi_name_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable pi_name_ref="pkginfo_${which}_${arch}_${report//-/_}_pimap_mvm" } @@ -1197,7 +1193,6 @@ function pkginfo_destructor() { # Adder callback used by mvm_declare for pkginfo mvms. function pkginfo_adder() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n map_ref=${1}; shift local mark @@ -1306,14 +1301,12 @@ function pkginfo_c_process_file() { esac local pkg version_slot throw_away v s - # shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else while read -r pkg version_slot throw_away; do pkg_debug_enable "${pkg}" pkg_debug "${which} ${arch} ${report}: ${version_slot}" v=${version_slot%%:*} s=${version_slot##*:} mvm_c_add "${pkg}" "${s}" "${v}" - # shellcheck disable=SC2034 # it's a reference to external variable pkg_set_ref["${pkg}"]='x' mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}" pkg_debug_disable @@ -1412,9 +1405,7 @@ function ver_min_max() { max=${v} fi done - # shellcheck disable=SC2034 # it's a reference to external variable min_ref=${min} - # shellcheck disable=SC2034 # it's a reference to external variable max_ref=${max} } @@ -1556,7 +1547,6 @@ function consistency_check_for_package() { function consistency_checks() { local which pkg_slots_set_mvm_var_name pkg_slot_verminmax_mvm_var_name which=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n all_pkgs_ref=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift pkg_slot_verminmax_mvm_var_name=${1}; shift @@ -1634,7 +1624,6 @@ function consistency_checks() { done local cc_slots_set_var_name s cc_min cc_max verminmax - # shellcheck disable=SC2034 # used by name below local -A empty_map=() local -a verminmax_map_var_names verminmaxes local cc_slot_verminmax_map_var_name @@ -1739,7 +1728,6 @@ function read_package_sources() { # 2 - name of the package tags map mvm variable function handle_package_changes() { local pkg_to_tags_mvm_var_name - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n renamed_old_to_new_map_ref=${1}; shift pkg_to_tags_mvm_var_name=${1}; shift @@ -1880,7 +1868,6 @@ function handle_package_changes() { local hpc_changed hpc_slot_changed hpc_update_dir_non_slot hpc_category_dir local which slots_set_var_name_var_name slot_verminmax_map_var_name_var_name filtered_slots_set_var_name verminmax local -A hpc_old_filtered_slots_set hpc_new_filtered_slots_set - # shellcheck disable=SC2034 # used by name below, in a special case empty_map_or_set=() while [[ ${pkg_idx} -lt ${#old_pkgs[@]} ]]; do old_name=${old_pkgs["${pkg_idx}"]} @@ -2129,7 +2116,6 @@ function handle_package_changes() { # 1 - name of the set variable # 2 - name of the variable where the element will be stored function get_first_from_set() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n set_ref=${1}; shift local -n return_ref=${1}; shift @@ -2138,7 +2124,6 @@ function get_first_from_set() { return_ref=${item} return 0 done - # shellcheck disable=SC2034 # it's a reference to external variable return_ref='' } @@ -2181,7 +2166,6 @@ function handle_pkg_update() { fi generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" - # shellcheck disable=SC2034 # these variables are used by name local hpu_update_dir hpu_update_dir_non_slot update_dir_non_slot "${new_pkg}" hpu_update_dir_non_slot update_dir "${new_pkg}" "${old_s}" "${new_s}" hpu_update_dir @@ -2297,7 +2281,6 @@ function handle_pkg_as_is() { # Nothing relevant has changed, return early. return 0 fi - # shellcheck disable=SC2034 # ref to an external variable changed_ref=x lines+=( '0:TODO: review occurences' ) if [[ ${old_pkg} != "${new_pkg}" ]]; then @@ -2410,7 +2393,6 @@ function tags_for_pkg() { tags_ref=() else local -n tags_in_mvm_ref=${tfp_tags_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable tags_ref=( "${tags_in_mvm_ref[@]}" ) pkg_debug "tags available: ${tags_in_mvm_ref[*]}" fi @@ -2714,7 +2696,6 @@ function update_dir_non_slot() { # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - # shellcheck disable=SC2034 # it's a reference to external variable dir_ref="${REPORTS_DIR}/updates/${pkg}" } @@ -2744,7 +2725,6 @@ function update_dir() { local ud_non_slot_dir update_dir_non_slot "${pkg}" ud_non_slot_dir - # shellcheck disable=SC2034 # it's a reference to external variable dir_ref="${ud_non_slot_dir}/${slot_dir}" } @@ -2786,7 +2766,6 @@ function handle_gentoo_sync() { mvm_declare hgs_pkg_to_tags_mvm process_listings hgs_pkg_to_tags_mvm - # shellcheck disable=SC2034 # passed to other function through a name local -A hgs_renames_old_to_new_map=() process_profile_updates_directory hgs_renames_old_to_new_map diff --git a/pkg_auto/impl/print_profile_tree.sh b/pkg_auto/impl/print_profile_tree.sh index 96ce744840..757adb7c9c 100755 --- a/pkg_auto/impl/print_profile_tree.sh +++ b/pkg_auto/impl/print_profile_tree.sh @@ -69,7 +69,6 @@ function get_repo_from_profile_path() { path=${1}; shift local -n repo_dir_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable repo_dir_ref="${path%/profiles/*}" } @@ -78,7 +77,6 @@ function repo_path_to_name() { path=${1}; shift local -n name_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable name_ref=${repo_data_r["${path}"]:-''} } @@ -138,7 +136,6 @@ function process_profile() { done <"${parent_file}" fi - # shellcheck disable=SC2034 # it's a reference to external variable children_ref=( "${children[@]}" ) } @@ -152,7 +149,6 @@ function get_profile_name() { repo_path=${repo_data["${repo_name}"]} profile_name=${profile_path#"${repo_path}/profiles/"} - # shellcheck disable=SC2034 # it's a reference to external variable profile_name_ref="${profile_name}" } diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh index b220060574..267b550d11 100644 --- a/pkg_auto/impl/util.sh +++ b/pkg_auto/impl/util.sh @@ -37,7 +37,6 @@ function dirname_out() { dir_ref='.' return 0 fi - # shellcheck disable=SC2034 # it's a reference to external variable dir_ref=${dn} } @@ -68,7 +67,6 @@ function basename_out() { cleaned_up=${cleaned_up//+(\/)/\/} # keep last component dn=${cleaned_up##*/} - # shellcheck disable=SC2034 # it's a reference to external variable base_ref=${dn} } @@ -86,7 +84,6 @@ THIS=$(realpath "${THIS}") THIS_DIR=$(realpath "${THIS_DIR}") dirname_out "${BASH_SOURCE[0]}" PKG_AUTO_IMPL_DIR PKG_AUTO_IMPL_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}") -# shellcheck disable=SC2034 # may be used by scripts sourcing this file PKG_AUTO_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}/..") # Prints an info line. @@ -165,7 +162,6 @@ function join_by() { printf -v "${output_var_name}" '%s' "${first}" "${@/#/${delimiter}}"; else local -n output_ref=${output_var_name} - # shellcheck disable=SC2034 # it's a reference to external variable output_ref='' fi } @@ -207,7 +203,6 @@ function strip_out() { t=${l} t=${t/#+([[:space:]])} t=${t/%+([[:space:]])} - # shellcheck disable=SC2034 # it's a reference to external variable out_ref=${t} } @@ -217,10 +212,8 @@ function strip_out() { # # 1 - name of an array variable, where the architectures will be stored function get_valid_arches() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n arches_ref=${1}; shift - # shellcheck disable=SC2034 # it's a reference to external variable arches_ref=( 'amd64' 'arm64' ) } @@ -235,7 +228,6 @@ function get_valid_arches() { # 2 - separator string # @ - strings function all_pairs() { - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays local -n pairs_ref=${1}; shift local sep=${1}; shift @@ -269,11 +261,8 @@ function all_pairs() { function sets_split() { local -n first_set_ref=${1}; shift local -n second_set_ref=${1}; shift - # shellcheck disable=SC2178 # shellcheck does not grok references local -n only_in_first_set_ref=${1}; shift - # shellcheck disable=SC2178 # shellcheck does not grok references local -n only_in_second_set_ref=${1}; shift - # shellcheck disable=SC2178 # shellcheck does not grok references local -n common_set_ref=${1}; shift only_in_first_set_ref=() @@ -285,10 +274,8 @@ function sets_split() { for item in "${!first_set_ref[@]}"; do mark=${second_set_ref["${item}"]:-} if [[ -z ${mark} ]]; then - # shellcheck disable=SC2034 # it's a reference to external variable only_in_first_set_ref["${item}"]=x else - # shellcheck disable=SC2034 # it's a reference to external variable common_set_ref["${item}"]=x fi done @@ -296,7 +283,6 @@ function sets_split() { for item in "${!second_set_ref[@]}"; do mark=${first_set_ref["${item}"]:-} if [[ -z ${mark} ]]; then - # shellcheck disable=SC2034 # it's a reference to external variable only_in_second_set_ref["${item}"]=x fi done @@ -319,7 +305,6 @@ function gen_varname() { fi local -n name_ref=${1}; shift - # shellcheck disable=SC2034 # shellcheck does not grok references name_ref="${prefix}_${__UTIL_SH_COUNTER}" __UTIL_SH_COUNTER=$((__UTIL_SH_COUNTER + 1)) } From ecd24bd5ebb9911424fbf85029fdce83bebc6ef3 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 10 Apr 2025 17:48:12 +0200 Subject: [PATCH 16/18] pkg-auto: Fix a shellcheck warning --- pkg_auto/impl/pkg_auto_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index a2d7b8851a..dbab438027 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -2742,7 +2742,7 @@ function grep_pkg() { pkg=${1}; shift # rest are directories - GIT_PAGER= git -C "${scripts}" grep "${pkg}"'\(-[0-9]\|[^a-zA-Z0-9_-]\|$\)' -- "${@}" || : + GIT_PAGER='' git -C "${scripts}" grep "${pkg}"'\(-[0-9]\|[^a-zA-Z0-9_-]\|$\)' -- "${@}" || : } # Prints the passed files preceding and following with BEGIN ENTRY and From a8922be627edb57a2ffdbc049be4582d6770adaf Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 29 Apr 2025 14:07:13 +0200 Subject: [PATCH 17/18] pkg-auto: Simplify error checking in emerge output Co-authored-by: James Le Cuirot --- pkg_auto/impl/inside_sdk_container_lib.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index acd2b25863..1798a2732e 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -325,12 +325,9 @@ function ensure_no_errors() { files+=( "${!name}" ) done - local file - for file in "${files[@]}"; do - if cat "${file}" | grep --quiet --fixed-strings 'ERROR'; then - fail "there are errors in emerge output warnings files" - fi - done + if grep --quiet --fixed-strings 'ERROR' "${files[@]}"; then + fail "there are errors in emerge output warnings files" + fi } # Stores a path to a package.provided file inside the given root From 1039ed0571a3ae0b312b59430c165fcfa61a6d14 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 29 Apr 2025 14:07:35 +0200 Subject: [PATCH 18/18] pkg-auto: Simplify the use of sed Co-authored-by: James Le Cuirot --- pkg_auto/impl/inside_sdk_container_lib.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index 1798a2732e..ec7480dd74 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -247,7 +247,7 @@ function packages_for_board() { local name=${arch^^}_BOARD_EO_F - cat "${!name}" | sed "${@}" | sort + sed "${@}" "${!name}" | sort } # Prints package name, slot and version information for SDK.