From 4c5e5500558455b8546a1458904b13fa2e583027 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 9 Jun 2023 10:15:28 +0200 Subject: [PATCH 1/6] pkg-auto: Add package automation scripts This adds some scripts I have been using for over a year to deal with the weekly package updates. It comes with a `README.md` which describes a workflow similar to my own. The `sync_packages.sh` and `update_packages.sh` scripts are currently not used anywhere. The idea behind them was to use them for Github Action, but that will come as a follow-up PR. --- pkg_auto/Makefile | 2 + pkg_auto/README.md | 59 + pkg_auto/config_template | 47 + pkg_auto/download_sdk_and_listings.sh | 265 ++ pkg_auto/generate_config.sh | 170 ++ pkg_auto/generate_reports.sh | 56 + pkg_auto/impl/cleanups.sh | 153 + pkg_auto/impl/inside_sdk_container_lib.sh | 485 +++ pkg_auto/impl/mvm.sh | 573 ++++ pkg_auto/impl/pkg_auto_lib.sh | 3305 +++++++++++++++++++++ pkg_auto/impl/print_profile_tree.sh | 274 ++ pkg_auto/impl/sort_packages_list.py | 206 ++ pkg_auto/impl/sync_with_gentoo.sh | 316 ++ pkg_auto/impl/util.sh | 253 ++ pkg_auto/inside_sdk_container.sh | 102 + pkg_auto/sync_packages.sh | 60 + pkg_auto/update_packages.sh | 60 + 17 files changed, 6386 insertions(+) create mode 100644 pkg_auto/Makefile create mode 100644 pkg_auto/README.md create mode 100644 pkg_auto/config_template create mode 100755 pkg_auto/download_sdk_and_listings.sh create mode 100755 pkg_auto/generate_config.sh create mode 100755 pkg_auto/generate_reports.sh create mode 100644 pkg_auto/impl/cleanups.sh create mode 100644 pkg_auto/impl/inside_sdk_container_lib.sh create mode 100644 pkg_auto/impl/mvm.sh create mode 100644 pkg_auto/impl/pkg_auto_lib.sh create mode 100755 pkg_auto/impl/print_profile_tree.sh create mode 100755 pkg_auto/impl/sort_packages_list.py create mode 100755 pkg_auto/impl/sync_with_gentoo.sh create mode 100644 pkg_auto/impl/util.sh create mode 100755 pkg_auto/inside_sdk_container.sh create mode 100755 pkg_auto/sync_packages.sh create mode 100755 pkg_auto/update_packages.sh diff --git a/pkg_auto/Makefile b/pkg_auto/Makefile new file mode 100644 index 0000000000..b2795d456a --- /dev/null +++ b/pkg_auto/Makefile @@ -0,0 +1,2 @@ +shellcheck: + docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh diff --git a/pkg_auto/README.md b/pkg_auto/README.md new file mode 100644 index 0000000000..d7c80500d7 --- /dev/null +++ b/pkg_auto/README.md @@ -0,0 +1,59 @@ +Scripts for package update automation +===================================== + +A quick start from blank state: + +- clone scripts repo and create worktrees for the old state and new state (and optionally for the branch with package automation scripts if they are not a part of the `main` branch yet): + - `git clone https://github.com/flatcar/scripts.git scripts/main` + - `cd scripts/main` + - `PKG_AUTO="${PWD}/pkg_auto"` + - `git worktree add --branch weekly-updates ../weekly-updates origin/buildbot/weekly-portage-stable-package-updates-2024-09-23` + - if package automation is still not merged into the `main` branch then: + - `git worktree add --branch pkg-auto ../pkg-auto origin/krnowak/pkg-auto` + - `PKG_AUTO="${PWD}/../pkg-auto/pkg_auto"` +- prepare for generating reports (create a directory, download necessary stuff, create config): + - `mkdir ../../weekly-updates` + - `cd ../../weekly-updates` + - `"${PKG_AUTO}/download_sdk_and_listings.sh" -s ../../scripts/main -x aux-cleanups aux` + - call `"${PKG_AUTO}/download_sdk_and_listings.sh" -h` to get help + - `"${PKG_AUTO}/generate_config.sh" -a aux -n weekly-updates -o main -r reports -s ../../scripts/main -x file,wd-cleanups config` + - call `"${PKG_AUTO}/generate_config.sh" -h` to get help +- generate the reports: + - `"${PKG_AUTO}/generate_reports.sh" -w wd config` + - if the command above fails, the `reports` directory (see the `-r reports` flag in the call to `generate_config.sh` above) will have some reports that may contain hints as to why the failure happened + - the `reports` directory may also contain files like `warnings` or `manual-work-needed` + - the items in `warnings` file should be addressed and the report generation should be rerun, see below + - the items in `manual-work-needed` are things to be done while processing the updates +- in order to rerun the report generation, stuff from previous run should be removed beforehand: + - `source wd-cleanups` + - `rm -rf reports` + - `"${PKG_AUTO}/generate_reports.sh" -w wd config` +- if generating reports succeeded, process the updates, update the PR with the changelogs and update summaries: + - this is the manual part, described below +- after everything is done (like the PR got merged), things needs cleaning up: + - `source wd-cleanups` + - `rm -rf reports` + - `source aux-cleanups` + +Processing the updates (the manual part) +======================================== + +The generated directory with reports will contain the `updates` directory. Within there are two files: `summary_stubs` and `changelog_stubs`. The rest of the entries are categories and packages that were updated. The first file, `summary_stubs`, contains a list of packages that have changed and TODO items associated to each package. It is mostly used for being pasted into the pull request description as an aid for the reviewers. The latter, `changelog_stubs`, can serve as a base for changelog that should be added to the `scripts` repo. + +For each package in the `summary_stubs` there are TODO items. These are basically of four kinds: + +- to review the changes in the ebuild +- to review the changes not in the ebuild (metadata, patch files) +- to review the occurences of the package name in the scripts repository +- to add a link to the release notes in case of a package update + +It is possible that none of the changes in the package are relevant to Flatcar (like when a package got stabilized for hppa, for instance), then the package should be just dropped from the `summary_stubs`. Note that the package update is relevant, so as such should stay in the file. + +The entries in `changelog_stubs` should be reviewed about relevancy (minor SDK-only packages should likely be dropped, they are seldom of interest to end-users) and the remaining entries should be updated with proper links to release notes. + +There may be also entries in `manual-work-needed` which may need addressing. Most often the reason is that the new package was added, or an existing package stopped being pulled in. This would need adding an entry to the `summary_stubs`. + +Other scripts +============= + +There are other scripts in this directory. `inside_sdk_container.sh` is a script executed by `generate_reports.sh` inside the SDK to collect the package information. `sync_packages.sh` is a script that updates packages and saves the result to a new branch. `update_packages.sh` is `sync_packages.sh` + `generate_reports.sh`. diff --git a/pkg_auto/config_template b/pkg_auto/config_template new file mode 100644 index 0000000000..57ef5ac0eb --- /dev/null +++ b/pkg_auto/config_template @@ -0,0 +1,47 @@ +# Path to the toplevel directory of the scripts repo. +scripts: .. + +# Path to the directory with auxiliary files. +aux: ../../downloads + +# Path to the directory where update reports will be stored. +reports: ../../reports + +# Base scripts branch for old state. +# +# Old state serves as an state before the updates. +# +# Optional, defaults to origin/main +#old-base: origin/main + +# Base scripts branch for new state. +# +# New state serves as an state that's either after the updates or to-be-updated. +# +# Optional, defaults to old-base value +#new-base: origin/main + +# Cleanups mode with semicolon-separated list of extra arguments. +# +# Can be: +# +# file,${path_to_file} +# stores cleanup steps in the file. Can be later sourced to perform cleanups. +# +# trap +# executes cleanups on script exit +# +# ignore +# performs no cleanups at all +# +# Optional, if not specified, defaults to ignore. +#cleanups: file,../../x-cleanups +#cleanups: trap +#cleanups: ignore + +# Override image name to use for an SDK container. +# +# Optional, defaults to +# ghcr.io/flatcar/flatcar-sdk-all:${last_nightly_version_id_in_base}-${last_nightly_build_id_in_base} +#amd64-sdk-img= +#arm64-sdk-img= diff --git a/pkg_auto/download_sdk_and_listings.sh b/pkg_auto/download_sdk_and_listings.sh new file mode 100755 index 0000000000..11b06f79ef --- /dev/null +++ b/pkg_auto/download_sdk_and_listings.sh @@ -0,0 +1,265 @@ +#!/bin/bash + +## +## Downloads package SDKs from bincache and loads them with +## docker. Downloads package listings from bincache. Version can be +## taken either from the latest nightly tag in the passed scripts +## directory (with the -s option) or from specified version ID and +## build ID (with -v and -b options). The results are written to the +## passed downloads directory. +## +## Parameters: +## -b : build ID, conflicts with -s +## -h: this help +## -s : scripts repo directory, conflicts with -v and -b +## -v : version ID, conflicts with -s +## -nd: skip downloading of docker images +## -p: download packages SDK images instead of the standard one (valid +## only when downloading docker images) +## -nl: skip downloading of listings +## -x : cleanup file +## +## Positional: +## 1: downloads directory +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" + +CLEANUP_FILE= +SCRIPTS= +VERSION_ID= +BUILD_ID= +SKIP_DOCKER= +SKIP_LISTINGS= +PKGS_SDK= + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -b) + if [[ -n ${SCRIPTS} ]]; then + fail '-b cannot be used at the same time with -s' + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -b' + fi + BUILD_ID=${2} + shift 2 + ;; + -h) + print_help + exit 0 + ;; + -p) + PKGS_SDK=x + shift + ;; + -s) + if [[ -n ${VERSION_ID} ]] || [[ -n ${BUILD_ID} ]]; then + fail '-s cannot be used at the same time with -v or -b' + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -s' + fi + SCRIPTS=${2} + shift 2 + ;; + -v) + if [[ -n ${SCRIPTS} ]]; then + fail '-v cannot be used at the same time with -s' + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -v' + fi + VERSION_ID=${2} + shift 2 + ;; + -x) + if [[ -z ${2:-} ]]; then + fail 'missing value for -x' + fi + CLEANUP_FILE=${2} + shift 2 + ;; + -nd) + SKIP_DOCKER=x + shift + ;; + -nl) + SKIP_LISTINGS=x + shift + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 1 ]]; then + fail 'Expected one positional parameter: a downloads directory' +fi + +DOWNLOADS_DIR=$(realpath "${1}"); shift + +if [[ -z ${SCRIPTS} ]] && [[ -z ${VERSION_ID} ]]; then + fail 'need to pass either -s or -v (latter with the optional -b too)' +fi + +if [[ -n ${CLEANUP_FILE} ]]; then + dirname_out "${CLEANUP_FILE}" cleanup_dir + # shellcheck disable=SC2154 # cleanup_dir is assigned in dirname_out + mkdir -p "${cleanup_dir}" + unset cleanup_dir + setup_cleanups file "${CLEANUP_FILE}" +else + setup_cleanups ignore +fi + +if [[ ! -d "${DOWNLOADS_DIR}" ]]; then + add_cleanup "rmdir ${DOWNLOADS_DIR@Q}" + mkdir "${DOWNLOADS_DIR}" +fi + +function download() { + local url output + url="${1}"; shift + output="${1}"; shift + + info "Downloading ${url}" + curl \ + --fail \ + --show-error \ + --location \ + --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + "${url}" >"${output}" +} + +if [[ -n ${SCRIPTS} ]]; then + # shellcheck disable=SC1091 # sourcing generated file + VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") + # shellcheck disable=SC1091 # sourcing generated file + BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") +fi + +ver_plus="${VERSION_ID}${BUILD_ID:++}${BUILD_ID}" +ver_dash="${VERSION_ID}${BUILD_ID:+-}${BUILD_ID}" + +exts=(zst bz2 gz) + +# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds +zst_cmds=( + zstd +) + +# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds +bz2_cmds=( + lbunzip2 + pbunzip2 + bunzip2 +) + +# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds +gz_cmds=( + unpigz + gunzip +) + +function download_sdk() { + local image_name=${1}; shift + local tarball_name=${1}; shift + local url_dir=${1}; shift + + if docker images --format '{{.Repository}}:{{.Tag}}' | grep -q -x -F "${image_name}"; then + return 0 + fi + + info "No ${image_name} available in docker, pulling it from bincache" + local ext full_tarball_name tb + for ext in "${exts[@]}"; do + full_tarball_name="${tarball_name}.tar.${ext}" + tb="${DOWNLOADS_DIR}/${full_tarball_name}" + if [[ -s ${tb} ]]; then + break; + else + add_cleanup "rm -f ${tb@Q}" + if download "${url_dir}/${full_tarball_name}" "${tb}"; then + break + fi + fi + done + info "Loading ${image_name} into docker" + cmds_name="${ext}_cmds" + if ! declare -p "${cmds_name}" >/dev/null 2>/dev/null; then + fail "Failed to extract ${tb@Q} - no tools to extract ${ext@Q} files" + fi + declare -n cmds="${ext}_cmds" + loaded= + for cmd in "${cmds[@]}"; do + if ! command -v "${cmd}" >/dev/null; then + info "${cmd@Q} is not available" + continue + fi + info "Using ${cmd@Q} to extract the tarball" + "${cmd}" -d -c "${tb}" | docker load + add_cleanup "docker rmi ${image_name@Q}" + loaded=x + break + done + if [[ -z ${loaded} ]]; then + fail "Failed to extract ${tb@Q} - no known available tool to extract it" + fi + unset -n cmds +} + +URL_DIR="https://bincache.flatcar-linux.net/containers/${ver_dash}" + +if [[ -z ${SKIP_DOCKER} ]] && [[ -z ${PKGS_SDK} ]]; then + download_sdk "ghcr.io/flatcar/flatcar-sdk-all:${ver_dash}" "flatcar-sdk-all-${ver_dash}" "${URL_DIR}" +fi + +declare -a dsal_arches +get_valid_arches dsal_arches + +for arch in "${dsal_arches[@]}"; do + if [[ -z ${SKIP_DOCKER} ]] && [[ -n ${PKGS_SDK} ]]; then + download_sdk "flatcar-packages-${arch}:${ver_dash}" "flatcar-packages-${arch}-${ver_dash}.tar.${ext}" "${URL_DIR}" + fi + + if [[ -z ${SKIP_LISTINGS} ]]; then + listing_dir="${DOWNLOADS_DIR}/${arch}" + add_cleanup "rmdir ${listing_dir@Q}" + mkdir "${listing_dir}" + base_url="https://bincache.flatcar-linux.net/images/${arch}/${ver_plus}" + + for infix in '' 'rootfs-included-sysexts'; do + index_html="${listing_dir}/${infix}${infix:+-}index.html" + url="${base_url}${infix:+/}${infix}" + add_cleanup "rm -f ${index_html@Q}" + download "${url}/" "${index_html}" + + # get names of all files ending with _packages.txt + mapfile -t listing_files < <(grep -F '_packages.txt"' "${index_html}" | sed -e 's#.*"\(\./\)\?\([^"]*\)".*#\2#') + + for listing in "${listing_files[@]}"; do + info "Downloading ${listing} for ${arch}" + listing_path="${listing_dir}/${listing}" + add_cleanup "rm -f ${listing_path@Q}" + download "${url}/${listing}" "${listing_path}" + done + done + fi +done +info 'Done' diff --git a/pkg_auto/generate_config.sh b/pkg_auto/generate_config.sh new file mode 100755 index 0000000000..ce354be9c3 --- /dev/null +++ b/pkg_auto/generate_config.sh @@ -0,0 +1,170 @@ +#!/bin/bash + +set -euo pipefail + +## +## Generate a config. +## +## Parameters: +## -a: aux directory +## -d: debug package - list many times +## -h: this help +## -i: SDK image override in form of ${arch}:${name}, the name part +## should be a valid docker image with an optional tag +## -ip: add SDK image overrides using flatcar-packages images +## -n: new base +## -o: old base +## -r: reports directory +## -s: scripts directory +## -x: cleanup opts +## +## Positional: +## 1: path for config file +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" + +# shellcheck disable=SC2034 # used by name below +gc_aux_directory='' +# shellcheck disable=SC2034 # used by name below +gc_new_base='' +# shellcheck disable=SC2034 # used by name below +gc_old_base='' +# shellcheck disable=SC2034 # used by name below +gc_reports_directory='' +# shellcheck disable=SC2034 # used by name below +gc_scripts_directory='' +# shellcheck disable=SC2034 # used by name below +gc_cleanup_opts='' +# gc_${arch}_sdk_img are declared on demand +gc_debug_packages=() + +declare -A opt_map +opt_map=( + ['-a']=gc_aux_directory + ['-n']=gc_new_base + ['-o']=gc_old_base + ['-r']=gc_reports_directory + ['-s']=gc_scripts_directory + ['-x']=gc_cleanup_opts +) + +declare -a gc_arches +get_valid_arches gc_arches + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -d) + if [[ -z ${2:-} ]]; then + fail 'missing value for -d' + fi + gc_debug_packages+=( "${2}" ) + shift 2 + ;; + -h) + print_help + exit 0 + ;; + -i) + if [[ -z ${2:-} ]]; then + fail 'missing value for -i' + fi + arch=${2%%:*} + image_name=${2#*:} + var_name="gc_${arch}_sdk_img" + unset arch + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${var_name}" + unset var_name + # shellcheck disable=SC2178 # shellcheck does not grok refs + ref=${image_name} + unset image_name + unset -n ref + shift 2 + ;; + -ip) + for arch in "${gc_arches[@]}"; do + var_name="gc_${arch}_sdk_img" + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${var_name}" + unset var_name + # shellcheck disable=SC2178 # shellcheck does not grok refs + ref="flatcar-packages-${arch}" + unset -n ref + done + unset arch + shift + ;; + --) + shift + break + ;; + -*) + var_name=${opt_map["${1}"]:-} + if [[ -z ${var_name} ]]; then + fail "unknown flag '${1}'" + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${var_name}" + # shellcheck disable=SC2178 # shellcheck does not grok refs + ref=${2} + unset -n ref + unset var_name + shift 2 + ;; + *) + break + ;; + esac +done + +join_by gc_debug_packages_csv ',' "${gc_debug_packages[@]}" + +declare -a pairs +pairs=( + 'scripts' gc_scripts_directory + 'aux' gc_aux_directory + 'reports' gc_reports_directory + 'old-base' gc_old_base + 'new-base' gc_new_base + 'cleanups' gc_cleanup_opts +) + +for arch in "${gc_arches[@]}"; do + pairs+=( "${arch}-sdk-img" "gc_${arch}_sdk_img" ) +done + +pairs+=( 'debug-packages' gc_debug_packages_csv ) + + +if [[ ${#} -ne 1 ]]; then + fail 'expected one positional parameters: a path for the config' +fi + +config=${1}; shift + +{ + opt_idx=0 + name_idx=1 + while [[ ${name_idx} -lt "${#pairs[@]}" ]]; do + opt=${pairs["${opt_idx}"]} + name=${pairs["${name_idx}"]} + opt_idx=$((opt_idx + 2)) + name_idx=$((name_idx + 2)) + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${name}" + if [[ -n ${ref:-} ]]; then + printf '%s: %s\n' "${opt}" "${ref}" + fi + unset -n ref + done + unset opt_idx name_idx +} >"${config}" + +info 'Done, but note that the config is not guaranteed to be valid!' diff --git a/pkg_auto/generate_reports.sh b/pkg_auto/generate_reports.sh new file mode 100755 index 0000000000..0b6df1ab4f --- /dev/null +++ b/pkg_auto/generate_reports.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + +## +## Generates reports. +## +## Parameters: +## -w: path to use for workdir +## -h: this help +## +## Positional: +## 1: config file +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/pkg_auto_lib.sh" + +workdir='' + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -w) + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + workdir=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 1 ]]; then + fail 'expected one positional parameter: a config file' +fi + +config_file=${1}; shift + +setup_workdir_with_config "${workdir}" "${config_file}" +generate_package_update_reports diff --git a/pkg_auto/impl/cleanups.sh b/pkg_auto/impl/cleanups.sh new file mode 100644 index 0000000000..66d6d050e2 --- /dev/null +++ b/pkg_auto/impl/cleanups.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# +# Cleanups +# +# This is basically a command stack to be executed in some deferred +# time. So last command added will be the first to be executed at +# cleanup time. +# +# Cleanups are implemented through two functions, setup_cleanups and +# add_cleanup, prefixed with _${type}_. So for type "foo" the +# functions would be _foo_setup_cleanups and _foo_add_cleanup. +# +# setup_cleanup may take some extra parameters that are specific to +# the type. For example file type takes a path where the commands will +# be stored. +# +# add_cleanup takes one or more command to add to the cleanup stack. +# + +if [[ -z ${__CLEANUPS_SH_INCLUDED__:-} ]]; then +__CLEANUPS_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Sets up cleanup stack of a given type. A type may need some extra +# parameter, which comes after a comma. Possible types are: +# +# - file: requires extra argument about cleanup file location; an +# example could be "file,/path/to/cleanups-file" +# - trap: executed on shell exit +# - ignore: noop +# +# Params: +# +# 1 - type of cleanup +function setup_cleanups() { + local kind + kind=${1}; shift + + if [[ -n ${_cleanups_sh_cleanup_kind_:-} ]]; then + fail "cannot set cleanups to '${kind}', they are already set up to '${_cleanups_sh_cleanup_kind_}'" + fi + + declare -g _cleanups_sh_cleanup_kind_ + + _ensure_valid_cleanups_sh_cleanup_kind_ "${kind}" + _cleanups_sh_cleanup_kind_=${kind} + _call_cleanup_func setup_cleanups "${@}" +} + +# Adds commands to the cleanup stack. +# +# Params: +# +# @ - commands, one per parameter +function add_cleanup() { + _call_cleanup_func add_cleanup "${@}" +} + +# +# Implementation details. +# + +# "file" cleanups + +function _file_setup_cleanups() { + if [[ ${#} -ne 1 ]]; then + fail 'missing cleanup file location argument for file cleanups' + fi + + declare -g _file_cleanup_file + _file_cleanup_file=$(realpath "${1}"); shift + add_cleanup "rm -f ${_file_cleanup_file@Q}" +} + +function _file_add_cleanup() { + local fac_cleanup_dir tmpfile + dirname_out "${_file_cleanup_file}" fac_cleanup_dir + + tmpfile=$(mktemp -p "${fac_cleanup_dir}") + printf '%s\n' "${@}" >"${tmpfile}" + if [[ -f "${_file_cleanup_file}" ]]; then + cat "${_file_cleanup_file}" >>"${tmpfile}" + fi + mv -f "${tmpfile}" "${_file_cleanup_file}" +} + +# "trap" cleanups + +function _trap_update_trap() { + # shellcheck disable=SC2064 # using double quotes on purpose instead of single quotes + trap "${_trap_cleanup_actions}" EXIT +} + +function _trap_setup_cleanups() { + declare -g _trap_cleanup_actions + _trap_cleanup_actions=':' + + declare -g -A _trap_cleanup_snapshots + _trap_cleanup_snapshots=() + + _trap_update_trap +} + +function _trap_add_cleanup() { + local tac_joined + join_by tac_joined ' ; ' "${@/%/' || :'}" + _trap_cleanup_actions="${tac_joined} ; ${_trap_cleanup_actions}" + _trap_update_trap +} + +# "ignore" cleanups + +function _ignore_setup_cleanups() { + : +} + +function _ignore_add_cleanup() { + : +} + +function _ensure_valid_cleanups_sh_cleanup_kind_() { + local kind + kind=${1}; shift + + local -a functions=( + setup_cleanups + add_cleanup + ) + + local func + for func in "${functions[@]/#/_${kind}_}"; do + if ! declare -pF "${func}" >/dev/null 2>/dev/null; then + fail "kind '${kind}' is not a valid cleanup kind, function '${func}' is not defined" + fi + done +} + +function _call_cleanup_func() { + local func_name + func_name=${1}; shift + if [[ -z "${_cleanups_sh_cleanup_kind_}" ]]; then + _cleanups_sh_cleanup_kind_='trap' + fi + + local func + func="_${_cleanups_sh_cleanup_kind_}_${func_name}" + + "${func}" "${@}" +} + +fi diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh new file mode 100644 index 0000000000..04f6367a88 --- /dev/null +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -0,0 +1,485 @@ +#!/bin/bash + +if [[ -z ${__INSIDE_SDK_CONTAINER_LIB_SH_INCLUDED__:-} ]]; then +__INSIDE_SDK_CONTAINER_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Invokes emerge to get a report about built packages for a given +# metapackage in the given root that has a portage configuration. +# +# Params: +# +# 1 - root filesystem with the portage config +# 2 - metapackage to get the deps from +function emerge_pretend() { + local root package + root=${1}; shift + package=${1}; shift + + # Probably a bunch of those flags are not necessary, but I'm not + # touching it - they seem to be working. :) + local -a emerge_opts=( + --config-root="${root}" + --root="${root}" + --sysroot="${root}" + --pretend + --columns + --nospinner + --oneshot + --color n + --emptytree + --verbose + --verbose-conflicts + --verbose-slot-rebuilds y + --changed-deps y + --changed-deps-report y + --changed-slot y + --changed-use + --newuse + --complete-graph y + --deep + --rebuild-if-new-slot y + --rebuild-if-unbuilt y + --with-bdeps y + --dynamic-deps y + --update + --ignore-built-slot-operator-deps y + --selective n + --keep-going y + ) + local rv + rv=0 + emerge "${emerge_opts[@]}" "${package}" || rv=${?} + if [[ ${rv} -ne 0 ]]; then + echo "WARNING: emerge exited with status ${rv}" >&2 + fi +} + +# Gets package list for SDK. +function package_info_for_sdk() { + local root + root='/' + + ignore_crossdev_stuff "${root}" + emerge_pretend "${root}" coreos-devel/sdk-depends + revert_crossdev_stuff "${root}" +} + +# Gets package list for board of a given architecture. +# +# Params: +# +# 1 - architecture +function package_info_for_board() { + local arch + arch=${1}; shift + + local root + root="/build/${arch}-usr" + + # Ignore crossdev stuff in both SDK root and board root - emerge + # may query SDK stuff for the board packages. + ignore_crossdev_stuff / + ignore_crossdev_stuff "${root}" + emerge_pretend "${root}" coreos-devel/board-packages + revert_crossdev_stuff "${root}" + revert_crossdev_stuff / +} + +# Set the directory where the emerge output and the results of +# processing it will be stored. EO stands for "emerge output" +# +# Params: +# +# 1 - directory path +function set_eo() { + local dir=${1}; shift + + SDK_EO="${dir}/sdk-emerge-output" + BOARD_EO="${dir}/board-emerge-output" + # shellcheck disable=SC2034 # used indirectly in cat_eo_f + SDK_EO_F="${SDK_EO}-filtered" + # shellcheck disable=SC2034 # used indirectly in cat_eo_f + BOARD_EO_F="${BOARD_EO}-filtered" + # shellcheck disable=SC2034 # used indirectly in cat_eo_w + SDK_EO_W="${SDK_EO}-warnings" + # shellcheck disable=SC2034 # used indirectly in cat_eo_w + BOARD_EO_W="${BOARD_EO}-warnings" +} + +# Print the contents of file, path of which is stored in a variable of +# a given name. +# +# Params: +# +# 1 - name of the variable +function cat_var() { + local var_name + var_name=${1}; shift + local -n ref="${var_name}" + + if [[ -z "${ref+isset}" ]]; then + fail "${var_name} unset" + fi + if [[ ! -e "${ref}" ]]; then + fail "${ref} does not exist" + fi + + cat "${ref}" +} + +# Print contents of the emerge output of a given kind. Kind can be +# either either sdk or board. +# +# Params: +# +# 1 - kind +function cat_eo() { + local kind + kind=${1}; shift + + cat_var "${kind^^}_EO" +} + +# Print contents of the filtered emerge output of a given kind. Kind +# can be either either sdk or board. The filtered emerge output +# contains only lines with package information. +# +# Params: +# +# 1 - kind +function cat_eo_f() { + local kind + kind=${1}; shift + cat_var "${kind^^}_EO_F" +} + +# Print contents of a file that stores warnings that were printed by +# emerge. The warnings are specific to a kind (sdk or board). +# +# Params: +# +# 1 - kind +function cat_eo_w() { + local kind + kind=${1}; shift + + cat_var "${kind^^}_EO_W" +} + +# JSON output would be more verbose, but probably would not require +# these abominations below. But, alas, emerge doesn't have that yet. + +# status package name version slot repo target (opt) keyvals size +# |--------------| |----------| |#-g1-----------#--#-g2-#| |--|-g------| |-g----------#-#-g-----| |---| +# [ebuild R ~] virtual/rust [1.71.1:0/llvm-16::coreos] to /some/root USE="-rustfmt" FOO="bar" 0 KiB +# +# Actually, there can also be another "version slot repo" part between +# the first "version slot repo" and "target" part. +STATUS_RE='\[[^]]*]' # 0 groups +PACKAGE_NAME_RE='[^[:space:]]*' # 0 groups +VER_SLOT_REPO_RE='\[\([^]]\+\)::\([^]]\+\)]' # 2 groups +TARGET_RE='to[[:space:]]\+\([^[:space:]]\)\+' # 1 group +KEYVALS_RE='\([[:space:]]*[A-Za-z0-9_]*="[^"]*"\)*' # 1 group (but containing only the last pair!) +SIZE_RE='[[:digit:]]\+[[:space:]]*[[:alpha:]]*B' # 0 groups +SPACES_RE='[[:space:]]\+' # 0 groups +NONSPACES_RE='[^[:space:]]\+' # 0 groups +NONSPACES_WITH_COLON_RE='[^[:space:]]*:' # 0 groups + +FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${VER_SLOT_REPO_RE}"'\)\{1,2\}\('"${SPACES_RE}${TARGET_RE}"'\)\?\('"${SPACES_RE}${KEYVALS_RE}"'\)*'"${SPACES_RE}${SIZE_RE}"'$' + +# Filters sdk reports to get the package information. +function filter_sdk_eo() { + cat_eo sdk | xgrep -e "${FULL_LINE_RE}" +} + +# Filters board reports for a given arch to get the package +# information. +# +# Params: +# +# 1 - architecture +function filter_board_eo() { + local arch + arch=${1}; shift + + # Replace ${arch}-usr in the output with a generic word BOARD. + cat_eo board | \ + xgrep -e "${FULL_LINE_RE}" | \ + sed -e "s#/build/${arch}-usr/#/build/BOARD/#" +} + +# Filters sdk reports to get anything but the package information +# (i.e. junk). +function junk_sdk_eo() { + cat_eo sdk | xgrep -v -e "${FULL_LINE_RE}" +} + +# Filters board reports to get anything but the package information +# (i.e. junk). +function junk_board_eo() { + cat_eo board | xgrep -v -e "${FULL_LINE_RE}" +} + +# More regexp-like abominations follow. + +# There may also be a line like: +# +# [blocks B ] / is masked +# (like for sys-libs/glibc and cross-x86_64-cros-linux-gnu/glibc), +# because it has no keywords. In theory, we could try updating +# /etc/portage/package.mask/cross- file created by the +# crossdev tool to unmask the new version, but it's an unnecessary +# hassle - native and cross package are supposed to be the same ebuild +# anyway, so update information about cross package is redundant. +# +# Params: +# +# 1 - root directory +# 2 - ID of the crossdev repository (optional, defaults to x-crossdev) +function ignore_crossdev_stuff() { + local root crossdev_repo_id + root=${1}; shift + crossdev_repo_id=${1:-x-crossdev}; shift || : + + local crossdev_repo_path + crossdev_repo_path=$(portageq get_repo_path "${root}" "${crossdev_repo_id}") + + local ics_path ics_dir + get_provided_file "${root}" ics_path + dirname_out "${ics_path}" ics_dir + + sudo mkdir -p "${ics_dir}" + env --chdir="${crossdev_repo_path}" find . -type l | \ + cut -d/ -f2-3 | \ + sed -e 's/$/-9999/' | \ + sudo tee "${ics_path}" >/dev/null +} + +# Reverts effects of the ignore_crossdev_stuff function. +# +# Params: +# +# 1 - root directory +function revert_crossdev_stuff() { + local root + root=${1}; shift + + local ics_path ics_dir + get_provided_file "${root}" ics_path + dirname_out "${ics_path}" ics_dir + + sudo rm -f "${ics_path}" + if dir_is_empty "${ics_dir}"; then + sudo rmdir "${ics_dir}" + fi +} + +# Checks if the expected reports were generated by emerge. +function ensure_valid_reports() { + local kind var_name + for kind in sdk board; do + var_name="${kind^^}_EO_F" + if [[ ! -s ${!var_name} ]]; then + fail "report files are missing or are empty" + fi + done +} + +# Drops the empty warning files in given directory. +# +# Params: +# +# 1 - path to the directory +function clean_empty_warning_files() { + local dir + dir=${1}; shift + + local file + for file in "${dir}/"*'-warnings'; do + if [[ ! -s ${file} ]]; then + rm -f "${file}" + fi + done +} + +fi diff --git a/pkg_auto/impl/mvm.sh b/pkg_auto/impl/mvm.sh new file mode 100644 index 0000000000..eded4aa54a --- /dev/null +++ b/pkg_auto/impl/mvm.sh @@ -0,0 +1,573 @@ +#!/bin/bash + +# +# "mvm" stands for "multi-valued map", so these are maps of scalars +# (strings, numbers) to other container (arrays or maps) +# +# mvm is implemented with a map that contains some predefined keys, +# like "name", "constructor", "storage", etc. +# +# The "storage" field is the actual "map" part of the "mvm", at the +# values stored in it are names of the global variables being the +# "multi-valued" part of the "mvm". In the code these variables are +# referred to as "mvc" meaning "multi-value container". +# +# The "constructor" and "destructor" fields are here to properly +# implement creating and destroying mvcs. The "adder" field is for +# adding elements to an mvc. +# +# There is also a "counter" field which, together with the "name" +# field, is used for creating the names for mvc variables. +# +# The "extras" field is for user-defined mapping. The mvm will clear +# the mapping itself, but if the values are anything else than simple +# scalars (e.g. names of variables) then the cleanup of those is +# user's task. +# +# There is also an optional field named "iteration_helper" which is a +# callback invoked when iterating over the mvm. +# +# In order to implement a new mvc type, the following functions need +# to be implemented: +# +# _constructor - takes an mvc name; should create an mvc with the +# passed name. +# _destructor - takes an mvc name; should unset an mvc with the +# passed name, should likely take care of cleaning +# up the values stored in the mvc +# _adder - takes an mvc name and values to be added; should add +# the values to the mvc +# _iteration_helper - optional; takes a key, an mvc name, a +# callback and extra arguments to be +# forwarded to the callback; should invoke +# the callback with the extra arguments, the +# key, the mvc name and optionally some +# extra arguments the helper deems useful + +if [[ -z ${__MVM_SH_INCLUDED__:-} ]]; then +__MVM_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Used for creating unique names for extras and storage maps. +MVM_COUNTER=0 + +# Array mvm, the default. Provides an iteration helper that sends all +# the array values to the iteration callback. + +function mvm_mvc_array_constructor() { + local array_var_name + array_var_name=${1}; shift + + declare -g -a "${array_var_name}" + + local -n array_ref=${array_var_name} + array_ref=() +} + +function mvm_mvc_array_destructor() { + local array_var_name + array_var_name=${1}; shift + + unset "${array_var_name}" +} + +function mvm_mvc_array_adder() { + local array_var_name + array_var_name=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n array_ref=${array_var_name} + + array_ref+=( "${@}" ) +} + +# iteration_helper is optional +function mvm_mvc_array_iteration_helper() { + local key array_var_name callback + key=${1}; shift + array_var_name=${1}; shift + callback=${1}; shift + # rest are extra args passed to cb + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n array_ref=${array_var_name} + "${callback}" "${@}" "${key}" "${array_var_name}" "${array_ref[@]}" +} + +# Map mvm. When adding elements to the mvc, it is expected that the +# number of items passed will be even. Odd elements will be used as +# keys, even elements will be used as values. +# +# No iteration helper. + +function mvm_mvc_map_constructor() { + local map_var_name + map_var_name=${1}; shift + + declare -g -A "${map_var_name}" + + local -n map_ref=${map_var_name} + map_ref=() +} + +function mvm_mvc_map_destructor() { + local map_var_name + map_var_name=${1}; shift + + unset "${map_var_name}" +} + +function mvm_mvc_map_adder() { + local map_var_name + map_var_name=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n map_ref=${map_var_name} + + while [[ ${#} -gt 1 ]]; do + # shellcheck disable=SC2034 # it's a reference to external variable + map_ref["${1}"]=${2} + shift 2 + done +} + +# Set mvm. Behaves like array mvm, but all elements in each set are +# unique and the order of elements is not guaranteed to be the same as +# order of insertions. + +function mvm_mvc_set_constructor() { + local set_var_name + set_var_name=${1}; shift + + declare -g -A "${set_var_name}" + + # shellcheck disable=SC2178 # shellcheck does not grok refs + local -n set_ref=${set_var_name} + set_ref=() +} + +function mvm_mvc_set_destructor() { + local set_var_name + set_var_name=${1} + + unset "${set_var_name}" +} + +function mvm_mvc_set_adder() { + local set_var_name + set_var_name=${1}; shift + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${set_var_name} + while [[ ${#} -gt 0 ]]; do + set_ref["${1}"]=x + shift + done +} + +# iteration_helper is optional +function mvm_mvc_set_iteration_helper() { + local key map_var_name callback + + key=${1}; shift + set_var_name=${1}; shift + callback=${1}; shift + # rest are extra args passed to cb + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${set_var_name} + "${callback}" "${@}" "${key}" "${set_var_name}" "${!set_ref[@]}" +} + +# mvm API + +# Creates a new mvm with a passed name, optionally type and +# extras. The name must be globally unique. The type is optional. If +# no type is passed, an array mvm will be assumed. Otherwise the type +# must be valid, i.e. it must provide a constructor, a destructor, an +# adder and, optionally, an iteration helper. The built in types are +# "mvm_mvc_array", "mvm_mvc_set" and "mvm_mvc_map". If any extras are +# passed, they must be preceded with a double dash to avoid ambiguity +# between type and a first extras key. Extras are expected to be even +# in count, odd elements will be used as keys, even elements will be +# used as values. +# +# Params: +# +# 1 - name of the mvm +# @ - optional mvc type, optionally followed by double dash and extras +# key-value pairs. +function mvm_declare() { + local mvm_var_name + mvm_var_name=${1}; shift + + if declare -p "${mvm_var_name}" >/dev/null 2>/dev/null; then + fail "variable ${mvm_var_name} already exists, declaring mvm for it would clobber it" + fi + + local value_handler_prefix + value_handler_prefix='' + if [[ ${#} -gt 0 ]]; then + if [[ ${1} != '--' ]]; then + value_handler_prefix=${1} + shift + fi + if [[ ${#} -gt 0 ]]; then + if [[ ${1} != '--' ]]; then + fail "missing double-dash separator between optional value handler prefix and extra key value pairs for '${mvm_var_name}'" + fi + shift + fi + fi + if [[ -z ${value_handler_prefix} ]]; then + value_handler_prefix=mvm_mvc_array + fi + # rest are key value pairs for extras + + mvm_debug "${mvm_var_name}" "using prefix ${value_handler_prefix}" + + local constructor destructor adder iteration_helper + constructor="${value_handler_prefix}_constructor" + destructor="${value_handler_prefix}_destructor" + adder="${value_handler_prefix}_adder" + iteration_helper="${value_handler_prefix}_iteration_helper" + + local func + for func in "${constructor}" "${destructor}" "${adder}"; do + if ! declare -pF "${func}" >/dev/null 2>/dev/null; then + fail "'${func}' is not a function, is '${value_handler_prefix}' a valid prefix?" + fi + done + + if ! declare -pF "${iteration_helper}" >/dev/null 2>/dev/null; then + mvm_debug "${mvm_var_name}" "no interation helper available" + iteration_helper='' + fi + + local extras_idx storage_idx extras_map_var_name storage_map_var_name + extras_idx=$((MVM_COUNTER)) + storage_idx=$((MVM_COUNTER + 1)) + extras_map_var_name="mvm_stuff_${extras_idx}" + storage_map_var_name="mvm_stuff_${storage_idx}" + + MVM_COUNTER=$((MVM_COUNTER + 2)) + + declare -g -A "${mvm_var_name}" "${extras_map_var_name}" "${storage_map_var_name}" + + mvm_debug "${mvm_var_name}" "extras map: ${extras_map_var_name}, storage_map: ${storage_map_var_name}" + + local -n storage_map_ref=${storage_map_var_name} + storage_map_ref=() + + local -n mvm_ref=${mvm_var_name} + # shellcheck disable=SC2034 # it's a reference to external variable + mvm_ref=( + ['name']="${mvm_var_name}" + ['constructor']="${constructor}" + ['destructor']="${destructor}" + ['adder']="${adder}" + ['iteration_helper']="${iteration_helper}" + ['counter']=0 + ['extras']="${extras_map_var_name}" + ['storage']="${storage_map_var_name}" + ) + local -n extras_map_ref=${extras_map_var_name} + while [[ ${#} -gt 1 ]]; do + mvm_debug "${mvm_var_name}" "adding ${1} -> ${2} pair to extras" + extras_map_ref["${1}"]=${2} + shift 2 + done + if [[ ${#} -gt 0 ]]; then + fail "odd number of parameters for extra key value information for '${mvm_var_name}'" + fi +} + +# Takes a name of mvm, a callback, and extra parameters that will be +# forwarded to the callback. Before invoking the callback, the +# function will declare a local variable called "mvm" which is a +# reference to the variable with the passed name. The "mvm" variable +# can be used for easy access to the map within the callback. +# +# The convention is that the function foo_barize will use mvm_call to +# invoke a callback named foo_c_barize. The foo_c_barize function can +# invoke other _c_ infixed functions, like mvm_c_get_extra or +# mvm_c_get. +# +# Params: +# +# 1 - name of mvm variable +# 2 - name of the callback +# @ - arguments for the callback +function mvm_call() { + local name func + name=${1}; shift + func=${1}; shift + # rest are func args + + mvm_debug "${name}" "invoking ${func} with args: ${*@Q}" + + # The "mvm" variable can be used by ${func} now. + local -n mvm=${name} + "${func}" "${@}" +} + +# Internal function that generates a name for mvc based on passed name +# and counter. +function __mvm_mvc_name() { + local name counter mvc_name_var_name + name=${1}; shift + counter=${1}; shift + mvc_name_var_name=${1}; shift + local -n mvc_name_ref=${mvc_name_var_name} + + # shellcheck disable=SC2034 # it's a reference to external variable + mvc_name_ref="mvm_${name}_mvc_${counter}" +} + +# Destroy the mvm with passed name. +# +# Params: +# +# 1 - name of mvm to destroy +function mvm_unset() { + mvm_call "${1}" mvm_c_unset "${@:2}" +} + +# Helper function for mvm_unset invoked through mvm_call. +function mvm_c_unset() { + local counter name extras_map_var_name storage_map_var_name destructor mvm_mcu_mvc_name + + counter=${mvm['counter']} + name=${mvm['name']} + extras_map_var_name=${mvm['extras']} + storage_map_var_name=${mvm['storage']} + destructor=${mvm['destructor']} + + while [[ ${counter} -gt 0 ]]; do + counter=$((counter - 1)) + __mvm_mvc_name "${name}" "${counter}" mvm_mcu_mvc_name + "${destructor}" "${mvm_mcu_mvc_name}" + done + unset "${storage_map_var_name}" + unset "${extras_map_var_name}" + unset "${name}" +} + +# Gets an value from extras map for a given key. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - extra key +# 3 - name of a variable where the extra value will be stored +function mvm_get_extra() { + mvm_call "${1}" mvm_c_get_extra "${@:2}" +} + +# Helper function for mvm_get_extra invoked through mvm_call. +function mvm_c_get_extra() { + local extra extra_var_name + extra=${1}; shift + extra_var_name=${1}; shift + local -n extra_ref=${extra_var_name} + + local extras_map_var_name + extras_map_var_name=${mvm['extras']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n extras_map_ref=${extras_map_var_name} + + # shellcheck disable=SC2034 # it's a reference to external variable + extra_ref=${extras_map_ref["${extra}"]:-} +} + +# Gets a name of the mvc for a given key. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - key +# 3 - name of a variable where the mvc name will be stored +function mvm_get() { + mvm_call "${1}" mvm_c_get "${@:2}" +} + +# Helper function for mvm_get invoked through mvm_call. +function mvm_c_get() { + local key value_var_name + key=${1}; shift + value_var_name=${1}; shift + local -n value_ref=${value_var_name} + + local storage_map_var_name + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + + # shellcheck disable=SC2034 # it's a reference to external variable + value_ref=${storage_map_ref["${key}"]:-} +} + +# Internal function for creating a new mvc. +function __mvm_c_make_new_mvc() { + local key mvc_name_var_name + key=${1}; shift + mvc_name_var_name=${1}; shift + + local name counter storage_map_var_name + name=${mvm['name']} + counter=${mvm['counter']} + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + + __mvm_mvc_name "${name}" "${counter}" "${mvc_name_var_name}" + + local constructor + constructor=${mvm['constructor']} + + "${constructor}" "${!mvc_name_var_name}" + mvm['counter']=$((counter + 1)) + storage_map_ref["${key}"]=${!mvc_name_var_name} +} + +# Adds passed elements to the mvm under the given key. If an mvc for +# the key didn't exist in the mvm, it gets created. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - key +# @ - elements +function mvm_add() { + mvm_call "${1}" mvm_c_add "${@:2}" +} + +# Helper function for mvm_add invoked through mvm_call. +function mvm_c_add() { + local key + key=${1}; shift + # rest are values to add + + local adder mvm_mca_mvc_name + adder=${mvm['adder']} + mvm_c_get "${key}" mvm_mca_mvc_name + + if [[ -z ${mvm_mca_mvc_name} ]]; then + __mvm_c_make_new_mvc "${key}" mvm_mca_mvc_name + fi + "${adder}" "${mvm_mca_mvc_name}" "${@}" +} + +# Removes the key from the mvm. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - key +function mvm_remove() { + mvm_call "${1}" mvm_c_remove "${@:2}" +} + +# Helper function for mvm_remove invoked through mvm_call. +function mvm_c_remove() { + local key + key=${1}; shift + + local storage_map_var_name + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + + if [[ -z ${storage_map_ref["${key}"]:-} ]]; then + return 0 + fi + + local var_name=${storage_map_ref["${key}"]} + unset "storage_map_ref[${key}]" + + local destructor + destructor=${mvm['destructor']} + + "${destructor}" "${var_name}" +} + +# Iterates over the key-mvc pairs and invokes a callback for each. The +# function also takes some extra parameters to forward to the +# callback. The callback will receive, in order, extra parameters, a +# key, an mvc name, and possibly some extra parameters from the +# iteration helper, if such exists for the mvm. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - callback +# @ - extra parameters forwarded to the callback +function mvm_iterate() { + mvm_call "${1}" mvm_c_iterate "${@:2}" +} + +# Helper function for mvm_iterate invoked through mvm_call. +function mvm_c_iterate() { + local callback + callback=${1}; shift + # rest are extra args passed to callback + + local storage_map_var_name helper + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + helper=${mvm['iteration_helper']} + + local key value + if [[ -n "${helper}" ]]; then + for key in "${!storage_map_ref[@]}"; do + value=${storage_map_ref["${key}"]} + "${helper}" "${key}" "${value}" "${callback}" "${@}" + done + else + for key in "${!storage_map_ref[@]}"; do + value=${storage_map_ref["${key}"]} + "${callback}" "${@}" "${key}" "${value}" + done + fi +} + +# debugging + +declare -A MVM_DEBUG_NAMES=() + +# Enables printing debugging info for a specified mvm. +# +# Params: +# +# 1 - name of the mvm variable +function mvm_debug_enable() { + local mvm_var_name=${1}; shift + MVM_DEBUG_NAMES["${mvm_var_name}"]=x +} + +# Print debugging info about the mvm if debugging for it was enabled +# beforehand. +# +# Params: +# +# 1 - name of the mvm variable +# @ - strings to be printed +function mvm_debug() { + local name=${1}; shift + + if [[ -n ${MVM_DEBUG_NAMES["${name}"]:-} ]]; then + info "MVM_DEBUG(${name}): ${*}" + fi +} + +# Disables printing debugging info for a specified mvm. +# +# Params: +# +# 1 - name of the mvm variable +function mvm_debug_disable() { + local mvm_var_name=${1}; shift + unset "MVM_DEBUG_NAMES[${mvm_var_name}]" +} + +fi diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh new file mode 100644 index 0000000000..1c8adba2d6 --- /dev/null +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -0,0 +1,3305 @@ +#!/bin/bash + +# +# TODO: +# +# - Generate a report about missing build deps of board packages in +# sdk. These reports can be generated by processing sdk-pkgs-kv and +# board-bdeps reports, I think. +# +# - Mount Gentoo repo into the SDK container and set up emerge to use +# Gentoo as a primary repo, and portage-stable and coreos-overlay as +# overlays. That way if an updated package pulls in a new package we +# can notice it when it comes from Gentoo (emerge reports also +# source repo like sys-libs/glibc-2.35-r5::gentoo or something like +# this). This would make this script more robust. +# +# - Instead of having a list of packages to update, rather update them +# all in a single commit and have a list of exclusions. The reason +# is that, at this point, almost all of the packages in +# portage-stable are automatically updated, exclusions being usually +# temporary, so it would be better to have a short file with +# temporary exclusions rather than a long file with some commented +# out entries. This probably would render the sort_packages_list.py +# script unnecessary. On the other hand, the current mode of +# operation may be still useful for the coreos-overlay packages, +# because none of them are under automation (some are, though, under +# an ad-hoc automation via github actions). +# +# - Handle package appearance or disappearance. Currently, when a +# package ends up being unused (so it exists, but is not picked up, +# because some other package stopped depending on it) or removed, +# the package ends up in the manual-work-needed file. This probably +# could be handled as an entry in the summary stubs about being +# dropped. +# +# - Find unused packages and eclasses. +# +# - The rename handling should probably also change all instances of +# the old name everywhere outside portage-stable, otherwise emerge +# may fail when some our ebuild still uses the old name, maybe. +# + +# Needed to be enabled here to parse some globs inside the functions. +shopt -s extglob +# Saner defaults. +shopt -s nullglob +shopt -s dotglob + +if [[ -z ${__PKG_AUTO_LIB_SH_INCLUDED__:-} ]]; then +__PKG_AUTO_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" + +# Sets up the workdir using the passed config. The config can be +# created basing on the config_template file or using the +# generate_config script. +# +# The path to the workdir can be empty - the function will then create +# a temporary directory. +# +# This also sets the WORKDIR global variable, allowing other function +# to be invoked. +# +# Params: +# +# 1 - path to the workdir +# 2 - path to the config file +function setup_workdir_with_config() { + local workdir file + workdir=${1}; shift + config_file=${1}; shift + + local cfg_scripts cfg_aux cfg_reports cfg_old_base cfg_new_base + local -a cfg_cleanups cfg_debug_packages + local -A cfg_overrides + + # some defaults + cfg_old_base='origin/main' + cfg_new_base='' + cfg_cleanups=('ignore') + cfg_overrides=() + cfg_debug_packages=() + + local line key value swwc_stripped var_name arch + while read -r line; do + strip_out "${line%%:*}" swwc_stripped + key=${swwc_stripped} + strip_out "${line#*:}" swwc_stripped + value=${swwc_stripped} + if [[ -z ${value} ]]; then + fail "empty value for ${key} in config" + fi + case ${key} in + scripts|aux|reports) + var_name="cfg_${key//-/_}" + local -n var=${var_name} + var=$(realpath "${value}") + unset -n var + ;; + old-base|new-base) + var_name="cfg_${key//-/_}" + local -n var=${var_name} + var=${value} + unset -n var + ;; + cleanups|debug-packages) + var_name="cfg_${key//-/_}" + mapfile -t "${var_name}" <<<"${value//,/$'\n'}" + ;; + *-sdk-img) + arch=${key%%-*} + # shellcheck disable=SC2034 # used by name below + cfg_overrides["${arch}"]=${value} + ;; + esac + done < <(cat_meaningful "${config_file}") + if [[ -z "${cfg_new_base}" ]]; then + cfg_new_base=${cfg_old_base} + fi + for key in scripts aux reports; do + var_name="cfg_${key//-/_}" + if [[ -z "${!var_name}" ]]; then + fail "${key} was not specified in config" + fi + done + + setup_cleanups "${cfg_cleanups[@]}" + setup_workdir "${workdir}" + add_cleanup "rm -f ${WORKDIR@Q}/config" + cp -a "${config_file}" "${WORKDIR}/config" + setup_worktrees_in_workdir "${cfg_scripts}" "${cfg_old_base}" "${cfg_new_base}" "${cfg_reports}" "${cfg_aux}" + override_sdk_image_names cfg_overrides + add_debug_packages "${cfg_debug_packages[@]}" +} + +# Goes over the list of automatically updated packages and synces them +# with packages from Gentoo repo. Cleans up missing packages. +# +# The function can only be called after the workdir has been set up +# with setup_workdir_with_config. +# +# Params: +# +# 1 - a path to the Gentoo repo +function perform_sync_with_gentoo() { + local gentoo + gentoo=$(realpath "${1}"); shift + + run_sync "${gentoo}" + handle_missing_in_scripts + handle_missing_in_gentoo "${gentoo}" +} + +# Generates package update reports. Duh. +# +# The function can only be called after the workdir has been set up +# with setup_workdir_with_config. +# +# The location of the reports is specified in the config that was +# passed to setup_workdir_with_config. +function generate_package_update_reports() { + generate_sdk_reports + handle_gentoo_sync +} + +# Saves the new state to a git branch in scripts. +# +# The function can only be called after the workdir has been set up +# with setup_workdir_with_config. +# +# Params: +# +# 1 - name of the new branch +function save_new_state() { + local branch_name + branch_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + info "saving new state to branch ${branch_name}" + # shellcheck disable=SC2153 # SCRIPTS is not a misspelling, it comes from globals file + git -C "${SCRIPTS}" branch --force "${branch_name}" "${NEW_STATE_BRANCH}" +} + +# +# +# Implementation details, do not use directly in scripts sourcing this file. +# +# + +# Creates a workdir, the path to which is stored in WORKDIR global +# variable. +# +# 1 - predefined work directory path (optional) +function setup_workdir() { + local workdir + workdir=${1:-} + if [[ -z "${workdir}" ]]; then + workdir=$(mktemp --tmpdir --directory 'pkg-auto-workdir.XXXXXXXX') + else + if [[ -e ${workdir} ]] && [[ ! -d ${workdir} ]]; then + fail "Expected ${workdir@Q} to be a directory" + fi + if [[ -d ${workdir} ]] && ! dir_is_empty "${workdir}"; then + fail "Expected ${workdir@Q} to be an empty directory" + fi + fi + + declare -g WORKDIR + WORKDIR=$(realpath "${workdir}") + add_cleanup "rmdir ${WORKDIR@Q}" + mkdir -p "${WORKDIR}" + + setup_initial_globals_file +} + +# Sets up worktrees for the old and new state inside WORKDIR. Creates +# the globals file inside WORKDIR. +# +# 1 - path to scripts repo +# 2 - base for the old state worktree (e.g. origin/main) +# 3 - base for the new state worktree (e.g. origin/main) +# 4 - path to reports directory +function setup_worktrees_in_workdir() { + local scripts old_base new_base reports_dir aux_dir + scripts=${1}; shift + old_base=${1}; shift + new_base=${1}; shift + reports_dir=${1}; shift + aux_dir=${1}; shift + + local old_state new_state + old_state="${WORKDIR}/old_state" + new_state="${WORKDIR}/new_state" + + # create reports directory now - there may be some developer + # warnings afoot + mkdir -p "${reports_dir}" + + setup_worktree "${scripts}" "${old_base}" "old-state-${RANDOM}" "${old_state}" + setup_worktree "${scripts}" "${new_base}" "new-state-${RANDOM}" "${new_state}" + extend_globals_file "${scripts}" "${old_state}" "${new_state}" "${reports_dir}" "${aux_dir}" +} + +# Adds overridden SDK image names to the globals file. +# +# Params: +# +# 1 - name of a map variable; should be a mapping of architecture to +# the image name +function override_sdk_image_names() { + local -n overrides_map_ref=${1} + + if [[ ${#overrides_map_ref[@]} -eq 0 ]]; then + return 0 + fi + + local arch image_name upcase_arch + local -a lines + lines=() + for arch in "${!overrides_map_ref[@]}"; do + image_name=${overrides_map_ref["${arch}"]} + upcase_arch=${arch^^} + if [[ ${#lines[@]} -eq 0 ]]; then + # separate overrides from initial values + lines+=( '' ) + fi + lines+=( "${upcase_arch}_SDK_IMAGE=${image_name@Q}" ) + done + append_to_globals "${lines[@]}" +} + +# Adds information about packages to be debugged to the globals file. +# +# Params: +# +# @ - a list of packages to be debugged +function add_debug_packages() { + local -a prepared lines + prepared=( "${@@Q}" ) + prepared=( "${prepared[@]/#/' ['}" ) + prepared=( "${prepared[@]/%/']=x'}" ) + lines=( + '' + 'local -A DEBUG_PACKAGES' + '' + 'DEBUG_PACKAGES=(' + "${prepared[@]}" + ')' + ) + append_to_globals "${lines[@]}" +} + +# Appends passed lines to the globals file. +# +# Params: +# +# @ - lines to append +function append_to_globals() { + local globals_file + globals_file="${WORKDIR}/globals" + if [[ ! -e "${globals_file}" ]]; then + fail "globals not set yet in workdir" + fi + + lines_to_file "${globals_file}" "${@}" +} + +# Processes the update files in portage-stable and coreos-overlay to +# figure out potential package renames. The results are stored in the +# passed map. +# +# Params: +# +# 1 - name of a map variable; will be a mapping of old package name to +# new package name +function process_profile_updates_directory() { + local from_to_map_var_name=${1}; shift + + local -a ppud_ordered_names + get_ordered_update_filenames ppud_ordered_names + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local bf ps_f co_f pkg f line old new + local -a fields + local -A from_to_f=() + mvm_declare ppud_to_from_set_mvm mvm_mvc_set + for bf in "${ppud_ordered_names[@]}"; do + # coreos-overlay updates may overwrite updates from + # portage-stable, but only from the file of the same name + # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file + ps_f=${NEW_PORTAGE_STABLE}/profiles/updates/${bf} + # shellcheck disable=SC2153 # NEW_COREOS_OVERLAY is not a misspelling, it comes from globals file + co_f=${NEW_COREOS_OVERLAY}/profiles/updates/${bf} + for f in "${ps_f}" "${co_f}"; do + if [[ ! -f ${f} ]]; then + continue + fi + while read -r line; do + if [[ ${line} != 'move '* ]]; then + # other possibility is "slotmove" - we don't care + # about those. + continue + fi + mapfile -t fields <<<"${line// /$'\n'}" + if [[ ${#fields[@]} -ne 3 ]]; then + fail_lines \ + "Malformed line ${line@Q} in updates file ${f@Q}." \ + "The line should have 3 fields, has ${#fields[*]}." + fi + from_to_f["${fields[1]}"]=${fields[2]} + done <"${f}" + done + for old in "${!from_to_f[@]}"; do + new=${from_to_f["${old}"]} + update_rename_maps "${from_to_map_var_name}" ppud_to_from_set_mvm "${old}" "${new}" + done + done + + mvm_unset ppud_to_from_set_mvm +} + +# Gets a merged and ordered list of update files from portage-stable +# and coreos-overlay and stores the in the passed array. The files +# have names like Q1-2018, Q1-2023, Q2-2019 and so on. We need to sort +# them by year, then by quarter. +# +# Params: +# +# 1 - name of a array variable where the ordered names will be stored +function get_ordered_update_filenames() { + local ordered_names_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -A names_set=() + local f + + for f in "${NEW_PORTAGE_STABLE}/profiles/updates/"* "${NEW_COREOS_OVERLAY}/profiles/updates/"*; do + names_set["${f##*/}"]=x + done + + mapfile -t "${ordered_names_var_name}" < <(printf '%s\n' "${!names_set[@]}" | sort --field-separator=- --key=2n --key=1n) +} + +# Updates the rename map with the new "old to new package rename". It +# tries to be smart about the rename sequences (though not sure if +# this is necessary, really). If in older update file package foo was +# renamed to bar and in current update file the bar package is renamed +# to quux, then this function adds an entry about the "bar to quux" +# rename, but also updates the older entry about "foo to bar" rename +# to "foo to quux" rename. +# +# Params: +# +# 1 - name of the renames map variable; should be a mapping of old to +# new names +# 2 - name of the set mvm variable; should be a mapping of new name to +# a set of old names (a reverse mapping to the renames map) +# 3 - old name +# 4 - new name +function update_rename_maps() { + local -n ft_map=${1}; shift + local tf_set_mvm_var_name=${1}; shift + local old_name=${1}; shift + local new_name=${1}; shift + + local prev_new_name=${ft_map["${old_name}"]:-} + + if [[ -n ${prev_new_name} ]] && [[ ${prev_new_name} != "${new_name}" ]]; then + fail_lines \ + "Invalid package rename from ${old_name@Q} to ${new_name@Q}." \ + "There was already a rename from ${old_name@Q} to ${prev_new_name@Q}." + fi + + local -a new_set=() + + local urm_set_var_name + mvm_get "${tf_set_mvm_var_name}" "${old_name}" urm_set_var_name + if [[ -n ${urm_set_var_name} ]]; then + local -n old_set=${urm_set_var_name} + new_set+=( "${!old_set[@]}" ) + unset -n old_set + fi + new_set+=( "${old_name}" ) + mvm_add "${tf_set_mvm_var_name}" "${new_name}" "${new_set[@]}" + local old + + for old in "${new_set[@]}"; do + ft_map["${old}"]=${new_name} + done + unset -n ft_map +} + +# Sets up a worktree and necessary cleanups. +# +# Params: +# +# 1 - path to the git repo +# 2 - name of a branch to be used as a base of a new worktree branch +# 3 - name of the new worktree branch +# 4 - path where the new worktree will be created +function setup_worktree() { + local repo base branch worktree_dir + repo=${1}; shift + base=${1}; shift + branch=${1}; shift + worktree_dir=${1}; shift + + add_cleanup \ + "git -C ${worktree_dir@Q} reset --hard HEAD" \ + "git -C ${worktree_dir@Q} clean -ffdx" \ + "git -C ${repo@Q} worktree remove ${worktree_dir@Q}" \ + "git -C ${repo@Q} branch -D ${branch@Q}" + + git -C "${repo}" worktree add -b "${branch}" "${worktree_dir}" "${base}" +} + +# Creates an initial globals file. It's initial because it contains +# data known up-front, so mostly things that are defined in one place +# to avoid repeating them everywhere. +# +# More stuff will be added later to the globals file based on config +# or else. +function setup_initial_globals_file() { + local sync_script pkg_list_sort_script + sync_script="${PKG_AUTO_IMPL_DIR}/sync_with_gentoo.sh" + pkg_list_sort_script="${PKG_AUTO_IMPL_DIR}/sort_packages_list.py" + + local globals_file + globals_file="${WORKDIR}/globals" + + local -a sigf_arches + get_valid_arches sigf_arches + + add_cleanup "rm -f ${globals_file@Q}" + cat <"${globals_file}" +local -a GIT_ENV_VARS ARCHES WHICH REPORTS +local SDK_PKGS BOARD_PKGS +local SYNC_SCRIPT PKG_LIST_SORT_SCRIPT + +GIT_ENV_VARS=( + GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} +) + +SYNC_SCRIPT=${sync_script@Q} +PKG_LIST_SORT_SCRIPT=${pkg_list_sort_script@Q} + +ARCHES=( ${sigf_arches[*]@Q} ) +WHICH=('old' 'new') +SDK_PKGS='sdk-pkgs' +BOARD_PKGS='board-pkgs' +REPORTS=( "\${SDK_PKGS}" "\${BOARD_PKGS}" ) +EOF +} + +# Extend the globals file with information from config and other +# information derived from it. +# +# Params: +# +# 1 - path to scripts repository +# 2 - path to scripts worktree with old state +# 3 - path to scripts worktree with new state +# 4 - path to reports directory +# 5 - path to aux directory +function extend_globals_file() { + local scripts old_state new_state reports_dir aux_dir + scripts=${1}; shift + old_state=${1}; shift + new_state=${1}; shift + reports_dir=${1}; shift + aux_dir=${1}; shift + + local globals_file + globals_file="${WORKDIR}/globals" + if [[ ! -e "${globals_file}" ]]; then + fail 'an initial version of globals file should already exist' + fi + + local old_state_branch new_state_branch + old_state_branch=$(git -C "${old_state}" rev-parse --abbrev-ref HEAD) + new_state_branch=$(git -C "${new_state}" rev-parse --abbrev-ref HEAD) + + local portage_stable_suffix old_portage_stable new_portage_stable + portage_stable_suffix='sdk_container/src/third_party/portage-stable' + old_portage_stable="${old_state}/${portage_stable_suffix}" + new_portage_stable="${new_state}/${portage_stable_suffix}" + + local coreos_overlay_suffix old_coreos_overlay new_coreos_overlay + coreos_overlay_suffix='sdk_container/src/third_party/coreos-overlay' + old_coreos_overlay="${old_state}/${coreos_overlay_suffix}" + new_coreos_overlay="${new_state}/${coreos_overlay_suffix}" + + cat <>"${globals_file}" + +local SCRIPTS OLD_STATE NEW_STATE OLD_STATE_BRANCH NEW_STATE_BRANCH +local PORTAGE_STABLE_SUFFIX OLD_PORTAGE_STABLE NEW_PORTAGE_STABLE REPORTS_DIR +local NEW_STATE_PACKAGES_LIST AUX_DIR +local COREOS_OVERLAY_SUFFIX OLD_COREOS_OVERLAY NEW_COREOS_OVERLAY + +SCRIPTS=${scripts@Q} +OLD_STATE=${old_state@Q} +NEW_STATE=${new_state@Q} +OLD_STATE_BRANCH=${old_state_branch@Q} +NEW_STATE_BRANCH=${new_state_branch@Q} +PORTAGE_STABLE_SUFFIX=${portage_stable_suffix@Q} +OLD_PORTAGE_STABLE=${old_portage_stable@Q} +NEW_PORTAGE_STABLE=${new_portage_stable@Q} +REPORTS_DIR=${reports_dir@Q} + +COREOS_OVERLAY_SUFFIX=${coreos_overlay_suffix@Q} +OLD_COREOS_OVERLAY=${old_coreos_overlay@Q} +NEW_COREOS_OVERLAY=${new_coreos_overlay@Q} + +NEW_STATE_PACKAGES_LIST="\${NEW_STATE}/.github/workflows/portage-stable-packages-list" + +AUX_DIR=${aux_dir@Q} +EOF + + # shellcheck disable=SC1090 # generated file + source "${globals_file}" + + local last_nightly_version_id last_nightly_build_id + # shellcheck disable=SC1091,SC2153 # sourcing generated file, NEW_STATE is not misspelled + last_nightly_version_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") + # shellcheck disable=SC1091 # sourcing generated file + last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") + + local -a locals definitions + locals=() + definitions=() + local sdk_image_name + sdk_image_name="ghcr.io/flatcar/flatcar-sdk-all:${last_nightly_version_id}-${last_nightly_build_id}" + local arch sdk_image_var_name + for arch in "${ARCHES[@]}"; do + sdk_image_var_name="${arch^^}_SDK_IMAGE" + locals+=( "${sdk_image_var_name@Q}" ) + definitions+=( "${sdk_image_var_name}=${sdk_image_name@Q}" ) + done + + append_to_globals \ + '' \ + "local ${locals[*]}" \ + '' \ + "${definitions[@]}" + + local -A listing_kinds + local packages_file tag filename stripped old + + for arch in "${ARCHES[@]}"; do + # shellcheck disable=SC2153 # AUX_DIR is not a misspelling, it comes from globals file + for packages_file in "${AUX_DIR}/${arch}/"*_packages.txt; do + filename=${packages_file##*/} + stripped=${filename%_packages.txt} + case ${stripped} in + 'flatcar_developer_container') + tag='dev' + ;; + 'flatcar_production_image') + tag='prod' + ;; + 'flatcar-'*) + tag="sysext-${stripped#flatcar-}" + ;; + 'oem-'*) + tag=${stripped#oem-} + ;; + *'-flatcar') + tag=${stripped%-flatcar} + ;; + *) + devel_warn "Unknown listing file ${packages_file@Q}" + continue + ;; + esac + old=${listing_kinds["${tag}"]:-} + if [[ -n ${old} ]]; then + if [[ ${old} != "${filename}" ]]; then + devel_warn "Two different packages files (${old@Q} and ${filename@Q} for a single tag ${tag@Q}" + fi + else + listing_kinds["${tag}"]=${filename} + fi + done + done + + local -a sorted_tags sorted_lines + mapfile -t sorted_tags < <(printf '%s\n' "${!listing_kinds[@]}" | sort) + for tag in "${sorted_tags[@]}"; do + filename=${listing_kinds["${tag}"]} + sorted_lines+=(" [${tag@Q}]=${filename@Q}") + done + + append_to_globals \ + '' \ + 'local -A LISTING_KINDS' \ + '' \ + 'LISTING_KINDS=(' \ + "${sorted_lines[@]}" \ + ')' +} + +# Sets up environment variables for some git commands. +# +# Make sure to call the following beforehand: +# +# local -x "${GIT_ENV_VARS[@]}" +# +# The GIT_ENV_VARS array comes from the globals file. +function setup_git_env() { + local bot_name bot_email role what + + # shellcheck disable=SC2034 # used indirectly + bot_name='Flatcar Buildbot' + # shellcheck disable=SC2034 # used indirectly + bot_email='buildbot@flatcar-linux.org' + for role in AUTHOR COMMITTER; do + for what in name email; do + local -n var="GIT_${role}_${what^^}" + local -n value="bot_${what}" + # shellcheck disable=SC2034 # it's a reference to external variable + var=${value} + unset -n value + unset -n var + done + done +} + +# Goes over the packages list and synces them with the passed Gentoo +# repo. +# +# Params: +# +# 1 - path to the Gentoo repo +function run_sync() { + local gentoo + gentoo=${1}; shift + + local -a missing_in_scripts missing_in_gentoo + missing_in_scripts=() + missing_in_gentoo=() + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -x "${GIT_ENV_VARS[@]}" + setup_git_env + + local -a packages_to_update + packages_to_update=() + + local package + while read -r package; do + # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file + if [[ ! -e "${NEW_PORTAGE_STABLE}/${package}" ]]; then + # If this happens, it means that the package was moved to overlay + # or dropped, the list ought to be updated. + missing_in_scripts+=("${package}") + continue + fi + if [[ ! -e "${gentoo}/${package}" ]]; then + # If this happens, it means that the package was obsoleted or moved + # in Gentoo. The obsoletion needs to be handled in the case-by-case + # manner, while move should be handled by doing the same move + # in portage-stable. The build should not break because of the move, + # because most likely it's already reflected in the profiles/updates + # directory. + missing_in_gentoo+=("${package}") + continue + fi + packages_to_update+=( "${package}" ) + done < <(cat_meaningful "${NEW_STATE_PACKAGES_LIST}") + # shellcheck disable=SC2153 # SYNC_SCRIPT is not a misspelling + env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${packages_to_update[@]}" + + save_missing_in_scripts "${missing_in_scripts[@]}" + save_missing_in_gentoo "${missing_in_gentoo[@]}" +} + +# A helper function that prints the contents of a file skipping empty +# lines and lines starting with a hash. +# +# Params: +# +# 1 - path to a file to print +function cat_meaningful() { + local file + file=${1}; shift + + xgrep '^[^#]' "${file}" +} + +# Saves a list of package names to a file and adds a cleanup for +# it. The names can be loaded again with load_simple_package_list. +# +# Params: +# +# 1 - path to a file where package names will be stored +# @ - the package names +function save_simple_package_list() { + local file + file=${1}; shift + # rest are packages + + add_cleanup "rm -f ${file@Q}" + if [[ ${#} -eq 0 ]]; then + truncate --size=0 "${file}" + else + printf '%s\n' "${@}" >"${file}" + fi +} + +# Loads a list of packages saved previously with +# save_simple_package_list. +# +# Params: +# +# 1 - path to a file where packages were stored +# 2 - name of an array variable; will contain package names +function load_simple_package_list() { + local file packages_var_name + file=${1}; shift + packages_var_name=${1}; shift + + mapfile -t "${packages_var_name}" <"${file}" +} + +# General function for saving missing packages. Takes care of creating +# a directory for the listing. +# +# Params: +# +# 1 - path to a directory which will contain the listing +# 2 - name of the listing file +function save_missing_packages() { + local dir file + dir=${1}; shift + file=${1}; shift + + create_cleanup_dir "${dir}" + save_simple_package_list "${dir}/${file}" "${@}" +} + +# Creates a directory and adds a cleanup if the directory was missing. +# +# Params: +# +# 1 - path to the directory +function create_cleanup_dir() { + local dir + dir=${1}; shift + if [[ ! -d "${dir}" ]]; then + add_cleanup "rmdir ${dir@Q}" + mkdir "${dir}" + fi +} + +# Saves a list of package names that we missing in scripts repo (which +# means that we were asked to sync a package that isn't in scripts to +# begin with). +# +# Params: +# +# @ - package names +function save_missing_in_scripts() { + save_missing_packages "${WORKDIR}/missing_in_scripts" "saved_list" "${@}" +} + +# Saves a list of package names that we missing in Gentoo repo (which +# means that we were asked to sync a possibly obsolete or renamed +# package). +# +# Params: +# +# @ - package names +function save_missing_in_gentoo() { + save_missing_packages "${WORKDIR}/missing_in_gentoo" "saved_list" "${@}" +} + +# Loads a list of package names that were missing in scripts repo. +# +# Params: +# +# 1 - name of an array variable; will contain package names +function load_missing_in_scripts() { + local packages_var_name + packages_var_name=${1}; shift + + load_simple_package_list "${WORKDIR}/missing_in_scripts/saved_list" "${packages_var_name}" +} + +# Loads a list of package names that were missing in Gentoo repo. +# +# Params: +# +# 1 - name of an array variable; will contain package names +function load_missing_in_gentoo() { + local packages_var_name + packages_var_name=${1}; shift + + load_simple_package_list "${WORKDIR}/missing_in_gentoo/saved_list" "${packages_var_name}" +} + +# Handles package names that were missing in scripts by dropping them +# from the listing of packages that should be updated automatically. +function handle_missing_in_scripts() { + local -a hmis_missing_in_scripts + hmis_missing_in_scripts=() + load_missing_in_scripts hmis_missing_in_scripts + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + if [[ ${#hmis_missing_in_scripts[@]} -eq 0 ]]; then + return 0; + fi + + # Remove missing in scripts entries from package automation + local dir + dir="${WORKDIR}/missing_in_scripts" + create_cleanup_dir "${dir}" + local missing_re + join_by missing_re '\|' "${missing_in_scripts[@]}" + add_cleanup "rm -f ${dir@Q}/pkg_list" + xgrep --invert-match --line-regexp --fixed-strings --regexp="${missing_re}" "${NEW_STATE_PACKAGES_LIST}" >"${dir}/pkg_list" + # shellcheck disable=SC2153 # PKG_LIST_SORT_SCRIPT is not a misspelling + "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" + + local -x "${GIT_ENV_VARS[@]}" + setup_git_env + + git -C "${NEW_STATE}" add "${NEW_STATE_PACKAGES_LIST}" + git -C "${NEW_STATE}" commit --quiet --message '.github: Drop missing packages from automation' + info_lines 'dropped missing packages from automation' "${missing_in_scripts[@]/#/- }" +} + +# Helper function to print lines to a file clobbering the old +# contents. +# +# Params: +# +# 1 - path to the file +# @ - lines to print +function lines_to_file_truncate() { + truncate --size=0 "${1}" + lines_to_file "${@}" +} + +# Helper function to append lines to a file. +# +# Params: +# +# 1 - path to the file +# @ - lines to print +function lines_to_file() { + printf '%s\n' "${@:2}" >>"${1}" +} + +# Adds lines to "manual work needed" file in reports. +# +# Params: +# +# @ - lines to add +function manual() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + pkg_debug_lines 'manual work needed:' "${@}" + # shellcheck disable=SC2153 # REPORTS_DIR is not a misspelling, it comes from globals file + lines_to_file "${REPORTS_DIR}/manual-work-needed" "${@}" +} + +# Adds lines to "warnings" file in reports. Should be used to report +# some issues with the processed packages. +# +# Params: +# +# @ - lines to add +function pkg_warn() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + pkg_debug_lines 'pkg warn:' "${@}" + lines_to_file "${REPORTS_DIR}/warnings" "${@}" +} + +# Adds lines to "developer warnings" file in reports. Should be used +# to report some failed assumption in the automation, or bugs. +# +# Params: +# +# @ - lines to add +function devel_warn() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + pkg_debug_lines 'developer warn:' "${@}" + lines_to_file "${REPORTS_DIR}/developer-warnings" "${@}" +} + +# Handles package names that were missing from Gentoo by either +# renaming and syncing them if a rename exists or by adding the +# package to the "manual work needed" file. +function handle_missing_in_gentoo() { + local gentoo + gentoo=${1}; shift + + local -a hmig_missing_in_gentoo + hmig_missing_in_gentoo=() + load_missing_in_gentoo hmig_missing_in_gentoo + + if [[ ${#hmig_missing_in_gentoo[@]} -eq 0 ]]; then + return 0; + fi + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -A hmig_rename_map=() + process_profile_updates_directory hmig_rename_map + + local -a renamed_from renamed_to + renamed_from=() + renamed_to=() + + local -x "${GIT_ENV_VARS[@]}" + setup_git_env + + local missing new_name hmig_old_basename hmig_new_basename ebuild ebuild_version_ext new_ebuild_filename + for missing in "${hmig_missing_in_gentoo[@]}"; do + new_name=${hmig_rename_map["${missing}"]:-} + if [[ -z "${new_name}" ]]; then + manual "- package ${missing} is gone from Gentoo and no rename found" + continue + fi + mkdir -p "${NEW_PORTAGE_STABLE}/${new_name%/*}" + git -C "${NEW_STATE}" mv "${NEW_PORTAGE_STABLE}/${missing}" "${NEW_PORTAGE_STABLE}/${new_name}" + basename_out "${missing}" hmig_old_basename + basename_out "${new_name}" hmig_new_basename + if [[ "${hmig_old_basename}" != "${hmig_new_basename}" ]]; then + for ebuild in "${NEW_PORTAGE_STABLE}/${new_name}/${hmig_old_basename}-"*'.ebuild'; do + # 1.2.3-r4.ebuild + ebuild_version_ext=${ebuild##*/"${hmig_old_basename}-"} + new_ebuild_filename="${hmig_new_basename}-${ebuild_version_ext}" + git -C "${NEW_STATE}" mv "${ebuild}" "${NEW_PORTAGE_STABLE}/${new_name}/${new_ebuild_filename}" + done + fi + git -C "${NEW_STATE}" commit --quiet --message "${new_name}: Renamed from ${missing}" + info "renamed ${missing} to ${new_name}" + renamed_from+=("${missing}") + renamed_to+=("${new_name}") + done + + if [[ ${#renamed_from[@]} -eq 0 ]]; then + return 0 + fi + + env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${renamed_to[@]}" + + local dir renamed_re + dir="${WORKDIR}/missing_in_gentoo" + create_cleanup_dir "${dir}" + join_by renamed_re '\|' "${renamed_from[@]}" + add_cleanup "rm -f ${dir@Q}/pkg_list" + { + xgrep --invert-match --line-regexp --regexp="${renamed_re}" "${NEW_STATE_PACKAGES_LIST}" + printf '%s\n' "${renamed_to[@]}" + } >"${dir}/pkg_list" + "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" + git -C "${NEW_STATE}" add "${NEW_STATE_PACKAGES_LIST}" + git -C "${NEW_STATE}" commit --quiet --message '.github: Update package names in automation' + info 'updated packages names in automation' +} + +# Process the package listings stored in the aux directory to find out +# the package tags that describe the kind of image the package is used +# in (base image, developer container, sysext image, etc.) +# +# Params: +# +# 1 - a name to an array mvm variable; will be a mapping of a package +# name to an array of tags +function process_listings() { + local pkg_to_tags_mvm_var_name + pkg_to_tags_mvm_var_name=${1} + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local eclass ver_ere pkg_ere + # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling + eclass="${PKG_AUTO_DIR}/../${PORTAGE_STABLE_SUFFIX}/eclass/eapi7-ver.eclass" + # line is like ' re=""' + ver_ere=$(grep -e 're=' "${eclass}" || fail "no 're=' line found in eapi7-ver.eclass") + if [[ -z ${ver_ere} ]]; then + fail 'empty version regex from eapi7-ver.eclass' + fi + # strip everything until first quotes + ver_ere=${ver_ere#*'"'} + # strip last quote + ver_ere=${ver_ere%'"'*} + # regexp begins with ^ and ends with $, so strip them too + ver_ere=${ver_ere#'^'} + ver_ere=${ver_ere%'$'} + pkg_ere='[a-z0-9]*-?[a-z0-9]*/[a-z0-9A-Z_+-]*' + + #mvm_debug_enable pl_pkg_to_tags_set_mvm + mvm_declare pl_pkg_to_tags_set_mvm mvm_mvc_set + + local arch kind file listing pkg + for arch in "${ARCHES[@]}"; do + # shellcheck disable=SC2153 # LISTING_KINDS is not a misspelling, it comes from globals file + for kind in "${!LISTING_KINDS[@]}"; do + file=${LISTING_KINDS["${kind}"]} + listing="${AUX_DIR}/${arch}/${file}" + if [[ ! -e "${listing}" ]]; then + # some listings are arch-specific, so they will be + # missing for other arches + continue + fi + # lines are like as follows: + # + # acct-group/adm-0-r2::portage-stable + while read -r pkg; do + pkg_debug_enable "${pkg}" + pkg_debug "processing listings: adding tag ${kind^^}" + pkg_debug_disable + mvm_add pl_pkg_to_tags_set_mvm "${pkg}" "${kind^^}" + done < <(sed -E -e 's#^('"${pkg_ere}"')-'"${ver_ere}"'::.*#\1#' "${listing}") + done + done + + mvm_iterate pl_pkg_to_tags_set_mvm set_mvm_to_array_mvm_cb "${pkg_to_tags_mvm_var_name}" + mvm_unset pl_pkg_to_tags_set_mvm + #mvm_debug_disable pl_pkg_to_tags_set_mvm + if pkg_debug_possible; then + mvm_iterate "${pkg_to_tags_mvm_var_name}" debug_dump_package_tags "${pkg_to_tags_mvm_var_name}" + fi +} + +# A debug function that prints the package tags. Used as a callback to +# mvm_iterate. +# +# Params: +# +# 1 - name of the array mvm variable (extra arg of the callback) +# 2 - name of the package +# 3 - name of the array variable holding tags (unused) +# @ - tags +function debug_dump_package_tags() { + local map_name=${1}; shift + local pkg=${1}; shift + shift # we don't care about array variable name + # rest are array elements, which are tags + pkg_debug_enable "${pkg}" + pkg_debug "tags for ${pkg} stored in ${map_name}: ${*}" + pkg_debug_disable +} + +# A callback to mvm_iterate that turns a set mvm to an array mvm. It +# makes sure that the tag for the production image (or base image) is +# always first in the array. +# +# Params: +# +# 1 - name of the array mvm variable that will be filled (extra arg of +# the callback) +# 2 - name of the package +# 3 - name of the set variable holding tags +# @ - tags +function set_mvm_to_array_mvm_cb() { + local pkg_to_tags_mvm_var_name pkg + pkg_to_tags_mvm_var_name=${1}; shift + pkg=${1}; shift + local -n set_ref=${1}; shift + # rest are set items + + local removed + removed='' + local -a prod_item + prod_item=() + if [[ -n ${set_ref['PROD']:-} ]]; then + prod_item+=('PROD') + unset "set_ref['PROD']" + removed=x + fi + local -a sorted_items + mapfile -t sorted_items < <(printf '%s\n' "${!set_ref[@]}" | sort) + if [[ -n ${removed} ]]; then + set_ref['PROD']=x + fi + + mvm_add "${pkg_to_tags_mvm_var_name}" "${pkg}" "${prod_item[@]}" "${sorted_items[@]}" +} + +# Generate package reports inside SDKs for all arches and states. In +# case of failure, whatever reports where generated so far will be +# stored in salvaged-reports subdirectory of the reports directory. +# Otherwise they will end up in reports-from-sdk subdirectory. +function generate_sdk_reports() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" + mkdir "${WORKDIR}/pkg-reports" + + local arch sdk_image_var_name sdk_image_name + local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch + local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy + local -a report_files run_sdk_container_args + for arch in "${ARCHES[@]}"; do + sdk_image_var_name="${arch^^}_SDK_IMAGE" + sdk_image_name=${!sdk_image_var_name} + if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${sdk_image_name}"; then + fail "No SDK image named ${sdk_image_name@Q} available locally, pull it before running this script" + fi + + # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file + for sdk_run_kind in "${WHICH[@]}"; do + state_var_name="${sdk_run_kind^^}_STATE" + sdk_run_state="${!state_var_name}_sdk_run_${arch}" + state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" + sdk_run_state_branch="${!state_branch_var_name}-sdk-run-${arch}" + + add_cleanup \ + "git -C ${sdk_run_state@Q} reset --hard HEAD" \ + "git -C ${sdk_run_state@Q} clean -ffdx" \ + "git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ + "git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" + git -C "${SCRIPTS}" \ + worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}" + + pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX") + add_cleanup "rm -rf ${pkg_auto_copy@Q}" + cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}" + local -a run_sdk_container_args=( + -C "${sdk_image_name}" + -n "pkg-${sdk_run_kind}-${arch}" + -a "${arch}" + -U + -m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto" + --rm + ./pkg_auto/inside_sdk_container.sh "${arch}" pkg-reports + ) + rv=0 + env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?} + if [[ ${rv} -ne 0 ]]; then + { + salvaged_dir="${REPORTS_DIR}/salvaged-reports" + info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue" + info + for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do + info "from ${file}:" + echo + cat "${file}" + echo + done + info + info 'whatever reports generated by the failed run are saved in' + info "${salvaged_dir@Q} directory" + info + } >&2 + rm -rf "${salvaged_dir}" + cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}" + fail "copying done, stopping now" + fi + sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}-${arch}" + report_files=() + for full_file in "${sdk_run_state}/pkg-reports/"*; do + file=${full_file##"${sdk_run_state}/pkg-reports/"} + report_files+=( "${sdk_reports_dir}/${file}" ) + done + add_cleanup \ + "rm -f ${report_files[*]@Q}" \ + "rmdir ${sdk_reports_dir@Q}" + mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}" + done + done + cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk" +} + +source "${PKG_AUTO_IMPL_DIR}/mvm.sh" + +# pkginfo mvm is a map mvm that has the following Go-like type: +# +# map[pkg]map[slot]version +# +# pkg, slot and version are strings + +# Generate a name for pkginfo mvm based on passed information. +# +# Params: +# +# 1 - which state it refers to (old or new) +# 2 - architecture +# 3 - which report (board packages or SDK packages) +# 4 - name of a variable that will contain the name +function pkginfo_name() { + local which arch report + + which=${1}; shift + arch=${1}; shift + report=${1}; shift + local -n pi_name_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + pi_name_ref="pkginfo_${which}_${arch}_${report//-/_}_pimap_mvm" +} + +# Constructor callback used by mvm_declare for pkginfo mvms. +function pkginfo_constructor() { + mvm_mvc_map_constructor "${@}" +} + +# Destructor callback used by mvm_declare for pkginfo mvms. +function pkginfo_destructor() { + mvm_mvc_map_destructor "${@}" +} + +# Adder callback used by mvm_declare for pkginfo mvms. +function pkginfo_adder() { + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n map_ref=${1}; shift + + local mark + while [[ ${#} -gt 1 ]]; do + mark=${map_ref["${1}"]:-} + if [[ -n "${mark}" ]]; then + fail "multiple versions for a single slot for a package in a single report" + fi + map_ref["${1}"]=${2} + shift 2 + done +} + +# Creates a pkginfo mvm. +# +# Params: +# +# 1 - which state it refers to (old or new) +# 2 - architecture +# 3 - which report (board packages or SDK packages) +# 4 - name of a variable that will contain the name of the created +# pkginfo mvm +function pkginfo_declare() { + local which arch report pi_name_var_name + which=${1}; shift + arch=${1}; shift + report=${1}; shift + pi_name_var_name=${1}; shift + + pkginfo_name "${which}" "${arch}" "${report}" "${pi_name_var_name}" + + local -a extras + extras=( + 'which' "${which}" + 'arch' "${arch}" + 'report' "${report}" + ) + + mvm_declare "${!pi_name_var_name}" pkginfo -- "${extras[@]}" +} + +# Destroys a pkginfo mvm. +# +# Params: +# +# 1 - which state it refers to (old or new) +# 2 - architecture +# 3 - which report (board packages or SDK packages) +function pkginfo_unset() { + local which arch report + which=${1}; shift + arch=${1}; shift + report=${1}; shift + + local piu_pi_name + pkginfo_name "${which}" "${arch}" "${report}" piu_pi_name + + mvm_unset "${piu_pi_name}" +} + +# Processes the report file associated to the passed pkginfo mvm. The +# pkginfo mvm is filled with info about packages, slots and +# versions. Additional information is put into passed package set and +# package to slots set mvm. +# +# Params: +# +# 1 - name of the pkginfo mvm variable +# 2 - name of the set variable, will contain packages in the report +# 3 - name of the set mvm variable, will contain a map of package to +# slots +function pkginfo_process_file() { + mvm_call "${1}" pkginfo_c_process_file "${@:2}" +} + +# Helper function for pkginfo_process_file, used by mvm_call. +function pkginfo_c_process_file() { + local pkg_slots_set_mvm_var_name + local -n pkg_set_ref=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local which arch report + mvm_c_get_extra 'which' which + mvm_c_get_extra 'arch' arch + mvm_c_get_extra 'report' report + + local pkg version_slot throw_away v s + # shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else + while read -r pkg version_slot throw_away; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} ${arch} ${report}: ${version_slot}" + v=${version_slot%%:*} + s=${version_slot##*:} + mvm_c_add "${pkg}" "${s}" "${v}" + # shellcheck disable=SC2034 # it's a reference to external variable + pkg_set_ref["${pkg}"]='x' + mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}" + pkg_debug_disable + done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" +} + +# Gets a profile of the pkginfo mvm. The "profile" is a confusing +# misnomer as it has nothing to do with Gentoo profiles, but rather a +# description of the pkginfo (which is a which-arch-report triplet) +# that is used for reporting. +function pkginfo_profile() { + mvm_call "${1}" pkginfo_c_profile "${@:2}" +} + +# Helper function for pkginfo_profile, used by mvm_call. +function pkginfo_c_profile() { + local profile_var_name + profile_var_name=${1}; shift + + local which arch report + mvm_c_get_extra 'which' which + mvm_c_get_extra 'arch' arch + mvm_c_get_extra 'report' report + + printf -v "${profile_var_name}" '%s-%s-%s' "${which}" "${arch}" "${report}" +} + +# Creates pkginfo maps for all the reports and processes the +# reports. Additional information is stored in passed packages array +# and packages to slots set mvm variables. +# +# Params: +# +# 1 - name of the array variable, will contain sorted packages from +# all the reports +# 2 - name of the set mvm variable, will contain a map of package to +# slots +function read_reports() { + local all_pkgs_var_name pkg_slots_set_mvm_var_name + all_pkgs_var_name=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -A rr_all_packages_set + rr_all_packages_set=() + + local arch which report rr_pimap_mvm_var_name + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in "${REPORTS[@]}"; do + pkginfo_declare "${which}" "${arch}" "${report}" rr_pimap_mvm_var_name + pkginfo_process_file "${rr_pimap_mvm_var_name}" rr_all_packages_set "${pkg_slots_set_mvm_var_name}" + done + done + done + + mapfile -t "${all_pkgs_var_name}" < <(printf '%s\n' "${!rr_all_packages_set[@]}" | sort) +} + +# Destroys the pkginfo maps for all the reports. +function unset_report_mvms() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local arch which report + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in "${REPORTS[@]}"; do + pkginfo_unset "${which}" "${arch}" "${report}" + done + done + done +} + +### +### BEGIN GENTOO VER COMP HACKS +### + +# shellcheck disable=SC2034 # it's here only for the eapi7-ver.eclass +EAPI=6 +function die() { + fail "$*" +} + +# This brings in ver_test function. +# +# shellcheck disable=SC1091 # sourcing external file +source "${PKG_AUTO_DIR}/../sdk_container/src/third_party/portage-stable/eclass/eapi7-ver.eclass" + +unset EAPI + +# symbolic names for use with gentoo_ver_cmp +GV_LT=1 +GV_EQ=2 +GV_GT=3 + +# Compare two versions. The result can be compared against GV_LT, GV_EQ and GV_GT variables. +# +# Params: +# +# 1 - version 1 +# 2 - version 2 +# 3 - name of variable to store the result in +function gentoo_ver_cmp_out() { + local v1 v2 + v1=${1}; shift + v2=${1}; shift + local -n out_ref=${1}; shift + + out_ref=0 + _ver_compare "${v1}" "${v2}" || out_ref=${?} + case ${out_ref} in + 1|2|3) + return 0 + ;; + *) + fail "unexpected return value ${out_ref} from _ver_compare for ${v1} and ${v2}" + ;; + esac +} + +### +### END GENTOO VER COMP HACKS +### + +# Finds out the highest and the lowest version from the passed versions. +# +# Params: +# +# 1 - name of a variable where the min version will be stored +# 2 - name of a variable where the min version will be stored +# @ - the versions +function ver_min_max() { + local -n min_ref=${1}; shift + local -n max_ref=${1}; shift + + local min max v + min='' + max='' + for v; do + if [[ -z ${min} ]] || ver_test "${v}" -lt "${min}"; then + min=${v} + fi + if [[ -z ${max} ]] || ver_test "${v}" -gt "${max}"; then + max=${v} + fi + done + # shellcheck disable=SC2034 # it's a reference to external variable + min_ref=${min} + # shellcheck disable=SC2034 # it's a reference to external variable + max_ref=${max} +} + +# Does consistency checks on two profiles for a package using an +# additional map for slots information for a package. Checks if common +# slots for the package in both profiles are using the same +# versions. This is to catch version discrepancies that sometimes +# happen when e.g. a package gets stabilized for one arch, but not for +# the other. +# +# While at it, store package, slot and version range information into +# the passed map. +# +# 1 - package +# 2 - name of the pkginfo mvm for profile 1 +# 3 - name of the pkginfo mvm for profile 2 +# 4 - name of the pkg to slots to version range map mvm # TODO: This should be the last parameter +# 5 - name of the pkg to all slots set mvm +function consistency_check_for_package() { + local pkg pi1_pimap_mvm_var_name pi2_pimap_mvm_var_name pkg_slot_verminmax_map_mvm_var_name pkg_slots_set_mvm_var_name + pkg=${1}; shift + pi1_pimap_mvm_var_name=${1}; shift + pi2_pimap_mvm_var_name=${1}; shift + pkg_slot_verminmax_map_mvm_var_name=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + + local ccfp_slot_version1_map_var_name ccfp_slot_version2_map_var_name + mvm_get "${pi1_pimap_mvm_var_name}" "${pkg}" ccfp_slot_version1_map_var_name + mvm_get "${pi2_pimap_mvm_var_name}" "${pkg}" ccfp_slot_version2_map_var_name + + local -A empty_map + empty_map=() + + local -n slot_version1_map=${ccfp_slot_version1_map_var_name:-empty_map} + local -n slot_version2_map=${ccfp_slot_version2_map_var_name:-empty_map} + + local ccfp_slots_set_var_name + mvm_get "${pkg_slots_set_mvm_var_name}" "${pkg}" ccfp_slots_set_var_name + local -n slots_set_ref=${ccfp_slots_set_var_name} + + local -a profile_1_slots profile_2_slots common_slots + profile_1_slots=() + profile_2_slots=() + common_slots=() + + local ccfp_profile_1 ccfp_profile_2 + pkginfo_profile "${pi1_pimap_mvm_var_name}" ccfp_profile_1 + pkginfo_profile "${pi2_pimap_mvm_var_name}" ccfp_profile_2 + + local s v1 v2 ccfp_min ccfp_max mm + pkg_debug "all slots iterated over: ${!slots_set_ref[*]}" + for s in "${!slots_set_ref[@]}"; do + v1=${slot_version1_map["${s}"]:-} + v2=${slot_version2_map["${s}"]:-} + pkg_debug "v1: ${v1}, v2: ${v2}" + + if [[ -n ${v1} ]] && [[ -n ${v2} ]]; then + pkg_debug "${s} is a common slot for ${ccfp_profile_1} and ${ccfp_profile_2}" + common_slots+=( "${s}" ) + if [[ ${v1} != "${v2}" ]]; then + pkg_warn \ + "- version mismatch:" \ + " - package: ${pkg}" \ + " - slot: ${s}" \ + " - profile 1: ${ccfp_profile_1}" \ + " - version: ${v1}" \ + " - profile 1: ${ccfp_profile_2}" \ + " - version: ${v2}" + fi + ver_min_max ccfp_min ccfp_max "${v1}" "${v2}" + mm="${ccfp_min}:${ccfp_max}" + elif [[ -n ${v1} ]]; then + # only side1 has the slot + pkg_debug "${s} is a slot only in ${ccfp_profile_1}" + profile_1_slots+=( "${s}" ) + mm="${v1}:${v1}" + elif [[ -n ${v2} ]]; then + # only side 2 has the slot + pkg_debug "${s} is a slot only in ${ccfp_profile_2}" + profile_2_slots+=( "${s}" ) + mm="${v2}:${v2}" + else + pkg_debug "${s} is a slot absent from both ${ccfp_profile_1} and ${ccfp_profile_2}" + continue + fi + + mvm_add "${pkg_slot_verminmax_map_mvm_var_name}" "${pkg}" "${s}" "${mm}" + done + pkg_debug "common slots: ${common_slots[*]}" + pkg_debug "profile 1 slots: ${profile_1_slots[*]}" + pkg_debug "profile 2 slots: ${profile_2_slots[*]}" + local s1 s2 + if [[ ${#common_slots[@]} -gt 0 ]]; then + if [[ ${#profile_1_slots[@]} -gt 0 ]] || [[ ${#profile_2_slots[@]} -gt 0 ]]; then + pkg_warn \ + "- suspicious:" \ + " - package: ${pkg}" \ + " - profile 1: ${ccfp_profile_1}" \ + " - profile 2: ${ccfp_profile_2}" \ + " - common slots: ${common_slots[*]}" \ + " - slots only in profile 1: ${profile_1_slots[*]}" \ + " - slots only in profile 2: ${profile_2_slots[*]}" \ + " - what: there are slots that exist only on one profile while both profiles also have some common slots" + fi + elif [[ ${#profile_1_slots[@]} -eq 1 ]] && [[ ${#profile_2_slots[@]} -eq 1 ]]; then + s1=${profile_1_slots[0]} + s2=${profile_2_slots[0]} + v1=${slot_version1_map["${s1}"]:-} + v2=${slot_version2_map["${s2}"]:-} + if [[ ${v1} != "${v2}" ]]; then + pkg_warn \ + "- version mismatch:" \ + " - package ${pkg}" \ + " - profile 1: ${ccfp_profile_1}" \ + " - slot: ${profile_1_slots[0]}" \ + " - version: ${v1}" \ + " - profile 2: ${ccfp_profile_2}" \ + " - slot: ${profile_2_slots[0]}" \ + " - version: ${v2}" + fi + fi +} + +# Do consistency checks for the following pairs of profiles for a passed state: +# +# ${arch} sdk <-> ${arch} board +# ${arch1} board <-> ${arch2} board +# ${arch1} sdk <-> ${arch2} sdk +# +# While at it, store package, slot and version range information into +# the passed map. +# +# Params: +# +# 1 - which state should be checked (old or new) +# 2 - name of an array variable that contains all the package names +# 3 - name of the pkg to all slots set mvm variable +# 4 - name of the pkg to slot to version range map mvm variable +function consistency_checks() { + local which pkg_slots_set_mvm_var_name pkg_slot_verminmax_mvm_var_name + which=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n all_pkgs_ref=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + pkg_slot_verminmax_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local cc_pimap_mvm_1_var_name cc_pimap_mvm_2_var_name pkg + local -a all_mvm_names=() + local arch name + # ${arch} SDK <-> ${arch} board + for arch in "${ARCHES[@]}"; do + # currently we only have amd64 SDK, so skip others + if [[ ${arch} != 'amd64' ]]; then + continue + fi + pkginfo_name "${which}" "${arch}" "${SDK_PKGS}" cc_pimap_mvm_1_var_name + pkginfo_name "${which}" "${arch}" "${BOARD_PKGS}" cc_pimap_mvm_2_var_name + name="cc_${arch}_sdk_board_pkg_slot_verminmax_map_mvm" + all_mvm_names+=( "${name}" ) + mvm_declare "${name}" mvm_mvc_map + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} ${arch} sdk <-> ${arch} board" + consistency_check_for_package "${pkg}" "${cc_pimap_mvm_1_var_name}" "${cc_pimap_mvm_2_var_name}" "${name}" "${pkg_slots_set_mvm_var_name}" + pkg_debug_disable + done + done + + # We want to check consistency between each pair of arches. + local -a cc_all_arch_pairs + all_pairs cc_all_arch_pairs ':' "${ARCHES[@]}" + + local pair arch1 arch2 + # ${arch1} board <-> ${arch2} board + for pair in "${cc_all_arch_pairs[@]}"; do + arch1=${pair%:*} + arch2=${pair#*:} + pkginfo_name "${which}" "${arch1}" "${BOARD_PKGS}" cc_pimap_mvm_1_var_name + pkginfo_name "${which}" "${arch2}" "${BOARD_PKGS}" cc_pimap_mvm_2_var_name + name="cc_${arch1}_${arch2}_board_pkg_slot_verminmax_map_mvm" + all_mvm_names+=( "${name}" ) + mvm_declare "${name}" mvm_mvc_map + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} ${arch1} board <-> ${arch2} board" + consistency_check_for_package "${pkg}" "${cc_pimap_mvm_1_var_name}" "${cc_pimap_mvm_2_var_name}" "${name}" "${pkg_slots_set_mvm_var_name}" + pkg_debug_disable + done + done + + # ${arch1} sdk <-> ${arch2} sdk + for pair in "${cc_all_arch_pairs[@]}"; do + arch1=${pair%:*} + arch2=${pair#*:} + # We currently only have amd64 SDK, so this loop will + # effectively iterate zero times. When we get the arm64 SDK + # too, this if could be dropped. Getting the listing of arm64 + # packages inside amd64 SDK is going to be problem to solve, + # though. + if [[ ${arch1} != 'amd64' ]] || [[ ${arch2} != 'amd64' ]]; then + continue + fi + pkginfo_name "${which}" "${arch1}" "${SDK_PKGS}" cc_pimap_mvm_1_var_name + pkginfo_name "${which}" "${arch2}" "${SDK_PKGS}" cc_pimap_mvm_2_var_name + name="cc_${arch1}_${arch2}_sdk_pkg_slot_verminmax_map_mvm" + all_mvm_names+=( "${name}" ) + mvm_declare "${name}" mvm_mvc_map + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} ${arch1} sdk <-> ${arch2} sdk" + consistency_check_for_package "${pkg}" "${cc_pimap_mvm_1_var_name}" "${cc_pimap_mvm_2_var_name}" "${name}" "${pkg_slots_set_mvm_var_name}" + pkg_debug_disable + done + done + + local cc_slots_set_var_name s cc_min cc_max verminmax + # shellcheck disable=SC2034 # used by name below + local -A empty_map=() + local -a verminmax_map_var_names verminmaxes + local cc_slot_verminmax_map_var_name + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} verminmax stuff" + verminmax_map_var_names=() + for name in "${all_mvm_names[@]}"; do + mvm_get "${name}" "${pkg}" cc_slot_verminmax_map_var_name + verminmax_map_var_names+=("${cc_slot_verminmax_map_var_name}") + done + if pkg_debug_possible; then + for name in "${verminmax_map_var_names[@]}"; do + local -n slot_verminmax_map_ref=${name:-empty_map} + pkg_debug "all slots in ${name}: ${!slot_verminmax_map_ref[*]}" + pkg_debug "all vmms in ${name}: ${slot_verminmax_map_ref[*]}" + unset -n slot_verminmax_map_ref + done + fi + mvm_get "${pkg_slots_set_mvm_var_name}" "${pkg}" cc_slots_set_var_name + local -n slots_set_ref=${cc_slots_set_var_name} + pkg_debug "all slots iterated over: ${!slots_set_ref[*]}" + for s in "${!slots_set_ref[@]}"; do + verminmaxes=() + for name in "${verminmax_map_var_names[@]}"; do + local -n slot_verminmax_map_ref=${name:-empty_map} + verminmax=${slot_verminmax_map_ref["${s}"]:-} + if [[ -n ${verminmax} ]]; then + verminmaxes+=( "${verminmax}" ) + fi + unset -n slot_verminmax_map_ref + done + if [[ ${#verminmaxes[@]} -gt 1 ]]; then + ver_min_max cc_min cc_max "${verminmaxes[@]%%:*}" "${verminmaxes[@]##*:}" + verminmax="${cc_min}:${cc_max}" + elif [[ ${#verminmaxes[@]} -eq 1 ]]; then + verminmax=${verminmaxes[0]} + else + continue + fi + pkg_debug "adding vmm ${verminmax} for slot ${s}" + mvm_add "${pkg_slot_verminmax_mvm_var_name}" "${pkg}" "${s}" "${verminmax}" + done + unset -n slots_set_ref + pkg_debug_disable + done + for name in "${all_mvm_names[@]}"; do + mvm_unset "${name}" + done +} + +# Read a report describing from which repo the package came and store +# in the passed map. +# +# Params: +# +# 1 - name of a map variable, will contain a mapping of package name +# to repository name +function read_package_sources() { + local -n package_sources_map_ref=${1}; shift + + local arch which report pkg repo saved_repo + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in sdk-package-repos board-package-repos; do + while read -r pkg repo; do + saved_repo=${package_sources_map_ref["${pkg}"]:-} + if [[ -n ${saved_repo} ]]; then + if [[ ${saved_repo} != "${repo}" ]]; then + pkg_warn \ + '- different repos used for the package:' \ + " - package: ${pkg}" \ + ' - repos:' \ + " - ${saved_repo}" \ + " - ${repo}" + fi + else + package_sources_map_ref["${pkg}"]=${repo} + fi + done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" + done + done + done +} + +# This monstrosity takes renames map and package tags information, +# reads the reports, does consistency checks and uses the information +# from previous steps to write out package differences between the old +# and new state into the reports directory. +# +# Params: +# +# 1 - name of the renames map variable +# 2 - name of the package tags map mvm variable +function handle_package_changes() { + local pkg_to_tags_mvm_var_name + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n renamed_old_to_new_map_ref=${1}; shift + pkg_to_tags_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a hpc_all_pkgs + hpc_all_pkgs=() + + # map[package]map[slot]interface{} + mvm_declare hpc_pkg_slots_set_mvm mvm_mvc_set + read_reports hpc_all_pkgs hpc_pkg_slots_set_mvm + + # map[package]map[slot]string (string being "min version:max version") + mvm_declare hpc_old_pkg_slot_verminmax_map_mvm mvm_mvc_map + mvm_declare hpc_new_pkg_slot_verminmax_map_mvm mvm_mvc_map + consistency_checks old hpc_all_pkgs hpc_pkg_slots_set_mvm hpc_old_pkg_slot_verminmax_map_mvm + consistency_checks new hpc_all_pkgs hpc_pkg_slots_set_mvm hpc_new_pkg_slot_verminmax_map_mvm + + unset_report_mvms + + # TODO: when we handle moving packages between repos, then there + # should be two maps, for old and new state + local -A hpc_package_sources_map + hpc_package_sources_map=() + read_package_sources hpc_package_sources_map + + mkdir -p "${REPORTS_DIR}/updates" + + local -a old_pkgs new_pkgs + old_pkgs=() + new_pkgs=() + + # The following loop fills the old_pkgs and new_pkgs arrays sorted + # package names, where old package name at index I has it's new + # counterpart at the same index. For the most part, both old and + # new names will be the same, since the renames are rather rare. + + # map[package]index + local -A added_pkg_to_index_map=() + local pkg other + for pkg in "${hpc_all_pkgs[@]}"; do + other=${renamed_old_to_new_map_ref["${pkg}"]:-} + if [[ -n "${other}" ]]; then + # There seem be a rename from ${pkg} to ${other} + pkg_debug_enable "${pkg}" "${other}" + pkg_debug "${pkg} renamed to ${other}" + pkg_debug_disable + local other_idx + other_idx=${added_pkg_to_index_map["${other}"]:-} + if [[ -n ${other_idx} ]]; then + # Looks like we have already processed the ${other} + # name. In this case, both old_pkgs[${other_idx}] and + # new_pkgs[${other_idx}] should just be + # ${other}. Since ${pkg} is the old name (${other} is + # new), we update old_pkgs to hold the old name. Just + # make sure that old_pkgs indeed had the new name + # first. + local other_old + other_old=${old_pkgs["${other_idx}"]} + if [[ ${other_old} = "${other}" ]]; then + old_pkgs["${other_idx}"]=${pkg} + else + manual \ + '- there seem to be two old packages in our repos that are supposed to be renamed to the same name:' \ + " - old package 1: ${pkg}" \ + " - old package 2: ${other_old}" \ + " - new package: ${other}" + fi + unset other_idx other_old + continue + else + unset other_idx + fi + + # Looks like we haven't processed the ${other} name yet, + # it probably will come up later, which will be taken care + # of by the "pkg_debug 'handled already through some + # rename'" part below, after else. + + local pkg_idx + # doesn't matter if it's length of new_pkgs or old_pkgs, + # both are assumed to have the same length + pkg_idx=${#old_pkgs[@]} + old_pkgs+=("${pkg}") + new_pkgs+=("${other}") + added_pkg_to_index_map["${pkg}"]=${pkg_idx} + added_pkg_to_index_map["${other}"]=${pkg_idx} + unset pkg_idx + else + pkg_debug_enable "${pkg}" + if [[ -n ${added_pkg_to_index_map["${pkg}"]:-} ]]; then + pkg_debug 'handled already through some rename' + else + pkg_debug "${pkg} is not renamed" + local pkg_idx + # doesn't matter if it's length of new_pkgs or old_pkgs, + # both are assumed to have the same length + pkg_idx=${#old_pkgs[@]} + old_pkgs+=("${pkg}") + new_pkgs+=("${pkg}") + added_pkg_to_index_map["${pkg}"]=${pkg_idx} + fi + pkg_debug_disable + fi + done + unset added_pkg_to_index_map + + # The loop below goes over the pairs of old and new package + # names. For each name there will be some checks done (like does + # this package even exist). Each name in the pair has a set of + # used slots associated with it (the most common situation is that + # each have just one slot, but there are some packages that we + # have multiple slots installed, like + # app-text/docbook-xml-dtd). Some of the slots will appear in both + # old and new package name, sometimes there will be slots + # available only in the old state or only in the new state. Each + # slot for each package name has an associated min version and max + # version. So for common slots we usually compare min version for + # old package with max version for new package. Any + # inconsistencies with the versions should be reported by + # now. There are some edge cases with the slots that are not + # handled by the automation - in such cases there will be a + # "manual action needed" report. + + local pkg_idx=0 + local old_name new_name old_repo new_repo + local hpc_old_slots_set_var_name hpc_new_slots_set_var_name + local hpc_old_slot_verminmax_map_var_name hpc_new_slot_verminmax_map_var_name + local s hpc_old_s hpc_new_s + local old_verminmax new_verminmax + local old_version new_version + local hpc_cmp_result + local -A hpc_only_old_slots_set hpc_only_new_slots_set hpc_common_slots_set + local -a lines + local hpc_update_dir + local -A empty_map_or_set + local hpc_changed hpc_slot_changed hpc_update_dir_non_slot hpc_category_dir + local which slots_set_var_name_var_name slot_verminmax_map_var_name_var_name filtered_slots_set_var_name verminmax + local -A hpc_old_filtered_slots_set hpc_new_filtered_slots_set + # shellcheck disable=SC2034 # used by name below, in a special case + empty_map_or_set=() + while [[ ${pkg_idx} -lt ${#old_pkgs[@]} ]]; do + old_name=${old_pkgs["${pkg_idx}"]} + new_name=${new_pkgs["${pkg_idx}"]} + if [[ ${old_name} = "${new_name}" ]]; then + info "handling update of ${new_name}" + else + info "handling update of ${new_name} (renamed from ${old_name})" + fi + pkg_debug_enable "${old_name}" "${new_name}" + pkg_debug 'handling updates' + pkg_idx=$((pkg_idx + 1)) + old_repo=${hpc_package_sources_map["${old_name}"]:-} + new_repo=${hpc_package_sources_map["${new_name}"]:-} + if [[ -z ${old_repo} ]]; then + pkg_warn \ + '- package not in old state' \ + " - old package: ${old_name}" \ + " - new package: ${new_name}" + pkg_debug_disable + continue + fi + if [[ -z ${new_repo} ]]; then + pkg_warn \ + '- package not in new state' \ + " - old package: ${old_name}" \ + " - new package: ${new_name}" + pkg_debug_disable + continue + fi + if [[ ${old_repo} != "${new_repo}" ]]; then + # This is pretty much an arbitrary limitation and I don't + # remember any more why we have it. + pkg_warn \ + '- package has moved between repos? unsupported for now' \ + " - old package and repo: ${old_name} ${old_repo}" \ + " - new package and repo: ${new_name} ${new_repo}" + pkg_debug_disable + continue + fi + if [[ ${new_repo} != 'portage-stable' ]]; then + # coreos-overlay packages will need a separate handling + pkg_debug 'not a portage-stable package' + pkg_debug_disable + continue + fi + + mvm_get hpc_pkg_slots_set_mvm "${old_name}" hpc_old_slots_set_var_name + mvm_get hpc_pkg_slots_set_mvm "${new_name}" hpc_new_slots_set_var_name + : "${hpc_old_slots_set_var_name:=empty_map_or_set}" + : "${hpc_new_slots_set_var_name:=empty_map_or_set}" + mvm_get hpc_old_pkg_slot_verminmax_map_mvm "${old_name}" hpc_old_slot_verminmax_map_var_name + mvm_get hpc_new_pkg_slot_verminmax_map_mvm "${new_name}" hpc_new_slot_verminmax_map_var_name + : "${hpc_old_slot_verminmax_map_var_name:=empty_map_or_set}" + : "${hpc_new_slot_verminmax_map_var_name:=empty_map_or_set}" + local -n old_slot_verminmax_map_ref=${hpc_old_slot_verminmax_map_var_name} + local -n new_slot_verminmax_map_ref=${hpc_new_slot_verminmax_map_var_name} + + # Filter out slots for old and new package name that comes out + # without versions. This may happen, because we collect all + # slot names for the package name, without differentiating + # whether such a slot existed in the old state or still exists + # in the new state. If slot didn't exist in either one then it + # will come without version information. Such a slot is + # dropped. An example would be an update of sys-devel/binutils + # from 2.42 to 2.43. Each binutils version has a separate slot + # which is named after the version. So the slots set would be + # (2.42 2.43). Slot "2.42" does not exist in the new state any + # more, "2.43" does not yet exist in the old state. So those + # slots for those states will be dropped. Thus filtered slots + # set for the old state will only contain 2.42, while for the + # new state - only 2.43. + for which in old new; do + slots_set_var_name_var_name="hpc_${which}_slots_set_var_name" + slot_verminmax_map_var_name_var_name="hpc_${which}_slot_verminmax_map_var_name" + filtered_slots_set_var_name="hpc_${which}_filtered_slots_set" + local -n which_slots_set_ref=${!slots_set_var_name_var_name} + local -n which_slot_verminmax_map_ref=${!slot_verminmax_map_var_name_var_name} + local -n which_filtered_slots_set_ref=${filtered_slots_set_var_name} + pkg_debug "all unfiltered slots for ${which} name: ${!which_slots_set_ref[*]}" + which_filtered_slots_set_ref=() + for s in "${!which_slots_set_ref[@]}"; do + verminmax=${which_slot_verminmax_map_ref["${s}"]:-} + if [[ -n ${verminmax} ]]; then + which_filtered_slots_set_ref["${s}"]=x + fi + done + pkg_debug "all filtered slots for ${which} name: ${!which_filtered_slots_set_ref[*]}" + unset -n which_filtered_slots_set_ref + unset -n which_slot_verminmax_map_ref + unset -n which_slots_set_ref + done + + hpc_only_old_slots_set=() + hpc_only_new_slots_set=() + hpc_common_slots_set=() + sets_split \ + hpc_old_filtered_slots_set hpc_new_filtered_slots_set \ + hpc_only_old_slots_set hpc_only_new_slots_set hpc_common_slots_set + pkg_debug "all common slots: ${!hpc_common_slots_set[*]}" + pkg_debug "slots only for old name: ${!hpc_only_old_slots_set[*]}" + pkg_debug "slots only for new name: ${!hpc_only_new_slots_set[*]}" + + update_dir_non_slot "${new_name}" hpc_update_dir_non_slot + mkdir -p "${hpc_update_dir_non_slot}" + + # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE comes from globals file + generate_non_ebuild_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" + generate_full_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" + generate_package_mention_reports "${NEW_STATE}" "${old_name}" "${new_name}" + + hpc_changed= + pkg_debug 'going over common slots' + for s in "${!hpc_common_slots_set[@]}"; do + old_verminmax=${old_slot_verminmax_map_ref["${s}"]:-} + new_verminmax=${new_slot_verminmax_map_ref["${s}"]:-} + pkg_debug "slot: ${s}, vmm old: ${old_verminmax}, vmm new: ${new_verminmax}" + if [[ -z "${old_verminmax}" ]] || [[ -z "${new_verminmax}" ]]; then + devel_warn \ + "- no minmax info available for old and/or new:" \ + " - old package: ${old_name}" \ + " - slot: ${s}" \ + " - minmax: ${old_verminmax}" \ + " - new package: ${new_name}" \ + " - slot: ${s}" \ + " - minmax: ${new_verminmax}" + continue + fi + update_dir "${new_name}" "${s}" "${s}" hpc_update_dir + mkdir -p "${hpc_update_dir}" + old_version=${old_verminmax%%:*} + new_version=${new_verminmax##*:} + gentoo_ver_cmp_out "${new_version}" "${old_version}" hpc_cmp_result + case ${hpc_cmp_result} in + "${GV_GT}") + handle_pkg_update "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + "${GV_EQ}") + hpc_slot_changed= + handle_pkg_as_is "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" hpc_slot_changed + if [[ -z ${hpc_slot_changed} ]]; then + rm -rf "${hpc_update_dir}" + else + hpc_changed=x + fi + ;; + "${GV_LT}") + handle_pkg_downgrade "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + esac + done + # A "sys-devel/binutils update" case - one old slot and one + # new slot, but different from each other. + if [[ ${#hpc_only_old_slots_set[@]} -eq 1 ]] && [[ ${#hpc_only_new_slots_set[@]} -eq 1 ]]; then + get_first_from_set hpc_only_old_slots_set hpc_old_s + old_verminmax=${old_slot_verminmax_map_ref["${hpc_old_s}"]:-} + get_first_from_set hpc_only_new_slots_set hpc_new_s + new_verminmax=${new_slot_verminmax_map_ref["${hpc_new_s}"]:-} + pkg_debug "jumping from slot ${hpc_old_s} (vmm: ${old_verminmax}) to slot ${hpc_new_s} (vmm: ${new_verminmax})" + if [[ -z "${old_verminmax}" ]] || [[ -z "${new_verminmax}" ]]; then + devel_warn \ + "- no verminmax info available for old and/or new:" \ + " - old package: ${old_name}" \ + " - slot: ${hpc_old_s}" \ + " - minmax: ${old_verminmax}" \ + " - new package: ${new_name}" \ + " - slot: ${hpc_new_s}" \ + " - minmax: ${new_verminmax}" + else + update_dir "${new_name}" "${hpc_old_s}" "${hpc_new_s}" hpc_update_dir + mkdir -p "${hpc_update_dir}" + old_version=${old_verminmax%%:*} + new_version=${new_verminmax##*:} + gentoo_ver_cmp_out "${new_version}" "${old_version}" hpc_cmp_result + case ${hpc_cmp_result} in + "${GV_GT}") + handle_pkg_update "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hpc_old_s}" "${hpc_new_s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + "${GV_EQ}") + hpc_slot_changed= + handle_pkg_as_is "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hpc_old_s}" "${hpc_new_s}" "${old_version}" hpc_slot_changed + if [[ -z ${hpc_slot_changed} ]]; then + rm -rf "${hpc_update_dir}" + else + hpc_changed=x + fi + ;; + "${GV_LT}") + handle_pkg_downgrade "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hpc_old_s}" "${hpc_new_s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + esac + fi + elif [[ ${#hpc_only_old_slots_set[@]} -gt 0 ]] || [[ ${#hpc_only_new_slots_set[@]} -gt 0 ]]; then + pkg_debug 'complicated slots situation, needs manual intervention' + lines=( + '- handle package update:' + ' - old package name:' + " - name: ${old_name}" + ' - slots:' + ) + for s in "${!hpc_old_filtered_slots_set[@]}"; do + old_verminmax=${old_slot_verminmax_map_ref["${s}"]:-} + lines+=(" - ${s}, minmax: ${old_verminmax}") + done + lines+=( + ' - new package name:' + " - name: ${new_name}" + ' - slots:' + ) + for s in "${!hpc_new_filtered_slots_set[@]}"; do + new_verminmax=${new_slot_verminmax_map_ref["${s}"]:-} + lines+=(" - ${s}, minmax: ${new_verminmax}") + done + manual "${lines[@]}" + fi + unset -n new_slot_verminmax_map_ref old_slot_verminmax_map_ref + # if nothing changed, drop the entire update directory for the + # package, and possibly the parent directory if it became + # empty (parent directory being a category directory, like + # sys-apps) + if [[ -z ${hpc_changed} ]]; then + pkg_debug 'no changes, dropping reports' + rm -rf "${hpc_update_dir_non_slot}" + dirname_out "${hpc_update_dir_non_slot}" hpc_category_dir + if dir_is_empty "${hpc_category_dir}"; then + rmdir "${hpc_category_dir}" + fi + fi + pkg_debug_disable + done + + mvm_unset hpc_new_pkg_slot_verminmax_map_mvm + mvm_unset hpc_old_pkg_slot_verminmax_map_mvm + mvm_unset hpc_pkg_slots_set_mvm +} + +# Gets the first item from the passed set. +# +# Mostly intended to "unwrap" a single-element set. +# +# Params: +# +# 1 - name of the set variable +# 2 - name of the variable where the element will be stored +function get_first_from_set() { + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${1}; shift + local -n return_ref=${1}; shift + + local item + for item in "${!set_ref[@]}"; do + return_ref=${item} + return 0 + done + # shellcheck disable=SC2034 # it's a reference to external variable + return_ref='' +} + +# Does the set operation on two passed sets - both set differences and +# an intersection. +# +# Params: +# +# 1 - name of the first set variable +# 2 - name of the second set variable +# 3 - name of the set variable that will contain elements that exist +# in first set, but not the second +# 4 - name of the set variable that will contain elements that exist +# in second set, but not the first +# 5 - name of the set variable that will contain elements that exist +# in both first and second sets +function sets_split() { + local -n first_set_ref=${1}; shift + local -n second_set_ref=${1}; shift + local -n only_in_first_set_ref=${1}; shift + local -n only_in_second_set_ref=${1}; shift + local -n common_set_ref=${1}; shift + + only_in_first_set_ref=() + only_in_second_set_ref=() + common_set_ref=() + + local item mark + + for item in "${!first_set_ref[@]}"; do + mark=${second_set_ref["${item}"]:-} + if [[ -z "${mark}" ]]; then + # shellcheck disable=SC2034 # it's a reference to external variable + only_in_first_set_ref["${item}"]=x + else + # shellcheck disable=SC2034 # it's a reference to external variable + common_set_ref["${item}"]=x + fi + done + + for item in "${!second_set_ref[@]}"; do + mark=${first_set_ref["${item}"]:-} + if [[ -z "${mark}" ]]; then + # shellcheck disable=SC2034 # it's a reference to external variable + only_in_second_set_ref["${item}"]=x + fi + done +} + +# Write information to reports directory about the package update +# (meaning specifically that the new version is greater than the old +# one). +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - old package name +# 3 - new package name +# 4 - old package slot +# 5 - new package slot +# 6 - old version +# 7 - new version +function handle_pkg_update() { + local pkg_to_tags_mvm_var_name old_pkg new_pkg old_s new_s old new + pkg_to_tags_mvm_var_name=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + old=${1}; shift + new=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local old_no_r new_no_r + old_no_r=${old%-r+([0-9])} + new_no_r=${new%-r+([0-9])} + + local pkg_name + pkg_name=${new_pkg#*/} + local -a lines + lines=( "from ${old} to ${new}") + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( "renamed from ${old_pkg}" ) + fi + # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file + generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" + + # shellcheck disable=SC2034 # these variables are used by name + local hpu_update_dir hpu_update_dir_non_slot + update_dir_non_slot "${new_pkg}" hpu_update_dir_non_slot + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpu_update_dir + if [[ -s "${hpu_update_dir}/ebuild.diff" ]]; then + lines+=( 'TODO: review ebuild.diff' ) + fi + if [[ -s "${hpu_update_dir_non_slot}/other.diff" ]]; then + lines+=( 'TODO: review other.diff' ) + fi + lines+=( 'TODO: review occurences' ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( 'TODO: review occurences-for-old-name' ) + fi + + local -a hpu_tags + tags_for_pkg "${pkg_to_tags_mvm_var_name}" "${new_pkg}" hpu_tags + + if ver_test "${new_no_r}" -gt "${old_no_r}"; then + # version bump + generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpu_tags[@]}" + lines+=( 'release notes: TODO' ) + fi + + generate_summary_stub "${new_pkg}" "${hpu_tags[@]}" -- "${lines[@]}" +} + +# Write information to reports directory about the modified package +# (meaning specifically that the new version is equal than the old +# one). +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - old package name +# 3 - new package name +# 4 - old package slot +# 5 - new package slot +# 6 - version +# 7 - name of a "bool" variable where info is stored if relevant files +# has changed (empty means nothing changed, non-empty means +# something has changed) +function handle_pkg_as_is() { + local pkg_to_tags_mvm_var_name old_pkg new_pkg old_s new_s v + pkg_to_tags_mvm_var_name=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + v=${1}; shift + local -n changed_ref=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local hpai_update_dir + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpai_update_dir + + local pkg_name + pkg_name=${new_pkg#/} + local -a lines + lines=( "still at ${v}" ) + + local renamed + renamed= + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( "renamed from ${old_pkg}" ) + renamed=x + fi + generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${v}" "${v}" + local hpai_update_dir_non_slot hpai_update_dir + update_dir_non_slot "${new_pkg}" hpai_update_dir_non_slot + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpai_update_dir + local modified + modified= + if [[ -s "${hpai_update_dir}/ebuild.diff" ]]; then + lines+=( 'TODO: review ebuild.diff' ) + modified=x + fi + if [[ -s "${hpai_update_dir_non_slot}/other.diff" ]]; then + lines+=( 'TODO: review other.diff' ) + modified=x + fi + if [[ -z ${renamed} ]] && [[ -z ${modified} ]]; then + # Nothing relevant has changed, return early. + return 0 + fi + # shellcheck disable=SC2034 # ref to an external variable + changed_ref=x + lines+=( 'TODO: review occurences' ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( 'TODO: review occurences-for-old-name' ) + fi + + local -a hpai_tags + tags_for_pkg "${pkg_to_tags_mvm_var_name}" "${new_pkg}" hpai_tags + generate_summary_stub "${new_pkg}" "${hpai_tags[@]}" -- "${lines[@]}" +} + +# Write information to reports directory about the package downgrade +# (meaning specifically that the new version is lower than the old +# one). +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - old package name +# 3 - new package name +# 4 - old package slot +# 5 - new package slot +# 6 - old version +# 7 - new version +function handle_pkg_downgrade() { + local pkg_to_tags_mvm_var_name old_pkg new_pkg old_s new_s old new + pkg_to_tags_mvm_var_name=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + old=${1}; shift + new=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local old_no_r new_no_r + old_no_r=${old%-r+([0-9])} + new_no_r=${new%-r+([0-9])} + + local pkg_name + pkg_name=${new_pkg#*/} + local -a lines + lines=( "downgraded from ${old} to ${new}" ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( "renamed from ${old_pkg}" ) + fi + generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" + + local hpd_update_dir hpd_update_dir_non_slot + update_dir_non_slot "${new_pkg}" hpd_update_dir_non_slot + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpd_update_dir + if [[ -s "${hpd_update_dir}/ebuild.diff" ]]; then + lines+=( 'TODO: review ebuild.diff' ) + fi + if [[ -s "${hpd_update_dir_non_slot}/other.diff" ]]; then + lines+=( 'TODO: review other.diff' ) + fi + lines+=( 'TODO: review occurences' ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( 'TODO: review occurences-for-old-name' ) + fi + + local -a hpd_tags + tags_for_pkg "${pkg_to_tags_mvm_var_name}" "${new_pkg}" hpd_tags + + if ver_test "${new_no_r}" -lt "${old_no_r}"; then + # version bump + generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpd_tags[@]}" + lines+=( "release notes: TODO" ) + fi + + generate_summary_stub "${new_pkg}" "${hpd_tags[@]}" -- "${lines[@]}" +} + +# Retrieves tags for a package. +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - package name +# 3 - name of the array variable, where the tags will be stored +function tags_for_pkg() { + local pkg_to_tags_mvm_var_name pkg + pkg_to_tags_mvm_var_name=${1}; shift + pkg=${1}; shift + local -n tags_ref=${1}; shift + + local tfp_tags_var_name + mvm_get "${pkg_to_tags_mvm_var_name}" "${pkg}" tfp_tags_var_name + + pkg_debug_enable "${pkg}" + pkg_debug "checking for tags in ${pkg_to_tags_mvm_var_name}" + if [[ -z ${tfp_tags_var_name} ]]; then + pkg_debug "no tags available" + tags_ref=() + else + local -n tags_in_mvm=${tfp_tags_var_name} + # shellcheck disable=SC2034 # it's a reference to external variable + tags_ref=( "${tags_in_mvm[@]}" ) + pkg_debug "tags available: ${tags_in_mvm[*]}" + fi + pkg_debug_disable +} + +# Adds a changelog stub to changelog file in reports directory. +# +# Params: +# 1 - package name (shortened, without the category) +# 2 - version +# @ - package tags +function generate_changelog_entry_stub() { + local pkg_name v + pkg_name=${1}; shift + v=${1}; shift + # rest are tags + + local -a applied_tags=() + for tag; do + case ${tag} in + PROD) + applied_tags+=( 'base' ) + ;; + *) + # add lower-cased tag + applied_tags+=( "${tag,,}" ) + ;; + esac + done + local gces_tags='' + if [[ ${#applied_tags[@]} -gt 0 ]]; then + join_by gces_tags ', ' "${applied_tags[@]}" + else + # no tags, it means it's an SDK package + gces_tags='SDK' + fi + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + printf '%s %s: %s ([%s](TODO))\n' '-' "${gces_tags}" "${pkg_name}" "${v}" >>"${REPORTS_DIR}/updates/changelog_stubs" +} + +# Adds a stub to the summary file in reports directory. +# +# Params: +# 1 - package +# @ - tags followed by double dash followed by lines to append to the +# file +function generate_summary_stub() { + local pkg + pkg=${1}; shift + # rest are tags separated followed by double dash followed by lines + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a tags + tags=() + while [[ ${#} -gt 0 ]]; do + if [[ ${1} = '--' ]]; then + shift + break + fi + tags+=( "${1}" ) + shift + done + # rest are lines + + { + printf '%s %s:' '-' "${pkg}" + if [[ ${#tags[@]} -gt 0 ]]; then + printf ' [%s]' "${tags[@]}" + fi + printf '\n' + if [[ ${#} -gt 0 ]]; then + printf ' - %s\n' "${@}" + printf '\n' + fi + } >>"${REPORTS_DIR}/updates/summary_stubs" +} + +# Generate diffs between directories in old state and new state for a +# package. +# +# Params: +# +# 1 - path to portage-stable in old state +# 2 - path to portage-stable in new state +# 3 - old package name +# 4 - new package name +function generate_full_diffs() { + local old_ps new_ps old_pkg new_pkg + old_ps=${1}; shift + new_ps=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + + local old_path new_path + old_path="${old_ps}/${old_pkg}" + new_path="${new_ps}/${new_pkg}" + + local gfd_update_dir + update_dir_non_slot "${new_pkg}" gfd_update_dir + + local -a common_diff_opts=( + --recursive + --unified=3 + ) + xdiff "${common_diff_opts[@]}" --new-file "${old_path}" "${new_path}" >"${gfd_update_dir}/full.diff" + xdiff "${common_diff_opts[@]}" --brief "${old_path}" "${new_path}" >"${gfd_update_dir}/brief-summary" +} + +# Generate a diff between non-ebuild, non-Manifest files for old and +# new package. +# +# Params: +# +# 1 - path to portage-stable in old state +# 2 - path to portage-stable in new state +# 3 - old package name +# 4 - new package name +function generate_non_ebuild_diffs() { + local old_ps new_ps old_pkg new_pkg + old_ps=${1}; shift + new_ps=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + + local old_path new_path + old_path="${old_ps}/${old_pkg}" + new_path="${new_ps}/${new_pkg}" + + local gned_update_dir + update_dir_non_slot "${new_pkg}" gned_update_dir + + local -a diff_opts=( + --recursive + --unified=3 + # Show contents of deleted or added files too. + --new-file + # Ignore ebuilds and the Manifest file. + --exclude='*.ebuild' + --exclude='Manifest' + ) + xdiff "${diff_opts[@]}" "${old_path}" "${new_path}" >"${gned_update_dir}/other.diff" +} + +# Generate a diff between specific ebuilds for old and new package. +# +# Params: +# +# 1 - path to portage-stable in old state +# 2 - path to portage-stable in new state +# 3 - old package name +# 4 - new package name +# 5 - old package slot +# 6 - new package slot +# 7 - old package version +# 8 - new package version +function generate_ebuild_diff() { + local old_ps new_ps old_pkg new_pkg old_s new_s old new + old_ps=${1}; shift + new_ps=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + old=${1}; shift + new=${1}; shift + + local old_pkg_name new_pkg_name + old_pkg_name=${old_pkg#*/} + new_pkg_name=${new_pkg#*/} + + local old_path new_path + old_path="${old_ps}/${old_pkg}/${old_pkg_name}-${old}.ebuild" + new_path="${new_ps}/${new_pkg}/${new_pkg_name}-${new}.ebuild" + + local ged_update_dir + update_dir "${new_pkg}" "${old_s}" "${new_s}" ged_update_dir + xdiff --unified=3 "${old_path}" "${new_path}" >"${ged_update_dir}/ebuild.diff" +} + +# Generate a report with information where the old and new packages +# are mentioned in entire scripts repository. May result in two +# separate reports if the package got renamed. +# +# 1 - path to scripts repo +# 2 - old package name +# 3 - new package name +function generate_package_mention_reports() { + local scripts old_pkg new_pkg + scripts=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + + local gpr_update_dir + update_dir_non_slot "${new_pkg}" gpr_update_dir + + generate_mention_report_for_package "${scripts}" "${new_pkg}" >"${gpr_update_dir}/occurences" + + if [[ ${old_pkg} != "${new_pkg}" ]]; then + generate_mention_report_for_package "${scripts}" "${old_pkg}" >"${gpr_update_dir}/occurences-for-old-name" + fi +} + +# Generate a report with information where the package is mentioned in +# entire scripts repository. +# +# 1 - path to scripts repo +# 3 - package name +function generate_mention_report_for_package() { + local scripts pkg + scripts=${1}; shift + pkg=${1}; shift + + local ps co + ps='sdk_container/src/third_party/portage-stable' + co='sdk_container/src/third_party/coreos-overlay' + + yell "${pkg} in overlay profiles" + grep_pkg "${scripts}" "${pkg}" "${co}/profiles" + + yell "${pkg} in Gentoo profiles" + grep_pkg "${scripts}" "${pkg}" "${ps}/profiles" + + # shellcheck disable=SC2164 # we use set -e, so the script will exit if it fails + pushd "${scripts}/${co}" >/dev/null + + yell "${pkg} in env overrides" + cat_entries "coreos/config/env/${pkg}"@(|-+([0-9])*) + + yell "${pkg} in user patches" + local dir + for dir in "coreos/user-patches/${pkg}"@(|-+([0-9])*); do + echo "BEGIN DIRECTORY: ${dir}" + cat_entries "${dir}"/* + echo "END DIRECTORY: ${dir}" + done + + # shellcheck disable=SC2164 # we use set -e, so the script will exit if it fails + popd >/dev/null + + yell "${pkg} in overlay (outside profiles)" + grep_pkg "${scripts}" "${pkg}" "${co}" ":(exclude)${co}/profiles" + + yell "${pkg} in Gentoo (outside profiles)" + grep_pkg "${scripts}" "${pkg}" "${ps}" ":(exclude)${ps}/profiles" + + yell "${pkg} in scripts (outside overlay and Gentoo)" + grep_pkg "${scripts}" "${pkg}" ":(exclude)${ps}" ":(exclude)${co}" +} + +# Gets a toplevel update reports directory for a package. This is +# where occurences and non-ebuild diffs are stored. +# +# Params: +# +# 1 - package name +# 2 - name of a variable where the path will be stored +function update_dir_non_slot() { + local pkg + pkg=${1}; shift + local -n dir_ref=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + # shellcheck disable=SC2034 # it's a reference to external variable + dir_ref="${REPORTS_DIR}/updates/${pkg}" +} + +# Gets a slot specific update reports directory for a package. This is +# where ebuild diffs are stored. +# +# Params: +# +# 1 - package name +# 2 - old slot +# 3 - new slot +# 4 - name of a variable where the path will be stored +function update_dir() { + local pkg old_s new_s + pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + local -n dir_ref=${1}; shift + + # slots may have slashes in them - replace them with "-slash-" + local slot_dir + if [[ ${old_s} = "${new_s}" ]]; then + slot_dir=${old_s//\//-slash-} + else + slot_dir="${old_s//\//-slash-}-to-${new_s//\//-slash-}" + fi + + local ud_non_slot_dir + update_dir_non_slot "${pkg}" ud_non_slot_dir + # shellcheck disable=SC2034 # it's a reference to external variable + dir_ref="${ud_non_slot_dir}/${slot_dir}" +} + +# Greps for a package name in selected directories of the passed +# repo. It prints, so the invocation needs to be captured. +# +# Params: +# +# 1 - path to scripts repo +# 2 - package name +# @ - directories in the repo to limit the search for +function grep_pkg() { + local scripts pkg + scripts=${1}; shift + pkg=${1}; shift + # rest are directories + + git -C "${scripts}" grep "${pkg}"'\(-[0-9]\|[^a-zA-Z0-9_-]\|$\)' -- "${@}" || : +} + +# Prints the passed files preceding and following with BEGIN ENTRY and +# END ENTRY markers. +# +# Params: +# +# @ - the files to print +function cat_entries() { + for entry; do + echo "BEGIN ENTRY: ${entry}" + cat "${entry}" + echo "END ENTRY: ${entry}" + done +} + +# Reads the listings and renames, handles updates of both packages and +# non-packages (eclasses, licenses, profiles, etc.) +function handle_gentoo_sync() { + #mvm_debug_enable hgs_pkg_to_tags_mvm + mvm_declare hgs_pkg_to_tags_mvm + process_listings hgs_pkg_to_tags_mvm + + # shellcheck disable=SC2034 # passed to other function through a name + local -A hgs_renames_old_to_new_map=() + process_profile_updates_directory hgs_renames_old_to_new_map + + handle_package_changes hgs_renames_old_to_new_map hgs_pkg_to_tags_mvm + + mvm_unset hgs_pkg_to_tags_mvm + #mvm_debug_disable hgs_pkg_to_tags_mvm + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local old_head new_head + # shellcheck disable=SC2153 # OLD_STATE is not a misspelling + old_head=$(git -C "${OLD_STATE}" rev-parse HEAD) + new_head=$(git -C "${NEW_STATE}" rev-parse HEAD) + + local -A non_package_updates_set + non_package_updates_set=() + local path in_ps category + if [[ "${old_head}" != "${new_head}" ]]; then + while read -r path; do + if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then + continue + fi + in_ps=${path#"${PORTAGE_STABLE_SUFFIX}/"} + category=${in_ps%%/*} + case "${category}" in + eclass) + if [[ ${in_ps} != 'eclass/'+([^/])'.eclass' ]]; then + fail "unexpected updated file inside eclass directory: '${path}'" + fi + non_package_updates_set["${in_ps}"]=x + ;; + licenses|metadata|profiles|scripts) + non_package_updates_set["${category}"]=x + ;; + virtual|*-*) + # Package update, already handled + : + ;; + *) + fail "unexpected updated file '${path}'" + ;; + esac + done < <(git -C "${NEW_STATE}" diff-tree --no-commit-id --name-only -r "${old_head}" "${new_head}") + fi + + local entry + for entry in "${!non_package_updates_set[@]}"; do + case "${entry}" in + eclass/*) + handle_eclass "${entry}" + ;; + licenses) + handle_licenses + ;; + metadata) + info "not handling metadata updates, skipping" + ;; + profiles) + handle_profiles + ;; + scripts) + handle_scripts + ;; + *) + fail "unknown non-package update for ${entry}" + ;; + esac + done + sort_summary_stubs + sort_changelog_stubs +} + +# Sorts entries in the summary file if it exists. +function sort_summary_stubs() { + if [[ -f "${REPORTS_DIR}/updates/summary_stubs" ]]; then + sort_like_summary_stubs "${REPORTS_DIR}/updates/summary_stubs" + fi +} + +# Sorts entries in the summary file. +# +# Lines look like as follows: +# +# -BEGIN- +# - dev-lang/python: [DEV] +# - from 3.11.4 to 3.11.5 +# - no changes in ebuild +# - release notes: TODO +# +# - app-emulation/qemu: +# - from 8.0.3 to 8.0.4 +# - no changes in ebuild +# - release notes: TODO +# +# -END- +function sort_like_summary_stubs() { + local f + f=${1}; shift + + mvm_declare groups_mvm + + local -a lines entries + lines=() + entries=() + local -A dups + dups=() + + local REPLY line entry sss_lines_name dup_count + while read -r; do + if [[ -z ${REPLY} ]]; then + if [[ ${#lines[@]} -gt 0 ]]; then + line=${lines[0]} + entry=${line#-+([[:space:]])} + entry=${entry%%:*} + dup_count=${dups["${entry}"]:-0} + if [[ ${dup_count} -gt 0 ]]; then + dup_count=$((dup_count + 1)) + mvm_add groups_mvm "${entry}@${dup_count}" "${lines[@]}" + dups["${entry}"]=${dup_count} + else + mvm_get groups_mvm "${entry}" sss_lines_name + if [[ -n ${sss_lines_name} ]]; then + local -n lines_ref=${sss_lines_name} + mvm_add groups_mvm "${entry}@1" "${lines_ref[@]}" + unset -n lines_ref + mvm_remove groups_mvm "${entry}" + mvm_add groups_mvm "${entry}@2" "${lines[@]}" + dups["${entry}"]=2 + else + mvm_add groups_mvm "${entry}" "${lines[@]}" + entries+=( "${entry}" ) + fi + fi + lines=() + fi + else + lines+=( "${REPLY}" ) + fi + done < <(cat "${f}"; echo) # echo for final empty line, just in case + + if [[ ${#entries[@]} -eq 0 ]]; then + return 0 + fi + + local idx + { + while read -r line; do + dup_count=${dups["${line}"]:-0} + if [[ ${dup_count} -gt 0 ]]; then + idx=0 + while [[ ${idx} -lt ${dup_count} ]]; do + idx=$((idx + 1)) + mvm_get groups_mvm "${line}@${idx}" sss_lines_name + local -n lines_ref=${sss_lines_name} + printf '%s\n' "${lines_ref[@]}" '' + unset -n lines_ref + done + else + mvm_get groups_mvm "${line}" sss_lines_name + local -n lines_ref=${sss_lines_name} + printf '%s\n' "${lines_ref[@]}" '' + unset -n lines_ref + fi + done < <(printf '%s\n' "${entries[@]}" | csort) + } >"${f}" + mvm_unset groups_mvm +} + +# Sorts entries in changelog stub if it exists. +function sort_changelog_stubs() { + if [[ -f "${REPORTS_DIR}/updates/changelog_stubs" ]]; then + sort_like_changelog_stubs "${REPORTS_DIR}/updates/changelog_stubs" + fi +} + +# Sorts entries in changelog stub. +function sort_like_changelog_stubs() { + local f t + f=${1}; shift + t="${f}.tmp" + csort --output="${t}" "${f}" + mv -f "${t}" "${f}" +} + +# Invokes sort with C locale. Meant to be used in bash pipelines. +# +# Params: +# +# @ - additional parameters to passed to sort +function csort() { + LC_ALL=C sort "${@}" +} + +# Handle an eclass update. Basically generate a diff. +# +# Params: +# +# 1 - path to eclass file within an ebuild repo +function handle_eclass() { + local eclass + eclass=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a lines + lines=() + if [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]] && [[ -e "${NEW_PORTAGE_STABLE}/${eclass}" ]]; then + mkdir -p "${REPORTS_DIR}/updates/${eclass}" + xdiff --unified=3 "${OLD_PORTAGE_STABLE}/${eclass}" "${NEW_PORTAGE_STABLE}/${eclass}" >"${REPORTS_DIR}/updates/${eclass}/eclass.diff" + lines+=( 'TODO: review the diff' ) + elif [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]]; then + lines+=( 'unused, dropped' ) + else + lines+=( 'added from Gentoo' ) + fi + generate_summary_stub "${eclass}" -- "${lines[@]}" +} + +# Handle profile changes. Generates three different diffs - changes in +# relevant profiles (ancestors of the profiles used by board packages +# and SDK), a full diff between all the profiles, and a list of +# possibly irrelevant files that has changed too. +function handle_profiles() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a files + files=() + local arch which report + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in sdk-profiles board-profiles; do + files+=("${WORKDIR}/pkg-reports/${which}-${arch}/${report}") + done + done + done + local -A profile_dirs_set + profile_dirs_set=() + + local line + while read -r line; do + profile_dirs_set["${line}"]=x + done < <(xgrep --no-filename '^portage-stable:' "${files[@]}" | cut -d: -f2-) + + local -a diff_opts + diff_opts=( + --recursive + --unified=3 + --new-file # treat absent files as empty + ) + + local out_dir + out_dir="${REPORTS_DIR}/updates/profiles" + mkdir -p "${out_dir}" + + xdiff "${diff_opts[@]}" \ + "${OLD_PORTAGE_STABLE}/profiles" "${NEW_PORTAGE_STABLE}/profiles" >"${out_dir}/full.diff" + + local relevant + relevant='' + local -a relevant_lines possibly_irrelevant_files + relevant_lines=() + possibly_irrelevant_files=() + local REPLY path dir mark + while read -r; do + if [[ ${REPLY} = "diff "* ]]; then + path=${REPLY##*"${NEW_PORTAGE_STABLE}/profiles/"} + dirname_out "${path}" dir + relevant='' + mark=${profile_dirs_set["${dir}"]:-} + if [[ -n "${mark}" ]]; then + relevant=x + else + case ${dir} in + .|desc|desc/*|updates|updates/*) + relevant=x + ;; + esac + fi + if [[ -z ${relevant} ]]; then + possibly_irrelevant_files+=( "profiles/${path}" ) + fi + fi + if [[ -n ${relevant} ]]; then + relevant_lines+=( "${REPLY}" ) + fi + done <"${out_dir}/full.diff" + lines_to_file_truncate "${out_dir}/relevant.diff" "${relevant_lines[@]}" + lines_to_file_truncate "${out_dir}/possibly-irrelevant-files" "${possibly_irrelevant_files[@]}" + generate_summary_stub profiles -- 'TODO: review the diffs' +} + +# Handles changes in license directory. Generates brief reports and +# diffs about dropped, added or modified licenses. +function handle_licenses() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a dropped added changed + dropped=() + added=() + changed=() + + local line hl_stripped + # Lines are: + # + # Only in /licenses: BSL-1.1 + # + # or + # + # Files /licenses/BSL-1.1 and /licenses/BSL-1.1 differ + while read -r line; do + if [[ ${line} = 'Only in '* ]]; then + # shellcheck disable=SC2153 # OLD_STATE is not a misspelling, it comes from globals file + strip_out "${line##*:}" hl_stripped + if [[ ${line} = *"${OLD_STATE}"* ]]; then + dropped+=( "${hl_stripped}" ) + elif [[ ${line} = *"${NEW_STATE}"* ]]; then + added+=( "${hl_stripped}" ) + else + devel_warn "- unhandled license change: ${line}" + fi + elif [[ ${line} = 'Files '*' differ' ]]; then + line=${line##"Files ${OLD_PORTAGE_STABLE}/licenses/"} + line=${line%% *} + strip_out "${line}" hl_stripped + changed+=( "${hl_stripped}" ) + else + devel_warn \ + '- unhandled diff --brief line:' \ + " - ${line}" + fi + done < <(xdiff --brief --recursive "${OLD_PORTAGE_STABLE}/licenses" "${NEW_PORTAGE_STABLE}/licenses") + + local out_dir + out_dir="${REPORTS_DIR}/updates/licenses" + mkdir -p "${out_dir}" + + lines_to_file_truncate \ + "${out_dir}/brief-summary" \ + '- removed:' \ + "${dropped[@]/#/ - }" \ + '- added:' \ + "${added[@]/#/ - }" \ + '- modified:' \ + "${changed[@]/#/ - }" + truncate --size=0 "${out_dir}/modified.diff" + + local c + for c in "${changed[@]}"; do + xdiff --unified=3 "${OLD_PORTAGE_STABLE}/licenses/${c}" "${NEW_PORTAGE_STABLE}/licenses/${c}" >>"${out_dir}/modified.diff" + done + local -a lines + lines=() + + local joined + if [[ ${#dropped[@]} -gt 0 ]]; then + join_by joined ', ' "${dropped[@]}" + lines+=( "dropped ${joined}" ) + fi + if [[ ${#added[@]} -gt 0 ]]; then + join_by joined ', ' "${added[@]}" + lines+=( "added ${joined}" ) + fi + if [[ ${#changed[@]} -gt 0 ]]; then + join_by joined ', ' "${changed[@]}" + lines+=( "updated ${joined}" ) + fi + generate_summary_stub licenses -- "${lines[@]}" +} + +# Generates reports about changes inside the scripts directory. +function handle_scripts() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local out_dir + out_dir="${REPORTS_DIR}/updates/scripts" + mkdir -p "${out_dir}" + + xdiff --unified=3 --recursive "${OLD_PORTAGE_STABLE}/scripts" "${NEW_PORTAGE_STABLE}/scripts" >"${out_dir}/scripts.diff" + generate_summary_stub scripts -- 'TODO: review the diffs' +} + +# Enables debug logs when specific packages are processed. +# +# It is expected that globals were already sourced, otherwise +# debugging won't be enabled at all. +# +# Params: +# +# @ - package names to enable debugging for +function pkg_debug_enable() { + local -A pkg_set + pkg_set=() + local -a vals + vals=() + local pkg + for pkg; do + if [[ -n ${pkg_set["${pkg}"]:-} ]]; then + continue + fi + pkg_set["${pkg}"]=x + if [[ -n ${DEBUG_PACKAGES["${pkg}"]:-} ]]; then + vals+=( "${pkg}" ) + fi + done + if [[ ${#vals[@]} -gt 0 ]]; then + declare -g PKG_AUTO_LIB_DEBUG + join_by PKG_AUTO_LIB_DEBUG ',' "${vals[@]}" + fi +} + +# Returns true or false whether any debugging has been enabled. +function pkg_debug_possible() { + local ret=0 + [[ ${#DEBUG_PACKAGES[@]} -gt 0 ]] || ret=1 + return ${ret} +} + +# Disables debug logs to be printed. +function pkg_debug_disable() { + unset PKG_AUTO_LIB_DEBUG +} + +# Prints passed parameters if debugging is enabled. +# +# Params: +# +# @ - parameters to print +function pkg_debug() { + if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then + info "DEBUG(${PKG_AUTO_LIB_DEBUG}): ${*}" + fi +} + +# Prints passed lines if debugging is enabled. +# +# Params: +# +# @ - lines to print +function pkg_debug_lines() { + if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then + info_lines "${@/#/"DEBUG(${PKG_AUTO_LIB_DEBUG}): "}" + fi +} + +fi diff --git a/pkg_auto/impl/print_profile_tree.sh b/pkg_auto/impl/print_profile_tree.sh new file mode 100755 index 0000000000..96ce744840 --- /dev/null +++ b/pkg_auto/impl/print_profile_tree.sh @@ -0,0 +1,274 @@ +#!/bin/bash + +## +## Prints profile information in form of an inheritance tree and/or +## evaluation order. +## +## Parameters: +## -h: this help +## -ni: no inheritance tree +## -ne: no evaluation order +## -nh: no headers +## +## Environment variables: +## ROOT +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +: "${ROOT:=/}" + +print_inheritance_tree=x +print_evaluation_order=x +print_headers=x + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -ni) + print_inheritance_tree= + ;; + -ne) + print_evaluation_order= + ;; + -nh) + print_headers= + ;; + *) + fail "unknown flag ${1}" + ;; + esac + shift +done + +all_repo_names=() +read -a all_repo_names -r < <(portageq get_repos "${ROOT}") + +declare -A repo_data repo_data_r +# name to path +repo_data=() +# path to name +repo_data_r=() + +for repo_name in "${all_repo_names[@]}"; do + repo_path=$(portageq get_repo_path "${ROOT}" "${repo_name}") + repo_path=$(realpath "${repo_path}") + repo_data["${repo_name}"]="${repo_path}" + repo_data_r["${repo_path}"]="${repo_name}" +done + +unset all_repo_names + +function get_repo_from_profile_path() { + local path + path=${1}; shift + local -n repo_dir_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + repo_dir_ref="${path%/profiles/*}" +} + +function repo_path_to_name() { + local path + path=${1}; shift + local -n name_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + name_ref=${repo_data_r["${path}"]:-''} +} + +function repeat_string() { + local str ntimes out_str_var_name + str=${1}; shift + ntimes=${1}; shift + out_str_var_name=${1}; shift + local -n out_str_ref="${out_str_var_name}" + + if [[ ${ntimes} -eq 0 ]]; then + out_str_ref="" + return 0 + elif [[ ${ntimes} -eq 1 ]]; then + out_str_ref="${str}" + return 0 + fi + local add_one + add_one=$((ntimes % 2)) + repeat_string "${str}${str}" $((ntimes / 2)) "${out_str_var_name}" + if [[ add_one -gt 0 ]]; then + out_str_ref+="${str}" + fi +} + +function process_profile() { + local repo_name profile_path + repo_name=${1}; shift + profile_path=${1}; shift + local -n children_ref=${1}; shift + + local parent_file line pp_new_repo_name new_profile_path pp_new_repo_path + local -a children + + parent_file="${profile_path}/parent" + children=() + if [[ -e ${parent_file} ]]; then + while read -r line; do + if [[ ${line} = *:* ]]; then + pp_new_repo_name=${line%%:*} + if [[ -z ${pp_new_repo_name} ]]; then + pp_new_repo_name=${repo_name} + fi + pp_new_repo_path=${repo_data["${pp_new_repo_name}"]} + new_profile_path="${pp_new_repo_path}/profiles/${line#*:}" + children+=( "${pp_new_repo_name}" "${new_profile_path}" ) + elif [[ ${line} = /* ]]; then + pp_new_repo_path= + get_repo_from_profile_path "${line}" pp_new_repo_path + pp_new_repo_name= + repo_path_to_name "${pp_new_repo_path}" pp_new_repo_name + children+=( "${pp_new_repo_name}" "${line}" ) + else + pp_new_repo_path=$(realpath "${profile_path}/${line}") + children+=( "${repo_name}" "${pp_new_repo_path}" ) + fi + done <"${parent_file}" + fi + + # shellcheck disable=SC2034 # it's a reference to external variable + children_ref=( "${children[@]}" ) +} + +function get_profile_name() { + local repo_name profile_path + repo_name="${1}"; shift + profile_path="${1}"; shift + local -n profile_name_ref=${1}; shift + + local repo_path profile_name + repo_path=${repo_data["${repo_name}"]} + profile_name=${profile_path#"${repo_path}/profiles/"} + + # shellcheck disable=SC2034 # it's a reference to external variable + profile_name_ref="${profile_name}" +} + +make_profile_path="${ROOT%/}/etc/portage/make.profile" +top_profile_dir_path=$(realpath "${make_profile_path}") +top_repo_path= +get_repo_from_profile_path "${top_profile_dir_path}" top_repo_path +top_repo_name= +repo_path_to_name "${top_repo_path}" top_repo_name + +if [[ -n ${print_inheritance_tree} ]]; then + +set -- '0' "${top_repo_name}" "${top_profile_dir_path}" + +profile_tree=() + +while [[ ${#} -gt 2 ]]; do + indent=${1}; shift + repo_name=${1}; shift + profile_path=${1}; shift + + lines= + fork= + if [[ ${indent} -gt 0 ]]; then + if [[ ${indent} -gt 1 ]]; then + repeat_string '| ' $((indent - 1)) lines + fi + fork='+-' + fi + g_profile_name= + get_profile_name "${repo_name}" "${profile_path}" g_profile_name + profile_tree+=( "${lines}${fork}${repo_name}:${g_profile_name}" ) + g_profile_children=() + + process_profile "${repo_name}" "${profile_path}" g_profile_children + + new_profiles=() + new_indent=$((indent + 1)) + pc_idx=0 + while [[ $((pc_idx + 1)) -lt "${#g_profile_children[@]}" ]]; do + new_repo_name=${g_profile_children["${pc_idx}"]} + new_profile_path=${g_profile_children[$((pc_idx + 1))]} + new_profiles+=( "${new_indent}" "${new_repo_name}" "${new_profile_path}" ) + pc_idx=$((pc_idx + 2)) + done + + set -- "${new_profiles[@]}" "${@}" +done + +if [[ -n ${print_headers} ]]; then + echo + echo 'profile inheritance tree:' + echo +fi +for line in "${profile_tree[@]}"; do + echo "${line}" +done + +fi + +if [[ -n ${print_evaluation_order} ]]; then + +set -- "${top_repo_name}" "${top_profile_dir_path}" '0' + +profile_eval=() + +while [[ ${#} -gt 2 ]]; do + repo_name=${1}; shift + profile_path=${1}; shift + num_parents=${1}; shift + # each parent is a repo name and profile path, so two items for each parent + num_parent_items=$((num_parents * 2)) + parents=( "${@:1:${num_parent_items}}" ) + shift "${num_parent_items}" + g_profile_children=() + + process_profile "${repo_name}" "${profile_path}" g_profile_children + + new_args=() + if [[ ${#g_profile_children[@]} -eq 0 ]]; then + to_evaluate=( "${repo_name}" "${profile_path}" "${parents[@]}" ) + te_idx=0 + while [[ $((te_idx + 1)) -lt "${#to_evaluate[@]}" ]]; do + new_repo_name=${to_evaluate["${te_idx}"]} + new_profile_path=${to_evaluate[$((te_idx + 1))]} + g_new_profile_name= + get_profile_name "${new_repo_name}" "${new_profile_path}" g_new_profile_name + profile_eval+=( "${new_repo_name}:${g_new_profile_name}" ) + te_idx=$((te_idx + 2)) + done + else + last_idx=$(( ${#g_profile_children[@]} - 2 )) + pc_idx=0 + while [[ $((pc_idx + 1)) -lt "${#g_profile_children[@]}" ]]; do + new_repo_name=${g_profile_children["${pc_idx}"]} + new_profile_path=${g_profile_children[$((pc_idx + 1))]} + new_args+=( "${new_repo_name}" "${new_profile_path}" ) + if [[ pc_idx -eq last_idx ]]; then + new_args+=( $((num_parents + 1)) "${repo_name}" "${profile_path}" "${parents[@]}" ) + else + new_args+=( 0 ) + fi + pc_idx=$((pc_idx + 2)) + done + fi + + set -- "${new_args[@]}" "${@}" +done + +if [[ -n ${print_headers} ]]; then + echo + echo 'profile evaluation order:' + echo +fi +for line in "${profile_eval[@]}"; do + echo "${line}" +done + +fi diff --git a/pkg_auto/impl/sort_packages_list.py b/pkg_auto/impl/sort_packages_list.py new file mode 100755 index 0000000000..909cd0b6ff --- /dev/null +++ b/pkg_auto/impl/sort_packages_list.py @@ -0,0 +1,206 @@ +#!/usr/bin/python3 + +# The package list file is a document that consists of a header and an +# empty-line-separated list of package groups. Header is a list of all +# lines until the first package group. Package group is a list of +# packages of the same category and possibly some related +# comments. The comments are usually about packages that are +# temporarily excluded from the list. So the comments usually have two +# parts - the free form part and a commented-out package list part; +# the example would be: +# +# # Temporarily excluded from automated updates, because reasons. +# # +# # sys-devel/whatever +# +# The script tries to preserve the comments and its ordering, so it +# associates the free form part to the package name. +# +# The script also deduplicates the packages while sorting. An edge +# case is when a package appears multiple times and is not +# commented-out at least once - all commented out entries are dropped. +# +# Implementation-wise, the document has a list of lines being a +# header, a list of free form comments and a map of category name to a +# group. A group is a list of packages, where each package has a name, +# information if it's commented out and may have a free form comment +# associated with it. + +import re +import sys + +class FreeForm: + def __init__(self, lines): + self.lines = lines + +class Pkg: + def __init__(self, idx, name, out): + self.free_form_idx = idx + self.name = name + self.commented_out = out + +class Group: + def __init__(self): + self.pkgs = [] + self.pkg_names_set = set() + +class Document: + def __init__(self): + self.header = [] + self.free_forms = [] + self.groups = {} + +class Reader: + category_or_pkg_pattern = re.compile("^[a-z0-9-]+(?:/[A-Za-z0-9-_+]+)?$") + parsing_header = 1 + parsing_group = 2 + parsing_comment = 3 + + def __init__(self, doc): + self.doc = doc + self.parsing_stage = Reader.parsing_header + self.current_comments = [] + self.free_form_idx_for_next_pkg = None + + def get_group(self, category): + if category not in self.doc.groups: + new_group = Group() + self.doc.groups[category] = new_group + return new_group + return self.doc.groups[category] + + def add_pkg_impl(self, idx, name, out): + category = name.split('/', 1)[0] + group = self.get_group(category) + if name in group.pkg_names_set: + if not out: + for pkg in group.pkgs: + if pkg.name == name: + pkg.commented_out = False + break + else: + group.pkg_names_set.add(name) + group.pkgs += [Pkg(idx, name, out)] + return True + return False + + def add_pkg(self, name): + if self.add_pkg_impl(self.free_form_idx_for_next_pkg, name, False): + self.free_form_idx_for_next_pkg = None + + class CommentBatch: + def __init__(self, ff_lines, p_lines): + self.free_form_lines = ff_lines + self.pkg_lines = p_lines + + def get_batches(self): + batches = [] + free_form_lines = [] + pkg_lines = [] + for line in self.current_comments: + line = line.lstrip('#').strip() + if not line: + if not pkg_lines: + free_form_lines += [line] + elif Reader.category_or_pkg_pattern.match(line): + pkg_lines += [line] + else: + if pkg_lines: + while not free_form_lines[-1]: + free_form_lines = free_form_lines[:-1] + batches += [Reader.CommentBatch(free_form_lines, pkg_lines)] + free_form_lines = [] + pkg_lines = [] + free_form_lines += [line] + self.current_comments = [] + if free_form_lines or pkg_lines: + batches += [Reader.CommentBatch(free_form_lines, pkg_lines)] + return batches + + def process_current_comments(self): + for batch in self.get_batches(): + free_form_idx = None + if batch.free_form_lines: + free_form_idx = len(self.doc.free_forms) + self.doc.free_forms += [FreeForm(batch.free_form_lines)] + if batch.pkg_lines: + for line in batch.pkg_lines: + self.add_pkg_impl(free_form_idx, line, True) + else: + self.free_form_idx_for_next_pkg = free_form_idx + + def read(self, input): + while line := input.readline(): + line = line.strip() + if self.parsing_stage == Reader.parsing_header: + if not line: + self.parsing_stage = Reader.parsing_group + elif line.startswith('#'): + self.doc.header += [line] + else: + self.parsing_stage = Reader.parsing_group + self.add_pkg(line) + elif self.parsing_stage == Reader.parsing_group: + if not line: + pass + elif line.startswith('#'): + self.current_comments += [line] + self.parsing_stage = Reader.parsing_comment + else: + self.add_pkg(line) + elif self.parsing_stage == Reader.parsing_comment: + if not line: + self.parsing_stage = Reader.parsing_group + self.process_current_comments() + elif line.startswith('#'): + self.current_comments += [line] + else: + self.parsing_stage = Reader.parsing_group + self.process_current_comments() + self.add_pkg(line) + if self.current_comments: + self.process_current_comments() + +class Writer: + def __init__(self, doc): + self.doc = doc + + def write(self, output): + output_lines = [] + if self.doc.header: + output_lines += self.doc.header + output_lines += [''] + for category in sorted(self.doc.groups): + last_free_form_idx = None + for pkg in sorted(self.doc.groups[category].pkgs, key=lambda pkg: pkg.name): + if pkg.free_form_idx != last_free_form_idx: + last_free_form_idx = pkg.free_form_idx + if pkg.free_form_idx is not None: + for line in self.doc.free_forms[pkg.free_form_idx].lines: + if line: + output_lines += [f"# {line}"] + else: + output_lines += ['#'] + if pkg.commented_out: + output_lines += [f"# {pkg.name}"] + else: + output_lines += [f"{pkg.name}"] + output_lines += [''] + while not output_lines[0]: + output_lines = output_lines[1:] + while not output_lines[-1]: + output_lines = output_lines[:-1] + for line in output_lines: + print(line, file=output) + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"1 argument expected, got {len(sys.argv) - 1}", file=sys.stderr) + sys.exit(1) + filename = sys.argv[1] + doc = Document() + with open(filename, 'r', encoding='UTF-8') as file: + reader = Reader(doc) + reader.read(file) + writer = Writer(doc) + writer.write(sys.stdout) diff --git a/pkg_auto/impl/sync_with_gentoo.sh b/pkg_auto/impl/sync_with_gentoo.sh new file mode 100755 index 0000000000..2080f78c05 --- /dev/null +++ b/pkg_auto/impl/sync_with_gentoo.sh @@ -0,0 +1,316 @@ +#!/bin/bash + +## +## Used for syncing with gentoo. Needs to be called from the +## toplevel-directory of portage-stable. If syncing everything or +## syncing metadata/glsa specifically, it is expected that the Gentoo +## repo will have the GLSA files stored in metadata/glsa too. +## +## Parameters: +## -h: this help +## -b: be brief, print only names of changed entries and errors +## -s: skip adding source git commit hash information to commits +## +## Positional: +## 0: Gentoo repository +## #: Entries to update (can be a package name, eclass, category, some special +## directories like profiles or . for everything) +## +## Example invocations: +## +## sync_with_gentoo -h +## +## Print a help message. +## +## sync_with_gentoo dev-libs/nettle app-crypt/argon2 +## +## This will update the packages, each in a separate commit. The +## commit message will contain the commit hash from gentoo repo. +## +## sync_with_gentoo dev-libs +## +## This will update all the packages in dev-libs category. The +## commit message will contain the commit hash from gentoo repo. +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +BRIEF= +SKIP_GIT_INFO= + +while true; do + case ${1-} in + -h) + print_help + exit 0 + ;; + -b) + BRIEF=x + shift + ;; + -s) + SKIP_GIT_INFO=x + shift + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ $# -lt 2 ]]; then + fail 'expected at least two positional parameters: a Gentoo repository and at least one package, use -h to print help' +fi + +if [[ ! -e 'profiles/repo_name' ]]; then + fail 'sync is only possible from ebuild packages top-level directory (a directory from which "./profiles/repo_name" is accessible)' +fi + +function vcall() { + if [[ -z ${BRIEF} ]]; then + "${@}" + fi +} + +function bcall() { + if [[ -n ${BRIEF} ]]; then + "${@}" + fi +} + +GENTOO=$(realpath "${1}"); shift +# rest are package names + +if [[ $(realpath '.') = "${GENTOO}" ]]; then + fail 'trying to sync within a Gentoo repo?' +fi + +if [[ -z ${SKIP_GIT_INFO} ]] && [[ ! -e ${GENTOO}/.git ]]; then + info "Skipping adding source git commit hash information to commits, ${GENTOO@Q} is not a git repository" + SKIP_GIT_INFO=x +fi + +glsa_repo=${GENTOO}/metadata/glsa +if [[ -z ${SKIP_GIT_INFO} ]] && [[ -e ${glsa_repo} ]] && [[ ! -e ${glsa_repo}/.git ]] && [[ $(git -C "${GENTOO}" status --porcelain -- metadata/glsa) = '?? metadata/glsa' ]]; then + info "Skipping adding source git commit hash information to commits, ${glsa_repo@Q} exists, but it is not a git repository and is not a part of Gentoo git repository" + SKIP_GIT_INFO=x +fi +unset glsa_repo + +# Synchronizes given path with its Gentoo counterpart. Returns true if +# there were changes. +# +# Params: +# +# 1 - path within ebuild repo +function sync_git_prepare() { + local path + path=${1}; shift + + local gentoo_path + gentoo_path="${GENTOO}/${path}" + + if [[ ! -e "${gentoo_path}" ]]; then + info "no ${path@Q} in Gentoo repository" + if [[ ${path} = 'metadata/glsa' ]]; then + info "did you forget to clone https://gitweb.gentoo.org/data/glsa.git/ into ${gentoo_path@Q}?" + fi + return 1 + fi + + local -a rsync_opts=( --archive --delete-before ) + + case ${path} in + profiles) + rsync_opts+=( --exclude /profiles/repo_name ) + ;; + esac + + local parent + dirname_out "${path}" parent + mkdir --parents "${parent}" + rsync "${rsync_opts[@]}" "${gentoo_path}" "${parent}" + if [[ -n $(git status --porcelain -- "${path}") ]]; then + bcall info "updated ${path}" + git add "${path}" + return 0 + fi + return 1 +} + +# Creates a git commit. If checking Gentoo commit ID is enabled the +# given path is used to get the ID of the commit with the last change +# in the path. Name parameter is used for denoting which part has +# changed, and sync parameter to denote if the commit is about adding +# new package or updating an existing one. +# +# Params: +# +# 1 - path +# 2 - name +# 3 - not empty if existing package was updated, or an empty string if +# the package is new +function commit_with_gentoo_sha() { + local path name sync + path=${1}; shift + name=${1}; shift + sync=${1:-}; shift + + local -a commit_extra=() + if [[ -z ${SKIP_GIT_INFO} ]]; then + local commit + + commit=$(git -C "${GENTOO}/${path}" log --pretty=oneline -1 -- . | cut -f1 -d' ') + commit_extra+=( --message "It's from Gentoo commit ${commit}." ) + unset commit + fi + commit_msg="${name}: Add from Gentoo" + if [[ -n "${sync}" ]]; then + commit_msg="${name}: Sync with Gentoo" + fi + git commit --quiet --message "${commit_msg}" "${commit_extra[@]}" + GIT_PAGER='cat' vcall git show --stat +} + +# Simple path sync and commit; takes the contents from Gentoo at the +# given path and puts it in the repo. +# +# 1 - path to sync +# 2 - name for commit message +function path_sync() { + local path name + path=${1}; shift + name=${1}; shift + + local sync + sync='' + if [[ -e "${path}" ]]; then + sync='x' + fi + + if sync_git_prepare "${path}"; then + commit_with_gentoo_sha "${path}" "${name}" "${sync}" + else + vcall info "no changes in ${path}" + fi +} + +# Goes over the given directory and syncs its subdirectories or +# files. No commit is created. +function prepare_dir() { + local dir + dir=${1}; shift + + local pkg mod='' + for pkg in "${dir}/"*; do + if sync_git_prepare "${pkg}"; then + mod=x + fi + done + if [[ -n ${mod} ]]; then + return 0 + fi + return 1 +} + +# Synces entire category of packages and creates a commit. Note that +# if the category already exists, no new packages will be added. +# +# Params: +# +# 1 - path to the category directory +function category_sync() { + local path + path=${1}; shift + + if [[ ! -e "${path}" ]]; then + if sync_git_prepare "${path}"; then + commit_with_gentoo_sha "${path}" "${path}" + fi + else + if prepare_dir "${path}"; then + commit_with_gentoo_sha "${path}" "${path}" 'x' + fi + fi + +} + +# Synces entire repo. No new packages will be added. +function everything_sync() { + local path mod + + for path in *; do + case ${path} in + licenses|profiles|scripts) + if sync_git_prepare "${path}"; then + mod=x + fi + ;; + metadata) + # do only metadata updates + if sync_git_prepare metadata/glsa; then + mod=x + fi + ;; + eclass|virtual|*-*) + if prepare_dir "${path}"; then + mod=x + fi + ;; + changelog|*.md) + # ignore those + : + ;; + *) + info "Unknown entry ${path@Q}, ignoring" + ;; + esac + done + if [[ -n ${mod} ]]; then + commit_with_gentoo_sha '.' '*' 'x' + fi +} + +shopt -s extglob + +for cpn; do + cpn=${cpn%%*(/)} + case ${cpn} in + .) + everything_sync + ;; + licenses|profiles|scripts) + path_sync "${cpn}" "${cpn}" + ;; + eclass/*.eclass) + path_sync "${cpn}" "${cpn%.eclass}" + ;; + metadata/glsa) + path_sync "${cpn}" "${cpn}" + ;; + metadata) + fail "metadata directory can't be synced, did you mean metadata/glsa?" + ;; + virtual/*/*|*-*/*/*) + fail "invalid thing to sync: ${cpn}" + ;; + virtual/*|*-*/*) + path_sync "${cpn}" "${cpn}" + ;; + eclass|virtual|*-*) + category_sync "${cpn}" + ;; + *) + fail "invalid thing to sync: ${cpn}" + ;; + esac +done diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh new file mode 100644 index 0000000000..83e0ee94ce --- /dev/null +++ b/pkg_auto/impl/util.sh @@ -0,0 +1,253 @@ +#!/bin/bash + +if [[ -z ${__UTIL_SH_INCLUDED__:-} ]]; then +__UTIL_SH_INCLUDED__=x + +# Works like dirname, but without spawning new processes. +# +# Params: +# +# 1 - path to operate on +# 2 - name of a variable which will contain a dirname of the path +function dirname_out() { + local path dir_var_name + path=${1}; shift + dir_var_name=${1}; shift + local -n dir_ref=${dir_var_name} + + if [[ -z ${path} ]]; then + dir_ref='.' + return 0 + fi + local cleaned_up dn + # strip trailing slashes + cleaned_up=${path%%*(/)} + # strip duplicated slashes + cleaned_up=${cleaned_up//+(\/)/\/} + # strip last component + dn=${cleaned_up%/*} + if [[ -z ${dn} ]]; then + dir_ref='/' + return 0 + fi + if [[ ${cleaned_up} = "${dn}" ]]; then + dir_ref='.' + return 0 + fi + # shellcheck disable=SC2034 # it's a reference to external variable + dir_ref=${dn} +} + +# Works like basename, but without spawning new processes. +# +# Params: +# +# 1 - path to operate on +# 2 - name of a variable which will contain a basename of the path +function basename_out() { + local path base_var_name + path=${1}; shift + base_var_name=${1}; shift + local -n base_ref=${base_var_name} + + if [[ -z ${path} ]]; then + base_ref='' + return 0 + fi + local cleaned_up dn + # strip trailing slashes + cleaned_up=${path%%*(/)} + if [[ -z ${cleaned_up} ]]; then + base_ref='/' + return 0 + fi + # strip duplicated slashes + cleaned_up=${cleaned_up//+(\/)/\/} + # keep last component + dn=${cleaned_up##*/} + # shellcheck disable=SC2034 # it's a reference to external variable + base_ref=${dn} +} + +if [[ ${BASH_SOURCE[-1]##*/} = 'util.sh' ]]; then + THIS=${BASH} + basename_out "${THIS}" THIS_NAME + THIS_DIR=. +else + THIS=${BASH_SOURCE[-1]} + basename_out "${THIS}" THIS_NAME + dirname_out "${THIS}" THIS_DIR +fi + +THIS=$(realpath "${THIS}") +THIS_DIR=$(realpath "${THIS_DIR}") +dirname_out "${BASH_SOURCE[0]}" PKG_AUTO_IMPL_DIR +PKG_AUTO_IMPL_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}") +# shellcheck disable=SC2034 # may be used by scripts sourcing this file +PKG_AUTO_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}/..") + +# Prints an info line. +# +# Params: +# +# @ - strings to print +function info() { + printf '%s: %s\n' "${THIS_NAME}" "${*}" +} + +# Prints info lines. +# +# Params: +# +# @ - lines to print +function info_lines() { + printf '%s\n' "${@/#/"${THIS_NAME}: "}" +} + +# Prints an info to stderr and fails the execution. +# +# Params: +# +# @ - strings to print +function fail() { + info "${@}" >&2 + exit 1 +} + +# Prints infos to stderr and fails the execution. +# +# Params: +# +# @ - lines to print +function fail_lines() { + info_lines "${@}" >&2 + exit 1 +} + +# Yells a message. +# +# Params: +# +# @ - strings to yell +function yell() { + echo + echo '!!!!!!!!!!!!!!!!!!' + echo " ${*}" + echo '!!!!!!!!!!!!!!!!!!' + echo +} + +# Prints help. Help is taken from the lines prefixed with double +# hashes in the top sourcer of this file. +function print_help() { + if [[ ${THIS} != "${BASH}" ]]; then + grep '^##' "${THIS}" | sed -e 's/##[[:space:]]*//' + fi +} + +# Joins passed strings with a given delimiter. +# +# Params: +# +# 1 - name of a variable that will contain the joined result +# 2 - delimiter +# @ - strings to join +function join_by() { + local output_var_name delimiter first + + output_var_name=${1}; shift + delimiter=${1}; shift + first=${1-} + if shift; then + printf -v "${output_var_name}" '%s' "${first}" "${@/#/${delimiter}}"; + else + local -n output_ref=${output_var_name} + # shellcheck disable=SC2034 # it's a reference to external variable + output_ref='' + fi +} + +# Checks if directory is empty, returns true if so, otherwise false. +# +# Params: +# +# 1 - path to a directory +function dir_is_empty() { + local dir + dir=${1}; shift + + [[ -z $(echo "${dir}"/*) ]] +} + +# Just like diff, but ignores the return value. +function xdiff() { + diff "${@}" || : +} + +# Just like grep, but ignores the return value. +function xgrep() { + grep "${@}" || : +} + +# Strips leading and trailing whitespace from the passed parameter. +# +# Params: +# +# 1 - string to strip +# 2 - name of a variable where the result of stripping will be stored +function strip_out() { + local l + l=${1}; shift + local -n out_ref=${1}; shift + + local t + t=${l} + t=${t/#+([[:space:]])} + t=${t/%+([[:space:]])} + # shellcheck disable=SC2034 # it's a reference to external variable + out_ref=${t} +} + +# Gets supported architectures. +# +# Params: +# +# 1 - name of an array variable, where the architectures will be stored +function get_valid_arches() { + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n arches_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + arches_ref=( 'amd64' 'arm64' ) +} + +# Generates all pairs from a given sequence of strings. Each pair will +# be stored in the given variable and items in the pair will be +# separated by the given separator. For N strings, (N * N - N) / 2 +# pairs will be generatated. +# +# Params: +# +# 1 - name of an array variable where the pairs will be stored +# 2 - separator string +# @ - strings +function all_pairs() { + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n pairs_ref=${1}; shift + local sep=${1}; shift + + # indices in ${@} are 1-based, 0 gives script name or something + local idx=1 next_idx + + pairs_ref=() + while [[ ${idx} -lt ${#} ]]; do + next_idx=$((idx + 1)) + while [[ ${next_idx} -le ${#} ]]; do + pairs_ref+=( "${!idx}${sep}${!next_idx}" ) + next_idx=$((next_idx + 1)) + done + idx=$((idx+1)) + done +} + +fi diff --git a/pkg_auto/inside_sdk_container.sh b/pkg_auto/inside_sdk_container.sh new file mode 100755 index 0000000000..8651917916 --- /dev/null +++ b/pkg_auto/inside_sdk_container.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +## +## Gathers information about SDK and board packages. Also collects +## info about actual build deps of board packages, which may be useful +## for verifying if SDK provides those. +## +## Reports generated: +## sdk-pkgs - contains package information for SDK +## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK +## board-pkgs - contains package information for board for chosen architecture +## board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies +## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order +## board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order +## sdk-package-repos - contains package information with their repos for SDK +## board-package-repos - contains package information with their repos for board +## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports +## board-emerge-output - contains raw emerge output for board being a base for other reports +## sdk-emerge-output-filtered - contains only lines with package information for SDK +## board-emerge-output-filtered - contains only lines with package information for board +## sdk-emerge-output-junk - contains only junk lines for SDK +## board-emerge-output-junk - contains only junk lines for board +## *-warnings - warnings printed by emerge or other tools +## +## Parameters: +## -h: this help +## +## Positional: +## 1 - architecture (amd64 or arm64) +## 2 - reports directory +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/inside_sdk_container_lib.sh" + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 2 ]]; then + fail 'Expected two parameters: board architecture and reports directory' +fi + +arch=${1}; shift +reports_dir=${1}; shift + +mkdir -p "${reports_dir}" + +set_eo "${reports_dir}" + +echo 'Running pretend-emerge to get complete report for SDK' +package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}" +echo 'Running pretend-emerge to get complete report for board' +package_info_for_board "${arch}" >"${BOARD_EO}" 2>"${BOARD_EO_W}" + +ensure_no_errors + +echo 'Separating emerge info from junk in SDK emerge output' +filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}" +junk_sdk_eo >"${SDK_EO}-junk" 2>>"${SDK_EO_W}" +echo 'Separating emerge info from junk in board emerge output' +filter_board_eo "${arch}" >"${BOARD_EO_F}" 2>>"${BOARD_EO_W}" +junk_board_eo >"${BOARD_EO}-junk" 2>>"${BOARD_EO_W}" + +ensure_valid_reports + +echo 'Generating SDK packages listing' +versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings" +echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)' +versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings" +echo 'Generating board packages listing' +versions_board >"${reports_dir}/board-pkgs" 2>"${reports_dir}/board-pkgs-warnings" +echo 'Generating board packages bdeps listing' +board_bdeps >"${reports_dir}/board-bdeps" 2>"${reports_dir}/board-bdeps-warnings" +echo 'Generating SDK profiles evaluation list' +ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings" +echo 'Generating board profiles evaluation list' +ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/board-profiles" 2>"${reports_dir}/board-profiles-warnings" +echo 'Generating SDK package source information' +package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings" +echo 'Generating board package source information' +package_sources_board >"${reports_dir}/board-package-repos" 2>"${reports_dir}/board-package-repos-warnings" + +echo "Cleaning empty warning files" +clean_empty_warning_files "${reports_dir}" diff --git a/pkg_auto/sync_packages.sh b/pkg_auto/sync_packages.sh new file mode 100755 index 0000000000..03edc578e8 --- /dev/null +++ b/pkg_auto/sync_packages.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +## +## Synces the packages with Gentoo. +## +## Parameters: +## -f: remove reports directory if it exists at startup +## -w: path to use for workdir +## -h: this help +## +## Positional: +## 1: config file +## 2: new branch name with updates +## 3: gentoo repo +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/pkg_auto_lib.sh" + +workdir='' + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -w) + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + workdir=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 3 ]]; then + fail 'expected three positional parameters: a config file, a final branch name and a path to Gentoo repo' +fi + +config_file=${1}; shift +saved_branch_name=${1}; shift +gentoo=${1}; shift + +setup_workdir_with_config "${workdir}" "${config_file}" +perform_sync_with_gentoo "${gentoo}" +save_new_state "${saved_branch_name}" diff --git a/pkg_auto/update_packages.sh b/pkg_auto/update_packages.sh new file mode 100755 index 0000000000..2eb5a9c217 --- /dev/null +++ b/pkg_auto/update_packages.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +## +## Updates the packages +## +## Parameters: +## -w: path to use for workdir +## -h: this help +## +## Positional: +## 1: config file +## 2: new branch name with updates +## 3: gentoo repo +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/pkg_auto_lib.sh" + +workdir='' + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -w) + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + workdir=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 3 ]]; then + fail 'expected three positional parameters: a config file, a final branch name and a path to Gentoo repo' +fi + +config_file=${1}; shift +saved_branch_name=${1}; shift +gentoo=${1}; shift + +setup_workdir_with_config "${workdir}" "${config_file}" +perform_sync_with_gentoo "${gentoo}" +save_new_state "${saved_branch_name}" +generate_package_update_reports From dd09caba17150b8495405cbc6225849a294af28a Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 27 Nov 2024 15:53:21 +0100 Subject: [PATCH 2/6] pkg-auto: Import a stripped-down version of eapi7-ver.eclass into impl The eclass was removed from Gentoo, so we followed suit. This broke the pkg-auto code. Thus I imported the eclass into the impl directory as gentoo_ver.sh, threw away all the unnecessary parts and moved some from pkg_auto_lib.sh to the new file. This allowed me to also drop a hack where I was grepping for the version regexp in the eclass. Now I'm just exporting it. --- pkg_auto/impl/gentoo_ver.sh | 151 ++++++++++++++++++++++++++++++++++ pkg_auto/impl/pkg_auto_lib.sh | 69 ++-------------- 2 files changed, 156 insertions(+), 64 deletions(-) create mode 100644 pkg_auto/impl/gentoo_ver.sh diff --git a/pkg_auto/impl/gentoo_ver.sh b/pkg_auto/impl/gentoo_ver.sh new file mode 100644 index 0000000000..1b53ba8ada --- /dev/null +++ b/pkg_auto/impl/gentoo_ver.sh @@ -0,0 +1,151 @@ +# Copyright 1999-2021 Gentoo Authors +# Distributed under the terms of the GNU General Public License v2 + +# @AUTHOR: +# Ulrich Müller +# Michał Górny + +# It is a cut-down and modified version of the now-gone eapi7-ver.eclass. + +if [[ -z ${__GENTOO_VER_SH_INCLUDED__:-} ]]; then +__GENTOO_VER_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +VER_ERE="^([0-9]+(\.[0-9]+)*)([a-z]?)((_(alpha|beta|pre|rc|p)[0-9]*)*)(-r[0-9]+)?$" + +# @FUNCTION: _ver_compare_int +# @USAGE: +# @RETURN: 0 if -eq , 1 if -lt , 3 if -gt +# @INTERNAL +# @DESCRIPTION: +# Compare two non-negative integers and , of arbitrary length. +# If is equal to, less than, or greater than , return 0, 1, or 3 +# as exit status, respectively. +_ver_compare_int() { + local a=$1 b=$2 d=$(( ${#1}-${#2} )) + + # Zero-pad to equal length if necessary. + if [[ ${d} -gt 0 ]]; then + printf -v b "%0${d}d%s" 0 "${b}" + elif [[ ${d} -lt 0 ]]; then + printf -v a "%0$(( -d ))d%s" 0 "${a}" + fi + + [[ ${a} > ${b} ]] && return 3 + [[ ${a} == "${b}" ]] +} + +# @FUNCTION: _ver_compare +# @USAGE: +# @RETURN: 1 if < , 2 if = , 3 if > +# @INTERNAL +# @DESCRIPTION: +# Compare two versions and . If is less than, equal to, +# or greater than , return 1, 2, or 3 as exit status, respectively. +_ver_compare() { + local va=${1} vb=${2} a an al as ar b bn bl bs br re LC_ALL=C + + re=${VER_ERE} + + [[ ${va} =~ ${re} ]] || fail "${FUNCNAME}: invalid version: ${va}" + an=${BASH_REMATCH[1]} + al=${BASH_REMATCH[3]} + as=${BASH_REMATCH[4]} + ar=${BASH_REMATCH[7]} + + [[ ${vb} =~ ${re} ]] || fail "${FUNCNAME}: invalid version: ${vb}" + bn=${BASH_REMATCH[1]} + bl=${BASH_REMATCH[3]} + bs=${BASH_REMATCH[4]} + br=${BASH_REMATCH[7]} + + # Compare numeric components (PMS algorithm 3.2) + # First component + _ver_compare_int "${an%%.*}" "${bn%%.*}" || return + + while [[ ${an} == *.* && ${bn} == *.* ]]; do + # Other components (PMS algorithm 3.3) + an=${an#*.} + bn=${bn#*.} + a=${an%%.*} + b=${bn%%.*} + if [[ ${a} == 0* || ${b} == 0* ]]; then + # Remove any trailing zeros + [[ ${a} =~ 0+$ ]] && a=${a%"${BASH_REMATCH[0]}"} + [[ ${b} =~ 0+$ ]] && b=${b%"${BASH_REMATCH[0]}"} + [[ ${a} > ${b} ]] && return 3 + [[ ${a} < ${b} ]] && return 1 + else + _ver_compare_int "${a}" "${b}" || return + fi + done + [[ ${an} == *.* ]] && return 3 + [[ ${bn} == *.* ]] && return 1 + + # Compare letter components (PMS algorithm 3.4) + [[ ${al} > ${bl} ]] && return 3 + [[ ${al} < ${bl} ]] && return 1 + + # Compare suffixes (PMS algorithm 3.5) + as=${as#_}${as:+_} + bs=${bs#_}${bs:+_} + while [[ -n ${as} && -n ${bs} ]]; do + # Compare each suffix (PMS algorithm 3.6) + a=${as%%_*} + b=${bs%%_*} + if [[ ${a%%[0-9]*} == "${b%%[0-9]*}" ]]; then + _ver_compare_int "${a##*[a-z]}" "${b##*[a-z]}" || return + else + # Check for p first + [[ ${a%%[0-9]*} == p ]] && return 3 + [[ ${b%%[0-9]*} == p ]] && return 1 + # Hack: Use that alpha < beta < pre < rc alphabetically + [[ ${a} > ${b} ]] && return 3 || return 1 + fi + as=${as#*_} + bs=${bs#*_} + done + if [[ -n ${as} ]]; then + [[ ${as} == p[_0-9]* ]] && return 3 || return 1 + elif [[ -n ${bs} ]]; then + [[ ${bs} == p[_0-9]* ]] && return 1 || return 3 + fi + + # Compare revision components (PMS algorithm 3.7) + _ver_compare_int "${ar#-r}" "${br#-r}" || return + + return 2 +} + +# symbolic names for use with gentoo_ver_cmp_out +GV_LT=1 +GV_EQ=2 +GV_GT=3 + +# Compare two versions. The result can be compared against GV_LT, GV_EQ and GV_GT variables. +# +# Params: +# +# 1 - version 1 +# 2 - version 2 +# 3 - name of variable to store the result in +function gentoo_ver_cmp_out() { + local v1 v2 + v1=${1}; shift + v2=${1}; shift + local -n out_ref=${1}; shift + + out_ref=0 + _ver_compare "${v1}" "${v2}" || out_ref=${?} + case ${out_ref} in + 1|2|3) + return 0 + ;; + *) + fail "unexpected return value ${out_ref} from _ver_compare for ${v1} and ${v2}" + ;; + esac +} + +fi diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 1c8adba2d6..00acf39270 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -51,6 +51,7 @@ __PKG_AUTO_LIB_SH_INCLUDED__=x source "$(dirname "${BASH_SOURCE[0]}")/util.sh" source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" +source "${PKG_AUTO_IMPL_DIR}/gentoo_ver.sh" # Sets up the workdir using the passed config. The config can be # created basing on the config_template file or using the @@ -1032,19 +1033,10 @@ function process_listings() { # shellcheck disable=SC1091 # generated file source "${WORKDIR}/globals" - local eclass ver_ere pkg_ere - # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling - eclass="${PKG_AUTO_DIR}/../${PORTAGE_STABLE_SUFFIX}/eclass/eapi7-ver.eclass" - # line is like ' re=""' - ver_ere=$(grep -e 're=' "${eclass}" || fail "no 're=' line found in eapi7-ver.eclass") - if [[ -z ${ver_ere} ]]; then - fail 'empty version regex from eapi7-ver.eclass' - fi - # strip everything until first quotes - ver_ere=${ver_ere#*'"'} - # strip last quote - ver_ere=${ver_ere%'"'*} - # regexp begins with ^ and ends with $, so strip them too + local ver_ere pkg_ere + # VER_ERE comes from gentoo_ver.sh + ver_ere=${VER_ERE} + # regexp begins with ^ and ends with $, so strip them ver_ere=${ver_ere#'^'} ver_ere=${ver_ere%'$'} pkg_ere='[a-z0-9]*-?[a-z0-9]*/[a-z0-9A-Z_+-]*' @@ -1439,57 +1431,6 @@ function unset_report_mvms() { done } -### -### BEGIN GENTOO VER COMP HACKS -### - -# shellcheck disable=SC2034 # it's here only for the eapi7-ver.eclass -EAPI=6 -function die() { - fail "$*" -} - -# This brings in ver_test function. -# -# shellcheck disable=SC1091 # sourcing external file -source "${PKG_AUTO_DIR}/../sdk_container/src/third_party/portage-stable/eclass/eapi7-ver.eclass" - -unset EAPI - -# symbolic names for use with gentoo_ver_cmp -GV_LT=1 -GV_EQ=2 -GV_GT=3 - -# Compare two versions. The result can be compared against GV_LT, GV_EQ and GV_GT variables. -# -# Params: -# -# 1 - version 1 -# 2 - version 2 -# 3 - name of variable to store the result in -function gentoo_ver_cmp_out() { - local v1 v2 - v1=${1}; shift - v2=${1}; shift - local -n out_ref=${1}; shift - - out_ref=0 - _ver_compare "${v1}" "${v2}" || out_ref=${?} - case ${out_ref} in - 1|2|3) - return 0 - ;; - *) - fail "unexpected return value ${out_ref} from _ver_compare for ${v1} and ${v2}" - ;; - esac -} - -### -### END GENTOO VER COMP HACKS -### - # Finds out the highest and the lowest version from the passed versions. # # Params: From ae67b87fba0167703c3a39094811b3d5032fd4e0 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 27 Nov 2024 15:56:43 +0100 Subject: [PATCH 3/6] pkg-auto: Move some implementation detail down the file I try to keep "public API" at the top of the file. --- pkg_auto/impl/mvm.sh | 252 +++++++++++++++++++++---------------------- 1 file changed, 126 insertions(+), 126 deletions(-) diff --git a/pkg_auto/impl/mvm.sh b/pkg_auto/impl/mvm.sh index eded4aa54a..74a8756493 100644 --- a/pkg_auto/impl/mvm.sh +++ b/pkg_auto/impl/mvm.sh @@ -52,132 +52,6 @@ source "$(dirname "${BASH_SOURCE[0]}")/util.sh" # Used for creating unique names for extras and storage maps. MVM_COUNTER=0 -# Array mvm, the default. Provides an iteration helper that sends all -# the array values to the iteration callback. - -function mvm_mvc_array_constructor() { - local array_var_name - array_var_name=${1}; shift - - declare -g -a "${array_var_name}" - - local -n array_ref=${array_var_name} - array_ref=() -} - -function mvm_mvc_array_destructor() { - local array_var_name - array_var_name=${1}; shift - - unset "${array_var_name}" -} - -function mvm_mvc_array_adder() { - local array_var_name - array_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays - local -n array_ref=${array_var_name} - - array_ref+=( "${@}" ) -} - -# iteration_helper is optional -function mvm_mvc_array_iteration_helper() { - local key array_var_name callback - key=${1}; shift - array_var_name=${1}; shift - callback=${1}; shift - # rest are extra args passed to cb - - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays - local -n array_ref=${array_var_name} - "${callback}" "${@}" "${key}" "${array_var_name}" "${array_ref[@]}" -} - -# Map mvm. When adding elements to the mvc, it is expected that the -# number of items passed will be even. Odd elements will be used as -# keys, even elements will be used as values. -# -# No iteration helper. - -function mvm_mvc_map_constructor() { - local map_var_name - map_var_name=${1}; shift - - declare -g -A "${map_var_name}" - - local -n map_ref=${map_var_name} - map_ref=() -} - -function mvm_mvc_map_destructor() { - local map_var_name - map_var_name=${1}; shift - - unset "${map_var_name}" -} - -function mvm_mvc_map_adder() { - local map_var_name - map_var_name=${1}; shift - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays - local -n map_ref=${map_var_name} - - while [[ ${#} -gt 1 ]]; do - # shellcheck disable=SC2034 # it's a reference to external variable - map_ref["${1}"]=${2} - shift 2 - done -} - -# Set mvm. Behaves like array mvm, but all elements in each set are -# unique and the order of elements is not guaranteed to be the same as -# order of insertions. - -function mvm_mvc_set_constructor() { - local set_var_name - set_var_name=${1}; shift - - declare -g -A "${set_var_name}" - - # shellcheck disable=SC2178 # shellcheck does not grok refs - local -n set_ref=${set_var_name} - set_ref=() -} - -function mvm_mvc_set_destructor() { - local set_var_name - set_var_name=${1} - - unset "${set_var_name}" -} - -function mvm_mvc_set_adder() { - local set_var_name - set_var_name=${1}; shift - - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays - local -n set_ref=${set_var_name} - while [[ ${#} -gt 0 ]]; do - set_ref["${1}"]=x - shift - done -} - -# iteration_helper is optional -function mvm_mvc_set_iteration_helper() { - local key map_var_name callback - - key=${1}; shift - set_var_name=${1}; shift - callback=${1}; shift - # rest are extra args passed to cb - - # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays - local -n set_ref=${set_var_name} - "${callback}" "${@}" "${key}" "${set_var_name}" "${!set_ref[@]}" -} - # mvm API # Creates a new mvm with a passed name, optionally type and @@ -570,4 +444,130 @@ function mvm_debug_disable() { unset "MVM_DEBUG_NAMES[${mvm_var_name}]" } +# Array mvm, the default. Provides an iteration helper that sends all +# the array values to the iteration callback. + +function mvm_mvc_array_constructor() { + local array_var_name + array_var_name=${1}; shift + + declare -g -a "${array_var_name}" + + local -n array_ref=${array_var_name} + array_ref=() +} + +function mvm_mvc_array_destructor() { + local array_var_name + array_var_name=${1}; shift + + unset "${array_var_name}" +} + +function mvm_mvc_array_adder() { + local array_var_name + array_var_name=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n array_ref=${array_var_name} + + array_ref+=( "${@}" ) +} + +# iteration_helper is optional +function mvm_mvc_array_iteration_helper() { + local key array_var_name callback + key=${1}; shift + array_var_name=${1}; shift + callback=${1}; shift + # rest are extra args passed to cb + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n array_ref=${array_var_name} + "${callback}" "${@}" "${key}" "${array_var_name}" "${array_ref[@]}" +} + +# Map mvm. When adding elements to the mvc, it is expected that the +# number of items passed will be even. Odd elements will be used as +# keys, even elements will be used as values. +# +# No iteration helper. + +function mvm_mvc_map_constructor() { + local map_var_name + map_var_name=${1}; shift + + declare -g -A "${map_var_name}" + + local -n map_ref=${map_var_name} + map_ref=() +} + +function mvm_mvc_map_destructor() { + local map_var_name + map_var_name=${1}; shift + + unset "${map_var_name}" +} + +function mvm_mvc_map_adder() { + local map_var_name + map_var_name=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n map_ref=${map_var_name} + + while [[ ${#} -gt 1 ]]; do + # shellcheck disable=SC2034 # it's a reference to external variable + map_ref["${1}"]=${2} + shift 2 + done +} + +# Set mvm. Behaves like array mvm, but all elements in each set are +# unique and the order of elements is not guaranteed to be the same as +# order of insertions. + +function mvm_mvc_set_constructor() { + local set_var_name + set_var_name=${1}; shift + + declare -g -A "${set_var_name}" + + # shellcheck disable=SC2178 # shellcheck does not grok refs + local -n set_ref=${set_var_name} + set_ref=() +} + +function mvm_mvc_set_destructor() { + local set_var_name + set_var_name=${1} + + unset "${set_var_name}" +} + +function mvm_mvc_set_adder() { + local set_var_name + set_var_name=${1}; shift + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${set_var_name} + while [[ ${#} -gt 0 ]]; do + set_ref["${1}"]=x + shift + done +} + +# iteration_helper is optional +function mvm_mvc_set_iteration_helper() { + local key map_var_name callback + + key=${1}; shift + set_var_name=${1}; shift + callback=${1}; shift + # rest are extra args passed to cb + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${set_var_name} + "${callback}" "${@}" "${key}" "${set_var_name}" "${!set_ref[@]}" +} + fi From 4eba9bec28d598d83e71c2be59c00a48a1ffe140 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 27 Nov 2024 16:49:41 +0100 Subject: [PATCH 4/6] pkg-auto: Add a missing function --- pkg_auto/impl/gentoo_ver.sh | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/pkg_auto/impl/gentoo_ver.sh b/pkg_auto/impl/gentoo_ver.sh index 1b53ba8ada..9db3ab7d1c 100644 --- a/pkg_auto/impl/gentoo_ver.sh +++ b/pkg_auto/impl/gentoo_ver.sh @@ -118,6 +118,38 @@ _ver_compare() { return 2 } +# @FUNCTION: ver_test +# @USAGE: [] +# @DESCRIPTION: +# Check if the relation is true. If is not specified, +# default to ${PVR}. can be -gt, -ge, -eq, -ne, -le, -lt. +# Both versions must conform to the PMS version syntax (with optional +# revision parts), and the comparison is performed according to +# the algorithm specified in the PMS. +ver_test() { + local va op vb + + if [[ $# -eq 3 ]]; then + va=${1} + shift + else + va=${PVR} + fi + + [[ $# -eq 2 ]] || fail "${FUNCNAME}: bad number of arguments" + + op=${1} + vb=${2} + + case ${op} in + -eq|-ne|-lt|-le|-gt|-ge) ;; + *) fail "${FUNCNAME}: invalid operator: ${op}" ;; + esac + + _ver_compare "${va}" "${vb}" + test $? "${op}" 2 +} + # symbolic names for use with gentoo_ver_cmp_out GV_LT=1 GV_EQ=2 From 04b78d965796b5722da5f20b15f9346b9b8d1de9 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 27 Nov 2024 17:17:58 +0100 Subject: [PATCH 5/6] pkg-auto: Address shellcheck complaints --- pkg_auto/impl/gentoo_ver.sh | 10 +++++----- pkg_auto/impl/pkg_auto_lib.sh | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg_auto/impl/gentoo_ver.sh b/pkg_auto/impl/gentoo_ver.sh index 9db3ab7d1c..1df280be27 100644 --- a/pkg_auto/impl/gentoo_ver.sh +++ b/pkg_auto/impl/gentoo_ver.sh @@ -48,13 +48,13 @@ _ver_compare() { re=${VER_ERE} - [[ ${va} =~ ${re} ]] || fail "${FUNCNAME}: invalid version: ${va}" + [[ ${va} =~ ${re} ]] || fail "${FUNCNAME[0]}: invalid version: ${va}" an=${BASH_REMATCH[1]} al=${BASH_REMATCH[3]} as=${BASH_REMATCH[4]} ar=${BASH_REMATCH[7]} - [[ ${vb} =~ ${re} ]] || fail "${FUNCNAME}: invalid version: ${vb}" + [[ ${vb} =~ ${re} ]] || fail "${FUNCNAME[0]}: invalid version: ${vb}" bn=${BASH_REMATCH[1]} bl=${BASH_REMATCH[3]} bs=${BASH_REMATCH[4]} @@ -136,14 +136,14 @@ ver_test() { va=${PVR} fi - [[ $# -eq 2 ]] || fail "${FUNCNAME}: bad number of arguments" + [[ $# -eq 2 ]] || fail "${FUNCNAME[0]}: bad number of arguments" op=${1} vb=${2} case ${op} in -eq|-ne|-lt|-le|-gt|-ge) ;; - *) fail "${FUNCNAME}: invalid operator: ${op}" ;; + *) fail "${FUNCNAME[0]}: invalid operator: ${op}" ;; esac _ver_compare "${va}" "${vb}" @@ -171,7 +171,7 @@ function gentoo_ver_cmp_out() { out_ref=0 _ver_compare "${v1}" "${v2}" || out_ref=${?} case ${out_ref} in - 1|2|3) + "${GV_LT}"|"${GV_EQ}"|"${GV_GT}") return 0 ;; *) diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 00acf39270..437efd619e 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -2814,6 +2814,7 @@ function handle_gentoo_sync() { local path in_ps category if [[ "${old_head}" != "${new_head}" ]]; then while read -r path; do + # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then continue fi From 7d72d5d8a05c13fc86416da5674e6c0b5489cd67 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Tue, 3 Dec 2024 15:04:45 +0100 Subject: [PATCH 6/6] pkg-auto: Update README --- pkg_auto/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg_auto/README.md b/pkg_auto/README.md index d7c80500d7..b433166438 100644 --- a/pkg_auto/README.md +++ b/pkg_auto/README.md @@ -8,9 +8,6 @@ A quick start from blank state: - `cd scripts/main` - `PKG_AUTO="${PWD}/pkg_auto"` - `git worktree add --branch weekly-updates ../weekly-updates origin/buildbot/weekly-portage-stable-package-updates-2024-09-23` - - if package automation is still not merged into the `main` branch then: - - `git worktree add --branch pkg-auto ../pkg-auto origin/krnowak/pkg-auto` - - `PKG_AUTO="${PWD}/../pkg-auto/pkg_auto"` - prepare for generating reports (create a directory, download necessary stuff, create config): - `mkdir ../../weekly-updates` - `cd ../../weekly-updates` @@ -53,6 +50,8 @@ The entries in `changelog_stubs` should be reviewed about relevancy (minor SDK-o There may be also entries in `manual-work-needed` which may need addressing. Most often the reason is that the new package was added, or an existing package stopped being pulled in. This would need adding an entry to the `summary_stubs`. +Another thing that to do is to check [the security reports](https://github.com/flatcar/Flatcar/issues?q=is%3Aopen+is%3Aissue+label%3Aadvisory). If the updated package brings a fix for a security issue, it should be mentioned in the summary and an entry in a separate security changelog should be added. + Other scripts =============