diff --git a/pkg_auto/Makefile b/pkg_auto/Makefile new file mode 100644 index 0000000000..b2795d456a --- /dev/null +++ b/pkg_auto/Makefile @@ -0,0 +1,2 @@ +shellcheck: + docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh diff --git a/pkg_auto/config_template b/pkg_auto/config_template new file mode 100644 index 0000000000..c5c667d814 --- /dev/null +++ b/pkg_auto/config_template @@ -0,0 +1,47 @@ +# Path to the toplevel directory of the scripts repo. +scripts: .. + +# Path to the directory with auxiliary files. +aux: ../../downloads + +# Path to the directory where update reports will be stored. +reports: ../../reports + +# Base scripts branch for old state. +# +# Old state serves as an state before the updates. +# +# Optional, defaults to origin/main +#old-base: origin/main + +# Base scripts branch for new state. +# +# New state serves as an state that's either after the updates or to-be-updated. +# +# Optional, defaults to old-base value +#new-base: origin/main + +# Cleanups mode with semicolon-separated list of extra arguments. +# +# Can be: +# +# file,${path_to_file} +# stores cleanup steps in the file. Can be later sourced to perform cleanups. +# +# trap +# executes cleanups on script exit +# +# ignore +# performs no cleanups at all +# +# Optional, if not specified, defaults to ignore. +#cleanups: file,../../x-cleanups +#cleanups: trap +#cleanups: ignore + +# Override image name to use for an SDK container. +# +# Optional, defaults to +# flatcar-packages-${arch}:${last_nightly_version_id_in_base}-${last_nightly_build_id_in_base} +#amd64-sdk-img= +#arm64-sdk-img= diff --git a/pkg_auto/download_sdk_and_listings.sh b/pkg_auto/download_sdk_and_listings.sh new file mode 100755 index 0000000000..cfb0f57256 --- /dev/null +++ b/pkg_auto/download_sdk_and_listings.sh @@ -0,0 +1,238 @@ +#!/bin/bash + +## +## Downloads package SDKs from bincache and loads them with +## docker. Downloads package listings from bincache. Version can be +## taken either from the latest nightly tag in the passed scripts +## directory (with the -s option) or from specified version ID and +## build ID (with -v and -b options). The results are written to the +## passed downloads directory. +## +## Parameters: +## -b : build ID, conflicts with -s +## -h: this help +## -s : scripts repo directory, conflicts with -v and -b +## -v : version ID, conflicts with -s +## -nd: skip downloading of docker images +## -nl: skip downloading of listings +## -x : cleanup file +## +## Positional: +## 1: downloads directory +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" + +CLEANUP_FILE= +SCRIPTS= +VERSION_ID= +BUILD_ID= +SKIP_DOCKER= +SKIP_LISTINGS= + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -b) + if [[ -n ${SCRIPTS} ]]; then + fail '-b cannot be used at the same time with -s' + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -b' + fi + BUILD_ID=${2} + shift 2 + ;; + -h) + print_help + exit 0 + ;; + -s) + if [[ -n ${VERSION_ID} ]] || [[ -n ${BUILD_ID} ]]; then + fail '-s cannot be used at the same time with -v or -b' + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -s' + fi + SCRIPTS=${2} + shift 2 + ;; + -v) + if [[ -n ${SCRIPTS} ]]; then + fail '-v cannot be used at the same time with -s' + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -v' + fi + VERSION_ID=${2} + shift 2 + ;; + -x) + if [[ -z ${2:-} ]]; then + fail 'missing value for -x' + fi + CLEANUP_FILE=${2} + shift 2 + ;; + -nd) + SKIP_DOCKER=x + shift + ;; + -nl) + SKIP_LISTINGS=x + shift + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 1 ]]; then + fail 'Expected one positional parameter: a downloads directory' +fi + +DOWNLOADS_DIR=$(realpath "${1}"); shift + +if [[ -z ${SCRIPTS} ]] && [[ -z ${VERSION_ID} ]]; then + fail 'need to pass either -s or -v (latter with the optional -b too)' +fi + +if [[ -n ${CLEANUP_FILE} ]]; then + dirname_out "${CLEANUP_FILE}" cleanup_dir + # shellcheck disable=SC2154 # cleanup_dir is assigned in dirname_out + mkdir -p "${cleanup_dir}" + unset cleanup_dir + setup_cleanups file "${CLEANUP_FILE}" +else + setup_cleanups ignore +fi + +if [[ ! -d "${DOWNLOADS_DIR}" ]]; then + add_cleanup "rmdir ${DOWNLOADS_DIR@Q}" + mkdir "${DOWNLOADS_DIR}" +fi + +function download() { + local url output + url="${1}"; shift + output="${1}"; shift + + info "Downloading ${url}" + curl \ + --fail \ + --show-error \ + --location \ + --retry-delay 1 \ + --retry 60 \ + --retry-connrefused \ + --retry-max-time 60 \ + --connect-timeout 20 \ + "${url}" >"${output}" +} + +if [[ -n ${SCRIPTS} ]]; then + # shellcheck disable=SC1091 # sourcing generated file + VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") + # shellcheck disable=SC1091 # sourcing generated file + BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") +fi + +ver_plus="${VERSION_ID}${BUILD_ID:++}${BUILD_ID}" +ver_dash="${VERSION_ID}${BUILD_ID:+-}${BUILD_ID}" + +exts=(zst bz2 gz) + +# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds +zst_cmds=( + zstd +) + +# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds +bz2_cmds=( + lbunzip2 + pbunzip2 + bunzip2 +) + +# shellcheck disable=SC2034 # used indirectly as cmds_name and cmds +gz_cmds=( + unpigz + gunzip +) + +for arch in amd64 arm64; do + if [[ -z ${SKIP_DOCKER} ]]; then + packages_image_name="flatcar-packages-${arch}:${ver_dash}" + if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep -q -x -F "${packages_image_name}"; then + info "No ${packages_image_name} available in docker, pulling it from bincache" + for ext in "${exts[@]}"; do + tb="${DOWNLOADS_DIR}/packages-sdk-${arch}.tar.${ext}" + if [[ -s ${tb} ]]; then + break; + else + add_cleanup "rm -f ${tb@Q}" + if download "https://bincache.flatcar-linux.net/containers/${ver_dash}/flatcar-packages-${arch}-${ver_dash}.tar.${ext}" "${tb}"; then + break + fi + fi + done + info "Loading ${packages_image_name} into docker" + cmds_name="${ext}_cmds" + if ! declare -p "${cmds_name}" >/dev/null 2>/dev/null; then + fail "Failed to extract ${tb@Q} - no tools to extract ${ext@Q} files" + fi + declare -n cmds="${ext}_cmds" + loaded= + for cmd in "${cmds[@]}"; do + if ! command -v "${cmd}" >/dev/null; then + info "${cmd@Q} is not available" + continue + fi + info "Using ${cmd@Q} to extract the tarball" + "${cmd}" -d -c "${tb}" | docker load + add_cleanup "docker rmi ${packages_image_name@Q}" + loaded=x + break + done + if [[ -z ${loaded} ]]; then + fail "Failed to extract ${tb@Q} - no known available tool to extract it" + fi + unset -n cmds + fi + fi + + if [[ -z ${SKIP_LISTINGS} ]]; then + listing_dir="${DOWNLOADS_DIR}/${arch}" + add_cleanup "rmdir ${listing_dir@Q}" + mkdir "${listing_dir}" + base_url="https://bincache.flatcar-linux.net/images/${arch}/${ver_plus}" + + for infix in '' 'rootfs-included-sysexts'; do + index_html="${listing_dir}/${infix}${infix:+-}index.html" + url="${base_url}${infix:+/}${infix}" + add_cleanup "rm -f ${index_html@Q}" + download "${url}/" "${index_html}" + + # get names of all files ending with _packages.txt + mapfile -t listing_files < <(grep -F '_packages.txt"' "${index_html}" | sed -e 's#.*"\(\./\)\?\([^"]*\)".*#\2#') + + for listing in "${listing_files[@]}"; do + info "Downloading ${listing} for ${arch}" + listing_path="${listing_dir}/${listing}" + add_cleanup "rm -f ${listing_path@Q}" + download "${url}/${listing}" "${listing_path}" + done + done + fi +done +info 'Done' diff --git a/pkg_auto/generate_config.sh b/pkg_auto/generate_config.sh new file mode 100755 index 0000000000..b3b64e23ad --- /dev/null +++ b/pkg_auto/generate_config.sh @@ -0,0 +1,148 @@ +#!/bin/bash + +set -euo pipefail + +## +## Generate a config. +## +## Parameters: +## -a: aux directory +## -d: debug package - list many times +## -h: this help +## -i: SDK image override in form of ${arch}:${name}, +## -n: new base +## -o: old base +## -r: reports directory +## -s: scripts directory +## -x: cleanup opts +## +## Positional: +## 1: path for config file +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" + +# shellcheck disable=SC2034 # used by name below +gc_aux_directory='' +# shellcheck disable=SC2034 # used by name below +gc_new_base='' +# shellcheck disable=SC2034 # used by name below +gc_old_base='' +# shellcheck disable=SC2034 # used by name below +gc_reports_directory='' +# shellcheck disable=SC2034 # used by name below +gc_scripts_directory='' +# shellcheck disable=SC2034 # used by name below +gc_cleanup_opts='' +# gc_${arch}_sdk_img are declared on demand +gc_debug_packages=() + +declare -A opt_map +opt_map=( + ['-a']=gc_aux_directory + ['-n']=gc_new_base + ['-o']=gc_old_base + ['-r']=gc_reports_directory + ['-s']=gc_scripts_directory + ['-x']=gc_cleanup_opts +) + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -d) + if [[ -z ${2:-} ]]; then + fail 'missing value for -d' + fi + gc_debug_packages+=( "${2}" ) + shift 2 + ;; + -h) + print_help + exit 0 + ;; + -i) + if [[ -z ${2:-} ]]; then + fail 'missing value for -i' + fi + arch=${2%%:*} + image_name=${2#*:} + var_name="gc_${arch}_sdk_img" + unset arch + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${var_name}" + unset var_name + # shellcheck disable=SC2178 # shellcheck does not grok refs + ref=${image_name} + unset image_name + unset -n ref + shift 2 + ;; + --) + shift + break + ;; + -*) + var_name=${opt_map["${1}"]:-} + if [[ -z ${var_name} ]]; then + fail "unknown flag '${1}'" + fi + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${var_name}" + # shellcheck disable=SC2178 # shellcheck does not grok refs + ref=${2} + unset -n ref + unset var_name + shift 2 + ;; + *) + break + ;; + esac +done + +join_by gc_debug_packages_csv ',' "${gc_debug_packages[@]}" + +declare -a pairs +pairs=( + 'scripts' gc_scripts_directory + 'aux' gc_aux_directory + 'reports' gc_reports_directory + 'old-base' gc_old_base + 'new-base' gc_new_base + 'cleanups' gc_cleanup_opts + 'amd64-sdk-img' gc_amd64_sdk_img + 'arm64-sdk-img' gc_arm64_sdk_img + 'debug-packages' gc_debug_packages_csv +) + +if [[ ${#} -ne 1 ]]; then + fail 'expected one positional parameters: a path for the config' +fi + +config=${1}; shift + +{ + opt_idx=0 + name_idx=1 + while [[ ${name_idx} -lt "${#pairs[@]}" ]]; do + opt=${pairs["${opt_idx}"]} + name=${pairs["${name_idx}"]} + opt_idx=$((opt_idx + 2)) + name_idx=$((name_idx + 2)) + # shellcheck disable=SC2178 # shellcheck does not grok refs + declare -n ref="${name}" + if [[ -n ${ref:-} ]]; then + printf '%s: %s\n' "${opt}" "${ref}" + fi + unset -n ref + done + unset opt_idx name_idx +} >"${config}" + +info 'Done, but note that the config is not guaranteed to be valid!' diff --git a/pkg_auto/generate_reports.sh b/pkg_auto/generate_reports.sh new file mode 100755 index 0000000000..0b6df1ab4f --- /dev/null +++ b/pkg_auto/generate_reports.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +set -euo pipefail + +## +## Generates reports. +## +## Parameters: +## -w: path to use for workdir +## -h: this help +## +## Positional: +## 1: config file +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/pkg_auto_lib.sh" + +workdir='' + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -w) + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + workdir=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 1 ]]; then + fail 'expected one positional parameter: a config file' +fi + +config_file=${1}; shift + +setup_workdir_with_config "${workdir}" "${config_file}" +generate_package_update_reports diff --git a/pkg_auto/impl/cleanups.sh b/pkg_auto/impl/cleanups.sh new file mode 100644 index 0000000000..66d6d050e2 --- /dev/null +++ b/pkg_auto/impl/cleanups.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +# +# Cleanups +# +# This is basically a command stack to be executed in some deferred +# time. So last command added will be the first to be executed at +# cleanup time. +# +# Cleanups are implemented through two functions, setup_cleanups and +# add_cleanup, prefixed with _${type}_. So for type "foo" the +# functions would be _foo_setup_cleanups and _foo_add_cleanup. +# +# setup_cleanup may take some extra parameters that are specific to +# the type. For example file type takes a path where the commands will +# be stored. +# +# add_cleanup takes one or more command to add to the cleanup stack. +# + +if [[ -z ${__CLEANUPS_SH_INCLUDED__:-} ]]; then +__CLEANUPS_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Sets up cleanup stack of a given type. A type may need some extra +# parameter, which comes after a comma. Possible types are: +# +# - file: requires extra argument about cleanup file location; an +# example could be "file,/path/to/cleanups-file" +# - trap: executed on shell exit +# - ignore: noop +# +# Params: +# +# 1 - type of cleanup +function setup_cleanups() { + local kind + kind=${1}; shift + + if [[ -n ${_cleanups_sh_cleanup_kind_:-} ]]; then + fail "cannot set cleanups to '${kind}', they are already set up to '${_cleanups_sh_cleanup_kind_}'" + fi + + declare -g _cleanups_sh_cleanup_kind_ + + _ensure_valid_cleanups_sh_cleanup_kind_ "${kind}" + _cleanups_sh_cleanup_kind_=${kind} + _call_cleanup_func setup_cleanups "${@}" +} + +# Adds commands to the cleanup stack. +# +# Params: +# +# @ - commands, one per parameter +function add_cleanup() { + _call_cleanup_func add_cleanup "${@}" +} + +# +# Implementation details. +# + +# "file" cleanups + +function _file_setup_cleanups() { + if [[ ${#} -ne 1 ]]; then + fail 'missing cleanup file location argument for file cleanups' + fi + + declare -g _file_cleanup_file + _file_cleanup_file=$(realpath "${1}"); shift + add_cleanup "rm -f ${_file_cleanup_file@Q}" +} + +function _file_add_cleanup() { + local fac_cleanup_dir tmpfile + dirname_out "${_file_cleanup_file}" fac_cleanup_dir + + tmpfile=$(mktemp -p "${fac_cleanup_dir}") + printf '%s\n' "${@}" >"${tmpfile}" + if [[ -f "${_file_cleanup_file}" ]]; then + cat "${_file_cleanup_file}" >>"${tmpfile}" + fi + mv -f "${tmpfile}" "${_file_cleanup_file}" +} + +# "trap" cleanups + +function _trap_update_trap() { + # shellcheck disable=SC2064 # using double quotes on purpose instead of single quotes + trap "${_trap_cleanup_actions}" EXIT +} + +function _trap_setup_cleanups() { + declare -g _trap_cleanup_actions + _trap_cleanup_actions=':' + + declare -g -A _trap_cleanup_snapshots + _trap_cleanup_snapshots=() + + _trap_update_trap +} + +function _trap_add_cleanup() { + local tac_joined + join_by tac_joined ' ; ' "${@/%/' || :'}" + _trap_cleanup_actions="${tac_joined} ; ${_trap_cleanup_actions}" + _trap_update_trap +} + +# "ignore" cleanups + +function _ignore_setup_cleanups() { + : +} + +function _ignore_add_cleanup() { + : +} + +function _ensure_valid_cleanups_sh_cleanup_kind_() { + local kind + kind=${1}; shift + + local -a functions=( + setup_cleanups + add_cleanup + ) + + local func + for func in "${functions[@]/#/_${kind}_}"; do + if ! declare -pF "${func}" >/dev/null 2>/dev/null; then + fail "kind '${kind}' is not a valid cleanup kind, function '${func}' is not defined" + fi + done +} + +function _call_cleanup_func() { + local func_name + func_name=${1}; shift + if [[ -z "${_cleanups_sh_cleanup_kind_}" ]]; then + _cleanups_sh_cleanup_kind_='trap' + fi + + local func + func="_${_cleanups_sh_cleanup_kind_}_${func_name}" + + "${func}" "${@}" +} + +fi diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh new file mode 100644 index 0000000000..04f6367a88 --- /dev/null +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -0,0 +1,485 @@ +#!/bin/bash + +if [[ -z ${__INSIDE_SDK_CONTAINER_LIB_SH_INCLUDED__:-} ]]; then +__INSIDE_SDK_CONTAINER_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Invokes emerge to get a report about built packages for a given +# metapackage in the given root that has a portage configuration. +# +# Params: +# +# 1 - root filesystem with the portage config +# 2 - metapackage to get the deps from +function emerge_pretend() { + local root package + root=${1}; shift + package=${1}; shift + + # Probably a bunch of those flags are not necessary, but I'm not + # touching it - they seem to be working. :) + local -a emerge_opts=( + --config-root="${root}" + --root="${root}" + --sysroot="${root}" + --pretend + --columns + --nospinner + --oneshot + --color n + --emptytree + --verbose + --verbose-conflicts + --verbose-slot-rebuilds y + --changed-deps y + --changed-deps-report y + --changed-slot y + --changed-use + --newuse + --complete-graph y + --deep + --rebuild-if-new-slot y + --rebuild-if-unbuilt y + --with-bdeps y + --dynamic-deps y + --update + --ignore-built-slot-operator-deps y + --selective n + --keep-going y + ) + local rv + rv=0 + emerge "${emerge_opts[@]}" "${package}" || rv=${?} + if [[ ${rv} -ne 0 ]]; then + echo "WARNING: emerge exited with status ${rv}" >&2 + fi +} + +# Gets package list for SDK. +function package_info_for_sdk() { + local root + root='/' + + ignore_crossdev_stuff "${root}" + emerge_pretend "${root}" coreos-devel/sdk-depends + revert_crossdev_stuff "${root}" +} + +# Gets package list for board of a given architecture. +# +# Params: +# +# 1 - architecture +function package_info_for_board() { + local arch + arch=${1}; shift + + local root + root="/build/${arch}-usr" + + # Ignore crossdev stuff in both SDK root and board root - emerge + # may query SDK stuff for the board packages. + ignore_crossdev_stuff / + ignore_crossdev_stuff "${root}" + emerge_pretend "${root}" coreos-devel/board-packages + revert_crossdev_stuff "${root}" + revert_crossdev_stuff / +} + +# Set the directory where the emerge output and the results of +# processing it will be stored. EO stands for "emerge output" +# +# Params: +# +# 1 - directory path +function set_eo() { + local dir=${1}; shift + + SDK_EO="${dir}/sdk-emerge-output" + BOARD_EO="${dir}/board-emerge-output" + # shellcheck disable=SC2034 # used indirectly in cat_eo_f + SDK_EO_F="${SDK_EO}-filtered" + # shellcheck disable=SC2034 # used indirectly in cat_eo_f + BOARD_EO_F="${BOARD_EO}-filtered" + # shellcheck disable=SC2034 # used indirectly in cat_eo_w + SDK_EO_W="${SDK_EO}-warnings" + # shellcheck disable=SC2034 # used indirectly in cat_eo_w + BOARD_EO_W="${BOARD_EO}-warnings" +} + +# Print the contents of file, path of which is stored in a variable of +# a given name. +# +# Params: +# +# 1 - name of the variable +function cat_var() { + local var_name + var_name=${1}; shift + local -n ref="${var_name}" + + if [[ -z "${ref+isset}" ]]; then + fail "${var_name} unset" + fi + if [[ ! -e "${ref}" ]]; then + fail "${ref} does not exist" + fi + + cat "${ref}" +} + +# Print contents of the emerge output of a given kind. Kind can be +# either either sdk or board. +# +# Params: +# +# 1 - kind +function cat_eo() { + local kind + kind=${1}; shift + + cat_var "${kind^^}_EO" +} + +# Print contents of the filtered emerge output of a given kind. Kind +# can be either either sdk or board. The filtered emerge output +# contains only lines with package information. +# +# Params: +# +# 1 - kind +function cat_eo_f() { + local kind + kind=${1}; shift + cat_var "${kind^^}_EO_F" +} + +# Print contents of a file that stores warnings that were printed by +# emerge. The warnings are specific to a kind (sdk or board). +# +# Params: +# +# 1 - kind +function cat_eo_w() { + local kind + kind=${1}; shift + + cat_var "${kind^^}_EO_W" +} + +# JSON output would be more verbose, but probably would not require +# these abominations below. But, alas, emerge doesn't have that yet. + +# status package name version slot repo target (opt) keyvals size +# |--------------| |----------| |#-g1-----------#--#-g2-#| |--|-g------| |-g----------#-#-g-----| |---| +# [ebuild R ~] virtual/rust [1.71.1:0/llvm-16::coreos] to /some/root USE="-rustfmt" FOO="bar" 0 KiB +# +# Actually, there can also be another "version slot repo" part between +# the first "version slot repo" and "target" part. +STATUS_RE='\[[^]]*]' # 0 groups +PACKAGE_NAME_RE='[^[:space:]]*' # 0 groups +VER_SLOT_REPO_RE='\[\([^]]\+\)::\([^]]\+\)]' # 2 groups +TARGET_RE='to[[:space:]]\+\([^[:space:]]\)\+' # 1 group +KEYVALS_RE='\([[:space:]]*[A-Za-z0-9_]*="[^"]*"\)*' # 1 group (but containing only the last pair!) +SIZE_RE='[[:digit:]]\+[[:space:]]*[[:alpha:]]*B' # 0 groups +SPACES_RE='[[:space:]]\+' # 0 groups +NONSPACES_RE='[^[:space:]]\+' # 0 groups +NONSPACES_WITH_COLON_RE='[^[:space:]]*:' # 0 groups + +FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${VER_SLOT_REPO_RE}"'\)\{1,2\}\('"${SPACES_RE}${TARGET_RE}"'\)\?\('"${SPACES_RE}${KEYVALS_RE}"'\)*'"${SPACES_RE}${SIZE_RE}"'$' + +# Filters sdk reports to get the package information. +function filter_sdk_eo() { + cat_eo sdk | xgrep -e "${FULL_LINE_RE}" +} + +# Filters board reports for a given arch to get the package +# information. +# +# Params: +# +# 1 - architecture +function filter_board_eo() { + local arch + arch=${1}; shift + + # Replace ${arch}-usr in the output with a generic word BOARD. + cat_eo board | \ + xgrep -e "${FULL_LINE_RE}" | \ + sed -e "s#/build/${arch}-usr/#/build/BOARD/#" +} + +# Filters sdk reports to get anything but the package information +# (i.e. junk). +function junk_sdk_eo() { + cat_eo sdk | xgrep -v -e "${FULL_LINE_RE}" +} + +# Filters board reports to get anything but the package information +# (i.e. junk). +function junk_board_eo() { + cat_eo board | xgrep -v -e "${FULL_LINE_RE}" +} + +# More regexp-like abominations follow. + +# There may also be a line like: +# +# [blocks B ] / is masked +# (like for sys-libs/glibc and cross-x86_64-cros-linux-gnu/glibc), +# because it has no keywords. In theory, we could try updating +# /etc/portage/package.mask/cross- file created by the +# crossdev tool to unmask the new version, but it's an unnecessary +# hassle - native and cross package are supposed to be the same ebuild +# anyway, so update information about cross package is redundant. +# +# Params: +# +# 1 - root directory +# 2 - ID of the crossdev repository (optional, defaults to x-crossdev) +function ignore_crossdev_stuff() { + local root crossdev_repo_id + root=${1}; shift + crossdev_repo_id=${1:-x-crossdev}; shift || : + + local crossdev_repo_path + crossdev_repo_path=$(portageq get_repo_path "${root}" "${crossdev_repo_id}") + + local ics_path ics_dir + get_provided_file "${root}" ics_path + dirname_out "${ics_path}" ics_dir + + sudo mkdir -p "${ics_dir}" + env --chdir="${crossdev_repo_path}" find . -type l | \ + cut -d/ -f2-3 | \ + sed -e 's/$/-9999/' | \ + sudo tee "${ics_path}" >/dev/null +} + +# Reverts effects of the ignore_crossdev_stuff function. +# +# Params: +# +# 1 - root directory +function revert_crossdev_stuff() { + local root + root=${1}; shift + + local ics_path ics_dir + get_provided_file "${root}" ics_path + dirname_out "${ics_path}" ics_dir + + sudo rm -f "${ics_path}" + if dir_is_empty "${ics_dir}"; then + sudo rmdir "${ics_dir}" + fi +} + +# Checks if the expected reports were generated by emerge. +function ensure_valid_reports() { + local kind var_name + for kind in sdk board; do + var_name="${kind^^}_EO_F" + if [[ ! -s ${!var_name} ]]; then + fail "report files are missing or are empty" + fi + done +} + +# Drops the empty warning files in given directory. +# +# Params: +# +# 1 - path to the directory +function clean_empty_warning_files() { + local dir + dir=${1}; shift + + local file + for file in "${dir}/"*'-warnings'; do + if [[ ! -s ${file} ]]; then + rm -f "${file}" + fi + done +} + +fi diff --git a/pkg_auto/impl/mvm.sh b/pkg_auto/impl/mvm.sh new file mode 100644 index 0000000000..eded4aa54a --- /dev/null +++ b/pkg_auto/impl/mvm.sh @@ -0,0 +1,573 @@ +#!/bin/bash + +# +# "mvm" stands for "multi-valued map", so these are maps of scalars +# (strings, numbers) to other container (arrays or maps) +# +# mvm is implemented with a map that contains some predefined keys, +# like "name", "constructor", "storage", etc. +# +# The "storage" field is the actual "map" part of the "mvm", at the +# values stored in it are names of the global variables being the +# "multi-valued" part of the "mvm". In the code these variables are +# referred to as "mvc" meaning "multi-value container". +# +# The "constructor" and "destructor" fields are here to properly +# implement creating and destroying mvcs. The "adder" field is for +# adding elements to an mvc. +# +# There is also a "counter" field which, together with the "name" +# field, is used for creating the names for mvc variables. +# +# The "extras" field is for user-defined mapping. The mvm will clear +# the mapping itself, but if the values are anything else than simple +# scalars (e.g. names of variables) then the cleanup of those is +# user's task. +# +# There is also an optional field named "iteration_helper" which is a +# callback invoked when iterating over the mvm. +# +# In order to implement a new mvc type, the following functions need +# to be implemented: +# +# _constructor - takes an mvc name; should create an mvc with the +# passed name. +# _destructor - takes an mvc name; should unset an mvc with the +# passed name, should likely take care of cleaning +# up the values stored in the mvc +# _adder - takes an mvc name and values to be added; should add +# the values to the mvc +# _iteration_helper - optional; takes a key, an mvc name, a +# callback and extra arguments to be +# forwarded to the callback; should invoke +# the callback with the extra arguments, the +# key, the mvc name and optionally some +# extra arguments the helper deems useful + +if [[ -z ${__MVM_SH_INCLUDED__:-} ]]; then +__MVM_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +# Used for creating unique names for extras and storage maps. +MVM_COUNTER=0 + +# Array mvm, the default. Provides an iteration helper that sends all +# the array values to the iteration callback. + +function mvm_mvc_array_constructor() { + local array_var_name + array_var_name=${1}; shift + + declare -g -a "${array_var_name}" + + local -n array_ref=${array_var_name} + array_ref=() +} + +function mvm_mvc_array_destructor() { + local array_var_name + array_var_name=${1}; shift + + unset "${array_var_name}" +} + +function mvm_mvc_array_adder() { + local array_var_name + array_var_name=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n array_ref=${array_var_name} + + array_ref+=( "${@}" ) +} + +# iteration_helper is optional +function mvm_mvc_array_iteration_helper() { + local key array_var_name callback + key=${1}; shift + array_var_name=${1}; shift + callback=${1}; shift + # rest are extra args passed to cb + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n array_ref=${array_var_name} + "${callback}" "${@}" "${key}" "${array_var_name}" "${array_ref[@]}" +} + +# Map mvm. When adding elements to the mvc, it is expected that the +# number of items passed will be even. Odd elements will be used as +# keys, even elements will be used as values. +# +# No iteration helper. + +function mvm_mvc_map_constructor() { + local map_var_name + map_var_name=${1}; shift + + declare -g -A "${map_var_name}" + + local -n map_ref=${map_var_name} + map_ref=() +} + +function mvm_mvc_map_destructor() { + local map_var_name + map_var_name=${1}; shift + + unset "${map_var_name}" +} + +function mvm_mvc_map_adder() { + local map_var_name + map_var_name=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n map_ref=${map_var_name} + + while [[ ${#} -gt 1 ]]; do + # shellcheck disable=SC2034 # it's a reference to external variable + map_ref["${1}"]=${2} + shift 2 + done +} + +# Set mvm. Behaves like array mvm, but all elements in each set are +# unique and the order of elements is not guaranteed to be the same as +# order of insertions. + +function mvm_mvc_set_constructor() { + local set_var_name + set_var_name=${1}; shift + + declare -g -A "${set_var_name}" + + # shellcheck disable=SC2178 # shellcheck does not grok refs + local -n set_ref=${set_var_name} + set_ref=() +} + +function mvm_mvc_set_destructor() { + local set_var_name + set_var_name=${1} + + unset "${set_var_name}" +} + +function mvm_mvc_set_adder() { + local set_var_name + set_var_name=${1}; shift + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${set_var_name} + while [[ ${#} -gt 0 ]]; do + set_ref["${1}"]=x + shift + done +} + +# iteration_helper is optional +function mvm_mvc_set_iteration_helper() { + local key map_var_name callback + + key=${1}; shift + set_var_name=${1}; shift + callback=${1}; shift + # rest are extra args passed to cb + + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${set_var_name} + "${callback}" "${@}" "${key}" "${set_var_name}" "${!set_ref[@]}" +} + +# mvm API + +# Creates a new mvm with a passed name, optionally type and +# extras. The name must be globally unique. The type is optional. If +# no type is passed, an array mvm will be assumed. Otherwise the type +# must be valid, i.e. it must provide a constructor, a destructor, an +# adder and, optionally, an iteration helper. The built in types are +# "mvm_mvc_array", "mvm_mvc_set" and "mvm_mvc_map". If any extras are +# passed, they must be preceded with a double dash to avoid ambiguity +# between type and a first extras key. Extras are expected to be even +# in count, odd elements will be used as keys, even elements will be +# used as values. +# +# Params: +# +# 1 - name of the mvm +# @ - optional mvc type, optionally followed by double dash and extras +# key-value pairs. +function mvm_declare() { + local mvm_var_name + mvm_var_name=${1}; shift + + if declare -p "${mvm_var_name}" >/dev/null 2>/dev/null; then + fail "variable ${mvm_var_name} already exists, declaring mvm for it would clobber it" + fi + + local value_handler_prefix + value_handler_prefix='' + if [[ ${#} -gt 0 ]]; then + if [[ ${1} != '--' ]]; then + value_handler_prefix=${1} + shift + fi + if [[ ${#} -gt 0 ]]; then + if [[ ${1} != '--' ]]; then + fail "missing double-dash separator between optional value handler prefix and extra key value pairs for '${mvm_var_name}'" + fi + shift + fi + fi + if [[ -z ${value_handler_prefix} ]]; then + value_handler_prefix=mvm_mvc_array + fi + # rest are key value pairs for extras + + mvm_debug "${mvm_var_name}" "using prefix ${value_handler_prefix}" + + local constructor destructor adder iteration_helper + constructor="${value_handler_prefix}_constructor" + destructor="${value_handler_prefix}_destructor" + adder="${value_handler_prefix}_adder" + iteration_helper="${value_handler_prefix}_iteration_helper" + + local func + for func in "${constructor}" "${destructor}" "${adder}"; do + if ! declare -pF "${func}" >/dev/null 2>/dev/null; then + fail "'${func}' is not a function, is '${value_handler_prefix}' a valid prefix?" + fi + done + + if ! declare -pF "${iteration_helper}" >/dev/null 2>/dev/null; then + mvm_debug "${mvm_var_name}" "no interation helper available" + iteration_helper='' + fi + + local extras_idx storage_idx extras_map_var_name storage_map_var_name + extras_idx=$((MVM_COUNTER)) + storage_idx=$((MVM_COUNTER + 1)) + extras_map_var_name="mvm_stuff_${extras_idx}" + storage_map_var_name="mvm_stuff_${storage_idx}" + + MVM_COUNTER=$((MVM_COUNTER + 2)) + + declare -g -A "${mvm_var_name}" "${extras_map_var_name}" "${storage_map_var_name}" + + mvm_debug "${mvm_var_name}" "extras map: ${extras_map_var_name}, storage_map: ${storage_map_var_name}" + + local -n storage_map_ref=${storage_map_var_name} + storage_map_ref=() + + local -n mvm_ref=${mvm_var_name} + # shellcheck disable=SC2034 # it's a reference to external variable + mvm_ref=( + ['name']="${mvm_var_name}" + ['constructor']="${constructor}" + ['destructor']="${destructor}" + ['adder']="${adder}" + ['iteration_helper']="${iteration_helper}" + ['counter']=0 + ['extras']="${extras_map_var_name}" + ['storage']="${storage_map_var_name}" + ) + local -n extras_map_ref=${extras_map_var_name} + while [[ ${#} -gt 1 ]]; do + mvm_debug "${mvm_var_name}" "adding ${1} -> ${2} pair to extras" + extras_map_ref["${1}"]=${2} + shift 2 + done + if [[ ${#} -gt 0 ]]; then + fail "odd number of parameters for extra key value information for '${mvm_var_name}'" + fi +} + +# Takes a name of mvm, a callback, and extra parameters that will be +# forwarded to the callback. Before invoking the callback, the +# function will declare a local variable called "mvm" which is a +# reference to the variable with the passed name. The "mvm" variable +# can be used for easy access to the map within the callback. +# +# The convention is that the function foo_barize will use mvm_call to +# invoke a callback named foo_c_barize. The foo_c_barize function can +# invoke other _c_ infixed functions, like mvm_c_get_extra or +# mvm_c_get. +# +# Params: +# +# 1 - name of mvm variable +# 2 - name of the callback +# @ - arguments for the callback +function mvm_call() { + local name func + name=${1}; shift + func=${1}; shift + # rest are func args + + mvm_debug "${name}" "invoking ${func} with args: ${*@Q}" + + # The "mvm" variable can be used by ${func} now. + local -n mvm=${name} + "${func}" "${@}" +} + +# Internal function that generates a name for mvc based on passed name +# and counter. +function __mvm_mvc_name() { + local name counter mvc_name_var_name + name=${1}; shift + counter=${1}; shift + mvc_name_var_name=${1}; shift + local -n mvc_name_ref=${mvc_name_var_name} + + # shellcheck disable=SC2034 # it's a reference to external variable + mvc_name_ref="mvm_${name}_mvc_${counter}" +} + +# Destroy the mvm with passed name. +# +# Params: +# +# 1 - name of mvm to destroy +function mvm_unset() { + mvm_call "${1}" mvm_c_unset "${@:2}" +} + +# Helper function for mvm_unset invoked through mvm_call. +function mvm_c_unset() { + local counter name extras_map_var_name storage_map_var_name destructor mvm_mcu_mvc_name + + counter=${mvm['counter']} + name=${mvm['name']} + extras_map_var_name=${mvm['extras']} + storage_map_var_name=${mvm['storage']} + destructor=${mvm['destructor']} + + while [[ ${counter} -gt 0 ]]; do + counter=$((counter - 1)) + __mvm_mvc_name "${name}" "${counter}" mvm_mcu_mvc_name + "${destructor}" "${mvm_mcu_mvc_name}" + done + unset "${storage_map_var_name}" + unset "${extras_map_var_name}" + unset "${name}" +} + +# Gets an value from extras map for a given key. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - extra key +# 3 - name of a variable where the extra value will be stored +function mvm_get_extra() { + mvm_call "${1}" mvm_c_get_extra "${@:2}" +} + +# Helper function for mvm_get_extra invoked through mvm_call. +function mvm_c_get_extra() { + local extra extra_var_name + extra=${1}; shift + extra_var_name=${1}; shift + local -n extra_ref=${extra_var_name} + + local extras_map_var_name + extras_map_var_name=${mvm['extras']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n extras_map_ref=${extras_map_var_name} + + # shellcheck disable=SC2034 # it's a reference to external variable + extra_ref=${extras_map_ref["${extra}"]:-} +} + +# Gets a name of the mvc for a given key. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - key +# 3 - name of a variable where the mvc name will be stored +function mvm_get() { + mvm_call "${1}" mvm_c_get "${@:2}" +} + +# Helper function for mvm_get invoked through mvm_call. +function mvm_c_get() { + local key value_var_name + key=${1}; shift + value_var_name=${1}; shift + local -n value_ref=${value_var_name} + + local storage_map_var_name + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + + # shellcheck disable=SC2034 # it's a reference to external variable + value_ref=${storage_map_ref["${key}"]:-} +} + +# Internal function for creating a new mvc. +function __mvm_c_make_new_mvc() { + local key mvc_name_var_name + key=${1}; shift + mvc_name_var_name=${1}; shift + + local name counter storage_map_var_name + name=${mvm['name']} + counter=${mvm['counter']} + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + + __mvm_mvc_name "${name}" "${counter}" "${mvc_name_var_name}" + + local constructor + constructor=${mvm['constructor']} + + "${constructor}" "${!mvc_name_var_name}" + mvm['counter']=$((counter + 1)) + storage_map_ref["${key}"]=${!mvc_name_var_name} +} + +# Adds passed elements to the mvm under the given key. If an mvc for +# the key didn't exist in the mvm, it gets created. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - key +# @ - elements +function mvm_add() { + mvm_call "${1}" mvm_c_add "${@:2}" +} + +# Helper function for mvm_add invoked through mvm_call. +function mvm_c_add() { + local key + key=${1}; shift + # rest are values to add + + local adder mvm_mca_mvc_name + adder=${mvm['adder']} + mvm_c_get "${key}" mvm_mca_mvc_name + + if [[ -z ${mvm_mca_mvc_name} ]]; then + __mvm_c_make_new_mvc "${key}" mvm_mca_mvc_name + fi + "${adder}" "${mvm_mca_mvc_name}" "${@}" +} + +# Removes the key from the mvm. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - key +function mvm_remove() { + mvm_call "${1}" mvm_c_remove "${@:2}" +} + +# Helper function for mvm_remove invoked through mvm_call. +function mvm_c_remove() { + local key + key=${1}; shift + + local storage_map_var_name + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + + if [[ -z ${storage_map_ref["${key}"]:-} ]]; then + return 0 + fi + + local var_name=${storage_map_ref["${key}"]} + unset "storage_map_ref[${key}]" + + local destructor + destructor=${mvm['destructor']} + + "${destructor}" "${var_name}" +} + +# Iterates over the key-mvc pairs and invokes a callback for each. The +# function also takes some extra parameters to forward to the +# callback. The callback will receive, in order, extra parameters, a +# key, an mvc name, and possibly some extra parameters from the +# iteration helper, if such exists for the mvm. +# +# Params: +# +# 1 - name of the mvm variable +# 2 - callback +# @ - extra parameters forwarded to the callback +function mvm_iterate() { + mvm_call "${1}" mvm_c_iterate "${@:2}" +} + +# Helper function for mvm_iterate invoked through mvm_call. +function mvm_c_iterate() { + local callback + callback=${1}; shift + # rest are extra args passed to callback + + local storage_map_var_name helper + storage_map_var_name=${mvm['storage']} + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n storage_map_ref=${storage_map_var_name} + helper=${mvm['iteration_helper']} + + local key value + if [[ -n "${helper}" ]]; then + for key in "${!storage_map_ref[@]}"; do + value=${storage_map_ref["${key}"]} + "${helper}" "${key}" "${value}" "${callback}" "${@}" + done + else + for key in "${!storage_map_ref[@]}"; do + value=${storage_map_ref["${key}"]} + "${callback}" "${@}" "${key}" "${value}" + done + fi +} + +# debugging + +declare -A MVM_DEBUG_NAMES=() + +# Enables printing debugging info for a specified mvm. +# +# Params: +# +# 1 - name of the mvm variable +function mvm_debug_enable() { + local mvm_var_name=${1}; shift + MVM_DEBUG_NAMES["${mvm_var_name}"]=x +} + +# Print debugging info about the mvm if debugging for it was enabled +# beforehand. +# +# Params: +# +# 1 - name of the mvm variable +# @ - strings to be printed +function mvm_debug() { + local name=${1}; shift + + if [[ -n ${MVM_DEBUG_NAMES["${name}"]:-} ]]; then + info "MVM_DEBUG(${name}): ${*}" + fi +} + +# Disables printing debugging info for a specified mvm. +# +# Params: +# +# 1 - name of the mvm variable +function mvm_debug_disable() { + local mvm_var_name=${1}; shift + unset "MVM_DEBUG_NAMES[${mvm_var_name}]" +} + +fi diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh new file mode 100644 index 0000000000..dcb26e7368 --- /dev/null +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -0,0 +1,3255 @@ +#!/bin/bash + +# +# TODO: +# +# - Generate a report about missing build deps of board packages in +# sdk. These reports can be generated by processing sdk-pkgs-kv and +# board-bdeps reports, I think. +# +# - Mount Gentoo repo into the SDK container and set up emerge to use +# Gentoo as a primary repo, and portage-stable and coreos-overlay as +# overlays. That way if an updated package pulls in a new package we +# can notice it when it comes from Gentoo (emerge reports also +# source repo like sys-libs/glibc-2.35-r5::gentoo or something like +# this). This would make this script more robust. +# +# - Instead of having a list of packages to update, rather update them +# all in a single commit and have a list of exclusions. The reason +# is that, at this point, almost all of the packages in +# portage-stable are automatically updated, exclusions being usually +# temporary, so it would be better to have a short file with +# temporary exclusions rather than a long file with some commented +# out entries. This probably would render the sort_packages_list.py +# script unnecessary. On the other hand, the current mode of +# operation may be still useful for the coreos-overlay packages, +# because none of them are under automation (some are, though, under +# an ad-hoc automation via github actions). +# +# - Handle package appearance or disappearance. Currently, when a +# package ends up being unused (so it exists, but is not picked up, +# because some other package stopped depending on it) or removed, +# the package ends up in the manual-work-needed file. This probably +# could be handled as an entry in the summary stubs about being +# dropped. +# +# - Find unused packages and eclasses. +# +# - The rename handling should probably also change all instances of +# the old name everywhere outside portage-stable, otherwise emerge +# may fail when some our ebuild still uses the old name, maybe. +# + +# Needed to be enabled here to parse some globs inside the functions. +shopt -s extglob +# Saner defaults. +shopt -s nullglob +shopt -s dotglob + +if [[ -z ${__PKG_AUTO_LIB_SH_INCLUDED__:-} ]]; then +__PKG_AUTO_LIB_SH_INCLUDED__=x + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" +source "${PKG_AUTO_IMPL_DIR}/cleanups.sh" + +# Sets up the workdir using the passed config. The config can be +# created basing on the config_template file or using the +# generate_config script. +# +# The path to the workdir can be empty - the function will then create +# a temporary directory. +# +# This also sets the WORKDIR global variable, allowing other function +# to be invoked. +# +# Params: +# +# 1 - path to the workdir +# 2 - path to the config file +function setup_workdir_with_config() { + local workdir file + workdir=${1}; shift + config_file=${1}; shift + + local cfg_scripts cfg_aux cfg_reports cfg_old_base cfg_new_base + local -a cfg_cleanups cfg_debug_packages + local -A cfg_overrides + + # some defaults + cfg_old_base='origin/main' + cfg_new_base='' + cfg_cleanups=('ignore') + cfg_overrides=() + cfg_debug_packages=() + + local line key value swwc_stripped var_name arch + while read -r line; do + strip_out "${line%%:*}" swwc_stripped + key=${swwc_stripped} + strip_out "${line#*:}" swwc_stripped + value=${swwc_stripped} + if [[ -z ${value} ]]; then + fail "empty value for ${key} in config" + fi + case ${key} in + scripts|aux|reports) + var_name="cfg_${key//-/_}" + local -n var=${var_name} + var=$(realpath "${value}") + unset -n var + ;; + old-base|new-base) + var_name="cfg_${key//-/_}" + local -n var=${var_name} + var=${value} + unset -n var + ;; + cleanups|debug-packages) + var_name="cfg_${key//-/_}" + mapfile -t "${var_name}" <<<"${value//,/$'\n'}" + ;; + *-sdk-img) + arch=${key%%-*} + # shellcheck disable=SC2034 # used by name below + cfg_overrides["${arch}"]=${value} + ;; + esac + done < <(cat_meaningful "${config_file}") + if [[ -z "${cfg_new_base}" ]]; then + cfg_new_base=${cfg_old_base} + fi + for key in scripts aux reports; do + var_name="cfg_${key//-/_}" + if [[ -z "${!var_name}" ]]; then + fail "${key} was not specified in config" + fi + done + + setup_cleanups "${cfg_cleanups[@]}" + setup_workdir "${workdir}" + add_cleanup "rm -f ${WORKDIR@Q}/config" + cp -a "${config_file}" "${WORKDIR}/config" + setup_worktrees_in_workdir "${cfg_scripts}" "${cfg_old_base}" "${cfg_new_base}" "${cfg_reports}" "${cfg_aux}" + override_sdk_image_names cfg_overrides + add_debug_packages "${cfg_debug_packages[@]}" +} + +# Goes over the list of automatically updated packages and synces them +# with packages from Gentoo repo. Cleans up missing packages. +# +# The function can only be called after the workdir has been set up +# with setup_workdir_with_config. +# +# Params: +# +# 1 - a path to the Gentoo repo +function perform_sync_with_gentoo() { + local gentoo + gentoo=$(realpath "${1}"); shift + + run_sync "${gentoo}" + handle_missing_in_scripts + handle_missing_in_gentoo "${gentoo}" +} + +# Generates package update reports. Duh. +# +# The function can only be called after the workdir has been set up +# with setup_workdir_with_config. +# +# The location of the reports is specified in the config that was +# passed to setup_workdir_with_config. +function generate_package_update_reports() { + generate_sdk_reports + handle_gentoo_sync +} + +# Saves the new state to a git branch in scripts. +# +# The function can only be called after the workdir has been set up +# with setup_workdir_with_config. +# +# Params: +# +# 1 - name of the new branch +function save_new_state() { + local branch_name + branch_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + info "saving new state to branch ${branch_name}" + # shellcheck disable=SC2153 # SCRIPTS is not a misspelling, it comes from globals file + git -C "${SCRIPTS}" branch --force "${branch_name}" "${NEW_STATE_BRANCH}" +} + +# +# +# Implementation details, do not use directly in scripts sourcing this file. +# +# + +# Creates a workdir, the path to which is stored in WORKDIR global +# variable. +# +# 1 - predefined work directory path (optional) +function setup_workdir() { + local workdir + workdir=${1:-} + if [[ -z "${workdir}" ]]; then + workdir=$(mktemp --tmpdir --directory 'pkg-auto-workdir.XXXXXXXX') + else + if [[ -e ${workdir} ]] && [[ ! -d ${workdir} ]]; then + fail "Expected ${workdir@Q} to be a directory" + fi + if [[ -d ${workdir} ]] && ! dir_is_empty "${workdir}"; then + fail "Expected ${workdir@Q} to be an empty directory" + fi + fi + + declare -g WORKDIR + WORKDIR=$(realpath "${workdir}") + add_cleanup "rmdir ${WORKDIR@Q}" + mkdir -p "${WORKDIR}" + + setup_initial_globals_file +} + +# Sets up worktrees for the old and new state inside WORKDIR. Creates +# the globals file inside WORKDIR. +# +# 1 - path to scripts repo +# 2 - base for the old state worktree (e.g. origin/main) +# 3 - base for the new state worktree (e.g. origin/main) +# 4 - path to reports directory +function setup_worktrees_in_workdir() { + local scripts old_base new_base reports_dir aux_dir + scripts=${1}; shift + old_base=${1}; shift + new_base=${1}; shift + reports_dir=${1}; shift + aux_dir=${1}; shift + + local old_state new_state + old_state="${WORKDIR}/old_state" + new_state="${WORKDIR}/new_state" + + # create reports directory now - there may be some developer + # warnings afoot + mkdir -p "${reports_dir}" + + setup_worktree "${scripts}" "${old_base}" "old-state-${RANDOM}" "${old_state}" + setup_worktree "${scripts}" "${new_base}" "new-state-${RANDOM}" "${new_state}" + extend_globals_file "${scripts}" "${old_state}" "${new_state}" "${reports_dir}" "${aux_dir}" +} + +# Adds overridden SDK image names to the globals file. +# +# Params: +# +# 1 - name of a map variable; should be a mapping of architecture to +# the image name +function override_sdk_image_names() { + local -n overrides_map_ref=${1} + + if [[ ${#overrides_map_ref[@]} -eq 0 ]]; then + return 0 + fi + + local arch image_name upcase_arch + local -a lines + lines=() + for arch in "${!overrides_map_ref[@]}"; do + image_name=${overrides_map_ref["${arch}"]} + upcase_arch=${arch^^} + if [[ ${#lines[@]} -eq 0 ]]; then + # separate overrides from initial values + lines+=( '' ) + fi + lines+=( "${upcase_arch}_PACKAGES_IMAGE=${image_name@Q}" ) + done + append_to_globals "${lines[@]}" +} + +# Adds information about packages to be debugged to the globals file. +# +# Params: +# +# @ - a list of packages to be debugged +function add_debug_packages() { + local -a prepared lines + prepared=( "${@@Q}" ) + prepared=( "${prepared[@]/#/' ['}" ) + prepared=( "${prepared[@]/%/']=x'}" ) + lines=( + '' + 'local -A DEBUG_PACKAGES' + '' + 'DEBUG_PACKAGES=(' + "${prepared[@]}" + ')' + ) + append_to_globals "${lines[@]}" +} + +# Appends passed lines to the globals file. +# +# Params: +# +# @ - lines to append +function append_to_globals() { + local globals_file + globals_file="${WORKDIR}/globals" + if [[ ! -e "${globals_file}" ]]; then + fail "globals not set yet in workdir" + fi + + lines_to_file "${globals_file}" "${@}" +} + +# Processes the update files in portage-stable and coreos-overlay to +# figure out potential package renames. The results are stored in the +# passed map. +# +# Params: +# +# 1 - name of a map variable; will be a mapping of old package name to +# new package name +function process_profile_updates_directory() { + local from_to_map_var_name=${1}; shift + + local -a ppud_ordered_names + get_ordered_update_filenames ppud_ordered_names + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local bf ps_f co_f pkg f line old new + local -a fields + local -A from_to_f=() + mvm_declare ppud_to_from_set_mvm mvm_mvc_set + for bf in "${ppud_ordered_names[@]}"; do + # coreos-overlay updates may overwrite updates from + # portage-stable, but only from the file of the same name + # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file + ps_f=${NEW_PORTAGE_STABLE}/profiles/updates/${bf} + # shellcheck disable=SC2153 # NEW_COREOS_OVERLAY is not a misspelling, it comes from globals file + co_f=${NEW_COREOS_OVERLAY}/profiles/updates/${bf} + for f in "${ps_f}" "${co_f}"; do + if [[ ! -f ${f} ]]; then + continue + fi + while read -r line; do + if [[ ${line} != 'move '* ]]; then + # other possibility is "slotmove" - we don't care + # about those. + continue + fi + mapfile -t fields <<<"${line// /$'\n'}" + if [[ ${#fields[@]} -ne 3 ]]; then + fail_lines \ + "Malformed line ${line@Q} in updates file ${f@Q}." \ + "The line should have 3 fields, has ${#fields[*]}." + fi + from_to_f["${fields[1]}"]=${fields[2]} + done <"${f}" + done + for old in "${!from_to_f[@]}"; do + new=${from_to_f["${old}"]} + update_rename_maps "${from_to_map_var_name}" ppud_to_from_set_mvm "${old}" "${new}" + done + done + + mvm_unset ppud_to_from_set_mvm +} + +# Gets a merged and ordered list of update files from portage-stable +# and coreos-overlay and stores the in the passed array. The files +# have names like Q1-2018, Q1-2023, Q2-2019 and so on. We need to sort +# them by year, then by quarter. +# +# Params: +# +# 1 - name of a array variable where the ordered names will be stored +function get_ordered_update_filenames() { + local ordered_names_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -A names_set=() + local f + + for f in "${NEW_PORTAGE_STABLE}/profiles/updates/"* "${NEW_COREOS_OVERLAY}/profiles/updates/"*; do + names_set["${f##*/}"]=x + done + + mapfile -t "${ordered_names_var_name}" < <(printf '%s\n' "${!names_set[@]}" | sort --field-separator=- --key=2n --key=1n) +} + +# Updates the rename map with the new "old to new package rename". It +# tries to be smart about the rename sequences (though not sure if +# this is necessary, really). If in older update file package foo was +# renamed to bar and in current update file the bar package is renamed +# to quux, then this function adds an entry about the "bar to quux" +# rename, but also updates the older entry about "foo to bar" rename +# to "foo to quux" rename. +# +# Params: +# +# 1 - name of the renames map variable; should be a mapping of old to +# new names +# 2 - name of the set mvm variable; should be a mapping of new name to +# a set of old names (a reverse mapping to the renames map) +# 3 - old name +# 4 - new name +function update_rename_maps() { + local -n ft_map=${1}; shift + local tf_set_mvm_var_name=${1}; shift + local old_name=${1}; shift + local new_name=${1}; shift + + local prev_new_name=${ft_map["${old_name}"]:-} + + if [[ -n ${prev_new_name} ]] && [[ ${prev_new_name} != "${new_name}" ]]; then + fail_lines \ + "Invalid package rename from ${old_name@Q} to ${new_name@Q}." \ + "There was already a rename from ${old_name@Q} to ${prev_new_name@Q}." + fi + + local -a new_set=() + + local urm_set_var_name + mvm_get "${tf_set_mvm_var_name}" "${old_name}" urm_set_var_name + if [[ -n ${urm_set_var_name} ]]; then + local -n old_set=${urm_set_var_name} + new_set+=( "${!old_set[@]}" ) + unset -n old_set + fi + new_set+=( "${old_name}" ) + mvm_add "${tf_set_mvm_var_name}" "${new_name}" "${new_set[@]}" + local old + + for old in "${new_set[@]}"; do + ft_map["${old}"]=${new_name} + done + unset -n ft_map +} + +# Sets up a worktree and necessary cleanups. +# +# Params: +# +# 1 - path to the git repo +# 2 - name of a branch to be used as a base of a new worktree branch +# 3 - name of the new worktree branch +# 4 - path where the new worktree will be created +function setup_worktree() { + local repo base branch worktree_dir + repo=${1}; shift + base=${1}; shift + branch=${1}; shift + worktree_dir=${1}; shift + + add_cleanup \ + "git -C ${worktree_dir@Q} reset --hard HEAD" \ + "git -C ${worktree_dir@Q} clean -ffdx" \ + "git -C ${repo@Q} worktree remove ${worktree_dir@Q}" \ + "git -C ${repo@Q} branch -D ${branch@Q}" + + git -C "${repo}" worktree add -b "${branch}" "${worktree_dir}" "${base}" +} + +# Creates an initial globals file. It's initial because it contains +# data known up-front, so mostly things that are defined in one place +# to avoid repeating them everywhere. +# +# More stuff will be added later to the globals file based on config +# or else. +function setup_initial_globals_file() { + local sync_script pkg_list_sort_script + sync_script="${PKG_AUTO_IMPL_DIR}/sync_with_gentoo.sh" + pkg_list_sort_script="${PKG_AUTO_IMPL_DIR}/sort_packages_list.py" + + local globals_file + globals_file="${WORKDIR}/globals" + + local -a sigf_arches + sigf_arches=(amd64 arm64) + + add_cleanup "rm -f ${globals_file@Q}" + cat <"${globals_file}" +local -a GIT_ENV_VARS ARCHES WHICH REPORTS +local SDK_PKGS BOARD_PKGS +local SYNC_SCRIPT PKG_LIST_SORT_SCRIPT + +GIT_ENV_VARS=( + GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} +) + +SYNC_SCRIPT=${sync_script@Q} +PKG_LIST_SORT_SCRIPT=${pkg_list_sort_script@Q} + +ARCHES=( ${sigf_arches[*]@Q} ) +WHICH=('old' 'new') +SDK_PKGS='sdk-pkgs' +BOARD_PKGS='board-pkgs' +REPORTS=( "\${SDK_PKGS}" "\${BOARD_PKGS}" ) +EOF +} + +# Extend the globals file with information from config and other +# information derived from it. +# +# Params: +# +# 1 - path to scripts repository +# 2 - path to scripts worktree with old state +# 3 - path to scripts worktree with new state +# 4 - path to reports directory +# 5 - path to aux directory +function extend_globals_file() { + local scripts old_state new_state reports_dir aux_dir + scripts=${1}; shift + old_state=${1}; shift + new_state=${1}; shift + reports_dir=${1}; shift + aux_dir=${1}; shift + + local globals_file + globals_file="${WORKDIR}/globals" + if [[ ! -e "${globals_file}" ]]; then + fail 'an initial version of globals file should already exist' + fi + + local old_state_branch new_state_branch + old_state_branch=$(git -C "${old_state}" rev-parse --abbrev-ref HEAD) + new_state_branch=$(git -C "${new_state}" rev-parse --abbrev-ref HEAD) + + local portage_stable_suffix old_portage_stable new_portage_stable + portage_stable_suffix='sdk_container/src/third_party/portage-stable' + old_portage_stable="${old_state}/${portage_stable_suffix}" + new_portage_stable="${new_state}/${portage_stable_suffix}" + + local coreos_overlay_suffix old_coreos_overlay new_coreos_overlay + coreos_overlay_suffix='sdk_container/src/third_party/coreos-overlay' + old_coreos_overlay="${old_state}/${coreos_overlay_suffix}" + new_coreos_overlay="${new_state}/${coreos_overlay_suffix}" + + cat <>"${globals_file}" + +local SCRIPTS OLD_STATE NEW_STATE OLD_STATE_BRANCH NEW_STATE_BRANCH +local PORTAGE_STABLE_SUFFIX OLD_PORTAGE_STABLE NEW_PORTAGE_STABLE REPORTS_DIR +local NEW_STATE_PACKAGES_LIST AUX_DIR +local COREOS_OVERLAY_SUFFIX OLD_COREOS_OVERLAY NEW_COREOS_OVERLAY + +SCRIPTS=${scripts@Q} +OLD_STATE=${old_state@Q} +NEW_STATE=${new_state@Q} +OLD_STATE_BRANCH=${old_state_branch@Q} +NEW_STATE_BRANCH=${new_state_branch@Q} +PORTAGE_STABLE_SUFFIX=${portage_stable_suffix@Q} +OLD_PORTAGE_STABLE=${old_portage_stable@Q} +NEW_PORTAGE_STABLE=${new_portage_stable@Q} +REPORTS_DIR=${reports_dir@Q} + +COREOS_OVERLAY_SUFFIX=${coreos_overlay_suffix@Q} +OLD_COREOS_OVERLAY=${old_coreos_overlay@Q} +NEW_COREOS_OVERLAY=${new_coreos_overlay@Q} + +NEW_STATE_PACKAGES_LIST="\${NEW_STATE}/.github/workflows/portage-stable-packages-list" + +AUX_DIR=${aux_dir@Q} +EOF + + # shellcheck disable=SC1090 # generated file + source "${globals_file}" + + local last_nightly_version_id last_nightly_build_id + # shellcheck disable=SC1091,SC2153 # sourcing generated file, NEW_STATE is not misspelled + last_nightly_version_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") + # shellcheck disable=SC1091 # sourcing generated file + last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") + + local -a locals definitions + locals=() + definitions=() + local packages_image_var_name packages_image_name + for arch in "${ARCHES[@]}"; do + packages_image_var_name="${arch^^}_PACKAGES_IMAGE" + packages_image_name="flatcar-packages-${arch}:${last_nightly_version_id}-${last_nightly_build_id}" + locals+=( "${packages_image_var_name@Q}" ) + definitions+=( "${packages_image_var_name}=${packages_image_name@Q}" ) + done + + append_to_globals \ + '' \ + "local ${locals[*]}" \ + '' \ + "${definitions[@]}" + + local -A listing_kinds + local packages_file tag filename stripped old + + for arch in "${ARCHES[@]}"; do + # shellcheck disable=SC2153 # AUX_DIR is not a misspelling, it comes from globals file + for packages_file in "${AUX_DIR}/${arch}/"*_packages.txt; do + filename=${packages_file##*/} + stripped=${filename%_packages.txt} + case ${stripped} in + 'flatcar_developer_container') + tag='dev' + ;; + 'flatcar_production_image') + tag='prod' + ;; + 'flatcar-'*) + tag="sysext-${stripped#flatcar-}" + ;; + 'oem-'*) + tag=${stripped#oem-} + ;; + *'-flatcar') + tag=${stripped%-flatcar} + ;; + *) + devel_warn "Unknown listing file ${packages_file@Q}" + continue + ;; + esac + old=${listing_kinds["${tag}"]:-} + if [[ -n ${old} ]]; then + if [[ ${old} != "${filename}" ]]; then + devel_warn "Two different packages files (${old@Q} and ${filename@Q} for a single tag ${tag@Q}" + fi + else + listing_kinds["${tag}"]=${filename} + fi + done + done + + local -a sorted_tags sorted_lines + mapfile -t sorted_tags < <(printf '%s\n' "${!listing_kinds[@]}" | sort) + for tag in "${sorted_tags[@]}"; do + filename=${listing_kinds["${tag}"]} + sorted_lines+=(" [${tag@Q}]=${filename@Q}") + done + + append_to_globals \ + '' \ + 'local -A LISTING_KINDS' \ + '' \ + 'LISTING_KINDS=(' \ + "${sorted_lines[@]}" \ + ')' +} + +# Sets up environment variables for some git commands. +# +# Make sure to call the following beforehand: +# +# local -x "${GIT_ENV_VARS[@]}" +# +# The GIT_ENV_VARS array comes from the globals file. +function setup_git_env() { + local bot_name bot_email role what + + # shellcheck disable=SC2034 # used indirectly + bot_name='Flatcar Buildbot' + # shellcheck disable=SC2034 # used indirectly + bot_email='buildbot@flatcar-linux.org' + for role in AUTHOR COMMITTER; do + for what in name email; do + local -n var="GIT_${role}_${what^^}" + local -n value="bot_${what}" + # shellcheck disable=SC2034 # it's a reference to external variable + var=${value} + unset -n value + unset -n var + done + done +} + +# Goes over the packages list and synces them with the passed Gentoo +# repo. +# +# Params: +# +# 1 - path to the Gentoo repo +function run_sync() { + local gentoo + gentoo=${1}; shift + + local -a missing_in_scripts missing_in_gentoo + missing_in_scripts=() + missing_in_gentoo=() + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -x "${GIT_ENV_VARS[@]}" + setup_git_env + + local -a packages_to_update + packages_to_update=() + + local package + while read -r package; do + # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file + if [[ ! -e "${NEW_PORTAGE_STABLE}/${package}" ]]; then + # If this happens, it means that the package was moved to overlay + # or dropped, the list ought to be updated. + missing_in_scripts+=("${package}") + continue + fi + if [[ ! -e "${gentoo}/${package}" ]]; then + # If this happens, it means that the package was obsoleted or moved + # in Gentoo. The obsoletion needs to be handled in the case-by-case + # manner, while move should be handled by doing the same move + # in portage-stable. The build should not break because of the move, + # because most likely it's already reflected in the profiles/updates + # directory. + missing_in_gentoo+=("${package}") + continue + fi + packages_to_update+=( "${package}" ) + done < <(cat_meaningful "${NEW_STATE_PACKAGES_LIST}") + # shellcheck disable=SC2153 # SYNC_SCRIPT is not a misspelling + env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${packages_to_update[@]}" + + save_missing_in_scripts "${missing_in_scripts[@]}" + save_missing_in_gentoo "${missing_in_gentoo[@]}" +} + +# A helper function that prints the contents of a file skipping empty +# lines and lines starting with a hash. +# +# Params: +# +# 1 - path to a file to print +function cat_meaningful() { + local file + file=${1}; shift + + xgrep '^[^#]' "${file}" +} + +# Saves a list of package names to a file and adds a cleanup for +# it. The names can be loaded again with load_simple_package_list. +# +# Params: +# +# 1 - path to a file where package names will be stored +# @ - the package names +function save_simple_package_list() { + local file + file=${1}; shift + # rest are packages + + add_cleanup "rm -f ${file@Q}" + if [[ ${#} -eq 0 ]]; then + truncate --size=0 "${file}" + else + printf '%s\n' "${@}" >"${file}" + fi +} + +# Loads a list of packages saved previously with +# save_simple_package_list. +# +# Params: +# +# 1 - path to a file where packages were stored +# 2 - name of an array variable; will contain package names +function load_simple_package_list() { + local file packages_var_name + file=${1}; shift + packages_var_name=${1}; shift + + mapfile -t "${packages_var_name}" <"${file}" +} + +# General function for saving missing packages. Takes care of creating +# a directory for the listing. +# +# Params: +# +# 1 - path to a directory which will contain the listing +# 2 - name of the listing file +function save_missing_packages() { + local dir file + dir=${1}; shift + file=${1}; shift + + create_cleanup_dir "${dir}" + save_simple_package_list "${dir}/${file}" "${@}" +} + +# Creates a directory and adds a cleanup if the directory was missing. +# +# Params: +# +# 1 - path to the directory +function create_cleanup_dir() { + local dir + dir=${1}; shift + if [[ ! -d "${dir}" ]]; then + add_cleanup "rmdir ${dir@Q}" + mkdir "${dir}" + fi +} + +# Saves a list of package names that we missing in scripts repo (which +# means that we were asked to sync a package that isn't in scripts to +# begin with). +# +# Params: +# +# @ - package names +function save_missing_in_scripts() { + save_missing_packages "${WORKDIR}/missing_in_scripts" "saved_list" "${@}" +} + +# Saves a list of package names that we missing in Gentoo repo (which +# means that we were asked to sync a possibly obsolete or renamed +# package). +# +# Params: +# +# @ - package names +function save_missing_in_gentoo() { + save_missing_packages "${WORKDIR}/missing_in_gentoo" "saved_list" "${@}" +} + +# Loads a list of package names that were missing in scripts repo. +# +# Params: +# +# 1 - name of an array variable; will contain package names +function load_missing_in_scripts() { + local packages_var_name + packages_var_name=${1}; shift + + load_simple_package_list "${WORKDIR}/missing_in_scripts/saved_list" "${packages_var_name}" +} + +# Loads a list of package names that were missing in Gentoo repo. +# +# Params: +# +# 1 - name of an array variable; will contain package names +function load_missing_in_gentoo() { + local packages_var_name + packages_var_name=${1}; shift + + load_simple_package_list "${WORKDIR}/missing_in_gentoo/saved_list" "${packages_var_name}" +} + +# Handles package names that were missing in scripts by dropping them +# from the listing of packages that should be updated automatically. +function handle_missing_in_scripts() { + local -a hmis_missing_in_scripts + hmis_missing_in_scripts=() + load_missing_in_scripts hmis_missing_in_scripts + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + if [[ ${#hmis_missing_in_scripts[@]} -eq 0 ]]; then + return 0; + fi + + # Remove missing in scripts entries from package automation + local dir + dir="${WORKDIR}/missing_in_scripts" + create_cleanup_dir "${dir}" + local missing_re + join_by missing_re '\|' "${missing_in_scripts[@]}" + add_cleanup "rm -f ${dir@Q}/pkg_list" + xgrep --invert-match --line-regexp --fixed-strings --regexp="${missing_re}" "${NEW_STATE_PACKAGES_LIST}" >"${dir}/pkg_list" + # shellcheck disable=SC2153 # PKG_LIST_SORT_SCRIPT is not a misspelling + "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" + + local -x "${GIT_ENV_VARS[@]}" + setup_git_env + + git -C "${NEW_STATE}" add "${NEW_STATE_PACKAGES_LIST}" + git -C "${NEW_STATE}" commit --quiet --message '.github: Drop missing packages from automation' + info_lines 'dropped missing packages from automation' "${missing_in_scripts[@]/#/- }" +} + +# Helper function to print lines to a file clobbering the old +# contents. +# +# Params: +# +# 1 - path to the file +# @ - lines to print +function lines_to_file_truncate() { + truncate --size=0 "${1}" + lines_to_file "${@}" +} + +# Helper function to append lines to a file. +# +# Params: +# +# 1 - path to the file +# @ - lines to print +function lines_to_file() { + printf '%s\n' "${@:2}" >>"${1}" +} + +# Adds lines to "manual work needed" file in reports. +# +# Params: +# +# @ - lines to add +function manual() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + pkg_debug_lines 'manual work needed:' "${@}" + # shellcheck disable=SC2153 # REPORTS_DIR is not a misspelling, it comes from globals file + lines_to_file "${REPORTS_DIR}/manual-work-needed" "${@}" +} + +# Adds lines to "warnings" file in reports. Should be used to report +# some issues with the processed packages. +# +# Params: +# +# @ - lines to add +function pkg_warn() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + pkg_debug_lines 'pkg warn:' "${@}" + lines_to_file "${REPORTS_DIR}/warnings" "${@}" +} + +# Adds lines to "developer warnings" file in reports. Should be used +# to report some failed assumption in the automation, or bugs. +# +# Params: +# +# @ - lines to add +function devel_warn() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + pkg_debug_lines 'developer warn:' "${@}" + lines_to_file "${REPORTS_DIR}/developer-warnings" "${@}" +} + +# Handles package names that were missing from Gentoo by either +# renaming and syncing them if a rename exists or by adding the +# package to the "manual work needed" file. +function handle_missing_in_gentoo() { + local gentoo + gentoo=${1}; shift + + local -a hmig_missing_in_gentoo + hmig_missing_in_gentoo=() + load_missing_in_gentoo hmig_missing_in_gentoo + + if [[ ${#hmig_missing_in_gentoo[@]} -eq 0 ]]; then + return 0; + fi + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -A hmig_rename_map=() + process_profile_updates_directory hmig_rename_map + + local -a renamed_from renamed_to + renamed_from=() + renamed_to=() + + local -x "${GIT_ENV_VARS[@]}" + setup_git_env + + local missing new_name hmig_old_basename hmig_new_basename ebuild ebuild_version_ext new_ebuild_filename + for missing in "${hmig_missing_in_gentoo[@]}"; do + new_name=${hmig_rename_map["${missing}"]:-} + if [[ -z "${new_name}" ]]; then + manual "- package ${missing} is gone from Gentoo and no rename found" + continue + fi + mkdir -p "${NEW_PORTAGE_STABLE}/${new_name%/*}" + git -C "${NEW_STATE}" mv "${NEW_PORTAGE_STABLE}/${missing}" "${NEW_PORTAGE_STABLE}/${new_name}" + basename_out "${missing}" hmig_old_basename + basename_out "${new_name}" hmig_new_basename + if [[ "${hmig_old_basename}" != "${hmig_new_basename}" ]]; then + for ebuild in "${NEW_PORTAGE_STABLE}/${new_name}/${hmig_old_basename}-"*'.ebuild'; do + # 1.2.3-r4.ebuild + ebuild_version_ext=${ebuild##*/"${hmig_old_basename}-"} + new_ebuild_filename="${hmig_new_basename}-${ebuild_version_ext}" + git -C "${NEW_STATE}" mv "${ebuild}" "${NEW_PORTAGE_STABLE}/${new_name}/${new_ebuild_filename}" + done + fi + git -C "${NEW_STATE}" commit --quiet --message "${new_name}: Renamed from ${missing}" + info "renamed ${missing} to ${new_name}" + renamed_from+=("${missing}") + renamed_to+=("${new_name}") + done + + if [[ ${#renamed_from[@]} -eq 0 ]]; then + return 0 + fi + + env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${renamed_to[@]}" + + local dir renamed_re + dir="${WORKDIR}/missing_in_gentoo" + create_cleanup_dir "${dir}" + join_by renamed_re '\|' "${renamed_from[@]}" + add_cleanup "rm -f ${dir@Q}/pkg_list" + { + xgrep --invert-match --line-regexp --regexp="${renamed_re}" "${NEW_STATE_PACKAGES_LIST}" + printf '%s\n' "${renamed_to[@]}" + } >"${dir}/pkg_list" + "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" + git -C "${NEW_STATE}" add "${NEW_STATE_PACKAGES_LIST}" + git -C "${NEW_STATE}" commit --quiet --message '.github: Update package names in automation' + info 'updated packages names in automation' +} + +# Process the package listings stored in the aux directory to find out +# the package tags that describe the kind of image the package is used +# in (base image, developer container, sysext image, etc.) +# +# Params: +# +# 1 - a name to an array mvm variable; will be a mapping of a package +# name to an array of tags +function process_listings() { + local pkg_to_tags_mvm_var_name + pkg_to_tags_mvm_var_name=${1} + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local eclass ver_ere pkg_ere + # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling + eclass="${PKG_AUTO_DIR}/../${PORTAGE_STABLE_SUFFIX}/eclass/eapi7-ver.eclass" + # line is like ' re=""' + ver_ere=$(grep -e 're=' "${eclass}" || fail "no 're=' line found in eapi7-ver.eclass") + if [[ -z ${ver_ere} ]]; then + fail 'empty version regex from eapi7-ver.eclass' + fi + # strip everything until first quotes + ver_ere=${ver_ere#*'"'} + # strip last quote + ver_ere=${ver_ere%'"'*} + # regexp begins with ^ and ends with $, so strip them too + ver_ere=${ver_ere#'^'} + ver_ere=${ver_ere%'$'} + pkg_ere='[a-z0-9]*-?[a-z0-9]*/[a-z0-9A-Z_+-]*' + + #mvm_debug_enable pl_pkg_to_tags_set_mvm + mvm_declare pl_pkg_to_tags_set_mvm mvm_mvc_set + + local arch kind file listing pkg + for arch in "${ARCHES[@]}"; do + # shellcheck disable=SC2153 # LISTING_KINDS is not a misspelling, it comes from globals file + for kind in "${!LISTING_KINDS[@]}"; do + file=${LISTING_KINDS["${kind}"]} + listing="${AUX_DIR}/${arch}/${file}" + if [[ ! -e "${listing}" ]]; then + # some listings are arch-specific, so they will be + # missing for other arches + continue + fi + # lines are like as follows: + # + # acct-group/adm-0-r2::portage-stable + while read -r pkg; do + pkg_debug_enable "${pkg}" + pkg_debug "processing listings: adding tag ${kind^^}" + pkg_debug_disable + mvm_add pl_pkg_to_tags_set_mvm "${pkg}" "${kind^^}" + done < <(sed -E -e 's#^('"${pkg_ere}"')-'"${ver_ere}"'::.*#\1#' "${listing}") + done + done + + mvm_iterate pl_pkg_to_tags_set_mvm set_mvm_to_array_mvm_cb "${pkg_to_tags_mvm_var_name}" + mvm_unset pl_pkg_to_tags_set_mvm + #mvm_debug_disable pl_pkg_to_tags_set_mvm + if pkg_debug_possible; then + mvm_iterate "${pkg_to_tags_mvm_var_name}" debug_dump_package_tags "${pkg_to_tags_mvm_var_name}" + fi +} + +# A debug function that prints the package tags. Used as a callback to +# mvm_iterate. +# +# Params: +# +# 1 - name of the array mvm variable (extra arg of the callback) +# 2 - name of the package +# 3 - name of the array variable holding tags (unused) +# @ - tags +function debug_dump_package_tags() { + local map_name=${1}; shift + local pkg=${1}; shift + shift # we don't care about array variable name + # rest are array elements, which are tags + pkg_debug_enable "${pkg}" + pkg_debug "tags for ${pkg} stored in ${map_name}: ${*}" + pkg_debug_disable +} + +# A callback to mvm_iterate that turns a set mvm to an array mvm. It +# makes sure that the tag for the production image (or base image) is +# always first in the array. +# +# Params: +# +# 1 - name of the array mvm variable that will be filled (extra arg of +# the callback) +# 2 - name of the package +# 3 - name of the set variable holding tags +# @ - tags +function set_mvm_to_array_mvm_cb() { + local pkg_to_tags_mvm_var_name pkg + pkg_to_tags_mvm_var_name=${1}; shift + pkg=${1}; shift + local -n set_ref=${1}; shift + # rest are set items + + local removed + removed='' + local -a prod_item + prod_item=() + if [[ -n ${set_ref['PROD']:-} ]]; then + prod_item+=('PROD') + unset "set_ref['PROD']" + removed=x + fi + local -a sorted_items + mapfile -t sorted_items < <(printf '%s\n' "${!set_ref[@]}" | sort) + if [[ -n ${removed} ]]; then + set_ref['PROD']=x + fi + + mvm_add "${pkg_to_tags_mvm_var_name}" "${pkg}" "${prod_item[@]}" "${sorted_items[@]}" +} + +# Generate package reports inside SDKs for all arches and states. In +# case of failure, whatever reports where generated so far will be +# stored in salvaged-reports subdirectory of the reports directory. +# Otherwise they will end up in reports-from-sdk subdirectory. +function generate_sdk_reports() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" + mkdir "${WORKDIR}/pkg-reports" + + local arch packages_image_var_name packages_image_name + local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch + local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy + local -a report_files run_sdk_container_args + for arch in "${ARCHES[@]}"; do + packages_image_var_name="${arch^^}_PACKAGES_IMAGE" + packages_image_name=${!packages_image_var_name} + if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${packages_image_name}"; then + fail "No SDK image named '${packages_image_name}' available locally, pull it before running this script" + fi + + # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file + for sdk_run_kind in "${WHICH[@]}"; do + state_var_name="${sdk_run_kind^^}_STATE" + sdk_run_state="${!state_var_name}_sdk_run_${arch}" + state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" + sdk_run_state_branch="${!state_branch_var_name}-sdk-run-${arch}" + + add_cleanup \ + "git -C ${sdk_run_state@Q} reset --hard HEAD" \ + "git -C ${sdk_run_state@Q} clean -ffdx" \ + "git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ + "git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" + git -C "${SCRIPTS}" \ + worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}" + + pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX") + add_cleanup "rm -rf ${pkg_auto_copy@Q}" + cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}" + local -a run_sdk_container_args=( + -C "${packages_image_name}" + -n "pkg-${sdk_run_kind}-${arch}" + -a "${arch}" + -U + -m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto" + --rm + ./pkg_auto/inside_sdk_container.sh "${arch}" pkg-reports + ) + rv=0 + env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?} + if [[ ${rv} -ne 0 ]]; then + { + salvaged_dir="${REPORTS_DIR}/salvaged-reports" + info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue" + info + for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do + info "from ${file}:" + echo + cat "${file}" + echo + done + info + info 'whatever reports generated by the failed run are saved in' + info "${salvaged_dir@Q} directory" + info + } >&2 + rm -rf "${salvaged_dir}" + cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}" + fail "copying done, stopping now" + fi + sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}-${arch}" + report_files=() + for full_file in "${sdk_run_state}/pkg-reports/"*; do + file=${full_file##"${sdk_run_state}/pkg-reports/"} + report_files+=( "${sdk_reports_dir}/${file}" ) + done + add_cleanup \ + "rm -f ${report_files[*]@Q}" \ + "rmdir ${sdk_reports_dir@Q}" + mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}" + done + done + cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk" +} + +source "${PKG_AUTO_IMPL_DIR}/mvm.sh" + +# pkginfo mvm is a map mvm that has the following Go-like type: +# +# map[pkg]map[slot]version +# +# pkg, slot and version are strings + +# Generate a name for pkginfo mvm based on passed information. +# +# Params: +# +# 1 - which state it refers to (old or new) +# 2 - architecture +# 3 - which report (board packages or SDK packages) +# 4 - name of a variable that will contain the name +function pkginfo_name() { + local which arch report + + which=${1}; shift + arch=${1}; shift + report=${1}; shift + local -n pi_name_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + pi_name_ref="pkginfo_${which}_${arch}_${report//-/_}_pimap_mvm" +} + +# Constructor callback used by mvm_declare for pkginfo mvms. +function pkginfo_constructor() { + mvm_mvc_map_constructor "${@}" +} + +# Destructor callback used by mvm_declare for pkginfo mvms. +function pkginfo_destructor() { + mvm_mvc_map_destructor "${@}" +} + +# Adder callback used by mvm_declare for pkginfo mvms. +function pkginfo_adder() { + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n map_ref=${1}; shift + + local mark + while [[ ${#} -gt 1 ]]; do + mark=${map_ref["${1}"]:-} + if [[ -n "${mark}" ]]; then + fail "multiple versions for a single slot for a package in a single report" + fi + map_ref["${1}"]=${2} + shift 2 + done +} + +# Creates a pkginfo mvm. +# +# Params: +# +# 1 - which state it refers to (old or new) +# 2 - architecture +# 3 - which report (board packages or SDK packages) +# 4 - name of a variable that will contain the name of the created +# pkginfo mvm +function pkginfo_declare() { + local which arch report pi_name_var_name + which=${1}; shift + arch=${1}; shift + report=${1}; shift + pi_name_var_name=${1}; shift + + pkginfo_name "${which}" "${arch}" "${report}" "${pi_name_var_name}" + + local -a extras + extras=( + 'which' "${which}" + 'arch' "${arch}" + 'report' "${report}" + ) + + mvm_declare "${!pi_name_var_name}" pkginfo -- "${extras[@]}" +} + +# Destroys a pkginfo mvm. +# +# Params: +# +# 1 - which state it refers to (old or new) +# 2 - architecture +# 3 - which report (board packages or SDK packages) +function pkginfo_unset() { + local which arch report + which=${1}; shift + arch=${1}; shift + report=${1}; shift + + local piu_pi_name + pkginfo_name "${which}" "${arch}" "${report}" piu_pi_name + + mvm_unset "${piu_pi_name}" +} + +# Processes the report file associated to the passed pkginfo mvm. The +# pkginfo mvm is filled with info about packages, slots and +# versions. Additional information is put into passed package set and +# package to slots set mvm. +# +# Params: +# +# 1 - name of the pkginfo mvm variable +# 2 - name of the set variable, will contain packages in the report +# 3 - name of the set mvm variable, will contain a map of package to +# slots +function pkginfo_process_file() { + mvm_call "${1}" pkginfo_c_process_file "${@:2}" +} + +# Helper function for pkginfo_process_file, used by mvm_call. +function pkginfo_c_process_file() { + local pkg_slots_set_mvm_var_name + local -n pkg_set_ref=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local which arch report + mvm_c_get_extra 'which' which + mvm_c_get_extra 'arch' arch + mvm_c_get_extra 'report' report + + local pkg version_slot throw_away v s + # shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else + while read -r pkg version_slot throw_away; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} ${arch} ${report}: ${version_slot}" + v=${version_slot%%:*} + s=${version_slot##*:} + mvm_c_add "${pkg}" "${s}" "${v}" + # shellcheck disable=SC2034 # it's a reference to external variable + pkg_set_ref["${pkg}"]='x' + mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}" + pkg_debug_disable + done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" +} + +# Gets a profile of the pkginfo mvm. The "profile" is a confusing +# misnomer as it has nothing to do with Gentoo profiles, but rather a +# description of the pkginfo (which is a which-arch-report triplet) +# that is used for reporting. +function pkginfo_profile() { + mvm_call "${1}" pkginfo_c_profile "${@:2}" +} + +# Helper function for pkginfo_profile, used by mvm_call. +function pkginfo_c_profile() { + local profile_var_name + profile_var_name=${1}; shift + + local which arch report + mvm_c_get_extra 'which' which + mvm_c_get_extra 'arch' arch + mvm_c_get_extra 'report' report + + printf -v "${profile_var_name}" '%s-%s-%s' "${which}" "${arch}" "${report}" +} + +# Creates pkginfo maps for all the reports and processes the +# reports. Additional information is stored in passed packages array +# and packages to slots set mvm variables. +# +# Params: +# +# 1 - name of the array variable, will contain sorted packages from +# all the reports +# 2 - name of the set mvm variable, will contain a map of package to +# slots +function read_reports() { + local all_pkgs_var_name pkg_slots_set_mvm_var_name + all_pkgs_var_name=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -A rr_all_packages_set + rr_all_packages_set=() + + local arch which report rr_pimap_mvm_var_name + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in "${REPORTS[@]}"; do + pkginfo_declare "${which}" "${arch}" "${report}" rr_pimap_mvm_var_name + pkginfo_process_file "${rr_pimap_mvm_var_name}" rr_all_packages_set "${pkg_slots_set_mvm_var_name}" + done + done + done + + mapfile -t "${all_pkgs_var_name}" < <(printf '%s\n' "${!rr_all_packages_set[@]}" | sort) +} + +# Destroys the pkginfo maps for all the reports. +function unset_report_mvms() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local arch which report + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in "${REPORTS[@]}"; do + pkginfo_unset "${which}" "${arch}" "${report}" + done + done + done +} + +### +### BEGIN GENTOO VER COMP HACKS +### + +# shellcheck disable=SC2034 # it's here only for the eapi7-ver.eclass +EAPI=6 +function die() { + fail "$*" +} + +# This brings in ver_test function. +# +# shellcheck disable=SC1091 # sourcing external file +source "${PKG_AUTO_DIR}/../sdk_container/src/third_party/portage-stable/eclass/eapi7-ver.eclass" + +unset EAPI + +# symbolic names for use with gentoo_ver_cmp +GV_LT=1 +GV_EQ=2 +GV_GT=3 + +# Compare two versions. The result can be compared against GV_LT, GV_EQ and GV_GT variables. +# +# Params: +# +# 1 - version 1 +# 2 - version 2 +# 3 - name of variable to store the result in +function gentoo_ver_cmp_out() { + local v1 v2 + v1=${1}; shift + v2=${1}; shift + local -n out_ref=${1}; shift + + out_ref=0 + _ver_compare "${v1}" "${v2}" || out_ref=${?} + case ${out_ref} in + 1|2|3) + return 0 + ;; + *) + fail "unexpected return value ${out_ref} from _ver_compare for ${v1} and ${v2}" + ;; + esac +} + +### +### END GENTOO VER COMP HACKS +### + +# Finds out the highest and the lowest version from the passed versions. +# +# Params: +# +# 1 - name of a variable where the min version will be stored +# 2 - name of a variable where the min version will be stored +# @ - the versions +function ver_min_max() { + local -n min_ref=${1}; shift + local -n max_ref=${1}; shift + + local min max v + min='' + max='' + for v; do + if [[ -z ${min} ]] || ver_test "${v}" -lt "${min}"; then + min=${v} + fi + if [[ -z ${max} ]] || ver_test "${v}" -gt "${max}"; then + max=${v} + fi + done + # shellcheck disable=SC2034 # it's a reference to external variable + min_ref=${min} + # shellcheck disable=SC2034 # it's a reference to external variable + max_ref=${max} +} + +# Does consistency checks on two profiles for a package using an +# additional map for slots information for a package. Checks if common +# slots for the package in both profiles are using the same +# versions. This is to catch version discrepancies that sometimes +# happen when e.g. a package gets stabilized for one arch, but not for +# the other. +# +# While at it, store package, slot and version range information into +# the passed map. +# +# 1 - package +# 2 - name of the pkginfo mvm for profile 1 +# 3 - name of the pkginfo mvm for profile 2 +# 4 - name of the pkg to slots to version range map mvm # TODO: This should be the last parameter +# 5 - name of the pkg to all slots set mvm +function consistency_check_for_package() { + local pkg pi1_pimap_mvm_var_name pi2_pimap_mvm_var_name pkg_slot_verminmax_map_mvm_var_name pkg_slots_set_mvm_var_name + pkg=${1}; shift + pi1_pimap_mvm_var_name=${1}; shift + pi2_pimap_mvm_var_name=${1}; shift + pkg_slot_verminmax_map_mvm_var_name=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + + local ccfp_slot_version1_map_var_name ccfp_slot_version2_map_var_name + mvm_get "${pi1_pimap_mvm_var_name}" "${pkg}" ccfp_slot_version1_map_var_name + mvm_get "${pi2_pimap_mvm_var_name}" "${pkg}" ccfp_slot_version2_map_var_name + + local -A empty_map + empty_map=() + + local -n slot_version1_map=${ccfp_slot_version1_map_var_name:-empty_map} + local -n slot_version2_map=${ccfp_slot_version2_map_var_name:-empty_map} + + local ccfp_slots_set_var_name + mvm_get "${pkg_slots_set_mvm_var_name}" "${pkg}" ccfp_slots_set_var_name + local -n slots_set_ref=${ccfp_slots_set_var_name} + + local -a profile_1_slots profile_2_slots common_slots + profile_1_slots=() + profile_2_slots=() + common_slots=() + + local ccfp_profile_1 ccfp_profile_2 + pkginfo_profile "${pi1_pimap_mvm_var_name}" ccfp_profile_1 + pkginfo_profile "${pi2_pimap_mvm_var_name}" ccfp_profile_2 + + local s v1 v2 ccfp_min ccfp_max mm + pkg_debug "all slots iterated over: ${!slots_set_ref[*]}" + for s in "${!slots_set_ref[@]}"; do + v1=${slot_version1_map["${s}"]:-} + v2=${slot_version2_map["${s}"]:-} + pkg_debug "v1: ${v1}, v2: ${v2}" + + if [[ -n ${v1} ]] && [[ -n ${v2} ]]; then + pkg_debug "${s} is a common slot for ${ccfp_profile_1} and ${ccfp_profile_2}" + common_slots+=( "${s}" ) + if [[ ${v1} != "${v2}" ]]; then + pkg_warn \ + "- version mismatch:" \ + " - package: ${pkg}" \ + " - slot: ${s}" \ + " - profile 1: ${ccfp_profile_1}" \ + " - version: ${v1}" \ + " - profile 1: ${ccfp_profile_2}" \ + " - version: ${v2}" + fi + ver_min_max ccfp_min ccfp_max "${v1}" "${v2}" + mm="${ccfp_min}:${ccfp_max}" + elif [[ -n ${v1} ]]; then + # only side1 has the slot + pkg_debug "${s} is a slot only in ${ccfp_profile_1}" + profile_1_slots+=( "${s}" ) + mm="${v1}:${v1}" + elif [[ -n ${v2} ]]; then + # only side 2 has the slot + pkg_debug "${s} is a slot only in ${ccfp_profile_2}" + profile_2_slots+=( "${s}" ) + mm="${v2}:${v2}" + else + pkg_debug "${s} is a slot absent from both ${ccfp_profile_1} and ${ccfp_profile_2}" + continue + fi + + mvm_add "${pkg_slot_verminmax_map_mvm_var_name}" "${pkg}" "${s}" "${mm}" + done + pkg_debug "common slots: ${common_slots[*]}" + pkg_debug "profile 1 slots: ${profile_1_slots[*]}" + pkg_debug "profile 2 slots: ${profile_2_slots[*]}" + local s1 s2 + if [[ ${#common_slots[@]} -gt 0 ]]; then + if [[ ${#profile_1_slots[@]} -gt 0 ]] || [[ ${#profile_2_slots[@]} -gt 0 ]]; then + pkg_warn \ + "- suspicious:" \ + " - package: ${pkg}" \ + " - profile 1: ${ccfp_profile_1}" \ + " - profile 2: ${ccfp_profile_2}" \ + " - common slots: ${common_slots[*]}" \ + " - slots only in profile 1: ${profile_1_slots[*]}" \ + " - slots only in profile 2: ${profile_2_slots[*]}" \ + " - what: there are slots that exist only on one profile while both profiles also have some common slots" + fi + elif [[ ${#profile_1_slots[@]} -eq 1 ]] && [[ ${#profile_2_slots[@]} -eq 1 ]]; then + s1=${profile_1_slots[0]} + s2=${profile_2_slots[0]} + v1=${slot_version1_map["${s1}"]:-} + v2=${slot_version2_map["${s2}"]:-} + if [[ ${v1} != "${v2}" ]]; then + pkg_warn \ + "- version mismatch:" \ + " - package ${pkg}" \ + " - profile 1: ${ccfp_profile_1}" \ + " - slot: ${profile_1_slots[0]}" \ + " - version: ${v1}" \ + " - profile 2: ${ccfp_profile_2}" \ + " - slot: ${profile_2_slots[0]}" \ + " - version: ${v2}" + fi + fi +} + +# Do consistency checks for the following pairs of profiles for a passed state: +# +# amd64 sdk <-> amd64 board +# amd64 board <-> arm64 board +# +# These are not yet checked: +# +# amd64 sdk <-> arm64 sdk +# arm64 sdk <-> arm64 board +# +# While at it, store package, slot and version range information into +# the passed map. +# +# Params: +# +# 1 - which state should be checked (old or new) +# 2 - name of an array variable that contains all the package names +# 3 - name of the pkg to all slots set mvm variable +# 4 - name of the pkg to slot to version range map mvm variable +function consistency_checks() { + local which all_pkgs_var_name pkg_slots_set_mvm_var_name pkg_slot_verminmax_mvm_var_name + which=${1}; shift + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n all_pkgs_ref=${1}; shift + pkg_slots_set_mvm_var_name=${1}; shift + pkg_slot_verminmax_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local cc_pimap_mvm_1_var_name cc_pimap_mvm_2_var_name pkg + + # amd64 sdk <-> amd64 board + pkginfo_name "${which}" amd64 "${SDK_PKGS}" cc_pimap_mvm_1_var_name + pkginfo_name "${which}" amd64 "${BOARD_PKGS}" cc_pimap_mvm_2_var_name + mvm_declare cc_amd64_sdk_board_pkg_slot_verminmax_map_mvm mvm_mvc_map + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} amd64 sdk <-> amd64 board" + consistency_check_for_package "${pkg}" "${cc_pimap_mvm_1_var_name}" "${cc_pimap_mvm_2_var_name}" cc_amd64_sdk_board_pkg_slot_verminmax_map_mvm "${pkg_slots_set_mvm_var_name}" + pkg_debug_disable + done + + # amd64 board <-> arm64 board + pkginfo_name "${which}" amd64 "${BOARD_PKGS}" cc_pimap_mvm_1_var_name + pkginfo_name "${which}" arm64 "${BOARD_PKGS}" cc_pimap_mvm_2_var_name + mvm_declare cc_amd64_arm64_board_pkg_slot_verminmax_map_mvm mvm_mvc_map + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} amd64 board <-> arm64 board" + consistency_check_for_package "${pkg}" "${cc_pimap_mvm_1_var_name}" "${cc_pimap_mvm_2_var_name}" cc_amd64_arm64_board_pkg_slot_verminmax_map_mvm "${pkg_slots_set_mvm_var_name}" + pkg_debug_disable + done + + local cc_slot_verminmax1_map_var_name cc_slot_verminmax2_map_var_name + local cc_slots_set_var_name s verminmax1 verminmax2 cc_min cc_max verminmax + local -A empty_map + # shellcheck disable=SC2034 # used indirectly below + empty_map=() + for pkg in "${all_pkgs_ref[@]}"; do + pkg_debug_enable "${pkg}" + pkg_debug "${which} verminmax stuff" + mvm_get cc_amd64_sdk_board_pkg_slot_verminmax_map_mvm "${pkg}" cc_slot_verminmax1_map_var_name + mvm_get cc_amd64_arm64_board_pkg_slot_verminmax_map_mvm "${pkg}" cc_slot_verminmax2_map_var_name + mvm_get "${pkg_slots_set_mvm_var_name}" "${pkg}" cc_slots_set_var_name + local -n slot_verminmax1_map_ref=${cc_slot_verminmax1_map_var_name:-empty_map} + pkg_debug "all slots in 1: ${!slot_verminmax1_map_ref[*]}" + pkg_debug "all vmms in 1: ${slot_verminmax1_map_ref[*]}" + local -n slot_verminmax2_map_ref=${cc_slot_verminmax2_map_var_name:-empty_map} + pkg_debug "all slots in 2: ${!slot_verminmax2_map_ref[*]}" + pkg_debug "all vmms in 2: ${slot_verminmax2_map_ref[*]}" + local -n slots_set_ref=${cc_slots_set_var_name} + pkg_debug "all slots iterated over: ${!slots_set_ref[*]}" + for s in "${!slots_set_ref[@]}"; do + verminmax1=${slot_verminmax1_map_ref["${s}"]:-} + verminmax2=${slot_verminmax2_map_ref["${s}"]:-} + pkg_debug "slot: ${s}, vmm1: ${verminmax1}, vmm2: ${verminmax2}" + if [[ -n "${verminmax1}" ]] && [[ -n "${verminmax2}" ]]; then + ver_min_max \ + cc_min cc_max \ + "${verminmax1%%:*}" "${verminmax1##*:}" "${verminmax2%%:*}" "${verminmax2##*:}" + verminmax="${cc_min}:${cc_max}" + elif [[ -n "${verminmax1}" ]]; then + verminmax=${verminmax1} + elif [[ -n "${verminmax2}" ]]; then + verminmax=${verminmax2} + else + continue + fi + pkg_debug "adding vmm ${verminmax} for slot ${s}" + mvm_add "${pkg_slot_verminmax_mvm_var_name}" "${pkg}" "${s}" "${verminmax}" + done + unset -n slots_set_ref slot_verminmax2_map_ref slot_verminmax1_map_ref + pkg_debug_disable + done + mvm_unset cc_amd64_arm64_board_pkg_slot_verminmax_map_mvm + mvm_unset cc_amd64_sdk_board_pkg_slot_verminmax_map_mvm +} + +# Read a report describing from which repo the package came and store +# in the passed map. +# +# Params: +# +# 1 - name of a map variable, will contain a mapping of package name +# to repository name +function read_package_sources() { + local -n package_sources_map_ref=${1}; shift + + local arch which report pkg repo saved_repo + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in sdk-package-repos board-package-repos; do + while read -r pkg repo; do + saved_repo=${package_sources_map_ref["${pkg}"]:-} + if [[ -n ${saved_repo} ]]; then + if [[ ${saved_repo} != "${repo}" ]]; then + pkg_warn \ + '- different repos used for the package:' \ + " - package: ${pkg}" \ + ' - repos:' \ + " - ${saved_repo}" \ + " - ${repo}" + fi + else + package_sources_map_ref["${pkg}"]=${repo} + fi + done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" + done + done + done +} + +# This monstrosity takes renames map and package tags information, +# reads the reports, does consistency checks and uses the information +# from previous steps to write out package differences between the old +# and new state into the reports directory. +# +# Params: +# +# 1 - name of the renames map variable +# 2 - name of the package tags map mvm variable +function handle_package_changes() { + local pkg_to_tags_mvm_var_name + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n renamed_old_to_new_map_ref=${1}; shift + pkg_to_tags_mvm_var_name=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a hpc_all_pkgs + hpc_all_pkgs=() + + # map[package]map[slot]interface{} + mvm_declare hpc_pkg_slots_set_mvm mvm_mvc_set + read_reports hpc_all_pkgs hpc_pkg_slots_set_mvm + + # map[package]map[slot]string (string being "min version:max version") + mvm_declare hpc_old_pkg_slot_verminmax_map_mvm mvm_mvc_map + mvm_declare hpc_new_pkg_slot_verminmax_map_mvm mvm_mvc_map + consistency_checks old hpc_all_pkgs hpc_pkg_slots_set_mvm hpc_old_pkg_slot_verminmax_map_mvm + consistency_checks new hpc_all_pkgs hpc_pkg_slots_set_mvm hpc_new_pkg_slot_verminmax_map_mvm + + unset_report_mvms + + # TODO: when we handle moving packages between repos, then there + # should be two maps, for old and new state + local -A hpc_package_sources_map + hpc_package_sources_map=() + read_package_sources hpc_package_sources_map + + mkdir -p "${REPORTS_DIR}/updates" + + local -a old_pkgs new_pkgs + old_pkgs=() + new_pkgs=() + + # The following loop fills the old_pkgs and new_pkgs arrays sorted + # package names, where old package name at index I has it's new + # counterpart at the same index. For the most part, both old and + # new names will be the same, since the renames are rather rare. + + # map[package]index + local -A added_pkg_to_index_map=() + local pkg other + for pkg in "${hpc_all_pkgs[@]}"; do + other=${renamed_old_to_new_map_ref["${pkg}"]:-} + if [[ -n "${other}" ]]; then + # There seem be a rename from ${pkg} to ${other} + pkg_debug_enable "${pkg}" "${other}" + pkg_debug "${pkg} renamed to ${other}" + pkg_debug_disable + local other_idx + other_idx=${added_pkg_to_index_map["${other}"]:-} + if [[ -n ${other_idx} ]]; then + # Looks like we have already processed the ${other} + # name. In this case, both old_pkgs[${other_idx}] and + # new_pkgs[${other_idx}] should just be + # ${other}. Since ${pkg} is the old name (${other} is + # new), we update old_pkgs to hold the old name. Just + # make sure that old_pkgs indeed had the new name + # first. + local other_old + other_old=${old_pkgs["${other_idx}"]} + if [[ ${other_old} = "${other}" ]]; then + old_pkgs["${other_idx}"]=${pkg} + else + manual \ + '- there seem to be two old packages in our repos that are supposed to be renamed to the same name:' \ + " - old package 1: ${pkg}" \ + " - old package 2: ${other_old}" \ + " - new package: ${other}" + fi + unset other_idx other_old + continue + else + unset other_idx + fi + + # Looks like we haven't processed the ${other} name yet, + # it probably will come up later, which will be taken care + # of by the "pkg_debug 'handled already through some + # rename'" part below, after else. + + local pkg_idx + # doesn't matter if it's length of new_pkgs or old_pkgs, + # both are assumed to have the same length + pkg_idx=${#old_pkgs[@]} + old_pkgs+=("${pkg}") + new_pkgs+=("${other}") + added_pkg_to_index_map["${pkg}"]=${pkg_idx} + added_pkg_to_index_map["${other}"]=${pkg_idx} + unset pkg_idx + else + pkg_debug_enable "${pkg}" + if [[ -n ${added_pkg_to_index_map["${pkg}"]:-} ]]; then + pkg_debug 'handled already through some rename' + else + pkg_debug "${pkg} is not renamed" + local pkg_idx + # doesn't matter if it's length of new_pkgs or old_pkgs, + # both are assumed to have the same length + pkg_idx=${#old_pkgs[@]} + old_pkgs+=("${pkg}") + new_pkgs+=("${pkg}") + added_pkg_to_index_map["${pkg}"]=${pkg_idx} + fi + pkg_debug_disable + fi + done + unset added_pkg_to_index_map + + # The loop below goes over the pairs of old and new package + # names. For each name there will be some checks done (like does + # this package even exist). Each name in the pair has a set of + # used slots associated with it (the most common situation is that + # each have just one slot, but there are some packages that we + # have multiple slots installed, like + # app-text/docbook-xml-dtd). Some of the slots will appear in both + # old and new package name, sometimes there will be slots + # available only in the old state or only in the new state. Each + # slot for each package name has an associated min version and max + # version. So for common slots we usually compare min version for + # old package with max version for new package. Any + # inconsistencies with the versions should be reported by + # now. There are some edge cases with the slots that are not + # handled by the automation - in such cases there will be a + # "manual action needed" report. + + local pkg_idx=0 + local old_name new_name old_repo new_repo + local hpc_old_slots_set_var_name hpc_new_slots_set_var_name + local hpc_old_slot_verminmax_map_var_name hpc_new_slot_verminmax_map_var_name + local s hpc_old_s hpc_new_s + local old_verminmax new_verminmax + local old_version new_version + local hpc_cmp_result + local -A hpc_only_old_slots_set hpc_only_new_slots_set hpc_common_slots_set + local -a lines + local hpc_update_dir + local -A empty_map_or_set + local hpc_changed hpc_slot_changed hpc_update_dir_non_slot hpc_category_dir + local which slots_set_var_name_var_name slot_verminmax_map_var_name_var_name filtered_slots_set_var_name verminmax + local -A hpc_old_filtered_slots_set hpc_new_filtered_slots_set + # shellcheck disable=SC2034 # used by name below, in a special case + empty_map_or_set=() + while [[ ${pkg_idx} -lt ${#old_pkgs[@]} ]]; do + old_name=${old_pkgs["${pkg_idx}"]} + new_name=${new_pkgs["${pkg_idx}"]} + if [[ ${old_name} = "${new_name}" ]]; then + info "handling update of ${new_name}" + else + info "handling update of ${new_name} (renamed from ${old_name})" + fi + pkg_debug_enable "${old_name}" "${new_name}" + pkg_debug 'handling updates' + pkg_idx=$((pkg_idx + 1)) + old_repo=${hpc_package_sources_map["${old_name}"]:-} + new_repo=${hpc_package_sources_map["${new_name}"]:-} + if [[ -z ${old_repo} ]]; then + pkg_warn \ + '- package not in old state' \ + " - old package: ${old_name}" \ + " - new package: ${new_name}" + pkg_debug_disable + continue + fi + if [[ -z ${new_repo} ]]; then + pkg_warn \ + '- package not in new state' \ + " - old package: ${old_name}" \ + " - new package: ${new_name}" + pkg_debug_disable + continue + fi + if [[ ${old_repo} != "${new_repo}" ]]; then + # This is pretty much an arbitrary limitation and I don't + # remember any more why we have it. + pkg_warn \ + '- package has moved between repos? unsupported for now' \ + " - old package and repo: ${old_name} ${old_repo}" \ + " - new package and repo: ${new_name} ${new_repo}" + pkg_debug_disable + continue + fi + if [[ ${new_repo} != 'portage-stable' ]]; then + # coreos-overlay packages will need a separate handling + pkg_debug 'not a portage-stable package' + pkg_debug_disable + continue + fi + + mvm_get hpc_pkg_slots_set_mvm "${old_name}" hpc_old_slots_set_var_name + mvm_get hpc_pkg_slots_set_mvm "${new_name}" hpc_new_slots_set_var_name + : "${hpc_old_slots_set_var_name:=empty_map_or_set}" + : "${hpc_new_slots_set_var_name:=empty_map_or_set}" + mvm_get hpc_old_pkg_slot_verminmax_map_mvm "${old_name}" hpc_old_slot_verminmax_map_var_name + mvm_get hpc_new_pkg_slot_verminmax_map_mvm "${new_name}" hpc_new_slot_verminmax_map_var_name + : "${hpc_old_slot_verminmax_map_var_name:=empty_map_or_set}" + : "${hpc_new_slot_verminmax_map_var_name:=empty_map_or_set}" + local -n old_slot_verminmax_map_ref=${hpc_old_slot_verminmax_map_var_name} + local -n new_slot_verminmax_map_ref=${hpc_new_slot_verminmax_map_var_name} + + # Filter out slots for old and new package name that comes out + # without versions. This may happen, because we collect all + # slot names for the package name, without differentiating + # whether such a slot existed in the old state or still exists + # in the new state. If slot didn't exist in either one then it + # will come without version information. Such a slot is + # dropped. An example would be an update of sys-devel/binutils + # from 2.42 to 2.43. Each binutils version has a separate slot + # which is named after the version. So the slots set would be + # (2.42 2.43). Slot "2.42" does not exist in the new state any + # more, "2.43" does not yet exist in the old state. So those + # slots for those states will be dropped. Thus filtered slots + # set for the old state will only contain 2.42, while for the + # new state - only 2.43. + for which in old new; do + slots_set_var_name_var_name="hpc_${which}_slots_set_var_name" + slot_verminmax_map_var_name_var_name="hpc_${which}_slot_verminmax_map_var_name" + filtered_slots_set_var_name="hpc_${which}_filtered_slots_set" + local -n which_slots_set_ref=${!slots_set_var_name_var_name} + local -n which_slot_verminmax_map_ref=${!slot_verminmax_map_var_name_var_name} + local -n which_filtered_slots_set_ref=${filtered_slots_set_var_name} + pkg_debug "all unfiltered slots for ${which} name: ${!which_slots_set_ref[*]}" + which_filtered_slots_set_ref=() + for s in "${!which_slots_set_ref[@]}"; do + verminmax=${which_slot_verminmax_map_ref["${s}"]:-} + if [[ -n ${verminmax} ]]; then + which_filtered_slots_set_ref["${s}"]=x + fi + done + pkg_debug "all filtered slots for ${which} name: ${!which_filtered_slots_set_ref[*]}" + unset -n which_filtered_slots_set_ref + unset -n which_slot_verminmax_map_ref + unset -n which_slots_set_ref + done + + hpc_only_old_slots_set=() + hpc_only_new_slots_set=() + hpc_common_slots_set=() + sets_split \ + hpc_old_filtered_slots_set hpc_new_filtered_slots_set \ + hpc_only_old_slots_set hpc_only_new_slots_set hpc_common_slots_set + pkg_debug "all common slots: ${!hpc_common_slots_set[*]}" + pkg_debug "slots only for old name: ${!hpc_only_old_slots_set[*]}" + pkg_debug "slots only for new name: ${!hpc_only_new_slots_set[*]}" + + update_dir_non_slot "${new_name}" hpc_update_dir_non_slot + mkdir -p "${hpc_update_dir_non_slot}" + + # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE comes from globals file + generate_non_ebuild_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" + generate_full_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" + generate_package_mention_reports "${NEW_STATE}" "${old_name}" "${new_name}" + + hpc_changed= + pkg_debug 'going over common slots' + for s in "${!hpc_common_slots_set[@]}"; do + old_verminmax=${old_slot_verminmax_map_ref["${s}"]:-} + new_verminmax=${new_slot_verminmax_map_ref["${s}"]:-} + pkg_debug "slot: ${s}, vmm old: ${old_verminmax}, vmm new: ${new_verminmax}" + if [[ -z "${old_verminmax}" ]] || [[ -z "${new_verminmax}" ]]; then + devel_warn \ + "- no minmax info available for old and/or new:" \ + " - old package: ${old_name}" \ + " - slot: ${s}" \ + " - minmax: ${old_verminmax}" \ + " - new package: ${new_name}" \ + " - slot: ${s}" \ + " - minmax: ${new_verminmax}" + continue + fi + update_dir "${new_name}" "${s}" "${s}" hpc_update_dir + mkdir -p "${hpc_update_dir}" + old_version=${old_verminmax%%:*} + new_version=${new_verminmax##*:} + gentoo_ver_cmp_out "${new_version}" "${old_version}" hpc_cmp_result + case ${hpc_cmp_result} in + "${GV_GT}") + handle_pkg_update "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + "${GV_EQ}") + hpc_slot_changed= + handle_pkg_as_is "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" hpc_slot_changed + if [[ -z ${hpc_slot_changed} ]]; then + rm -rf "${hpc_update_dir}" + else + hpc_changed=x + fi + ;; + "${GV_LT}") + handle_pkg_downgrade "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + esac + done + # A "sys-devel/binutils update" case - one old slot and one + # new slot, but different from each other. + if [[ ${#hpc_only_old_slots_set[@]} -eq 1 ]] && [[ ${#hpc_only_new_slots_set[@]} -eq 1 ]]; then + get_first_from_set hpc_only_old_slots_set hpc_old_s + old_verminmax=${old_slot_verminmax_map_ref["${hpc_old_s}"]:-} + get_first_from_set hpc_only_new_slots_set hpc_new_s + new_verminmax=${new_slot_verminmax_map_ref["${hpc_new_s}"]:-} + pkg_debug "jumping from slot ${hpc_old_s} (vmm: ${old_verminmax}) to slot ${hpc_new_s} (vmm: ${new_verminmax})" + if [[ -z "${old_verminmax}" ]] || [[ -z "${new_verminmax}" ]]; then + devel_warn \ + "- no verminmax info available for old and/or new:" \ + " - old package: ${old_name}" \ + " - slot: ${hpc_old_s}" \ + " - minmax: ${old_verminmax}" \ + " - new package: ${new_name}" \ + " - slot: ${hpc_new_s}" \ + " - minmax: ${new_verminmax}" + else + update_dir "${new_name}" "${hpc_old_s}" "${hpc_new_s}" hpc_update_dir + mkdir -p "${hpc_update_dir}" + old_version=${old_verminmax%%:*} + new_version=${new_verminmax##*:} + gentoo_ver_cmp_out "${new_version}" "${old_version}" hpc_cmp_result + case ${hpc_cmp_result} in + "${GV_GT}") + handle_pkg_update "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hpc_old_s}" "${hpc_new_s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + "${GV_EQ}") + hpc_slot_changed= + handle_pkg_as_is "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hpc_old_s}" "${hpc_new_s}" "${old_version}" hpc_slot_changed + if [[ -z ${hpc_slot_changed} ]]; then + rm -rf "${hpc_update_dir}" + else + hpc_changed=x + fi + ;; + "${GV_LT}") + handle_pkg_downgrade "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hpc_old_s}" "${hpc_new_s}" "${old_version}" "${new_version}" + hpc_changed=x + ;; + esac + fi + elif [[ ${#hpc_only_old_slots_set[@]} -gt 0 ]] || [[ ${#hpc_only_new_slots_set[@]} -gt 0 ]]; then + pkg_debug 'complicated slots situation, needs manual intervention' + lines=( + '- handle package update:' + ' - old package name:' + " - name: ${old_name}" + ' - slots:' + ) + for s in "${!hpc_old_filtered_slots_set[@]}"; do + old_verminmax=${old_slot_verminmax_map_ref["${s}"]:-} + lines+=(" - ${s}, minmax: ${old_verminmax}") + done + lines+=( + ' - new package name:' + " - name: ${new_name}" + ' - slots:' + ) + for s in "${!hpc_new_filtered_slots_set[@]}"; do + new_verminmax=${new_slot_verminmax_map_ref["${s}"]:-} + lines+=(" - ${s}, minmax: ${new_verminmax}") + done + manual "${lines[@]}" + fi + unset -n new_slot_verminmax_map_ref old_slot_verminmax_map_ref + # if nothing changed, drop the entire update directory for the + # package, and possibly the parent directory if it became + # empty (parent directory being a category directory, like + # sys-apps) + if [[ -z ${hpc_changed} ]]; then + pkg_debug 'no changes, dropping reports' + rm -rf "${hpc_update_dir_non_slot}" + dirname_out "${hpc_update_dir_non_slot}" hpc_category_dir + if dir_is_empty "${hpc_category_dir}"; then + rmdir "${hpc_category_dir}" + fi + fi + pkg_debug_disable + done + + mvm_unset hpc_new_pkg_slot_verminmax_map_mvm + mvm_unset hpc_old_pkg_slot_verminmax_map_mvm + mvm_unset hpc_pkg_slots_set_mvm +} + +# Gets the first item from the passed set. +# +# Mostly intended to "unwrap" a single-element set. +# +# Params: +# +# 1 - name of the set variable +# 2 - name of the variable where the element will be stored +function get_first_from_set() { + # shellcheck disable=SC2178 # shellcheck doesn't grok references to arrays + local -n set_ref=${1}; shift + local -n return_ref=${1}; shift + + local item + for item in "${!set_ref[@]}"; do + return_ref=${item} + return 0 + done + # shellcheck disable=SC2034 # it's a reference to external variable + return_ref='' +} + +# Does the set operation on two passed sets - both set differences and +# an intersection. +# +# Params: +# +# 1 - name of the first set variable +# 2 - name of the second set variable +# 3 - name of the set variable that will contain elements that exist +# in first set, but not the second +# 4 - name of the set variable that will contain elements that exist +# in second set, but not the first +# 5 - name of the set variable that will contain elements that exist +# in both first and second sets +function sets_split() { + local -n first_set_ref=${1}; shift + local -n second_set_ref=${1}; shift + local -n only_in_first_set_ref=${1}; shift + local -n only_in_second_set_ref=${1}; shift + local -n common_set_ref=${1}; shift + + only_in_first_set_ref=() + only_in_second_set_ref=() + common_set_ref=() + + local item mark + + for item in "${!first_set_ref[@]}"; do + mark=${second_set_ref["${item}"]:-} + if [[ -z "${mark}" ]]; then + # shellcheck disable=SC2034 # it's a reference to external variable + only_in_first_set_ref["${item}"]=x + else + # shellcheck disable=SC2034 # it's a reference to external variable + common_set_ref["${item}"]=x + fi + done + + for item in "${!second_set_ref[@]}"; do + mark=${first_set_ref["${item}"]:-} + if [[ -z "${mark}" ]]; then + # shellcheck disable=SC2034 # it's a reference to external variable + only_in_second_set_ref["${item}"]=x + fi + done +} + +# Write information to reports directory about the package update +# (meaning specifically that the new version is greater than the old +# one). +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - old package name +# 3 - new package name +# 4 - old package slot +# 5 - new package slot +# 6 - old version +# 7 - new version +function handle_pkg_update() { + local pkg_to_tags_mvm_var_name old_pkg new_pkg old_s new_s old new + pkg_to_tags_mvm_var_name=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + old=${1}; shift + new=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local old_no_r new_no_r + old_no_r=${old%-r+([0-9])} + new_no_r=${new%-r+([0-9])} + + local pkg_name + pkg_name=${new_pkg#*/} + local -a lines + lines=( "from ${old} to ${new}") + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( "renamed from ${old_pkg}" ) + fi + # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file + generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" + + # shellcheck disable=SC2034 # these variables are used by name + local hpu_update_dir hpu_update_dir_non_slot + update_dir_non_slot "${new_pkg}" hpu_update_dir_non_slot + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpu_update_dir + if [[ -s "${hpu_update_dir}/ebuild.diff" ]]; then + lines+=( 'TODO: review ebuild.diff' ) + fi + if [[ -s "${hpu_update_dir_non_slot}/other.diff" ]]; then + lines+=( 'TODO: review other.diff' ) + fi + lines+=( 'TODO: review occurences' ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( 'TODO: review occurences-for-old-name' ) + fi + + local -a hpu_tags + tags_for_pkg "${pkg_to_tags_mvm_var_name}" "${new_pkg}" hpu_tags + + if ver_test "${new_no_r}" -gt "${old_no_r}"; then + # version bump + generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpu_tags[@]}" + lines+=( 'release notes: TODO' ) + fi + + generate_summary_stub "${new_pkg}" "${hpu_tags[@]}" -- "${lines[@]}" +} + +# Write information to reports directory about the modified package +# (meaning specifically that the new version is equal than the old +# one). +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - old package name +# 3 - new package name +# 4 - old package slot +# 5 - new package slot +# 6 - version +# 7 - name of a "bool" variable where info is stored if relevant files +# has changed (empty means nothing changed, non-empty means +# something has changed) +function handle_pkg_as_is() { + local pkg_to_tags_mvm_var_name old_pkg new_pkg old_s new_s v + pkg_to_tags_mvm_var_name=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + v=${1}; shift + local -n changed_ref=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local hpai_update_dir + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpai_update_dir + + local pkg_name + pkg_name=${new_pkg#/} + local -a lines + lines=( "still at ${v}" ) + + local renamed + renamed= + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( "renamed from ${old_pkg}" ) + renamed=x + fi + generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${v}" "${v}" + local hpai_update_dir_non_slot hpai_update_dir + update_dir_non_slot "${new_pkg}" hpai_update_dir_non_slot + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpai_update_dir + local modified + modified= + if [[ -s "${hpai_update_dir}/ebuild.diff" ]]; then + lines+=( 'TODO: review ebuild.diff' ) + modified=x + fi + if [[ -s "${hpai_update_dir_non_slot}/other.diff" ]]; then + lines+=( 'TODO: review other.diff' ) + modified=x + fi + if [[ -z ${renamed} ]] && [[ -z ${modified} ]]; then + # Nothing relevant has changed, return early. + return 0 + fi + # shellcheck disable=SC2034 # ref to an external variable + changed_ref=x + lines+=( 'TODO: review occurences' ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( 'TODO: review occurences-for-old-name' ) + fi + + local -a hpai_tags + tags_for_pkg "${pkg_to_tags_mvm_var_name}" "${new_pkg}" hpai_tags + generate_summary_stub "${new_pkg}" "${hpai_tags[@]}" -- "${lines[@]}" +} + +# Write information to reports directory about the package downgrade +# (meaning specifically that the new version is lower than the old +# one). +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - old package name +# 3 - new package name +# 4 - old package slot +# 5 - new package slot +# 6 - old version +# 7 - new version +function handle_pkg_downgrade() { + local pkg_to_tags_mvm_var_name old_pkg new_pkg old_s new_s old new + pkg_to_tags_mvm_var_name=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + old=${1}; shift + new=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local old_no_r new_no_r + old_no_r=${old%-r+([0-9])} + new_no_r=${new%-r+([0-9])} + + local pkg_name + pkg_name=${new_pkg#*/} + local -a lines + lines=( "downgraded from ${old} to ${new}" ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( "renamed from ${old_pkg}" ) + fi + generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" + + local hpd_update_dir hpd_update_dir_non_slot + update_dir_non_slot "${new_pkg}" hpd_update_dir_non_slot + update_dir "${new_pkg}" "${old_s}" "${new_s}" hpd_update_dir + if [[ -s "${hpd_update_dir}/ebuild.diff" ]]; then + lines+=( 'TODO: review ebuild.diff' ) + fi + if [[ -s "${hpd_update_dir_non_slot}/other.diff" ]]; then + lines+=( 'TODO: review other.diff' ) + fi + lines+=( 'TODO: review occurences' ) + if [[ ${old_pkg} != "${new_pkg}" ]]; then + lines+=( 'TODO: review occurences-for-old-name' ) + fi + + local -a hpd_tags + tags_for_pkg "${pkg_to_tags_mvm_var_name}" "${new_pkg}" hpd_tags + + if ver_test "${new_no_r}" -lt "${old_no_r}"; then + # version bump + generate_changelog_entry_stub "${pkg_name}" "${new_no_r}" "${hpd_tags[@]}" + lines+=( "release notes: TODO" ) + fi + + generate_summary_stub "${new_pkg}" "${hpd_tags[@]}" -- "${lines[@]}" +} + +# Retrieves tags for a package. +# +# Params: +# +# 1 - name of the package tags set mvm variable +# 2 - package name +# 3 - name of the array variable, where the tags will be stored +function tags_for_pkg() { + local pkg_to_tags_mvm_var_name pkg + pkg_to_tags_mvm_var_name=${1}; shift + pkg=${1}; shift + local -n tags_ref=${1}; shift + + local tfp_tags_var_name + mvm_get "${pkg_to_tags_mvm_var_name}" "${pkg}" tfp_tags_var_name + + pkg_debug_enable "${pkg}" + pkg_debug "checking for tags in ${pkg_to_tags_mvm_var_name}" + if [[ -z ${tfp_tags_var_name} ]]; then + pkg_debug "no tags available" + tags_ref=() + else + local -n tags_in_mvm=${tfp_tags_var_name} + # shellcheck disable=SC2034 # it's a reference to external variable + tags_ref=( "${tags_in_mvm[@]}" ) + pkg_debug "tags available: ${tags_in_mvm[*]}" + fi + pkg_debug_disable +} + +# Adds a changelog stub to changelog file in reports directory. +# +# Params: +# 1 - package name (shortened, without the category) +# 2 - version +# @ - package tags +function generate_changelog_entry_stub() { + local pkg_name v + pkg_name=${1}; shift + v=${1}; shift + # rest are tags + + local -a applied_tags=() + for tag; do + case ${tag} in + PROD) + applied_tags+=( 'base' ) + ;; + *) + # add lower-cased tag + applied_tags+=( "${tag,,}" ) + ;; + esac + done + local gces_tags='' + if [[ ${#applied_tags[@]} -gt 0 ]]; then + join_by gces_tags ', ' "${applied_tags[@]}" + else + # no tags, it means it's an SDK package + gces_tags='SDK' + fi + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + printf '%s %s: %s ([%s](TODO))\n' '-' "${gces_tags}" "${pkg_name}" "${v}" >>"${REPORTS_DIR}/updates/changelog_stubs" +} + +# Adds a stub to the summary file in reports directory. +# +# Params: +# 1 - package +# @ - tags followed by double dash followed by lines to append to the +# file +function generate_summary_stub() { + local pkg + pkg=${1}; shift + # rest are tags separated followed by double dash followed by lines + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a tags + tags=() + while [[ ${#} -gt 0 ]]; do + if [[ ${1} = '--' ]]; then + shift + break + fi + tags+=( "${1}" ) + shift + done + # rest are lines + + { + printf '%s %s:' '-' "${pkg}" + if [[ ${#tags[@]} -gt 0 ]]; then + printf ' [%s]' "${tags[@]}" + fi + printf '\n' + if [[ ${#} -gt 0 ]]; then + printf ' - %s\n' "${@}" + printf '\n' + fi + } >>"${REPORTS_DIR}/updates/summary_stubs" +} + +# Generate diffs between directories in old state and new state for a +# package. +# +# Params: +# +# 1 - path to portage-stable in old state +# 2 - path to portage-stable in new state +# 3 - old package name +# 4 - new package name +function generate_full_diffs() { + local old_ps new_ps old_pkg new_pkg + old_ps=${1}; shift + new_ps=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + + local old_path new_path + old_path="${old_ps}/${old_pkg}" + new_path="${new_ps}/${new_pkg}" + + local gfd_update_dir + update_dir_non_slot "${new_pkg}" gfd_update_dir + + local -a common_diff_opts=( + --recursive + --unified=3 + ) + xdiff "${common_diff_opts[@]}" --new-file "${old_path}" "${new_path}" >"${gfd_update_dir}/full.diff" + xdiff "${common_diff_opts[@]}" --brief "${old_path}" "${new_path}" >"${gfd_update_dir}/brief-summary" +} + +# Generate a diff between non-ebuild, non-Manifest files for old and +# new package. +# +# Params: +# +# 1 - path to portage-stable in old state +# 2 - path to portage-stable in new state +# 3 - old package name +# 4 - new package name +function generate_non_ebuild_diffs() { + local old_ps new_ps old_pkg new_pkg + old_ps=${1}; shift + new_ps=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + + local old_path new_path + old_path="${old_ps}/${old_pkg}" + new_path="${new_ps}/${new_pkg}" + + local gned_update_dir + update_dir_non_slot "${new_pkg}" gned_update_dir + + local -a diff_opts=( + --recursive + --unified=3 + # Show contents of deleted or added files too. + --new-file + # Ignore ebuilds and the Manifest file. + --exclude='*.ebuild' + --exclude='Manifest' + ) + xdiff "${diff_opts[@]}" "${old_path}" "${new_path}" >"${gned_update_dir}/other.diff" +} + +# Generate a diff between specific ebuilds for old and new package. +# +# Params: +# +# 1 - path to portage-stable in old state +# 2 - path to portage-stable in new state +# 3 - old package name +# 4 - new package name +# 5 - old package slot +# 6 - new package slot +# 7 - old package version +# 8 - new package version +function generate_ebuild_diff() { + local old_ps new_ps old_pkg new_pkg old_s new_s old new + old_ps=${1}; shift + new_ps=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + old=${1}; shift + new=${1}; shift + + local old_pkg_name new_pkg_name + old_pkg_name=${old_pkg#*/} + new_pkg_name=${new_pkg#*/} + + local old_path new_path + old_path="${old_ps}/${old_pkg}/${old_pkg_name}-${old}.ebuild" + new_path="${new_ps}/${new_pkg}/${new_pkg_name}-${new}.ebuild" + + local ged_update_dir + update_dir "${new_pkg}" "${old_s}" "${new_s}" ged_update_dir + xdiff --unified=3 "${old_path}" "${new_path}" >"${ged_update_dir}/ebuild.diff" +} + +# Generate a report with information where the old and new packages +# are mentioned in entire scripts repository. May result in two +# separate reports if the package got renamed. +# +# 1 - path to scripts repo +# 2 - old package name +# 3 - new package name +function generate_package_mention_reports() { + local scripts old_pkg new_pkg + scripts=${1}; shift + old_pkg=${1}; shift + new_pkg=${1}; shift + + local gpr_update_dir + update_dir_non_slot "${new_pkg}" gpr_update_dir + + generate_mention_report_for_package "${scripts}" "${new_pkg}" >"${gpr_update_dir}/occurences" + + if [[ ${old_pkg} != "${new_pkg}" ]]; then + generate_mention_report_for_package "${scripts}" "${old_pkg}" >"${gpr_update_dir}/occurences-for-old-name" + fi +} + +# Generate a report with information where the package is mentioned in +# entire scripts repository. +# +# 1 - path to scripts repo +# 3 - package name +function generate_mention_report_for_package() { + local scripts pkg + scripts=${1}; shift + pkg=${1}; shift + + local ps co + ps='sdk_container/src/third_party/portage-stable' + co='sdk_container/src/third_party/coreos-overlay' + + yell "${pkg} in overlay profiles" + grep_pkg "${scripts}" "${pkg}" "${co}/profiles" + + yell "${pkg} in Gentoo profiles" + grep_pkg "${scripts}" "${pkg}" "${ps}/profiles" + + # shellcheck disable=SC2164 # we use set -e, so the script will exit if it fails + pushd "${scripts}/${co}" >/dev/null + + yell "${pkg} in env overrides" + cat_entries "coreos/config/env/${pkg}"@(|-+([0-9])*) + + yell "${pkg} in user patches" + local dir + for dir in "coreos/user-patches/${pkg}"@(|-+([0-9])*); do + echo "BEGIN DIRECTORY: ${dir}" + cat_entries "${dir}"/* + echo "END DIRECTORY: ${dir}" + done + + # shellcheck disable=SC2164 # we use set -e, so the script will exit if it fails + popd >/dev/null + + yell "${pkg} in overlay (outside profiles)" + grep_pkg "${scripts}" "${pkg}" "${co}" ":(exclude)${co}/profiles" + + yell "${pkg} in Gentoo (outside profiles)" + grep_pkg "${scripts}" "${pkg}" "${ps}" ":(exclude)${ps}/profiles" + + yell "${pkg} in scripts (outside overlay and Gentoo)" + grep_pkg "${scripts}" "${pkg}" ":(exclude)${ps}" ":(exclude)${co}" +} + +# Gets a toplevel update reports directory for a package. This is +# where occurences and non-ebuild diffs are stored. +# +# Params: +# +# 1 - package name +# 2 - name of a variable where the path will be stored +function update_dir_non_slot() { + local pkg + pkg=${1}; shift + local -n dir_ref=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + # shellcheck disable=SC2034 # it's a reference to external variable + dir_ref="${REPORTS_DIR}/updates/${pkg}" +} + +# Gets a slot specific update reports directory for a package. This is +# where ebuild diffs are stored. +# +# Params: +# +# 1 - package name +# 2 - old slot +# 3 - new slot +# 4 - name of a variable where the path will be stored +function update_dir() { + local pkg old_s new_s + pkg=${1}; shift + old_s=${1}; shift + new_s=${1}; shift + local -n dir_ref=${1}; shift + + # slots may have slashes in them - replace them with "-slash-" + local slot_dir + if [[ ${old_s} = "${new_s}" ]]; then + slot_dir=${old_s//\//-slash-} + else + slot_dir="${old_s//\//-slash-}-to-${new_s//\//-slash-}" + fi + + local ud_non_slot_dir + update_dir_non_slot "${pkg}" ud_non_slot_dir + # shellcheck disable=SC2034 # it's a reference to external variable + dir_ref="${ud_non_slot_dir}/${slot_dir}" +} + +# Greps for a package name in selected directories of the passed +# repo. It prints, so the invocation needs to be captured. +# +# Params: +# +# 1 - path to scripts repo +# 2 - package name +# @ - directories in the repo to limit the search for +function grep_pkg() { + local scripts pkg + scripts=${1}; shift + pkg=${1}; shift + # rest are directories + + git -C "${scripts}" grep "${pkg}"'\(-[0-9]\|[^a-zA-Z0-9_-]\|$\)' -- "${@}" || : +} + +# Prints the passed files preceding and following with BEGIN ENTRY and +# END ENTRY markers. +# +# Params: +# +# @ - the files to print +function cat_entries() { + for entry; do + echo "BEGIN ENTRY: ${entry}" + cat "${entry}" + echo "END ENTRY: ${entry}" + done +} + +# Reads the listings and renames, handles updates of both packages and +# non-packages (eclasses, licenses, profiles, etc.) +function handle_gentoo_sync() { + #mvm_debug_enable hgs_pkg_to_tags_mvm + mvm_declare hgs_pkg_to_tags_mvm + process_listings hgs_pkg_to_tags_mvm + + # shellcheck disable=SC2034 # passed to other function through a name + local -A hgs_renames_old_to_new_map=() + process_profile_updates_directory hgs_renames_old_to_new_map + + handle_package_changes hgs_renames_old_to_new_map hgs_pkg_to_tags_mvm + + mvm_unset hgs_pkg_to_tags_mvm + #mvm_debug_disable hgs_pkg_to_tags_mvm + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local old_head new_head + # shellcheck disable=SC2153 # OLD_STATE is not a misspelling + old_head=$(git -C "${OLD_STATE}" rev-parse HEAD) + new_head=$(git -C "${NEW_STATE}" rev-parse HEAD) + + local -A non_package_updates_set + non_package_updates_set=() + local path in_ps category + if [[ "${old_head}" != "${new_head}" ]]; then + while read -r path; do + if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then + continue + fi + in_ps=${path#"${PORTAGE_STABLE_SUFFIX}/"} + category=${in_ps%%/*} + case "${category}" in + eclass) + if [[ ${in_ps} != 'eclass/'+([^/])'.eclass' ]]; then + fail "unexpected updated file inside eclass directory: '${path}'" + fi + non_package_updates_set["${in_ps}"]=x + ;; + licenses|metadata|profiles|scripts) + non_package_updates_set["${category}"]=x + ;; + virtual|*-*) + # Package update, already handled + : + ;; + *) + fail "unexpected updated file '${path}'" + ;; + esac + done < <(git -C "${NEW_STATE}" diff-tree --no-commit-id --name-only -r "${old_head}" "${new_head}") + fi + + local entry + for entry in "${!non_package_updates_set[@]}"; do + case "${entry}" in + eclass/*) + handle_eclass "${entry}" + ;; + licenses) + handle_licenses + ;; + metadata) + info "not handling metadata updates, skipping" + ;; + profiles) + handle_profiles + ;; + scripts) + handle_scripts + ;; + *) + fail "unknown non-package update for ${entry}" + ;; + esac + done + sort_summary_stubs + sort_changelog_stubs +} + +# Sorts entries in the summary file if it exists. +function sort_summary_stubs() { + if [[ -f "${REPORTS_DIR}/updates/summary_stubs" ]]; then + sort_like_summary_stubs "${REPORTS_DIR}/updates/summary_stubs" + fi +} + +# Sorts entries in the summary file. +# +# Lines look like as follows: +# +# -BEGIN- +# - dev-lang/python: [DEV] +# - from 3.11.4 to 3.11.5 +# - no changes in ebuild +# - release notes: TODO +# +# - app-emulation/qemu: +# - from 8.0.3 to 8.0.4 +# - no changes in ebuild +# - release notes: TODO +# +# -END- +function sort_like_summary_stubs() { + local f + f=${1}; shift + + mvm_declare groups_mvm + + local -a lines entries + lines=() + entries=() + local -A dups + dups=() + + local REPLY line entry sss_lines_name dup_count + while read -r; do + if [[ -z ${REPLY} ]]; then + if [[ ${#lines[@]} -gt 0 ]]; then + line=${lines[0]} + entry=${line#-+([[:space:]])} + entry=${entry%%:*} + dup_count=${dups["${entry}"]:-0} + if [[ ${dup_count} -gt 0 ]]; then + dup_count=$((dup_count + 1)) + mvm_add groups_mvm "${entry}@${dup_count}" "${lines[@]}" + dups["${entry}"]=${dup_count} + else + mvm_get groups_mvm "${entry}" sss_lines_name + if [[ -n ${sss_lines_name} ]]; then + local -n lines_ref=${sss_lines_name} + mvm_add groups_mvm "${entry}@1" "${lines_ref[@]}" + unset -n lines_ref + mvm_remove groups_mvm "${entry}" + mvm_add groups_mvm "${entry}@2" "${lines[@]}" + dups["${entry}"]=2 + else + mvm_add groups_mvm "${entry}" "${lines[@]}" + entries+=( "${entry}" ) + fi + fi + lines=() + fi + else + lines+=( "${REPLY}" ) + fi + done < <(cat "${f}"; echo) # echo for final empty line, just in case + + if [[ ${#entries[@]} -eq 0 ]]; then + return 0 + fi + + local idx + { + while read -r line; do + dup_count=${dups["${line}"]:-0} + if [[ ${dup_count} -gt 0 ]]; then + idx=0 + while [[ ${idx} -lt ${dup_count} ]]; do + idx=$((idx + 1)) + mvm_get groups_mvm "${line}@${idx}" sss_lines_name + local -n lines_ref=${sss_lines_name} + printf '%s\n' "${lines_ref[@]}" '' + unset -n lines_ref + done + else + mvm_get groups_mvm "${line}" sss_lines_name + local -n lines_ref=${sss_lines_name} + printf '%s\n' "${lines_ref[@]}" '' + unset -n lines_ref + fi + done < <(printf '%s\n' "${entries[@]}" | csort) + } >"${f}" + mvm_unset groups_mvm +} + +# Sorts entries in changelog stub if it exists. +function sort_changelog_stubs() { + if [[ -f "${REPORTS_DIR}/updates/changelog_stubs" ]]; then + sort_like_changelog_stubs "${REPORTS_DIR}/updates/changelog_stubs" + fi +} + +# Sorts entries in changelog stub. +function sort_like_changelog_stubs() { + local f t + f=${1}; shift + t="${f}.tmp" + csort --output="${t}" "${f}" + mv -f "${t}" "${f}" +} + +# Invokes sort with C locale. Meant to be used in bash pipelines. +# +# Params: +# +# @ - additional parameters to passed to sort +function csort() { + LC_ALL=C sort "${@}" +} + +# Handle an eclass update. Basically generate a diff. +# +# Params: +# +# 1 - path to eclass file within an ebuild repo +function handle_eclass() { + local eclass + eclass=${1}; shift + + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a lines + lines=() + if [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]] && [[ -e "${NEW_PORTAGE_STABLE}/${eclass}" ]]; then + mkdir -p "${REPORTS_DIR}/updates/${eclass}" + xdiff --unified=3 "${OLD_PORTAGE_STABLE}/${eclass}" "${NEW_PORTAGE_STABLE}/${eclass}" >"${REPORTS_DIR}/updates/${eclass}/eclass.diff" + lines+=( 'TODO: review the diff' ) + elif [[ -e "${OLD_PORTAGE_STABLE}/${eclass}" ]]; then + lines+=( 'unused, dropped' ) + else + lines+=( 'added from Gentoo' ) + fi + generate_summary_stub "${eclass}" -- "${lines[@]}" +} + +# Handle profile changes. Generates three different diffs - changes in +# relevant profiles (ancestors of the profiles used by board packages +# and SDK), a full diff between all the profiles, and a list of +# possibly irrelevant files that has changed too. +function handle_profiles() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a files + files=() + local arch which report + for arch in "${ARCHES[@]}"; do + for which in "${WHICH[@]}"; do + for report in sdk-profiles board-profiles; do + files+=("${WORKDIR}/pkg-reports/${which}-${arch}/${report}") + done + done + done + local -A profile_dirs_set + profile_dirs_set=() + + local line + while read -r line; do + profile_dirs_set["${line}"]=x + done < <(xgrep --no-filename '^portage-stable:' "${files[@]}" | cut -d: -f2-) + + local -a diff_opts + diff_opts=( + --recursive + --unified=3 + --new-file # treat absent files as empty + ) + + local out_dir + out_dir="${REPORTS_DIR}/updates/profiles" + mkdir -p "${out_dir}" + + xdiff "${diff_opts[@]}" \ + "${OLD_PORTAGE_STABLE}/profiles" "${NEW_PORTAGE_STABLE}/profiles" >"${out_dir}/full.diff" + + local relevant + relevant='' + local -a relevant_lines possibly_irrelevant_files + relevant_lines=() + possibly_irrelevant_files=() + local REPLY path dir mark + while read -r; do + if [[ ${REPLY} = "diff "* ]]; then + path=${REPLY##*"${NEW_PORTAGE_STABLE}/profiles/"} + dirname_out "${path}" dir + relevant='' + mark=${profile_dirs_set["${dir}"]:-} + if [[ -n "${mark}" ]]; then + relevant=x + else + case ${dir} in + .|desc|desc/*|updates|updates/*) + relevant=x + ;; + esac + fi + if [[ -z ${relevant} ]]; then + possibly_irrelevant_files+=( "profiles/${path}" ) + fi + fi + if [[ -n ${relevant} ]]; then + relevant_lines+=( "${REPLY}" ) + fi + done <"${out_dir}/full.diff" + lines_to_file_truncate "${out_dir}/relevant.diff" "${relevant_lines[@]}" + lines_to_file_truncate "${out_dir}/possibly-irrelevant-files" "${possibly_irrelevant_files[@]}" + generate_summary_stub profiles -- 'TODO: review the diffs' +} + +# Handles changes in license directory. Generates brief reports and +# diffs about dropped, added or modified licenses. +function handle_licenses() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local -a dropped added changed + dropped=() + added=() + changed=() + + local line hl_stripped + # Lines are: + # + # Only in /licenses: BSL-1.1 + # + # or + # + # Files /licenses/BSL-1.1 and /licenses/BSL-1.1 differ + while read -r line; do + if [[ ${line} = 'Only in '* ]]; then + # shellcheck disable=SC2153 # OLD_STATE is not a misspelling, it comes from globals file + strip_out "${line##*:}" hl_stripped + if [[ ${line} = *"${OLD_STATE}"* ]]; then + dropped+=( "${hl_stripped}" ) + elif [[ ${line} = *"${NEW_STATE}"* ]]; then + added+=( "${hl_stripped}" ) + else + devel_warn "- unhandled license change: ${line}" + fi + elif [[ ${line} = 'Files '*' differ' ]]; then + line=${line##"Files ${OLD_PORTAGE_STABLE}/licenses/"} + line=${line%% *} + strip_out "${line}" hl_stripped + changed+=( "${hl_stripped}" ) + else + devel_warn \ + '- unhandled diff --brief line:' \ + " - ${line}" + fi + done < <(xdiff --brief --recursive "${OLD_PORTAGE_STABLE}/licenses" "${NEW_PORTAGE_STABLE}/licenses") + + local out_dir + out_dir="${REPORTS_DIR}/updates/licenses" + mkdir -p "${out_dir}" + + lines_to_file_truncate \ + "${out_dir}/brief-summary" \ + '- removed:' \ + "${dropped[@]/#/ - }" \ + '- added:' \ + "${added[@]/#/ - }" \ + '- modified:' \ + "${changed[@]/#/ - }" + truncate --size=0 "${out_dir}/modified.diff" + + local c + for c in "${changed[@]}"; do + xdiff --unified=3 "${OLD_PORTAGE_STABLE}/licenses/${c}" "${NEW_PORTAGE_STABLE}/licenses/${c}" >>"${out_dir}/modified.diff" + done + local -a lines + lines=() + + local joined + if [[ ${#dropped[@]} -gt 0 ]]; then + join_by joined ', ' "${dropped[@]}" + lines+=( "dropped ${joined}" ) + fi + if [[ ${#added[@]} -gt 0 ]]; then + join_by joined ', ' "${added[@]}" + lines+=( "added ${joined}" ) + fi + if [[ ${#changed[@]} -gt 0 ]]; then + join_by joined ', ' "${changed[@]}" + lines+=( "updated ${joined}" ) + fi + generate_summary_stub licenses -- "${lines[@]}" +} + +# Generates reports about changes inside the scripts directory. +function handle_scripts() { + # shellcheck disable=SC1091 # generated file + source "${WORKDIR}/globals" + + local out_dir + out_dir="${REPORTS_DIR}/updates/scripts" + mkdir -p "${out_dir}" + + xdiff --unified=3 --recursive "${OLD_PORTAGE_STABLE}/scripts" "${NEW_PORTAGE_STABLE}/scripts" >"${out_dir}/scripts.diff" + generate_summary_stub scripts -- 'TODO: review the diffs' +} + +# Enables debug logs when specific packages are processed. +# +# It is expected that globals were already sourced, otherwise +# debugging won't be enabled at all. +# +# Params: +# +# @ - package names to enable debugging for +function pkg_debug_enable() { + local -A pkg_set + pkg_set=() + local -a vals + vals=() + local pkg + for pkg; do + if [[ -n ${pkg_set["${pkg}"]:-} ]]; then + continue + fi + pkg_set["${pkg}"]=x + if [[ -n ${DEBUG_PACKAGES["${pkg}"]:-} ]]; then + vals+=( "${pkg}" ) + fi + done + if [[ ${#vals[@]} -gt 0 ]]; then + declare -g PKG_AUTO_LIB_DEBUG + join_by PKG_AUTO_LIB_DEBUG ',' "${vals[@]}" + fi +} + +# Returns true or false whether any debugging has been enabled. +function pkg_debug_possible() { + local ret=0 + [[ ${#DEBUG_PACKAGES[@]} -gt 0 ]] || ret=1 + return ${ret} +} + +# Disables debug logs to be printed. +function pkg_debug_disable() { + unset PKG_AUTO_LIB_DEBUG +} + +# Prints passed parameters if debugging is enabled. +# +# Params: +# +# @ - parameters to print +function pkg_debug() { + if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then + info "DEBUG(${PKG_AUTO_LIB_DEBUG}): ${*}" + fi +} + +# Prints passed lines if debugging is enabled. +# +# Params: +# +# @ - lines to print +function pkg_debug_lines() { + if [[ -n ${PKG_AUTO_LIB_DEBUG:-} ]]; then + info_lines "${@/#/"DEBUG(${PKG_AUTO_LIB_DEBUG}): "}" + fi +} + +fi diff --git a/pkg_auto/impl/print_profile_tree.sh b/pkg_auto/impl/print_profile_tree.sh new file mode 100755 index 0000000000..96ce744840 --- /dev/null +++ b/pkg_auto/impl/print_profile_tree.sh @@ -0,0 +1,274 @@ +#!/bin/bash + +## +## Prints profile information in form of an inheritance tree and/or +## evaluation order. +## +## Parameters: +## -h: this help +## -ni: no inheritance tree +## -ne: no evaluation order +## -nh: no headers +## +## Environment variables: +## ROOT +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +: "${ROOT:=/}" + +print_inheritance_tree=x +print_evaluation_order=x +print_headers=x + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -ni) + print_inheritance_tree= + ;; + -ne) + print_evaluation_order= + ;; + -nh) + print_headers= + ;; + *) + fail "unknown flag ${1}" + ;; + esac + shift +done + +all_repo_names=() +read -a all_repo_names -r < <(portageq get_repos "${ROOT}") + +declare -A repo_data repo_data_r +# name to path +repo_data=() +# path to name +repo_data_r=() + +for repo_name in "${all_repo_names[@]}"; do + repo_path=$(portageq get_repo_path "${ROOT}" "${repo_name}") + repo_path=$(realpath "${repo_path}") + repo_data["${repo_name}"]="${repo_path}" + repo_data_r["${repo_path}"]="${repo_name}" +done + +unset all_repo_names + +function get_repo_from_profile_path() { + local path + path=${1}; shift + local -n repo_dir_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + repo_dir_ref="${path%/profiles/*}" +} + +function repo_path_to_name() { + local path + path=${1}; shift + local -n name_ref=${1}; shift + + # shellcheck disable=SC2034 # it's a reference to external variable + name_ref=${repo_data_r["${path}"]:-''} +} + +function repeat_string() { + local str ntimes out_str_var_name + str=${1}; shift + ntimes=${1}; shift + out_str_var_name=${1}; shift + local -n out_str_ref="${out_str_var_name}" + + if [[ ${ntimes} -eq 0 ]]; then + out_str_ref="" + return 0 + elif [[ ${ntimes} -eq 1 ]]; then + out_str_ref="${str}" + return 0 + fi + local add_one + add_one=$((ntimes % 2)) + repeat_string "${str}${str}" $((ntimes / 2)) "${out_str_var_name}" + if [[ add_one -gt 0 ]]; then + out_str_ref+="${str}" + fi +} + +function process_profile() { + local repo_name profile_path + repo_name=${1}; shift + profile_path=${1}; shift + local -n children_ref=${1}; shift + + local parent_file line pp_new_repo_name new_profile_path pp_new_repo_path + local -a children + + parent_file="${profile_path}/parent" + children=() + if [[ -e ${parent_file} ]]; then + while read -r line; do + if [[ ${line} = *:* ]]; then + pp_new_repo_name=${line%%:*} + if [[ -z ${pp_new_repo_name} ]]; then + pp_new_repo_name=${repo_name} + fi + pp_new_repo_path=${repo_data["${pp_new_repo_name}"]} + new_profile_path="${pp_new_repo_path}/profiles/${line#*:}" + children+=( "${pp_new_repo_name}" "${new_profile_path}" ) + elif [[ ${line} = /* ]]; then + pp_new_repo_path= + get_repo_from_profile_path "${line}" pp_new_repo_path + pp_new_repo_name= + repo_path_to_name "${pp_new_repo_path}" pp_new_repo_name + children+=( "${pp_new_repo_name}" "${line}" ) + else + pp_new_repo_path=$(realpath "${profile_path}/${line}") + children+=( "${repo_name}" "${pp_new_repo_path}" ) + fi + done <"${parent_file}" + fi + + # shellcheck disable=SC2034 # it's a reference to external variable + children_ref=( "${children[@]}" ) +} + +function get_profile_name() { + local repo_name profile_path + repo_name="${1}"; shift + profile_path="${1}"; shift + local -n profile_name_ref=${1}; shift + + local repo_path profile_name + repo_path=${repo_data["${repo_name}"]} + profile_name=${profile_path#"${repo_path}/profiles/"} + + # shellcheck disable=SC2034 # it's a reference to external variable + profile_name_ref="${profile_name}" +} + +make_profile_path="${ROOT%/}/etc/portage/make.profile" +top_profile_dir_path=$(realpath "${make_profile_path}") +top_repo_path= +get_repo_from_profile_path "${top_profile_dir_path}" top_repo_path +top_repo_name= +repo_path_to_name "${top_repo_path}" top_repo_name + +if [[ -n ${print_inheritance_tree} ]]; then + +set -- '0' "${top_repo_name}" "${top_profile_dir_path}" + +profile_tree=() + +while [[ ${#} -gt 2 ]]; do + indent=${1}; shift + repo_name=${1}; shift + profile_path=${1}; shift + + lines= + fork= + if [[ ${indent} -gt 0 ]]; then + if [[ ${indent} -gt 1 ]]; then + repeat_string '| ' $((indent - 1)) lines + fi + fork='+-' + fi + g_profile_name= + get_profile_name "${repo_name}" "${profile_path}" g_profile_name + profile_tree+=( "${lines}${fork}${repo_name}:${g_profile_name}" ) + g_profile_children=() + + process_profile "${repo_name}" "${profile_path}" g_profile_children + + new_profiles=() + new_indent=$((indent + 1)) + pc_idx=0 + while [[ $((pc_idx + 1)) -lt "${#g_profile_children[@]}" ]]; do + new_repo_name=${g_profile_children["${pc_idx}"]} + new_profile_path=${g_profile_children[$((pc_idx + 1))]} + new_profiles+=( "${new_indent}" "${new_repo_name}" "${new_profile_path}" ) + pc_idx=$((pc_idx + 2)) + done + + set -- "${new_profiles[@]}" "${@}" +done + +if [[ -n ${print_headers} ]]; then + echo + echo 'profile inheritance tree:' + echo +fi +for line in "${profile_tree[@]}"; do + echo "${line}" +done + +fi + +if [[ -n ${print_evaluation_order} ]]; then + +set -- "${top_repo_name}" "${top_profile_dir_path}" '0' + +profile_eval=() + +while [[ ${#} -gt 2 ]]; do + repo_name=${1}; shift + profile_path=${1}; shift + num_parents=${1}; shift + # each parent is a repo name and profile path, so two items for each parent + num_parent_items=$((num_parents * 2)) + parents=( "${@:1:${num_parent_items}}" ) + shift "${num_parent_items}" + g_profile_children=() + + process_profile "${repo_name}" "${profile_path}" g_profile_children + + new_args=() + if [[ ${#g_profile_children[@]} -eq 0 ]]; then + to_evaluate=( "${repo_name}" "${profile_path}" "${parents[@]}" ) + te_idx=0 + while [[ $((te_idx + 1)) -lt "${#to_evaluate[@]}" ]]; do + new_repo_name=${to_evaluate["${te_idx}"]} + new_profile_path=${to_evaluate[$((te_idx + 1))]} + g_new_profile_name= + get_profile_name "${new_repo_name}" "${new_profile_path}" g_new_profile_name + profile_eval+=( "${new_repo_name}:${g_new_profile_name}" ) + te_idx=$((te_idx + 2)) + done + else + last_idx=$(( ${#g_profile_children[@]} - 2 )) + pc_idx=0 + while [[ $((pc_idx + 1)) -lt "${#g_profile_children[@]}" ]]; do + new_repo_name=${g_profile_children["${pc_idx}"]} + new_profile_path=${g_profile_children[$((pc_idx + 1))]} + new_args+=( "${new_repo_name}" "${new_profile_path}" ) + if [[ pc_idx -eq last_idx ]]; then + new_args+=( $((num_parents + 1)) "${repo_name}" "${profile_path}" "${parents[@]}" ) + else + new_args+=( 0 ) + fi + pc_idx=$((pc_idx + 2)) + done + fi + + set -- "${new_args[@]}" "${@}" +done + +if [[ -n ${print_headers} ]]; then + echo + echo 'profile evaluation order:' + echo +fi +for line in "${profile_eval[@]}"; do + echo "${line}" +done + +fi diff --git a/pkg_auto/impl/sort_packages_list.py b/pkg_auto/impl/sort_packages_list.py new file mode 100755 index 0000000000..909cd0b6ff --- /dev/null +++ b/pkg_auto/impl/sort_packages_list.py @@ -0,0 +1,206 @@ +#!/usr/bin/python3 + +# The package list file is a document that consists of a header and an +# empty-line-separated list of package groups. Header is a list of all +# lines until the first package group. Package group is a list of +# packages of the same category and possibly some related +# comments. The comments are usually about packages that are +# temporarily excluded from the list. So the comments usually have two +# parts - the free form part and a commented-out package list part; +# the example would be: +# +# # Temporarily excluded from automated updates, because reasons. +# # +# # sys-devel/whatever +# +# The script tries to preserve the comments and its ordering, so it +# associates the free form part to the package name. +# +# The script also deduplicates the packages while sorting. An edge +# case is when a package appears multiple times and is not +# commented-out at least once - all commented out entries are dropped. +# +# Implementation-wise, the document has a list of lines being a +# header, a list of free form comments and a map of category name to a +# group. A group is a list of packages, where each package has a name, +# information if it's commented out and may have a free form comment +# associated with it. + +import re +import sys + +class FreeForm: + def __init__(self, lines): + self.lines = lines + +class Pkg: + def __init__(self, idx, name, out): + self.free_form_idx = idx + self.name = name + self.commented_out = out + +class Group: + def __init__(self): + self.pkgs = [] + self.pkg_names_set = set() + +class Document: + def __init__(self): + self.header = [] + self.free_forms = [] + self.groups = {} + +class Reader: + category_or_pkg_pattern = re.compile("^[a-z0-9-]+(?:/[A-Za-z0-9-_+]+)?$") + parsing_header = 1 + parsing_group = 2 + parsing_comment = 3 + + def __init__(self, doc): + self.doc = doc + self.parsing_stage = Reader.parsing_header + self.current_comments = [] + self.free_form_idx_for_next_pkg = None + + def get_group(self, category): + if category not in self.doc.groups: + new_group = Group() + self.doc.groups[category] = new_group + return new_group + return self.doc.groups[category] + + def add_pkg_impl(self, idx, name, out): + category = name.split('/', 1)[0] + group = self.get_group(category) + if name in group.pkg_names_set: + if not out: + for pkg in group.pkgs: + if pkg.name == name: + pkg.commented_out = False + break + else: + group.pkg_names_set.add(name) + group.pkgs += [Pkg(idx, name, out)] + return True + return False + + def add_pkg(self, name): + if self.add_pkg_impl(self.free_form_idx_for_next_pkg, name, False): + self.free_form_idx_for_next_pkg = None + + class CommentBatch: + def __init__(self, ff_lines, p_lines): + self.free_form_lines = ff_lines + self.pkg_lines = p_lines + + def get_batches(self): + batches = [] + free_form_lines = [] + pkg_lines = [] + for line in self.current_comments: + line = line.lstrip('#').strip() + if not line: + if not pkg_lines: + free_form_lines += [line] + elif Reader.category_or_pkg_pattern.match(line): + pkg_lines += [line] + else: + if pkg_lines: + while not free_form_lines[-1]: + free_form_lines = free_form_lines[:-1] + batches += [Reader.CommentBatch(free_form_lines, pkg_lines)] + free_form_lines = [] + pkg_lines = [] + free_form_lines += [line] + self.current_comments = [] + if free_form_lines or pkg_lines: + batches += [Reader.CommentBatch(free_form_lines, pkg_lines)] + return batches + + def process_current_comments(self): + for batch in self.get_batches(): + free_form_idx = None + if batch.free_form_lines: + free_form_idx = len(self.doc.free_forms) + self.doc.free_forms += [FreeForm(batch.free_form_lines)] + if batch.pkg_lines: + for line in batch.pkg_lines: + self.add_pkg_impl(free_form_idx, line, True) + else: + self.free_form_idx_for_next_pkg = free_form_idx + + def read(self, input): + while line := input.readline(): + line = line.strip() + if self.parsing_stage == Reader.parsing_header: + if not line: + self.parsing_stage = Reader.parsing_group + elif line.startswith('#'): + self.doc.header += [line] + else: + self.parsing_stage = Reader.parsing_group + self.add_pkg(line) + elif self.parsing_stage == Reader.parsing_group: + if not line: + pass + elif line.startswith('#'): + self.current_comments += [line] + self.parsing_stage = Reader.parsing_comment + else: + self.add_pkg(line) + elif self.parsing_stage == Reader.parsing_comment: + if not line: + self.parsing_stage = Reader.parsing_group + self.process_current_comments() + elif line.startswith('#'): + self.current_comments += [line] + else: + self.parsing_stage = Reader.parsing_group + self.process_current_comments() + self.add_pkg(line) + if self.current_comments: + self.process_current_comments() + +class Writer: + def __init__(self, doc): + self.doc = doc + + def write(self, output): + output_lines = [] + if self.doc.header: + output_lines += self.doc.header + output_lines += [''] + for category in sorted(self.doc.groups): + last_free_form_idx = None + for pkg in sorted(self.doc.groups[category].pkgs, key=lambda pkg: pkg.name): + if pkg.free_form_idx != last_free_form_idx: + last_free_form_idx = pkg.free_form_idx + if pkg.free_form_idx is not None: + for line in self.doc.free_forms[pkg.free_form_idx].lines: + if line: + output_lines += [f"# {line}"] + else: + output_lines += ['#'] + if pkg.commented_out: + output_lines += [f"# {pkg.name}"] + else: + output_lines += [f"{pkg.name}"] + output_lines += [''] + while not output_lines[0]: + output_lines = output_lines[1:] + while not output_lines[-1]: + output_lines = output_lines[:-1] + for line in output_lines: + print(line, file=output) + +if __name__ == "__main__": + if len(sys.argv) != 2: + print(f"1 argument expected, got {len(sys.argv) - 1}", file=sys.stderr) + sys.exit(1) + filename = sys.argv[1] + doc = Document() + with open(filename, 'r', encoding='UTF-8') as file: + reader = Reader(doc) + reader.read(file) + writer = Writer(doc) + writer.write(sys.stdout) diff --git a/pkg_auto/impl/sync_with_gentoo.sh b/pkg_auto/impl/sync_with_gentoo.sh new file mode 100755 index 0000000000..2080f78c05 --- /dev/null +++ b/pkg_auto/impl/sync_with_gentoo.sh @@ -0,0 +1,316 @@ +#!/bin/bash + +## +## Used for syncing with gentoo. Needs to be called from the +## toplevel-directory of portage-stable. If syncing everything or +## syncing metadata/glsa specifically, it is expected that the Gentoo +## repo will have the GLSA files stored in metadata/glsa too. +## +## Parameters: +## -h: this help +## -b: be brief, print only names of changed entries and errors +## -s: skip adding source git commit hash information to commits +## +## Positional: +## 0: Gentoo repository +## #: Entries to update (can be a package name, eclass, category, some special +## directories like profiles or . for everything) +## +## Example invocations: +## +## sync_with_gentoo -h +## +## Print a help message. +## +## sync_with_gentoo dev-libs/nettle app-crypt/argon2 +## +## This will update the packages, each in a separate commit. The +## commit message will contain the commit hash from gentoo repo. +## +## sync_with_gentoo dev-libs +## +## This will update all the packages in dev-libs category. The +## commit message will contain the commit hash from gentoo repo. +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/util.sh" + +BRIEF= +SKIP_GIT_INFO= + +while true; do + case ${1-} in + -h) + print_help + exit 0 + ;; + -b) + BRIEF=x + shift + ;; + -s) + SKIP_GIT_INFO=x + shift + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ $# -lt 2 ]]; then + fail 'expected at least two positional parameters: a Gentoo repository and at least one package, use -h to print help' +fi + +if [[ ! -e 'profiles/repo_name' ]]; then + fail 'sync is only possible from ebuild packages top-level directory (a directory from which "./profiles/repo_name" is accessible)' +fi + +function vcall() { + if [[ -z ${BRIEF} ]]; then + "${@}" + fi +} + +function bcall() { + if [[ -n ${BRIEF} ]]; then + "${@}" + fi +} + +GENTOO=$(realpath "${1}"); shift +# rest are package names + +if [[ $(realpath '.') = "${GENTOO}" ]]; then + fail 'trying to sync within a Gentoo repo?' +fi + +if [[ -z ${SKIP_GIT_INFO} ]] && [[ ! -e ${GENTOO}/.git ]]; then + info "Skipping adding source git commit hash information to commits, ${GENTOO@Q} is not a git repository" + SKIP_GIT_INFO=x +fi + +glsa_repo=${GENTOO}/metadata/glsa +if [[ -z ${SKIP_GIT_INFO} ]] && [[ -e ${glsa_repo} ]] && [[ ! -e ${glsa_repo}/.git ]] && [[ $(git -C "${GENTOO}" status --porcelain -- metadata/glsa) = '?? metadata/glsa' ]]; then + info "Skipping adding source git commit hash information to commits, ${glsa_repo@Q} exists, but it is not a git repository and is not a part of Gentoo git repository" + SKIP_GIT_INFO=x +fi +unset glsa_repo + +# Synchronizes given path with its Gentoo counterpart. Returns true if +# there were changes. +# +# Params: +# +# 1 - path within ebuild repo +function sync_git_prepare() { + local path + path=${1}; shift + + local gentoo_path + gentoo_path="${GENTOO}/${path}" + + if [[ ! -e "${gentoo_path}" ]]; then + info "no ${path@Q} in Gentoo repository" + if [[ ${path} = 'metadata/glsa' ]]; then + info "did you forget to clone https://gitweb.gentoo.org/data/glsa.git/ into ${gentoo_path@Q}?" + fi + return 1 + fi + + local -a rsync_opts=( --archive --delete-before ) + + case ${path} in + profiles) + rsync_opts+=( --exclude /profiles/repo_name ) + ;; + esac + + local parent + dirname_out "${path}" parent + mkdir --parents "${parent}" + rsync "${rsync_opts[@]}" "${gentoo_path}" "${parent}" + if [[ -n $(git status --porcelain -- "${path}") ]]; then + bcall info "updated ${path}" + git add "${path}" + return 0 + fi + return 1 +} + +# Creates a git commit. If checking Gentoo commit ID is enabled the +# given path is used to get the ID of the commit with the last change +# in the path. Name parameter is used for denoting which part has +# changed, and sync parameter to denote if the commit is about adding +# new package or updating an existing one. +# +# Params: +# +# 1 - path +# 2 - name +# 3 - not empty if existing package was updated, or an empty string if +# the package is new +function commit_with_gentoo_sha() { + local path name sync + path=${1}; shift + name=${1}; shift + sync=${1:-}; shift + + local -a commit_extra=() + if [[ -z ${SKIP_GIT_INFO} ]]; then + local commit + + commit=$(git -C "${GENTOO}/${path}" log --pretty=oneline -1 -- . | cut -f1 -d' ') + commit_extra+=( --message "It's from Gentoo commit ${commit}." ) + unset commit + fi + commit_msg="${name}: Add from Gentoo" + if [[ -n "${sync}" ]]; then + commit_msg="${name}: Sync with Gentoo" + fi + git commit --quiet --message "${commit_msg}" "${commit_extra[@]}" + GIT_PAGER='cat' vcall git show --stat +} + +# Simple path sync and commit; takes the contents from Gentoo at the +# given path and puts it in the repo. +# +# 1 - path to sync +# 2 - name for commit message +function path_sync() { + local path name + path=${1}; shift + name=${1}; shift + + local sync + sync='' + if [[ -e "${path}" ]]; then + sync='x' + fi + + if sync_git_prepare "${path}"; then + commit_with_gentoo_sha "${path}" "${name}" "${sync}" + else + vcall info "no changes in ${path}" + fi +} + +# Goes over the given directory and syncs its subdirectories or +# files. No commit is created. +function prepare_dir() { + local dir + dir=${1}; shift + + local pkg mod='' + for pkg in "${dir}/"*; do + if sync_git_prepare "${pkg}"; then + mod=x + fi + done + if [[ -n ${mod} ]]; then + return 0 + fi + return 1 +} + +# Synces entire category of packages and creates a commit. Note that +# if the category already exists, no new packages will be added. +# +# Params: +# +# 1 - path to the category directory +function category_sync() { + local path + path=${1}; shift + + if [[ ! -e "${path}" ]]; then + if sync_git_prepare "${path}"; then + commit_with_gentoo_sha "${path}" "${path}" + fi + else + if prepare_dir "${path}"; then + commit_with_gentoo_sha "${path}" "${path}" 'x' + fi + fi + +} + +# Synces entire repo. No new packages will be added. +function everything_sync() { + local path mod + + for path in *; do + case ${path} in + licenses|profiles|scripts) + if sync_git_prepare "${path}"; then + mod=x + fi + ;; + metadata) + # do only metadata updates + if sync_git_prepare metadata/glsa; then + mod=x + fi + ;; + eclass|virtual|*-*) + if prepare_dir "${path}"; then + mod=x + fi + ;; + changelog|*.md) + # ignore those + : + ;; + *) + info "Unknown entry ${path@Q}, ignoring" + ;; + esac + done + if [[ -n ${mod} ]]; then + commit_with_gentoo_sha '.' '*' 'x' + fi +} + +shopt -s extglob + +for cpn; do + cpn=${cpn%%*(/)} + case ${cpn} in + .) + everything_sync + ;; + licenses|profiles|scripts) + path_sync "${cpn}" "${cpn}" + ;; + eclass/*.eclass) + path_sync "${cpn}" "${cpn%.eclass}" + ;; + metadata/glsa) + path_sync "${cpn}" "${cpn}" + ;; + metadata) + fail "metadata directory can't be synced, did you mean metadata/glsa?" + ;; + virtual/*/*|*-*/*/*) + fail "invalid thing to sync: ${cpn}" + ;; + virtual/*|*-*/*) + path_sync "${cpn}" "${cpn}" + ;; + eclass|virtual|*-*) + category_sync "${cpn}" + ;; + *) + fail "invalid thing to sync: ${cpn}" + ;; + esac +done diff --git a/pkg_auto/impl/util.sh b/pkg_auto/impl/util.sh new file mode 100644 index 0000000000..55ff223020 --- /dev/null +++ b/pkg_auto/impl/util.sh @@ -0,0 +1,211 @@ +#!/bin/bash + +if [[ -z ${__UTIL_SH_INCLUDED__:-} ]]; then +__UTIL_SH_INCLUDED__=x + +# Works like dirname, but without spawning new processes. +# +# Params: +# +# 1 - path to operate on +# 2 - name of a variable which will contain a dirname of the path +function dirname_out() { + local path dir_var_name + path=${1}; shift + dir_var_name=${1}; shift + local -n dir_ref=${dir_var_name} + + if [[ -z ${path} ]]; then + dir_ref='.' + return 0 + fi + local cleaned_up dn + # strip trailing slashes + cleaned_up=${path%%*(/)} + # strip duplicated slashes + cleaned_up=${cleaned_up//+(\/)/\/} + # strip last component + dn=${cleaned_up%/*} + if [[ -z ${dn} ]]; then + dir_ref='/' + return 0 + fi + if [[ ${cleaned_up} = "${dn}" ]]; then + dir_ref='.' + return 0 + fi + # shellcheck disable=SC2034 # it's a reference to external variable + dir_ref=${dn} +} + +# Works like basename, but without spawning new processes. +# +# Params: +# +# 1 - path to operate on +# 2 - name of a variable which will contain a basename of the path +function basename_out() { + local path base_var_name + path=${1}; shift + base_var_name=${1}; shift + local -n base_ref=${base_var_name} + + if [[ -z ${path} ]]; then + base_ref='' + return 0 + fi + local cleaned_up dn + # strip trailing slashes + cleaned_up=${path%%*(/)} + if [[ -z ${cleaned_up} ]]; then + base_ref='/' + return 0 + fi + # strip duplicated slashes + cleaned_up=${cleaned_up//+(\/)/\/} + # keep last component + dn=${cleaned_up##*/} + # shellcheck disable=SC2034 # it's a reference to external variable + base_ref=${dn} +} + +if [[ ${BASH_SOURCE[-1]##*/} = 'util.sh' ]]; then + THIS=${BASH} + basename_out "${THIS}" THIS_NAME + THIS_DIR=. +else + THIS=${BASH_SOURCE[-1]} + basename_out "${THIS}" THIS_NAME + dirname_out "${THIS}" THIS_DIR +fi + +THIS=$(realpath "${THIS}") +THIS_DIR=$(realpath "${THIS_DIR}") +dirname_out "${BASH_SOURCE[0]}" PKG_AUTO_IMPL_DIR +PKG_AUTO_IMPL_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}") +# shellcheck disable=SC2034 # may be used by scripts sourcing this file +PKG_AUTO_DIR=$(realpath "${PKG_AUTO_IMPL_DIR}/..") + +# Prints an info line. +# +# Params: +# +# @ - strings to print +function info() { + printf '%s: %s\n' "${THIS_NAME}" "${*}" +} + +# Prints info lines. +# +# Params: +# +# @ - lines to print +function info_lines() { + printf '%s\n' "${@/#/"${THIS_NAME}: "}" +} + +# Prints an info to stderr and fails the execution. +# +# Params: +# +# @ - strings to print +function fail() { + info "${@}" >&2 + exit 1 +} + +# Prints infos to stderr and fails the execution. +# +# Params: +# +# @ - lines to print +function fail_lines() { + info_lines "${@}" >&2 + exit 1 +} + +# Yells a message. +# +# Params: +# +# @ - strings to yell +function yell() { + echo + echo '!!!!!!!!!!!!!!!!!!' + echo " ${*}" + echo '!!!!!!!!!!!!!!!!!!' + echo +} + +# Prints help. Help is taken from the lines prefixed with double +# hashes in the top sourcer of this file. +function print_help() { + if [[ ${THIS} != "${BASH}" ]]; then + grep '^##' "${THIS}" | sed -e 's/##[[:space:]]*//' + fi +} + +# Joins passed strings with a given delimiter. +# +# Params: +# +# 1 - name of a variable that will contain the joined result +# 2 - delimiter +# @ - strings to join +function join_by() { + local output_var_name delimiter first + + output_var_name=${1}; shift + delimiter=${1}; shift + first=${1-} + if shift; then + printf -v "${output_var_name}" '%s' "${first}" "${@/#/${delimiter}}"; + else + local -n output_ref=${output_var_name} + # shellcheck disable=SC2034 # it's a reference to external variable + output_ref='' + fi +} + +# Checks if directory is empty, returns true if so, otherwise false. +# +# Params: +# +# 1 - path to a directory +function dir_is_empty() { + local dir + dir=${1}; shift + + [[ -z $(echo "${dir}"/*) ]] +} + +# Just like diff, but ignores the return value. +function xdiff() { + diff "${@}" || : +} + +# Just like grep, but ignores the return value. +function xgrep() { + grep "${@}" || : +} + +# Strips leading and trailing whitespace from the passed parameter. +# +# Params: +# +# 1 - string to strip +# 2 - name of a variable where the result of stripping will be stored +function strip_out() { + local l + l=${1}; shift + local -n out_ref=${1}; shift + + local t + t=${l} + t=${t/#+([[:space:]])} + t=${t/%+([[:space:]])} + # shellcheck disable=SC2034 # it's a reference to external variable + out_ref=${t} +} + +fi diff --git a/pkg_auto/inside_sdk_container.sh b/pkg_auto/inside_sdk_container.sh new file mode 100755 index 0000000000..8651917916 --- /dev/null +++ b/pkg_auto/inside_sdk_container.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +## +## Gathers information about SDK and board packages. Also collects +## info about actual build deps of board packages, which may be useful +## for verifying if SDK provides those. +## +## Reports generated: +## sdk-pkgs - contains package information for SDK +## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK +## board-pkgs - contains package information for board for chosen architecture +## board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies +## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order +## board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order +## sdk-package-repos - contains package information with their repos for SDK +## board-package-repos - contains package information with their repos for board +## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports +## board-emerge-output - contains raw emerge output for board being a base for other reports +## sdk-emerge-output-filtered - contains only lines with package information for SDK +## board-emerge-output-filtered - contains only lines with package information for board +## sdk-emerge-output-junk - contains only junk lines for SDK +## board-emerge-output-junk - contains only junk lines for board +## *-warnings - warnings printed by emerge or other tools +## +## Parameters: +## -h: this help +## +## Positional: +## 1 - architecture (amd64 or arm64) +## 2 - reports directory +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/inside_sdk_container_lib.sh" + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 2 ]]; then + fail 'Expected two parameters: board architecture and reports directory' +fi + +arch=${1}; shift +reports_dir=${1}; shift + +mkdir -p "${reports_dir}" + +set_eo "${reports_dir}" + +echo 'Running pretend-emerge to get complete report for SDK' +package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}" +echo 'Running pretend-emerge to get complete report for board' +package_info_for_board "${arch}" >"${BOARD_EO}" 2>"${BOARD_EO_W}" + +ensure_no_errors + +echo 'Separating emerge info from junk in SDK emerge output' +filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}" +junk_sdk_eo >"${SDK_EO}-junk" 2>>"${SDK_EO_W}" +echo 'Separating emerge info from junk in board emerge output' +filter_board_eo "${arch}" >"${BOARD_EO_F}" 2>>"${BOARD_EO_W}" +junk_board_eo >"${BOARD_EO}-junk" 2>>"${BOARD_EO_W}" + +ensure_valid_reports + +echo 'Generating SDK packages listing' +versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings" +echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)' +versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings" +echo 'Generating board packages listing' +versions_board >"${reports_dir}/board-pkgs" 2>"${reports_dir}/board-pkgs-warnings" +echo 'Generating board packages bdeps listing' +board_bdeps >"${reports_dir}/board-bdeps" 2>"${reports_dir}/board-bdeps-warnings" +echo 'Generating SDK profiles evaluation list' +ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings" +echo 'Generating board profiles evaluation list' +ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/board-profiles" 2>"${reports_dir}/board-profiles-warnings" +echo 'Generating SDK package source information' +package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings" +echo 'Generating board package source information' +package_sources_board >"${reports_dir}/board-package-repos" 2>"${reports_dir}/board-package-repos-warnings" + +echo "Cleaning empty warning files" +clean_empty_warning_files "${reports_dir}" diff --git a/pkg_auto/sync_packages.sh b/pkg_auto/sync_packages.sh new file mode 100755 index 0000000000..03edc578e8 --- /dev/null +++ b/pkg_auto/sync_packages.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +## +## Synces the packages with Gentoo. +## +## Parameters: +## -f: remove reports directory if it exists at startup +## -w: path to use for workdir +## -h: this help +## +## Positional: +## 1: config file +## 2: new branch name with updates +## 3: gentoo repo +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/pkg_auto_lib.sh" + +workdir='' + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -w) + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + workdir=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 3 ]]; then + fail 'expected three positional parameters: a config file, a final branch name and a path to Gentoo repo' +fi + +config_file=${1}; shift +saved_branch_name=${1}; shift +gentoo=${1}; shift + +setup_workdir_with_config "${workdir}" "${config_file}" +perform_sync_with_gentoo "${gentoo}" +save_new_state "${saved_branch_name}" diff --git a/pkg_auto/update_packages.sh b/pkg_auto/update_packages.sh new file mode 100755 index 0000000000..2eb5a9c217 --- /dev/null +++ b/pkg_auto/update_packages.sh @@ -0,0 +1,60 @@ +#!/bin/bash + +## +## Updates the packages +## +## Parameters: +## -w: path to use for workdir +## -h: this help +## +## Positional: +## 1: config file +## 2: new branch name with updates +## 3: gentoo repo +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" +source "${PKG_AUTO_IMPL_DIR}/pkg_auto_lib.sh" + +workdir='' + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -w) + if [[ -z ${2:-} ]]; then + fail 'missing value for -w' + fi + workdir=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 3 ]]; then + fail 'expected three positional parameters: a config file, a final branch name and a path to Gentoo repo' +fi + +config_file=${1}; shift +saved_branch_name=${1}; shift +gentoo=${1}; shift + +setup_workdir_with_config "${workdir}" "${config_file}" +perform_sync_with_gentoo "${gentoo}" +save_new_state "${saved_branch_name}" +generate_package_update_reports