From 310a8716f49622d56271374b64da414772e90366 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Mon, 13 Jan 2025 14:22:19 +0100 Subject: [PATCH 1/4] pkg-auto: Fix syncing eclass files eclass is a file, as opposed to the package, which is a directory. So doing `git -C eclass/foo.eclass log -- .` will fail because we can't do "cd eclass/foo.eclass", which is what `-C` is trying to do. --- pkg_auto/impl/sync_with_gentoo.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_auto/impl/sync_with_gentoo.sh b/pkg_auto/impl/sync_with_gentoo.sh index 2080f78c05..0ec280f895 100755 --- a/pkg_auto/impl/sync_with_gentoo.sh +++ b/pkg_auto/impl/sync_with_gentoo.sh @@ -169,7 +169,7 @@ function commit_with_gentoo_sha() { if [[ -z ${SKIP_GIT_INFO} ]]; then local commit - commit=$(git -C "${GENTOO}/${path}" log --pretty=oneline -1 -- . | cut -f1 -d' ') + commit=$(git -C "${GENTOO}" log --pretty=oneline -1 -- "${path}" | cut -f1 -d' ') commit_extra+=( --message "It's from Gentoo commit ${commit}." ) unset commit fi From 9fe7006812b454895b49adef9d14f5324997defb Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 15 Jan 2025 17:22:45 +0100 Subject: [PATCH 2/4] pkg-auto: Allow emerging multiple packages for reports --- pkg_auto/impl/inside_sdk_container_lib.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index 04f6367a88..4e30391426 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -11,11 +11,10 @@ source "$(dirname "${BASH_SOURCE[0]}")/util.sh" # Params: # # 1 - root filesystem with the portage config -# 2 - metapackage to get the deps from +# @ - packages and metapackages to get the deps from function emerge_pretend() { - local root package + local root root=${1}; shift - package=${1}; shift # Probably a bunch of those flags are not necessary, but I'm not # touching it - they seem to be working. :) @@ -50,7 +49,7 @@ function emerge_pretend() { ) local rv rv=0 - emerge "${emerge_opts[@]}" "${package}" || rv=${?} + emerge "${emerge_opts[@]}" "${@}" || rv=${?} if [[ ${rv} -ne 0 ]]; then echo "WARNING: emerge exited with status ${rv}" >&2 fi From 6536b2cf3d2ed77191308ce814062cb39f4bd931 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Wed, 15 Jan 2025 17:23:23 +0100 Subject: [PATCH 3/4] pkg-auto: Emerge dev-lang/rust for SDK reports The coreos-devel/sdk-depends is a metapackage that is emerged in stage4 of the catalyst SDK build. But the fsscript that is being run at the end of the stage4 build also pulls in dev-lang/rust package. Thus pull both packages for the report. This should result in updated dev-lang/rust to appear in the reports. --- pkg_auto/impl/inside_sdk_container_lib.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg_auto/impl/inside_sdk_container_lib.sh b/pkg_auto/impl/inside_sdk_container_lib.sh index 4e30391426..13f4e456ea 100644 --- a/pkg_auto/impl/inside_sdk_container_lib.sh +++ b/pkg_auto/impl/inside_sdk_container_lib.sh @@ -61,7 +61,10 @@ function package_info_for_sdk() { root='/' ignore_crossdev_stuff "${root}" - emerge_pretend "${root}" coreos-devel/sdk-depends + # stage4 build of SDK builds coreos-devel/sdk-depends, fsscript + # pulls in cross toolchains with crossdev (which we have just + # ignored) and dev-lang/rust + emerge_pretend "${root}" coreos-devel/sdk-depends dev-lang/rust revert_crossdev_stuff "${root}" } From 020f9fe19ca7e6282d7f80d64cb8077d8dc9ddf2 Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Thu, 16 Jan 2025 19:57:13 +0100 Subject: [PATCH 4/4] pkg-auto: Add some tools These tools I found useful when I had to investigate why report generation failed. Since report generation failed, I had no reports about ebuild diffs or package occurences. The new scripts, `diff_pkg.sh` and `occurences.sh`, allowed me to get these reports for the troublesome package. --- pkg_auto/diff_pkg.sh | 145 +++++++++++++++++++++++++++++++++++++++++ pkg_auto/occurences.sh | 43 ++++++++++++ 2 files changed, 188 insertions(+) create mode 100755 pkg_auto/diff_pkg.sh create mode 100755 pkg_auto/occurences.sh diff --git a/pkg_auto/diff_pkg.sh b/pkg_auto/diff_pkg.sh new file mode 100755 index 0000000000..a98538a425 --- /dev/null +++ b/pkg_auto/diff_pkg.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/util.sh" + +## +## Prints a diff of a package in old and new scripts +## directories. Operates in one of two modes. First mode is an +## "ebuild" mode where it shows a diff of two ebuilds. The other mode +## is an "other" mode where it shows diffs of all non-ebuild files +## (ignoring Manifest files too). +## +## Parameters: +## -h: this help +## -r : New package name (useful when package got renamed) +## +## Positional: +## 1: mode, "ebuild" (or "e") or "other" (or "o") +## 2: path to old scripts repository +## 3: path to new scripts repository +## 4: package name (but see -r flag too) +## 5: old version (for "ebuild" mode only) +## 6: new version (for "ebuild" mode only) +## + +renamed= + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + -r) + renamed=${2} + shift 2 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag ${1@Q}" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -lt 1 ]]; then + fail 'Use -h to get help' +fi + +mode=${1}; shift + +case ${mode} in + ebuild|ebuil|ebui|ebu|eb|e) + mode=e + if [[ ${#} -ne 5 ]]; then + fail 'Expected five positional parameters: a path to old and new scripts repositories, a package name, and old a new version of package' + fi + ;; + other|othe|oth|ot|o) + mode=o + # expect at least three parameters, if more are given, they + # will be ignored, to allow changing mode from ebuild to other + # without the need for removing versions from the command + # invocation + if [[ ${#} -lt 3 ]]; then + fail 'Expected three positional parameters: a path to old and new scripts repositories, and a package name' + fi + ;; +esac + +old_scripts=${1} +new_scripts=${2} +# old and new package name +old_package=${3} +new_package=${renamed:-${3}} + +gentoo_path=sdk_container/src/third_party/portage-stable +overlay_path=sdk_container/src/third_party/coreos-overlay +old_gentoo_path=${old_scripts}/${gentoo_path}/${old_package} +old_overlay_path=${old_scripts}/${overlay_path}/${old_package} +new_gentoo_path=${new_scripts}/${gentoo_path}/${new_package} +new_overlay_path=${new_scripts}/${overlay_path}/${new_package} + +if [[ -e ${old_gentoo_path} ]] && [[ -e ${old_overlay_path} ]]; then + fail "Package ${old_package@Q} exists in both gentoo and overlay in old scripts" +fi + +if [[ -e ${new_gentoo_path} ]] && [[ -e ${new_overlay_path} ]]; then + fail "Package ${new_package@Q} exists in both gentoo and overlay in new scripts" +fi + +if [[ ${mode} = e ]]; then + old_version=${4} + new_version=${5} + old_gentoo_ebuild=${old_gentoo_path}/${old_package#*/}-${old_version}.ebuild + old_overlay_ebuild=${old_overlay_path}/${old_package#*/}-${old_version}.ebuild + new_gentoo_ebuild=${new_gentoo_path}/${new_package#*/}-${new_version}.ebuild + new_overlay_ebuild=${new_overlay_path}/${new_package#*/}-${new_version}.ebuild + + old_path='' + new_path='' + + if [[ -e ${old_gentoo_ebuild} ]]; then + old_path=${old_gentoo_ebuild} + fi + if [[ -e ${old_overlay_ebuild} ]]; then + old_path=${old_overlay_ebuild} + fi + + if [[ -e ${new_gentoo_ebuild} ]]; then + new_path=${new_gentoo_ebuild} + fi + if [[ -e ${new_overlay_ebuild} ]]; then + new_path=${new_overlay_ebuild} + fi + + if [[ -z ${old_path} ]]; then + fail "Old version ${old_version@Q} does not exist neither in overlay nor in gentoo" + fi + + if [[ -z ${new_path} ]]; then + fail "New version ${new_version@Q} does not exist neither in overlay nor in gentoo" + fi + + diff --color --unified=3 "${old_path}" "${new_path}" || : +else + old_path=${old_gentoo_path} + new_path=${new_gentoo_path} + + if [[ -e ${old_overlay_path} ]]; then + old_path=${old_overlay_path} + fi + + if [[ -e ${new_overlay_path} ]]; then + new_path=${new_overlay_path} + fi + + diff --color --recursive --unified=3 --new-file --exclude='*.ebuild' --exclude='Manifest' "${old_path}" "${new_path}" || : +fi diff --git a/pkg_auto/occurences.sh b/pkg_auto/occurences.sh new file mode 100755 index 0000000000..2422bf2c78 --- /dev/null +++ b/pkg_auto/occurences.sh @@ -0,0 +1,43 @@ +#!/bin/bash + +## +## Print all the places in the scripts repository where the given +## package is mentioned. It's sort of like grep, but a bit smarter and +## with a slightly better output. +## +## Parameters: +## -h: this help +## +## Positional: +## 1: scripts repo +## 2: package name +## + +set -euo pipefail + +source "$(dirname "${BASH_SOURCE[0]}")/impl/pkg_auto_lib.sh" + +while [[ ${#} -gt 0 ]]; do + case ${1} in + -h) + print_help + exit 0 + ;; + --) + shift + break + ;; + -*) + fail "unknown flag '${1}'" + ;; + *) + break + ;; + esac +done + +if [[ ${#} -ne 2 ]]; then + fail 'Expected two positional parameters: a path to scripts repository and a package name' +fi + +generate_mention_report_for_package "${1}" "${2}"