mirror of
https://github.com/flatcar/scripts.git
synced 2025-08-07 13:06:59 +02:00
pkg-auto: Rework reports generation
Reports generation used to be executed four times. The number of runs was a result of cartesian product of two sets - old and new state, and of amd64 and arm64 architectures. It was pretty much a slow process because egencache was called implicitly four times, and it was running in a single-threaded fashion, and also SDK reports were duplicated (they were the same for old-amd64 and old-arm64, and the same for new-amd64 and new-arm64 runs). This changes the generation, so it is being run only two times - once for old state and once for new state. Every run generates SDK packages reports and per-architecture board packages reports. Egencache will now utilize more threads too.
This commit is contained in:
Krzesimir Nowak
parent
00572476c4
commit
80d12ea75f
@ -97,79 +97,24 @@ function package_info_for_board() {
|
||||
# 1 - directory path
|
||||
function set_eo() {
|
||||
local dir=${1}; shift
|
||||
# rest are architectures
|
||||
|
||||
# shellcheck disable=SC2034 # used externally
|
||||
declare -g EGENCACHE_W="${dir}/egencache-warnings"
|
||||
declare -g SDK_EO="${dir}/sdk-emerge-output"
|
||||
declare -g BOARD_EO="${dir}/board-emerge-output"
|
||||
# shellcheck disable=SC2034 # used indirectly in cat_eo_f
|
||||
declare -g SDK_EO_F="${SDK_EO}-filtered"
|
||||
# shellcheck disable=SC2034 # used indirectly in cat_eo_f
|
||||
declare -g BOARD_EO_F="${BOARD_EO}-filtered"
|
||||
# shellcheck disable=SC2034 # used indirectly in cat_eo_w
|
||||
declare -g SDK_EO_W="${SDK_EO}-warnings"
|
||||
# shellcheck disable=SC2034 # used indirectly in cat_eo_w
|
||||
declare -g BOARD_EO_W="${BOARD_EO}-warnings"
|
||||
}
|
||||
declare -g SDK_EO_J="${SDK_EO}-junk"
|
||||
|
||||
# Print the contents of file, path of which is stored in a variable of
|
||||
# a given name.
|
||||
#
|
||||
# Params:
|
||||
#
|
||||
# 1 - name of the variable
|
||||
function cat_var() {
|
||||
local var_name
|
||||
var_name=${1}; shift
|
||||
local -n ref="${var_name}"
|
||||
|
||||
if [[ -z "${ref+isset}" ]]; then
|
||||
fail "${var_name} unset"
|
||||
fi
|
||||
if [[ ! -e "${ref}" ]]; then
|
||||
fail "${ref} does not exist"
|
||||
fi
|
||||
|
||||
cat "${ref}"
|
||||
}
|
||||
|
||||
# Print contents of the emerge output of a given kind. Kind can be
|
||||
# either either sdk or board.
|
||||
#
|
||||
# Params:
|
||||
#
|
||||
# 1 - kind
|
||||
function cat_eo() {
|
||||
local kind
|
||||
kind=${1}; shift
|
||||
|
||||
cat_var "${kind^^}_EO"
|
||||
}
|
||||
|
||||
# Print contents of the filtered emerge output of a given kind. Kind
|
||||
# can be either either sdk or board. The filtered emerge output
|
||||
# contains only lines with package information.
|
||||
#
|
||||
# Params:
|
||||
#
|
||||
# 1 - kind
|
||||
function cat_eo_f() {
|
||||
local kind
|
||||
kind=${1}; shift
|
||||
cat_var "${kind^^}_EO_F"
|
||||
}
|
||||
|
||||
# Print contents of a file that stores warnings that were printed by
|
||||
# emerge. The warnings are specific to a kind (sdk or board).
|
||||
#
|
||||
# Params:
|
||||
#
|
||||
# 1 - kind
|
||||
function cat_eo_w() {
|
||||
local kind
|
||||
kind=${1}; shift
|
||||
|
||||
cat_var "${kind^^}_EO_W"
|
||||
local arch
|
||||
local board_eo
|
||||
for arch; do
|
||||
board_eo=${dir}/${arch}-board-emerge-output
|
||||
declare -g "${arch^^}_BOARD_EO=${board_eo}"
|
||||
declare -g "${arch^^}_BOARD_EO_F=${board_eo}-filtered"
|
||||
declare -g "${arch^^}_BOARD_EO_W=${board_eo}-warnings"
|
||||
declare -g "${arch^^}_BOARD_EO_J=${board_eo}-junk"
|
||||
done
|
||||
}
|
||||
|
||||
# JSON output would be more verbose, but probably would not require
|
||||
@ -195,7 +140,7 @@ FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${V
|
||||
|
||||
# Filters sdk reports to get the package information.
|
||||
function filter_sdk_eo() {
|
||||
cat_eo sdk | xgrep -e "${FULL_LINE_RE}"
|
||||
cat "${SDK_EO}" | xgrep -e "${FULL_LINE_RE}"
|
||||
}
|
||||
|
||||
# Filters board reports for a given arch to get the package
|
||||
@ -205,11 +150,12 @@ function filter_sdk_eo() {
|
||||
#
|
||||
# 1 - architecture
|
||||
function filter_board_eo() {
|
||||
local arch
|
||||
local arch name
|
||||
arch=${1}; shift
|
||||
name=${arch^^}_BOARD_EO
|
||||
|
||||
# Replace ${arch}-usr in the output with a generic word BOARD.
|
||||
cat_eo board | \
|
||||
cat "${!name}" | \
|
||||
xgrep -e "${FULL_LINE_RE}" | \
|
||||
sed -e "s#/build/${arch}-usr/#/build/BOARD/#"
|
||||
}
|
||||
@ -217,13 +163,17 @@ function filter_board_eo() {
|
||||
# Filters sdk reports to get anything but the package information
|
||||
# (i.e. junk).
|
||||
function junk_sdk_eo() {
|
||||
cat_eo sdk | xgrep -v -e "${FULL_LINE_RE}"
|
||||
cat "${SDK_EO}" | xgrep -v -e "${FULL_LINE_RE}"
|
||||
}
|
||||
|
||||
# Filters board reports to get anything but the package information
|
||||
# (i.e. junk).
|
||||
function junk_board_eo() {
|
||||
cat_eo board | xgrep -v -e "${FULL_LINE_RE}"
|
||||
local arch name
|
||||
arch=${1}; shift
|
||||
name=${arch^^}_BOARD_EO
|
||||
|
||||
cat "${!name}" | xgrep -v -e "${FULL_LINE_RE}"
|
||||
}
|
||||
|
||||
# More regexp-like abominations follow.
|
||||
@ -276,21 +226,6 @@ PKG_REPO_SED_FILTERS=(
|
||||
-e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}${SPACES_RE}.*/\1 \3/"
|
||||
)
|
||||
|
||||
# Applies some sed filter over the emerge output of a given kind.
|
||||
# Results are printed.
|
||||
#
|
||||
# Params:
|
||||
#
|
||||
# 1 - kind (sdk or board)
|
||||
# @ - parameters passed to sed
|
||||
function sed_eo_and_sort() {
|
||||
local kind
|
||||
kind=${1}; shift
|
||||
# rest goes to sed
|
||||
|
||||
cat_eo_f "${kind}" | sed "${@}" | sort
|
||||
}
|
||||
|
||||
# Applies some sed filter over the SDK emerge output. Results are
|
||||
# printed.
|
||||
#
|
||||
@ -298,9 +233,7 @@ function sed_eo_and_sort() {
|
||||
#
|
||||
# @ - parameters passed to sed
|
||||
function packages_for_sdk() {
|
||||
# args are passed to sed_eo_and_sort
|
||||
|
||||
sed_eo_and_sort sdk "${@}"
|
||||
cat "${SDK_EO_F}" | sed "${@}" | sort
|
||||
}
|
||||
|
||||
# Applies some sed filter over the board emerge output. Results are
|
||||
@ -310,9 +243,12 @@ function packages_for_sdk() {
|
||||
#
|
||||
# @ - parameters passed to sed
|
||||
function packages_for_board() {
|
||||
# args are passed to sed_eo_and_sort
|
||||
local arch=${1}; shift
|
||||
# rest goes to sed
|
||||
|
||||
sed_eo_and_sort board "${@}"
|
||||
local name=${arch^^}_BOARD_EO_F
|
||||
|
||||
cat "${!name}" | sed "${@}" | sort
|
||||
}
|
||||
|
||||
# Prints package name, slot and version information for SDK.
|
||||
@ -336,24 +272,26 @@ function versions_sdk_with_key_values() {
|
||||
|
||||
# Prints package name, slot and version information for board.
|
||||
function versions_board() {
|
||||
local arch=${1}; shift
|
||||
local -a sed_opts
|
||||
sed_opts=(
|
||||
-e '/to \/build\/BOARD\// ! d'
|
||||
"${PKG_VER_SLOT_SED_FILTERS[@]}"
|
||||
)
|
||||
packages_for_board "${sed_opts[@]}"
|
||||
packages_for_board "${arch}" "${sed_opts[@]}"
|
||||
}
|
||||
|
||||
# Prints package name, slot, version and key-values information for
|
||||
# build dependencies of board. Key-values may be something like
|
||||
# USE="foo bar -baz".
|
||||
function board_bdeps() {
|
||||
local arch=${1}; shift
|
||||
local -a sed_opts
|
||||
sed_opts=(
|
||||
-e '/to \/build\/BOARD\// d'
|
||||
"${PKG_VER_SLOT_KV_SED_FILTERS[@]}"
|
||||
)
|
||||
packages_for_board "${sed_opts[@]}"
|
||||
packages_for_board "${arch}" "${sed_opts[@]}"
|
||||
}
|
||||
|
||||
# Print package name and source repository names information for SDK.
|
||||
@ -368,21 +306,29 @@ function package_sources_sdk() {
|
||||
# Print package name and source repository names information for
|
||||
# board.
|
||||
function package_sources_board() {
|
||||
local arch=${1}; shift
|
||||
local -a sed_opts
|
||||
sed_opts=(
|
||||
"${PKG_REPO_SED_FILTERS[@]}"
|
||||
)
|
||||
packages_for_board "${sed_opts[@]}"
|
||||
packages_for_board "${arch}" "${sed_opts[@]}"
|
||||
}
|
||||
|
||||
# Checks if no errors were produced by emerge when generating
|
||||
# reports. It is assumed that emerge will print a line with "ERROR" in
|
||||
# it to denote a failure.
|
||||
function ensure_no_errors() {
|
||||
local kind
|
||||
local -a files=( "${SDK_EO_W}" )
|
||||
local arch name
|
||||
|
||||
for kind in sdk board; do
|
||||
if cat_eo_w "${kind}" | grep --quiet --fixed-strings 'ERROR'; then
|
||||
for arch; do
|
||||
name=${arch^^}_BOARD_EO_W
|
||||
files+=( "${!name}" )
|
||||
done
|
||||
|
||||
local file
|
||||
for file in "${files[@]}"; do
|
||||
if cat "${file}" | grep --quiet --fixed-strings 'ERROR'; then
|
||||
fail "there are errors in emerge output warnings files"
|
||||
fi
|
||||
done
|
||||
@ -460,10 +406,17 @@ function revert_crossdev_stuff() {
|
||||
|
||||
# Checks if the expected reports were generated by emerge.
|
||||
function ensure_valid_reports() {
|
||||
local kind var_name
|
||||
for kind in sdk board; do
|
||||
var_name="${kind^^}_EO_F"
|
||||
if [[ ! -s ${!var_name} ]]; then
|
||||
local -a files=( "${SDK_EO_F}" )
|
||||
local arch name
|
||||
|
||||
for arch; do
|
||||
name=${arch^^}_BOARD_EO_F
|
||||
files+=( "${!name}" )
|
||||
done
|
||||
|
||||
local file
|
||||
for file in "${files[@]}"; do
|
||||
if [[ ! -s ${file} ]]; then
|
||||
fail "report files are missing or are empty"
|
||||
fi
|
||||
done
|
||||
@ -486,10 +439,28 @@ function clean_empty_warning_files() {
|
||||
done
|
||||
}
|
||||
|
||||
function get_num_proc() {
|
||||
local -n num_proc_ref=${1}; shift
|
||||
local -i num_proc=1 rv=1
|
||||
|
||||
# stolen from portage
|
||||
[[ rv -eq 0 ]] || { rv=0; num_proc=$(getconf _NPROCESSORS_ONLN 2>/dev/null) || rv=1; }
|
||||
[[ rv -eq 0 ]] || { rv=0; num_proc=$(sysctl -n hw.ncpu 2>/dev/null) || rv=1; }
|
||||
# stolen from common.sh
|
||||
[[ rv -eq 0 ]] || { rv=0; num_proc=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null) || rv=1; }
|
||||
[[ rv -eq 0 ]] || { rv=0; num_proc=1; }
|
||||
|
||||
num_proc_ref=${num_proc}
|
||||
}
|
||||
|
||||
function generate_cache_for() {
|
||||
local repo=${1}; shift
|
||||
|
||||
egencache --repo "${repo}" --update
|
||||
local -i gcf_num_proc
|
||||
local load_avg
|
||||
get_num_proc gcf_num_proc
|
||||
load_avg=$(bc <<< "${gcf_num_proc} * 0.75")
|
||||
egencache --repo "${repo}" --jobs="${gcf_num_proc}" --load-average="${load_avg}" --update
|
||||
}
|
||||
|
||||
function copy_cache_to_reports() {
|
||||
|
||||
@ -1114,78 +1114,96 @@ function generate_sdk_reports() {
|
||||
add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports"
|
||||
mkdir "${WORKDIR}/pkg-reports"
|
||||
|
||||
local arch sdk_image_var_name sdk_image_name
|
||||
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${SDK_IMAGE}"; then
|
||||
fail "No SDK image named ${SDK_IMAGE@Q} available locally, pull it before running this script"
|
||||
fi
|
||||
|
||||
local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch
|
||||
local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy
|
||||
local -a report_files run_sdk_container_args
|
||||
for arch in "${ARCHES[@]}"; do
|
||||
sdk_image_var_name="SDK_IMAGE"
|
||||
sdk_image_name=${!sdk_image_var_name}
|
||||
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${sdk_image_name}"; then
|
||||
fail "No SDK image named ${sdk_image_name@Q} available locally, pull it before running this script"
|
||||
local pkg_auto_copy rv
|
||||
local sdk_reports_dir top_dir dir entry full_path
|
||||
local -a dir_queue all_dirs all_files
|
||||
|
||||
# shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file
|
||||
for sdk_run_kind in "${WHICH[@]}"; do
|
||||
state_var_name="${sdk_run_kind^^}_STATE"
|
||||
sdk_run_state="${!state_var_name}_sdk_run"
|
||||
state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH"
|
||||
sdk_run_state_branch="${!state_branch_var_name}-sdk-run"
|
||||
|
||||
add_cleanup \
|
||||
"git -C ${sdk_run_state@Q} reset --hard HEAD" \
|
||||
"git -C ${sdk_run_state@Q} clean -ffdx" \
|
||||
"git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \
|
||||
"git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}"
|
||||
git -C "${SCRIPTS}" \
|
||||
worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}"
|
||||
|
||||
pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX")
|
||||
add_cleanup "rm -rf ${pkg_auto_copy@Q}"
|
||||
cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}"
|
||||
local -a run_sdk_container_args=(
|
||||
-C "${SDK_IMAGE}"
|
||||
-n "pkg-${sdk_run_kind}"
|
||||
-U
|
||||
-m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto"
|
||||
--rm
|
||||
./pkg_auto/inside_sdk_container.sh pkg-reports "${ARCHES[@]}"
|
||||
)
|
||||
rv=0
|
||||
env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?}
|
||||
unset run_sdk_container_args
|
||||
if [[ ${rv} -ne 0 ]]; then
|
||||
local salvaged_dir
|
||||
salvaged_dir="${REPORTS_DIR}/salvaged-reports"
|
||||
{
|
||||
info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue"
|
||||
info
|
||||
for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do
|
||||
info "from ${file}:"
|
||||
echo
|
||||
cat "${file}"
|
||||
echo
|
||||
done
|
||||
info
|
||||
info 'whatever reports generated by the failed run are saved in'
|
||||
info "${salvaged_dir@Q} directory"
|
||||
info
|
||||
} >&2
|
||||
rm -rf "${salvaged_dir}"
|
||||
cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}"
|
||||
unset salvaged_dir
|
||||
fail "copying done, stopping now"
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file
|
||||
for sdk_run_kind in "${WHICH[@]}"; do
|
||||
state_var_name="${sdk_run_kind^^}_STATE"
|
||||
sdk_run_state="${!state_var_name}_sdk_run_${arch}"
|
||||
state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH"
|
||||
sdk_run_state_branch="${!state_branch_var_name}-sdk-run-${arch}"
|
||||
|
||||
add_cleanup \
|
||||
"git -C ${sdk_run_state@Q} reset --hard HEAD" \
|
||||
"git -C ${sdk_run_state@Q} clean -ffdx" \
|
||||
"git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \
|
||||
"git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}"
|
||||
git -C "${SCRIPTS}" \
|
||||
worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}"
|
||||
|
||||
pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX")
|
||||
add_cleanup "rm -rf ${pkg_auto_copy@Q}"
|
||||
cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}"
|
||||
local -a run_sdk_container_args=(
|
||||
-C "${sdk_image_name}"
|
||||
-n "pkg-${sdk_run_kind}-${arch}"
|
||||
-a "${arch}"
|
||||
-U
|
||||
-m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto"
|
||||
--rm
|
||||
./pkg_auto/inside_sdk_container.sh "${arch}" pkg-reports
|
||||
)
|
||||
rv=0
|
||||
env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?}
|
||||
if [[ ${rv} -ne 0 ]]; then
|
||||
{
|
||||
salvaged_dir="${REPORTS_DIR}/salvaged-reports"
|
||||
info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue"
|
||||
info
|
||||
for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do
|
||||
info "from ${file}:"
|
||||
echo
|
||||
cat "${file}"
|
||||
echo
|
||||
done
|
||||
info
|
||||
info 'whatever reports generated by the failed run are saved in'
|
||||
info "${salvaged_dir@Q} directory"
|
||||
info
|
||||
} >&2
|
||||
rm -rf "${salvaged_dir}"
|
||||
cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}"
|
||||
fail "copying done, stopping now"
|
||||
sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}"
|
||||
top_dir="${sdk_run_state}/pkg-reports"
|
||||
dir_queue=( "${top_dir}" )
|
||||
all_dirs=()
|
||||
all_files=()
|
||||
while [[ ${#dir_queue[@]} -gt 0 ]]; do
|
||||
dir=${dir_queue[0]}
|
||||
dir_queue=( "${dir_queue[@]:1}" )
|
||||
entry=${dir#"${top_dir}"}
|
||||
if [[ -z ${entry} ]]; then
|
||||
all_dirs=( "${sdk_reports_dir}" "${all_dirs[@]}" )
|
||||
else
|
||||
entry=${entry#/}
|
||||
all_dirs=( "${sdk_reports_dir}/${entry}" "${all_dirs[@]}" )
|
||||
fi
|
||||
sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}-${arch}"
|
||||
report_files=()
|
||||
for full_file in "${sdk_run_state}/pkg-reports/"*; do
|
||||
file=${full_file##"${sdk_run_state}/pkg-reports/"}
|
||||
report_files+=( "${sdk_reports_dir}/${file}" )
|
||||
for full_path in "${dir}/"*; do
|
||||
if [[ -d ${full_path} ]]; then
|
||||
dir_queue+=( "${full_path}" )
|
||||
else
|
||||
entry=${full_path##"${top_dir}/"}
|
||||
all_files+=( "${sdk_reports_dir}/${entry}" )
|
||||
fi
|
||||
done
|
||||
add_cleanup \
|
||||
"rm -f ${report_files[*]@Q}" \
|
||||
"rmdir ${sdk_reports_dir@Q}"
|
||||
mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}"
|
||||
done
|
||||
add_cleanup \
|
||||
"rm -f ${all_files[*]@Q}" \
|
||||
"rmdir ${all_dirs[*]@Q}"
|
||||
mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}"
|
||||
done
|
||||
|
||||
cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk"
|
||||
}
|
||||
|
||||
@ -1319,6 +1337,24 @@ function pkginfo_c_process_file() {
|
||||
mvm_c_get_extra 'arch' arch
|
||||
mvm_c_get_extra 'report' report
|
||||
|
||||
local report_file
|
||||
case ${report}:${arch} in
|
||||
"${SDK_PKGS}:arm64")
|
||||
# short-circuit it, there's no arm64 sdk
|
||||
return 0
|
||||
;;
|
||||
"${SDK_PKGS}:amd64")
|
||||
report_file="${WORKDIR}/pkg-reports/${which}/${report}"
|
||||
;;
|
||||
"${BOARD_PKGS}:"*)
|
||||
report_file="${WORKDIR}/pkg-reports/${which}/${arch}-${report}"
|
||||
;;
|
||||
*)
|
||||
local c=${report}:${arch}
|
||||
devel_warn "unknown report-architecture combination (${c@Q})"
|
||||
return 0
|
||||
esac
|
||||
|
||||
local pkg version_slot throw_away v s
|
||||
# shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else
|
||||
while read -r pkg version_slot throw_away; do
|
||||
@ -1331,7 +1367,7 @@ function pkginfo_c_process_file() {
|
||||
pkg_set_ref["${pkg}"]='x'
|
||||
mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}"
|
||||
pkg_debug_disable
|
||||
done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}"
|
||||
done <"${report_file}"
|
||||
}
|
||||
|
||||
# Gets a profile of the pkginfo mvm. The "profile" is a confusing
|
||||
@ -1710,28 +1746,36 @@ function consistency_checks() {
|
||||
function read_package_sources() {
|
||||
local -n package_sources_map_ref=${1}; shift
|
||||
|
||||
local arch which report pkg repo saved_repo
|
||||
for arch in "${ARCHES[@]}"; do
|
||||
for which in "${WHICH[@]}"; do
|
||||
for report in sdk-package-repos board-package-repos; do
|
||||
while read -r pkg repo; do
|
||||
saved_repo=${package_sources_map_ref["${pkg}"]:-}
|
||||
if [[ -n ${saved_repo} ]]; then
|
||||
if [[ ${saved_repo} != "${repo}" ]]; then
|
||||
pkg_warn \
|
||||
'- different repos used for the package:' \
|
||||
" - package: ${pkg}" \
|
||||
' - repos:' \
|
||||
" - ${saved_repo}" \
|
||||
" - ${repo}"
|
||||
fi
|
||||
else
|
||||
package_sources_map_ref["${pkg}"]=${repo}
|
||||
fi
|
||||
done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}"
|
||||
done
|
||||
# shellcheck disable=SC1091 # generated file
|
||||
source "${WORKDIR}/globals"
|
||||
|
||||
local -a files=()
|
||||
local which arch
|
||||
for which in "${WHICH[@]}"; do
|
||||
files+=( "${WORKDIR}/pkg-reports/${which}/sdk-package-repos" )
|
||||
for arch in "${ARCHES[@]}"; do
|
||||
files+=( "${WORKDIR}/pkg-reports/${which}/${arch}-board-package-repos" )
|
||||
done
|
||||
done
|
||||
|
||||
local file pkg repo saved_repo
|
||||
for file in "${files[@]}"; do
|
||||
while read -r pkg repo; do
|
||||
saved_repo=${package_sources_map_ref["${pkg}"]:-}
|
||||
if [[ -n ${saved_repo} ]]; then
|
||||
if [[ ${saved_repo} != "${repo}" ]]; then
|
||||
pkg_warn \
|
||||
'- different repos used for the package:' \
|
||||
" - package: ${pkg}" \
|
||||
' - repos:' \
|
||||
" - ${saved_repo}" \
|
||||
" - ${repo}"
|
||||
fi
|
||||
else
|
||||
package_sources_map_ref["${pkg}"]=${repo}
|
||||
fi
|
||||
done <"${file}"
|
||||
done
|
||||
}
|
||||
|
||||
# This monstrosity takes renames map and package tags information,
|
||||
@ -2950,14 +2994,12 @@ function handle_profiles() {
|
||||
# shellcheck disable=SC1091 # generated file
|
||||
source "${WORKDIR}/globals"
|
||||
|
||||
local -a files
|
||||
files=()
|
||||
local arch which report
|
||||
for arch in "${ARCHES[@]}"; do
|
||||
for which in "${WHICH[@]}"; do
|
||||
for report in sdk-profiles board-profiles; do
|
||||
files+=("${WORKDIR}/pkg-reports/${which}-${arch}/${report}")
|
||||
done
|
||||
local -a files=()
|
||||
local which arch
|
||||
for which in "${WHICH[@]}"; do
|
||||
files+=("${WORKDIR}/pkg-reports/${which}/sdk-profiles")
|
||||
for arch in "${ARCHES[@]}"; do
|
||||
files+=("${WORKDIR}/pkg-reports/${which}/${arch}-board-profiles")
|
||||
done
|
||||
done
|
||||
local -A profile_dirs_set
|
||||
|
||||
@ -1,33 +1,34 @@
|
||||
#!/bin/bash
|
||||
|
||||
##
|
||||
## Gathers information about SDK and board packages. Also collects
|
||||
## info about actual build deps of board packages, which may be useful
|
||||
## for verifying if SDK provides those.
|
||||
## Gathers information about SDK packages and board packages for each
|
||||
## passed architecture. Also collects info about actual build deps of
|
||||
## board packages, which may be useful for verifying if SDK provides
|
||||
## those.
|
||||
##
|
||||
## Reports generated:
|
||||
## sdk-pkgs - contains package information for SDK
|
||||
## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK
|
||||
## board-pkgs - contains package information for board for chosen architecture
|
||||
## board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies
|
||||
## ${arch}-board-pkgs - contains package information for board for chosen architecture
|
||||
## ${arch}-board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies
|
||||
## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order
|
||||
## board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order
|
||||
## ${arch}-board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order
|
||||
## sdk-package-repos - contains package information with their repos for SDK
|
||||
## board-package-repos - contains package information with their repos for board
|
||||
## ${arch}-board-package-repos - contains package information with their repos for board
|
||||
## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports
|
||||
## board-emerge-output - contains raw emerge output for board being a base for other reports
|
||||
## ${arch}-board-emerge-output - contains raw emerge output for board being a base for other reports
|
||||
## sdk-emerge-output-filtered - contains only lines with package information for SDK
|
||||
## board-emerge-output-filtered - contains only lines with package information for board
|
||||
## ${arch}-board-emerge-output-filtered - contains only lines with package information for board
|
||||
## sdk-emerge-output-junk - contains only junk lines for SDK
|
||||
## board-emerge-output-junk - contains only junk lines for board
|
||||
## ${arch}-board-emerge-output-junk - contains only junk lines for board
|
||||
## *-warnings - warnings printed by emerge or other tools
|
||||
##
|
||||
## Parameters:
|
||||
## -h: this help
|
||||
##
|
||||
## Positional:
|
||||
## 1 - architecture (amd64 or arm64)
|
||||
## 2 - reports directory
|
||||
## 1 - reports directory
|
||||
## # - architectures (currently only amd64 or arm64 are valid)
|
||||
##
|
||||
|
||||
set -euo pipefail
|
||||
@ -54,16 +55,16 @@ while [[ ${#} -gt 0 ]]; do
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ${#} -ne 2 ]]; then
|
||||
fail 'Expected two parameters: board architecture and reports directory'
|
||||
if [[ ${#} -lt 2 ]]; then
|
||||
fail 'Expected at least two parameters: reports directory and one or more board architectures'
|
||||
fi
|
||||
|
||||
arch=${1}; shift
|
||||
reports_dir=${1}; shift
|
||||
# rest are architectures
|
||||
|
||||
mkdir -p "${reports_dir}"
|
||||
|
||||
set_eo "${reports_dir}"
|
||||
set_eo "${reports_dir}" "${@}"
|
||||
|
||||
echo 'Running egencache for portage-stable'
|
||||
generate_cache_for 'portage-stable' 2>"${EGENCACHE_W}"
|
||||
@ -77,36 +78,48 @@ copy_cache_to_reports 'coreos-overlay' "${reports_dir}" 2>>"${EGENCACHE_W}"
|
||||
|
||||
echo 'Running pretend-emerge to get complete report for SDK'
|
||||
package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}"
|
||||
echo 'Running pretend-emerge to get complete report for board'
|
||||
package_info_for_board "${arch}" >"${BOARD_EO}" 2>"${BOARD_EO_W}"
|
||||
for arch; do
|
||||
be=${arch^^}_BOARD_EO
|
||||
bew=${arch^^}_BOARD_EO_W
|
||||
echo "Running pretend-emerge to get complete report for ${arch} board"
|
||||
package_info_for_board "${arch}" >"${!be}" 2>"${!bew}"
|
||||
done
|
||||
|
||||
ensure_no_errors
|
||||
ensure_no_errors "${@}"
|
||||
|
||||
echo 'Separating emerge info from junk in SDK emerge output'
|
||||
filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}"
|
||||
junk_sdk_eo >"${SDK_EO}-junk" 2>>"${SDK_EO_W}"
|
||||
echo 'Separating emerge info from junk in board emerge output'
|
||||
filter_board_eo "${arch}" >"${BOARD_EO_F}" 2>>"${BOARD_EO_W}"
|
||||
junk_board_eo >"${BOARD_EO}-junk" 2>>"${BOARD_EO_W}"
|
||||
junk_sdk_eo >"${SDK_EO_J}" 2>>"${SDK_EO_W}"
|
||||
for arch; do
|
||||
bej=${arch^^}_BOARD_EO_J
|
||||
bef=${arch^^}_BOARD_EO_F
|
||||
bew=${arch^^}_BOARD_EO_W
|
||||
echo "Separating emerge info from junk in ${arch} board emerge output"
|
||||
filter_board_eo "${arch}" >"${!bef}" 2>>"${!bew}"
|
||||
junk_board_eo "${arch}" >"${!bej}" 2>>"${!bew}"
|
||||
done
|
||||
|
||||
ensure_valid_reports
|
||||
ensure_valid_reports "${@}"
|
||||
|
||||
echo 'Generating SDK packages listing'
|
||||
versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings"
|
||||
echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)'
|
||||
versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings"
|
||||
echo 'Generating board packages listing'
|
||||
versions_board >"${reports_dir}/board-pkgs" 2>"${reports_dir}/board-pkgs-warnings"
|
||||
echo 'Generating board packages bdeps listing'
|
||||
board_bdeps >"${reports_dir}/board-bdeps" 2>"${reports_dir}/board-bdeps-warnings"
|
||||
echo 'Generating SDK profiles evaluation list'
|
||||
ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings"
|
||||
echo 'Generating board profiles evaluation list'
|
||||
ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/board-profiles" 2>"${reports_dir}/board-profiles-warnings"
|
||||
echo 'Generating SDK package source information'
|
||||
package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings"
|
||||
echo 'Generating board package source information'
|
||||
package_sources_board >"${reports_dir}/board-package-repos" 2>"${reports_dir}/board-package-repos-warnings"
|
||||
|
||||
for arch; do
|
||||
echo "Generating ${arch} board packages listing"
|
||||
versions_board "${arch}" >"${reports_dir}/${arch}-board-pkgs" 2>"${reports_dir}/${arch}-board-pkgs-warnings"
|
||||
echo "Generating ${arch} board packages bdeps listing"
|
||||
board_bdeps "${arch}" >"${reports_dir}/${arch}-board-bdeps" 2>"${reports_dir}/${arch}-board-bdeps-warnings"
|
||||
echo "Generating ${arch} board profiles evaluation list"
|
||||
ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/${arch}-board-profiles" 2>"${reports_dir}/${arch}-board-profiles-warnings"
|
||||
echo "Generating ${arch} board package source information"
|
||||
package_sources_board "${arch}" >"${reports_dir}/${arch}-board-package-repos" 2>"${reports_dir}/${arch}-board-package-repos-warnings"
|
||||
done
|
||||
|
||||
echo "Cleaning empty warning files"
|
||||
clean_empty_warning_files "${reports_dir}"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user