mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-08-07 13:06:59 +02:00
Code Issues Packages Projects Releases Wiki Activity

pkg-auto: Rework reports generation

Browse Source 
Reports generation used to be executed four times. The number of runs
was a result of cartesian product of two sets - old and new state, and
of amd64 and arm64 architectures. It was pretty much a slow process
because egencache was called implicitly four times, and it was running
in a single-threaded fashion, and also SDK reports were duplicated
(they were the same for old-amd64 and old-arm64, and the same for
new-amd64 and new-arm64 runs).

This changes the generation, so it is being run only two times - once
for old state and once for new state. Every run generates SDK packages
reports and per-architecture board packages reports. Egencache will
now utilize more threads too.
This commit is contained in:
Krzesimir Nowak 2025-04-10 18:09:52 +02:00
parent 00572476c4
commit 80d12ea75f
3 changed files with 255 additions and 229 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

175
pkg_auto/impl/inside_sdk_container_lib.sh
View File

@ -97,79 +97,24 @@ function package_info_for_board() {
# 1 - directory path # 1 - directory path
function set_eo() { function set_eo() {
local dir=${1}; shift local dir=${1}; shift
# rest are architectures
# shellcheck disable=SC2034 # used externally # shellcheck disable=SC2034 # used externally
declare -g EGENCACHE_W="${dir}/egencache-warnings" declare -g EGENCACHE_W="${dir}/egencache-warnings"
declare -g SDK_EO="${dir}/sdk-emerge-output" declare -g SDK_EO="${dir}/sdk-emerge-output"
declare -g BOARD_EO="${dir}/board-emerge-output"
# shellcheck disable=SC2034 # used indirectly in cat_eo_f
declare -g SDK_EO_F="${SDK_EO}-filtered" declare -g SDK_EO_F="${SDK_EO}-filtered"
# shellcheck disable=SC2034 # used indirectly in cat_eo_f
declare -g BOARD_EO_F="${BOARD_EO}-filtered"
# shellcheck disable=SC2034 # used indirectly in cat_eo_w
declare -g SDK_EO_W="${SDK_EO}-warnings" declare -g SDK_EO_W="${SDK_EO}-warnings"
# shellcheck disable=SC2034 # used indirectly in cat_eo_w declare -g SDK_EO_J="${SDK_EO}-junk"
declare -g BOARD_EO_W="${BOARD_EO}-warnings"
}
# Print the contents of file, path of which is stored in a variable of local arch
# a given name. local board_eo
# for arch; do
# Params: board_eo=${dir}/${arch}-board-emerge-output
# declare -g "${arch^^}_BOARD_EO=${board_eo}"
# 1 - name of the variable declare -g "${arch^^}_BOARD_EO_F=${board_eo}-filtered"
function cat_var() { declare -g "${arch^^}_BOARD_EO_W=${board_eo}-warnings"
local var_name declare -g "${arch^^}_BOARD_EO_J=${board_eo}-junk"
var_name=${1}; shift done
local -n ref="${var_name}"
if [[ -z "${ref+isset}" ]]; then
fail "${var_name} unset"
fi
if [[ ! -e "${ref}" ]]; then
fail "${ref} does not exist"
fi
cat "${ref}"
}
# Print contents of the emerge output of a given kind. Kind can be
# either either sdk or board.
#
# Params:
#
# 1 - kind
function cat_eo() {
local kind
kind=${1}; shift
cat_var "${kind^^}_EO"
}
# Print contents of the filtered emerge output of a given kind. Kind
# can be either either sdk or board. The filtered emerge output
# contains only lines with package information.
#
# Params:
#
# 1 - kind
function cat_eo_f() {
local kind
kind=${1}; shift
cat_var "${kind^^}_EO_F"
}
# Print contents of a file that stores warnings that were printed by
# emerge. The warnings are specific to a kind (sdk or board).
#
# Params:
#
# 1 - kind
function cat_eo_w() {
local kind
kind=${1}; shift
cat_var "${kind^^}_EO_W"
} }
# JSON output would be more verbose, but probably would not require # JSON output would be more verbose, but probably would not require
@ -195,7 +140,7 @@ FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${V
# Filters sdk reports to get the package information. # Filters sdk reports to get the package information.
function filter_sdk_eo() { function filter_sdk_eo() {
cat_eo sdk | xgrep -e "${FULL_LINE_RE}" cat "${SDK_EO}" | xgrep -e "${FULL_LINE_RE}"
} }
# Filters board reports for a given arch to get the package # Filters board reports for a given arch to get the package
@ -205,11 +150,12 @@ function filter_sdk_eo() {
# #
# 1 - architecture # 1 - architecture
function filter_board_eo() { function filter_board_eo() {
local arch local arch name
arch=${1}; shift arch=${1}; shift
name=${arch^^}_BOARD_EO
# Replace ${arch}-usr in the output with a generic word BOARD. # Replace ${arch}-usr in the output with a generic word BOARD.
cat_eo board | \ cat "${!name}" | \
xgrep -e "${FULL_LINE_RE}" | \ xgrep -e "${FULL_LINE_RE}" | \
sed -e "s#/build/${arch}-usr/#/build/BOARD/#" sed -e "s#/build/${arch}-usr/#/build/BOARD/#"
} }
@ -217,13 +163,17 @@ function filter_board_eo() {
# Filters sdk reports to get anything but the package information # Filters sdk reports to get anything but the package information
# (i.e. junk). # (i.e. junk).
function junk_sdk_eo() { function junk_sdk_eo() {
cat_eo sdk | xgrep -v -e "${FULL_LINE_RE}" cat "${SDK_EO}" | xgrep -v -e "${FULL_LINE_RE}"
} }
# Filters board reports to get anything but the package information # Filters board reports to get anything but the package information
# (i.e. junk). # (i.e. junk).
function junk_board_eo() { function junk_board_eo() {
cat_eo board | xgrep -v -e "${FULL_LINE_RE}" local arch name
arch=${1}; shift
name=${arch^^}_BOARD_EO
cat "${!name}" | xgrep -v -e "${FULL_LINE_RE}"
} }
# More regexp-like abominations follow. # More regexp-like abominations follow.
@ -276,21 +226,6 @@ PKG_REPO_SED_FILTERS=(
-e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}${SPACES_RE}.*/\1 \3/" -e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}${SPACES_RE}.*/\1 \3/"
) )
# Applies some sed filter over the emerge output of a given kind.
# Results are printed.
#
# Params:
#
# 1 - kind (sdk or board)
# @ - parameters passed to sed
function sed_eo_and_sort() {
local kind
kind=${1}; shift
# rest goes to sed
cat_eo_f "${kind}" | sed "${@}" | sort
}
# Applies some sed filter over the SDK emerge output. Results are # Applies some sed filter over the SDK emerge output. Results are
# printed. # printed.
# #
@ -298,9 +233,7 @@ function sed_eo_and_sort() {
# #
# @ - parameters passed to sed # @ - parameters passed to sed
function packages_for_sdk() { function packages_for_sdk() {
# args are passed to sed_eo_and_sort cat "${SDK_EO_F}" | sed "${@}" | sort
sed_eo_and_sort sdk "${@}"
} }
# Applies some sed filter over the board emerge output. Results are # Applies some sed filter over the board emerge output. Results are
@ -310,9 +243,12 @@ function packages_for_sdk() {
# #
# @ - parameters passed to sed # @ - parameters passed to sed
function packages_for_board() { function packages_for_board() {
# args are passed to sed_eo_and_sort local arch=${1}; shift
# rest goes to sed
sed_eo_and_sort board "${@}" local name=${arch^^}_BOARD_EO_F
cat "${!name}" | sed "${@}" | sort
} }
# Prints package name, slot and version information for SDK. # Prints package name, slot and version information for SDK.
@ -336,24 +272,26 @@ function versions_sdk_with_key_values() {
# Prints package name, slot and version information for board. # Prints package name, slot and version information for board.
function versions_board() { function versions_board() {
local arch=${1}; shift
local -a sed_opts local -a sed_opts
sed_opts=( sed_opts=(
-e '/to \/build\/BOARD\// ! d' -e '/to \/build\/BOARD\// ! d'
"${PKG_VER_SLOT_SED_FILTERS[@]}" "${PKG_VER_SLOT_SED_FILTERS[@]}"
) )
packages_for_board "${sed_opts[@]}" packages_for_board "${arch}" "${sed_opts[@]}"
} }
# Prints package name, slot, version and key-values information for # Prints package name, slot, version and key-values information for
# build dependencies of board. Key-values may be something like # build dependencies of board. Key-values may be something like
# USE="foo bar -baz". # USE="foo bar -baz".
function board_bdeps() { function board_bdeps() {
local arch=${1}; shift
local -a sed_opts local -a sed_opts
sed_opts=( sed_opts=(
-e '/to \/build\/BOARD\// d' -e '/to \/build\/BOARD\// d'
"${PKG_VER_SLOT_KV_SED_FILTERS[@]}" "${PKG_VER_SLOT_KV_SED_FILTERS[@]}"
) )
packages_for_board "${sed_opts[@]}" packages_for_board "${arch}" "${sed_opts[@]}"
} }
# Print package name and source repository names information for SDK. # Print package name and source repository names information for SDK.
@ -368,21 +306,29 @@ function package_sources_sdk() {
# Print package name and source repository names information for # Print package name and source repository names information for
# board. # board.
function package_sources_board() { function package_sources_board() {
local arch=${1}; shift
local -a sed_opts local -a sed_opts
sed_opts=( sed_opts=(
"${PKG_REPO_SED_FILTERS[@]}" "${PKG_REPO_SED_FILTERS[@]}"
) )
packages_for_board "${sed_opts[@]}" packages_for_board "${arch}" "${sed_opts[@]}"
} }
# Checks if no errors were produced by emerge when generating # Checks if no errors were produced by emerge when generating
# reports. It is assumed that emerge will print a line with "ERROR" in # reports. It is assumed that emerge will print a line with "ERROR" in
# it to denote a failure. # it to denote a failure.
function ensure_no_errors() { function ensure_no_errors() {
local kind local -a files=( "${SDK_EO_W}" )
local arch name
for kind in sdk board; do for arch; do
if cat_eo_w "${kind}" | grep --quiet --fixed-strings 'ERROR'; then name=${arch^^}_BOARD_EO_W
files+=( "${!name}" )
done
local file
for file in "${files[@]}"; do
if cat "${file}" | grep --quiet --fixed-strings 'ERROR'; then
fail "there are errors in emerge output warnings files" fail "there are errors in emerge output warnings files"
fi fi
done done
@ -460,10 +406,17 @@ function revert_crossdev_stuff() {
# Checks if the expected reports were generated by emerge. # Checks if the expected reports were generated by emerge.
function ensure_valid_reports() { function ensure_valid_reports() {
local kind var_name local -a files=( "${SDK_EO_F}" )
for kind in sdk board; do local arch name
var_name="${kind^^}_EO_F"
if [[ ! -s ${!var_name} ]]; then for arch; do
name=${arch^^}_BOARD_EO_F
files+=( "${!name}" )
done
local file
for file in "${files[@]}"; do
if [[ ! -s ${file} ]]; then
fail "report files are missing or are empty" fail "report files are missing or are empty"
fi fi
done done
@ -486,10 +439,28 @@ function clean_empty_warning_files() {
done done
} }
function get_num_proc() {
local -n num_proc_ref=${1}; shift
local -i num_proc=1 rv=1
# stolen from portage
[[ rv -eq 0 ]] || { rv=0; num_proc=$(getconf _NPROCESSORS_ONLN 2>/dev/null) || rv=1; }
[[ rv -eq 0 ]] || { rv=0; num_proc=$(sysctl -n hw.ncpu 2>/dev/null) || rv=1; }
# stolen from common.sh
[[ rv -eq 0 ]] || { rv=0; num_proc=$(grep -c "^processor" /proc/cpuinfo 2>/dev/null) || rv=1; }
[[ rv -eq 0 ]] || { rv=0; num_proc=1; }
num_proc_ref=${num_proc}
}
function generate_cache_for() { function generate_cache_for() {
local repo=${1}; shift local repo=${1}; shift
egencache --repo "${repo}" --update local -i gcf_num_proc
local load_avg
get_num_proc gcf_num_proc
load_avg=$(bc <<< "${gcf_num_proc} * 0.75")
egencache --repo "${repo}" --jobs="${gcf_num_proc}" --load-average="${load_avg}" --update
} }
function copy_cache_to_reports() { function copy_cache_to_reports() {

232
pkg_auto/impl/pkg_auto_lib.sh
View File

@ -1114,78 +1114,96 @@ function generate_sdk_reports() {
add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports"
mkdir "${WORKDIR}/pkg-reports" mkdir "${WORKDIR}/pkg-reports"
local arch sdk_image_var_name sdk_image_name if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${SDK_IMAGE}"; then
fail "No SDK image named ${SDK_IMAGE@Q} available locally, pull it before running this script"
fi
local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch local sdk_run_kind state_var_name sdk_run_state state_branch_var_name sdk_run_state_branch
local file full_file rv sdk_reports_dir salvaged_dir pkg_auto_copy local pkg_auto_copy rv
local -a report_files run_sdk_container_args local sdk_reports_dir top_dir dir entry full_path
for arch in "${ARCHES[@]}"; do local -a dir_queue all_dirs all_files
sdk_image_var_name="SDK_IMAGE"
sdk_image_name=${!sdk_image_var_name} # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file
if ! docker images --format '{{.Repository}}:{{.Tag}}' | grep --quiet --line-regexp --fixed-strings "${sdk_image_name}"; then for sdk_run_kind in "${WHICH[@]}"; do
fail "No SDK image named ${sdk_image_name@Q} available locally, pull it before running this script" state_var_name="${sdk_run_kind^^}_STATE"
sdk_run_state="${!state_var_name}_sdk_run"
state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH"
sdk_run_state_branch="${!state_branch_var_name}-sdk-run"
add_cleanup \
"git -C ${sdk_run_state@Q} reset --hard HEAD" \
"git -C ${sdk_run_state@Q} clean -ffdx" \
"git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \
"git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}"
git -C "${SCRIPTS}" \
worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}"
pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX")
add_cleanup "rm -rf ${pkg_auto_copy@Q}"
cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}"
local -a run_sdk_container_args=(
-C "${SDK_IMAGE}"
-n "pkg-${sdk_run_kind}"
-U
-m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto"
--rm
./pkg_auto/inside_sdk_container.sh pkg-reports "${ARCHES[@]}"
)
rv=0
env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?}
unset run_sdk_container_args
if [[ ${rv} -ne 0 ]]; then
local salvaged_dir
salvaged_dir="${REPORTS_DIR}/salvaged-reports"
{
info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue"
info
for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do
info "from ${file}:"
echo
cat "${file}"
echo
done
info
info 'whatever reports generated by the failed run are saved in'
info "${salvaged_dir@Q} directory"
info
} >&2
rm -rf "${salvaged_dir}"
cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}"
unset salvaged_dir
fail "copying done, stopping now"
fi fi
sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}"
# shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file top_dir="${sdk_run_state}/pkg-reports"
for sdk_run_kind in "${WHICH[@]}"; do dir_queue=( "${top_dir}" )
state_var_name="${sdk_run_kind^^}_STATE" all_dirs=()
sdk_run_state="${!state_var_name}_sdk_run_${arch}" all_files=()
state_branch_var_name="${sdk_run_kind^^}_STATE_BRANCH" while [[ ${#dir_queue[@]} -gt 0 ]]; do
sdk_run_state_branch="${!state_branch_var_name}-sdk-run-${arch}" dir=${dir_queue[0]}
dir_queue=( "${dir_queue[@]:1}" )
add_cleanup \ entry=${dir#"${top_dir}"}
"git -C ${sdk_run_state@Q} reset --hard HEAD" \ if [[ -z ${entry} ]]; then
"git -C ${sdk_run_state@Q} clean -ffdx" \ all_dirs=( "${sdk_reports_dir}" "${all_dirs[@]}" )
"git -C ${SCRIPTS@Q} worktree remove ${sdk_run_state@Q}" \ else
"git -C ${SCRIPTS@Q} branch -D ${sdk_run_state_branch@Q}" entry=${entry#/}
git -C "${SCRIPTS}" \ all_dirs=( "${sdk_reports_dir}/${entry}" "${all_dirs[@]}" )
worktree add -b "${sdk_run_state_branch}" "${sdk_run_state}" "${!state_branch_var_name}"
pkg_auto_copy=$(mktemp --tmpdir="${WORKDIR}" --directory "pkg-auto-copy.XXXXXXXX")
add_cleanup "rm -rf ${pkg_auto_copy@Q}"
cp -a "${PKG_AUTO_DIR}"/* "${pkg_auto_copy}"
local -a run_sdk_container_args=(
-C "${sdk_image_name}"
-n "pkg-${sdk_run_kind}-${arch}"
-a "${arch}"
-U
-m "${pkg_auto_copy}:/mnt/host/source/src/scripts/pkg_auto"
--rm
./pkg_auto/inside_sdk_container.sh "${arch}" pkg-reports
)
rv=0
env --chdir "${sdk_run_state}" ./run_sdk_container "${run_sdk_container_args[@]}" || rv=${?}
if [[ ${rv} -ne 0 ]]; then
{
salvaged_dir="${REPORTS_DIR}/salvaged-reports"
info "run_sdk_container finished with exit status ${rv}, printing the warnings below for a clue"
info
for file in "${sdk_run_state}/pkg-reports/"*'-warnings'; do
info "from ${file}:"
echo
cat "${file}"
echo
done
info
info 'whatever reports generated by the failed run are saved in'
info "${salvaged_dir@Q} directory"
info
} >&2
rm -rf "${salvaged_dir}"
cp -a "${sdk_run_state}/pkg-reports" "${salvaged_dir}"
fail "copying done, stopping now"
fi fi
sdk_reports_dir="${WORKDIR}/pkg-reports/${sdk_run_kind}-${arch}" for full_path in "${dir}/"*; do
report_files=() if [[ -d ${full_path} ]]; then
for full_file in "${sdk_run_state}/pkg-reports/"*; do dir_queue+=( "${full_path}" )
file=${full_file##"${sdk_run_state}/pkg-reports/"} else
report_files+=( "${sdk_reports_dir}/${file}" ) entry=${full_path##"${top_dir}/"}
all_files+=( "${sdk_reports_dir}/${entry}" )
fi
done done
add_cleanup \
"rm -f ${report_files[*]@Q}" \
"rmdir ${sdk_reports_dir@Q}"
mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}"
done done
add_cleanup \
"rm -f ${all_files[*]@Q}" \
"rmdir ${all_dirs[*]@Q}"
mv "${sdk_run_state}/pkg-reports" "${sdk_reports_dir}"
done done
cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk" cp -a "${WORKDIR}/pkg-reports" "${REPORTS_DIR}/reports-from-sdk"
} }
@ -1319,6 +1337,24 @@ function pkginfo_c_process_file() {
mvm_c_get_extra 'arch' arch mvm_c_get_extra 'arch' arch
mvm_c_get_extra 'report' report mvm_c_get_extra 'report' report
local report_file
case ${report}:${arch} in
"${SDK_PKGS}:arm64")
# short-circuit it, there's no arm64 sdk
return 0
;;
"${SDK_PKGS}:amd64")
report_file="${WORKDIR}/pkg-reports/${which}/${report}"
;;
"${BOARD_PKGS}:"*)
report_file="${WORKDIR}/pkg-reports/${which}/${arch}-${report}"
;;
*)
local c=${report}:${arch}
devel_warn "unknown report-architecture combination (${c@Q})"
return 0
esac
local pkg version_slot throw_away v s local pkg version_slot throw_away v s
# shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else # shellcheck disable=SC2034 # throw_away is unused, it's here for read to store the rest of the line if there is something else
while read -r pkg version_slot throw_away; do while read -r pkg version_slot throw_away; do
@ -1331,7 +1367,7 @@ function pkginfo_c_process_file() {
pkg_set_ref["${pkg}"]='x' pkg_set_ref["${pkg}"]='x'
mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}" mvm_add "${pkg_slots_set_mvm_var_name}" "${pkg}" "${s}"
pkg_debug_disable pkg_debug_disable
done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}" done <"${report_file}"
} }
# Gets a profile of the pkginfo mvm. The "profile" is a confusing # Gets a profile of the pkginfo mvm. The "profile" is a confusing
@ -1710,28 +1746,36 @@ function consistency_checks() {
function read_package_sources() { function read_package_sources() {
local -n package_sources_map_ref=${1}; shift local -n package_sources_map_ref=${1}; shift
local arch which report pkg repo saved_repo # shellcheck disable=SC1091 # generated file
for arch in "${ARCHES[@]}"; do source "${WORKDIR}/globals"
for which in "${WHICH[@]}"; do
for report in sdk-package-repos board-package-repos; do local -a files=()
while read -r pkg repo; do local which arch
saved_repo=${package_sources_map_ref["${pkg}"]:-} for which in "${WHICH[@]}"; do
if [[ -n ${saved_repo} ]]; then files+=( "${WORKDIR}/pkg-reports/${which}/sdk-package-repos" )
if [[ ${saved_repo} != "${repo}" ]]; then for arch in "${ARCHES[@]}"; do
pkg_warn \ files+=( "${WORKDIR}/pkg-reports/${which}/${arch}-board-package-repos" )
'- different repos used for the package:' \
" - package: ${pkg}" \
' - repos:' \
" - ${saved_repo}" \
" - ${repo}"
fi
else
package_sources_map_ref["${pkg}"]=${repo}
fi
done <"${WORKDIR}/pkg-reports/${which}-${arch}/${report}"
done
done done
done done
local file pkg repo saved_repo
for file in "${files[@]}"; do
while read -r pkg repo; do
saved_repo=${package_sources_map_ref["${pkg}"]:-}
if [[ -n ${saved_repo} ]]; then
if [[ ${saved_repo} != "${repo}" ]]; then
pkg_warn \
'- different repos used for the package:' \
" - package: ${pkg}" \
' - repos:' \
" - ${saved_repo}" \
" - ${repo}"
fi
else
package_sources_map_ref["${pkg}"]=${repo}
fi
done <"${file}"
done
} }
# This monstrosity takes renames map and package tags information, # This monstrosity takes renames map and package tags information,
@ -2950,14 +2994,12 @@ function handle_profiles() {
# shellcheck disable=SC1091 # generated file # shellcheck disable=SC1091 # generated file
source "${WORKDIR}/globals" source "${WORKDIR}/globals"
local -a files local -a files=()
files=() local which arch
local arch which report for which in "${WHICH[@]}"; do
for arch in "${ARCHES[@]}"; do files+=("${WORKDIR}/pkg-reports/${which}/sdk-profiles")
for which in "${WHICH[@]}"; do for arch in "${ARCHES[@]}"; do
for report in sdk-profiles board-profiles; do files+=("${WORKDIR}/pkg-reports/${which}/${arch}-board-profiles")
files+=("${WORKDIR}/pkg-reports/${which}-${arch}/${report}")
done
done done
done done
local -A profile_dirs_set local -A profile_dirs_set

77
pkg_auto/inside_sdk_container.sh
View File

@ -1,33 +1,34 @@
#!/bin/bash #!/bin/bash
## ##
## Gathers information about SDK and board packages. Also collects ## Gathers information about SDK packages and board packages for each
## info about actual build deps of board packages, which may be useful ## passed architecture. Also collects info about actual build deps of
## for verifying if SDK provides those. ## board packages, which may be useful for verifying if SDK provides
## those.
## ##
## Reports generated: ## Reports generated:
## sdk-pkgs - contains package information for SDK ## sdk-pkgs - contains package information for SDK
## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK ## sdk-pkgs-kv - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) for SDK
## board-pkgs - contains package information for board for chosen architecture ## ${arch}-board-pkgs - contains package information for board for chosen architecture
## board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies ## ${arch}-board-bdeps - contains package information with key values (USE, PYTHON_TARGETS, CPU_FLAGS_X86) of board build dependencies
## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order ## sdk-profiles - contains a list of profiles used by the SDK, in evaluation order
## board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order ## ${arch}-board-profiles - contains a list of profiles used by the board for the chosen architecture, in evaluation order
## sdk-package-repos - contains package information with their repos for SDK ## sdk-package-repos - contains package information with their repos for SDK
## board-package-repos - contains package information with their repos for board ## ${arch}-board-package-repos - contains package information with their repos for board
## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports ## sdk-emerge-output - contains raw emerge output for SDK being a base for other reports
## board-emerge-output - contains raw emerge output for board being a base for other reports ## ${arch}-board-emerge-output - contains raw emerge output for board being a base for other reports
## sdk-emerge-output-filtered - contains only lines with package information for SDK ## sdk-emerge-output-filtered - contains only lines with package information for SDK
## board-emerge-output-filtered - contains only lines with package information for board ## ${arch}-board-emerge-output-filtered - contains only lines with package information for board
## sdk-emerge-output-junk - contains only junk lines for SDK ## sdk-emerge-output-junk - contains only junk lines for SDK
## board-emerge-output-junk - contains only junk lines for board ## ${arch}-board-emerge-output-junk - contains only junk lines for board
## *-warnings - warnings printed by emerge or other tools ## *-warnings - warnings printed by emerge or other tools
## ##
## Parameters: ## Parameters:
## -h: this help ## -h: this help
## ##
## Positional: ## Positional:
## 1 - architecture (amd64 or arm64) ## 1 - reports directory
## 2 - reports directory ## # - architectures (currently only amd64 or arm64 are valid)
## ##
set -euo pipefail set -euo pipefail
@ -54,16 +55,16 @@ while [[ ${#} -gt 0 ]]; do
esac esac
done done
if [[ ${#} -ne 2 ]]; then if [[ ${#} -lt 2 ]]; then
fail 'Expected two parameters: board architecture and reports directory' fail 'Expected at least two parameters: reports directory and one or more board architectures'
fi fi
arch=${1}; shift
reports_dir=${1}; shift reports_dir=${1}; shift
# rest are architectures
mkdir -p "${reports_dir}" mkdir -p "${reports_dir}"
set_eo "${reports_dir}" set_eo "${reports_dir}" "${@}"
echo 'Running egencache for portage-stable' echo 'Running egencache for portage-stable'
generate_cache_for 'portage-stable' 2>"${EGENCACHE_W}" generate_cache_for 'portage-stable' 2>"${EGENCACHE_W}"
@ -77,36 +78,48 @@ copy_cache_to_reports 'coreos-overlay' "${reports_dir}" 2>>"${EGENCACHE_W}"
echo 'Running pretend-emerge to get complete report for SDK' echo 'Running pretend-emerge to get complete report for SDK'
package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}" package_info_for_sdk >"${SDK_EO}" 2>"${SDK_EO_W}"
echo 'Running pretend-emerge to get complete report for board' for arch; do
package_info_for_board "${arch}" >"${BOARD_EO}" 2>"${BOARD_EO_W}" be=${arch^^}_BOARD_EO
bew=${arch^^}_BOARD_EO_W
echo "Running pretend-emerge to get complete report for ${arch} board"
package_info_for_board "${arch}" >"${!be}" 2>"${!bew}"
done
ensure_no_errors ensure_no_errors "${@}"
echo 'Separating emerge info from junk in SDK emerge output' echo 'Separating emerge info from junk in SDK emerge output'
filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}" filter_sdk_eo >"${SDK_EO_F}" 2>>"${SDK_EO_W}"
junk_sdk_eo >"${SDK_EO}-junk" 2>>"${SDK_EO_W}" junk_sdk_eo >"${SDK_EO_J}" 2>>"${SDK_EO_W}"
echo 'Separating emerge info from junk in board emerge output' for arch; do
filter_board_eo "${arch}" >"${BOARD_EO_F}" 2>>"${BOARD_EO_W}" bej=${arch^^}_BOARD_EO_J
junk_board_eo >"${BOARD_EO}-junk" 2>>"${BOARD_EO_W}" bef=${arch^^}_BOARD_EO_F
bew=${arch^^}_BOARD_EO_W
echo "Separating emerge info from junk in ${arch} board emerge output"
filter_board_eo "${arch}" >"${!bef}" 2>>"${!bew}"
junk_board_eo "${arch}" >"${!bej}" 2>>"${!bew}"
done
ensure_valid_reports ensure_valid_reports "${@}"
echo 'Generating SDK packages listing' echo 'Generating SDK packages listing'
versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings" versions_sdk >"${reports_dir}/sdk-pkgs" 2>"${reports_dir}/sdk-pkgs-warnings"
echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)' echo 'Generating SDK packages listing with key-values (USE, PYTHON_TARGETS CPU_FLAGS_X86, etc)'
versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings" versions_sdk_with_key_values >"${reports_dir}/sdk-pkgs-kv" 2>"${reports_dir}/sdk-pkgs-kv-warnings"
echo 'Generating board packages listing'
versions_board >"${reports_dir}/board-pkgs" 2>"${reports_dir}/board-pkgs-warnings"
echo 'Generating board packages bdeps listing'
board_bdeps >"${reports_dir}/board-bdeps" 2>"${reports_dir}/board-bdeps-warnings"
echo 'Generating SDK profiles evaluation list' echo 'Generating SDK profiles evaluation list'
ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings" ROOT=/ "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/sdk-profiles" 2>"${reports_dir}/sdk-profiles-warnings"
echo 'Generating board profiles evaluation list'
ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/board-profiles" 2>"${reports_dir}/board-profiles-warnings"
echo 'Generating SDK package source information' echo 'Generating SDK package source information'
package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings" package_sources_sdk >"${reports_dir}/sdk-package-repos" 2>"${reports_dir}/sdk-package-repos-warnings"
echo 'Generating board package source information'
package_sources_board >"${reports_dir}/board-package-repos" 2>"${reports_dir}/board-package-repos-warnings" for arch; do
echo "Generating ${arch} board packages listing"
versions_board "${arch}" >"${reports_dir}/${arch}-board-pkgs" 2>"${reports_dir}/${arch}-board-pkgs-warnings"
echo "Generating ${arch} board packages bdeps listing"
board_bdeps "${arch}" >"${reports_dir}/${arch}-board-bdeps" 2>"${reports_dir}/${arch}-board-bdeps-warnings"
echo "Generating ${arch} board profiles evaluation list"
ROOT="/build/${arch}-usr" "${PKG_AUTO_IMPL_DIR}/print_profile_tree.sh" -ni -nh >"${reports_dir}/${arch}-board-profiles" 2>"${reports_dir}/${arch}-board-profiles-warnings"
echo "Generating ${arch} board package source information"
package_sources_board "${arch}" >"${reports_dir}/${arch}-board-package-repos" 2>"${reports_dir}/${arch}-board-package-repos-warnings"
done
echo "Cleaning empty warning files" echo "Cleaning empty warning files"
clean_empty_warning_files "${reports_dir}" clean_empty_warning_files "${reports_dir}"