mirror of
https://github.com/flatcar/scripts.git
synced 2026-01-16 22:12:37 +01:00
5471c71745
This will come in handy for spawning jobs for handling package updates. Since we don't want to spawn as many jobs as there are packages, then limiting ourselves to the job count matching the processor or core count sounds like a better idea. Signed-off-by: Krzesimir Nowak <knowak@microsoft.com>
457 lines
13 KiB
Bash
457 lines
13 KiB
Bash
#!/bin/bash
|
|
|
|
if [[ -z ${__INSIDE_SDK_CONTAINER_LIB_SH_INCLUDED__:-} ]]; then
|
|
__INSIDE_SDK_CONTAINER_LIB_SH_INCLUDED__=x
|
|
|
|
source "$(dirname "${BASH_SOURCE[0]}")/util.sh"
|
|
|
|
# Invokes emerge to get a report about built packages for a given
|
|
# metapackage in the given root that has a portage configuration.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - root filesystem with the portage config
|
|
# @ - packages and metapackages to get the deps from
|
|
function emerge_pretend() {
|
|
local root
|
|
root=${1}; shift
|
|
|
|
# Probably a bunch of those flags are not necessary, but I'm not
|
|
# touching it - they seem to be working. :)
|
|
local -a emerge_opts=(
|
|
--config-root="${root}"
|
|
--root="${root}"
|
|
--sysroot="${root}"
|
|
--pretend
|
|
--columns
|
|
--nospinner
|
|
--oneshot
|
|
--color n
|
|
--emptytree
|
|
--verbose
|
|
--verbose-conflicts
|
|
--verbose-slot-rebuilds y
|
|
--changed-deps y
|
|
--changed-deps-report y
|
|
--changed-slot y
|
|
--changed-use
|
|
--newuse
|
|
--complete-graph y
|
|
--deep
|
|
--rebuild-if-new-slot y
|
|
--rebuild-if-unbuilt y
|
|
--with-bdeps y
|
|
--dynamic-deps y
|
|
--update
|
|
--ignore-built-slot-operator-deps y
|
|
--selective n
|
|
--keep-going y
|
|
)
|
|
local rv
|
|
rv=0
|
|
emerge "${emerge_opts[@]}" "${@}" || rv=${?}
|
|
if [[ ${rv} -ne 0 ]]; then
|
|
echo "WARNING: emerge exited with status ${rv}" >&2
|
|
fi
|
|
}
|
|
|
|
# Gets package list for SDK.
|
|
function package_info_for_sdk() {
|
|
local root
|
|
root='/'
|
|
|
|
ignore_crossdev_stuff "${root}"
|
|
# stage4 build of SDK builds coreos-devel/sdk-depends, fsscript
|
|
# pulls in cross toolchains with crossdev (which we have just
|
|
# ignored) and dev-lang/rust
|
|
emerge_pretend "${root}" coreos-devel/sdk-depends dev-lang/rust
|
|
revert_crossdev_stuff "${root}"
|
|
}
|
|
|
|
# Gets package list for board of a given architecture.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - architecture
|
|
function package_info_for_board() {
|
|
local arch
|
|
arch=${1}; shift
|
|
|
|
local root
|
|
root="/build/${arch}-usr"
|
|
|
|
# Ignore crossdev stuff in both SDK root and board root - emerge
|
|
# may query SDK stuff for the board packages.
|
|
ignore_crossdev_stuff /
|
|
ignore_crossdev_stuff "${root}"
|
|
emerge_pretend "${root}" coreos-devel/board-packages
|
|
revert_crossdev_stuff "${root}"
|
|
revert_crossdev_stuff /
|
|
}
|
|
|
|
# Set the directory where the emerge output and the results of
|
|
# processing it will be stored. EO stands for "emerge output"
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - directory path
|
|
function set_eo() {
|
|
local dir=${1}; shift
|
|
# rest are architectures
|
|
|
|
declare -g EGENCACHE_W="${dir}/egencache-warnings"
|
|
declare -g SDK_EO="${dir}/sdk-emerge-output"
|
|
declare -g SDK_EO_F="${SDK_EO}-filtered"
|
|
declare -g SDK_EO_W="${SDK_EO}-warnings"
|
|
declare -g SDK_EO_J="${SDK_EO}-junk"
|
|
|
|
local arch
|
|
local board_eo
|
|
for arch; do
|
|
board_eo=${dir}/${arch}-board-emerge-output
|
|
declare -g "${arch^^}_BOARD_EO=${board_eo}"
|
|
declare -g "${arch^^}_BOARD_EO_F=${board_eo}-filtered"
|
|
declare -g "${arch^^}_BOARD_EO_W=${board_eo}-warnings"
|
|
declare -g "${arch^^}_BOARD_EO_J=${board_eo}-junk"
|
|
done
|
|
}
|
|
|
|
# JSON output would be more verbose, but probably would not require
|
|
# these abominations below. But, alas, emerge doesn't have that yet.
|
|
|
|
# status package name version slot repo target (opt) keyvals size
|
|
# |--------------| |----------| |#-g1-----------#--#-g2-#| |--|-g------| |-g----------#-#-g-----| |---|
|
|
# [ebuild R ~] virtual/rust [1.71.1:0/llvm-16::coreos] to /some/root USE="-rustfmt" FOO="bar" 0 KiB
|
|
#
|
|
# Actually, there can also be another "version slot repo" part between
|
|
# the first "version slot repo" and "target" part.
|
|
STATUS_RE='\[[^]]*]' # 0 groups
|
|
PACKAGE_NAME_RE='[^[:space:]]*' # 0 groups
|
|
VER_SLOT_REPO_RE='\[\([^]]\+\)::\([^]]\+\)]' # 2 groups
|
|
TARGET_RE='to[[:space:]]\+\([^[:space:]]\)\+' # 1 group
|
|
KEYVALS_RE='\([[:space:]]*[A-Za-z0-9_]*="[^"]*"\)*' # 1 group (but containing only the last pair!)
|
|
SIZE_RE='[[:digit:]]\+[[:space:]]*[[:alpha:]]*B' # 0 groups
|
|
SPACES_RE='[[:space:]]\+' # 0 groups
|
|
NONSPACES_RE='[^[:space:]]\+' # 0 groups
|
|
NONSPACES_WITH_COLON_RE='[^[:space:]]*:' # 0 groups
|
|
|
|
FULL_LINE_RE='^'"${STATUS_RE}${SPACES_RE}${PACKAGE_NAME_RE}"'\('"${SPACES_RE}${VER_SLOT_REPO_RE}"'\)\{1,2\}\('"${SPACES_RE}${TARGET_RE}"'\)\?\('"${SPACES_RE}${KEYVALS_RE}"'\)*'"${SPACES_RE}${SIZE_RE}"'$'
|
|
|
|
# Filters sdk reports to get the package information.
|
|
function filter_sdk_eo() {
|
|
cat "${SDK_EO}" | xgrep -e "${FULL_LINE_RE}"
|
|
}
|
|
|
|
# Filters board reports for a given arch to get the package
|
|
# information.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - architecture
|
|
function filter_board_eo() {
|
|
local arch name
|
|
arch=${1}; shift
|
|
name=${arch^^}_BOARD_EO
|
|
|
|
# Replace ${arch}-usr in the output with a generic word BOARD.
|
|
cat "${!name}" | \
|
|
xgrep -e "${FULL_LINE_RE}" | \
|
|
sed -e "s#/build/${arch}-usr/#/build/BOARD/#"
|
|
}
|
|
|
|
# Filters sdk reports to get anything but the package information
|
|
# (i.e. junk).
|
|
function junk_sdk_eo() {
|
|
cat "${SDK_EO}" | xgrep -v -e "${FULL_LINE_RE}"
|
|
}
|
|
|
|
# Filters board reports to get anything but the package information
|
|
# (i.e. junk).
|
|
function junk_board_eo() {
|
|
local arch name
|
|
arch=${1}; shift
|
|
name=${arch^^}_BOARD_EO
|
|
|
|
cat "${!name}" | xgrep -v -e "${FULL_LINE_RE}"
|
|
}
|
|
|
|
# More regexp-like abominations follow.
|
|
|
|
# There may also be a line like:
|
|
#
|
|
# [blocks B ] <dev-util/gdbus-codegen-2.76.4 ("<dev-util/gdbus-codegen-2.76.4" is soft blocking dev-libs/glib-2.76.4)
|
|
#
|
|
# But currently we don't care about those - they land in junk.
|
|
|
|
SLOT_INFO_SED_FILTERS=(
|
|
# if there is no slot information in version, add :0
|
|
#
|
|
# assumption here is that version is a second word
|
|
-e "/^${NONSPACES_RE}${SPACES_RE}${NONSPACES_WITH_COLON_RE}/ ! s/^\(${NONSPACES_RE}${SPACES_RE}${NONSPACES_RE}\)/\1:0/"
|
|
)
|
|
|
|
PKG_VER_SLOT_SED_FILTERS=(
|
|
# from line like:
|
|
#
|
|
# [ebuild R ~] virtual/rust [1.71.1:0/llvm-16::coreos] USE="-rustfmt" 0 KiB
|
|
#
|
|
# extract package name, version and optionally a slot if it exists, the result would be:
|
|
#
|
|
# virtual/rust 1.71.1:0/llvm-16
|
|
-e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}.*/\1 \2/"
|
|
"${SLOT_INFO_SED_FILTERS[@]}"
|
|
)
|
|
|
|
PKG_VER_SLOT_KV_SED_FILTERS=(
|
|
# from line like:
|
|
#
|
|
# [ebuild R ~] virtual/rust [1.71.1:0/llvm-16::coreos] USE="-rustfmt" 0 KiB
|
|
#
|
|
# extract package name, version, optionally a slot if it exists and key value pairs if any, the result would be:
|
|
#
|
|
# virtual/rust 1.71.1:0/llvm-16 USE="-rustfmt"
|
|
-e "s/${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}\(${SPACES_RE}${VER_SLOT_REPO_RE}\)\?\(${SPACES_RE}${TARGET_RE}\)\?${SPACES_RE}\(${KEYVALS_RE}\)${SPACES_RE}${SIZE_RE}\$/\1 \2 \9/"
|
|
"${SLOT_INFO_SED_FILTERS[@]}"
|
|
)
|
|
|
|
PKG_REPO_SED_FILTERS=(
|
|
# from line like:
|
|
#
|
|
# [ebuild R ~] virtual/rust [1.71.1:0/llvm-16::coreos] USE="-rustfmt" 0 KiB
|
|
#
|
|
# extract package name and repo, the result would be:
|
|
#
|
|
# virtual/rust coreos
|
|
-e "s/^${STATUS_RE}${SPACES_RE}\(${PACKAGE_NAME_RE}\)${SPACES_RE}${VER_SLOT_REPO_RE}${SPACES_RE}.*/\1 \3/"
|
|
)
|
|
|
|
# Applies some sed filter over the SDK emerge output. Results are
|
|
# printed.
|
|
#
|
|
# Params:
|
|
#
|
|
# @ - parameters passed to sed
|
|
function packages_for_sdk() {
|
|
cat "${SDK_EO_F}" | sed "${@}" | sort
|
|
}
|
|
|
|
# Applies some sed filter over the board emerge output. Results are
|
|
# printed.
|
|
#
|
|
# Params:
|
|
#
|
|
# @ - parameters passed to sed
|
|
function packages_for_board() {
|
|
local arch=${1}; shift
|
|
# rest goes to sed
|
|
|
|
local name=${arch^^}_BOARD_EO_F
|
|
|
|
sed "${@}" "${!name}" | sort
|
|
}
|
|
|
|
# Prints package name, slot and version information for SDK.
|
|
function versions_sdk() {
|
|
local -a sed_opts
|
|
sed_opts=(
|
|
"${PKG_VER_SLOT_SED_FILTERS[@]}"
|
|
)
|
|
packages_for_sdk "${sed_opts[@]}"
|
|
}
|
|
|
|
# Prints package name, slot, version and key-values information for
|
|
# SDK. Key-values may be something like USE="foo bar -baz".
|
|
function versions_sdk_with_key_values() {
|
|
local -a sed_opts
|
|
sed_opts=(
|
|
"${PKG_VER_SLOT_KV_SED_FILTERS[@]}"
|
|
)
|
|
packages_for_sdk "${sed_opts[@]}"
|
|
}
|
|
|
|
# Prints package name, slot and version information for board.
|
|
function versions_board() {
|
|
local arch=${1}; shift
|
|
local -a sed_opts
|
|
sed_opts=(
|
|
-e '/to \/build\/BOARD\// ! d'
|
|
"${PKG_VER_SLOT_SED_FILTERS[@]}"
|
|
)
|
|
packages_for_board "${arch}" "${sed_opts[@]}"
|
|
}
|
|
|
|
# Prints package name, slot, version and key-values information for
|
|
# build dependencies of board. Key-values may be something like
|
|
# USE="foo bar -baz".
|
|
function board_bdeps() {
|
|
local arch=${1}; shift
|
|
local -a sed_opts
|
|
sed_opts=(
|
|
-e '/to \/build\/BOARD\// d'
|
|
"${PKG_VER_SLOT_KV_SED_FILTERS[@]}"
|
|
)
|
|
packages_for_board "${arch}" "${sed_opts[@]}"
|
|
}
|
|
|
|
# Print package name and source repository names information for SDK.
|
|
function package_sources_sdk() {
|
|
local -a sed_opts
|
|
sed_opts=(
|
|
"${PKG_REPO_SED_FILTERS[@]}"
|
|
)
|
|
packages_for_sdk "${sed_opts[@]}"
|
|
}
|
|
|
|
# Print package name and source repository names information for
|
|
# board.
|
|
function package_sources_board() {
|
|
local arch=${1}; shift
|
|
local -a sed_opts
|
|
sed_opts=(
|
|
"${PKG_REPO_SED_FILTERS[@]}"
|
|
)
|
|
packages_for_board "${arch}" "${sed_opts[@]}"
|
|
}
|
|
|
|
# Checks if no errors were produced by emerge when generating
|
|
# reports. It is assumed that emerge will print a line with "ERROR" in
|
|
# it to denote a failure.
|
|
function ensure_no_errors() {
|
|
local -a files=( "${SDK_EO_W}" )
|
|
local arch name
|
|
|
|
for arch; do
|
|
name=${arch^^}_BOARD_EO_W
|
|
files+=( "${!name}" )
|
|
done
|
|
|
|
if grep --quiet --fixed-strings 'ERROR' "${files[@]}"; then
|
|
fail "there are errors in emerge output warnings files"
|
|
fi
|
|
}
|
|
|
|
# Stores a path to a package.provided file inside the given root
|
|
# filesystem portage configuration. Mostly used to ignore
|
|
# cross-toolchains.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - path to root filesystem with the portage configuration
|
|
# 2 - name of a variable where the path will be stored
|
|
function get_provided_file() {
|
|
local root path_var_name
|
|
root=${1}; shift
|
|
path_var_name=${1}; shift
|
|
local -n path_ref="${path_var_name}"
|
|
|
|
path_ref="${root}/etc/portage/profile/package.provided/ignore_cross_packages"
|
|
}
|
|
|
|
# Marks packages coming from crossdev repo as provided at a very high
|
|
# version. We do this, because updating their native counterparts will
|
|
# cause emerge to complain that cross-<triplet>/<package> is masked
|
|
# (like for sys-libs/glibc and cross-x86_64-cros-linux-gnu/glibc),
|
|
# because it has no keywords. In theory, we could try updating
|
|
# <ROOT>/etc/portage/package.mask/cross-<triplet> file created by the
|
|
# crossdev tool to unmask the new version, but it's an unnecessary
|
|
# hassle - native and cross package are supposed to be the same ebuild
|
|
# anyway, so update information about cross package is redundant.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - root directory
|
|
# 2 - ID of the crossdev repository (optional, defaults to x-crossdev)
|
|
function ignore_crossdev_stuff() {
|
|
local root crossdev_repo_id
|
|
root=${1}; shift
|
|
crossdev_repo_id=${1:-x-crossdev}; shift || :
|
|
|
|
local crossdev_repo_path
|
|
crossdev_repo_path=$(portageq get_repo_path "${root}" "${crossdev_repo_id}")
|
|
|
|
local ics_path ics_dir
|
|
get_provided_file "${root}" ics_path
|
|
dirname_out "${ics_path}" ics_dir
|
|
|
|
sudo mkdir -p "${ics_dir}"
|
|
env --chdir="${crossdev_repo_path}" find . -type l | \
|
|
cut -d/ -f2-3 | \
|
|
sed -e 's/$/-9999/' | \
|
|
sudo tee "${ics_path}" >/dev/null
|
|
}
|
|
|
|
# Reverts effects of the ignore_crossdev_stuff function.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - root directory
|
|
function revert_crossdev_stuff() {
|
|
local root
|
|
root=${1}; shift
|
|
|
|
local ics_path ics_dir
|
|
get_provided_file "${root}" ics_path
|
|
dirname_out "${ics_path}" ics_dir
|
|
|
|
sudo rm -f "${ics_path}"
|
|
if dir_is_empty "${ics_dir}"; then
|
|
sudo rmdir "${ics_dir}"
|
|
fi
|
|
}
|
|
|
|
# Checks if the expected reports were generated by emerge.
|
|
function ensure_valid_reports() {
|
|
local -a files=( "${SDK_EO_F}" )
|
|
local arch name
|
|
|
|
for arch; do
|
|
name=${arch^^}_BOARD_EO_F
|
|
files+=( "${!name}" )
|
|
done
|
|
|
|
local file
|
|
for file in "${files[@]}"; do
|
|
if [[ ! -s ${file} ]]; then
|
|
fail "report files are missing or are empty"
|
|
fi
|
|
done
|
|
}
|
|
|
|
# Drops the empty warning files in given directory.
|
|
#
|
|
# Params:
|
|
#
|
|
# 1 - path to the directory
|
|
function clean_empty_warning_files() {
|
|
local dir
|
|
dir=${1}; shift
|
|
|
|
local file
|
|
for file in "${dir}/"*'-warnings'; do
|
|
if [[ ! -s ${file} ]]; then
|
|
rm -f "${file}"
|
|
fi
|
|
done
|
|
}
|
|
|
|
function generate_cache_for() {
|
|
local repo=${1}; shift
|
|
|
|
local -i gcf_num_proc
|
|
local load_avg
|
|
get_num_proc gcf_num_proc
|
|
load_avg=$(bc <<< "${gcf_num_proc} * 0.75")
|
|
egencache --repo "${repo}" --jobs="${gcf_num_proc}" --load-average="${load_avg}" --update
|
|
}
|
|
|
|
function copy_cache_to_reports() {
|
|
local repo=${1}; shift
|
|
local reports_dir=${1}; shift
|
|
|
|
local repo_dir
|
|
repo_dir=$(portageq get_repo_path / "${repo}")
|
|
cp -a "${repo_dir}/metadata/md5-cache" "${reports_dir}/${repo}-cache"
|
|
}
|
|
|
|
fi
|