mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-24 23:21:17 +02:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #634 from flatcar/krnowak/report-generation

Browse Source 
Update report generation
This commit is contained in:
Krzesimir Nowak 2023-02-02 09:44:43 +01:00 committed by GitHub
commit e979082b17
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 207 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

91
build_library/build_image_util.sh
View File

@ -244,12 +244,60 @@ systemd_enable() {
write_contents() {
info "Writing ${2##*/}"
pushd "$1" >/dev/null
# %M - file permissions
# %n - number of hard links to file
# %u - file's user name
# %g - file's group name
# %s - size in bytes
# %Tx - modification time (Y - year, m - month, d - day, H - hours, M - minutes)
# %P - file's path
# %l - symlink target (empty if not a symlink)
sudo TZ=UTC find -printf \
'%M %2n %-7u %-7g %7s %TY-%Tm-%Td %TH:%TM ./%P -> %l\n' \
| sed -e 's/ -> $//' > "$2"
popd >/dev/null
}
# Generate a listing that can be used by other tools to analyze
# image/file size changes.
write_contents_with_technical_details() {
info "Writing ${2##*/}"
pushd "$1" >/dev/null
# %M - file permissions
# %D - ID of a device where file resides
# %i - inode number
# %n - number of hard links to file
# %s - size in bytes
# %P - file's path
sudo find -printf \
'%M %D %i %n %s ./%P\n' > "$2"
popd >/dev/null
}
# Generate a report like the following:
#
# File Size Used Avail Use% Type
# /boot 127M 62M 65M 50% vfat
# /usr 983M 721M 212M 78% ext2
# / 6,0G 13M 5,6G 1% ext4
# SUM 7,0G 796M 5,9G 12% -
write_disk_space_usage() {
info "Writing ${2##*/}"
pushd "${1}" >/dev/null
# The sed's first command turns './<path>' into '/<path> ', second
# command replaces '- ' with 'SUM' for the total row. All this to
# keep the numbers neatly aligned in columns.
sudo df \
--human-readable \
--total \
--output='file,size,used,avail,pcent,fstype' \
./boot ./usr ./ | \
sed \
-e 's#^\.\(/[^ ]*\)#\1 #' \
-e 's/^- /SUM/' >"${2}"
popd >/dev/null
}
# "equery list" a potentially uninstalled board package
query_available_package() {
local pkg="$1"
@ -624,11 +672,15 @@ finish_image() {
local disk_layout="$2"
local root_fs_dir="$3"
local image_contents="$4"
local image_kernel="$5"
local pcr_policy="$6"
local image_grub="$7"
local image_shim="$8"
local image_kconfig="$9"
local image_contents_wtd="$5"
local image_kernel="$6"
local pcr_policy="$7"
local image_grub="$8"
local image_shim="$9"
local image_kconfig="${10}"
local image_initrd_contents="${11}"
local image_initrd_contents_wtd="${12}"
local image_disk_space_usage="${13}"
local install_grub=0
local disk_img="${BUILD_DIR}/${image_name}"
@ -730,8 +782,6 @@ EOF
"${BUILD_DIR}/${image_kconfig}"
fi
write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}"
# Zero all fs free space to make it more compressible so auto-update
# payloads become smaller, not fatal since it won't work on linux < 3.2
sudo fstrim "${root_fs_dir}" || true
@ -828,4 +878,31 @@ EOF
popd >/dev/null
rm -rf "${BUILD_DIR}/pcrs"
fi
# Mount the final image again, as readonly, to generate some reports.
"${BUILD_LIBRARY_DIR}/disk_util" --disk_layout="${disk_layout}" \
mount --read_only "${disk_img}" "${root_fs_dir}"
trap "cleanup_mounts '${root_fs_dir}'" EXIT
write_contents "${root_fs_dir}" "${BUILD_DIR}/${image_contents}"
write_contents_with_technical_details "${root_fs_dir}" "${BUILD_DIR}/${image_contents_wtd}"
if [[ -n "${image_initrd_contents}" ]] || [[ -n "${image_initrd_contents_wtd}" ]]; then
"${BUILD_LIBRARY_DIR}/extract-initramfs-from-vmlinuz.sh" "${root_fs_dir}/boot/flatcar/vmlinuz-a" "${BUILD_DIR}/tmp_initrd_contents"
if [[ -n "${image_initrd_contents}" ]]; then
write_contents "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents}"
fi
if [[ -n "${image_initrd_contents_wtd}" ]]; then
write_contents_with_technical_details "${BUILD_DIR}/tmp_initrd_contents" "${BUILD_DIR}/${image_initrd_contents_wtd}"
fi
rm -rf "${BUILD_DIR}/tmp_initrd_contents"
fi
if [[ -n "${image_disk_space_usage}" ]]; then
write_disk_space_usage "${root_fs_dir}" "${BUILD_DIR}/${image_disk_space_usage}"
fi
cleanup_mounts "${root_fs_dir}"
trap - EXIT
}

4
build_library/dev_container_util.sh
View File

@ -77,6 +77,7 @@ create_dev_container() {
info "Building developer image ${image_name}"
local root_fs_dir="${BUILD_DIR}/rootfs"
local image_contents="${image_name%.bin}_contents.txt"
local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
local image_packages="${image_name%.bin}_packages.txt"
local image_licenses="${image_name%.bin}_licenses.json"
@ -104,7 +105,7 @@ create_dev_container() {
# The remount services are provided by coreos-base/coreos-init
systemd_enable "${root_fs_dir}" "multi-user.target" "remount-usr.service"
finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}"
finish_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${image_contents}" "${image_contents_wtd}"
declare -a files_to_evaluate
declare -a compressed_images
@ -115,6 +116,7 @@ create_dev_container() {
upload_image -d "${BUILD_DIR}/${image_name}.DIGESTS" \
"${BUILD_DIR}/${image_contents}" \
"${BUILD_DIR}/${image_contents_wtd}" \
"${BUILD_DIR}/${image_packages}" \
"${BUILD_DIR}/${image_licenses}" \
"${compressed_images[@]}" \

107
build_library/extract-initramfs-from-vmlinuz.sh Executable file
View File

@ -0,0 +1,107 @@
#!/bin/bash
# Does as it says on the tin.
#
# Example: extract-initramfs-from-vmlinuz /boot/flatcar/vmlinuz-a out-dir
#
# This will create one or more out-dir/rootfs-N directories that contain the contents of the initramfs.
set -euo pipefail
# check for unzstd. Will abort the script with an error message if the tool is not present.
unzstd -V >/dev/null
fail() {
echo "${*}" >&2
exit 1
}
# Stolen from extract-vmlinux and modified.
try_decompress() {
local header="${1}"
local no_idea="${2}"
local tool="${3}"
local image="${4}"
local tmp="${5}"
local output_basename="${6}"
local pos
local tool_filename=$(echo "${tool}" | cut -f1 -d' ')
# The obscure use of the "tr" filter is to work around older versions of
# "grep" that report the byte offset of the line instead of the pattern.
# Try to find the header and decompress from here.
for pos in $(tr "${header}\n${no_idea}" "\n${no_idea}=" < "${image}" |
grep --text --byte-offset --only-matching "^${no_idea}")
do
pos=${pos%%:*}
# Disable error handling, because we will be potentially
# giving the tool garbage or a valid archive with some garbage
# appended to it. So let the tool extract the valid archive
# and then complain about the garbage at the end, but don't
# fail the script because of it.
set +e; tail "-c+${pos}" "${image}" | "${tool}" >"${tmp}/out" 2>/dev/null; set -e;
if [ -s "${tmp}/out" ]; then
mv "${tmp}/out" "${output_basename}-${tool_filename}-at-${pos}"
else
rm -f "${tmp}/out"
fi
done
}
try_unzstd_decompress() {
local image="${1}"
local tmp="${2}"
local output_basename="${3}"
try_decompress '(\265/\375' xxx unzstd "${image}" "${tmp}" "${output_basename}"
}
me="${0##*/}"
if [[ $# -ne 2 ]]; then
fail "Usage: ${me} <vmlinuz> <output_directory>"
fi
image="${1}"
out="${2}"
if [[ ! -s "${image}" ]]; then
fail "The image file '${image}' either does not exist or is empty"
fi
mkdir -p "${out}"
tmp=$(mktemp --directory /tmp/eifv-XXXXXX)
trap "rm -rf ${tmp}" EXIT
tmp_dec="${tmp}/decompress"
mkdir "${tmp_dec}"
fr_prefix="${tmp}/first-round"
ROOTFS_IDX=0
perform_round() {
local image="${1}"
local tmp_dec="${2}"
local round_prefix="${3}"
try_unzstd_decompress "${image}" "${tmp_dec}" "${round_prefix}"
for rnd in "${round_prefix}"*; do
if [[ $(file --brief "${rnd}") =~ 'cpio archive' ]]; then
mkdir -p "${out}/rootfs-${ROOTFS_IDX}"
while cpio --quiet --extract --make-directories --directory="${out}/rootfs-${ROOTFS_IDX}" --nonmatching 'dev/*'; do
ROOTFS_IDX=$(( ROOTFS_IDX + 1 ))
mkdir -p "${out}/rootfs-${ROOTFS_IDX}"
done <${rnd}
rmdir "${out}/rootfs-${ROOTFS_IDX}"
fi
done
}
shopt -s nullglob
perform_round "${image}" "${tmp_dec}" "${fr_prefix}"
for fr in "${fr_prefix}"*; do
fr_files="${fr}-files"
fr_dec="${fr_files}/decompress"
mkdir -p "${fr_dec}"
sr_prefix="${fr_files}/second-round"
perform_round "${fr}" "${fr_dec}" "${sr_prefix}"
done
if [[ ${ROOTFS_IDX} -eq 0 ]]; then
fail "no initramfs found in ${image}"
fi
echo "done, found ${ROOTFS_IDX} rootfs(es)"

14
build_library/prod_image_util.sh
View File

@ -65,6 +65,7 @@ create_prod_image() {
info "Building production image ${image_name}"
local root_fs_dir="${BUILD_DIR}/rootfs"
local image_contents="${image_name%.bin}_contents.txt"
local image_contents_wtd="${image_name%.bin}_contents_wtd.txt"
local image_packages="${image_name%.bin}_packages.txt"
local image_sbom="${image_name%.bin}_sbom.json"
local image_licenses="${image_name%.bin}_licenses.json"
@ -73,6 +74,9 @@ create_prod_image() {
local image_pcr_policy="${image_name%.bin}_pcr_policy.zip"
local image_grub="${image_name%.bin}.grub"
local image_shim="${image_name%.bin}.shim"
local image_initrd_contents="${image_name%.bin}_initrd_contents.txt"
local image_initrd_contents_wtd="${image_name%.bin}_initrd_contents_wtd.txt"
local image_disk_usage="${image_name%.bin}_disk_usage.txt"
start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}"
@ -131,15 +135,20 @@ EOF
"${disk_layout}" \
"${root_fs_dir}" \
"${image_contents}" \
"${image_contents_wtd}" \
"${image_kernel}" \
"${image_pcr_policy}" \
"${image_grub}" \
"${image_shim}" \
"${image_kconfig}"
"${image_kconfig}" \
"${image_initrd_contents}" \
"${image_initrd_contents_wtd}" \
"${image_disk_usage}"
# Upload
local to_upload=(
"${BUILD_DIR}/${image_contents}"
"${BUILD_DIR}/${image_contents_wtd}"
"${BUILD_DIR}/${image_packages}"
"${BUILD_DIR}/${image_sbom}"
"${BUILD_DIR}/${image_licenses}"
@ -147,6 +156,9 @@ EOF
"${BUILD_DIR}/${image_pcr_policy}"
"${BUILD_DIR}/${image_grub}"
"${BUILD_DIR}/${image_kconfig}"
"${BUILD_DIR}/${image_initrd_contents}"
"${BUILD_DIR}/${image_initrd_contents_wtd}"
"${BUILD_DIR}/${image_disk_usage}"
)
local files_to_evaluate=( "${BUILD_DIR}/${image_name}" )