#!/bin/bash # # Copyright (c) 2021 The Flatcar Maintainers. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. # >>> This file is supposed to be SOURCED from the repository ROOT. <<< # # image_changes() should be called w/ the positional INPUT parameters below. # OS image differences display stub. # This script will display the differences between the last released image and the currently built one. # # PREREQUISITES: # # 1. Artifacts describing the built image (kernel config, contents, packages, etc.) must be present in build cache server. # 2. Scripts repo version tag of OS image version to be built is available and checked out. # # INPUT: # # 1. Architecture (ARCH) of the TARGET OS image ("arm64", "amd64"). # 2. What to compare against, must be either "last-release" or "last-nightly". # # OPTIONAL INPUT: # # (none) # # OUTPUT: # # 1. Currently the script prints the image differences compared to the last release and the changelog for the release notes but doesn't store it yet in the buildcache. # Run a subshell, so the traps, environment changes and global # variables are not spilled into the caller. function image_changes() ( set -euo pipefail local arch what arch=${1}; shift what=${1}; shift local -a package_diff_env package_diff_params local -a size_changes_env size_changes_params local -a show_changes_env show_changes_params local version_description local -a var_names=( package_diff_env package_diff_params size_changes_env size_changes_params show_changes_env show_changes_params version_description ) case ${what} in last-release) local git_tag git_tag_for_release . git_tag prepare_env_vars_and_params_for_release "${arch}" "${git_tag}" "${var_names[@]}" ;; last-nightly) local git_tag git_tag_for_nightly . git_tag prepare_env_vars_and_params_for_bincache "${arch}" "${git_tag}" "${var_names[@]}" ;; *) echo "invalid argument '${what}', expected 'last-nightly' or 'last-release'" >&2 exit 1 ;; esac local ic_head_tag version head_git_tag . ic_head_tag version=$(source sdk_container/.repo/manifests/version.txt; echo "${FLATCAR_VERSION}") package_diff_env+=( "FROM_B=bincache" "BOARD_B=${arch}-usr" # CHANNEL_B is unused ) package_diff_params+=( "${version}" ) # Nothing to add to size changes env. size_changes_params+=( "bincache:${arch}:${version}" ) show_changes_env+=( # Provide a python3 command for the CVE DB parsing "PATH=${PATH}:${PWD}/ci-automation/python-bin" # Override the default locations of repositories. "SCRIPTS_REPO=." "COREOS_OVERLAY_REPO=../coreos-overlay" "PORTAGE_STABLE_REPO=../portage-stable" ) show_changes_params+=( # The show-changes script expects a tag name, so using git tag # here instead of the vernum variable. "NEW_VERSION=${ic_head_tag}" ) local fbs_repo='../flatcar-build-scripts' rm -rf "${fbs_repo}" git clone \ --depth 1 \ --single-branch \ "https://github.com/flatcar/flatcar-build-scripts" \ "${fbs_repo}" if [[ -z "${BUILDCACHE_SERVER:-}" ]]; then local BUILDCACHE_SERVER=$(source ci-automation/ci-config.env; echo "${BUILDCACHE_SERVER}") fi echo "Image URL: http://${BUILDCACHE_SERVER}/images/${arch}/${version}/flatcar_production_image.bin.bz2" echo local -a oemids get_oem_id_list . oemids generate_image_changes_report \ "${version_description}" '-' "${fbs_repo}" \ "${package_diff_env[@]}" --- "${package_diff_params[@]}" -- \ "${size_changes_env[@]}" --- "${size_changes_params[@]}" -- \ "${show_changes_env[@]}" --- "${show_changes_params[@]}" -- \ "${oemids[@]}" ) # -- # Gets a git tag that can be passed to # prepare_env_vars_and_params_for_release. # # 1 - scripts repo # 2 - name of a variable to store the result in function git_tag_for_release() { local scripts_repo git_tag_var_name scripts_repo=${1}; shift git_tag_var_name=${1}; shift head_git_tag "${scripts_repo}" "${git_tag_var_name}" } function head_git_tag() { local scripts_repo scripts_repo=${1}; shift local -n git_tag_ref="${1}"; shift git_tag_ref=$(git -C "${scripts_repo}" tag --points-at HEAD) if [[ -z ${git_tag_ref} ]]; then echo 'expected git HEAD commit to contain a tag' >&2 exit 1 fi } # Gets a git tag of a previous nightly that can be passed to # prepare_env_vars_and_params_for_bincache. # # 1 - scripts repo # 2 - name of a variable to store the result in function git_tag_for_nightly() { local scripts_repo scripts_repo=${1}; shift local -n git_tag_ref="${1}"; shift local head_tag search_object head_tag=$(git -C "${scripts_repo}" tag --points-at HEAD) search_object='HEAD' if [[ ${head_tag} = *-nightly-* ]] && [[ ! ${head_tag} = *-INTERMEDIATE ]]; then # HEAD is a nightly, pick an earlier commit to avoid comparing with itself search_object='HEAD^' fi git_tag_ref=$(git -C "${scripts_repo}" describe --tags --abbrev=0 --match='*-nightly-*' --exclude='*-INTERMEDIATE' "${search_object}") } # Gets a list of OEMs that are using sysexts. # # 1 - scripts repo # 2 - name of an array variable to store the result in function get_oem_id_list() { local scripts_repo scripts_repo=${1}; shift local -n list_var_ref=${1}; shift local -a ebuilds ebuilds=( "${scripts_repo}/sdk_container/src/third_party/coreos-overlay/coreos-base/common-oem-files/common-oem-files-"*'.ebuild' ) list_var_ref=() if [[ ${#ebuilds[@]} -eq 0 ]]; then return 0 fi local line mode # 0 = none OEMIDS line found yet # 1 = OEMIDS line found mode=0 while read -r line; do case ${mode} in 0) if [[ ${line} = 'OEMIDS=(' ]]; then mode=1 fi ;; 1) if [[ ${line} = ')' ]]; then break fi list_var_ref+=( "${line}" ) ;; esac done <"${ebuilds[0]}" } # Generates reports with passed parameters. The report is redirected # into the passed report file. # # 1 - version description (a free form string that describes a version of image that current version is compared against) # 2 - report file (can be relative) # 3 - flatcar-build-scripts directory (can be relative, will be realpathed) # @ - package-diff env vars --- package-diff version B param -- size-change-report.sh env vars --- size-change-report.sh spec B param -- show-changes env vars --- show-changes param overrides -- list of OEM ids # # Example: # # generate_image_changes_report \\ # 'Alpha 3456.0.0' reports/images.txt ../flatcar-build-scripts .. \\ # FROM_A=release BOARD_A=amd64-usr CHANNEL_A=alpha FROM_B=bincache BOARD_B=amd64-usr --- \\ # 3456.0.0 3478.0.0+my-changes -- \\ # --- \\ # release:amd64-usr:3456.0.0 bincache:amd64:3478.0.0+my-changes -- \\ # "PATH=${PATH}:${PWD}/ci-automation/python-bin" --- \\ # NEW_VERSION=main-3478.0.0-my-changes NEW_CHANNEL=alpha NEW_CHANNEL_PREV_VERSION=3456.0.0 OLD_CHANNEL=alpha OLD_VERSION='' -- \\ # azure vmware function generate_image_changes_report() ( set -euo pipefail local version_description=${1}; shift local report_output=${1}; shift local flatcar_build_scripts_repo=${1}; shift # rest is forwarded verbatim to print_image_reports local print_image_reports_invocation=( print_image_reports "${flatcar_build_scripts_repo}" "${version_description}" "${@}" ) # Using "|| :" to avoid failing the job. if [[ ${report_output} = '-' ]]; then "${print_image_reports_invocation[@]}" || : else { "${print_image_reports_invocation[@]}" || : } >"${report_output}" fi ) # -- # Prepares the tool parameters, so they compare against the last # release relative to the git tag. The git tag should be in form of # --, which is the usual format used in # scripts repo. function prepare_env_vars_and_params_for_release() { local arch git_tag arch=${1}; shift git_tag=${1}; shift local -n package_diff_env_ref="${1}"; shift local -n package_diff_params_ref="${1}"; shift local -n size_changes_env_ref="${1}"; shift local -n size_changes_params_ref="${1}"; shift local -n show_changes_env_ref="${1}"; shift local -n show_changes_params_ref="${1}"; shift local -n version_description_ref="${1}"; shift local ppfr_channel ppfr_version_id ppfr_build_id ppfr_version ppfr_vernum split_tag "${git_tag}" ppfr_channel ppfr_version_id ppfr_build_id ppfr_version ppfr_vernum if [[ ${ppfr_channel} = 'main' ]]; then ppfr_channel='alpha' fi local board new_channel new_channel_prev_version channel_a version_a board="${arch}-usr" new_channel="${ppfr_channel}" new_channel_prev_version=$(channel_version "${new_channel}" "${board}") channel_a='' version_a='' get_channel_a_and_version_a "${new_channel}" "${new_channel_prev_version}" "${ppfr_version}" "${board}" channel_a version_a package_diff_env_ref=( # For A. "FROM_A=release" "BOARD_A=${board}" "CHANNEL_A=${channel_a}" ) package_diff_params_ref=( # For A. "${version_a}" ) # Nothing to prepend to size_changes_env. size_changes_env_ref=() # First parts of the size-changes-report specs, the kind is # appended at call sites. size_changes_params_ref=( # For A. "release:${channel_a}:${board}:${version_a}" ) # Nothing to prepend to show_changes_env. show_changes_env_ref=() show_changes_params=( "NEW_CHANNEL=${new_channel}" "NEW_CHANNEL_PREV_VERSION=${new_channel_prev_version}" # Channel transition stuff "OLD_CHANNEL=${channel_a}" "OLD_VERSION=${version_a}" ) version_description_ref="${channel_a} ${version_a}" } # -- # Prepares the tool parameters, so they compare against the last # nightly relative to the git tag. The git tag should be in form of # --, which is the usual format used in # scripts repo. function prepare_env_vars_and_params_for_bincache() { local arch git_tag arch=${1}; shift git_tag=${1}; shift local -n package_diff_env_ref="${1}"; shift local -n package_diff_params_ref="${1}"; shift local -n size_changes_env_ref="${1}"; shift local -n size_changes_params_ref="${1}"; shift local -n show_changes_env_ref="${1}"; shift local -n show_changes_params_ref="${1}"; shift local -n version_description_ref="${1}"; shift local board board="${arch}-usr" local ppfb_channel ppfb_version_id ppfb_build_id ppfb_version ppfb_vernum split_tag "${git_tag}" ppfb_channel ppfb_version_id ppfb_build_id ppfb_version ppfb_vernum package_diff_env_ref=( # For A. "FROM_A=bincache" "BOARD_A=${board}" # CHANNEL_A is unused. ) package_diff_params_ref=( # For A. "${ppfb_version}" ) # Nothing to prepend to size_changes_env. size_changes_env_ref=() # First parts of the size-changes-report specs, the kind is # appended at call sites. size_changes_params_ref=( # For A. "bincache:${arch}:${ppfb_version}" ) # Nothing to prepend to show_changes_env. show_changes_env_ref=() show_changes_params=( "NEW_CHANNEL=${ppfb_channel}" "NEW_CHANNEL_PREV_VERSION=${ppfb_vernum}" # Channel transition stuff, we set the old channel to be the # same as the new channel to say that there was no channel # transition. Such would not make any sense here. "OLD_CHANNEL=${ppfb_channel}" "OLD_VERSION=${ppfb_vernum}" ) version_description_ref="development version ${ppfb_channel} ${ppfb_version}" } # -- function split_tag() { local git_tag git_tag=${1}; shift local -n channel_ref=${1}; shift local -n version_id_ref=${1}; shift local -n build_id_ref=${1}; shift local -n version_ref=${1}; shift local -n vernum_ref=${1}; shift local channel version_id build_id version vernum channel=${git_tag%%-*} version_id=${git_tag#*-} version_id=${version_id%%-*} build_id=${git_tag#"${channel}-${version_id}"} if [[ -n ${build_id} ]]; then build_id=${build_id#-} version="${version_id}+${build_id}" vernum="${version_id}-${build_id}" else version="${version_id}" vernum="${version_id}" fi channel_ref=${channel} version_id_ref=${version_id} build_id_ref=${build_id} version_ref=${version} vernum_ref=${vernum} } # -- function get_channel_a_and_version_a() { local new_channel=${1}; shift local new_channel_prev_version=${1}; shift local new_channel_new_version=${1}; shift local board=${1}; shift local gcaava_channel_a_varname=${1}; shift local gcaava_version_a_varname=${1}; shift local -n gcaava_channel_a_ref="${gcaava_channel_a_varname}" local -n gcaava_version_a_ref="${gcaava_version_a_varname}" local major_a major_b channel version major_a=$(echo "${new_channel_prev_version}" | cut -d . -f 1) major_b=$(echo "${new_channel_new_version}" | cut -d . -f 1) # When the major version for the new channel is different, a transition has happened and we can find the previous release in the old channel if [ "${major_a}" != "${major_b}" ]; then case "${new_channel}" in lts) channel=stable ;; stable) channel=beta ;; *) channel=alpha ;; esac version=$(channel_version "${channel}" "${board}") else channel="${new_channel}" version="${new_channel_prev_version}" fi gcaava_channel_a_ref=${channel} gcaava_version_a_ref=${version} } # -- # Gets the latest release for given channel and board. For lts channel # gets a version of the latest LTS. function channel_version() { local channel=${1}; shift local board=${1}; shift curl \ -fsSL \ --retry-delay 1 \ --retry 60 \ --retry-connrefused \ --retry-max-time 60 \ --connect-timeout 20 \ "https://${channel}.release.flatcar-linux.net/${board}/current/version.txt" | \ grep -m 1 'FLATCAR_VERSION=' | cut -d = -f 2- } # -- # Prints some reports using scripts from the passed path to # flatcar-build-scripts repo. The environment and parameters for the # scripts are passed as follows: # # print_image_reports \\ # --- -- \\ # --- -- \\ # --- -- \\ # # # Env vars are passed to the called scripts verbatim. Parameters are # described below. # # Parameters for package-diff: # # Passed directly to the script, so there should only be two # parameters - for version A and version B. # # Parameters for size-change-report.sh: # # Passed directly to the script after appending a ':' to them, # so there should be only two parameters being specs without the final # "kind" part. # # Parameters for show-changes: # # Should come in format of key=value, just like env vars. It's # expected that the following key-value pairs will be specified - for # NEW_CHANNEL, NEW_CHANNEL_PREV_VERSION, NEW_VERSION, OLD_CHANNEL and # OLD_VERSION. function print_image_reports() { local flatcar_build_scripts_repo=${1}; shift local previous_version_description=${1}; shift local -a package_diff_env=() package_diff_params=() local -a size_change_report_env=() size_change_report_params=() local -a show_changes_env=() show_changes_params=() local -a oemids local params_shift=0 split_to_env_and_params \ package_diff_env package_diff_params params_shift \ "${@}" shift "${params_shift}" split_to_env_and_params \ size_change_report_env size_change_report_params params_shift \ "${@}" shift "${params_shift}" split_to_env_and_params \ show_changes_env show_changes_params params_shift \ "${@}" shift "${params_shift}" oemids=( "${@}" ) flatcar_build_scripts_repo=$(realpath "${flatcar_build_scripts_repo}") local size_changes_invocation=( env "${size_change_report_env[@]}" "${flatcar_build_scripts_repo}/size-change-report.sh" ) yell "Image differences compared to ${previous_version_description}" underline "Package updates, compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE=flatcar_production_image_packages.txt \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 underline "Image file changes, compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt FILESONLY=1 CUTKERNEL=1 \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 underline "Image file size changes, compared to ${previous_version_description}:" if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:wtd}" 2>&1; then "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:old}" 2>&1 fi underline "Image kernel config changes, compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE=flatcar_production_image_kernel_config.txt \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 underline "Image file size change (includes /boot, /usr and the default rootfs partitions), compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE=flatcar_production_image_contents.txt CALCSIZE=1 \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 yell "Init ramdisk differences compared to ${previous_version_description}" underline "Image init ramdisk file changes, compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE=flatcar_production_image_initrd_contents.txt FILESONLY=1 CUTKERNEL=1 \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 underline "Image init ramdisk file size changes, compared to ${previous_version_description}:" if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-wtd}" 2>&1; then "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:initrd-old}" 2>&1 fi echo echo "Take the total size difference with a grain of salt as normally initrd is compressed, so the actual difference will be smaller." echo "To see the actual difference in size, see if there was a report for /boot/flatcar/vmlinuz-a." echo "Note that vmlinuz-a also contains the kernel code, which might have changed too, so the reported difference does not accurately describe the change in initrd." echo local oemid for oemid in "${oemids[@]}"; do yell "Sysext changes for OEM ${oemid} compared to ${previous_version_description}" underline "Package updates, compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE="oem-${oemid}_packages.txt" \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 underline "Image file changes, compared to ${previous_version_description}:" env \ "${package_diff_env[@]}" FILE="oem-${oemid}_contents.txt" FILESONLY=1 CUTKERNEL=1 \ "${flatcar_build_scripts_repo}/package-diff" "${package_diff_params[@]}" 2>&1 underline "Image file size changes, compared to ${previous_version_description}:" if ! "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-wtd}"; then "${size_changes_invocation[@]}" "${size_change_report_params[@]/%/:oem-${oemid}-old}" 2>&1 fi done local param for param in "${show_changes_params[@]}"; do local "SHOW_CHANGES_${param}" done # The first changelog we print is always against the previous # version of the new channel (is only same as old channel and old # version without a transition) yell "Changelog against ${SHOW_CHANGES_NEW_CHANNEL}-${SHOW_CHANGES_NEW_CHANNEL_PREV_VERSION}" env \ "${show_changes_env[@]}" \ "${flatcar_build_scripts_repo}/show-changes" \ "${SHOW_CHANGES_NEW_CHANNEL}-${SHOW_CHANGES_NEW_CHANNEL_PREV_VERSION}" \ "${SHOW_CHANGES_NEW_VERSION}" 2>&1 # See if a channel transition happened and print the changelog # against old channel and old version which is the previous # release if [ "${SHOW_CHANGES_OLD_CHANNEL}" != "${SHOW_CHANGES_NEW_CHANNEL}" ]; then yell "Changelog against ${SHOW_CHANGES_OLD_CHANNEL}-${SHOW_CHANGES_OLD_VERSION}" env \ "${show_changes_env[@]}" \ "${flatcar_build_scripts_repo}/show-changes" \ "${SHOW_CHANGES_OLD_CHANNEL}-${SHOW_CHANGES_OLD_VERSION}" \ "${SHOW_CHANGES_NEW_VERSION}" 2>&1 fi } # -- function yell() { local msg msg=${1}; shift local msg_len msg_len=${#msg} local y_str repeat_string '!' $((msg_len + 6)) y_str printf '\n%s\n!! %s !!\n%s\n\n' "${y_str}" "${msg}" "${y_str}" } function underline() { local msg msg=${1}; shift local msg_len msg_len=${#msg} local u_str repeat_string '=' "${msg_len}" u_str printf '\n%s\n%s\n\n' "${msg}" "${u_str}" } function repeat_string() { local str ntimes out_str_var_name str="${1}"; shift ntimes="${1}"; shift out_str_var_name="${1}"; shift local -n out_str_ref="${out_str_var_name}" if [[ ${ntimes} -eq 0 ]]; then out_str_ref="" return 0 elif [[ ${ntimes} -eq 1 ]]; then out_str_ref="${str}" return 0 fi local add_one add_one=$((ntimes % 2)) repeat_string "${str}${str}" $((ntimes / 2)) "${out_str_var_name}" if [[ add_one -gt 0 ]]; then out_str_ref+="${str}" fi } # 1 - name of an array variable for environment variables # 2 - name of an array variable for parameters # 3 - name of a scalar variable for shift number # @ - [ env var key=value pair… [ --- [ parameter… [ -- [ garbage… ] ] ] ] ] function split_to_env_and_params() { local steap_env_var_name=${1}; shift local steap_params_var_name=${1}; shift local steap_to_shift_var_name=${1}; shift local -n steap_env_var_ref="${steap_env_var_name}" local -n steap_params_var_ref="${steap_params_var_name}" local -n steap_to_shift_var_ref="${steap_to_shift_var_name}" local kv param to_shift=0 local -a env params env=() params=() # rest of parameters are key-value pairs followed by triple dash # followed by parameters followed by double dash or nothing for kv; do if [[ "${kv}" == '--' ]]; then break fi to_shift=$((to_shift + 1)) if [[ "${kv}" == '---' ]]; then break fi env+=( "${kv}" ) done shift "${to_shift}" for param; do to_shift=$((to_shift + 1)) if [[ "${param}" == '--' ]]; then break fi params+=( "${param}" ) done steap_env_var_ref=( "${env[@]}" ) steap_params_var_ref=( "${params[@]}" ) steap_to_shift_var_ref=${to_shift} } # --