mirrors/flatcar-scripts
1
0
Fork 0
mirror of https://github.com/flatcar/scripts.git synced 2025-09-24 07:01:13 +02:00
Code Issues Packages Projects Releases Wiki Activity

pkg-auto: Add a setup for shellcheck's "source" checking

Browse Source 
That way shellcheck sources some prepared files and learns about some
variables the sourced files define. Thanks to that, we can remove some
of the "shellcheck disable" clauses.
This commit is contained in:
Krzesimir Nowak 2025-04-11 14:56:21 +02:00
parent 13e84333b8
commit 296efcdb22
6 changed files with 112 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
pkg_auto/Makefile
View File

@ -1,2 +1,4 @@
all: shellcheck
shellcheck:
docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh
docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --source-path=SCRIPTDIR/impl/for-shellcheck --external-sources --check-sourced *.sh impl/*.sh

4
pkg_auto/download_sdk_and_listings.sh
View File

@ -148,9 +148,9 @@ function download() {
}
if [[ -n ${SCRIPTS} ]]; then
# shellcheck disable=SC1091 # sourcing generated file
# shellcheck source=for-shellcheck/version.txt
VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}")
# shellcheck disable=SC1091 # sourcing generated file
# shellcheck source=for-shellcheck/version.txt
BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}")
fi

2
pkg_auto/impl/for-shellcheck/README.md Normal file
View File

@ -0,0 +1,2 @@
Shellcheck is being pointed here when processing source bash
directives for generated files.

68
pkg_auto/impl/for-shellcheck/globals Normal file
View File

@ -0,0 +1,68 @@
local -a GIT_ENV_VARS ARCHES WHICH REPORTS
local SDK_PKGS BOARD_PKGS
local SYNC_SCRIPT PKG_LIST_SORT_SCRIPT
GIT_ENV_VARS=(
GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL}
)
SYNC_SCRIPT='/path/to/sync_with_gentoo.sh'
PKG_LIST_SORT_SCRIPT='/path/to/sort_packages_list.py'
ARCHES=( 'amd64' 'arm64' )
WHICH=('old' 'new')
SDK_PKGS='sdk-pkgs'
BOARD_PKGS='board-pkgs'
REPORTS=( "${SDK_PKGS}" "${BOARD_PKGS}" )
local SCRIPTS OLD_STATE NEW_STATE OLD_STATE_BRANCH NEW_STATE_BRANCH
local PORTAGE_STABLE_SUFFIX OLD_PORTAGE_STABLE NEW_PORTAGE_STABLE REPORTS_DIR
local NEW_STATE_PACKAGES_LIST AUX_DIR
local COREOS_OVERLAY_SUFFIX OLD_COREOS_OVERLAY NEW_COREOS_OVERLAY
SCRIPTS='/path/to/scripts'
OLD_STATE='/path/to/old_state'
NEW_STATE='/path/to/new_state'
OLD_STATE_BRANCH='old-state'
NEW_STATE_BRANCH='new-state'
PORTAGE_STABLE_SUFFIX='sdk_container/src/third_party/portage-stable'
OLD_PORTAGE_STABLE='/path/to/old_state/portage-stable'
NEW_PORTAGE_STABLE='/path/to/new_state/portage-stable'
REPORTS_DIR='/path/to/reports'
COREOS_OVERLAY_SUFFIX='sdk_container/src/third_party/coreos-overlay'
OLD_COREOS_OVERLAY='/path/to/old_state/coreos-overlay'
NEW_COREOS_OVERLAY='/path/to/new_state/coreos-overlay'
NEW_STATE_PACKAGES_LIST="${NEW_STATE}/.github/workflows/portage-stable-packages-list"
AUX_DIR='/path/to/aux'
local 'SDK_IMAGE'
SDK_IMAGE='ghcr.io/flatcar/flatcar-sdk-all:1234.5.6-build-tag'
local -A LISTING_KINDS
LISTING_KINDS=(
['akamai']='oem-akamai_packages.txt'
['ami']='oem-ami_packages.txt'
['azure']='oem-azure_packages.txt'
['containerd']='containerd-flatcar_packages.txt'
['dev']='flatcar_developer_container_packages.txt'
['digitalocean']='oem-digitalocean_packages.txt'
['docker']='docker-flatcar_packages.txt'
['gce']='oem-gce_packages.txt'
['hetzner']='oem-hetzner_packages.txt'
['kubevirt']='oem-kubevirt_packages.txt'
['openstack']='oem-openstack_packages.txt'
['packet']='oem-packet_packages.txt'
['prod']='flatcar_production_image_packages.txt'
['proxmoxve']='oem-proxmoxve_packages.txt'
['qemu']='oem-qemu_packages.txt'
['scaleway']='oem-scaleway_packages.txt'
['sysext-podman']='flatcar-podman_packages.txt'
['sysext-python']='flatcar-python_packages.txt'
['sysext-zfs']='flatcar-zfs_packages.txt'
['vmware']='oem-vmware_packages.txt'
)

4
pkg_auto/impl/for-shellcheck/version.txt Normal file
View File

@ -0,0 +1,4 @@
FLATCAR_VERSION=1234.5.6+build-tag
FLATCAR_VERSION_ID=1234.5.6
FLATCAR_BUILD_ID="build-tag"
FLATCAR_SDK_VERSION=1234.5.6+build-tag

84
pkg_auto/impl/pkg_auto_lib.sh
View File

@ -175,10 +175,9 @@ function save_new_state() {
local branch_name
branch_name=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
info "saving new state to branch ${branch_name}"
# shellcheck disable=SC2153 # SCRIPTS is not a misspelling, it comes from globals file
git -C "${SCRIPTS}" branch --force "${branch_name}" "${NEW_STATE_BRANCH}"
}
@ -282,7 +281,7 @@ function process_profile_updates_directory() {
local -a ppud_ordered_names
get_ordered_update_filenames ppud_ordered_names
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local bf ps_f co_f pkg f line old new
@ -292,9 +291,7 @@ function process_profile_updates_directory() {
for bf in "${ppud_ordered_names[@]}"; do
# coreos-overlay updates may overwrite updates from
# portage-stable, but only from the file of the same name
# shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file
ps_f=${NEW_PORTAGE_STABLE}/profiles/updates/${bf}
# shellcheck disable=SC2153 # NEW_COREOS_OVERLAY is not a misspelling, it comes from globals file
co_f=${NEW_COREOS_OVERLAY}/profiles/updates/${bf}
for f in "${ps_f}" "${co_f}"; do
if [[ ! -f ${f} ]]; then
@ -335,7 +332,7 @@ function process_profile_updates_directory() {
function get_ordered_update_filenames() {
local ordered_names_var_name=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -A names_set=()
@ -523,13 +520,13 @@ NEW_STATE_PACKAGES_LIST="\${NEW_STATE}/.github/workflows/portage-stable-packages
AUX_DIR=${aux_dir@Q}
EOF
# shellcheck disable=SC1090 # generated file
# shellcheck source=for-shellcheck/globals
source "${globals_file}"
local last_nightly_version_id last_nightly_build_id
# shellcheck disable=SC1091,SC2153 # sourcing generated file, NEW_STATE is not misspelled
# shellcheck source=for-shellcheck/version.txt
last_nightly_version_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}")
# shellcheck disable=SC1091 # sourcing generated file
# shellcheck source=for-shellcheck/version.txt
last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}")
local -a locals=() definitions=()
@ -548,7 +545,6 @@ EOF
local packages_file tag filename stripped old
for arch in "${ARCHES[@]}"; do
# shellcheck disable=SC2153 # AUX_DIR is not a misspelling, it comes from globals file
for packages_file in "${AUX_DIR}/${arch}/"*_packages.txt; do
filename=${packages_file##*/}
stripped=${filename%_packages.txt}
@ -640,7 +636,7 @@ function run_sync() {
missing_in_scripts=()
missing_in_gentoo=()
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -x "${GIT_ENV_VARS[@]}"
@ -651,7 +647,6 @@ function run_sync() {
local package
while read -r package; do
# shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file
if [[ ! -e "${NEW_PORTAGE_STABLE}/${package}" ]]; then
# If this happens, it means that the package was moved to overlay
# or dropped, the list ought to be updated.
@ -670,7 +665,6 @@ function run_sync() {
fi
packages_to_update+=( "${package}" )
done < <(cat_meaningful "${NEW_STATE_PACKAGES_LIST}")
# shellcheck disable=SC2153 # SYNC_SCRIPT is not a misspelling
env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${packages_to_update[@]}"
save_missing_in_scripts "${missing_in_scripts[@]}"
@ -808,7 +802,7 @@ function handle_missing_in_scripts() {
hmis_missing_in_scripts=()
load_missing_in_scripts hmis_missing_in_scripts
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
if [[ ${#hmis_missing_in_scripts[@]} -eq 0 ]]; then
@ -823,7 +817,6 @@ function handle_missing_in_scripts() {
join_by missing_re '\|' "${missing_in_scripts[@]}"
add_cleanup "rm -f ${dir@Q}/pkg_list"
xgrep --invert-match --line-regexp --fixed-strings --regexp="${missing_re}" "${NEW_STATE_PACKAGES_LIST}" >"${dir}/pkg_list"
# shellcheck disable=SC2153 # PKG_LIST_SORT_SCRIPT is not a misspelling
"${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}"
local -x "${GIT_ENV_VARS[@]}"
@ -862,11 +855,10 @@ function lines_to_file() {
#
# @ - lines to add
function manual() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
pkg_debug_lines 'manual work needed:' "${@}"
# shellcheck disable=SC2153 # REPORTS_DIR is not a misspelling, it comes from globals file
lines_to_file "${REPORTS_DIR}/manual-work-needed" "${@}"
}
@ -877,7 +869,7 @@ function manual() {
#
# @ - lines to add
function pkg_warn() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
pkg_debug_lines 'pkg warn:' "${@}"
@ -891,7 +883,7 @@ function pkg_warn() {
#
# @ - lines to add
function devel_warn() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
pkg_debug_lines 'developer warn:' "${@}"
@ -913,7 +905,7 @@ function handle_missing_in_gentoo() {
return 0;
fi
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -A hmig_rename_map=()
@ -984,7 +976,7 @@ function process_listings() {
local pkg_to_tags_mvm_var_name
pkg_to_tags_mvm_var_name=${1}
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
#mvm_debug_enable pl_pkg_to_tags_set_mvm
@ -992,7 +984,6 @@ function process_listings() {
local arch kind file listing pkg
for arch in "${ARCHES[@]}"; do
# shellcheck disable=SC2153 # LISTING_KINDS is not a misspelling, it comes from globals file
for kind in "${!LISTING_KINDS[@]}"; do
file=${LISTING_KINDS["${kind}"]}
listing="${AUX_DIR}/${arch}/${file}"
@ -1068,7 +1059,7 @@ function set_mvm_to_array_mvm_cb() {
# stored in salvaged-reports subdirectory of the reports directory.
# Otherwise they will end up in reports-from-sdk subdirectory.
function generate_sdk_reports() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports"
@ -1083,7 +1074,6 @@ function generate_sdk_reports() {
local sdk_reports_dir top_dir dir entry full_path
local -a dir_queue all_dirs all_files
# shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file
for sdk_run_kind in "${WHICH[@]}"; do
state_var_name="${sdk_run_kind^^}_STATE"
sdk_run_state="${!state_var_name}_sdk_run"
@ -1289,7 +1279,7 @@ function pkginfo_c_process_file() {
local -n pkg_set_ref=${1}; shift
pkg_slots_set_mvm_var_name=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local which arch report
@ -1366,7 +1356,7 @@ function read_reports() {
all_pkgs_var_name=${1}; shift
pkg_slots_set_mvm_var_name=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -A rr_all_packages_set
@ -1387,7 +1377,7 @@ function read_reports() {
# Destroys the pkginfo maps for all the reports.
function unset_report_mvms() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local arch which report
@ -1571,7 +1561,7 @@ function consistency_checks() {
pkg_slots_set_mvm_var_name=${1}; shift
pkg_slot_verminmax_mvm_var_name=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local cc_pimap_mvm_1_var_name cc_pimap_mvm_2_var_name pkg
@ -1706,7 +1696,7 @@ function consistency_checks() {
function read_package_sources() {
local -n package_sources_map_ref=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -a files=()
@ -1753,7 +1743,7 @@ function handle_package_changes() {
local -n renamed_old_to_new_map_ref=${1}; shift
pkg_to_tags_mvm_var_name=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -a hpc_all_pkgs
@ -1997,7 +1987,6 @@ function handle_package_changes() {
update_dir_non_slot "${new_name}" hpc_update_dir_non_slot
mkdir -p "${hpc_update_dir_non_slot}"
# shellcheck disable=SC2153 # OLD_PORTAGE_STABLE comes from globals file
generate_non_ebuild_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}"
generate_full_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}"
generate_package_mention_reports "${NEW_STATE}" "${old_name}" "${new_name}"
@ -2176,7 +2165,7 @@ function handle_pkg_update() {
old=${1}; shift
new=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local old_no_r new_no_r
@ -2190,7 +2179,6 @@ function handle_pkg_update() {
if [[ ${old_pkg} != "${new_pkg}" ]]; then
lines+=( "0:renamed from ${old_pkg}" )
fi
# shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file
generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}"
# shellcheck disable=SC2034 # these variables are used by name
@ -2258,7 +2246,7 @@ function handle_pkg_as_is() {
v=${1}; shift
local -n changed_ref=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local hpai_update_dir
@ -2344,7 +2332,7 @@ function handle_pkg_downgrade() {
old=${1}; shift
new=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local old_no_r new_no_r
@ -2461,7 +2449,7 @@ function generate_changelog_entry_stub() {
gces_tags='SDK'
fi
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
printf '%s %s: %s ([%s](TODO))\n' '-' "${gces_tags}" "${pkg_name}" "${v}" >>"${REPORTS_DIR}/updates/changelog_stubs"
@ -2478,7 +2466,7 @@ function generate_summary_stub() {
pkg=${1}; shift
# rest are tags separated followed by double dash followed by lines
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -a tags
@ -2623,6 +2611,7 @@ function generate_cache_diff_report() {
local old=${1}; shift
local new=${1}; shift
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local old_entry=${old_cache_dir}/${old_pkg}-${old}
@ -2722,7 +2711,7 @@ function update_dir_non_slot() {
pkg=${1}; shift
local -n dir_ref=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
# shellcheck disable=SC2034 # it's a reference to external variable
@ -2806,11 +2795,10 @@ function handle_gentoo_sync() {
mvm_unset hgs_pkg_to_tags_mvm
#mvm_debug_disable hgs_pkg_to_tags_mvm
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local old_head new_head
# shellcheck disable=SC2153 # OLD_STATE is not a misspelling
old_head=$(git -C "${OLD_STATE}" rev-parse HEAD)
new_head=$(git -C "${NEW_STATE}" rev-parse HEAD)
@ -2819,7 +2807,6 @@ function handle_gentoo_sync() {
local path in_ps category
if [[ "${old_head}" != "${new_head}" ]]; then
while read -r path; do
# shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling
if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then
continue
fi
@ -3003,7 +2990,7 @@ function handle_eclass() {
local eclass
eclass=${1}; shift
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -a lines
@ -3025,7 +3012,7 @@ function handle_eclass() {
# and SDK), a full diff between all the profiles, and a list of
# possibly irrelevant files that has changed too.
function handle_profiles() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -a files=()
@ -3095,14 +3082,10 @@ function handle_profiles() {
# Handles changes in license directory. Generates brief reports and
# diffs about dropped, added or modified licenses.
function handle_licenses() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local -a dropped added changed
dropped=()
added=()
changed=()
local -a dropped=() added=() changed=()
local line hl_stripped
# Lines are:
#
@ -3113,7 +3096,6 @@ function handle_licenses() {
# Files <PORTAGE_STABLE_1>/licenses/BSL-1.1 and <PORTAGE_STABLE_2>/licenses/BSL-1.1 differ
while read -r line; do
if [[ ${line} = 'Only in '* ]]; then
# shellcheck disable=SC2153 # OLD_STATE is not a misspelling, it comes from globals file
strip_out "${line##*:}" hl_stripped
if [[ ${line} = *"${OLD_STATE}"* ]]; then
dropped+=( "${hl_stripped}" )
@ -3173,7 +3155,7 @@ function handle_licenses() {
# Generates reports about changes inside the scripts directory.
function handle_scripts() {
# shellcheck disable=SC1091 # generated file
# shellcheck source=for-shellcheck/globals
source "${WORKDIR}/globals"
local out_dir