From 296efcdb22a2cc82139794cc2d15c79c5bab40ff Mon Sep 17 00:00:00 2001 From: Krzesimir Nowak Date: Fri, 11 Apr 2025 14:56:21 +0200 Subject: [PATCH] pkg-auto: Add a setup for shellcheck's "source" checking That way shellcheck sources some prepared files and learns about some variables the sourced files define. Thanks to that, we can remove some of the "shellcheck disable" clauses. --- pkg_auto/Makefile | 4 +- pkg_auto/download_sdk_and_listings.sh | 4 +- pkg_auto/impl/for-shellcheck/README.md | 2 + pkg_auto/impl/for-shellcheck/globals | 68 +++++++++++++++++++ pkg_auto/impl/for-shellcheck/version.txt | 4 ++ pkg_auto/impl/pkg_auto_lib.sh | 84 ++++++++++-------------- 6 files changed, 112 insertions(+), 54 deletions(-) create mode 100644 pkg_auto/impl/for-shellcheck/README.md create mode 100644 pkg_auto/impl/for-shellcheck/globals create mode 100644 pkg_auto/impl/for-shellcheck/version.txt diff --git a/pkg_auto/Makefile b/pkg_auto/Makefile index b2795d456a..f71092b3c7 100644 --- a/pkg_auto/Makefile +++ b/pkg_auto/Makefile @@ -1,2 +1,4 @@ +all: shellcheck + shellcheck: - docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --external-sources --check-sourced *.sh impl/*.sh + docker run --rm -v "$$PWD:/mnt" koalaman/shellcheck:latest --norc --shell=bash --source-path=SCRIPTDIR --source-path=SCRIPTDIR/impl --source-path=SCRIPTDIR/impl/for-shellcheck --external-sources --check-sourced *.sh impl/*.sh diff --git a/pkg_auto/download_sdk_and_listings.sh b/pkg_auto/download_sdk_and_listings.sh index 11b06f79ef..8cfafb7e6d 100755 --- a/pkg_auto/download_sdk_and_listings.sh +++ b/pkg_auto/download_sdk_and_listings.sh @@ -148,9 +148,9 @@ function download() { } if [[ -n ${SCRIPTS} ]]; then - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt VERSION_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt BUILD_ID=$(source "${SCRIPTS}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") fi diff --git a/pkg_auto/impl/for-shellcheck/README.md b/pkg_auto/impl/for-shellcheck/README.md new file mode 100644 index 0000000000..223cd8c9cf --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/README.md @@ -0,0 +1,2 @@ +Shellcheck is being pointed here when processing source bash +directives for generated files. diff --git a/pkg_auto/impl/for-shellcheck/globals b/pkg_auto/impl/for-shellcheck/globals new file mode 100644 index 0000000000..f2060a0238 --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/globals @@ -0,0 +1,68 @@ +local -a GIT_ENV_VARS ARCHES WHICH REPORTS +local SDK_PKGS BOARD_PKGS +local SYNC_SCRIPT PKG_LIST_SORT_SCRIPT + +GIT_ENV_VARS=( + GIT_{AUTHOR,COMMITTER}_{NAME,EMAIL} +) + +SYNC_SCRIPT='/path/to/sync_with_gentoo.sh' +PKG_LIST_SORT_SCRIPT='/path/to/sort_packages_list.py' + +ARCHES=( 'amd64' 'arm64' ) +WHICH=('old' 'new') +SDK_PKGS='sdk-pkgs' +BOARD_PKGS='board-pkgs' +REPORTS=( "${SDK_PKGS}" "${BOARD_PKGS}" ) + +local SCRIPTS OLD_STATE NEW_STATE OLD_STATE_BRANCH NEW_STATE_BRANCH +local PORTAGE_STABLE_SUFFIX OLD_PORTAGE_STABLE NEW_PORTAGE_STABLE REPORTS_DIR +local NEW_STATE_PACKAGES_LIST AUX_DIR +local COREOS_OVERLAY_SUFFIX OLD_COREOS_OVERLAY NEW_COREOS_OVERLAY + +SCRIPTS='/path/to/scripts' +OLD_STATE='/path/to/old_state' +NEW_STATE='/path/to/new_state' +OLD_STATE_BRANCH='old-state' +NEW_STATE_BRANCH='new-state' +PORTAGE_STABLE_SUFFIX='sdk_container/src/third_party/portage-stable' +OLD_PORTAGE_STABLE='/path/to/old_state/portage-stable' +NEW_PORTAGE_STABLE='/path/to/new_state/portage-stable' +REPORTS_DIR='/path/to/reports' + +COREOS_OVERLAY_SUFFIX='sdk_container/src/third_party/coreos-overlay' +OLD_COREOS_OVERLAY='/path/to/old_state/coreos-overlay' +NEW_COREOS_OVERLAY='/path/to/new_state/coreos-overlay' + +NEW_STATE_PACKAGES_LIST="${NEW_STATE}/.github/workflows/portage-stable-packages-list" + +AUX_DIR='/path/to/aux' + +local 'SDK_IMAGE' + +SDK_IMAGE='ghcr.io/flatcar/flatcar-sdk-all:1234.5.6-build-tag' + +local -A LISTING_KINDS + +LISTING_KINDS=( + ['akamai']='oem-akamai_packages.txt' + ['ami']='oem-ami_packages.txt' + ['azure']='oem-azure_packages.txt' + ['containerd']='containerd-flatcar_packages.txt' + ['dev']='flatcar_developer_container_packages.txt' + ['digitalocean']='oem-digitalocean_packages.txt' + ['docker']='docker-flatcar_packages.txt' + ['gce']='oem-gce_packages.txt' + ['hetzner']='oem-hetzner_packages.txt' + ['kubevirt']='oem-kubevirt_packages.txt' + ['openstack']='oem-openstack_packages.txt' + ['packet']='oem-packet_packages.txt' + ['prod']='flatcar_production_image_packages.txt' + ['proxmoxve']='oem-proxmoxve_packages.txt' + ['qemu']='oem-qemu_packages.txt' + ['scaleway']='oem-scaleway_packages.txt' + ['sysext-podman']='flatcar-podman_packages.txt' + ['sysext-python']='flatcar-python_packages.txt' + ['sysext-zfs']='flatcar-zfs_packages.txt' + ['vmware']='oem-vmware_packages.txt' +) diff --git a/pkg_auto/impl/for-shellcheck/version.txt b/pkg_auto/impl/for-shellcheck/version.txt new file mode 100644 index 0000000000..2804576264 --- /dev/null +++ b/pkg_auto/impl/for-shellcheck/version.txt @@ -0,0 +1,4 @@ +FLATCAR_VERSION=1234.5.6+build-tag +FLATCAR_VERSION_ID=1234.5.6 +FLATCAR_BUILD_ID="build-tag" +FLATCAR_SDK_VERSION=1234.5.6+build-tag diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 1f7b33c7bd..7181c188d0 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -175,10 +175,9 @@ function save_new_state() { local branch_name branch_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" info "saving new state to branch ${branch_name}" - # shellcheck disable=SC2153 # SCRIPTS is not a misspelling, it comes from globals file git -C "${SCRIPTS}" branch --force "${branch_name}" "${NEW_STATE_BRANCH}" } @@ -282,7 +281,7 @@ function process_profile_updates_directory() { local -a ppud_ordered_names get_ordered_update_filenames ppud_ordered_names - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local bf ps_f co_f pkg f line old new @@ -292,9 +291,7 @@ function process_profile_updates_directory() { for bf in "${ppud_ordered_names[@]}"; do # coreos-overlay updates may overwrite updates from # portage-stable, but only from the file of the same name - # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file ps_f=${NEW_PORTAGE_STABLE}/profiles/updates/${bf} - # shellcheck disable=SC2153 # NEW_COREOS_OVERLAY is not a misspelling, it comes from globals file co_f=${NEW_COREOS_OVERLAY}/profiles/updates/${bf} for f in "${ps_f}" "${co_f}"; do if [[ ! -f ${f} ]]; then @@ -335,7 +332,7 @@ function process_profile_updates_directory() { function get_ordered_update_filenames() { local ordered_names_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A names_set=() @@ -523,13 +520,13 @@ NEW_STATE_PACKAGES_LIST="\${NEW_STATE}/.github/workflows/portage-stable-packages AUX_DIR=${aux_dir@Q} EOF - # shellcheck disable=SC1090 # generated file + # shellcheck source=for-shellcheck/globals source "${globals_file}" local last_nightly_version_id last_nightly_build_id - # shellcheck disable=SC1091,SC2153 # sourcing generated file, NEW_STATE is not misspelled + # shellcheck source=for-shellcheck/version.txt last_nightly_version_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_VERSION_ID}") - # shellcheck disable=SC1091 # sourcing generated file + # shellcheck source=for-shellcheck/version.txt last_nightly_build_id=$(source "${NEW_STATE}/sdk_container/.repo/manifests/version.txt"; printf '%s' "${FLATCAR_BUILD_ID}") local -a locals=() definitions=() @@ -548,7 +545,6 @@ EOF local packages_file tag filename stripped old for arch in "${ARCHES[@]}"; do - # shellcheck disable=SC2153 # AUX_DIR is not a misspelling, it comes from globals file for packages_file in "${AUX_DIR}/${arch}/"*_packages.txt; do filename=${packages_file##*/} stripped=${filename%_packages.txt} @@ -640,7 +636,7 @@ function run_sync() { missing_in_scripts=() missing_in_gentoo=() - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -x "${GIT_ENV_VARS[@]}" @@ -651,7 +647,6 @@ function run_sync() { local package while read -r package; do - # shellcheck disable=SC2153 # NEW_PORTAGE_STABLE is not a misspelling, it comes from globals file if [[ ! -e "${NEW_PORTAGE_STABLE}/${package}" ]]; then # If this happens, it means that the package was moved to overlay # or dropped, the list ought to be updated. @@ -670,7 +665,6 @@ function run_sync() { fi packages_to_update+=( "${package}" ) done < <(cat_meaningful "${NEW_STATE_PACKAGES_LIST}") - # shellcheck disable=SC2153 # SYNC_SCRIPT is not a misspelling env --chdir="${NEW_PORTAGE_STABLE}" "${SYNC_SCRIPT}" -b -- "${gentoo}" "${packages_to_update[@]}" save_missing_in_scripts "${missing_in_scripts[@]}" @@ -808,7 +802,7 @@ function handle_missing_in_scripts() { hmis_missing_in_scripts=() load_missing_in_scripts hmis_missing_in_scripts - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" if [[ ${#hmis_missing_in_scripts[@]} -eq 0 ]]; then @@ -823,7 +817,6 @@ function handle_missing_in_scripts() { join_by missing_re '\|' "${missing_in_scripts[@]}" add_cleanup "rm -f ${dir@Q}/pkg_list" xgrep --invert-match --line-regexp --fixed-strings --regexp="${missing_re}" "${NEW_STATE_PACKAGES_LIST}" >"${dir}/pkg_list" - # shellcheck disable=SC2153 # PKG_LIST_SORT_SCRIPT is not a misspelling "${PKG_LIST_SORT_SCRIPT}" "${dir}/pkg_list" >"${NEW_STATE_PACKAGES_LIST}" local -x "${GIT_ENV_VARS[@]}" @@ -862,11 +855,10 @@ function lines_to_file() { # # @ - lines to add function manual() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'manual work needed:' "${@}" - # shellcheck disable=SC2153 # REPORTS_DIR is not a misspelling, it comes from globals file lines_to_file "${REPORTS_DIR}/manual-work-needed" "${@}" } @@ -877,7 +869,7 @@ function manual() { # # @ - lines to add function pkg_warn() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'pkg warn:' "${@}" @@ -891,7 +883,7 @@ function pkg_warn() { # # @ - lines to add function devel_warn() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" pkg_debug_lines 'developer warn:' "${@}" @@ -913,7 +905,7 @@ function handle_missing_in_gentoo() { return 0; fi - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A hmig_rename_map=() @@ -984,7 +976,7 @@ function process_listings() { local pkg_to_tags_mvm_var_name pkg_to_tags_mvm_var_name=${1} - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" #mvm_debug_enable pl_pkg_to_tags_set_mvm @@ -992,7 +984,6 @@ function process_listings() { local arch kind file listing pkg for arch in "${ARCHES[@]}"; do - # shellcheck disable=SC2153 # LISTING_KINDS is not a misspelling, it comes from globals file for kind in "${!LISTING_KINDS[@]}"; do file=${LISTING_KINDS["${kind}"]} listing="${AUX_DIR}/${arch}/${file}" @@ -1068,7 +1059,7 @@ function set_mvm_to_array_mvm_cb() { # stored in salvaged-reports subdirectory of the reports directory. # Otherwise they will end up in reports-from-sdk subdirectory. function generate_sdk_reports() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" add_cleanup "rmdir ${WORKDIR@Q}/pkg-reports" @@ -1083,7 +1074,6 @@ function generate_sdk_reports() { local sdk_reports_dir top_dir dir entry full_path local -a dir_queue all_dirs all_files - # shellcheck disable=SC2153 # WHICH is not a misspelling, it comes from globals file for sdk_run_kind in "${WHICH[@]}"; do state_var_name="${sdk_run_kind^^}_STATE" sdk_run_state="${!state_var_name}_sdk_run" @@ -1289,7 +1279,7 @@ function pkginfo_c_process_file() { local -n pkg_set_ref=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local which arch report @@ -1366,7 +1356,7 @@ function read_reports() { all_pkgs_var_name=${1}; shift pkg_slots_set_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -A rr_all_packages_set @@ -1387,7 +1377,7 @@ function read_reports() { # Destroys the pkginfo maps for all the reports. function unset_report_mvms() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local arch which report @@ -1571,7 +1561,7 @@ function consistency_checks() { pkg_slots_set_mvm_var_name=${1}; shift pkg_slot_verminmax_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local cc_pimap_mvm_1_var_name cc_pimap_mvm_2_var_name pkg @@ -1706,7 +1696,7 @@ function consistency_checks() { function read_package_sources() { local -n package_sources_map_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a files=() @@ -1753,7 +1743,7 @@ function handle_package_changes() { local -n renamed_old_to_new_map_ref=${1}; shift pkg_to_tags_mvm_var_name=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a hpc_all_pkgs @@ -1997,7 +1987,6 @@ function handle_package_changes() { update_dir_non_slot "${new_name}" hpc_update_dir_non_slot mkdir -p "${hpc_update_dir_non_slot}" - # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE comes from globals file generate_non_ebuild_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" generate_full_diffs "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_name}" "${new_name}" generate_package_mention_reports "${NEW_STATE}" "${old_name}" "${new_name}" @@ -2176,7 +2165,7 @@ function handle_pkg_update() { old=${1}; shift new=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_no_r new_no_r @@ -2190,7 +2179,6 @@ function handle_pkg_update() { if [[ ${old_pkg} != "${new_pkg}" ]]; then lines+=( "0:renamed from ${old_pkg}" ) fi - # shellcheck disable=SC2153 # OLD_PORTAGE_STABLE is not a misspelling, it comes from globals file generate_ebuild_diff "${OLD_PORTAGE_STABLE}" "${NEW_PORTAGE_STABLE}" "${old_pkg}" "${new_pkg}" "${old_s}" "${new_s}" "${old}" "${new}" # shellcheck disable=SC2034 # these variables are used by name @@ -2258,7 +2246,7 @@ function handle_pkg_as_is() { v=${1}; shift local -n changed_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local hpai_update_dir @@ -2344,7 +2332,7 @@ function handle_pkg_downgrade() { old=${1}; shift new=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_no_r new_no_r @@ -2461,7 +2449,7 @@ function generate_changelog_entry_stub() { gces_tags='SDK' fi - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" printf '%s %s: %s ([%s](TODO))\n' '-' "${gces_tags}" "${pkg_name}" "${v}" >>"${REPORTS_DIR}/updates/changelog_stubs" @@ -2478,7 +2466,7 @@ function generate_summary_stub() { pkg=${1}; shift # rest are tags separated followed by double dash followed by lines - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a tags @@ -2623,6 +2611,7 @@ function generate_cache_diff_report() { local old=${1}; shift local new=${1}; shift + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_entry=${old_cache_dir}/${old_pkg}-${old} @@ -2722,7 +2711,7 @@ function update_dir_non_slot() { pkg=${1}; shift local -n dir_ref=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" # shellcheck disable=SC2034 # it's a reference to external variable @@ -2806,11 +2795,10 @@ function handle_gentoo_sync() { mvm_unset hgs_pkg_to_tags_mvm #mvm_debug_disable hgs_pkg_to_tags_mvm - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local old_head new_head - # shellcheck disable=SC2153 # OLD_STATE is not a misspelling old_head=$(git -C "${OLD_STATE}" rev-parse HEAD) new_head=$(git -C "${NEW_STATE}" rev-parse HEAD) @@ -2819,7 +2807,6 @@ function handle_gentoo_sync() { local path in_ps category if [[ "${old_head}" != "${new_head}" ]]; then while read -r path; do - # shellcheck disable=SC2153 # PORTAGE_STABLE_SUFFIX is not a misspelling if [[ ${path} != "${PORTAGE_STABLE_SUFFIX}/"* ]]; then continue fi @@ -3003,7 +2990,7 @@ function handle_eclass() { local eclass eclass=${1}; shift - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a lines @@ -3025,7 +3012,7 @@ function handle_eclass() { # and SDK), a full diff between all the profiles, and a list of # possibly irrelevant files that has changed too. function handle_profiles() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local -a files=() @@ -3095,14 +3082,10 @@ function handle_profiles() { # Handles changes in license directory. Generates brief reports and # diffs about dropped, added or modified licenses. function handle_licenses() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" - local -a dropped added changed - dropped=() - added=() - changed=() - + local -a dropped=() added=() changed=() local line hl_stripped # Lines are: # @@ -3113,7 +3096,6 @@ function handle_licenses() { # Files /licenses/BSL-1.1 and /licenses/BSL-1.1 differ while read -r line; do if [[ ${line} = 'Only in '* ]]; then - # shellcheck disable=SC2153 # OLD_STATE is not a misspelling, it comes from globals file strip_out "${line##*:}" hl_stripped if [[ ${line} = *"${OLD_STATE}"* ]]; then dropped+=( "${hl_stripped}" ) @@ -3173,7 +3155,7 @@ function handle_licenses() { # Generates reports about changes inside the scripts directory. function handle_scripts() { - # shellcheck disable=SC1091 # generated file + # shellcheck source=for-shellcheck/globals source "${WORKDIR}/globals" local out_dir