diff --git a/pkg_auto/impl/md5_cache_diff_lib.sh b/pkg_auto/impl/md5_cache_diff_lib.sh index 3dcab5ca63..af99a09dfd 100644 --- a/pkg_auto/impl/md5_cache_diff_lib.sh +++ b/pkg_auto/impl/md5_cache_diff_lib.sh @@ -283,25 +283,123 @@ function diff_report_dedent() { ((dr_ref[DR_INDENT_IDX]--)) } +# TODO: documentation +declare -gri DLF_LLVM_SLOTS_IDX=0 DLF_PYTHON_SLOTS_IDX=1 DLF_RUST_SLOTS_IDX=2 DLF_LLVM_TARGETS_IDX=3 +# computed with diff_lib_generate_maps +declare -gri DLF_RELEVANT_IUSES_SET_IDX=4 DLF_QUESTIONABLE_PREFIXES_IDX=5 DLF_IRRELEVANT_IUSES_SET_IDX=6 DLF_RELEVANT_PYTHON_URS_SET_IDX=7 DLF_QUESTIONABLE_PYTHON_UR_PREFIXES_IDX=8 + +# TODO: documentation +function diff_lib_filters_declare() { + struct_declare -ga "${@}" "( 'EMPTY_ARRAY' 'EMPTY_ARRAY' 'EMPTY_ARRAY' 'EMPTY_ARRAY' 'EMPTY_MAP' 'EMPTY_ARRAY' 'EMPTY_MAP' 'EMPTY_MAP' 'EMPTY_ARRAY' )" +} + +# TODO: documentation +function diff_lib_filters_unset() { + local name + local -i idx + for name; do + local -n filters_ref=${name} + for idx in DLF_LLVM_SLOTS_IDX DLF_PYTHON_SLOTS_IDX DLF_RUST_SLOTS_IDX DLF_LLVM_TARGETS_IDX DLF_QUESTIONABLE_PREFIXES_IDX DLF_QUESTIONABLE_PYTHON_UR_PREFIXES_IDX; do + name=${filters_ref[idx]} + if [[ ${name} != 'EMPTY_ARRAY' ]]; then + unset "${name}" + fi + done + for idx in DLF_RELEVANT_IUSES_SET_IDX DLF_IRRELEVANT_IUSES_SET_IDX DLF_RELEVANT_PYTHON_URS_SET_IDX; do + name=${filters_ref[idx]} + if [[ ${name} != 'EMPTY_MAP' ]]; then + unset "${name}" + fi + done + unset -n filters_ref + done + unset "${@}" +} + +function diff_lib_generate_maps() { + local -n filters_ref=${1}; shift + + local dlgm_relevant_iuses_set_name dlgm_questionable_prefixes_name dlgm_irrelevant_iuses_set_name + local dlgm_relevant_urs_set_name dlgm_questionable_ur_prefixes_name + gen_varname dlgm_relevant_iuses_set_name + gen_varname dlgm_questionable_prefixes_name + gen_varname dlgm_irrelevant_iuses_set_name + gen_varname dlgm_relevant_python_urs_set_name + gen_varname dlgm_questionable_python_ur_prefixes_name + declare -gA "${dlgm_relevant_iuses_set_name}=()" "${dlgm_irrelevant_iuses_set_name}=()" + declare -ga "${dlgm_questionable_prefixes_name}=()" + declare -gA "${dlgm_relevant_python_urs_set_name}=()" + declare -ga "${dlgm_questionable_python_ur_prefixes_name}=()" + + local -n relevant_iuses_set_ref=${dlgm_relevant_iuses_set_name} + local -n questionable_prefixes_ref=${dlgm_questionable_prefixes_name} + local -n irrelevant_iuses_set_ref=${dlgm_irrelevant_iuses_set_name} + local -n relevant_python_urs_set_ref=${dlgm_relevant_python_urs_set_name} + local -n questionable_python_ur_prefixes_ref=${dlgm_questionable_python_ur_prefixes_name} + + local -n python_slots_ref=${filters_ref[DLF_PYTHON_SLOTS_IDX]} + local slot + for slot in "${python_slots_ref[@]}"; do + # slots is something like 3.11, we want 3_11 + slot=${slot//./_} + relevant_iuses_set_ref["python_targets_python${slot}"]=x + relevant_iuses_set_ref["python_single_target_python${slot}"]=x + relevant_python_urs_set_ref["python_targets_python${slot}"]=x + done + unset -n python_slots_ref + # not python_targets_python, so we can also catch python_targets_pypy prefix + questionable_prefixes_ref+=( 'python_targets_' 'python_single_target_' ) + questionable_python_ur_prefixes_ref+=( 'python_targets_' ) + + local -n llvm_slots_ref=${filters_ref[DLF_LLVM_SLOTS_IDX]} + for slot in "${llvm_slots_ref[@]}"; do + relevant_iuses_set_ref["llvm_slot_${slot}"]=x + done + unset -n llvm_slots_ref + questionable_prefixes_ref+=( 'llvm_slot_' ) + + local -n llvm_targets_ref=${filters_ref[DLF_LLVM_TARGETS_IDX]} + local target + for target in "${llvm_targets_ref[@]}"; do + target="llvm_targets_${target}" + relevant_iuses_set_ref["${target}"]=x + done + questionable_prefixes_ref+=( 'llvm_targets_' ) + unset -n llvm_targets_ref + + # maybe we will find more generally irrelevant iuses, maybe "doc" or "full-test"? + irrelevant_iuses_set_ref['test']=x + + unset -n relevant_iuses_set_ref questionable_prefixes_ref irrelevant_iuses_set_ref relevant_python_urs_set_ref questionable_python_ur_prefixes_ref + + filters_ref[DLF_RELEVANT_IUSES_SET_IDX]=${dlgm_relevant_iuses_set_name} + filters_ref[DLF_QUESTIONABLE_PREFIXES_IDX]=${dlgm_questionable_prefixes_name} + filters_ref[DLF_IRRELEVANT_IUSES_SET_IDX]=${dlgm_irrelevant_iuses_set_name} + filters_ref[DLF_RELEVANT_PYTHON_URS_SET_IDX]=${dlgm_relevant_python_urs_set_name} + filters_ref[DLF_QUESTIONABLE_PYTHON_UR_PREFIXES_IDX]=${dlgm_questionable_python_ur_prefixes_name} +} + # Diffs two cache files. # # Params: # # 1 - old cache file # 2 - new cache file -# 3 - diff report where the diff will be written to +# 3 - filters +# 4 - diff report where the diff will be written to function diff_cache_data() { local old_var_name=${1}; shift local new_var_name=${1}; shift + local diff_lib_filters_var_name=${1}; shift local dr_var_name=${1}; shift __mcdl_diff_eapi "${old_var_name}" "${new_var_name}" "${dr_var_name}" __mcdl_diff_keywords "${old_var_name}" "${new_var_name}" "${dr_var_name}" - __mcdl_diff_iuse "${old_var_name}" "${new_var_name}" "${dr_var_name}" + __mcdl_diff_iuse "${old_var_name}" "${new_var_name}" "${diff_lib_filters_var_name}" "${dr_var_name}" local -i idx for idx in PCF_BDEPEND_IDX PCF_DEPEND_IDX PCF_IDEPEND_IDX PCF_PDEPEND_IDX PCF_RDEPEND_IDX PCF_LICENSE_IDX; do - __mcdl_diff_deps "${old_var_name}" "${new_var_name}" ${idx} "${dr_var_name}" + __mcdl_diff_deps "${old_var_name}" "${new_var_name}" "${diff_lib_filters_var_name}" ${idx} "${dr_var_name}" done } @@ -325,6 +423,7 @@ function __mcdl_diff_eapi() { function __mcdl_diff_iuse() { local -n old_ref=${1}; shift local -n new_ref=${1}; shift + local diff_lib_filters_var_name=${1}; shift local dr_var_name=${1}; shift local old_iuses_var_name=${old_ref[PCF_IUSE_IDX]} @@ -357,10 +456,16 @@ function __mcdl_diff_iuse() { local iuse for iuse in "${!removed_iuses[@]}"; do + if __mcdl_is_iuse_irrelevant "${diff_lib_filters_var_name}" "${iuse}"; then + continue + fi diff_report_append "${dr_var_name}" "removed IUSE flag ${iuse@Q}" diff_report_append_indented "${dr_var_name}" "TODO: describe removed IUSE flag" done for iuse in "${!added_iuses[@]}"; do + if __mcdl_is_iuse_irrelevant "${diff_lib_filters_var_name}" "${iuse}"; then + continue + fi diff_report_append "${dr_var_name}" "added IUSE flag ${iuse@Q}" diff_report_append_indented "${dr_var_name}" "TODO: describe added IUSE flag" done @@ -611,22 +716,196 @@ function __mcdl_iuse_stack_to_string() { str_ref=${str_ref:0:$((${#str_ref} - 4))} } +# similar to __mcdl_iuse_stack_to_string but strips numbering of +# unnamed groups +function __mcdl_iuse_stack_to_string_for_matching() { + local -n stack_ref=${1}; shift + local -n str_ref=${1}; shift + + str_ref='' + # we always ignore the first element - it's a name of the toplevel + # unnamed group, so stack of length 1 means no groups were + # encountered + if [[ ${#stack_ref[@]} -le 1 ]]; then + return 0 + fi + + local iuse + # we always ignore the first element - it's a name of the toplevel + # unnamed group + for iuse in "${stack_ref[@]:1}"; do + if [[ ${iuse} = 'unnamed-all-of-'* ]]; then + iuse='unnamed-all-of-X' + fi + str_ref+="${iuse@Q} -> " + done + str_ref=${str_ref:0:$((${#str_ref} - 4))} +} + +function __mcdl_is_iuse_stack_irrelevant() { + local diff_lib_filters_var_name=${1}; shift + local -n stack_ref=${1}; shift + local i + + for i in "${stack_ref[@]}"; do + i=${i%\?} + if __mcdl_is_iuse_irrelevant "${diff_lib_filters_var_name}" "${i}"; then + return 0 + fi + done + return 1 +} + +function __mcdl_is_iuse_irrelevant() { + local diff_lib_filters_var_name=${1}; shift + local iuse=${1}; shift + + local -n filters_ref=${diff_lib_filters_var_name} + + local -n irrelevant_iuses_set_ref=${filters_ref[DLF_IRRELEVANT_IUSES_SET_IDX]} + if [[ -n ${irrelevant_iuses_set_ref["${iuse}"]:-} ]]; then + return 0 + fi + unset -n irrelevant_iuses_set_ref + + local -n relevant_iuses_set_ref=${filters_ref[DLF_RELEVANT_IUSES_SET_IDX]} + if [[ -n ${relevant_iuses_set_ref["${iuse}"]:-} ]]; then + return 1 + fi + unset -n relevant_iuses_set_ref + + local -n questionable_prefixes_ref=${filters_ref[DLF_QUESTIONABLE_PREFIXES_IDX]} + local prefix + for prefix in "${questionable_prefixes_ref[@]}"; do + if [[ ${iuse} = "${prefix}"* ]]; then + return 0 + fi + done + + return 1 +} + +function __mcdl_is_pds_irrelevant() { + local diff_lib_filters_var_name=${1}; shift + local -n pds_ref=${1}; shift + local stack_var_name=${1}; shift + + local n=${pds_ref[PDS_NAME_IDX]} + + if __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" "${stack_var_name}"; then + return 0 + fi + + local -n stack_ref=${stack_var_name}; shift + case ${n} in + 'dev-lang/python') + local op=${pds_ref[PDS_OP_IDX]} + local slot=${pds_ref[PDS_SLOT_IDX]} relevant_slot + local matched='' + if [[ -z ${op} && -n ${slot} ]]; then + local -n filters_ref=${diff_lib_filters_var_name} + local -n python_slots_ref=${filters_ref[DLF_PYTHON_SLOTS_IDX]} + unset -n filters_ref + for relevant_slot in "${python_slots_ref[@]}"; do + if [[ ${slot} = "${relevant_slot}" ]]; then + matched=x + break + fi + done + unset -n python_slots_ref + fi + if [[ -z ${matched} ]]; then + return 0 + fi + ;; + 'dev-lang/rust'|'dev-lang/rust-bin') + local op=${pds_ref[PDS_OP_IDX]} + local slot=${pds_ref[PDS_SLOT_IDX]} relevant_slot + local matched='' + if [[ -z ${op} && -n ${slot} ]]; then + local -n filters_ref=${diff_lib_filters_var_name} + local -n rust_slots_ref=${filters_ref[DLF_RUST_SLOTS_IDX]} + unset -n filters_ref + for relevant_slot in "${rust_slots_ref[@]}"; do + if [[ ${slot} = "${relevant_slot}" ]]; then + matched=x + break + fi + done + unset -n rust_slots_ref + fi + if [[ -z ${matched} ]]; then + return 0 + fi + ;; + 'dev-python/'*) + # ignore dev-python/foo in situations like: + # + # || ( dev-python/foo[python_targets_python3_xx(-)] ) + # + # or + # + # || ( ( dev-lang/python:3.xx dev-python/foo[python_targets_python3_xx(-)]) ) + if [[ ( ${#stack_ref[@]} -gt 1 && ${stack_ref[-1]} = 'unnamed-all-of-'* && ${stack_ref[-2]} = '||' ) || + ( ${#stack_ref[@]} -gt 0 && ${stack_ref[-1]} = '||' ) ]]; then + local irrelevant= + local -n urs=${pds_ref[PDS_UR_IDX]} + if [[ ${#urs[@]} -eq 1 ]]; then + local -n ur=${urs[0]} + local ur_name=${ur[UR_NAME_IDX]} + local -n filters_ref=${diff_lib_filters_var_name} + local -n relevant_python_urs_map_ref=${filters_ref[DLF_RELEVANT_PYTHON_URS_SET_IDX]} + if [[ -z ${relevant_python_urs_map_ref["${ur_name}"]:-} ]]; then + local -n questionable_python_ur_prefixes=${filters_ref[DLF_QUESTIONABLE_PYTHON_UR_PREFIXES_IDX]} + local prefix + for prefix in "${questionable_python_ur_prefixes[@]}"; do + if [[ ${ur_name} = "${prefix}"* ]]; then + irrelevant=x + break + fi + done + unset -n questionable_python_ur_prefixes + fi + unset -n relevant_python_urs_map_ref filters_ref ur + fi + unset -n urs + if [[ -n ${irrelevant} ]]; then + return 0 + fi + fi + ;; + esac + return 1 +} + function __mcdl_pds_diff() { local -n old_pds_ref=${1}; shift local -n new_pds_ref=${1}; shift + local diff_lib_filters_var_name=${1}; shift local old_stack_name=${1}; shift local new_stack_name=${1}; shift local dr_var_name=${1}; shift + if __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" "${old_stack_name}" && __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" "${new_stack_name}"; then + return 0 + fi + local name=${new_pds_ref[PDS_NAME_IDX]} - local old_iuses new_iuses - __mcdl_iuse_stack_to_string "${old_stack_name}" old_iuses - __mcdl_iuse_stack_to_string "${new_stack_name}" new_iuses + local old_iuses new_iuses same_iuses='' + __mcdl_iuse_stack_to_string_for_matching "${old_stack_name}" old_iuses + __mcdl_iuse_stack_to_string_for_matching "${new_stack_name}" new_iuses diff_report_declare local_pds_dr - if [[ ${old_iuses} != "${new_iuses}" ]]; then + if [[ ${old_iuses} = "${new_iuses}" ]]; then + same_iuses=x + fi + + __mcdl_iuse_stack_to_string "${old_stack_name}" old_iuses + __mcdl_iuse_stack_to_string "${new_stack_name}" new_iuses + + if [[ -z ${same_iuses} ]]; then if [[ -z ${new_iuses} ]]; then diff_report_append local_pds_dr "dropped all USE conditionals" else @@ -707,10 +986,16 @@ function __mcdl_pds_diff() { local -A only_old_urs=() only_new_urs=() common_urs=() sets_split old_name_index new_name_index only_old_urs only_new_urs common_urs for use_name in "${!only_old_urs[@]}"; do + if __mcdl_is_iuse_irrelevant "${diff_lib_filters_var_name}" "${use_name}"; then + continue + fi diff_report_append local_pds_dr "dropped ${use_name} use requirement" done local pd_mode_str pd_pretend_str for use_name in "${!only_new_urs[@]}"; do + if __mcdl_is_iuse_irrelevant "${diff_lib_filters_var_name}" "${use_name}"; then + continue + fi idx=${new_name_index["${use_name}"]} local -n ur_ref=${new_urs_ref[${idx}]} __mcdl_ur_mode_description "${ur_ref[UR_MODE_IDX]}" pd_mode_str @@ -1222,6 +1507,7 @@ function __mcdl_debug_iuse_stack() { function __mcdl_diff_deps() { local -n old_ref=${1}; shift local -n new_ref=${1}; shift + local diff_lib_filters_var_name=${1}; shift local deps_idx=${1}; shift local dr_var_name=${1}; shift @@ -1301,32 +1587,36 @@ function __mcdl_diff_deps() { local t1=${item1:0:1} v1=${item1:2} case ${t1} in 'l') - local use_str - __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' - diff_report_append local_dr "dropped license ${v1@Q}${use_str}" - unset use_str + if ! __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" old_iuse_stack; then + local use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "dropped license ${v1@Q}${use_str}" + unset use_str + fi ;; 'p') - local p_str use_str - __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' - pds_to_string "${v1}" p_str - local -n p_ref=${v1} - local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} - unset -n p_ref - local what='' - case ${p_blocks} in - "${PDS_NO_BLOCK}") - what='dependency' - ;; - "${PDS_WEAK_BLOCK}") - what='weak blocker' - ;; - "${PDS_STRONG_BLOCK}") - what='strong blocker' - ;; - esac - diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" - unset what p_blocks use_str p_str + if ! __mcdl_is_pds_irrelevant "${diff_lib_filters_var_name}" "${v1}" old_iuse_stack; then + local p_str use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + pds_to_string "${v1}" p_str + local -n p_ref=${v1} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + fi ;; 'i') # This will be stored in prev item and used in 'o' @@ -1366,32 +1656,36 @@ function __mcdl_diff_deps() { local t2=${item2:0:1} v2=${item2:2} case ${t2} in 'l') - local use_str - __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' - diff_report_append local_dr "added license ${v2@Q}${use_str}" - unset use_str + if ! __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" new_iuse_stack; then + local use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "added license ${v2@Q}${use_str}" + unset use_str + fi ;; 'p') - local p_str use_str - __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' - pds_to_string "${v2}" p_str - local -n p_ref=${v2} - local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} - unset -n p_ref - local what='' - case ${p_blocks} in - "${PDS_NO_BLOCK}") - what='dependency' - ;; - "${PDS_WEAK_BLOCK}") - what='weak blocker' - ;; - "${PDS_STRONG_BLOCK}") - what='strong blocker' - ;; - esac - diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" - unset what p_blocks use_str p_str + if ! __mcdl_is_pds_irrelevant "${diff_lib_filters_var_name}" "${v2}" new_iuse_stack; then + local p_str use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + pds_to_string "${v2}" p_str + local -n p_ref=${v2} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + fi ;; 'i') # This will be stored in prev item and used in 'o' @@ -1473,7 +1767,7 @@ function __mcdl_diff_deps() { __mcdl_debug_iuse_stack new_iuse_stack "new iuse stack after dropping common last name" ;; 'p') - __mcdl_pds_diff "${item1:2}" "${item2:2}" old_iuse_stack new_iuse_stack local_dr + __mcdl_pds_diff "${item1:2}" "${item2:2}" "${diff_lib_filters_var_name}" old_iuse_stack new_iuse_stack local_dr ;; *) fail "item ${item1} or ${item2} is bad" @@ -1496,32 +1790,36 @@ function __mcdl_diff_deps() { local t1=${item1:0:1} v1=${item1:2} case ${t1} in 'l') - local use_str - __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' - diff_report_append local_dr "dropped license ${v1@Q}${use_str}" - unset use_str + if ! __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" old_iuse_stack; then + local use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "dropped license ${v1@Q}${use_str}" + unset use_str + fi ;; 'p') - local p_str use_str - __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' - pds_to_string "${v1}" p_str - local -n p_ref=${v1} - local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} - unset -n p_ref - local what='' - case ${p_blocks} in - "${PDS_NO_BLOCK}") - what='dependency' - ;; - "${PDS_WEAK_BLOCK}") - what='weak blocker' - ;; - "${PDS_STRONG_BLOCK}") - what='strong blocker' - ;; - esac - diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" - unset what p_blocks use_str p_str + if ! __mcdl_is_pds_irrelevant "${diff_lib_filters_var_name}" "${v1}" old_iuse_stack; then + local p_str use_str + __mcdl_iuse_stack_to_string_ps old_iuse_stack use_str ' for USE ' '' + pds_to_string "${v1}" p_str + local -n p_ref=${v1} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "dropped a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + fi ;; 'i') # This will be stored in prev item and used in 'o' @@ -1561,32 +1859,36 @@ function __mcdl_diff_deps() { local t2=${item2:0:1} v2=${item2:2} case ${t2} in 'l') - local use_str - __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' - diff_report_append local_dr "added license ${v2@Q}${use_str}" - unset use_str + if ! __mcdl_is_iuse_stack_irrelevant "${diff_lib_filters_var_name}" new_iuse_stack; then + local use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + diff_report_append local_dr "added license ${v2@Q}${use_str}" + unset use_str + fi ;; 'p') - local p_str use_str - __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' - pds_to_string "${v2}" p_str - local -n p_ref=${v2} - local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} - unset -n p_ref - local what='' - case ${p_blocks} in - "${PDS_NO_BLOCK}") - what='dependency' - ;; - "${PDS_WEAK_BLOCK}") - what='weak blocker' - ;; - "${PDS_STRONG_BLOCK}") - what='strong blocker' - ;; - esac - diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" - unset what p_blocks use_str p_str + if ! __mcdl_is_pds_irrelevant "${diff_lib_filters_var_name}" "${v2}" new_iuse_stack; then + local p_str use_str + __mcdl_iuse_stack_to_string_ps new_iuse_stack use_str ' for USE ' '' + pds_to_string "${v2}" p_str + local -n p_ref=${v2} + local -i p_blocks=${p_ref[PDS_BLOCKS_IDX]} + unset -n p_ref + local what='' + case ${p_blocks} in + "${PDS_NO_BLOCK}") + what='dependency' + ;; + "${PDS_WEAK_BLOCK}") + what='weak blocker' + ;; + "${PDS_STRONG_BLOCK}") + what='strong blocker' + ;; + esac + diff_report_append local_dr "added a ${what} ${p_str@Q}${use_str}" + unset what p_blocks use_str p_str + fi ;; 'i') # This will be stored in prev item and used in 'o' diff --git a/pkg_auto/impl/pkg_auto_lib.sh b/pkg_auto/impl/pkg_auto_lib.sh index 4958e445ba..ee229bdab1 100644 --- a/pkg_auto/impl/pkg_auto_lib.sh +++ b/pkg_auto/impl/pkg_auto_lib.sh @@ -1936,7 +1936,12 @@ function read_package_sources() { # # BOM_PKG_SOURCES_MAP_IDX - mapping of package name to the repository # name -declare -gri BOM_PKG_TO_TAGS_MVM_IDX=0 BOM_PKG_SLOTS_SET_MVM_IDX=1 BOM_OLD_PKG_SLOT_VERMINMAX_MAP_MVM_IDX=2 BOM_NEW_PKG_SLOT_VERMINMAX_MAP_MVM_IDX=3 BOM_PKG_SOURCES_MAP_IDX=4 +# +# BOM_DIFF_LIB_FILTERS_IDX - filtering information used to skip +# irrelevant parts when processing +# md5-cache entries for automated change +# reporting +declare -gri BOM_PKG_TO_TAGS_MVM_IDX=0 BOM_PKG_SLOTS_SET_MVM_IDX=1 BOM_OLD_PKG_SLOT_VERMINMAX_MAP_MVM_IDX=2 BOM_NEW_PKG_SLOT_VERMINMAX_MAP_MVM_IDX=3 BOM_PKG_SOURCES_MAP_IDX=4 BOM_DIFF_LIB_FILTERS_IDX=5 # Declare bunch of maps variables. # @@ -1944,7 +1949,7 @@ declare -gri BOM_PKG_TO_TAGS_MVM_IDX=0 BOM_PKG_SLOTS_SET_MVM_IDX=1 BOM_OLD_PKG_S # # @ - names of variables to be used for bunch of maps function bunch_of_maps_declare() { - struct_declare -ga "${@}" "( '' '' '' '' '' )" + struct_declare -ga "${@}" "( '' '' '' '' '' '' )" } # Unset bunch of maps variables. @@ -2100,6 +2105,7 @@ function handle_one_package_change() { local old_pkg_slot_verminmax_map_mvm_var_name=${bunch_of_maps_ref[BOM_OLD_PKG_SLOT_VERMINMAX_MAP_MVM_IDX]} local new_pkg_slot_verminmax_map_mvm_var_name=${bunch_of_maps_ref[BOM_NEW_PKG_SLOT_VERMINMAX_MAP_MVM_IDX]} local -n pkg_sources_map_ref=${bunch_of_maps_ref[BOM_PKG_SOURCES_MAP_IDX]} + local diff_lib_filters_var_name=${bunch_of_maps_ref[BOM_DIFF_LIB_FILTERS_IDX]} # The function goes over a pair of old and new package names. For # each name there will be some checks done (like does this package @@ -2260,12 +2266,12 @@ function handle_one_package_change() { gentoo_ver_cmp_out "${new_version}" "${old_version}" hopc_cmp_result case ${hopc_cmp_result} in "${GV_GT}") - handle_pkg_update hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${old_version}" "${new_version}" + handle_pkg_update hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${diff_lib_filters_var_name}" "${old_name}" "${new_name}" "${old_version}" "${new_version}" hopc_changed=x ;; "${GV_EQ}") hopc_slot_changed= - handle_pkg_as_is hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${old_version}" hopc_slot_changed + handle_pkg_as_is hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${diff_lib_filters_var_name}" "${old_name}" "${new_name}" "${old_version}" hopc_slot_changed if [[ -z ${hopc_slot_changed} ]]; then rm -rf "${update_dir}" else @@ -2273,7 +2279,7 @@ function handle_one_package_change() { fi ;; "${GV_LT}") - handle_pkg_downgrade hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" "${new_version}" + handle_pkg_downgrade hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${diff_lib_filters_var_name}" "${old_name}" "${new_name}" "${s}" "${s}" "${old_version}" "${new_version}" hopc_changed=x ;; esac @@ -2307,12 +2313,12 @@ function handle_one_package_change() { gentoo_ver_cmp_out "${new_version}" "${old_version}" hopc_cmp_result case ${hopc_cmp_result} in "${GV_GT}") - handle_pkg_update hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${old_version}" "${new_version}" + handle_pkg_update hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${diff_lib_filters_var_name}" "${old_name}" "${new_name}" "${old_version}" "${new_version}" hopc_changed=x ;; "${GV_EQ}") hopc_slot_changed= - handle_pkg_as_is hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${old_version}" hopc_slot_changed + handle_pkg_as_is hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${diff_lib_filters_var_name}" "${old_name}" "${new_name}" "${old_version}" hopc_slot_changed if [[ -z ${hopc_slot_changed} ]]; then rm -rf "${update_dir}" else @@ -2320,7 +2326,7 @@ function handle_one_package_change() { fi ;; "${GV_LT}") - handle_pkg_downgrade hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${old_name}" "${new_name}" "${hopc_old_s}" "${hopc_new_s}" "${old_version}" "${new_version}" + handle_pkg_downgrade hopc_package_output_paths "${pkg_to_tags_mvm_var_name}" "${diff_lib_filters_var_name}" "${old_name}" "${new_name}" "${hopc_old_s}" "${hopc_new_s}" "${old_version}" "${new_version}" hopc_changed=x ;; esac @@ -2368,6 +2374,115 @@ function handle_one_package_change() { pkg_debug_disable } +function get_diff_lib_filters() { + local old_pkg_slot_verminmax_map_mvm_name=${1}; shift + local new_pkg_slot_verminmax_map_mvm_name=${1}; shift + local -n filter_name_ref=${1}; shift + + # {old,new}_pkg_slot_verminmax_map_mvm_name are + # map[package]map[slot]string (string being "min version:max + # version") + + local gdlf_name + gen_varname gdlf_name + diff_lib_filters_declare "${gdlf_name}" + + local gdlf_llvm_slots_name gdlf_python_slots_name gdlf_rust_slots_name gdlf_llvm_targets_name + gen_varname gdlf_llvm_slots_name + gen_varname gdlf_python_slots_name + gen_varname gdlf_rust_slots_name + gen_varname gdlf_llvm_targets_name + + local -A llvm_slots_set=() python_slots_set=() rust_slots_set=() + + local gdlf_slots_verminmax_map_name REPLY slot verminmax version + for which in "${WHICH[@]}"; do + local -n portage_stable_path_ref=${which^^}_PORTAGE_STABLE + # get the newest stable llvm slot from llvm eclasses in portage-stable + while read -r; do + llvm_slots_set["${REPLY}"]=x + done < <(grep --extended-regexp --line-regexp '_LLVM_NEWEST_STABLE=[0-9]+' "${portage_stable_path_ref}"/eclass/llvm-r*.eclass | sed 's/.*=\([0-9]\+\)/\1/') + unset -n portage_stable_path_ref + + local -n the_mvm_name_ref=${which}_pkg_slot_verminmax_map_mvm_name + # get used python slot from our reports + mvm_get "${the_mvm_name_ref}" dev-lang/python gdlf_slots_verminmax_map_name + local -n slots_map=${gdlf_slots_verminmax_map_name} + for slot in "${!slots_map[@]}"; do + python_slots_set["${slot}"]=x + done + unset -n slots_map + + # get rust slot - we deduce it from dev-lang/rust-common + # version, because dev-lang/rust in reports tends to stay on + # the old slot for some reason, while in reality, during + # rebuild, a new slot matching the version of + # dev-lang/rust-common will be picked up + mvm_get "${the_mvm_name_ref}" dev-lang/rust-common gdlf_slots_verminmax_map_name + local -n slots_map=${gdlf_slots_verminmax_map_name} + # there should be just one slot, "0", and one verminmax + for verminmax in "${slots_map[@]}"; do + for version in "${verminmax#:*}" "${verminmax%:*}"; do + # strip all the extra stuff from the version to get something resembling a slot + version=${version%%-*} + version=${version%%_*} + rust_slots_set["${version}"]=x + done + done + unset -n slots_map + + unset -n the_mvm_name_ref + done + + # this exists so we can clearly bail out when we add a new arch + # (like riscv) + local -A arch_to_target_map=( + ["amd64"]='X86' + ["arm64"]='AArch64' + ) + local arch + for arch in "${ARCHES[@]}"; do + if [[ -z ${arch_to_target_map["${arch}"]:-} ]]; then + fail "no LLVM target for arch ${arch} specified" + fi + done + + pkg_debug_lines \ + "Slots used for diff filtering:" \ + "LLVM: ${!llvm_slots_set[*]}" \ + "python: ${!python_slots_set[*]}" \ + "rust: ${!rust_slots_set[*]}" \ + "LLVM targets: ${arch_to_target_map[*]}" + + local -n array_ref=${gdlf_llvm_slots_name} + array_ref=( "${!llvm_slots_set[@]}" ) + unset -n array_ref + + local -n array_ref=${gdlf_python_slots_name} + array_ref=( "${!python_slots_set[@]}" ) + unset -n array_ref + + local -n array_ref=${gdlf_rust_slots_name} + array_ref=( "${!rust_slots_set[@]}" ) + unset -n array_ref + + local -n array_ref=${gdlf_llvm_targets_name} + array_ref=( "${arch_to_target_map[@]}" ) + unset -n array_ref + + local -n filters_ref=${gdlf_name} + filters_ref[DLF_LLVM_SLOTS_IDX]=${gdlf_llvm_slots_name} + filters_ref[DLF_PYTHON_SLOTS_IDX]=${gdlf_python_slots_name} + filters_ref[DLF_RUST_SLOTS_IDX]=${gdlf_rust_slots_name} + filters_ref[DLF_LLVM_TARGETS_IDX]=${gdlf_llvm_targets_name} + unset -n filters_ref + + diff_lib_generate_maps "${gdlf_name}" + + filter_name_ref=${gdlf_name} +} + + # Reads the reports, does consistency checks, runs jobs to process all # the packages, and writes out reports into the reports directory. # @@ -2386,7 +2501,7 @@ function handle_package_changes() { local -a hpc_all_pkgs hpc_all_pkgs=() - # map[package]map[slot]interface{} + # map[package]map[slot]struct{} mvm_declare hpc_pkg_slots_set_mvm mvm_mvc_set read_reports hpc_all_pkgs hpc_pkg_slots_set_mvm @@ -2487,12 +2602,16 @@ function handle_package_changes() { done unset added_pkg_to_index_map + local hpc_diff_lib_filters_name + get_diff_lib_filters hpc_old_pkg_slot_verminmax_map_mvm hpc_new_pkg_slot_verminmax_map_mvm hpc_diff_lib_filters_name + bunch_of_maps_declare hpc_bunch_of_maps hpc_bunch_of_maps[BOM_PKG_TO_TAGS_MVM_IDX]=${pkg_to_tags_mvm_var_name} hpc_bunch_of_maps[BOM_PKG_SLOTS_SET_MVM_IDX]=hpc_pkg_slots_set_mvm hpc_bunch_of_maps[BOM_OLD_PKG_SLOT_VERMINMAX_MAP_MVM_IDX]=hpc_old_pkg_slot_verminmax_map_mvm hpc_bunch_of_maps[BOM_NEW_PKG_SLOT_VERMINMAX_MAP_MVM_IDX]=hpc_new_pkg_slot_verminmax_map_mvm hpc_bunch_of_maps[BOM_PKG_SOURCES_MAP_IDX]=hpc_package_sources_map + hpc_bunch_of_maps[BOM_DIFF_LIB_FILTERS_IDX]="${hpc_diff_lib_filters_name}" # We will be spawning as many jobs below as there are available # processors/cores. Each job has its own work directory and will @@ -2646,6 +2765,8 @@ function handle_package_changes() { pkg_job_state_unset "${pkg_job_state_names[@]}" bunch_of_maps_unset hpc_bunch_of_maps + diff_lib_filters_unset "${hpc_diff_lib_filters_name}" + mvm_unset hpc_new_pkg_slot_verminmax_map_mvm mvm_unset hpc_old_pkg_slot_verminmax_map_mvm mvm_unset hpc_pkg_slots_set_mvm @@ -2690,6 +2811,7 @@ function get_first_from_set() { function handle_pkg_update() { local -n package_output_paths_ref=${1}; shift local pkg_to_tags_mvm_var_name=${1}; shift + local diff_lib_filters_var_name=${1}; shift local old_pkg=${1}; shift local new_pkg=${1}; shift local old=${1}; shift @@ -2712,7 +2834,7 @@ function handle_pkg_update() { local diff_report_name gen_varname diff_report_name diff_report_declare "${diff_report_name}" - generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" + generate_cache_diff_report "${diff_report_name}" "${diff_lib_filters_var_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" local -n diff_report_ref=${diff_report_name} local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} @@ -2764,6 +2886,7 @@ function handle_pkg_update() { function handle_pkg_as_is() { local -n package_output_paths_ref=${1}; shift local pkg_to_tags_mvm_var_name=${1}; shift + local diff_lib_filters_var_name=${1}; shift local old_pkg=${1}; shift local new_pkg=${1}; shift local v=${1}; shift @@ -2789,7 +2912,7 @@ function handle_pkg_as_is() { local diff_report_name gen_varname diff_report_name diff_report_declare "${diff_report_name}" - generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${v}" "${v}" + generate_cache_diff_report "${diff_report_name}" "${diff_lib_filters_var_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${v}" "${v}" local -n diff_report_ref=${diff_report_name} local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} @@ -2842,6 +2965,7 @@ function handle_pkg_as_is() { function handle_pkg_downgrade() { local -n package_output_paths_ref=${1}; shift local pkg_to_tags_mvm_var_name=${1}; shift + local diff_lib_filters_var_name=${1}; shift local old_pkg=${1}; shift local new_pkg=${1}; shift local old=${1}; shift @@ -2865,7 +2989,7 @@ function handle_pkg_downgrade() { local diff_report_name gen_varname diff_report_name diff_report_declare "${diff_report_name}" - generate_cache_diff_report "${diff_report_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" + generate_cache_diff_report "${diff_report_name}" "${diff_lib_filters_var_name}" "${WORKDIR}/pkg-reports/old/portage-stable-cache" "${WORKDIR}/pkg-reports/new/portage-stable-cache" "${old_pkg}" "${new_pkg}" "${old}" "${new}" local -n diff_report_ref=${diff_report_name} local -n diff_lines_ref=${diff_report_ref[${DR_LINES_IDX}]} @@ -3098,6 +3222,7 @@ function generate_ebuild_diff() { function generate_cache_diff_report() { local diff_report_var_name=${1}; shift + local diff_lib_filters_var_name=${1}; shift local old_cache_dir=${1}; shift local new_cache_dir=${1}; shift local old_pkg=${1}; shift @@ -3118,7 +3243,7 @@ function generate_cache_diff_report() { parse_cache_file "${old_cache_name}" "${old_entry}" "${ARCHES[@]}" parse_cache_file "${new_cache_name}" "${new_entry}" "${ARCHES[@]}" - diff_cache_data "${old_cache_name}" "${new_cache_name}" "${diff_report_var_name}" + diff_cache_data "${old_cache_name}" "${new_cache_name}" "${diff_lib_filters_var_name}" "${diff_report_var_name}" cache_file_unset "${old_cache_name}" "${new_cache_name}" }