armbian_build/lib/functions/compilation/patch/patching.sh
Ricardo Pardini 76e276c6a9
armbian-next: Python patching delusion, pt1 & pt2 & pt3
- WiP: Python patching delusion, pt 1: finding & parsing patches; apply & git commit with pygit2; Markdown summaries (also for aggregation); git-to-patches tool
  - Python: Markdown aggregation and patching summaries; collapsible; SummarizedMarkdownWriter
  - Python: Markdown aggregation and patching summaries
  - Python: reorg a bit into common/armbian_utils; define the `ASSET_LOG_BASE` in preparation for Markdown delusion
  - Python patching: initial apply patches & initial commit patches to git (using pygit2)
  - Python patching: add basic `series.conf` support
  - Python patching: force use of utf-8; better error handling; use realpath of dirs
  - Python patching: `git-to-patches` initial hack. not proud. half-reused some of the patches-to-git
  - Python patching: "tag" the git commits with info for extracting later; introduce REWRITE_PATCHES/rewrite_patches_in_place
  - Python patching: commented-out, recover-bad-patches hacks
  - Python patching: shorten the signature
  - Python patching: allow BASE_GIT_TAG as well as BASE_GIT_REVISION
  - Python patching: git-archeology for patches missing descriptions; avoid UTF-8 in header/desc (not diff)
  - Python patching: use modern-er email.utils.parsedate_to_datetime to parse commit date
  - Python patching: unify PatchInPatchFile; better git-commiting; re-exporting patches from Git (directly)
  - Python patching: switch to GitPython
    - GitPython is like 100x slower than pygit2, but actually allows for date & committer
    - also allows to remove untracked files before starting
  - Python aggregation: fix missing `AGGREGATED_APT_SOURCES_DICT`
  - Python patching: add `unidecode` dependency to pip3 install
  - Python patching: don't try archeology if SRC is not a Git Repo (eg, in Docker)
  - Python patching: don't try archeology if not applying patches to git
- WiP: Python patching delusion, pt2: actually use for u-boot & kernel patching
  - Python patching: much better problem handling/logging; lenient with recreations (kernel)
  - Python patching: don't force SHOW_LOG for u-boot patching
  - Python patching: don't bomb for no reason when there are no patches to apply
  - Python patching: fully (?) switch kernel patching to Python
  - Python patching: more logging fixups
  - Python patching: capture `kernel_git_revision` from `fetch_from_repo()`'s `checked_out_revision`
  - Python patching: fully switch u-boot patching to Python
  - Python aggregation/patching: colored logging; patching: always reset to git revision
  - Python aggregation/patching: better logging; introduce u-boot Python patching
- Python patching pt3: recovers and better Markdown
  - Python patching: detect, and rescue, `wrong_strip_level` problem; don't try to export patches that didn't apply, bitch instead
  - Python patching: Markdown patching summary table, complete with emoji
  - Python patching: include the problem breakdown in Markdown summary
  - Python patching: sanity check against half-bare, half-mbox patches
  - Python patching: try to recover from 1) bad utf-8 encoded patches; 2) bad unidiff patches; add a few sanity checks
  - Python patching: try, and fail, to apply badly utf-8 encoded patches directly as bytes [reverted]
  - Python patching: try to recover from patch *parse* failures; show summary; better logging
      - set `GIT_ARCHEOLOGY=yes` to do archeology, default not

- armbian-next: Python `pip` dependencies handling, similar to `hostdeps`
  - same scheme for Dockerfile caching
  - @TODO: still using global/shared environment; should move to a dir under `cache` or some kinda venv
- WiP: add `python3-pip` to hostdeps; remove `python-setuptools`
  - remove `python-setuptools` (Python2, no longer exists in Sid) from hostdeps
  - add `python3-pip` to hostdeps; part of virtualenv saga
- WiP: split `kernel.sh` a bit, into `kernel-patching.sh`, `kernel-config.sh` and `kernel-make.sh`
  - `advanced_patch()`: rename vars for clarity; no real changes
- Python patching: introduce FAST_ARCHEOLOGY; still trying for Markdown links
2023-02-18 07:40:52 -03:00

192 lines
8.0 KiB
Bash

#!/usr/bin/env bash
# advanced_patch <patch_kind> <{patch_dir}> <board> <target> <branch> <description>
#
# parameters:
# <patch_kind>: u-boot, kernel, atf
# <{patch_dir}>: u-boot: u-boot, u-boot-neo; kernel: sun4i-default, sunxi-next, ...
# <board>: cubieboard, cubieboard2, cubietruck, ...
# <target>: optional subdirectory
# <description>: additional description text
# calls:
# ${patch_kind} ${patch_dir} $board $target $branch $description
# kernel: advanced_patch "kernel" "$KERNELPATCHDIR" "$BOARD" "" "$BRANCH" "$LINUXFAMILY-$BRANCH"
# u-boot: advanced_patch "u-boot" "$BOOTPATCHDIR" "$BOARD" "$target_patchdir" "$BRANCH" "${LINUXFAMILY}-${BOARD}-${BRANCH}"
function advanced_patch() {
local patch_kind="$1"
local patch_dir="$2"
local board="$3"
local target="$4"
local branch="$5"
local description="$6"
display_alert "Started patching process for" "${patch_kind} $description" "info"
display_alert "Looking for user patches in" "userpatches/${patch_kind}/${patch_dir}" "info"
local names=()
local dirs=(
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}/target_${target}:[\e[33mu\e[0m][\e[34mt\e[0m]"
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}/board_${board}:[\e[33mu\e[0m][\e[35mb\e[0m]"
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}/branch_${branch}:[\e[33mu\e[0m][\e[33mb\e[0m]"
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}:[\e[33mu\e[0m][\e[32mc\e[0m]"
"$SRC/patch/${patch_kind}/${patch_dir}/target_${target}:[\e[32ml\e[0m][\e[34mt\e[0m]" # used for u-boot "spi" stuff
"$SRC/patch/${patch_kind}/${patch_dir}/board_${board}:[\e[32ml\e[0m][\e[35mb\e[0m]" # used for u-boot board-specific stuff
"$SRC/patch/${patch_kind}/${patch_dir}/branch_${branch}:[\e[32ml\e[0m][\e[33mb\e[0m]" # NOT used, I think.
"$SRC/patch/${patch_kind}/${patch_dir}:[\e[32ml\e[0m][\e[32mc\e[0m]" # used for everything
)
local links=()
# required for "for" command
# @TODO these shopts leak for the rest of the build script! either make global, or restore them after this function
shopt -s nullglob dotglob
# get patch file names
for dir in "${dirs[@]}"; do
for patch in ${dir%%:*}/*.patch; do
names+=($(basename "${patch}"))
done
# add linked patch directories
if [[ -d ${dir%%:*} ]]; then
local findlinks
findlinks=$(find "${dir%%:*}" -maxdepth 1 -type l -print0 2>&1 | xargs -0)
[[ -n $findlinks ]] && readarray -d '' links < <(find "${findlinks}" -maxdepth 1 -type f -follow -print -iname "*.patch" -print | grep "\.patch$" | sed "s|${dir%%:*}/||g" 2>&1)
fi
done
# merge static and linked
names=("${names[@]}" "${links[@]}")
# remove duplicates
local names_s=($(echo "${names[@]}" | tr ' ' '\n' | LC_ALL=C sort -u | tr '\n' ' '))
# apply patches
for name in "${names_s[@]}"; do
for dir in "${dirs[@]}"; do
if [[ -f ${dir%%:*}/$name ]]; then
if [[ -s ${dir%%:*}/$name ]]; then
process_patch_file "${dir%%:*}/$name" "${dir##*:}"
else
display_alert "* ${dir##*:} $name" "skipped"
fi
break # next name
fi
done
done
}
# process_patch_file <file> <description>
#
# parameters:
# <file>: path to patch file
# <status>: additional status text
#
process_patch_file() {
local patch="${1}"
local status="${2}"
local -i patch_date
local relative_patch="${patch##"${SRC}"/}" # ${FOO##prefix} remove prefix from FOO
# report_fashtash_should_execute is report_fasthash returns true only if we're supposed to apply the patch on disk.
if report_fashtash_should_execute file "${patch}" "Apply patch ${relative_patch}"; then
# get the modification date of the patch. make it not less than MIN_PATCH_AGE, if set.
patch_date=$(get_file_modification_time "${patch}")
# shellcheck disable=SC2154 # patch_minimum_target_mtime can be declared in outer scope
if [[ "${patch_minimum_target_mtime}" != "" ]]; then
if [[ ${patch_date} -lt ${patch_minimum_target_mtime} ]]; then
display_alert "Patch before minimum date" "${patch_date} -lt ${patch_minimum_target_mtime}" "timestamp"
patch_date=${patch_minimum_target_mtime}
fi
fi
# detect and remove files which patch will create
lsdiff -s --strip=1 "${patch}" | grep '^+' | awk '{print $2}' | xargs -I % sh -c 'rm -f %'
# store an array of the files that patch will add or modify, we'll set their modification times after the fact
declare -a patched_files
mapfile -t patched_files < <(lsdiff -s --strip=1 "${patch}" | grep -e '^+' -e '^!' | awk '{print $2}')
# @TODO: try patching with `git am` first, so git contains the patch commit info/msg. -- For future git-based hashing.
# shellcheck disable=SC2015 # noted, thanks. I need to handle exit code here.
patch --batch -p1 -N --input="${patch}" --quiet --reject-file=- && { # "-" discards rejects
# Fix the dates on the patched files
set_files_modification_time "${patch_date}" "${patched_files[@]}"
display_alert "* $status ${relative_patch}" "" "info"
} || {
display_alert "* $status ${relative_patch}" "failed" "wrn"
[[ $EXIT_PATCHING_ERROR == yes ]] && exit_with_error "Aborting due to" "EXIT_PATCHING_ERROR"
}
mark_fasthash_done # will do git commit, associate fasthash to real hash.
fi
return 0 # short-circuit above, avoid exiting with error
}
# apply_patch_series <target dir> <full path to series_file_full_path file>
apply_patch_series() {
local target_dir="${1}"
local series_file_full_path="${2}"
local included_list skip_list skip_count counter=1 base_dir
base_dir="$(dirname "${series_file_full_path}")"
included_list="$(awk '$0 !~ /^#.*|^-.*|^$/' "${series_file_full_path}")"
included_count=$(echo -n "${included_list}" | wc -w)
skip_list="$(awk '$0 ~ /^-.*/{print $NF}' "${series_file_full_path}")"
skip_count=$(echo -n "${skip_list}" | wc -w)
display_alert "apply a series of " "[$(echo -n "$included_list" | wc -w)] patches" "info"
[[ ${skip_count} -gt 0 ]] && display_alert "skipping" "[${skip_count}] patches" "warn"
cd "${target_dir}" || exit 1
for p in $included_list; do
process_patch_file "${base_dir}/${p}" "${counter}/${included_count}"
counter=$((counter + 1))
done
display_alert "done applying patch series " "[$(echo -n "$included_list" | wc -w)] patches" "info"
}
userpatch_create() {
# create commit to start from clean source
git add .
git -c user.name='Armbian User' -c user.email='user@example.org' commit -q -m "Cleaning working copy"
mkdir -p "${DEST}/patch"
local patch="$DEST/patch/$1-$LINUXFAMILY-$BRANCH.patch"
# apply previous user debug mode created patches
if [[ -f $patch ]]; then
display_alert "Applying existing $1 patch" "$patch" "wrn" && patch --batch --silent -p1 -N < "${patch}"
# read title of a patch in case Git is configured
if [[ -n $(git config user.email) ]]; then
COMMIT_MESSAGE=$(cat "${patch}" | grep Subject | sed -n -e '0,/PATCH/s/.*PATCH]//p' | xargs)
display_alert "Patch name extracted" "$COMMIT_MESSAGE" "wrn"
fi
fi
# prompt to alter source
display_alert "Make your changes in this directory:" "$(pwd)" "wrn"
display_alert "Press <Enter> after you are done" "waiting" "wrn"
read -r < /dev/tty
tput cuu1
git add .
# create patch out of changes
if ! git diff-index --quiet --cached HEAD; then
# If Git is configured, create proper patch and ask for a name
if [[ -n $(git config user.email) ]]; then
display_alert "Add / change patch name" "$COMMIT_MESSAGE" "wrn"
read -e -p "Patch description: " -i "$COMMIT_MESSAGE" COMMIT_MESSAGE
[[ -z "$COMMIT_MESSAGE" ]] && COMMIT_MESSAGE="Patching something"
git commit -s -m "$COMMIT_MESSAGE"
git format-patch -1 HEAD --stdout --signature="Created with Armbian build tools https://github.com/armbian/build" > "${patch}"
PATCHFILE=$(git format-patch -1 HEAD)
rm $PATCHFILE # delete the actual file
# create a symlink to have a nice name ready
find $DEST/patch/ -type l -delete # delete any existing
ln -sf $patch $DEST/patch/$PATCHFILE
else
git diff --staged > "${patch}"
fi
display_alert "You will find your patch here:" "$patch" "info"
else
display_alert "No changes found, skipping patch creation" "" "wrn"
fi
git reset --soft HEAD~
for i in {3..1..1}; do echo -n "$i." && sleep 1; done
}