mirror of
https://github.com/armbian/build.git
synced 2025-08-09 20:56:57 +02:00
- WiP: Python patching delusion, pt 1: finding & parsing patches; apply & git commit with pygit2; Markdown summaries (also for aggregation); git-to-patches tool - Python: Markdown aggregation and patching summaries; collapsible; SummarizedMarkdownWriter - Python: Markdown aggregation and patching summaries - Python: reorg a bit into common/armbian_utils; define the `ASSET_LOG_BASE` in preparation for Markdown delusion - Python patching: initial apply patches & initial commit patches to git (using pygit2) - Python patching: add basic `series.conf` support - Python patching: force use of utf-8; better error handling; use realpath of dirs - Python patching: `git-to-patches` initial hack. not proud. half-reused some of the patches-to-git - Python patching: "tag" the git commits with info for extracting later; introduce REWRITE_PATCHES/rewrite_patches_in_place - Python patching: commented-out, recover-bad-patches hacks - Python patching: shorten the signature - Python patching: allow BASE_GIT_TAG as well as BASE_GIT_REVISION - Python patching: git-archeology for patches missing descriptions; avoid UTF-8 in header/desc (not diff) - Python patching: use modern-er email.utils.parsedate_to_datetime to parse commit date - Python patching: unify PatchInPatchFile; better git-commiting; re-exporting patches from Git (directly) - Python patching: switch to GitPython - GitPython is like 100x slower than pygit2, but actually allows for date & committer - also allows to remove untracked files before starting - Python aggregation: fix missing `AGGREGATED_APT_SOURCES_DICT` - Python patching: add `unidecode` dependency to pip3 install - Python patching: don't try archeology if SRC is not a Git Repo (eg, in Docker) - Python patching: don't try archeology if not applying patches to git - WiP: Python patching delusion, pt2: actually use for u-boot & kernel patching - Python patching: much better problem handling/logging; lenient with recreations (kernel) - Python patching: don't force SHOW_LOG for u-boot patching - Python patching: don't bomb for no reason when there are no patches to apply - Python patching: fully (?) switch kernel patching to Python - Python patching: more logging fixups - Python patching: capture `kernel_git_revision` from `fetch_from_repo()`'s `checked_out_revision` - Python patching: fully switch u-boot patching to Python - Python aggregation/patching: colored logging; patching: always reset to git revision - Python aggregation/patching: better logging; introduce u-boot Python patching - Python patching pt3: recovers and better Markdown - Python patching: detect, and rescue, `wrong_strip_level` problem; don't try to export patches that didn't apply, bitch instead - Python patching: Markdown patching summary table, complete with emoji - Python patching: include the problem breakdown in Markdown summary - Python patching: sanity check against half-bare, half-mbox patches - Python patching: try to recover from 1) bad utf-8 encoded patches; 2) bad unidiff patches; add a few sanity checks - Python patching: try, and fail, to apply badly utf-8 encoded patches directly as bytes [reverted] - Python patching: try to recover from patch *parse* failures; show summary; better logging - set `GIT_ARCHEOLOGY=yes` to do archeology, default not - armbian-next: Python `pip` dependencies handling, similar to `hostdeps` - same scheme for Dockerfile caching - @TODO: still using global/shared environment; should move to a dir under `cache` or some kinda venv - WiP: add `python3-pip` to hostdeps; remove `python-setuptools` - remove `python-setuptools` (Python2, no longer exists in Sid) from hostdeps - add `python3-pip` to hostdeps; part of virtualenv saga - WiP: split `kernel.sh` a bit, into `kernel-patching.sh`, `kernel-config.sh` and `kernel-make.sh` - `advanced_patch()`: rename vars for clarity; no real changes - Python patching: introduce FAST_ARCHEOLOGY; still trying for Markdown links
192 lines
8.0 KiB
Bash
192 lines
8.0 KiB
Bash
#!/usr/bin/env bash
|
|
# advanced_patch <patch_kind> <{patch_dir}> <board> <target> <branch> <description>
|
|
#
|
|
# parameters:
|
|
# <patch_kind>: u-boot, kernel, atf
|
|
# <{patch_dir}>: u-boot: u-boot, u-boot-neo; kernel: sun4i-default, sunxi-next, ...
|
|
# <board>: cubieboard, cubieboard2, cubietruck, ...
|
|
# <target>: optional subdirectory
|
|
# <description>: additional description text
|
|
|
|
# calls:
|
|
# ${patch_kind} ${patch_dir} $board $target $branch $description
|
|
# kernel: advanced_patch "kernel" "$KERNELPATCHDIR" "$BOARD" "" "$BRANCH" "$LINUXFAMILY-$BRANCH"
|
|
# u-boot: advanced_patch "u-boot" "$BOOTPATCHDIR" "$BOARD" "$target_patchdir" "$BRANCH" "${LINUXFAMILY}-${BOARD}-${BRANCH}"
|
|
function advanced_patch() {
|
|
local patch_kind="$1"
|
|
local patch_dir="$2"
|
|
local board="$3"
|
|
local target="$4"
|
|
local branch="$5"
|
|
local description="$6"
|
|
|
|
display_alert "Started patching process for" "${patch_kind} $description" "info"
|
|
display_alert "Looking for user patches in" "userpatches/${patch_kind}/${patch_dir}" "info"
|
|
|
|
local names=()
|
|
local dirs=(
|
|
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}/target_${target}:[\e[33mu\e[0m][\e[34mt\e[0m]"
|
|
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}/board_${board}:[\e[33mu\e[0m][\e[35mb\e[0m]"
|
|
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}/branch_${branch}:[\e[33mu\e[0m][\e[33mb\e[0m]"
|
|
"$USERPATCHES_PATH/${patch_kind}/${patch_dir}:[\e[33mu\e[0m][\e[32mc\e[0m]"
|
|
|
|
"$SRC/patch/${patch_kind}/${patch_dir}/target_${target}:[\e[32ml\e[0m][\e[34mt\e[0m]" # used for u-boot "spi" stuff
|
|
"$SRC/patch/${patch_kind}/${patch_dir}/board_${board}:[\e[32ml\e[0m][\e[35mb\e[0m]" # used for u-boot board-specific stuff
|
|
"$SRC/patch/${patch_kind}/${patch_dir}/branch_${branch}:[\e[32ml\e[0m][\e[33mb\e[0m]" # NOT used, I think.
|
|
"$SRC/patch/${patch_kind}/${patch_dir}:[\e[32ml\e[0m][\e[32mc\e[0m]" # used for everything
|
|
)
|
|
local links=()
|
|
|
|
# required for "for" command
|
|
# @TODO these shopts leak for the rest of the build script! either make global, or restore them after this function
|
|
shopt -s nullglob dotglob
|
|
# get patch file names
|
|
for dir in "${dirs[@]}"; do
|
|
for patch in ${dir%%:*}/*.patch; do
|
|
names+=($(basename "${patch}"))
|
|
done
|
|
# add linked patch directories
|
|
if [[ -d ${dir%%:*} ]]; then
|
|
local findlinks
|
|
findlinks=$(find "${dir%%:*}" -maxdepth 1 -type l -print0 2>&1 | xargs -0)
|
|
[[ -n $findlinks ]] && readarray -d '' links < <(find "${findlinks}" -maxdepth 1 -type f -follow -print -iname "*.patch" -print | grep "\.patch$" | sed "s|${dir%%:*}/||g" 2>&1)
|
|
fi
|
|
done
|
|
|
|
# merge static and linked
|
|
names=("${names[@]}" "${links[@]}")
|
|
# remove duplicates
|
|
local names_s=($(echo "${names[@]}" | tr ' ' '\n' | LC_ALL=C sort -u | tr '\n' ' '))
|
|
# apply patches
|
|
for name in "${names_s[@]}"; do
|
|
for dir in "${dirs[@]}"; do
|
|
if [[ -f ${dir%%:*}/$name ]]; then
|
|
if [[ -s ${dir%%:*}/$name ]]; then
|
|
process_patch_file "${dir%%:*}/$name" "${dir##*:}"
|
|
else
|
|
display_alert "* ${dir##*:} $name" "skipped"
|
|
fi
|
|
break # next name
|
|
fi
|
|
done
|
|
done
|
|
}
|
|
|
|
# process_patch_file <file> <description>
|
|
#
|
|
# parameters:
|
|
# <file>: path to patch file
|
|
# <status>: additional status text
|
|
#
|
|
process_patch_file() {
|
|
local patch="${1}"
|
|
local status="${2}"
|
|
local -i patch_date
|
|
local relative_patch="${patch##"${SRC}"/}" # ${FOO##prefix} remove prefix from FOO
|
|
|
|
# report_fashtash_should_execute is report_fasthash returns true only if we're supposed to apply the patch on disk.
|
|
if report_fashtash_should_execute file "${patch}" "Apply patch ${relative_patch}"; then
|
|
|
|
# get the modification date of the patch. make it not less than MIN_PATCH_AGE, if set.
|
|
patch_date=$(get_file_modification_time "${patch}")
|
|
# shellcheck disable=SC2154 # patch_minimum_target_mtime can be declared in outer scope
|
|
if [[ "${patch_minimum_target_mtime}" != "" ]]; then
|
|
if [[ ${patch_date} -lt ${patch_minimum_target_mtime} ]]; then
|
|
display_alert "Patch before minimum date" "${patch_date} -lt ${patch_minimum_target_mtime}" "timestamp"
|
|
patch_date=${patch_minimum_target_mtime}
|
|
fi
|
|
fi
|
|
|
|
# detect and remove files which patch will create
|
|
lsdiff -s --strip=1 "${patch}" | grep '^+' | awk '{print $2}' | xargs -I % sh -c 'rm -f %'
|
|
|
|
# store an array of the files that patch will add or modify, we'll set their modification times after the fact
|
|
declare -a patched_files
|
|
mapfile -t patched_files < <(lsdiff -s --strip=1 "${patch}" | grep -e '^+' -e '^!' | awk '{print $2}')
|
|
|
|
# @TODO: try patching with `git am` first, so git contains the patch commit info/msg. -- For future git-based hashing.
|
|
# shellcheck disable=SC2015 # noted, thanks. I need to handle exit code here.
|
|
patch --batch -p1 -N --input="${patch}" --quiet --reject-file=- && { # "-" discards rejects
|
|
# Fix the dates on the patched files
|
|
set_files_modification_time "${patch_date}" "${patched_files[@]}"
|
|
display_alert "* $status ${relative_patch}" "" "info"
|
|
} || {
|
|
display_alert "* $status ${relative_patch}" "failed" "wrn"
|
|
[[ $EXIT_PATCHING_ERROR == yes ]] && exit_with_error "Aborting due to" "EXIT_PATCHING_ERROR"
|
|
}
|
|
mark_fasthash_done # will do git commit, associate fasthash to real hash.
|
|
fi
|
|
|
|
return 0 # short-circuit above, avoid exiting with error
|
|
}
|
|
|
|
# apply_patch_series <target dir> <full path to series_file_full_path file>
|
|
apply_patch_series() {
|
|
local target_dir="${1}"
|
|
local series_file_full_path="${2}"
|
|
local included_list skip_list skip_count counter=1 base_dir
|
|
base_dir="$(dirname "${series_file_full_path}")"
|
|
included_list="$(awk '$0 !~ /^#.*|^-.*|^$/' "${series_file_full_path}")"
|
|
included_count=$(echo -n "${included_list}" | wc -w)
|
|
skip_list="$(awk '$0 ~ /^-.*/{print $NF}' "${series_file_full_path}")"
|
|
skip_count=$(echo -n "${skip_list}" | wc -w)
|
|
display_alert "apply a series of " "[$(echo -n "$included_list" | wc -w)] patches" "info"
|
|
[[ ${skip_count} -gt 0 ]] && display_alert "skipping" "[${skip_count}] patches" "warn"
|
|
cd "${target_dir}" || exit 1
|
|
|
|
for p in $included_list; do
|
|
process_patch_file "${base_dir}/${p}" "${counter}/${included_count}"
|
|
counter=$((counter + 1))
|
|
done
|
|
display_alert "done applying patch series " "[$(echo -n "$included_list" | wc -w)] patches" "info"
|
|
}
|
|
|
|
userpatch_create() {
|
|
# create commit to start from clean source
|
|
git add .
|
|
git -c user.name='Armbian User' -c user.email='user@example.org' commit -q -m "Cleaning working copy"
|
|
|
|
mkdir -p "${DEST}/patch"
|
|
local patch="$DEST/patch/$1-$LINUXFAMILY-$BRANCH.patch"
|
|
|
|
# apply previous user debug mode created patches
|
|
if [[ -f $patch ]]; then
|
|
display_alert "Applying existing $1 patch" "$patch" "wrn" && patch --batch --silent -p1 -N < "${patch}"
|
|
# read title of a patch in case Git is configured
|
|
if [[ -n $(git config user.email) ]]; then
|
|
COMMIT_MESSAGE=$(cat "${patch}" | grep Subject | sed -n -e '0,/PATCH/s/.*PATCH]//p' | xargs)
|
|
display_alert "Patch name extracted" "$COMMIT_MESSAGE" "wrn"
|
|
fi
|
|
fi
|
|
|
|
# prompt to alter source
|
|
display_alert "Make your changes in this directory:" "$(pwd)" "wrn"
|
|
display_alert "Press <Enter> after you are done" "waiting" "wrn"
|
|
read -r < /dev/tty
|
|
tput cuu1
|
|
git add .
|
|
# create patch out of changes
|
|
if ! git diff-index --quiet --cached HEAD; then
|
|
# If Git is configured, create proper patch and ask for a name
|
|
if [[ -n $(git config user.email) ]]; then
|
|
display_alert "Add / change patch name" "$COMMIT_MESSAGE" "wrn"
|
|
read -e -p "Patch description: " -i "$COMMIT_MESSAGE" COMMIT_MESSAGE
|
|
[[ -z "$COMMIT_MESSAGE" ]] && COMMIT_MESSAGE="Patching something"
|
|
git commit -s -m "$COMMIT_MESSAGE"
|
|
git format-patch -1 HEAD --stdout --signature="Created with Armbian build tools https://github.com/armbian/build" > "${patch}"
|
|
PATCHFILE=$(git format-patch -1 HEAD)
|
|
rm $PATCHFILE # delete the actual file
|
|
# create a symlink to have a nice name ready
|
|
find $DEST/patch/ -type l -delete # delete any existing
|
|
ln -sf $patch $DEST/patch/$PATCHFILE
|
|
else
|
|
git diff --staged > "${patch}"
|
|
fi
|
|
display_alert "You will find your patch here:" "$patch" "info"
|
|
else
|
|
display_alert "No changes found, skipping patch creation" "" "wrn"
|
|
fi
|
|
git reset --soft HEAD~
|
|
for i in {3..1..1}; do echo -n "$i." && sleep 1; done
|
|
}
|