armbian_build/lib/functions/compilation/kernel-make.sh
Ricardo Pardini 76e276c6a9
armbian-next: Python patching delusion, pt1 & pt2 & pt3
- WiP: Python patching delusion, pt 1: finding & parsing patches; apply & git commit with pygit2; Markdown summaries (also for aggregation); git-to-patches tool
  - Python: Markdown aggregation and patching summaries; collapsible; SummarizedMarkdownWriter
  - Python: Markdown aggregation and patching summaries
  - Python: reorg a bit into common/armbian_utils; define the `ASSET_LOG_BASE` in preparation for Markdown delusion
  - Python patching: initial apply patches & initial commit patches to git (using pygit2)
  - Python patching: add basic `series.conf` support
  - Python patching: force use of utf-8; better error handling; use realpath of dirs
  - Python patching: `git-to-patches` initial hack. not proud. half-reused some of the patches-to-git
  - Python patching: "tag" the git commits with info for extracting later; introduce REWRITE_PATCHES/rewrite_patches_in_place
  - Python patching: commented-out, recover-bad-patches hacks
  - Python patching: shorten the signature
  - Python patching: allow BASE_GIT_TAG as well as BASE_GIT_REVISION
  - Python patching: git-archeology for patches missing descriptions; avoid UTF-8 in header/desc (not diff)
  - Python patching: use modern-er email.utils.parsedate_to_datetime to parse commit date
  - Python patching: unify PatchInPatchFile; better git-commiting; re-exporting patches from Git (directly)
  - Python patching: switch to GitPython
    - GitPython is like 100x slower than pygit2, but actually allows for date & committer
    - also allows to remove untracked files before starting
  - Python aggregation: fix missing `AGGREGATED_APT_SOURCES_DICT`
  - Python patching: add `unidecode` dependency to pip3 install
  - Python patching: don't try archeology if SRC is not a Git Repo (eg, in Docker)
  - Python patching: don't try archeology if not applying patches to git
- WiP: Python patching delusion, pt2: actually use for u-boot & kernel patching
  - Python patching: much better problem handling/logging; lenient with recreations (kernel)
  - Python patching: don't force SHOW_LOG for u-boot patching
  - Python patching: don't bomb for no reason when there are no patches to apply
  - Python patching: fully (?) switch kernel patching to Python
  - Python patching: more logging fixups
  - Python patching: capture `kernel_git_revision` from `fetch_from_repo()`'s `checked_out_revision`
  - Python patching: fully switch u-boot patching to Python
  - Python aggregation/patching: colored logging; patching: always reset to git revision
  - Python aggregation/patching: better logging; introduce u-boot Python patching
- Python patching pt3: recovers and better Markdown
  - Python patching: detect, and rescue, `wrong_strip_level` problem; don't try to export patches that didn't apply, bitch instead
  - Python patching: Markdown patching summary table, complete with emoji
  - Python patching: include the problem breakdown in Markdown summary
  - Python patching: sanity check against half-bare, half-mbox patches
  - Python patching: try to recover from 1) bad utf-8 encoded patches; 2) bad unidiff patches; add a few sanity checks
  - Python patching: try, and fail, to apply badly utf-8 encoded patches directly as bytes [reverted]
  - Python patching: try to recover from patch *parse* failures; show summary; better logging
      - set `GIT_ARCHEOLOGY=yes` to do archeology, default not

- armbian-next: Python `pip` dependencies handling, similar to `hostdeps`
  - same scheme for Dockerfile caching
  - @TODO: still using global/shared environment; should move to a dir under `cache` or some kinda venv
- WiP: add `python3-pip` to hostdeps; remove `python-setuptools`
  - remove `python-setuptools` (Python2, no longer exists in Sid) from hostdeps
  - add `python3-pip` to hostdeps; part of virtualenv saga
- WiP: split `kernel.sh` a bit, into `kernel-patching.sh`, `kernel-config.sh` and `kernel-make.sh`
  - `advanced_patch()`: rename vars for clarity; no real changes
- Python patching: introduce FAST_ARCHEOLOGY; still trying for Markdown links
2023-02-18 07:40:52 -03:00

68 lines
3.6 KiB
Bash

#!/usr/bin/env bash
function run_kernel_make_internal() {
set -e
declare -a common_make_params_quoted common_make_envs full_command
# Prepare distcc, if enabled.
declare -a -g DISTCC_EXTRA_ENVS=()
declare -a -g DISTCC_CROSS_COMPILE_PREFIX=()
declare -a -g DISTCC_MAKE_J_PARALLEL=()
prepare_distcc_compilation_config
common_make_envs=(
"CCACHE_BASEDIR=\"$(pwd)\"" # Base directory for ccache, for cache reuse # @TODO: experiment with this and the source path to maximize hit rate
"PATH=\"${toolchain}:${PATH}\"" # Insert the toolchain first into the PATH.
"DPKG_COLORS=always" # Use colors for dpkg @TODO no dpkg is done anymore, remove?
"XZ_OPT='--threads=0'" # Use parallel XZ compression
"TERM='${TERM}'" # Pass the terminal type, so that 'make menuconfig' can work.
)
# If CCACHE_DIR is set, pass it to the kernel build; Pass the ccache dir explicitly, since we'll run under "env -i"
if [[ -n "${CCACHE_DIR}" ]]; then
common_make_envs+=("CCACHE_DIR='${CCACHE_DIR}'")
fi
# Add the distcc envs, if any.
common_make_envs+=("${DISTCC_EXTRA_ENVS[@]}")
common_make_params_quoted=(
# @TODO: introduce O=path/to/binaries, so sources and bins are not in the same dir; this has high impact in headers packaging though.
"${DISTCC_MAKE_J_PARALLEL[@]}" # Parallel compile, "-j X" for X cpus; determined by distcc, or is just "$CTHREADS" if distcc is not enabled.
"ARCH=${ARCHITECTURE}" # Key param. Everything depends on this.
"LOCALVERSION=-${LINUXFAMILY}" # Change the internal kernel version to include the family. Changing this causes recompiles # @TODO change to "localversion" file
"CROSS_COMPILE=${CCACHE} ${DISTCC_CROSS_COMPILE_PREFIX[@]} ${KERNEL_COMPILER}" # added as prefix to every compiler invocation by make
"KCFLAGS=-fdiagnostics-color=always -Wno-error=misleading-indentation" # Force GCC colored messages, downgrade misleading indentation to warning
"SOURCE_DATE_EPOCH=${kernel_base_revision_ts}" # https://reproducible-builds.org/docs/source-date-epoch/ and https://www.kernel.org/doc/html/latest/kbuild/reproducible-builds.html
"KBUILD_BUILD_TIMESTAMP=${kernel_base_revision_date}" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-timestamp
"KBUILD_BUILD_USER=armbian" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host
"KBUILD_BUILD_HOST=next" # https://www.kernel.org/doc/html/latest/kbuild/kbuild.html#kbuild-build-user-kbuild-build-host
"KGZIP=pigz" "KBZIP2=pbzip2" # Parallel compression, use explicit parallel compressors https://lore.kernel.org/lkml/20200901151002.988547791@linuxfoundation.org/
)
# last statement, so it passes the result to calling function. "env -i" is used for empty env
full_command=("${KERNEL_MAKE_RUNNER:-run_host_command_logged}" "env" "-i" "${common_make_envs[@]}"
make "${common_make_params_quoted[@]@Q}" "$@" "${make_filter}")
"${full_command[@]}" # and exit with it's code, since it's the last statement
}
function run_kernel_make() {
KERNEL_MAKE_RUNNER="run_host_command_logged" KERNEL_MAKE_UNBUFFER="unbuffer" run_kernel_make_internal "$@"
}
function run_kernel_make_dialog() {
KERNEL_MAKE_RUNNER="run_host_command_dialog" run_kernel_make_internal "$@"
}
function run_kernel_make_long_running() {
local seconds_start=${SECONDS} # Bash has a builtin SECONDS that is seconds since start of script
KERNEL_MAKE_RUNNER="run_host_command_logged_long_running" KERNEL_MAKE_UNBUFFER="unbuffer" run_kernel_make_internal "$@"
display_alert "Kernel Make '$*' took" "$((SECONDS - seconds_start)) seconds" "debug"
}