armbian_build/lib/functions/general/python-tools.sh

# This whole thing is a big "I refuse to use venv in a simple bash script" delusion.
# If you know to tame it, teach me. I'd rather not know about PYTHONUSERBASE and such.
# --rpardini

function early_prepare_pip3_dependencies_for_python_tools() {
	# This is like a stupid version of requirements.txt
	declare -a -g python3_pip_dependencies=(
		"unidiff==0.7.4"      # for parsing unified diff
		"GitPython==3.1.30"   # for manipulating git repos
		"unidecode==1.3.6"    # for converting strings to ascii
		"coloredlogs==15.0.1" # for colored logging
	)
	return 0
}

# call: prepare_python_and_pip # this defines global PYTHON3_INFO dict and PYTHON3_VARS array
function prepare_python_and_pip() {
	assert_prepared_host # this needs a prepared host to work; avoid fake errors about "python3-pip" not being installed
	# First determine with python3 to use; requires knowing the HOSTRELEASE.
	[[ -z "${HOSTRELEASE}" ]] && exit_with_error "HOSTRELEASE is not set"

	# fake-memoize this, it's expensive and does not need to be done twice
	declare -g _already_prepared_python_and_pip="${_already_prepared_python_and_pip:-no}"
	if [[ "${_already_prepared_python_and_pip}" == "yes" ]]; then
		display_alert "All Python preparation done before" "skipping python prep" "debug"
		return 0
	fi

	declare python3_binary_path="/usr/bin/python3"
	# Determine what version of python3;  focal-like OS's have Python 3.8, but we need 3.9.
	if [[ "focal ulyana ulyssa uma una" == *"$HOSTRELEASE"* ]]; then
		python3_binary_path="/usr/bin/python3.9"
		display_alert "Using  '${python3_binary_path}' for" "'$HOSTRELEASE' has outdated python3, using python3.9" "warn"
	fi

	# Check that the actual python3 --version is 3.9 at least
	declare python3_version python3_full_version
	python3_full_version="$("${python3_binary_path}" --version)" # "cut" below masks errors, do it twice.
	python3_version="$("${python3_binary_path}" --version | cut -d' ' -f2)"
	display_alert "Python3 version" "${python3_version} - '${python3_full_version}'" "info"
	if ! linux-version compare "${python3_version}" ge "3.9"; then
		exit_with_error "Python3 version is too old (${python3_version}), need at least 3.9"
	fi

	# Check actual pip3 version
	#   Note: we don't use "/usr/bin/pip3" at all, since it's commonly missing. instead "python -m pip"
	#   The hostdep package python3-pip is still required, and other crazy might impact this.
	#   We might need to install our own pip if it gets bad enough.
	declare pip3_version
	pip3_version="$("${python3_binary_path}" -m pip --version)"
	display_alert "pip3 version" "${pip3_version}" "info"

	# Hash the contents of the dependencies array + the Python version + the release
	declare python3_pip_dependencies_hash
	early_prepare_pip3_dependencies_for_python_tools
	python3_pip_dependencies_hash="$(echo "${HOSTRELEASE}" "${python3_version}" "${pip3_version}" "${python3_pip_dependencies[*]}" | sha256sum | cut -d' ' -f1)"

	declare non_cache_dir="/armbian-pip"
	declare python_pip_cache="${SRC}/cache/pip"

	if [[ "${deploy_to_non_cache_dir:-"no"}" == "yes" ]]; then
		display_alert "Using non-cache dir" "PIP: ${non_cache_dir}" "warn"
		python_pip_cache="${non_cache_dir}"
	else
		# if the non-cache dir exists, copy it into place, if not already existing...
		if [[ -d "${non_cache_dir}" && ! -d "${python_pip_cache}" ]]; then
			display_alert "Deploying pip cache from Docker image" "${non_cache_dir} -> ${python_pip_cache}" "info"
			run_host_command_logged cp -pr "${non_cache_dir}" "${python_pip_cache}"
		fi
	fi

	declare python_hash_base="${python_pip_cache}/pip_pkg_hash"
	declare python_hash_file="${python_hash_base}_${python3_pip_dependencies_hash}"
	declare python3_user_base="${python_pip_cache}/base"
	declare python3_pycache="${python_pip_cache}/pycache"

	# declare a readonly global dict with all needed info for executing stuff using this setup
	declare -r -g -A PYTHON3_INFO=(
		[BIN]="${python3_binary_path}"
		[USERBASE]="${python3_user_base}"
		[PYCACHEPREFIX]="${python3_pycache}"
		[HASH]="${python3_pip_dependencies_hash}"
		[DEPS]="${python3_pip_dependencies[*]}"
		[VERSION]="${python3_version}"
		[PIP_VERSION]="${pip3_version}"
	)

	# declare a readonly global array for ENV vars to invoke python3 with
	declare -r -g -a PYTHON3_VARS=(
		"PYTHONUSERBASE=${PYTHON3_INFO[USERBASE]}"
		"PYTHONUNBUFFERED=yes"
		"PYTHONPYCACHEPREFIX=${PYTHON3_INFO[PYCACHEPREFIX]}"
	)

	# If the hash file exists, we're done.
	if [[ -f "${python_hash_file}" ]]; then
		display_alert "Using cached pip packages for Python tools" "${python3_pip_dependencies_hash}" "info"
	else
		display_alert "Installing pip packages for Python tools" "${python3_pip_dependencies_hash:0:10}" "info"
		# remove the old hashes matching base, don't leave junk behind
		run_host_command_logged rm -fv "${python_hash_base}*"

		# @TODO: when running with sudo:
		# WARNING: The directory '/home/human/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.
		# --root-user-action=ignore requires pip 22.1+

		run_host_command_logged env -i "${PYTHON3_VARS[@]@Q}" "${PYTHON3_INFO[BIN]}" -m pip install \
			--no-warn-script-location --user "${python3_pip_dependencies[@]}"

		# Create the hash file
		run_host_command_logged touch "${python_hash_file}"
	fi

	_already_prepared_python_and_pip="yes"
	return 0
}

# Called during early_prepare_host_dependencies(); when building a Dockerfile, host_release is set to the Docker image name.
function host_deps_add_extra_python() {
	# check host_release is set, or bail.
	[[ -z "${host_release}" ]] && exit_with_error "host_release is not set"

	# host_release is from outer scope (
	# Determine what version of python3;  focal-like OS's have Python 3.8, but we need 3.9.
	if [[ "focal ulyana ulyssa uma una" == *"${host_release}"* ]]; then
		display_alert "Using Python 3.9 for" "hostdeps: '${host_release}' has outdated python3, using python3.9" "warn"
		host_dependencies+=("python3.9-dev")
	else
		display_alert "Using Python3 for" "hostdeps: '${host_release}' has python3 >= 3.9" "debug"
	fi
}