armbian_build/tools/repository/repo

#!/usr/bin/env bash

# Adding package
#
# @arg $1 string component
# @arg $2 string incoming folder
# @arg $3 string description
# @arg $4 input folder
#
adding_packages() {
# add deb files to repository if they are not already there
for f in "${4}${2}"/*.deb; do
	local package name version arch
	# read package
	package=$(dpkg-deb -I "${f}")
	name=$(echo "${package}" | awk /Package/'{print $2}')
	version=$(echo "${package}" | awk /Version/'{print $2}')
	arch=$(echo "${package}" | awk /Architecture/'{print $2}')
	# add if not already there
	aptly repo search -architectures="${arch}" -config="${CONFIG}" "${1}" \
	'Name (% '${name}'), $Version (='${version}'), $Architecture (='${arch}')' &> /dev/null
	if [[ $? -ne 0 ]]; then
		echo -e "Checking and adding \x1B[92m$name\x1B[0m to repository \x1B[92m$release $3\x1B[0m"
		aptly repo add -force-replace=true -config="${CONFIG}" "${1}" "${f}" &> /dev/null
	fi
done
}


# publishing repository
#
# $1: Input folder
# $2: Output folder
# $3: Command
# $4: GPG password
# $5: jammy,sid
#
publishing() {
# read comma delimited distros into array
IFS=', ' read -r -a DISTROS <<< "$5"
local errors=0
# publish all, update selected
local distributions=("jessy" "xenial" "stretch" "bionic" "focal" "hirsute" "impish" "jammy" "lunar" "kinetic" "buster" "bullseye" "bookworm" "sid")
for release in "${distributions[@]}"; do
	local forceoverwrite=""

	ADDING_PACKAGES="false"
	[[ " ${DISTROS[@]} " =~ " ${release} " ]] && ADDING_PACKAGES="true"

	[[ $(find ${1} -type f -name "*.deb" 2> /dev/null | wc -l) -eq 0 ]] && continue

	# let's drop from publish if exits
	if [[ -n $(aptly publish list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
		aptly publish drop -config="${CONFIG}" "${release}" > /dev/null 2>&1
	fi
	# create local repository if not exist
	if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}") ]]; then
		aptly repo create -config="${CONFIG}" -distribution="${release}" \
		-component="main,${release}-utils,${release}-desktop" -comment="Armbian main repository" "${release}"  > /dev/null 2>&1
	fi
	if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
		aptly repo create -config="${CONFIG}" -distribution="${release}" \
		-component="${release}-utils" -comment="Armbian ${release} utilities" "${release}-utils" > /dev/null 2>&1
	fi
	if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
		aptly repo create -config="${CONFIG}" -distribution="${release}" \
		-component="${release}-desktop" -comment="Armbian ${release} desktop" "${release}-desktop" > /dev/null 2>&1
	fi
	# adding main
	if find "$1"/ -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "$release" "" "main" "$1"
	else
		aptly repo add -config="${CONFIG}" "${release}" "example.deb" > /dev/null
	fi
	local COMPONENTS="main"
	# adding release-specific main
	if find "${1}/${release}" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}" "/${release}" "release packages" "$1"
	else
		# workaround - add dummy package to not trigger error
		aptly repo add -config="${CONFIG}" "${release}" "example.deb" > /dev/null
	fi
	# adding release-specific utils
	if find "${1}/extra/${release}-utils" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}-utils" "/extra/${release}-utils" "release utils" "$1"
	else
		aptly repo add -config="${CONFIG}" "${release}-utils" "example.deb" > /dev/null
	fi
	COMPONENTS="${COMPONENTS} ${release}-utils"
	# adding release-specific desktop
	if find "${1}/extra/${release}-desktop" -maxdepth 1 -type f -name "*.deb" 2> /dev/null | grep -q .; then
		[[ "${ADDING_PACKAGES}" == true ]] && adding_packages "${release}-desktop" "/extra/${release}-desktop" "desktop" "$1"
	else
		# workaround - add dummy package to not trigger error
		aptly repo add -config="${CONFIG}" "${release}-desktop" "example.deb" > /dev/null
	fi
	COMPONENTS="${COMPONENTS} ${release}-desktop"
	local mainnum utilnum desknum
	mainnum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}" | grep "Number of packages" | awk '{print $NF}')
	utilnum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | grep "Number of packages" | awk '{print $NF}')
	desknum=$(aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | grep "Number of packages" | awk '{print $NF}')
	if [ $mainnum -gt 0 ] && [ $utilnum -gt 0 ] && [ $desknum -gt 0 ]; then
		# write repo sync control file
		sudo date +%s > ${2}/public/.control
		# publish
		echo "Publishing ${release}"
		aptly publish \
			-acquire-by-hash \
			-architectures="armhf,arm64,amd64,riscv64,i386,all" \
			-passphrase="${4}" \
			-origin="Armbian" \
			-label="Armbian" \
			-config="${CONFIG}" \
			-component="${COMPONENTS// /,}" \
			-distribution="${release}" repo "${release}" ${COMPONENTS//main/} > /dev/null
		if [[ $? -ne 0 ]]; then
			echo "Publishing failed ${release}"
			errors=$((errors + 1))
			exit 0
		fi
	else
		errors=$((errors + 1))
		local err_txt=": All components must be present: main, utils and desktop for first build"
	fi
done
# cleanup
aptly db cleanup -config="${CONFIG}" > /dev/null
# key
cp armbian.key "${2}"/public/
# display what we have
(aptly repo list -config="${CONFIG}") | grep -E packages
# remove debs if no errors found
if [[ $errors -eq 0 ]]; then
		echo "Purging incoming debs"
		sudo find "${1}" -name "*.deb" -type f -delete
else
	display_alert "There were some problems $err_txt" "leaving incoming directory intact" "err"
fi
}


#
# $1: Input folder
# $2: Output folder
# $3: Command
# $4: GPG password
# $5: jammy,sid
#
repo-manipulate() {

# read comma delimited distros into array
IFS=', ' read -r -a DISTROS <<< "$5"

case $3 in

	serve)

		sudo aptly serve -listen=$(ip -f inet addr | grep -Po 'inet \K[\d.]+' | grep -v 127.0.0.1 | head -1):80 -config="${CONFIG}"
		return 0
	;;

	html)
		cat header.html
		for release in "${DISTROS[@]}"; do
		echo "<thead><tr><td colspan=3><h2>$release</h2></tr><tr><th>Main</th><th>Utils</th><th>Desktop</th></tr></thead>"
		echo "<tbody><tr><td width=33% valing=top>"
		aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7 | sed 's/.*/&<br>/'
		echo "</td><td width=33% valign=top>" | sudo tee -a ${filename}
		aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7 | sed 's/.*/&<br>/'
		echo "</td><td width=33% valign=top>" | sudo tee -a ${filename}
		aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7 | sed 's/.*/&<br>/'
		echo "</td></tr></tbody>"
		done
		cat footer.html
		return 0
	;;

	show)

		for release in "${DISTROS[@]}"; do
			echo "Displaying repository contents for $release"
			aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7
			echo "Displaying repository contents for $release-utils"
			aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
			echo "Displaying repository contents for $release-desktop"
			aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
		done
		return 0
	;;

	unique)
		# which package should be removed from all repositories
		IFS=$'\n'
		while true; do
			LIST=()
			for release in "${DISTROS[@]}"; do
				LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}" | tail -n +7))
				LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7))
				LIST+=($(aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7))
			done
			LIST=($(echo "${LIST[@]}" | tr ' ' '\n' | sort -u))
			new_list=()
			# create a human readable menu
			for ((n = 0; n < $((${#LIST[@]})); n++)); do
				new_list+=("${LIST[$n]}")
				new_list+=("")
			done
			LIST=("${new_list[@]}")
			LIST_LENGTH=$((${#LIST[@]} / 2))
			exec 3>&1
			TARGET_VERSION=$(dialog --cancel-label "Cancel" --backtitle "BACKTITLE" --no-collapse --title \
			"Remove packages from repositories" --clear --menu "Delete" $((9 + LIST_LENGTH)) 82 65 "${LIST[@]}" 2>&1 1>&3)
			exitstatus=$?
			exec 3>&-
			if [[ $exitstatus -eq 0 ]]; then
				for release in "${DISTROS[@]}"; do
					aptly repo remove -config="${CONFIG}" "${release}" "$TARGET_VERSION"
					aptly repo remove -config="${CONFIG}" "${release}-utils" "$TARGET_VERSION"
					aptly repo remove -config="${CONFIG}" "${release}-desktop" "$TARGET_VERSION"
				done
			else
				return 1
			fi
			aptly db cleanup -config="${CONFIG}" > /dev/null 2>&1
			# remove empty folders
			find $2/public -type d -empty -print -exec rm -rf {} \;
		done
		;;

	update)
		publishing "$1" "$2" "$3" "$4" "$5"
		;;

	*)
		echo -e "Unknown command"
		return 1
		;;
esac
}


# defaults
input="input"
output="output"
command="show"
releases="jammy,sid"

help()
{
echo "Armbian wrapper for Aptly v1.0

(c) Igor Pecovnik, igor@armbian.com

License: (MIT) <https://mit-license.org/>

Usage: $0 [ -short | --long ]

-h --help displays this
-i --input [input folder]
-o --output [output folder]
-p --password [GPG password]
-r --repository [jammy,sid,bullseye,...]
-c --command

          [show] displays packages in each repository
          [html] displays packages in each repository in html form
          [serve] serve repository - useful for local diagnostics
          [unique] manually select which package should be removed from all repositories
          [update] search for packages in input folder
	"
    exit 2
}

SHORT=i:,o:,c:,p:,r:,h
LONG=input:,output:,command:,password:,releases:,help
OPTS=$(getopt -a -n repo --options $SHORT --longoptions $LONG -- "$@")

VALID_ARGUMENTS=$# # Returns the count of arguments that are in short or long options

eval set -- "$OPTS"

while :
do
  case "$1" in
    -i | --input )
      input="$2"
      shift 2
      ;;
    -o | --output )
      output="$2"
      shift 2
      ;;
    -c | --command )
      command="$2"
      shift 2
      ;;
    -p | --password )
      password="$2"
      shift 2
      ;;
    -r | --releases )
      releases="$2"
      shift 2
      ;;
    -h | --help)
      help
      ;;
    --)
      shift;
      break
      ;;
    *)
      echo "Unexpected option: $1"
      help
      ;;
  esac
done

# define job name
[[ $(cat /var/run/repomanagement 2>/dev/null) == "$output" ]] && echo "Running. Try again later" && exit 0
echo "${output}" | sudo tee /var/run/repomanagement &>/dev/null

# redefine output folder in Aptly
TempDir="$(mktemp -d || exit 1)"
sed 's|"rootDir": ".*"|"rootDir": "'$output'"|g' aptly.conf > "${TempDir}"/aptly.conf
CONFIG="${TempDir}/aptly.conf"

# main
repo-manipulate "$input" "$output" "$command" "$password" "$releases"
RETURN=$?
sudo rm /var/run/repomanagement
exit $RETURN