Merge branch 'dedup-cleanup' into 'master'

Misc de-duplication and cleanups

See merge request archlinux/archlinux-docker!85

.gitlab-ci.yml

@@ -80,9 +80,6 @@ rootfs:
     - releases@archlinux/archlinux-docker
     - schedules@archlinux/archlinux-docker
     - tags@archlinux/archlinux-docker
-  parallel:
-    matrix:
-      - GROUP: [base, base-devel]
 
 rootfs:secure:
   extends: .rootfs
@@ -153,9 +150,6 @@ image:publish:secure:
     - vm
   only:
     - tags@archlinux/archlinux-docker
-  parallel:
-    matrix:
-      - GROUP: [base, base-devel]
   before_script:
     - pacman -Syu --noconfirm podman
     - podman login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_ACCESS_TOKEN" "docker.io"
@@ -167,26 +161,18 @@ image:publish:secure:
         -f "$CI_PROJECT_DIR/Dockerfile.$GROUP"
         -t "archlinux:$GROUP-$BUILD_VERSION"
         "$CI_PROJECT_DIR"
-    - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:$GROUP"
+    - |
-    - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:$GROUP-$BUILD_VERSION"
+      for host in "docker.io" "quay.io" "ghcr.io"; do
-    - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:$GROUP"
+        podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:$GROUP"
-    - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:$GROUP-$BUILD_VERSION"
+        podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:$GROUP-$BUILD_VERSION"
-    - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:$GROUP"
+        podman push "$host/archlinux/archlinux:$GROUP"
-    - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:$GROUP-$BUILD_VERSION"
+        podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "$host/archlinux/archlinux:$GROUP-$BUILD_VERSION"
-    - podman push "docker.io/archlinux/archlinux:$GROUP"
+
-    - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "docker.io/archlinux/archlinux:$GROUP-$BUILD_VERSION"
+        if [[ "$GROUP" == "base" ]]; then
-    - podman push "quay.io/archlinux/archlinux:$GROUP"
+          podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:latest"
-    - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "quay.io/archlinux/archlinux:$GROUP-$BUILD_VERSION"
+          podman push "$host/archlinux/archlinux:latest"
-    - podman push "ghcr.io/archlinux/archlinux:$GROUP"
+        fi
-    - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "ghcr.io/archlinux/archlinux:$GROUP-$BUILD_VERSION"
+      done
-    - if [[ "$GROUP" == "base" ]]; then
-        podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:latest";
-        podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:latest";
-        podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:latest";
-        podman push "docker.io/archlinux/archlinux:latest";
-        podman push "quay.io/archlinux/archlinux:latest";
-        podman push "ghcr.io/archlinux/archlinux:latest";
-      fi
 
 
 .test:
@@ -257,15 +243,9 @@ release:
         curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst
         echo "Uploading ${group}.tar.zst.SHA256"
         curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst.SHA256
-        sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > output/Dockerfile.${group}
         package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.zst)
-        sed -i "s|TEMPLATE_ROOTFS_RELEASE_URL|https://gitlab.archlinux.org/archlinux/archlinux-docker/-/releases/v${BUILD_VERSION}|" output/Dockerfile.${group}
+        scripts/make-dockerfile.sh "${group}" "output" "curl -sOJL \"${package_url}\"" "${group}"
-        sed -i "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"\$(curl -sOJL -w \"%{filename_effective}\" \"${package_url}\")\"|" output/Dockerfile.${group}
+        sed -i "/^COPY ${group}.tar.zst \/$/d" output/Dockerfile.${group}
-        sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.zst.SHA256)|" output/Dockerfile.${group}
-        sed -i "s|TEMPLATE_TITLE|Arch Linux ${group} Image|" output/Dockerfile.${group}
-        sed -i "s|TEMPLATE_VERSION_ID|${BUILD_VERSION}|" output/Dockerfile.${group}
-        sed -i "s|TEMPLATE_REVISION|${CI_COMMIT_SHA}|" output/Dockerfile.${group}
-        sed -i "s|TEMPLATE_CREATED|$(date -Is)|" output/Dockerfile.${group}
       done
     - >
       curl -sSf --request POST -o commit-response.json
@@ -337,7 +317,16 @@ publish-dockerhub:
     - git reset --hard FETCH_HEAD
     - head="release/${BUILD_VERSION}"
     - git checkout -b "$head"
-    - envsubst < ../docker-library.template > library/archlinux
+    - cp ../docker-library.template library/archlinux
+    - |
+      for group in base base-devel; do
+        test "${group}" = "base" && extra="latest, " || extra=""
+        echo "Tags: ${extra}${group}, ${group}-${BUILD_VERSION}" >> library/archlinux
+        echo "GitCommit: ${BUILD_COMMIT}" >> library/archlinux
+        echo "GitFetch: refs/tags/v${BUILD_VERSION}" >> library/archlinux
+        echo "File: Dockerfile.${group}" >> library/archlinux
+        echo >> library/archlinux
+      done
     - git diff
     - git add library/archlinux
     - maintainers="$(grep \(@ ../docker-library.template | cut -d\( -f2 | cut -d\) -f1 | xargs)"

Dockerfile.template

@@ -15,13 +15,11 @@ COPY TEMPLATE_ROOTFS_FILE /
 
 RUN apk add --no-cache curl tar zstd
 
-# TEMPLATE_ROOTFS_RELEASE_URL
 RUN TEMPLATE_ROOTFS_DOWNLOAD && \
-    echo "TEMPLATE_ROOTFS_HASH" > /tmp/rootfs.tar.sha256 && \
+    echo "TEMPLATE_ROOTFS_HASH" > /tmp/sha256sums.txt && \
-    cat /tmp/rootfs.tar.sha256 && \
+    sha256sum -c /tmp/sha256sums.txt && \
-    sha256sum -c /tmp/rootfs.tar.sha256 && \
     mkdir /rootfs && \
-    tar -C /rootfs --extract --file "${ROOTFS}"
+    tar -C /rootfs --extract --file TEMPLATE_ROOTFS_FILE
 
 FROM scratch AS root
 

Makefile

@@ -2,76 +2,20 @@ OCITOOL=podman # or docker
 BUILDDIR=$(shell pwd)/build
 OUTPUTDIR=$(shell pwd)/output
 
-define rootfs
-	mkdir -vp $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks
-	find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(BUILDDIR)/alpm-hooks{} \;
-
-	mkdir -vp $(BUILDDIR)/var/lib/pacman/ $(OUTPUTDIR)
-	install -Dm644 /usr/share/devtools/pacman.conf.d/extra.conf $(BUILDDIR)/etc/pacman.conf
-	cat pacman-conf.d-noextract.conf >> $(BUILDDIR)/etc/pacman.conf
-
-	sed 's/Include = /&rootfs/g' < $(BUILDDIR)/etc/pacman.conf > pacman.conf
-
-	fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \
-		--noconfirm --dbpath $(BUILDDIR)/var/lib/pacman \
-		--config pacman.conf \
-		--noscriptlet \
-		--hookdir $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks/ $(2)
-
-	cp --recursive --preserve=timestamps rootfs/* $(BUILDDIR)/
-
-	fakechroot -- fakeroot -- chroot $(BUILDDIR) update-ca-trust
-	fakechroot -- fakeroot -- chroot $(BUILDDIR) sh -c 'pacman-key --init && pacman-key --populate && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*"'
-
-	ln -fs /usr/lib/os-release $(BUILDDIR)/etc/os-release
-
-	# add system users
-	fakechroot -- fakeroot -- chroot $(BUILDDIR) /usr/bin/systemd-sysusers --root "/"
-
-	# remove passwordless login for root (see CVE-2019-5021 for reference)
-	sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow"
-
-	# fakeroot to map the gid/uid of the builder process to root
-	# fixes #22
-	fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar
-
-	cd $(OUTPUTDIR); zstd --long -T0 -8 $(1).tar; sha256sum $(1).tar.zst > $(1).tar.zst.SHA256
-endef
-
-define dockerfile
-	sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.zst|" \
-	    -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \
-	    -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$(1).tar.zst\"|" \
-	    -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.zst.SHA256)|" \
-	    -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \
-	    -e "s|TEMPLATE_VERSION_ID|dev|" \
-	    -e "s|TEMPLATE_REVISION|$$(git rev-parse HEAD)|" \
-	    -e "s|TEMPLATE_CREATED|$$(date -Is)|" \
-	    Dockerfile.template > $(OUTPUTDIR)/Dockerfile.$(1)
-endef
-
 .PHONY: clean
 clean:
 	rm -rf $(BUILDDIR) $(OUTPUTDIR)
 
-$(OUTPUTDIR)/base.tar.zst:
+.PRECIOUS: $(OUTPUTDIR)/%.tar.zst
-	$(call rootfs,base,base)
+$(OUTPUTDIR)/%.tar.zst:
+	scripts/make-rootfs.sh $(*) $(BUILDDIR) $(OUTPUTDIR)
 
-$(OUTPUTDIR)/base-devel.tar.zst:
+.PRECIOUS: $(OUTPUTDIR)/Dockerfile.%
-	$(call rootfs,base-devel,base base-devel)
+$(OUTPUTDIR)/Dockerfile.%: $(OUTPUTDIR)/%.tar.zst
-
+	scripts/make-dockerfile.sh $(*) $(OUTPUTDIR) "true" "Dev"
-$(OUTPUTDIR)/Dockerfile.base: $(OUTPUTDIR)/base.tar.zst
-	$(call dockerfile,base)
-
-$(OUTPUTDIR)/Dockerfile.base-devel: $(OUTPUTDIR)/base-devel.tar.zst
-	$(call dockerfile,base-devel)
 
 # The following is for local builds only, it is not used by the CI/CD pipeline
 
-.PHONY: image-base
+all: image-base image-base-devel
-image-base: $(OUTPUTDIR)/Dockerfile.base
+image-%: $(OUTPUTDIR)/Dockerfile.%
-	${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.base -t archlinux/archlinux:base $(OUTPUTDIR)
+	${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.$(*) -t archlinux/archlinux:$(*) $(OUTPUTDIR)
-
-.PHONY: image-base-devel
-image-base-devel: $(OUTPUTDIR)/Dockerfile.base-devel
-	${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.base-devel -t archlinux/archlinux:base-devel $(OUTPUTDIR)

docker-library.template

@@ -5,13 +5,3 @@ Maintainers: Santiago Torres-Arias <santiago@archlinux.org> (@SantiagoTorres),
              Justin Kromlinger <hashworks@archlinux.org> (@hashworks)
 GitRepo: https://gitlab.archlinux.org/archlinux/archlinux-docker.git
 
-Tags: latest, base, base-${BUILD_VERSION}
-GitCommit: ${BUILD_COMMIT}
-GitFetch: refs/tags/v${BUILD_VERSION}
-File: Dockerfile.base
-
-Tags: base-devel, base-devel-${BUILD_VERSION}
-GitCommit: ${BUILD_COMMIT}
-GitFetch: refs/tags/v${BUILD_VERSION}
-File: Dockerfile.base-devel
-

scripts/make-dockerfile.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -euo pipefail
+
+declare -r GROUP="$1"
+declare -r OUTPUTDIR="$2"
+declare -r DOWNLOAD="$3"
+declare -r TITLE="$4"
+
+# Do not use these directly in the sed below - it will mask git failures
+BUILD_VERSION="${BUILD_VERSION:-dev}"
+CI_COMMIT_SHA="${CI_COMMIT_SHA:-$(git rev-parse HEAD)}"
+
+sed -e "s|TEMPLATE_ROOTFS_FILE|$GROUP.tar.zst|" \
+    -e "s|TEMPLATE_ROOTFS_DOWNLOAD|$DOWNLOAD|" \
+    -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \
+    -e "s|TEMPLATE_TITLE|Arch Linux $TITLE Image|" \
+    -e "s|TEMPLATE_VERSION_ID|$BUILD_VERSION|" \
+    -e "s|TEMPLATE_REVISION|$CI_COMMIT_SHA|" \
+    -e "s|TEMPLATE_CREATED|$(date -Is)|" \
+    Dockerfile.template > "$OUTPUTDIR/Dockerfile.$GROUP"

scripts/make-rootfs.sh

@@ -0,0 +1,54 @@
+#!/bin/bash
+
+set -euo pipefail
+
+declare -r WRAPPER="fakechroot -- fakeroot"
+
+declare -r GROUP="$1"
+declare -r BUILDDIR="$2"
+declare -r OUTPUTDIR="$3"
+
+mkdir -vp "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks"
+find /usr/share/libalpm/hooks -exec ln -sf /dev/null "$BUILDDIR/alpm-hooks"{} \;
+
+mkdir -vp "$BUILDDIR/var/lib/pacman/" "$OUTPUTDIR"
+install -Dm644 /usr/share/devtools/pacman.conf.d/extra.conf "$BUILDDIR/etc/pacman.conf"
+cat pacman-conf.d-noextract.conf >> "$BUILDDIR/etc/pacman.conf"
+
+sed 's/Include = /&rootfs/g' < "$BUILDDIR/etc/pacman.conf" > pacman.conf
+
+cp --recursive --preserve=timestamps rootfs/* "$BUILDDIR/"
+ln -fs /usr/lib/os-release "$BUILDDIR/etc/os-release"
+
+$WRAPPER -- \
+    pacman -Sy -r "$BUILDDIR" \
+        --noconfirm --dbpath "$BUILDDIR/var/lib/pacman" \
+        --config pacman.conf \
+        --noscriptlet \
+        --hookdir "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks/" base "$GROUP"
+
+$WRAPPER -- chroot "$BUILDDIR" update-ca-trust
+$WRAPPER -- chroot "$BUILDDIR" pacman-key --init
+$WRAPPER -- chroot "$BUILDDIR" pacman-key --populate
+
+# add system users
+$WRAPPER -- chroot "$BUILDDIR" /usr/bin/systemd-sysusers --root "/"
+
+# remove passwordless login for root (see CVE-2019-5021 for reference)
+sed -i -e 's/^root::/root:!:/' "$BUILDDIR/etc/shadow"
+
+# fakeroot to map the gid/uid of the builder process to root
+# fixes #22
+fakeroot -- \
+    tar \
+        --numeric-owner \
+        --xattrs \
+        --acls \
+        --exclude-from=exclude \
+        -C "$BUILDDIR" \
+        -c . \
+        -f "$OUTPUTDIR/$GROUP.tar"
+
+cd "$OUTPUTDIR"
+zstd --long -T0 -8 "$GROUP.tar"
+sha256sum "$GROUP.tar.zst" > "$GROUP.tar.zst.SHA256"