From 069a4dee59eef648ed5bd43f5840facdb998ed7f Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Thu, 21 Sep 2023 11:45:40 +0100 Subject: [PATCH 01/11] Drop duplicate parallel/matrix The option was moved the to template and an earlier commit reintroduced/copied them back seemingly by mistake. Cc: nl6720 Fixes: ca9957f ("Do not use secure runners on forks") Signed-off-by: Emil Velikov --- .gitlab-ci.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 678414d..d11c227 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,9 +80,6 @@ rootfs: - releases@archlinux/archlinux-docker - schedules@archlinux/archlinux-docker - tags@archlinux/archlinux-docker - parallel: - matrix: - - GROUP: [base, base-devel] rootfs:secure: extends: .rootfs @@ -153,9 +150,6 @@ image:publish:secure: - vm only: - tags@archlinux/archlinux-docker - parallel: - matrix: - - GROUP: [base, base-devel] before_script: - pacman -Syu --noconfirm podman - podman login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_ACCESS_TOKEN" "docker.io" From b213655855777c4b4730a52c53930f6dc2b4ab72 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Thu, 21 Sep 2023 12:01:11 +0100 Subject: [PATCH 02/11] De-duplicate podman tag/push commands The commands are identical across the three hosts. Just flesh that into a simple for loop. Signed-off-by: Emil Velikov --- .gitlab-ci.yml | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d11c227..a7333b4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,26 +161,18 @@ image:publish:secure: -f "$CI_PROJECT_DIR/Dockerfile.$GROUP" -t "archlinux:$GROUP-$BUILD_VERSION" "$CI_PROJECT_DIR" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:$GROUP" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:$GROUP" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:$GROUP" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman push "docker.io/archlinux/archlinux:$GROUP" - - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "docker.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman push "quay.io/archlinux/archlinux:$GROUP" - - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "quay.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman push "ghcr.io/archlinux/archlinux:$GROUP" - - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "ghcr.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - if [[ "$GROUP" == "base" ]]; then - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:latest"; - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:latest"; - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:latest"; - podman push "docker.io/archlinux/archlinux:latest"; - podman push "quay.io/archlinux/archlinux:latest"; - podman push "ghcr.io/archlinux/archlinux:latest"; - fi + - | + for host in "docker.io" "quay.io" "ghcr.io"; do + podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:$GROUP" + podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:$GROUP-$BUILD_VERSION" + podman push "$host/archlinux/archlinux:$GROUP" + podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "$host/archlinux/archlinux:$GROUP-$BUILD_VERSION" + + if [[ "$GROUP" == "base" ]]; then + podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:latest" + podman push "$host/archlinux/archlinux:latest" + fi + done .test: From f737d64397f5fa3e0a11fccf771a0c0958abb60a Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Thu, 21 Sep 2023 12:41:55 +0100 Subject: [PATCH 03/11] Move shell scripts out of the Makefile Embedding one pieces of code into another (shell script into a makefiles in this case) is rarely pretty. Split things up, as appropriate. While here, simplify the rootfs in a few ways: - pass only the extra non-base (and effectively group name) package - add a handy variable for the fakeroot/fakechroot combo - split and rewrap long lines As a bonus point, this makes it easier to use pattern rules in the makefile - which will be handy for the upcoming multilib-devel group/target. Plus we can check the scripts via shellcheck/etc CI stage, as follow-up. Signed-off-by: Emil Velikov --- Makefile | 56 +++---------------------------------- scripts/make-dockerfile.sh | 16 +++++++++++ scripts/make-rootfs.sh | 57 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 52 deletions(-) create mode 100755 scripts/make-dockerfile.sh create mode 100755 scripts/make-rootfs.sh diff --git a/Makefile b/Makefile index 38c8e37..049d931 100644 --- a/Makefile +++ b/Makefile @@ -2,69 +2,21 @@ OCITOOL=podman # or docker BUILDDIR=$(shell pwd)/build OUTPUTDIR=$(shell pwd)/output -define rootfs - mkdir -vp $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks - find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(BUILDDIR)/alpm-hooks{} \; - - mkdir -vp $(BUILDDIR)/var/lib/pacman/ $(OUTPUTDIR) - install -Dm644 /usr/share/devtools/pacman.conf.d/extra.conf $(BUILDDIR)/etc/pacman.conf - cat pacman-conf.d-noextract.conf >> $(BUILDDIR)/etc/pacman.conf - - sed 's/Include = /&rootfs/g' < $(BUILDDIR)/etc/pacman.conf > pacman.conf - - fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ - --noconfirm --dbpath $(BUILDDIR)/var/lib/pacman \ - --config pacman.conf \ - --noscriptlet \ - --hookdir $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks/ $(2) - - cp --recursive --preserve=timestamps rootfs/* $(BUILDDIR)/ - - fakechroot -- fakeroot -- chroot $(BUILDDIR) update-ca-trust - fakechroot -- fakeroot -- chroot $(BUILDDIR) sh -c 'pacman-key --init && pacman-key --populate && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*"' - - ln -fs /usr/lib/os-release $(BUILDDIR)/etc/os-release - - # add system users - fakechroot -- fakeroot -- chroot $(BUILDDIR) /usr/bin/systemd-sysusers --root "/" - - # remove passwordless login for root (see CVE-2019-5021 for reference) - sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" - - # fakeroot to map the gid/uid of the builder process to root - # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar - - cd $(OUTPUTDIR); zstd --long -T0 -8 $(1).tar; sha256sum $(1).tar.zst > $(1).tar.zst.SHA256 -endef - -define dockerfile - sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.zst|" \ - -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \ - -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$(1).tar.zst\"|" \ - -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.zst.SHA256)|" \ - -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \ - -e "s|TEMPLATE_VERSION_ID|dev|" \ - -e "s|TEMPLATE_REVISION|$$(git rev-parse HEAD)|" \ - -e "s|TEMPLATE_CREATED|$$(date -Is)|" \ - Dockerfile.template > $(OUTPUTDIR)/Dockerfile.$(1) -endef - .PHONY: clean clean: rm -rf $(BUILDDIR) $(OUTPUTDIR) $(OUTPUTDIR)/base.tar.zst: - $(call rootfs,base,base) + scripts/make-rootfs.sh base $(BUILDDIR) $(OUTPUTDIR) $(OUTPUTDIR)/base-devel.tar.zst: - $(call rootfs,base-devel,base base-devel) + scripts/make-rootfs.sh base-devel $(BUILDDIR) $(OUTPUTDIR) $(OUTPUTDIR)/Dockerfile.base: $(OUTPUTDIR)/base.tar.zst - $(call dockerfile,base) + scripts/make-dockerfile.sh base $(OUTPUTDIR) $(OUTPUTDIR)/Dockerfile.base-devel: $(OUTPUTDIR)/base-devel.tar.zst - $(call dockerfile,base-devel) + scripts/make-dockerfile.sh base-devel $(OUTPUTDIR) # The following is for local builds only, it is not used by the CI/CD pipeline diff --git a/scripts/make-dockerfile.sh b/scripts/make-dockerfile.sh new file mode 100755 index 0000000..43263e3 --- /dev/null +++ b/scripts/make-dockerfile.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -euo pipefail + +declare -r GROUP="$1" +declare -r OUTPUTDIR="$2" + +sed -e "s|TEMPLATE_ROOTFS_FILE|$GROUP.tar.zst|" \ + -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \ + -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$GROUP.tar.zst\"|" \ + -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \ + -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \ + -e "s|TEMPLATE_VERSION_ID|dev|" \ + -e "s|TEMPLATE_REVISION|$(git rev-parse HEAD)|" \ + -e "s|TEMPLATE_CREATED|$(date -Is)|" \ + Dockerfile.template > "$OUTPUTDIR/Dockerfile.$GROUP" diff --git a/scripts/make-rootfs.sh b/scripts/make-rootfs.sh new file mode 100755 index 0000000..15f12d9 --- /dev/null +++ b/scripts/make-rootfs.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -euo pipefail + +declare -r WRAPPER="fakechroot -- fakeroot" + +declare -r GROUP="$1" +declare -r BUILDDIR="$2" +declare -r OUTPUTDIR="$3" + +mkdir -vp "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks" +find /usr/share/libalpm/hooks -exec ln -sf /dev/null "$BUILDDIR/alpm-hooks"{} \; + +mkdir -vp "$BUILDDIR/var/lib/pacman/" "$OUTPUTDIR" +install -Dm644 /usr/share/devtools/pacman.conf.d/extra.conf "$BUILDDIR/etc/pacman.conf" +cat pacman-conf.d-noextract.conf >> "$BUILDDIR/etc/pacman.conf" + +sed 's/Include = /&rootfs/g' < "$BUILDDIR/etc/pacman.conf" > pacman.conf + +$WRAPPER -- \ + pacman -Sy -r "$BUILDDIR" \ + --noconfirm --dbpath "$BUILDDIR/var/lib/pacman" \ + --config pacman.conf \ + --noscriptlet \ + --hookdir "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks/" base "$GROUP" + +cp --recursive --preserve=timestamps rootfs/* "$BUILDDIR/" + +$WRAPPER -- chroot "$BUILDDIR" update-ca-trust +$WRAPPER -- chroot "$BUILDDIR" pacman-key --init +$WRAPPER -- chroot "$BUILDDIR" pacman-key --populate + +rm -rf "$BUILDDIR/etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" + +ln -fs /usr/lib/os-release "$BUILDDIR/etc/os-release" + +# add system users +$WRAPPER -- chroot "$BUILDDIR" /usr/bin/systemd-sysusers --root "/" + +# remove passwordless login for root (see CVE-2019-5021 for reference) +sed -i -e 's/^root::/root:!:/' "$BUILDDIR/etc/shadow" + +# fakeroot to map the gid/uid of the builder process to root +# fixes #22 +fakeroot -- \ + tar \ + --numeric-owner \ + --xattrs \ + --acls \ + --exclude-from=exclude \ + -C "$BUILDDIR" \ + -c . \ + -f "$OUTPUTDIR/$GROUP.tar" + +cd "$OUTPUTDIR" +zstd --long -T0 -8 "$GROUP.tar" +sha256sum "$GROUP.tar.zst" > "$GROUP.tar.zst.SHA256" From 0067757d21a8c92457bbd0c2220e84aee3578c65 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 25 Sep 2023 16:21:01 +0100 Subject: [PATCH 04/11] scripts/make-rootfs.sh: drop pacman gnupg removal The files are explicitly omitted from the tarball via the excludes file. Signed-off-by: Emil Velikov --- scripts/make-rootfs.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/make-rootfs.sh b/scripts/make-rootfs.sh index 15f12d9..cf59108 100755 --- a/scripts/make-rootfs.sh +++ b/scripts/make-rootfs.sh @@ -30,8 +30,6 @@ $WRAPPER -- chroot "$BUILDDIR" update-ca-trust $WRAPPER -- chroot "$BUILDDIR" pacman-key --init $WRAPPER -- chroot "$BUILDDIR" pacman-key --populate -rm -rf "$BUILDDIR/etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" - ln -fs /usr/lib/os-release "$BUILDDIR/etc/os-release" # add system users From c4b62bb18fb306857ea364cd86c0fca61d0d4c64 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 25 Sep 2023 16:31:16 +0100 Subject: [PATCH 05/11] scripts/make-rootfs.sh: move non-wrapper calls further up Move the cp/ln calls outside of the WRAPPER call block. The files that are referenced are either disowned by pacman or are explicitly "backup" files, such that pacman will not override them. Signed-off-by: Emil Velikov --- scripts/make-rootfs.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/make-rootfs.sh b/scripts/make-rootfs.sh index cf59108..8843c53 100755 --- a/scripts/make-rootfs.sh +++ b/scripts/make-rootfs.sh @@ -17,6 +17,9 @@ cat pacman-conf.d-noextract.conf >> "$BUILDDIR/etc/pacman.conf" sed 's/Include = /&rootfs/g' < "$BUILDDIR/etc/pacman.conf" > pacman.conf +cp --recursive --preserve=timestamps rootfs/* "$BUILDDIR/" +ln -fs /usr/lib/os-release "$BUILDDIR/etc/os-release" + $WRAPPER -- \ pacman -Sy -r "$BUILDDIR" \ --noconfirm --dbpath "$BUILDDIR/var/lib/pacman" \ @@ -24,14 +27,10 @@ $WRAPPER -- \ --noscriptlet \ --hookdir "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks/" base "$GROUP" -cp --recursive --preserve=timestamps rootfs/* "$BUILDDIR/" - $WRAPPER -- chroot "$BUILDDIR" update-ca-trust $WRAPPER -- chroot "$BUILDDIR" pacman-key --init $WRAPPER -- chroot "$BUILDDIR" pacman-key --populate -ln -fs /usr/lib/os-release "$BUILDDIR/etc/os-release" - # add system users $WRAPPER -- chroot "$BUILDDIR" /usr/bin/systemd-sysusers --root "/" From 2f43fc413a67f535bd3b07422240e94783bccc86 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 25 Sep 2023 16:34:41 +0100 Subject: [PATCH 06/11] scripts/make-dockerfile.sh: tweak sha256 sum handling Drop the spurious cat, we already echo the value the like above so it's pretty clear what it is in the logs. While here, rename the file to the what's commonly used by us (see the ISOs) and others, while removing the misleading "rootfs.tar" - we're having a tar.zst tarball. Signed-off-by: Emil Velikov --- Dockerfile.template | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index 9430d80..262015c 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -17,9 +17,8 @@ RUN apk add --no-cache curl tar zstd # TEMPLATE_ROOTFS_RELEASE_URL RUN TEMPLATE_ROOTFS_DOWNLOAD && \ - echo "TEMPLATE_ROOTFS_HASH" > /tmp/rootfs.tar.sha256 && \ - cat /tmp/rootfs.tar.sha256 && \ - sha256sum -c /tmp/rootfs.tar.sha256 && \ + echo "TEMPLATE_ROOTFS_HASH" > /tmp/sha256sums.txt && \ + sha256sum -c /tmp/sha256sums.txt && \ mkdir /rootfs && \ tar -C /rootfs --extract --file "${ROOTFS}" From a972ead2e07afd10e0125e4de0d8fe9f643588e3 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 25 Sep 2023 16:41:00 +0100 Subject: [PATCH 07/11] scripts/make-dockerfile.sh: rework TEMPLATE_ROOTFS_DOWNLOAD handling Currently the TEMPLATE_ROOTFS_DOWNLOAD handling is overly complicated. For the local builds, we set a ROOTFS=$GROUP.tar.zst. While for remote builds, we: - invoke curl to fetch the remote tarball - ok - do curl and shell escaping contortions to prints the filename - ehhh - that we already now - and rely upon to not change, otherwise sha256sum will fail Just use a dummy "true", for the local builds and a normal curl command otherwise. v2: - don't call curl in a sub-shell - no longer needed Signed-off-by: Emil Velikov --- .gitlab-ci.yml | 5 +++-- Dockerfile.template | 2 +- scripts/make-dockerfile.sh | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a7333b4..038afa8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -243,10 +243,11 @@ release: curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst echo "Uploading ${group}.tar.zst.SHA256" curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst.SHA256 - sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > output/Dockerfile.${group} + sed "/COPY TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > output/Dockerfile.${group} + sed -i "s|TEMPLATE_ROOTFS_FILE|${group}.tar.zst|" output/Dockerfile.${group} package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.zst) sed -i "s|TEMPLATE_ROOTFS_RELEASE_URL|https://gitlab.archlinux.org/archlinux/archlinux-docker/-/releases/v${BUILD_VERSION}|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"\$(curl -sOJL -w \"%{filename_effective}\" \"${package_url}\")\"|" output/Dockerfile.${group} + sed -i "s|TEMPLATE_ROOTFS_DOWNLOAD|curl -sOJL \"${package_url}\"|" output/Dockerfile.${group} sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.zst.SHA256)|" output/Dockerfile.${group} sed -i "s|TEMPLATE_TITLE|Arch Linux ${group} Image|" output/Dockerfile.${group} sed -i "s|TEMPLATE_VERSION_ID|${BUILD_VERSION}|" output/Dockerfile.${group} diff --git a/Dockerfile.template b/Dockerfile.template index 262015c..507f57b 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -20,7 +20,7 @@ RUN TEMPLATE_ROOTFS_DOWNLOAD && \ echo "TEMPLATE_ROOTFS_HASH" > /tmp/sha256sums.txt && \ sha256sum -c /tmp/sha256sums.txt && \ mkdir /rootfs && \ - tar -C /rootfs --extract --file "${ROOTFS}" + tar -C /rootfs --extract --file TEMPLATE_ROOTFS_FILE FROM scratch AS root diff --git a/scripts/make-dockerfile.sh b/scripts/make-dockerfile.sh index 43263e3..782c567 100755 --- a/scripts/make-dockerfile.sh +++ b/scripts/make-dockerfile.sh @@ -7,7 +7,7 @@ declare -r OUTPUTDIR="$2" sed -e "s|TEMPLATE_ROOTFS_FILE|$GROUP.tar.zst|" \ -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \ - -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$GROUP.tar.zst\"|" \ + -e "s|TEMPLATE_ROOTFS_DOWNLOAD|true|" \ -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \ -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \ -e "s|TEMPLATE_VERSION_ID|dev|" \ From 64cb18a0c364f4d3b7cdd5cbedb8f35c61f25bbb Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 25 Sep 2023 17:06:55 +0100 Subject: [PATCH 08/11] scripts/make-dockerfile.sh: drop TEMPLATE_ROOTFS_RELEASE_URL The field is an inline comment, which by default is not present in the podman log. Plus we do have the exact URL list a couple of lines further down. Haven't seen any other Dockerfile have one either, so let's nuke it. Signed-off-by: Emil Velikov --- .gitlab-ci.yml | 1 - Dockerfile.template | 1 - scripts/make-dockerfile.sh | 1 - 3 files changed, 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 038afa8..1a19745 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -246,7 +246,6 @@ release: sed "/COPY TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > output/Dockerfile.${group} sed -i "s|TEMPLATE_ROOTFS_FILE|${group}.tar.zst|" output/Dockerfile.${group} package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.zst) - sed -i "s|TEMPLATE_ROOTFS_RELEASE_URL|https://gitlab.archlinux.org/archlinux/archlinux-docker/-/releases/v${BUILD_VERSION}|" output/Dockerfile.${group} sed -i "s|TEMPLATE_ROOTFS_DOWNLOAD|curl -sOJL \"${package_url}\"|" output/Dockerfile.${group} sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.zst.SHA256)|" output/Dockerfile.${group} sed -i "s|TEMPLATE_TITLE|Arch Linux ${group} Image|" output/Dockerfile.${group} diff --git a/Dockerfile.template b/Dockerfile.template index 507f57b..c691df8 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -15,7 +15,6 @@ COPY TEMPLATE_ROOTFS_FILE / RUN apk add --no-cache curl tar zstd -# TEMPLATE_ROOTFS_RELEASE_URL RUN TEMPLATE_ROOTFS_DOWNLOAD && \ echo "TEMPLATE_ROOTFS_HASH" > /tmp/sha256sums.txt && \ sha256sum -c /tmp/sha256sums.txt && \ diff --git a/scripts/make-dockerfile.sh b/scripts/make-dockerfile.sh index 782c567..545d61c 100755 --- a/scripts/make-dockerfile.sh +++ b/scripts/make-dockerfile.sh @@ -6,7 +6,6 @@ declare -r GROUP="$1" declare -r OUTPUTDIR="$2" sed -e "s|TEMPLATE_ROOTFS_FILE|$GROUP.tar.zst|" \ - -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \ -e "s|TEMPLATE_ROOTFS_DOWNLOAD|true|" \ -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \ -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \ From 0a6910cade1e3771adef3b702721bac826bca025 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Thu, 21 Sep 2023 13:41:47 +0100 Subject: [PATCH 09/11] Use pattern rules in the makefile Less duplication ftw, specially with multilib-devel on the horizon. Unfortunately we'd need to tag the file pattern rules as precious, otherwise make thinks they are intermediate files and nukes them. Signed-off-by: Emil Velikov --- Makefile | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 049d931..31a0b81 100644 --- a/Makefile +++ b/Makefile @@ -6,24 +6,16 @@ OUTPUTDIR=$(shell pwd)/output clean: rm -rf $(BUILDDIR) $(OUTPUTDIR) -$(OUTPUTDIR)/base.tar.zst: - scripts/make-rootfs.sh base $(BUILDDIR) $(OUTPUTDIR) +.PRECIOUS: $(OUTPUTDIR)/%.tar.zst +$(OUTPUTDIR)/%.tar.zst: + scripts/make-rootfs.sh $(*) $(BUILDDIR) $(OUTPUTDIR) -$(OUTPUTDIR)/base-devel.tar.zst: - scripts/make-rootfs.sh base-devel $(BUILDDIR) $(OUTPUTDIR) - -$(OUTPUTDIR)/Dockerfile.base: $(OUTPUTDIR)/base.tar.zst - scripts/make-dockerfile.sh base $(OUTPUTDIR) - -$(OUTPUTDIR)/Dockerfile.base-devel: $(OUTPUTDIR)/base-devel.tar.zst - scripts/make-dockerfile.sh base-devel $(OUTPUTDIR) +.PRECIOUS: $(OUTPUTDIR)/Dockerfile.% +$(OUTPUTDIR)/Dockerfile.%: $(OUTPUTDIR)/%.tar.zst + scripts/make-dockerfile.sh $(*) $(OUTPUTDIR) "true" "Dev" # The following is for local builds only, it is not used by the CI/CD pipeline -.PHONY: image-base -image-base: $(OUTPUTDIR)/Dockerfile.base - ${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.base -t archlinux/archlinux:base $(OUTPUTDIR) - -.PHONY: image-base-devel -image-base-devel: $(OUTPUTDIR)/Dockerfile.base-devel - ${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.base-devel -t archlinux/archlinux:base-devel $(OUTPUTDIR) +all: image-base image-base-devel +image-%: $(OUTPUTDIR)/Dockerfile.% + ${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.$(*) -t archlinux/archlinux:$(*) $(OUTPUTDIR) From d1254eeee40214ab5de307e80e4c705c4603b883 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 2 Oct 2023 12:46:17 +0100 Subject: [PATCH 10/11] scripts/make-dockerfile.sh: reuse in the release stage Bonus point, we actually error out when the git command fails. v2: - fix curl quoting - sed match-complete-line-and-remove - inline update make-dockerfile.sh variables Signed-off-by: Emil Velikov --- .gitlab-ci.yml | 10 ++-------- scripts/make-dockerfile.sh | 14 ++++++++++---- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1a19745..8ccf14f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -243,15 +243,9 @@ release: curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst echo "Uploading ${group}.tar.zst.SHA256" curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst.SHA256 - sed "/COPY TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > output/Dockerfile.${group} - sed -i "s|TEMPLATE_ROOTFS_FILE|${group}.tar.zst|" output/Dockerfile.${group} package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.zst) - sed -i "s|TEMPLATE_ROOTFS_DOWNLOAD|curl -sOJL \"${package_url}\"|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.zst.SHA256)|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_TITLE|Arch Linux ${group} Image|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_VERSION_ID|${BUILD_VERSION}|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_REVISION|${CI_COMMIT_SHA}|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_CREATED|$(date -Is)|" output/Dockerfile.${group} + scripts/make-dockerfile.sh "${group}" "output" "curl -sOJL \"${package_url}\"" "${group}" + sed -i "/^COPY ${group}.tar.zst \/$/d" output/Dockerfile.${group} done - > curl -sSf --request POST -o commit-response.json diff --git a/scripts/make-dockerfile.sh b/scripts/make-dockerfile.sh index 545d61c..b90adaf 100755 --- a/scripts/make-dockerfile.sh +++ b/scripts/make-dockerfile.sh @@ -4,12 +4,18 @@ set -euo pipefail declare -r GROUP="$1" declare -r OUTPUTDIR="$2" +declare -r DOWNLOAD="$3" +declare -r TITLE="$4" + +# Do not use these directly in the sed below - it will mask git failures +BUILD_VERSION="${BUILD_VERSION:-dev}" +CI_COMMIT_SHA="${CI_COMMIT_SHA:-$(git rev-parse HEAD)}" sed -e "s|TEMPLATE_ROOTFS_FILE|$GROUP.tar.zst|" \ - -e "s|TEMPLATE_ROOTFS_DOWNLOAD|true|" \ + -e "s|TEMPLATE_ROOTFS_DOWNLOAD|$DOWNLOAD|" \ -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \ - -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \ - -e "s|TEMPLATE_VERSION_ID|dev|" \ - -e "s|TEMPLATE_REVISION|$(git rev-parse HEAD)|" \ + -e "s|TEMPLATE_TITLE|Arch Linux $TITLE Image|" \ + -e "s|TEMPLATE_VERSION_ID|$BUILD_VERSION|" \ + -e "s|TEMPLATE_REVISION|$CI_COMMIT_SHA|" \ -e "s|TEMPLATE_CREATED|$(date -Is)|" \ Dockerfile.template > "$OUTPUTDIR/Dockerfile.$GROUP" From ed0440cca2b886765c2185427f0e750feb52e112 Mon Sep 17 00:00:00 2001 From: Emil Velikov Date: Mon, 25 Sep 2023 23:37:01 +0100 Subject: [PATCH 11/11] Parametrise Github/Dockerhub dockerfile Not a big win as-is. It will make the upcoming introduction of multilib-devel a bit saner. Signed-off-by: Emil Velikov --- .gitlab-ci.yml | 11 ++++++++++- docker-library.template | 10 ---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8ccf14f..891cee8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -317,7 +317,16 @@ publish-dockerhub: - git reset --hard FETCH_HEAD - head="release/${BUILD_VERSION}" - git checkout -b "$head" - - envsubst < ../docker-library.template > library/archlinux + - cp ../docker-library.template library/archlinux + - | + for group in base base-devel; do + test "${group}" = "base" && extra="latest, " || extra="" + echo "Tags: ${extra}${group}, ${group}-${BUILD_VERSION}" >> library/archlinux + echo "GitCommit: ${BUILD_COMMIT}" >> library/archlinux + echo "GitFetch: refs/tags/v${BUILD_VERSION}" >> library/archlinux + echo "File: Dockerfile.${group}" >> library/archlinux + echo >> library/archlinux + done - git diff - git add library/archlinux - maintainers="$(grep \(@ ../docker-library.template | cut -d\( -f2 | cut -d\) -f1 | xargs)" diff --git a/docker-library.template b/docker-library.template index c2f5dc2..fa3a4f1 100644 --- a/docker-library.template +++ b/docker-library.template @@ -5,13 +5,3 @@ Maintainers: Santiago Torres-Arias (@SantiagoTorres), Justin Kromlinger (@hashworks) GitRepo: https://gitlab.archlinux.org/archlinux/archlinux-docker.git -Tags: latest, base, base-${BUILD_VERSION} -GitCommit: ${BUILD_COMMIT} -GitFetch: refs/tags/v${BUILD_VERSION} -File: Dockerfile.base - -Tags: base-devel, base-devel-${BUILD_VERSION} -GitCommit: ${BUILD_COMMIT} -GitFetch: refs/tags/v${BUILD_VERSION} -File: Dockerfile.base-devel -