diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 678414d..891cee8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -80,9 +80,6 @@ rootfs: - releases@archlinux/archlinux-docker - schedules@archlinux/archlinux-docker - tags@archlinux/archlinux-docker - parallel: - matrix: - - GROUP: [base, base-devel] rootfs:secure: extends: .rootfs @@ -153,9 +150,6 @@ image:publish:secure: - vm only: - tags@archlinux/archlinux-docker - parallel: - matrix: - - GROUP: [base, base-devel] before_script: - pacman -Syu --noconfirm podman - podman login -u "$DOCKERHUB_USERNAME" -p "$DOCKERHUB_ACCESS_TOKEN" "docker.io" @@ -167,26 +161,18 @@ image:publish:secure: -f "$CI_PROJECT_DIR/Dockerfile.$GROUP" -t "archlinux:$GROUP-$BUILD_VERSION" "$CI_PROJECT_DIR" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:$GROUP" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:$GROUP" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:$GROUP" - - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman push "docker.io/archlinux/archlinux:$GROUP" - - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "docker.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman push "quay.io/archlinux/archlinux:$GROUP" - - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "quay.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - podman push "ghcr.io/archlinux/archlinux:$GROUP" - - podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "ghcr.io/archlinux/archlinux:$GROUP-$BUILD_VERSION" - - if [[ "$GROUP" == "base" ]]; then - podman tag "archlinux:$GROUP-$BUILD_VERSION" "docker.io/archlinux/archlinux:latest"; - podman tag "archlinux:$GROUP-$BUILD_VERSION" "quay.io/archlinux/archlinux:latest"; - podman tag "archlinux:$GROUP-$BUILD_VERSION" "ghcr.io/archlinux/archlinux:latest"; - podman push "docker.io/archlinux/archlinux:latest"; - podman push "quay.io/archlinux/archlinux:latest"; - podman push "ghcr.io/archlinux/archlinux:latest"; - fi + - | + for host in "docker.io" "quay.io" "ghcr.io"; do + podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:$GROUP" + podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:$GROUP-$BUILD_VERSION" + podman push "$host/archlinux/archlinux:$GROUP" + podman push --sign-by-sigstore=<(sed "s/TEMPLATE_OIDC_ID_TOKEN/${SIGSTORE_ID_TOKEN}/" sigstore-param-file.yaml) "$host/archlinux/archlinux:$GROUP-$BUILD_VERSION" + + if [[ "$GROUP" == "base" ]]; then + podman tag "archlinux:$GROUP-$BUILD_VERSION" "$host/archlinux/archlinux:latest" + podman push "$host/archlinux/archlinux:latest" + fi + done .test: @@ -257,15 +243,9 @@ release: curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst echo "Uploading ${group}.tar.zst.SHA256" curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.zst.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.zst.SHA256 - sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > output/Dockerfile.${group} package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.zst) - sed -i "s|TEMPLATE_ROOTFS_RELEASE_URL|https://gitlab.archlinux.org/archlinux/archlinux-docker/-/releases/v${BUILD_VERSION}|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"\$(curl -sOJL -w \"%{filename_effective}\" \"${package_url}\")\"|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.zst.SHA256)|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_TITLE|Arch Linux ${group} Image|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_VERSION_ID|${BUILD_VERSION}|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_REVISION|${CI_COMMIT_SHA}|" output/Dockerfile.${group} - sed -i "s|TEMPLATE_CREATED|$(date -Is)|" output/Dockerfile.${group} + scripts/make-dockerfile.sh "${group}" "output" "curl -sOJL \"${package_url}\"" "${group}" + sed -i "/^COPY ${group}.tar.zst \/$/d" output/Dockerfile.${group} done - > curl -sSf --request POST -o commit-response.json @@ -337,7 +317,16 @@ publish-dockerhub: - git reset --hard FETCH_HEAD - head="release/${BUILD_VERSION}" - git checkout -b "$head" - - envsubst < ../docker-library.template > library/archlinux + - cp ../docker-library.template library/archlinux + - | + for group in base base-devel; do + test "${group}" = "base" && extra="latest, " || extra="" + echo "Tags: ${extra}${group}, ${group}-${BUILD_VERSION}" >> library/archlinux + echo "GitCommit: ${BUILD_COMMIT}" >> library/archlinux + echo "GitFetch: refs/tags/v${BUILD_VERSION}" >> library/archlinux + echo "File: Dockerfile.${group}" >> library/archlinux + echo >> library/archlinux + done - git diff - git add library/archlinux - maintainers="$(grep \(@ ../docker-library.template | cut -d\( -f2 | cut -d\) -f1 | xargs)" diff --git a/Dockerfile.template b/Dockerfile.template index 9430d80..c691df8 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -15,13 +15,11 @@ COPY TEMPLATE_ROOTFS_FILE / RUN apk add --no-cache curl tar zstd -# TEMPLATE_ROOTFS_RELEASE_URL RUN TEMPLATE_ROOTFS_DOWNLOAD && \ - echo "TEMPLATE_ROOTFS_HASH" > /tmp/rootfs.tar.sha256 && \ - cat /tmp/rootfs.tar.sha256 && \ - sha256sum -c /tmp/rootfs.tar.sha256 && \ + echo "TEMPLATE_ROOTFS_HASH" > /tmp/sha256sums.txt && \ + sha256sum -c /tmp/sha256sums.txt && \ mkdir /rootfs && \ - tar -C /rootfs --extract --file "${ROOTFS}" + tar -C /rootfs --extract --file TEMPLATE_ROOTFS_FILE FROM scratch AS root diff --git a/Makefile b/Makefile index 38c8e37..31a0b81 100644 --- a/Makefile +++ b/Makefile @@ -2,76 +2,20 @@ OCITOOL=podman # or docker BUILDDIR=$(shell pwd)/build OUTPUTDIR=$(shell pwd)/output -define rootfs - mkdir -vp $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks - find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(BUILDDIR)/alpm-hooks{} \; - - mkdir -vp $(BUILDDIR)/var/lib/pacman/ $(OUTPUTDIR) - install -Dm644 /usr/share/devtools/pacman.conf.d/extra.conf $(BUILDDIR)/etc/pacman.conf - cat pacman-conf.d-noextract.conf >> $(BUILDDIR)/etc/pacman.conf - - sed 's/Include = /&rootfs/g' < $(BUILDDIR)/etc/pacman.conf > pacman.conf - - fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ - --noconfirm --dbpath $(BUILDDIR)/var/lib/pacman \ - --config pacman.conf \ - --noscriptlet \ - --hookdir $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks/ $(2) - - cp --recursive --preserve=timestamps rootfs/* $(BUILDDIR)/ - - fakechroot -- fakeroot -- chroot $(BUILDDIR) update-ca-trust - fakechroot -- fakeroot -- chroot $(BUILDDIR) sh -c 'pacman-key --init && pacman-key --populate && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*"' - - ln -fs /usr/lib/os-release $(BUILDDIR)/etc/os-release - - # add system users - fakechroot -- fakeroot -- chroot $(BUILDDIR) /usr/bin/systemd-sysusers --root "/" - - # remove passwordless login for root (see CVE-2019-5021 for reference) - sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" - - # fakeroot to map the gid/uid of the builder process to root - # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar - - cd $(OUTPUTDIR); zstd --long -T0 -8 $(1).tar; sha256sum $(1).tar.zst > $(1).tar.zst.SHA256 -endef - -define dockerfile - sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.zst|" \ - -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \ - -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$(1).tar.zst\"|" \ - -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.zst.SHA256)|" \ - -e "s|TEMPLATE_TITLE|Arch Linux Dev Image|" \ - -e "s|TEMPLATE_VERSION_ID|dev|" \ - -e "s|TEMPLATE_REVISION|$$(git rev-parse HEAD)|" \ - -e "s|TEMPLATE_CREATED|$$(date -Is)|" \ - Dockerfile.template > $(OUTPUTDIR)/Dockerfile.$(1) -endef - .PHONY: clean clean: rm -rf $(BUILDDIR) $(OUTPUTDIR) -$(OUTPUTDIR)/base.tar.zst: - $(call rootfs,base,base) +.PRECIOUS: $(OUTPUTDIR)/%.tar.zst +$(OUTPUTDIR)/%.tar.zst: + scripts/make-rootfs.sh $(*) $(BUILDDIR) $(OUTPUTDIR) -$(OUTPUTDIR)/base-devel.tar.zst: - $(call rootfs,base-devel,base base-devel) - -$(OUTPUTDIR)/Dockerfile.base: $(OUTPUTDIR)/base.tar.zst - $(call dockerfile,base) - -$(OUTPUTDIR)/Dockerfile.base-devel: $(OUTPUTDIR)/base-devel.tar.zst - $(call dockerfile,base-devel) +.PRECIOUS: $(OUTPUTDIR)/Dockerfile.% +$(OUTPUTDIR)/Dockerfile.%: $(OUTPUTDIR)/%.tar.zst + scripts/make-dockerfile.sh $(*) $(OUTPUTDIR) "true" "Dev" # The following is for local builds only, it is not used by the CI/CD pipeline -.PHONY: image-base -image-base: $(OUTPUTDIR)/Dockerfile.base - ${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.base -t archlinux/archlinux:base $(OUTPUTDIR) - -.PHONY: image-base-devel -image-base-devel: $(OUTPUTDIR)/Dockerfile.base-devel - ${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.base-devel -t archlinux/archlinux:base-devel $(OUTPUTDIR) +all: image-base image-base-devel +image-%: $(OUTPUTDIR)/Dockerfile.% + ${OCITOOL} build -f $(OUTPUTDIR)/Dockerfile.$(*) -t archlinux/archlinux:$(*) $(OUTPUTDIR) diff --git a/docker-library.template b/docker-library.template index c2f5dc2..fa3a4f1 100644 --- a/docker-library.template +++ b/docker-library.template @@ -5,13 +5,3 @@ Maintainers: Santiago Torres-Arias (@SantiagoTorres), Justin Kromlinger (@hashworks) GitRepo: https://gitlab.archlinux.org/archlinux/archlinux-docker.git -Tags: latest, base, base-${BUILD_VERSION} -GitCommit: ${BUILD_COMMIT} -GitFetch: refs/tags/v${BUILD_VERSION} -File: Dockerfile.base - -Tags: base-devel, base-devel-${BUILD_VERSION} -GitCommit: ${BUILD_COMMIT} -GitFetch: refs/tags/v${BUILD_VERSION} -File: Dockerfile.base-devel - diff --git a/scripts/make-dockerfile.sh b/scripts/make-dockerfile.sh new file mode 100755 index 0000000..b90adaf --- /dev/null +++ b/scripts/make-dockerfile.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -euo pipefail + +declare -r GROUP="$1" +declare -r OUTPUTDIR="$2" +declare -r DOWNLOAD="$3" +declare -r TITLE="$4" + +# Do not use these directly in the sed below - it will mask git failures +BUILD_VERSION="${BUILD_VERSION:-dev}" +CI_COMMIT_SHA="${CI_COMMIT_SHA:-$(git rev-parse HEAD)}" + +sed -e "s|TEMPLATE_ROOTFS_FILE|$GROUP.tar.zst|" \ + -e "s|TEMPLATE_ROOTFS_DOWNLOAD|$DOWNLOAD|" \ + -e "s|TEMPLATE_ROOTFS_HASH|$(cat $OUTPUTDIR/$GROUP.tar.zst.SHA256)|" \ + -e "s|TEMPLATE_TITLE|Arch Linux $TITLE Image|" \ + -e "s|TEMPLATE_VERSION_ID|$BUILD_VERSION|" \ + -e "s|TEMPLATE_REVISION|$CI_COMMIT_SHA|" \ + -e "s|TEMPLATE_CREATED|$(date -Is)|" \ + Dockerfile.template > "$OUTPUTDIR/Dockerfile.$GROUP" diff --git a/scripts/make-rootfs.sh b/scripts/make-rootfs.sh new file mode 100755 index 0000000..8843c53 --- /dev/null +++ b/scripts/make-rootfs.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +set -euo pipefail + +declare -r WRAPPER="fakechroot -- fakeroot" + +declare -r GROUP="$1" +declare -r BUILDDIR="$2" +declare -r OUTPUTDIR="$3" + +mkdir -vp "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks" +find /usr/share/libalpm/hooks -exec ln -sf /dev/null "$BUILDDIR/alpm-hooks"{} \; + +mkdir -vp "$BUILDDIR/var/lib/pacman/" "$OUTPUTDIR" +install -Dm644 /usr/share/devtools/pacman.conf.d/extra.conf "$BUILDDIR/etc/pacman.conf" +cat pacman-conf.d-noextract.conf >> "$BUILDDIR/etc/pacman.conf" + +sed 's/Include = /&rootfs/g' < "$BUILDDIR/etc/pacman.conf" > pacman.conf + +cp --recursive --preserve=timestamps rootfs/* "$BUILDDIR/" +ln -fs /usr/lib/os-release "$BUILDDIR/etc/os-release" + +$WRAPPER -- \ + pacman -Sy -r "$BUILDDIR" \ + --noconfirm --dbpath "$BUILDDIR/var/lib/pacman" \ + --config pacman.conf \ + --noscriptlet \ + --hookdir "$BUILDDIR/alpm-hooks/usr/share/libalpm/hooks/" base "$GROUP" + +$WRAPPER -- chroot "$BUILDDIR" update-ca-trust +$WRAPPER -- chroot "$BUILDDIR" pacman-key --init +$WRAPPER -- chroot "$BUILDDIR" pacman-key --populate + +# add system users +$WRAPPER -- chroot "$BUILDDIR" /usr/bin/systemd-sysusers --root "/" + +# remove passwordless login for root (see CVE-2019-5021 for reference) +sed -i -e 's/^root::/root:!:/' "$BUILDDIR/etc/shadow" + +# fakeroot to map the gid/uid of the builder process to root +# fixes #22 +fakeroot -- \ + tar \ + --numeric-owner \ + --xattrs \ + --acls \ + --exclude-from=exclude \ + -C "$BUILDDIR" \ + -c . \ + -f "$OUTPUTDIR/$GROUP.tar" + +cd "$OUTPUTDIR" +zstd --long -T0 -8 "$GROUP.tar" +sha256sum "$GROUP.tar.zst" > "$GROUP.tar.zst.SHA256"