diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 0fa7348..0000000 --- a/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!archlinux.tar -!archlinux.tar.xz diff --git a/.gitignore b/.gitignore index 6a9306a..76f4628 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,2 @@ -*~ -*.orig -/.idea -/archlinux.tar -/archlinux.tar.xz -rootfs/etc/pacman.conf +build +output diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eed2256..cc56a69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,45 +1,146 @@ -stages: - - rootfs - - docker - - test +default: + image: "archlinux:latest" -roofs: - stage: rootfs - image: archlinux:latest +stages: + - lint + - rootfs + - image + - test + - upload + - release + - publish + +lint: + stage: lint + image: hadolint/hadolint:latest + # DL3007: We use the latest tag for multistage build + script: hadolint --ignore DL3007 --ignore DL3020 Dockerfile.template + +get_version: + stage: .pre script: + - | + # If we're building a tagged release, use the tag (without the 'v' prefix) as the + # BUILD_VERSION. Otherwise, determine a new BUILD_VERSION. + if [[ -n "$CI_COMMIT_TAG" ]]; then + echo "BUILD_VERSION=${CI_COMMIT_TAG/v/}" > build.env + else + echo "BUILD_VERSION=$(date +%Y%m%d).0.$CI_JOB_ID" > build.env + fi + - export $(< build.env) + - echo "PACKAGE_REGISTRY_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/rootfs/${BUILD_VERSION}" >> build.env + artifacts: + reports: + dotenv: build.env + +.rootfs: + stage: rootfs + before_script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot - - make compress-rootfs artifacts: paths: - - archlinux.tar.xz - expire_in: 10m + - output/* + expire_in: 2h -docker: - stage: docker +rootfs: + extends: .rootfs + except: + - master + - add-base-devel-tags + - schedules + - tags + parallel: + matrix: + - GROUP: [base, base-devel] + script: + - make $PWD/output/$GROUP.tar.xz $PWD/output/Dockerfile.$GROUP + +rootfs:secure: + extends: .rootfs + tags: + - secure + only: + - master + - add-base-devel-tags + - schedules + except: + - tags + parallel: + matrix: + - GROUP: [base, base-devel] + script: + - make $PWD/output/$GROUP.tar.xz $PWD/output/Dockerfile.$GROUP + +.image: + stage: image image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] script: - - test -f archlinux.tar.xz - # kaniko can't process .tar.xz archives - # https://github.com/GoogleContainerTools/kaniko/issues/1107 - - unxz archlinux.tar.xz - - test -f archlinux.tar - - sed -i 's/archlinux\.tar\.xz/archlinux\.tar/g' Dockerfile - - echo "Building ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - /kaniko/executor --whitelist-var-run="false" - --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile - --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} + --context $CI_PROJECT_DIR/output + --dockerfile $CI_PROJECT_DIR/output/Dockerfile.$GROUP + --destination $CI_REGISTRY_IMAGE:$GROUP-$CI_COMMIT_REF_SLUG -test: +image:build: + extends: .image + except: + - master + - add-base-devel-tags + - schedules + - tags + parallel: + matrix: + - GROUP: [base, base-devel] + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + +image:build:secure: + extends: .image + tags: + - secure + only: + - master + - add-base-devel-tags + - schedules + except: + - tags + parallel: + matrix: + - GROUP: [base, base-devel] + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json + +image:publish:secure: + extends: .image + tags: + - secure + only: + - tags + parallel: + matrix: + - GROUP: [base, base-devel] + before_script: + - echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR/ci/$GROUP + --dockerfile $CI_PROJECT_DIR/ci/$GROUP/Dockerfile + --destination archlinux/archlinux:$GROUP-$BUILD_VERSION + +.test: stage: test - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} - needs: - - job: docker - artifacts: false + dependencies: [] + only: + variables: + # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663 + # This is fine as at this point we're sure that the release works anyway. + - $GITLAB_USER_EMAIL != "project10185_bot2@example.com" + except: + refs: + - tags script: - pacman -Sy - pacman -Qqk @@ -48,3 +149,94 @@ test: - id -u http - locale | grep -q UTF-8 +test:base: + extends: .test + image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + +test:base-devel: + extends: .test + image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + after_script: + - gcc -v + - g++ -v + - make -v + +release: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest + tags: + - secure + only: + refs: + - schedules + variables: + - $SCHEDULED_PUBLISH == "TRUE" + before_script: + - apk add jq curl + script: + - | + for group in base base-devel; do + sed -i "s|${group}.tar.xz|${group}-${BUILD_VERSION}.tar.xz|" output/${group}.tar.xz.SHA256 + echo "Uploading ${group}.tar.xz" + curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz + echo "Uploading ${group}.tar.xz.SHA256" + curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 + sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile + package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.xz) + sed -i "s|TEMPLATE_ROOTFS_URL|${package_url}|" ci/${group}/Dockerfile + sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile + done + - > + curl -sSf --request POST + --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" + --form "branch=add-base-devel-tags" + --form "commit_message=Release ${BUILD_VERSION}" + --form "actions[][action]=update" + --form "actions[][file_path]=ci/base/Dockerfile" + --form "actions[][content]=> rootfs/etc/pacman.conf + mkdir -vp $(BUILDDIR)/var/lib/pacman/ $(OUTPUTDIR) + install -Dm644 /usr/share/devtools/pacman-extra.conf $(BUILDDIR)/etc/pacman.conf + cat pacman-conf.d-noextract.conf >> $(BUILDDIR)/etc/pacman.conf fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ - --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ - --config rootfs/etc/pacman.conf \ + --noconfirm --dbpath $(BUILDDIR)/var/lib/pacman \ + --config $(BUILDDIR)/etc/pacman.conf \ --noscriptlet \ - --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) + --hookdir $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks/ $(2) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ - + # remove passwordless login for root (see CVE-2019-5021 for reference) sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" # fakeroot to map the gid/uid of the builder process to root # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar - rm -rf $(BUILDDIR) alpm-hooks + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar -archlinux.tar: rootfs + cd $(OUTPUTDIR); xz -9 -T0 -f $(1).tar; sha256sum $(1).tar.xz > $(1).tar.xz.SHA256 +endef -compress-rootfs: archlinux.tar - xz -9 -T"$(XZ_THREADS)" -f archlinux.tar +define dockerfile + sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.xz|" \ + -e "s|TEMPLATE_ROOTFS_URL|file:///$(1).tar.xz|" \ + -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.xz.SHA256)|" \ + Dockerfile.template > $(OUTPUTDIR)/Dockerfile.$(1) +endef -docker-image: compress-rootfs - docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . +.PHONY: clean +clean: + rm -rf $(BUILDDIR) $(OUTPUTDIR) -docker-image-test: docker-image - # FIXME: /etc/mtab is hidden by docker so the stricter -Qkk fails - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Sy && /usr/bin/pacman -Qqk" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm docker && docker -v" # Ensure that the image does not include a private key - ! docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) pacman-key --lsign-key pierre@archlinux.de - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/id -u http" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm grep && locale | grep -q UTF-8" +$(OUTPUTDIR)/base.tar.xz: + $(call rootfs,base,base) -docker-push: - docker login -u $(DOCKER_USER) - docker push $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) +$(OUTPUTDIR)/base-devel.tar.xz: + $(call rootfs,base-devel,base base-devel) -.PHONY: rootfs docker-image docker-image-test docker-push +$(OUTPUTDIR)/Dockerfile.base: + $(call dockerfile,base) + +$(OUTPUTDIR)/Dockerfile.base-devel: + $(call dockerfile,base-devel) + +.PHONY: docker-image-base +image-base: $(OUTPUTDIR)/base.tar.xz $(OUTPUTDIR)/Dockerfile.base + docker build -f $(OUTPUTDIR)/Dockerfile.base -t archlinux/archlinux:base $(OUTPUTDIR) + +.PHONY: docker-image-base-devel +image-base-devel: $(OUTPUTDIR)/base-devel.tar.xz $(OUTPUTDIR)/Dockerfile.base-devel + docker build -f $(OUTPUTDIR)/Dockerfile.base-devel -t archlinux/archlinux:base-devel $(OUTPUTDIR) diff --git a/README.md b/README.md index 8e80316..cae1c75 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,26 @@ -# Docker Base Image for Arch Linux [![Build Status](https://travis-ci.org/archlinux/archlinux-docker.svg?branch=master)](https://travis-ci.org/archlinux/archlinux-docker) -This repository contains all scripts and files needed to create a Docker base image for the Arch Linux distribution. +# Arch Linux Docker Image +[![pipeline status](https://gitlab.archlinux.org/archlinux/archlinux-docker/badges/master/pipeline.svg)](https://gitlab.archlinux.org/archlinux/archlinux-docker/-/commits/master) + +This repository contains all scripts and files needed to create a Docker image for Arch Linux. + ## Dependencies Install the following Arch Linux packages: + * make * devtools * docker * fakechroot * fakeroot + +Make sure your user can directly interact with Docker (ie. `docker info` works). + ## Usage -Run `make docker-image` as root to build the base image. -## Purpose -* Provide the Arch experience in a Docker Image +Run `make docker-image-base` to build the image `archlinux/archlinux:base` with the +`base` group installed. You can also run `make docker-image-base-devel` to +build the image `archlinux/archlinux:base-devel` with the `base-devel` group installed. + +## Principles +* Provide the Arch experience in a Docker image * Provide the most simple but complete image to base every other upon * `pacman` needs to work out of the box * All installed packages have to be kept unmodified diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile new file mode 100644 index 0000000..097fcdb --- /dev/null +++ b/ci/base-devel/Dockerfile @@ -0,0 +1,32 @@ +# We're using a multistage Docker build here in order to allow us to release a self-verifying +# Docker image when built on the official Docker infrastructure. +# They require us to verify the source integrity in some way while making sure that this is a +# reproducible build. +# See https://github.com/docker-library/official-images#image-build +# In order to achieve this, we externally host the rootfs archives and their checksums and then +# just download and verify it in the first stage of this Dockerfile. +# The second stage is for actually configuring the system a little bit. +# Some templating is done in order to allow us to easily build different configurations and to +# allow us to automate the releaes process. +FROM archlinux:latest AS verify +SHELL ["/bin/bash", "-c"] +RUN ROOTFS="$(curl -OJL --continue-at - -w "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/53/download)" && \ + sha256sum -c <<< "452e26d9775a76e95bd582b96c742844a3e27b90147e1f6e1bc6dd3b82e8a558 base-devel-20201025.0.7220.tar.xz" && \ + mkdir /rootfs && \ + tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" + +FROM scratch AS root +COPY --from=verify /rootfs/ / + +# manually run all alpm hooks that can't be run inside the fakechroot +RUN ldconfig && update-ca-trust && locale-gen +RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' + +# update /etc/os-release +RUN ln -s /usr/lib/os-release /etc/os-release + +# initialize the archlinux keyring, but discard any private key that may be shipped. +RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" + +ENV LANG=en_US.UTF-8 +CMD ["/usr/bin/bash"] diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile new file mode 100644 index 0000000..b33f495 --- /dev/null +++ b/ci/base/Dockerfile @@ -0,0 +1,32 @@ +# We're using a multistage Docker build here in order to allow us to release a self-verifying +# Docker image when built on the official Docker infrastructure. +# They require us to verify the source integrity in some way while making sure that this is a +# reproducible build. +# See https://github.com/docker-library/official-images#image-build +# In order to achieve this, we externally host the rootfs archives and their checksums and then +# just download and verify it in the first stage of this Dockerfile. +# The second stage is for actually configuring the system a little bit. +# Some templating is done in order to allow us to easily build different configurations and to +# allow us to automate the releaes process. +FROM archlinux:latest AS verify +SHELL ["/bin/bash", "-c"] +RUN ROOTFS="$(curl -OJL --continue-at - -w "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/51/download)" && \ + sha256sum -c <<< "175387448f7992b2760e758bdb75bfd45de7d2bf5ad2940add9e19a96ffb4129 base-20201025.0.7220.tar.xz" && \ + mkdir /rootfs && \ + tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" + +FROM scratch AS root +COPY --from=verify /rootfs/ / + +# manually run all alpm hooks that can't be run inside the fakechroot +RUN ldconfig && update-ca-trust && locale-gen +RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' + +# update /etc/os-release +RUN ln -s /usr/lib/os-release /etc/os-release + +# initialize the archlinux keyring, but discard any private key that may be shipped. +RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" + +ENV LANG=en_US.UTF-8 +CMD ["/usr/bin/bash"] diff --git a/ci/get-public-download-for-generic-package.sh b/ci/get-public-download-for-generic-package.sh new file mode 100755 index 0000000..53922b0 --- /dev/null +++ b/ci/get-public-download-for-generic-package.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +set -eu + +package_name=$1 + +package_id=$(curl -sSf --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") +package_file_id=$(curl -sSf --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"$package_name\") | .id") +echo "https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${package_file_id}/download" diff --git a/exclude b/exclude index 0a45d22..e847b05 100644 --- a/exclude +++ b/exclude @@ -5,6 +5,7 @@ ./dev ./etc/hostname ./etc/machine-id +./etc/resolv.conf ./etc/pacman.d/gnupg/openpgp-revocs.d/* ./etc/pacman.d/gnupg/private-keys-v1.d/* ./etc/pacman.d/gnupg/pubring.gpg~ @@ -13,4 +14,5 @@ ./tmp/* ./var/cache/pacman/pkg/* ./var/lib/pacman/sync/* -./var/tmp/* \ No newline at end of file +./var/tmp/* +./alpm-hooks diff --git a/packages b/packages deleted file mode 100644 index 2c07590..0000000 --- a/packages +++ /dev/null @@ -1,10 +0,0 @@ -sed -gzip -pacman -systemd -gawk -file -grep -tar -procps-ng -licenses diff --git a/rootfs/etc/pacman.d/mirrorlist b/rootfs/etc/pacman.d/mirrorlist index 3735559..31adb47 100644 --- a/rootfs/etc/pacman.d/mirrorlist +++ b/rootfs/etc/pacman.d/mirrorlist @@ -1,2 +1,3 @@ +Server = https://mirror.pkgbuild.com/$repo/os/$arch Server = https://mirror.rackspace.com/archlinux/$repo/os/$arch Server = https://mirror.leaseweb.net/archlinux/$repo/os/$arch