From d51a887efb57340be5b4a7c33b6969d758d88ffe Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sat, 29 Aug 2020 04:51:05 +0200 Subject: [PATCH] Add base-devel tag --- .dockerignore | 3 - .gitignore | 4 +- .gitlab-ci.yml | 169 ++++++++++++++++++++++++++---- Dockerfile => Dockerfile.template | 8 +- Makefile | 70 +++++++++---- README.md | 18 +++- base-devel/Dockerfile | 0 base/Dockerfile | 0 ci/release.py | 86 +++++++++++++++ rootfs/etc/pacman.d/mirrorlist | 1 + 10 files changed, 305 insertions(+), 54 deletions(-) delete mode 100644 .dockerignore rename Dockerfile => Dockerfile.template (59%) create mode 100644 base-devel/Dockerfile create mode 100644 base/Dockerfile create mode 100755 ci/release.py diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 0fa7348..0000000 --- a/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!archlinux.tar -!archlinux.tar.xz diff --git a/.gitignore b/.gitignore index 6a9306a..94f5842 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *~ *.orig /.idea -/archlinux.tar -/archlinux.tar.xz +/base.tar* +/base-devel.tar* rootfs/etc/pacman.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eed2256..97520a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,44 +1,91 @@ stages: + - lint - rootfs - docker - test + - release + - publish -roofs: +lint: + stage: lint + image: hadolint/hadolint:latest + script: hadolint --ignore DL3020 Dockerfile.template + +rootfs:base: stage: rootfs image: archlinux:latest + needs: + - job: "lint" + before_script: + - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot - - make compress-rootfs + - make base.tar.xz artifacts: paths: - - archlinux.tar.xz + - base.tar.xz expire_in: 10m + reports: + dotenv: build.env -docker: +rootfs:base-devel: + stage: rootfs + image: archlinux:latest + needs: + - job: "lint" + before_script: + - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env + script: + - pacman -Syu --noconfirm make devtools fakechroot fakeroot + - make base-devel.tar.xz + artifacts: + paths: + - base-devel.tar.xz + expire_in: 10m + reports: + dotenv: build.env + +docker:base: stage: docker image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] - script: - - test -f archlinux.tar.xz - # kaniko can't process .tar.xz archives - # https://github.com/GoogleContainerTools/kaniko/issues/1107 - - unxz archlinux.tar.xz - - test -f archlinux.tar - - sed -i 's/archlinux\.tar\.xz/archlinux\.tar/g' Dockerfile - - echo "Building ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" + needs: + - job: "rootfs:base" + before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base + - unxz base.tar.xz + script: - /kaniko/executor --whitelist-var-run="false" --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile - --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} + --dockerfile $CI_PROJECT_DIR/Dockerfile.base + --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG -test: - stage: test - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} +docker:base-devel: + stage: docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] needs: - - job: docker + - job: "rootfs:base-devel" + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel + - unxz base-devel.tar.xz + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel + --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + +test:base: + stage: test + image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + needs: + - job: "docker:base" artifacts: false script: - pacman -Sy @@ -48,3 +95,89 @@ test: - id -u http - locale | grep -q UTF-8 +test:base-devel: + stage: test + image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + needs: + - job: "docker:base-devel" + artifacts: false + script: + - pacman -Sy + - pacman -Qqk + - pacman -Syu --noconfirm docker grep + - docker -v + - id -u http + - locale | grep -q UTF-8 + - gcc -v + - g++ -v + - make -v + +release: + stage: release + image: archlinux:latest + only: + refs: + - master + - add-base-devel-tags + variables: + - $SCHEDULED_PUBLISH == "TRUE" + needs: + - job: "test:base" + - job: "test:base-devel" + before_script: + - pacman -Syu python-gitlab + script: + - python ci/release.py + tags: + - secure + +# Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base +# publish:org:base: +# stage: publish +# image: +# name: gcr.io/go-containerregistry/crane:debug +# entrypoint: [""] +# needs: +# - job: "test:base" +# artifacts: true +# tags: +# - secure +# variables: +# GIT_STRATEGY: none +# before_script: +# - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux +# script: +# - crane cp $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG archlinux/archlinux:base +# - crane tag archlinux/archlinux:base latest +# - crane tag archlinux/archlinux:base base-$BUILD_DATE +# only: +# variables: +# - $SCHEDULED_PUBLISH == "TRUE" + +# Publish base-devel to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base-devel +# publish:org:base-devel: +# stage: publish +# image: +# name: gcr.io/go-containerregistry/crane:debug +# entrypoint: [""] +# needs: +# - job: "test:base-devel" +# artifacts: true +# tags: +# - secure +# variables: +# GIT_STRATEGY: none +# before_script: +# - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux +# script: +# - crane cp $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG archlinux/archlinux:base-devel +# - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_DATE +# only: +# variables: +# - $SCHEDULED_PUBLISH == "TRUE" + +# Publish to the official Docker namespace: https://hub.docker.com/_/archlinux +# publish:official: +# TODO No idea right now how we're going to automatically do the official Docker Hub pull request diff --git a/Dockerfile b/Dockerfile.template similarity index 59% rename from Dockerfile rename to Dockerfile.template index 02fbabc..ffe6415 100644 --- a/Dockerfile +++ b/Dockerfile.template @@ -1,15 +1,15 @@ -FROM scratch -ADD archlinux.tar.xz / +FROM scratch AS base +ADD TEMPLATE_LOCATION_HERE / # manually run all alpm hooks that can't be run inside the fakechroot -RUN ldconfig && update-ca-trust && locale-gen +RUN ldconfig && update-ca-trust && locale-gen RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' # update /etc/os-release RUN ln -s /usr/lib/os-release /etc/os-release # initialize the archlinux keyring, but discard any private key that may be shipped. -RUN pacman-key --init && pacman-key --populate archlinux && rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}* +RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 61fb0db..1736e13 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,14 @@ DOCKER_USER:=pierres -DOCKER_ORGANIZATION=archlinux -DOCKER_IMAGE:=base BUILDDIR=build PWD=$(shell pwd) -XZ_THREADS ?= 0 - +.PHONY: hooks hooks: mkdir -p alpm-hooks/usr/share/libalpm/hooks find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(PWD)/alpm-hooks{} \; -rootfs: hooks +.PHONY: rootfs-base +rootfs-base: hooks mkdir -vp $(BUILDDIR)/var/lib/pacman/ cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf @@ -18,35 +16,61 @@ rootfs: hooks --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ --config rootfs/etc/pacman.conf \ --noscriptlet \ - --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) + --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ - + # remove passwordless login for root (see CVE-2019-5021 for reference) sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" # fakeroot to map the gid/uid of the builder process to root # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base.tar rm -rf $(BUILDDIR) alpm-hooks -archlinux.tar: rootfs +.PHONY: rootfs-base-devel +rootfs-base-devel: hooks + mkdir -vp $(BUILDDIR)/var/lib/pacman/ + cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf + cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf + fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ + --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ + --config rootfs/etc/pacman.conf \ + --noscriptlet \ + --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base base-devel + cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ -compress-rootfs: archlinux.tar - xz -9 -T"$(XZ_THREADS)" -f archlinux.tar + # remove passwordless login for root (see CVE-2019-5021 for reference) + sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" -docker-image: compress-rootfs - docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . + # fakeroot to map the gid/uid of the builder process to root + # fixes #22 + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base-devel.tar + rm -rf $(BUILDDIR) alpm-hooks -docker-image-test: docker-image - # FIXME: /etc/mtab is hidden by docker so the stricter -Qkk fails - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Sy && /usr/bin/pacman -Qqk" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm docker && docker -v" # Ensure that the image does not include a private key - ! docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) pacman-key --lsign-key pierre@archlinux.de - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/id -u http" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm grep && locale | grep -q UTF-8" +base.tar.xz: rootfs-base + xz -9 -T0 -f base.tar -docker-push: +base-devel.tar.xz: rootfs-base-devel + xz -9 -T0 -f base-devel.tar + +.PHONY: docker-image-base +docker-image-base: base.tar.xz + unxz base.tar.xz + sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base + docker build -f Dockerfile.base -t archlinux/archlinux:base . + +.PHONY: docker-image-base-devel +docker-image-base-devel: base-devel.tar.xz + unxz base-devel.tar.xz + sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel + docker build -f Dockerfile.base-devel -t archlinux/archlinux:base-devel . + +.PHONY: docker-push-base +docker-push-base: docker login -u $(DOCKER_USER) - docker push $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) + docker push archlinux/archlinux:base -.PHONY: rootfs docker-image docker-image-test docker-push +.PHONY: docker-push-base-devel +docker-push-base-devel: + docker login -u $(DOCKER_USER) + docker push archlinux/archlinux:base-devel diff --git a/README.md b/README.md index 8e80316..30b0e77 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,26 @@ -# Docker Base Image for Arch Linux [![Build Status](https://travis-ci.org/archlinux/archlinux-docker.svg?branch=master)](https://travis-ci.org/archlinux/archlinux-docker) -This repository contains all scripts and files needed to create a Docker base image for the Arch Linux distribution. +# Docker Base Image for Arch Linux +[![pipeline status](https://gitlab.archlinux.org/archlinux/archlinux-docker/badges/master/pipeline.svg)](https://gitlab.archlinux.org/archlinux/archlinux-docker/-/commits/master) + +This repository contains all scripts and files needed to create a Docker base image for Arch Linux. + ## Dependencies Install the following Arch Linux packages: + * make * devtools * docker * fakechroot * fakeroot + +Make sure your user can directly interact with Docker (ie. `docker info` works). + ## Usage -Run `make docker-image` as root to build the base image. +Run `make docker-image-base` to build the image `archlinux:base` with the +`base` group installed. You can also run `make docker-image-base-devel` to +build the image `archlinux:base-devel` with the `base-devel` group installed. + ## Purpose -* Provide the Arch experience in a Docker Image +* Provide the Arch experience in a Docker image * Provide the most simple but complete image to base every other upon * `pacman` needs to work out of the box * All installed packages have to be kept unmodified diff --git a/base-devel/Dockerfile b/base-devel/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/ci/release.py b/ci/release.py new file mode 100755 index 0000000..5896f20 --- /dev/null +++ b/ci/release.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +""" +Should only be called from GitLab CI! + +Required env vars: + - GITLAB_PROJECT_TOKEN + - BUILD_DATE + - CI_PROJECT_ID + - CI_PROJECT_URL +""" + +import os +from pathlib import Path +import gitlab + +token = os.environ['GITLAB_PROJECT_TOKEN'] +build_date = os.environ['BUILD_DATE'] +project_id = os.environ['CI_PROJECT_ID'] +project_url = os.environ['CI_PROJECT_URL'] + +if __name__ == "__main__": + gl = gitlab.Gitlab("https://gitlab.archlinux.org", token) + project = gl.projects.get(project_id) + + print("Uploading base.tar.xz") + base_uploaded_url = project.upload( + f"base-{build_date}.tar.xz", filepath="base.tar.xz" + )["url"] + base_template = Path("Dockerfile.template").read_text() + base_full_url = f"{project_url}{base_uploaded_url}" + base_replaced = base_template.replace("TEMPLATE_LOCATION_HERE", base_full_url) + + print("Uploading base-devel.tar.xz") + base_devel_uploaded_url = project.upload( + f"base-devel-{build_date}.tar.xz", filepath="base-devel.tar.xz" + )["url"] + base_devel_template = Path("Dockerfile.template").read_text() + base_devel_full_url = f"{project_url}{base_devel_uploaded_url}" + base_devel_replaced = base_devel_template.replace( + "TEMPLATE_LOCATION_HERE", base_devel_full_url + ) + + print("Templating Dockerfiles") + data = { + "branch": "add-base-devel-tags", + "commit_message": f"Release {build_date}", + "actions": [ + { + "action": "update", + "file_path": "base/Dockerfile", + "content": base_replaced, + }, + { + "action": "update", + "file_path": "base-devel/Dockerfile", + "content": base_devel_replaced, + }, + ], + } + project.commits.create(data) + + print("Creating release") + release = project.releases.create( + { + "name": f"Release {build_date}", + "tag_name": build_date, + "description": f"Release {build_date}", + "ref": "add-base-devel-tags", + "assets": { + "links": [ + { + "name": "base.tar.xz", + "url": base_full_url, + "link_type": "package", + }, + { + "name": "base-devel.tar.xz", + "url": base_devel_full_url, + "link_type": "package", + } + ] + }, + } + ) + print("Created release", release.get_id()) diff --git a/rootfs/etc/pacman.d/mirrorlist b/rootfs/etc/pacman.d/mirrorlist index 3735559..31adb47 100644 --- a/rootfs/etc/pacman.d/mirrorlist +++ b/rootfs/etc/pacman.d/mirrorlist @@ -1,2 +1,3 @@ +Server = https://mirror.pkgbuild.com/$repo/os/$arch Server = https://mirror.rackspace.com/archlinux/$repo/os/$arch Server = https://mirror.leaseweb.net/archlinux/$repo/os/$arch