From 0535b255ae0fc293a52c24f7d835576fec069b51 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Fri, 23 Mar 2018 20:55:32 -0400 Subject: [PATCH 01/26] ENH: makefile/Dockerfile: allow for non-root build The old makefile required commands to be run as a superuser, which may become problematic when deploying on the archlinux infrastructure. Use fakeroot and regular pacman commands to allow for to bootstrap a rootfs. --- Dockerfile | 2 ++ Makefile | 16 ++++++++-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index cde2238..6d64765 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,6 @@ FROM scratch ADD archlinux.tar / +RUN locale-gen +RUN pacman-key --init && pacman-key --populate archlinux ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 9a94322..df67303 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,16 @@ DOCKER_USER:=pierres DOCKER_ORGANIZATION=archlinux DOCKER_IMAGE:=base +BUILDDIR=build rootfs: - $(eval TMPDIR := $(shell mktemp -d)) - env -i pacstrap -C /usr/share/devtools/pacman-extra.conf -c -d -G -M $(TMPDIR) $(shell cat packages) - cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(TMPDIR)/ - arch-chroot $(TMPDIR) locale-gen - arch-chroot $(TMPDIR) pacman-key --init - arch-chroot $(TMPDIR) pacman-key --populate archlinux - tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(TMPDIR) -c . -f archlinux.tar - rm -rf $(TMPDIR) + mkdir -vp $(BUILDDIR)/var/lib/pacman/ + fakeroot -- pacman -Syu -r $(BUILDDIR) \ + --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ + --hookdir rootfs/usr/share/libalpm/hooks/ $(shell cat packages) + cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ + tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar + rm -rf $(BUILDDIR) docker-image: rootfs docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . From 6b583c70e7b75cd62e5688d521d2aa333f4d0f8c Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 27 Mar 2018 00:48:01 -0400 Subject: [PATCH 02/26] WIP: Makefile: allow for rootless build --- Dockerfile | 1 + Makefile | 7 +++- rootfs/etc/pacman.conf | 91 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 rootfs/etc/pacman.conf diff --git a/Dockerfile b/Dockerfile index 6d64765..8e684d0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM scratch ADD archlinux.tar / +RUN update-ca-certs RUN locale-gen RUN pacman-key --init && pacman-key --populate archlinux ENV LANG=en_US.UTF-8 diff --git a/Makefile b/Makefile index df67303..aee8e83 100644 --- a/Makefile +++ b/Makefile @@ -5,10 +5,13 @@ BUILDDIR=build rootfs: mkdir -vp $(BUILDDIR)/var/lib/pacman/ - fakeroot -- pacman -Syu -r $(BUILDDIR) \ + fakechroot -- fakeroot -- pacman -Syu -r $(BUILDDIR) \ --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ - --hookdir rootfs/usr/share/libalpm/hooks/ $(shell cat packages) + --config rootfs/etc/pacman.conf \ + --noscriptlet \ + --hookdir $(PWD)/rootfs/usr/share/libalpm/hooks/ $(shell cat packages) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ + rm -r build/var/cache/pacman/pkg tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar rm -rf $(BUILDDIR) diff --git a/rootfs/etc/pacman.conf b/rootfs/etc/pacman.conf new file mode 100644 index 0000000..9f62156 --- /dev/null +++ b/rootfs/etc/pacman.conf @@ -0,0 +1,91 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +#HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -C - -f %u > %o +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +#UseDelta = 0.7 +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +#TotalDownload +CheckSpace +#VerbosePkgLists + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[core] +Include = rootfs/etc/pacman.d/mirrorlist + +[extra] +Include = rootfs/etc/pacman.d/mirrorlist + +[community] +Include = rootfs//etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. + +#[multilib] +#Include = /etc/pacman.d/mirrorlist +# +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs From 0afc2ce276a0f57588dd00a5305f7bd9e6471498 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 17 Apr 2018 20:05:57 -0400 Subject: [PATCH 03/26] ENH: update root-less build This commit adds: - Systemd sysusers hook is ran on the dockerfile - The dockerfile also populates the archlinux keyring - Remove the libalpm hooks as they are uneccessary now - Moved the package cleanup before the tar step on the makefile --- Dockerfile | 1 + Makefile | 12 ++++++++---- rootfs/usr/share/libalpm/hooks/package-cleanup.hook | 11 ----------- 3 files changed, 9 insertions(+), 15 deletions(-) delete mode 100644 rootfs/usr/share/libalpm/hooks/package-cleanup.hook diff --git a/Dockerfile b/Dockerfile index 8e684d0..a09622b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,5 +3,6 @@ ADD archlinux.tar / RUN update-ca-certs RUN locale-gen RUN pacman-key --init && pacman-key --populate archlinux +RUN /usr/share/libalpm/scripts/systemd-hook sysusers ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index aee8e83..0e50cd5 100644 --- a/Makefile +++ b/Makefile @@ -3,17 +3,21 @@ DOCKER_ORGANIZATION=archlinux DOCKER_IMAGE:=base BUILDDIR=build -rootfs: +hooks: + mkdir -p alpm-hooks/usr/share/libalpm/hooks + find /usr/share/libalpm/hooks -exec ln -s /dev/null $(PWD)/alpm-hooks{} \; + +rootfs: hooks mkdir -vp $(BUILDDIR)/var/lib/pacman/ - fakechroot -- fakeroot -- pacman -Syu -r $(BUILDDIR) \ + fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ --config rootfs/etc/pacman.conf \ --noscriptlet \ - --hookdir $(PWD)/rootfs/usr/share/libalpm/hooks/ $(shell cat packages) + --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ rm -r build/var/cache/pacman/pkg tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar - rm -rf $(BUILDDIR) + rm -rf $(BUILDDIR) alpm-hooks docker-image: rootfs docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . diff --git a/rootfs/usr/share/libalpm/hooks/package-cleanup.hook b/rootfs/usr/share/libalpm/hooks/package-cleanup.hook deleted file mode 100644 index 448d871..0000000 --- a/rootfs/usr/share/libalpm/hooks/package-cleanup.hook +++ /dev/null @@ -1,11 +0,0 @@ -[Trigger] -Operation = Install -Operation = Upgrade -Type = Package -Target = * - -[Action] -Description = Cleaning up package cache... -Depends = coreutils -When = PostTransaction -Exec = /usr/bin/rm -rf /var/cache/pacman/pkg From 6f9523185903ac4a7d6a3cc016b1bb2800e1e081 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Sun, 3 Jun 2018 21:38:45 -0400 Subject: [PATCH 04/26] FIX:Makefile: add PWD environment variable The previous makefile assumed that PWD was populated. This doesn't happen in certain cases (e.g., when running as a systemd unit). Make sure we have the PWD variable set before running any target. --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 0e50cd5..e89e414 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ DOCKER_USER:=pierres DOCKER_ORGANIZATION=archlinux DOCKER_IMAGE:=base BUILDDIR=build +PWD=$(shell pwd) hooks: mkdir -p alpm-hooks/usr/share/libalpm/hooks From 5e9c8ffad051ba8bc02726cc10d8c2bb9b1ac6bc Mon Sep 17 00:00:00 2001 From: Christian Rebischke Date: Mon, 4 Jun 2018 17:21:38 +0200 Subject: [PATCH 05/26] added fakechroot as dependency for ci-test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e89e414..6ebbd53 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ docker-image-test: docker-image ci-test: docker run --rm --privileged --tmpfs=/tmp:exec --tmpfs=/run/shm -v /run/docker.sock:/run/docker.sock \ -v $(PWD):/app -w /app $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) \ - sh -c 'pacman -Syu --noconfirm make devtools docker && make docker-image-test' + sh -c 'pacman -Syu --noconfirm make devtools docker fakechroot && make docker-image-test' docker-push: docker login -u $(DOCKER_USER) From b0401164819c01764db528587da933cf829d02d6 Mon Sep 17 00:00:00 2001 From: Christian Rebischke Date: Mon, 4 Jun 2018 17:25:51 +0200 Subject: [PATCH 06/26] removed fakechroot again --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6ebbd53..e89e414 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ docker-image-test: docker-image ci-test: docker run --rm --privileged --tmpfs=/tmp:exec --tmpfs=/run/shm -v /run/docker.sock:/run/docker.sock \ -v $(PWD):/app -w /app $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) \ - sh -c 'pacman -Syu --noconfirm make devtools docker fakechroot && make docker-image-test' + sh -c 'pacman -Syu --noconfirm make devtools docker && make docker-image-test' docker-push: docker login -u $(DOCKER_USER) From 602cda864eca6777f6c950c74fa00f8dcaf9c5c0 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 4 Jun 2018 12:43:19 -0400 Subject: [PATCH 07/26] FIX: add fakeroot and fakechroot deps for ci-test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index e89e414..3b83b24 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ docker-image-test: docker-image ci-test: docker run --rm --privileged --tmpfs=/tmp:exec --tmpfs=/run/shm -v /run/docker.sock:/run/docker.sock \ -v $(PWD):/app -w /app $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) \ - sh -c 'pacman -Syu --noconfirm make devtools docker && make docker-image-test' + sh -c 'pacman -Syu --noconfirm fakechroot fakeroot make devtools docker && make docker-image-test' docker-push: docker login -u $(DOCKER_USER) From f65025bafe9cb1dd1d4bd1546d3e0adc98619868 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 4 Jun 2018 13:17:30 -0400 Subject: [PATCH 08/26] FIX:fix docker build commands I commited the wrong Dockerfile. --- Dockerfile | 8 ++++---- Makefile | 2 +- rootfs/etc/pacman.conf => pacman.conf | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) rename rootfs/etc/pacman.conf => pacman.conf (98%) diff --git a/Dockerfile b/Dockerfile index a09622b..c0060f6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ FROM scratch ADD archlinux.tar / -RUN update-ca-certs -RUN locale-gen -RUN pacman-key --init && pacman-key --populate archlinux -RUN /usr/share/libalpm/scripts/systemd-hook sysusers + +# manually run all alpm hooks that can't be run inside the fakechroot +RUN ldconfig && update-ca-trust && locale-gen && /usr/share/libalpm/scripts/systemd-hook sysusers && pacman-key --init && pacman-key --populate archlinux + ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 3b83b24..8e47df6 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ rootfs: hooks mkdir -vp $(BUILDDIR)/var/lib/pacman/ fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ - --config rootfs/etc/pacman.conf \ + --config pacman.conf \ --noscriptlet \ --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ diff --git a/rootfs/etc/pacman.conf b/pacman.conf similarity index 98% rename from rootfs/etc/pacman.conf rename to pacman.conf index 9f62156..4cf98ed 100644 --- a/rootfs/etc/pacman.conf +++ b/pacman.conf @@ -76,7 +76,7 @@ Include = rootfs/etc/pacman.d/mirrorlist Include = rootfs/etc/pacman.d/mirrorlist [community] -Include = rootfs//etc/pacman.d/mirrorlist +Include = rootfs/etc/pacman.d/mirrorlist # If you want to run 32 bit applications on your x86_64 system, # enable the multilib repositories as required here. From ebcde63f6c329d751ac6fa8475b0a0f66a329062 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 4 Jun 2018 14:04:27 -0400 Subject: [PATCH 09/26] FIX:Dockerfile/Makefile: fix ci-test requirements The old Dockerfile and Makefile pair didn't meet the requirements of the ci-test. Namely, the (very sensible )requirement of not shipping a private key was missing, the sysusers hook wasn't executed properly and the /var/cache/pacman/pkg folder was remoed (and it's specified in the pacman MTREE package) --- Dockerfile | 7 ++++++- Makefile | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index c0060f6..8a5c283 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,12 @@ FROM scratch ADD archlinux.tar / # manually run all alpm hooks that can't be run inside the fakechroot -RUN ldconfig && update-ca-trust && locale-gen && /usr/share/libalpm/scripts/systemd-hook sysusers && pacman-key --init && pacman-key --populate archlinux +RUN ldconfig && update-ca-trust && locale-gen +RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' + +# initialize the archilnux keyring, but discard any private key that may be shipped. +RUN pacman-key --init && pacman-key --populate archlinux +RUN rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pugring.gpg~,gnupg.S.}* ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 8e47df6..452fb23 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,6 @@ rootfs: hooks --noscriptlet \ --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ - rm -r build/var/cache/pacman/pkg tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar rm -rf $(BUILDDIR) alpm-hooks From 8ec2549c3e7ed4edf62380cc6daf7d9d725a2465 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 25 Jun 2019 19:21:44 -0400 Subject: [PATCH 10/26] Dockerfile,Makefile: use a compressed tarball When building the image for publishing in the infrastructure, we need a rootfs that's <100MB, due to github filesize restrictions (this, plus the fact that there's no git-lfs support for building in the docker library toolchain). Compress the rootfs image so that it can be hosted on github. --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index cde2238..0afff82 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ FROM scratch -ADD archlinux.tar / +ADD archlinux.tar.xz / ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 9a94322..4d37d02 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ rootfs: arch-chroot $(TMPDIR) locale-gen arch-chroot $(TMPDIR) pacman-key --init arch-chroot $(TMPDIR) pacman-key --populate archlinux - tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(TMPDIR) -c . -f archlinux.tar + tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(TMPDIR) -c . -xf archlinux.tar.xz rm -rf $(TMPDIR) docker-image: rootfs From b2e07647e948d1d0fb71c811d87db24198addddf Mon Sep 17 00:00:00 2001 From: Linux Ninja Date: Sat, 10 Aug 2019 02:57:40 -0400 Subject: [PATCH 11/26] add fakechroot dependency in README.md, prevent 'already exists' error when running ln -s by adding -f in Makefile, combine two layers of Dockerfile and fix typo of pugring.gpg~ Signed-off-by: Linux Ninja --- Dockerfile | 5 ++--- Makefile | 2 +- README.md | 1 + 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index e071867..65f92c1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,9 +5,8 @@ ADD archlinux.tar.xz / RUN ldconfig && update-ca-trust && locale-gen RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' -# initialize the archilnux keyring, but discard any private key that may be shipped. -RUN pacman-key --init && pacman-key --populate archlinux -RUN rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pugring.gpg~,gnupg.S.}* +# initialize the archlinux keyring, but discard any private key that may be shipped. +RUN pacman-key --init && pacman-key --populate archlinux && rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}* ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 4cfd2ee..22a7c25 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ PWD=$(shell pwd) hooks: mkdir -p alpm-hooks/usr/share/libalpm/hooks - find /usr/share/libalpm/hooks -exec ln -s /dev/null $(PWD)/alpm-hooks{} \; + find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(PWD)/alpm-hooks{} \; rootfs: hooks mkdir -vp $(BUILDDIR)/var/lib/pacman/ diff --git a/README.md b/README.md index 4dfbc53..d319da5 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ Install the following Arch Linux packages: * make * devtools * docker +* fakechroot ## Usage Run `make docker-image` as root to build the base image. ## Purpose From eaa38cb44e7c24d7256de973dc4b832d99a40c1c Mon Sep 17 00:00:00 2001 From: Linux Ninja Date: Sat, 10 Aug 2019 03:16:54 -0400 Subject: [PATCH 12/26] add xz params to tighten archive size (saves 5M currently), use all available CPU cores, overwrite existing file Signed-off-by: Linux Ninja --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 22a7c25..5b62979 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ rootfs: hooks rm -rf $(BUILDDIR) alpm-hooks compress-rootfs: rootfs - xz archlinux.tar + xz -9e -T0 -f archlinux.tar docker-image: compress-rootfs docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . From 19fb8aeb466e06a872155aad74b61e7a843d2241 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Sun, 6 Oct 2019 12:31:21 -0400 Subject: [PATCH 13/26] Makefile: add fix for CVE-2019-5021 The previous instances of the docker image allowed for passwordless root login. Update the default shadow setting so the root account to disallow this. --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index 4cfd2ee..8588436 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,10 @@ rootfs: hooks --noscriptlet \ --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ + + # remove passwordless login for root (see CVE-2019-5021 for reference) + sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" + tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar rm -rf $(BUILDDIR) alpm-hooks From a2b08c5fc19eabce752840e69b9cfb387ffe6657 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Sun, 6 Oct 2019 12:32:39 -0400 Subject: [PATCH 14/26] Makefile:compress:use the existing archlinux.tar The requirements for the compress rule used to rebuild the rootfs regardless. Update the requirement to use the existing archlinux archive for compression --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8588436..9f7ff04 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,7 @@ rootfs: hooks tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar rm -rf $(BUILDDIR) alpm-hooks -compress-rootfs: rootfs +compress-rootfs: archlinux.tar xz archlinux.tar docker-image: compress-rootfs From 368a504fa3bb4dcc04131fd8cbac486e1f880839 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Wed, 23 Oct 2019 20:22:50 -0400 Subject: [PATCH 15/26] Makefile: use fakeroot for tar-ing rootfs When tar-ing the rootfs, the permissions preserved are from the worker process. This is not the intended behavior, as we want to preserve the permissions of the root user to most of the files. Use fakeroot during the tar process to automagically map the uid of the user to that of root's. Fixes #22 Signed-off-by: Santiago Torres --- Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9f7ff04..8a5274a 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,9 @@ rootfs: hooks # remove passwordless login for root (see CVE-2019-5021 for reference) sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" - tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar + # fakeroot to map the gid/uid of the builder process to root + # fixes #22 + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar rm -rf $(BUILDDIR) alpm-hooks compress-rootfs: archlinux.tar From ed424bc02f3183b7de110d544ad9c749b5d85771 Mon Sep 17 00:00:00 2001 From: Christian Rebischke Date: Sat, 12 Oct 2019 15:29:12 +0200 Subject: [PATCH 16/26] add Github Actions Signed-off-by: Christian Rebischke --- .github/workflows/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 .github/workflows/main.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..9d120d3 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,12 @@ +name: Github-Actions + +on: [push, pull_request] + +jobs: + ci-test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + - name: make ci-test + run: make ci-test From e5ee97ef26e20fb689b4ecd8d9cef62744c7845e Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Wed, 23 Oct 2019 21:04:36 -0400 Subject: [PATCH 17/26] github-actions: add rootfs target to make This is relevant for the no-root-build. --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9d120d3..f43c10f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,4 +9,4 @@ jobs: steps: - uses: actions/checkout@v1 - name: make ci-test - run: make ci-test + run: make rootfs ci-test From 2fc6d9f3eeba6c108915a6a633c53494a3550729 Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Thu, 28 Feb 2019 14:53:48 -0500 Subject: [PATCH 18/26] pacstrap: usie NoExtract to prune dead weight A docker container does not need tons of locales in order to do its job, nor does it need extensive (or any) documentation. With this change, a bare pacstrapped directory drops from 500MB to 390MB --- .gitignore | 1 + Makefile | 2 ++ pacman-conf.d-noextract.conf | 9 +++++++++ 3 files changed, 12 insertions(+) create mode 100644 pacman-conf.d-noextract.conf diff --git a/.gitignore b/.gitignore index a583ac7..e573ef4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *.orig /.idea /archlinux.tar +rootfs/etc/pacman.conf diff --git a/Makefile b/Makefile index 8a5274a..8a85f42 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,8 @@ hooks: rootfs: hooks mkdir -vp $(BUILDDIR)/var/lib/pacman/ + cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf + cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ --config pacman.conf \ diff --git a/pacman-conf.d-noextract.conf b/pacman-conf.d-noextract.conf new file mode 100644 index 0000000..76b762b --- /dev/null +++ b/pacman-conf.d-noextract.conf @@ -0,0 +1,9 @@ +[options] +NoExtract = usr/share/help/* !usr/share/help/en* +NoExtract = usr/share/gtk-doc/html/* usr/share/doc/* +NoExtract = usr/share/locale/* usr/share/X11/locale/* usr/share/i18n/* +NoExtract = !*locale*/en*/* !usr/share/i18n/charmaps/UTF-8.gz !usr/share/*locale*/locale.* +NoExtract = !usr/share/*locales/en_?? !usr/share/*locales/i18n* !usr/share/*locales/iso* +NoExtract = !usr/share/*locales/trans* +NoExtract = usr/share/man/* usr/share/info/* +NoExtract = usr/share/vim/vim*/lang/* From 4ebfbe58833529a3ea8bab334b979fb237f870e8 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 26 Nov 2019 16:02:21 -0500 Subject: [PATCH 19/26] packages: add base group --- Makefile | 2 +- packages | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8a85f42..ac188a8 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ rootfs: hooks cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ - --config pacman.conf \ + --config rootfs/etc/pacman.conf \ --noscriptlet \ --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ diff --git a/packages b/packages index 39dffd0..5ef6e83 100644 --- a/packages +++ b/packages @@ -2,3 +2,4 @@ sed gzip pacman systemd +gawk From 9091d1ebb9cca5b86a0298dc3169fc7dd3efdc33 Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 26 Nov 2019 17:02:56 -0500 Subject: [PATCH 20/26] Makefile: add archlinux.tar target, force xz --- Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ac188a8..b1800c4 100644 --- a/Makefile +++ b/Makefile @@ -27,8 +27,10 @@ rootfs: hooks fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar rm -rf $(BUILDDIR) alpm-hooks +archlinux.tar: rootfs + compress-rootfs: archlinux.tar - xz archlinux.tar + xz -f archlinux.tar docker-image: compress-rootfs docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . @@ -36,8 +38,7 @@ docker-image: compress-rootfs docker-image-test: docker-image # FIXME: /etc/mtab is hidden by docker so the stricter -Qkk fails docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Sy && /usr/bin/pacman -Qqk" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm docker && docker -v" - # Ensure that the image does not include a private key + docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm docker && docker -v" # Ensure that the image does not include a private key ! docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) pacman-key --lsign-key pierre@archlinux.de docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/id -u http" docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm grep && locale | grep -q UTF-8" From 8f683e3cdc7ae7a6504c525829edf2b039959d8f Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Tue, 26 Nov 2019 17:03:20 -0500 Subject: [PATCH 21/26] packages: +likely necessary pkgs from base group --- packages | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages b/packages index 5ef6e83..2c07590 100644 --- a/packages +++ b/packages @@ -3,3 +3,8 @@ gzip pacman systemd gawk +file +grep +tar +procps-ng +licenses From 9bd11c95c2b13af854388498ac948f363bd78bcc Mon Sep 17 00:00:00 2001 From: Santiago Torres Date: Mon, 8 Jun 2020 10:54:44 -0400 Subject: [PATCH 22/26] Dockerfile: manually symlink os-release --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e071867..5470c0b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,9 +2,12 @@ FROM scratch ADD archlinux.tar.xz / # manually run all alpm hooks that can't be run inside the fakechroot -RUN ldconfig && update-ca-trust && locale-gen +RUN ldconfig && update-ca-trust && locale-gen RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' +# update /etc/os-release +RUN ln -s /usr/lib/os-release /etc/os-release + # initialize the archilnux keyring, but discard any private key that may be shipped. RUN pacman-key --init && pacman-key --populate archlinux RUN rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pugring.gpg~,gnupg.S.}* From 026d2788dffdf2de5880aaa71e6c0b7d6b823dcb Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 30 Jul 2020 17:55:18 +0200 Subject: [PATCH 23/26] Add fakeroot as a build dependency --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index d319da5..8e80316 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Install the following Arch Linux packages: * devtools * docker * fakechroot +* fakeroot ## Usage Run `make docker-image` as root to build the base image. ## Purpose From cbaf17832edb9767a623034b7fe1dceb7b245854 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 30 Jul 2020 17:55:36 +0200 Subject: [PATCH 24/26] Add the ability to adjust the thread count of xz, drop -e `xz --extreme` would only save us about 1MB but take ~50% longer. --- Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6bbf628..dfe2e9c 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,8 @@ DOCKER_IMAGE:=base BUILDDIR=build PWD=$(shell pwd) +XZ_THREADS ?= 0 + hooks: mkdir -p alpm-hooks/usr/share/libalpm/hooks find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(PWD)/alpm-hooks{} \; @@ -30,8 +32,7 @@ rootfs: hooks archlinux.tar: rootfs compress-rootfs: archlinux.tar - xz -f archlinux.tar - xz -9e -T0 -f archlinux.tar + xz -9 -T"$(XZ_THREADS)" -f archlinux.tar docker-image: compress-rootfs docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . From 0fa3f77f16c0f27c05fafd97ead6387289e4fd38 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Tue, 28 Jul 2020 22:05:30 +0200 Subject: [PATCH 25/26] Add GitLabCI definition Additionally drops the GitHub and Travis CI definitions since they are no longer needed. --- .dockerignore | 3 +++ .github/workflows/main.yml | 12 ---------- .gitignore | 1 + .gitlab-ci.yml | 48 ++++++++++++++++++++++++++++++++++++++ .travis.yml | 9 ------- Makefile | 7 +----- exclude | 29 ++++++++++++----------- 7 files changed, 69 insertions(+), 40 deletions(-) create mode 100644 .dockerignore delete mode 100644 .github/workflows/main.yml create mode 100644 .gitlab-ci.yml delete mode 100644 .travis.yml diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0fa7348 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,3 @@ +* +!archlinux.tar +!archlinux.tar.xz diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml deleted file mode 100644 index f43c10f..0000000 --- a/.github/workflows/main.yml +++ /dev/null @@ -1,12 +0,0 @@ -name: Github-Actions - -on: [push, pull_request] - -jobs: - ci-test: - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v1 - - name: make ci-test - run: make rootfs ci-test diff --git a/.gitignore b/.gitignore index e573ef4..6a9306a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.orig /.idea /archlinux.tar +/archlinux.tar.xz rootfs/etc/pacman.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..c092ee1 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,48 @@ +stages: + - rootfs + - docker + - test + +roofs: + stage: rootfs + image: archlinux:latest + script: + - pacman -Syu --noconfirm make devtools fakechroot fakeroot + - make archlinux.tar + artifacts: + paths: + - archlinux.tar + expire_in: 10m + +docker: + stage: docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - test -f archlinux.tar + # kaniko can't process .tar.xz archives + # https://github.com/GoogleContainerTools/kaniko/issues/1107 + - sed -i 's/archlinux\.tar\.xz/archlinux\.tar/g' Dockerfile + - echo "Building ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/Dockerfile + --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} + +test: + stage: test + image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} + needs: + - job: docker + artifacts: false + script: + - pacman -Sy + - pacman -Qqk + - pacman -Syu --noconfirm docker grep + - docker -v + - id -u http + - locale | grep -q UTF-8 + diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index cfc4d72..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -sudo: required - -language: bash - -services: - - docker - -script: - - make ci-test diff --git a/Makefile b/Makefile index dfe2e9c..61fb0db 100644 --- a/Makefile +++ b/Makefile @@ -45,13 +45,8 @@ docker-image-test: docker-image docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/id -u http" docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm grep && locale | grep -q UTF-8" -ci-test: - docker run --rm --privileged --tmpfs=/tmp:exec --tmpfs=/run/shm -v /run/docker.sock:/run/docker.sock \ - -v $(PWD):/app -w /app $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) \ - sh -c 'pacman -Syu --noconfirm fakechroot fakeroot make devtools docker && make docker-image-test' - docker-push: docker login -u $(DOCKER_USER) docker push $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) -.PHONY: rootfs docker-image docker-image-test ci-test docker-push +.PHONY: rootfs docker-image docker-image-test docker-push diff --git a/exclude b/exclude index 1517525..0a45d22 100644 --- a/exclude +++ b/exclude @@ -1,13 +1,16 @@ -.dockerenv -.dockerinit -etc/hostname -etc/machine-id -etc/pacman.d/gnupg/openpgp-revocs.d/* -etc/pacman.d/gnupg/private-keys-v1.d/* -etc/pacman.d/gnupg/pubring.gpg~ -etc/pacman.d/gnupg/S.* -root/* -tmp/* -var/cache/pacman/pkg/* -var/lib/pacman/sync/* -var/tmp/* +./.dockerenv +./.dockerinit +./sys +./proc +./dev +./etc/hostname +./etc/machine-id +./etc/pacman.d/gnupg/openpgp-revocs.d/* +./etc/pacman.d/gnupg/private-keys-v1.d/* +./etc/pacman.d/gnupg/pubring.gpg~ +./etc/pacman.d/gnupg/S.* +./root/* +./tmp/* +./var/cache/pacman/pkg/* +./var/lib/pacman/sync/* +./var/tmp/* \ No newline at end of file From 59e29b7a9b94ad84516e7f67ff433cf196a81ae1 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 30 Jul 2020 18:05:57 +0200 Subject: [PATCH 26/26] Switch to compress-rootfs in CI since xz is now using --threads --- .gitlab-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c092ee1..eed2256 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -8,10 +8,10 @@ roofs: image: archlinux:latest script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot - - make archlinux.tar + - make compress-rootfs artifacts: paths: - - archlinux.tar + - archlinux.tar.xz expire_in: 10m docker: @@ -20,9 +20,11 @@ docker: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] script: - - test -f archlinux.tar + - test -f archlinux.tar.xz # kaniko can't process .tar.xz archives # https://github.com/GoogleContainerTools/kaniko/issues/1107 + - unxz archlinux.tar.xz + - test -f archlinux.tar - sed -i 's/archlinux\.tar\.xz/archlinux\.tar/g' Dockerfile - echo "Building ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json