From d47ca225eefdd3a80689d60acaf3ede4e5f298cb Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Wed, 13 Apr 2022 17:48:07 +0200 Subject: [PATCH 1/2] Replace xz with zstd Closes #63. Using `zstd -T0 -8` instead of `gz -T0 -9` results in a larger rootfs file, but requires significantly less time and memory: ``` zstd -3 115M 1.60user 0.18system 0:00.32elapsed 557%CPU (0avgtext+0avgdata 130212maxresident)k zstd -6 107M 5.03user 0.24system 0:00.72elapsed 729%CPU (0avgtext+0avgdata 149660maxresident)k zstd -7 106M 7.33user 0.29system 0:01.04elapsed 728%CPU (0avgtext+0avgdata 174368maxresident)k zstd -8 105M 8.45user 0.27system 0:01.23elapsed 707%CPU (0avgtext+0avgdata 173008maxresident)k zstd -9 104M 9.35user 0.37system 0:01.42elapsed 683%CPU (0avgtext+0avgdata 335920maxresident)k zstd -13 104M 30.57user 0.31system 0:04.76elapsed 648%CPU (0avgtext+0avgdata 498740maxresident)k zstd -19 91M 132.06user 0.53system 0:21.98elapsed 603%CPU (0avgtext+0avgdata 1106328maxresident)k xz -9 81M 105.71user 0.58system 0:58.24elapsed 182%CPU (0avgtext+0avgdata 2006964maxresident)k ``` Additionally this drops bash from the build Dockerfile, since `SHELL` is not OCI compliant. --- Dockerfile.template | 8 +++++--- Makefile | 18 +++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index 891a056..56bf2a9 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -10,11 +10,13 @@ # allow us to automate the releaes process. FROM alpine:3.12 AS verify COPY TEMPLATE_ROOTFS_FILE / -RUN apk add --no-cache curl bash -SHELL ["/bin/bash", "-c"] +RUN apk add --no-cache curl tar zstd + # TEMPLATE_ROOTFS_RELEASE_URL RUN TEMPLATE_ROOTFS_DOWNLOAD && \ - sha256sum -c <<< "TEMPLATE_ROOTFS_HASH" && \ + echo "TEMPLATE_ROOTFS_HASH" > /tmp/rootfs.tar.sha256 && \ + cat /tmp/rootfs.tar.sha256 && \ + sha256sum -c /tmp/rootfs.tar.sha256 && \ mkdir /rootfs && \ tar -C /rootfs --extract --file "${ROOTFS}" diff --git a/Makefile b/Makefile index 64d706f..66a4208 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ define rootfs cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ -fakechroot -- fakeroot -- chroot $(BUILDDIR) update-ca-trust + fakechroot -- fakeroot -- chroot $(BUILDDIR) update-ca-trust fakechroot -- fakeroot -- chroot $(BUILDDIR) locale-gen fakechroot -- fakeroot -- chroot $(BUILDDIR) sh -c 'pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*"' @@ -34,14 +34,14 @@ fakechroot -- fakeroot -- chroot $(BUILDDIR) update-ca-trust # fixes #22 fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar - cd $(OUTPUTDIR); xz -9 -T0 -f $(1).tar; sha256sum $(1).tar.xz > $(1).tar.xz.SHA256 + cd $(OUTPUTDIR); zstd -T0 -8 $(1).tar; sha256sum $(1).tar.zst > $(1).tar.zst.SHA256 endef define dockerfile - sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.xz|" \ + sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.zst|" \ -e "s|TEMPLATE_ROOTFS_RELEASE_URL|Local build|" \ - -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$(1).tar.xz\"|" \ - -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.xz.SHA256)|" \ + -e "s|TEMPLATE_ROOTFS_DOWNLOAD|ROOTFS=\"$(1).tar.zst\"|" \ + -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.zst.SHA256)|" \ Dockerfile.template > $(OUTPUTDIR)/Dockerfile.$(1) endef @@ -49,16 +49,16 @@ endef clean: rm -rf $(BUILDDIR) $(OUTPUTDIR) -$(OUTPUTDIR)/base.tar.xz: +$(OUTPUTDIR)/base.tar.zst: $(call rootfs,base,base) -$(OUTPUTDIR)/base-devel.tar.xz: +$(OUTPUTDIR)/base-devel.tar.zst: $(call rootfs,base-devel,base base-devel) -$(OUTPUTDIR)/Dockerfile.base: $(OUTPUTDIR)/base.tar.xz +$(OUTPUTDIR)/Dockerfile.base: $(OUTPUTDIR)/base.tar.zst $(call dockerfile,base) -$(OUTPUTDIR)/Dockerfile.base-devel: $(OUTPUTDIR)/base-devel.tar.xz +$(OUTPUTDIR)/Dockerfile.base-devel: $(OUTPUTDIR)/base-devel.tar.zst $(call dockerfile,base-devel) .PHONY: docker-image-base From 47a614ade94cfabf844eec6e9fd46fd69363a378 Mon Sep 17 00:00:00 2001 From: Justin Kromlinger Date: Thu, 14 Apr 2022 01:02:30 +0200 Subject: [PATCH 2/2] Add --long to zstd call --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 66a4208..222643d 100644 --- a/Makefile +++ b/Makefile @@ -34,7 +34,7 @@ define rootfs # fixes #22 fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar - cd $(OUTPUTDIR); zstd -T0 -8 $(1).tar; sha256sum $(1).tar.zst > $(1).tar.zst.SHA256 + cd $(OUTPUTDIR); zstd --long -T0 -8 $(1).tar; sha256sum $(1).tar.zst > $(1).tar.zst.SHA256 endef define dockerfile