From d51a887efb57340be5b4a7c33b6969d758d88ffe Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sat, 29 Aug 2020 04:51:05 +0200 Subject: [PATCH 01/92] Add base-devel tag --- .dockerignore | 3 - .gitignore | 4 +- .gitlab-ci.yml | 169 ++++++++++++++++++++++++++---- Dockerfile => Dockerfile.template | 8 +- Makefile | 70 +++++++++---- README.md | 18 +++- base-devel/Dockerfile | 0 base/Dockerfile | 0 ci/release.py | 86 +++++++++++++++ rootfs/etc/pacman.d/mirrorlist | 1 + 10 files changed, 305 insertions(+), 54 deletions(-) delete mode 100644 .dockerignore rename Dockerfile => Dockerfile.template (59%) create mode 100644 base-devel/Dockerfile create mode 100644 base/Dockerfile create mode 100755 ci/release.py diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index 0fa7348..0000000 --- a/.dockerignore +++ /dev/null @@ -1,3 +0,0 @@ -* -!archlinux.tar -!archlinux.tar.xz diff --git a/.gitignore b/.gitignore index 6a9306a..94f5842 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *~ *.orig /.idea -/archlinux.tar -/archlinux.tar.xz +/base.tar* +/base-devel.tar* rootfs/etc/pacman.conf diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index eed2256..97520a6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,44 +1,91 @@ stages: + - lint - rootfs - docker - test + - release + - publish -roofs: +lint: + stage: lint + image: hadolint/hadolint:latest + script: hadolint --ignore DL3020 Dockerfile.template + +rootfs:base: stage: rootfs image: archlinux:latest + needs: + - job: "lint" + before_script: + - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot - - make compress-rootfs + - make base.tar.xz artifacts: paths: - - archlinux.tar.xz + - base.tar.xz expire_in: 10m + reports: + dotenv: build.env -docker: +rootfs:base-devel: + stage: rootfs + image: archlinux:latest + needs: + - job: "lint" + before_script: + - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env + script: + - pacman -Syu --noconfirm make devtools fakechroot fakeroot + - make base-devel.tar.xz + artifacts: + paths: + - base-devel.tar.xz + expire_in: 10m + reports: + dotenv: build.env + +docker:base: stage: docker image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] - script: - - test -f archlinux.tar.xz - # kaniko can't process .tar.xz archives - # https://github.com/GoogleContainerTools/kaniko/issues/1107 - - unxz archlinux.tar.xz - - test -f archlinux.tar - - sed -i 's/archlinux\.tar\.xz/archlinux\.tar/g' Dockerfile - - echo "Building ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG}" + needs: + - job: "rootfs:base" + before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base + - unxz base.tar.xz + script: - /kaniko/executor --whitelist-var-run="false" --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile - --destination ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} + --dockerfile $CI_PROJECT_DIR/Dockerfile.base + --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG -test: - stage: test - image: ${CI_REGISTRY_IMAGE}:${CI_COMMIT_REF_SLUG} +docker:base-devel: + stage: docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] needs: - - job: docker + - job: "rootfs:base-devel" + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + - sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel + - unxz base-devel.tar.xz + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel + --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + +test:base: + stage: test + image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + needs: + - job: "docker:base" artifacts: false script: - pacman -Sy @@ -48,3 +95,89 @@ test: - id -u http - locale | grep -q UTF-8 +test:base-devel: + stage: test + image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + needs: + - job: "docker:base-devel" + artifacts: false + script: + - pacman -Sy + - pacman -Qqk + - pacman -Syu --noconfirm docker grep + - docker -v + - id -u http + - locale | grep -q UTF-8 + - gcc -v + - g++ -v + - make -v + +release: + stage: release + image: archlinux:latest + only: + refs: + - master + - add-base-devel-tags + variables: + - $SCHEDULED_PUBLISH == "TRUE" + needs: + - job: "test:base" + - job: "test:base-devel" + before_script: + - pacman -Syu python-gitlab + script: + - python ci/release.py + tags: + - secure + +# Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base +# publish:org:base: +# stage: publish +# image: +# name: gcr.io/go-containerregistry/crane:debug +# entrypoint: [""] +# needs: +# - job: "test:base" +# artifacts: true +# tags: +# - secure +# variables: +# GIT_STRATEGY: none +# before_script: +# - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux +# script: +# - crane cp $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG archlinux/archlinux:base +# - crane tag archlinux/archlinux:base latest +# - crane tag archlinux/archlinux:base base-$BUILD_DATE +# only: +# variables: +# - $SCHEDULED_PUBLISH == "TRUE" + +# Publish base-devel to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base-devel +# publish:org:base-devel: +# stage: publish +# image: +# name: gcr.io/go-containerregistry/crane:debug +# entrypoint: [""] +# needs: +# - job: "test:base-devel" +# artifacts: true +# tags: +# - secure +# variables: +# GIT_STRATEGY: none +# before_script: +# - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY +# - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux +# script: +# - crane cp $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG archlinux/archlinux:base-devel +# - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_DATE +# only: +# variables: +# - $SCHEDULED_PUBLISH == "TRUE" + +# Publish to the official Docker namespace: https://hub.docker.com/_/archlinux +# publish:official: +# TODO No idea right now how we're going to automatically do the official Docker Hub pull request diff --git a/Dockerfile b/Dockerfile.template similarity index 59% rename from Dockerfile rename to Dockerfile.template index 02fbabc..ffe6415 100644 --- a/Dockerfile +++ b/Dockerfile.template @@ -1,15 +1,15 @@ -FROM scratch -ADD archlinux.tar.xz / +FROM scratch AS base +ADD TEMPLATE_LOCATION_HERE / # manually run all alpm hooks that can't be run inside the fakechroot -RUN ldconfig && update-ca-trust && locale-gen +RUN ldconfig && update-ca-trust && locale-gen RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' # update /etc/os-release RUN ln -s /usr/lib/os-release /etc/os-release # initialize the archlinux keyring, but discard any private key that may be shipped. -RUN pacman-key --init && pacman-key --populate archlinux && rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}* +RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" ENV LANG=en_US.UTF-8 CMD ["/usr/bin/bash"] diff --git a/Makefile b/Makefile index 61fb0db..1736e13 100644 --- a/Makefile +++ b/Makefile @@ -1,16 +1,14 @@ DOCKER_USER:=pierres -DOCKER_ORGANIZATION=archlinux -DOCKER_IMAGE:=base BUILDDIR=build PWD=$(shell pwd) -XZ_THREADS ?= 0 - +.PHONY: hooks hooks: mkdir -p alpm-hooks/usr/share/libalpm/hooks find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(PWD)/alpm-hooks{} \; -rootfs: hooks +.PHONY: rootfs-base +rootfs-base: hooks mkdir -vp $(BUILDDIR)/var/lib/pacman/ cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf @@ -18,35 +16,61 @@ rootfs: hooks --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ --config rootfs/etc/pacman.conf \ --noscriptlet \ - --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ $(shell cat packages) + --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ - + # remove passwordless login for root (see CVE-2019-5021 for reference) sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" # fakeroot to map the gid/uid of the builder process to root # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f archlinux.tar + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base.tar rm -rf $(BUILDDIR) alpm-hooks -archlinux.tar: rootfs +.PHONY: rootfs-base-devel +rootfs-base-devel: hooks + mkdir -vp $(BUILDDIR)/var/lib/pacman/ + cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf + cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf + fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ + --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ + --config rootfs/etc/pacman.conf \ + --noscriptlet \ + --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base base-devel + cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ -compress-rootfs: archlinux.tar - xz -9 -T"$(XZ_THREADS)" -f archlinux.tar + # remove passwordless login for root (see CVE-2019-5021 for reference) + sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" -docker-image: compress-rootfs - docker build -t $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) . + # fakeroot to map the gid/uid of the builder process to root + # fixes #22 + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base-devel.tar + rm -rf $(BUILDDIR) alpm-hooks -docker-image-test: docker-image - # FIXME: /etc/mtab is hidden by docker so the stricter -Qkk fails - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Sy && /usr/bin/pacman -Qqk" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm docker && docker -v" # Ensure that the image does not include a private key - ! docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) pacman-key --lsign-key pierre@archlinux.de - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/id -u http" - docker run --rm $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) sh -c "/usr/bin/pacman -Syu --noconfirm grep && locale | grep -q UTF-8" +base.tar.xz: rootfs-base + xz -9 -T0 -f base.tar -docker-push: +base-devel.tar.xz: rootfs-base-devel + xz -9 -T0 -f base-devel.tar + +.PHONY: docker-image-base +docker-image-base: base.tar.xz + unxz base.tar.xz + sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base + docker build -f Dockerfile.base -t archlinux/archlinux:base . + +.PHONY: docker-image-base-devel +docker-image-base-devel: base-devel.tar.xz + unxz base-devel.tar.xz + sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel + docker build -f Dockerfile.base-devel -t archlinux/archlinux:base-devel . + +.PHONY: docker-push-base +docker-push-base: docker login -u $(DOCKER_USER) - docker push $(DOCKER_ORGANIZATION)/$(DOCKER_IMAGE) + docker push archlinux/archlinux:base -.PHONY: rootfs docker-image docker-image-test docker-push +.PHONY: docker-push-base-devel +docker-push-base-devel: + docker login -u $(DOCKER_USER) + docker push archlinux/archlinux:base-devel diff --git a/README.md b/README.md index 8e80316..30b0e77 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,26 @@ -# Docker Base Image for Arch Linux [![Build Status](https://travis-ci.org/archlinux/archlinux-docker.svg?branch=master)](https://travis-ci.org/archlinux/archlinux-docker) -This repository contains all scripts and files needed to create a Docker base image for the Arch Linux distribution. +# Docker Base Image for Arch Linux +[![pipeline status](https://gitlab.archlinux.org/archlinux/archlinux-docker/badges/master/pipeline.svg)](https://gitlab.archlinux.org/archlinux/archlinux-docker/-/commits/master) + +This repository contains all scripts and files needed to create a Docker base image for Arch Linux. + ## Dependencies Install the following Arch Linux packages: + * make * devtools * docker * fakechroot * fakeroot + +Make sure your user can directly interact with Docker (ie. `docker info` works). + ## Usage -Run `make docker-image` as root to build the base image. +Run `make docker-image-base` to build the image `archlinux:base` with the +`base` group installed. You can also run `make docker-image-base-devel` to +build the image `archlinux:base-devel` with the `base-devel` group installed. + ## Purpose -* Provide the Arch experience in a Docker Image +* Provide the Arch experience in a Docker image * Provide the most simple but complete image to base every other upon * `pacman` needs to work out of the box * All installed packages have to be kept unmodified diff --git a/base-devel/Dockerfile b/base-devel/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/base/Dockerfile b/base/Dockerfile new file mode 100644 index 0000000..e69de29 diff --git a/ci/release.py b/ci/release.py new file mode 100755 index 0000000..5896f20 --- /dev/null +++ b/ci/release.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +""" +Should only be called from GitLab CI! + +Required env vars: + - GITLAB_PROJECT_TOKEN + - BUILD_DATE + - CI_PROJECT_ID + - CI_PROJECT_URL +""" + +import os +from pathlib import Path +import gitlab + +token = os.environ['GITLAB_PROJECT_TOKEN'] +build_date = os.environ['BUILD_DATE'] +project_id = os.environ['CI_PROJECT_ID'] +project_url = os.environ['CI_PROJECT_URL'] + +if __name__ == "__main__": + gl = gitlab.Gitlab("https://gitlab.archlinux.org", token) + project = gl.projects.get(project_id) + + print("Uploading base.tar.xz") + base_uploaded_url = project.upload( + f"base-{build_date}.tar.xz", filepath="base.tar.xz" + )["url"] + base_template = Path("Dockerfile.template").read_text() + base_full_url = f"{project_url}{base_uploaded_url}" + base_replaced = base_template.replace("TEMPLATE_LOCATION_HERE", base_full_url) + + print("Uploading base-devel.tar.xz") + base_devel_uploaded_url = project.upload( + f"base-devel-{build_date}.tar.xz", filepath="base-devel.tar.xz" + )["url"] + base_devel_template = Path("Dockerfile.template").read_text() + base_devel_full_url = f"{project_url}{base_devel_uploaded_url}" + base_devel_replaced = base_devel_template.replace( + "TEMPLATE_LOCATION_HERE", base_devel_full_url + ) + + print("Templating Dockerfiles") + data = { + "branch": "add-base-devel-tags", + "commit_message": f"Release {build_date}", + "actions": [ + { + "action": "update", + "file_path": "base/Dockerfile", + "content": base_replaced, + }, + { + "action": "update", + "file_path": "base-devel/Dockerfile", + "content": base_devel_replaced, + }, + ], + } + project.commits.create(data) + + print("Creating release") + release = project.releases.create( + { + "name": f"Release {build_date}", + "tag_name": build_date, + "description": f"Release {build_date}", + "ref": "add-base-devel-tags", + "assets": { + "links": [ + { + "name": "base.tar.xz", + "url": base_full_url, + "link_type": "package", + }, + { + "name": "base-devel.tar.xz", + "url": base_devel_full_url, + "link_type": "package", + } + ] + }, + } + ) + print("Created release", release.get_id()) diff --git a/rootfs/etc/pacman.d/mirrorlist b/rootfs/etc/pacman.d/mirrorlist index 3735559..31adb47 100644 --- a/rootfs/etc/pacman.d/mirrorlist +++ b/rootfs/etc/pacman.d/mirrorlist @@ -1,2 +1,3 @@ +Server = https://mirror.pkgbuild.com/$repo/os/$arch Server = https://mirror.rackspace.com/archlinux/$repo/os/$arch Server = https://mirror.leaseweb.net/archlinux/$repo/os/$arch From 9b52d5674a5b59386183289dca7ec54249d8466f Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Mon, 31 Aug 2020 22:59:57 +0200 Subject: [PATCH 02/92] Generate and verify checksum for the rootfs --- .gitlab-ci.yml | 13 ++++++------- Dockerfile.template | 10 +++++++++- Makefile | 24 ++++++++++++++++++------ ci/release.py | 23 +++++++++++++++++++---- 4 files changed, 52 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 97520a6..aaa3642 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -9,7 +9,8 @@ stages: lint: stage: lint image: hadolint/hadolint:latest - script: hadolint --ignore DL3020 Dockerfile.template + # DL3007: We use the latest tag for multistage build + script: hadolint --ignore DL3007 --ignore DL3020 Dockerfile.template rootfs:base: stage: rootfs @@ -20,10 +21,11 @@ rootfs:base: - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot - - make base.tar.xz + - make dockerfile-image-base artifacts: paths: - base.tar.xz + - Dockerfile.base expire_in: 10m reports: dotenv: build.env @@ -37,10 +39,11 @@ rootfs:base-devel: - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot - - make base-devel.tar.xz + - make dockerfile-image-base-devel artifacts: paths: - base-devel.tar.xz + - Dockerfile.base-devel expire_in: 10m reports: dotenv: build.env @@ -54,8 +57,6 @@ docker:base: - job: "rootfs:base" before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base - - unxz base.tar.xz script: - /kaniko/executor --whitelist-var-run="false" @@ -72,8 +73,6 @@ docker:base-devel: - job: "rootfs:base-devel" before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json - - sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel - - unxz base-devel.tar.xz script: - /kaniko/executor --whitelist-var-run="false" diff --git a/Dockerfile.template b/Dockerfile.template index ffe6415..45612ef 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,5 +1,13 @@ +FROM archlinux:latest AS verify +COPY TEMPLATE_ROOTFS_FILE / +SHELL ["/bin/bash", "-c"] +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ + sha256sum -c <<< "TEMPLATE_ROOTFS_HASH" && \ + mkdir /rootfs && \ + tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" + FROM scratch AS base -ADD TEMPLATE_LOCATION_HERE / +COPY --from=verify /rootfs/ / # manually run all alpm hooks that can't be run inside the fakechroot RUN ldconfig && update-ca-trust && locale-gen diff --git a/Makefile b/Makefile index 1736e13..9ed96df 100644 --- a/Makefile +++ b/Makefile @@ -49,20 +49,32 @@ rootfs-base-devel: hooks base.tar.xz: rootfs-base xz -9 -T0 -f base.tar + sha256sum base.tar.xz > base.tar.xz.SHA256 base-devel.tar.xz: rootfs-base-devel xz -9 -T0 -f base-devel.tar + sha256sum base-devel.tar.xz > base-devel.tar.xz.SHA256 + +.PHONY: dockerfile-image-base +dockerfile-image-base: base.tar.xz + sed -e "s/TEMPLATE_ROOTFS_FILE/base.tar.xz/" \ + -e "s/TEMPLATE_ROOTFS_URL/file:\/\/\/base.tar.xz/" \ + -e "s/TEMPLATE_ROOTFS_HASH/$$(cat base.tar.xz.SHA256)/" \ + Dockerfile.template > Dockerfile.base + +.PHONY: dockerfile-image-base-devel +dockerfile-image-base-devel: base-devel.tar.xz + sed -e "s/TEMPLATE_ROOTFS_FILE/base-devel.tar.xz/" \ + -e "s/TEMPLATE_ROOTFS_URL/file:\/\/\/base-devel.tar.xz/" \ + -e "s/TEMPLATE_ROOTFS_HASH/$$(cat base-devel.tar.xz.SHA256)/" \ + Dockerfile.template > Dockerfile.base-devel .PHONY: docker-image-base -docker-image-base: base.tar.xz - unxz base.tar.xz - sed "s/TEMPLATE_LOCATION_HERE/base.tar/" Dockerfile.template > Dockerfile.base +docker-image-base: dockerfile-image-base docker build -f Dockerfile.base -t archlinux/archlinux:base . .PHONY: docker-image-base-devel -docker-image-base-devel: base-devel.tar.xz - unxz base-devel.tar.xz - sed "s/TEMPLATE_LOCATION_HERE/base-devel.tar/" Dockerfile.template > Dockerfile.base-devel +docker-image-base-devel: dockerfile-image-base-devel docker build -f Dockerfile.base-devel -t archlinux/archlinux:base-devel . .PHONY: docker-push-base diff --git a/ci/release.py b/ci/release.py index 5896f20..b4b5fed 100755 --- a/ci/release.py +++ b/ci/release.py @@ -11,6 +11,7 @@ Required env vars: """ import os +import re from pathlib import Path import gitlab @@ -24,22 +25,36 @@ if __name__ == "__main__": project = gl.projects.get(project_id) print("Uploading base.tar.xz") + base_filename = f"base-{build_date}.tar.xz" base_uploaded_url = project.upload( - f"base-{build_date}.tar.xz", filepath="base.tar.xz" + base_filename, filepath="base.tar.xz" )["url"] base_template = Path("Dockerfile.template").read_text() base_full_url = f"{project_url}{base_uploaded_url}" - base_replaced = base_template.replace("TEMPLATE_LOCATION_HERE", base_full_url) + base_replaced = base_template.replace("TEMPLATE_ROOTFS_URL", base_full_url) + base_hash = f"{Path('base.tar.xz.SHA256').read_text()[0:64]} {base_filename}" + base_replaced = base_replaced.replace( + "TEMPLATE_ROOTFS_HASH", base_hash + ) + # Remove the line containing TEMPLATE_ROOTFS_FILE + base_replaced = re.sub(".*TEMPLATE_ROOTFS_FILE.*\n", "", base_replaced) print("Uploading base-devel.tar.xz") + base_devel_filename = f"base-devel-{build_date}.tar.xz" base_devel_uploaded_url = project.upload( - f"base-devel-{build_date}.tar.xz", filepath="base-devel.tar.xz" + base_devel_filename, filepath="base-devel.tar.xz" )["url"] base_devel_template = Path("Dockerfile.template").read_text() base_devel_full_url = f"{project_url}{base_devel_uploaded_url}" base_devel_replaced = base_devel_template.replace( - "TEMPLATE_LOCATION_HERE", base_devel_full_url + "TEMPLATE_ROOTFS_URL", base_devel_full_url ) + base_devel_hash = f"{Path('base-devel.tar.xz.SHA256').read_text()[0:64]} {base_devel_filename}" + base_devel_replaced = base_devel_replaced.replace( + "TEMPLATE_ROOTFS_HASH", base_devel_hash + ) + # Remove the line containing TEMPLATE_ROOTFS_FILE + base_devel_replaced = re.sub(".*TEMPLATE_ROOTFS_FILE.*\n", "", base_devel_replaced) print("Templating Dockerfiles") data = { From 72d1d5eb2de1368980229573d3b9fd11becffac2 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Wed, 14 Oct 2020 01:15:22 +0200 Subject: [PATCH 03/92] Observe security concept --- .gitlab-ci.yml | 210 ++++++++++++++++++++++++++++++------------------- 1 file changed, 130 insertions(+), 80 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aaa3642..231e405 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,7 +1,7 @@ stages: - lint - rootfs - - docker + - image - test - release - publish @@ -12,51 +12,78 @@ lint: # DL3007: We use the latest tag for multistage build script: hadolint --ignore DL3007 --ignore DL3020 Dockerfile.template -rootfs:base: +.rootfs: stage: rootfs image: archlinux:latest - needs: - - job: "lint" before_script: - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env - script: - pacman -Syu --noconfirm make devtools fakechroot fakeroot + script: - make dockerfile-image-base artifacts: paths: - base.tar.xz + - base-devel.tar.xz - Dockerfile.base - expire_in: 10m + - Dockerfile.base-devel + expire_in: 2h reports: dotenv: build.env +rootfs:base: + extends: .rootfs + except: + - master + - schedules + - tags + script: + - make dockerfile-image-base + rootfs:base-devel: - stage: rootfs - image: archlinux:latest - needs: - - job: "lint" - before_script: - - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env + extends: .rootfs + except: + - master + - schedules + - tags script: - - pacman -Syu --noconfirm make devtools fakechroot fakeroot - make dockerfile-image-base-devel - artifacts: - paths: - - base-devel.tar.xz - - Dockerfile.base-devel - expire_in: 10m - reports: - dotenv: build.env -docker:base: - stage: docker +rootfs:base:secure: + extends: .rootfs + tags: + - secure + only: + - master + - schedules + - tags + script: + - make dockerfile-image-base + +rootfs:base-devel:secure: + extends: .rootfs + tags: + - secure + only: + - master + - schedules + - tags + script: + - make dockerfile-image-base-devel + +.image: + stage: image image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] - needs: - - job: "rootfs:base" before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json + +image:base: + extends: .image + except: + - master + - schedules + - tags script: - /kaniko/executor --whitelist-var-run="false" @@ -64,15 +91,12 @@ docker:base: --dockerfile $CI_PROJECT_DIR/Dockerfile.base --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG -docker:base-devel: - stage: docker - image: - name: gcr.io/kaniko-project/executor:debug - entrypoint: [""] - needs: - - job: "rootfs:base-devel" - before_script: - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json +image:base-devel: + extends: .image + except: + - master + - schedules + - tags script: - /kaniko/executor --whitelist-var-run="false" @@ -80,55 +104,81 @@ docker:base-devel: --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG -test:base: - stage: test - image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG - needs: - - job: "docker:base" - artifacts: false - script: - - pacman -Sy - - pacman -Qqk - - pacman -Syu --noconfirm docker grep - - docker -v - - id -u http - - locale | grep -q UTF-8 - -test:base-devel: - stage: test - image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG - needs: - - job: "docker:base-devel" - artifacts: false - script: - - pacman -Sy - - pacman -Qqk - - pacman -Syu --noconfirm docker grep - - docker -v - - id -u http - - locale | grep -q UTF-8 - - gcc -v - - g++ -v - - make -v - -release: - stage: release - image: archlinux:latest - only: - refs: - - master - - add-base-devel-tags - variables: - - $SCHEDULED_PUBLISH == "TRUE" - needs: - - job: "test:base" - - job: "test:base-devel" - before_script: - - pacman -Syu python-gitlab - script: - - python ci/release.py +image:base:secure: + extends: .image tags: - secure + only: + - master + - schedules + - tags + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/Dockerfile.base + --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + +image:base-devel:secure: + extends: .image + tags: + - secure + only: + - master + - schedules + - tags + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel + --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + +# test:base: +# stage: test +# image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG +# dependencies: [] +# script: +# - pacman -Sy +# - pacman -Qqk +# - pacman -Syu --noconfirm docker grep +# - docker -v +# - id -u http +# - locale | grep -q UTF-8 +# +# test:base-devel: +# stage: test +# image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG +# dependencies: [] +# script: +# - pacman -Sy +# - pacman -Qqk +# - pacman -Syu --noconfirm docker grep +# - docker -v +# - id -u http +# - locale | grep -q UTF-8 +# - gcc -v +# - g++ -v +# - make -v +# +# release: +# stage: release +# image: archlinux:latest +# only: +# refs: +# - master +# - add-base-devel-tags +# variables: +# - $SCHEDULED_PUBLISH == "TRUE" +# needs: +# - job: "test:base" +# - job: "test:base-devel" +# before_script: +# - pacman -Syu python-gitlab +# script: +# - python ci/release.py +# tags: +# - secure # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base # publish:org:base: From dcf7c44d1221f936253768bd471c86d8ada0632a Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Wed, 14 Oct 2020 01:16:28 +0200 Subject: [PATCH 04/92] Ignore Dockerfile.base and Dockerfile.base-devel We don't be checking these in as they are generated by the build. --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 94f5842..49803f4 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ /base.tar* /base-devel.tar* rootfs/etc/pacman.conf +Dockerfile.base +Dockerfile.base-devel From 4fbd8d2d7dd4adf2cbca1a728faaa08e792f1458 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 18 Oct 2020 03:24:15 +0200 Subject: [PATCH 05/92] Don't use xz -9 - it takes too long to no benefit --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 9ed96df..1c8870b 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,11 @@ rootfs-base-devel: hooks rm -rf $(BUILDDIR) alpm-hooks base.tar.xz: rootfs-base - xz -9 -T0 -f base.tar + xz -T0 -f base.tar sha256sum base.tar.xz > base.tar.xz.SHA256 base-devel.tar.xz: rootfs-base-devel - xz -9 -T0 -f base-devel.tar + xz -T0 -f base-devel.tar sha256sum base-devel.tar.xz > base-devel.tar.xz.SHA256 .PHONY: dockerfile-image-base From 13793e559d547ba8377d6a1b2570f6d7401b904d Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 18 Oct 2020 04:47:36 +0200 Subject: [PATCH 06/92] Try copying underlying /etc/resolv.conf --- Dockerfile.template | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.template b/Dockerfile.template index 45612ef..5917be8 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -8,6 +8,7 @@ RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effecti FROM scratch AS base COPY --from=verify /rootfs/ / +COPY --from=verify /etc/resolv.conf /etc/resolv.conf # manually run all alpm hooks that can't be run inside the fakechroot RUN ldconfig && update-ca-trust && locale-gen From 2706f3bb9a408cddf2504ccbe14c742d70e8be80 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 18 Oct 2020 05:02:53 +0200 Subject: [PATCH 07/92] Add note for kaniko and resolv.conf --- Dockerfile.template | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Dockerfile.template b/Dockerfile.template index 5917be8..ab0f23e 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -8,6 +8,8 @@ RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effecti FROM scratch AS base COPY --from=verify /rootfs/ / + +# Fix kaniko DNS problems as kaniko uses the current layer's resolv.conf. COPY --from=verify /etc/resolv.conf /etc/resolv.conf # manually run all alpm hooks that can't be run inside the fakechroot From 3bf05f92aa63c31536da7d2001950af0d6fcdad8 Mon Sep 17 00:00:00 2001 From: Kristian Klausen Date: Sun, 18 Oct 2020 16:09:28 +0200 Subject: [PATCH 08/92] Exclude /etc/resolv.conf --- Dockerfile.template | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index ab0f23e..b9c722d 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -4,14 +4,12 @@ SHELL ["/bin/bash", "-c"] RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ sha256sum -c <<< "TEMPLATE_ROOTFS_HASH" && \ mkdir /rootfs && \ - tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" + # Fix kaniko DNS problems as kaniko uses the current layer's resolv.conf. \ + tar -C /rootfs --extract --auto-compress --exclude=./etc/resolv.conf --file "${ROOTFS}" FROM scratch AS base COPY --from=verify /rootfs/ / -# Fix kaniko DNS problems as kaniko uses the current layer's resolv.conf. -COPY --from=verify /etc/resolv.conf /etc/resolv.conf - # manually run all alpm hooks that can't be run inside the fakechroot RUN ldconfig && update-ca-trust && locale-gen RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' From f114a6d4542840ca02b9f1738e563f125a3c7c6c Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 01:05:18 +0200 Subject: [PATCH 09/92] Revert "Don't use xz -9 - it takes too long to no benefit" This reverts commit 4fbd8d2d7dd4adf2cbca1a728faaa08e792f1458. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1c8870b..9ed96df 100644 --- a/Makefile +++ b/Makefile @@ -48,11 +48,11 @@ rootfs-base-devel: hooks rm -rf $(BUILDDIR) alpm-hooks base.tar.xz: rootfs-base - xz -T0 -f base.tar + xz -9 -T0 -f base.tar sha256sum base.tar.xz > base.tar.xz.SHA256 base-devel.tar.xz: rootfs-base-devel - xz -T0 -f base-devel.tar + xz -9 -T0 -f base-devel.tar sha256sum base-devel.tar.xz > base-devel.tar.xz.SHA256 .PHONY: dockerfile-image-base From ecb01053cc02e66e2d28ba7d26a6b153a7e869ff Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 01:13:04 +0200 Subject: [PATCH 10/92] Use ./exclude mechanism for /etc/resolv.conf --- Dockerfile.template | 3 +-- exclude | 3 ++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile.template b/Dockerfile.template index b9c722d..45612ef 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -4,8 +4,7 @@ SHELL ["/bin/bash", "-c"] RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ sha256sum -c <<< "TEMPLATE_ROOTFS_HASH" && \ mkdir /rootfs && \ - # Fix kaniko DNS problems as kaniko uses the current layer's resolv.conf. \ - tar -C /rootfs --extract --auto-compress --exclude=./etc/resolv.conf --file "${ROOTFS}" + tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" FROM scratch AS base COPY --from=verify /rootfs/ / diff --git a/exclude b/exclude index 0a45d22..7074803 100644 --- a/exclude +++ b/exclude @@ -5,6 +5,7 @@ ./dev ./etc/hostname ./etc/machine-id +./etc/resolv.conf ./etc/pacman.d/gnupg/openpgp-revocs.d/* ./etc/pacman.d/gnupg/private-keys-v1.d/* ./etc/pacman.d/gnupg/pubring.gpg~ @@ -13,4 +14,4 @@ ./tmp/* ./var/cache/pacman/pkg/* ./var/lib/pacman/sync/* -./var/tmp/* \ No newline at end of file +./var/tmp/* From 3f2721cb6b295319c3db745ec212e617b4f646cf Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 02:33:44 +0200 Subject: [PATCH 11/92] Change ci-only Dockerfile location --- base-devel/Dockerfile | 0 base/Dockerfile | 0 ci/base-devel/Dockerfile | 2 ++ ci/base/Dockerfile | 2 ++ ci/release.py | 4 ++-- 5 files changed, 6 insertions(+), 2 deletions(-) delete mode 100644 base-devel/Dockerfile delete mode 100644 base/Dockerfile create mode 100644 ci/base-devel/Dockerfile create mode 100644 ci/base/Dockerfile diff --git a/base-devel/Dockerfile b/base-devel/Dockerfile deleted file mode 100644 index e69de29..0000000 diff --git a/base/Dockerfile b/base/Dockerfile deleted file mode 100644 index e69de29..0000000 diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile new file mode 100644 index 0000000..44879eb --- /dev/null +++ b/ci/base-devel/Dockerfile @@ -0,0 +1,2 @@ +# Don't delete. +# This is a placeholder so that our `ci/release.py` will work properly. diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile new file mode 100644 index 0000000..44879eb --- /dev/null +++ b/ci/base/Dockerfile @@ -0,0 +1,2 @@ +# Don't delete. +# This is a placeholder so that our `ci/release.py` will work properly. diff --git a/ci/release.py b/ci/release.py index b4b5fed..5918847 100755 --- a/ci/release.py +++ b/ci/release.py @@ -63,12 +63,12 @@ if __name__ == "__main__": "actions": [ { "action": "update", - "file_path": "base/Dockerfile", + "file_path": "ci/base/Dockerfile", "content": base_replaced, }, { "action": "update", - "file_path": "base-devel/Dockerfile", + "file_path": "ci/base-devel/Dockerfile", "content": base_devel_replaced, }, ], From 2cbbe5a7201944c58f35af2fccc068dbe0b43d0c Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 03:32:25 +0200 Subject: [PATCH 12/92] Deduplicate a lot of code --- .gitignore | 10 ++---- Makefile | 101 +++++++++++++++++++---------------------------------- exclude | 1 + 3 files changed, 38 insertions(+), 74 deletions(-) diff --git a/.gitignore b/.gitignore index 49803f4..76f4628 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,2 @@ -*~ -*.orig -/.idea -/base.tar* -/base-devel.tar* -rootfs/etc/pacman.conf -Dockerfile.base -Dockerfile.base-devel +build +output diff --git a/Makefile b/Makefile index 9ed96df..0bb7a87 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,18 @@ -DOCKER_USER:=pierres -BUILDDIR=build -PWD=$(shell pwd) +BUILDDIR=$(shell pwd)/build +OUTPUTDIR=$(shell pwd)/output -.PHONY: hooks -hooks: - mkdir -p alpm-hooks/usr/share/libalpm/hooks - find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(PWD)/alpm-hooks{} \; +define rootfs + mkdir -vp $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks + find /usr/share/libalpm/hooks -exec ln -sf /dev/null $(BUILDDIR)/alpm-hooks{} \; -.PHONY: rootfs-base -rootfs-base: hooks - mkdir -vp $(BUILDDIR)/var/lib/pacman/ - cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf - cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf + mkdir -vp $(BUILDDIR)/var/lib/pacman/ $(OUTPUTDIR) + install -Dm644 /usr/share/devtools/pacman-extra.conf $(BUILDDIR)/etc/pacman.conf + cat pacman-conf.d-noextract.conf >> $(BUILDDIR)/etc/pacman.conf fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ - --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ - --config rootfs/etc/pacman.conf \ + --noconfirm --dbpath $(BUILDDIR)/var/lib/pacman \ + --config $(BUILDDIR)/etc/pacman.conf \ --noscriptlet \ - --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base + --hookdir $(BUILDDIR)/alpm-hooks/usr/share/libalpm/hooks/ $(2) cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ # remove passwordless login for root (see CVE-2019-5021 for reference) @@ -24,65 +20,38 @@ rootfs-base: hooks # fakeroot to map the gid/uid of the builder process to root # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base.tar - rm -rf $(BUILDDIR) alpm-hooks + fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f $(OUTPUTDIR)/$(1).tar -.PHONY: rootfs-base-devel -rootfs-base-devel: hooks - mkdir -vp $(BUILDDIR)/var/lib/pacman/ - cp /usr/share/devtools/pacman-extra.conf rootfs/etc/pacman.conf - cat pacman-conf.d-noextract.conf >> rootfs/etc/pacman.conf - fakechroot -- fakeroot -- pacman -Sy -r $(BUILDDIR) \ - --noconfirm --dbpath $(PWD)/$(BUILDDIR)/var/lib/pacman \ - --config rootfs/etc/pacman.conf \ - --noscriptlet \ - --hookdir $(PWD)/alpm-hooks/usr/share/libalpm/hooks/ base base-devel - cp --recursive --preserve=timestamps --backup --suffix=.pacnew rootfs/* $(BUILDDIR)/ + cd $(OUTPUTDIR); xz -9 -T0 -f $(1).tar; sha256sum $(1).tar.xz > $(1).tar.xz.SHA256 +endef - # remove passwordless login for root (see CVE-2019-5021 for reference) - sed -i -e 's/^root::/root:!:/' "$(BUILDDIR)/etc/shadow" +define dockerfile + sed -e "s|TEMPLATE_ROOTFS_FILE|$(1).tar.xz|" \ + -e "s|TEMPLATE_ROOTFS_URL|file:///$(1).tar.xz|" \ + -e "s|TEMPLATE_ROOTFS_HASH|$$(cat $(OUTPUTDIR)/$(1).tar.xz.SHA256)|" \ + Dockerfile.template > $(OUTPUTDIR)/Dockerfile.$(1) +endef - # fakeroot to map the gid/uid of the builder process to root - # fixes #22 - fakeroot -- tar --numeric-owner --xattrs --acls --exclude-from=exclude -C $(BUILDDIR) -c . -f base-devel.tar - rm -rf $(BUILDDIR) alpm-hooks +.PHONY: clean +clean: + rm -rf $(BUILDDIR) $(OUTPUTDIR) -base.tar.xz: rootfs-base - xz -9 -T0 -f base.tar - sha256sum base.tar.xz > base.tar.xz.SHA256 +$(OUTPUTDIR)/base.tar.xz: + $(call rootfs,base,base) -base-devel.tar.xz: rootfs-base-devel - xz -9 -T0 -f base-devel.tar - sha256sum base-devel.tar.xz > base-devel.tar.xz.SHA256 +$(OUTPUTDIR)/base-devel.tar.xz: + $(call rootfs,base,base base-devel) -.PHONY: dockerfile-image-base -dockerfile-image-base: base.tar.xz - sed -e "s/TEMPLATE_ROOTFS_FILE/base.tar.xz/" \ - -e "s/TEMPLATE_ROOTFS_URL/file:\/\/\/base.tar.xz/" \ - -e "s/TEMPLATE_ROOTFS_HASH/$$(cat base.tar.xz.SHA256)/" \ - Dockerfile.template > Dockerfile.base +$(OUTPUTDIR)/Dockerfile.base: + $(call dockerfile,base) -.PHONY: dockerfile-image-base-devel -dockerfile-image-base-devel: base-devel.tar.xz - sed -e "s/TEMPLATE_ROOTFS_FILE/base-devel.tar.xz/" \ - -e "s/TEMPLATE_ROOTFS_URL/file:\/\/\/base-devel.tar.xz/" \ - -e "s/TEMPLATE_ROOTFS_HASH/$$(cat base-devel.tar.xz.SHA256)/" \ - Dockerfile.template > Dockerfile.base-devel +$(OUTPUTDIR)/Dockerfile.base-devel: + $(call dockerfile,base-devel) .PHONY: docker-image-base -docker-image-base: dockerfile-image-base - docker build -f Dockerfile.base -t archlinux/archlinux:base . +image-base: $(OUTPUTDIR)/base.tar.xz $(OUTPUTDIR)/Dockerfile.base + docker build -f $(OUTPUTDIR)/Dockerfile.base -t archlinux/archlinux:base $(OUTPUTDIR) .PHONY: docker-image-base-devel -docker-image-base-devel: dockerfile-image-base-devel - docker build -f Dockerfile.base-devel -t archlinux/archlinux:base-devel . - -.PHONY: docker-push-base -docker-push-base: - docker login -u $(DOCKER_USER) - docker push archlinux/archlinux:base - -.PHONY: docker-push-base-devel -docker-push-base-devel: - docker login -u $(DOCKER_USER) - docker push archlinux/archlinux:base-devel +image-base-devel: $(OUTPUTDIR)/base-devel.tar.xz $(OUTPUTDIR)/Dockerfile.base-devel + docker build -f $(OUTPUTDIR)/Dockerfile.base-devel -t archlinux/archlinux:base-devel $(OUTPUTDIR) diff --git a/exclude b/exclude index 7074803..e847b05 100644 --- a/exclude +++ b/exclude @@ -15,3 +15,4 @@ ./var/cache/pacman/pkg/* ./var/lib/pacman/sync/* ./var/tmp/* +./alpm-hooks From 36bc03260de1e7eda8907d11800c7a62e7435ec2 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 03:35:07 +0200 Subject: [PATCH 13/92] Use new Makefile targets to .gitlab-ci.yml --- .gitlab-ci.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 231e405..c178fe6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,8 +18,6 @@ lint: before_script: - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env - pacman -Syu --noconfirm make devtools fakechroot fakeroot - script: - - make dockerfile-image-base artifacts: paths: - base.tar.xz @@ -37,7 +35,7 @@ rootfs:base: - schedules - tags script: - - make dockerfile-image-base + - make image-base rootfs:base-devel: extends: .rootfs @@ -46,7 +44,7 @@ rootfs:base-devel: - schedules - tags script: - - make dockerfile-image-base-devel + - make image-base-devel rootfs:base:secure: extends: .rootfs @@ -57,7 +55,7 @@ rootfs:base:secure: - schedules - tags script: - - make dockerfile-image-base + - make image-base rootfs:base-devel:secure: extends: .rootfs @@ -68,7 +66,7 @@ rootfs:base-devel:secure: - schedules - tags script: - - make dockerfile-image-base-devel + - make image-base-devel .image: stage: image From 8d8d296f6d1176bb2b8ee2a393369f0140d4bf53 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 03:41:22 +0200 Subject: [PATCH 14/92] Generate correct artifacts --- .gitlab-ci.yml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c178fe6..5216410 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,10 +20,7 @@ lint: - pacman -Syu --noconfirm make devtools fakechroot fakeroot artifacts: paths: - - base.tar.xz - - base-devel.tar.xz - - Dockerfile.base - - Dockerfile.base-devel + - output/* expire_in: 2h reports: dotenv: build.env @@ -35,7 +32,7 @@ rootfs:base: - schedules - tags script: - - make image-base + - make output/base.tar.xz output/Dockerfile.base rootfs:base-devel: extends: .rootfs @@ -44,7 +41,7 @@ rootfs:base-devel: - schedules - tags script: - - make image-base-devel + - make output/base-devel.tar.xz output/Dockerfile.base-devel rootfs:base:secure: extends: .rootfs @@ -55,7 +52,7 @@ rootfs:base:secure: - schedules - tags script: - - make image-base + - make output/base.tar.xz output/Dockerfile.base rootfs:base-devel:secure: extends: .rootfs @@ -66,7 +63,7 @@ rootfs:base-devel:secure: - schedules - tags script: - - make image-base-devel + - make output/base-devel.tar.xz output/Dockerfile.base-devel .image: stage: image From 2d20a3f878164e4c845076cdfc860a64fc83a7be Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 03:50:00 +0200 Subject: [PATCH 15/92] Fix make target calls --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5216410..285b7a5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -32,7 +32,7 @@ rootfs:base: - schedules - tags script: - - make output/base.tar.xz output/Dockerfile.base + - make $PWD/output/base.tar.xz $PWD/output/Dockerfile.base rootfs:base-devel: extends: .rootfs @@ -41,7 +41,7 @@ rootfs:base-devel: - schedules - tags script: - - make output/base-devel.tar.xz output/Dockerfile.base-devel + - make $PWD/output/base-devel.tar.xz $PWD/output/Dockerfile.base-devel rootfs:base:secure: extends: .rootfs @@ -52,7 +52,7 @@ rootfs:base:secure: - schedules - tags script: - - make output/base.tar.xz output/Dockerfile.base + - make $PWD/output/base.tar.xz $PWD/output/Dockerfile.base rootfs:base-devel:secure: extends: .rootfs @@ -63,7 +63,7 @@ rootfs:base-devel:secure: - schedules - tags script: - - make output/base-devel.tar.xz output/Dockerfile.base-devel + - make $PWD/output/base-devel.tar.xz $PWD/output/Dockerfile.base-devel .image: stage: image From 94d242fb8796146f93fdd0b252de42c115159928 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 03:55:27 +0200 Subject: [PATCH 16/92] Make kaniko use proper output paths --- .gitlab-ci.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 285b7a5..aa00946 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -82,8 +82,8 @@ image:base: script: - /kaniko/executor --whitelist-var-run="false" - --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile.base + --context $CI_PROJECT_DIR/output + --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG image:base-devel: @@ -95,8 +95,8 @@ image:base-devel: script: - /kaniko/executor --whitelist-var-run="false" - --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel + --context $CI_PROJECT_DIR/output + --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG image:base:secure: @@ -110,8 +110,8 @@ image:base:secure: script: - /kaniko/executor --whitelist-var-run="false" - --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile.base + --context $CI_PROJECT_DIR/output + --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG image:base-devel:secure: @@ -125,8 +125,8 @@ image:base-devel:secure: script: - /kaniko/executor --whitelist-var-run="false" - --context $CI_PROJECT_DIR - --dockerfile $CI_PROJECT_DIR/Dockerfile.base-devel + --context $CI_PROJECT_DIR/output + --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG # test:base: From 80d8c5ee918afa9ae403eab85bcefd8f8efc0e89 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:05:15 +0200 Subject: [PATCH 17/92] Fix incorrect name --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 0bb7a87..e430121 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ $(OUTPUTDIR)/base.tar.xz: $(call rootfs,base,base) $(OUTPUTDIR)/base-devel.tar.xz: - $(call rootfs,base,base base-devel) + $(call rootfs,base-devel,base base-devel) $(OUTPUTDIR)/Dockerfile.base: $(call dockerfile,base) From dc895e48cf5319eececd61e8d8b34928cfe57ba5 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:10:52 +0200 Subject: [PATCH 18/92] Rename second layer to root --- Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.template b/Dockerfile.template index 45612ef..9d67451 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -6,7 +6,7 @@ RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effecti mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" -FROM scratch AS base +FROM scratch AS root COPY --from=verify /rootfs/ / # manually run all alpm hooks that can't be run inside the fakechroot From cf6a172694a9821ec2e1f24fb7095471377548e4 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:15:13 +0200 Subject: [PATCH 19/92] Update README for all the new changes --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 30b0e77..cae1c75 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -# Docker Base Image for Arch Linux +# Arch Linux Docker Image [![pipeline status](https://gitlab.archlinux.org/archlinux/archlinux-docker/badges/master/pipeline.svg)](https://gitlab.archlinux.org/archlinux/archlinux-docker/-/commits/master) -This repository contains all scripts and files needed to create a Docker base image for Arch Linux. +This repository contains all scripts and files needed to create a Docker image for Arch Linux. ## Dependencies Install the following Arch Linux packages: @@ -15,11 +15,11 @@ Install the following Arch Linux packages: Make sure your user can directly interact with Docker (ie. `docker info` works). ## Usage -Run `make docker-image-base` to build the image `archlinux:base` with the +Run `make docker-image-base` to build the image `archlinux/archlinux:base` with the `base` group installed. You can also run `make docker-image-base-devel` to -build the image `archlinux:base-devel` with the `base-devel` group installed. +build the image `archlinux/archlinux:base-devel` with the `base-devel` group installed. -## Purpose +## Principles * Provide the Arch experience in a Docker image * Provide the most simple but complete image to base every other upon * `pacman` needs to work out of the box From 21ba6bc967be1749b200f1807377f9d9f929fc0a Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:18:12 +0200 Subject: [PATCH 20/92] Re-add test stage --- .gitlab-ci.yml | 54 +++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa00946..43ac720 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -129,33 +129,33 @@ image:base-devel:secure: --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG -# test:base: -# stage: test -# image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG -# dependencies: [] -# script: -# - pacman -Sy -# - pacman -Qqk -# - pacman -Syu --noconfirm docker grep -# - docker -v -# - id -u http -# - locale | grep -q UTF-8 -# -# test:base-devel: -# stage: test -# image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG -# dependencies: [] -# script: -# - pacman -Sy -# - pacman -Qqk -# - pacman -Syu --noconfirm docker grep -# - docker -v -# - id -u http -# - locale | grep -q UTF-8 -# - gcc -v -# - g++ -v -# - make -v -# +test:base: + stage: test + image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + dependencies: [] + script: + - pacman -Sy + - pacman -Qqk + - pacman -Syu --noconfirm docker grep + - docker -v + - id -u http + - locale | grep -q UTF-8 + +test:base-devel: + stage: test + image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + dependencies: [] + script: + - pacman -Sy + - pacman -Qqk + - pacman -Syu --noconfirm docker grep + - docker -v + - id -u http + - locale | grep -q UTF-8 + - gcc -v + - g++ -v + - make -v + # release: # stage: release # image: archlinux:latest From 7da8f99ca4abaf8bdf0f751d849e3f5b1d33da0a Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:27:02 +0200 Subject: [PATCH 21/92] Re-enable release stage --- .gitlab-ci.yml | 33 +++++++++++++++----------------- ci/release.py | 52 ++++++++++++++++++++------------------------------ 2 files changed, 36 insertions(+), 49 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 43ac720..118184a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -156,24 +156,21 @@ test:base-devel: - g++ -v - make -v -# release: -# stage: release -# image: archlinux:latest -# only: -# refs: -# - master -# - add-base-devel-tags -# variables: -# - $SCHEDULED_PUBLISH == "TRUE" -# needs: -# - job: "test:base" -# - job: "test:base-devel" -# before_script: -# - pacman -Syu python-gitlab -# script: -# - python ci/release.py -# tags: -# - secure +release: + stage: release + image: archlinux:latest + only: + refs: + - master + - add-base-devel-tags + variables: + - $SCHEDULED_PUBLISH == "TRUE" + before_script: + - pacman -Syu python-gitlab + script: + - python ci/release.py + tags: + - secure # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base # publish:org:base: diff --git a/ci/release.py b/ci/release.py index 5918847..d93d4f7 100755 --- a/ci/release.py +++ b/ci/release.py @@ -20,41 +20,31 @@ build_date = os.environ['BUILD_DATE'] project_id = os.environ['CI_PROJECT_ID'] project_url = os.environ['CI_PROJECT_URL'] + +def upload(name): + print(f"Uploading {name}.tar.xz") + filename = f"{name}-{build_date}.tar.xz" + uploaded_url = project.upload( + filename, filepath="output/{name}.tar.xz" + )["url"] + template = Path("Dockerfile.template").read_text() + full_url = f"{project_url}{uploaded_url}" + replaced = template.replace("TEMPLATE_ROOTFS_URL", full_url) + hash = f"{Path('output/{name}.tar.xz.SHA256').read_text()[0:64]} {filename}" + replaced = replaced.replace( + "TEMPLATE_ROOTFS_HASH", hash + ) + # Remove the line containing TEMPLATE_ROOTFS_FILE + replaced = re.sub(".*TEMPLATE_ROOTFS_FILE.*\n", "", replaced) + return replaced, full_url + + if __name__ == "__main__": gl = gitlab.Gitlab("https://gitlab.archlinux.org", token) project = gl.projects.get(project_id) - print("Uploading base.tar.xz") - base_filename = f"base-{build_date}.tar.xz" - base_uploaded_url = project.upload( - base_filename, filepath="base.tar.xz" - )["url"] - base_template = Path("Dockerfile.template").read_text() - base_full_url = f"{project_url}{base_uploaded_url}" - base_replaced = base_template.replace("TEMPLATE_ROOTFS_URL", base_full_url) - base_hash = f"{Path('base.tar.xz.SHA256').read_text()[0:64]} {base_filename}" - base_replaced = base_replaced.replace( - "TEMPLATE_ROOTFS_HASH", base_hash - ) - # Remove the line containing TEMPLATE_ROOTFS_FILE - base_replaced = re.sub(".*TEMPLATE_ROOTFS_FILE.*\n", "", base_replaced) - - print("Uploading base-devel.tar.xz") - base_devel_filename = f"base-devel-{build_date}.tar.xz" - base_devel_uploaded_url = project.upload( - base_devel_filename, filepath="base-devel.tar.xz" - )["url"] - base_devel_template = Path("Dockerfile.template").read_text() - base_devel_full_url = f"{project_url}{base_devel_uploaded_url}" - base_devel_replaced = base_devel_template.replace( - "TEMPLATE_ROOTFS_URL", base_devel_full_url - ) - base_devel_hash = f"{Path('base-devel.tar.xz.SHA256').read_text()[0:64]} {base_devel_filename}" - base_devel_replaced = base_devel_replaced.replace( - "TEMPLATE_ROOTFS_HASH", base_devel_hash - ) - # Remove the line containing TEMPLATE_ROOTFS_FILE - base_devel_replaced = re.sub(".*TEMPLATE_ROOTFS_FILE.*\n", "", base_devel_replaced) + base_replaced, base_full_url = upload("base") + base_devel_replaced, base_devel_full_url = upload("base-devel") print("Templating Dockerfiles") data = { From 0e58892c756f1462fa96c219d74c2265c9854f83 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:50:18 +0200 Subject: [PATCH 22/92] Run secure jobs also on add-base-devel-tags for testing --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 118184a..f908862 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -105,6 +105,7 @@ image:base:secure: - secure only: - master + - add-base-devel-tags - schedules - tags script: @@ -120,6 +121,7 @@ image:base-devel:secure: - secure only: - master + - add-base-devel-tags - schedules - tags script: From 24157ab0192f4c29a0ca713a76571837a25c452a Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 04:54:08 +0200 Subject: [PATCH 23/92] Properly exclude add-base-devel-tags branch --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f908862..fd18eb1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -77,6 +77,7 @@ image:base: extends: .image except: - master + - add-base-devel-tags - schedules - tags script: @@ -90,6 +91,7 @@ image:base-devel: extends: .image except: - master + - add-base-devel-tags - schedules - tags script: From 2de94ea296f3b8e4e4d9bcfd6845aa3da88a11e6 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 05:03:48 +0200 Subject: [PATCH 24/92] Run release on schedules --- .gitlab-ci.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fd18eb1..3a2dd07 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -165,8 +165,7 @@ release: image: archlinux:latest only: refs: - - master - - add-base-devel-tags + - schedules variables: - $SCHEDULED_PUBLISH == "TRUE" before_script: From 5a83584a67ddbb6bc4bd8d5a72f6efdcd8ea2981 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 05:18:46 +0200 Subject: [PATCH 25/92] Move tags keyword up --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3a2dd07..a781f27 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -163,6 +163,8 @@ test:base-devel: release: stage: release image: archlinux:latest + tags: + - secure only: refs: - schedules @@ -172,8 +174,6 @@ release: - pacman -Syu python-gitlab script: - python ci/release.py - tags: - - secure # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base # publish:org:base: From 7747e4616e3e9f49d8b1e3b6925e12694a61def7 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 19:10:49 +0200 Subject: [PATCH 26/92] Add CI_JOB_ID GitLab CI variable to BUILD_DATE --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a781f27..1f2cf4a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ lint: stage: rootfs image: archlinux:latest before_script: - - echo "BUILD_DATE=$(date +%Y%m%d)" > build.env + - echo "BUILD_DATE=$(date +%Y%m%d).$CI_JOB_ID" > build.env - pacman -Syu --noconfirm make devtools fakechroot fakeroot artifacts: paths: From 559d5053b3caa13097b4c5c8d7a31bbc363a36ed Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 19:15:52 +0200 Subject: [PATCH 27/92] Define archlinux:latest as default image --- .gitlab-ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1f2cf4a..12f62f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,3 +1,6 @@ +default: + image: "archlinux:latest" + stages: - lint - rootfs @@ -14,7 +17,6 @@ lint: .rootfs: stage: rootfs - image: archlinux:latest before_script: - echo "BUILD_DATE=$(date +%Y%m%d).$CI_JOB_ID" > build.env - pacman -Syu --noconfirm make devtools fakechroot fakeroot @@ -162,7 +164,6 @@ test:base-devel: release: stage: release - image: archlinux:latest tags: - secure only: From 2f411f062cf2582bb7763a18823929381ccfc9d2 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 19:18:08 +0200 Subject: [PATCH 28/92] Debug --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12f62f5..58ac219 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -169,8 +169,8 @@ release: only: refs: - schedules - variables: - - $SCHEDULED_PUBLISH == "TRUE" + # variables: + # - $SCHEDULED_PUBLISH == "TRUE" before_script: - pacman -Syu python-gitlab script: From 03fe0b4250e9145308cc2e9c026d4801597f51f2 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 19:20:05 +0200 Subject: [PATCH 29/92] Undebug --- .gitlab-ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 58ac219..12f62f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -169,8 +169,8 @@ release: only: refs: - schedules - # variables: - # - $SCHEDULED_PUBLISH == "TRUE" + variables: + - $SCHEDULED_PUBLISH == "TRUE" before_script: - pacman -Syu python-gitlab script: From c786b40a38872a42774b52a1ff0a8de514cb50d2 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 19:26:55 +0200 Subject: [PATCH 30/92] Use pacman --noconfirm --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 12f62f5..8d52872 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,7 +172,7 @@ release: variables: - $SCHEDULED_PUBLISH == "TRUE" before_script: - - pacman -Syu python-gitlab + - pacman -Syu --noconfirm python-gitlab script: - python ci/release.py From dba6a9053a9d87fea55b2e8afd74adda8df16582 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 19:44:03 +0200 Subject: [PATCH 31/92] Fix format string --- ci/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release.py b/ci/release.py index d93d4f7..5fdc7f0 100755 --- a/ci/release.py +++ b/ci/release.py @@ -25,7 +25,7 @@ def upload(name): print(f"Uploading {name}.tar.xz") filename = f"{name}-{build_date}.tar.xz" uploaded_url = project.upload( - filename, filepath="output/{name}.tar.xz" + filename, filepath=f"output/{name}.tar.xz" )["url"] template = Path("Dockerfile.template").read_text() full_url = f"{project_url}{uploaded_url}" From b0ecf075d4ae01a5520f616812df8247c89d7751 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 22:11:22 +0200 Subject: [PATCH 32/92] Fix another format string --- ci/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release.py b/ci/release.py index 5fdc7f0..7a44d79 100755 --- a/ci/release.py +++ b/ci/release.py @@ -30,7 +30,7 @@ def upload(name): template = Path("Dockerfile.template").read_text() full_url = f"{project_url}{uploaded_url}" replaced = template.replace("TEMPLATE_ROOTFS_URL", full_url) - hash = f"{Path('output/{name}.tar.xz.SHA256').read_text()[0:64]} {filename}" + hash = f"Path('output/{name}.tar.xz.SHA256').read_text()[0:64] {filename}" replaced = replaced.replace( "TEMPLATE_ROOTFS_HASH", hash ) From d6d626619ee72cc119cddd56cdb4fac0068d0a9c Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 22:34:24 +0200 Subject: [PATCH 33/92] Refactor to use BUILD_VERSION instead of BUILD_DATE --- .gitlab-ci.yml | 120 +++++++++++++++++++++++-------------------------- ci/release.py | 14 +++--- 2 files changed, 64 insertions(+), 70 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8d52872..6a28c45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -18,7 +18,7 @@ lint: .rootfs: stage: rootfs before_script: - - echo "BUILD_DATE=$(date +%Y%m%d).$CI_JOB_ID" > build.env + - echo "BUILD_VERSION=$(date +%Y%m%d).$CI_JOB_ID" > build.env - pacman -Syu --noconfirm make devtools fakechroot fakeroot artifacts: paths: @@ -117,7 +117,7 @@ image:base:secure: --whitelist-var-run="false" --context $CI_PROJECT_DIR/output --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base - --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + --destination $CI_REGISTRY_IMAGE:base image:base-devel:secure: extends: .image @@ -133,31 +133,49 @@ image:base-devel:secure: --whitelist-var-run="false" --context $CI_PROJECT_DIR/output --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel - --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + --destination $CI_REGISTRY_IMAGE:base-devel + +.test: + script: + - pacman -Sy + - pacman -Qqk + - pacman -Syu --noconfirm docker grep + - docker -v + - id -u http + - locale | grep -q UTF-8 test:base: + extends: .test stage: test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG dependencies: [] - script: - - pacman -Sy - - pacman -Qqk - - pacman -Syu --noconfirm docker grep - - docker -v - - id -u http - - locale | grep -q UTF-8 test:base-devel: + extends: .test stage: test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG dependencies: [] - script: - - pacman -Sy - - pacman -Qqk - - pacman -Syu --noconfirm docker grep - - docker -v - - id -u http - - locale | grep -q UTF-8 + after_script: + - gcc -v + - g++ -v + - make -v + +test:base:secure: + extends: .test + stage: test + tags: + - secure + image: $CI_REGISTRY_IMAGE:base + dependencies: [] + +test:base-devel:secure: + extends: .test + stage: test + tags: + - secure + image: $CI_REGISTRY_IMAGE:base-devel + dependencies: [] + after_script: - gcc -v - g++ -v - make -v @@ -176,52 +194,28 @@ release: script: - python ci/release.py -# Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base -# publish:org:base: -# stage: publish -# image: -# name: gcr.io/go-containerregistry/crane:debug -# entrypoint: [""] -# needs: -# - job: "test:base" -# artifacts: true -# tags: -# - secure -# variables: -# GIT_STRATEGY: none -# before_script: -# - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY -# - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux -# script: -# - crane cp $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG archlinux/archlinux:base -# - crane tag archlinux/archlinux:base latest -# - crane tag archlinux/archlinux:base base-$BUILD_DATE -# only: -# variables: -# - $SCHEDULED_PUBLISH == "TRUE" - -# Publish base-devel to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux:base-devel -# publish:org:base-devel: -# stage: publish -# image: -# name: gcr.io/go-containerregistry/crane:debug -# entrypoint: [""] -# needs: -# - job: "test:base-devel" -# artifacts: true -# tags: -# - secure -# variables: -# GIT_STRATEGY: none -# before_script: -# - crane auth login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY -# - crane auth login -u $SOME_TECHNICAL_DOCKER_HUB_USER -p $SOME_TECHNICAL_DOCKER_HUB_PASSWORD archlinux/archlinux -# script: -# - crane cp $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG archlinux/archlinux:base-devel -# - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_DATE -# only: -# variables: -# - $SCHEDULED_PUBLISH == "TRUE" +# Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux +publish:base: + stage: publish + tags: + - secure + image: + name: gcr.io/go-containerregistry/crane:debug + entrypoint: [""] + variables: + GIT_STRATEGY: none + before_script: + - echo $CI_REGISTRY_PASSWORD | crane auth login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - cat $DOCKER_ACCESS_TOKEN | crane auth login -u $DOCKER_USERNAME --password-stdin index.docker.io + script: + - crane cp $CI_REGISTRY_IMAGE:base archlinux/archlinux:base + - crane tag archlinux/archlinux:base latest + - crane tag archlinux/archlinux:base base-$BUILD_VERSION + - crane cp $CI_REGISTRY_IMAGE:base-devel archlinux/archlinux:base-devel + - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_VERSION + only: + variables: + - $SCHEDULED_PUBLISH == "TRUE" # Publish to the official Docker namespace: https://hub.docker.com/_/archlinux # publish:official: diff --git a/ci/release.py b/ci/release.py index 7a44d79..120c843 100755 --- a/ci/release.py +++ b/ci/release.py @@ -5,7 +5,7 @@ Should only be called from GitLab CI! Required env vars: - GITLAB_PROJECT_TOKEN - - BUILD_DATE + - BUILD_VERSION - CI_PROJECT_ID - CI_PROJECT_URL """ @@ -16,14 +16,14 @@ from pathlib import Path import gitlab token = os.environ['GITLAB_PROJECT_TOKEN'] -build_date = os.environ['BUILD_DATE'] +build_version = os.environ['BUILD_VERSION'] project_id = os.environ['CI_PROJECT_ID'] project_url = os.environ['CI_PROJECT_URL'] def upload(name): print(f"Uploading {name}.tar.xz") - filename = f"{name}-{build_date}.tar.xz" + filename = f"{name}-{build_version}.tar.xz" uploaded_url = project.upload( filename, filepath=f"output/{name}.tar.xz" )["url"] @@ -49,7 +49,7 @@ if __name__ == "__main__": print("Templating Dockerfiles") data = { "branch": "add-base-devel-tags", - "commit_message": f"Release {build_date}", + "commit_message": f"Release {build_version}", "actions": [ { "action": "update", @@ -68,9 +68,9 @@ if __name__ == "__main__": print("Creating release") release = project.releases.create( { - "name": f"Release {build_date}", - "tag_name": build_date, - "description": f"Release {build_date}", + "name": f"Release {build_version}", + "tag_name": build_version, + "description": f"Release {build_version}", "ref": "add-base-devel-tags", "assets": { "links": [ From 33d42dd0e59709a5fc2a3a3052e41ebdc77e38e7 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Mon, 19 Oct 2020 22:42:21 +0200 Subject: [PATCH 34/92] Use proper CI rules --- .gitlab-ci.yml | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6a28c45..ce2c2fe 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,6 +31,7 @@ rootfs:base: extends: .rootfs except: - master + - add-base-devel-tags - schedules - tags script: @@ -40,6 +41,7 @@ rootfs:base-devel: extends: .rootfs except: - master + - add-base-devel-tags - schedules - tags script: @@ -51,6 +53,7 @@ rootfs:base:secure: - secure only: - master + - add-base-devel-tags - schedules - tags script: @@ -62,6 +65,7 @@ rootfs:base-devel:secure: - secure only: - master + - add-base-devel-tags - schedules - tags script: @@ -136,6 +140,8 @@ image:base-devel:secure: --destination $CI_REGISTRY_IMAGE:base-devel .test: + dependencies: [] + stage: test script: - pacman -Sy - pacman -Qqk @@ -146,15 +152,21 @@ image:base-devel:secure: test:base: extends: .test - stage: test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG - dependencies: [] + except: + - master + - add-base-devel-tags + - schedules + - tags test:base-devel: extends: .test - stage: test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG - dependencies: [] + except: + - master + - add-base-devel-tags + - schedules + - tags after_script: - gcc -v - g++ -v @@ -162,19 +174,25 @@ test:base-devel: test:base:secure: extends: .test - stage: test tags: - secure + only: + - master + - add-base-devel-tags + - schedules + - tags image: $CI_REGISTRY_IMAGE:base - dependencies: [] test:base-devel:secure: extends: .test - stage: test tags: - secure + only: + - master + - add-base-devel-tags + - schedules + - tags image: $CI_REGISTRY_IMAGE:base-devel - dependencies: [] after_script: - gcc -v - g++ -v @@ -195,7 +213,7 @@ release: - python ci/release.py # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux -publish:base: +publish: stage: publish tags: - secure @@ -214,6 +232,8 @@ publish:base: - crane cp $CI_REGISTRY_IMAGE:base-devel archlinux/archlinux:base-devel - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_VERSION only: + refs: + - schedules variables: - $SCHEDULED_PUBLISH == "TRUE" From c13ad972f981ea5b01e6708f47e766cdf2e4aebc Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Mon, 19 Oct 2020 20:49:45 +0000 Subject: [PATCH 35/92] Release 20201019.6288 --- ci/base-devel/Dockerfile | 24 ++++++++++++++++++++++-- ci/base/Dockerfile | 24 ++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 44879eb..46298a3 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,2 +1,22 @@ -# Don't delete. -# This is a placeholder so that our `ci/release.py` will work properly. +FROM archlinux:latest AS verify +SHELL ["/bin/bash", "-c"] +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/77ce9d3fc8a189f8d55946db6b45ab2d/base-devel-20201019.6288.tar.xz)" && \ + sha256sum -c <<< "Path('output/base-devel.tar.xz.SHA256').read_text()[0:64] base-devel-20201019.6288.tar.xz" && \ + mkdir /rootfs && \ + tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" + +FROM scratch AS root +COPY --from=verify /rootfs/ / + +# manually run all alpm hooks that can't be run inside the fakechroot +RUN ldconfig && update-ca-trust && locale-gen +RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' + +# update /etc/os-release +RUN ln -s /usr/lib/os-release /etc/os-release + +# initialize the archlinux keyring, but discard any private key that may be shipped. +RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" + +ENV LANG=en_US.UTF-8 +CMD ["/usr/bin/bash"] diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 44879eb..43344cb 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,2 +1,22 @@ -# Don't delete. -# This is a placeholder so that our `ci/release.py` will work properly. +FROM archlinux:latest AS verify +SHELL ["/bin/bash", "-c"] +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/c2404963a8e1847c3e01cde076cc6a9b/base-20201019.6288.tar.xz)" && \ + sha256sum -c <<< "Path('output/base.tar.xz.SHA256').read_text()[0:64] base-20201019.6288.tar.xz" && \ + mkdir /rootfs && \ + tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" + +FROM scratch AS root +COPY --from=verify /rootfs/ / + +# manually run all alpm hooks that can't be run inside the fakechroot +RUN ldconfig && update-ca-trust && locale-gen +RUN sh -c 'ls usr/lib/sysusers.d/*.conf | /usr/share/libalpm/scripts/systemd-hook sysusers ' + +# update /etc/os-release +RUN ln -s /usr/lib/os-release /etc/os-release + +# initialize the archlinux keyring, but discard any private key that may be shipped. +RUN pacman-key --init && pacman-key --populate archlinux && bash -c "rm -rf etc/pacman.d/gnupg/{openpgp-revocs.d/,private-keys-v1.d/,pubring.gpg~,gnupg.S.}*" + +ENV LANG=en_US.UTF-8 +CMD ["/usr/bin/bash"] From fe15ce7922cfb537e77ab26dae5bd09f07f6a03a Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 01:51:46 +0200 Subject: [PATCH 36/92] Better way to build official release --- .gitlab-ci.yml | 84 ++++++++++++++++++++++++++------------------------ ci/release.py | 5 +-- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ce2c2fe..5884f2a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -55,6 +55,7 @@ rootfs:base:secure: - master - add-base-devel-tags - schedules + except: - tags script: - make $PWD/output/base.tar.xz $PWD/output/Dockerfile.base @@ -67,6 +68,7 @@ rootfs:base-devel:secure: - master - add-base-devel-tags - schedules + except: - tags script: - make $PWD/output/base-devel.tar.xz $PWD/output/Dockerfile.base-devel @@ -115,13 +117,14 @@ image:base:secure: - master - add-base-devel-tags - schedules + except: - tags script: - /kaniko/executor --whitelist-var-run="false" --context $CI_PROJECT_DIR/output --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base - --destination $CI_REGISTRY_IMAGE:base + --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG image:base-devel:secure: extends: .image @@ -131,13 +134,44 @@ image:base-devel:secure: - master - add-base-devel-tags - schedules + except: - tags script: - /kaniko/executor --whitelist-var-run="false" --context $CI_PROJECT_DIR/output --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel - --destination $CI_REGISTRY_IMAGE:base-devel + --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + +image:base:publish:secure: + extends: .image + tags: + - secure + only: + - tags + before_script: + - echo "{\"auths\":{\"index.docker.io\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR/ci/base + --dockerfile $CI_PROJECT_DIR/ci/base/Dockerfile + --destination archlinux/archlinux:base-$BUILD_VERSION + +image:base-devel:publish:secure: + extends: .image + tags: + - secure + only: + - tags + before_script: + - echo "{\"auths\":{\"index.docker.io\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR/ci/base-devel + --dockerfile $CI_PROJECT_DIR/ci/base-devel/Dockerfile + --destination archlinux/archlinux:base-devel-$BUILD_VERSION .test: dependencies: [] @@ -172,32 +206,6 @@ test:base-devel: - g++ -v - make -v -test:base:secure: - extends: .test - tags: - - secure - only: - - master - - add-base-devel-tags - - schedules - - tags - image: $CI_REGISTRY_IMAGE:base - -test:base-devel:secure: - extends: .test - tags: - - secure - only: - - master - - add-base-devel-tags - - schedules - - tags - image: $CI_REGISTRY_IMAGE:base-devel - after_script: - - gcc -v - - g++ -v - - make -v - release: stage: release tags: @@ -222,20 +230,14 @@ publish: entrypoint: [""] variables: GIT_STRATEGY: none - before_script: - - echo $CI_REGISTRY_PASSWORD | crane auth login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY - - cat $DOCKER_ACCESS_TOKEN | crane auth login -u $DOCKER_USERNAME --password-stdin index.docker.io - script: - - crane cp $CI_REGISTRY_IMAGE:base archlinux/archlinux:base - - crane tag archlinux/archlinux:base latest - - crane tag archlinux/archlinux:base base-$BUILD_VERSION - - crane cp $CI_REGISTRY_IMAGE:base-devel archlinux/archlinux:base-devel - - crane tag archlinux/archlinux:base-devel base-devel-$BUILD_VERSION only: - refs: - - schedules - variables: - - $SCHEDULED_PUBLISH == "TRUE" + - tags + before_script: + - echo $DOCKER_ACCESS_TOKEN | crane auth login -u $DOCKER_USERNAME --password-stdin index.docker.io + script: + - crane tag archlinux/archlinux:base-$BUILD_VERSION base + - crane tag archlinux/archlinux:base-$BUILD_VERSION latest + - crane tag archlinux/archlinux:base-devel-$BUILD_VERSION base-devel # Publish to the official Docker namespace: https://hub.docker.com/_/archlinux # publish:official: diff --git a/ci/release.py b/ci/release.py index 120c843..6e608a2 100755 --- a/ci/release.py +++ b/ci/release.py @@ -30,9 +30,10 @@ def upload(name): template = Path("Dockerfile.template").read_text() full_url = f"{project_url}{uploaded_url}" replaced = template.replace("TEMPLATE_ROOTFS_URL", full_url) - hash = f"Path('output/{name}.tar.xz.SHA256').read_text()[0:64] {filename}" + rootfs_sha256 = Path('output/{name}.tar.xz.SHA256').read_text()[0:64] + hash_string = f"{rootfs_sha256} {filename}" replaced = replaced.replace( - "TEMPLATE_ROOTFS_HASH", hash + "TEMPLATE_ROOTFS_HASH", hash_string ) # Remove the line containing TEMPLATE_ROOTFS_FILE replaced = re.sub(".*TEMPLATE_ROOTFS_FILE.*\n", "", replaced) From b6d16e18291eae99524d364e2ece89c8d8badafb Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 02:04:56 +0200 Subject: [PATCH 37/92] Remove restriction on test stage --- .gitlab-ci.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5884f2a..7669997 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,6 +145,7 @@ image:base-devel:secure: image:base:publish:secure: extends: .image + dependencies: [] tags: - secure only: @@ -160,6 +161,7 @@ image:base:publish:secure: image:base-devel:publish:secure: extends: .image + dependencies: [] tags: - secure only: @@ -174,8 +176,8 @@ image:base-devel:publish:secure: --destination archlinux/archlinux:base-devel-$BUILD_VERSION .test: - dependencies: [] stage: test + dependencies: [] script: - pacman -Sy - pacman -Qqk @@ -188,18 +190,12 @@ test:base: extends: .test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG except: - - master - - add-base-devel-tags - - schedules - tags test:base-devel: extends: .test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG except: - - master - - add-base-devel-tags - - schedules - tags after_script: - gcc -v From ff691ecc2ab6566cc153f6e44765fbd378c5adb1 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 02:22:37 +0200 Subject: [PATCH 38/92] Fix another format string --- ci/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release.py b/ci/release.py index 6e608a2..d9ae95d 100755 --- a/ci/release.py +++ b/ci/release.py @@ -30,7 +30,7 @@ def upload(name): template = Path("Dockerfile.template").read_text() full_url = f"{project_url}{uploaded_url}" replaced = template.replace("TEMPLATE_ROOTFS_URL", full_url) - rootfs_sha256 = Path('output/{name}.tar.xz.SHA256').read_text()[0:64] + rootfs_sha256 = Path(f"output/{name}.tar.xz.SHA256").read_text()[0:64] hash_string = f"{rootfs_sha256} {filename}" replaced = replaced.replace( "TEMPLATE_ROOTFS_HASH", hash_string From d525b6da10913493a49062ca97e7adfdffbd30df Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 00:29:45 +0000 Subject: [PATCH 39/92] Release 20201020.6342 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 46298a3..24f3bf4 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/77ce9d3fc8a189f8d55946db6b45ab2d/base-devel-20201019.6288.tar.xz)" && \ - sha256sum -c <<< "Path('output/base-devel.tar.xz.SHA256').read_text()[0:64] base-devel-20201019.6288.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/6b3e77b855f15944e92d8d5ef3c12f29/base-devel-20201020.6342.tar.xz)" && \ + sha256sum -c <<< "6ea0f422d9376909a8becce5d3ce8591817840b2183c45c08d2b3e12aa356212 base-devel-20201020.6342.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 43344cb..e8c4c02 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/c2404963a8e1847c3e01cde076cc6a9b/base-20201019.6288.tar.xz)" && \ - sha256sum -c <<< "Path('output/base.tar.xz.SHA256').read_text()[0:64] base-20201019.6288.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/8c78f3b00c369b9f1fb8b6b4e9be617b/base-20201020.6342.tar.xz)" && \ + sha256sum -c <<< "00b28d7bed8425202d4f6b61373064362350db12518839bea56e3d256e281ccd base-20201020.6342.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From d5681dcb99a5a63c7b4c66c89f8ffe63f8ee16a1 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 02:48:55 +0200 Subject: [PATCH 40/92] Try to use GitLab access token for pushing --- .gitlab-ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7669997..ecaa8af 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -119,6 +119,8 @@ image:base:secure: - schedules except: - tags + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" @@ -136,6 +138,8 @@ image:base-devel:secure: - schedules except: - tags + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" From bb6ff3c73b2ace6fc010016d1a445b6a08b524cb Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 02:50:56 +0200 Subject: [PATCH 41/92] Add 'v' prefix for git tags --- ci/release.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release.py b/ci/release.py index d9ae95d..d751f7e 100755 --- a/ci/release.py +++ b/ci/release.py @@ -70,7 +70,7 @@ if __name__ == "__main__": release = project.releases.create( { "name": f"Release {build_version}", - "tag_name": build_version, + "tag_name": f"v{build_version}", "description": f"Release {build_version}", "ref": "add-base-devel-tags", "assets": { From 24985d5d2e02ff92bd0dd758b38d92b02b4b46f7 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 00:58:32 +0000 Subject: [PATCH 42/92] Release 20201020.6383 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 24f3bf4..12b01bb 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/6b3e77b855f15944e92d8d5ef3c12f29/base-devel-20201020.6342.tar.xz)" && \ - sha256sum -c <<< "6ea0f422d9376909a8becce5d3ce8591817840b2183c45c08d2b3e12aa356212 base-devel-20201020.6342.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/fe83ec937dfbd47b8982a867e66a01d8/base-devel-20201020.6383.tar.xz)" && \ + sha256sum -c <<< "4bff512b512003491af2d7d546c47c0d39c860092664c6e376aeb8d1e7a71347 base-devel-20201020.6383.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index e8c4c02..f373792 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/8c78f3b00c369b9f1fb8b6b4e9be617b/base-20201020.6342.tar.xz)" && \ - sha256sum -c <<< "00b28d7bed8425202d4f6b61373064362350db12518839bea56e3d256e281ccd base-20201020.6342.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/43f8631c144b0406b5bd1a160241ebb6/base-20201020.6383.tar.xz)" && \ + sha256sum -c <<< "7993f11890d929bc324585754ab52ecf35fb9f905ab7c10e877b67e9083e83a5 base-20201020.6383.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From d89a5f781d7099f59aea1d24c4448ddc93e10380 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 03:24:22 +0200 Subject: [PATCH 43/92] Don't run CI for generated files --- .gitlab-ci.yml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecaa8af..3b523c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -155,7 +155,7 @@ image:base:publish:secure: only: - tags before_script: - - echo "{\"auths\":{\"index.docker.io\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json + - echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" @@ -171,7 +171,7 @@ image:base-devel:publish:secure: only: - tags before_script: - - echo "{\"auths\":{\"index.docker.io\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json + - echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" @@ -194,13 +194,21 @@ test:base: extends: .test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG except: - - tags + refs: + - tags + changes: + - ci/base/Dockerfile + - ci/base-devel/Dockerfile test:base-devel: extends: .test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG except: - - tags + refs: + - tags + changes: + - ci/base/Dockerfile + - ci/base-devel/Dockerfile after_script: - gcc -v - g++ -v From 6383077e8134b2645ea676b47e93501d5af3a7d8 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 03:28:26 +0200 Subject: [PATCH 44/92] Debug --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b523c3..34e1145 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -197,8 +197,8 @@ test:base: refs: - tags changes: - - ci/base/Dockerfile - - ci/base-devel/Dockerfile + - "ci/base/Dockerfile" + - "ci/base-devel/Dockerfile" test:base-devel: extends: .test @@ -207,8 +207,8 @@ test:base-devel: refs: - tags changes: - - ci/base/Dockerfile - - ci/base-devel/Dockerfile + - "ci/base/Dockerfile" + - "ci/base-devel/Dockerfile" after_script: - gcc -v - g++ -v From 40d427519842b92ceb7d70821c37e0ed9cada2e9 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 03:37:31 +0200 Subject: [PATCH 45/92] Unrestrict test jobs --- .gitlab-ci.yml | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 34e1145..173daeb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -194,21 +194,13 @@ test:base: extends: .test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG except: - refs: - - tags - changes: - - "ci/base/Dockerfile" - - "ci/base-devel/Dockerfile" + - tags test:base-devel: extends: .test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG except: - refs: - - tags - changes: - - "ci/base/Dockerfile" - - "ci/base-devel/Dockerfile" + - tags after_script: - gcc -v - g++ -v From 3ec9c84f70b87f3006963511c1cec1b4b595b658 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 01:44:44 +0000 Subject: [PATCH 46/92] Release 20201020.6441 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 12b01bb..5af4199 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/fe83ec937dfbd47b8982a867e66a01d8/base-devel-20201020.6383.tar.xz)" && \ - sha256sum -c <<< "4bff512b512003491af2d7d546c47c0d39c860092664c6e376aeb8d1e7a71347 base-devel-20201020.6383.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/f07d6ddc7fc4b3611b6ea7b58515f5a2/base-devel-20201020.6441.tar.xz)" && \ + sha256sum -c <<< "5e67b9b2ffada0161a68555173e765d85630196a34470d31535cbf8c699f7acc base-devel-20201020.6441.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index f373792..84a4959 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/43f8631c144b0406b5bd1a160241ebb6/base-20201020.6383.tar.xz)" && \ - sha256sum -c <<< "7993f11890d929bc324585754ab52ecf35fb9f905ab7c10e877b67e9083e83a5 base-20201020.6383.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/548e5076c542e793d547a419ee2c7e9e/base-20201020.6441.tar.xz)" && \ + sha256sum -c <<< "73025d04499975075b3346ce2b9f686bc60ad52402d26b1e5e193c5b9e666f64 base-20201020.6441.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 026333d88020a09715057e13fe05b1092fa4fbb4 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 03:49:55 +0200 Subject: [PATCH 47/92] We need the env in the publish image job after all --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 173daeb..de6cc34 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -149,7 +149,6 @@ image:base-devel:secure: image:base:publish:secure: extends: .image - dependencies: [] tags: - secure only: @@ -165,7 +164,6 @@ image:base:publish:secure: image:base-devel:publish:secure: extends: .image - dependencies: [] tags: - secure only: From 10e066082d7098cb03ef7b2af2947a1e1f5a6553 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 01:57:30 +0000 Subject: [PATCH 48/92] Release 20201020.6467 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 5af4199..8d9a995 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/f07d6ddc7fc4b3611b6ea7b58515f5a2/base-devel-20201020.6441.tar.xz)" && \ - sha256sum -c <<< "5e67b9b2ffada0161a68555173e765d85630196a34470d31535cbf8c699f7acc base-devel-20201020.6441.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/9e782ae1da761b3bec8a4008ee8bedfc/base-devel-20201020.6467.tar.xz)" && \ + sha256sum -c <<< "c03455a6aa9e4a479fe936aa9905748136a9bb8a0cd705a5dd8b9d8cf37a6f43 base-devel-20201020.6467.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 84a4959..618b669 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/548e5076c542e793d547a419ee2c7e9e/base-20201020.6441.tar.xz)" && \ - sha256sum -c <<< "73025d04499975075b3346ce2b9f686bc60ad52402d26b1e5e193c5b9e666f64 base-20201020.6441.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/0ae1b4b3655fc0d3982527269fa9021b/base-20201020.6467.tar.xz)" && \ + sha256sum -c <<< "b731085938bbb1f1b1d48d82080da47f9a120453c76e66baefd4be9f84baaf11 base-20201020.6467.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 2d6948afc57c47159a4c9fad002e185bf8326ae4 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 04:18:17 +0200 Subject: [PATCH 49/92] Debug --- .gitlab-ci.yml | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index de6cc34..ba9d4dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,17 +15,30 @@ lint: # DL3007: We use the latest tag for multistage build script: hadolint --ignore DL3007 --ignore DL3020 Dockerfile.template +get_version: + stage: .pre + script: + - | + # If we're building a tagged release, use the tag (without the 'v' prefix) as the + # BUILD_VERSION. Otherwise, determine a new BUILD_VERSION. + if [[ -n "$CI_COMMIT_TAG" ]]; then + echo "BUILD_VERSION=${CI_COMMIT_TAG/v/}" > build.env + else + echo "BUILD_VERSION=$(date +%Y%m%d).$CI_JOB_ID" > build.env + fi + - export $(< build.env) + artifacts: + reports: + dotenv: build.env + .rootfs: stage: rootfs before_script: - - echo "BUILD_VERSION=$(date +%Y%m%d).$CI_JOB_ID" > build.env - pacman -Syu --noconfirm make devtools fakechroot fakeroot artifacts: paths: - output/* expire_in: 2h - reports: - dotenv: build.env rootfs:base: extends: .rootfs @@ -119,8 +132,8 @@ image:base:secure: - schedules except: - tags - before_script: - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json + # before_script: + # - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" @@ -138,8 +151,8 @@ image:base-devel:secure: - schedules except: - tags - before_script: - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json + # before_script: + # - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" From 3b8874980bbb25b4672648822a366c8ae058dde3 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 02:26:11 +0000 Subject: [PATCH 50/92] Release 20201020.6493 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 8d9a995..96f76a4 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/9e782ae1da761b3bec8a4008ee8bedfc/base-devel-20201020.6467.tar.xz)" && \ - sha256sum -c <<< "c03455a6aa9e4a479fe936aa9905748136a9bb8a0cd705a5dd8b9d8cf37a6f43 base-devel-20201020.6467.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/cfdfa305d34b89838ed3e69089287405/base-devel-20201020.6493.tar.xz)" && \ + sha256sum -c <<< "15603329c2b8805f141dcf15f307880ce9ff2b2f99f527d401eed32ad3dd96b7 base-devel-20201020.6493.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 618b669..4949f07 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/0ae1b4b3655fc0d3982527269fa9021b/base-20201020.6467.tar.xz)" && \ - sha256sum -c <<< "b731085938bbb1f1b1d48d82080da47f9a120453c76e66baefd4be9f84baaf11 base-20201020.6467.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/fde4f8d9d3c7ff5acc339a8043619ab9/base-20201020.6493.tar.xz)" && \ + sha256sum -c <<< "ac4b8cb88d4210d294d32c7c42eb90bca08dddca844b10b377f2565853c9fdb9 base-20201020.6493.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 8f7837598bb9fdc6c31ae22990464e7c795d757c Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 04:37:38 +0200 Subject: [PATCH 51/92] Generate docker auth again --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ba9d4dd..2c04700 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -132,8 +132,8 @@ image:base:secure: - schedules except: - tags - # before_script: - # - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" @@ -151,8 +151,8 @@ image:base-devel:secure: - schedules except: - tags - # before_script: - # - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json + before_script: + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" From 7f684cdd75e32e3ad0aed42939727fb0ca24e002 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 02:45:20 +0000 Subject: [PATCH 52/92] Release 20201020.6523 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 96f76a4..e882e9b 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/cfdfa305d34b89838ed3e69089287405/base-devel-20201020.6493.tar.xz)" && \ - sha256sum -c <<< "15603329c2b8805f141dcf15f307880ce9ff2b2f99f527d401eed32ad3dd96b7 base-devel-20201020.6493.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/1e35d4772c86844091c6d77d1d0cf6d8/base-devel-20201020.6523.tar.xz)" && \ + sha256sum -c <<< "bb0c31be47f8179e3479f52e9da2a180dda2d3744ab29e375922124d0e974165 base-devel-20201020.6523.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 4949f07..aaa164f 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/fde4f8d9d3c7ff5acc339a8043619ab9/base-20201020.6493.tar.xz)" && \ - sha256sum -c <<< "ac4b8cb88d4210d294d32c7c42eb90bca08dddca844b10b377f2565853c9fdb9 base-20201020.6493.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/9c5da71afe55581a4ac6ebd946406919/base-20201020.6523.tar.xz)" && \ + sha256sum -c <<< "c7fb68fb7f26f2d7fed45f748058f8b81872b6121ad518add7373285fa48d16b base-20201020.6523.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 6172cf1aa6907945b417047bec7770f75304a362 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Tue, 20 Oct 2020 05:17:29 +0200 Subject: [PATCH 53/92] Exclude test job when job was made by project access token See https://gitlab.com/gitlab-org/gitlab/-/issues/259663 --- .gitlab-ci.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2c04700..7e3c2ce 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -204,14 +204,26 @@ image:base-devel:publish:secure: test:base: extends: .test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG + only: + variables: + # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663 + # This is fine as at this point we're sure that the release works anyway. + - $GITLAB_USER_EMAIL != "project10185_bot2@example.com" except: - - tags + refs: + - tags test:base-devel: extends: .test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG + only: + variables: + # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663 + # This is fine as at this point we're sure that the release works anyway. + - $GITLAB_USER_EMAIL != "project10185_bot2@example.com" except: - - tags + refs: + - tags after_script: - gcc -v - g++ -v From 447f68507e8fbee2ea174b5241b285b967e5e974 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Tue, 20 Oct 2020 03:27:02 +0000 Subject: [PATCH 54/92] Release 20201020.6556 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index e882e9b..e8eeac3 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/1e35d4772c86844091c6d77d1d0cf6d8/base-devel-20201020.6523.tar.xz)" && \ - sha256sum -c <<< "bb0c31be47f8179e3479f52e9da2a180dda2d3744ab29e375922124d0e974165 base-devel-20201020.6523.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/49a5060351ef28a3cd8494f591ceed3b/base-devel-20201020.6556.tar.xz)" && \ + sha256sum -c <<< "84c0bb4c1ab1d937980acd9cf2165930db43180c35674b08c32e6810fdcb2e1b base-devel-20201020.6556.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index aaa164f..d1ec152 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,7 @@ FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/9c5da71afe55581a4ac6ebd946406919/base-20201020.6523.tar.xz)" && \ - sha256sum -c <<< "c7fb68fb7f26f2d7fed45f748058f8b81872b6121ad518add7373285fa48d16b base-20201020.6523.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/93e7d135858872fa3aa626a5fe2719ea/base-20201020.6556.tar.xz)" && \ + sha256sum -c <<< "c980d72136d9db1d82be7844a09341f0ccfc211a2a0ac94d1c471d9aa4f2fba3 base-20201020.6556.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 60059beb7261584362fc49a3c19d7ec7a2210aa3 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 02:47:41 +0200 Subject: [PATCH 55/92] Get rid of unused packages file --- packages | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 packages diff --git a/packages b/packages deleted file mode 100644 index 2c07590..0000000 --- a/packages +++ /dev/null @@ -1,10 +0,0 @@ -sed -gzip -pacman -systemd -gawk -file -grep -tar -procps-ng -licenses From e791991ce6aa2e451fa0d6d7133f468dec21ed64 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 02:48:34 +0200 Subject: [PATCH 56/92] Check whether we can use matrices to get rid of a lot of duplication GitLab 13.5 introduced support for one-dimensional matrices: https://about.gitlab.com/releases/2020/10/22/gitlab-13-5-released/#allow-one-dimensional-parallel-matrices Hopefully this works as intended. :) --- .gitlab-ci.yml | 162 ++++++++++++++----------------------------------- 1 file changed, 46 insertions(+), 116 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 7e3c2ce..3d48950 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -40,27 +40,20 @@ get_version: - output/* expire_in: 2h -rootfs:base: +rootfs: extends: .rootfs except: - master - add-base-devel-tags - schedules - tags + parallel: + matrix: + - GROUP: [base, base-devel] script: - - make $PWD/output/base.tar.xz $PWD/output/Dockerfile.base + - make $PWD/output/$GROUP.tar.xz $PWD/output/Dockerfile.$GROUP -rootfs:base-devel: - extends: .rootfs - except: - - master - - add-base-devel-tags - - schedules - - tags - script: - - make $PWD/output/base-devel.tar.xz $PWD/output/Dockerfile.base-devel - -rootfs:base:secure: +rootfs:secure: extends: .rootfs tags: - secure @@ -70,59 +63,38 @@ rootfs:base:secure: - schedules except: - tags + parallel: + matrix: + - GROUP: [base, base-devel] script: - - make $PWD/output/base.tar.xz $PWD/output/Dockerfile.base - -rootfs:base-devel:secure: - extends: .rootfs - tags: - - secure - only: - - master - - add-base-devel-tags - - schedules - except: - - tags - script: - - make $PWD/output/base-devel.tar.xz $PWD/output/Dockerfile.base-devel + - make $PWD/output/$GROUP.tar.xz $PWD/output/Dockerfile.$GROUP .image: stage: image image: name: gcr.io/kaniko-project/executor:debug entrypoint: [""] + script: + - /kaniko/executor + --whitelist-var-run="false" + --context $CI_PROJECT_DIR/output + --dockerfile $CI_PROJECT_DIR/output/Dockerfile.$GROUP + --destination $CI_REGISTRY_IMAGE:$GROUP-$CI_COMMIT_REF_SLUG + +image:build: + extends: .image + except: + - master + - add-base-devel-tags + - schedules + - tags + parallel: + matrix: + - GROUP: [base, base-devel] before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"}}}" > /kaniko/.docker/config.json -image:base: - extends: .image - except: - - master - - add-base-devel-tags - - schedules - - tags - script: - - /kaniko/executor - --whitelist-var-run="false" - --context $CI_PROJECT_DIR/output - --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base - --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG - -image:base-devel: - extends: .image - except: - - master - - add-base-devel-tags - - schedules - - tags - script: - - /kaniko/executor - --whitelist-var-run="false" - --context $CI_PROJECT_DIR/output - --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel - --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG - -image:base:secure: +image:build:secure: extends: .image tags: - secure @@ -132,67 +104,41 @@ image:base:secure: - schedules except: - tags + parallel: + matrix: + - GROUP: [base, base-devel] before_script: - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json - script: - - /kaniko/executor - --whitelist-var-run="false" - --context $CI_PROJECT_DIR/output - --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base - --destination $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG -image:base-devel:secure: - extends: .image - tags: - - secure - only: - - master - - add-base-devel-tags - - schedules - except: - - tags - before_script: - - echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$GITLAB_PROJECT_USER\",\"password\":\"$GITLAB_PROJECT_TOKEN\"}}}" > /kaniko/.docker/config.json - script: - - /kaniko/executor - --whitelist-var-run="false" - --context $CI_PROJECT_DIR/output - --dockerfile $CI_PROJECT_DIR/output/Dockerfile.base-devel - --destination $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG - -image:base:publish:secure: +image:publish:secure: extends: .image tags: - secure only: - tags + parallel: + matrix: + - GROUP: [base, base-devel] before_script: - echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json script: - /kaniko/executor --whitelist-var-run="false" - --context $CI_PROJECT_DIR/ci/base - --dockerfile $CI_PROJECT_DIR/ci/base/Dockerfile - --destination archlinux/archlinux:base-$BUILD_VERSION - -image:base-devel:publish:secure: - extends: .image - tags: - - secure - only: - - tags - before_script: - - echo "{\"auths\":{\"https://index.docker.io/v1/\":{\"username\":\"$DOCKER_USERNAME\",\"password\":\"$DOCKER_ACCESS_TOKEN\"}}}" > /kaniko/.docker/config.json - script: - - /kaniko/executor - --whitelist-var-run="false" - --context $CI_PROJECT_DIR/ci/base-devel - --dockerfile $CI_PROJECT_DIR/ci/base-devel/Dockerfile - --destination archlinux/archlinux:base-devel-$BUILD_VERSION + --context $CI_PROJECT_DIR/ci/$GROUP + --dockerfile $CI_PROJECT_DIR/ci/$GROUP/Dockerfile + --destination archlinux/archlinux:$GROUP-$BUILD_VERSION .test: stage: test dependencies: [] + only: + variables: + # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663 + # This is fine as at this point we're sure that the release works anyway. + - $GITLAB_USER_EMAIL != "project10185_bot2@example.com" + except: + refs: + - tags script: - pacman -Sy - pacman -Qqk @@ -204,26 +150,10 @@ image:base-devel:publish:secure: test:base: extends: .test image: $CI_REGISTRY_IMAGE:base-$CI_COMMIT_REF_SLUG - only: - variables: - # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663 - # This is fine as at this point we're sure that the release works anyway. - - $GITLAB_USER_EMAIL != "project10185_bot2@example.com" - except: - refs: - - tags test:base-devel: extends: .test image: $CI_REGISTRY_IMAGE:base-devel-$CI_COMMIT_REF_SLUG - only: - variables: - # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/259663 - # This is fine as at this point we're sure that the release works anyway. - - $GITLAB_USER_EMAIL != "project10185_bot2@example.com" - except: - refs: - - tags after_script: - gcc -v - g++ -v From ef0c0eae64bd93644d241a50d82a859149d5520c Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:00:26 +0200 Subject: [PATCH 57/92] Get rid of Python deploy script --- .gitlab-ci.yml | 44 +++++++++++++++++++++--- ci/release.py | 92 -------------------------------------------------- 2 files changed, 39 insertions(+), 97 deletions(-) delete mode 100755 ci/release.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3d48950..6c77aec 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ stages: - rootfs - image - test + - upload - release - publish @@ -27,6 +28,7 @@ get_version: echo "BUILD_VERSION=$(date +%Y%m%d).$CI_JOB_ID" > build.env fi - export $(< build.env) + - echo "PACKAGE_REGISTRY_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/rootfs/${BUILD_VERSION}" >> build.env artifacts: reports: dotenv: build.env @@ -159,8 +161,9 @@ test:base-devel: - g++ -v - make -v -release: - stage: release +upload_and_commit_rootfs: + stage: upload + image: curlimages/curl:latest tags: - secure only: @@ -168,10 +171,41 @@ release: - schedules variables: - $SCHEDULED_PUBLISH == "TRUE" - before_script: - - pacman -Syu --noconfirm python-gitlab script: - - python ci/release.py + - for group in base base-devel; do + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz + sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile + sed -i "s|TEMPLATE_ROOTFS_URL|${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz|" ci/${group}/Dockerfile + sed -i "s|${group}.tar.xz|${group}-${BUILD_VERSION}.tar.xz|" output/${group}.tar.xz.SHA256 + sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile + done + curl --request POST + --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" + --form "branch-add-base-devel-tags" + --form "commit_message=Release ${BUILD_VERSION}" + --form "actions[][action]=update" + --form "actions[][file_path]=ci/base/Dockerfile" + --form "actions[][content]= Date: Fri, 23 Oct 2020 04:01:21 +0200 Subject: [PATCH 58/92] Fix multiline YAML --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6c77aec..c7e50a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -172,7 +172,8 @@ upload_and_commit_rootfs: variables: - $SCHEDULED_PUBLISH == "TRUE" script: - - for group in base base-devel; do + - | + for group in base base-devel; do curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile sed -i "s|TEMPLATE_ROOTFS_URL|${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz|" ci/${group}/Dockerfile From 34f172d65aa84e2dc2109d7038f4a2cb918bbe5c Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:10:36 +0200 Subject: [PATCH 59/92] Add big block of documentation to Dockerfile --- Dockerfile.template | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/Dockerfile.template b/Dockerfile.template index 9d67451..6ccf941 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -1,3 +1,13 @@ +# We're using a multistage Docker build here in order to allow us to release a self-verifying +# Docker image when built on the official Docker infrastructure. +# They require us to verify the source integrity in some way while making sure that this is a +# reproducible build. +# See https://github.com/docker-library/official-images#image-build +# In order to achieve this, we externally host the rootfs archives and their checksums and then +# just download and verify it in the first stage of this Dockerfile. +# The second stage is for actually configuring the system a little bit. +# Some templating is done in order to allow us to easily build different configurations and to +# allow us to automate the releaes process. FROM archlinux:latest AS verify COPY TEMPLATE_ROOTFS_FILE / SHELL ["/bin/bash", "-c"] From 53b90611f16f216e735783b1746d84bbba5d13d3 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:10:58 +0200 Subject: [PATCH 60/92] Upload SHA256 integrity files beside actual artifacts --- .gitlab-ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index c7e50a9..a25d8de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -174,11 +174,12 @@ upload_and_commit_rootfs: script: - | for group in base base-devel; do - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz + sed -i "s|${group}.tar.xz|${group}-${BUILD_VERSION}.tar.xz|" output/${group}.tar.xz.SHA256 sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile sed -i "s|TEMPLATE_ROOTFS_URL|${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz|" ci/${group}/Dockerfile - sed -i "s|${group}.tar.xz|${group}-${BUILD_VERSION}.tar.xz|" output/${group}.tar.xz.SHA256 sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz + curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 done curl --request POST --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" @@ -206,7 +207,9 @@ release: - release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz"}' + --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256"}' --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz"}' + --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz.SHA256"}' # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux publish: From 05f8b8f3eb5ad1a75cbdee04f83e1d8f786052cf Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:31:46 +0200 Subject: [PATCH 61/92] Fix version format to look like n.n.n This is expected of the new generic package API: https://docs.gitlab.com/ee/user/packages/generic_packages/ --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a25d8de..e680f71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -25,7 +25,7 @@ get_version: if [[ -n "$CI_COMMIT_TAG" ]]; then echo "BUILD_VERSION=${CI_COMMIT_TAG/v/}" > build.env else - echo "BUILD_VERSION=$(date +%Y%m%d).$CI_JOB_ID" > build.env + echo "BUILD_VERSION=$(date +%Y%m%d).0.$CI_JOB_ID" > build.env fi - export $(< build.env) - echo "PACKAGE_REGISTRY_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/rootfs/${BUILD_VERSION}" >> build.env From 0aa9aba8d29ff3a52ede7c331b3c3453d1599097 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:31:57 +0200 Subject: [PATCH 62/92] Fix newline handling --- .gitlab-ci.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e680f71..e8fce1c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,17 +181,17 @@ upload_and_commit_rootfs: curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 done - curl --request POST - --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" - --form "branch-add-base-devel-tags" - --form "commit_message=Release ${BUILD_VERSION}" - --form "actions[][action]=update" - --form "actions[][file_path]=ci/base/Dockerfile" - --form "actions[][content]= Date: Fri, 23 Oct 2020 04:34:32 +0200 Subject: [PATCH 63/92] Make YAML happy --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e8fce1c..4f388da 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -181,7 +181,8 @@ upload_and_commit_rootfs: curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 done - - curl --request POST + - > + curl --request POST --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" --form "branch-add-base-devel-tags" --form "commit_message=Release ${BUILD_VERSION}" From 197fc6ff3632fa16f0b0eff51b663d081bcd8c9e Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:42:55 +0200 Subject: [PATCH 64/92] Fix typo --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4f388da..fa0e716 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -184,7 +184,7 @@ upload_and_commit_rootfs: - > curl --request POST --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" - --form "branch-add-base-devel-tags" + --form "branch=add-base-devel-tags" --form "commit_message=Release ${BUILD_VERSION}" --form "actions[][action]=update" --form "actions[][file_path]=ci/base/Dockerfile" From 5b3de15ee6ac5b429673e3e585be22fd00633f34 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Fri, 23 Oct 2020 02:49:38 +0000 Subject: [PATCH 65/92] Release 20201023.0.6754 --- ci/base-devel/Dockerfile | 14 ++++++++++++-- ci/base/Dockerfile | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index e8eeac3..a47757f 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -1,7 +1,17 @@ +# We're using a multistage Docker build here in order to allow us to release a self-verifying +# Docker image when built on the official Docker infrastructure. +# They require us to verify the source integrity in some way while making sure that this is a +# reproducible build. +# See https://github.com/docker-library/official-images#image-build +# In order to achieve this, we externally host the rootfs archives and their checksums and then +# just download and verify it in the first stage of this Dockerfile. +# The second stage is for actually configuring the system a little bit. +# Some templating is done in order to allow us to easily build different configurations and to +# allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/49a5060351ef28a3cd8494f591ceed3b/base-devel-20201020.6556.tar.xz)" && \ - sha256sum -c <<< "84c0bb4c1ab1d937980acd9cf2165930db43180c35674b08c32e6810fdcb2e1b base-devel-20201020.6556.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6754/base-devel-20201023.0.6754.tar.xz)" && \ + sha256sum -c <<< "7c547a1d692e7513ac6af1634bc39e7ceb5ff2fb8a16a8fcc53916d4bd557351 base-devel-20201023.0.6754.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index d1ec152..1b30acb 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -1,7 +1,17 @@ +# We're using a multistage Docker build here in order to allow us to release a self-verifying +# Docker image when built on the official Docker infrastructure. +# They require us to verify the source integrity in some way while making sure that this is a +# reproducible build. +# See https://github.com/docker-library/official-images#image-build +# In order to achieve this, we externally host the rootfs archives and their checksums and then +# just download and verify it in the first stage of this Dockerfile. +# The second stage is for actually configuring the system a little bit. +# Some templating is done in order to allow us to easily build different configurations and to +# allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/uploads/93e7d135858872fa3aa626a5fe2719ea/base-20201020.6556.tar.xz)" && \ - sha256sum -c <<< "c980d72136d9db1d82be7844a09341f0ccfc211a2a0ac94d1c471d9aa4f2fba3 base-20201020.6556.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6754/base-20201023.0.6754.tar.xz)" && \ + sha256sum -c <<< "ded8467a888c78ab0838fb7841a58b45c5004685fde82631ff787d4b1ce60dd7 base-20201023.0.6754.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From def049d8a3fa8cbc682095b7eff407587e36c7c8 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 04:58:44 +0200 Subject: [PATCH 66/92] Debug --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index fa0e716..5d18e7c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,6 +29,8 @@ get_version: fi - export $(< build.env) - echo "PACKAGE_REGISTRY_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/rootfs/${BUILD_VERSION}" >> build.env + - export $(< build.env) + - echo ${PACKAGE_REGISTRY_URL} artifacts: reports: dotenv: build.env From aeac512739048b1365b28f01458660a0908edabd Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 05:04:31 +0200 Subject: [PATCH 67/92] More debug --- .gitlab-ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5d18e7c..0508d4e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -207,7 +207,15 @@ release: variables: - $SCHEDULED_PUBLISH == "TRUE" script: - - release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" + - | + echo release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" + echo --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" + echo --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz"}' + echo --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256"}' + echo --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz"}' + echo --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz.SHA256"}' + - > + release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz"}' --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256"}' From 1e99d443b783716915af07c8b61c81c7ea3d29ca Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 05:08:38 +0200 Subject: [PATCH 68/92] Fix variables --- .gitlab-ci.yml | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0508d4e..5a9a5cc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -29,8 +29,6 @@ get_version: fi - export $(< build.env) - echo "PACKAGE_REGISTRY_URL=${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/rootfs/${BUILD_VERSION}" >> build.env - - export $(< build.env) - - echo ${PACKAGE_REGISTRY_URL} artifacts: reports: dotenv: build.env @@ -207,20 +205,12 @@ release: variables: - $SCHEDULED_PUBLISH == "TRUE" script: - - | - echo release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" - echo --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" - echo --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz"}' - echo --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256"}' - echo --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz"}' - echo --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz.SHA256"}' - - > - release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" + - release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" - --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz"}' - --assets-link '{"name":"base-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256"}' - --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz"}' - --assets-link '{"name":"base-devel-${BUILD_VERSION}.tar.xz.SHA256","url":"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz.SHA256"}' + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\"}" + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256\"}" + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz\"}" + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz.SHA256\"}" # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux publish: From c54a082b4604c20110ecf032fc02922ebb116cf8 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Fri, 23 Oct 2020 03:15:42 +0000 Subject: [PATCH 69/92] Release 20201023.0.6809 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index a47757f..03e59b8 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6754/base-devel-20201023.0.6754.tar.xz)" && \ - sha256sum -c <<< "7c547a1d692e7513ac6af1634bc39e7ceb5ff2fb8a16a8fcc53916d4bd557351 base-devel-20201023.0.6754.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6809/base-devel-20201023.0.6809.tar.xz)" && \ + sha256sum -c <<< "3ddaf0bbdd144c7517e1e03956f952368ade55ec2ad0ae3c690b95eab1bfd356 base-devel-20201023.0.6809.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 1b30acb..2a53af3 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6754/base-20201023.0.6754.tar.xz)" && \ - sha256sum -c <<< "ded8467a888c78ab0838fb7841a58b45c5004685fde82631ff787d4b1ce60dd7 base-20201023.0.6754.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6809/base-20201023.0.6809.tar.xz)" && \ + sha256sum -c <<< "e7915c4873ce10476f8b9b7d29e43a26340b678cf5e737b610752de61215a8ca base-20201023.0.6809.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From bcab556065e1318ffaaf7f2da4b3e1a5e8539c4d Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 06:15:02 +0200 Subject: [PATCH 70/92] Implement hacky way to get public download URLs --- .gitlab-ci.yml | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5a9a5cc..a3388ca 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -204,13 +204,25 @@ release: - schedules variables: - $SCHEDULED_PUBLISH == "TRUE" + before_script: + - apk add jq script: - - release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" + - | + package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\" | .id)") + web_path=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}" | jq "._links.web_path") + base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz\") | .id") + base_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz.SHA256\") | .id") + base_devel_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz\") | .id") + base_devel_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz.SHA256\") | .id") + + # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" + # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! + release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" - --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\"}" - --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz.SHA256\"}" - --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz\"}" - --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"${PACKAGE_REGISTRY_URL}/base-devel-${BUILD_VERSION}.tar.xz.SHA256\"}" + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_id}/download\"}" + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_sha_id}/download\"}" + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_id}/download\"}" + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_sha_id}/download\"}" # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux publish: From 6aab81717a62c742d0c9d706bd9af86ab7b7eb1a Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Fri, 23 Oct 2020 04:22:14 +0000 Subject: [PATCH 71/92] Release 20201023.0.6841 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 03e59b8..35e57a3 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6809/base-devel-20201023.0.6809.tar.xz)" && \ - sha256sum -c <<< "3ddaf0bbdd144c7517e1e03956f952368ade55ec2ad0ae3c690b95eab1bfd356 base-devel-20201023.0.6809.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6841/base-devel-20201023.0.6841.tar.xz)" && \ + sha256sum -c <<< "1573b42decd4c94e931621733c04d58acb2c01e3c943aa7d2554daf0222deed3 base-devel-20201023.0.6841.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 2a53af3..4553e41 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6809/base-20201023.0.6809.tar.xz)" && \ - sha256sum -c <<< "e7915c4873ce10476f8b9b7d29e43a26340b678cf5e737b610752de61215a8ca base-20201023.0.6809.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6841/base-20201023.0.6841.tar.xz)" && \ + sha256sum -c <<< "3c1fe4e4a1153863b70589fe60e064e33b5aa70d1de84eb8291cda7606ddcd71 base-20201023.0.6841.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From bb2c6330d8ac755154420d1b491c69ff868db61e Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 06:24:48 +0200 Subject: [PATCH 72/92] Add curl --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index a3388ca..ff2560a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -205,7 +205,7 @@ release: variables: - $SCHEDULED_PUBLISH == "TRUE" before_script: - - apk add jq + - apk add jq curl script: - | package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\" | .id)") From f4276f9af54af74c4c50bab412ea748981e42d01 Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Fri, 23 Oct 2020 04:31:53 +0000 Subject: [PATCH 73/92] Release 20201023.0.6866 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 35e57a3..b9a6c63 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6841/base-devel-20201023.0.6841.tar.xz)" && \ - sha256sum -c <<< "1573b42decd4c94e931621733c04d58acb2c01e3c943aa7d2554daf0222deed3 base-devel-20201023.0.6841.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6866/base-devel-20201023.0.6866.tar.xz)" && \ + sha256sum -c <<< "46ed17385a81ce4895337d2d6e5d5d1ff570e18fdf763da3dc39de321ac755fa base-devel-20201023.0.6866.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 4553e41..23ec3c4 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6841/base-20201023.0.6841.tar.xz)" && \ - sha256sum -c <<< "3c1fe4e4a1153863b70589fe60e064e33b5aa70d1de84eb8291cda7606ddcd71 base-20201023.0.6841.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6866/base-20201023.0.6866.tar.xz)" && \ + sha256sum -c <<< "33bef86d6bfddc60e710b062216233cfc5fd0014314d85aa1b44e063cc14f47b base-20201023.0.6866.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 6833b127eb378504b2c701f6df99eccbda1a40ec Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 06:31:55 +0200 Subject: [PATCH 74/92] We don't actually need the web_path We can simply construct it ourselves. --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ff2560a..77925a9 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,7 +209,6 @@ release: script: - | package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\" | .id)") - web_path=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}" | jq "._links.web_path") base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz\") | .id") base_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz.SHA256\") | .id") base_devel_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz\") | .id") From 583324fa847e91056276c421fe8b67387f0831ea Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Fri, 23 Oct 2020 06:33:29 +0200 Subject: [PATCH 75/92] Fix syntax --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 77925a9..adb0991 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -208,7 +208,7 @@ release: - apk add jq curl script: - | - package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\" | .id)") + package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz\") | .id") base_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz.SHA256\") | .id") base_devel_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz\") | .id") From 9b9dd39ba83fae7b406952d88804709afeab8a7d Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Fri, 23 Oct 2020 04:41:20 +0000 Subject: [PATCH 76/92] Release 20201023.0.6898 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index b9a6c63..751ef3e 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6866/base-devel-20201023.0.6866.tar.xz)" && \ - sha256sum -c <<< "46ed17385a81ce4895337d2d6e5d5d1ff570e18fdf763da3dc39de321ac755fa base-devel-20201023.0.6866.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6898/base-devel-20201023.0.6898.tar.xz)" && \ + sha256sum -c <<< "89509dddaf5c75d125cf7455c52e473008576c52596387c943e8798606f1bda6 base-devel-20201023.0.6898.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 23ec3c4..c92eedb 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6866/base-20201023.0.6866.tar.xz)" && \ - sha256sum -c <<< "33bef86d6bfddc60e710b062216233cfc5fd0014314d85aa1b44e063cc14f47b base-20201023.0.6866.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6898/base-20201023.0.6898.tar.xz)" && \ + sha256sum -c <<< "9c89d7518a3269b7dfa1d5e25a61faadd3d8d0ddd94661a8e14decc457a2b0d0 base-20201023.0.6898.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From ff3bc769ba78cf7e6157d513fe613feab3900519 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 02:32:32 +0100 Subject: [PATCH 77/92] Fix typo --- .gitlab-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index adb0991..5ca4045 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -209,10 +209,10 @@ release: script: - | package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") - base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz\") | .id") - base_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz.SHA256\") | .id") - base_devel_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz\") | .id") - base_devel_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${pacakge_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz.SHA256\") | .id") + base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz\") | .id") + base_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz.SHA256\") | .id") + base_devel_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz\") | .id") + base_devel_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz.SHA256\") | .id") # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! From a4474f560413b8abcc8e6c1be254919d232463cd Mon Sep 17 00:00:00 2001 From: automatic-release-token Date: Sun, 25 Oct 2020 01:39:35 +0000 Subject: [PATCH 78/92] Release 20201025.0.7009 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 751ef3e..ff2edec 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6898/base-devel-20201023.0.6898.tar.xz)" && \ - sha256sum -c <<< "89509dddaf5c75d125cf7455c52e473008576c52596387c943e8798606f1bda6 base-devel-20201023.0.6898.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7009/base-devel-20201025.0.7009.tar.xz)" && \ + sha256sum -c <<< "ccbfb69101748c01547838c92c51e95a358f99c9635be63fae53f836178922fa base-devel-20201025.0.7009.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index c92eedb..7a4b1ff 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201023.0.6898/base-20201023.0.6898.tar.xz)" && \ - sha256sum -c <<< "9c89d7518a3269b7dfa1d5e25a61faadd3d8d0ddd94661a8e14decc457a2b0d0 base-20201023.0.6898.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7009/base-20201025.0.7009.tar.xz)" && \ + sha256sum -c <<< "a034d14a389bb242039abbe1f7c5521d727a5cd7dc7c9d79b018169d1ef5757b base-20201025.0.7009.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 7c705e36e01c70dadddfa02f9cb4cc0e3a92fd2f Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 02:41:48 +0100 Subject: [PATCH 79/92] Fix line breaks --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5ca4045..f3fe895 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -216,11 +216,11 @@ release: # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! - release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" - --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" - --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_id}/download\"}" - --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_sha_id}/download\"}" - --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_id}/download\"}" + release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" \ + --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" \ + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_id}/download\"}" \ + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_sha_id}/download\"}" \ + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_id}/download\"}" \ --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_sha_id}/download\"}" # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux From b23b74dcb16599a2127ba059166f5a766e03b9ee Mon Sep 17 00:00:00 2001 From: Arch Linux Docker release bot Date: Sun, 25 Oct 2020 01:48:52 +0000 Subject: [PATCH 80/92] Release 20201025.0.7033 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index ff2edec..5e4bb1b 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7009/base-devel-20201025.0.7009.tar.xz)" && \ - sha256sum -c <<< "ccbfb69101748c01547838c92c51e95a358f99c9635be63fae53f836178922fa base-devel-20201025.0.7009.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7033/base-devel-20201025.0.7033.tar.xz)" && \ + sha256sum -c <<< "f1b7b3ae69dc5f6d013d9cfb949a740d113891e87f35e9c63a67f0fe41b08628 base-devel-20201025.0.7033.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 7a4b1ff..d0526f7 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7009/base-20201025.0.7009.tar.xz)" && \ - sha256sum -c <<< "a034d14a389bb242039abbe1f7c5521d727a5cd7dc7c9d79b018169d1ef5757b base-20201025.0.7009.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7033/base-20201025.0.7033.tar.xz)" && \ + sha256sum -c <<< "b0c5922d77e11cabdea4b404b58db4e8d019092c328385cbd8fe47d336be51b9 base-20201025.0.7033.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From afb314e64b0c6ead905b62bc6ecd07a96a974a6e Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 03:23:20 +0100 Subject: [PATCH 81/92] Get URL via external script --- .gitlab-ci.yml | 24 +++++++++---------- ci/get-public-download-for-generic-package.sh | 9 +++++++ 2 files changed, 21 insertions(+), 12 deletions(-) create mode 100755 ci/get-public-download-for-generic-package.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index f3fe895..e50bf88 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -175,11 +175,12 @@ upload_and_commit_rootfs: - | for group in base base-devel; do sed -i "s|${group}.tar.xz|${group}-${BUILD_VERSION}.tar.xz|" output/${group}.tar.xz.SHA256 - sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile - sed -i "s|TEMPLATE_ROOTFS_URL|${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz|" ci/${group}/Dockerfile - sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 + sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile + package_url=$(ci/get-public-download-for-generic-package.sh ${group}-$(BUILD_VERSION).tar.xz) + sed -i "s|TEMPLATE_ROOTFS_URL|${package_url}|" ci/${group}/Dockerfile + sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile done - > curl --request POST @@ -208,20 +209,19 @@ release: - apk add jq curl script: - | - package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") - base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz\") | .id") - base_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-${BUILD_VERSION}.tar.xz.SHA256\") | .id") - base_devel_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz\") | .id") - base_devel_sha_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"base-devel-${BUILD_VERSION}.tar.xz.SHA256\") | .id") + base_url=$(ci/get-public-download-for-generic-package.sh base-$(BUILD_VERSION).tar.xz) + base_sha_url=$(ci/get-public-download-for-generic-package.sh base-$(BUILD_VERSION).tar.xz.SHA256) + base_devel_url=$(ci/get-public-download-for-generic-package.sh base-$(BUILD_VERSION).tar.xz) + base_devel_sha_url=$(ci/get-public-download-for-generic-package.sh base-devel-$(BUILD_VERSION).tar.xz.SHA256) # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" \ --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" \ - --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_id}/download\"}" \ - --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_sha_id}/download\"}" \ - --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_id}/download\"}" \ - --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_devel_sha_id}/download\"}" + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"${base_url}\"}" \ + --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"${base_sha_url}\"}" \ + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz\",\"url\":\"${base_devel_url}\"}" \ + --assets-link "{\"name\":\"base-devel-${BUILD_VERSION}.tar.xz.SHA256\",\"url\":\"${base_devel_sha_url}\"}" # Publish base to the Arch Linux group namespace: https://hub.docker.com/r/archlinux/archlinux publish: diff --git a/ci/get-public-download-for-generic-package.sh b/ci/get-public-download-for-generic-package.sh new file mode 100755 index 0000000..3070513 --- /dev/null +++ b/ci/get-public-download-for-generic-package.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -eu + +package_name=$1 + +package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") +base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"$package_name\") | .id") +echo "https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_id}/download" From 682a053a760128767094e4c76cb4a287340c11ff Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 03:33:42 +0100 Subject: [PATCH 82/92] Fix syntax slip-up --- .gitlab-ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e50bf88..57eacf4 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -178,7 +178,7 @@ upload_and_commit_rootfs: curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile - package_url=$(ci/get-public-download-for-generic-package.sh ${group}-$(BUILD_VERSION).tar.xz) + package_url=$(ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.xz) sed -i "s|TEMPLATE_ROOTFS_URL|${package_url}|" ci/${group}/Dockerfile sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile done @@ -209,10 +209,10 @@ release: - apk add jq curl script: - | - base_url=$(ci/get-public-download-for-generic-package.sh base-$(BUILD_VERSION).tar.xz) - base_sha_url=$(ci/get-public-download-for-generic-package.sh base-$(BUILD_VERSION).tar.xz.SHA256) - base_devel_url=$(ci/get-public-download-for-generic-package.sh base-$(BUILD_VERSION).tar.xz) - base_devel_sha_url=$(ci/get-public-download-for-generic-package.sh base-devel-$(BUILD_VERSION).tar.xz.SHA256) + base_url=$(ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + base_sha_url=$(ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz.SHA256) + base_devel_url=$(ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + base_devel_sha_url=$(ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.xz.SHA256) # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! From 6830497ed8962a3b95fe7ceae08416edcaa89b83 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 03:44:24 +0100 Subject: [PATCH 83/92] Let's hope that sh is enough --- .gitlab-ci.yml | 10 +++++----- ci/get-public-download-for-generic-package.sh | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 57eacf4..65301e0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -178,7 +178,7 @@ upload_and_commit_rootfs: curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile - package_url=$(ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.xz) + package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.xz) sed -i "s|TEMPLATE_ROOTFS_URL|${package_url}|" ci/${group}/Dockerfile sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile done @@ -209,10 +209,10 @@ release: - apk add jq curl script: - | - base_url=$(ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) - base_sha_url=$(ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz.SHA256) - base_devel_url=$(ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) - base_devel_sha_url=$(ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.xz.SHA256) + base_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + base_sha_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz.SHA256) + base_devel_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + base_devel_sha_url=$(./ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.xz.SHA256) # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! diff --git a/ci/get-public-download-for-generic-package.sh b/ci/get-public-download-for-generic-package.sh index 3070513..de77c9f 100755 --- a/ci/get-public-download-for-generic-package.sh +++ b/ci/get-public-download-for-generic-package.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh set -eu From 1a1089e341fab77e84e3d7ac88a229e3b71f9bb3 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 03:53:53 +0100 Subject: [PATCH 84/92] Merge things down a bit --- .gitlab-ci.yml | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 65301e0..191ce6d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -161,9 +161,9 @@ test:base-devel: - g++ -v - make -v -upload_and_commit_rootfs: - stage: upload - image: curlimages/curl:latest +release: + stage: release + image: registry.gitlab.com/gitlab-org/release-cli:latest tags: - secure only: @@ -171,6 +171,8 @@ upload_and_commit_rootfs: - schedules variables: - $SCHEDULED_PUBLISH == "TRUE" + before_script: + - apk add jq curl script: - | for group in base base-devel; do @@ -194,20 +196,6 @@ upload_and_commit_rootfs: --form "actions[][file_path]=ci/base-devel/Dockerfile" --form "actions[][content]= Date: Sun, 25 Oct 2020 03:00:51 +0000 Subject: [PATCH 85/92] Release 20201025.0.7116 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 5e4bb1b..fd0884b 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7033/base-devel-20201025.0.7033.tar.xz)" && \ - sha256sum -c <<< "f1b7b3ae69dc5f6d013d9cfb949a740d113891e87f35e9c63a67f0fe41b08628 base-devel-20201025.0.7033.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/41/download)" && \ + sha256sum -c <<< "9309c517f81fd00bb87e79f051f93eae9b7ae7e628813ad4b30af7a2d9f4369d base-devel-20201025.0.7116.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index d0526f7..fef6ebc 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/api/v4/projects/10185/packages/generic/rootfs/20201025.0.7033/base-20201025.0.7033.tar.xz)" && \ - sha256sum -c <<< "b0c5922d77e11cabdea4b404b58db4e8d019092c328385cbd8fe47d336be51b9 base-20201025.0.7033.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/39/download)" && \ + sha256sum -c <<< "b388140813c91862043a6d8cf3719764fcd940f5350a0720cc681ab8af0b0384 base-20201025.0.7116.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 1b964f97154d48edbd106cabbbb775992a686be1 Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 04:16:09 +0100 Subject: [PATCH 86/92] Better debug --- .gitlab-ci.yml | 13 ++++++++++--- ci/get-public-download-for-generic-package.sh | 6 +++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 191ce6d..99b3a3e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -177,15 +177,17 @@ release: - | for group in base base-devel; do sed -i "s|${group}.tar.xz|${group}-${BUILD_VERSION}.tar.xz|" output/${group}.tar.xz.SHA256 - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz - curl --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 + echo "Uploading ${group}.tar.xz" + curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz + echo "Uploading ${group}.tar.xz.SHA256" + curl -sSf --header "JOB-TOKEN: ${CI_JOB_TOKEN}" --upload-file output/${group}.tar.xz.SHA256 ${PACKAGE_REGISTRY_URL}/${group}-${BUILD_VERSION}.tar.xz.SHA256 sed "/TEMPLATE_ROOTFS_FILE/d" Dockerfile.template > ci/${group}/Dockerfile package_url=$(./ci/get-public-download-for-generic-package.sh ${group}-${BUILD_VERSION}.tar.xz) sed -i "s|TEMPLATE_ROOTFS_URL|${package_url}|" ci/${group}/Dockerfile sed -i "s|TEMPLATE_ROOTFS_HASH|$(cat output/${group}.tar.xz.SHA256)|" ci/${group}/Dockerfile done - > - curl --request POST + curl -sSf --request POST --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" --form "branch=add-base-devel-tags" --form "commit_message=Release ${BUILD_VERSION}" @@ -198,12 +200,17 @@ release: "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/repository/commits" - | base_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + echo "${base_url}" base_sha_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz.SHA256) + echo "${base_sha_url}" base_devel_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + echo "${base_devel_url}" base_devel_sha_url=$(./ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.xz.SHA256) + echo "${base_devel_sha_url}" # TODO: We should actually be able to do something like \"url\":\"${PACKAGE_REGISTRY_URL}/base-${BUILD_VERSION}.tar.xz\" # But it doesn't appear that those downloads are public. I consider this a bug and hopefully it's fixed in a future version! + echo "Creating release" release-cli create --name "Release ${BUILD_VERSION}" --description "Release ${BUILD_VERSION}" \ --tag-name v${BUILD_VERSION} --ref "add-base-devel-tags" \ --assets-link "{\"name\":\"base-${BUILD_VERSION}.tar.xz\",\"url\":\"${base_url}\"}" \ diff --git a/ci/get-public-download-for-generic-package.sh b/ci/get-public-download-for-generic-package.sh index de77c9f..53922b0 100755 --- a/ci/get-public-download-for-generic-package.sh +++ b/ci/get-public-download-for-generic-package.sh @@ -4,6 +4,6 @@ set -eu package_name=$1 -package_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") -base_id=$(curl --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"$package_name\") | .id") -echo "https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${base_id}/download" +package_id=$(curl -sSf --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages" | jq ".[] | select(.version == \"${BUILD_VERSION}\") | .id") +package_file_id=$(curl -sSf --header "PRIVATE-TOKEN: ${GITLAB_PROJECT_TOKEN}" "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/${package_id}/package_files" | jq ".[] | select(.file_name == \"$package_name\") | .id") +echo "https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/${package_file_id}/download" From 81dd765c7ad8d5f1b2291bb998f1c3ef585a4fc0 Mon Sep 17 00:00:00 2001 From: Arch Linux Docker release bot Date: Sun, 25 Oct 2020 03:22:53 +0000 Subject: [PATCH 87/92] Release 20201025.0.7144 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index fd0884b..4d0beeb 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/41/download)" && \ - sha256sum -c <<< "9309c517f81fd00bb87e79f051f93eae9b7ae7e628813ad4b30af7a2d9f4369d base-devel-20201025.0.7116.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/45/download)" && \ + sha256sum -c <<< "4118073b71fa0f4dbb8fd267c91a39632be275b37f5ccad2d49443cfa86e3fbb base-devel-20201025.0.7144.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index fef6ebc..59c7744 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/39/download)" && \ - sha256sum -c <<< "b388140813c91862043a6d8cf3719764fcd940f5350a0720cc681ab8af0b0384 base-20201025.0.7116.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/43/download)" && \ + sha256sum -c <<< "9bd0ff530a32b54e4cdf50554c1ae4be858ea39f235d9fdf6517bb974d1a6a8a base-20201025.0.7144.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 0ed3e802494a32023d9b9f058dd0b5f233e7304d Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 04:25:36 +0100 Subject: [PATCH 88/92] Fix duplicated name --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 99b3a3e..cc56a69 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -203,7 +203,7 @@ release: echo "${base_url}" base_sha_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz.SHA256) echo "${base_sha_url}" - base_devel_url=$(./ci/get-public-download-for-generic-package.sh base-${BUILD_VERSION}.tar.xz) + base_devel_url=$(./ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.xz) echo "${base_devel_url}" base_devel_sha_url=$(./ci/get-public-download-for-generic-package.sh base-devel-${BUILD_VERSION}.tar.xz.SHA256) echo "${base_devel_sha_url}" From bfac114869cf03dbf940ea88860e5b1cbfd97d10 Mon Sep 17 00:00:00 2001 From: Arch Linux Docker release bot Date: Sun, 25 Oct 2020 03:32:52 +0000 Subject: [PATCH 89/92] Release 20201025.0.7175 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index 4d0beeb..e43e34f 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/45/download)" && \ - sha256sum -c <<< "4118073b71fa0f4dbb8fd267c91a39632be275b37f5ccad2d49443cfa86e3fbb base-devel-20201025.0.7144.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/49/download)" && \ + sha256sum -c <<< "c6738ec91d88ab9c0b32d67ece506de3de36122a6c2672dfff7b08687e6ed442 base-devel-20201025.0.7175.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 59c7744..2df734d 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/43/download)" && \ - sha256sum -c <<< "9bd0ff530a32b54e4cdf50554c1ae4be858ea39f235d9fdf6517bb974d1a6a8a base-20201025.0.7144.tar.xz" && \ +RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/47/download)" && \ + sha256sum -c <<< "d72adf698b3f8f5c5dc4354a25e6523e47be14919b33ed7784f7c3dd148e5026 base-20201025.0.7175.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From ca6e2dbddfbe287784ed8b6bca73ed7966f18eae Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 04:49:26 +0100 Subject: [PATCH 90/92] Proper way to get effective filename --- Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.template b/Dockerfile.template index 6ccf941..bd9aa78 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -11,7 +11,7 @@ FROM archlinux:latest AS verify COPY TEMPLATE_ROOTFS_FILE / SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ +RUN ROOTFS="$(curl -OJL -w "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ sha256sum -c <<< "TEMPLATE_ROOTFS_HASH" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 4443f7f8bd4e3ae62c0861a84c1a168a32bdfefe Mon Sep 17 00:00:00 2001 From: Sven-Hendrik Haase Date: Sun, 25 Oct 2020 05:09:47 +0100 Subject: [PATCH 91/92] Turns out we need --continue-at - for local builds still --- Dockerfile.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile.template b/Dockerfile.template index bd9aa78..ad79c50 100644 --- a/Dockerfile.template +++ b/Dockerfile.template @@ -11,7 +11,7 @@ FROM archlinux:latest AS verify COPY TEMPLATE_ROOTFS_FILE / SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl -OJL -w "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ +RUN ROOTFS="$(curl -OJL --continue-at - -w "%{filename_effective}" TEMPLATE_ROOTFS_URL)" && \ sha256sum -c <<< "TEMPLATE_ROOTFS_HASH" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" From 9a9b2ad4096b758c15ddf22d4d06369a17e30bde Mon Sep 17 00:00:00 2001 From: Arch Linux Docker release bot Date: Sun, 25 Oct 2020 04:16:29 +0000 Subject: [PATCH 92/92] Release 20201025.0.7220 --- ci/base-devel/Dockerfile | 4 ++-- ci/base/Dockerfile | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/base-devel/Dockerfile b/ci/base-devel/Dockerfile index e43e34f..097fcdb 100644 --- a/ci/base-devel/Dockerfile +++ b/ci/base-devel/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/49/download)" && \ - sha256sum -c <<< "c6738ec91d88ab9c0b32d67ece506de3de36122a6c2672dfff7b08687e6ed442 base-devel-20201025.0.7175.tar.xz" && \ +RUN ROOTFS="$(curl -OJL --continue-at - -w "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/53/download)" && \ + sha256sum -c <<< "452e26d9775a76e95bd582b96c742844a3e27b90147e1f6e1bc6dd3b82e8a558 base-devel-20201025.0.7220.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}" diff --git a/ci/base/Dockerfile b/ci/base/Dockerfile index 2df734d..b33f495 100644 --- a/ci/base/Dockerfile +++ b/ci/base/Dockerfile @@ -10,8 +10,8 @@ # allow us to automate the releaes process. FROM archlinux:latest AS verify SHELL ["/bin/bash", "-c"] -RUN ROOTFS="$(curl --continue-at - --remote-name --write-out "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/47/download)" && \ - sha256sum -c <<< "d72adf698b3f8f5c5dc4354a25e6523e47be14919b33ed7784f7c3dd148e5026 base-20201025.0.7175.tar.xz" && \ +RUN ROOTFS="$(curl -OJL --continue-at - -w "%{filename_effective}" https://gitlab.archlinux.org/archlinux/archlinux-docker/-/package_files/51/download)" && \ + sha256sum -c <<< "175387448f7992b2760e758bdb75bfd45de7d2bf5ad2940add9e19a96ffb4129 base-20201025.0.7220.tar.xz" && \ mkdir /rootfs && \ tar -C /rootfs --extract --auto-compress --file "${ROOTFS}"