From ec8c8dbafcdaf63d036bdba92fa153d4d1c90100 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Thu, 20 Apr 2023 19:20:13 +0400 Subject: [PATCH] chore: fix container image reproducibility Also provide make targets to sign the images. Signed-off-by: Andrey Smirnov --- .drone.jsonnet | 4 ++++ Dockerfile | 21 +++++++++++++++++++-- Makefile | 30 ++++++++++++++++++++++++++++++ hack/images.sh | 17 ----------------- 4 files changed, 53 insertions(+), 19 deletions(-) delete mode 100755 hack/images.sh diff --git a/.drone.jsonnet b/.drone.jsonnet index 559f06dc3..876250b2a 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -505,6 +505,8 @@ local integration_qemu_csi = Step('e2e-csi', target='e2e-qemu', privileged=true, local integration_images = Step('images', target='images', depends_on=[load_artifacts], environment={ IMAGE_REGISTRY: local_registry }); local integration_sbcs = Step('sbcs', target='sbcs', depends_on=[integration_images], environment={ IMAGE_REGISTRY: local_registry }); +local integration_reproducibility_test = Step('reproducibility-test', target='reproducibility-test', depends_on=[load_artifacts], environment={ IMAGE_REGISTRY: local_registry }); + local push_edge = { name: 'push-edge', image: 'autonomy/build-container:latest', @@ -554,6 +556,7 @@ local integration_pipelines = [ Pipeline('integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race]) + integration_trigger(['integration-qemu-race']), Pipeline('integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi]) + integration_trigger(['integration-qemu-csi']), Pipeline('integration-images', default_pipeline_steps + [integration_images, integration_sbcs]) + integration_trigger(['integration-images']), + Pipeline('integration-reproducibility-test', default_pipeline_steps + [integration_reproducibility_test]) + integration_trigger(['integration-reproducibility']), // cron pipelines, triggered on schedule events Pipeline('cron-integration-qemu', default_pipeline_steps + [integration_qemu, push_edge], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']), @@ -575,6 +578,7 @@ local integration_pipelines = [ Pipeline('cron-integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race], [default_cron_pipeline]) + cron_trigger(['nightly']), Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']), Pipeline('cron-integration-images', default_pipeline_steps + [integration_images, integration_sbcs], [default_cron_pipeline]) + cron_trigger(['nightly']), + Pipeline('cron-integration-reproducibility-test', default_pipeline_steps + [integration_reproducibility_test], [default_cron_pipeline]) + cron_trigger(['nightly']), ]; diff --git a/Dockerfile b/Dockerfile index 0ff8564d2..9f8a403cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -364,6 +364,7 @@ ARG GO_LDFLAGS ARG GOAMD64 RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=amd64 GOAMD64=${GOAMD64} go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-linux-amd64 RUN chmod +x /talosctl-linux-amd64 +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-linux-amd64 FROM base AS talosctl-linux-arm64-build WORKDIR /src/cmd/talosctl @@ -371,6 +372,7 @@ ARG GO_BUILDFLAGS ARG GO_LDFLAGS RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=arm64 go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-linux-arm64 RUN chmod +x /talosctl-linux-arm64 +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-linux-arm64 FROM base AS talosctl-linux-armv7-build WORKDIR /src/cmd/talosctl @@ -378,6 +380,7 @@ ARG GO_BUILDFLAGS ARG GO_LDFLAGS RUN --mount=type=cache,target=/.cache GOOS=linux GOARCH=arm GOARM=7 go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-linux-armv7 RUN chmod +x /talosctl-linux-armv7 +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-linux-armv7 FROM scratch AS talosctl-linux COPY --from=talosctl-linux-amd64-build /talosctl-linux-amd64 /talosctl-linux-amd64 @@ -400,6 +403,7 @@ ARG GO_LDFLAGS ARG GOAMD64 RUN --mount=type=cache,target=/.cache GOOS=darwin GOARCH=amd64 GOAMD64=${GOAMD64} go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-darwin-amd64 RUN chmod +x /talosctl-darwin-amd64 +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-darwin-amd64 FROM base AS talosctl-darwin-arm64-build WORKDIR /src/cmd/talosctl @@ -407,6 +411,7 @@ ARG GO_BUILDFLAGS ARG GO_LDFLAGS RUN --mount=type=cache,target=/.cache GOOS=darwin GOARCH=arm64 go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-darwin-arm64 RUN chmod +x /talosctl-darwin-arm64 +RUN touch --date="@${SOURCE_DATE_EPOCH}" talosctl-darwin-arm64 FROM scratch AS talosctl-darwin COPY --from=talosctl-darwin-amd64-build /talosctl-darwin-amd64 /talosctl-darwin-amd64 @@ -418,6 +423,7 @@ ARG GO_BUILDFLAGS ARG GO_LDFLAGS ARG GOAMD64 RUN --mount=type=cache,target=/.cache GOOS=windows GOARCH=amd64 GOAMD64=${GOAMD64} go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-windows-amd64.exe +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-windows-amd64.exe FROM scratch AS talosctl-windows COPY --from=talosctl-windows-amd64-build /talosctl-windows-amd64.exe /talosctl-windows-amd64.exe @@ -428,12 +434,14 @@ ARG GO_BUILDFLAGS ARG GO_LDFLAGS ARG GOAMD64 RUN --mount=type=cache,target=/.cache GOOS=freebsd GOARCH=amd64 GOAMD64=${GOAMD64} go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-freebsd-amd64 +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-freebsd-amd64 FROM base AS talosctl-freebsd-arm64-build WORKDIR /src/cmd/talosctl ARG GO_BUILDFLAGS ARG GO_LDFLAGS RUN --mount=type=cache,target=/.cache GOOS=freebsd GOARCH=arm64 go build ${GO_BUILDFLAGS} -ldflags "${GO_LDFLAGS}" -o /talosctl-freebsd-arm64 +RUN touch --date="@${SOURCE_DATE_EPOCH}" /talosctl-freebsd-arm64 FROM scratch AS talosctl-freebsd COPY --from=talosctl-freebsd-amd64-build /talosctl-freebsd-amd64 /talosctl-freebsd-amd64 @@ -550,6 +558,8 @@ RUN ln -s /etc/ssl /rootfs/usr/local/share/ca-certificates RUN ln -s /etc/ssl /rootfs/etc/ca-certificates FROM rootfs-base-${TARGETARCH} AS rootfs-base +RUN find /rootfs -print0 \ + | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" FROM rootfs-base-arm64 AS rootfs-squashfs-arm64 RUN find /rootfs -print0 \ @@ -663,6 +673,8 @@ COPY --from=pkg-grub / / COPY --from=unicode-pf2 /usr/share/grub/unicode.pf2 /usr/share/grub/unicode.pf2 FROM alpine:3.17.2 AS installer-image +ARG SOURCE_DATE_EPOCH +ENV SOURCE_DATE_EPOCH ${SOURCE_DATE_EPOCH} RUN apk add --no-cache --update --no-scripts \ bash \ cpio \ @@ -681,13 +693,18 @@ COPY --from=install-artifacts / / COPY --from=installer-build /installer /bin/installer COPY --chmod=0644 hack/extra-modules.conf /etc/modules.d/10-extra-modules.conf RUN ln -s /bin/installer /bin/talosctl +RUN find /bin /etc /lib /usr /sbin | grep -Ev '/etc/hosts|/etc/resolv.conf' \ + | xargs -r touch --date="@${SOURCE_DATE_EPOCH}" --no-dereference + +FROM scratch AS installer-image-squashed +COPY --from=installer-image / / ARG TAG ENV VERSION ${TAG} LABEL "alpha.talos.dev/version"="${VERSION}" LABEL org.opencontainers.image.source https://github.com/siderolabs/talos ENTRYPOINT ["/bin/installer"] -FROM installer-image AS installer +FROM installer-image-squashed AS installer ONBUILD RUN apk add --no-cache --update \ cpio \ squashfs-tools \ @@ -708,7 +725,7 @@ ONBUILD RUN find /rootfs \ && rm -rf /initramfs ONBUILD WORKDIR / -FROM installer-image AS imager +FROM installer-image-squashed AS imager # The test target performs tests on the source code. diff --git a/Makefile b/Makefile index d1369963c..77c620933 100644 --- a/Makefile +++ b/Makefile @@ -477,3 +477,33 @@ push-%: login ## Pushes the installer, imager, talos and talosctl images to the .PHONY: clean clean: ## Cleans up all artifacts. @-rm -rf $(ARTIFACTS) + +.PHONY: image-list +image-list: ## Prints a list of all images built by this Makefile with digests. + @echo -n installer talos imager talosctl | xargs -d ' ' -I{} sh -c 'echo $(REGISTRY_AND_USERNAME)/{}:$(IMAGE_TAG)' | xargs -I{} sh -c 'echo {}@$$(crane digest {})' + +.PHONY: sign-images +sign-images: ## Run cosign to sign all images built by this Makefile. + @$(MAKE) --quiet image-list | xargs -I{} sh -c 'cosign sign --yes {}' + +.PHONY: reproducibility-test +reproducibility-test: + @$(MAKE) reproducibility-test-local-initramfs + @$(MAKE) reproducibility-test-docker-installer INSTALLER_ARCH=targetarch PLATFORM=linux/amd64 + @$(MAKE) reproducibility-test-docker-talos reproducibility-test-docker-imager reproducibility-test-docker-talosctl PLATFORM=linux/amd64 + +reproducibility-test-docker-%: + @rm -rf _out1/ _out2/ + @mkdir -p _out1/ _out2/ + @$(MAKE) docker-$* DEST=_out1/ + @$(MAKE) docker-$* DEST=_out2/ TARGET_ARGS="--no-cache" + @find _out1/ -type f | xargs -IFILE diffoscope FILE `echo FILE | sed 's/_out1/_out2/'` + @rm -rf _out1/ _out2/ + +reproducibility-test-local-%: + @rm -rf _out1/ _out2/ + @mkdir -p _out1/ _out2/ + @$(MAKE) local-$* DEST=_out1/ + @$(MAKE) local-$* DEST=_out2/ TARGET_ARGS="--no-cache" + @find _out1/ -type f | xargs -IFILE diffoscope FILE `echo FILE | sed 's/_out1/_out2/'` + @rm -rf _out1/ _out2/ diff --git a/hack/images.sh b/hack/images.sh deleted file mode 100755 index 18610a47a..000000000 --- a/hack/images.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash - -set -e - -CRANE="${CRANE:-crane}" -REGISTRY="${REGISTRY:-ghcr.io}" -USERNAME="${USERNAME:-siderolabs}" -TAG="${TAG:-latest}" - -IMAGES="installer talos imager" - -for image in $IMAGES; do - ref="${REGISTRY}/${USERNAME}/${image}:${TAG}" - digest=$(${CRANE} digest ${ref}) - - echo "${ref}@${digest}" -done