From 4c0c626b786f14c5eabdc65e88d2aae92829bf73 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 26 Apr 2024 17:32:15 +0400 Subject: [PATCH] feat: use zstd compression in place of xz Initramfs and kernel are compressed with zstd. Extensions are compressed with zstd for Talos 1.8+. Signed-off-by: Andrey Smirnov --- .drone.jsonnet | 63 +++-- Dockerfile | 15 +- Makefile | 10 +- cmd/installer/pkg/install/extensions.go | 2 + go.mod | 2 +- hack/release.toml | 218 +----------------- .../test/extensions/extension-patch-filter.jq | 6 +- internal/pkg/extensions/compress.go | 13 +- internal/pkg/extensions/extensions_test.go | 3 +- internal/pkg/extensions/kernel_modules.go | 21 +- pkg/imager/extensions/extensions.go | 7 +- pkg/imager/extensions/rebuild.go | 21 +- pkg/imager/imager.go | 3 + pkg/imager/post.go | 14 ++ pkg/imager/profile/default.go | 44 ++-- pkg/imager/profile/outformat_enumer.go | 12 +- pkg/imager/profile/output.go | 1 + pkg/machinery/gendata/data/pkgs | 2 +- pkg/machinery/imager/quirks/quirks.go | 12 + pkg/machinery/imager/quirks/quirks_test.go | 32 +++ 20 files changed, 203 insertions(+), 298 deletions(-) diff --git a/.drone.jsonnet b/.drone.jsonnet index f0de639e5..8be3f89fa 100644 --- a/.drone.jsonnet +++ b/.drone.jsonnet @@ -368,17 +368,10 @@ local ExtensionsStep(with_e2e=true) = // generates the extension list patch manifest local extensions_patch_manifest = Step( - 'extensions-patch-manifest', - with_make=false, - environment=creds_env_vars, + 'installer-with-extensions', + environment={ IMAGE_REGISTRY: local_registry }, depends_on=[ extensions_artifacts, - ], - extra_commands=[ - // create a patch file to pass to the downstream build - // ignore nvidia extensions, testing nvidia extensions needs a machine with nvidia graphics card - 'jq -R < _out/extensions-metadata | jq -s -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > _out/extensions-patch.yaml', - 'cat _out/extensions-patch.yaml', ] ); @@ -1056,35 +1049,35 @@ local release = { files: [ '_out/akamai-amd64.raw.gz', '_out/akamai-arm64.raw.gz', - '_out/aws-amd64.raw.xz', - '_out/aws-arm64.raw.xz', - '_out/azure-amd64.vhd.xz', - '_out/azure-arm64.vhd.xz', + '_out/aws-amd64.raw.zst', + '_out/aws-arm64.raw.zst', + '_out/azure-amd64.vhd.zst', + '_out/azure-arm64.vhd.zst', '_out/cloud-images.json', '_out/digital-ocean-amd64.raw.gz', '_out/digital-ocean-arm64.raw.gz', - '_out/exoscale-amd64.qcow2.xz', - '_out/exoscale-arm64.qcow2.xz', + '_out/exoscale-amd64.qcow2.zst', + '_out/exoscale-arm64.qcow2.zst', '_out/gcp-amd64.raw.tar.gz', '_out/gcp-arm64.raw.tar.gz', - '_out/hcloud-amd64.raw.xz', - '_out/hcloud-arm64.raw.xz', - '_out/initramfs-amd64.xz', - '_out/initramfs-arm64.xz', + '_out/hcloud-amd64.raw.zst', + '_out/hcloud-arm64.raw.zst', + '_out/initramfs-amd64.zst', + '_out/initramfs-arm64.zst', '_out/metal-amd64.iso', '_out/metal-arm64.iso', - '_out/metal-amd64.raw.xz', - '_out/metal-arm64.raw.xz', - '_out/nocloud-amd64.raw.xz', - '_out/nocloud-arm64.raw.xz', - '_out/opennebula-amd64.raw.xz', - '_out/opennebula-arm64.raw.xz', - '_out/openstack-amd64.raw.xz', - '_out/openstack-arm64.raw.xz', - '_out/oracle-amd64.qcow2.xz', - '_out/oracle-arm64.qcow2.xz', - '_out/scaleway-amd64.raw.xz', - '_out/scaleway-arm64.raw.xz', + '_out/metal-amd64.raw.zst', + '_out/metal-arm64.raw.zst', + '_out/nocloud-amd64.raw.zst', + '_out/nocloud-arm64.raw.zst', + '_out/opennebula-amd64.raw.zst', + '_out/opennebula-arm64.raw.zst', + '_out/openstack-amd64.raw.zst', + '_out/openstack-arm64.raw.zst', + '_out/oracle-amd64.qcow2.zst', + '_out/oracle-arm64.qcow2.zst', + '_out/scaleway-amd64.raw.zst', + '_out/scaleway-arm64.raw.zst', '_out/sd-boot-amd64.efi', '_out/sd-boot-arm64.efi', '_out/sd-stub-amd64.efi', @@ -1099,14 +1092,14 @@ local release = { '_out/talosctl-linux-arm64', '_out/talosctl-linux-armv7', '_out/talosctl-windows-amd64.exe', - '_out/upcloud-amd64.raw.xz', - '_out/upcloud-arm64.raw.xz', + '_out/upcloud-amd64.raw.zst', + '_out/upcloud-arm64.raw.zst', '_out/vmware-amd64.ova', '_out/vmware-arm64.ova', '_out/vmlinuz-amd64', '_out/vmlinuz-arm64', - '_out/vultr-amd64.raw.xz', - '_out/vultr-arm64.raw.xz', + '_out/vultr-amd64.raw.zst', + '_out/vultr-arm64.raw.zst', ], checksum: ['sha256', 'sha512'], }, diff --git a/Dockerfile b/Dockerfile index b65a830d1..ec99b8cc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -688,14 +688,16 @@ RUN find /rootfs -print0 \ | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" FROM rootfs-base-arm64 AS rootfs-squashfs-arm64 +ARG ZSTD_COMPRESSION_LEVEL RUN find /rootfs -print0 \ | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" -RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress +RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress FROM rootfs-base-amd64 AS rootfs-squashfs-amd64 +ARG ZSTD_COMPRESSION_LEVEL RUN find /rootfs -print0 \ | xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}" -RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress +RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress FROM scratch AS squashfs-arm64 COPY --from=rootfs-squashfs-arm64 /rootfs.sqsh / @@ -710,6 +712,7 @@ COPY --from=rootfs-base /rootfs / FROM build AS initramfs-archive-arm64 WORKDIR /initramfs +ARG ZSTD_COMPRESSION_LEVEL COPY --from=squashfs-arm64 /rootfs.sqsh . COPY --from=init-build-arm64 /init . RUN find . -print0 \ @@ -718,11 +721,12 @@ RUN set -o pipefail \ && find . 2>/dev/null \ | LC_ALL=c sort \ | cpio --reproducible -H newc -o \ - | xz -v -C crc32 -0 -e -T 0 -z \ + | zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \ > /initramfs.xz FROM build AS initramfs-archive-amd64 WORKDIR /initramfs +ARG ZSTD_COMPRESSION_LEVEL COPY --from=squashfs-amd64 /rootfs.sqsh . COPY --from=init-build-amd64 /init . RUN find . -print0 \ @@ -731,7 +735,7 @@ RUN set -o pipefail \ && find . 2>/dev/null \ | LC_ALL=c sort \ | cpio --reproducible -H newc -o \ - | xz -v -C crc32 -0 -e -T 0 -z \ + | zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \ > /initramfs.xz FROM initramfs-archive-${TARGETARCH} AS initramfs-archive @@ -800,7 +804,8 @@ RUN apk add --no-cache --update --no-scripts \ util-linux \ xfsprogs \ xorriso \ - xz + xz \ + zstd ARG TARGETARCH ENV TARGETARCH ${TARGETARCH} COPY --from=installer-build /installer /bin/installer diff --git a/Makefile b/Makefile index 1bbe29881..38b0804a7 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,13 @@ DOCKER_LOGIN_ENABLED ?= true NAME = Talos CLOUD_IMAGES_EXTRA_ARGS ?= "" +ZSTD_COMPRESSION_LEVEL ?= 18 ARTIFACTS := _out TOOLS ?= ghcr.io/siderolabs/tools:v1.8.0-alpha.0 PKGS_PREFIX ?= ghcr.io/siderolabs -PKGS ?= v1.8.0-alpha.0-7-g718a7da +PKGS ?= v1.8.0-alpha.0-8-gca6249b EXTRAS ?= v1.8.0-alpha.0 PKG_FHS ?= $(PKGS_PREFIX)/fhs:$(PKGS) @@ -202,6 +203,7 @@ COMMON_ARGS += --build-arg=PKG_RASPBERYPI_FIRMWARE=$(PKG_RASPBERYPI_FIRMWARE) COMMON_ARGS += --build-arg=PKG_KERNEL=$(PKG_KERNEL) COMMON_ARGS += --build-arg=PKG_TALOSCTL_CNI_BUNDLE_INSTALL=$(PKG_TALOSCTL_CNI_BUNDLE_INSTALL) COMMON_ARGS += --build-arg=ABBREV_TAG=$(ABBREV_TAG) +COMMON_ARGS += --build-arg=ZSTD_COMPRESSION_LEVEL=$(ZSTD_COMPRESSION_LEVEL) CI_ARGS ?= @@ -524,6 +526,12 @@ provision-tests-track-%: REGISTRY=$(IMAGE_REGISTRY) \ ARTIFACTS=$(ARTIFACTS) +installer-with-extensions: $(ARTIFACTS)/extensions-metadata + $(MAKE) image-installer \ + IMAGER_ARGS="--base-installer-image=$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG) $(shell cat $(ARTIFACTS)/extensions-metadata | grep -vE 'tailscale|xen-guest-agent|nvidia' | xargs -n 1 echo --system-extension-image)" + crane push $(ARTIFACTS)/installer-amd64.tar $(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions + echo -n "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions" | jq -Rs -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > $(ARTIFACTS)/extensions-patch.yaml + # Assets for releases .PHONY: $(ARTIFACTS)/$(TALOS_RELEASE) diff --git a/cmd/installer/pkg/install/extensions.go b/cmd/installer/pkg/install/extensions.go index 66836bc1e..541c460d7 100644 --- a/cmd/installer/pkg/install/extensions.go +++ b/cmd/installer/pkg/install/extensions.go @@ -11,6 +11,7 @@ import ( "github.com/siderolabs/talos/pkg/imager/extensions" "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/imager/quirks" ) func (i *Installer) installExtensions() error { @@ -19,6 +20,7 @@ func (i *Installer) installExtensions() error { Arch: i.options.Arch, ExtensionTreePath: constants.SystemExtensionsPath, Printf: log.Printf, + Quirks: quirks.New(i.options.Version), } return builder.Build() diff --git a/go.mod b/go.mod index 084692a0c..004fe5056 100644 --- a/go.mod +++ b/go.mod @@ -92,6 +92,7 @@ require ( github.com/jeromer/syslogparser v1.1.0 github.com/jsimonetti/rtnetlink v1.4.1 github.com/jxskiss/base62 v1.1.0 + github.com/klauspost/compress v1.17.7 github.com/klauspost/cpuid/v2 v2.2.7 github.com/linode/go-metadata v0.2.0 github.com/martinlindhe/base36 v1.1.1 @@ -267,7 +268,6 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/josharian/native v1.1.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.17.7 // indirect github.com/kylelemons/godebug v1.1.0 // indirect github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect github.com/lucasb-eyer/go-colorful v1.2.0 // indirect diff --git a/hack/release.toml b/hack/release.toml index 3563cdef3..97f6b8697 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -6,7 +6,7 @@ github_repo = "siderolabs/talos" match_deps = "^github.com/((talos-systems|siderolabs)/[a-zA-Z0-9-]+)$" # previous release -previous = "v1.6.0" +previous = "v1.7.0" pre_release = true @@ -17,221 +17,15 @@ preface = """\ [notes.updates] title = "Component Updates" description = """\ -Linux: 6.6.28 -etcd: 3.5.11 -Kubernetes: 1.30.0 -containerd: 1.7.15 -runc: 1.1.12 -Flannel: 0.25.1 - Talos is built with Go 1.22.2. """ - [notes.device_selectors] - title = "Device Selectors" + + [notes.zstd] + title = "ZSTD Compression" description = """\ -Talos Linux now supports `physical: true` qualifier for device selectors, it selects non-virtual network interfaces (i.e. `en0` is selected, while `bond0` is not). -""" - - [notes.dns-resolve-cache] - title = "DNS Caching" - description = """\ -Talos Linux now provides a caching DNS resolver for host workloads (including host networking pods). It can be disabled with: - -```yaml -machine: - features: - hostDNS: - enabled: false -``` - -You can also enable dns caching for k8s pods with: - -```yaml -machine: - features: - hostDNS: - enabled: true - forwardKubeDNSToHost: true -``` - -Please note that on running cluster you will have to kill CoreDNS pods for this change to apply. - -If you want to can also enable the resolving of member addresses through their host and node names: - -```yaml -machine: - features: - hostDNS: - enabled: true - resolveMemberNames: true -``` -""" - - [notes.secureboot-image] - title = "Secure Boot Image" - description = """\ -Talos Linux now provides a way to configure systemd-boot ISO 'secure-boot-enroll' option while generating a SecureBoot ISO image: - -```yaml -output: - kind: iso - isoOptions: - sdBootEnrollKeys: force # default is still if-safe - outFormat: raw -``` -""" - - [notes.rsa-service-account] - title = "Kubernetes API Server Service Account Key" - description = """\ -Talos Linux starting from this release uses RSA key for Kubernetes API Server Service Account instead of ECDSA key to provide better compatibility with external OpenID Connect implementations. -""" - - [notes.opennebula] - title = "OpenNebula" - description = """\ -Talos Linux now supports OpenNebula platform. -""" - - [notes.extensions] - title = "Extension Services Config" - description = """\ -Talos now supports supplying configuration files and environment variables for extension services. -The extension service configuration is a separate config document. An example is shown below: - -```yaml ---- -apiVersion: v1alpha1 -kind: ExtensionServiceConfig -name: nut-client -configFiles: - - content: MONITOR ${upsmonHost} 1 remote pass password - mountPath: /usr/local/etc/nut/upsmon.conf -environment: - - UPS_NAME=ups -``` - -For documentation, see [Extension Services Config Files](https://www.talos.dev/v1.7/reference/configuration/extensions/extensionserviceconfig/). - -**Note**: The use of `environmentFile` in extension service spec is now deprecated and will be removed in a future release of Talos. -Use `ExtensionServiceConfig` instead. -""" - - [notes.k8supgrade] - title = "Kubernetes Upgrade" - description = """\ -The command `talosctl upgrade-k8s` now supports specifying custom image references for Kubernetes components via `--*-image` flags. -The default behavior is unchanged, and the flags are optional. -""" - - [notes.kubespan] - title = "KubeSpan" - description = """\ -Talos Linux disables by default a KubeSpan feature to harvest additional endpoints from KubeSpan members. -This feature turned out to be less helpful than expected and caused unnecessary performance issues. - -Previous behavior can be restored with: - -```yaml -machine: - network: - kubespan: - harvestExtraEndpoints: true -``` -""" - - [notes.sbc] - title = "SBC" - description = """\ -Talos has split the SBC's (Single Board Computers) into separate repositories. -There will not be any more SBC specific release assets as part of Talos release. - -The default Talos Installer image will stop working for SBC's and will fail the upgrade, if used, starting from Talos v1.7.0. - -The SBC's images and installers can be generated on the fly using [Image Factory](https://factory.talos.dev) or using [Imager](https://www.talos.dev/latest/talos-guides/install/boot-assets/) for custom images. -The list of official SBC's images supported by Image Factory can be found in the [Overlays](https://github.com/siderolabs/overlays/) repository. -""" - - [notes.syslog] - title = "Syslog" - description = """\ -Talos Linux now starts a basic syslog receiver listening on `/dev/log`. -The receiver can mostly parse both RFC3164 and RFC5424 messages and writes them as JSON formatted message. -The logs can be viewed via `talosctl logs syslogd`. - -This is mostly implemented for extension services that log to syslog. -""" - - [notes.ntp] - title = "Time Sync" - description = """\ -Default NTP server was updated to be `time.cloudflare.com` instead of `pool.ntp.org`. -Default server is only used if the user does not specify any NTP servers in the configuration. - -Talos Linux can now sync to PTP devices (e.g. provided by the hypervisor) skipping the network time servers. -In order to activate PTP sync, set `machine.time.servers` to the PTP device name (e.g. `/dev/ptp0`): - -```yaml -machine: - time: - servers: - - /dev/ptp0 -``` -""" - - [notes.ca-rotation] - title = "CA Rotation" - description = """\ -Talos Linux now supports rotating the root CA certificate and key for Talos API and Kubernetes API. -""" - - [notes.watchdog] - title = "Hardware Watchdog Timers" - description = """\ -Talos Linux now supports hardware watchdog timers configuration. -If enabled, and the machine becomes unresponsive, the hardware watchdog will reset the machine. - -The watchdog can be enabled with the following configuration document: - -```yaml -apiVersion: v1alpha1 -kind: WatchdogTimerConfig -device: /dev/watchdog0 -timeout: 3m0s -``` -""" - - [notes.logging] - title = "Logging" - description = """\ -Talos Linux now supports setting extra tags when sending logs in JSON format: - -```yaml -machine: - logging: - destinations: - - endpoint: "udp://127.0.0.1:12345/" - format: "json_lines" - extraTags: - server: s03-rack07 -``` -""" - - [notes.platforms] - title = "Platforms" - description = """\ -Talos Linux now supports [Akamai Connected Cloud](https://www.linode.com/) provider (platform `akamai`). -""" - - [notes.iptables] - title = "IPTables" - description = """\ -Talos Linux now forces `kubelet` and `kube-proxy` to use `iptables-nft` instead of `iptables-legacy` (`xtables`) which was the default -before Talos 1.7.0. - -Container images based on `iptables-wrapper` should work without changes, but if there was a direct call to `legacy` mode of `iptables`, make sure -to update to use `iptables-nft`. +Talos Linux now compresses kernel and initramfs using ZSTD. +Linux arm64 kernel is now compressed (previously it was uncompressed). """ [make_deps] diff --git a/hack/test/extensions/extension-patch-filter.jq b/hack/test/extensions/extension-patch-filter.jq index 45f4745e3..d626154fb 100644 --- a/hack/test/extensions/extension-patch-filter.jq +++ b/hack/test/extensions/extension-patch-filter.jq @@ -2,11 +2,7 @@ { "machine": { "install": { - "extensions": [ - { - "image": map(select(. | contains("nvidia") or contains("tailscale") or contains("xen-guest-agent") | not)) | .[] - } - ], + "image": . }, "sysctls": { "user.max_user_namespaces": "11255" diff --git a/internal/pkg/extensions/compress.go b/internal/pkg/extensions/compress.go index e1d11edc3..e2c7dde50 100644 --- a/internal/pkg/extensions/compress.go +++ b/internal/pkg/extensions/compress.go @@ -13,6 +13,7 @@ import ( "path/filepath" "github.com/siderolabs/talos/pkg/machinery/constants" + "github.com/siderolabs/talos/pkg/machinery/imager/quirks" ) // List of globs and destinations for early CPU ucode. @@ -38,7 +39,7 @@ var initramfsPaths = []string{ // // Components which should be placed to the initramfs are moved to the initramfsPath. // Ucode components are moved into a separate designated location. -func (ext *Extension) Compress(squashPath, initramfsPath string) (string, error) { +func (ext *Extension) Compress(squashPath, initramfsPath string, quirks quirks.Quirks) (string, error) { if err := ext.handleUcode(initramfsPath); err != nil { return "", err } @@ -53,7 +54,15 @@ func (ext *Extension) Compress(squashPath, initramfsPath string) (string, error) squashPath = filepath.Join(squashPath, fmt.Sprintf("%s.sqsh", ext.directory)) - cmd := exec.Command("mksquashfs", ext.rootfsPath, squashPath, "-all-root", "-noappend", "-comp", "xz", "-Xdict-size", "100%", "-no-progress") + var compressArgs []string + + if quirks.UseZSTDCompression() { + compressArgs = []string{"-comp", "zstd", "-Xcompression-level", "18"} + } else { + compressArgs = []string{"-comp", "xz", "-Xdict-size", "100%"} + } + + cmd := exec.Command("mksquashfs", append([]string{ext.rootfsPath, squashPath, "-all-root", "-noappend", "-no-progress"}, compressArgs...)...) cmd.Stderr = os.Stderr return squashPath, cmd.Run() diff --git a/internal/pkg/extensions/extensions_test.go b/internal/pkg/extensions/extensions_test.go index 50468abc7..fdf362636 100644 --- a/internal/pkg/extensions/extensions_test.go +++ b/internal/pkg/extensions/extensions_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/siderolabs/talos/internal/pkg/extensions" + "github.com/siderolabs/talos/pkg/machinery/imager/quirks" "github.com/siderolabs/talos/pkg/machinery/version" ) @@ -43,7 +44,7 @@ func TestCompress(t *testing.T) { require.NoError(t, err) squashDest, initramfsDest := t.TempDir(), t.TempDir() - squashFile, err := ext.Compress(squashDest, initramfsDest) + squashFile, err := ext.Compress(squashDest, initramfsDest, quirks.New("")) assert.NoError(t, err) assert.FileExists(t, squashFile) diff --git a/internal/pkg/extensions/kernel_modules.go b/internal/pkg/extensions/kernel_modules.go index 4f892ca1a..3afa57e22 100644 --- a/internal/pkg/extensions/kernel_modules.go +++ b/internal/pkg/extensions/kernel_modules.go @@ -6,6 +6,7 @@ package extensions import ( + "bytes" "errors" "fmt" "io" @@ -15,6 +16,7 @@ import ( "path/filepath" "strings" + "github.com/klauspost/compress/zstd" "github.com/u-root/u-root/pkg/cpio" "github.com/ulikunitz/xz" @@ -36,6 +38,23 @@ func (ext *Extension) KernelModuleDirectory() string { return filepath.Join(ext.rootfsPath, constants.KernelModulesPath) } +func autoDecompress(r io.Reader) (io.Reader, error) { + var magic [4]byte + + if _, err := r.Read(magic[:]); err != nil { + return nil, err + } + + src := io.MultiReader(bytes.NewReader(magic[:]), r) + + // xz magic + if bytes.Equal(magic[:], []byte{0xfd, '7', 'z', 'X'}) { + return xz.NewReader(src) + } + + return zstd.NewReader(src) +} + // GenerateKernelModuleDependencyTreeExtension generates a kernel module dependency tree extension. // //nolint:gocyclo @@ -60,7 +79,7 @@ func GenerateKernelModuleDependencyTreeExtension(extensionPathsWithKernelModules return initramfsxz.Close() }) - r, err := xz.NewReader(initramfsxz) + r, err := autoDecompress(initramfsxz) if err != nil { return nil, err } diff --git a/pkg/imager/extensions/extensions.go b/pkg/imager/extensions/extensions.go index 1ec898f29..95d8a4640 100644 --- a/pkg/imager/extensions/extensions.go +++ b/pkg/imager/extensions/extensions.go @@ -13,6 +13,7 @@ import ( "github.com/siderolabs/talos/internal/pkg/extensions" "github.com/siderolabs/talos/pkg/machinery/constants" extinterface "github.com/siderolabs/talos/pkg/machinery/extensions" + "github.com/siderolabs/talos/pkg/machinery/imager/quirks" ) // Builder rebuilds initramfs.xz with extensions. @@ -25,6 +26,8 @@ type Builder struct { ExtensionTreePath string // Printf is used for logging. Printf func(format string, v ...any) + // Quirks for the Talos version being used. + Quirks quirks.Quirks } // Build rebuilds the initramfs.xz with extensions. @@ -86,7 +89,7 @@ func (builder *Builder) Build() error { return err } - return builder.rebuildInitramfs(tempDir) + return builder.rebuildInitramfs(tempDir, builder.Quirks) } func (builder *Builder) validateExtensions(extensions []*extensions.Extension) error { @@ -107,7 +110,7 @@ func (builder *Builder) compressExtensions(extensions []*extensions.Extension, t builder.Printf("compressing system extensions") for _, ext := range extensions { - path, err := ext.Compress(tempDir, tempDir) + path, err := ext.Compress(tempDir, tempDir, builder.Quirks) if err != nil { return nil, fmt.Errorf("error compressing extension %q: %w", ext.Manifest.Metadata.Name, err) } diff --git a/pkg/imager/extensions/rebuild.go b/pkg/imager/extensions/rebuild.go index 08b2b82ce..98af7374b 100644 --- a/pkg/imager/extensions/rebuild.go +++ b/pkg/imager/extensions/rebuild.go @@ -10,14 +10,16 @@ import ( "io" "os" "os/exec" + + "github.com/siderolabs/talos/pkg/machinery/imager/quirks" ) // rebuildInitramfs rebuilds finalized initramfs with extensions. // // If uncompressedListing is not empty, contents will be prepended to the initramfs uncompressed. -// Contents from compressedListing will be appended to the initramfs compressed (xz) as a second block. +// Contents from compressedListing will be appended to the initramfs compressed (xz/zstd) as a second block. // Original initramfs.xz contents will stay without changes. -func (builder *Builder) rebuildInitramfs(tempDir string) error { +func (builder *Builder) rebuildInitramfs(tempDir string, quirks quirks.Quirks) error { compressedListing, uncompressedListing, err := buildInitramfsContents(tempDir) if err != nil { return err @@ -29,18 +31,18 @@ func (builder *Builder) rebuildInitramfs(tempDir string) error { } } - if err = builder.appendCompressedInitramfs(tempDir, compressedListing); err != nil { + if err = builder.appendCompressedInitramfs(tempDir, compressedListing, quirks); err != nil { return fmt.Errorf("error appending compressed initramfs: %w", err) } return nil } -func (builder *Builder) appendCompressedInitramfs(tempDir string, compressedListing []byte) error { +func (builder *Builder) appendCompressedInitramfs(tempDir string, compressedListing []byte, quirks quirks.Quirks) error { builder.Printf("creating system extensions initramfs archive and compressing it") // the code below runs the equivalent of: - // find $tempDir -print | cpio -H newc --create --reproducible | xz -v -C crc32 -0 -e -T 0 -z + // find $tempDir -print | cpio -H newc --create --reproducible | { xz -v -C crc32 -0 -e -T 0 -z || zstd -T0 -18 -c --quiet } pipeR, pipeW, err := os.Pipe() if err != nil { @@ -73,7 +75,14 @@ func (builder *Builder) appendCompressedInitramfs(tempDir string, compressedList defer destination.Close() //nolint:errcheck // append compressed initramfs.sysext to the original initramfs.xz, kernel can read such format - cmd2 := exec.Command("xz", "-v", "-C", "crc32", "-0", "-e", "-T", "0", "-z", "--quiet") + var cmd2 *exec.Cmd + + if quirks.UseZSTDCompression() { + cmd2 = exec.Command("zstd", "-T0", "-18", "-c", "--quiet") + } else { + cmd2 = exec.Command("xz", "-v", "-C", "crc32", "-0", "-e", "-T", "0", "-z", "--quiet") + } + cmd2.Dir = tempDir cmd2.Stdin = pipeR cmd2.Stdout = destination diff --git a/pkg/imager/imager.go b/pkg/imager/imager.go index c5f7de380..19bfa6b4c 100644 --- a/pkg/imager/imager.go +++ b/pkg/imager/imager.go @@ -152,6 +152,8 @@ func (i *Imager) Execute(ctx context.Context, outputPath string, report *reporte return i.postProcessXz(outputAssetPath, report) case profile.OutFormatGZ: return i.postProcessGz(outputAssetPath, report) + case profile.OutFormatZSTD: + return i.postProcessZstd(outputAssetPath, report) case profile.OutFormatTar: return i.postProcessTar(outputAssetPath, report) case profile.OutFormatUnknown: @@ -305,6 +307,7 @@ func (i *Imager) buildInitramfs(ctx context.Context, report *reporter.Reporter) Arch: i.prof.Arch, ExtensionTreePath: extensionsCheckoutDir, Printf: printf, + Quirks: quirks.New(i.prof.Version), } if err := builder.Build(); err != nil { diff --git a/pkg/imager/post.go b/pkg/imager/post.go index e9047e2e0..29b37be67 100644 --- a/pkg/imager/post.go +++ b/pkg/imager/post.go @@ -62,3 +62,17 @@ func (i *Imager) postProcessXz(filename string, report *reporter.Reporter) (stri return filename + ".xz", nil } + +func (i *Imager) postProcessZstd(filename string, report *reporter.Reporter) (string, error) { + report.Report(reporter.Update{Message: "compressing .zst", Status: reporter.StatusRunning}) + + out := filename + ".zst" + + if _, err := cmd.Run("zstd", "-T0", "--rm", "-18", "--quiet", "--force", "-o", out, filename); err != nil { + return "", err + } + + report.Report(reporter.Update{Message: fmt.Sprintf("compression done: %s", out), Status: reporter.StatusSucceeded}) + + return filename + ".zst", nil +} diff --git a/pkg/imager/profile/default.go b/pkg/imager/profile/default.go index 51f70ea68..227347b2a 100644 --- a/pkg/imager/profile/default.go +++ b/pkg/imager/profile/default.go @@ -48,7 +48,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -60,7 +60,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(true), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -101,7 +101,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: DefaultRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -113,7 +113,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: DefaultRAWDiskSize, DiskFormat: DiskFormatVPC, @@ -138,7 +138,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: 10 * 1024 * mib, DiskFormat: DiskFormatQCOW2, @@ -163,7 +163,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -175,7 +175,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -187,7 +187,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -199,7 +199,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -211,7 +211,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: DefaultRAWDiskSize, DiskFormat: DiskFormatQCOW2, @@ -224,7 +224,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -236,7 +236,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: DefaultRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -260,7 +260,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: DefaultRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -275,7 +275,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -289,7 +289,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -303,7 +303,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -317,7 +317,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -331,7 +331,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -345,7 +345,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -359,7 +359,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -373,7 +373,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, @@ -387,7 +387,7 @@ var Default = map[string]Profile{ SecureBoot: pointer.To(false), Output: Output{ Kind: OutKindImage, - OutFormat: OutFormatXZ, + OutFormat: OutFormatZSTD, ImageOptions: &ImageOptions{ DiskSize: MinRAWDiskSize, DiskFormat: DiskFormatRaw, diff --git a/pkg/imager/profile/outformat_enumer.go b/pkg/imager/profile/outformat_enumer.go index f062e9d03..8b0960fcd 100644 --- a/pkg/imager/profile/outformat_enumer.go +++ b/pkg/imager/profile/outformat_enumer.go @@ -7,11 +7,11 @@ import ( "strings" ) -const _OutFormatName = "unknownraw.tar.gz.xz.gz" +const _OutFormatName = "unknownraw.tar.gz.xz.gz.zst" -var _OutFormatIndex = [...]uint8{0, 7, 10, 17, 20, 23} +var _OutFormatIndex = [...]uint8{0, 7, 10, 17, 20, 23, 27} -const _OutFormatLowerName = "unknownraw.tar.gz.xz.gz" +const _OutFormatLowerName = "unknownraw.tar.gz.xz.gz.zst" func (i OutFormat) String() string { if i < 0 || i >= OutFormat(len(_OutFormatIndex)-1) { @@ -29,9 +29,10 @@ func _OutFormatNoOp() { _ = x[OutFormatTar-(2)] _ = x[OutFormatXZ-(3)] _ = x[OutFormatGZ-(4)] + _ = x[OutFormatZSTD-(5)] } -var _OutFormatValues = []OutFormat{OutFormatUnknown, OutFormatRaw, OutFormatTar, OutFormatXZ, OutFormatGZ} +var _OutFormatValues = []OutFormat{OutFormatUnknown, OutFormatRaw, OutFormatTar, OutFormatXZ, OutFormatGZ, OutFormatZSTD} var _OutFormatNameToValueMap = map[string]OutFormat{ _OutFormatName[0:7]: OutFormatUnknown, @@ -44,6 +45,8 @@ var _OutFormatNameToValueMap = map[string]OutFormat{ _OutFormatLowerName[17:20]: OutFormatXZ, _OutFormatName[20:23]: OutFormatGZ, _OutFormatLowerName[20:23]: OutFormatGZ, + _OutFormatName[23:27]: OutFormatZSTD, + _OutFormatLowerName[23:27]: OutFormatZSTD, } var _OutFormatNames = []string{ @@ -52,6 +55,7 @@ var _OutFormatNames = []string{ _OutFormatName[10:17], _OutFormatName[17:20], _OutFormatName[20:23], + _OutFormatName[23:27], } // OutFormatString retrieves an enum value from the enum constants string name. diff --git a/pkg/imager/profile/output.go b/pkg/imager/profile/output.go index 61576775e..0198a9f38 100644 --- a/pkg/imager/profile/output.go +++ b/pkg/imager/profile/output.go @@ -76,6 +76,7 @@ const ( OutFormatTar // .tar.gz OutFormatXZ // .xz OutFormatGZ // .gz + OutFormatZSTD // .zst ) //go:generate enumer -type DiskFormat -linecomment -text diff --git a/pkg/machinery/gendata/data/pkgs b/pkg/machinery/gendata/data/pkgs index 835657470..556a12508 100644 --- a/pkg/machinery/gendata/data/pkgs +++ b/pkg/machinery/gendata/data/pkgs @@ -1 +1 @@ -v1.8.0-alpha.0-7-g718a7da \ No newline at end of file +v1.8.0-alpha.0-8-gca6249b \ No newline at end of file diff --git a/pkg/machinery/imager/quirks/quirks.go b/pkg/machinery/imager/quirks/quirks.go index dab696261..7d516e9ff 100644 --- a/pkg/machinery/imager/quirks/quirks.go +++ b/pkg/machinery/imager/quirks/quirks.go @@ -62,3 +62,15 @@ func (q Quirks) SupportsOverlay() bool { return q.v.GTE(minVersionOverlay) } + +var minVersionZstd = semver.MustParse("1.8.0") + +// UseZSTDCompression returns true if the Talos should use zstd compression in place of xz. +func (q Quirks) UseZSTDCompression() bool { + // if the version doesn't parse, we assume it's latest Talos + if q.v == nil { + return true + } + + return q.v.GTE(minVersionZstd) +} diff --git a/pkg/machinery/imager/quirks/quirks_test.go b/pkg/machinery/imager/quirks/quirks_test.go index a5fc1d75f..9dc45b797 100644 --- a/pkg/machinery/imager/quirks/quirks_test.go +++ b/pkg/machinery/imager/quirks/quirks_test.go @@ -99,3 +99,35 @@ func TestSupportsOverlay(t *testing.T) { }) } } + +func TestSupportsZstd(t *testing.T) { + for _, test := range []struct { + version string + + expected bool + }{ + { + version: "1.7.3", + expected: false, + }, + { + expected: true, + }, + { + version: "1.6.2", + expected: false, + }, + { + version: "1.8.0-alpha.0", + expected: true, + }, + { + version: "v1.8.3", + expected: true, + }, + } { + t.Run(test.version, func(t *testing.T) { + assert.Equal(t, test.expected, quirks.New(test.version).UseZSTDCompression()) + }) + } +}