feat: use zstd compression in place of xz

Initramfs and kernel are compressed with zstd.

Extensions are compressed with zstd for Talos 1.8+.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
This commit is contained in:
Andrey Smirnov 2024-04-26 17:32:15 +04:00
parent 98906ed6ea
commit 4c0c626b78
No known key found for this signature in database
GPG Key ID: FE042E3D4085A811
20 changed files with 203 additions and 298 deletions

View File

@ -368,17 +368,10 @@ local ExtensionsStep(with_e2e=true) =
// generates the extension list patch manifest
local extensions_patch_manifest = Step(
'extensions-patch-manifest',
with_make=false,
environment=creds_env_vars,
'installer-with-extensions',
environment={ IMAGE_REGISTRY: local_registry },
depends_on=[
extensions_artifacts,
],
extra_commands=[
// create a patch file to pass to the downstream build
// ignore nvidia extensions, testing nvidia extensions needs a machine with nvidia graphics card
'jq -R < _out/extensions-metadata | jq -s -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > _out/extensions-patch.yaml',
'cat _out/extensions-patch.yaml',
]
);
@ -1056,35 +1049,35 @@ local release = {
files: [
'_out/akamai-amd64.raw.gz',
'_out/akamai-arm64.raw.gz',
'_out/aws-amd64.raw.xz',
'_out/aws-arm64.raw.xz',
'_out/azure-amd64.vhd.xz',
'_out/azure-arm64.vhd.xz',
'_out/aws-amd64.raw.zst',
'_out/aws-arm64.raw.zst',
'_out/azure-amd64.vhd.zst',
'_out/azure-arm64.vhd.zst',
'_out/cloud-images.json',
'_out/digital-ocean-amd64.raw.gz',
'_out/digital-ocean-arm64.raw.gz',
'_out/exoscale-amd64.qcow2.xz',
'_out/exoscale-arm64.qcow2.xz',
'_out/exoscale-amd64.qcow2.zst',
'_out/exoscale-arm64.qcow2.zst',
'_out/gcp-amd64.raw.tar.gz',
'_out/gcp-arm64.raw.tar.gz',
'_out/hcloud-amd64.raw.xz',
'_out/hcloud-arm64.raw.xz',
'_out/initramfs-amd64.xz',
'_out/initramfs-arm64.xz',
'_out/hcloud-amd64.raw.zst',
'_out/hcloud-arm64.raw.zst',
'_out/initramfs-amd64.zst',
'_out/initramfs-arm64.zst',
'_out/metal-amd64.iso',
'_out/metal-arm64.iso',
'_out/metal-amd64.raw.xz',
'_out/metal-arm64.raw.xz',
'_out/nocloud-amd64.raw.xz',
'_out/nocloud-arm64.raw.xz',
'_out/opennebula-amd64.raw.xz',
'_out/opennebula-arm64.raw.xz',
'_out/openstack-amd64.raw.xz',
'_out/openstack-arm64.raw.xz',
'_out/oracle-amd64.qcow2.xz',
'_out/oracle-arm64.qcow2.xz',
'_out/scaleway-amd64.raw.xz',
'_out/scaleway-arm64.raw.xz',
'_out/metal-amd64.raw.zst',
'_out/metal-arm64.raw.zst',
'_out/nocloud-amd64.raw.zst',
'_out/nocloud-arm64.raw.zst',
'_out/opennebula-amd64.raw.zst',
'_out/opennebula-arm64.raw.zst',
'_out/openstack-amd64.raw.zst',
'_out/openstack-arm64.raw.zst',
'_out/oracle-amd64.qcow2.zst',
'_out/oracle-arm64.qcow2.zst',
'_out/scaleway-amd64.raw.zst',
'_out/scaleway-arm64.raw.zst',
'_out/sd-boot-amd64.efi',
'_out/sd-boot-arm64.efi',
'_out/sd-stub-amd64.efi',
@ -1099,14 +1092,14 @@ local release = {
'_out/talosctl-linux-arm64',
'_out/talosctl-linux-armv7',
'_out/talosctl-windows-amd64.exe',
'_out/upcloud-amd64.raw.xz',
'_out/upcloud-arm64.raw.xz',
'_out/upcloud-amd64.raw.zst',
'_out/upcloud-arm64.raw.zst',
'_out/vmware-amd64.ova',
'_out/vmware-arm64.ova',
'_out/vmlinuz-amd64',
'_out/vmlinuz-arm64',
'_out/vultr-amd64.raw.xz',
'_out/vultr-arm64.raw.xz',
'_out/vultr-amd64.raw.zst',
'_out/vultr-arm64.raw.zst',
],
checksum: ['sha256', 'sha512'],
},

View File

@ -688,14 +688,16 @@ RUN find /rootfs -print0 \
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
FROM rootfs-base-arm64 AS rootfs-squashfs-arm64
ARG ZSTD_COMPRESSION_LEVEL
RUN find /rootfs -print0 \
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress
FROM rootfs-base-amd64 AS rootfs-squashfs-amd64
ARG ZSTD_COMPRESSION_LEVEL
RUN find /rootfs -print0 \
| xargs -0r touch --no-dereference --date="@${SOURCE_DATE_EPOCH}"
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress
RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp zstd -Xcompression-level ${ZSTD_COMPRESSION_LEVEL} -no-progress
FROM scratch AS squashfs-arm64
COPY --from=rootfs-squashfs-arm64 /rootfs.sqsh /
@ -710,6 +712,7 @@ COPY --from=rootfs-base /rootfs /
FROM build AS initramfs-archive-arm64
WORKDIR /initramfs
ARG ZSTD_COMPRESSION_LEVEL
COPY --from=squashfs-arm64 /rootfs.sqsh .
COPY --from=init-build-arm64 /init .
RUN find . -print0 \
@ -718,11 +721,12 @@ RUN set -o pipefail \
&& find . 2>/dev/null \
| LC_ALL=c sort \
| cpio --reproducible -H newc -o \
| xz -v -C crc32 -0 -e -T 0 -z \
| zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \
> /initramfs.xz
FROM build AS initramfs-archive-amd64
WORKDIR /initramfs
ARG ZSTD_COMPRESSION_LEVEL
COPY --from=squashfs-amd64 /rootfs.sqsh .
COPY --from=init-build-amd64 /init .
RUN find . -print0 \
@ -731,7 +735,7 @@ RUN set -o pipefail \
&& find . 2>/dev/null \
| LC_ALL=c sort \
| cpio --reproducible -H newc -o \
| xz -v -C crc32 -0 -e -T 0 -z \
| zstd -c -T0 -${ZSTD_COMPRESSION_LEVEL} \
> /initramfs.xz
FROM initramfs-archive-${TARGETARCH} AS initramfs-archive
@ -800,7 +804,8 @@ RUN apk add --no-cache --update --no-scripts \
util-linux \
xfsprogs \
xorriso \
xz
xz \
zstd
ARG TARGETARCH
ENV TARGETARCH ${TARGETARCH}
COPY --from=installer-build /installer /bin/installer

View File

@ -13,12 +13,13 @@ DOCKER_LOGIN_ENABLED ?= true
NAME = Talos
CLOUD_IMAGES_EXTRA_ARGS ?= ""
ZSTD_COMPRESSION_LEVEL ?= 18
ARTIFACTS := _out
TOOLS ?= ghcr.io/siderolabs/tools:v1.8.0-alpha.0
PKGS_PREFIX ?= ghcr.io/siderolabs
PKGS ?= v1.8.0-alpha.0-7-g718a7da
PKGS ?= v1.8.0-alpha.0-8-gca6249b
EXTRAS ?= v1.8.0-alpha.0
PKG_FHS ?= $(PKGS_PREFIX)/fhs:$(PKGS)
@ -202,6 +203,7 @@ COMMON_ARGS += --build-arg=PKG_RASPBERYPI_FIRMWARE=$(PKG_RASPBERYPI_FIRMWARE)
COMMON_ARGS += --build-arg=PKG_KERNEL=$(PKG_KERNEL)
COMMON_ARGS += --build-arg=PKG_TALOSCTL_CNI_BUNDLE_INSTALL=$(PKG_TALOSCTL_CNI_BUNDLE_INSTALL)
COMMON_ARGS += --build-arg=ABBREV_TAG=$(ABBREV_TAG)
COMMON_ARGS += --build-arg=ZSTD_COMPRESSION_LEVEL=$(ZSTD_COMPRESSION_LEVEL)
CI_ARGS ?=
@ -524,6 +526,12 @@ provision-tests-track-%:
REGISTRY=$(IMAGE_REGISTRY) \
ARTIFACTS=$(ARTIFACTS)
installer-with-extensions: $(ARTIFACTS)/extensions-metadata
$(MAKE) image-installer \
IMAGER_ARGS="--base-installer-image=$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG) $(shell cat $(ARTIFACTS)/extensions-metadata | grep -vE 'tailscale|xen-guest-agent|nvidia' | xargs -n 1 echo --system-extension-image)"
crane push $(ARTIFACTS)/installer-amd64.tar $(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions
echo -n "$(REGISTRY_AND_USERNAME)/installer:$(IMAGE_TAG)-amd64-extensions" | jq -Rs -f hack/test/extensions/extension-patch-filter.jq | yq eval ".[] | split_doc" -P > $(ARTIFACTS)/extensions-patch.yaml
# Assets for releases
.PHONY: $(ARTIFACTS)/$(TALOS_RELEASE)

View File

@ -11,6 +11,7 @@ import (
"github.com/siderolabs/talos/pkg/imager/extensions"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
)
func (i *Installer) installExtensions() error {
@ -19,6 +20,7 @@ func (i *Installer) installExtensions() error {
Arch: i.options.Arch,
ExtensionTreePath: constants.SystemExtensionsPath,
Printf: log.Printf,
Quirks: quirks.New(i.options.Version),
}
return builder.Build()

2
go.mod
View File

@ -92,6 +92,7 @@ require (
github.com/jeromer/syslogparser v1.1.0
github.com/jsimonetti/rtnetlink v1.4.1
github.com/jxskiss/base62 v1.1.0
github.com/klauspost/compress v1.17.7
github.com/klauspost/cpuid/v2 v2.2.7
github.com/linode/go-metadata v0.2.0
github.com/martinlindhe/base36 v1.1.1
@ -267,7 +268,6 @@ require (
github.com/josharian/intern v1.0.0 // indirect
github.com/josharian/native v1.1.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/kylelemons/godebug v1.1.0 // indirect
github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
github.com/lucasb-eyer/go-colorful v1.2.0 // indirect

View File

@ -6,7 +6,7 @@ github_repo = "siderolabs/talos"
match_deps = "^github.com/((talos-systems|siderolabs)/[a-zA-Z0-9-]+)$"
# previous release
previous = "v1.6.0"
previous = "v1.7.0"
pre_release = true
@ -17,221 +17,15 @@ preface = """\
[notes.updates]
title = "Component Updates"
description = """\
Linux: 6.6.28
etcd: 3.5.11
Kubernetes: 1.30.0
containerd: 1.7.15
runc: 1.1.12
Flannel: 0.25.1
Talos is built with Go 1.22.2.
"""
[notes.device_selectors]
title = "Device Selectors"
[notes.zstd]
title = "ZSTD Compression"
description = """\
Talos Linux now supports `physical: true` qualifier for device selectors, it selects non-virtual network interfaces (i.e. `en0` is selected, while `bond0` is not).
"""
[notes.dns-resolve-cache]
title = "DNS Caching"
description = """\
Talos Linux now provides a caching DNS resolver for host workloads (including host networking pods). It can be disabled with:
```yaml
machine:
features:
hostDNS:
enabled: false
```
You can also enable dns caching for k8s pods with:
```yaml
machine:
features:
hostDNS:
enabled: true
forwardKubeDNSToHost: true
```
Please note that on running cluster you will have to kill CoreDNS pods for this change to apply.
If you want to can also enable the resolving of member addresses through their host and node names:
```yaml
machine:
features:
hostDNS:
enabled: true
resolveMemberNames: true
```
"""
[notes.secureboot-image]
title = "Secure Boot Image"
description = """\
Talos Linux now provides a way to configure systemd-boot ISO 'secure-boot-enroll' option while generating a SecureBoot ISO image:
```yaml
output:
kind: iso
isoOptions:
sdBootEnrollKeys: force # default is still if-safe
outFormat: raw
```
"""
[notes.rsa-service-account]
title = "Kubernetes API Server Service Account Key"
description = """\
Talos Linux starting from this release uses RSA key for Kubernetes API Server Service Account instead of ECDSA key to provide better compatibility with external OpenID Connect implementations.
"""
[notes.opennebula]
title = "OpenNebula"
description = """\
Talos Linux now supports OpenNebula platform.
"""
[notes.extensions]
title = "Extension Services Config"
description = """\
Talos now supports supplying configuration files and environment variables for extension services.
The extension service configuration is a separate config document. An example is shown below:
```yaml
---
apiVersion: v1alpha1
kind: ExtensionServiceConfig
name: nut-client
configFiles:
- content: MONITOR ${upsmonHost} 1 remote pass password
mountPath: /usr/local/etc/nut/upsmon.conf
environment:
- UPS_NAME=ups
```
For documentation, see [Extension Services Config Files](https://www.talos.dev/v1.7/reference/configuration/extensions/extensionserviceconfig/).
**Note**: The use of `environmentFile` in extension service spec is now deprecated and will be removed in a future release of Talos.
Use `ExtensionServiceConfig` instead.
"""
[notes.k8supgrade]
title = "Kubernetes Upgrade"
description = """\
The command `talosctl upgrade-k8s` now supports specifying custom image references for Kubernetes components via `--*-image` flags.
The default behavior is unchanged, and the flags are optional.
"""
[notes.kubespan]
title = "KubeSpan"
description = """\
Talos Linux disables by default a KubeSpan feature to harvest additional endpoints from KubeSpan members.
This feature turned out to be less helpful than expected and caused unnecessary performance issues.
Previous behavior can be restored with:
```yaml
machine:
network:
kubespan:
harvestExtraEndpoints: true
```
"""
[notes.sbc]
title = "SBC"
description = """\
Talos has split the SBC's (Single Board Computers) into separate repositories.
There will not be any more SBC specific release assets as part of Talos release.
The default Talos Installer image will stop working for SBC's and will fail the upgrade, if used, starting from Talos v1.7.0.
The SBC's images and installers can be generated on the fly using [Image Factory](https://factory.talos.dev) or using [Imager](https://www.talos.dev/latest/talos-guides/install/boot-assets/) for custom images.
The list of official SBC's images supported by Image Factory can be found in the [Overlays](https://github.com/siderolabs/overlays/) repository.
"""
[notes.syslog]
title = "Syslog"
description = """\
Talos Linux now starts a basic syslog receiver listening on `/dev/log`.
The receiver can mostly parse both RFC3164 and RFC5424 messages and writes them as JSON formatted message.
The logs can be viewed via `talosctl logs syslogd`.
This is mostly implemented for extension services that log to syslog.
"""
[notes.ntp]
title = "Time Sync"
description = """\
Default NTP server was updated to be `time.cloudflare.com` instead of `pool.ntp.org`.
Default server is only used if the user does not specify any NTP servers in the configuration.
Talos Linux can now sync to PTP devices (e.g. provided by the hypervisor) skipping the network time servers.
In order to activate PTP sync, set `machine.time.servers` to the PTP device name (e.g. `/dev/ptp0`):
```yaml
machine:
time:
servers:
- /dev/ptp0
```
"""
[notes.ca-rotation]
title = "CA Rotation"
description = """\
Talos Linux now supports rotating the root CA certificate and key for Talos API and Kubernetes API.
"""
[notes.watchdog]
title = "Hardware Watchdog Timers"
description = """\
Talos Linux now supports hardware watchdog timers configuration.
If enabled, and the machine becomes unresponsive, the hardware watchdog will reset the machine.
The watchdog can be enabled with the following configuration document:
```yaml
apiVersion: v1alpha1
kind: WatchdogTimerConfig
device: /dev/watchdog0
timeout: 3m0s
```
"""
[notes.logging]
title = "Logging"
description = """\
Talos Linux now supports setting extra tags when sending logs in JSON format:
```yaml
machine:
logging:
destinations:
- endpoint: "udp://127.0.0.1:12345/"
format: "json_lines"
extraTags:
server: s03-rack07
```
"""
[notes.platforms]
title = "Platforms"
description = """\
Talos Linux now supports [Akamai Connected Cloud](https://www.linode.com/) provider (platform `akamai`).
"""
[notes.iptables]
title = "IPTables"
description = """\
Talos Linux now forces `kubelet` and `kube-proxy` to use `iptables-nft` instead of `iptables-legacy` (`xtables`) which was the default
before Talos 1.7.0.
Container images based on `iptables-wrapper` should work without changes, but if there was a direct call to `legacy` mode of `iptables`, make sure
to update to use `iptables-nft`.
Talos Linux now compresses kernel and initramfs using ZSTD.
Linux arm64 kernel is now compressed (previously it was uncompressed).
"""
[make_deps]

View File

@ -2,11 +2,7 @@
{
"machine": {
"install": {
"extensions": [
{
"image": map(select(. | contains("nvidia") or contains("tailscale") or contains("xen-guest-agent") | not)) | .[]
}
],
"image": .
},
"sysctls": {
"user.max_user_namespaces": "11255"

View File

@ -13,6 +13,7 @@ import (
"path/filepath"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
)
// List of globs and destinations for early CPU ucode.
@ -38,7 +39,7 @@ var initramfsPaths = []string{
//
// Components which should be placed to the initramfs are moved to the initramfsPath.
// Ucode components are moved into a separate designated location.
func (ext *Extension) Compress(squashPath, initramfsPath string) (string, error) {
func (ext *Extension) Compress(squashPath, initramfsPath string, quirks quirks.Quirks) (string, error) {
if err := ext.handleUcode(initramfsPath); err != nil {
return "", err
}
@ -53,7 +54,15 @@ func (ext *Extension) Compress(squashPath, initramfsPath string) (string, error)
squashPath = filepath.Join(squashPath, fmt.Sprintf("%s.sqsh", ext.directory))
cmd := exec.Command("mksquashfs", ext.rootfsPath, squashPath, "-all-root", "-noappend", "-comp", "xz", "-Xdict-size", "100%", "-no-progress")
var compressArgs []string
if quirks.UseZSTDCompression() {
compressArgs = []string{"-comp", "zstd", "-Xcompression-level", "18"}
} else {
compressArgs = []string{"-comp", "xz", "-Xdict-size", "100%"}
}
cmd := exec.Command("mksquashfs", append([]string{ext.rootfsPath, squashPath, "-all-root", "-noappend", "-no-progress"}, compressArgs...)...)
cmd.Stderr = os.Stderr
return squashPath, cmd.Run()

View File

@ -13,6 +13,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/siderolabs/talos/internal/pkg/extensions"
"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
"github.com/siderolabs/talos/pkg/machinery/version"
)
@ -43,7 +44,7 @@ func TestCompress(t *testing.T) {
require.NoError(t, err)
squashDest, initramfsDest := t.TempDir(), t.TempDir()
squashFile, err := ext.Compress(squashDest, initramfsDest)
squashFile, err := ext.Compress(squashDest, initramfsDest, quirks.New(""))
assert.NoError(t, err)
assert.FileExists(t, squashFile)

View File

@ -6,6 +6,7 @@
package extensions
import (
"bytes"
"errors"
"fmt"
"io"
@ -15,6 +16,7 @@ import (
"path/filepath"
"strings"
"github.com/klauspost/compress/zstd"
"github.com/u-root/u-root/pkg/cpio"
"github.com/ulikunitz/xz"
@ -36,6 +38,23 @@ func (ext *Extension) KernelModuleDirectory() string {
return filepath.Join(ext.rootfsPath, constants.KernelModulesPath)
}
func autoDecompress(r io.Reader) (io.Reader, error) {
var magic [4]byte
if _, err := r.Read(magic[:]); err != nil {
return nil, err
}
src := io.MultiReader(bytes.NewReader(magic[:]), r)
// xz magic
if bytes.Equal(magic[:], []byte{0xfd, '7', 'z', 'X'}) {
return xz.NewReader(src)
}
return zstd.NewReader(src)
}
// GenerateKernelModuleDependencyTreeExtension generates a kernel module dependency tree extension.
//
//nolint:gocyclo
@ -60,7 +79,7 @@ func GenerateKernelModuleDependencyTreeExtension(extensionPathsWithKernelModules
return initramfsxz.Close()
})
r, err := xz.NewReader(initramfsxz)
r, err := autoDecompress(initramfsxz)
if err != nil {
return nil, err
}

View File

@ -13,6 +13,7 @@ import (
"github.com/siderolabs/talos/internal/pkg/extensions"
"github.com/siderolabs/talos/pkg/machinery/constants"
extinterface "github.com/siderolabs/talos/pkg/machinery/extensions"
"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
)
// Builder rebuilds initramfs.xz with extensions.
@ -25,6 +26,8 @@ type Builder struct {
ExtensionTreePath string
// Printf is used for logging.
Printf func(format string, v ...any)
// Quirks for the Talos version being used.
Quirks quirks.Quirks
}
// Build rebuilds the initramfs.xz with extensions.
@ -86,7 +89,7 @@ func (builder *Builder) Build() error {
return err
}
return builder.rebuildInitramfs(tempDir)
return builder.rebuildInitramfs(tempDir, builder.Quirks)
}
func (builder *Builder) validateExtensions(extensions []*extensions.Extension) error {
@ -107,7 +110,7 @@ func (builder *Builder) compressExtensions(extensions []*extensions.Extension, t
builder.Printf("compressing system extensions")
for _, ext := range extensions {
path, err := ext.Compress(tempDir, tempDir)
path, err := ext.Compress(tempDir, tempDir, builder.Quirks)
if err != nil {
return nil, fmt.Errorf("error compressing extension %q: %w", ext.Manifest.Metadata.Name, err)
}

View File

@ -10,14 +10,16 @@ import (
"io"
"os"
"os/exec"
"github.com/siderolabs/talos/pkg/machinery/imager/quirks"
)
// rebuildInitramfs rebuilds finalized initramfs with extensions.
//
// If uncompressedListing is not empty, contents will be prepended to the initramfs uncompressed.
// Contents from compressedListing will be appended to the initramfs compressed (xz) as a second block.
// Contents from compressedListing will be appended to the initramfs compressed (xz/zstd) as a second block.
// Original initramfs.xz contents will stay without changes.
func (builder *Builder) rebuildInitramfs(tempDir string) error {
func (builder *Builder) rebuildInitramfs(tempDir string, quirks quirks.Quirks) error {
compressedListing, uncompressedListing, err := buildInitramfsContents(tempDir)
if err != nil {
return err
@ -29,18 +31,18 @@ func (builder *Builder) rebuildInitramfs(tempDir string) error {
}
}
if err = builder.appendCompressedInitramfs(tempDir, compressedListing); err != nil {
if err = builder.appendCompressedInitramfs(tempDir, compressedListing, quirks); err != nil {
return fmt.Errorf("error appending compressed initramfs: %w", err)
}
return nil
}
func (builder *Builder) appendCompressedInitramfs(tempDir string, compressedListing []byte) error {
func (builder *Builder) appendCompressedInitramfs(tempDir string, compressedListing []byte, quirks quirks.Quirks) error {
builder.Printf("creating system extensions initramfs archive and compressing it")
// the code below runs the equivalent of:
// find $tempDir -print | cpio -H newc --create --reproducible | xz -v -C crc32 -0 -e -T 0 -z
// find $tempDir -print | cpio -H newc --create --reproducible | { xz -v -C crc32 -0 -e -T 0 -z || zstd -T0 -18 -c --quiet }
pipeR, pipeW, err := os.Pipe()
if err != nil {
@ -73,7 +75,14 @@ func (builder *Builder) appendCompressedInitramfs(tempDir string, compressedList
defer destination.Close() //nolint:errcheck
// append compressed initramfs.sysext to the original initramfs.xz, kernel can read such format
cmd2 := exec.Command("xz", "-v", "-C", "crc32", "-0", "-e", "-T", "0", "-z", "--quiet")
var cmd2 *exec.Cmd
if quirks.UseZSTDCompression() {
cmd2 = exec.Command("zstd", "-T0", "-18", "-c", "--quiet")
} else {
cmd2 = exec.Command("xz", "-v", "-C", "crc32", "-0", "-e", "-T", "0", "-z", "--quiet")
}
cmd2.Dir = tempDir
cmd2.Stdin = pipeR
cmd2.Stdout = destination

View File

@ -152,6 +152,8 @@ func (i *Imager) Execute(ctx context.Context, outputPath string, report *reporte
return i.postProcessXz(outputAssetPath, report)
case profile.OutFormatGZ:
return i.postProcessGz(outputAssetPath, report)
case profile.OutFormatZSTD:
return i.postProcessZstd(outputAssetPath, report)
case profile.OutFormatTar:
return i.postProcessTar(outputAssetPath, report)
case profile.OutFormatUnknown:
@ -305,6 +307,7 @@ func (i *Imager) buildInitramfs(ctx context.Context, report *reporter.Reporter)
Arch: i.prof.Arch,
ExtensionTreePath: extensionsCheckoutDir,
Printf: printf,
Quirks: quirks.New(i.prof.Version),
}
if err := builder.Build(); err != nil {

View File

@ -62,3 +62,17 @@ func (i *Imager) postProcessXz(filename string, report *reporter.Reporter) (stri
return filename + ".xz", nil
}
func (i *Imager) postProcessZstd(filename string, report *reporter.Reporter) (string, error) {
report.Report(reporter.Update{Message: "compressing .zst", Status: reporter.StatusRunning})
out := filename + ".zst"
if _, err := cmd.Run("zstd", "-T0", "--rm", "-18", "--quiet", "--force", "-o", out, filename); err != nil {
return "", err
}
report.Report(reporter.Update{Message: fmt.Sprintf("compression done: %s", out), Status: reporter.StatusSucceeded})
return filename + ".zst", nil
}

View File

@ -48,7 +48,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -60,7 +60,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(true),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -101,7 +101,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: DefaultRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -113,7 +113,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: DefaultRAWDiskSize,
DiskFormat: DiskFormatVPC,
@ -138,7 +138,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: 10 * 1024 * mib,
DiskFormat: DiskFormatQCOW2,
@ -163,7 +163,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -175,7 +175,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -187,7 +187,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -199,7 +199,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -211,7 +211,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: DefaultRAWDiskSize,
DiskFormat: DiskFormatQCOW2,
@ -224,7 +224,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -236,7 +236,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: DefaultRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -260,7 +260,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: DefaultRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -275,7 +275,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -289,7 +289,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -303,7 +303,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -317,7 +317,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -331,7 +331,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -345,7 +345,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -359,7 +359,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -373,7 +373,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,
@ -387,7 +387,7 @@ var Default = map[string]Profile{
SecureBoot: pointer.To(false),
Output: Output{
Kind: OutKindImage,
OutFormat: OutFormatXZ,
OutFormat: OutFormatZSTD,
ImageOptions: &ImageOptions{
DiskSize: MinRAWDiskSize,
DiskFormat: DiskFormatRaw,

View File

@ -7,11 +7,11 @@ import (
"strings"
)
const _OutFormatName = "unknownraw.tar.gz.xz.gz"
const _OutFormatName = "unknownraw.tar.gz.xz.gz.zst"
var _OutFormatIndex = [...]uint8{0, 7, 10, 17, 20, 23}
var _OutFormatIndex = [...]uint8{0, 7, 10, 17, 20, 23, 27}
const _OutFormatLowerName = "unknownraw.tar.gz.xz.gz"
const _OutFormatLowerName = "unknownraw.tar.gz.xz.gz.zst"
func (i OutFormat) String() string {
if i < 0 || i >= OutFormat(len(_OutFormatIndex)-1) {
@ -29,9 +29,10 @@ func _OutFormatNoOp() {
_ = x[OutFormatTar-(2)]
_ = x[OutFormatXZ-(3)]
_ = x[OutFormatGZ-(4)]
_ = x[OutFormatZSTD-(5)]
}
var _OutFormatValues = []OutFormat{OutFormatUnknown, OutFormatRaw, OutFormatTar, OutFormatXZ, OutFormatGZ}
var _OutFormatValues = []OutFormat{OutFormatUnknown, OutFormatRaw, OutFormatTar, OutFormatXZ, OutFormatGZ, OutFormatZSTD}
var _OutFormatNameToValueMap = map[string]OutFormat{
_OutFormatName[0:7]: OutFormatUnknown,
@ -44,6 +45,8 @@ var _OutFormatNameToValueMap = map[string]OutFormat{
_OutFormatLowerName[17:20]: OutFormatXZ,
_OutFormatName[20:23]: OutFormatGZ,
_OutFormatLowerName[20:23]: OutFormatGZ,
_OutFormatName[23:27]: OutFormatZSTD,
_OutFormatLowerName[23:27]: OutFormatZSTD,
}
var _OutFormatNames = []string{
@ -52,6 +55,7 @@ var _OutFormatNames = []string{
_OutFormatName[10:17],
_OutFormatName[17:20],
_OutFormatName[20:23],
_OutFormatName[23:27],
}
// OutFormatString retrieves an enum value from the enum constants string name.

View File

@ -76,6 +76,7 @@ const (
OutFormatTar // .tar.gz
OutFormatXZ // .xz
OutFormatGZ // .gz
OutFormatZSTD // .zst
)
//go:generate enumer -type DiskFormat -linecomment -text

View File

@ -1 +1 @@
v1.8.0-alpha.0-7-g718a7da
v1.8.0-alpha.0-8-gca6249b

View File

@ -62,3 +62,15 @@ func (q Quirks) SupportsOverlay() bool {
return q.v.GTE(minVersionOverlay)
}
var minVersionZstd = semver.MustParse("1.8.0")
// UseZSTDCompression returns true if the Talos should use zstd compression in place of xz.
func (q Quirks) UseZSTDCompression() bool {
// if the version doesn't parse, we assume it's latest Talos
if q.v == nil {
return true
}
return q.v.GTE(minVersionZstd)
}

View File

@ -99,3 +99,35 @@ func TestSupportsOverlay(t *testing.T) {
})
}
}
func TestSupportsZstd(t *testing.T) {
for _, test := range []struct {
version string
expected bool
}{
{
version: "1.7.3",
expected: false,
},
{
expected: true,
},
{
version: "1.6.2",
expected: false,
},
{
version: "1.8.0-alpha.0",
expected: true,
},
{
version: "v1.8.3",
expected: true,
},
} {
t.Run(test.version, func(t *testing.T) {
assert.Equal(t, test.expected, quirks.New(test.version).UseZSTDCompression())
})
}
}