fix: allow META encoded values to be compressed

Fixes #8186

This is planned to be backported to Talos 1.6.3.

This allows to pass large META values (YAML for platform network
configuration) which might otherwise exceed the limit for kernel
command line params.

Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

cmd/installer/pkg/install/meta_value.go

@@ -103,7 +103,7 @@ func (s *MetaValues) GetSlice() []string {
 
 // Encode returns the encoded values.
 func (s *MetaValues) Encode() string {
-	return s.values.Encode()
+	return s.values.Encode(false)
 }
 
 // Decode the values from the given string.

pkg/imager/imager.go

@@ -20,6 +20,7 @@ import (
 	"github.com/siderolabs/talos/internal/pkg/secureboot/uki"
 	"github.com/siderolabs/talos/pkg/imager/extensions"
 	"github.com/siderolabs/talos/pkg/imager/profile"
+	"github.com/siderolabs/talos/pkg/imager/quirks"
 	"github.com/siderolabs/talos/pkg/imager/utils"
 	"github.com/siderolabs/talos/pkg/machinery/config/merge"
 	"github.com/siderolabs/talos/pkg/machinery/constants"
@@ -276,7 +277,10 @@ func (i *Imager) buildCmdline() error {
 	// meta values can be written only to the "image" output
 	if len(i.prof.Customization.MetaContents) > 0 && i.prof.Output.Kind != profile.OutKindImage {
 		// pass META values as kernel talos.environment args which will be passed via the environment to the installer
-		cmdline.Append(constants.KernelParamEnvironment, constants.MetaValuesEnvVar+"="+i.prof.Customization.MetaContents.Encode())
+		cmdline.Append(
+			constants.KernelParamEnvironment,
+			constants.MetaValuesEnvVar+"="+i.prof.Customization.MetaContents.Encode(quirks.New(i.prof.Version).SupportsCompressedEncodedMETA()),
+		)
 	}
 
 	// apply customization

pkg/imager/quirks/quirks.go

@@ -33,3 +33,15 @@ func (q Quirks) SupportsResetGRUBOption() bool {
 
 	return q.v.GTE(minVersionResetOption)
 }
+
+var minVersionCompressedMETA = semver.MustParse("1.6.3")
+
+// SupportsCompressedEncodedMETA returns true if the Talos version supports compressed and encoded META as an environment variable.
+func (q Quirks) SupportsCompressedEncodedMETA() bool {
+	// if the version doesn't parse, we assume it's latest Talos
+	if q.v == nil {
+		return true
+	}
+
+	return q.v.GTE(minVersionCompressedMETA)
+}

pkg/imager/quirks/quirks_test.go

@@ -35,3 +35,31 @@ func TestSupportsResetOption(t *testing.T) {
 		})
 	}
 }
+
+func TestSupportsCompressedEncodedMETA(t *testing.T) {
+	for _, test := range []struct {
+		version string
+
+		expected bool
+	}{
+		{
+			version:  "1.6.3",
+			expected: true,
+		},
+		{
+			version:  "1.7.0",
+			expected: true,
+		},
+		{
+			expected: true,
+		},
+		{
+			version:  "1.6.2",
+			expected: false,
+		},
+	} {
+		t.Run(test.version, func(t *testing.T) {
+			assert.Equal(t, test.expected, quirks.New(test.version).SupportsCompressedEncodedMETA())
+		})
+	}
+}

pkg/machinery/meta/meta.go

@@ -6,8 +6,11 @@
 package meta
 
 import (
+	"bytes"
+	"compress/gzip"
 	"encoding/base64"
 	"fmt"
+	"io"
 	"strconv"
 	"strings"
 
@@ -49,8 +52,29 @@ type Values []Value
 //
 // Each Value is encoded a k=v, split by ';' character.
 // The result is base64 encoded.
-func (v Values) Encode() string {
-	return base64.StdEncoding.EncodeToString([]byte(strings.Join(xslices.Map(v, Value.String), ";")))
+func (v Values) Encode(allowGzip bool) string {
+	raw := []byte(strings.Join(xslices.Map(v, Value.String), ";"))
+
+	if allowGzip && len(raw) > 256 {
+		var buf bytes.Buffer
+
+		gzW, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+		if err != nil {
+			panic(err)
+		}
+
+		if _, err := gzW.Write(raw); err != nil {
+			panic(err)
+		}
+
+		if err := gzW.Close(); err != nil {
+			panic(err)
+		}
+
+		raw = buf.Bytes()
+	}
+
+	return base64.StdEncoding.EncodeToString(raw)
 }
 
 // DecodeValues parses a string representation of Values for the environment variable.
@@ -66,6 +90,25 @@ func DecodeValues(s string) (Values, error) {
 		return nil, nil
 	}
 
+	// do un-gzip if needed
+	if hasGzipMagic(b) {
+		gzR, err := gzip.NewReader(bytes.NewReader(b))
+		if err != nil {
+			return nil, err
+		}
+
+		defer gzR.Close() //nolint:errcheck
+
+		b, err = io.ReadAll(gzR)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := gzR.Close(); err != nil {
+			return nil, err
+		}
+	}
+
 	parts := strings.Split(string(b), ";")
 
 	result := make(Values, 0, len(parts))
@@ -82,3 +125,12 @@ func DecodeValues(s string) (Values, error) {
 
 	return result, nil
 }
+
+func hasGzipMagic(b []byte) bool {
+	if len(b) < 10 {
+		return false
+	}
+
+	// See https://en.wikipedia.org/wiki/Gzip#File_format.
+	return b[0] == 0x1f && b[1] == 0x8b
+}

pkg/machinery/meta/meta_test.go

@@ -5,6 +5,8 @@
 package meta_test
 
 import (
+	"fmt"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -35,15 +37,74 @@ func TestValue(t *testing.T) {
 func TestEncodeDecodeValues(t *testing.T) {
 	t.Parallel()
 
-	values := make(meta.Values, 2)
+	for _, allowGzip := range []bool{false, true} {
+		allowGzip := allowGzip
 
-	require.NoError(t, values[0].Parse("10=foo"))
-	require.NoError(t, values[1].Parse("0xb=bar"))
+		t.Run(fmt.Sprintf("allowGzip=%v", allowGzip), func(t *testing.T) {
+			t.Parallel()
 
-	encoded := values.Encode()
+			for _, test := range []struct {
+				name string
 
-	decoded, err := meta.DecodeValues(encoded)
-	require.NoError(t, err)
+				values []string
 
-	assert.Equal(t, values, decoded)
+				expectedEncodedSize int
+				expectedGzippedSize int
+			}{
+				{
+					name: "empty",
+				},
+				{
+					name: "simple",
+					values: []string{
+						"10=foo",
+						"0xb=bar",
+					},
+
+					expectedEncodedSize: 20,
+					expectedGzippedSize: 20,
+				},
+				{
+					name: "huge",
+					values: []string{
+						"10=" + strings.Repeat("foobar", 256),
+						"0xb=" + strings.Repeat("baz", 256),
+					},
+
+					expectedEncodedSize: 3084,
+					expectedGzippedSize: 80,
+				},
+			} {
+				test := test
+
+				t.Run(test.name, func(t *testing.T) {
+					t.Parallel()
+
+					values := make(meta.Values, len(test.values))
+
+					for i, v := range test.values {
+						require.NoError(t, values[i].Parse(v))
+					}
+
+					if len(values) == 0 {
+						values = nil
+					}
+
+					encoded := values.Encode(allowGzip)
+
+					switch {
+					case test.expectedEncodedSize > 0 && !allowGzip:
+						assert.Equal(t, test.expectedEncodedSize, len(encoded))
+					case test.expectedGzippedSize > 0 && allowGzip:
+						assert.Equal(t, test.expectedGzippedSize, len(encoded))
+					}
+
+					decoded, err := meta.DecodeValues(encoded)
+					require.NoError(t, err)
+
+					assert.Equal(t, values, decoded)
+				})
+			}
+		})
+	}
 }

website/content/v1.7/advanced/metal-network-configuration.md

@@ -403,7 +403,7 @@ kernel command line: ... talos.environment=INSTALLER_META_BASE64=MHhhPWZvbw==
 When PXE booting, the value of `INSTALLER_META_BASE64` should be set manually:
 
 ```bash
-echo -n "0xa=$(cat network.yaml)" | base64
+echo -n "0xa=$(cat network.yaml)" | gzip -9 | base64
 ```
 
 The resulting base64 string should be passed as an environment variable `INSTALLER_META_BASE64` to the initial boot of Talos: `talos.environment=INSTALLER_META_BASE64=<base64-encoded value>`.