From e0dfbb8fba3c50652d0ecbae1db0b0660d0766a6 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 22 Jan 2024 17:48:04 +0400 Subject: [PATCH] fix: allow META encoded values to be compressed Fixes #8186 This is planned to be backported to Talos 1.6.3. This allows to pass large META values (YAML for platform network configuration) which might otherwise exceed the limit for kernel command line params. Signed-off-by: Andrey Smirnov --- cmd/installer/pkg/install/meta_value.go | 2 +- pkg/imager/imager.go | 6 +- pkg/imager/quirks/quirks.go | 12 +++ pkg/imager/quirks/quirks_test.go | 28 +++++++ pkg/machinery/meta/meta.go | 56 +++++++++++++- pkg/machinery/meta/meta_test.go | 75 +++++++++++++++++-- .../advanced/metal-network-configuration.md | 2 +- 7 files changed, 169 insertions(+), 12 deletions(-) diff --git a/cmd/installer/pkg/install/meta_value.go b/cmd/installer/pkg/install/meta_value.go index 534b44779..32ef01f75 100644 --- a/cmd/installer/pkg/install/meta_value.go +++ b/cmd/installer/pkg/install/meta_value.go @@ -103,7 +103,7 @@ func (s *MetaValues) GetSlice() []string { // Encode returns the encoded values. func (s *MetaValues) Encode() string { - return s.values.Encode() + return s.values.Encode(false) } // Decode the values from the given string. diff --git a/pkg/imager/imager.go b/pkg/imager/imager.go index 2911c494e..55024151e 100644 --- a/pkg/imager/imager.go +++ b/pkg/imager/imager.go @@ -20,6 +20,7 @@ import ( "github.com/siderolabs/talos/internal/pkg/secureboot/uki" "github.com/siderolabs/talos/pkg/imager/extensions" "github.com/siderolabs/talos/pkg/imager/profile" + "github.com/siderolabs/talos/pkg/imager/quirks" "github.com/siderolabs/talos/pkg/imager/utils" "github.com/siderolabs/talos/pkg/machinery/config/merge" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -276,7 +277,10 @@ func (i *Imager) buildCmdline() error { // meta values can be written only to the "image" output if len(i.prof.Customization.MetaContents) > 0 && i.prof.Output.Kind != profile.OutKindImage { // pass META values as kernel talos.environment args which will be passed via the environment to the installer - cmdline.Append(constants.KernelParamEnvironment, constants.MetaValuesEnvVar+"="+i.prof.Customization.MetaContents.Encode()) + cmdline.Append( + constants.KernelParamEnvironment, + constants.MetaValuesEnvVar+"="+i.prof.Customization.MetaContents.Encode(quirks.New(i.prof.Version).SupportsCompressedEncodedMETA()), + ) } // apply customization diff --git a/pkg/imager/quirks/quirks.go b/pkg/imager/quirks/quirks.go index 63768e000..9d1c40738 100644 --- a/pkg/imager/quirks/quirks.go +++ b/pkg/imager/quirks/quirks.go @@ -33,3 +33,15 @@ func (q Quirks) SupportsResetGRUBOption() bool { return q.v.GTE(minVersionResetOption) } + +var minVersionCompressedMETA = semver.MustParse("1.6.3") + +// SupportsCompressedEncodedMETA returns true if the Talos version supports compressed and encoded META as an environment variable. +func (q Quirks) SupportsCompressedEncodedMETA() bool { + // if the version doesn't parse, we assume it's latest Talos + if q.v == nil { + return true + } + + return q.v.GTE(minVersionCompressedMETA) +} diff --git a/pkg/imager/quirks/quirks_test.go b/pkg/imager/quirks/quirks_test.go index 852325e65..fbf85a9dc 100644 --- a/pkg/imager/quirks/quirks_test.go +++ b/pkg/imager/quirks/quirks_test.go @@ -35,3 +35,31 @@ func TestSupportsResetOption(t *testing.T) { }) } } + +func TestSupportsCompressedEncodedMETA(t *testing.T) { + for _, test := range []struct { + version string + + expected bool + }{ + { + version: "1.6.3", + expected: true, + }, + { + version: "1.7.0", + expected: true, + }, + { + expected: true, + }, + { + version: "1.6.2", + expected: false, + }, + } { + t.Run(test.version, func(t *testing.T) { + assert.Equal(t, test.expected, quirks.New(test.version).SupportsCompressedEncodedMETA()) + }) + } +} diff --git a/pkg/machinery/meta/meta.go b/pkg/machinery/meta/meta.go index 4e27db571..0a802bb81 100644 --- a/pkg/machinery/meta/meta.go +++ b/pkg/machinery/meta/meta.go @@ -6,8 +6,11 @@ package meta import ( + "bytes" + "compress/gzip" "encoding/base64" "fmt" + "io" "strconv" "strings" @@ -49,8 +52,29 @@ type Values []Value // // Each Value is encoded a k=v, split by ';' character. // The result is base64 encoded. -func (v Values) Encode() string { - return base64.StdEncoding.EncodeToString([]byte(strings.Join(xslices.Map(v, Value.String), ";"))) +func (v Values) Encode(allowGzip bool) string { + raw := []byte(strings.Join(xslices.Map(v, Value.String), ";")) + + if allowGzip && len(raw) > 256 { + var buf bytes.Buffer + + gzW, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + panic(err) + } + + if _, err := gzW.Write(raw); err != nil { + panic(err) + } + + if err := gzW.Close(); err != nil { + panic(err) + } + + raw = buf.Bytes() + } + + return base64.StdEncoding.EncodeToString(raw) } // DecodeValues parses a string representation of Values for the environment variable. @@ -66,6 +90,25 @@ func DecodeValues(s string) (Values, error) { return nil, nil } + // do un-gzip if needed + if hasGzipMagic(b) { + gzR, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + defer gzR.Close() //nolint:errcheck + + b, err = io.ReadAll(gzR) + if err != nil { + return nil, err + } + + if err := gzR.Close(); err != nil { + return nil, err + } + } + parts := strings.Split(string(b), ";") result := make(Values, 0, len(parts)) @@ -82,3 +125,12 @@ func DecodeValues(s string) (Values, error) { return result, nil } + +func hasGzipMagic(b []byte) bool { + if len(b) < 10 { + return false + } + + // See https://en.wikipedia.org/wiki/Gzip#File_format. + return b[0] == 0x1f && b[1] == 0x8b +} diff --git a/pkg/machinery/meta/meta_test.go b/pkg/machinery/meta/meta_test.go index 183ac4c24..c7dbb4ae5 100644 --- a/pkg/machinery/meta/meta_test.go +++ b/pkg/machinery/meta/meta_test.go @@ -5,6 +5,8 @@ package meta_test import ( + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -35,15 +37,74 @@ func TestValue(t *testing.T) { func TestEncodeDecodeValues(t *testing.T) { t.Parallel() - values := make(meta.Values, 2) + for _, allowGzip := range []bool{false, true} { + allowGzip := allowGzip - require.NoError(t, values[0].Parse("10=foo")) - require.NoError(t, values[1].Parse("0xb=bar")) + t.Run(fmt.Sprintf("allowGzip=%v", allowGzip), func(t *testing.T) { + t.Parallel() - encoded := values.Encode() + for _, test := range []struct { + name string - decoded, err := meta.DecodeValues(encoded) - require.NoError(t, err) + values []string - assert.Equal(t, values, decoded) + expectedEncodedSize int + expectedGzippedSize int + }{ + { + name: "empty", + }, + { + name: "simple", + values: []string{ + "10=foo", + "0xb=bar", + }, + + expectedEncodedSize: 20, + expectedGzippedSize: 20, + }, + { + name: "huge", + values: []string{ + "10=" + strings.Repeat("foobar", 256), + "0xb=" + strings.Repeat("baz", 256), + }, + + expectedEncodedSize: 3084, + expectedGzippedSize: 80, + }, + } { + test := test + + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + values := make(meta.Values, len(test.values)) + + for i, v := range test.values { + require.NoError(t, values[i].Parse(v)) + } + + if len(values) == 0 { + values = nil + } + + encoded := values.Encode(allowGzip) + + switch { + case test.expectedEncodedSize > 0 && !allowGzip: + assert.Equal(t, test.expectedEncodedSize, len(encoded)) + case test.expectedGzippedSize > 0 && allowGzip: + assert.Equal(t, test.expectedGzippedSize, len(encoded)) + } + + decoded, err := meta.DecodeValues(encoded) + require.NoError(t, err) + + assert.Equal(t, values, decoded) + }) + } + }) + } } diff --git a/website/content/v1.7/advanced/metal-network-configuration.md b/website/content/v1.7/advanced/metal-network-configuration.md index db4384c56..bbbab45c7 100644 --- a/website/content/v1.7/advanced/metal-network-configuration.md +++ b/website/content/v1.7/advanced/metal-network-configuration.md @@ -403,7 +403,7 @@ kernel command line: ... talos.environment=INSTALLER_META_BASE64=MHhhPWZvbw== When PXE booting, the value of `INSTALLER_META_BASE64` should be set manually: ```bash -echo -n "0xa=$(cat network.yaml)" | base64 +echo -n "0xa=$(cat network.yaml)" | gzip -9 | base64 ``` The resulting base64 string should be passed as an environment variable `INSTALLER_META_BASE64` to the initial boot of Talos: `talos.environment=INSTALLER_META_BASE64=`.