mirrors/talos
1
0
Fork 0
mirror of https://github.com/siderolabs/talos.git synced 2025-10-09 22:51:12 +02:00
Code Issues Packages Projects Releases Wiki Activity

feat: allow configuring etcd listen addresses

Browse Source 
This introduces new configuration settings to configure
advertised/listen subnets. For backwards compatibility when using no
settings or old 'subnet' argument, etcd still listens on all addresses.

If new `advertisedSubnets` is being used, this automatically limits etcd
listen addresses to the same value. `listenSubnets` can be configured
also explicitly e.g. to listen on additional addresses for some other
scenarios (e.g. accessing etcd from outside of the cluster).

See #5668

One more thing left (for a separate PR) is to update etcd advertised
URLs on the fly.

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
This commit is contained in:
Andrey Smirnov 2022-08-04 23:43:48 +04:00
parent 4c3485ae3f
commit dce923f747
No known key found for this signature in database
GPG Key ID: 7B26396447AB6DFD
16 changed files with 492 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
internal/app/machined/pkg/controllers/etcd/config.go
View File

@ -93,11 +93,8 @@ func (ctrl *ConfigController) Run(ctx context.Context, r controller.Runtime, log
}
if err = safe.WriterModify(ctx, r, etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID), func(status *etcd.Config) error {
if machineConfig.Config().Cluster().Etcd().Subnet() != "" {
status.TypedSpec().ValidSubnets = []string{machineConfig.Config().Cluster().Etcd().Subnet()}
} else {
status.TypedSpec().ValidSubnets = []string{"0.0.0.0/0", "::/0"}
}
status.TypedSpec().AdvertiseValidSubnets = machineConfig.Config().Cluster().Etcd().AdvertisedSubnets()
status.TypedSpec().ListenValidSubnets = machineConfig.Config().Cluster().Etcd().ListenSubnets()
status.TypedSpec().Image = machineConfig.Config().Cluster().Etcd().Image()
status.TypedSpec().ExtraArgs = machineConfig.Config().Cluster().Etcd().ExtraArgs()

98
internal/app/machined/pkg/controllers/etcd/config_test.go
View File

@ -40,31 +40,91 @@ func (suite *ConfigSuite) TestReconcile() {
machineType.SetMachineType(machine.TypeControlPlane)
suite.Require().NoError(suite.State().Create(suite.Ctx(), machineType))
cfg := &v1alpha1.Config{
ClusterConfig: &v1alpha1.ClusterConfig{
EtcdConfig: &v1alpha1.EtcdConfig{
for _, tt := range []struct {
name string
etcdConfig *v1alpha1.EtcdConfig
expectedConfig etcd.ConfigSpec
}{
{
name: "default config",
etcdConfig: &v1alpha1.EtcdConfig{
ContainerImage: "foo/bar:v1.0.0",
},
expectedConfig: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{},
AdvertiseValidSubnets: nil,
ListenValidSubnets: nil,
},
},
{
name: "legacy subnet",
etcdConfig: &v1alpha1.EtcdConfig{
ContainerImage: "foo/bar:v1.0.0",
EtcdExtraArgs: map[string]string{
"arg": "value",
},
EtcdSubnet: "10.0.0.0/8",
},
expectedConfig: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{
"arg": "value",
},
AdvertiseValidSubnets: []string{"10.0.0.0/8"},
ListenValidSubnets: nil,
},
},
{
name: "advertised subnets",
etcdConfig: &v1alpha1.EtcdConfig{
ContainerImage: "foo/bar:v1.0.0",
EtcdAdvertisedSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"},
},
expectedConfig: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{},
AdvertiseValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"},
ListenValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"},
},
},
{
name: "advertised and listen subnets",
etcdConfig: &v1alpha1.EtcdConfig{
ContainerImage: "foo/bar:v1.0.0",
EtcdAdvertisedSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"},
EtcdListenSubnets: []string{"10.0.0.0/8"},
},
expectedConfig: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{},
AdvertiseValidSubnets: []string{"10.0.0.0/8", "192.168.0.0/24"},
ListenValidSubnets: []string{"10.0.0.0/8"},
},
},
} {
suite.Run(tt.name, func() {
cfg := &v1alpha1.Config{
ClusterConfig: &v1alpha1.ClusterConfig{
EtcdConfig: tt.etcdConfig,
},
}
machineConfig := config.NewMachineConfig(cfg)
suite.Require().NoError(suite.State().Create(suite.Ctx(), machineConfig))
suite.AssertWithin(3*time.Second, 100*time.Millisecond, ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) {
etcdConfig, err := safe.StateGet[*etcd.Config](suite.Ctx(), suite.State(), etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID).Metadata())
if err != nil {
assert.NoError(err)
return
}
assert.Equal(tt.expectedConfig, *etcdConfig.TypedSpec())
}))
suite.Require().NoError(suite.State().Destroy(suite.Ctx(), machineConfig.Metadata()))
})
}
machineConfig := config.NewMachineConfig(cfg)
suite.Require().NoError(suite.State().Create(suite.Ctx(), machineConfig))
suite.AssertWithin(3*time.Second, 100*time.Millisecond, ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) {
etcdConfig, err := safe.StateGet[*etcd.Config](suite.Ctx(), suite.State(), etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID).Metadata())
if err != nil {
assert.NoError(err)
return
}
assert.Equal("foo/bar:v1.0.0", etcdConfig.TypedSpec().Image)
assert.Equal(map[string]string{"arg": "value"}, etcdConfig.TypedSpec().ExtraArgs)
assert.Equal([]string{"10.0.0.0/8"}, etcdConfig.TypedSpec().ValidSubnets)
}))
}

109
internal/app/machined/pkg/controllers/etcd/spec.go
View File

@ -7,7 +7,7 @@ package etcd
import (
"context"
"fmt"
"net/netip"
stdnet "net"
"github.com/cosi-project/runtime/pkg/controller"
"github.com/cosi-project/runtime/pkg/resource"
@ -16,7 +16,10 @@ import (
"github.com/siderolabs/go-pointer"
"github.com/talos-systems/net"
"go.uber.org/zap"
"inet.af/netaddr"
"github.com/talos-systems/talos/pkg/machinery/generic/slices"
"github.com/talos-systems/talos/pkg/machinery/nethelpers"
"github.com/talos-systems/talos/pkg/machinery/resources/etcd"
"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
"github.com/talos-systems/talos/pkg/machinery/resources/network"
@ -48,7 +51,7 @@ func (ctrl *SpecController) Inputs() []controller.Input {
{
Namespace: network.NamespaceName,
Type: network.NodeAddressType,
ID: pointer.To(network.FilteredNodeAddressID(network.NodeAddressCurrentID, k8s.NodeAddressFilterNoK8s)),
ID: pointer.To(network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s)),
Kind: controller.InputWeak,
},
}
@ -93,50 +96,96 @@ func (ctrl *SpecController) Run(ctx context.Context, r controller.Runtime, logge
return fmt.Errorf("error getting hostname status: %w", err)
}
cidrs := make([]string, 0, len(etcdConfig.TypedSpec().ValidSubnets)+len(etcdConfig.TypedSpec().ExcludeSubnets))
cidrs = append(cidrs, etcdConfig.TypedSpec().ValidSubnets...)
for _, subnet := range etcdConfig.TypedSpec().ExcludeSubnets {
cidrs = append(cidrs, "!"+subnet)
}
// we have trigger on NodeAddresses, but we don't use them directly as they contain
// some addresses which are not assigned to the node (like AWS ExternalIP).
// we need to find solution for that later, for now just pull addresses directly
ips, err := net.IPAddrs()
nodeAddrs, err := safe.ReaderGet[*network.NodeAddress](
ctx,
r,
resource.NewMetadata(
network.NamespaceName,
network.NodeAddressType,
network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s),
resource.VersionUndefined,
),
)
if err != nil {
return fmt.Errorf("error listing IPs: %w", err)
if state.IsNotFoundError(err) {
continue
}
return fmt.Errorf("error getting addresses: %w", err)
}
listenAddress := netip.IPv4Unspecified()
addrs := nodeAddrs.TypedSpec().IPs()
for _, ip := range ips {
if ip.To4() == nil {
listenAddress = netip.IPv6Unspecified()
// need at least a single address
if len(addrs) == 0 {
continue
}
advertisedCIDRs := make([]string, 0, len(etcdConfig.TypedSpec().AdvertiseValidSubnets)+len(etcdConfig.TypedSpec().AdvertiseExcludeSubnets))
advertisedCIDRs = append(advertisedCIDRs, etcdConfig.TypedSpec().AdvertiseValidSubnets...)
advertisedCIDRs = append(advertisedCIDRs, slices.Map(etcdConfig.TypedSpec().AdvertiseExcludeSubnets, func(cidr string) string { return "!" + cidr })...)
listenCIDRs := make([]string, 0, len(etcdConfig.TypedSpec().ListenValidSubnets)+len(etcdConfig.TypedSpec().ListenExcludeSubnets))
listenCIDRs = append(listenCIDRs, etcdConfig.TypedSpec().ListenValidSubnets...)
listenCIDRs = append(listenCIDRs, slices.Map(etcdConfig.TypedSpec().ListenExcludeSubnets, func(cidr string) string { return "!" + cidr })...)
defaultListenAddress := netaddr.IPv4(0, 0, 0, 0)
loopbackAddress := netaddr.IPv4(127, 0, 0, 1)
for _, ip := range addrs {
if ip.Is6() {
defaultListenAddress = netaddr.IPv6Unspecified()
loopbackAddress = netaddr.MustParseIP("::1")
break
}
}
// we use stdnet.IP here to re-use already existing functions in talos-systems/net
// once talos-systems/net is migrated to netaddr or netip, we can use it here
ips = net.IPFilter(ips, network.NotSideroLinkStdIP)
var (
advertisedIPs []netaddr.IP
listenPeerIPs []netaddr.IP
listenClientIPs []netaddr.IP
)
ips, err = net.FilterIPs(ips, cidrs)
if err != nil {
return fmt.Errorf("error filtering IPs: %w", err)
if len(advertisedCIDRs) > 0 {
// TODO: this should eventually be rewritten with `net.FilterIPs` on netaddrs, but for now we'll keep same code and do the conversion.
var stdIPs []stdnet.IP
stdIPs, err = net.FilterIPs(nethelpers.MapNetAddrToStd(addrs), advertisedCIDRs)
if err != nil {
return fmt.Errorf("error filtering IPs: %w", err)
}
advertisedIPs = nethelpers.MapStdToNetAddr(stdIPs)
} else {
// if advertise subnet is not set, advertise the first address
advertisedIPs = []netaddr.IP{addrs[0]}
}
if len(ips) == 0 {
if len(listenCIDRs) > 0 {
// TODO: this should eventually be rewritten with `net.FilterIPs` on netaddrs, but for now we'll keep same code and do the conversion.
var stdIPs []stdnet.IP
stdIPs, err = net.FilterIPs(nethelpers.MapNetAddrToStd(addrs), listenCIDRs)
if err != nil {
return fmt.Errorf("error filtering IPs: %w", err)
}
listenPeerIPs = nethelpers.MapStdToNetAddr(stdIPs)
listenClientIPs = append([]netaddr.IP{loopbackAddress}, listenPeerIPs...)
} else {
listenPeerIPs = []netaddr.IP{defaultListenAddress}
listenClientIPs = []netaddr.IP{defaultListenAddress}
}
if len(advertisedIPs) == 0 || len(listenPeerIPs) == 0 {
continue
}
if err = safe.WriterModify(ctx, r, etcd.NewSpec(etcd.NamespaceName, etcd.SpecID), func(status *etcd.Spec) error {
status.TypedSpec().AdvertisedAddress, _ = netip.AddrFromSlice(ips[0])
status.TypedSpec().AdvertisedAddress = status.TypedSpec().AdvertisedAddress.Unmap()
status.TypedSpec().ListenAddress = listenAddress
status.TypedSpec().AdvertisedAddresses = advertisedIPs
status.TypedSpec().ListenClientAddresses = listenClientIPs
status.TypedSpec().ListenPeerAddresses = listenPeerIPs
status.TypedSpec().Name = hostnameStatus.TypedSpec().Hostname
status.TypedSpec().Image = etcdConfig.TypedSpec().Image
status.TypedSpec().ExtraArgs = etcdConfig.TypedSpec().ExtraArgs

138
internal/app/machined/pkg/controllers/etcd/spec_test.go
View File

@ -5,7 +5,6 @@
package etcd_test
import (
"net/netip"
"testing"
"time"
@ -13,10 +12,12 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"inet.af/netaddr"
"github.com/talos-systems/talos/internal/app/machined/pkg/controllers/ctest"
etcdctrl "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/etcd"
"github.com/talos-systems/talos/pkg/machinery/resources/etcd"
"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
"github.com/talos-systems/talos/pkg/machinery/resources/network"
)
@ -35,33 +36,124 @@ type SpecSuite struct {
}
func (suite *SpecSuite) TestReconcile() {
etcdConfig := etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID)
*etcdConfig.TypedSpec() = etcd.ConfigSpec{
ValidSubnets: []string{"0.0.0.0/0", "::/0"},
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{
"arg": "value",
},
}
suite.Require().NoError(suite.State().Create(suite.Ctx(), etcdConfig))
hostnameStatus := network.NewHostnameStatus(network.NamespaceName, network.HostnameID)
hostnameStatus.TypedSpec().Hostname = "worker1"
hostnameStatus.TypedSpec().Domainname = "some.domain"
suite.Require().NoError(suite.State().Create(suite.Ctx(), hostnameStatus))
suite.AssertWithin(3*time.Second, 100*time.Millisecond, ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) {
etcdSpec, err := safe.StateGet[*etcd.Spec](suite.Ctx(), suite.State(), etcd.NewSpec(etcd.NamespaceName, etcd.SpecID).Metadata())
if err != nil {
assert.NoError(err)
addresses := network.NewNodeAddress(
network.NamespaceName,
network.FilteredNodeAddressID(network.NodeAddressRoutedID, k8s.NodeAddressFilterNoK8s),
)
return
}
addresses.TypedSpec().Addresses = []netaddr.IPPrefix{
netaddr.MustParseIPPrefix("10.0.0.5/24"),
netaddr.MustParseIPPrefix("192.168.1.1/24"),
netaddr.MustParseIPPrefix("2001:0db8:85a3:0000:0000:8a2e:0370:7334/64"),
netaddr.MustParseIPPrefix("2002:0db8:85a3:0000:0000:8a2e:0370:7335/64"),
}
assert.Equal("foo/bar:v1.0.0", etcdSpec.TypedSpec().Image)
assert.Equal(map[string]string{"arg": "value"}, etcdSpec.TypedSpec().ExtraArgs)
assert.NotEqual(netip.Addr{}, etcdSpec.TypedSpec().AdvertisedAddress)
assert.True(etcdSpec.TypedSpec().ListenAddress.IsUnspecified())
}))
suite.Require().NoError(suite.State().Create(suite.Ctx(), addresses))
for _, tt := range []struct {
name string
cfg etcd.ConfigSpec
expected etcd.SpecSpec
}{
{
name: "defaults",
cfg: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{
"arg": "value",
},
},
expected: etcd.SpecSpec{
Name: "worker1",
Image: "foo/bar:v1.0.0",
ExtraArgs: map[string]string{
"arg": "value",
},
AdvertisedAddresses: []netaddr.IP{
netaddr.MustParseIP("10.0.0.5"),
},
ListenPeerAddresses: []netaddr.IP{
netaddr.IPv6Unspecified(),
},
ListenClientAddresses: []netaddr.IP{
netaddr.IPv6Unspecified(),
},
},
},
{
name: "only advertised",
cfg: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
AdvertiseValidSubnets: []string{
"192.168.0.0/16",
},
},
expected: etcd.SpecSpec{
Name: "worker1",
Image: "foo/bar:v1.0.0",
AdvertisedAddresses: []netaddr.IP{
netaddr.MustParseIP("192.168.1.1"),
},
ListenPeerAddresses: []netaddr.IP{
netaddr.IPv6Unspecified(),
},
ListenClientAddresses: []netaddr.IP{
netaddr.IPv6Unspecified(),
},
},
},
{
name: "advertised and listen",
cfg: etcd.ConfigSpec{
Image: "foo/bar:v1.0.0",
AdvertiseValidSubnets: []string{
"192.168.0.0/16",
"2001::/16",
},
ListenValidSubnets: []string{
"192.168.0.0/16",
},
},
expected: etcd.SpecSpec{
Name: "worker1",
Image: "foo/bar:v1.0.0",
AdvertisedAddresses: []netaddr.IP{
netaddr.MustParseIP("192.168.1.1"),
netaddr.MustParseIP("2001:0db8:85a3:0000:0000:8a2e:0370:7334"),
},
ListenPeerAddresses: []netaddr.IP{
netaddr.MustParseIP("192.168.1.1"),
},
ListenClientAddresses: []netaddr.IP{
netaddr.MustParseIP("::1"),
netaddr.MustParseIP("192.168.1.1"),
},
},
},
} {
suite.Run(tt.name, func() {
etcdConfig := etcd.NewConfig(etcd.NamespaceName, etcd.ConfigID)
*etcdConfig.TypedSpec() = tt.cfg
suite.Require().NoError(suite.State().Create(suite.Ctx(), etcdConfig))
suite.AssertWithin(3*time.Second, 100*time.Millisecond, ctest.WrapRetry(func(assert *assert.Assertions, require *require.Assertions) {
etcdSpec, err := safe.StateGet[*etcd.Spec](suite.Ctx(), suite.State(), etcd.NewSpec(etcd.NamespaceName, etcd.SpecID).Metadata())
if err != nil {
assert.NoError(err)
return
}
assert.Equal(tt.expected, *etcdSpec.TypedSpec())
}))
suite.Require().NoError(suite.State().Destroy(suite.Ctx(), etcdConfig.Metadata()))
})
}
}

51
internal/app/machined/pkg/system/services/etcd.go
View File

@ -26,6 +26,7 @@ import (
"github.com/talos-systems/go-retry/retry"
clientv3 "go.etcd.io/etcd/client/v3"
snapshot "go.etcd.io/etcd/etcdutl/v3/snapshot"
"inet.af/netaddr"
"github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
"github.com/talos-systems/talos/internal/app/machined/pkg/runtime/v1alpha1/bootloader"
@ -45,6 +46,7 @@ import (
machineapi "github.com/talos-systems/talos/pkg/machinery/api/machine"
"github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine"
"github.com/talos-systems/talos/pkg/machinery/constants"
"github.com/talos-systems/talos/pkg/machinery/generic/slices"
"github.com/talos-systems/talos/pkg/machinery/nethelpers"
etcdresource "github.com/talos-systems/talos/pkg/machinery/resources/etcd"
"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
@ -300,7 +302,7 @@ func addMember(ctx context.Context, r runtime.Runtime, addrs []string, name stri
return list, add.Member.ID, nil
}
func buildInitialCluster(ctx context.Context, r runtime.Runtime, name, ip string) (initial string, learnerMemberID uint64, err error) {
func buildInitialCluster(ctx context.Context, r runtime.Runtime, name string, peerAddrs []string) (initial string, learnerMemberID uint64, err error) {
var (
id uint64
lastNag time.Time
@ -311,10 +313,7 @@ func buildInitialCluster(ctx context.Context, r runtime.Runtime, name, ip string
retry.WithJitter(time.Second),
retry.WithErrorLogging(true),
).RetryWithContext(ctx, func(ctx context.Context) error {
var (
peerAddrs = []string{"https://" + nethelpers.JoinHostPort(ip, +constants.EtcdPeerPort)}
resp *clientv3.MemberListResponse
)
var resp *clientv3.MemberListResponse
if time.Since(lastNag) > 30*time.Second {
lastNag = time.Now()
@ -396,8 +395,8 @@ func (e *Etcd) argsForInit(ctx context.Context, r runtime.Runtime, spec *etcdres
"auto-tls": "false",
"peer-auto-tls": "false",
"data-dir": constants.EtcdDataPath,
"listen-peer-urls": "https://" + nethelpers.JoinHostPort(spec.ListenAddress.String(), constants.EtcdPeerPort),
"listen-client-urls": "https://" + nethelpers.JoinHostPort(spec.ListenAddress.String(), constants.EtcdClientPort),
"listen-peer-urls": formatEtcdURLs(spec.ListenPeerAddresses, constants.EtcdPeerPort),
"listen-client-urls": formatEtcdURLs(spec.ListenClientAddresses, constants.EtcdClientPort),
"client-cert-auth": "true",
"cert-file": constants.EtcdCert,
"key-file": constants.EtcdKey,
@ -427,12 +426,12 @@ func (e *Etcd) argsForInit(ctx context.Context, r runtime.Runtime, spec *etcdres
}
if ok {
initialCluster := fmt.Sprintf("%s=https://%s", spec.Name, nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdPeerPort))
initialCluster := fmt.Sprintf("%s=%s", spec.Name, formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort))
if upgraded {
denyListArgs.Set("initial-cluster-state", "existing")
initialCluster, e.learnerMemberID, err = buildInitialCluster(ctx, r, spec.Name, spec.AdvertisedAddress.String())
initialCluster, e.learnerMemberID, err = buildInitialCluster(ctx, r, spec.Name, getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort))
if err != nil {
return err
}
@ -446,13 +445,13 @@ func (e *Etcd) argsForInit(ctx context.Context, r runtime.Runtime, spec *etcdres
if !extraArgs.Contains("initial-advertise-peer-urls") {
denyListArgs.Set("initial-advertise-peer-urls",
fmt.Sprintf("https://%s", nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdPeerPort)),
formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort),
)
}
if !extraArgs.Contains("advertise-client-urls") {
denyListArgs.Set("advertise-client-urls",
fmt.Sprintf("https://%s", nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdClientPort)),
formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdClientPort),
)
}
@ -472,8 +471,8 @@ func (e *Etcd) argsForControlPlane(ctx context.Context, r runtime.Runtime, spec
"auto-tls": "false",
"peer-auto-tls": "false",
"data-dir": constants.EtcdDataPath,
"listen-peer-urls": "https://" + nethelpers.JoinHostPort(spec.ListenAddress.String(), constants.EtcdPeerPort),
"listen-client-urls": "https://" + nethelpers.JoinHostPort(spec.ListenAddress.String(), constants.EtcdClientPort),
"listen-peer-urls": formatEtcdURLs(spec.ListenPeerAddresses, constants.EtcdPeerPort),
"listen-client-urls": formatEtcdURLs(spec.ListenClientAddresses, constants.EtcdClientPort),
"client-cert-auth": "true",
"cert-file": constants.EtcdCert,
"key-file": constants.EtcdKey,
@ -515,9 +514,9 @@ func (e *Etcd) argsForControlPlane(ctx context.Context, r runtime.Runtime, spec
var initialCluster string
if e.Bootstrap {
initialCluster = fmt.Sprintf("%s=https://%s", spec.Name, nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdPeerPort))
initialCluster = fmt.Sprintf("%s=%s", spec.Name, formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort))
} else {
initialCluster, e.learnerMemberID, err = buildInitialCluster(ctx, r, spec.Name, spec.AdvertisedAddress.String())
initialCluster, e.learnerMemberID, err = buildInitialCluster(ctx, r, spec.Name, getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort))
if err != nil {
return fmt.Errorf("failed to build initial etcd cluster: %w", err)
}
@ -528,14 +527,14 @@ func (e *Etcd) argsForControlPlane(ctx context.Context, r runtime.Runtime, spec
if !extraArgs.Contains("initial-advertise-peer-urls") {
denyListArgs.Set("initial-advertise-peer-urls",
fmt.Sprintf("https://%s", nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdPeerPort)),
formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort),
)
}
}
if !extraArgs.Contains("advertise-client-urls") {
denyListArgs.Set("advertise-client-urls",
fmt.Sprintf("https://%s", nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdClientPort)),
formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdClientPort),
)
}
@ -566,9 +565,9 @@ func (e *Etcd) recoverFromSnapshot(spec *etcdresource.SpecSpec) error {
Name: spec.Name,
OutputDataDir: constants.EtcdDataPath,
PeerURLs: []string{"https://" + nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdPeerPort)},
PeerURLs: getEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort),
InitialCluster: fmt.Sprintf("%s=https://%s", spec.Name, nethelpers.JoinHostPort(spec.AdvertisedAddress.String(), constants.EtcdPeerPort)),
InitialCluster: fmt.Sprintf("%s=%s", spec.Name, formatEtcdURLs(spec.AdvertisedAddresses, constants.EtcdPeerPort)),
SkipHashCheck: e.RecoverSkipHashCheck,
}); err != nil {
@ -696,3 +695,17 @@ func BootstrapEtcd(ctx context.Context, r runtime.Runtime, req *machineapi.Boots
return nil
}
func formatEtcdURL(addr netaddr.IP, port int) string {
return fmt.Sprintf("https://%s", nethelpers.JoinHostPort(addr.String(), port))
}
func getEtcdURLs(addrs []netaddr.IP, port int) []string {
return slices.Map(addrs, func(addr netaddr.IP) string {
return formatEtcdURL(addr, port)
})
}
func formatEtcdURLs(addrs []netaddr.IP, port int) string {
return strings.Join(getEtcdURLs(addrs, port), ",")
}

3
pkg/machinery/config/provider.go
View File

@ -463,7 +463,8 @@ type Etcd interface {
Image() string
CA() *x509.PEMEncodedCertificateAndKey
ExtraArgs() map[string]string
Subnet() string
AdvertisedSubnets() []string
ListenSubnets() []string
}
// Token defines the requirements for a config that pertains to Kubernetes

30
pkg/machinery/config/types/v1alpha1/v1alpha1_etcdconfig.go
View File

@ -43,7 +43,31 @@ func (e *EtcdConfig) ExtraArgs() map[string]string {
return e.EtcdExtraArgs
}
// Subnet implements the config.Etcd interface.
func (e *EtcdConfig) Subnet() string {
return e.EtcdSubnet
// AdvertisedSubnets implements the config.Etcd interface.
func (e *EtcdConfig) AdvertisedSubnets() []string {
if len(e.EtcdAdvertisedSubnets) > 0 {
return e.EtcdAdvertisedSubnets
}
if e.EtcdSubnet != "" {
return []string{e.EtcdSubnet}
}
return nil
}
// ListenSubnets implements the config.Etcd interface.
func (e *EtcdConfig) ListenSubnets() []string {
if len(e.EtcdListenSubnets) > 0 {
return e.EtcdListenSubnets
}
// if advertised subnets are set, use them
if len(e.EtcdAdvertisedSubnets) > 0 {
return e.EtcdAdvertisedSubnets
}
// nothing set, rely on defaults (listen on all interfaces)
return nil
}

30
pkg/machinery/config/types/v1alpha1/v1alpha1_types.go
View File

@ -350,7 +350,7 @@ var (
clusterEtcdImageExample = (&EtcdConfig{}).Image()
clusterEtcdSubnetExample = (&EtcdConfig{EtcdSubnet: "10.0.0.0/8"}).Subnet()
clusterEtcdAdvertisedSubnetsExample = (&EtcdConfig{EtcdAdvertisedSubnets: []string{"10.0.0.0/8"}}).AdvertisedSubnets()
clusterCoreDNSExample = &CoreDNS{
CoreDNSImage: (&CoreDNS{}).Image(),
@ -1707,12 +1707,32 @@ type EtcdConfig struct {
// "advertise-client-urls": "https://1.2.3.4:2379",
// }
EtcdExtraArgs map[string]string `yaml:"extraArgs,omitempty"`
// description: |
// The subnet from which the advertise URL should be.
// docgen:nodoc
//
// examples:
// - value: clusterEtcdSubnetExample
// Deprecated: use EtcdAdvertistedSubnets
EtcdSubnet string `yaml:"subnet,omitempty"`
// description: |
// The `advertisedSubnets` field configures the networks to pick etcd advertised IP from.
//
// IPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.
// Negative subnet matches should be specified last to filter out IPs picked by positive matches.
// If not specified, advertised IP is selected as the first routable address of the node.
//
// examples:
// - value: clusterEtcdAdvertisedSubnetsExample
EtcdAdvertisedSubnets []string `yaml:"advertisedSubnets,omitempty"`
// description: |
// The `listenSubnets` field configures the networks for the etcd to listen for peer and client connections.
//
// If `listenSubnets` is not set, but `advertisedSubnets` is set, `listenSubnets` defaults to
// `advertisedSubnets`.
//
// If neither `advertisedSubnets` nor `listenSubnets` is set, `listenSubnets` defaults to listen on all addresses.
//
// IPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.
// Negative subnet matches should be specified last to filter out IPs picked by positive matches.
// If not specified, advertised IP is selected as the first routable address of the node.
EtcdListenSubnets []string `yaml:"listenSubnets,omitempty"`
}
// ClusterNetworkConfig represents kube networking configuration options.

19
pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go
View File

@ -1248,7 +1248,7 @@ func init() {
FieldName: "etcd",
},
}
EtcdConfigDoc.Fields = make([]encoder.Doc, 4)
EtcdConfigDoc.Fields = make([]encoder.Doc, 6)
EtcdConfigDoc.Fields[0].Name = "image"
EtcdConfigDoc.Fields[0].Type = "string"
EtcdConfigDoc.Fields[0].Note = ""
@ -1269,13 +1269,18 @@ func init() {
EtcdConfigDoc.Fields[2].Description = "Extra arguments to supply to etcd.\nNote that the following args are not allowed:\n\n- `name`\n- `data-dir`\n- `initial-cluster-state`\n- `listen-peer-urls`\n- `listen-client-urls`\n- `cert-file`\n- `key-file`\n- `trusted-ca-file`\n- `peer-client-cert-auth`\n- `peer-cert-file`\n- `peer-trusted-ca-file`\n- `peer-key-file`"
EtcdConfigDoc.Fields[2].Comments[encoder.LineComment] = "Extra arguments to supply to etcd."
EtcdConfigDoc.Fields[3].Name = "subnet"
EtcdConfigDoc.Fields[3].Type = "string"
EtcdConfigDoc.Fields[3].Note = ""
EtcdConfigDoc.Fields[3].Description = "The subnet from which the advertise URL should be."
EtcdConfigDoc.Fields[3].Comments[encoder.LineComment] = "The subnet from which the advertise URL should be."
EtcdConfigDoc.Fields[4].Name = "advertisedSubnets"
EtcdConfigDoc.Fields[4].Type = "[]string"
EtcdConfigDoc.Fields[4].Note = ""
EtcdConfigDoc.Fields[4].Description = "The `advertisedSubnets` field configures the networks to pick etcd advertised IP from.\n\nIPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.\nNegative subnet matches should be specified last to filter out IPs picked by positive matches.\nIf not specified, advertised IP is selected as the first routable address of the node."
EtcdConfigDoc.Fields[4].Comments[encoder.LineComment] = "The `advertisedSubnets` field configures the networks to pick etcd advertised IP from."
EtcdConfigDoc.Fields[3].AddExample("", clusterEtcdSubnetExample)
EtcdConfigDoc.Fields[4].AddExample("", clusterEtcdAdvertisedSubnetsExample)
EtcdConfigDoc.Fields[5].Name = "listenSubnets"
EtcdConfigDoc.Fields[5].Type = "[]string"
EtcdConfigDoc.Fields[5].Note = ""
EtcdConfigDoc.Fields[5].Description = "The `listenSubnets` field configures the networks for the etcd to listen for peer and client connections.\n\nIf `listenSubnets` is not set, but `advertisedSubnets` is set, `listenSubnets` defaults to\n`advertisedSubnets`.\n\nIf neither `advertisedSubnets` nor `listenSubnets` is set, `listenSubnets` defaults to listen on all addresses.\n\nIPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.\nNegative subnet matches should be specified last to filter out IPs picked by positive matches.\nIf not specified, advertised IP is selected as the first routable address of the node."
EtcdConfigDoc.Fields[5].Comments[encoder.LineComment] = "The `listenSubnets` field configures the networks for the etcd to listen for peer and client connections."
ClusterNetworkConfigDoc.Type = "ClusterNetworkConfig"
ClusterNetworkConfigDoc.Comments[encoder.LineComment] = "ClusterNetworkConfig represents kube networking configuration options."

33
pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go
View File

@ -310,10 +310,8 @@ func (c *ClusterConfig) Validate() error {
result = multierror.Append(result, ecp.Validate())
}
if c.EtcdConfig != nil && c.EtcdConfig.EtcdSubnet != "" {
if _, _, err := net.ParseCIDR(c.EtcdConfig.EtcdSubnet); err != nil {
result = multierror.Append(result, fmt.Errorf("%q is not a valid subnet", c.EtcdConfig.EtcdSubnet))
}
if c.EtcdConfig != nil {
result = multierror.Append(result, c.EtcdConfig.Validate())
}
result = multierror.Append(result, c.ClusterInlineManifests.Validate(), c.ClusterDiscoveryConfig.Validate(c))
@ -793,3 +791,30 @@ func (k *KubeletConfig) Validate() ([]string, error) {
return nil, result.ErrorOrNil()
}
// Validate kubelet configuration.
func (e *EtcdConfig) Validate() error {
var result *multierror.Error
if e.EtcdSubnet != "" && len(e.EtcdAdvertisedSubnets) > 0 {
result = multierror.Append(result, fmt.Errorf("etcd subnet can't be set when advertised subnets are set"))
}
for _, cidr := range e.AdvertisedSubnets() {
cidr = strings.TrimPrefix(cidr, "!")
if _, err := talosnet.ParseCIDR(cidr); err != nil {
result = multierror.Append(result, fmt.Errorf("etcd advertised subnet is not valid: %q", cidr))
}
}
for _, cidr := range e.ListenSubnets() {
cidr = strings.TrimPrefix(cidr, "!")
if _, err := talosnet.ParseCIDR(cidr); err != nil {
result = multierror.Append(result, fmt.Errorf("etcd listen subnet is not valid: %q", cidr))
}
}
return result.ErrorOrNil()
}

18
pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go
View File

@ -961,7 +961,14 @@ func TestValidate(t *testing.T) {
},
},
EtcdConfig: &v1alpha1.EtcdConfig{
EtcdSubnet: "10.0.0.0/8",
EtcdAdvertisedSubnets: []string{
"10.0.0.0/8",
"!1.1.1.1/32",
},
EtcdListenSubnets: []string{
"10.0.0.0/8",
"1.1.1.1/32",
},
},
},
},
@ -981,11 +988,16 @@ func TestValidate(t *testing.T) {
},
},
EtcdConfig: &v1alpha1.EtcdConfig{
EtcdSubnet: "10.0.0.0",
EtcdAdvertisedSubnets: []string{
"1234:",
},
EtcdListenSubnets: []string{
"10",
},
},
},
},
expectedError: "1 error occurred:\n\t* \"10.0.0.0\" is not a valid subnet\n\n",
expectedError: "2 errors occurred:\n\t* etcd advertised subnet is not valid: \"1234:\"\n\t* etcd listen subnet is not valid: \"10\"\n\n",
},
{
name: "GoodKubeletSubnet",

10
pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go
View File

@ -859,6 +859,16 @@ func (in *EtcdConfig) DeepCopyInto(out *EtcdConfig) {
(*out)[key] = val
}
}
if in.EtcdAdvertisedSubnets != nil {
in, out := &in.EtcdAdvertisedSubnets, &out.EtcdAdvertisedSubnets
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.EtcdListenSubnets != nil {
in, out := &in.EtcdListenSubnets, &out.EtcdListenSubnets
*out = make([]string, len(*in))
copy(*out, *in)
}
return
}

12
pkg/machinery/resources/etcd/config.go
View File

@ -26,10 +26,14 @@ type Config = typed.Resource[ConfigSpec, ConfigRD]
//
//gotagsrewrite:gen
type ConfigSpec struct {
ValidSubnets []string `yaml:"validSubnets,omitempty" protobuf:"1"`
ExcludeSubnets []string `yaml:"excludeSubnets" protobuf:"2"`
Image string `yaml:"image" protobuf:"3"`
ExtraArgs map[string]string `yaml:"extraArgs" protobuf:"4"`
AdvertiseValidSubnets []string `yaml:"advertiseValidSubnets,omitempty" protobuf:"1"`
AdvertiseExcludeSubnets []string `yaml:"advertiseExcludeSubnets" protobuf:"2"`
ListenValidSubnets []string `yaml:"listenValidSubnets,omitempty" protobuf:"5"`
ListenExcludeSubnets []string `yaml:"listenExcludeSubnets" protobuf:"6"`
Image string `yaml:"image" protobuf:"3"`
ExtraArgs map[string]string `yaml:"extraArgs" protobuf:"4"`
}
// NewConfig initializes a Config resource.

36
pkg/machinery/resources/etcd/deep_copy.generated.go
View File

@ -6,16 +6,28 @@
package etcd
import (
"inet.af/netaddr"
)
// DeepCopy generates a deep copy of ConfigSpec.
func (o ConfigSpec) DeepCopy() ConfigSpec {
var cp ConfigSpec = o
if o.ValidSubnets != nil {
cp.ValidSubnets = make([]string, len(o.ValidSubnets))
copy(cp.ValidSubnets, o.ValidSubnets)
if o.AdvertiseValidSubnets != nil {
cp.AdvertiseValidSubnets = make([]string, len(o.AdvertiseValidSubnets))
copy(cp.AdvertiseValidSubnets, o.AdvertiseValidSubnets)
}
if o.ExcludeSubnets != nil {
cp.ExcludeSubnets = make([]string, len(o.ExcludeSubnets))
copy(cp.ExcludeSubnets, o.ExcludeSubnets)
if o.AdvertiseExcludeSubnets != nil {
cp.AdvertiseExcludeSubnets = make([]string, len(o.AdvertiseExcludeSubnets))
copy(cp.AdvertiseExcludeSubnets, o.AdvertiseExcludeSubnets)
}
if o.ListenValidSubnets != nil {
cp.ListenValidSubnets = make([]string, len(o.ListenValidSubnets))
copy(cp.ListenValidSubnets, o.ListenValidSubnets)
}
if o.ListenExcludeSubnets != nil {
cp.ListenExcludeSubnets = make([]string, len(o.ListenExcludeSubnets))
copy(cp.ListenExcludeSubnets, o.ListenExcludeSubnets)
}
if o.ExtraArgs != nil {
cp.ExtraArgs = make(map[string]string, len(o.ExtraArgs))
@ -35,6 +47,18 @@ func (o PKIStatusSpec) DeepCopy() PKIStatusSpec {
// DeepCopy generates a deep copy of SpecSpec.
func (o SpecSpec) DeepCopy() SpecSpec {
var cp SpecSpec = o
if o.AdvertisedAddresses != nil {
cp.AdvertisedAddresses = make([]netaddr.IP, len(o.AdvertisedAddresses))
copy(cp.AdvertisedAddresses, o.AdvertisedAddresses)
}
if o.ListenPeerAddresses != nil {
cp.ListenPeerAddresses = make([]netaddr.IP, len(o.ListenPeerAddresses))
copy(cp.ListenPeerAddresses, o.ListenPeerAddresses)
}
if o.ListenClientAddresses != nil {
cp.ListenClientAddresses = make([]netaddr.IP, len(o.ListenClientAddresses))
copy(cp.ListenClientAddresses, o.ListenClientAddresses)
}
if o.ExtraArgs != nil {
cp.ExtraArgs = make(map[string]string, len(o.ExtraArgs))
for k2, v2 := range o.ExtraArgs {

26
pkg/machinery/resources/etcd/spec.go
View File

@ -5,12 +5,11 @@
package etcd
import (
"net/netip"
"github.com/cosi-project/runtime/pkg/resource"
"github.com/cosi-project/runtime/pkg/resource/meta"
"github.com/cosi-project/runtime/pkg/resource/protobuf"
"github.com/cosi-project/runtime/pkg/resource/typed"
"inet.af/netaddr"
"github.com/talos-systems/talos/pkg/machinery/proto"
)
@ -28,11 +27,12 @@ type Spec = typed.Resource[SpecSpec, SpecRD]
//
//gotagsrewrite:gen
type SpecSpec struct {
Name string `yaml:"name" protobuf:"1"`
AdvertisedAddress netip.Addr `yaml:"advertisedAddress" protobuf:"2"`
ListenAddress netip.Addr `yaml:"listenAddress" protobuf:"5"`
Image string `yaml:"image" protobuf:"3"`
ExtraArgs map[string]string `yaml:"extraArgs" protobuf:"4"`
Name string `yaml:"name" protobuf:"1"`
AdvertisedAddresses []netaddr.IP `yaml:"advertisedAddresses" protobuf:"2"`
ListenPeerAddresses []netaddr.IP `yaml:"listenPeerAddresses" protobuf:"5"`
ListenClientAddresses []netaddr.IP `yaml:"listenClientAddresses" protobuf:"6"`
Image string `yaml:"image" protobuf:"3"`
ExtraArgs map[string]string `yaml:"extraArgs" protobuf:"4"`
}
// NewSpec initializes a Spec resource.
@ -58,8 +58,16 @@ func (SpecRD) ResourceDefinition(resource.Metadata, SpecSpec) meta.ResourceDefin
JSONPath: "{.name}",
},
{
Name: "AdvertisedAddress",
JSONPath: "{.advertisedAddress}",
Name: "AdvertisedAddresses",
JSONPath: "{.advertisedAddresses}",
},
{
Name: "ListenPeerAddresses",
JSONPath: "{.listenPeerAddresses}",
},
{
Name: "ListenClientAddresses",
JSONPath: "{.listenClientAddresses}",
},
},
}

16
website/content/v1.2/reference/configuration.md
View File

@ -581,8 +581,9 @@ etcd:
extraArgs:
election-timeout: "5000"
# # The subnet from which the advertise URL should be.
# subnet: 10.0.0.0/8
# # The `advertisedSubnets` field configures the networks to pick etcd advertised IP from.
# advertisedSubnets:
# - 10.0.0.0/8
{{< /highlight >}}</details> | |
|`coreDNS` |<a href="#coredns">CoreDNS</a> |Core DNS specific configuration options. <details><summary>Show example(s)</summary>{{< highlight yaml >}}
coreDNS:
@ -1568,8 +1569,9 @@ ca:
extraArgs:
election-timeout: "5000"
# # The subnet from which the advertise URL should be.
# subnet: 10.0.0.0/8
# # The `advertisedSubnets` field configures the networks to pick etcd advertised IP from.
# advertisedSubnets:
# - 10.0.0.0/8
{{< /highlight >}}
@ -1584,9 +1586,11 @@ ca:
key: LS0tIEVYQU1QTEUgS0VZIC0tLQ==
{{< /highlight >}}</details> | |
|`extraArgs` |map[string]string |<details><summary>Extra arguments to supply to etcd.</summary>Note that the following args are not allowed:<br /><br />- `name`<br />- `data-dir`<br />- `initial-cluster-state`<br />- `listen-peer-urls`<br />- `listen-client-urls`<br />- `cert-file`<br />- `key-file`<br />- `trusted-ca-file`<br />- `peer-client-cert-auth`<br />- `peer-cert-file`<br />- `peer-trusted-ca-file`<br />- `peer-key-file`</details> | |
|`subnet` |string |The subnet from which the advertise URL should be. <details><summary>Show example(s)</summary>{{< highlight yaml >}}
subnet: 10.0.0.0/8
|`advertisedSubnets` |[]string |<details><summary>The `advertisedSubnets` field configures the networks to pick etcd advertised IP from.</summary><br />IPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.<br />Negative subnet matches should be specified last to filter out IPs picked by positive matches.<br />If not specified, advertised IP is selected as the first routable address of the node.</details> <details><summary>Show example(s)</summary>{{< highlight yaml >}}
advertisedSubnets:
- 10.0.0.0/8
{{< /highlight >}}</details> | |
|`listenSubnets` |[]string |<details><summary>The `listenSubnets` field configures the networks for the etcd to listen for peer and client connections.</summary><br />If `listenSubnets` is not set, but `advertisedSubnets` is set, `listenSubnets` defaults to<br />`advertisedSubnets`.<br /><br />If neither `advertisedSubnets` nor `listenSubnets` is set, `listenSubnets` defaults to listen on all addresses.<br /><br />IPs can be excluded from the list by using negative match with `!`, e.g `!10.0.0.0/8`.<br />Negative subnet matches should be specified last to filter out IPs picked by positive matches.<br />If not specified, advertised IP is selected as the first routable address of the node.</details> | |