diff --git a/cmd/osctl/cmd/cluster.go b/cmd/osctl/cmd/cluster.go index b2baa4157..55dd9f21a 100644 --- a/cmd/osctl/cmd/cluster.go +++ b/cmd/osctl/cmd/cluster.go @@ -178,9 +178,6 @@ func createNodes(requests []*node.Request) (err error) { // node comes up // 1 <- 2 <- 3 <- 4 <- 5 ... req.Input.Index = idx - if req.Input.Index > 0 { - req.Input.Index-- - } if req.IP != nil { req.Input.IP = req.IP } diff --git a/cmd/osctl/cmd/config.go b/cmd/osctl/cmd/config.go index b03a82a67..02bfc645f 100644 --- a/cmd/osctl/cmd/config.go +++ b/cmd/osctl/cmd/config.go @@ -144,7 +144,6 @@ var configGenerateCmd = &cobra.Command{ if input.Index == 0 { udType = generate.TypeInit } else { - input.Index-- udType = generate.TypeControlPlane } diff --git a/pkg/userdata/generate/controlplane.go b/pkg/userdata/generate/controlplane.go index 02e4cb018..449a0e5eb 100644 --- a/pkg/userdata/generate/controlplane.go +++ b/pkg/userdata/generate/controlplane.go @@ -28,7 +28,7 @@ services: bootstrapToken: token: '{{ .KubeadmTokens.BootstrapToken }}' unsafeSkipCAVerification: true - apiServerEndpoint: {{ index .MasterIPs .Index }}:6443 + apiServerEndpoint: "{{ .GetControlPlaneEndpoint "6443" }}" nodeRegistration: taints: [] kubeletExtraArgs: diff --git a/pkg/userdata/generate/generate.go b/pkg/userdata/generate/generate.go index 4016d3495..8acd6128c 100644 --- a/pkg/userdata/generate/generate.go +++ b/pkg/userdata/generate/generate.go @@ -12,15 +12,28 @@ import ( "encoding/base64" "encoding/pem" "errors" + "fmt" "net" - "strings" "text/template" "github.com/talos-systems/talos/pkg/constants" "github.com/talos-systems/talos/pkg/crypto/x509" + tnet "github.com/talos-systems/talos/pkg/net" "github.com/talos-systems/talos/pkg/userdata/token" ) +// DefaultIPv4PodNet is the network to be used for kubernetes Pods when using IPv4-based master nodes +const DefaultIPv4PodNet = "10.244.0.0/16" + +// DefaultIPv4ServiceNet is the network to be used for kubernetes Services when using IPv4-based master nodes +const DefaultIPv4ServiceNet = "10.96.0.0/12" + +// DefaultIPv6PodNet is the network to be used for kubernetes Pods when using IPv6-based master nodes +const DefaultIPv6PodNet = "fc00:db8:10::/56" + +// DefaultIPv6ServiceNet is the network to be used for kubernetes Services when using IPv6-based master nodes +const DefaultIPv6ServiceNet = "fc00:db8:20::/112" + // CertStrings holds the string representation of a certificate and key. type CertStrings struct { Crt string @@ -29,19 +42,61 @@ type CertStrings struct { // Input holds info about certs, ips, and node type. type Input struct { - Certs *Certs - MasterIPs []string - Index int + Certs *Certs + MasterIPs []string + ClusterName string ServiceDomain string PodNet []string ServiceNet []string - Endpoints string KubernetesVersion string KubeadmTokens *KubeadmTokens TrustdInfo *TrustdInfo InitToken *token.Token - IP net.IP + + // + // Runtime variables + // + + // Index is the index of the current master + Index int + + // IP is the IP address of the current master + IP net.IP +} + +// Endpoints returns the formatted set of Master IP addresses +func (i *Input) Endpoints() (out string) { + if i == nil || len(i.MasterIPs) < 1 { + panic("cannot Endpoints without any Master IPs") + } + for index, addr := range i.MasterIPs { + if index > 0 { + out += ", " + } + out += fmt.Sprintf(`"%s"`, addr) + } + return +} + +// GetControlPlaneEndpoint returns the formatted host:port of the first master node +func (i *Input) GetControlPlaneEndpoint(port string) string { + + if i == nil || len(i.MasterIPs) < 1 { + panic("cannot GetControlPlaneEndpoint without any Master IPs") + } + + // Each master after the first should reference the next-lower master index. + // Thus, master-2 references master-1 and master-3 references master-2. + refMaster := 0 + if i.Index > 1 { + refMaster = i.Index - 1 + } + + if port == "" { + return tnet.FormatAddress(i.MasterIPs[refMaster]) + } + return net.JoinHostPort(i.MasterIPs[refMaster], port) } // Certs holds the base64 encoded keys and certificates. @@ -118,11 +173,34 @@ func genToken(lenFirst int, lenSecond int) (string, error) { return tokenTemp[0] + "." + tokenTemp[1], nil } +func isIPv6(addrs ...string) bool { + for _, a := range addrs { + if ip := net.ParseIP(a); ip != nil { + if ip.To4() == nil { + return true + } + } + } + return false +} + // NewInput generates the sensitive data required to generate all userdata // types. // nolint: gocyclo func NewInput(clustername string, masterIPs []string) (input *Input, err error) { + var loopbackIP, podNet, serviceNet string + + if isIPv6(masterIPs...) { + loopbackIP = "::1" + podNet = DefaultIPv6PodNet + serviceNet = DefaultIPv6ServiceNet + } else { + loopbackIP = "127.0.0.1" + podNet = DefaultIPv4PodNet + serviceNet = DefaultIPv4ServiceNet + } + //Gen trustd token strings kubeadmBootstrapToken, err := genToken(6, 16) if err != nil { @@ -184,7 +262,7 @@ func NewInput(clustername string, masterIPs []string) (input *Input, err error) if err != nil { return nil, err } - ips := []net.IP{net.ParseIP("127.0.0.1")} + ips := []net.IP{net.ParseIP(loopbackIP)} opts = []x509.Option{x509.IPAddresses(ips)} csr, err := x509.NewCertificateSigningRequest(adminKeyEC, opts...) if err != nil { @@ -231,11 +309,10 @@ func NewInput(clustername string, masterIPs []string) (input *Input, err error) input = &Input{ Certs: certs, MasterIPs: masterIPs, - PodNet: []string{"10.244.0.0/16"}, - ServiceNet: []string{"10.96.0.0/12"}, + PodNet: []string{podNet}, + ServiceNet: []string{serviceNet}, ServiceDomain: "cluster.local", ClusterName: clustername, - Endpoints: strings.Join(masterIPs, ", "), KubernetesVersion: constants.KubernetesVersion, KubeadmTokens: kubeadmTokens, TrustdInfo: trustdInfo, diff --git a/pkg/userdata/generate/generate_test.go b/pkg/userdata/generate/generate_test.go index 48d4ac231..4f0bc2a31 100644 --- a/pkg/userdata/generate/generate_test.go +++ b/pkg/userdata/generate/generate_test.go @@ -5,6 +5,7 @@ package generate_test import ( + "fmt" "net" "testing" @@ -15,7 +16,8 @@ import ( ) var ( - input *generate.Input + input *generate.Input + inputv6 *generate.Input ) type GenerateSuite struct { @@ -30,8 +32,13 @@ func (suite *GenerateSuite) SetupSuite() { var err error input, err = generate.NewInput("test", []string{"10.0.1.5", "10.0.1.6", "10.0.1.7"}) suite.Require().NoError(err) + + inputv6, err = generate.NewInput("test", []string{"2001:db8::1", "2001:db8::2", "2001:db8::3"}) + suite.Require().NoError(err) } +// TODO: this is triggering a false positive for the dupl test, between TestGenerateControlPlaneSuccess +// nolint: dupl func (suite *GenerateSuite) TestGenerateInitSuccess() { input.IP = net.ParseIP("10.0.1.5") dataString, err := generate.Userdata(generate.TypeInit, input) @@ -39,8 +46,17 @@ func (suite *GenerateSuite) TestGenerateInitSuccess() { data := &userdata.UserData{} err = yaml.Unmarshal([]byte(dataString), data) suite.Require().NoError(err) + + inputv6.IP = net.ParseIP("2001:db8::1") + dataString, err = generate.Userdata(generate.TypeInit, inputv6) + suite.Require().NoError(err) + data = &userdata.UserData{} + err = yaml.Unmarshal([]byte(dataString), data) + suite.Require().NoError(err) } +// TODO: this is triggering a false positive for the dupl test, between TestGenerateInitSuccess +// nolint: dupl func (suite *GenerateSuite) TestGenerateControlPlaneSuccess() { input.IP = net.ParseIP("10.0.1.6") dataString, err := generate.Userdata(generate.TypeControlPlane, input) @@ -48,6 +64,13 @@ func (suite *GenerateSuite) TestGenerateControlPlaneSuccess() { data := &userdata.UserData{} err = yaml.Unmarshal([]byte(dataString), data) suite.Require().NoError(err) + + inputv6.IP = net.ParseIP("2001:db8::2") + dataString, err = generate.Userdata(generate.TypeControlPlane, inputv6) + suite.Require().NoError(err) + data = &userdata.UserData{} + err = yaml.Unmarshal([]byte(dataString), data) + suite.Require().NoError(err) } func (suite *GenerateSuite) TestGenerateWorkerSuccess() { @@ -56,9 +79,63 @@ func (suite *GenerateSuite) TestGenerateWorkerSuccess() { data := &userdata.UserData{} err = yaml.Unmarshal([]byte(dataString), data) suite.Require().NoError(err) + + dataString, err = generate.Userdata(generate.TypeJoin, inputv6) + suite.Require().NoError(err) + data = &userdata.UserData{} + err = yaml.Unmarshal([]byte(dataString), data) + suite.Require().NoError(err) } func (suite *GenerateSuite) TestGenerateTalosconfigSuccess() { _, err := generate.Talosconfig(input) suite.Require().NoError(err) + + _, err = generate.Talosconfig(inputv6) + suite.Require().NoError(err) +} + +func (suite *GenerateSuite) TestGetControlPlaneEndpoint() { + ep := input.GetControlPlaneEndpoint("6443") + suite.Require().Equal(input.MasterIPs[0]+":6443", ep) + + ep = input.GetControlPlaneEndpoint("443") + suite.Require().Equal(input.MasterIPs[0]+":443", ep) + + ep = inputv6.GetControlPlaneEndpoint("6443") + suite.Require().Equal(fmt.Sprintf("[%s]:6443", inputv6.MasterIPs[0]), ep) + + ep = input.GetControlPlaneEndpoint("") + suite.Require().Equal(input.MasterIPs[0], ep) + + ep = inputv6.GetControlPlaneEndpoint("") + suite.Require().Equal(fmt.Sprintf("[%s]", inputv6.MasterIPs[0]), ep) + + inputv6.IP = net.ParseIP("2001:db8::1") + inputv6.Index = 0 + suite.Require().Equal( + fmt.Sprintf("[%s]", inputv6.MasterIPs[0]), + inputv6.GetControlPlaneEndpoint(""), + ) + + inputv6.IP = net.ParseIP("2001:db8::2") + inputv6.Index = 1 + suite.Require().Equal( + fmt.Sprintf("[%s]", inputv6.MasterIPs[0]), + inputv6.GetControlPlaneEndpoint(""), + ) + + inputv6.IP = net.ParseIP("2001:db8::3") + inputv6.Index = 2 + suite.Require().Equal( + fmt.Sprintf("[%s]", inputv6.MasterIPs[1]), + inputv6.GetControlPlaneEndpoint(""), + ) + + inputv6.IP = net.ParseIP("2001:db8::d") + inputv6.Index = 0 + suite.Require().Equal( + fmt.Sprintf("[%s]", inputv6.MasterIPs[0]), + inputv6.GetControlPlaneEndpoint(""), + ) } diff --git a/pkg/userdata/generate/init.go b/pkg/userdata/generate/init.go index 36c54e6b7..22f10b6ed 100644 --- a/pkg/userdata/generate/init.go +++ b/pkg/userdata/generate/init.go @@ -36,9 +36,9 @@ services: kind: ClusterConfiguration clusterName: {{ .ClusterName }} kubernetesVersion: {{ .KubernetesVersion }} - controlPlaneEndpoint: {{ .IP }}:443 + controlPlaneEndpoint: "{{ .GetControlPlaneEndpoint "443" }}" apiServer: - certSANs: [ {{ range $i,$ip := .MasterIPs }}{{if $i}},{{end}}"{{$ip}}"{{end}}, "127.0.0.1" ] + certSANs: [ {{ range $i,$ip := .MasterIPs }}{{if $i}},{{end}}"{{$ip}}"{{end}}, "127.0.0.1", "::1" ] extraArgs: runtime-config: settings.k8s.io/v1alpha1=true feature-gates: ExperimentalCriticalPodAnnotation=true @@ -51,8 +51,8 @@ services: feature-gates: ExperimentalCriticalPodAnnotation=true networking: dnsDomain: {{ .ServiceDomain }} - podSubnet: {{ index .PodNet 0 }} - serviceSubnet: {{ index .ServiceNet 0 }} + podSubnet: "{{ index .PodNet 0 }}" + serviceSubnet: "{{ index .ServiceNet 0 }}" --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration @@ -67,5 +67,5 @@ services: trustd: token: '{{ .TrustdInfo.Token }}' endpoints: [ {{ .Endpoints }} ] - certSANs: [ "{{ .IP }}", "127.0.0.1" ] + certSANs: [ "{{ .IP }}", "127.0.0.1", "::1" ] ` diff --git a/pkg/userdata/generate/join.go b/pkg/userdata/generate/join.go index 8b479f692..c7a5a5eab 100644 --- a/pkg/userdata/generate/join.go +++ b/pkg/userdata/generate/join.go @@ -18,7 +18,7 @@ services: bootstrapToken: token: '{{ .KubeadmTokens.BootstrapToken }}' unsafeSkipCAVerification: true - apiServerEndpoint: {{ index .MasterIPs 0 }}:443 + apiServerEndpoint: "{{ .GetControlPlaneEndpoint "443" }}" nodeRegistration: taints: [] kubeletExtraArgs: diff --git a/pkg/userdata/generate/talosconfig.go b/pkg/userdata/generate/talosconfig.go index ff65d2938..d2db51032 100644 --- a/pkg/userdata/generate/talosconfig.go +++ b/pkg/userdata/generate/talosconfig.go @@ -12,7 +12,7 @@ func Talosconfig(in *Input) (string, error) { const talosconfigTempl = `context: {{ .ClusterName }} contexts: {{ .ClusterName }}: - target: {{ index .MasterIPs 0 }} + target: {{ .GetControlPlaneEndpoint "" }} ca: {{ .Certs.OsCert }} crt: {{ .Certs.AdminCert }} key: {{ .Certs.AdminKey }}