fix: format IPv6 host entries properly

This reworks a bunch of the formatting for the userdata generation to
output a cleaner talos config when using IPv6 masters and `osctl config
generate`.

Please note that this changes the scope of concern for master indexing,
keeping `osctl` blissfully unaware of the master-reference chaining.
All it does is report the index of the master it is trying to generate.
The generator itself handles the reference chaining.

Fixes #916, fixes #917, and fixes #918

Signed-off-by: Seán C McCord <ulexus@gmail.com>

cmd/osctl/cmd/cluster.go

@@ -178,9 +178,6 @@ func createNodes(requests []*node.Request) (err error) {
 			// node comes up
 			// 1 <- 2 <- 3 <- 4 <- 5 ...
 			req.Input.Index = idx
-			if req.Input.Index > 0 {
-				req.Input.Index--
-			}
 			if req.IP != nil {
 				req.Input.IP = req.IP
 			}

cmd/osctl/cmd/config.go

@@ -144,7 +144,6 @@ var configGenerateCmd = &cobra.Command{
 			if input.Index == 0 {
 				udType = generate.TypeInit
 			} else {
-				input.Index--
 				udType = generate.TypeControlPlane
 			}
 

pkg/userdata/generate/controlplane.go

@@ -28,7 +28,7 @@ services:
         bootstrapToken:
           token: '{{ .KubeadmTokens.BootstrapToken }}'
           unsafeSkipCAVerification: true
-          apiServerEndpoint: {{ index .MasterIPs .Index }}:6443
+          apiServerEndpoint: "{{ .GetControlPlaneEndpoint "6443" }}"
       nodeRegistration:
         taints: []
         kubeletExtraArgs:

pkg/userdata/generate/generate.go

@@ -12,15 +12,28 @@ import (
 	"encoding/base64"
 	"encoding/pem"
 	"errors"
+	"fmt"
 	"net"
-	"strings"
 	"text/template"
 
 	"github.com/talos-systems/talos/pkg/constants"
 	"github.com/talos-systems/talos/pkg/crypto/x509"
+	tnet "github.com/talos-systems/talos/pkg/net"
 	"github.com/talos-systems/talos/pkg/userdata/token"
 )
 
+// DefaultIPv4PodNet is the network to be used for kubernetes Pods when using IPv4-based master nodes
+const DefaultIPv4PodNet = "10.244.0.0/16"
+
+// DefaultIPv4ServiceNet is the network to be used for kubernetes Services when using IPv4-based master nodes
+const DefaultIPv4ServiceNet = "10.96.0.0/12"
+
+// DefaultIPv6PodNet is the network to be used for kubernetes Pods when using IPv6-based master nodes
+const DefaultIPv6PodNet = "fc00:db8:10::/56"
+
+// DefaultIPv6ServiceNet is the network to be used for kubernetes Services when using IPv6-based master nodes
+const DefaultIPv6ServiceNet = "fc00:db8:20::/112"
+
 // CertStrings holds the string representation of a certificate and key.
 type CertStrings struct {
 	Crt string
@@ -31,19 +44,61 @@ type CertStrings struct {
 type Input struct {
 	Certs     *Certs
 	MasterIPs []string
-	Index             int
+
 	ClusterName       string
 	ServiceDomain     string
 	PodNet            []string
 	ServiceNet        []string
-	Endpoints         string
 	KubernetesVersion string
 	KubeadmTokens     *KubeadmTokens
 	TrustdInfo        *TrustdInfo
 	InitToken         *token.Token
+
+	//
+	// Runtime variables
+	//
+
+	// Index is the index of the current master
+	Index int
+
+	// IP is the IP address of the current master
 	IP net.IP
 }
 
+// Endpoints returns the formatted set of Master IP addresses
+func (i *Input) Endpoints() (out string) {
+	if i == nil || len(i.MasterIPs) < 1 {
+		panic("cannot Endpoints without any Master IPs")
+	}
+	for index, addr := range i.MasterIPs {
+		if index > 0 {
+			out += ", "
+		}
+		out += fmt.Sprintf(`"%s"`, addr)
+	}
+	return
+}
+
+// GetControlPlaneEndpoint returns the formatted host:port of the first master node
+func (i *Input) GetControlPlaneEndpoint(port string) string {
+
+	if i == nil || len(i.MasterIPs) < 1 {
+		panic("cannot GetControlPlaneEndpoint without any Master IPs")
+	}
+
+	// Each master after the first should reference the next-lower master index.
+	// Thus, master-2 references master-1 and master-3 references master-2.
+	refMaster := 0
+	if i.Index > 1 {
+		refMaster = i.Index - 1
+	}
+
+	if port == "" {
+		return tnet.FormatAddress(i.MasterIPs[refMaster])
+	}
+	return net.JoinHostPort(i.MasterIPs[refMaster], port)
+}
+
 // Certs holds the base64 encoded keys and certificates.
 type Certs struct {
 	AdminCert string
@@ -118,11 +173,34 @@ func genToken(lenFirst int, lenSecond int) (string, error) {
 	return tokenTemp[0] + "." + tokenTemp[1], nil
 }
 
+func isIPv6(addrs ...string) bool {
+	for _, a := range addrs {
+		if ip := net.ParseIP(a); ip != nil {
+			if ip.To4() == nil {
+				return true
+			}
+		}
+	}
+	return false
+}
+
 // NewInput generates the sensitive data required to generate all userdata
 // types.
 // nolint: gocyclo
 func NewInput(clustername string, masterIPs []string) (input *Input, err error) {
 
+	var loopbackIP, podNet, serviceNet string
+
+	if isIPv6(masterIPs...) {
+		loopbackIP = "::1"
+		podNet = DefaultIPv6PodNet
+		serviceNet = DefaultIPv6ServiceNet
+	} else {
+		loopbackIP = "127.0.0.1"
+		podNet = DefaultIPv4PodNet
+		serviceNet = DefaultIPv4ServiceNet
+	}
+
 	//Gen trustd token strings
 	kubeadmBootstrapToken, err := genToken(6, 16)
 	if err != nil {
@@ -184,7 +262,7 @@ func NewInput(clustername string, masterIPs []string) (input *Input, err error)
 	if err != nil {
 		return nil, err
 	}
-	ips := []net.IP{net.ParseIP("127.0.0.1")}
+	ips := []net.IP{net.ParseIP(loopbackIP)}
 	opts = []x509.Option{x509.IPAddresses(ips)}
 	csr, err := x509.NewCertificateSigningRequest(adminKeyEC, opts...)
 	if err != nil {
@@ -231,11 +309,10 @@ func NewInput(clustername string, masterIPs []string) (input *Input, err error)
 	input = &Input{
 		Certs:             certs,
 		MasterIPs:         masterIPs,
-		PodNet:            []string{"10.244.0.0/16"},
+		PodNet:            []string{podNet},
-		ServiceNet:        []string{"10.96.0.0/12"},
+		ServiceNet:        []string{serviceNet},
 		ServiceDomain:     "cluster.local",
 		ClusterName:       clustername,
-		Endpoints:         strings.Join(masterIPs, ", "),
 		KubernetesVersion: constants.KubernetesVersion,
 		KubeadmTokens:     kubeadmTokens,
 		TrustdInfo:        trustdInfo,

pkg/userdata/generate/generate_test.go

@@ -5,6 +5,7 @@
 package generate_test
 
 import (
+	"fmt"
 	"net"
 	"testing"
 
@@ -16,6 +17,7 @@ import (
 
 var (
 	input   *generate.Input
+	inputv6 *generate.Input
 )
 
 type GenerateSuite struct {
@@ -30,8 +32,13 @@ func (suite *GenerateSuite) SetupSuite() {
 	var err error
 	input, err = generate.NewInput("test", []string{"10.0.1.5", "10.0.1.6", "10.0.1.7"})
 	suite.Require().NoError(err)
+
+	inputv6, err = generate.NewInput("test", []string{"2001:db8::1", "2001:db8::2", "2001:db8::3"})
+	suite.Require().NoError(err)
 }
 
+// TODO: this is triggering a false positive for the dupl test, between TestGenerateControlPlaneSuccess
+// nolint: dupl
 func (suite *GenerateSuite) TestGenerateInitSuccess() {
 	input.IP = net.ParseIP("10.0.1.5")
 	dataString, err := generate.Userdata(generate.TypeInit, input)
@@ -39,8 +46,17 @@ func (suite *GenerateSuite) TestGenerateInitSuccess() {
 	data := &userdata.UserData{}
 	err = yaml.Unmarshal([]byte(dataString), data)
 	suite.Require().NoError(err)
+
+	inputv6.IP = net.ParseIP("2001:db8::1")
+	dataString, err = generate.Userdata(generate.TypeInit, inputv6)
+	suite.Require().NoError(err)
+	data = &userdata.UserData{}
+	err = yaml.Unmarshal([]byte(dataString), data)
+	suite.Require().NoError(err)
 }
 
+// TODO: this is triggering a false positive for the dupl test, between TestGenerateInitSuccess
+// nolint: dupl
 func (suite *GenerateSuite) TestGenerateControlPlaneSuccess() {
 	input.IP = net.ParseIP("10.0.1.6")
 	dataString, err := generate.Userdata(generate.TypeControlPlane, input)
@@ -48,6 +64,13 @@ func (suite *GenerateSuite) TestGenerateControlPlaneSuccess() {
 	data := &userdata.UserData{}
 	err = yaml.Unmarshal([]byte(dataString), data)
 	suite.Require().NoError(err)
+
+	inputv6.IP = net.ParseIP("2001:db8::2")
+	dataString, err = generate.Userdata(generate.TypeControlPlane, inputv6)
+	suite.Require().NoError(err)
+	data = &userdata.UserData{}
+	err = yaml.Unmarshal([]byte(dataString), data)
+	suite.Require().NoError(err)
 }
 
 func (suite *GenerateSuite) TestGenerateWorkerSuccess() {
@@ -56,9 +79,63 @@ func (suite *GenerateSuite) TestGenerateWorkerSuccess() {
 	data := &userdata.UserData{}
 	err = yaml.Unmarshal([]byte(dataString), data)
 	suite.Require().NoError(err)
+
+	dataString, err = generate.Userdata(generate.TypeJoin, inputv6)
+	suite.Require().NoError(err)
+	data = &userdata.UserData{}
+	err = yaml.Unmarshal([]byte(dataString), data)
+	suite.Require().NoError(err)
 }
 
 func (suite *GenerateSuite) TestGenerateTalosconfigSuccess() {
 	_, err := generate.Talosconfig(input)
 	suite.Require().NoError(err)
+
+	_, err = generate.Talosconfig(inputv6)
+	suite.Require().NoError(err)
+}
+
+func (suite *GenerateSuite) TestGetControlPlaneEndpoint() {
+	ep := input.GetControlPlaneEndpoint("6443")
+	suite.Require().Equal(input.MasterIPs[0]+":6443", ep)
+
+	ep = input.GetControlPlaneEndpoint("443")
+	suite.Require().Equal(input.MasterIPs[0]+":443", ep)
+
+	ep = inputv6.GetControlPlaneEndpoint("6443")
+	suite.Require().Equal(fmt.Sprintf("[%s]:6443", inputv6.MasterIPs[0]), ep)
+
+	ep = input.GetControlPlaneEndpoint("")
+	suite.Require().Equal(input.MasterIPs[0], ep)
+
+	ep = inputv6.GetControlPlaneEndpoint("")
+	suite.Require().Equal(fmt.Sprintf("[%s]", inputv6.MasterIPs[0]), ep)
+
+	inputv6.IP = net.ParseIP("2001:db8::1")
+	inputv6.Index = 0
+	suite.Require().Equal(
+		fmt.Sprintf("[%s]", inputv6.MasterIPs[0]),
+		inputv6.GetControlPlaneEndpoint(""),
+	)
+
+	inputv6.IP = net.ParseIP("2001:db8::2")
+	inputv6.Index = 1
+	suite.Require().Equal(
+		fmt.Sprintf("[%s]", inputv6.MasterIPs[0]),
+		inputv6.GetControlPlaneEndpoint(""),
+	)
+
+	inputv6.IP = net.ParseIP("2001:db8::3")
+	inputv6.Index = 2
+	suite.Require().Equal(
+		fmt.Sprintf("[%s]", inputv6.MasterIPs[1]),
+		inputv6.GetControlPlaneEndpoint(""),
+	)
+
+	inputv6.IP = net.ParseIP("2001:db8::d")
+	inputv6.Index = 0
+	suite.Require().Equal(
+		fmt.Sprintf("[%s]", inputv6.MasterIPs[0]),
+		inputv6.GetControlPlaneEndpoint(""),
+	)
 }

pkg/userdata/generate/init.go

@@ -36,9 +36,9 @@ services:
       kind: ClusterConfiguration
       clusterName: {{ .ClusterName }}
       kubernetesVersion: {{ .KubernetesVersion }}
-      controlPlaneEndpoint: {{ .IP }}:443
+      controlPlaneEndpoint: "{{ .GetControlPlaneEndpoint "443" }}"
       apiServer:
-        certSANs: [ {{ range $i,$ip := .MasterIPs }}{{if $i}},{{end}}"{{$ip}}"{{end}}, "127.0.0.1" ]
+        certSANs: [ {{ range $i,$ip := .MasterIPs }}{{if $i}},{{end}}"{{$ip}}"{{end}}, "127.0.0.1", "::1" ]
         extraArgs:
           runtime-config: settings.k8s.io/v1alpha1=true
           feature-gates: ExperimentalCriticalPodAnnotation=true
@@ -51,8 +51,8 @@ services:
           feature-gates: ExperimentalCriticalPodAnnotation=true
       networking:
         dnsDomain: {{ .ServiceDomain }}
-        podSubnet: {{ index .PodNet 0 }}
+        podSubnet: "{{ index .PodNet 0 }}"
-        serviceSubnet: {{ index .ServiceNet 0 }}
+        serviceSubnet: "{{ index .ServiceNet 0 }}"
       ---
       apiVersion: kubelet.config.k8s.io/v1beta1
       kind: KubeletConfiguration
@@ -67,5 +67,5 @@ services:
   trustd:
     token: '{{ .TrustdInfo.Token }}'
     endpoints: [ {{ .Endpoints }} ]
-    certSANs: [ "{{ .IP }}", "127.0.0.1" ]
+    certSANs: [ "{{ .IP }}", "127.0.0.1", "::1" ]
 `

pkg/userdata/generate/join.go

@@ -18,7 +18,7 @@ services:
         bootstrapToken:
           token: '{{ .KubeadmTokens.BootstrapToken }}'
           unsafeSkipCAVerification: true
-          apiServerEndpoint: {{ index .MasterIPs 0 }}:443
+          apiServerEndpoint: "{{ .GetControlPlaneEndpoint "443" }}"
       nodeRegistration:
         taints: []
         kubeletExtraArgs:

pkg/userdata/generate/talosconfig.go

@@ -12,7 +12,7 @@ func Talosconfig(in *Input) (string, error) {
 const talosconfigTempl = `context: {{ .ClusterName }}
 contexts:
   {{ .ClusterName }}:
-    target: {{ index .MasterIPs 0 }}
+    target: {{ .GetControlPlaneEndpoint "" }}
     ca: {{ .Certs.OsCert }}
     crt: {{ .Certs.AdminCert }}
     key: {{ .Certs.AdminKey }}