mirror of
https://github.com/siderolabs/talos.git
synced 2025-08-06 14:47:05 +02:00
7f0300f108
Bump all dependencies, many small changes due to new golangci-lint version. Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>
472 lines
12 KiB
Go
472 lines
12 KiB
Go
// This Source Code Form is subject to the terms of the Mozilla Public
|
|
// License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
//go:build integration_k8s
|
|
|
|
package k8s
|
|
|
|
import (
|
|
"context"
|
|
"crypto/tls"
|
|
_ "embed"
|
|
"fmt"
|
|
"net"
|
|
"net/netip"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/siderolabs/gen/ensure"
|
|
"github.com/siderolabs/gen/xslices"
|
|
"github.com/siderolabs/go-pointer"
|
|
"github.com/siderolabs/go-retry/retry"
|
|
"github.com/stretchr/testify/assert"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
podsecurity "k8s.io/pod-security-admission/api"
|
|
|
|
"github.com/siderolabs/talos/internal/integration/base"
|
|
"github.com/siderolabs/talos/pkg/cluster"
|
|
"github.com/siderolabs/talos/pkg/cluster/check"
|
|
machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine"
|
|
"github.com/siderolabs/talos/pkg/machinery/client"
|
|
"github.com/siderolabs/talos/pkg/machinery/config/generate"
|
|
"github.com/siderolabs/talos/pkg/machinery/config/machine"
|
|
"github.com/siderolabs/talos/pkg/machinery/constants"
|
|
"github.com/siderolabs/talos/pkg/machinery/version"
|
|
)
|
|
|
|
// TinkSuite verifies Talos-in-Kubernetes.
|
|
type TinkSuite struct {
|
|
base.K8sSuite
|
|
}
|
|
|
|
// SuiteName ...
|
|
func (suite *TinkSuite) SuiteName() string {
|
|
return "k8s.TinkSuite"
|
|
}
|
|
|
|
//go:embed testdata/local-path-storage.yaml
|
|
var localPathStorageYAML []byte
|
|
|
|
const (
|
|
tinkK8sPort = "k8s-api"
|
|
tinkTalosPort = "talos-api"
|
|
)
|
|
|
|
// TestDeploy verifies that tink can be deployed with a single control-plane node.
|
|
func (suite *TinkSuite) TestDeploy() {
|
|
if testing.Short() {
|
|
suite.T().Skip("skipping in short mode")
|
|
}
|
|
|
|
if suite.Cluster == nil {
|
|
suite.T().Skip("without full cluster state reaching out to the node IP is not reliable")
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
|
suite.T().Cleanup(cancel)
|
|
|
|
localPathStorage := suite.ParseManifests(localPathStorageYAML)
|
|
|
|
suite.T().Cleanup(func() {
|
|
cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cleanupCancel()
|
|
|
|
suite.DeleteManifests(cleanUpCtx, localPathStorage)
|
|
})
|
|
|
|
suite.ApplyManifests(ctx, localPathStorage)
|
|
|
|
const (
|
|
namespace = "talos-in-talos"
|
|
service = "talos"
|
|
ss = "talos-cp"
|
|
)
|
|
|
|
talosImage := fmt.Sprintf("%s:%s", suite.TalosImage, version.Tag)
|
|
|
|
suite.T().Logf("deploying Talos-in-Kubernetes from image %s", talosImage)
|
|
|
|
tinkManifests := suite.getTinkManifests(namespace, service, ss, talosImage)
|
|
|
|
suite.T().Cleanup(func() {
|
|
cleanUpCtx, cleanupCancel := context.WithTimeout(context.Background(), time.Minute)
|
|
defer cleanupCancel()
|
|
|
|
suite.DeleteManifests(cleanUpCtx, tinkManifests)
|
|
})
|
|
|
|
suite.ApplyManifests(ctx, tinkManifests)
|
|
|
|
// wait for the control-plane pod to be running
|
|
suite.Require().NoError(suite.WaitForPodToBeRunning(ctx, time.Minute, namespace, ss+"-0"))
|
|
|
|
// read back Service to figure out the ports
|
|
svc, err := suite.Clientset.CoreV1().Services(namespace).Get(ctx, service, metav1.GetOptions{})
|
|
suite.Require().NoError(err)
|
|
|
|
var k8sPort, talosPort int
|
|
|
|
for _, portSpec := range svc.Spec.Ports {
|
|
switch portSpec.Name {
|
|
case tinkK8sPort:
|
|
k8sPort = int(portSpec.NodePort)
|
|
case tinkTalosPort:
|
|
talosPort = int(portSpec.NodePort)
|
|
}
|
|
}
|
|
|
|
suite.Require().NotZero(k8sPort)
|
|
suite.Require().NotZero(talosPort)
|
|
|
|
// find pod IP
|
|
pod, err := suite.Clientset.CoreV1().Pods(namespace).Get(ctx, ss+"-0", metav1.GetOptions{})
|
|
suite.Require().NoError(err)
|
|
|
|
suite.Require().NotEmpty(pod.Status.PodIP)
|
|
|
|
podIP := netip.MustParseAddr(pod.Status.PodIP)
|
|
|
|
// grab any random lbNode IP
|
|
lbNode := suite.RandomDiscoveredNodeInternalIP()
|
|
|
|
talosEndpoint := net.JoinHostPort(lbNode, strconv.Itoa(talosPort))
|
|
|
|
in, err := generate.NewInput(namespace,
|
|
fmt.Sprintf("https://%s", net.JoinHostPort(lbNode, strconv.Itoa(k8sPort))),
|
|
constants.DefaultKubernetesVersion,
|
|
generate.WithAdditionalSubjectAltNames([]string{lbNode}),
|
|
generate.WithHostDNSForwardKubeDNSToHost(true),
|
|
)
|
|
suite.Require().NoError(err)
|
|
|
|
// override pod/service subnets, as Talos-in-Talos would use it for "host" addresses
|
|
in.PodNet = []string{"192.168.0.0/20"}
|
|
in.ServiceNet = []string{"192.168.128.0/20"}
|
|
|
|
cpCfg, err := in.Config(machine.TypeControlPlane)
|
|
suite.Require().NoError(err)
|
|
|
|
cpCfgBytes, err := cpCfg.Bytes()
|
|
suite.Require().NoError(err)
|
|
|
|
readyErr := suite.waitForEndpointReady(talosEndpoint)
|
|
if readyErr != nil {
|
|
suite.LogPodLogs(ctx, namespace, ss+"-0")
|
|
}
|
|
|
|
suite.Require().NoError(readyErr)
|
|
|
|
insecureClient, err := client.New(ctx,
|
|
client.WithEndpoints(talosEndpoint),
|
|
client.WithTLSConfig(&tls.Config{InsecureSkipVerify: true}),
|
|
)
|
|
suite.Require().NoError(err)
|
|
|
|
suite.T().Log("applying initial configuration")
|
|
|
|
_, err = insecureClient.ApplyConfiguration(ctx, &machineapi.ApplyConfigurationRequest{
|
|
Data: cpCfgBytes,
|
|
Mode: machineapi.ApplyConfigurationRequest_AUTO,
|
|
})
|
|
suite.Require().NoError(err)
|
|
|
|
// bootstrap
|
|
talosconfig, err := in.Talosconfig()
|
|
suite.Require().NoError(err)
|
|
|
|
talosconfig.Contexts[talosconfig.Context].Endpoints = []string{talosEndpoint}
|
|
talosconfig.Contexts[talosconfig.Context].Nodes = []string{podIP.String()}
|
|
|
|
suite.T().Logf("talosconfig = %s", string(ensure.Value(talosconfig.Bytes())))
|
|
|
|
readyErr = suite.waitForEndpointReady(talosEndpoint)
|
|
if readyErr != nil {
|
|
suite.LogPodLogs(ctx, namespace, ss+"-0")
|
|
}
|
|
|
|
suite.Require().NoError(readyErr)
|
|
|
|
talosClient, err := client.New(ctx,
|
|
client.WithConfigContext(talosconfig.Contexts[talosconfig.Context]),
|
|
)
|
|
suite.Require().NoError(err)
|
|
|
|
suite.T().Log("bootstrapping")
|
|
|
|
suite.Require().EventuallyWithT(func(collect *assert.CollectT) {
|
|
asrt := assert.New(collect)
|
|
|
|
asrt.NoError(talosClient.Bootstrap(ctx, &machineapi.BootstrapRequest{}))
|
|
}, time.Minute, 100*time.Millisecond)
|
|
|
|
clusterAccess := &tinkClusterAccess{
|
|
KubernetesClient: cluster.KubernetesClient{
|
|
ClientProvider: &cluster.ConfigClientProvider{
|
|
TalosConfig: talosconfig,
|
|
},
|
|
},
|
|
|
|
nodeIP: podIP,
|
|
}
|
|
|
|
suite.Require().NoError(
|
|
check.Wait(
|
|
ctx,
|
|
clusterAccess,
|
|
check.DefaultClusterChecks(),
|
|
check.StderrReporter(),
|
|
),
|
|
)
|
|
}
|
|
|
|
type tinkClusterAccess struct {
|
|
cluster.KubernetesClient
|
|
|
|
nodeIP netip.Addr
|
|
}
|
|
|
|
func (access *tinkClusterAccess) Nodes() []cluster.NodeInfo {
|
|
return []cluster.NodeInfo{
|
|
{
|
|
InternalIP: access.nodeIP,
|
|
IPs: []netip.Addr{access.nodeIP},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (access *tinkClusterAccess) NodesByType(typ machine.Type) []cluster.NodeInfo {
|
|
switch typ {
|
|
case machine.TypeControlPlane:
|
|
return []cluster.NodeInfo{
|
|
{
|
|
InternalIP: access.nodeIP,
|
|
IPs: []netip.Addr{access.nodeIP},
|
|
},
|
|
}
|
|
case machine.TypeWorker, machine.TypeInit:
|
|
return nil
|
|
case machine.TypeUnknown:
|
|
fallthrough
|
|
default:
|
|
panic(fmt.Sprintf("unexpected machine type: %s", typ))
|
|
}
|
|
}
|
|
|
|
func (suite *TinkSuite) waitForEndpointReady(endpoint string) error {
|
|
return retry.Constant(30*time.Second, retry.WithUnits(10*time.Millisecond)).Retry(func() error {
|
|
c, err := tls.Dial("tcp", endpoint,
|
|
&tls.Config{
|
|
InsecureSkipVerify: true,
|
|
},
|
|
)
|
|
|
|
if c != nil {
|
|
c.Close() //nolint:errcheck
|
|
}
|
|
|
|
return retry.ExpectedError(err)
|
|
})
|
|
}
|
|
|
|
func (suite *TinkSuite) getTinkManifests(namespace, serviceName, ssName, talosImage string) []unstructured.Unstructured {
|
|
labels := map[string]string{
|
|
"app": "talos-cp",
|
|
}
|
|
|
|
tinkManifests := []runtime.Object{
|
|
&corev1.Namespace{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "Namespace",
|
|
APIVersion: "v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: namespace,
|
|
Labels: map[string]string{
|
|
podsecurity.EnforceLevelLabel: string(podsecurity.LevelPrivileged),
|
|
},
|
|
},
|
|
},
|
|
&corev1.Service{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "Service",
|
|
APIVersion: "v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: serviceName,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: corev1.ServiceSpec{
|
|
Type: corev1.ServiceTypeNodePort,
|
|
Selector: labels,
|
|
Ports: []corev1.ServicePort{
|
|
{
|
|
Name: tinkK8sPort,
|
|
Protocol: corev1.ProtocolTCP,
|
|
Port: constants.DefaultControlPlanePort,
|
|
TargetPort: intstr.FromString(tinkK8sPort),
|
|
},
|
|
{
|
|
Name: tinkTalosPort,
|
|
Protocol: corev1.ProtocolTCP,
|
|
Port: constants.ApidPort,
|
|
TargetPort: intstr.FromString(tinkTalosPort),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
statefulSet := &appsv1.StatefulSet{
|
|
TypeMeta: metav1.TypeMeta{
|
|
Kind: "StatefulSet",
|
|
APIVersion: "apps/v1",
|
|
},
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: ssName,
|
|
Namespace: namespace,
|
|
},
|
|
Spec: appsv1.StatefulSetSpec{
|
|
ServiceName: serviceName,
|
|
Replicas: pointer.To(int32(1)),
|
|
Selector: &metav1.LabelSelector{
|
|
MatchLabels: labels,
|
|
},
|
|
Template: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: labels,
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
Containers: []corev1.Container{
|
|
{
|
|
Name: "talos",
|
|
Image: talosImage,
|
|
ImagePullPolicy: corev1.PullAlways,
|
|
SecurityContext: &corev1.SecurityContext{
|
|
Privileged: pointer.To(true),
|
|
ReadOnlyRootFilesystem: pointer.To(true),
|
|
SeccompProfile: &corev1.SeccompProfile{
|
|
Type: corev1.SeccompProfileTypeUnconfined,
|
|
},
|
|
},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "PLATFORM",
|
|
Value: "container",
|
|
},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("1Gi"),
|
|
corev1.ResourceCPU: resource.MustParse("750m"),
|
|
},
|
|
},
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
ContainerPort: constants.ApidPort,
|
|
Name: tinkTalosPort,
|
|
},
|
|
{
|
|
ContainerPort: constants.DefaultControlPlanePort,
|
|
Name: tinkK8sPort,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, ephemeralMount := range []string{"run", "system", "tmp"} {
|
|
statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts = append(
|
|
statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts,
|
|
corev1.VolumeMount{
|
|
MountPath: "/" + ephemeralMount,
|
|
Name: ephemeralMount,
|
|
},
|
|
)
|
|
|
|
statefulSet.Spec.Template.Spec.Volumes = append(
|
|
statefulSet.Spec.Template.Spec.Volumes,
|
|
corev1.Volume{
|
|
Name: ephemeralMount,
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{},
|
|
},
|
|
},
|
|
)
|
|
}
|
|
|
|
type overlayMountSpec struct {
|
|
MountPoint string
|
|
Size string
|
|
}
|
|
|
|
for _, overlayMount := range append(
|
|
[]overlayMountSpec{
|
|
{
|
|
MountPoint: constants.StateMountPoint,
|
|
Size: "100Mi",
|
|
},
|
|
{
|
|
MountPoint: constants.EphemeralMountPoint,
|
|
Size: "6Gi",
|
|
},
|
|
},
|
|
xslices.Map(
|
|
xslices.Filter(constants.Overlays, func(overlay constants.SELinuxLabeledPath) bool { return overlay.Path != "/opt" }), // /opt/cni/bin contains CNI binaries
|
|
func(mnt constants.SELinuxLabeledPath) overlayMountSpec {
|
|
return overlayMountSpec{
|
|
MountPoint: mnt.Path,
|
|
Size: "100Mi",
|
|
}
|
|
},
|
|
)...,
|
|
) {
|
|
name := strings.ReplaceAll(strings.TrimLeft(overlayMount.MountPoint, "/"), "/", "-")
|
|
|
|
statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts = append(
|
|
statefulSet.Spec.Template.Spec.Containers[0].VolumeMounts,
|
|
corev1.VolumeMount{
|
|
MountPath: overlayMount.MountPoint,
|
|
Name: name,
|
|
},
|
|
)
|
|
|
|
statefulSet.Spec.VolumeClaimTemplates = append(
|
|
statefulSet.Spec.VolumeClaimTemplates,
|
|
corev1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: name,
|
|
},
|
|
Spec: corev1.PersistentVolumeClaimSpec{
|
|
AccessModes: []corev1.PersistentVolumeAccessMode{
|
|
corev1.ReadWriteOnce,
|
|
},
|
|
Resources: corev1.VolumeResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceStorage: resource.MustParse(overlayMount.Size),
|
|
},
|
|
},
|
|
},
|
|
})
|
|
}
|
|
|
|
tinkManifests = append(tinkManifests, statefulSet)
|
|
|
|
return xslices.Map(tinkManifests, suite.ToUnstructured)
|
|
}
|
|
|
|
func init() {
|
|
allSuites = append(allSuites, new(TinkSuite))
|
|
}
|