talos/internal/integration/base/k8s.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

//go:build integration_k8s

package base

import (
	"bufio"
	"bytes"
	"context"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"slices"
	"strings"
	"time"

	"github.com/siderolabs/gen/channel"
	"github.com/siderolabs/gen/xslices"
	"github.com/siderolabs/go-pointer"
	"github.com/siderolabs/go-retry/retry"
	corev1 "k8s.io/api/core/v1"
	eventsv1 "k8s.io/api/events/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/yaml"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/discovery/cached/memory"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/informers"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/restmapper"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"
	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
	"k8s.io/client-go/tools/remotecommand"
	watchtools "k8s.io/client-go/tools/watch"
	"k8s.io/client-go/util/jsonpath"
	"k8s.io/kubectl/pkg/scheme"

	taloskubernetes "github.com/siderolabs/talos/pkg/kubernetes"
	"github.com/siderolabs/talos/pkg/machinery/constants"
)

// K8sSuite is a base suite for K8s tests.
type K8sSuite struct {
	APISuite

	Clientset       *kubernetes.Clientset
	DynamicClient   dynamic.Interface
	DiscoveryClient *discovery.DiscoveryClient
	RestConfig      *rest.Config
	Mapper          *restmapper.DeferredDiscoveryRESTMapper
}

// SetupSuite initializes Kubernetes client.
func (k8sSuite *K8sSuite) SetupSuite() {
	k8sSuite.APISuite.SetupSuite()

	kubeconfig, err := k8sSuite.Client.Kubeconfig(context.Background())
	k8sSuite.Require().NoError(err)

	config, err := clientcmd.BuildConfigFromKubeconfigGetter("", func() (*clientcmdapi.Config, error) {
		return clientcmd.Load(kubeconfig)
	})
	k8sSuite.Require().NoError(err)

	if k8sSuite.K8sEndpoint != "" {
		config.Host = k8sSuite.K8sEndpoint
	}

	k8sSuite.RestConfig = config
	k8sSuite.Clientset, err = kubernetes.NewForConfig(config)
	k8sSuite.Require().NoError(err)

	k8sSuite.DynamicClient, err = dynamic.NewForConfig(config)
	k8sSuite.Require().NoError(err)

	k8sSuite.DiscoveryClient, err = discovery.NewDiscoveryClientForConfig(config)
	k8sSuite.Require().NoError(err)

	k8sSuite.Mapper = restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(k8sSuite.DiscoveryClient))
}

// GetK8sNodeByInternalIP returns the kubernetes node by its internal ip or error if it is not found.
func (k8sSuite *K8sSuite) GetK8sNodeByInternalIP(ctx context.Context, internalIP string) (*corev1.Node, error) {
	nodeList, err := k8sSuite.Clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}

	for _, item := range nodeList.Items {
		for _, address := range item.Status.Addresses {
			if address.Type == corev1.NodeInternalIP {
				if address.Address == internalIP {
					return &item, nil
				}
			}
		}
	}

	return nil, fmt.Errorf("node with internal IP %s not found", internalIP)
}

// CleanupFailedPods deletes all pods in kube-system namespace with the status Failed.
func (k8sSuite *K8sSuite) CleanupFailedPods(ctx context.Context, internalIP string) {
	nodeName, err := k8sSuite.GetK8sNodeByInternalIP(ctx, internalIP)
	if err != nil {
		k8sSuite.T().Logf("failed to get node by internal IP %s: %v", internalIP, err)

		return
	}

	pods, err := k8sSuite.Clientset.CoreV1().Pods("kube-system").List(ctx, metav1.ListOptions{
		FieldSelector: fields.OneTermEqualSelector("spec.nodeName", nodeName.Name).String(),
	})
	if err != nil {
		k8sSuite.T().Logf("failed to list pods in kube-system namespace: %v", err)

		return
	}

	for _, pod := range pods.Items {
		if pod.Status.Phase == corev1.PodFailed {
			if err := k8sSuite.Clientset.CoreV1().Pods("kube-system").Delete(ctx, pod.Name, metav1.DeleteOptions{}); err != nil {
				k8sSuite.T().Logf("failed to delete pod %s: %v", pod.Name, err)
			} else {
				k8sSuite.T().Logf("deleted pod %s", pod.Name)
			}
		}
	}
}

// WaitForK8sNodeReadinessStatus waits for node to have the given status.
// It retries until the node with the name is found and matches the expected condition.
func (k8sSuite *K8sSuite) WaitForK8sNodeReadinessStatus(ctx context.Context, nodeName string, checkFn func(corev1.ConditionStatus) bool) error {
	return retry.Constant(5 * time.Minute).Retry(func() error {
		readinessStatus, err := k8sSuite.GetK8sNodeReadinessStatus(ctx, nodeName)
		if errors.IsNotFound(err) {
			return retry.ExpectedError(err)
		}

		if taloskubernetes.IsRetryableError(err) {
			return retry.ExpectedError(err)
		}

		if err != nil {
			return err
		}

		if !checkFn(readinessStatus) {
			return retry.ExpectedErrorf("node readiness status is %s", readinessStatus)
		}

		return nil
	})
}

// GetK8sNodeReadinessStatus returns the node readiness status of the node.
func (k8sSuite *K8sSuite) GetK8sNodeReadinessStatus(ctx context.Context, nodeName string) (corev1.ConditionStatus, error) {
	node, err := k8sSuite.Clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
	if err != nil {
		return "", err
	}

	for _, condition := range node.Status.Conditions {
		if condition.Type == corev1.NodeReady {
			return condition.Status, nil
		}
	}

	return "", fmt.Errorf("node %s has no readiness condition", nodeName)
}

// DeleteResource deletes the resource with the given GroupVersionResource, namespace and name.
// Does not return an error if the resource is not found.
func (k8sSuite *K8sSuite) DeleteResource(ctx context.Context, gvr schema.GroupVersionResource, ns, name string) error {
	err := k8sSuite.DynamicClient.Resource(gvr).Namespace(ns).Delete(ctx, name, metav1.DeleteOptions{})
	if errors.IsNotFound(err) {
		return nil
	}

	return err
}

// EnsureResourceIsDeleted ensures that the resource with the given GroupVersionResource, namespace and name does not exist on Kubernetes.
// It repeatedly checks the resource for the given duration.
func (k8sSuite *K8sSuite) EnsureResourceIsDeleted(
	ctx context.Context,
	duration time.Duration,
	gvr schema.GroupVersionResource,
	ns, name string,
) error {
	return retry.Constant(duration).RetryWithContext(ctx, func(ctx context.Context) error {
		_, err := k8sSuite.DynamicClient.Resource(gvr).Namespace(ns).Get(ctx, name, metav1.GetOptions{})
		if errors.IsNotFound(err) {
			return nil
		}

		return err
	})
}

// WaitForEventExists waits for the event with the given namespace and check condition to exist on Kubernetes.
func (k8sSuite *K8sSuite) WaitForEventExists(ctx context.Context, ns string, checkFn func(event eventsv1.Event) bool) error {
	return retry.Constant(15*time.Second).RetryWithContext(ctx, func(ctx context.Context) error {
		events, err := k8sSuite.Clientset.EventsV1().Events(ns).List(ctx, metav1.ListOptions{})

		filteredEvents := xslices.Filter(events.Items, func(item eventsv1.Event) bool {
			return checkFn(item)
		})

		if len(filteredEvents) == 0 {
			return retry.ExpectedError(err)
		}

		return nil
	})
}

type podInfo interface {
	Name() string
	WithNodeName(nodeName string) podInfo
	WithQuiet(quiet bool) podInfo
	WithNamespace(namespace string) podInfo
	WithHostVolumeMount(hostPath, mountPath string) podInfo
	Create(ctx context.Context, waitTimeout time.Duration) error
	Delete(ctx context.Context) error
	Exec(ctx context.Context, command string) (string, string, error)
}

type pod struct {
	name      string
	namespace string
	pod       *corev1.Pod
	quiet     bool

	suite *K8sSuite
}

func (p *pod) Name() string {
	return p.name
}

func (p *pod) WithNodeName(nodeName string) podInfo {
	p.pod.Spec.NodeName = nodeName

	return p
}

func (p *pod) WithQuiet(quiet bool) podInfo {
	p.quiet = quiet

	return p
}

func (p *pod) WithNamespace(namespace string) podInfo {
	p.namespace = namespace

	return p
}

func (p *pod) WithHostVolumeMount(hostPath, mountPath string) podInfo {
	name := filepath.Base(hostPath)

	p.pod.Spec.Volumes = append(p.pod.Spec.Volumes, corev1.Volume{
		Name: name,
		VolumeSource: corev1.VolumeSource{
			HostPath: &corev1.HostPathVolumeSource{
				Path: hostPath,
				Type: pointer.To(corev1.HostPathDirectoryOrCreate),
			},
		},
	})

	p.pod.Spec.Containers[0].VolumeMounts = append(p.pod.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{
		Name:      name,
		MountPath: mountPath,
	})

	return p
}

func (p *pod) Create(ctx context.Context, waitTimeout time.Duration) error {
	_, err := p.suite.Clientset.CoreV1().Pods(p.namespace).Create(ctx, p.pod, metav1.CreateOptions{})
	if err != nil {
		return err
	}

	return p.suite.WaitForPodToBeRunning(ctx, waitTimeout, p.namespace, p.name)
}

func (p *pod) Exec(ctx context.Context, command string) (string, string, error) {
	cmd := []string{
		"/bin/sh",
		"-c",
		command,
	}
	req := p.suite.Clientset.CoreV1().RESTClient().Post().Resource("pods").Name(p.name).
		Namespace(p.namespace).SubResource("exec")

	option := &corev1.PodExecOptions{
		Command: cmd,
		Stdin:   false,
		Stdout:  true,
		Stderr:  true,
		TTY:     false,
	}

	req.VersionedParams(
		option,
		scheme.ParameterCodec,
	)

	websocketExec, err := remotecommand.NewWebSocketExecutor(p.suite.RestConfig, "GET", req.URL().String())
	if err != nil {
		return "", "", err
	}

	var stdout, stderr strings.Builder

	err = websocketExec.StreamWithContext(ctx, remotecommand.StreamOptions{
		Stdout: &stdout,
		Stderr: &stderr,
	})
	if err != nil && !p.quiet {
		p.suite.T().Logf(
			"error executing command in pod %s/%s: %v\n\ncommand %q stdout:\n%s\n\ncommand %q stderr:\n%s",
			p.namespace,
			p.name,
			err,
			command,
			stdout.String(),
			command,
			stderr.String(),
		)
	}

	return stdout.String(), stderr.String(), err
}

func (p *pod) Delete(ctx context.Context) error {
	err := p.suite.Clientset.CoreV1().Pods(p.namespace).Delete(ctx, p.name, metav1.DeleteOptions{
		GracePeriodSeconds: pointer.To[int64](0),
	})
	if err != nil && errors.IsNotFound(err) {
		return nil
	}

	return err
}

// NewPrivilegedPod creates a new pod definition with a random suffix
// in the kube-system namespace with privileged security context.
func (k8sSuite *K8sSuite) NewPrivilegedPod(name string) (podInfo, error) {
	randomSuffix := make([]byte, 4)

	if _, err := rand.Read(randomSuffix); err != nil {
		return nil, fmt.Errorf("failed to generate random suffix: %w", err)
	}

	podName := fmt.Sprintf("%s-%x", name, randomSuffix)

	return &pod{
		name:      podName,
		namespace: "kube-system",
		pod: &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: podName,
			},
			Spec: corev1.PodSpec{
				Containers: []corev1.Container{
					{
						Name:  podName,
						Image: "alpine",
						Command: []string{
							"/bin/sh",
							"-c",
							"--",
						},
						Args: []string{
							"trap : TERM INT; (tail -f /dev/null) & wait",
						},
						SecurityContext: &corev1.SecurityContext{
							Privileged: pointer.To(true),
						},
						// lvm commands even though executed in the host mount namespace, still need access to /dev 🤷🏼,
						// otherwise lvcreate commands hangs on semop syscall
						VolumeMounts: []corev1.VolumeMount{
							{
								Name:      "dev",
								MountPath: "/dev",
							},
							{
								Name:      "host",
								MountPath: "/host",
							},
						},
					},
				},
				Volumes: []corev1.Volume{
					{
						Name: "dev",
						VolumeSource: corev1.VolumeSource{
							HostPath: &corev1.HostPathVolumeSource{
								Path: "/dev",
							},
						},
					},
					{
						Name: "host",
						VolumeSource: corev1.VolumeSource{
							HostPath: &corev1.HostPathVolumeSource{
								Path: "/",
							},
						},
					},
				},
				HostNetwork: true,
				HostIPC:     true,
				HostPID:     true,
			},
		},

		suite: k8sSuite,
	}, nil
}

// NewPod creates a new pod definition with a random suffix
// in the default namespace.
func (k8sSuite *K8sSuite) NewPod(name string) (podInfo, error) {
	randomSuffix := make([]byte, 4)

	if _, err := rand.Read(randomSuffix); err != nil {
		return nil, fmt.Errorf("failed to generate random suffix: %w", err)
	}

	podName := fmt.Sprintf("%s-%x", name, randomSuffix)

	return &pod{
		name:      podName,
		namespace: "default",
		pod: &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name: podName,
			},
			Spec: corev1.PodSpec{
				Containers: []corev1.Container{
					{
						Name:  podName,
						Image: "alpine",
						Command: []string{
							"/bin/sh",
							"-c",
							"--",
						},
						Args: []string{
							"trap : TERM INT; (tail -f /dev/null) & wait",
						},
					},
				},
			},
		},

		suite: k8sSuite,
	}, nil
}

// WaitForPodToBeRunning waits for the pod with the given namespace and name to be running.
func (k8sSuite *K8sSuite) WaitForPodToBeRunning(ctx context.Context, timeout time.Duration, namespace, podName string) error {
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	watcher, err := k8sSuite.Clientset.CoreV1().Pods(namespace).Watch(ctx, metav1.ListOptions{
		FieldSelector: fields.OneTermEqualSelector("metadata.name", podName).String(),
	})
	if err != nil {
		return err
	}

	defer watcher.Stop()

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case event := <-watcher.ResultChan():
			if event.Type == watch.Error {
				return fmt.Errorf("error watching pod: %v", event.Object)
			}

			pod, ok := event.Object.(*corev1.Pod)
			if !ok {
				continue
			}

			if pod.Name == podName && pod.Status.Phase == corev1.PodRunning {
				return nil
			}
		}
	}
}

// LogPodLogsByLabel logs the logs of the pod with the given namespace and label.
func (k8sSuite *K8sSuite) LogPodLogsByLabel(ctx context.Context, namespace, label, value string) {
	ctx, cancel := context.WithTimeout(ctx, time.Minute)
	defer cancel()

	podList, err := k8sSuite.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
		LabelSelector: fmt.Sprintf("%s=%s", label, value),
	})
	k8sSuite.Require().NoError(err)

	for _, pod := range podList.Items {
		k8sSuite.LogPodLogs(ctx, namespace, pod.Name)
	}
}

// LogPodLogs logs the logs of the pod with the given namespace and name.
func (k8sSuite *K8sSuite) LogPodLogs(ctx context.Context, namespace, podName string) {
	ctx, cancel := context.WithTimeout(ctx, time.Minute)
	defer cancel()

	req := k8sSuite.Clientset.CoreV1().Pods(namespace).GetLogs(podName, &corev1.PodLogOptions{})

	readCloser, err := req.Stream(ctx)
	if err != nil {
		k8sSuite.T().Logf("failed to get pod logs: %s", err)
	}

	defer readCloser.Close() //nolint:errcheck

	scanner := bufio.NewScanner(readCloser)

	for scanner.Scan() {
		k8sSuite.T().Logf("%s/%s: %s", namespace, podName, scanner.Text())
	}
}

// HelmInstall installs the Helm chart with the given namespace, repository, version, release name, chart name and values.
func (k8sSuite *K8sSuite) HelmInstall(ctx context.Context, namespace, repository, version, releaseName, chartName string, valuesBytes []byte) error {
	tempFile := filepath.Join(k8sSuite.T().TempDir(), "values.yaml")

	if err := os.WriteFile(tempFile, valuesBytes, 0o644); err != nil {
		return err
	}

	defer os.Remove(tempFile) //nolint:errcheck

	args := []string{
		"upgrade",
		"--install",
		"--cleanup-on-fail",
		"--create-namespace",
		"--namespace",
		namespace,
		"--wait",
		"--timeout",
		k8sSuite.CSITestTimeout,
		"--repo",
		repository,
		"--version",
		version,
		"--values",
		tempFile,
		releaseName,
		chartName,
	}

	cmd := exec.Command(k8sSuite.HelmPath, args...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	k8sSuite.T().Logf("running helm command: %s", strings.Join(cmd.Args, " "))

	return cmd.Run()
}

// WaitForResource waits for the resource with the given group, kind, version, namespace and jsonpath field selector to have the given expected value.
// mostly a restructuring of `kubectl wait` from https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/wait/wait.go
//
//nolint:gocyclo
func (k8sSuite *K8sSuite) WaitForResource(ctx context.Context, namespace, group, kind, version, resourceName, jsonPathSelector, expectedValue string) error {
	j := jsonpath.New("wait").AllowMissingKeys(true)

	if jsonPathSelector == "" {
		return fmt.Errorf("jsonpath condition is empty")
	}

	if err := j.Parse(jsonPathSelector); err != nil {
		return fmt.Errorf("error parsing jsonpath condition: %v", err)
	}

	mapping, err := k8sSuite.Mapper.RESTMapping(schema.GroupKind{
		Group: group,
		Kind:  kind,
	}, version)
	if err != nil {
		return fmt.Errorf("error creating mapping for resource %s/%s/%s", group, kind, version)
	}

	dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(namespace)

	fieldSelector := fields.OneTermEqualSelector("metadata.name", resourceName).String()

	lw := &cache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			options.FieldSelector = fieldSelector

			return dr.List(ctx, options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			options.FieldSelector = fieldSelector

			return dr.Watch(ctx, options)
		},
	}

	preconditionFunc := func(store cache.Store) (bool, error) {
		var exists bool

		_, exists, err = store.Get(&metav1.ObjectMeta{Namespace: namespace, Name: resourceName})
		if err != nil {
			return true, err
		}

		if !exists {
			return true, errors.NewNotFound(mapping.Resource.GroupResource(), resourceName)
		}

		return false, nil
	}
	if _, err = watchtools.UntilWithSync(ctx, lw, &unstructured.Unstructured{}, preconditionFunc, func(event watch.Event) (bool, error) {
		obj, ok := event.Object.(*unstructured.Unstructured)
		if !ok {
			return false, fmt.Errorf("error converting object to unstructured")
		}

		queryObj := obj.UnstructuredContent()

		k8sSuite.T().Logf("waiting for resource %s/%s/%s/%s to have field %s with value %s", group, version, kind, resourceName, jsonPathSelector, expectedValue)

		parseResults, err := j.FindResults(queryObj)
		if err != nil {
			return false, fmt.Errorf("error finding results: %v", err)
		}

		if len(parseResults) == 0 || len(parseResults[0]) == 0 {
			return false, nil
		}

		if len(parseResults) > 1 {
			return false, fmt.Errorf("given jsonpath expression matches more than one list")
		}

		if len(parseResults[0]) > 1 {
			return false, fmt.Errorf("given jsonpath expression matches more than one value")
		}

		switch parseResults[0][0].Interface().(type) {
		case map[string]interface{}, []interface{}:
			return false, fmt.Errorf("jsonpath leads to a nested object or list which is not supported")
		}

		s := fmt.Sprintf("%v", parseResults[0][0].Interface())

		return strings.TrimSpace(s) == strings.TrimSpace(expectedValue), nil
	}); err != nil {
		return err
	}

	return nil
}

// WaitForResourceToBeAvailable waits for the resource with the given namespace, group, kind, version and name to be available.
func (k8sSuite *K8sSuite) WaitForResourceToBeAvailable(ctx context.Context, duration time.Duration, namespace, group, kind, version, resourceName string) error {
	return retry.Constant(duration).Retry(func() error {
		mapping, err := k8sSuite.Mapper.RESTMapping(schema.GroupKind{
			Group: group,
			Kind:  kind,
		}, version)
		if err != nil {
			return fmt.Errorf("error creating mapping for resource %s/%s/%s", group, kind, version)
		}

		dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(namespace)

		_, err = dr.Get(ctx, resourceName, metav1.GetOptions{})
		if errors.IsNotFound(err) {
			k8sSuite.T().Logf("resource %s/%s/%s/%s not found, retrying", group, version, kind, resourceName)

			return retry.ExpectedError(err)
		}

		return err
	})
}

// GetUnstructuredResource gets the unstructured resource with the given namespace, group, kind, version and name.
func (k8sSuite *K8sSuite) GetUnstructuredResource(ctx context.Context, namespace, group, kind, version, resourceName string) (*unstructured.Unstructured, error) {
	mapping, err := k8sSuite.Mapper.RESTMapping(schema.GroupKind{
		Group: group,
		Kind:  kind,
	}, version)
	if err != nil {
		return nil, fmt.Errorf("error creating mapping for resource %s/%s/%s", group, kind, version)
	}

	dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(namespace)

	result, err := dr.Get(ctx, resourceName, metav1.GetOptions{})
	if err != nil {
		return nil, fmt.Errorf("error getting resource %s/%s/%s/%s: %v", group, version, kind, resourceName, err)
	}

	return result, nil
}

// RunFIOTest runs the FIO test with the given storage class and size using kubestr.
func (k8sSuite *K8sSuite) RunFIOTest(ctx context.Context, storageClass, size string) error {
	args := []string{
		"--outfile",
		fmt.Sprintf("/tmp/fio-%s.json", storageClass),
		"--output",
		"json",
		"fio",
		"--storageclass",
		storageClass,
		"--size",
		size,
	}

	cmd := exec.Command(k8sSuite.KubeStrPath, args...)

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	k8sSuite.T().Logf("running kubestr command: %s", strings.Join(cmd.Args, " "))

	return cmd.Run()
}

// GetPodsWithLabel returns the pods with the given label in the specified namespace.
func (k8sSuite *K8sSuite) GetPodsWithLabel(ctx context.Context, namespace, label string) (*corev1.PodList, error) {
	podList, err := k8sSuite.Clientset.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{
		LabelSelector: label,
	})
	if err != nil {
		return nil, err
	}

	return podList, nil
}

// ParseManifests parses YAML manifest bytes into unstructured objects.
func (k8sSuite *K8sSuite) ParseManifests(manifests []byte) []unstructured.Unstructured {
	reader := yaml.NewYAMLReader(bufio.NewReader(bytes.NewReader(manifests)))

	var parsedManifests []unstructured.Unstructured

	for {
		yamlManifest, err := reader.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			k8sSuite.Require().NoError(err)
		}

		yamlManifest = bytes.TrimSpace(yamlManifest)

		if len(yamlManifest) == 0 {
			continue
		}

		jsonManifest, err := yaml.ToJSON(yamlManifest)
		if err != nil {
			k8sSuite.Require().NoError(err, "error converting manifest to JSON")
		}

		if bytes.Equal(jsonManifest, []byte("null")) || bytes.Equal(jsonManifest, []byte("{}")) {
			// skip YAML docs which contain only comments
			continue
		}

		var obj unstructured.Unstructured

		if err = json.Unmarshal(jsonManifest, &obj); err != nil {
			k8sSuite.Require().NoError(err, "error loading JSON manifest into unstructured")
		}

		parsedManifests = append(parsedManifests, obj)
	}

	return parsedManifests
}

// ApplyManifests applies the given manifests to the Kubernetes cluster.
func (k8sSuite *K8sSuite) ApplyManifests(ctx context.Context, manifests []unstructured.Unstructured) {
	for _, obj := range manifests {
		mapping, err := k8sSuite.Mapper.RESTMapping(obj.GetObjectKind().GroupVersionKind().GroupKind(), obj.GetObjectKind().GroupVersionKind().Version)
		if err != nil {
			k8sSuite.Require().NoError(err, "error creating mapping for object %s", obj.GetName())
		}

		if obj.GetNamespace() == "" {
			k8sSuite.T().Logf("namespace not set for object %s, kind %s", obj.GetName(), obj.GetObjectKind().GroupVersionKind())
		}

		dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace())

		_, err = dr.Apply(ctx, obj.GetName(), &obj, metav1.ApplyOptions{
			FieldManager: "talos-e2e",
		})
		k8sSuite.Require().NoError(err, "error creating object %s", obj.GetName())

		k8sSuite.T().Logf("created object %s/%s/%s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName())
	}
}

// DeleteManifests deletes the given manifests from the Kubernetes cluster.
func (k8sSuite *K8sSuite) DeleteManifests(ctx context.Context, manifests []unstructured.Unstructured) {
	// process in reverse orderd
	manifests = slices.Clone(manifests)
	slices.Reverse(manifests)

	for _, obj := range manifests {
		mapping, err := k8sSuite.Mapper.RESTMapping(obj.GetObjectKind().GroupVersionKind().GroupKind(), obj.GetObjectKind().GroupVersionKind().Version)
		if err != nil {
			k8sSuite.Require().NoError(err, "error creating mapping for object %s", obj.GetName())
		}

		dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(obj.GetNamespace())

		err = dr.Delete(ctx, obj.GetName(), metav1.DeleteOptions{})
		if errors.IsNotFound(err) {
			continue
		}

		k8sSuite.Require().NoError(err, "error deleting object %s", obj.GetName())

		// wait for the object to be deleted
		fieldSelector := fields.OneTermEqualSelector("metadata.name", obj.GetName()).String()
		lw := &cache.ListWatch{
			ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
				options.FieldSelector = fieldSelector

				return dr.List(ctx, options)
			},
			WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
				options.FieldSelector = fieldSelector

				return dr.Watch(ctx, options)
			},
		}

		preconditionFunc := func(store cache.Store) (bool, error) {
			var exists bool

			_, exists, err = store.Get(&metav1.ObjectMeta{Namespace: obj.GetNamespace(), Name: obj.GetName()})
			if err != nil {
				return true, err
			}

			if !exists {
				// since we're looking for it to disappear we just return here if it no longer exists
				return true, nil
			}

			return false, nil
		}

		_, err = watchtools.UntilWithSync(ctx, lw, &unstructured.Unstructured{}, preconditionFunc, func(event watch.Event) (bool, error) {
			return event.Type == watch.Deleted, nil
		})

		k8sSuite.Require().NoError(err, "error waiting for the object to be deleted %s/%s/%s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName())

		k8sSuite.T().Logf("deleted object %s/%s/%s", obj.GetObjectKind().GroupVersionKind(), obj.GetNamespace(), obj.GetName())
	}
}

// PatchK8sObject patches the kubernetes object with the given namespace, group, kind, version and name.
func (k8sSuite *K8sSuite) PatchK8sObject(ctx context.Context, namespace, group, kind, version, resourceName string, patchBytes []byte) {
	patchBytes, err := yaml.ToJSON(patchBytes)
	k8sSuite.Require().NoError(err, "error converting patch to JSON")

	mapping, err := k8sSuite.Mapper.RESTMapping(schema.GroupKind{
		Group: group,
		Kind:  kind,
	}, version)

	k8sSuite.Require().NoError(err, "error creating mapping for resource %s/%s/%s", group, kind, version)

	dr := k8sSuite.DynamicClient.Resource(mapping.Resource).Namespace(namespace)

	_, err = dr.Patch(ctx, resourceName, types.MergePatchType, patchBytes, metav1.PatchOptions{})
	k8sSuite.Require().NoError(err, "error patching resource %s/%s/%s/%s", group, version, kind, resourceName)
}

// ToUnstructured converts the given runtime.Object to unstructured.Unstructured.
func (k8sSuite *K8sSuite) ToUnstructured(obj runtime.Object) unstructured.Unstructured {
	unstructuredObj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(obj)
	if err != nil {
		k8sSuite.Require().NoError(err, "error converting object to unstructured")
	}

	u := unstructured.Unstructured{Object: unstructuredObj}
	u.SetGroupVersionKind(obj.GetObjectKind().GroupVersionKind())

	return u
}

// SetupNodeInformer sets up a node informer for the given node name.
func (k8sSuite *K8sSuite) SetupNodeInformer(ctx context.Context, nodeName string) <-chan *corev1.Node {
	const metadataKeyName = "metadata.name="

	watchCh := make(chan *corev1.Node)

	informerFactory := informers.NewSharedInformerFactoryWithOptions(
		k8sSuite.Clientset, constants.KubernetesInformerDefaultResyncPeriod,
		informers.WithTweakListOptions(func(options *metav1.ListOptions) {
			options.FieldSelector = metadataKeyName + nodeName
		}),
	)

	nodeInformer := informerFactory.Core().V1().Nodes().Informer()
	_, err := nodeInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc: func(obj any) {
			node, ok := obj.(*corev1.Node)
			if !ok {
				return
			}

			channel.SendWithContext(ctx, watchCh, node)
		},
		UpdateFunc: func(_, obj any) {
			node, ok := obj.(*corev1.Node)
			if !ok {
				return
			}

			channel.SendWithContext(ctx, watchCh, node)
		},
	})
	k8sSuite.Require().NoError(err)

	informerFactory.Start(ctx.Done())
	k8sSuite.T().Cleanup(informerFactory.Shutdown)

	result := informerFactory.WaitForCacheSync(ctx.Done())

	for k, v := range result {
		k8sSuite.Assert().True(v, "informer %q failed to sync", k.String())
	}

	return watchCh
}