feat: custom image settings for k8s upgrade

Allows to use custom registry/images.

Fixes: #8275

Co-authored-by:  @g3offrey
Signed-off-by: Matthieu STROHL <mstrohl@dive-in-it.com>
Signed-off-by: Andrey Smirnov <andrey.smirnov@siderolabs.com>

cmd/talosctl/cmd/talos/upgrade-k8s.go

@@ -49,6 +49,13 @@ func init() {
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.DryRun, "dry-run", false, "skip the actual upgrade and show the upgrade plan instead")
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.PrePullImages, "pre-pull-images", true, "pre-pull images before upgrade")
 	upgradeK8sCmd.Flags().BoolVar(&upgradeOptions.UpgradeKubelet, "upgrade-kubelet", true, "upgrade kubelet service")
+
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.KubeletImage, "kubelet-image", constants.KubeletImage, "kubelet image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.APIServerImage, "apiserver-image", constants.KubernetesAPIServerImage, "kube-apiserver image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ControllerManagerImage, "controller-manager-image", constants.KubernetesControllerManagerImage, "kube-controller-manager image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.SchedulerImage, "scheduler-image", constants.KubernetesSchedulerImage, "kube-scheduler image to use")
+	upgradeK8sCmd.Flags().StringVar(&upgradeOptions.ProxyImage, "proxy-image", constants.KubeProxyImage, "kube-proxy image to use")
+
 	addCommand(upgradeK8sCmd)
 }
 

hack/release.toml

@@ -93,6 +93,13 @@ config:
         - content: MONITOR ${upsmonHost} 1 remote pass password
           mountPath: /usr/local/etc/nut/upsmon.conf
 ```
+"""
+
+    [notes.k8supgrade]
+        title = "Kubernetes Upgrade"
+        description = """\
+The command `talosctl upgrade-k8s` now supports specifying custom image references for Kubernetes components via `--*-image` flags.
+The default behavior is unchanged, and the flags are optional.
 """
 
 [make_deps]

internal/integration/provision/provision.go

@@ -417,6 +417,12 @@ func (suite *BaseSuite) upgradeKubernetes(fromVersion, toVersion string, skipKub
 		UpgradeKubelet: !skipKubeletUpgrade,
 		PrePullImages:  true,
 
+		KubeletImage:           constants.KubeletImage,
+		APIServerImage:         constants.KubernetesAPIServerImage,
+		ControllerManagerImage: constants.KubernetesControllerManagerImage,
+		SchedulerImage:         constants.KubernetesSchedulerImage,
+		ProxyImage:             constants.KubeProxyImage,
+
 		EncoderOpt: encoder.WithComments(encoder.CommentsAll),
 	}
 

pkg/cluster/kubernetes/kubelet.go

@@ -22,7 +22,6 @@ import (
 	"github.com/siderolabs/talos/pkg/kubernetes"
 	"github.com/siderolabs/talos/pkg/machinery/client"
 	v1alpha1config "github.com/siderolabs/talos/pkg/machinery/config/types/v1alpha1"
-	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/resources/k8s"
 	"github.com/siderolabs/talos/pkg/machinery/resources/v1alpha1"
 )
@@ -199,7 +198,7 @@ func upgradeKubeletPatcher(
 			}
 		}
 
-		image := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion())
+		image := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion())
 
 		if oldImage == image {
 			return errUpdateSkipped

pkg/cluster/kubernetes/talos_managed.go

@@ -124,9 +124,9 @@ func Upgrade(ctx context.Context, cluster UpgradeProvider, options UpgradeOption
 
 func prePullImages(ctx context.Context, talosClient *client.Client, options UpgradeOptions) error {
 	for _, imageRef := range []string{
-		fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion()),
+		fmt.Sprintf("%s:v%s", options.APIServerImage, options.Path.ToVersion()),
-		fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion()),
+		fmt.Sprintf("%s:v%s", options.ControllerManagerImage, options.Path.ToVersion()),
-		fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion()),
+		fmt.Sprintf("%s:v%s", options.SchedulerImage, options.Path.ToVersion()),
 	} {
 		for _, node := range options.controlPlaneNodes {
 			options.Log(" > %q: pre-pulling %s", node, imageRef)
@@ -146,7 +146,7 @@ func prePullImages(ctx context.Context, talosClient *client.Client, options Upgr
 		return nil
 	}
 
-	imageRef := fmt.Sprintf("%s:v%s", constants.KubeletImage, options.Path.ToVersion())
+	imageRef := fmt.Sprintf("%s:v%s", options.KubeletImage, options.Path.ToVersion())
 
 	for _, node := range append(append([]string(nil), options.controlPlaneNodes...), options.workerNodes...) {
 		options.Log(" > %q: pre-pulling %s", node, imageRef)
@@ -206,7 +206,7 @@ func patchKubeProxy(options UpgradeOptions) func(config *v1alpha1config.Config)
 			config.ClusterConfig.ProxyConfig = &v1alpha1config.ProxyConfig{}
 		}
 
-		config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", constants.KubeProxyImage, options.Path.ToVersion())
+		config.ClusterConfig.ProxyConfig.ContainerImage = fmt.Sprintf("%s:v%s", options.ProxyImage, options.Path.ToVersion())
 
 		return nil
 	}
@@ -304,7 +304,23 @@ func upgradeStaticPodOnNode(ctx context.Context, cluster UpgradeProvider, option
 
 var errUpdateSkipped = errors.New("update skipped")
 
-//nolint:gocyclo,cyclop
+func staticPodImage(logUpdate func(oldImage string), imageName, containerImage, configImage string, options UpgradeOptions) (string, error) {
+	image := fmt.Sprintf("%s:v%s", imageName, options.Path.ToVersion())
+
+	if containerImage == image || configImage == image {
+		return "", errUpdateSkipped
+	}
+
+	logUpdate(containerImage)
+
+	if options.DryRun {
+		return "", errUpdateSkipped
+	}
+
+	return image, nil
+}
+
+//nolint:gocyclo
 func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResource resource.Resource) func(config *v1alpha1config.Config) error {
 	return func(config *v1alpha1config.Config) error {
 		if config.ClusterConfig == nil {
@@ -349,16 +365,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
 				config.ClusterConfig.APIServerConfig = &v1alpha1config.APIServerConfig{}
 			}
 
-			image := fmt.Sprintf("%s:v%s", constants.KubernetesAPIServerImage, options.Path.ToVersion())
+			image, err := staticPodImage(logUpdate,
-
+				options.APIServerImage,
-			if config.ClusterConfig.APIServerConfig.ContainerImage == image || configImage == image {
+				config.ClusterConfig.APIServerConfig.ContainerImage,
-				return errUpdateSkipped
+				configImage,
-			}
+				options)
-
+			if err != nil {
-			logUpdate(config.ClusterConfig.APIServerConfig.ContainerImage)
+				return err
-
-			if options.DryRun {
-				return errUpdateSkipped
 			}
 
 			config.ClusterConfig.APIServerConfig.ContainerImage = image
@@ -367,16 +380,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
 				config.ClusterConfig.ControllerManagerConfig = &v1alpha1config.ControllerManagerConfig{}
 			}
 
-			image := fmt.Sprintf("%s:v%s", constants.KubernetesControllerManagerImage, options.Path.ToVersion())
+			image, err := staticPodImage(logUpdate,
-
+				options.ControllerManagerImage,
-			if config.ClusterConfig.ControllerManagerConfig.ContainerImage == image || configImage == image {
+				config.ClusterConfig.ControllerManagerConfig.ContainerImage,
-				return errUpdateSkipped
+				configImage,
-			}
+				options)
-
+			if err != nil {
-			logUpdate(config.ClusterConfig.ControllerManagerConfig.ContainerImage)
+				return err
-
-			if options.DryRun {
-				return errUpdateSkipped
 			}
 
 			config.ClusterConfig.ControllerManagerConfig.ContainerImage = image
@@ -385,16 +395,13 @@ func upgradeStaticPodPatcher(options UpgradeOptions, service string, configResou
 				config.ClusterConfig.SchedulerConfig = &v1alpha1config.SchedulerConfig{}
 			}
 
-			image := fmt.Sprintf("%s:v%s", constants.KubernetesSchedulerImage, options.Path.ToVersion())
+			image, err := staticPodImage(logUpdate,
-
+				options.SchedulerImage,
-			if config.ClusterConfig.SchedulerConfig.ContainerImage == image || configImage == image {
+				config.ClusterConfig.SchedulerConfig.ContainerImage,
-				return errUpdateSkipped
+				configImage,
-			}
+				options)
-
+			if err != nil {
-			logUpdate(config.ClusterConfig.SchedulerConfig.ContainerImage)
+				return err
-
-			if options.DryRun {
-				return errUpdateSkipped
 			}
 
 			config.ClusterConfig.SchedulerConfig.ContainerImage = image

pkg/cluster/kubernetes/upgrade.go

@@ -32,6 +32,12 @@ type UpgradeOptions struct {
 	DryRun               bool
 	EncoderOpt           encoder.Option
 
+	KubeletImage           string
+	APIServerImage         string
+	ControllerManagerImage string
+	SchedulerImage         string
+	ProxyImage             string
+
 	controlPlaneNodes []string
 	workerNodes       []string
 }

website/content/v1.7/kubernetes-guides/upgrading-kubernetes.md

@@ -106,6 +106,8 @@ This command runs in several phases:
 
 If the command fails for any reason, it can be safely restarted to continue the upgrade process from the moment of the failure.
 
+> Note: When using custom/overridden Kubernetes component images, use flags `--*-image` to override the default image names.
+
 ## Manual Kubernetes Upgrade
 
 Kubernetes can be upgraded manually by following the steps outlined below.

website/content/v1.7/reference/cli.md

@@ -2887,11 +2887,16 @@ talosctl upgrade-k8s [flags]
 ### Options
 
 ```
+      --apiserver-image string            kube-apiserver image to use (default "registry.k8s.io/kube-apiserver")
+      --controller-manager-image string   kube-controller-manager image to use (default "registry.k8s.io/kube-controller-manager")
       --dry-run                           skip the actual upgrade and show the upgrade plan instead
       --endpoint string                   the cluster control plane endpoint
       --from string                       the Kubernetes control plane version to upgrade from
   -h, --help                              help for upgrade-k8s
+      --kubelet-image string              kubelet image to use (default "ghcr.io/siderolabs/kubelet")
       --pre-pull-images                   pre-pull images before upgrade (default true)
+      --proxy-image string                kube-proxy image to use (default "registry.k8s.io/kube-proxy")
+      --scheduler-image string            kube-scheduler image to use (default "registry.k8s.io/kube-scheduler")
       --to string                         the Kubernetes control plane version to upgrade to (default "1.29.1")
       --upgrade-kubelet                   upgrade kubelet service (default true)
       --with-docs                         patch all machine configs adding the documentation for each field (default true)