From 81acadf345d00a30f26cdc979dd06e7dd0086c7c Mon Sep 17 00:00:00 2001
From: Andrey Smirnov <smirnov.andrey@gmail.com>
Date: Fri, 5 Mar 2021 17:17:33 +0300
Subject: [PATCH] fix: ignore connection refused errors when
 updating/converting cp

Without loadbalancer, when api-server goes down, there will be
connection refused errors which should be retried.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
---
 pkg/cluster/kubernetes/convert.go       |  8 ++++----
 pkg/cluster/kubernetes/kubernetes.go    | 22 ++++++++++++++++++++++
 pkg/cluster/kubernetes/self_hosted.go   |  3 +--
 pkg/cluster/kubernetes/talos_managed.go |  3 +--
 4 files changed, 28 insertions(+), 8 deletions(-)

diff --git a/pkg/cluster/kubernetes/convert.go b/pkg/cluster/kubernetes/convert.go
index 14e308a2e..65991b3d1 100644
--- a/pkg/cluster/kubernetes/convert.go
+++ b/pkg/cluster/kubernetes/convert.go
@@ -458,7 +458,7 @@ func waitForStaticPods(ctx context.Context, cluster ConvertProvider, options *Co
 			LabelSelector: fmt.Sprintf("k8s-app = %s", k8sApp),
 		})
 		if err != nil {
-			if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+			if retryableError(err) {
 				return retry.ExpectedError(err)
 			}
 
@@ -538,7 +538,7 @@ func disablePodCheckpointer(ctx context.Context, cluster ConvertProvider) error
 
 		checkpoints, err = getActiveCheckpoints(ctx, k8sClient)
 		if err != nil {
-			if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+			if retryableError(err) {
 				return retry.ExpectedError(err)
 			}
 
@@ -601,7 +601,7 @@ func deleteDaemonset(ctx context.Context, cluster ConvertProvider, k8sApp string
 	if err = retry.Constant(time.Minute, retry.WithUnits(100*time.Millisecond)).Retry(func() error {
 		err = k8sClient.AppsV1().DaemonSets(namespace).Delete(ctx, k8sApp, v1.DeleteOptions{})
 		if err != nil {
-			if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+			if retryableError(err) {
 				return retry.ExpectedError(err)
 			}
 
@@ -622,7 +622,7 @@ func deleteDaemonset(ctx context.Context, cluster ConvertProvider, k8sApp string
 			LabelSelector: fmt.Sprintf("k8s-app = %s", k8sApp),
 		})
 		if err != nil {
-			if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+			if retryableError(err) {
 				return retry.ExpectedError(err)
 			}
 
diff --git a/pkg/cluster/kubernetes/kubernetes.go b/pkg/cluster/kubernetes/kubernetes.go
index 16dfea22b..e2370a920 100644
--- a/pkg/cluster/kubernetes/kubernetes.go
+++ b/pkg/cluster/kubernetes/kubernetes.go
@@ -4,3 +4,25 @@
 
 // Package kubernetes provides cluster-wide kubernetes utilities.
 package kubernetes
+
+import (
+	"errors"
+	"net"
+	"syscall"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+)
+
+func retryableError(err error) bool {
+	if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+		return true
+	}
+
+	netErr := &net.OpError{}
+
+	if errors.As(err, &netErr) {
+		return netErr.Temporary() || errors.Is(netErr.Err, syscall.ECONNREFUSED)
+	}
+
+	return false
+}
diff --git a/pkg/cluster/kubernetes/self_hosted.go b/pkg/cluster/kubernetes/self_hosted.go
index ebbf3c22e..0bd3ad44f 100644
--- a/pkg/cluster/kubernetes/self_hosted.go
+++ b/pkg/cluster/kubernetes/self_hosted.go
@@ -14,7 +14,6 @@ import (
 	"github.com/talos-systems/go-retry/retry"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
@@ -126,7 +125,7 @@ func updateDaemonset(ctx context.Context, clientset *kubernetes.Clientset, ds st
 	return retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error {
 		daemonset, err = clientset.AppsV1().DaemonSets(namespace).Get(ctx, ds, metav1.GetOptions{})
 		if err != nil {
-			if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+			if retryableError(err) {
 				return retry.ExpectedError(err)
 			}
 
diff --git a/pkg/cluster/kubernetes/talos_managed.go b/pkg/cluster/kubernetes/talos_managed.go
index ce8711182..35864828c 100644
--- a/pkg/cluster/kubernetes/talos_managed.go
+++ b/pkg/cluster/kubernetes/talos_managed.go
@@ -13,7 +13,6 @@ import (
 
 	"github.com/talos-systems/go-retry/retry"
 	"github.com/talos-systems/os-runtime/pkg/state"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/talos-systems/talos/pkg/cluster"
@@ -217,7 +216,7 @@ func checkPodStatus(ctx context.Context, cluster UpgradeProvider, service, node,
 		LabelSelector: fmt.Sprintf("k8s-app = %s", service),
 	})
 	if err != nil {
-		if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
+		if retryableError(err) {
 			return retry.ExpectedError(err)
 		}