mirrors/talos
1
0
Fork 0
mirror of https://github.com/siderolabs/talos.git synced 2025-09-07 15:01:11 +02:00
Code Issues Packages Projects Releases Wiki Activity

fix: ignore connection refused errors when updating/converting cp

Browse Source 
Without loadbalancer, when api-server goes down, there will be
connection refused errors which should be retried.

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>
This commit is contained in:
Andrey Smirnov 2021-03-05 17:17:33 +03:00 committed by talos-bot
parent db3785b930
commit 81acadf345
4 changed files with 28 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
pkg/cluster/kubernetes/convert.go
View File

@ -458,7 +458,7 @@ func waitForStaticPods(ctx context.Context, cluster ConvertProvider, options *Co
LabelSelector: fmt.Sprintf("k8s-app = %s", k8sApp), LabelSelector: fmt.Sprintf("k8s-app = %s", k8sApp),
}) })
if err != nil { if err != nil {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) { if retryableError(err) {
return retry.ExpectedError(err) return retry.ExpectedError(err)
} }
@ -538,7 +538,7 @@ func disablePodCheckpointer(ctx context.Context, cluster ConvertProvider) error
checkpoints, err = getActiveCheckpoints(ctx, k8sClient) checkpoints, err = getActiveCheckpoints(ctx, k8sClient)
if err != nil { if err != nil {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) { if retryableError(err) {
return retry.ExpectedError(err) return retry.ExpectedError(err)
} }
@ -601,7 +601,7 @@ func deleteDaemonset(ctx context.Context, cluster ConvertProvider, k8sApp string
if err = retry.Constant(time.Minute, retry.WithUnits(100*time.Millisecond)).Retry(func() error { if err = retry.Constant(time.Minute, retry.WithUnits(100*time.Millisecond)).Retry(func() error {
err = k8sClient.AppsV1().DaemonSets(namespace).Delete(ctx, k8sApp, v1.DeleteOptions{}) err = k8sClient.AppsV1().DaemonSets(namespace).Delete(ctx, k8sApp, v1.DeleteOptions{})
if err != nil { if err != nil {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) { if retryableError(err) {
return retry.ExpectedError(err) return retry.ExpectedError(err)
} }
@ -622,7 +622,7 @@ func deleteDaemonset(ctx context.Context, cluster ConvertProvider, k8sApp string
LabelSelector: fmt.Sprintf("k8s-app = %s", k8sApp), LabelSelector: fmt.Sprintf("k8s-app = %s", k8sApp),
}) })
if err != nil { if err != nil {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) { if retryableError(err) {
return retry.ExpectedError(err) return retry.ExpectedError(err)
} }

22
pkg/cluster/kubernetes/kubernetes.go
View File

@ -4,3 +4,25 @@
// Package kubernetes provides cluster-wide kubernetes utilities. // Package kubernetes provides cluster-wide kubernetes utilities.
package kubernetes package kubernetes
import (
"errors"
"net"
"syscall"
apierrors "k8s.io/apimachinery/pkg/api/errors"
)
func retryableError(err error) bool {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) {
return true
}
netErr := &net.OpError{}
if errors.As(err, &netErr) {
return netErr.Temporary() || errors.Is(netErr.Err, syscall.ECONNREFUSED)
}
return false
}

3
pkg/cluster/kubernetes/self_hosted.go
View File

@ -14,7 +14,6 @@ import (
"github.com/talos-systems/go-retry/retry" "github.com/talos-systems/go-retry/retry"
appsv1 "k8s.io/api/apps/v1" appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/apimachinery/pkg/util/strategicpatch"
@ -126,7 +125,7 @@ func updateDaemonset(ctx context.Context, clientset *kubernetes.Clientset, ds st
return retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error { return retry.Constant(5*time.Minute, retry.WithUnits(10*time.Second)).Retry(func() error {
daemonset, err = clientset.AppsV1().DaemonSets(namespace).Get(ctx, ds, metav1.GetOptions{}) daemonset, err = clientset.AppsV1().DaemonSets(namespace).Get(ctx, ds, metav1.GetOptions{})
if err != nil { if err != nil {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) { if retryableError(err) {
return retry.ExpectedError(err) return retry.ExpectedError(err)
} }

3
pkg/cluster/kubernetes/talos_managed.go
View File

@ -13,7 +13,6 @@ import (
"github.com/talos-systems/go-retry/retry" "github.com/talos-systems/go-retry/retry"
"github.com/talos-systems/os-runtime/pkg/state" "github.com/talos-systems/os-runtime/pkg/state"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/talos-systems/talos/pkg/cluster" "github.com/talos-systems/talos/pkg/cluster"
@ -217,7 +216,7 @@ func checkPodStatus(ctx context.Context, cluster UpgradeProvider, service, node,
LabelSelector: fmt.Sprintf("k8s-app = %s", service), LabelSelector: fmt.Sprintf("k8s-app = %s", service),
}) })
if err != nil { if err != nil {
if apierrors.IsTimeout(err) || apierrors.IsServerTimeout(err) || apierrors.IsInternalError(err) { if retryableError(err) {
return retry.ExpectedError(err) return retry.ExpectedError(err)
} }