From 0b41923c3608a815d14bccd84a6abcf1bc99db43 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 31 Oct 2022 17:37:07 +0400 Subject: [PATCH] fix: restore the StaticPodStatus resource It got broken with the changes to the kubelet now sourcing static pods from a HTTP internal server. As we don't want it to be broken, and to make health checks better, add a new check to make sure kubelet reports control plane static pods as running. This coupled with API server check should make it more thorough. Also add logging when static pod definitions are updated (they were previously there for file-based implementation). These logs are very helpful for troubleshooting. Signed-off-by: Andrey Smirnov --- go.mod | 4 +- go.sum | 8 +- .../pkg/controllers/k8s/kubelet_static_pod.go | 14 +- .../pkg/controllers/k8s/static_pod_server.go | 42 ++++- .../digitalocean/digitalocean_test.go | 2 +- .../digitalocean/testdata/expected.yaml | 174 +++++++++--------- pkg/cluster/check/default.go | 7 + pkg/cluster/check/kubernetes.go | 50 +++++ pkg/kubernetes/kubelet/kubelet.go | 2 +- pkg/machinery/go.mod | 4 +- pkg/machinery/go.sum | 7 +- 11 files changed, 204 insertions(+), 110 deletions(-) diff --git a/go.mod b/go.mod index fed0fe1f2..6093426f7 100644 --- a/go.mod +++ b/go.mod @@ -40,7 +40,7 @@ require ( github.com/containernetworking/plugins v1.1.1 github.com/coreos/go-iptables v0.6.0 github.com/coreos/go-semver v0.3.0 - github.com/cosi-project/runtime v0.2.0-alpha.1.0.20221009084302-e8a8fdcc7548 + github.com/cosi-project/runtime v0.2.0-alpha.2 github.com/docker/distribution v2.8.1+incompatible github.com/docker/docker v20.10.20+incompatible github.com/docker/go-connections v0.4.0 @@ -199,7 +199,7 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect - github.com/hashicorp/go-memdb v1.3.3 // indirect + github.com/hashicorp/go-memdb v1.3.4 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect diff --git a/go.sum b/go.sum index fdca976c7..1196364a3 100644 --- a/go.sum +++ b/go.sum @@ -335,8 +335,8 @@ github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzA github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/cosi-project/runtime v0.2.0-alpha.1.0.20221009084302-e8a8fdcc7548 h1:/CMoJlmVdr1XrAoo4cQDPF4rwB2Ap1WCa/BlFfkqOW0= -github.com/cosi-project/runtime v0.2.0-alpha.1.0.20221009084302-e8a8fdcc7548/go.mod h1:u60xdQ7/f8WkO0qsDwPRJuy+0edCDYvwh2xYUvDO+no= +github.com/cosi-project/runtime v0.2.0-alpha.2 h1:B27kw7knAukaHiXwKP51hGh4q2RMaY+wPF+beXBjZMI= +github.com/cosi-project/runtime v0.2.0-alpha.2/go.mod h1:jv79UECqzQaAeVwbdawUGmEBCX0fl7J7JBysnOYIM4U= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= @@ -651,8 +651,8 @@ github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjh github.com/hashicorp/go-immutable-radix v1.3.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-memdb v1.3.3 h1:oGfEWrFuxtIUF3W2q/Jzt6G85TrMk9ey6XfYLvVe1Wo= -github.com/hashicorp/go-memdb v1.3.3/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= +github.com/hashicorp/go-memdb v1.3.4 h1:XSL3NR682X/cVk2IeV0d70N4DZ9ljI885xAEU8IoK3c= +github.com/hashicorp/go-memdb v1.3.4/go.mod h1:uBTr1oQbtuMgd1SSGoR8YV27eT3sBHbYiNm53bMpgSg= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go index 3a75eb598..476e70f15 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_static_pod.go @@ -155,8 +155,6 @@ func (ctrl *KubeletStaticPodController) Run(ctx context.Context, r controller.Ru nodename := nodenameResource.(*k8s.Nodename).TypedSpec().Nodename - // render static pods first, and attempt to build kubelet client last, - // as if kubelet issues certs from the API server, API server should be launched first. kubeletClient, err = kubelet.NewClient(nodename, secrets.APIServerKubeletClient.Crt, secrets.APIServerKubeletClient.Key, rootSecrets.CA.Crt) if err != nil { return fmt.Errorf("error building kubelet client: %w", err) @@ -189,12 +187,18 @@ func (ctrl *KubeletStaticPodController) refreshPodStatus(ctx context.Context, r podsSeen := map[string]struct{}{} for _, pod := range podList.Items { - if pod.Metadata.Annotations.ConfigSource != "file" { + pod := pod + + switch pod.Metadata.Annotations.ConfigSource { + case "file": + // static pod from a file source + case "http": + // static pod from an HTTP source + default: + // anything else is not a static pod, skip it continue } - pod := pod - statusID := fmt.Sprintf("%s/%s", pod.Metadata.Namespace, pod.Metadata.Name) podsSeen[statusID] = struct{}{} diff --git a/internal/app/machined/pkg/controllers/k8s/static_pod_server.go b/internal/app/machined/pkg/controllers/k8s/static_pod_server.go index 2e8c45e7b..c0303d95f 100644 --- a/internal/app/machined/pkg/controllers/k8s/static_pod_server.go +++ b/internal/app/machined/pkg/controllers/k8s/static_pod_server.go @@ -13,6 +13,7 @@ import ( "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" "go.uber.org/zap" "gopkg.in/yaml.v3" @@ -23,6 +24,8 @@ import ( type StaticPodServerController struct { podList []byte podListMu sync.Mutex + + staticPodVersions map[string]string } // Name implements controller.Controller interface. @@ -65,6 +68,8 @@ type podList struct { // //nolint:gocyclo,cyclop func (ctrl *StaticPodServerController) Run(ctx context.Context, r controller.Runtime, logger *zap.Logger) error { + ctrl.staticPodVersions = map[string]string{} + shutdownServer, serverError, err := ctrl.createServer(ctx, r, logger) if err != nil { return fmt.Errorf("failed to start http server to serve static pod list: %w", err) @@ -79,7 +84,7 @@ func (ctrl *StaticPodServerController) Run(ctx context.Context, r controller.Run case err := <-serverError: return fmt.Errorf("http server closed unexpectedly: %w", err) case <-r.EventCh(): - staticPodList, err := buildPodList(ctx, r) + staticPodList, err := ctrl.buildPodList(ctx, r, logger) if err != nil { logger.Error("error building static pod list", zap.Error(err)) } @@ -91,8 +96,8 @@ func (ctrl *StaticPodServerController) Run(ctx context.Context, r controller.Run } } -func buildPodList(ctx context.Context, r controller.Runtime) ([]byte, error) { - staticPods, err := r.List(ctx, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined)) +func (ctrl *StaticPodServerController) buildPodList(ctx context.Context, r controller.Runtime, logger *zap.Logger) ([]byte, error) { + staticPods, err := safe.ReaderList[*k8s.StaticPod](ctx, r, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined)) if err != nil { return nil, fmt.Errorf("error listing static pods: %w", err) } @@ -102,10 +107,37 @@ func buildPodList(ctx context.Context, r controller.Runtime) ([]byte, error) { APIVersion: "v1", } - for _, staticPod := range staticPods.Items { - staticPodSpec := staticPod.(*k8s.StaticPod).TypedSpec() + touchedPodIDs := map[string]struct{}{} + + for iter := safe.IteratorFromList(staticPods); iter.Next(); { + id := iter.Value().Metadata().ID() + version := iter.Value().Metadata().Version().String() + + if oldVersion, exists := ctrl.staticPodVersions[id]; !exists || oldVersion != version { + ctrl.staticPodVersions[id] = version + + if !exists { + logger.Info("rendered new static pod", zap.String("id", id)) + } else { + logger.Info("rendered updated static pod", zap.String("id", id), zap.String("old_version", oldVersion), zap.String("new_version", version)) + } + } + + staticPodSpec := iter.Value().TypedSpec() pl.Items = append(pl.Items, staticPodSpec.Pod) + + touchedPodIDs[id] = struct{}{} + } + + for id := range ctrl.staticPodVersions { + if _, exists := touchedPodIDs[id]; exists { + continue + } + + logger.Info("removed static pod", zap.String("id", id)) + + delete(ctrl.staticPodVersions, id) } manifestContent, err := yaml.Marshal(pl) diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go index 5f26c06db..02b951fa7 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/digitalocean_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" "github.com/talos-systems/talos/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean" ) diff --git a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml index cb75870cd..08d86f559 100644 --- a/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml +++ b/internal/app/machined/pkg/runtime/v1alpha1/platform/digitalocean/testdata/expected.yaml @@ -1,96 +1,96 @@ addresses: -- address: 128.199.52.32/19 - linkName: eth0 - family: inet4 - scope: global - flags: permanent - layer: platform -- address: 2a03:b0c0:2:d0::1478:3001/64 - linkName: eth0 - family: inet6 - scope: global - flags: permanent - layer: platform -- address: 10.18.0.5/16 - linkName: eth0 - family: inet4 - scope: global - flags: permanent - layer: platform -- address: 10.133.0.2/16 - linkName: eth1 - family: inet4 - scope: global - flags: permanent - layer: platform + - address: 128.199.52.32/19 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 2a03:b0c0:2:d0::1478:3001/64 + linkName: eth0 + family: inet6 + scope: global + flags: permanent + layer: platform + - address: 10.18.0.5/16 + linkName: eth0 + family: inet4 + scope: global + flags: permanent + layer: platform + - address: 10.133.0.2/16 + linkName: eth1 + family: inet4 + scope: global + flags: permanent + layer: platform links: -- name: eth0 - logical: false - up: true - mtu: 0 - kind: "" - type: netrom - layer: platform -- name: eth1 - logical: false - up: true - mtu: 0 - kind: "" - type: netrom - layer: platform + - name: eth0 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform + - name: eth1 + logical: false + up: true + mtu: 0 + kind: "" + type: netrom + layer: platform routes: -- family: inet4 - dst: "" - src: "" - gateway: 128.199.32.1 - outLinkName: eth0 - table: main - priority: 1024 - scope: global - type: unicast - flags: "" - protocol: static - layer: platform -- family: inet4 - dst: 169.254.169.254/32 - src: "" - gateway: 128.199.32.1 - outLinkName: eth0 - table: main - priority: 512 - scope: global - type: unicast - flags: "" - protocol: static - layer: platform -- family: inet6 - dst: "" - src: "" - gateway: 2a03:b0c0:2:d0::1 - outLinkName: eth0 - table: main - priority: 1024 - scope: global - type: unicast - flags: "" - protocol: static - layer: platform + - family: inet4 + dst: "" + src: "" + gateway: 128.199.32.1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet4 + dst: 169.254.169.254/32 + src: "" + gateway: 128.199.32.1 + outLinkName: eth0 + table: main + priority: 512 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform + - family: inet6 + dst: "" + src: "" + gateway: 2a03:b0c0:2:d0::1 + outLinkName: eth0 + table: main + priority: 1024 + scope: global + type: unicast + flags: "" + protocol: static + layer: platform hostnames: -- hostname: debian-s-1vcpu-512mb-10gb-ams3-01 - domainname: "" - layer: platform + - hostname: debian-s-1vcpu-512mb-10gb-ams3-01 + domainname: "" + layer: platform resolvers: -- dnsServers: - - 67.207.67.2 - - 67.207.67.3 - layer: platform + - dnsServers: + - 67.207.67.2 + - 67.207.67.3 + layer: platform timeServers: [] operators: [] externalIPs: -- 128.199.52.32 + - 128.199.52.32 metadata: - platform: digital-ocean - hostname: debian-s-1vcpu-512mb-10gb-ams3-01 - region: ams3 - instanceId: "320206672" - providerId: digitalocean://320206672 + platform: digital-ocean + hostname: debian-s-1vcpu-512mb-10gb-ams3-01 + region: ams3 + instanceId: "320206672" + providerId: digitalocean://320206672 diff --git a/pkg/cluster/check/default.go b/pkg/cluster/check/default.go index ea720c950..ca9685014 100644 --- a/pkg/cluster/check/default.go +++ b/pkg/cluster/check/default.go @@ -71,6 +71,13 @@ func DefaultClusterChecks() []ClusterCheck { }, 10*time.Minute, 5*time.Second) }, + // wait for k8s control plane static pods + func(cluster ClusterInfo) conditions.Condition { + return conditions.PollingCondition("all control plane static pods to be running", func(ctx context.Context) error { + return K8sControlPlaneStaticPods(ctx, cluster) + }, 5*time.Minute, 5*time.Second) + }, + // wait for HA k8s control plane func(cluster ClusterInfo) conditions.Condition { return conditions.PollingCondition("all control plane components to be ready", func(ctx context.Context) error { diff --git a/pkg/cluster/check/kubernetes.go b/pkg/cluster/check/kubernetes.go index dfa2aa63a..6798ae8cf 100644 --- a/pkg/cluster/check/kubernetes.go +++ b/pkg/cluster/check/kubernetes.go @@ -9,13 +9,19 @@ import ( "context" "fmt" "net/netip" + "strings" + "github.com/cosi-project/runtime/pkg/resource" + "github.com/cosi-project/runtime/pkg/safe" + "github.com/siderolabs/gen/maps" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/talos-systems/talos/pkg/cluster" + "github.com/talos-systems/talos/pkg/machinery/client" "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/machinery/constants" + "github.com/talos-systems/talos/pkg/machinery/resources/k8s" ) // K8sAllNodesReportedAssertion checks whether all the nodes show up in node list. @@ -379,3 +385,47 @@ func ReplicaSetPresent(ctx context.Context, cluster cluster.K8sProvider, namespa return len(rss.Items) > 0, nil } + +// K8sControlPlaneStaticPods checks whether all the controlplane nodes are running required Kubernetes static pods. +// +//nolint:gocyclo,cyclop +func K8sControlPlaneStaticPods(ctx context.Context, cl ClusterInfo) error { + expectedNodes := append(cl.NodesByType(machine.TypeInit), cl.NodesByType(machine.TypeControlPlane)...) + + // using here new Talos COSI API, Talos 1.2+ required + c, err := cl.Client() + if err != nil { + return err + } + + for _, node := range expectedNodes { + expectedStaticPods := map[string]struct{}{ + "kube-system/kube-apiserver": {}, + "kube-system/kube-controller-manager": {}, + "kube-system/kube-scheduler": {}, + } + + items, err := safe.StateList[*k8s.StaticPodStatus]( + client.WithNode(ctx, node.InternalIP.String()), + c.COSI, + resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodStatusType, "", resource.VersionUndefined), + ) + if err != nil { + return fmt.Errorf("error listing static pods on node %s: %w", node.InternalIP, err) + } + + for iter := safe.IteratorFromList(items); iter.Next(); { + for expectedStaticPod := range expectedStaticPods { + if strings.HasPrefix(iter.Value().Metadata().ID(), expectedStaticPod) { + delete(expectedStaticPods, expectedStaticPod) + } + } + } + + if len(expectedStaticPods) > 0 { + return fmt.Errorf("missing static pods on node %s: %v", node.InternalIP, maps.Keys(expectedStaticPods)) + } + } + + return nil +} diff --git a/pkg/kubernetes/kubelet/kubelet.go b/pkg/kubernetes/kubelet/kubelet.go index 7243f82ee..20133d3b4 100644 --- a/pkg/kubernetes/kubelet/kubelet.go +++ b/pkg/kubernetes/kubelet/kubelet.go @@ -102,6 +102,6 @@ type Metadata struct { // Annotations are the annotations on a pod. type Annotations struct { // ConfigMapSource indicates where the resource is coming from. - // Its value is "file" for static pods and "api" for resources came from kube-apiserver. + // Its value is "file"/"http" for static pods and "api" for resources came from kube-apiserver. ConfigSource string `json:"kubernetes.io/config.source"` } diff --git a/pkg/machinery/go.mod b/pkg/machinery/go.mod index 6c6112865..ee17cde78 100644 --- a/pkg/machinery/go.mod +++ b/pkg/machinery/go.mod @@ -8,7 +8,7 @@ replace gopkg.in/yaml.v3 => github.com/unix4ever/yaml v0.0.0-20220527175918-f17b require ( github.com/containerd/go-cni v1.1.7 - github.com/cosi-project/runtime v0.2.0-alpha.1.0.20221009084302-e8a8fdcc7548 + github.com/cosi-project/runtime v0.2.0-alpha.2 github.com/dustin/go-humanize v1.0.0 github.com/evanphx/json-patch v5.6.0+incompatible github.com/ghodss/yaml v1.0.0 @@ -61,7 +61,7 @@ require ( go.uber.org/zap v1.23.0 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/net v0.0.0-20220923203811-8be639271d50 // indirect - golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 // indirect + golang.org/x/sync v0.1.0 // indirect golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect golang.org/x/text v0.3.8 // indirect gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect diff --git a/pkg/machinery/go.sum b/pkg/machinery/go.sum index 00e187618..4e886d2c4 100644 --- a/pkg/machinery/go.sum +++ b/pkg/machinery/go.sum @@ -22,8 +22,8 @@ github.com/containerd/go-cni v1.1.7 h1:1yKpVCQAXI21BJIy8q7Nyk4CWpIgUno6ib7JIDca7 github.com/containerd/go-cni v1.1.7/go.mod h1:Ve4Q0RB2Bw78D90OL0YVyDjqdTL7FKh9W+UPbhWiZXA= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= -github.com/cosi-project/runtime v0.2.0-alpha.1.0.20221009084302-e8a8fdcc7548 h1:/CMoJlmVdr1XrAoo4cQDPF4rwB2Ap1WCa/BlFfkqOW0= -github.com/cosi-project/runtime v0.2.0-alpha.1.0.20221009084302-e8a8fdcc7548/go.mod h1:u60xdQ7/f8WkO0qsDwPRJuy+0edCDYvwh2xYUvDO+no= +github.com/cosi-project/runtime v0.2.0-alpha.2 h1:B27kw7knAukaHiXwKP51hGh4q2RMaY+wPF+beXBjZMI= +github.com/cosi-project/runtime v0.2.0-alpha.2/go.mod h1:jv79UECqzQaAeVwbdawUGmEBCX0fl7J7JBysnOYIM4U= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -198,8 +198,9 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7 h1:ZrnxWX62AgTKOSagEqxvb3ffipvEDX2pl7E1TdqLqIc= golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=