From b6acd0a152c2a69d99bd48f57c328421e112d0fd Mon Sep 17 00:00:00 2001 From: Murali Reddy Date: Mon, 29 Jun 2020 18:26:19 +0530 Subject: [PATCH] stop processing service and endpoints updates if network service (#939) controller has already shutdown fixes panic seen in #937 --- .../proxy/network_services_controller.go | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/controllers/proxy/network_services_controller.go b/pkg/controllers/proxy/network_services_controller.go index c2489292..ebddb131 100644 --- a/pkg/controllers/proxy/network_services_controller.go +++ b/pkg/controllers/proxy/network_services_controller.go @@ -366,6 +366,9 @@ func (nsc *NetworkServicesController) Run(healthChan chan<- *healthcheck.Control for { select { case <-stopCh: + nsc.mu.Lock() + nsc.readyForUpdates = false + nsc.mu.Unlock() glog.Info("Shutting down network services controller") return @@ -790,13 +793,13 @@ func (nsc *NetworkServicesController) OnEndpointsUpdate(ep *api.Endpoints) { return } - glog.V(1).Infof("Received update to endpoint: %s/%s from watch API", ep.Namespace, ep.Name) - if !nsc.readyForUpdates { - glog.V(3).Infof("Skipping update to endpoint: %s/%s, controller still performing bootup full-sync", ep.Namespace, ep.Name) - return - } nsc.mu.Lock() defer nsc.mu.Unlock() + glog.V(1).Infof("Received update to endpoint: %s/%s from watch API", ep.Namespace, ep.Name) + if !nsc.readyForUpdates { + glog.V(3).Infof("Skipping update to endpoint: %s/%s as controller is not ready to process service and endpoints updates", ep.Namespace, ep.Name) + return + } // build new service and endpoints map to reflect the change newServiceMap := nsc.buildServicesInfo() @@ -814,14 +817,16 @@ func (nsc *NetworkServicesController) OnEndpointsUpdate(ep *api.Endpoints) { // OnServiceUpdate handle change in service update from the API server func (nsc *NetworkServicesController) OnServiceUpdate(svc *api.Service) { - glog.V(1).Infof("Received update to service: %s/%s from watch API", svc.Namespace, svc.Name) - if !nsc.readyForUpdates { - glog.V(3).Infof("Skipping update to service: %s/%s, controller still performing bootup full-sync", svc.Namespace, svc.Name) - return - } + nsc.mu.Lock() defer nsc.mu.Unlock() + glog.V(1).Infof("Received update to service: %s/%s from watch API", svc.Namespace, svc.Name) + if !nsc.readyForUpdates { + glog.V(3).Infof("Skipping update to service: %s/%s as controller is not ready to process service and endpoints updates", svc.Namespace, svc.Name) + return + } + // build new service and endpoints map to reflect the change newServiceMap := nsc.buildServicesInfo() newEndpointsMap := nsc.buildEndpointsInfo()