mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request #335 from rancher/fix/idempotent-cluster-start

Browse Source 
[Fix] clusterStart: hangs forever when trying to start a node that's already running (waiting for log message)
This commit is contained in:
Thorsten Klein 2020-08-14 13:39:54 +03:00 committed by GitHub
commit 9bf264aaa9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 35 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
pkg/cluster/cluster.go
View File

@ -548,38 +548,48 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
continue
}
// start node
if err := runtime.StartNode(ctx, node); err != nil {
log.Warningf("Failed to start node '%s': Try to start it manually", node.Name)
failed++
continue
}
// check if node is running already to avoid waiting forever when checking for the node log message
if !node.State.Running {
// asynchronously wait for this server node to be ready (by checking the logs for a specific log mesage)
if node.Role == k3d.ServerRole && startClusterOpts.WaitForServer {
serverNode := node
waitForServerWaitgroup.Go(func() error {
// TODO: avoid `level=fatal msg="starting kubernetes: preparing server: post join: a configuration change is already in progress (5)"`
// ... by scanning for this line in logs and restarting the container in case it appears
log.Debugf("Starting to wait for server node '%s'", serverNode.Name)
return NodeWaitForLogMessage(ctx, runtime, serverNode, k3d.ReadyLogMessageByRole[k3d.ServerRole], start)
})
// start node
if err := runtime.StartNode(ctx, node); err != nil {
log.Warningf("Failed to start node '%s': Try to start it manually", node.Name)
failed++
continue
}
// asynchronously wait for this server node to be ready (by checking the logs for a specific log mesage)
if node.Role == k3d.ServerRole && startClusterOpts.WaitForServer {
serverNode := node
waitForServerWaitgroup.Go(func() error {
// TODO: avoid `level=fatal msg="starting kubernetes: preparing server: post join: a configuration change is already in progress (5)"`
// ... by scanning for this line in logs and restarting the container in case it appears
log.Debugf("Starting to wait for server node '%s'", serverNode.Name)
return NodeWaitForLogMessage(ctx, runtime, serverNode, k3d.ReadyLogMessageByRole[k3d.ServerRole], start)
})
}
} else {
log.Infof("Node '%s' already running", node.Name)
}
}
// start serverlb
if serverlb != nil {
log.Debugln("Starting serverlb...")
if err := runtime.StartNode(ctx, serverlb); err != nil { // FIXME: we could run into a nullpointer exception here
log.Warningf("Failed to start serverlb '%s': Try to start it manually", serverlb.Name)
failed++
if !serverlb.State.Running {
log.Debugln("Starting serverlb...")
if err := runtime.StartNode(ctx, serverlb); err != nil { // FIXME: we could run into a nullpointer exception here
log.Warningf("Failed to start serverlb '%s': Try to start it manually", serverlb.Name)
failed++
}
waitForServerWaitgroup.Go(func() error {
// TODO: avoid `level=fatal msg="starting kubernetes: preparing server: post join: a configuration change is already in progress (5)"`
// ... by scanning for this line in logs and restarting the container in case it appears
log.Debugf("Starting to wait for loadbalancer node '%s'", serverlb.Name)
return NodeWaitForLogMessage(ctx, runtime, serverlb, k3d.ReadyLogMessageByRole[k3d.LoadBalancerRole], start)
})
} else {
log.Infof("Serverlb '%s' already running", serverlb.Name)
}
waitForServerWaitgroup.Go(func() error {
// TODO: avoid `level=fatal msg="starting kubernetes: preparing server: post join: a configuration change is already in progress (5)"`
// ... by scanning for this line in logs and restarting the container in case it appears
log.Debugf("Starting to wait for loadbalancer node '%s'", serverlb.Name)
return NodeWaitForLogMessage(ctx, runtime, serverlb, k3d.ReadyLogMessageByRole[k3d.LoadBalancerRole], start)
})
}
if err := waitForServerWaitgroup.Wait(); err != nil {