mirrors/k3d
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Enhancement] Edit CoreDNS ConfigMap on disk for more reliability (#814)

Browse Source
This commit is contained in:
Thorsten Klein 2021-10-22 15:07:01 +02:00 committed by GitHub
parent 376f0378af
commit 7113694ab5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 142 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
pkg/actions/nodehooks.go
View File

@ -22,11 +22,16 @@ THE SOFTWARE.
package actions
import (
"bytes"
"context"
"fmt"
"io/ioutil"
"os"
"github.com/rancher/k3d/v5/pkg/runtimes"
k3d "github.com/rancher/k3d/v5/pkg/types"
l "github.com/rancher/k3d/v5/pkg/logger"
)
type WriteFileAction struct {
@ -39,3 +44,35 @@ type WriteFileAction struct {
func (act WriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, act.Mode, node)
}
type RewriteFileAction struct {
Runtime runtimes.Runtime
Path string
RewriteFunc func([]byte) ([]byte, error)
Mode os.FileMode
}
func (act RewriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
reader, err := act.Runtime.ReadFromNode(ctx, act.Path, node)
if err != nil {
return fmt.Errorf("runtime failed to read '%s' from node '%s': %w", act.Path, node.Name, err)
}
defer reader.Close()
file, err := ioutil.ReadAll(reader)
if err != nil {
return fmt.Errorf("failed to read file: %w", err)
}
file = bytes.Trim(file[512:], "\x00") // trim control characters, etc.
file, err = act.RewriteFunc(file)
if err != nil {
return fmt.Errorf("error while rewriting %s in %s: %w", act.Path, node.Name, err)
}
l.Log().Tracef("Rewritten:\n%s", string(file))
return act.Runtime.WriteToNode(ctx, file, act.Path, act.Mode, node)
}

92
pkg/client/cluster.go
View File

@ -22,6 +22,7 @@ THE SOFTWARE.
package client
import (
"bytes"
"context"
_ "embed"
"errors"
@ -30,6 +31,7 @@ import (
"os"
"sort"
"strconv"
"strings"
"time"
gort "runtime"
@ -845,18 +847,10 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
}
}
// TODO: remove trace logs below
l.Log().Traceln("Servers before sort:")
for i, n := range servers {
l.Log().Tracef("Server %d - %s", i, n.Name)
}
// sort list of servers for properly ordered sequential start
sort.Slice(servers, func(i, j int) bool {
return servers[i].Name < servers[j].Name
})
l.Log().Traceln("Servers after sort:")
for i, n := range servers {
l.Log().Tracef("Server %d - %s", i, n.Name)
}
/*
* Init Node
@ -880,7 +874,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
for _, serverNode := range servers {
if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
Wait: true,
NodeHooks: clusterStartOpts.NodeHooks,
NodeHooks: append(clusterStartOpts.NodeHooks, serverNode.HookActions...),
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
@ -948,14 +942,65 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
})
postStartErrgrp.Go(func() error {
// add host.k3d.internal record to the CoreDNS Configmap
l.Log().Infoln("Injecting record for host.k3d.internal into CoreDNS configmap...")
if err := corednsAddHost(postStartErrgrpCtx, runtime, cluster, clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord); err != nil {
return err
hosts := fmt.Sprintf("%s %s\n", clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord)
net, err := runtime.GetNetwork(ctx, &cluster.Network)
if err != nil {
return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
}
for _, member := range net.Members {
hosts += fmt.Sprintf("%s %s\n", member.IP.String(), member.Name)
}
// add records for other containers in the cluster network to the CoreDNS configmap (e.g. useful for using registries from within Pods inside the cluster)
return prepCoreDNSInjectNetworkMembers(postStartErrgrpCtx, runtime, cluster)
l.Log().Infof("Injecting records for host.k3d.internal and for %d network members into CoreDNS configmap...", len(net.Members))
act := actions.RewriteFileAction{
Runtime: runtime,
Path: "/var/lib/rancher/k3s/server/manifests/coredns.yaml",
Mode: 0744,
RewriteFunc: func(input []byte) ([]byte, error) {
split, err := util.SplitYAML(input)
if err != nil {
return nil, fmt.Errorf("error splitting yaml: %w", err)
}
var outputBuf bytes.Buffer
outputEncoder := yaml.NewEncoder(&outputBuf)
for _, d := range split {
var doc map[string]interface{}
if err := yaml.Unmarshal(d, &doc); err != nil {
return nil, err
}
if kind, ok := doc["kind"]; ok {
if strings.ToLower(kind.(string)) == "configmap" {
configmapData := doc["data"].(map[interface{}]interface{})
configmapData["NodeHosts"] = hosts
}
}
if err := outputEncoder.Encode(doc); err != nil {
return nil, err
}
}
outputEncoder.Close()
return outputBuf.Bytes(), nil
},
}
// get the first server in the list and run action on it once it's ready for it
for _, n := range cluster.Nodes {
if n.Role == k3d.ServerRole {
ts, err := time.Parse("2006-01-02T15:04:05.999999999Z", n.State.Started)
if err != nil {
return err
}
if err := NodeWaitForLogMessage(ctx, runtime, n, "Cluster dns configmap", ts.Truncate(time.Second)); err != nil {
return err
}
return act.Run(ctx, n)
}
}
return nil
})
if err := postStartErrgrp.Wait(); err != nil {
@ -1075,21 +1120,6 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C
return nil
}
func prepCoreDNSInjectNetworkMembers(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
net, err := runtime.GetNetwork(ctx, &cluster.Network)
if err != nil {
return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
}
l.Log().Debugf("Adding %d network members to CoreDNS...", len(net.Members))
for _, member := range net.Members {
hostsEntry := fmt.Sprintf("%s %s", member.IP.String(), member.Name)
if err := corednsAddHost(ctx, runtime, cluster, member.IP.String(), member.Name); err != nil {
return fmt.Errorf("failed to add host entry \"%s\" into CoreDNS: %w", hostsEntry, err)
}
}
return nil
}
func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
success := false
for _, node := range cluster.Nodes {

14
pkg/client/node.go
View File

@ -409,6 +409,16 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
}
}
// execute lifecycle hook actions
for _, hook := range nodeStartOpts.NodeHooks {
if hook.Stage == k3d.LifecycleStagePostStart {
l.Log().Tracef("Node %s: Executing postStartAction '%s'", node.Name, reflect.TypeOf(hook))
if err := hook.Action.Run(ctx, node); err != nil {
l.Log().Errorf("Node %s: Failed executing postStartAction '%+v': %+v", node.Name, hook, err)
}
}
}
return nil
}
@ -444,7 +454,7 @@ func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node,
// DNS Fix
if fixes.FixEnabled(fixes.EnvFixDNS) {
l.Log().Debugf("ENABLING DNS MAGIC!!!")
l.Log().Debugln(">>> enabling dns magic")
if nodeStartOpts.EnvironmentInfo == nil || nodeStartOpts.EnvironmentInfo.HostGateway == nil {
return fmt.Errorf("Cannot enable DNS fix, as Host Gateway IP is missing!")
@ -465,7 +475,7 @@ func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node,
// CGroupsV2Fix
if fixes.FixEnabled(fixes.EnvFixCgroupV2) {
l.Log().Debugf("ENABLING CGROUPSV2 MAGIC!!!")
l.Log().Debugf(">>> enabling cgroupsv2 magic")
if nodeStartOpts.NodeHooks == nil {
nodeStartOpts.NodeHooks = []k3d.NodeHook{}

31
pkg/util/util.go
View File

@ -22,7 +22,13 @@ THE SOFTWARE.
package util
import "strings"
import (
"bytes"
"io"
"strings"
"gopkg.in/yaml.v2"
)
func RemoveElementFromStringSlice(slice []string, index int) []string {
slice[index] = slice[len(slice)-1]
@ -35,3 +41,26 @@ func ReplaceInAllElements(replacer *strings.Replacer, arr []string) []string {
}
return arr
}
func SplitYAML(resources []byte) ([][]byte, error) {
dec := yaml.NewDecoder(bytes.NewReader(resources))
var res [][]byte
for {
var value interface{}
err := dec.Decode(&value)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
valueBytes, err := yaml.Marshal(value)
if err != nil {
return nil, err
}
res = append(res, valueBytes)
}
return res, nil
}

2
tests/test_full_lifecycle.sh
View File

@ -49,6 +49,8 @@ $EXE cluster start "$clustername" --wait --timeout 360s || failed "cluster didn'
info "Checking that we have access to the cluster..."
check_clusters "$clustername" || failed "error checking cluster"
kubectl delete pod -n kube-system -l k8s-app=kube-dns > /dev/null 2>&1 # delete coredns to force reload of config (reload plugin uses default 30s, which will make tests below fail)
info "Checking that we have 2 nodes online..."
check_multi_node "$clustername" 2 || failed "failed to verify number of nodes"