[FIX/ENHANCEMENT] node/create: inherit registry config from existing nodes (#597)

pkg/client/node.go

@@ -25,7 +25,9 @@ package client
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"reflect"
 	"strings"
@@ -33,8 +35,10 @@ import (
 
 	dockerunits "github.com/docker/go-units"
 	"github.com/imdario/mergo"
+	"github.com/rancher/k3d/v4/pkg/actions"
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	"github.com/rancher/k3d/v4/pkg/runtimes/docker"
+	runtimeErrors "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	"github.com/rancher/k3d/v4/pkg/util"
 	log "github.com/sirupsen/logrus"
@@ -89,6 +93,25 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 
 	log.Debugf("Adding node %+v \n>>> to cluster %+v\n>>> based on existing node %+v", node, cluster, chosenNode)
 
+	// fetch registry config
+	registryConfigBytes := []byte{}
+	registryConfigReader, err := runtime.ReadFromNode(ctx, k3d.DefaultRegistriesFilePath, chosenNode)
+	if err != nil {
+		if !errors.Is(err, runtimeErrors.ErrRuntimeFileNotFound) {
+			log.Warnf("Failed to read registry config from node %s: %+v", node.Name, err)
+		}
+	} else {
+		defer registryConfigReader.Close()
+
+		var err error
+		registryConfigBytes, err = ioutil.ReadAll(registryConfigReader)
+		if err != nil {
+			log.Warnf("Failed to read registry config from node %s: %+v", node.Name, err)
+		}
+		registryConfigReader.Close()
+		registryConfigBytes = bytes.Trim(registryConfigBytes[512:], "\x00") // trim control characters, etc.
+	}
+
 	// merge node config of new node into existing node config
 	if err := mergo.MergeWithOverwrite(chosenNode, *node); err != nil {
 		log.Errorln("Failed to merge new node config into existing node config")
@@ -133,11 +156,27 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
 		}
 	}
 
+	// add node actions
+	if len(registryConfigBytes) != 0 {
+		if createNodeOpts.NodeHooks == nil {
+			createNodeOpts.NodeHooks = []k3d.NodeHook{}
+		}
+		createNodeOpts.NodeHooks = append(createNodeOpts.NodeHooks, k3d.NodeHook{
+			Stage: k3d.LifecycleStagePreStart,
+			Action: actions.WriteFileAction{
+				Runtime: runtime,
+				Content: registryConfigBytes,
+				Dest:    k3d.DefaultRegistriesFilePath,
+				Mode:    0644,
+			},
+		})
+	}
+
 	// clear status fields
 	node.State.Running = false
 	node.State.Status = ""
 
-	if err := NodeRun(ctx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
+	if err := NodeRun(ctx, runtime, node, createNodeOpts); err != nil {
 		return err
 	}
 
@@ -162,7 +201,7 @@ func NodeAddToClusterMulti(ctx context.Context, runtime runtimes.Runtime, nodes
 
 	nodeWaitGroup, ctx := errgroup.WithContext(ctx)
 	for _, node := range nodes {
-		if err := NodeAddToCluster(ctx, runtime, node, cluster, k3d.NodeCreateOpts{}); err != nil {
+		if err := NodeAddToCluster(ctx, runtime, node, cluster, createNodeOpts); err != nil {
 			return err
 		}
 		if createNodeOpts.Wait {
@@ -230,8 +269,9 @@ func NodeRun(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, node
 	}
 
 	if err := NodeStart(ctx, runtime, node, k3d.NodeStartOpts{
-		Wait:    nodeCreateOpts.Wait,
-		Timeout: nodeCreateOpts.Timeout,
+		Wait:      nodeCreateOpts.Wait,
+		Timeout:   nodeCreateOpts.Timeout,
+		NodeHooks: nodeCreateOpts.NodeHooks,
 	}); err != nil {
 		return err
 	}

pkg/runtimes/docker/util.go

@@ -26,6 +26,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 
@@ -35,6 +36,7 @@ import (
 	"github.com/docker/docker/client"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/pkg/errors"
+	runtimeErrors "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
 	log "github.com/sirupsen/logrus"
 )
@@ -140,6 +142,30 @@ func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, mo
 	return nil
 }
 
+// ReadFromNode reads from a given filepath inside the node container
+func (d Docker) ReadFromNode(ctx context.Context, path string, node *k3d.Node) (io.ReadCloser, error) {
+	log.Tracef("Reading path %s from node %s...", path, node.Name)
+	nodeContainer, err := getNodeContainer(ctx, node)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to find container for node '%s': %+v", node.Name, err)
+	}
+
+	docker, err := GetDockerClient()
+	if err != nil {
+		return nil, err
+	}
+
+	reader, _, err := docker.CopyFromContainer(ctx, nodeContainer.ID, path)
+	if err != nil {
+		if client.IsErrNotFound(err) {
+			return nil, errors.Wrap(runtimeErrors.ErrRuntimeFileNotFound, err.Error())
+		}
+		return nil, err
+	}
+
+	return reader, err
+}
+
 // GetDockerClient returns a docker client
 func GetDockerClient() (*client.Client, error) {
 	var err error

pkg/runtimes/errors/errors.go

@@ -19,7 +19,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 */
-package runtimes
+package errors
 
 import "errors"
 
@@ -34,3 +34,6 @@ var (
 	ErrRuntimeNetworkNotExists     = errors.New("network does not exist")
 	ErrRuntimeNetworkMultiSameName = errors.New("multiple networks with same name found")
 )
+
+// Container Filesystem Errors
+var ErrRuntimeFileNotFound = errors.New("file not found")

pkg/runtimes/runtime.go

@@ -71,6 +71,7 @@ type Runtime interface {
 	GetImages(context.Context) ([]string, error)
 	CopyToNode(context.Context, string, string, *k3d.Node) error               // @param context, source, destination, node
 	WriteToNode(context.Context, []byte, string, os.FileMode, *k3d.Node) error // @param context, content, destination, filemode, node
+	ReadFromNode(context.Context, string, *k3d.Node) (io.ReadCloser, error)    // @param context, filepath, node
 	GetHostIP(context.Context, string) (net.IP, error)
 	ConnectNodeToNetwork(context.Context, *k3d.Node, string) error      // @param context, node, network name
 	DisconnectNodeFromNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name