fix/workaround: add workaround for cgroupv2 until fixed in k3s (#579)

special thanks to @AkihiroSuda for the support on this!

CHANGELOG.md

@@ -1,5 +1,16 @@
 # Changelog
 
+## v4.4.3
+
+### Highlights
+
+- cgroupv2 support: to properly work on cgroupv2 systems, k3s has to move all the processes from the root cgroup to a new /init cgroup and enable subtree_control
+  - this is going to be included in the k3s agent code directly (<https://github.com/k3s-io/k3s/pull/3242>)
+  - for now we're overriding the container entrypoint with a script that does this (#579, compare <https://github.com/k3s-io/k3s/pull/3237>)
+  - thanks a lot for all the input and support @AkihiroSuda
+  - **Usage**: set the environment variable `K3D_FIX_CGROUPV2` to a `true` value before/when creating a cluster with k3d
+    - e.g. `export K3D_FIX_CGROUPV2=1`
+
 ## v4.4.2
 
 ### Fixes

cmd/root.go

@@ -32,8 +32,6 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
-	rt "runtime"
-
 	"github.com/rancher/k3d/v4/cmd/cluster"
 	cfg "github.com/rancher/k3d/v4/cmd/config"
 	"github.com/rancher/k3d/v4/cmd/image"
@@ -206,7 +204,9 @@ func initRuntime() {
 		log.Fatalln(err)
 	}
 	runtimes.SelectedRuntime = runtime
-	log.Debugf("Selected runtime is '%T' on GOOS '%s/%s'", runtimes.SelectedRuntime, rt.GOOS, rt.GOARCH)
+	if rtinfo, err := runtime.Info(); err == nil {
+		log.Debugf("Runtime Info:\n%+v", rtinfo)
+	}
 }
 
 func printVersion() {

pkg/actions/nodehooks.go

@@ -23,6 +23,7 @@ package actions
 
 import (
 	"context"
+	"os"
 
 	"github.com/rancher/k3d/v4/pkg/runtimes"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
@@ -32,8 +33,9 @@ type WriteFileAction struct {
 	Runtime runtimes.Runtime
 	Content []byte
 	Dest    string
+	Mode    os.FileMode
 }
 
 func (act WriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
-	return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, node)
+	return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, act.Mode, node)
 }

pkg/client/cluster.go

@@ -23,6 +23,7 @@ package client
 
 import (
 	"context"
+	_ "embed"
 	"errors"
 	"fmt"
 	"sort"
@@ -41,6 +42,7 @@ import (
 	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	"github.com/rancher/k3d/v4/pkg/types"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/types/fixes"
 	"github.com/rancher/k3d/v4/pkg/types/k3s"
 	"github.com/rancher/k3d/v4/pkg/util"
 	"github.com/rancher/k3d/v4/version"
@@ -205,6 +207,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 				Runtime: runtime,
 				Content: regCm,
 				Dest:    k3d.DefaultLocalRegistryHostingConfigmapTempPath,
+				Mode:    0644,
 			},
 		})
 
@@ -233,6 +236,23 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 				Runtime: runtime,
 				Content: regConfBytes,
 				Dest:    k3d.DefaultRegistriesFilePath,
+				Mode:    0644,
+			},
+		})
+	}
+
+	// FIXME: FixCgroupV2 - to be removed when fixed upstream
+	if fixes.FixCgroupV2Enabled() {
+
+		log.Debugln("experimental cgroupv2 fix enabled")
+
+		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
+			Stage: k3d.LifecycleStagePreStart,
+			Action: actions.WriteFileAction{
+				Runtime: runtime,
+				Content: fixes.CgroupV2Entrypoint,
+				Dest:    "/bin/entrypoint.sh",
+				Mode:    0744,
 			},
 		})
 	}

pkg/runtimes/docker/translate.go

@@ -33,6 +33,7 @@ import (
 	"github.com/docker/go-connections/nat"
 	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/types/fixes"
 	log "github.com/sirupsen/logrus"
 
 	dockercliopts "github.com/docker/cli/opts"
@@ -54,6 +55,15 @@ func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
 	containerConfig.Image = node.Image
 
 	/* Command & Arguments */
+	// FIXME: FixCgroupV2 - to be removed when fixed upstream
+	if fixes.FixCgroupV2Enabled() {
+		if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole {
+			containerConfig.Entrypoint = []string{
+				"/bin/entrypoint.sh",
+			}
+		}
+	}
+
 	containerConfig.Cmd = []string{}
 
 	containerConfig.Cmd = append(containerConfig.Cmd, node.Cmd...)  // contains k3s command and role-specific required flags/args

pkg/runtimes/docker/translate_test.go

@@ -31,6 +31,7 @@ import (
 	"github.com/docker/docker/api/types/network"
 	"github.com/docker/go-connections/nat"
 	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v4/pkg/types/fixes"
 )
 
 func TestTranslateNodeToContainer(t *testing.T) {
@@ -93,6 +94,11 @@ func TestTranslateNodeToContainer(t *testing.T) {
 		},
 	}
 
+	// TODO: // FIXME: FixCgroupV2 - to be removed when fixed upstream
+	if fixes.FixCgroupV2Enabled() {
+		expectedRepresentation.ContainerConfig.Entrypoint = []string{"/bin/entrypoint.sh"}
+	}
+
 	actualRepresentation, err := TranslateNodeToContainer(inputNode)
 	if err != nil {
 		t.Error(err)

pkg/runtimes/docker/util.go

@@ -96,7 +96,7 @@ func (d Docker) CopyToNode(ctx context.Context, src string, dest string, node *k
 }
 
 // WriteToNode writes a byte array to the selected node
-func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, node *k3d.Node) error {
+func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, mode os.FileMode, node *k3d.Node) error {
 
 	nodeContainer, err := getNodeContainer(ctx, node)
 	if err != nil {
@@ -116,7 +116,7 @@ func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, no
 	defer tarWriter.Close()
 	tarHeader := &tar.Header{
 		Name: dest,
-		Mode: 0644,
+		Mode: int64(mode),
 		Size: int64(len(content)),
 	}
 
@@ -140,7 +140,6 @@ func (d Docker) WriteToNode(ctx context.Context, content []byte, dest string, no
 	return nil
 }
 
-
 // GetDockerClient returns a docker client
 func GetDockerClient() (*client.Client, error) {
 	var err error

pkg/runtimes/runtime.go

@@ -27,6 +27,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"os"
 	"time"
 
 	"github.com/rancher/k3d/v4/pkg/runtimes/docker"
@@ -68,8 +69,8 @@ type Runtime interface {
 	ExecInNodeGetLogs(context.Context, *k3d.Node, []string) (*bufio.Reader, error)
 	GetNodeLogs(context.Context, *k3d.Node, time.Time) (io.ReadCloser, error)
 	GetImages(context.Context) ([]string, error)
-	CopyToNode(context.Context, string, string, *k3d.Node) error  // @param context, source, destination, node
+	CopyToNode(context.Context, string, string, *k3d.Node) error               // @param context, source, destination, node
-	WriteToNode(context.Context, []byte, string, *k3d.Node) error // @param context, content, destination, node
+	WriteToNode(context.Context, []byte, string, os.FileMode, *k3d.Node) error // @param context, content, destination, filemode, node
 	GetHostIP(context.Context, string) (net.IP, error)
 	ConnectNodeToNetwork(context.Context, *k3d.Node, string) error      // @param context, node, network name
 	DisconnectNodeFromNetwork(context.Context, *k3d.Node, string) error // @param context, node, network name

pkg/types/fixes/assets/cgroupv2-entrypoint.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+#########################################################################################################################################
+# DISCLAIMER																																																														#
+# Copied from https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/hack/dind#L28-L37															#
+# Permission granted by Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp> (https://github.com/rancher/k3d/issues/493#issuecomment-827405962)	#
+# Moby License Apache 2.0: https://github.com/moby/moby/blob/ed89041433a031cafc0a0f19cfe573c31688d377/LICENSE														#
+#########################################################################################################################################
+if [ -f /sys/fs/cgroup/cgroup.controllers ]; then
+	# move the processes from the root group to the /init group,
+  # otherwise writing subtree_control fails with EBUSY.
+  mkdir -p /sys/fs/cgroup/init
+  busybox xargs -rn1 < /sys/fs/cgroup/cgroup.procs > /sys/fs/cgroup/init/cgroup.procs || :
+  # enable controllers
+  sed -e 's/ / +/g' -e 's/^/+/' <"/sys/fs/cgroup/cgroup.controllers" >"/sys/fs/cgroup/cgroup.subtree_control"
+fi
+
+exec /bin/k3s "$@"

pkg/types/fixes/fixes.go

@@ -0,0 +1,52 @@
+/*
+Copyright © 2020 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package fixes
+
+import (
+	_ "embed"
+	"os"
+	"strconv"
+)
+
+/* NOTE
+ * This file includes types used for workarounds and hotfixes which are subject to change
+ * and may disappear anytime, e.g. when the fix was included in an upstream project
+ */
+
+/*
+ * Cgroupv2 fix as per https://github.com/k3s-io/k3s/pull/3237 & https://github.com/k3s-io/k3s/pull/3242
+ * FIXME: FixCgroupV2 - to be removed when fixed upstream
+ */
+
+// EnvFixCgroupV2 is the environment variable that k3d will check for to enable/disable the cgroupv2 workaround
+const EnvFixCgroupV2 = "K3D_FIX_CGROUPV2"
+
+//go:embed assets/cgroupv2-entrypoint.sh
+var CgroupV2Entrypoint []byte
+
+func FixCgroupV2Enabled() bool {
+	enabled, err := strconv.ParseBool(os.Getenv(EnvFixCgroupV2))
+	if err != nil {
+		return false
+	}
+	return enabled
+}