v5.0.3: changelog

- new special internal role `initServer` used only to determine the correct ready-log-message
- ready-log-messages now looked up by role and new `Intent` type (cluster-create/cluster-start/node-create/node-start), as especially for the init server there are different log messages indicating that we can proceed with the next step
- moving types around:
	- K3s env vars now under .../types/k3s/env.go
	- defaults now under .../types/defaults.go
	- ...
- improved waiting for log messages
	- not checking the whole log again and again in a loop
	- follow log with a single reader (and retry in case we see a fatal error, meaning that the K3s container will restart -> backoff after 10 tries)
	- BREAKING: new `*runtimeTypes.NodeLogsOpts` parameter in GetNodeLogs

.all-contributorsrc

@@ -121,6 +121,24 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "Shanduur",
+      "name": "Mateusz Urbanek",
+      "avatar_url": "https://avatars.githubusercontent.com/u/32583062?v=4",
+      "profile": "http://shanduur.github.io",
+      "contributions": [
+        "code"
+      ]
+    },
+    {
+      "login": "benjaminjb",
+      "name": "Benjamin Blattberg",
+      "avatar_url": "https://avatars.githubusercontent.com/u/4651855?v=4",
+      "profile": "https://github.com/benjaminjb",
+      "contributions": [
+        "code"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.drone.yml

@@ -14,7 +14,7 @@ platform:
 steps:
 
   - name: lint
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make ci-setup
       - make check-fmt lint
@@ -40,7 +40,7 @@ steps:
         - tag
 
   - name: build
-    image: golang:1.16
+    image: golang:1.17
     environment:
       GIT_TAG: "${DRONE_TAG}"
     commands:
@@ -73,6 +73,7 @@ steps:
         - tag
       ref:
         include:
+          # include only pre-release tags
           - "refs/tags/*rc*"
           - "refs/tags/*beta*"
           - "refs/tags/*alpha*"
@@ -97,64 +98,13 @@ steps:
         - tag
       ref:
         exclude:
+          # exclude pre-release tags
           - "refs/tags/*rc*"
           - "refs/tags/*beta*"
           - "refs/tags/*alpha*"
           - "refs/tags/*test*"
           - "refs/tags/*dev*"
 
-  - name: docker_build_push_dind
-    image: plugins/docker
-    environment:
-      DOCKER_BUILDKIT: "1"
-    settings:
-      repo: rancher/k3d
-      tags:
-        - latest-dind
-        - "${DRONE_TAG}-dind"
-      dockerfile: Dockerfile
-      target: dind
-      context: .
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      build_args:
-        - GIT_TAG_OVERRIDE=${DRONE_TAG}
-    depends_on:
-      - lint
-      - test
-      - build
-    when:
-      event:
-        - tag
-
-  - name: docker_build_push_binary
-    environment:
-      DOCKER_BUILDKIT: "1"
-    image: plugins/docker
-    settings:
-      repo: rancher/k3d
-      tags:
-        - latest
-        - "${DRONE_TAG}"
-      dockerfile: Dockerfile
-      target: binary-only
-      context: .
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-      build_args:
-        - GIT_TAG_OVERRIDE=${DRONE_TAG}
-    depends_on:
-      - lint
-      - test
-      - build
-    when:
-      event:
-        - tag
-
 services:
   # Starting the docker service to be used by dind
   - name: docker
@@ -168,14 +118,29 @@ volumes:
   - name: dockersock
     temp: {}
 
+
 ---
-#########################
-##### Documentation #####
-#########################
+###########################
+###### Docker Images ######
+###########################
+#
+# +++ Docker Images +++
+# Tagged using the auto_tag feature of the docker plugin
+# See http://plugins.drone.io/drone-plugins/drone-docker/#autotag
+# > if event type is `tag`
+# > > 1.0.0 produces docker tags 1, 1.0, 1.0.0
+# > > 1.0.0-rc.1 produces docker tags 1.0.0-rc.1
+# > if event type is `push` and target branch == default branch (main)
+# > > tag `latest`
+
+
+################################
+##### Docker Images: amd64 #####
+################################
 
 kind: pipeline
 type: docker
-name: docs
+name: linux_amd64
 
 platform:
   os: linux
@@ -183,93 +148,99 @@ platform:
 
 steps:
 
-  - name: build
-    image: python:3.9
-    commands:
-      - python3 -m pip install -r docs/requirements.txt
-      - mkdocs build --verbose --clean --strict
-    when:
-      branch:
-        - main
-      event:
-        - push
-
-  - name: publish
-    image: plugins/gh-pages
+  - name: build_push_binary
+    environment:
+      DOCKER_BUILDKIT: "1"
+    image: plugins/docker
     settings:
+      repo: rancher/k3d
+      auto_tag: true
+      auto_tag_suffix: linux-amd64
+      dockerfile: Dockerfile
+      target: binary-only
+      context: .
+      username:
+        from_secret: docker_username
       password:
-        from_secret: github_token
-      username: rancherio-gh-m
-      pages_directory: site/
-      target_branch: gh-pages
-    when:
-      branch:
-        - main
-      event:
-        - push
+        from_secret: docker_password
+      build_args:
+        - GIT_TAG_OVERRIDE=${DRONE_TAG}
 
-trigger:
-  event:
-    - push
-  branch:
-    - main
+  - name: build_push_dind
+    image: plugins/docker
+    environment:
+      DOCKER_BUILDKIT: "1"
+    settings:
+      repo: rancher/k3d
+      auto_tag: true
+      auto_tag_suffix: dind-linux-amd64
+      dockerfile: Dockerfile
+      target: dind
+      context: .
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - GIT_TAG_OVERRIDE=${DRONE_TAG}
+        - ARCH=amd64
 
----
-#####################
-##### k3d-proxy #####
-#####################
-
-kind: pipeline
-type: docker
-name: proxy_linux_amd64
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-  - name: build_push
+  - name: build_push_proxy
     image: plugins/docker
     settings:
       repo: rancher/k3d-proxy
-      tags:
-        - latest-linux-amd64
-        - "${DRONE_TAG}-linux-amd64"
+      auto_tag: true
+      auto_tag_suffix: linux-amd64
       dockerfile: proxy/Dockerfile
       context: proxy/
       username:
         from_secret: docker_username
       password:
         from_secret: docker_password
-    when:
-      event:
-        - tag
+
+  - name: build_push_tools
+    image: plugins/docker
+    settings:
+      repo: rancher/k3d-tools
+      auto_tag: true
+      auto_tag_suffix: linux-amd64
+      dockerfile: tools/Dockerfile
+      context: tools/
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
 
 trigger:
   event:
-    - tag
+    - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the docker plugin
+    - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`)
 
 depends_on:
   - main
 
 ---
 
+################################
+##### Docker Images: arm #####
+################################
+
 kind: pipeline
 type: docker
-name: proxy_linux_arm
+name: linux_arm
 
 platform:
   os: linux
   arch: arm
 
 steps:
-  - name: build_push
+
+  - name: build_push_proxy
     image: plugins/docker
     settings:
       repo: rancher/k3d-proxy
-      tags:
-        - latest-linux-arm
-        - "${DRONE_TAG}-linux-arm"
+      auto_tag: true
+      auto_tag_suffix: linux-arm
       dockerfile: proxy/Dockerfile
       context: proxy/
       username:
@@ -278,35 +249,87 @@ steps:
         from_secret: docker_password
       build_args:
         - ARCH=arm
-    when:
-      event:
-        - tag
+
+  - name: build_push_tools
+    image: plugins/docker
+    settings:
+      repo: rancher/k3d-tools
+      auto_tag: true
+      auto_tag_suffix: linux-arm
+      dockerfile: tools/Dockerfile
+      context: tools/
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
 
 trigger:
   event:
-    - tag
+    - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the docker plugin
+    - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`)
 
 depends_on:
   - main
 
 ---
 
+################################
+##### Docker Images: arm64 #####
+################################
+
 kind: pipeline
 type: docker
-name: proxy_linux_arm64
+name: linux_arm64
 
 platform:
   os: linux
   arch: arm64
 
 steps:
-  - name: build_push
+
+  - name: build_push_binary
+    environment:
+      DOCKER_BUILDKIT: "1"
+    image: plugins/docker
+    settings:
+      repo: rancher/k3d
+      auto_tag: true
+      auto_tag_suffix: linux-arm64
+      dockerfile: Dockerfile
+      target: binary-only
+      context: .
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - GIT_TAG_OVERRIDE=${DRONE_TAG}
+
+  - name: build_push_dind
+    image: plugins/docker
+    environment:
+      DOCKER_BUILDKIT: "1"
+    settings:
+      repo: rancher/k3d
+      auto_tag: true
+      auto_tag_suffix: dind-linux-arm64
+      dockerfile: Dockerfile
+      target: dind
+      context: .
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      build_args:
+        - GIT_TAG_OVERRIDE=${DRONE_TAG}
+        - ARCH=arm64
+
+  - name: build_push_proxy
     image: plugins/docker
     settings:
       repo: rancher/k3d-proxy
-      tags:
-        - latest-linux-arm64
-        - "${DRONE_TAG}-linux-arm64"
+      auto_tag: true
+      auto_tag_suffix: linux-arm64
       dockerfile: proxy/Dockerfile
       context: proxy/
       username:
@@ -315,195 +338,94 @@ steps:
         from_secret: docker_password
       build_args:
         - ARCH=arm64
-    when:
-      event:
-        - tag
+
+  - name: build_push_tools
+    image: plugins/docker
+    settings:
+      repo: rancher/k3d-tools
+      auto_tag: true
+      auto_tag_suffix: linux-arm64
+      dockerfile: tools/Dockerfile
+      context: tools/
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
 
 trigger:
   event:
-    - tag
+    - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the docker plugin
+    - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`)
 
 depends_on:
   - main
 
 ---
 
+##############################
+###### Docker Manifests ######
+##############################
 kind: pipeline
 type: docker
-name: proxy_manifest
+name: manifests
 
 platform:
   os: linux
   arch: amd64
 
 steps:
-  - name: push_manifest
+  - name: push_manifest_binary
     image: plugins/manifest
     settings:
       username:
         from_secret: docker_username
       password:
         from_secret: docker_password
-      target: "rancher/k3d-proxy:${DRONE_TAG}"
-      template: "rancher/k3d-proxy:${DRONE_TAG}-OS-ARCH"
-      platforms:
-        - linux/amd64
-        - linux/arm
-        - linux/arm64
-    when:
-      event:
-        - tag
+      spec: manifest.tmpl
+      auto_tag: true
+      ignore_missing: true # expected, as we dropped arm due to missing base image for that arch
 
-trigger:
-  event:
-    - tag
-
-depends_on:
-  - main
-  - proxy_linux_amd64
-  - proxy_linux_arm
-  - proxy_linux_arm64
-
----
-#####################
-##### k3d-tools #####
-#####################
-
-kind: pipeline
-type: docker
-name: tools_linux_amd64
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-  - name: build_push
-    image: plugins/docker
-    settings:
-      repo: rancher/k3d-tools
-      tags:
-        - latest-linux-amd64
-        - "${DRONE_TAG}-linux-amd64"
-      dockerfile: tools/Dockerfile
-      context: tools/
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-    when:
-      event:
-        - tag
-
-trigger:
-  event:
-    - tag
-
-depends_on:
-  - main
-
----
-
-kind: pipeline
-type: docker
-name: tools_linux_arm
-
-platform:
-  os: linux
-  arch: arm
-
-steps:
-  - name: build_push
-    image: plugins/docker
-    settings:
-      repo: rancher/k3d-tools
-      tags:
-        - latest-linux-arm
-        - "${DRONE_TAG}-linux-arm"
-      dockerfile: tools/Dockerfile
-      context: tools/
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-    when:
-      event:
-        - tag
-
-trigger:
-  event:
-    - tag
-
-depends_on:
-  - main
----
-
-kind: pipeline
-type: docker
-name: tools_linux_arm64
-
-platform:
-  os: linux
-  arch: arm64
-
-steps:
-  - name: build_push
-    image: plugins/docker
-    settings:
-      repo: rancher/k3d-tools
-      tags:
-        - latest-linux-arm64
-        - "${DRONE_TAG}-linux-arm64"
-      dockerfile: tools/Dockerfile
-      context: tools/
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-    when:
-      event:
-        - tag
-
-trigger:
-  event:
-    - tag
-
-depends_on:
-  - main
----
-
-kind: pipeline
-type: docker
-name: tools_manifest
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-  - name: push_manifest
+  - name: push_manifest_dind
     image: plugins/manifest
     settings:
       username:
         from_secret: docker_username
       password:
         from_secret: docker_password
-      target: "rancher/k3d-tools:${DRONE_TAG}"
-      template: "rancher/k3d-tools:${DRONE_TAG}-OS-ARCH"
-      platforms:
-        - linux/amd64
-        - linux/arm
-        - linux/arm64
-    when:
-      event:
-        - tag
+      spec: dind-manifest.tmpl
+      auto_tag: true
+      ignore_missing: true # expected, as we dropped arm due to missing base image for that arch
+
+  - name: push_manifest_proxy
+    image: plugins/manifest
+    settings:
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      spec: proxy/manifest.tmpl
+      auto_tag: true
+      ignore_missing: false
+
+  - name: push_manifest_tools
+    image: plugins/manifest
+    settings:
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      spec: tools/manifest.tmpl
+      auto_tag: true
+      ignore_missing: false
 
 trigger:
   event:
-    - tag
+    - tag # see note at the start of the "Docker Images" section: creates SemVer tagged images using the `auto_tag` option of the manifest plugin
+    - push # `auto_tag` option only creates the `latest` tag if target branch is default branch (i.e. `main`)
 
 depends_on:
   - main
-  - tools_linux_amd64
-  - tools_linux_arm
-  - tools_linux_arm64
+  - linux_amd64
+  - linux_arm
+  - linux_arm64
+

.github/workflows/docs.yml

@@ -0,0 +1,46 @@
+name: k3d.io
+
+on:
+  push:
+    branches:
+      - main
+    tags:
+      # only run on tags for real releases and special docs releases
+      - 'v[0-9]+.[0-9]+.[0-9]+'
+      - 'v[0-9]+.[0-9]+.[0-9]+-docs.[0-9]+'
+    # tags-ignore:
+    #   - "*rc*"
+    #   - "*beta*"
+    #   - "*alpha*"
+    #   - "*test*"
+    #   - "*dev*"
+
+jobs:
+  build:
+    runs-on: ubuntu-20.04
+    container:
+      image: python:3.9
+    steps:
+      - name: Checkout Project
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Install Requirements
+        run: pip install -r docs/requirements.txt
+      - name: Build with MkDocs (validation)
+        run: |
+          mkdocs build --verbose --clean --strict
+          rm -r site/
+      - name: Configure Git
+        if: startsWith(github.ref, 'refs/tags/')
+        id: git
+        run: |
+          git config --global user.name ghaction-k3d.io
+          git config --global user.email ghaction@k3d.io
+          echo ::set-output name=tag::${GITHUB_REF#refs/tags/}
+      - name: Build & Deploy with Mike (versioned)
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          mike deploy --update-aliases --push --rebase ${{ steps.git.outputs.tag }} stable
+      
+

CHANGELOG.md

@@ -1,7 +1,61 @@
 # Changelog
 
+## v5.0.3
+
+### Enhancements & Fixes
+
+- simplified way of getting a Docker API Client that works with Docker Contexts and `DOCKER_*` environment variable configuration (#829, @dragonflylee)
+  - fix: didn't honor `DOCKER_TLS` environment variables before
+
+## v5.0.2
+
+### Enhancements
+
+- CoreDNS Configmap is now edited in the auto-deploy manifest on disk instead of relying on `kubectl patch` command (#814)
+- refactor: add cmd subcommands in a single function call (#819, @moeryomenko)
+- handle ready-log-messages by type and intent & check them in single log streams instead of checking whole chunks every time (#818)
+
+### Fixes
+
+- fix: config file check failing with env var expansion because unexpanded input file was checked
+
+### Misc
+
+- cleanup: ensure that connections/streams are closed once unused (#818)
+- cleanup: split type definitions across multiple files to increase readability (#818)
+- docs: clarify `node create` help text about cluster reference (#808, @losinggeneration)
+- refactor: move from io/ioutil (deprecated) to io and os packages (#827, @Juneezee)
+
+## v5.0.1
+
+### Enhancement
+
+- add `HostFromClusterNetwork` field to `LocalRegistryHosting` configmap as per KEP-1755 (#754)
+
+### Fixes
+
+- fix: nilpointer exception on failed exec process with no returned logreader
+- make post-create cluster preparation (DNS stuff mostly) more resilient (#780)
+- fix v1alpha2 -> v1alpha3 config migration (and other related issues) (#799)
+
+### Misc
+
+- docs: fix typo (#784)
+- docs: fix usage of legacy `--k3s-agent/server-arg` flag
+
 ## v5.0.0
 
+This release contains a whole lot of new features, breaking changes as well as smaller fixes and improvements.
+The changelog shown here is likely not complete but gives a broad overview over the changes.
+For more details, please check the v5 milestone (<https://github.com/rancher/k3d/milestone/27>) or even the commit history.
+The docs have been updated, so you should also find the information you need there, with more to come!
+
+The demo repository has also been updated to work with k3d v5: <https://github.com/iwilltry42/k3d-demo>.
+
+**Info**: <https://k3d.io> is now versioned, so you can checkout different versions of the documentation by using the dropdown menu in the page title bar!
+
+**Feedback welcome!**
+
 ### Breaking Changes
 
 - new syntax for nodefilters
@@ -15,6 +69,7 @@
   - the `--port` flag has the `proxy` opt (see new nodefilter syntax above) set by default
   - to leverage the old behavior of direct port-mappings, use the `direct` opt on the port flag
   - the nodefilter `loadbalancer` will now do the same as `servers:*;agents:*` (proxied via the loadbalancer)
+- flag `--registries-create` transformed from bool flag to string flag: let's you define the name and port-binding of the newly created registry, e.g. `--registry-create myregistry.localhost:5001`
 
 ### Fixes
 
@@ -31,6 +86,7 @@
   - updated fork of `confd` to make usage of the file backend including a file watcher for auto-reloads
     - this also checks the config before applying it, so the lb doesn't crash on a faulty config
   - updating the loadbalancer writes the new config file and also checks if everything's going fine afterwards
+  - some settings of the loadbalancer can now be configured using `--lb-config-override`, see docs at <https://k3d.io/v5.0.0/design/defaults/#k3d-loadbalancer>
 - helper images can now be set explicitly via environment variables: `K3D_IMAGE_LOADBALANCER` & `K3D_IMAGE_TOOLS` (#638)
 - concurrently add new nodes to an existing cluster (remove some dumb code) (#640)
   - `--wait` is now the default for `k3d node create`
@@ -51,6 +107,8 @@
 - changed flags `--k3s-server-arg` & `--k3s-agent-arg` into `--k3s-arg` with nodefilter support (#605)
   - new config path `options.k3s.extraArgs`
 - config file: environment variables (`$VAR`, `${VAR}` will be expanded unconditionally) (#643)
+- docker context support (#601, @developer-guy & #674)
+- Feature flag using the environment variable `K3D_FIX_DNS` and setting it to a true value (e.g. `export K3D_FIX_DNS=1`) to forward DNS queries to your local machine, e.g. to use your local company DNS
 
 ### Misc
 
@@ -58,6 +116,28 @@
 - logs: really final output when creating/deleting nodes (so far, we were not outputting a final success message and the process was still doing stuff) (#640)
 - tests/e2e: add tests for v1alpha2 to v1alpha3 migration
 - docs: use v1alpha3 config version
+- docs: update general appearance and cleanup
+
+## v4.4.8
+
+## Enhancements
+
+- Improved DroneCI Pipeline for Multiarch Images and SemVer Tags (#712)
+  - **Important**: New images will not have the `v` prefix in the tag anymore!
+    - but now real releases will use the "hierarchical" SemVer tags, so you could e.g. subscribe to rancher/k3d-proxy:4 to get v4.x.x images for the proxy container
+
+## Fixes
+
+- clusterCreate: do not override hostIP if hostPort is missing (#693, @lukaszo)
+- imageImport: import all listed images, not only the first one (#701, @mszostok)
+- clusterCreate: when memory constraints are set, only pull the image used for checking the edac folder, if it's not present on the machine
+- fix: update k3d-tools dependencies and use API Version Negotiation, so it still works with older versions of the Docker Engine (#679)
+
+### Misc
+
+- install script: add darwin/arm64 support (#676, @colelawrence)
+- docs: fix go install command (#677, @Rots)
+- docs: add project overview (<https://k3d.io/internals/project/>) (#680)
 
 ## v4.4.7
 

CONTRIBUTING.md

@@ -7,6 +7,8 @@ We welcome everyone who likes to use and improve our software.
 
 Before starting to work with and on k3d, please read and understand our [**Code of Conduct**](./CODE_OF_CONDUCT.md).
 
+Get an Overview of the k3d project in the documentation: [k3d.io/internals/project](https://k3d.io/internals/project)
+
 Before opening an issue or a Pull-Request, please use GitHub's search function to check whether something similar is already in process and hook in there instead.
 
 ## Get Recognized

Dockerfile

@@ -3,7 +3,7 @@
 # -> golang image used solely for building the k3d binary  #
 # -> built executable can then be copied into other stages #
 ############################################################
-FROM golang:1.16 as builder
+FROM golang:1.17 as builder
 ARG GIT_TAG_OVERRIDE
 WORKDIR /app
 COPY . .
@@ -15,16 +15,20 @@ RUN make build -e GIT_TAG_OVERRIDE=${GIT_TAG_OVERRIDE} && bin/k3d version
 # -> used e.g. in our CI pipelines for testing        #
 #######################################################
 FROM docker:20.10-dind as dind
+ARG OS=linux
+ARG ARCH=amd64
 
 # install some basic packages needed for testing, etc.
-RUN apk update && apk add bash curl sudo jq git make netcat-openbsd
+RUN echo "building for ${OS}/${ARCH}" && \
+    apk update && \
+    apk add bash curl sudo jq git make netcat-openbsd
 
 # install kubectl to interact with the k3d cluster
-RUN curl -L https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl -o /usr/local/bin/kubectl && \
+RUN curl -L https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/${OS}/${ARCH}/kubectl -o /usr/local/bin/kubectl && \
     chmod +x /usr/local/bin/kubectl
 
 # install yq (yaml processor) from source, as the busybox yq had some issues
-RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_linux_amd64 -o /usr/bin/yq &&\
+RUN curl -L https://github.com/mikefarah/yq/releases/download/v4.9.6/yq_${OS}_${ARCH} -o /usr/bin/yq &&\
     chmod +x /usr/bin/yq
 COPY --from=builder /app/bin/k3d /bin/k3d
 
@@ -34,4 +38,4 @@ COPY --from=builder /app/bin/k3d /bin/k3d
 #########################################
 FROM scratch as binary-only
 COPY --from=builder /app/bin/k3d /bin/k3d
-ENTRYPOINT ["/bin/k3d"]
+ENTRYPOINT ["/bin/k3d"]

Makefile

@@ -26,8 +26,8 @@ ifeq ($(GIT_TAG),)
 GIT_TAG   := $(shell git describe --always)
 endif
 
-# Docker image tag derived from Git tag
-K3D_IMAGE_TAG := $(GIT_TAG)
+# Docker image tag derived from Git tag (with prefix "v" stripped off)
+K3D_IMAGE_TAG := $(GIT_TAG:v%=%)
 
 # get latest k3s version: grep the tag and replace + with - (difference between git and dockerhub tags)
 K3S_TAG		:= $(shell curl --silent "https://update.k3s.io/v1-release/channels/stable" | egrep -o '/v[^ ]+"' | sed -E 's/\/|\"//g' | sed -E 's/\+/\-/')
@@ -65,7 +65,7 @@ PKG       := $(shell go mod vendor)
 TAGS      :=
 TESTS     := ./...
 TESTFLAGS :=
-LDFLAGS   := -w -s -X github.com/rancher/k3d/v4/version.Version=${GIT_TAG} -X github.com/rancher/k3d/v4/version.K3sVersion=${K3S_TAG}
+LDFLAGS   := -w -s -X github.com/rancher/k3d/v5/version.Version=${GIT_TAG} -X github.com/rancher/k3d/v5/version.K3sVersion=${K3S_TAG}
 GCFLAGS   := 
 GOFLAGS   :=
 BINDIR    := $(CURDIR)/bin
@@ -74,7 +74,7 @@ BINARIES  := k3d
 # Set version of the k3d helper images for build
 ifneq ($(K3D_HELPER_VERSION),)
 $(info [INFO] Helper Image version set to ${K3D_HELPER_VERSION})
-LDFLAGS += -X github.com/rancher/k3d/v4/version.HelperVersionOverride=${K3D_HELPER_VERSION}
+LDFLAGS += -X github.com/rancher/k3d/v5/version.HelperVersionOverride=${K3D_HELPER_VERSION}
 endif
 
 # Rules for finding all go source files using 'DIRS' and 'REC_DIRS'
@@ -129,10 +129,10 @@ build-docker-%:
 
 # build helper images
 build-helper-images:
-	@echo "Building docker image rancher/k3d-proxy:$(GIT_TAG)"
-	DOCKER_BUILDKIT=1 docker build proxy/ -f proxy/Dockerfile -t rancher/k3d-proxy:$(GIT_TAG)
-	@echo "Building docker image rancher/k3d-tools:$(GIT_TAG)"
-	DOCKER_BUILDKIT=1 docker build --no-cache tools/ -f tools/Dockerfile -t rancher/k3d-tools:$(GIT_TAG) --build-arg GIT_TAG=$(GIT_TAG)
+	@echo "Building docker image rancher/k3d-proxy:$(K3D_IMAGE_TAG)"
+	DOCKER_BUILDKIT=1 docker build proxy/ -f proxy/Dockerfile -t rancher/k3d-proxy:$(K3D_IMAGE_TAG)
+	@echo "Building docker image rancher/k3d-tools:$(K3D_IMAGE_TAG)"
+	DOCKER_BUILDKIT=1 docker build --no-cache tools/ -f tools/Dockerfile -t rancher/k3d-tools:$(K3D_IMAGE_TAG) --build-arg GIT_TAG=$(GIT_TAG)
 
 ##############################
 ########## Cleaning ##########

README.md

@@ -4,16 +4,16 @@
 [![License](https://img.shields.io/github/license/rancher/k3d?style=flat-square)](./LICENSE.md)
 ![Downloads](https://img.shields.io/github/downloads/rancher/k3d/total.svg?style=flat-square)
 
-[![Go Module](https://img.shields.io/badge/Go%20Module-github.com%2Francher%2Fk3d%2Fv4-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/rancher/k3d/v4)
+[![Go Module](https://img.shields.io/badge/Go%20Module-github.com%2Francher%2Fk3d%2Fv5-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/rancher/k3d/v5)
 [![Go version](https://img.shields.io/github/go-mod/go-version/rancher/k3d?logo=go&logoColor=white&style=flat-square)](./go.mod)
 [![Go Report Card](https://goreportcard.com/badge/github.com/rancher/k3d?style=flat-square)](https://goreportcard.com/report/github.com/rancher/k3d)
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-12-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md)
 
-**Please Note:** `main` is now v4.0.0 and the code for v3.x can be found in the `main-v3` branch!
+**Please Note:** `main` is now v5.0.0 and the code for v4.x can be found in the `main-v4` branch!
 
 ## [k3s in docker](https://k3d.io)
 
@@ -21,7 +21,7 @@ k3s is the lightweight Kubernetes distribution by Rancher: [rancher/k3s](https:/
 
 k3d creates containerized k3s clusters. This means, that you can spin up a multi-node k3s cluster on a single machine using docker.
 
-[![asciicast](https://asciinema.org/a/347570.svg)](https://asciinema.org/a/347570)
+[![asciicast](https://asciinema.org/a/436420.svg)](https://asciinema.org/a/436420)
 
 ## Learning
 
@@ -35,8 +35,9 @@ k3d creates containerized k3s clusters. This means, that you can spin up a multi
 
 ## Releases
 
-**Note**: In May 2020 we upgraded from v1.7.x to **v3.0.0** after a complete rewrite of k3d!
-**Note**: In January 2021 we upgraded from v3.x.x to **v4.0.0** which includes some breaking changes!
+**Note**: In May 2020 we upgraded from v1.7.x to **v3.0.0** after a complete rewrite of k3d!  
+**Note**: In January 2021 we upgraded from v3.x.x to **v4.0.0** which includes some breaking changes!  
+**Note**: In September 2021 we upgraded from v4.4.8 to **v5.0.0** which includes some breaking changes!  
 
 | Platform | Stage | Version | Release Date |  |
 |-----------------|--------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------|---|
@@ -53,15 +54,15 @@ You have several options there:
   - wget: `wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash`
   - curl: `curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash`
 - use the install script to grab a specific release (via `TAG` environment variable):
-  - wget: `wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v4.0.0 bash`
-  - curl: `curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v4.0.0 bash`
+  - wget: `wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v5.0.0 bash`
+  - curl: `curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v5.0.0 bash`
 
 - use [Homebrew](https://brew.sh): `brew install k3d` (Homebrew is available for MacOS and Linux)
   - Formula can be found in [homebrew/homebrew-core](https://github.com/Homebrew/homebrew-core/blob/master/Formula/k3d.rb) and is mirrored to [homebrew/linuxbrew-core](https://github.com/Homebrew/linuxbrew-core/blob/master/Formula/k3d.rb)
 - install via [MacPorts](https://www.macports.org): `sudo port selfupdate && sudo port install k3d` (MacPorts is available for MacOS)
 - install via [AUR](https://aur.archlinux.org/) package [rancher-k3d-bin](https://aur.archlinux.org/packages/rancher-k3d-bin/): `yay -S rancher-k3d-bin`
 - grab a release from the [release tab](https://github.com/rancher/k3d/releases) and install it yourself.
-- install via go: `go install github.com/rancher/k3d` (**Note**: this will give you unreleased/bleeding-edge changes)
+- install via go: `go install github.com/rancher/k3d@latest` (**Note**: this will give you unreleased/bleeding-edge changes)
 - use [Chocolatey](https://chocolatey.org/): `choco install k3d` (Chocolatey package manager is available for Windows)
   - package source can be found in [erwinkersten/chocolatey-packages](https://github.com/erwinkersten/chocolatey-packages/tree/master/automatic/k3d)
 
@@ -69,7 +70,7 @@ or...
 
 ## Build
 
-1. Clone this repo, e.g. via `git clone git@github.com:rancher/k3d.git` or `go get github.com/rancher/k3d/v4@main`
+1. Clone this repo, e.g. via `git clone git@github.com:rancher/k3d.git` or `go get github.com/rancher/k3d/v5@main`
 2. Inside the repo run
    - 'make install-tools' to make sure required go packages are installed
 3. Inside the repo run one of the following commands
@@ -112,6 +113,8 @@ k3d is a community-driven project and so we welcome contributions of any form, b
 
 Please read our [**Contributing Guidelines**](./CONTRIBUTING.md) and the related [**Code of Conduct**](./CODE_OF_CONDUCT.md).
 
+You can find an overview of the k3d project (e.g. explanations and a repository guide) in the documentation: [k3d.io/internals/project](https://k3d.io/internals/project)
+
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md)
 
 ## Contributors ✨
@@ -137,6 +140,8 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <td align="center"><a href="http://wsl.dev"><img src="https://avatars2.githubusercontent.com/u/905874?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Nuno do Carmo</b></sub></a><br /><a href="#content-nunix" title="Content">🖋</a> <a href="#tutorial-nunix" title="Tutorials">✅</a> <a href="#question-nunix" title="Answering Questions">💬</a></td>
     <td align="center"><a href="https://github.com/erwinkersten"><img src="https://avatars0.githubusercontent.com/u/4391121?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Erwin Kersten</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=erwinkersten" title="Documentation">📖</a></td>
     <td align="center"><a href="http://www.alexsears.com"><img src="https://avatars.githubusercontent.com/u/3712883?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Alex Sears</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=searsaw" title="Documentation">📖</a></td>
+    <td align="center"><a href="http://shanduur.github.io"><img src="https://avatars.githubusercontent.com/u/32583062?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mateusz Urbanek</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=Shanduur" title="Code">💻</a></td>
+    <td align="center"><a href="https://github.com/benjaminjb"><img src="https://avatars.githubusercontent.com/u/4651855?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Benjamin Blattberg</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=benjaminjb" title="Code">💻</a></td>
   </tr>
 </table>
 

cmd/cluster/cluster.go

@@ -22,7 +22,8 @@ THE SOFTWARE.
 package cluster
 
 import (
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+
 	"github.com/spf13/cobra"
 )
 
@@ -36,19 +37,19 @@ func NewCmdCluster() *cobra.Command {
 		Long:  `Manage cluster(s)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
 
 	// add subcommands
-	cmd.AddCommand(NewCmdClusterCreate())
-	cmd.AddCommand(NewCmdClusterStart())
-	cmd.AddCommand(NewCmdClusterStop())
-	cmd.AddCommand(NewCmdClusterDelete())
-	cmd.AddCommand(NewCmdClusterList())
-	cmd.AddCommand(NewCmdClusterEdit())
+	cmd.AddCommand(NewCmdClusterCreate(),
+		NewCmdClusterStart(),
+		NewCmdClusterStop(),
+		NewCmdClusterDelete(),
+		NewCmdClusterList(),
+		NewCmdClusterEdit())
 
 	// add flags
 

cmd/cluster/clusterCreate.go

@@ -24,28 +24,28 @@ package cluster
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/docker/go-connections/nat"
+	"github.com/sirupsen/logrus"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
 
-	cliutil "github.com/rancher/k3d/v4/cmd/util"
-	k3dCluster "github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/config"
-	conf "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	"github.com/rancher/k3d/v4/version"
-
-	log "github.com/sirupsen/logrus"
+	cliutil "github.com/rancher/k3d/v5/cmd/util"
+	cliconfig "github.com/rancher/k3d/v5/cmd/util/config"
+	k3dCluster "github.com/rancher/k3d/v5/pkg/client"
+	"github.com/rancher/k3d/v5/pkg/config"
+	conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+	"github.com/rancher/k3d/v5/version"
 )
 
 var configFile string
@@ -58,74 +58,30 @@ Every cluster will consist of one or more containers:
 	- (optionally) 1 (or more) agent node containers (k3s)
 `
 
-var cfgViper = viper.New()
-var ppViper = viper.New()
+/*
+ * Viper for configuration handling
+ * we use two different instances of Viper here to handle
+ * - cfgViper: "static" configuration
+ * - ppViper: "pre-processed" configuration, where CLI input has to be pre-processed
+ *             to be treated as part of the SImpleConfig
+ */
+var (
+	cfgViper = viper.New()
+	ppViper  = viper.New()
+)
 
-func initConfig() {
+func initConfig() error {
 
 	// Viper for pre-processed config options
 	ppViper.SetEnvPrefix("K3D")
 
-	// viper for the general config (file, env and non pre-processed flags)
-	cfgViper.SetEnvPrefix("K3D")
-	cfgViper.AutomaticEnv()
+	if l.Log().GetLevel() >= logrus.DebugLevel {
 
-	cfgViper.SetConfigType("yaml")
-
-	// Set config file, if specified
-	if configFile != "" {
-
-		if _, err := os.Stat(configFile); err != nil {
-			log.Fatalf("Failed to stat config file %s: %+v", configFile, err)
-		}
-
-		// create temporary file to expand environment variables in the config without writing that back to the original file
-		// we're doing it here, because this happens just before absolutely all other processing
-		tmpfile, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("k3d-config-tmp-%s", filepath.Base(configFile)))
-		if err != nil {
-			log.Fatalf("error creating temp copy of configfile %s for variable expansion: %v", configFile, err)
-		}
-		defer tmpfile.Close()
-
-		originalcontent, err := ioutil.ReadFile(configFile)
-		if err != nil {
-			log.Fatalf("error reading config file %s: %v", configFile, err)
-		}
-		expandedcontent := os.ExpandEnv(string(originalcontent))
-		if _, err := tmpfile.WriteString(expandedcontent); err != nil {
-			log.Fatalf("error writing expanded config file contents to temp file %s: %v", tmpfile.Name(), err)
-		}
-
-		// use temp file with expanded variables
-		cfgViper.SetConfigFile(tmpfile.Name())
-
-		// try to read config into memory (viper map structure)
-		if err := cfgViper.ReadInConfig(); err != nil {
-			if _, ok := err.(viper.ConfigFileNotFoundError); ok {
-				log.Fatalf("Config file %s not found: %+v", configFile, err)
-			}
-			// config file found but some other error happened
-			log.Fatalf("Failed to read config file %s: %+v", configFile, err)
-		}
-
-		schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
-		if err != nil {
-			log.Fatalf("Cannot validate config file %s: %+v", configFile, err)
-		}
-
-		if err := config.ValidateSchemaFile(configFile, schema); err != nil {
-			log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
-		}
-
-		log.Infof("Using config file %s (%s#%s)", configFile, strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))
+		c, _ := yaml.Marshal(ppViper.AllSettings())
+		l.Log().Debugf("Additional CLI Configuration:\n%s", c)
 	}
-	if log.GetLevel() >= log.DebugLevel {
-		c, _ := yaml.Marshal(cfgViper.AllSettings())
-		log.Debugf("Configuration:\n%s", c)
 
-		c, _ = yaml.Marshal(ppViper.AllSettings())
-		log.Debugf("Additional CLI Configuration:\n%s", c)
-	}
+	return cliconfig.InitViperWithConfigFile(cfgViper, configFile)
 }
 
 // NewCmdClusterCreate returns a new cobra command
@@ -138,8 +94,7 @@ func NewCmdClusterCreate() *cobra.Command {
 		Long:  clusterCreateDescription,
 		Args:  cobra.RangeArgs(0, 1), // exactly one cluster name can be set (default: k3d.DefaultClusterName)
 		PreRunE: func(cmd *cobra.Command, args []string) error {
-			initConfig()
-			return nil
+			return initConfig()
 		},
 		Run: func(cmd *cobra.Command, args []string) {
 
@@ -154,27 +109,27 @@ func NewCmdClusterCreate() *cobra.Command {
 			}
 			cfg, err := config.FromViper(cfgViper)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 
 			if cfg.GetAPIVersion() != config.DefaultConfigApiVersion {
-				log.Warnf("Default config apiVersion is '%s', but you're using '%s': consider migrating.", config.DefaultConfigApiVersion, cfg.GetAPIVersion())
+				l.Log().Warnf("Default config apiVersion is '%s', but you're using '%s': consider migrating.", config.DefaultConfigApiVersion, cfg.GetAPIVersion())
 				cfg, err = config.Migrate(cfg, config.DefaultConfigApiVersion)
 				if err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 			}
 
 			simpleCfg := cfg.(conf.SimpleConfig)
 
-			log.Debugf("========== Simple Config ==========\n%+v\n==========================\n", simpleCfg)
+			l.Log().Debugf("========== Simple Config ==========\n%+v\n==========================\n", simpleCfg)
 
 			simpleCfg, err = applyCLIOverrides(simpleCfg)
 			if err != nil {
-				log.Fatalf("Failed to apply CLI overrides: %+v", err)
+				l.Log().Fatalf("Failed to apply CLI overrides: %+v", err)
 			}
 
-			log.Debugf("========== Merged Simple Config ==========\n%+v\n==========================\n", simpleCfg)
+			l.Log().Debugf("========== Merged Simple Config ==========\n%+v\n==========================\n", simpleCfg)
 
 			/**************************************
 			 * Transform, Process & Validate Configuration *
@@ -187,18 +142,18 @@ func NewCmdClusterCreate() *cobra.Command {
 
 			clusterConfig, err := config.TransformSimpleToClusterConfig(cmd.Context(), runtimes.SelectedRuntime, simpleCfg)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
-			log.Debugf("===== Merged Cluster Config =====\n%+v\n===== ===== =====\n", clusterConfig)
+			l.Log().Debugf("===== Merged Cluster Config =====\n%+v\n===== ===== =====\n", clusterConfig)
 
 			clusterConfig, err = config.ProcessClusterConfig(*clusterConfig)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
-			log.Debugf("===== Processed Cluster Config =====\n%+v\n===== ===== =====\n", clusterConfig)
+			l.Log().Debugf("===== Processed Cluster Config =====\n%+v\n===== ===== =====\n", clusterConfig)
 
 			if err := config.ValidateClusterConfig(cmd.Context(), runtimes.SelectedRuntime, *clusterConfig); err != nil {
-				log.Fatalln("Failed Cluster Configuration Validation: ", err)
+				l.Log().Fatalln("Failed Cluster Configuration Validation: ", err)
 			}
 
 			/**************************************
@@ -207,44 +162,44 @@ func NewCmdClusterCreate() *cobra.Command {
 
 			// check if a cluster with that name exists already
 			if _, err := k3dCluster.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster); err == nil {
-				log.Fatalf("Failed to create cluster '%s' because a cluster with that name already exists", clusterConfig.Cluster.Name)
+				l.Log().Fatalf("Failed to create cluster '%s' because a cluster with that name already exists", clusterConfig.Cluster.Name)
 			}
 
 			// create cluster
 			if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig {
-				log.Debugln("'--kubeconfig-update-default set: enabling wait-for-server")
+				l.Log().Debugln("'--kubeconfig-update-default set: enabling wait-for-server")
 				clusterConfig.ClusterCreateOpts.WaitForServer = true
 			}
 			//if err := k3dCluster.ClusterCreate(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, &clusterConfig.ClusterCreateOpts); err != nil {
 			if err := k3dCluster.ClusterRun(cmd.Context(), runtimes.SelectedRuntime, clusterConfig); err != nil {
 				// rollback if creation failed
-				log.Errorln(err)
+				l.Log().Errorln(err)
 				if simpleCfg.Options.K3dOptions.NoRollback { // TODO: move rollback mechanics to pkg/
-					log.Fatalln("Cluster creation FAILED, rollback deactivated.")
+					l.Log().Fatalln("Cluster creation FAILED, rollback deactivated.")
 				}
 				// rollback if creation failed
-				log.Errorln("Failed to create cluster >>> Rolling Back")
+				l.Log().Errorln("Failed to create cluster >>> Rolling Back")
 				if err := k3dCluster.ClusterDelete(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, k3d.ClusterDeleteOpts{SkipRegistryCheck: true}); err != nil {
-					log.Errorln(err)
-					log.Fatalln("Cluster creation FAILED, also FAILED to rollback changes!")
+					l.Log().Errorln(err)
+					l.Log().Fatalln("Cluster creation FAILED, also FAILED to rollback changes!")
 				}
-				log.Fatalln("Cluster creation FAILED, all changes have been rolled back!")
+				l.Log().Fatalln("Cluster creation FAILED, all changes have been rolled back!")
 			}
-			log.Infof("Cluster '%s' created successfully!", clusterConfig.Cluster.Name)
+			l.Log().Infof("Cluster '%s' created successfully!", clusterConfig.Cluster.Name)
 
 			/**************
 			 * Kubeconfig *
 			 **************/
 
-			if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && clusterConfig.KubeconfigOpts.SwitchCurrentContext {
-				log.Infoln("--kubeconfig-update-default=false --> sets --kubeconfig-switch-context=false")
+			if !clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && clusterConfig.KubeconfigOpts.SwitchCurrentContext {
+				l.Log().Infoln("--kubeconfig-update-default=false --> sets --kubeconfig-switch-context=false")
 				clusterConfig.KubeconfigOpts.SwitchCurrentContext = false
 			}
 
 			if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig {
-				log.Debugf("Updating default kubeconfig with a new context for cluster %s", clusterConfig.Cluster.Name)
+				l.Log().Debugf("Updating default kubeconfig with a new context for cluster %s", clusterConfig.Cluster.Name)
 				if _, err := k3dCluster.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, &clusterConfig.Cluster, "", &k3dCluster.WriteKubeConfigOptions{UpdateExisting: true, OverwriteExisting: false, UpdateCurrentContext: simpleCfg.Options.KubeconfigOptions.SwitchCurrentContext}); err != nil {
-					log.Warningln(err)
+					l.Log().Warningln(err)
 				}
 			}
 
@@ -253,7 +208,7 @@ func NewCmdClusterCreate() *cobra.Command {
 			 *****************/
 
 			// print information on how to use the cluster with kubectl
-			log.Infoln("You can now use it like this:")
+			l.Log().Infoln("You can now use it like this:")
 			if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && !clusterConfig.KubeconfigOpts.SwitchCurrentContext {
 				fmt.Printf("kubectl config use-context %s\n", fmt.Sprintf("%s-%s", k3d.DefaultObjectNamePrefix, clusterConfig.Cluster.Name))
 			} else if !clusterConfig.KubeconfigOpts.SwitchCurrentContext {
@@ -272,8 +227,8 @@ func NewCmdClusterCreate() *cobra.Command {
 	 ***************/
 
 	cmd.Flags().StringVarP(&configFile, "config", "c", "", "Path of a config file to use")
-	if err := cobra.MarkFlagFilename(cmd.Flags(), "config", "yaml", "yml"); err != nil {
-		log.Fatalln("Failed to mark flag 'config' as filename flag")
+	if err := cmd.MarkFlagFilename("config", "yaml", "yml"); err != nil {
+		l.Log().Fatalln("Failed to mark flag 'config' as filename flag")
 	}
 
 	/***********************
@@ -298,21 +253,24 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().StringArrayP("env", "e", nil, "Add environment variables to nodes (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -e \"HTTP_PROXY=my.proxy.com@server:0\" -e \"SOME_KEY=SOME_VAL@server:0\"`")
 	_ = ppViper.BindPFlag("cli.env", cmd.Flags().Lookup("env"))
 
-	cmd.Flags().StringArrayP("volume", "v", nil, "Mount volumes into the nodes (Format: `[SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -v /my/path@agent[0,1] -v /tmp/test:/tmp/other@server:0`")
+	cmd.Flags().StringArrayP("volume", "v", nil, "Mount volumes into the nodes (Format: `[SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 -v /my/path@agent:0,1 -v /tmp/test:/tmp/other@server:0`")
 	_ = ppViper.BindPFlag("cli.volumes", cmd.Flags().Lookup("volume"))
 
-	cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers (via the serverlb) to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent:0 -p 8081@agent[1]`")
+	cmd.Flags().StringArrayP("port", "p", nil, "Map ports from the node containers (via the serverlb) to the host (Format: `[HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]`)\n - Example: `k3d cluster create --agents 2 -p 8080:80@agent:0 -p 8081@agent:1`")
 	_ = ppViper.BindPFlag("cli.ports", cmd.Flags().Lookup("port"))
 
-	cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent[0,1]\" --k3s-node-label \"other.label=somevalue@server:0\"`")
+	cmd.Flags().StringArrayP("k3s-node-label", "", nil, "Add label to k3s node (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --k3s-node-label \"my.label@agent:0,1\" --k3s-node-label \"other.label=somevalue@server:0\"`")
 	_ = ppViper.BindPFlag("cli.k3s-node-labels", cmd.Flags().Lookup("k3s-node-label"))
 
-	cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent[0,1]\" --runtime-label \"other.label=somevalue@server:0\"`")
+	cmd.Flags().StringArrayP("runtime-label", "", nil, "Add label to container runtime (Format: `KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]`\n - Example: `k3d cluster create --agents 2 --runtime-label \"my.label@agent:0,1\" --runtime-label \"other.label=somevalue@server:0\"`")
 	_ = ppViper.BindPFlag("cli.runtime-labels", cmd.Flags().Lookup("runtime-label"))
 
+	cmd.Flags().String("registry-create", "", "Create a k3d-managed registry and connect it to the cluster (Format: `NAME[:HOST][:HOSTPORT]`\n - Example: `k3d cluster create --registry-create mycluster-registry:0.0.0.0:5432`")
+	_ = ppViper.BindPFlag("cli.registries.create", cmd.Flags().Lookup("registry-create"))
+
 	/* k3s */
 	cmd.Flags().StringArray("k3s-arg", nil, "Additional args passed to k3s command (Format: `ARG@NODEFILTER[;@NODEFILTER]`)\n - Example: `k3d cluster create --k3s-arg \"--disable=traefik@server:0\"")
-	_ = cfgViper.BindPFlag("cli.k3sargs", cmd.Flags().Lookup("k3s-arg"))
+	_ = ppViper.BindPFlag("cli.k3sargs", cmd.Flags().Lookup("k3s-arg"))
 
 	/******************
 	 * "Normal" Flags *
@@ -362,9 +320,6 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().Bool("no-rollback", false, "Disable the automatic rollback actions, if anything goes wrong")
 	_ = cfgViper.BindPFlag("options.k3d.disablerollback", cmd.Flags().Lookup("no-rollback"))
 
-	cmd.Flags().Bool("no-hostip", false, "Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS")
-	_ = cfgViper.BindPFlag("options.k3d.disablehostipinjection", cmd.Flags().Lookup("no-hostip"))
-
 	cmd.Flags().String("gpus", "", "GPU devices to add to the cluster node containers ('all' to pass all GPUs) [From docker]")
 	_ = cfgViper.BindPFlag("options.runtime.gpurequest", cmd.Flags().Lookup("gpus"))
 
@@ -382,11 +337,15 @@ func NewCmdClusterCreate() *cobra.Command {
 	cmd.Flags().StringArray("registry-use", nil, "Connect to one or more k3d-managed registries running locally")
 	_ = cfgViper.BindPFlag("registries.use", cmd.Flags().Lookup("registry-use"))
 
-	cmd.Flags().Bool("registry-create", false, "Create a k3d-managed registry and connect it to the cluster")
-	_ = cfgViper.BindPFlag("registries.create", cmd.Flags().Lookup("registry-create"))
-
 	cmd.Flags().String("registry-config", "", "Specify path to an extra registries.yaml file")
 	_ = cfgViper.BindPFlag("registries.config", cmd.Flags().Lookup("registry-config"))
+	if err := cmd.MarkFlagFilename("registry-config", "yaml", "yml"); err != nil {
+		l.Log().Fatalln("Failed to mark flag 'config' as filename flag")
+	}
+
+	/* Loadbalancer / Proxy */
+	cmd.Flags().StringSlice("lb-config-override", nil, "Use dotted YAML path syntax to override nginx loadbalancer settings")
+	_ = cfgViper.BindPFlag("options.k3d.loadbalancer.configoverrides", cmd.Flags().Lookup("lb-config-override"))
 
 	/* Subcommands */
 
@@ -421,20 +380,25 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 	// Overwrite if cli arg is set
 	if ppViper.IsSet("cli.api-port") {
 		if cfg.ExposeAPI.HostPort != "" {
-			log.Debugf("Overriding pre-defined kubeAPI Exposure Spec %+v with CLI argument %s", cfg.ExposeAPI, ppViper.GetString("cli.api-port"))
+			l.Log().Debugf("Overriding pre-defined kubeAPI Exposure Spec %+v with CLI argument %s", cfg.ExposeAPI, ppViper.GetString("cli.api-port"))
 		}
 		exposeAPI, err = cliutil.ParsePortExposureSpec(ppViper.GetString("cli.api-port"), k3d.DefaultAPIPort)
 		if err != nil {
-			return cfg, err
+			return cfg, fmt.Errorf("failed to parse API Port spec: %w", err)
 		}
 	}
 
 	// Set to random port if port is empty string
 	if len(exposeAPI.Binding.HostPort) == 0 {
-		exposeAPI, err = cliutil.ParsePortExposureSpec("random", k3d.DefaultAPIPort)
-		if err != nil {
-			return cfg, err
+		var freePort string
+		port, err := cliutil.GetFreePort()
+		freePort = strconv.Itoa(port)
+		if err != nil || port == 0 {
+			l.Log().Warnf("Failed to get random free port: %+v", err)
+			l.Log().Warnf("Falling back to internal port %s (may be blocked though)...", k3d.DefaultAPIPort)
+			freePort = k3d.DefaultAPIPort
 		}
+		exposeAPI.Binding.HostPort = freePort
 	}
 
 	cfg.ExposeAPI = conf.SimpleExposureOpts{
@@ -451,11 +415,11 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		// split node filter from the specified volume
 		volume, filters, err := cliutil.SplitFiltersFromFlag(volumeFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
-		if strings.Contains(volume, k3d.DefaultRegistriesFilePath) && (cfg.Registries.Create || cfg.Registries.Config != "" || len(cfg.Registries.Use) != 0) {
-			log.Warnf("Seems like you're mounting a file at '%s' while also using a referenced registries config or k3d-managed registries: Your mounted file will probably be overwritten!", k3d.DefaultRegistriesFilePath)
+		if strings.Contains(volume, k3d.DefaultRegistriesFilePath) && (cfg.Registries.Create != nil || cfg.Registries.Config != "" || len(cfg.Registries.Use) != 0) {
+			l.Log().Warnf("Seems like you're mounting a file at '%s' while also using a referenced registries config or k3d-managed registries: Your mounted file will probably be overwritten!", k3d.DefaultRegistriesFilePath)
 		}
 
 		// create new entry or append filter to existing entry
@@ -473,7 +437,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		})
 	}
 
-	log.Tracef("VolumeFilterMap: %+v", volumeFilterMap)
+	l.Log().Tracef("VolumeFilterMap: %+v", volumeFilterMap)
 
 	// -> PORTS
 	portFilterMap := make(map[string][]string, 1)
@@ -481,12 +445,12 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		// split node filter from the specified volume
 		portmap, filters, err := cliutil.SplitFiltersFromFlag(portFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
 		// create new entry or append filter to existing entry
 		if _, exists := portFilterMap[portmap]; exists {
-			log.Fatalln("Same Portmapping can not be used for multiple nodes")
+			l.Log().Fatalln("Same Portmapping can not be used for multiple nodes")
 		} else {
 			portFilterMap[portmap] = filters
 		}
@@ -499,7 +463,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		})
 	}
 
-	log.Tracef("PortFilterMap: %+v", portFilterMap)
+	l.Log().Tracef("PortFilterMap: %+v", portFilterMap)
 
 	// --k3s-node-label
 	// k3sNodeLabelFilterMap will add k3s node label to applied node filters
@@ -509,7 +473,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		// split node filter from the specified label
 		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
 		// create new entry or append filter to existing entry
@@ -527,7 +491,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		})
 	}
 
-	log.Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap)
+	l.Log().Tracef("K3sNodeLabelFilterMap: %+v", k3sNodeLabelFilterMap)
 
 	// --runtime-label
 	// runtimeLabelFilterMap will add container runtime label to applied node filters
@@ -537,7 +501,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		// split node filter from the specified label
 		label, nodeFilters, err := cliutil.SplitFiltersFromFlag(labelFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
 		cliutil.ValidateRuntimeLabelKey(strings.Split(label, "=")[0])
@@ -557,7 +521,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		})
 	}
 
-	log.Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap)
+	l.Log().Tracef("RuntimeLabelFilterMap: %+v", runtimeLabelFilterMap)
 
 	// --env
 	// envFilterMap will add container env vars to applied node filters
@@ -567,7 +531,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		// split node filter from the specified env var
 		env, filters, err := cliutil.SplitFiltersFromFlag(envFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
 		// create new entry or append filter to existing entry
@@ -585,7 +549,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		})
 	}
 
-	log.Tracef("EnvFilterMap: %+v", envFilterMap)
+	l.Log().Tracef("EnvFilterMap: %+v", envFilterMap)
 
 	// --k3s-arg
 	argFilterMap := make(map[string][]string, 1)
@@ -594,7 +558,7 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		// split node filter from the specified arg
 		arg, filters, err := cliutil.SplitFiltersFromFlag(argFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
 		// create new entry or append filter to existing entry
@@ -612,5 +576,24 @@ func applyCLIOverrides(cfg conf.SimpleConfig) (conf.SimpleConfig, error) {
 		})
 	}
 
+	// --registry-create
+	if ppViper.IsSet("cli.registries.create") {
+		flagvalue := ppViper.GetString("cli.registries.create")
+		fvSplit := strings.SplitN(flagvalue, ":", 2)
+		if cfg.Registries.Create == nil {
+			cfg.Registries.Create = &conf.SimpleConfigRegistryCreateConfig{}
+		}
+		cfg.Registries.Create.Name = fvSplit[0]
+		if len(fvSplit) > 1 {
+			exposeAPI, err = cliutil.ParsePortExposureSpec(fvSplit[1], "1234") // internal port is unused after all
+			if err != nil {
+				return cfg, fmt.Errorf("failed to registry port spec: %w", err)
+			}
+			cfg.Registries.Create.Host = exposeAPI.Host
+			cfg.Registries.Create.HostPort = exposeAPI.Binding.HostPort
+		}
+
+	}
+
 	return cfg, nil
 }

cmd/cluster/clusterDelete.go

@@ -26,16 +26,21 @@ import (
 	"os"
 	"path"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	k3dutil "github.com/rancher/k3d/v4/pkg/util"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	cliconfig "github.com/rancher/k3d/v5/cmd/util/config"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+	k3dutil "github.com/rancher/k3d/v5/pkg/util"
 
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 )
 
+var clusterDeleteConfigFile string
+var clusterDeleteCfgViper = viper.New()
+
 // NewCmdClusterDelete returns a new cobra command
 func NewCmdClusterDelete() *cobra.Command {
 
@@ -47,35 +52,38 @@ func NewCmdClusterDelete() *cobra.Command {
 		Long:              `Delete cluster(s).`,
 		Args:              cobra.MinimumNArgs(0), // 0 or n arguments; 0 = default cluster name
 		ValidArgsFunction: util.ValidArgsAvailableClusters,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			return cliconfig.InitViperWithConfigFile(clusterDeleteCfgViper, clusterDeleteConfigFile)
+		},
 		Run: func(cmd *cobra.Command, args []string) {
 			clusters := parseDeleteClusterCmd(cmd, args)
 
 			if len(clusters) == 0 {
-				log.Infoln("No clusters found")
+				l.Log().Infoln("No clusters found")
 			} else {
 				for _, c := range clusters {
 					if err := client.ClusterDelete(cmd.Context(), runtimes.SelectedRuntime, c, k3d.ClusterDeleteOpts{SkipRegistryCheck: false}); err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
-					log.Infoln("Removing cluster details from default kubeconfig...")
+					l.Log().Infoln("Removing cluster details from default kubeconfig...")
 					if err := client.KubeconfigRemoveClusterFromDefaultConfig(cmd.Context(), c); err != nil {
-						log.Warnln("Failed to remove cluster details from default kubeconfig")
-						log.Warnln(err)
+						l.Log().Warnln("Failed to remove cluster details from default kubeconfig")
+						l.Log().Warnln(err)
 					}
-					log.Infoln("Removing standalone kubeconfig file (if there is one)...")
+					l.Log().Infoln("Removing standalone kubeconfig file (if there is one)...")
 					configDir, err := k3dutil.GetConfigDirOrCreate()
 					if err != nil {
-						log.Warnf("Failed to delete kubeconfig file: %+v", err)
+						l.Log().Warnf("Failed to delete kubeconfig file: %+v", err)
 					} else {
 						kubeconfigfile := path.Join(configDir, fmt.Sprintf("kubeconfig-%s.yaml", c.Name))
 						if err := os.Remove(kubeconfigfile); err != nil {
 							if !os.IsNotExist(err) {
-								log.Warnf("Failed to delete kubeconfig file '%s'", kubeconfigfile)
+								l.Log().Warnf("Failed to delete kubeconfig file '%s'", kubeconfigfile)
 							}
 						}
 					}
 
-					log.Infof("Successfully deleted cluster %s!", c.Name)
+					l.Log().Infof("Successfully deleted cluster %s!", c.Name)
 				}
 			}
 
@@ -87,6 +95,15 @@ func NewCmdClusterDelete() *cobra.Command {
 	// add flags
 	cmd.Flags().BoolP("all", "a", false, "Delete all existing clusters")
 
+	/***************
+	 * Config File *
+	 ***************/
+
+	cmd.Flags().StringVarP(&clusterDeleteConfigFile, "config", "c", "", "Path of a config file to use")
+	if err := cmd.MarkFlagFilename("config", "yaml", "yml"); err != nil {
+		l.Log().Fatalln("Failed to mark flag 'config' as filename flag")
+	}
+
 	// done
 	return cmd
 }
@@ -94,20 +111,45 @@ func NewCmdClusterDelete() *cobra.Command {
 // parseDeleteClusterCmd parses the command input into variables required to delete clusters
 func parseDeleteClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 
-	// --all
 	var clusters []*k3d.Cluster
 
-	if all, err := cmd.Flags().GetBool("all"); err != nil {
-		log.Fatalln(err)
-	} else if all {
-		log.Infoln("Deleting all clusters...")
+	// --all
+	all, err := cmd.Flags().GetBool("all")
+	if err != nil {
+		l.Log().Fatalln(err)
+	}
+
+	// --config
+	if clusterDeleteConfigFile != "" {
+		// not allowed with --all or more args
+		if len(args) > 0 || all {
+			l.Log().Fatalln("failed to delete cluster: cannot use `--config` flag with additional arguments or `--all`")
+		}
+
+		if clusterDeleteCfgViper.GetString("name") == "" {
+			l.Log().Fatalln("failed to delete cluster via config file: no name in config file")
+		}
+
+		c, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Cluster{Name: clusterDeleteCfgViper.GetString("name")})
+		if err != nil {
+			l.Log().Fatalf("failed to delete cluster '%s': %v", clusterDeleteCfgViper.GetString("name"), err)
+		}
+
+		clusters = append(clusters, c)
+		return clusters
+	}
+
+	// --all was set
+	if all {
+		l.Log().Infoln("Deleting all clusters...")
 		clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		return clusters
 	}
 
+	// args only
 	clusternames := []string{k3d.DefaultClusterName}
 	if len(args) != 0 {
 		clusternames = args
@@ -119,7 +161,7 @@ func parseDeleteClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 			if err == client.ClusterGetNoNodesFoundError {
 				continue
 			}
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		clusters = append(clusters, c)
 	}

cmd/cluster/clusterEdit.go

@@ -22,13 +22,13 @@ THE SOFTWARE.
 package cluster
 
 import (
-	"github.com/rancher/k3d/v4/cmd/util"
-	cliutil "github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	conf "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	cliutil "github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	conf "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 )
 
@@ -37,23 +37,23 @@ func NewCmdClusterEdit() *cobra.Command {
 
 	// create new cobra command
 	cmd := &cobra.Command{
-		Use:               "edit NAME",
+		Use:               "edit CLUSTER",
 		Short:             "[EXPERIMENTAL] Edit cluster(s).",
 		Long:              `[EXPERIMENTAL] Edit cluster(s).`,
 		Args:              cobra.ExactArgs(1),
 		Aliases:           []string{"update"},
-		ValidArgsFunction: util.ValidArgsAvailableNodes,
+		ValidArgsFunction: util.ValidArgsAvailableClusters,
 		Run: func(cmd *cobra.Command, args []string) {
 
 			existingCluster, changeset := parseEditClusterCmd(cmd, args)
 
-			log.Debugf("===== Current =====\n%+v\n===== Changeset =====\n%+v\n", existingCluster, changeset)
+			l.Log().Debugf("===== Current =====\n%+v\n===== Changeset =====\n%+v\n", existingCluster, changeset)
 
 			if err := client.ClusterEditChangesetSimple(cmd.Context(), runtimes.SelectedRuntime, existingCluster, changeset); err != nil {
-				log.Fatalf("Failed to update the cluster: %v", err)
+				l.Log().Fatalf("Failed to update the cluster: %v", err)
 			}
 
-			log.Infof("Successfully updated %s", existingCluster.Name)
+			l.Log().Infof("Successfully updated %s", existingCluster.Name)
 
 		},
 	}
@@ -72,11 +72,11 @@ func parseEditClusterCmd(cmd *cobra.Command, args []string) (*k3d.Cluster, *conf
 
 	existingCluster, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Cluster{Name: args[0]})
 	if err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	}
 
 	if existingCluster == nil {
-		log.Infof("Cluster %s not found", args[0])
+		l.Log().Infof("Cluster %s not found", args[0])
 		return nil, nil
 	}
 
@@ -87,7 +87,7 @@ func parseEditClusterCmd(cmd *cobra.Command, args []string) (*k3d.Cluster, *conf
 	 */
 	portFlags, err := cmd.Flags().GetStringArray("port-add")
 	if err != nil {
-		log.Errorln(err)
+		l.Log().Errorln(err)
 		return nil, nil
 	}
 
@@ -100,12 +100,12 @@ func parseEditClusterCmd(cmd *cobra.Command, args []string) (*k3d.Cluster, *conf
 		// split node filter from the specified volume
 		portmap, filters, err := cliutil.SplitFiltersFromFlag(portFlag)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 
 		// create new entry or append filter to existing entry
 		if _, exists := portFilterMap[portmap]; exists {
-			log.Fatalln("Same Portmapping can not be used for multiple nodes")
+			l.Log().Fatalln("Same Portmapping can not be used for multiple nodes")
 		} else {
 			portFilterMap[portmap] = filters
 		}
@@ -118,7 +118,7 @@ func parseEditClusterCmd(cmd *cobra.Command, args []string) (*k3d.Cluster, *conf
 		})
 	}
 
-	log.Tracef("PortFilterMap: %+v", portFilterMap)
+	l.Log().Tracef("PortFilterMap: %+v", portFilterMap)
 
 	return existingCluster, &changeset
 }

cmd/cluster/clusterList.go

@@ -28,15 +28,14 @@ import (
 	"os"
 	"strings"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	k3cluster "github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/cmd/util"
+	k3cluster "github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
-	log "github.com/sirupsen/logrus"
-
 	"github.com/liggitt/tabwriter"
 )
 
@@ -83,14 +82,14 @@ func buildClusterList(ctx context.Context, args []string) []*k3d.Cluster {
 		// cluster name not specified : get all clusters
 		clusters, err = k3cluster.ClusterList(ctx, runtimes.SelectedRuntime)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 	} else {
 		for _, clusterName := range args {
 			// cluster name specified : get specific cluster
 			retrievedCluster, err := k3cluster.ClusterGet(ctx, runtimes.SelectedRuntime, &k3d.Cluster{Name: clusterName})
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 			clusters = append(clusters, retrievedCluster)
 		}
@@ -126,7 +125,7 @@ func PrintClusters(clusters []*k3d.Cluster, flags clusterFlags) {
 			}
 			_, err := fmt.Fprintf(tabwriter, "%s\n", strings.Join(headers, "\t"))
 			if err != nil {
-				log.Fatalln("Failed to print headers")
+				l.Log().Fatalln("Failed to print headers")
 			}
 		}
 	}

cmd/cluster/clusterStart.go

@@ -24,21 +24,22 @@ package cluster
 import (
 	"time"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	"github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	"github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
 // NewCmdClusterStart returns a new cobra command
 func NewCmdClusterStart() *cobra.Command {
 
-	startClusterOpts := types.ClusterStartOpts{}
+	startClusterOpts := types.ClusterStartOpts{
+		Intent: k3d.IntentClusterStart,
+	}
 
 	// create new command
 	cmd := &cobra.Command{
@@ -49,12 +50,18 @@ func NewCmdClusterStart() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			clusters := parseStartClusterCmd(cmd, args)
 			if len(clusters) == 0 {
-				log.Infoln("No clusters found")
+				l.Log().Infoln("No clusters found")
 			} else {
 				for _, c := range clusters {
-					if err := client.ClusterStart(cmd.Context(), runtimes.SelectedRuntime, c, startClusterOpts); err != nil {
-						log.Fatalln(err)
+					envInfo, err := client.GatherEnvironmentInfo(cmd.Context(), runtimes.SelectedRuntime, c)
+					if err != nil {
+						l.Log().Fatalf("failed to gather info about cluster environment: %v", err)
 					}
+					startClusterOpts.EnvironmentInfo = envInfo
+					if err := client.ClusterStart(cmd.Context(), runtimes.SelectedRuntime, c, startClusterOpts); err != nil {
+						l.Log().Fatalln(err)
+					}
+					l.Log().Infof("Started cluster '%s'", c.Name)
 				}
 			}
 		},
@@ -77,11 +84,11 @@ func parseStartClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 	var clusters []*k3d.Cluster
 
 	if all, err := cmd.Flags().GetBool("all"); err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	} else if all {
 		clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		return clusters
 	}
@@ -94,7 +101,7 @@ func parseStartClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 	for _, name := range clusternames {
 		cluster, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Cluster{Name: name})
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		clusters = append(clusters, cluster)
 	}

cmd/cluster/clusterStop.go

@@ -24,12 +24,11 @@ package cluster
 import (
 	"github.com/spf13/cobra"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
 // NewCmdClusterStop returns a new cobra command
@@ -44,11 +43,11 @@ func NewCmdClusterStop() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			clusters := parseStopClusterCmd(cmd, args)
 			if len(clusters) == 0 {
-				log.Infoln("No clusters found")
+				l.Log().Infoln("No clusters found")
 			} else {
 				for _, c := range clusters {
 					if err := client.ClusterStop(cmd.Context(), runtimes.SelectedRuntime, c); err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 				}
 			}
@@ -70,11 +69,11 @@ func parseStopClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 	var clusters []*k3d.Cluster
 
 	if all, err := cmd.Flags().GetBool("all"); err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	} else if all {
 		clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		return clusters
 	}
@@ -87,7 +86,7 @@ func parseStopClusterCmd(cmd *cobra.Command, args []string) []*k3d.Cluster {
 	for _, name := range clusternames {
 		cluster, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Cluster{Name: name})
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		clusters = append(clusters, cluster)
 	}

cmd/config/config.go

@@ -22,7 +22,8 @@ THE SOFTWARE.
 package config
 
 import (
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+
 	"github.com/spf13/cobra"
 )
 
@@ -34,14 +35,13 @@ func NewCmdConfig() *cobra.Command {
 		Long:  `Work with config file(s)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
 
-	cmd.AddCommand(NewCmdConfigInit())
-	cmd.AddCommand(NewCmdConfigMigrate())
+	cmd.AddCommand(NewCmdConfigInit(), NewCmdConfigMigrate())
 
 	return cmd
 }

cmd/config/configInit.go

@@ -25,8 +25,8 @@ import (
 	"fmt"
 	"os"
 
-	config "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
-	log "github.com/sirupsen/logrus"
+	config "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra"
 )
 
@@ -39,7 +39,7 @@ func NewCmdConfigInit() *cobra.Command {
 		Use:     "init",
 		Aliases: []string{"create"},
 		Run: func(cmd *cobra.Command, args []string) {
-			log.Infoln("COMING SOON: print a basic k3d config with default pre-filled.")
+			l.Log().Infoln("COMING SOON: print a basic k3d config with default pre-filled.")
 			if output == "-" {
 				fmt.Println(config.DefaultConfig)
 			} else {
@@ -51,16 +51,16 @@ func NewCmdConfigInit() *cobra.Command {
 					// create/overwrite file
 					file, err = os.Create(output)
 					if err != nil {
-						log.Fatalf("Failed to create/overwrite output file: %s", err)
+						l.Log().Fatalf("Failed to create/overwrite output file: %s", err)
 					}
 					// write content
 					if _, err = file.WriteString(config.DefaultConfig); err != nil {
-						log.Fatalf("Failed to write to output file: %+v", err)
+						l.Log().Fatalf("Failed to write to output file: %+v", err)
 					}
 				} else if err != nil {
-					log.Fatalf("Failed to stat output file: %+v", err)
+					l.Log().Fatalf("Failed to stat output file: %+v", err)
 				} else {
-					log.Errorln("Output file exists and --force was not set")
+					l.Log().Errorln("Output file exists and --force was not set")
 					os.Exit(1)
 				}
 			}
@@ -68,6 +68,9 @@ func NewCmdConfigInit() *cobra.Command {
 	}
 
 	cmd.Flags().StringVarP(&output, "output", "o", "k3d-default.yaml", "Write a default k3d config")
+	if err := cmd.MarkFlagFilename("output", "yaml", "yml"); err != nil {
+		l.Log().Fatalf("Failed to mark flag 'output' as filename flag: %v", err)
+	}
 	cmd.Flags().BoolVarP(&force, "force", "f", false, "Force overwrite of target file")
 
 	return cmd

cmd/config/configMigrate.go

@@ -25,8 +25,8 @@ import (
 	"os"
 	"strings"
 
-	"github.com/rancher/k3d/v4/pkg/config"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/pkg/config"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"gopkg.in/yaml.v2"
@@ -36,14 +36,15 @@ import (
 func NewCmdConfigMigrate() *cobra.Command {
 
 	cmd := &cobra.Command{
-		Use:  "migrate INPUT OUTPUT",
-		Args: cobra.RangeArgs(1, 2),
+		Use:     "migrate INPUT [OUTPUT]",
+		Aliases: []string{"update"},
+		Args:    cobra.RangeArgs(1, 2),
 		Run: func(cmd *cobra.Command, args []string) {
 
 			configFile := args[0]
 
 			if _, err := os.Stat(configFile); err != nil {
-				log.Fatalf("Failed to stat config file %s: %+v", configFile, err)
+				l.Log().Fatalf("Failed to stat config file %s: %+v", configFile, err)
 			}
 
 			cfgViper := viper.New()
@@ -54,38 +55,38 @@ func NewCmdConfigMigrate() *cobra.Command {
 			// try to read config into memory (viper map structure)
 			if err := cfgViper.ReadInConfig(); err != nil {
 				if _, ok := err.(viper.ConfigFileNotFoundError); ok {
-					log.Fatalf("Config file %s not found: %+v", configFile, err)
+					l.Log().Fatalf("Config file %s not found: %+v", configFile, err)
 				}
 				// config file found but some other error happened
-				log.Fatalf("Failed to read config file %s: %+v", configFile, err)
+				l.Log().Fatalf("Failed to read config file %s: %+v", configFile, err)
 			}
 
 			schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
 			if err != nil {
-				log.Fatalf("Cannot validate config file %s: %+v", configFile, err)
+				l.Log().Fatalf("Cannot validate config file %s: %+v", configFile, err)
 			}
 
 			if err := config.ValidateSchemaFile(configFile, schema); err != nil {
-				log.Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
+				l.Log().Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
 			}
 
-			log.Infof("Using config file %s (%s#%s)", cfgViper.ConfigFileUsed(), strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))
+			l.Log().Infof("Using config file %s (%s#%s)", cfgViper.ConfigFileUsed(), strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))
 
 			cfg, err := config.FromViper(cfgViper)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 
 			if cfg.GetAPIVersion() != config.DefaultConfigApiVersion {
 				cfg, err = config.Migrate(cfg, config.DefaultConfigApiVersion)
 				if err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 			}
 
 			yamlout, err := yaml.Marshal(cfg)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 
 			output := "-"
@@ -96,11 +97,11 @@ func NewCmdConfigMigrate() *cobra.Command {
 
 			if output == "-" {
 				if _, err := os.Stdout.Write(yamlout); err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 			} else {
-				if err := os.WriteFile(output, yamlout, os.ModeAppend); err != nil {
-					log.Fatalln(err)
+				if err := os.WriteFile(output, yamlout, os.ModePerm); err != nil {
+					l.Log().Fatalln(err)
 				}
 			}
 

cmd/config/configView.go

@@ -24,7 +24,6 @@ package config
 import (
 	"fmt"
 
-	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -35,9 +34,7 @@ func NewCmdConfigView() *cobra.Command {
 		Use:     "view",
 		Aliases: []string{"show"},
 		Run: func(cmd *cobra.Command, args []string) {
-			log.Debugln("print config")
 			fmt.Printf("%+v", viper.AllSettings())
-			log.Debugln("printed config")
 		},
 	}
 	return cmd

cmd/debug/debug.go

@@ -24,10 +24,11 @@ package debug
 import (
 	"fmt"
 
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	"github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	"github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 )
@@ -41,8 +42,8 @@ func NewCmdDebug() *cobra.Command {
 		Long:   `Debug k3d cluster(s)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
@@ -60,28 +61,29 @@ func NewCmdDebugLoadbalancer() *cobra.Command {
 		Long:    `Debug the loadbalancer`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
 
 	cmd.AddCommand(&cobra.Command{
-		Use:  "get-config CLUSTERNAME",
-		Args: cobra.ExactArgs(1), // cluster name
+		Use:               "get-config CLUSTERNAME",
+		Args:              cobra.ExactArgs(1), // cluster name
+		ValidArgsFunction: util.ValidArgsAvailableClusters,
 		Run: func(cmd *cobra.Command, args []string) {
 			c, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &types.Cluster{Name: args[0]})
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 
 			lbconf, err := client.GetLoadbalancerConfig(cmd.Context(), runtimes.SelectedRuntime, c)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 			yamlized, err := yaml.Marshal(lbconf)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 			fmt.Println(string(yamlized))
 		},

cmd/image/image.go

@@ -22,7 +22,7 @@ THE SOFTWARE.
 package image
 
 import (
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra"
 )
 
@@ -31,13 +31,14 @@ func NewCmdImage() *cobra.Command {
 
 	// create new cobra command
 	cmd := &cobra.Command{
-		Use:   "image",
-		Short: "Handle container images.",
-		Long:  `Handle container images.`,
+		Use:     "image",
+		Aliases: []string{"images"},
+		Short:   "Handle container images.",
+		Long:    `Handle container images.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}

cmd/image/imageImport.go

@@ -26,12 +26,11 @@ import (
 
 	"github.com/spf13/cobra"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	"github.com/rancher/k3d/v4/pkg/tools"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
 // NewCmdImageImport returns a new cobra command
@@ -56,24 +55,24 @@ That is, 'rancher/k3d-tools' is treated as 'rancher/k3d-tools:latest'.
 
 A file ARCHIVE always takes precedence.
 So if a file './rancher/k3d-tools' exists, k3d will try to import it instead of the IMAGE of the same name.`,
-		Aliases: []string{"images"},
+		Aliases: []string{"load"},
 		Args:    cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			images, clusters := parseLoadImageCmd(cmd, args)
-			log.Debugf("Importing image(s) [%+v] from runtime [%s] into cluster(s) [%+v]...", images, runtimes.SelectedRuntime, clusters)
+			l.Log().Debugf("Importing image(s) [%+v] from runtime [%s] into cluster(s) [%+v]...", images, runtimes.SelectedRuntime, clusters)
 			errOccured := false
 			for _, cluster := range clusters {
-				log.Infof("Importing image(s) into cluster '%s'", cluster.Name)
-				if err := tools.ImageImportIntoClusterMulti(cmd.Context(), runtimes.SelectedRuntime, images, &cluster, loadImageOpts); err != nil {
-					log.Errorf("Failed to import image(s) into cluster '%s': %+v", cluster.Name, err)
+				l.Log().Infof("Importing image(s) into cluster '%s'", cluster.Name)
+				if err := client.ImageImportIntoClusterMulti(cmd.Context(), runtimes.SelectedRuntime, images, &cluster, loadImageOpts); err != nil {
+					l.Log().Errorf("Failed to import image(s) into cluster '%s': %+v", cluster.Name, err)
 					errOccured = true
 				}
 			}
 			if errOccured {
-				log.Warnln("At least one error occured while trying to import the image(s) into the selected cluster(s)")
+				l.Log().Warnln("At least one error occured while trying to import the image(s) into the selected cluster(s)")
 				os.Exit(1)
 			}
-			log.Infof("Successfully imported %d image(s) into %d cluster(s)", len(images), len(clusters))
+			l.Log().Infof("Successfully imported %d image(s) into %d cluster(s)", len(images), len(clusters))
 		},
 	}
 
@@ -82,7 +81,7 @@ So if a file './rancher/k3d-tools' exists, k3d will try to import it instead of
 	 *********/
 	cmd.Flags().StringArrayP("cluster", "c", []string{k3d.DefaultClusterName}, "Select clusters to load the image to.")
 	if err := cmd.RegisterFlagCompletionFunc("cluster", util.ValidArgsAvailableClusters); err != nil {
-		log.Fatalln("Failed to register flag completion for '--cluster'", err)
+		l.Log().Fatalln("Failed to register flag completion for '--cluster'", err)
 	}
 
 	cmd.Flags().BoolVarP(&loadImageOpts.KeepTar, "keep-tarball", "k", false, "Do not delete the tarball containing the saved images from the shared volume")
@@ -100,7 +99,7 @@ func parseLoadImageCmd(cmd *cobra.Command, args []string) ([]string, []k3d.Clust
 	// --cluster
 	clusterNames, err := cmd.Flags().GetStringArray("cluster")
 	if err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	}
 	clusters := []k3d.Cluster{}
 	for _, clusterName := range clusterNames {
@@ -110,7 +109,7 @@ func parseLoadImageCmd(cmd *cobra.Command, args []string) ([]string, []k3d.Clust
 	// images
 	images := args
 	if len(images) == 0 {
-		log.Fatalln("No images specified!")
+		l.Log().Fatalln("No images specified!")
 	}
 
 	return images, clusters

cmd/kubeconfig/kubeconfig.go

@@ -22,7 +22,7 @@ THE SOFTWARE.
 package kubeconfig
 
 import (
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra"
 )
 
@@ -36,15 +36,14 @@ func NewCmdKubeconfig() *cobra.Command {
 		Long:  `Manage kubeconfig(s)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
 
 	// add subcommands
-	cmd.AddCommand(NewCmdKubeconfigGet())
-	cmd.AddCommand(NewCmdKubeconfigMerge())
+	cmd.AddCommand(NewCmdKubeconfigGet(), NewCmdKubeconfigMerge())
 
 	// add flags
 

cmd/kubeconfig/kubeconfigGet.go

@@ -25,13 +25,12 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
-
-	log "github.com/sirupsen/logrus"
 )
 
 type getKubeconfigFlags struct {
@@ -70,13 +69,13 @@ func NewCmdKubeconfigGet() *cobra.Command {
 			if getKubeconfigFlags.all {
 				clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime)
 				if err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 			} else {
 				for _, clusterName := range args {
 					retrievedCluster, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Cluster{Name: clusterName})
 					if err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 					clusters = append(clusters, retrievedCluster)
 				}
@@ -85,10 +84,10 @@ func NewCmdKubeconfigGet() *cobra.Command {
 			// get kubeconfigs from all clusters
 			errorGettingKubeconfig := false
 			for _, c := range clusters {
-				log.Debugf("Getting kubeconfig for cluster '%s'", c.Name)
+				l.Log().Debugf("Getting kubeconfig for cluster '%s'", c.Name)
 				fmt.Println("---") // YAML document separator
 				if _, err := client.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, c, "-", &writeKubeConfigOptions); err != nil {
-					log.Errorln(err)
+					l.Log().Errorln(err)
 					errorGettingKubeconfig = true
 				}
 			}

cmd/kubeconfig/kubeconfigMerge.go

@@ -27,15 +27,14 @@ import (
 	"path"
 	"strings"
 
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	k3dutil "github.com/rancher/k3d/v4/pkg/util"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+	k3dutil "github.com/rancher/k3d/v5/pkg/util"
 	"github.com/spf13/cobra"
 	"k8s.io/client-go/tools/clientcmd"
-
-	log "github.com/sirupsen/logrus"
 )
 
 type mergeKubeconfigFlags struct {
@@ -64,14 +63,14 @@ func NewCmdKubeconfigMerge() *cobra.Command {
 			var err error
 
 			if mergeKubeconfigFlags.targetDefault && mergeKubeconfigFlags.output != "" {
-				log.Fatalln("Cannot use both '--output' and '--kubeconfig-merge-default' at the same time")
+				l.Log().Fatalln("Cannot use both '--output' and '--kubeconfig-merge-default' at the same time")
 			}
 
 			// generate list of clusters
 			if mergeKubeconfigFlags.all {
 				clusters, err = client.ClusterList(cmd.Context(), runtimes.SelectedRuntime)
 				if err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 			} else {
 
@@ -83,7 +82,7 @@ func NewCmdKubeconfigMerge() *cobra.Command {
 				for _, clusterName := range clusternames {
 					retrievedCluster, err := client.ClusterGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Cluster{Name: clusterName})
 					if err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 					clusters = append(clusters, retrievedCluster)
 				}
@@ -94,18 +93,18 @@ func NewCmdKubeconfigMerge() *cobra.Command {
 			var outputs []string
 			outputDir, err := k3dutil.GetConfigDirOrCreate()
 			if err != nil {
-				log.Errorln(err)
-				log.Fatalln("Failed to save kubeconfig to local directory")
+				l.Log().Errorln(err)
+				l.Log().Fatalln("Failed to save kubeconfig to local directory")
 			}
 			for _, c := range clusters {
-				log.Debugf("Getting kubeconfig for cluster '%s'", c.Name)
+				l.Log().Debugf("Getting kubeconfig for cluster '%s'", c.Name)
 				output := mergeKubeconfigFlags.output
 				if output == "" && !mergeKubeconfigFlags.targetDefault {
 					output = path.Join(outputDir, fmt.Sprintf("kubeconfig-%s.yaml", c.Name))
 				}
 				output, err = client.KubeconfigGetWrite(cmd.Context(), runtimes.SelectedRuntime, c, output, &writeKubeConfigOptions)
 				if err != nil {
-					log.Errorln(err)
+					l.Log().Errorln(err)
 					errorGettingKubeconfig = true
 				} else {
 					outputs = append(outputs, output)
@@ -127,7 +126,7 @@ func NewCmdKubeconfigMerge() *cobra.Command {
 	// add flags
 	cmd.Flags().StringVarP(&mergeKubeconfigFlags.output, "output", "o", "", fmt.Sprintf("Define output [ - | FILE ] (default from $KUBECONFIG or %s", clientcmd.RecommendedHomeFile))
 	if err := cmd.MarkFlagFilename("output"); err != nil {
-		log.Fatalln("Failed to mark flag --output as filename")
+		l.Log().Fatalln("Failed to mark flag --output as filename")
 	}
 	cmd.Flags().BoolVarP(&mergeKubeconfigFlags.targetDefault, "kubeconfig-merge-default", "d", false, fmt.Sprintf("Merge into the default kubeconfig ($KUBECONFIG or %s)", clientcmd.RecommendedHomeFile))
 	cmd.Flags().BoolVarP(&writeKubeConfigOptions.UpdateExisting, "update", "u", true, "Update conflicting fields in existing kubeconfig")

cmd/node/node.go

@@ -22,7 +22,7 @@ THE SOFTWARE.
 package node
 
 import (
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra"
 )
 
@@ -36,19 +36,19 @@ func NewCmdNode() *cobra.Command {
 		Long:  `Manage node(s)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
 
 	// add subcommands
-	cmd.AddCommand(NewCmdNodeCreate())
-	cmd.AddCommand(NewCmdNodeStart())
-	cmd.AddCommand(NewCmdNodeStop())
-	cmd.AddCommand(NewCmdNodeDelete())
-	cmd.AddCommand(NewCmdNodeList())
-	cmd.AddCommand(NewCmdNodeEdit())
+	cmd.AddCommand(NewCmdNodeCreate(),
+		NewCmdNodeStart(),
+		NewCmdNodeStop(),
+		NewCmdNodeDelete(),
+		NewCmdNodeList(),
+		NewCmdNodeEdit())
 
 	// add flags
 

cmd/node/nodeCreate.go

@@ -29,13 +29,13 @@ import (
 	"github.com/spf13/cobra"
 
 	dockerunits "github.com/docker/go-units"
-	"github.com/rancher/k3d/v4/cmd/util"
-	cliutil "github.com/rancher/k3d/v4/cmd/util"
-	k3dc "github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	"github.com/rancher/k3d/v4/version"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	cliutil "github.com/rancher/k3d/v5/cmd/util"
+	k3dc "github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+	"github.com/rancher/k3d/v5/version"
 )
 
 // NewCmdNodeCreate returns a new cobra command
@@ -50,12 +50,19 @@ func NewCmdNodeCreate() *cobra.Command {
 		Long:  `Create a new containerized k3s node (k3s in docker).`,
 		Args:  cobra.ExactArgs(1), // exactly one name accepted // TODO: if not specified, inherit from cluster that the node shall belong to, if that is specified
 		Run: func(cmd *cobra.Command, args []string) {
-			nodes, cluster := parseCreateNodeCmd(cmd, args)
-			if err := k3dc.NodeAddToClusterMulti(cmd.Context(), runtimes.SelectedRuntime, nodes, cluster, createNodeOpts); err != nil {
-				log.Errorf("Failed to add nodes to cluster '%s'", cluster.Name)
-				log.Fatalln(err)
+			nodes, clusterName := parseCreateNodeCmd(cmd, args)
+			if strings.HasPrefix(clusterName, "https://") {
+				l.Log().Infof("Adding %d node(s) to the remote cluster '%s'...", len(nodes), clusterName)
+				if err := k3dc.NodeAddToClusterMultiRemote(cmd.Context(), runtimes.SelectedRuntime, nodes, clusterName, createNodeOpts); err != nil {
+					l.Log().Fatalf("failed to add %d node(s) to the remote cluster '%s': %v", len(nodes), clusterName, err)
+				}
+			} else {
+				l.Log().Infof("Adding %d node(s) to the runtime local cluster '%s'...", len(nodes), clusterName)
+				if err := k3dc.NodeAddToClusterMulti(cmd.Context(), runtimes.SelectedRuntime, nodes, &k3d.Cluster{Name: clusterName}, createNodeOpts); err != nil {
+					l.Log().Fatalf("failed to add %d node(s) to the runtime local cluster '%s': %v", len(nodes), clusterName, err)
+				}
 			}
-			log.Infof("Successfully created %d node(s)!", len(nodes))
+			l.Log().Infof("Successfully created %d node(s)!", len(nodes))
 		},
 	}
 
@@ -63,11 +70,11 @@ func NewCmdNodeCreate() *cobra.Command {
 	cmd.Flags().Int("replicas", 1, "Number of replicas of this node specification.")
 	cmd.Flags().String("role", string(k3d.AgentRole), "Specify node role [server, agent]")
 	if err := cmd.RegisterFlagCompletionFunc("role", util.ValidArgsNodeRoles); err != nil {
-		log.Fatalln("Failed to register flag completion for '--role'", err)
+		l.Log().Fatalln("Failed to register flag completion for '--role'", err)
 	}
-	cmd.Flags().StringP("cluster", "c", k3d.DefaultClusterName, "Select the cluster that the node shall connect to.")
+	cmd.Flags().StringP("cluster", "c", k3d.DefaultClusterName, "Cluster URL or k3d cluster name to connect to.")
 	if err := cmd.RegisterFlagCompletionFunc("cluster", util.ValidArgsAvailableClusters); err != nil {
-		log.Fatalln("Failed to register flag completion for '--cluster'", err)
+		l.Log().Fatalln("Failed to register flag completion for '--cluster'", err)
 	}
 
 	cmd.Flags().StringP("image", "i", fmt.Sprintf("%s:%s", k3d.DefaultK3sImageRepo, version.GetK3sVersion(false)), "Specify k3s image used for the node(s)")
@@ -79,69 +86,70 @@ func NewCmdNodeCreate() *cobra.Command {
 	cmd.Flags().StringSliceP("runtime-label", "", []string{}, "Specify container runtime labels in format \"foo=bar\"")
 	cmd.Flags().StringSliceP("k3s-node-label", "", []string{}, "Specify k3s node labels in format \"foo=bar\"")
 
+	cmd.Flags().StringSliceP("network", "n", []string{}, "Add node to (another) runtime network")
+
+	cmd.Flags().StringVarP(&createNodeOpts.ClusterToken, "token", "t", "", "Override cluster token (required when connecting to an external cluster)")
+
 	// done
 	return cmd
 }
 
-// parseCreateNodeCmd parses the command input into variables required to create a cluster
-func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cluster) {
+// parseCreateNodeCmd parses the command input into variables required to create a node
+func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, string) {
 
 	// --replicas
 	replicas, err := cmd.Flags().GetInt("replicas")
 	if err != nil {
-		log.Errorln("No replica count specified")
-		log.Fatalln(err)
+		l.Log().Errorln("No replica count specified")
+		l.Log().Fatalln(err)
 	}
 
 	// --role
 	roleStr, err := cmd.Flags().GetString("role")
 	if err != nil {
-		log.Errorln("No node role specified")
-		log.Fatalln(err)
+		l.Log().Errorln("No node role specified")
+		l.Log().Fatalln(err)
 	}
 	if _, ok := k3d.NodeRoles[roleStr]; !ok {
-		log.Fatalf("Unknown node role '%s'\n", roleStr)
+		l.Log().Fatalf("Unknown node role '%s'\n", roleStr)
 	}
 	role := k3d.NodeRoles[roleStr]
 
 	// --image
 	image, err := cmd.Flags().GetString("image")
 	if err != nil {
-		log.Errorln("No image specified")
-		log.Fatalln(err)
+		l.Log().Errorln("No image specified")
+		l.Log().Fatalln(err)
 	}
 
 	// --cluster
 	clusterName, err := cmd.Flags().GetString("cluster")
 	if err != nil {
-		log.Fatalln(err)
-	}
-	cluster := &k3d.Cluster{
-		Name: clusterName,
+		l.Log().Fatalln(err)
 	}
 
 	// --memory
 	memory, err := cmd.Flags().GetString("memory")
 	if err != nil {
-		log.Errorln("No memory specified")
-		log.Fatalln(err)
+		l.Log().Errorln("No memory specified")
+		l.Log().Fatalln(err)
 	}
 	if _, err := dockerunits.RAMInBytes(memory); memory != "" && err != nil {
-		log.Errorf("Provided memory limit value is invalid")
+		l.Log().Errorf("Provided memory limit value is invalid")
 	}
 
 	// --runtime-label
 	runtimeLabelsFlag, err := cmd.Flags().GetStringSlice("runtime-label")
 	if err != nil {
-		log.Errorln("No runtime-label specified")
-		log.Fatalln(err)
+		l.Log().Errorln("No runtime-label specified")
+		l.Log().Fatalln(err)
 	}
 
 	runtimeLabels := make(map[string]string, len(runtimeLabelsFlag)+1)
 	for _, label := range runtimeLabelsFlag {
 		labelSplitted := strings.Split(label, "=")
 		if len(labelSplitted) != 2 {
-			log.Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label)
+			l.Log().Fatalf("unknown runtime-label format format: %s, use format \"foo=bar\"", label)
 		}
 		cliutil.ValidateRuntimeLabelKey(labelSplitted[0])
 		runtimeLabels[labelSplitted[0]] = labelSplitted[1]
@@ -153,19 +161,25 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 	// --k3s-node-label
 	k3sNodeLabelsFlag, err := cmd.Flags().GetStringSlice("k3s-node-label")
 	if err != nil {
-		log.Errorln("No k3s-node-label specified")
-		log.Fatalln(err)
+		l.Log().Errorln("No k3s-node-label specified")
+		l.Log().Fatalln(err)
 	}
 
 	k3sNodeLabels := make(map[string]string, len(k3sNodeLabelsFlag))
 	for _, label := range k3sNodeLabelsFlag {
 		labelSplitted := strings.Split(label, "=")
 		if len(labelSplitted) != 2 {
-			log.Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label)
+			l.Log().Fatalf("unknown k3s-node-label format format: %s, use format \"foo=bar\"", label)
 		}
 		k3sNodeLabels[labelSplitted[0]] = labelSplitted[1]
 	}
 
+	// --network
+	networks, err := cmd.Flags().GetStringSlice("network")
+	if err != nil {
+		l.Log().Fatalf("failed to get --network string slice flag: %v", err)
+	}
+
 	// generate list of nodes
 	nodes := []*k3d.Node{}
 	for i := 0; i < replicas; i++ {
@@ -177,9 +191,10 @@ func parseCreateNodeCmd(cmd *cobra.Command, args []string) ([]*k3d.Node, *k3d.Cl
 			RuntimeLabels: runtimeLabels,
 			Restart:       true,
 			Memory:        memory,
+			Networks:      networks,
 		}
 		nodes = append(nodes, node)
 	}
 
-	return nodes, cluster
+	return nodes, clusterName
 }

cmd/node/nodeDelete.go

@@ -22,11 +22,11 @@ THE SOFTWARE.
 package node
 
 import (
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 )
 
@@ -52,14 +52,14 @@ func NewCmdNodeDelete() *cobra.Command {
 			nodeDeleteOpts := k3d.NodeDeleteOpts{SkipLBUpdate: flags.All} // do not update LB, if we're deleting all nodes anyway
 
 			if len(nodes) == 0 {
-				log.Infoln("No nodes found")
+				l.Log().Infoln("No nodes found")
 			} else {
 				for _, node := range nodes {
 					if err := client.NodeDelete(cmd.Context(), runtimes.SelectedRuntime, node, nodeDeleteOpts); err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 				}
-				log.Infof("Successfully deleted %d node(s)!", len(nodes))
+				l.Log().Infof("Successfully deleted %d node(s)!", len(nodes))
 			}
 		},
 	}
@@ -83,11 +83,11 @@ func parseDeleteNodeCmd(cmd *cobra.Command, args []string, flags *nodeDeleteFlag
 	// --all
 	if flags.All {
 		if !flags.IncludeRegistries {
-			log.Infoln("Didn't set '--registries', so won't delete registries.")
+			l.Log().Infoln("Didn't set '--registries', so won't delete registries.")
 		}
 		nodes, err = client.NodeList(cmd.Context(), runtimes.SelectedRuntime)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		include := k3d.ClusterInternalNodeRoles
 		exclude := []k3d.Role{}
@@ -99,13 +99,13 @@ func parseDeleteNodeCmd(cmd *cobra.Command, args []string, flags *nodeDeleteFlag
 	}
 
 	if !flags.All && len(args) < 1 {
-		log.Fatalln("Expecting at least one node name if `--all` is not set")
+		l.Log().Fatalln("Expecting at least one node name if `--all` is not set")
 	}
 
 	for _, name := range args {
 		node, err := client.NodeGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Node{Name: name})
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		nodes = append(nodes, node)
 	}

cmd/node/nodeEdit.go

@@ -23,11 +23,11 @@ package node
 
 import (
 	"github.com/docker/go-connections/nat"
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 )
 
@@ -36,7 +36,7 @@ func NewCmdNodeEdit() *cobra.Command {
 
 	// create new cobra command
 	cmd := &cobra.Command{
-		Use:               "edit NAME",
+		Use:               "edit NODE",
 		Short:             "[EXPERIMENTAL] Edit node(s).",
 		Long:              `[EXPERIMENTAL] Edit node(s).`,
 		Args:              cobra.ExactArgs(1),
@@ -46,13 +46,13 @@ func NewCmdNodeEdit() *cobra.Command {
 
 			existingNode, changeset := parseEditNodeCmd(cmd, args)
 
-			log.Debugf("===== Current =====\n%+v\n===== Changeset =====\n%+v\n", existingNode, changeset)
+			l.Log().Debugf("===== Current =====\n%+v\n===== Changeset =====\n%+v\n", existingNode, changeset)
 
 			if err := client.NodeEdit(cmd.Context(), runtimes.SelectedRuntime, existingNode, changeset); err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 
-			log.Infof("Successfully updated %s", existingNode.Name)
+			l.Log().Infof("Successfully updated %s", existingNode.Name)
 
 		},
 	}
@@ -71,16 +71,16 @@ func parseEditNodeCmd(cmd *cobra.Command, args []string) (*k3d.Node, *k3d.Node)
 
 	existingNode, err := client.NodeGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Node{Name: args[0]})
 	if err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	}
 
 	if existingNode == nil {
-		log.Infof("Node %s not found", args[0])
+		l.Log().Infof("Node %s not found", args[0])
 		return nil, nil
 	}
 
 	if existingNode.Role != k3d.LoadBalancerRole {
-		log.Fatalln("Currently only the loadbalancer can be updated!")
+		l.Log().Fatalln("Currently only the loadbalancer can be updated!")
 	}
 
 	changeset := &k3d.Node{}
@@ -90,7 +90,7 @@ func parseEditNodeCmd(cmd *cobra.Command, args []string) (*k3d.Node, *k3d.Node)
 	 */
 	portFlags, err := cmd.Flags().GetStringArray("port-add")
 	if err != nil {
-		log.Errorln(err)
+		l.Log().Errorln(err)
 		return nil, nil
 	}
 
@@ -101,7 +101,7 @@ func parseEditNodeCmd(cmd *cobra.Command, args []string) (*k3d.Node, *k3d.Node)
 
 		portmappings, err := nat.ParsePortSpec(flag)
 		if err != nil {
-			log.Fatalf("Failed to parse port spec '%s': %+v", flag, err)
+			l.Log().Fatalf("Failed to parse port spec '%s': %+v", flag, err)
 		}
 
 		for _, pm := range portmappings {

cmd/node/nodeList.go

@@ -26,13 +26,12 @@ import (
 	"strings"
 
 	"github.com/liggitt/tabwriter"
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
-
-	log "github.com/sirupsen/logrus"
 )
 
 type nodeListFlags struct {
@@ -46,7 +45,7 @@ func NewCmdNodeList() *cobra.Command {
 
 	// create new command
 	cmd := &cobra.Command{
-		Use:               "list [NAME [NAME...]]",
+		Use:               "list [NODE [NODE...]]",
 		Aliases:           []string{"ls", "get"},
 		Short:             "List node(s)",
 		Long:              `List node(s).`,
@@ -64,14 +63,14 @@ func NewCmdNodeList() *cobra.Command {
 			if len(nodes) == 0 { // Option a)  no name specified -> get all nodes
 				found, err := client.NodeList(cmd.Context(), runtimes.SelectedRuntime)
 				if err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 				existingNodes = append(existingNodes, found...)
 			} else { // Option b) cluster name specified -> get specific cluster
 				for _, node := range nodes {
 					found, err := client.NodeGet(cmd.Context(), runtimes.SelectedRuntime, node)
 					if err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 					existingNodes = append(existingNodes, found)
 				}

cmd/node/nodeStart.go

@@ -22,12 +22,11 @@ THE SOFTWARE.
 package node
 
 import (
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/cmd/util"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
-
-	log "github.com/sirupsen/logrus"
 )
 
 // NewCmdNodeStart returns a new cobra command
@@ -35,14 +34,14 @@ func NewCmdNodeStart() *cobra.Command {
 
 	// create new command
 	cmd := &cobra.Command{
-		Use:               "start NAME", // TODO: startNode: allow one or more names or --all
+		Use:               "start NODE", // TODO: startNode: allow one or more names or --all
 		Short:             "Start an existing k3d node",
 		Long:              `Start an existing k3d node.`,
 		ValidArgsFunction: util.ValidArgsAvailableNodes,
 		Run: func(cmd *cobra.Command, args []string) {
 			node := parseStartNodeCmd(cmd, args)
 			if err := runtimes.SelectedRuntime.StartNode(cmd.Context(), node); err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 		},
 	}
@@ -55,7 +54,7 @@ func NewCmdNodeStart() *cobra.Command {
 func parseStartNodeCmd(cmd *cobra.Command, args []string) *k3d.Node {
 	// node name // TODO: startNode: allow node filters, e.g. `k3d node start mycluster@agent` to start all agent nodes of cluster 'mycluster'
 	if len(args) == 0 || len(args[0]) == 0 {
-		log.Fatalln("No node name given")
+		l.Log().Fatalln("No node name given")
 	}
 
 	return &k3d.Node{Name: args[0]}

cmd/node/nodeStop.go

@@ -22,13 +22,12 @@ THE SOFTWARE.
 package node
 
 import (
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
 	"github.com/spf13/cobra"
 
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
 // NewCmdNodeStop returns a new cobra command
@@ -43,7 +42,7 @@ func NewCmdNodeStop() *cobra.Command {
 		Run: func(cmd *cobra.Command, args []string) {
 			node := parseStopNodeCmd(cmd, args)
 			if err := runtimes.SelectedRuntime.StopNode(cmd.Context(), node); err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 		},
 	}
@@ -56,7 +55,7 @@ func NewCmdNodeStop() *cobra.Command {
 func parseStopNodeCmd(cmd *cobra.Command, args []string) *k3d.Node {
 	// node name // TODO: allow node filters, e.g. `k3d node stop mycluster@agent` to stop all agent nodes of cluster 'mycluster'
 	if len(args) == 0 || len(args[0]) == 0 {
-		log.Fatalln("No node name given")
+		l.Log().Fatalln("No node name given")
 	}
 
 	return &k3d.Node{Name: args[0]}

cmd/registry/registry.go

@@ -22,7 +22,7 @@ THE SOFTWARE.
 package registry
 
 import (
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra"
 )
 
@@ -37,19 +37,18 @@ func NewCmdRegistry() *cobra.Command {
 		Long:    `Manage registry/registries`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Errorln("Couldn't get help text")
-				log.Fatalln(err)
+				l.Log().Errorln("Couldn't get help text")
+				l.Log().Fatalln(err)
 			}
 		},
 	}
 
 	// add subcommands
-	cmd.AddCommand(NewCmdRegistryCreate())
-	cmd.AddCommand(NewCmdRegistryStart())
-	cmd.AddCommand(NewCmdRegistryStop())
-	cmd.AddCommand(NewCmdRegistryDelete())
-	cmd.AddCommand(NewCmdRegistryList())
-	// cmd.AddCommand(NewCmdRegistryConnect()) // TODO: registry connect requires reload capabilities for containerd config
+	cmd.AddCommand(NewCmdRegistryCreate(),
+		NewCmdRegistryStart(),
+		NewCmdRegistryStop(),
+		NewCmdRegistryDelete(),
+		NewCmdRegistryList())
 
 	// add flags
 

cmd/registry/registryCreate.go

@@ -24,14 +24,14 @@ package registry
 import (
 	"fmt"
 
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 
-	"github.com/rancher/k3d/v4/pkg/client"
+	"github.com/rancher/k3d/v5/pkg/client"
 
-	cliutil "github.com/rancher/k3d/v4/cmd/util"
+	cliutil "github.com/rancher/k3d/v5/cmd/util"
 	"github.com/spf13/cobra"
 )
 
@@ -75,12 +75,12 @@ func NewCmdRegistryCreate() *cobra.Command {
 			reg, clusters := parseCreateRegistryCmd(cmd, args, flags, ppFlags)
 			regNode, err := client.RegistryRun(cmd.Context(), runtimes.SelectedRuntime, reg)
 			if err != nil {
-				log.Fatalln(err)
+				l.Log().Fatalln(err)
 			}
 			if err := client.RegistryConnectClusters(cmd.Context(), runtimes.SelectedRuntime, regNode, clusters); err != nil {
-				log.Errorln(err)
+				l.Log().Errorln(err)
 			}
-			log.Infof("Successfully created registry '%s'", reg.Host)
+			l.Log().Infof("Successfully created registry '%s'", reg.Host)
 			regString := fmt.Sprintf("%s:%s", reg.Host, reg.ExposureOpts.Binding.HostPort)
 			if !flags.NoHelp {
 				fmt.Println(fmt.Sprintf(helptext, regString, regString, regString, regString))
@@ -93,10 +93,10 @@ func NewCmdRegistryCreate() *cobra.Command {
 	// TODO: connecting to clusters requires non-existing config reload functionality in containerd
 	cmd.Flags().StringArrayVarP(&ppFlags.Clusters, "cluster", "c", nil, "[NotReady] Select the cluster(s) that the registry shall connect to.")
 	if err := cmd.RegisterFlagCompletionFunc("cluster", cliutil.ValidArgsAvailableClusters); err != nil {
-		log.Fatalln("Failed to register flag completion for '--cluster'", err)
+		l.Log().Fatalln("Failed to register flag completion for '--cluster'", err)
 	}
 	if err := cmd.Flags().MarkHidden("cluster"); err != nil {
-		log.Fatalln("Failed to hide --cluster flag on registry create command")
+		l.Log().Fatalln("Failed to hide --cluster flag on registry create command")
 	}
 
 	cmd.Flags().StringVarP(&flags.Image, "image", "i", fmt.Sprintf("%s:%s", k3d.DefaultRegistryImageRepo, k3d.DefaultRegistryImageTag), "Specify image used for the registry")
@@ -125,8 +125,8 @@ func parseCreateRegistryCmd(cmd *cobra.Command, args []string, flags *regCreateF
 	// --port
 	exposePort, err := cliutil.ParsePortExposureSpec(ppFlags.Port, k3d.DefaultRegistryPort)
 	if err != nil {
-		log.Errorln("Failed to parse registry port")
-		log.Fatalln(err)
+		l.Log().Errorln("Failed to parse registry port")
+		l.Log().Fatalln(err)
 	}
 
 	// set the name for the registry node

cmd/registry/registryDelete.go

@@ -22,11 +22,11 @@ THE SOFTWARE.
 package registry
 
 import (
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 )
 
@@ -44,17 +44,18 @@ func NewCmdRegistryDelete() *cobra.Command {
 		Use:               "delete (NAME | --all)",
 		Short:             "Delete registry/registries.",
 		Long:              `Delete registry/registries.`,
+		Aliases:           []string{"del", "rm"},
 		ValidArgsFunction: util.ValidArgsAvailableRegistries,
 		Run: func(cmd *cobra.Command, args []string) {
 
 			nodes := parseRegistryDeleteCmd(cmd, args, &flags)
 
 			if len(nodes) == 0 {
-				log.Infoln("No registries found")
+				l.Log().Infoln("No registries found")
 			} else {
 				for _, node := range nodes {
 					if err := client.NodeDelete(cmd.Context(), runtimes.SelectedRuntime, node, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 				}
 			}
@@ -79,18 +80,18 @@ func parseRegistryDeleteCmd(cmd *cobra.Command, args []string, flags *registryDe
 	if flags.All {
 		nodes, err = client.NodeList(cmd.Context(), runtimes.SelectedRuntime)
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 	}
 
 	if !flags.All && len(args) < 1 {
-		log.Fatalln("Expecting at least one registry name if `--all` is not set")
+		l.Log().Fatalln("Expecting at least one registry name if `--all` is not set")
 	}
 
 	for _, name := range args {
 		node, err := client.NodeGet(cmd.Context(), runtimes.SelectedRuntime, &k3d.Node{Name: name})
 		if err != nil {
-			log.Fatalln(err)
+			l.Log().Fatalln(err)
 		}
 		nodes = append(nodes, node)
 	}

cmd/registry/registryList.go

@@ -26,11 +26,11 @@ import (
 	"strings"
 
 	"github.com/liggitt/tabwriter"
-	"github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/util"
+	"github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 )
 
@@ -64,15 +64,15 @@ func NewCmdRegistryList() *cobra.Command {
 			if len(nodes) == 0 { // Option a)  no name specified -> get all registries
 				found, err := client.NodeList(cmd.Context(), runtimes.SelectedRuntime)
 				if err != nil {
-					log.Fatalln(err)
+					l.Log().Fatalln(err)
 				}
 				existingNodes = append(existingNodes, found...)
 			} else { // Option b) registry name(s) specified -> get specific registries
 				for _, node := range nodes {
-					log.Tracef("Node %s", node.Name)
+					l.Log().Tracef("Node %s", node.Name)
 					found, err := client.NodeGet(cmd.Context(), runtimes.SelectedRuntime, node)
 					if err != nil {
-						log.Fatalln(err)
+						l.Log().Fatalln(err)
 					}
 					existingNodes = append(existingNodes, found)
 				}

cmd/root.go

@@ -25,24 +25,24 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
-	"github.com/rancher/k3d/v4/cmd/cluster"
-	cfg "github.com/rancher/k3d/v4/cmd/config"
-	"github.com/rancher/k3d/v4/cmd/debug"
-	"github.com/rancher/k3d/v4/cmd/image"
-	"github.com/rancher/k3d/v4/cmd/kubeconfig"
-	"github.com/rancher/k3d/v4/cmd/node"
-	"github.com/rancher/k3d/v4/cmd/registry"
-	cliutil "github.com/rancher/k3d/v4/cmd/util"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	"github.com/rancher/k3d/v4/version"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/cmd/cluster"
+	cfg "github.com/rancher/k3d/v5/cmd/config"
+	"github.com/rancher/k3d/v5/cmd/debug"
+	"github.com/rancher/k3d/v5/cmd/image"
+	"github.com/rancher/k3d/v5/cmd/kubeconfig"
+	"github.com/rancher/k3d/v5/cmd/node"
+	"github.com/rancher/k3d/v5/cmd/registry"
+	cliutil "github.com/rancher/k3d/v5/cmd/util"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	"github.com/rancher/k3d/v5/version"
+	"github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/writer"
 )
 
@@ -56,51 +56,26 @@ type RootFlags struct {
 
 var flags = RootFlags{}
 
-// rootCmd represents the base command when called without any subcommands
-var rootCmd = &cobra.Command{
-	Use:   "k3d",
-	Short: "https://k3d.io/ -> Run k3s in Docker!",
-	Long: `https://k3d.io/
+func NewCmdK3d() *cobra.Command {
+
+	// rootCmd represents the base command when called without any subcommands
+	rootCmd := &cobra.Command{
+		Use:   "k3d",
+		Short: "https://k3d.io/ -> Run k3s in Docker!",
+		Long: `https://k3d.io/
 k3d is a wrapper CLI that helps you to easily create k3s clusters inside docker.
 Nodes of a k3d cluster are docker containers running a k3s image.
 All Nodes of a k3d cluster are part of the same docker network.`,
-	Run: func(cmd *cobra.Command, args []string) {
-		if flags.version {
-			printVersion()
-		} else {
-			if err := cmd.Usage(); err != nil {
-				log.Fatalln(err)
+		Run: func(cmd *cobra.Command, args []string) {
+			if flags.version {
+				printVersion()
+			} else {
+				if err := cmd.Usage(); err != nil {
+					l.Log().Fatalln(err)
+				}
 			}
-		}
-	},
-}
-
-// Execute adds all child commands to the root command and sets flags appropriately.
-// This is called by main.main(). It only needs to happen once to the rootCmd.
-func Execute() {
-	if len(os.Args) > 1 {
-		parts := os.Args[1:]
-		// Check if it's a built-in command, else try to execute it as a plugin
-		if _, _, err := rootCmd.Find(parts); err != nil {
-			pluginFound, err := cliutil.HandlePlugin(context.Background(), parts)
-			if err != nil {
-				log.Errorf("Failed to execute plugin '%+v'", parts)
-				log.Fatalln(err)
-			} else if pluginFound {
-				os.Exit(0)
-			}
-		}
+		},
 	}
-	if err := rootCmd.Execute(); err != nil {
-		log.Fatalln(err)
-	}
-}
-
-func GetRootCmd() *cobra.Command {
-	return rootCmd
-}
-
-func init() {
 
 	rootCmd.PersistentFlags().BoolVar(&flags.debugLogging, "verbose", false, "Enable verbose output (debug logging)")
 	rootCmd.PersistentFlags().BoolVar(&flags.traceLogging, "trace", false, "Enable super verbose output (trace logging)")
@@ -110,85 +85,107 @@ func init() {
 	rootCmd.Flags().BoolVar(&flags.version, "version", false, "Show k3d and default k3s version")
 
 	// add subcommands
-	rootCmd.AddCommand(NewCmdCompletion())
-	rootCmd.AddCommand(cluster.NewCmdCluster())
-	rootCmd.AddCommand(kubeconfig.NewCmdKubeconfig())
-	rootCmd.AddCommand(node.NewCmdNode())
-	rootCmd.AddCommand(image.NewCmdImage())
-	rootCmd.AddCommand(cfg.NewCmdConfig())
-	rootCmd.AddCommand(registry.NewCmdRegistry())
-	rootCmd.AddCommand(debug.NewCmdDebug())
-
-	rootCmd.AddCommand(&cobra.Command{
-		Use:   "version",
-		Short: "Show k3d and default k3s version",
-		Long:  "Show k3d and default k3s version",
-		Run: func(cmd *cobra.Command, args []string) {
-			printVersion()
+	rootCmd.AddCommand(NewCmdCompletion(rootCmd),
+		cluster.NewCmdCluster(),
+		kubeconfig.NewCmdKubeconfig(),
+		node.NewCmdNode(),
+		image.NewCmdImage(),
+		cfg.NewCmdConfig(),
+		registry.NewCmdRegistry(),
+		debug.NewCmdDebug(),
+		&cobra.Command{
+			Use:   "version",
+			Short: "Show k3d and default k3s version",
+			Long:  "Show k3d and default k3s version",
+			Run: func(cmd *cobra.Command, args []string) {
+				printVersion()
+			},
 		},
-	})
-
-	rootCmd.AddCommand(&cobra.Command{
-		Use:   "runtime-info",
-		Short: "Show runtime information",
-		Long:  "Show some information about the runtime environment (e.g. docker info)",
-		Run: func(cmd *cobra.Command, args []string) {
-			info, err := runtimes.SelectedRuntime.Info()
-			if err != nil {
-				log.Fatalln(err)
-			}
-			err = yaml.NewEncoder(os.Stdout).Encode(info)
-			if err != nil {
-				log.Fatalln(err)
-			}
-		},
-		Hidden: true,
-	})
+		&cobra.Command{
+			Use:   "runtime-info",
+			Short: "Show runtime information",
+			Long:  "Show some information about the runtime environment (e.g. docker info)",
+			Run: func(cmd *cobra.Command, args []string) {
+				info, err := runtimes.SelectedRuntime.Info()
+				if err != nil {
+					l.Log().Fatalln(err)
+				}
+				err = yaml.NewEncoder(os.Stdout).Encode(info)
+				if err != nil {
+					l.Log().Fatalln(err)
+				}
+			},
+			Hidden: true,
+		})
 
 	// Init
 	cobra.OnInitialize(initLogging, initRuntime)
+
+	return rootCmd
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	cmd := NewCmdK3d()
+	if len(os.Args) > 1 {
+		parts := os.Args[1:]
+		// Check if it's a built-in command, else try to execute it as a plugin
+		if _, _, err := cmd.Find(parts); err != nil {
+			pluginFound, err := cliutil.HandlePlugin(context.Background(), parts)
+			if err != nil {
+				l.Log().Errorf("Failed to execute plugin '%+v'", parts)
+				l.Log().Fatalln(err)
+			} else if pluginFound {
+				os.Exit(0)
+			}
+		}
+	}
+	if err := cmd.Execute(); err != nil {
+		l.Log().Fatalln(err)
+	}
 }
 
 // initLogging initializes the logger
 func initLogging() {
 	if flags.traceLogging {
-		log.SetLevel(log.TraceLevel)
+		l.Log().SetLevel(logrus.TraceLevel)
 	} else if flags.debugLogging {
-		log.SetLevel(log.DebugLevel)
+		l.Log().SetLevel(logrus.DebugLevel)
 	} else {
 		switch logLevel := strings.ToUpper(os.Getenv("LOG_LEVEL")); logLevel {
 		case "TRACE":
-			log.SetLevel(log.TraceLevel)
+			l.Log().SetLevel(logrus.TraceLevel)
 		case "DEBUG":
-			log.SetLevel(log.DebugLevel)
+			l.Log().SetLevel(logrus.DebugLevel)
 		case "WARN":
-			log.SetLevel(log.WarnLevel)
+			l.Log().SetLevel(logrus.WarnLevel)
 		case "ERROR":
-			log.SetLevel(log.ErrorLevel)
+			l.Log().SetLevel(logrus.ErrorLevel)
 		default:
-			log.SetLevel(log.InfoLevel)
+			l.Log().SetLevel(logrus.InfoLevel)
 		}
 	}
-	log.SetOutput(ioutil.Discard)
-	log.AddHook(&writer.Hook{
+	l.Log().SetOutput(io.Discard)
+	l.Log().AddHook(&writer.Hook{
 		Writer: os.Stderr,
-		LogLevels: []log.Level{
-			log.PanicLevel,
-			log.FatalLevel,
-			log.ErrorLevel,
-			log.WarnLevel,
+		LogLevels: []logrus.Level{
+			logrus.PanicLevel,
+			logrus.FatalLevel,
+			logrus.ErrorLevel,
+			logrus.WarnLevel,
 		},
 	})
-	log.AddHook(&writer.Hook{
+	l.Log().AddHook(&writer.Hook{
 		Writer: os.Stdout,
-		LogLevels: []log.Level{
-			log.InfoLevel,
-			log.DebugLevel,
-			log.TraceLevel,
+		LogLevels: []logrus.Level{
+			logrus.InfoLevel,
+			logrus.DebugLevel,
+			logrus.TraceLevel,
 		},
 	})
 
-	formatter := &log.TextFormatter{
+	formatter := &logrus.TextFormatter{
 		ForceColors: true,
 	}
 
@@ -196,18 +193,18 @@ func initLogging() {
 		formatter.FullTimestamp = true
 	}
 
-	log.SetFormatter(formatter)
+	l.Log().SetFormatter(formatter)
 
 }
 
 func initRuntime() {
 	runtime, err := runtimes.GetRuntime("docker")
 	if err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	}
 	runtimes.SelectedRuntime = runtime
 	if rtinfo, err := runtime.Info(); err == nil {
-		log.Debugf("Runtime Info:\n%+v", rtinfo)
+		l.Log().Debugf("Runtime Info:\n%+v", rtinfo)
 	}
 }
 
@@ -216,43 +213,82 @@ func printVersion() {
 	fmt.Printf("k3s version %s (default)\n", version.K3sVersion)
 }
 
-func generateFishCompletion(writer io.Writer) error {
-	return rootCmd.GenFishCompletion(writer, true)
-}
-
-// Completion
-var completionFunctions = map[string]func(io.Writer) error{
-	"bash": rootCmd.GenBashCompletion,
-	"zsh": func(writer io.Writer) error {
-		if err := rootCmd.GenZshCompletion(writer); err != nil {
-			return err
-		}
-
-		fmt.Fprintf(writer, "\n# source completion file\ncompdef _k3d k3d\n")
-
-		return nil
-	},
-	"psh":        rootCmd.GenPowerShellCompletion,
-	"powershell": rootCmd.GenPowerShellCompletion,
-	"fish":       generateFishCompletion,
-}
-
 // NewCmdCompletion creates a new completion command
-func NewCmdCompletion() *cobra.Command {
+func NewCmdCompletion(rootCmd *cobra.Command) *cobra.Command {
+
+	completionFunctions := map[string]func(io.Writer) error{
+		"bash": rootCmd.GenBashCompletion,
+		"zsh": func(writer io.Writer) error {
+			if err := rootCmd.GenZshCompletion(writer); err != nil {
+				return err
+			}
+
+			fmt.Fprintf(writer, "\n# source completion file\ncompdef _k3d k3d\n")
+
+			return nil
+		},
+		"psh":        rootCmd.GenPowerShellCompletion,
+		"powershell": rootCmd.GenPowerShellCompletionWithDesc,
+		"fish": func(writer io.Writer) error {
+			return rootCmd.GenFishCompletion(writer, true)
+		},
+	}
+
 	// create new cobra command
 	cmd := &cobra.Command{
 		Use:   "completion SHELL",
 		Short: "Generate completion scripts for [bash, zsh, fish, powershell | psh]",
-		Long:  `Generate completion scripts for [bash, zsh, fish, powershell | psh]`,
-		Args:  cobra.ExactArgs(1), // TODO: NewCmdCompletion: add support for 0 args = auto detection
+		Long: `To load completions:
+
+Bash:
+
+	$ source <(k3d completion bash)
+
+	# To load completions for each session, execute once:
+	# Linux:
+	$ k3d completion bash > /etc/bash_completion.d/k3d
+	# macOS:
+	$ k3d completion bash > /usr/local/etc/bash_completion.d/k3d
+
+Zsh:
+
+	# If shell completion is not already enabled in your environment,
+	# you will need to enable it.  You can execute the following once:
+
+	$ echo "autoload -U compinit; compinit" >> ~/.zshrc
+
+	# To load completions for each session, execute once:
+	$ k3d completion zsh > "${fpath[1]}/k3d"
+
+	# You will need to start a new shell for this setup to take effect.
+
+fish:
+
+	$ k3d completion fish | source
+
+	# To load completions for each session, execute once:
+	$ k3d completion fish > ~/.config/fish/completions/k3d.fish
+
+PowerShell:
+
+	PS> k3d completion powershell | Out-String | Invoke-Expression
+
+	# To load completions for every new session, run:
+	PS> k3d completion powershell > k3d.ps1
+	# and source this file from your PowerShell profile.
+`,
+		ValidArgs:             []string{"bash", "zsh", "fish", "powershell"},
+		ArgAliases:            []string{"psh"},
+		DisableFlagsInUseLine: true,
+		Args:                  cobra.ExactValidArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			if completionFunc, ok := completionFunctions[args[0]]; ok {
 				if err := completionFunc(os.Stdout); err != nil {
-					log.Fatalf("Failed to generate completion script for shell '%s'", args[0])
+					l.Log().Fatalf("Failed to generate completion script for shell '%s'", args[0])
 				}
 				return
 			}
-			log.Fatalf("Shell '%s' not supported for completion", args[0])
+			l.Log().Fatalf("Shell '%s' not supported for completion", args[0])
 		},
 	}
 	return cmd

cmd/util/completion.go

@@ -25,10 +25,10 @@ import (
 	"context"
 	"strings"
 
-	k3dcluster "github.com/rancher/k3d/v4/pkg/client"
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	k3dcluster "github.com/rancher/k3d/v5/pkg/client"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"github.com/spf13/cobra"
 )
 
@@ -39,7 +39,7 @@ func ValidArgsAvailableClusters(cmd *cobra.Command, args []string, toComplete st
 	var clusters []*k3d.Cluster
 	clusters, err := k3dcluster.ClusterList(context.Background(), runtimes.SelectedRuntime)
 	if err != nil {
-		log.Errorln("Failed to get list of clusters for shell completion")
+		l.Log().Errorln("Failed to get list of clusters for shell completion")
 		return nil, cobra.ShellCompDirectiveError
 	}
 
@@ -64,7 +64,7 @@ func ValidArgsAvailableNodes(cmd *cobra.Command, args []string, toComplete strin
 	var nodes []*k3d.Node
 	nodes, err := k3dcluster.NodeList(context.Background(), runtimes.SelectedRuntime)
 	if err != nil {
-		log.Errorln("Failed to get list of nodes for shell completion")
+		l.Log().Errorln("Failed to get list of nodes for shell completion")
 		return nil, cobra.ShellCompDirectiveError
 	}
 
@@ -89,7 +89,7 @@ func ValidArgsAvailableRegistries(cmd *cobra.Command, args []string, toComplete
 	var nodes []*k3d.Node
 	nodes, err := k3dcluster.NodeList(context.Background(), runtimes.SelectedRuntime)
 	if err != nil {
-		log.Errorln("Failed to get list of nodes for shell completion")
+		l.Log().Errorln("Failed to get list of nodes for shell completion")
 		return nil, cobra.ShellCompDirectiveError
 	}
 

cmd/util/config/config.go

@@ -0,0 +1,97 @@
+/*
+Copyright © 2020-2021 The k3d Author(s)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+*/
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/rancher/k3d/v5/pkg/config"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"gopkg.in/yaml.v2"
+)
+
+func InitViperWithConfigFile(cfgViper *viper.Viper, configFile string) error {
+
+	// viper for the general config (file, env and non pre-processed flags)
+	cfgViper.SetEnvPrefix("K3D")
+	cfgViper.AutomaticEnv()
+
+	cfgViper.SetConfigType("yaml")
+
+	// Set config file, if specified
+	if configFile != "" {
+
+		if _, err := os.Stat(configFile); err != nil {
+			l.Log().Fatalf("Failed to stat config file %s: %+v", configFile, err)
+		}
+
+		// create temporary file to expand environment variables in the config without writing that back to the original file
+		// we're doing it here, because this happens just before absolutely all other processing
+		tmpfile, err := os.CreateTemp(os.TempDir(), fmt.Sprintf("k3d-config-tmp-%s", filepath.Base(configFile)))
+		if err != nil {
+			l.Log().Fatalf("error creating temp copy of configfile %s for variable expansion: %v", configFile, err)
+		}
+		defer tmpfile.Close()
+
+		originalcontent, err := os.ReadFile(configFile)
+		if err != nil {
+			l.Log().Fatalf("error reading config file %s: %v", configFile, err)
+		}
+		expandedcontent := os.ExpandEnv(string(originalcontent))
+		if _, err := tmpfile.WriteString(expandedcontent); err != nil {
+			l.Log().Fatalf("error writing expanded config file contents to temp file %s: %v", tmpfile.Name(), err)
+		}
+
+		// use temp file with expanded variables
+		cfgViper.SetConfigFile(tmpfile.Name())
+
+		// try to read config into memory (viper map structure)
+		if err := cfgViper.ReadInConfig(); err != nil {
+			if _, ok := err.(viper.ConfigFileNotFoundError); ok {
+				l.Log().Fatalf("Config file %s not found: %+v", configFile, err)
+			}
+			// config file found but some other error happened
+			l.Log().Fatalf("Failed to read config file %s: %+v", configFile, err)
+		}
+
+		schema, err := config.GetSchemaByVersion(cfgViper.GetString("apiVersion"))
+		if err != nil {
+			l.Log().Fatalf("Cannot validate config file %s: %+v", configFile, err)
+		}
+
+		if err := config.ValidateSchemaFile(tmpfile.Name(), schema); err != nil {
+			l.Log().Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
+		}
+
+		l.Log().Infof("Using config file %s (%s#%s)", configFile, strings.ToLower(cfgViper.GetString("apiVersion")), strings.ToLower(cfgViper.GetString("kind")))
+	}
+	if l.Log().GetLevel() >= logrus.DebugLevel {
+		c, _ := yaml.Marshal(cfgViper.AllSettings())
+		l.Log().Debugf("Configuration:\n%s", c)
+	}
+	return nil
+}

cmd/util/filter.go

@@ -25,7 +25,7 @@ import (
 	"fmt"
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 )
 
 // SplitFiltersFromFlag separates a flag's value from the node filter, if there is one
@@ -50,10 +50,10 @@ func SplitFiltersFromFlag(flag string) (string, []string, error) {
 			// Case 1.1: Escaped backslash
 			if strings.HasSuffix(it, "\\\\") {
 				it = strings.TrimSuffix(it, "\\")
-				log.Warnf("The part '%s' of the flag input '%s' ends with a double backslash, so we assume you want to escape the backslash before the '@'. That's the only time we do this.", it, flag)
+				l.Log().Warnf("The part '%s' of the flag input '%s' ends with a double backslash, so we assume you want to escape the backslash before the '@'. That's the only time we do this.", it, flag)
 			} else {
 				// Case 1.2: Unescaped backslash -> Escaping the '@' -> remove suffix and append it to buffer, followed by the escaped @ sign
-				log.Tracef("Item '%s' just before an '@' ends with '\\', so we assume it's escaping a literal '@'", it)
+				l.Log().Tracef("Item '%s' just before an '@' ends with '\\', so we assume it's escaping a literal '@'", it)
 				buffer += strings.TrimSuffix(it, "\\") + "@"
 				continue
 			}

cmd/util/listings.go

@@ -29,8 +29,8 @@ import (
 	"strings"
 
 	"github.com/liggitt/tabwriter"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 	"gopkg.in/yaml.v2"
 )
 
@@ -55,7 +55,7 @@ func PrintNodes(nodes []*k3d.Node, outputFormat string, headers *[]string, nodeP
 		if headers != nil {
 			_, err := fmt.Fprintf(tabwriter, "%s\n", strings.Join(*headers, "\t"))
 			if err != nil {
-				log.Fatalln("Failed to print headers")
+				l.Log().Fatalln("Failed to print headers")
 			}
 		}
 	}

cmd/util/plugins.go

@@ -28,7 +28,7 @@ import (
 	"os/exec"
 	"strings"
 
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
 )
 
 // HandlePlugin takes care of finding and executing a plugin based on the longest prefix

cmd/util/ports.go

@@ -28,9 +28,9 @@ import (
 	"strconv"
 
 	"github.com/docker/go-connections/nat"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	"github.com/rancher/k3d/v4/pkg/util"
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+	"github.com/rancher/k3d/v5/pkg/util"
 )
 
 var apiPortRegexp = regexp.MustCompile(`^(?P<hostref>(?P<hostip>\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})|(?P<hostname>\S+):)?(?P<port>(\d{1,5}|random))$`)
@@ -55,7 +55,7 @@ func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureO
 
 	// check if there's a host reference
 	if submatches["hostname"] != "" {
-		log.Tracef("Port Exposure: found hostname: %s", submatches["hostname"])
+		l.Log().Tracef("Port Exposure: found hostname: %s", submatches["hostname"])
 		addrs, err := net.LookupHost(submatches["hostname"])
 		if err != nil {
 			return nil, fmt.Errorf("Failed to lookup host '%s' specified for Port Exposure: %+v", submatches["hostname"], err)
@@ -77,15 +77,15 @@ func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureO
 
 	// port: get a free one if there's none defined or set to random
 	if submatches["port"] == "" || submatches["port"] == "random" {
-		log.Debugf("Port Exposure Mapping didn't specify hostPort, choosing one randomly...")
+		l.Log().Debugf("Port Exposure Mapping didn't specify hostPort, choosing one randomly...")
 		freePort, err := GetFreePort()
 		if err != nil || freePort == 0 {
-			log.Warnf("Failed to get random free port: %+v", err)
-			log.Warnf("Falling back to internal port %s (may be blocked though)...", internalPort)
+			l.Log().Warnf("Failed to get random free port: %+v", err)
+			l.Log().Warnf("Falling back to internal port %s (may be blocked though)...", internalPort)
 			submatches["port"] = internalPort
 		} else {
 			submatches["port"] = strconv.Itoa(freePort)
-			log.Debugf("Got free port for Port Exposure: '%d'", freePort)
+			l.Log().Debugf("Got free port for Port Exposure: '%d'", freePort)
 		}
 	}
 
@@ -93,7 +93,7 @@ func ParsePortExposureSpec(exposedPortSpec, internalPort string) (*k3d.ExposureO
 
 	portMapping, err := nat.ParsePortSpec(realPortString)
 	if err != nil {
-		return nil, fmt.Errorf("Failed to parse port spec for Port Exposure '%s': %+v", realPortString, err)
+		return nil, fmt.Errorf("failed to parse port spec for Port Exposure '%s': %+v", realPortString, err)
 	}
 
 	api.Port = portMapping[0].Port // there can be only one due to our regexp
@@ -112,14 +112,12 @@ func ValidatePortMap(portmap string) (string, error) {
 func GetFreePort() (int, error) {
 	tcpAddress, err := net.ResolveTCPAddr("tcp", "localhost:0")
 	if err != nil {
-		log.Errorln("Failed to resolve address")
-		return 0, err
+		return 0, fmt.Errorf("failed to resolve address 'localhost:0': %w", err)
 	}
 
 	tcpListener, err := net.ListenTCP("tcp", tcpAddress)
 	if err != nil {
-		log.Errorln("Failed to create TCP Listener")
-		return 0, err
+		return 0, fmt.Errorf("failed to create tcp listener: %w", err)
 	}
 	defer tcpListener.Close()
 

cmd/util/runtimeLabels.go

@@ -24,12 +24,12 @@ package util
 import (
 	"strings"
 
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 )
 
 // validateRuntimeLabelKey validates a given label key is not reserved for internal k3d usage
 func ValidateRuntimeLabelKey(labelKey string) {
 	if strings.HasPrefix(labelKey, "k3s.") || strings.HasPrefix(labelKey, "k3d.") || labelKey == "app" {
-		log.Fatalf("runtime label \"%s\" is reserved for internal usage", labelKey)
+		l.Log().Fatalf("runtime label \"%s\" is reserved for internal usage", labelKey)
 	}
 }

cmd/util/volumes.go

@@ -27,9 +27,9 @@ import (
 	rt "runtime"
 	"strings"
 
-	"github.com/rancher/k3d/v4/pkg/runtimes"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
 
-	log "github.com/sirupsen/logrus"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 )
 
 // ValidateVolumeMount checks, if the source of volume mounts exists and if the destination is an absolute path
@@ -81,7 +81,7 @@ func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) (string,
 		}
 		if !isNamedVolume {
 			if _, err := os.Stat(src); err != nil {
-				log.Warnf("Failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount)
+				l.Log().Warnf("Failed to stat file/directory/named volume that you're trying to mount: '%s' in '%s' -> Please make sure it exists", src, volumeMount)
 			}
 		}
 	}
@@ -98,7 +98,7 @@ func ValidateVolumeMount(runtime runtimes.Runtime, volumeMount string) (string,
 func verifyNamedVolume(runtime runtimes.Runtime, volumeName string) error {
 	volumeName, err := runtime.GetVolume(volumeName)
 	if err != nil {
-		return err
+		return fmt.Errorf("Failed to verify named volume: %w", err)
 	}
 	if volumeName == "" {
 		return fmt.Errorf("Failed to find named volume '%s'", volumeName)

dind-manifest.tmpl

@@ -0,0 +1,27 @@
+image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}-dind
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}dind-linux-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}dind-linux-arm64
+    platform:
+      variant: v8
+      architecture: arm64
+      os: linux
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}dind-linux-arm
+    platform:
+      variant: v7
+      architecture: arm
+      os: linux
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}dind-linux-arm
+    platform:
+      variant: v6
+      architecture: arm
+      os: linux

docgen/README.md

@@ -6,12 +6,10 @@ The code will output files in [`../docs/usage/commands/`](../docs/usage/commands
 
 ## Run
 
-- may required a `replace github.com/rancher/k3d/v4 => PATH/TO/LOCAL/REPO` in the `go.mod`
-
 ```bash
 # ensure that you're in the docgen dir, as the relative path to the docs/ dir is hardcoded
 cd docgen
 
 # run
-go run ./main.go
+./run.sh
 ```

docgen/go.mod

@@ -3,17 +3,11 @@ module github.com/rancher/k3d/docgen
 go 1.16
 
 require (
-	github.com/Microsoft/go-winio v0.4.17 // indirect
-	github.com/containerd/cgroups v0.0.0-20210414074453-680c246289fb // indirect
-	github.com/containerd/containerd v1.5.0-rc.1 // indirect
-	github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5 // indirect
-	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/rancher/k3d/v4 v4.4.7-0.20210709062205-c5f7884f7870
-	github.com/spf13/cobra v1.1.3
-	golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 // indirect
+	github.com/rancher/k3d/v5 v5.0.0-00010101000000-000000000000
+	github.com/spf13/cobra v1.2.1
 	golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78 // indirect
 	golang.org/x/term v0.0.0-20210406210042-72f3dc4e9b72 // indirect
-	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.1 // indirect
 )
+
+replace github.com/rancher/k3d/v5 => /PATH/TO/YOUR/REPO/DIRECTORY

docgen/go.sum

@@ -44,13 +44,15 @@ github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX
 github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw=
-github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=
+github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA=
 github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
 github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A=
+github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k=
 github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
+github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8=
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -69,8 +71,9 @@ github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEY
 github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
 github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
-github.com/Microsoft/hcsshim v0.8.16 h1:8/auA4LFIZFTGrqfKhGBSXwM6/4X1fHa/xniyEHu8ac=
 github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
+github.com/Microsoft/hcsshim v0.8.18 h1:cYnKADiM1869gvBpos3YCteeT6sZLB48lB5dmMMs8Tg=
+github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
 github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
 github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
 github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@@ -78,6 +81,8 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/OpenPeeDeeP/depguard v1.0.0/go.mod h1:7/4sitnI9YlQgTLLk734QlzXT8DuHVnAyztLplQjk+o=
 github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
 github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/Shopify/logrus-bugsnag v0.0.0-20170309145241-6dbc35f2c30d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
+github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs=
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -92,13 +97,16 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
+github.com/beorn7/perks v0.0.0-20150223135152-b965b613227f/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw=
 github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
-github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA=
 github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM=
 github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
@@ -106,12 +114,19 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dR
 github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
 github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0 h1:s7+5BfS4WFJoVF9pnB8kBk03S7pZXRdKamnV0FOl5Sc=
+github.com/bugsnag/bugsnag-go v1.0.5-0.20150529004307-13fd6b8acda0/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8=
+github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b h1:otBG+dV+YK+Soembjv71DPz3uX/V/6MMlSyD9JBQ6kQ=
 github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50=
+github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0 h1:nvj0OLI3YqYXer/kZD8Ri1aaunCxIEsOst1BVJswV0o=
 github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
+github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -119,7 +134,10 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE
 github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
 github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
+github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004 h1:lkAMpLVBDaj17e85keuznYcH5rqI438v41pKcBl4ZxQ=
+github.com/cloudflare/cfssl v0.0.0-20180223231731-4e2dcbde5004/go.mod h1:yMWuSON2oQp+43nFtAV/uvKQIFpSPerB57DCt9t8sSA=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
 github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
@@ -127,16 +145,18 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z
 github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE=
 github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU=
 github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
+github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU=
 github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E=
 github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
+github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss=
 github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI=
 github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
 github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM=
 github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo=
 github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
-github.com/containerd/cgroups v0.0.0-20210414074453-680c246289fb h1:cq9suWES/pQHVg1N4u8ltT30HWScFmcAz4sB/wJyp/I=
-github.com/containerd/cgroups v0.0.0-20210414074453-680c246289fb/go.mod h1:sgGgnAnNasYdJ1ypnikP2SO7SM0Lfgkgwk3TUc9bDO4=
+github.com/containerd/cgroups v1.0.1 h1:iJnMvco9XGvKUvNQkv88bE4uJXxRQH18efbKo9w5vHQ=
+github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU=
 github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
 github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
@@ -150,37 +170,41 @@ github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX
 github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
-github.com/containerd/containerd v1.4.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ=
 github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU=
 github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
 github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
-github.com/containerd/containerd v1.5.0-rc.1 h1:7n80DQm69wXXjLGQo8sytMPC9Z+kG6B4s95hfbFLiXQ=
-github.com/containerd/containerd v1.5.0-rc.1/go.mod h1:kAwhYasTYKvQWPnWf8CoRDu3vikb17YocPLvHMQhBn4=
+github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
+github.com/containerd/containerd v1.5.5 h1:q1gxsZsGZ8ddVe98yO6pR21b5xQSMiR61lD0W96pgQo=
+github.com/containerd/containerd v1.5.5/go.mod h1:oSTh0QpT1w6jYcGmbiSbxv9OSQYaa88mPyWIuU79zyo=
 github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo=
 github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y=
 github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ=
-github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5 h1:k6Dn7shF+i1q4utvCyW4+o9REsCMAeRyORM5IhXMCnw=
-github.com/containerd/continuity v0.0.0-20210315143101-93e15499afd5/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
+github.com/containerd/continuity v0.1.0 h1:UFRRY5JemiAhPZrr/uE0n8fMTLcZsUvySPr1+D7pgr8=
+github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM=
 github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
 github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0=
 github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
+github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4=
 github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU=
+github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk=
 github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
 github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g=
 github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
+github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
 github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0=
 github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA=
 github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow=
-github.com/containerd/imgcrypt v1.1.1-0.20210412181126-0bed51b9522c/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
+github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms=
 github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c=
 github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
+github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY=
 github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
 github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8=
@@ -189,10 +213,12 @@ github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8h
 github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
 github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk=
 github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg=
+github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
 github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw=
 github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y=
 github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
 github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
+github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY=
 github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
 github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
@@ -203,7 +229,6 @@ github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgU
 github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
@@ -222,34 +247,42 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
 github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
 github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
 github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
 github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
+github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.0.0-20191128021309-1d7a30a10f73/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
 github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
 github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E=
-github.com/docker/cli v20.10.6+incompatible h1:LAyI6Lnwv+AUjtp2ZyN1lxqXBtkeFUqm4H7CZMWZuP8=
-github.com/docker/cli v20.10.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.8+incompatible h1:/zO/6y9IOpcehE49yMRTV9ea0nBpb8OeqSskXLNfH1E=
+github.com/docker/cli v20.10.8+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20171011171712-7484e51bf6af/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
-github.com/docker/docker v20.10.6+incompatible h1:oXI3Vas8TI8Eu/EjH4srKHJBVqraSzJybhxY7Om9faQ=
-github.com/docker/docker v20.10.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v20.10.8+incompatible h1:RVqD337BgQicVCzYrrlhLDWhq6OAD2PJDUg2LsEUvKM=
+github.com/docker/docker v20.10.8+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o=
+github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c=
+github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c h1:lzqkGL9b3znc+ZUgi7FlLnqjQhcXxkNM/quxIjBVMD0=
+github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c/go.mod h1:CADgU4DSXK5QUlFslkQu2yW2TKzFZcXq/leZfM0UH5Q=
 github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ=
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
+github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8=
 github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
@@ -260,6 +293,7 @@ github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZ
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
 github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/dvsekhvalnov/jose2go v0.0.0-20170216131308-f21a8cedbbae/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM=
 github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
@@ -271,15 +305,20 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y
 github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.6.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
+github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA=
+github.com/fvbommel/sortorder v1.0.2 h1:mV4o8B2hKboCdkJm+a7uX/SIpZob4JzUpc5GGnM45eo=
+github.com/fvbommel/sortorder v1.0.2/go.mod h1:uk88iVf1ovNn1iLfgUVU2F9o5eO30ui720w+kxuqRs0=
 github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -305,9 +344,11 @@ github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL9
 github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo=
 github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
 github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk=
+github.com/go-sql-driver/mysql v1.3.0 h1:pgwjLi/dvffoP9aabwkT3AKpXQM93QARkjFhDDqC1UE=
+github.com/go-sql-driver/mysql v1.3.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
-github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M=
+github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
 github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
@@ -329,6 +370,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
+github.com/gogo/protobuf v1.0.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
 github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
@@ -336,6 +378,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
 github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -390,6 +433,9 @@ github.com/golangci/revgrep v0.0.0-20180526074752-d9c87f5ffaf0/go.mod h1:qOQCunE
 github.com/golangci/unconvert v0.0.0-20180507085042-28b1c447d1f4/go.mod h1:Izgrg8RkN3rCIMLGE9CyYmU9pY2Jer6DgANEnZ/L/cQ=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93 h1:jc2UWq7CbdszqeH6qu1ougXMIUBfSy8Pbh/anURYbGI=
+github.com/google/certificate-transparency-go v1.0.10-0.20180222191210-5ab67e519c93/go.mod h1:QeJfpSbVSfYc7RgB3gJFj9cbuQMMchQxrWXz8Ruopmg=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -424,12 +470,16 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU=
+github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
 github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
 github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ=
+github.com/gorilla/mux v1.7.0/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
 github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw=
 github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs=
@@ -444,6 +494,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
+github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -482,6 +534,11 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
+github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8 h1:CZkYfurY6KGhVtlalI4QwQ6T0Cu6iuY3e0x5RLu96WE=
+github.com/jinzhu/gorm v0.0.0-20170222002820-5409931a1bb8/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
+github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d h1:jRQLvyVGL+iVtDElaEIDdKwpPqUIZJfzkNLV34htpEc=
+github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
+github.com/jinzhu/now v1.1.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
 github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -494,6 +551,7 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/loggo v0.0.0-20190526231331-6e530bcce5d8/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
@@ -519,12 +577,13 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v0.0.0-20150723085316-0dad96c0b94f/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0=
 github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/magiconair/properties v1.5.3/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.7.6/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
 github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
@@ -536,13 +595,19 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/mattn/go-sqlite3 v1.6.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
 github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/miekg/pkcs11 v1.0.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/pkcs11 v1.0.3 h1:iMwmD7I5225wv84WxIG/bmxz9AXjWvTWIbM/TYHvWtw=
 github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
+github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
@@ -550,12 +615,16 @@ github.com/mitchellh/go-ps v0.0.0-20170309133038-4fdf99ab2936/go.mod h1:r1VsdOzO
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20150613213606-2caf8efc9366/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v0.0.0-20180220230111-00c29f56e238/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag=
 github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A=
+github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
+github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
+github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc=
 github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM=
 github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM=
@@ -594,12 +663,16 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
 github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
@@ -614,16 +687,21 @@ github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5X
 github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
 github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
-github.com/opencontainers/runc v1.0.0-rc93 h1:x2UMpOOVf3kQ8arv/EsDGwim8PTNqzL1/EYDr/+scOM=
 github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
+github.com/opencontainers/runc v1.0.1 h1:G18PGckGdAm3yVQRWDVQ1rLSLntiniKJ0cNRT2Tm5gs=
+github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
 github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
 github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
 github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
+github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8=
+github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.1.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -642,21 +720,25 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA=
 github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.0-pre1.0.20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
@@ -668,11 +750,10 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI=
-github.com/rancher/k3d/v4 v4.4.7-0.20210709062205-c5f7884f7870 h1:G+QYoXAR02hyJiPv4GnxWFBI92/HkA65QRVi+SCNgmk=
-github.com/rancher/k3d/v4 v4.4.7-0.20210709062205-c5f7884f7870/go.mod h1:Cr4a6z5rTg/C+JwbT7OtWQedzswBlfRfLljYnbesoGE=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -714,32 +795,37 @@ github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/afero v1.6.0 h1:xoax2sJ2DT8S8xA2paPFjDCScCNeWsg75VG0DLRreiY=
 github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I=
+github.com/spf13/cast v0.0.0-20150508191742-4d07383ffe94/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.2.0/go.mod h1:r2rcYCSwa1IExKTDiTfzaxqT2FNHs8hODu4LnUfgKEg=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.1/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.2/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE=
-github.com/spf13/cobra v1.1.3 h1:xghbfqPkxzxP3C/f3n5DdpAbdKLj4ZE4BWQI362l53M=
-github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo=
+github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw=
+github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk=
+github.com/spf13/jwalterweatherman v0.0.0-20141219030609-3d60171a6431/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v0.0.0-20180109140146-7c0cea34c8ec/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk=
 github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.0/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v0.0.0-20150530192845-be5ff3e4840c/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.0.2/go.mod h1:A8kyI5cUJhb8N+3pkfONlcEcZbueH6nhAm0Fq7SrnBM=
 github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
-github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44=
 github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -758,6 +844,8 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG
 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
 github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
+github.com/theupdateframework/notary v0.7.0 h1:QyagRZ7wlSpjT5N2qQAh/pN+DVqgekv4DzbAiAiEL3c=
+github.com/theupdateframework/notary v0.7.0/go.mod h1:c9DRxcmhHmVLDay4/2fUYdISnHqbFDGRSlXPO0AhYWw=
 github.com/timakin/bodyclose v0.0.0-20190721030226-87058b9bfcec/go.mod h1:Qimiffbc6q9tBWlVV6x0P9sat/ao1xEkREYPPj9hphk=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
@@ -831,16 +919,20 @@ golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnf
 golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201117144127-c1f2f97bffc9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
 golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
+golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -913,6 +1005,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
@@ -924,12 +1017,11 @@ golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwY
 golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc=
 golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1 h1:4qWs8cYYH6PoEFy4dfhDFgoMGkwAcETd+MmPdCPMzUc=
-golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE=
+golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1007,6 +1099,7 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -1028,16 +1121,18 @@ golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1058,8 +1153,8 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE=
-golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs=
+golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20170915040203-e531a2a1c15f/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1192,6 +1287,7 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
+google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
@@ -1205,6 +1301,7 @@ google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaE
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0=
 google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0=
 google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.0.5/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1244,6 +1341,8 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/cenkalti/backoff.v2 v2.2.1 h1:eJ9UAg01/HIHG987TwxvnzK2MgxXq97YY6rYDpY9aII=
+gopkg.in/cenkalti/backoff.v2 v2.2.1/go.mod h1:S0QdOvT2AlerfSBkp0O+dk+bbIMaNbEmVk876gPCthU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -1256,11 +1355,12 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
 gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
-gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU=
 gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1 h1:d4KQkxAaAiRY2h5Zqis161Pv91A37uZyJOx73duwUwM=
+gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.1/go.mod h1:WbjuEoo1oadwzQ4apSDU+JTvmllEHtsNHS6y7vFc7iw=
 gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -1276,6 +1376,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
@@ -1290,38 +1391,44 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
-inet.af/netaddr v0.0.0-20210421205553-78c777480f22 h1:TX8hopxzHycFVkIsvu6DSpCWUCqDqOvyyPj/5IK1fUQ=
-inet.af/netaddr v0.0.0-20210421205553-78c777480f22/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
+inet.af/netaddr v0.0.0-20210903134321-85fa6c94624e h1:tvgqez5ZQoBBiBAGNU/fmJy247yB/7++kcLOEoMYup0=
+inet.af/netaddr v0.0.0-20210903134321-85fa6c94624e/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls=
 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
 k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ=
-k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y=
-k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU=
+k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8=
+k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY=
+k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY=
 k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
 k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=
-k8s.io/apimachinery v0.21.0 h1:3Fx+41if+IRavNcKOz09FwEXDBG6ORh6iMsTSelhkMA=
-k8s.io/apimachinery v0.21.0/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY=
+k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc=
+k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM=
+k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0=
 k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU=
 k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM=
+k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q=
 k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y=
 k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k=
-k8s.io/client-go v0.21.0 h1:n0zzzJsAQmJngpC0IhgFcApZyoGXPrDIAD601HD09ag=
-k8s.io/client-go v0.21.0/go.mod h1:nNBytTF9qPFDEhoqgEPaarobC8QPae13bElIVHzIglA=
+k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0=
+k8s.io/client-go v0.22.1 h1:jW0ZSHi8wW260FvcXHkIa0NLxFBQszTlhiAVsU5mopw=
+k8s.io/client-go v0.22.1/go.mod h1:BquC5A4UOo4qVDUtoc04/+Nxp1MeHcVc1HJm1KmG8kk=
 k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk=
 k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI=
+k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM=
 k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM=
 k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
 k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI=
+k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
 k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
-k8s.io/klog/v2 v2.8.0 h1:Q3gmuM9hKEjefWFFYF0Mat+YyFJvsUyYuwyNNJ5C9Ts=
-k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
+k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM=
+k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec=
 k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM=
-k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE=
+k8s.io/kube-openapi v0.0.0-20210421082810-95288971da7e/go.mod h1:vHXdDvt9+2spS2Rx9ql3I8tycm3H9FDfdUoIuKCefvw=
 k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk=
 k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
-k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 h1:u5rPykqiCpL+LBfjRkXvnK71gOgIdmq3eHUEkPrbeTI=
-k8s.io/utils v0.0.0-20210305010621-2afb4311ab10/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9 h1:imL9YgXQ9p7xmPzHFm/vVd/cF78jad+n4wK1ABwYtMM=
+k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
 mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIaQYRfC7CSLworTXY9RMqwhhCm+8Nc=
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20190209190245-fbb59629db34/go.mod h1:H6SUd1XjIs+qQCyskXg5OFSrilMRUkD8ePJpHKDPaeY=
@@ -1329,10 +1436,11 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
+sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.1 h1:nYqY2A6oy37sKLYuSBXuQhbj4JVclzJK13BOIvJG5XU=
-sigs.k8s.io/structured-merge-diff/v4 v4.1.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno=
+sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

docgen/main.go

@@ -1,17 +1,16 @@
 package main
 
 import (
-	"log"
-
-	"github.com/rancher/k3d/v4/cmd"
+	"github.com/rancher/k3d/v5/cmd"
+	l "github.com/rancher/k3d/v5/pkg/logger"
 	"github.com/spf13/cobra/doc"
 )
 
 func main() {
-	k3d := cmd.GetRootCmd()
+	k3d := cmd.NewCmdK3d()
 	k3d.DisableAutoGenTag = true
 
 	if err := doc.GenMarkdownTree(k3d, "../docs/usage/commands"); err != nil {
-		log.Fatalln(err)
+		l.Log().Fatalln(err)
 	}
 }

docgen/run.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+REPLACE_PLACEHOLDER="/PATH/TO/YOUR/REPO/DIRECTORY"
+
+CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+[ -d "$CURR_DIR" ] || { echo "FATAL: no current dir (maybe running in zsh?)";  exit 1; }
+
+REPO_DIR=${CURR_DIR%"/docgen"}
+
+echo "$REPO_DIR"
+
+sed -i "s%$REPLACE_PLACEHOLDER%$REPO_DIR%" "$CURR_DIR/go.mod"
+
+go mod tidy
+
+go mod vendor
+
+go run ./main.go
+
+sed -i "s%$REPO_DIR%$REPLACE_PLACEHOLDER%" "$CURR_DIR/go.mod"
+
+rm -r "$CURR_DIR/vendor"

docs/.pages

@@ -1,6 +1,6 @@
 nav:
   - index.md
   - usage
-  - internals
+  - design
   - faq
-collapse: false
+collapse: false

docs/design/.pages

@@ -0,0 +1,5 @@
+title: Design
+nav:
+  - project.md
+  - defaults.md
+  - networking.md

docs/design/defaults.md

@@ -0,0 +1,60 @@
+# Defaults
+
+## k3d reserved settings
+
+When you create a K3s cluster in Docker using k3d, we make use of some K3s configuration options, making them "reserved" for k3d.
+This means, that overriding those options with your own may break the cluster setup.
+
+### Environment Variables
+
+The following K3s environment variables are used to configure the cluster:
+
+| Variable | K3d Default | Configurable? |
+|----------|-------------|---------------|
+| `K3S_URL`| `https://$CLUSTERNAME-server-0:6443` | no |
+| `K3S_TOKEN`| random | yes (`--token`) |
+| `K3S_KUBECONFIG_OUTPUT`| `/output/kubeconfig.yaml` | no |
+
+## k3d Loadbalancer
+
+By default, k3d creates an Nginx loadbalancer alongside the clusters it creates to handle the port-forwarding.
+The loadbalancer can partly be configured using k3d-defined settings.
+
+| Nginx setting | k3d default | k3d setting |
+|-------------|-------------|-------------|
+| `proxy_timeout` (default for all server stanzas) | `600` (s) | `settings.defaultProxyTimeout` |  |
+|`worker_connections` | `1024` | `settings.workerConnections` |
+
+### Overrides
+
+- Example via CLI: `k3d cluster create --lb-config-override settings.defaultProxyTimeout=900`
+- Example via Config File:
+
+  ```yaml
+  # ... truncated ...
+  k3d:
+    loadbalancer:
+      configOverrides:
+        - settings.workerConnections=2048
+  ```
+
+## Multiple server nodes
+
+- by default, when `--server` > 1 and no `--datastore-x` option is set, the first server node (server-0) will be the initializing server node
+  - the initializing server node will have the `--cluster-init` flag appended
+  - all other server nodes will refer to the initializing server node via `--server https://<init-node>:6443`
+
+## API-Ports
+
+- by default, we expose the API-Port (`6443`) by forwarding traffic from the default server loadbalancer (nginx container) to the server node(s)
+- port `6443` of the loadbalancer is then mapped to a specific (`--api-port` flag) or a random (default) port on the host system
+
+## Kubeconfig
+
+- if `--kubeconfig-update-default` is set, we use the default loading rules to get the default kubeconfig:
+  - First: kubeconfig specified via the KUBECONFIG environment variable (error out if multiple are specified)
+  - Second: default kubeconfig in home directory (e.g. `$HOME/.kube/config`)
+
+## Networking
+
+- [by default, k3d creates a new (docker) network for every cluster](./networking)

docs/design/project.md

@@ -0,0 +1,109 @@
+# Project Overview
+
+## About This Page
+
+On this page we'll try to give an overview of all the moving bits and pieces in k3d to ease contributions to the project.
+
+## Directory Overview
+
+- [`.github/`](https://github.com/rancher/k3d/tree/main/.github)
+  - templates for issues and pull requests
+  - GitHub Action workflow definitions
+- [`cmd/`](https://github.com/rancher/k3d/tree/main/cmd)
+  - everything related to the actual k3d CLI, like the whole command tree, config initialization, argument parsing, etc.
+- [`docgen/`](https://github.com/rancher/k3d/tree/main/docgen)
+  - sub-module used to auto-generate the documentation for the CLI commands, which ends up in [`docs/usage/commands/`](https://github.com/rancher/k3d/tree/main/docs/usage/commands)
+- [`docs/`](https://github.com/rancher/k3d/tree/main/docs)
+  - all the resources used to build [k3d.io](https://k3d.io) using mkdocs
+- [`pkg/`](<https://github.com/rancher/k3d/tree/main/pkg>)
+  - the place where the magic happens.. here you find all the main logic of k3d
+  - all function calls within [`cmd/`](https://github.com/rancher/k3d/tree/main/cmd) that do non-trivial things are imported from here
+  - this (or rather sub-packages) is what other projects would import as a module to work with k3d without using the CLI
+- [`proxy/`](https://github.com/rancher/k3d/tree/main/proxy)
+  - configuration to build the [`rancher/k3d-proxy`](https://hub.docker.com/r/rancher/k3d-proxy/) container image which is used as a loadbalancer/proxy in front of (almost) every k3d cluster
+  - this is basically just a combination of NGINX with confd and some k3d-specific configuration details
+- [`tests/`](https://github.com/rancher/k3d/tree/main/tests)
+  - a set of bash scripts used for end-to-end (E2E) tests of k3d
+  - mostly used for all the functionality of the k3d CLI which cannot be tested using Go unit tests
+- [`tools/`](https://github.com/rancher/k3d/tree/main/tools)
+  - sub-module used to build the [`rancher/k3d-tools`](https://hub.docker.com/r/rancher/k3d-tools) container image which supports some k3d functionality like `k3d image import`
+- [`vendor/`](https://github.com/rancher/k3d/tree/main/vendor)
+  - result of `go mod vendor`, which contains all dependencies of k3d
+- [`version/`](https://github.com/rancher/k3d/tree/main/version)
+  - package used to code k3d/k3s versions into releases
+  - this is where `go build` injects the version tags when building k3d
+    - that's the output you see when issuing `k3d version`
+
+## Packages Overview
+
+- [`pkg/`](https://github.com/rancher/k3d/tree/main/pkg)
+  - [`actions/`](https://github.com/rancher/k3d/tree/main/pkg/actions)
+    - hook actions describing actions (commands, etc.) that run at specific stages of the node/cluster lifecycle
+      - e.g. writing configuration files to the container filesystem just before the node (container) starts
+  - [`client/`](https://github.com/rancher/k3d/tree/main/pkg/client)
+    - all the top level functionality to work with k3d primitives
+      - create/retrieve/update/delete/start/stop clusters, nodes, registries, etc. managed by k3d
+  - [`config/`](https://github.com/rancher/k3d/tree/main/pkg/config)
+    - everything related to the k3d configuration (files), like `SimpleConfig` and `ClusterConfig`
+  - [`runtimes/`](https://github.com/rancher/k3d/tree/main/pkg/runtimes)
+    - interface and implementations of runtimes that power k3d (currently, that's only Docker)
+    - functions in [`client/`](https://github.com/rancher/k3d/tree/main/pkg/client) eventually call runtime functions to "materialize" nodes and clusters
+  - [`tools/`](https://github.com/rancher/k3d/tree/main/pkg/tools)
+    - functions eventually calling the [`k3d-tools`](https://hub.docker.com/r/rancher/k3d-tools) container (see [`tools/`](https://github.com/rancher/k3d/tree/main/tools) in the repo root)
+  - [`types/`](https://github.com/rancher/k3d/tree/main/pkg/types)
+    - definition of all k3d primitives and many other details and defaults
+    - e.g. contains the definition of a `Node` or a `Cluster` in k3d
+  - [`util/`](https://github.com/rancher/k3d/tree/main/pkg/util)
+    - some helper functions e.g. for string manipulation/generation, regexp or other re-usable usages
+
+## Anatomy of a Cluster
+
+By default, every k3d cluster consists of at least 2 containers (nodes):
+
+1. (optional, but default and strongly recommended) loadbalancer
+
+   - image: [`rancher/k3d-proxy`](https://hub.docker.com/r/rancher/k3d-proxy/), built from [`proxy/`](https://github.com/rancher/k3d/tree/main/proxy)
+   - purpose: proxy and load balance requests from the outside (i.e. most of the times your local host) to the cluster
+     - by default, it e.g. proxies all the traffic for the Kubernetes API to port `6443` (default listening port of K3s) to all the server nodes in the cluster
+     - can be used for multiple port-mappings to one or more nodes in your cluster
+       - that way, port-mappings can also easily be added/removed after the cluster creation, as we can simply re-create the proxy without affecting cluster state
+
+2. (required, always present) primary server node
+
+   - image: [`rancher/k3s`](https://hub.docker.com/r/rancher/k3s/), built from [`github.com/k3s-io/k3s`](https://github.com/k3s-io/k3s)
+   - purpose: (initializing) server (formerly: master) node of the cluster
+     - runs the K3s executable (which runs containerd, the Kubernetes API Server, etcd/sqlite, etc.): `k3s server`
+     - in a multi-server setup, it initializes the cluster with an embedded etcd database (using the K3s `--cluster-init` flag)
+
+3. (optional) secondary server node(s)
+
+   - image: [`rancher/k3s`](https://hub.docker.com/r/rancher/k3s/), built from [`github.com/k3s-io/k3s`](https://github.com/k3s-io/k3s)
+
+4. (optional) agent node(s)
+
+   - image: [`rancher/k3s`](https://hub.docker.com/r/rancher/k3s/), built from [`github.com/k3s-io/k3s`](https://github.com/k3s-io/k3s)
+   - purpose: running the K3s agent process (kubelet, etc.): `k3s agent`
+
+## Automation (CI)
+
+The k3d repository mainly leverages the following two CI systems:
+
+- GitHub Actions
+  - 2 workflows in <https://github.com/rancher/k3d/tree/main/.github/workflows> to push the artifact to AUR (Arch Linux User Repository)
+  - logs/history can be seen in the Actions tab: <https://github.com/rancher/k3d/actions>
+- DroneCI
+  - a set of pipelines in a single file: <https://github.com/rancher/k3d/blob/main/.drone.yml>
+  - static code analysis
+  - build
+  - tests
+  - docker builds + pushes
+  - render + push docs
+  - (pre-) release to GitHub
+  - `push` events end up here (also does the releases, when a tag is pushed): <https://drone-publish.rancher.io/rancher/k3d>
+  - `pr`s end up here: <https://drone-pr.rancher.io/rancher/k3d>
+
+## Documentation
+
+The website [k3d.io](https://k3d.io) containing all the documentation for k3d is built using [`mkdocs`](https://www.mkdocs.org/), configured via the [`mkdocs.yml`](https://github.com/rancher/k3d/blob/main/mkdocs.yml) config file with all the content residing in the [`docs/`](https://github.com/rancher/k3d/tree/main/docs) directory (Markdown).  
+Use `mkdocs serve` in the repository root to build and serve the webpage locally.  
+Some parts of the documentation are being auto-generated, like [`docs/usage/commands/`](https://github.com/rancher/k3d/tree/main/docs/usage/commands) is auto-generated using Cobra's command docs generation functionality in [`docgen/`](https://github.com/rancher/k3d/tree/main/docgen).

docs/faq/.pages

@@ -1,3 +1,4 @@
 title: FAQ
 nav:
-  - faq.md
+  - faq.md
+collapse: true

docs/faq/faq.md

@@ -1,4 +1,4 @@
-# FAQ / Nice to know
+# FAQ
 
 ## Issues with BTRFS
 
@@ -28,8 +28,8 @@
 
       ```bash
       k3d cluster create \
-        --k3s-agent-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%' \
-        --k3s-agent-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%'
+        --k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \
+        --k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*'
       ```
 
 ## Restarting a multi-server cluster or the initializing server node fails
@@ -44,7 +44,7 @@
 
 - The Problem: Passing a feature flag to the Kubernetes API Server running inside k3s.
 - Example: you want to enable the EphemeralContainers feature flag in Kubernetes
-- Solution: `#!bash k3d cluster create --k3s-server-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true'`
+- Solution: `#!bash k3d cluster create --k3s-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true@server:*'`
   - **Note**: Be aware of where the flags require dashes (`--`) and where not.
     - the k3s flag (`--kube-apiserver-arg`) has the dashes
     - the kube-apiserver flag `feature-gates` doesn't have them (k3s adds them internally)
@@ -53,10 +53,10 @@
 
   ```bash
   k3d cluster create k3d-one \
-    --k3s-server-arg --cluster-cidr="10.118.0.0/17" \
-    --k3s-server-arg --service-cidr="10.118.128.0/17" \
-    --k3s-server-arg --disable=servicelb \
-    --k3s-server-arg --disable=traefik \
+    --k3s-arg "--cluster-cidr=10.118.0.0/17@server:*" \
+    --k3s-arg "--service-cidr=10.118.128.0/17@server:*" \
+    --k3s-arg "--disable=servicelb@server:*" \
+    --k3s-arg "--disable=traefik@server:*" \
     --verbose
   ```
 
@@ -105,8 +105,8 @@ Some can be fixed by passing the `HTTP_PROXY` environment variables to k3d, some
 
   ```bash
   k3d cluster create \
-    --k3s-server-arg "--kube-proxy-arg=conntrack-max-per-core=0" \
-    --k3s-agent-arg "--kube-proxy-arg=conntrack-max-per-core=0" \
+    --k3s-arg "--kube-proxy-arg=conntrack-max-per-core=0@server:*" \
+    --k3s-arg "--kube-proxy-arg=conntrack-max-per-core=0@agent:*" \
     --image rancher/k3s:v1.20.6-k3s
   ```
 

docs/index.md

@@ -2,14 +2,16 @@
 
 ![k3d](static/img/k3d_logo_black_blue.svg)
 
-**This page is targeting k3d v4.0.0 and newer!**
+## What is k3d?
 
 k3d is a lightweight wrapper to run [k3s](https://github.com/rancher/k3s) (Rancher Lab's minimal Kubernetes distribution) in docker.
 
 k3d makes it very easy to create single- and multi-node [k3s](https://github.com/rancher/k3s) clusters in docker, e.g. for local development on Kubernetes.
 
+**Note:** k3d is a **community-driven project**, that is supported by Rancher (SUSE) and it's not an official Rancher (SUSE) project.
+
 ??? Tip "View a quick demo"
-    <asciinema-player src="/static/asciicast/20200715_k3d.01.cast" cols=200 rows=32></asciinema-player>
+    <asciinema-player src="/static/asciicast/20210917_k3d_v5.0.0_01.cast" cols=200 rows=32></asciinema-player>
 
 ## Learning
 
@@ -25,7 +27,8 @@ k3d makes it very easy to create single- and multi-node [k3s](https://github.com
 
 ## Requirements
 
-- [docker](https://docs.docker.com/install/)
+- [**docker**](https://docs.docker.com/install/) to be able to use k3d at all
+- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl) to interact with the Kubernetes cluster
 
 ## Releases
 
@@ -51,8 +54,8 @@ You have several options there:
 
 Use the install script to grab a specific release (via `TAG` environment variable):
 
-- wget: `#!bash wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v4.0.0 bash`
-- curl: `#!bash curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v4.0.0 bash`
+- wget: `#!bash wget -q -O - https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v5.0.0 bash`
+- curl: `#!bash curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | TAG=v5.0.0 bash`
 
 ### Other Installers
 
@@ -78,10 +81,10 @@ Use the install script to grab a specific release (via `TAG` environment variabl
 
     - [asdf](https://asdf-vm.com): `asdf plugin-add k3d && asdf install k3d latest`
 
-      *Note*: `asdf plugin-add k3d`, then `asdf install k3d <tag>` with `<tag> = latest` or `4.x.x` for a specific version (maintained by [spencergilbert/asdf-k3d](https://github.com/spencergilbert/asdf-k3d))
+      *Note*: `asdf plugin-add k3d`, then `asdf install k3d <tag>` with `<tag> = latest` or `5.x.x` for a specific version (maintained by [spencergilbert/asdf-k3d](https://github.com/spencergilbert/asdf-k3d))
 
     - Others
-      - install via go: `#!bash go install github.com/rancher/k3d` (**Note**: this will give you unreleased/bleeding-edge changes)
+      - install via go: `#!bash go install github.com/rancher/k3d@latest` (**Note**: this will give you unreleased/bleeding-edge changes)
 
 ## Quick Start
 

docs/internals/.pages

@@ -1,4 +0,0 @@
-title: Internals
-nav:
-  - defaults.md
-  - networking.md

docs/internals/defaults.md

@@ -1,22 +0,0 @@
-# Defaults
-
-## Multiple server nodes
-
-- by default, when `--server` > 1 and no `--datastore-x` option is set, the first server node (server-0) will be the initializing server node
-  - the initializing server node will have the `--cluster-init` flag appended
-  - all other server nodes will refer to the initializing server node via `--server https://<init-node>:6443`
-
-## API-Ports
-
-- by default, we expose the API-Port (`6443`) by forwarding traffic from the default server loadbalancer (nginx container) to the server node(s)
-- port `6443` of the loadbalancer is then mapped to a specific (`--api-port` flag) or a random (default) port on the host system
-
-## Kubeconfig
-
-- if `--kubeconfig-update-default` is set, we use the default loading rules to get the default kubeconfig:
-  - First: kubeconfig specified via the KUBECONFIG environment variable (error out if multiple are specified)
-  - Second: default kubeconfig in home directory (e.g. `$HOME/.kube/config`)
-
-## Networking
-
-- [by default, k3d creates a new (docker) network for every cluster](./networking)

docs/requirements.txt

@@ -1,7 +1,8 @@
-mkdocs
-mkdocs-material
-pymdown-extensions
-mkdocs-git-revision-date-localized-plugin
-mkdocs-awesome-pages-plugin
-mdx_truly_sane_lists
-mkdocs-include-markdown-plugin # https://github.com/mondeja/mkdocs-include-markdown-plugin
+mkdocs==1.2.2
+mkdocs-material==7.2.6
+pymdown-extensions==8.2
+mkdocs-git-revision-date-localized-plugin==0.9.3
+mkdocs-awesome-pages-plugin==2.5.0
+mdx_truly_sane_lists==1.2 # https://github.com/radude/mdx_truly_sane_lists
+mkdocs-include-markdown-plugin==3.2.2 # https://github.com/mondeja/mkdocs-include-markdown-plugin
+mike==1.1.0 # versioned docs: https://github.com/jimporter/mike

docs/static/asciicast/20210917_k3d_v5.0.0_01.cast

@@ -0,0 +1,162 @@
+{"version": 2, "width": 213, "height": 45, "timestamp": 1631908903, "env": {"SHELL": "bash", "TERM": "xterm-256color"}}
+[0.018381, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[1.636481, "o", "k"]
+[1.702291, "o", "3"]
+[1.835268, "o", "d"]
+[2.024007, "o", " "]
+[2.111734, "o", "v"]
+[2.210891, "o", "e"]
+[2.343441, "o", "r"]
+[2.516933, "o", "s"]
+[2.583471, "o", "i"]
+[2.773563, "o", "o"]
+[2.927568, "o", "n"]
+[3.159219, "o", "\r\n\u001b[?2004l\r"]
+[3.179508, "o", "k3d version v5.0.0\r\nk3s version v1.21.4-k3s1 (default)\r\n"]
+[3.180754, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[4.57973, "o", "k"]
+[4.656235, "o", "3"]
+[4.763252, "o", "d"]
+[4.865396, "o", " "]
+[4.986278, "o", "c"]
+[5.051494, "o", "l"]
+[5.238737, "o", "u"]
+[5.292747, "o", "s"]
+[5.381595, "o", "t"]
+[5.503508, "o", "e"]
+[5.578881, "o", "r"]
+[5.666704, "o", " "]
+[5.766742, "o", "c"]
+[5.962787, "o", "r"]
+[6.029469, "o", "e"]
+[6.061464, "o", "a"]
+[6.184275, "o", "t"]
+[6.281805, "o", "e"]
+[6.445508, "o", " "]
+[6.666863, "o", "-"]
+[7.20248, "o", "-"]
+[7.334019, "o", "a"]
+[7.490134, "o", "g"]
+[7.566087, "o", "e"]
+[7.631634, "o", "n"]
+[7.729597, "o", "t"]
+[7.897099, "o", "s"]
+[8.049496, "o", " "]
+[8.280178, "o", "3"]
+[8.499599, "o", " "]
+[8.631147, "o", "d"]
+[8.707104, "o", "e"]
+[8.773508, "o", "m"]
+[8.91407, "o", "o"]
+[9.113612, "o", "\r\n\u001b[?2004l\r"]
+[9.132118, "o", "\u001b[36mINFO\u001b[0m[0000] Prep: Network                                \r\n"]
+[9.183203, "o", "\u001b[36mINFO\u001b[0m[0000] Created network 'k3d-demo'                   \r\n"]
+[9.187229, "o", "\u001b[36mINFO\u001b[0m[0000] Created volume 'k3d-demo-images'             \r\n"]
+[10.187972, "o", "\u001b[36mINFO\u001b[0m[0001] Creating node 'k3d-demo-server-0'            \r\n"]
+[10.281058, "o", "\u001b[36mINFO\u001b[0m[0001] Creating node 'k3d-demo-agent-0'             \r\n"]
+[10.368708, "o", "\u001b[36mINFO\u001b[0m[0001] Creating node 'k3d-demo-agent-1'             \r\n"]
+[10.455282, "o", "\u001b[36mINFO\u001b[0m[0001] Creating node 'k3d-demo-agent-2'             \r\n"]
+[10.536337, "o", "\u001b[36mINFO\u001b[0m[0001] Creating LoadBalancer 'k3d-demo-serverlb'    \r\n"]
+[10.609539, "o", "\u001b[36mINFO\u001b[0m[0001] Using the k3d-tools node to gather environment information \r\n"]
+[10.628592, "o", "\u001b[36mINFO\u001b[0m[0001] Starting new tools node...                   \r\n"]
+[10.702678, "o", "\u001b[36mINFO\u001b[0m[0001] Starting Node 'k3d-demo-tools'               \r\n"]
+[11.394216, "o", "\u001b[36mINFO\u001b[0m[0002] Deleted k3d-demo-tools                       \r\n"]
+[11.394427, "o", "\u001b[36mINFO\u001b[0m[0002] Starting cluster 'demo'                      \r\n\u001b[36mINFO\u001b[0m[0002] Starting servers...                          \r\n"]
+[11.404635, "o", "\u001b[36mINFO\u001b[0m[0002] Starting Node 'k3d-demo-server-0'            \r\n"]
+[16.378372, "o", "\u001b[36mINFO\u001b[0m[0007] Starting agents...                           \r\n"]
+[16.388922, "o", "\u001b[36mINFO\u001b[0m[0007] Starting Node 'k3d-demo-agent-0'             \r\n"]
+[16.389848, "o", "\u001b[36mINFO\u001b[0m[0007] Starting Node 'k3d-demo-agent-1'             \r\n"]
+[16.397254, "o", "\u001b[36mINFO\u001b[0m[0007] Starting Node 'k3d-demo-agent-2'             \r\n"]
+[31.590126, "o", "\u001b[36mINFO\u001b[0m[0022] Starting helpers...                          \r\n"]
+[31.637947, "o", "\u001b[36mINFO\u001b[0m[0022] Starting Node 'k3d-demo-serverlb'            \r\n"]
+[38.185432, "o", "\u001b[36mINFO\u001b[0m[0029] Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access \r\n"]
+[50.256861, "o", "\u001b[36mINFO\u001b[0m[0041] Cluster 'demo' created successfully!         \r\n\u001b[36mINFO\u001b[0m[0041] --kubeconfig-update-default=false --> sets --kubeconfig-switch-context=false \r\n"]
+[50.295453, "o", "\u001b[36mINFO\u001b[0m[0041] You can now use it like this:                \r\nkubectl config use-context k3d-demo\r\nkubectl cluster-info\r\n"]
+[50.299281, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[52.777117, "o", "k"]
+[52.873341, "o", "3"]
+[53.006105, "o", "d"]
+[53.147707, "o", " "]
+[53.245736, "o", "c"]
+[53.343772, "o", "l"]
+[53.551038, "o", "u"]
+[53.617941, "o", "s"]
+[53.724853, "o", "t"]
+[53.878933, "o", "e"]
+[53.956281, "o", "r"]
+[54.076303, "o", " "]
+[54.21845, "o", "l"]
+[54.339561, "o", "s"]
+[54.447647, "o", "\r\n\u001b[?2004l\r"]
+[54.47118, "o", "NAME   SERVERS   AGENTS   LOADBALANCER\r\ndemo   1/1       3/3      true\r\n"]
+[54.472506, "o", "\u001b[?2004h"]
+[54.472562, "o", "\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[54.838629, "o", "k"]
+[54.918551, "o", "3"]
+[55.015846, "o", "d"]
+[55.115834, "o", " "]
+[55.290514, "o", "n"]
+[55.378089, "o", "o"]
+[55.454292, "o", "d"]
+[55.508669, "o", "e"]
+[55.869687, "o", " "]
+[56.05605, "o", "l"]
+[56.176004, "o", "s"]
+[56.31685, "o", "\r\n\u001b[?2004l\r"]
+[56.341161, "o", "NAME                ROLE           CLUSTER   STATUS\r\nk3d-demo-agent-0    agent          demo      running\r\nk3d-demo-agent-1    agent          demo      running\r\nk3d-demo-agent-2    agent          demo      running\r\nk3d-demo-server-0   server         demo      running\r\nk3d-demo-serverlb   loadbalancer   demo      running\r\n"]
+[56.34231, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[57.733293, "o", "k"]
+[57.932149, "o", "u"]
+[58.059135, "o", "b"]
+[58.137901, "o", "e"]
+[58.23908, "o", "c"]
+[58.418996, "o", "t"]
+[58.496899, "o", "l"]
+[58.687091, "o", " "]
+[58.740349, "o", "g"]
+[58.832322, "o", "e"]
+[58.955499, "o", "t"]
+[59.067944, "o", " "]
+[59.246223, "o", "n"]
+[59.344781, "o", "o"]
+[59.426918, "o", "d"]
+[59.493282, "o", "e"]
+[59.672248, "o", "s"]
+[59.772331, "o", "\r\n\u001b[?2004l\r"]
+[60.41166, "o", "NAME                STATUS   ROLES                  AGE   VERSION\r\nk3d-demo-agent-2    Ready    <none>                 29s   v1.21.4+k3s1\r\nk3d-demo-server-0   Ready    control-plane,master   41s   v1.21.4+k3s1\r\nk3d-demo-agent-0    Ready    <none>                 31s   v1.21.4+k3s1\r\nk3d-demo-agent-1    Ready    <none>                 31s   v1.21.4+k3s1\r\n"]
+[60.414302, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[61.301105, "o", "k"]
+[61.534792, "o", "u"]
+[61.723192, "o", "b"]
+[61.800647, "o", "e"]
+[61.912191, "o", "c"]
+[62.111433, "o", "t"]
+[62.220654, "o", "l"]
+[62.400417, "o", " "]
+[62.434071, "o", "g"]
+[62.523052, "o", "e"]
+[62.634216, "o", "t"]
+[62.700412, "o", " "]
+[62.923073, "o", "p"]
+[63.120958, "o", "o"]
+[63.231192, "o", "d"]
+[63.287011, "o", "s"]
+[63.497854, "o", " "]
+[63.642017, "o", "-"]
+[63.896056, "o", "A"]
+[64.129633, "o", "\r\n\u001b[?2004l\r"]
+[64.180813, "o", "NAMESPACE     NAME                                      READY   STATUS              RESTARTS   AGE\r\nkube-system   coredns-7448499f4d-rrmh5                  1/1     Running             0          34s\r\nkube-system   metrics-server-86cbb8457f-6hkns           1/1     Running             0          34s\r\nkube-system   local-path-provisioner-5ff76fc89d-ltzd4   1/1     Running             0          34s\r\nkube-system   helm-install-traefik-crd-st9fm            0/1     Completed           0          34s\r\nkube-system   traefik-97b44b794-lgljm                   0/1     ContainerCreating   0          11s\r\nkube-system   helm-install-traefik-6t7fr                0/1     Completed           1          "]
+[64.181, "o", "34s\r\nkube-system   svclb-traefik-wztvf                       2/2     Running             0          11s\r\nkube-system   svclb-traefik-ksk54                       2/2     Running             0          11s\r\nkube-system   svclb-traefik-s286b                       2/2     Running             0          11s\r\nkube-system   svclb-traefik-ksbmz                       2/2     Running             0          11s\r\n"]
+[64.182931, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[66.050907, "o", "#"]
+[66.160953, "o", " "]
+[66.559434, "o", "P"]
+[66.768444, "o", "r"]
+[66.844975, "o", "o"]
+[67.022583, "o", "f"]
+[67.098851, "o", "i"]
+[67.286285, "o", "t"]
+[67.921864, "o", "."]
+[69.59588, "o", "\r\n\u001b[?2004l\r"]
+[69.596126, "o", "\u001b[?2004h\u001b]0;ThisCouldBeYou: ~\u0007\u001b[01;32mThisCouldBeYou\u001b[00m:\u001b[01;34m~\u001b[00m$ "]
+[70.123764, "o", "\u001b[?2004l\r\r\nexit\r\n"]

docs/static/css/extra.css

@@ -23,10 +23,28 @@
     position: relative;
 }
 
+/* This is equal to light mode */
 [data-md-color-primary=black] .md-tabs {
+
+    /* Set color of the tab bar */
     background-color: #0DCEFF;
 }
 
+/* Dark Mode */
+[data-md-color-scheme="slate"] .md-header {
+    /* keep black backgroud of title bar (header) */
+    background-color: black;
+}
+
+/* Tab Bar */
 .md-tabs {
     color: black;
 }
+
+.md-tabs__item {
+    font-weight: bolder;
+}
+
+.md-tabs__link--active {
+    text-decoration: underline;
+}

docs/usage/.pages

@@ -1,7 +1,9 @@
-title: Usage
+title: Guides
 nav:
-  - commands
   - configfile.md
   - kubeconfig.md
   - multiserver.md
-  - guides
+  - registries.md
+  - exposing_services.md
+  - advanced
+  - commands

docs/usage/advanced/.pages

@@ -0,0 +1,4 @@
+title: Advanced Guides
+nav:
+  - calico.md
+  - cuda.md

docs/usage/advanced/calico.md

@@ -20,14 +20,14 @@ Or you can directly use this [calico.yaml](calico.yaml) manifest
 
 On the k3s cluster creation :
 
-- add the flag `--flannel-backend=none`. For this, on k3d you need to forward this flag to k3s with the option `--k3s-server-arg`.
+- add the flag `--flannel-backend=none`. For this, on k3d you need to forward this flag to k3s with the option `--k3s-arg`.
 - mount (`--volume`) the calico descriptor in the auto deploy manifest directory of k3s `/var/lib/rancher/k3s/server/manifests/`
 
 So the command of the cluster creation is (when you are at root of the k3d repository)
 
 ```bash
 k3d cluster create "${clustername}" \
-  --k3s-server-arg '--flannel-backend=none' \
+  --k3s-arg '--flannel-backend=none@server:*' \
   --volume "$(pwd)/docs/usage/guides/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml"
 ```
 

docs/usage/commands.md

@@ -16,8 +16,7 @@ k3d
       -e, --env  # add environment variables to the nodes (quoted string, format: 'KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]', use flag multiple times)
       --gpus  # [from docker CLI] add GPU devices to the node containers (string, e.g. 'all')
       -i, --image  # specify which k3s image should be used for the nodes (string, default: 'docker.io/rancher/k3s:v1.20.0-k3s2', tag changes per build)
-      --k3s-agent-arg  # add additional arguments to the k3s agent (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/#k3s-agent-cli-help)
-      --k3s-server-arg  # add additional arguments to the k3s server (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help)
+      --k3s-arg  # add additional arguments to the k3s server/agent (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help & https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/#k3s-agent-cli-help)
       --kubeconfig-switch-context  # (implies --kubeconfig-update-default) automatically sets the current-context of your default kubeconfig to the new cluster's context (default: true)
       --kubeconfig-update-default  # enable the automated update of the default kubeconfig with the details of the newly created cluster (also sets '--wait=true') (default: true)
       -l, --label  # add (docker) labels to the node containers (format: 'KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]', use flag multiple times)

docs/usage/commands/k3d_cluster.md

@@ -29,6 +29,7 @@ k3d cluster [flags]
 * [k3d](k3d.md)	 - https://k3d.io/ -> Run k3s in Docker!
 * [k3d cluster create](k3d_cluster_create.md)	 - Create a new cluster
 * [k3d cluster delete](k3d_cluster_delete.md)	 - Delete cluster(s).
+* [k3d cluster edit](k3d_cluster_edit.md)	 - [EXPERIMENTAL] Edit cluster(s).
 * [k3d cluster list](k3d_cluster_list.md)	 - List cluster(s)
 * [k3d cluster start](k3d_cluster_start.md)	 - Start existing k3d cluster(s)
 * [k3d cluster stop](k3d_cluster_stop.md)	 - Stop existing k3d cluster(s)

docs/usage/commands/k3d_cluster_create.md

@@ -31,27 +31,30 @@ k3d cluster create NAME [flags]
   -i, --image string                                                   Specify k3s image that you want to use for the nodes
       --k3s-arg ARG@NODEFILTER[;@NODEFILTER]                           Additional args passed to k3s command (Format: ARG@NODEFILTER[;@NODEFILTER])
                                                                         - Example: `k3d cluster create --k3s-arg "--disable=traefik@server:0"
+      --k3s-node-label KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]        Add label to k3s node (Format: KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]
+                                                                        - Example: `k3d cluster create --agents 2 --k3s-node-label "my.label@agent:0,1" --k3s-node-label "other.label=somevalue@server:0"`
       --kubeconfig-switch-context                                      Directly switch the default kubeconfig's current-context to the new cluster's context (requires --kubeconfig-update-default) (default true)
       --kubeconfig-update-default                                      Directly update the default kubeconfig with the new cluster's context (default true)
-  -l, --label KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]                 Add label to node container (Format: KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]
-                                                                        - Example: `k3d cluster create --agents 2 -l "my.label@agent[0,1]" -l "other.label=somevalue@server:0"`
+      --lb-config-override strings                                     Use dotted YAML path syntax to override nginx loadbalancer settings
       --network string                                                 Join an existing network
-      --no-hostip                                                      Disable the automatic injection of the Host IP as 'host.k3d.internal' into the containers and CoreDNS
       --no-image-volume                                                Disable the creation of a volume for importing images
       --no-lb                                                          Disable the creation of a LoadBalancer in front of the server nodes
       --no-rollback                                                    Disable the automatic rollback actions, if anything goes wrong
-  -p, --port [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]   Map ports from the node containers to the host (Format: [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER])
-                                                                        - Example: `k3d cluster create --agents 2 -p 8080:80@agent:0 -p 8081@agent[1]`
+  -p, --port [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]   Map ports from the node containers (via the serverlb) to the host (Format: [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER])
+                                                                        - Example: `k3d cluster create --agents 2 -p 8080:80@agent:0 -p 8081@agent:1`
       --registry-config string                                         Specify path to an extra registries.yaml file
-      --registry-create                                                Create a k3d-managed registry and connect it to the cluster
+      --registry-create NAME[:HOST][:HOSTPORT]                         Create a k3d-managed registry and connect it to the cluster (Format: NAME[:HOST][:HOSTPORT]
+                                                                        - Example: `k3d cluster create --registry-create mycluster-registry:0.0.0.0:5432`
       --registry-use stringArray                                       Connect to one or more k3d-managed registries running locally
+      --runtime-label KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]         Add label to container runtime (Format: KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]
+                                                                        - Example: `k3d cluster create --agents 2 --runtime-label "my.label@agent:0,1" --runtime-label "other.label=somevalue@server:0"`
   -s, --servers int                                                    Specify how many servers you want to create
       --servers-memory string                                          Memory limit imposed on the server nodes [From docker]
       --subnet 172.28.0.0/16                                           [Experimental: IPAM] Define a subnet for the newly created container network (Example: 172.28.0.0/16)
       --timeout duration                                               Rollback changes if cluster couldn't be created in specified duration.
       --token string                                                   Specify a cluster token. By default, we generate one.
   -v, --volume [SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]              Mount volumes into the nodes (Format: [SOURCE:]DEST[@NODEFILTER[;NODEFILTER...]]
-                                                                        - Example: `k3d cluster create --agents 2 -v /my/path@agent[0,1] -v /tmp/test:/tmp/other@server:0`
+                                                                        - Example: `k3d cluster create --agents 2 -v /my/path@agent:0,1 -v /tmp/test:/tmp/other@server:0`
       --wait                                                           Wait for the server(s) to be ready before returning. Use '--timeout DURATION' to not wait forever. (default true)
 ```
 

docs/usage/commands/k3d_cluster_delete.md

@@ -13,8 +13,9 @@ k3d cluster delete [NAME [NAME ...] | --all] [flags]
 ### Options
 
 ```
-  -a, --all    Delete all existing clusters
-  -h, --help   help for delete
+  -a, --all             Delete all existing clusters
+  -c, --config string   Path of a config file to use
+  -h, --help            help for delete
 ```
 
 ### Options inherited from parent commands

docs/usage/commands/k3d_cluster_edit.md

@@ -0,0 +1,32 @@
+## k3d cluster edit
+
+[EXPERIMENTAL] Edit cluster(s).
+
+### Synopsis
+
+[EXPERIMENTAL] Edit cluster(s).
+
+```
+k3d cluster edit CLUSTER [flags]
+```
+
+### Options
+
+```
+  -h, --help                                                               help for edit
+      --port-add [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]   [EXPERIMENTAL] Map ports from the node containers (via the serverlb) to the host (Format: [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER])
+                                                                            - Example: `k3d node edit k3d-mycluster-serverlb --port-add 8080:80`
+```
+
+### Options inherited from parent commands
+
+```
+      --timestamps   Enable Log timestamps
+      --trace        Enable super verbose output (trace logging)
+      --verbose      Enable verbose output (debug logging)
+```
+
+### SEE ALSO
+
+* [k3d cluster](k3d_cluster.md)	 - Manage cluster(s)
+

docs/usage/commands/k3d_completion.md

@@ -4,10 +4,48 @@ Generate completion scripts for [bash, zsh, fish, powershell | psh]
 
 ### Synopsis
 
-Generate completion scripts for [bash, zsh, fish, powershell | psh]
+To load completions:
+
+Bash:
+
+	$ source <(k3d completion bash)
+
+	# To load completions for each session, execute once:
+	# Linux:
+	$ k3d completion bash > /etc/bash_completion.d/k3d
+	# macOS:
+	$ k3d completion bash > /usr/local/etc/bash_completion.d/k3d
+
+Zsh:
+
+	# If shell completion is not already enabled in your environment,
+	# you will need to enable it.  You can execute the following once:
+
+	$ echo "autoload -U compinit; compinit" >> ~/.zshrc
+
+	# To load completions for each session, execute once:
+	$ k3d completion zsh > "${fpath[1]}/k3d"
+
+	# You will need to start a new shell for this setup to take effect.
+
+fish:
+
+	$ k3d completion fish | source
+
+	# To load completions for each session, execute once:
+	$ k3d completion fish > ~/.config/fish/completions/k3d.fish
+
+PowerShell:
+
+	PS> k3d completion powershell | Out-String | Invoke-Expression
+
+	# To load completions for every new session, run:
+	PS> k3d completion powershell > k3d.ps1
+	# and source this file from your PowerShell profile.
+
 
 ```
-k3d completion SHELL [flags]
+k3d completion SHELL
 ```
 
 ### Options

docs/usage/commands/k3d_config_migrate.md

@@ -3,7 +3,7 @@
 
 
 ```
-k3d config migrate INPUT OUTPUT [flags]
+k3d config migrate INPUT [OUTPUT] [flags]
 ```
 
 ### Options

docs/usage/commands/k3d_node.md

@@ -29,6 +29,7 @@ k3d node [flags]
 * [k3d](k3d.md)	 - https://k3d.io/ -> Run k3s in Docker!
 * [k3d node create](k3d_node_create.md)	 - Create a new k3s node in docker
 * [k3d node delete](k3d_node_delete.md)	 - Delete node(s).
+* [k3d node edit](k3d_node_edit.md)	 - [EXPERIMENTAL] Edit node(s).
 * [k3d node list](k3d_node_list.md)	 - List node(s)
 * [k3d node start](k3d_node_start.md)	 - Start an existing k3d node
 * [k3d node stop](k3d_node_stop.md)	 - Stop an existing k3d node

docs/usage/commands/k3d_node_create.md

@@ -13,15 +13,18 @@ k3d node create NAME [flags]
 ### Options
 
 ```
-  -c, --cluster string           Select the cluster that the node shall connect to. (default "k3s-default")
+  -c, --cluster string           Cluster URL or k3d cluster name to connect to. (default "k3s-default")
   -h, --help                     help for create
-  -i, --image string             Specify k3s image used for the node(s) (default "docker.io/rancher/k3s:v1.20.0-k3s2")
+  -i, --image string             Specify k3s image used for the node(s) (default "docker.io/rancher/k3s:v1.21.4-k3s2")
       --k3s-node-label strings   Specify k3s node labels in format "foo=bar"
       --memory string            Memory limit imposed on the node [From docker]
+  -n, --network strings          Add node to (another) runtime network
       --replicas int             Number of replicas of this node specification. (default 1)
       --role string              Specify node role [server, agent] (default "agent")
+      --runtime-label strings    Specify container runtime labels in format "foo=bar"
       --timeout duration         Maximum waiting time for '--wait' before canceling/returning.
-      --wait                     Wait for the node(s) to be ready before returning.
+  -t, --token string             Override cluster token (required when connecting to an external cluster)
+      --wait                     Wait for the node(s) to be ready before returning. (default true)
 ```
 
 ### Options inherited from parent commands

docs/usage/commands/k3d_node_edit.md

@@ -0,0 +1,32 @@
+## k3d node edit
+
+[EXPERIMENTAL] Edit node(s).
+
+### Synopsis
+
+[EXPERIMENTAL] Edit node(s).
+
+```
+k3d node edit NODE [flags]
+```
+
+### Options
+
+```
+  -h, --help                                                               help for edit
+      --port-add [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER]   [EXPERIMENTAL] (serverlb only!) Map ports from the node container to the host (Format: [HOST:][HOSTPORT:]CONTAINERPORT[/PROTOCOL][@NODEFILTER])
+                                                                            - Example: `k3d node edit k3d-mycluster-serverlb --port-add 8080:80`
+```
+
+### Options inherited from parent commands
+
+```
+      --timestamps   Enable Log timestamps
+      --trace        Enable super verbose output (trace logging)
+      --verbose      Enable verbose output (debug logging)
+```
+
+### SEE ALSO
+
+* [k3d node](k3d_node.md)	 - Manage node(s)
+

docs/usage/commands/k3d_node_list.md

@@ -7,7 +7,7 @@ List node(s)
 List node(s).
 
 ```
-k3d node list [NAME [NAME...]] [flags]
+k3d node list [NODE [NODE...]] [flags]
 ```
 
 ### Options

docs/usage/commands/k3d_node_start.md

@@ -7,7 +7,7 @@ Start an existing k3d node
 Start an existing k3d node.
 
 ```
-k3d node start NAME [flags]
+k3d node start NODE [flags]
 ```
 
 ### Options

docs/usage/configfile.md

@@ -1,10 +1,9 @@
-# Config File
+# Using Config Files
+
+The config file feature is **available as of k3d v4.0.0**
 
 ## Introduction
 
-As of k3d v4.0.0, released in January 2021, k3d ships with configuration file support for the `k3d cluster create` command.  
-This allows you to define all the things that you defined with CLI flags before in a nice and tidy YAML (as a Kubernetes user, we know you love it ;) ).
-
 !!! info "Syntax & Semantics"
     The options defined in the config file are not 100% the same as the CLI flags.  
     This concerns naming and style/usage/structure, e.g.
@@ -62,26 +61,26 @@ kubeAPI: # same as `--api-port myhost.my.domain:6445` (where the name would reso
   hostPort: "6445" # where the Kubernetes API listening port will be mapped to on your host system
 image: rancher/k3s:v1.20.4-k3s1 # same as `--image rancher/k3s:v1.20.4-k3s1`
 network: my-custom-net # same as `--network my-custom-net`
+subnet: "172.28.0.0/16" # same as `--subnet 172.28.0.0/16`
 token: superSecretToken # same as `--token superSecretToken`
 volumes: # repeatable flags are represented as YAML lists
-  - volume: /my/host/path:/path/in/node # same as `--volume '/my/host/path:/path/in/node@server:0;agent[*]'`
+  - volume: /my/host/path:/path/in/node # same as `--volume '/my/host/path:/path/in/node@server:0;agent:*'`
     nodeFilters:
       - server:0
-      - agent[*]
+      - agent:*
 ports:
   - port: 8080:80 # same as `--port '8080:80@loadbalancer'`
     nodeFilters:
       - loadbalancer
-labels:
-  - label: foo=bar # same as `--label 'foo=bar@agent[1]'`
-    nodeFilters:
-      - agent[1]
 env:
   - envVar: bar=baz # same as `--env 'bar=baz@server:0'`
     nodeFilters:
       - server:0
 registries: # define how registries should be created or used
-  create: true # creates a default registry to be used with the cluster; same as `--registry-create`
+  create: # creates a default registry to be used with the cluster; same as `--registry-create registry.localhost`
+    name: registry.localhost
+    host: "0.0.0.0"
+    hostPort: "5000"
   use:
     - k3d-myotherregistry:5000 # some other k3d-managed registry; same as `--registry-use 'k3d-myotherregistry:5000'`
   config: | # define contents of the `registries.yaml` file (or reference a file); same as `--registry-config /path/to/config.yaml`
@@ -96,17 +95,27 @@ options:
     disableLoadbalancer: false # same as `--no-lb`
     disableImageVolume: false # same as `--no-image-volume`
     disableRollback: false # same as `--no-Rollback`
-    disableHostIPInjection: false # same as `--no-hostip`
+    loadbalancer:
+      configOverrides:
+        - settings.workerConnections=2048
   k3s: # options passed on to K3s itself
     extraArgs: # additional arguments passed to the `k3s server|agent` command; same as `--k3s-arg`
       - arg: --tls-san=my.host.domain
         nodeFilters:
-          - server[*]
+          - server:*
+    nodeLabels:
+      - label: foo=bar # same as `--k3s-node-label 'foo=bar@agent:1'` -> this results in a Kubernetes node label
+        nodeFilters:
+          - agent:1
   kubeconfig:
     updateDefaultKubeconfig: true # add new cluster to your default Kubeconfig; same as `--kubeconfig-update-default` (default: true)
     switchCurrentContext: true # also set current-context to the new cluster's context; same as `--kubeconfig-switch-context` (default: true)
   runtime: # runtime (docker) specific options
     gpuRequest: all # same as `--gpus all`
+    labels:
+      - label: bar=baz # same as `--runtime-label 'bar=baz@agent:1'` -> this results in a runtime (docker) container label
+        nodeFilters:
+          - agent:1
 
 ```
 
@@ -125,4 +134,4 @@ For example, you use the same config file to create three clusters which only ha
 ## References
 
 - k3d demo repository: <https://github.com/iwilltry42/k3d-demo/blob/main/README.md#config-file-support>
-- SUSE Blog: <https://www.suse.com/c/introduction-k3d-run-k3s-docker-src/> (Search fo `The “Configuration as Code” Way`)
+- SUSE Blog: <https://www.suse.com/c/introduction-k3d-run-k3s-docker-src/> (Search for `The “Configuration as Code” Way`)

docs/usage/guides/.pages

@@ -1,6 +0,0 @@
-title: Guides
-nav:
-  - exposing_services.md
-  - registries.md
-  - calico.md
-  - cuda.md

docs/usage/multiserver.md

@@ -1,12 +1,12 @@
 # Creating multi-server clusters
 
 !!! info "Important note"
-    For the best results (and less unexpected issues), choose 1, 3, 5, ... server nodes.
+    For the best results (and less unexpected issues), choose 1, 3, 5, ... server nodes. (Read more on etcd quorum on [etcd.io](https://etcd.io/docs/v3.3/faq/#why-an-odd-number-of-cluster-members))
     At least 2 cores and 4GiB of RAM are recommended.
 
-## Embedded etcd (old: dqlite)
+## Embedded etcd
 
-Create a cluster with 3 server nodes using k3s' embedded etcd (old: dqlite) database.
+Create a cluster with 3 server nodes using k3s' embedded etcd database.
 The first server to be created will use the `--cluster-init` flag and k3d will wait for it to be up and running before creating (and connecting) the other server nodes.
 
 ```bash
@@ -23,4 +23,4 @@ k3d node create newserver --cluster multiserver --role server
 
 !!! important "There's a trap!"
     If your cluster was initially created with only a single server node, then this will fail.  
-    That's because the initial server node was not started with the `--cluster-init` flag and thus is not using the etcd (old: dqlite) backend.
+    That's because the initial server node was not started with the `--cluster-init` flag and thus is not using the etcd backend.

docs/usage/registries.md

@@ -1,13 +1,10 @@
-# Registries
+# Using Image Registries
 
 ## Registries configuration file
 
 You can add registries by specifying them in a `registries.yaml` and referencing it at creation time:
 `#!bash k3d cluster create mycluster --registry-config "/home/YOU/my-registries.yaml"`.
 
-??? Tip "Pre v4.0.0 solution"
-    Before we added the `--registry-config` flag in k3d v4.0.0, you had to bind-mount the file to the correct location: `--volume "/home/YOU/my-registries.yaml:/etc/rancher/k3s/registries.yaml"`
-
 This file is a regular [k3s registries configuration file](https://rancher.com/docs/k3s/latest/en/installation/private-registry/), and looks like this:
 
 ```yaml
@@ -17,10 +14,7 @@ mirrors:
       - http://my.company.registry:5000
 ```
 
-In this example, an image with a name like `my.company.registry:5000/nginx:latest` would be
-_pulled_ from the registry running at `http://my.company.registry:5000`.
-
-Note well there is an important limitation: **this configuration file will only work with k3s >= v0.10.0**. It will fail silently with previous versions of k3s, but you find in the [section below](#k3s-old) an alternative solution.
+In this example, an image with a name like `my.company.registry:5000/nginx:latest` would be _pulled_ from the registry running at `http://my.company.registry:5000`.
 
 This file can also be used for providing additional information necessary for accessing some registries, like [authentication](#authenticated-registries) and [certificates](#secure-registries).
 
@@ -35,7 +29,8 @@ name: test
 servers: 1
 agents: 2
 registries:
-  create: true
+  create: 
+    name: myregistry
   config: |
     mirrors:
       "my.company.registry":
@@ -43,7 +38,7 @@ registries:
           - http://my.company.registry:5000
 ```
 
-Here, the config for the k3d-managed registry, created by the `create: true` flag will be merged with the config specified under `config: |`.
+Here, the config for the k3d-managed registry, created by the `create: {...}` option will be merged with the config specified under `config: |`.
 
 ### Authenticated registries
 
@@ -95,24 +90,21 @@ k3d cluster create \
 
 ### Using k3d-managed registries
 
-!!! info "Just ported!"
-      The k3d-managed registry is available again as of k3d v4.0.0 (January 2021)
-
 #### Create a dedicated registry together with your cluster
 
-1. `#!bash k3d cluster create mycluster --registry-create`: This creates your cluster `mycluster` together with a registry container called `k3d-mycluster-registry`
+1. `#!bash k3d cluster create mycluster --registry-create mycluster-registry`: This creates your cluster `mycluster` together with a registry container called `mycluster-registry`
 
   - k3d sets everything up in the cluster for containerd to be able to pull images from that registry (using the `registries.yaml` file)
   - the port, which the registry is listening on will be mapped to a random port on your host system
 
-2. Check the k3d command output or `#!bash docker ps -f name=k3d-mycluster-registry` to find the exposed port (let's use `12345` here)
-3. Pull some image (optional) `#!bash docker pull alpine:latest`, re-tag it to reference your newly created registry `#!bash docker tag alpine:latest k3d-mycluster-registry:12345/testimage:local` and push it `#!bash docker push k3d-mycluster-registry:12345/testimage:local`
-4. Use kubectl to create a new pod in your cluster using that image to see, if the cluster can pull from the new registry: `#!bash kubectl run --image k3d-mycluster-registry:12345/testimage:local testimage --command -- tail -f /dev/null` (creates a container that will not do anything but keep on running)
+2. Check the k3d command output or `#!bash docker ps -f name=mycluster-registry` to find the exposed port (let's use `12345` here)
+3. Pull some image (optional) `#!bash docker pull alpine:latest`, re-tag it to reference your newly created registry `#!bash docker tag alpine:latest mycluster-registry:12345/testimage:local` and push it `#!bash docker push mycluster-registry:12345/testimage:local`
+4. Use kubectl to create a new pod in your cluster using that image to see, if the cluster can pull from the new registry: `#!bash kubectl run --image mycluster-registry:12345/testimage:local testimage --command -- tail -f /dev/null` (creates a container that will not do anything but keep on running)
 
 #### Create a customized k3d-managed registry
 
 1. `#!bash k3d registry create myregistry.localhost --port 12345` creates a new registry called `k3d-myregistry.localhost` (could be used with automatic resolution of `*.localhost`, see next section - also, **note the `k3d-` prefix** that k3d adds to all resources it creates)
-2. `#!bash k3d cluster create newcluster --registry-use k3d-myregistry.localhost:12345` (make sure you use the **`k3d-` prefix** here) creates a new cluster set up to us that registry
+2. `#!bash k3d cluster create newcluster --registry-use k3d-myregistry.localhost:12345` (make sure you use the **`k3d-` prefix** here) creates a new cluster set up to use that registry
 3. continue with step 3 and 4 from the last section for testing
 
 <!-- Admonition to describe usage of a non-k3d-managed registry -->
@@ -129,13 +121,13 @@ k3d cluster create \
     docker container run -d --name registry.localhost -v local_registry:/var/lib/registry --restart always -p 5000:5000 registry:2
     ```
 
-    These commands will start your registry in `registry.localhost:5000`. In order to push to this registry, you will need to make it accessible as described in the next section.
+    These commands will start your registry container with name and port `registry.localhost:5000`. In order to push to this registry, you will need to make it accessible as described in the next section.
     Once your registry is up and running, we will need to add it to your `registries.yaml` configuration file.
     Finally, you have to connect the registry network to the k3d cluster network: `#!bash docker network connect k3d-k3s-default registry.localhost`. And then you can [test your local registry](#testing-your-registry).
 
 ### Pushing to your local registry address
 
-As per the guide above, the registry will be available at `registry.localhost:5000`.
+As per the guide above, the registry will be available as `registry.localhost:5000`.
 
 All the nodes in your k3d cluster can resolve this hostname (thanks to the DNS server provided by the Docker daemon) but, in order to be able to push to this registry, this hostname also has to be resolved by your host.
 
@@ -150,7 +142,9 @@ If your system does not provide/support tools that can auto-resolve specific nam
 127.0.0.1 k3d-registry.localhost
 ```
 
-Once again, this will only work with k3s >= v0.10.0 (see the some sections below when using k3s <= v0.9.1)
+!!! info "Just use localhost"
+    Alternatively, if you don't care about pretty names, just push directly to `localhost:5000` (or whatever port you used) and it will work.
+    If you later pull the image from the registry, only the repository path (e.g. `myrepo/myimage:mytag` in `registry.localhost:5000/myrepo/myimage:mytag`) matters to find your image in the targeted registry.
 
 ## Testing your registry
 
@@ -199,44 +193,3 @@ EOF
 ```
 
 Then you should check that the pod is running with `kubectl get pods -l "app=nginx-test-registry"`.
-
-## Configuring registries for k3s <= v0.9.1
-
-k3s servers below v0.9.1 do not recognize the `registries.yaml` file as described in the in the beginning, so you will need to embed the contents of that file in a `containerd` configuration file.  
-You will have to create your own `containerd` configuration file at some well-known path like `${HOME}/.k3d/config.toml.tmpl`, like this:
-
-??? registriesprev091 "config.toml.tmpl"
-
-    ```toml
-    # Original section: no changes
-    [plugins.opt]
-    path = "{{ .NodeConfig.Containerd.Opt }}"
-    [plugins.cri]
-    stream_server_address = "{{ .NodeConfig.AgentConfig.NodeName }}"
-    stream_server_port = "10010"
-    {{- if .IsRunningInUserNS }}
-    disable_cgroup = true
-    disable_apparmor = true
-    restrict_oom_score_adj = true
-    {{ end -}}
-    {{- if .NodeConfig.AgentConfig.PauseImage }}
-    sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-    {{ end -}}
-    {{- if not .NodeConfig.NoFlannel }}
-      [plugins.cri.cni]
-        bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-        conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-    {{ end -}}
-
-    # Added section: additional registries and the endpoints
-    [plugins.cri.registry.mirrors]
-      [plugins.cri.registry.mirrors."<b>registry.localhost:5000</b>"]
-        endpoint = ["http://<b>registry.localhost:5000</b>"]
-    ```
-
-and then mount it at `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl` (where `containerd` in your k3d nodes will load it) when creating the k3d cluster:
-
-```bash
-k3d cluster create mycluster \
-    --volume ${HOME}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl
-```

go.mod

@@ -1,21 +1,20 @@
-module github.com/rancher/k3d/v4
+module github.com/rancher/k3d/v5
 
-go 1.16
+go 1.17
 
 require (
-	github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 // indirect
-	github.com/Microsoft/hcsshim v0.8.14 // indirect
-	github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 // indirect
-	github.com/containerd/containerd v1.4.4 // indirect
-	github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e // indirect
-	github.com/docker/cli v20.10.6+incompatible
-	github.com/docker/docker v20.10.6+incompatible
-	github.com/docker/docker-credential-helpers v0.6.3 // indirect
+	github.com/Microsoft/go-winio v0.4.17 // indirect
+	github.com/Microsoft/hcsshim v0.8.18 // indirect
+	github.com/containerd/cgroups v1.0.1 // indirect
+	github.com/containerd/containerd v1.5.5
+	github.com/docker/cli v20.10.8+incompatible
+	github.com/docker/docker v20.10.8+incompatible
+	github.com/docker/docker-credential-helpers v0.6.4 // indirect
 	github.com/docker/go-connections v0.4.0
 	github.com/docker/go-metrics v0.0.1 // indirect
 	github.com/docker/go-units v0.4.0
 	github.com/fvbommel/sortorder v1.0.2 // indirect
-	github.com/go-test/deep v1.0.4
+	github.com/go-test/deep v1.0.7
 	github.com/heroku/docker-registry-client v0.0.0-20190909225348-afc9e1acc3d5
 	github.com/imdario/mergo v0.3.12
 	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de
@@ -24,21 +23,81 @@ require (
 	github.com/moby/sys/mount v0.2.0 // indirect
 	github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf // indirect
 	github.com/morikuni/aec v1.0.0 // indirect
-	github.com/opencontainers/runc v1.0.0-rc93 // indirect
+	github.com/opencontainers/runc v1.0.1 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.1.3
+	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.8.1
 	github.com/theupdateframework/notary v0.7.0 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect
+	golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools v2.2.0+incompatible
-	gotest.tools/v3 v3.0.3 // indirect
-	inet.af/netaddr v0.0.0-20210421205553-78c777480f22
-	k8s.io/client-go v0.21.0
+	inet.af/netaddr v0.0.0-20210903134321-85fa6c94624e
+	k8s.io/client-go v0.22.1
 	sigs.k8s.io/yaml v1.2.0
 )
+
+require github.com/spf13/pflag v1.0.5
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/docker/distribution v2.7.1+incompatible // indirect
+	github.com/docker/go v1.5.1-1.0.20160303222718-d30aec9fd63c // indirect
+	github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
+	github.com/fsnotify/fsnotify v1.4.9 // indirect
+	github.com/go-logr/logr v0.4.0 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/go-cmp v0.5.5 // indirect
+	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/gorilla/mux v1.7.3 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.0 // indirect
+	github.com/json-iterator/go v1.1.11 // indirect
+	github.com/magiconair/properties v1.8.5 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/miekg/pkcs11 v1.0.3 // indirect
+	github.com/mitchellh/mapstructure v1.4.1 // indirect
+	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/moby/sys/mountinfo v0.4.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/opencontainers/go-digest v1.0.0 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/pelletier/go-toml v1.9.3 // indirect
+	github.com/prometheus/client_golang v1.7.1 // indirect
+	github.com/prometheus/client_model v0.2.0 // indirect
+	github.com/prometheus/common v0.10.0 // indirect
+	github.com/prometheus/procfs v0.6.0 // indirect
+	github.com/spf13/afero v1.6.0 // indirect
+	github.com/spf13/cast v1.3.1 // indirect
+	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/subosito/gotenv v1.2.0 // indirect
+	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	go4.org/intern v0.0.0-20210108033219-3eb7198706b2 // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 // indirect
+	golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 // indirect
+	golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 // indirect
+	golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 // indirect
+	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d // indirect
+	golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c // indirect
+	google.golang.org/grpc v1.38.0 // indirect
+	google.golang.org/protobuf v1.26.0 // indirect
+	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.62.0 // indirect
+	k8s.io/apimachinery v0.22.1 // indirect
+	k8s.io/klog/v2 v2.9.0 // indirect
+	k8s.io/utils v0.0.0-20210707171843-4b05e18ac7d9 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.1.2 // indirect
+)

install.sh

@@ -45,7 +45,7 @@ runAsRoot() {
 # verifySupported checks that the os/arch combination is supported for
 # binary builds.
 verifySupported() {
-  local supported="darwin-386\ndarwin-amd64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nwindows-386\nwindows-amd64"
+  local supported="darwin-386\ndarwin-amd64\ndarwin-arm64\nlinux-386\nlinux-amd64\nlinux-arm\nlinux-arm64\nwindows-386\nwindows-amd64"
   if ! echo "${supported}" | grep -q "${OS}-${ARCH}"; then
     echo "No prebuilt binary for ${OS}-${ARCH}."
     echo "To build from source, go to $REPO_URL"

main.go

@@ -21,7 +21,7 @@ THE SOFTWARE.
 */
 package main
 
-import "github.com/rancher/k3d/v4/cmd"
+import "github.com/rancher/k3d/v5/cmd"
 
 func main() {
 	cmd.Execute()

manifest.tmpl

@@ -0,0 +1,27 @@
+image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}latest{{/if}}
+{{#if build.tags}}
+tags:
+{{#each build.tags}}
+  - {{this}}
+{{/each}}
+{{/if}}
+manifests:
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-amd64
+    platform:
+      architecture: amd64
+      os: linux
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm64
+    platform:
+      variant: v8
+      architecture: arm64
+      os: linux
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
+    platform:
+      variant: v7
+      architecture: arm
+      os: linux
+  - image: rancher/k3d:{{#if build.tag}}{{trimPrefix "v" build.tag}}-{{/if}}linux-arm
+    platform:
+      variant: v6
+      architecture: arm
+      os: linux

mkdocs.yml

@@ -25,7 +25,11 @@ theme:
   name: material
   language: en
   features:
-    - tabs
+    - navigation.top # show back to top button
+    - search.suggest # search suggestions: https://squidfunk.github.io/mkdocs-material/setup/setting-up-site-search/#search-suggestions
+    - search.highlight # highlight search term on target page: https://squidfunk.github.io/mkdocs-material/setup/setting-up-site-search/#search-suggestions
+    - navigation.expand
+    - navigation.tabs
   palette:
     - media: "(prefers-color-scheme: light)"
       scheme: default
@@ -71,6 +75,16 @@ plugins:
       type: date
   - awesome-pages # https://squidfunk.github.io/mkdocs-material/plugins/awesome-pages/
   - include-markdown # https://github.com/mondeja/mkdocs-include-markdown-plugin
+  - mike: # Versioned Docs: https://github.com/jimporter/mike
+      version_selector: true   # set to false to leave out the version selector
+      css_dir: static/css             # the directory to put the version selector's CSS
+      javascript_dir: static/js       # the directory to put the version selector's JS
+      canonical_version: null  # the version for <link rel="canonical">; `null` uses the version specified via `mike deploy`
+
+# Extra mkdocs-material settings
+extra:
+  version:
+    provider: mike
 
 # Other Settings
-strict: true # halt processing when a warning is raised
+strict: true # halt processing when a warning is raised

pkg/actions/nodehooks.go

@@ -22,11 +22,16 @@ THE SOFTWARE.
 package actions
 
 import (
+	"bytes"
 	"context"
+	"fmt"
+	"io"
 	"os"
 
-	"github.com/rancher/k3d/v4/pkg/runtimes"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/pkg/runtimes"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+
+	l "github.com/rancher/k3d/v5/pkg/logger"
 )
 
 type WriteFileAction struct {
@@ -39,3 +44,35 @@ type WriteFileAction struct {
 func (act WriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
 	return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, act.Mode, node)
 }
+
+type RewriteFileAction struct {
+	Runtime     runtimes.Runtime
+	Path        string
+	RewriteFunc func([]byte) ([]byte, error)
+	Mode        os.FileMode
+}
+
+func (act RewriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
+	reader, err := act.Runtime.ReadFromNode(ctx, act.Path, node)
+	if err != nil {
+		return fmt.Errorf("runtime failed to read '%s' from node '%s': %w", act.Path, node.Name, err)
+	}
+	defer reader.Close()
+
+	file, err := io.ReadAll(reader)
+	if err != nil {
+		return fmt.Errorf("failed to read file: %w", err)
+	}
+
+	file = bytes.Trim(file[512:], "\x00") // trim control characters, etc.
+
+	file, err = act.RewriteFunc(file)
+	if err != nil {
+		return fmt.Errorf("error while rewriting %s in %s: %w", act.Path, node.Name, err)
+	}
+
+	l.Log().Tracef("Rewritten:\n%s", string(file))
+
+	return act.Runtime.WriteToNode(ctx, file, act.Path, act.Mode, node)
+
+}

pkg/client/cluster.go

@@ -22,13 +22,16 @@ THE SOFTWARE.
 package client
 
 import (
+	"bytes"
 	"context"
 	_ "embed"
 	"errors"
 	"fmt"
-	"io/ioutil"
+	"io"
+	"os"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	gort "runtime"
@@ -36,16 +39,18 @@ import (
 	"github.com/docker/go-connections/nat"
 	"github.com/imdario/mergo"
 	copystruct "github.com/mitchellh/copystructure"
-	"github.com/rancher/k3d/v4/pkg/actions"
-	config "github.com/rancher/k3d/v4/pkg/config/v1alpha3"
-	k3drt "github.com/rancher/k3d/v4/pkg/runtimes"
-	"github.com/rancher/k3d/v4/pkg/runtimes/docker"
-	runtimeErr "github.com/rancher/k3d/v4/pkg/runtimes/errors"
-	"github.com/rancher/k3d/v4/pkg/types"
-	k3d "github.com/rancher/k3d/v4/pkg/types"
-	"github.com/rancher/k3d/v4/pkg/types/k3s"
-	"github.com/rancher/k3d/v4/pkg/util"
-	log "github.com/sirupsen/logrus"
+	"github.com/rancher/k3d/v5/pkg/actions"
+	config "github.com/rancher/k3d/v5/pkg/config/v1alpha3"
+	l "github.com/rancher/k3d/v5/pkg/logger"
+	k3drt "github.com/rancher/k3d/v5/pkg/runtimes"
+	"github.com/rancher/k3d/v5/pkg/runtimes/docker"
+	runtimeErr "github.com/rancher/k3d/v5/pkg/runtimes/errors"
+	"github.com/rancher/k3d/v5/pkg/types"
+	k3d "github.com/rancher/k3d/v5/pkg/types"
+	"github.com/rancher/k3d/v5/pkg/types/k3s"
+	"github.com/rancher/k3d/v5/pkg/util"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sync/errgroup"
 	"gopkg.in/yaml.v2"
 )
 
@@ -58,6 +63,9 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 		return fmt.Errorf("Failed Cluster Preparation: %+v", err)
 	}
 
+	// Create tools-node for later steps
+	go EnsureToolsNode(ctx, runtime, &clusterConfig.Cluster)
+
 	/*
 	 * Step 1: Create Containers
 	 */
@@ -68,15 +76,20 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 	/*
 	 * Step 2: Pre-Start Configuration
 	 */
-	// TODO: ClusterRun: add cluster configuration step here
+	envInfo, err := GatherEnvironmentInfo(ctx, runtime, &clusterConfig.Cluster)
+	if err != nil {
+		return fmt.Errorf("failed to gather environment information used for cluster creation: %w", err)
+	}
 
 	/*
 	 * Step 3: Start Containers
 	 */
 	if err := ClusterStart(ctx, runtime, &clusterConfig.Cluster, k3d.ClusterStartOpts{
-		WaitForServer: clusterConfig.ClusterCreateOpts.WaitForServer,
-		Timeout:       clusterConfig.ClusterCreateOpts.Timeout, // TODO: here we should consider the time used so far
-		NodeHooks:     clusterConfig.ClusterCreateOpts.NodeHooks,
+		WaitForServer:   clusterConfig.ClusterCreateOpts.WaitForServer,
+		Timeout:         clusterConfig.ClusterCreateOpts.Timeout, // TODO: here we should consider the time used so far
+		NodeHooks:       clusterConfig.ClusterCreateOpts.NodeHooks,
+		EnvironmentInfo: envInfo,
+		Intent:          k3d.IntentClusterCreate,
 	}); err != nil {
 		return fmt.Errorf("Failed Cluster Start: %+v", err)
 	}
@@ -88,19 +101,10 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
 	 * Additional Cluster Preparation *
 	 **********************************/
 
-	/*
-	 * Networking Magic
-	 */
-
-	// add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
-	if !clusterConfig.ClusterCreateOpts.PrepDisableHostIPInjection {
-		prepInjectHostIP(ctx, runtime, &clusterConfig.Cluster)
-	}
-
 	// create the registry hosting configmap
 	if len(clusterConfig.ClusterCreateOpts.Registries.Use) > 0 {
 		if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil {
-			log.Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err)
+			l.Log().Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err)
 		}
 	}
 
@@ -147,14 +151,14 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 
 	// Ensure referenced registries
 	for _, reg := range clusterConfig.ClusterCreateOpts.Registries.Use {
-		log.Debugf("Trying to find registry %s", reg.Host)
+		l.Log().Debugf("Trying to find registry %s", reg.Host)
 		regNode, err := runtime.GetNode(ctx, &k3d.Node{Name: reg.Host})
 		if err != nil {
 			return fmt.Errorf("Failed to find registry node '%s': %+v", reg.Host, err)
 		}
 		regFromNode, err := RegistryFromNode(regNode)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to translate node to registry spec: %w", err)
 		}
 		*reg = *regFromNode
 	}
@@ -172,7 +176,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 	}
 
 	// Use existing registries (including the new one, if created)
-	log.Tracef("Using Registries: %+v", clusterConfig.ClusterCreateOpts.Registries.Use)
+	l.Log().Tracef("Using Registries: %+v", clusterConfig.ClusterCreateOpts.Registries.Use)
 
 	var registryConfig *k3s.Registry
 
@@ -199,7 +203,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 		if err != nil {
 			return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err)
 		}
-		log.Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm))
+		l.Log().Tracef("Writing LocalRegistryHosting YAML:\n%s", string(regCm))
 		clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{
 			Stage: k3d.LifecycleStagePreStart,
 			Action: actions.WriteFileAction{
@@ -219,7 +223,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 			if err := RegistryMergeConfig(ctx, registryConfig, clusterConfig.ClusterCreateOpts.Registries.Config); err != nil {
 				return err
 			}
-			log.Tracef("Merged registry config: %+v", registryConfig)
+			l.Log().Tracef("Merged registry config: %+v", registryConfig)
 		} else {
 			registryConfig = clusterConfig.ClusterCreateOpts.Registries.Config
 		}
@@ -246,7 +250,7 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf
 
 // ClusterPrepNetwork creates a new cluster network, if needed or sets everything up to re-use an existing network
 func ClusterPrepNetwork(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterCreateOpts *k3d.ClusterCreateOpts) error {
-	log.Infoln("Prep: Network")
+	l.Log().Infoln("Prep: Network")
 
 	// error out if external cluster network should be used but no name was set
 	if cluster.Network.Name == "" && cluster.Network.External {
@@ -272,8 +276,7 @@ func ClusterPrepNetwork(ctx context.Context, runtime k3drt.Runtime, cluster *k3d
 	// create cluster network or use an existing one
 	network, networkExists, err := runtime.CreateNetworkIfNotPresent(ctx, &cluster.Network)
 	if err != nil {
-		log.Errorln("Failed to create cluster network")
-		return err
+		return fmt.Errorf("failed to create cluster network: %w", err)
 	}
 	cluster.Network = *network
 	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkID] = network.ID
@@ -281,7 +284,7 @@ func ClusterPrepNetwork(ctx context.Context, runtime k3drt.Runtime, cluster *k3d
 	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkIPRange] = cluster.Network.IPAM.IPPrefix.String()
 	clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal] = strconv.FormatBool(cluster.Network.External)
 	if networkExists {
-		log.Infof("Re-using existing network '%s' (%s)", network.Name, network.ID)
+		l.Log().Infof("Re-using existing network '%s' (%s)", network.Name, network.ID)
 		clusterCreateOpts.GlobalLabels[k3d.LabelNetworkExternal] = "true" // if the network wasn't created, we say that it's managed externally (important for cluster deletion)
 	}
 
@@ -295,11 +298,11 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
 	 */
 	imageVolumeName := fmt.Sprintf("%s-%s-images", k3d.DefaultObjectNamePrefix, cluster.Name)
 	if err := runtime.CreateVolume(ctx, imageVolumeName, map[string]string{k3d.LabelClusterName: cluster.Name}); err != nil {
-		log.Errorf("Failed to create image volume '%s' for cluster '%s'", imageVolumeName, cluster.Name)
-		return err
+		return fmt.Errorf("failed to create image volume '%s' for cluster '%s': %w", imageVolumeName, cluster.Name, err)
 	}
 
 	clusterCreateOpts.GlobalLabels[k3d.LabelImageVolume] = imageVolumeName
+	cluster.ImageVolume = imageVolumeName
 
 	// attach volume to nodes
 	for _, node := range cluster.Nodes {
@@ -313,7 +316,7 @@ func ClusterPrepImageVolume(ctx context.Context, runtime k3drt.Runtime, cluster
 // - a docker network
 func ClusterCreate(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterCreateOpts *k3d.ClusterCreateOpts) error {
 
-	log.Tracef(`
+	l.Log().Tracef(`
 ===== Creating Cluster =====
 
 Runtime:
@@ -344,16 +347,16 @@ ClusterCreatOpts:
 	 */
 	if cluster.KubeAPI.Host == k3d.DefaultAPIHost && runtime == k3drt.Docker {
 		if gort.GOOS == "windows" || gort.GOOS == "darwin" {
-			log.Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS)
+			l.Log().Tracef("Running on %s: checking if it's using docker-machine", gort.GOOS)
 			machineIP, err := runtime.(docker.Docker).GetDockerMachineIP()
 			if err != nil {
-				log.Warnf("Using docker-machine, but failed to get it's IP: %+v", err)
+				l.Log().Warnf("Using docker-machine, but failed to get it's IP: %+v", err)
 			} else if machineIP != "" {
-				log.Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP)
+				l.Log().Infof("Using the docker-machine IP %s to connect to the Kubernetes API", machineIP)
 				cluster.KubeAPI.Host = machineIP
 				cluster.KubeAPI.Binding.HostIP = machineIP
 			} else {
-				log.Traceln("Not using docker-machine")
+				l.Log().Traceln("Not using docker-machine")
 			}
 		}
 	}
@@ -377,7 +380,7 @@ ClusterCreatOpts:
 	// connection url is always the name of the first server node (index 0) // TODO: change this to the server loadbalancer
 	connectionURL := fmt.Sprintf("https://%s:%s", GenerateNodeName(cluster.Name, k3d.ServerRole, 0), k3d.DefaultAPIPort)
 	clusterCreateOpts.GlobalLabels[k3d.LabelClusterURL] = connectionURL
-	clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("K3S_TOKEN=%s", cluster.Token))
+	clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("%s=%s", k3s.EnvClusterToken, cluster.Token))
 
 	nodeSetup := func(node *k3d.Node) error {
 		// cluster specific settings
@@ -399,7 +402,7 @@ ClusterCreatOpts:
 			if cluster.Network.IPAM.Managed {
 				ip, err := GetIP(ctx, runtime, &cluster.Network)
 				if err != nil {
-					return err
+					return fmt.Errorf("failed to find free IP in network %s: %w", cluster.Network.Name, err)
 				}
 				cluster.Network.IPAM.IPsUsed = append(cluster.Network.IPAM.IPsUsed, ip) // make sure that we're not reusing the same IP next time
 				node.IP.Static = true
@@ -411,12 +414,12 @@ ClusterCreatOpts:
 
 			// the cluster has an init server node, but its not this one, so connect it to the init node
 			if cluster.InitNode != nil && !node.ServerOpts.IsInit {
-				node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
+				node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, connectionURL))
 				node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
 			}
 
 		} else if node.Role == k3d.AgentRole {
-			node.Env = append(node.Env, fmt.Sprintf("K3S_URL=%s", connectionURL))
+			node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, connectionURL))
 		}
 
 		node.Networks = []string{cluster.Network.Name}
@@ -424,12 +427,11 @@ ClusterCreatOpts:
 		node.GPURequest = clusterCreateOpts.GPURequest
 
 		// create node
-		log.Infof("Creating node '%s'", node.Name)
+		l.Log().Infof("Creating node '%s'", node.Name)
 		if err := NodeCreate(clusterCreateCtx, runtime, node, k3d.NodeCreateOpts{}); err != nil {
-			log.Errorln("Failed to create node")
-			return err
+			return fmt.Errorf("failed to create node: %w", err)
 		}
-		log.Debugf("Created node '%s'", node.Name)
+		l.Log().Debugf("Created node '%s'", node.Name)
 
 		// start node
 		//return NodeStart(clusterCreateCtx, runtime, node, k3d.NodeStartOpts{PreStartActions: clusterCreateOpts.NodeHookActions})
@@ -441,7 +443,7 @@ ClusterCreatOpts:
 
 	// create init node first
 	if cluster.InitNode != nil {
-		log.Infoln("Creating initializing server node")
+		l.Log().Infoln("Creating initializing server node")
 		cluster.InitNode.Args = append(cluster.InitNode.Args, "--cluster-init")
 		if cluster.InitNode.RuntimeLabels == nil {
 			cluster.InitNode.RuntimeLabels = map[string]string{}
@@ -457,7 +459,7 @@ ClusterCreatOpts:
 		}
 
 		if err := nodeSetup(cluster.InitNode); err != nil {
-			return err
+			return fmt.Errorf("failed init node setup: %w", err)
 		}
 		serverCount++
 
@@ -485,14 +487,14 @@ ClusterCreatOpts:
 		}
 		if node.Role == k3d.ServerRole || node.Role == k3d.AgentRole {
 			if err := nodeSetup(node); err != nil {
-				return err
+				return fmt.Errorf("failed setup of server/agent node %s: %w", node.Name, err)
 			}
 		}
 	}
 
 	// WARN, if there are exactly two server nodes: that means we're using etcd, but don't have fault tolerance
 	if serverCount == 2 {
-		log.Warnln("You're creating 2 server nodes: Please consider creating at least 3 to achieve etcd quorum & fault tolerance")
+		l.Log().Warnln("You're creating 2 server nodes: Please consider creating at least 3 to achieve etcd quorum & fault tolerance")
 	}
 
 	/*
@@ -501,9 +503,10 @@ ClusterCreatOpts:
 	// *** ServerLoadBalancer ***
 	if !clusterCreateOpts.DisableLoadBalancer {
 		if cluster.ServerLoadBalancer == nil {
+			l.Log().Infof("No loadbalancer specified, creating a default one...")
 			lbNode, err := LoadbalancerPrepare(ctx, runtime, cluster, &k3d.LoadbalancerCreateOpts{Labels: clusterCreateOpts.GlobalLabels})
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to prepare loadbalancer: %w", err)
 			}
 			cluster.Nodes = append(cluster.Nodes, lbNode) // append lbNode to list of cluster nodes, so it will be considered during rollback
 		}
@@ -521,7 +524,7 @@ ClusterCreatOpts:
 		// prepare to write config to lb container
 		configyaml, err := yaml.Marshal(cluster.ServerLoadBalancer.Config)
 		if err != nil {
-			return err
+			return fmt.Errorf("failed to marshal loadbalancer config: %w", err)
 		}
 
 		writeLbConfigAction := k3d.NodeHook{
@@ -536,12 +539,11 @@ ClusterCreatOpts:
 
 		cluster.ServerLoadBalancer.Node.HookActions = append(cluster.ServerLoadBalancer.Node.HookActions, writeLbConfigAction)
 
-		log.Infof("Creating LoadBalancer '%s'", cluster.ServerLoadBalancer.Node.Name)
+		l.Log().Infof("Creating LoadBalancer '%s'", cluster.ServerLoadBalancer.Node.Name)
 		if err := NodeCreate(ctx, runtime, cluster.ServerLoadBalancer.Node, k3d.NodeCreateOpts{}); err != nil {
 			return fmt.Errorf("error creating loadbalancer: %v", err)
 		}
-		log.Debugf("Created loadbalancer '%s'", cluster.ServerLoadBalancer.Node.Name)
-		return err
+		l.Log().Debugf("Created loadbalancer '%s'", cluster.ServerLoadBalancer.Node.Name)
 	}
 
 	return nil
@@ -550,18 +552,18 @@ ClusterCreatOpts:
 // ClusterDelete deletes an existing cluster
 func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, opts k3d.ClusterDeleteOpts) error {
 
-	log.Infof("Deleting cluster '%s'", cluster.Name)
+	l.Log().Infof("Deleting cluster '%s'", cluster.Name)
 	cluster, err := ClusterGet(ctx, runtime, cluster)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get cluster: %w", err)
 	}
-	log.Debugf("Cluster Details: %+v", cluster)
+	l.Log().Debugf("Cluster Details: %+v", cluster)
 
 	failed := 0
 	for _, node := range cluster.Nodes {
 		// registry: only delete, if not connected to other networks
 		if node.Role == k3d.RegistryRole && !opts.SkipRegistryCheck {
-			log.Tracef("Registry Node has %d networks: %+v", len(node.Networks), node)
+			l.Log().Tracef("Registry Node has %d networks: %+v", len(node.Networks), node)
 
 			// check if node is connected to other networks, that are not
 			// - the cluster network
@@ -575,21 +577,21 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 				if net == "bridge" || net == "host" {
 					continue
 				}
-				log.Tracef("net: %s", net)
+				l.Log().Tracef("net: %s", net)
 				connectedToOtherNet = true
 				break
 			}
 			if connectedToOtherNet {
-				log.Infof("Registry %s is also connected to other (non-default) networks (%+v), not deleting it...", node.Name, node.Networks)
+				l.Log().Infof("Registry %s is also connected to other (non-default) networks (%+v), not deleting it...", node.Name, node.Networks)
 				if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil {
-					log.Warnf("Failed to disconnect registry %s from cluster network %s", node.Name, cluster.Network.Name)
+					l.Log().Warnf("Failed to disconnect registry %s from cluster network %s", node.Name, cluster.Network.Name)
 				}
 				continue
 			}
 		}
 
 		if err := NodeDelete(ctx, runtime, node, k3d.NodeDeleteOpts{SkipLBUpdate: true}); err != nil {
-			log.Warningf("Failed to delete node '%s': Try to delete it manually", node.Name)
+			l.Log().Warningf("Failed to delete node '%s': Try to delete it manually", node.Name)
 			failed++
 			continue
 		}
@@ -598,48 +600,48 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 	// Delete the cluster network, if it was created for/by this cluster (and if it's not in use anymore)
 	if cluster.Network.Name != "" {
 		if !cluster.Network.External {
-			log.Infof("Deleting cluster network '%s'", cluster.Network.Name)
+			l.Log().Infof("Deleting cluster network '%s'", cluster.Network.Name)
 			if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil {
 				if errors.Is(err, runtimeErr.ErrRuntimeNetworkNotEmpty) { // there are still containers connected to that network
 
 					connectedNodes, err := runtime.GetNodesInNetwork(ctx, cluster.Network.Name) // check, if there are any k3d nodes connected to the cluster
 					if err != nil {
-						log.Warningf("Failed to check cluster network for connected nodes: %+v", err)
+						l.Log().Warningf("Failed to check cluster network for connected nodes: %+v", err)
 					}
 
 					if len(connectedNodes) > 0 { // there are still k3d-managed containers (aka nodes) connected to the network
 						connectedRegistryNodes := util.FilterNodesByRole(connectedNodes, k3d.RegistryRole)
 						if len(connectedRegistryNodes) == len(connectedNodes) { // only registry node(s) left in the network
 							for _, node := range connectedRegistryNodes {
-								log.Debugf("Disconnecting registry node %s from the network...", node.Name)
+								l.Log().Debugf("Disconnecting registry node %s from the network...", node.Name)
 								if err := runtime.DisconnectNodeFromNetwork(ctx, node, cluster.Network.Name); err != nil {
-									log.Warnf("Failed to disconnect registry %s from network %s", node.Name, cluster.Network.Name)
+									l.Log().Warnf("Failed to disconnect registry %s from network %s", node.Name, cluster.Network.Name)
 								} else {
 									if err := runtime.DeleteNetwork(ctx, cluster.Network.Name); err != nil {
-										log.Warningf("Failed to delete cluster network, even after disconnecting registry node(s): %+v", err)
+										l.Log().Warningf("Failed to delete cluster network, even after disconnecting registry node(s): %+v", err)
 									}
 								}
 							}
 						} else { // besides the registry node(s), there are still other nodes... maybe they still need a registry
-							log.Debugf("There are some non-registry nodes left in the network")
+							l.Log().Debugf("There are some non-registry nodes left in the network")
 						}
 					} else {
-						log.Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name)
+						l.Log().Warningf("Failed to delete cluster network '%s' because it's still in use: is there another cluster using it?", cluster.Network.Name)
 					}
 				} else {
-					log.Warningf("Failed to delete cluster network '%s': '%+v'", cluster.Network.Name, err)
+					l.Log().Warningf("Failed to delete cluster network '%s': '%+v'", cluster.Network.Name, err)
 				}
 			}
 		} else if cluster.Network.External {
-			log.Debugf("Skip deletion of cluster network '%s' because it's managed externally", cluster.Network.Name)
+			l.Log().Debugf("Skip deletion of cluster network '%s' because it's managed externally", cluster.Network.Name)
 		}
 	}
 
 	// delete image volume
 	if cluster.ImageVolume != "" {
-		log.Infof("Deleting image volume '%s'", cluster.ImageVolume)
+		l.Log().Infof("Deleting image volume '%s'", cluster.ImageVolume)
 		if err := runtime.DeleteVolume(ctx, cluster.ImageVolume); err != nil {
-			log.Warningf("Failed to delete image volume '%s' of cluster '%s': Try to delete it manually", cluster.ImageVolume, cluster.Name)
+			l.Log().Warningf("Failed to delete image volume '%s' of cluster '%s': Try to delete it manually", cluster.ImageVolume, cluster.Name)
 		}
 	}
 
@@ -652,26 +654,25 @@ func ClusterDelete(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clus
 
 // ClusterList returns a list of all existing clusters
 func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, error) {
-	log.Traceln("Listing Clusters...")
+	l.Log().Traceln("Listing Clusters...")
 	nodes, err := runtime.GetNodesByLabel(ctx, k3d.DefaultRuntimeLabels)
 	if err != nil {
-		log.Errorln("Failed to get clusters")
-		return nil, err
+		return nil, fmt.Errorf("runtime failed to list nodes: %w", err)
 	}
 
-	log.Debugf("Found %d nodes", len(nodes))
-	if log.GetLevel() == log.TraceLevel {
+	l.Log().Debugf("Found %d nodes", len(nodes))
+	if l.Log().GetLevel() == logrus.TraceLevel {
 		for _, node := range nodes {
-			log.Tracef("Found node %s of role %s", node.Name, node.Role)
+			l.Log().Tracef("Found node %s of role %s", node.Name, node.Role)
 		}
 	}
 
 	nodes = NodeFilterByRoles(nodes, k3d.ClusterInternalNodeRoles, k3d.ClusterExternalNodeRoles)
 
-	log.Tracef("Found %d cluster-internal nodes", len(nodes))
-	if log.GetLevel() == log.TraceLevel {
+	l.Log().Tracef("Found %d cluster-internal nodes", len(nodes))
+	if l.Log().GetLevel() == logrus.TraceLevel {
 		for _, node := range nodes {
-			log.Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName])
+			l.Log().Tracef("Found cluster-internal node %s of role %s belonging to cluster %s", node.Name, node.Role, node.RuntimeLabels[k3d.LabelClusterName])
 		}
 	}
 
@@ -698,11 +699,11 @@ func ClusterList(ctx context.Context, runtime k3drt.Runtime) ([]*k3d.Cluster, er
 	// enrich cluster structs with label values
 	for _, cluster := range clusters {
 		if err := populateClusterFieldsFromLabels(cluster); err != nil {
-			log.Warnf("Failed to populate cluster fields from node label values for cluster '%s'", cluster.Name)
-			log.Warnln(err)
+			l.Log().Warnf("Failed to populate cluster fields from node label values for cluster '%s'", cluster.Name)
+			l.Log().Warnln(err)
 		}
 	}
-	log.Debugf("Found %d clusters", len(clusters))
+	l.Log().Debugf("Found %d clusters", len(clusters))
 	return clusters, nil
 }
 
@@ -755,7 +756,7 @@ func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster
 	// get nodes that belong to the selected cluster
 	nodes, err := runtime.GetNodesByLabel(ctx, map[string]string{k3d.LabelClusterName: cluster.Name})
 	if err != nil {
-		log.Errorf("Failed to get nodes for cluster '%s'", cluster.Name)
+		l.Log().Errorf("Failed to get nodes for cluster '%s': %v", cluster.Name, err)
 	}
 
 	if len(nodes) == 0 {
@@ -793,18 +794,17 @@ func ClusterGet(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster
 			}
 		}
 
-		if cluster.ServerLoadBalancer.Node != nil {
+		if cluster.ServerLoadBalancer != nil && cluster.ServerLoadBalancer.Node != nil {
 			lbcfg, err := GetLoadbalancerConfig(ctx, runtime, cluster)
 			if err != nil {
-				return cluster, fmt.Errorf("error getting loadbalancer config for cluster %s: %w", cluster.Name, err)
+				l.Log().Errorf("error getting loadbalancer config from %s: %v", cluster.ServerLoadBalancer.Node.Name, err)
 			}
 			cluster.ServerLoadBalancer.Config = &lbcfg
 		}
 	}
 
 	if err := populateClusterFieldsFromLabels(cluster); err != nil {
-		log.Warnf("Failed to populate cluster fields from node labels")
-		log.Warnln(err)
+		l.Log().Warnf("Failed to populate cluster fields from node labels: %v", err)
 	}
 
 	return cluster, nil
@@ -820,12 +820,16 @@ func GenerateNodeName(cluster string, role k3d.Role, suffix int) string {
 }
 
 // ClusterStart starts a whole cluster (i.e. all nodes of the cluster)
-func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, startClusterOpts types.ClusterStartOpts) error {
-	log.Infof("Starting cluster '%s'", cluster.Name)
+func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts types.ClusterStartOpts) error {
+	l.Log().Infof("Starting cluster '%s'", cluster.Name)
 
-	if startClusterOpts.Timeout > 0*time.Second {
+	if clusterStartOpts.Intent == "" {
+		clusterStartOpts.Intent = k3d.IntentClusterStart
+	}
+
+	if clusterStartOpts.Timeout > 0*time.Second {
 		var cancel context.CancelFunc
-		ctx, cancel = context.WithTimeout(ctx, startClusterOpts.Timeout)
+		ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout)
 		defer cancel()
 	}
 
@@ -848,28 +852,21 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 		}
 	}
 
-	// TODO: remove trace logs below
-	log.Traceln("Servers before sort:")
-	for i, n := range servers {
-		log.Tracef("Server %d - %s", i, n.Name)
-	}
+	// sort list of servers for properly ordered sequential start
 	sort.Slice(servers, func(i, j int) bool {
 		return servers[i].Name < servers[j].Name
 	})
-	log.Traceln("Servers after sort:")
-	for i, n := range servers {
-		log.Tracef("Server %d - %s", i, n.Name)
-	}
 
 	/*
 	 * Init Node
 	 */
 	if initNode != nil {
-		log.Infoln("Starting the initializing server...")
-		if err := NodeStart(ctx, runtime, initNode, k3d.NodeStartOpts{
+		l.Log().Infoln("Starting the initializing server...")
+		if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
 			Wait:            true, // always wait for the init node
-			NodeHooks:       startClusterOpts.NodeHooks,
-			ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
+			NodeHooks:       clusterStartOpts.NodeHooks,
+			ReadyLogMessage: types.GetReadyLogMessage(initNode, clusterStartOpts.Intent), // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
+			EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
 		}); err != nil {
 			return fmt.Errorf("Failed to start initializing server node: %+v", err)
 		}
@@ -878,13 +875,13 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	/*
 	 * Server Nodes
 	 */
-	log.Infoln("Starting servers...")
-	nodeStartOpts := k3d.NodeStartOpts{
-		Wait:      true,
-		NodeHooks: startClusterOpts.NodeHooks,
-	}
+	l.Log().Infoln("Starting servers...")
 	for _, serverNode := range servers {
-		if err := NodeStart(ctx, runtime, serverNode, nodeStartOpts); err != nil {
+		if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
+			Wait:            true,
+			NodeHooks:       append(clusterStartOpts.NodeHooks, serverNode.HookActions...),
+			EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
+		}); err != nil {
 			return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
 		}
 	}
@@ -893,37 +890,126 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 	 * Agent Nodes
 	 */
 
-	failedAgents := 0
+	agentWG, aCtx := errgroup.WithContext(ctx)
 
-	log.Infoln("Starting agents...")
+	l.Log().Infoln("Starting agents...")
 	for _, agentNode := range agents {
-		if err := NodeStart(ctx, runtime, agentNode, nodeStartOpts); err != nil {
-			log.Warnf("Failed to start agent %s: %+v", agentNode.Name, err)
-			failedAgents++
-		}
+		currentAgentNode := agentNode
+		agentWG.Go(func() error {
+			return NodeStart(aCtx, runtime, currentAgentNode, &k3d.NodeStartOpts{
+				Wait:            true,
+				NodeHooks:       clusterStartOpts.NodeHooks,
+				EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
+			})
+		})
+	}
+	if err := agentWG.Wait(); err != nil {
+		return fmt.Errorf("Failed to add one or more agents: %w", err)
 	}
 
 	/*
 	 * Auxiliary/Helper Nodes
 	 */
 
-	log.Infoln("Starting helpers...")
-	failedHelpers := 0
+	helperWG, hCtx := errgroup.WithContext(ctx)
+	l.Log().Infoln("Starting helpers...")
 	for _, helperNode := range aux {
-		nodeStartOpts := k3d.NodeStartOpts{
-			NodeHooks: helperNode.HookActions,
-		}
-		if helperNode.Role == k3d.LoadBalancerRole {
-			nodeStartOpts.Wait = true
-		}
-		if err := NodeStart(ctx, runtime, helperNode, nodeStartOpts); err != nil {
-			log.Warnf("Failed to start helper %s: %+v", helperNode.Name, err)
-			failedHelpers++
-		}
+		currentHelperNode := helperNode
+
+		helperWG.Go(func() error {
+			nodeStartOpts := &k3d.NodeStartOpts{
+				NodeHooks:       currentHelperNode.HookActions,
+				EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
+			}
+			if currentHelperNode.Role == k3d.LoadBalancerRole {
+				nodeStartOpts.Wait = true
+			}
+
+			return NodeStart(hCtx, runtime, currentHelperNode, nodeStartOpts)
+		})
 	}
 
-	if failedAgents+failedHelpers > 0 {
-		log.Warnf("%d non-critical (agent or helper) nodes failed to start. You may want to start them manually.", failedAgents+failedHelpers)
+	if err := helperWG.Wait(); err != nil {
+		return fmt.Errorf("Failed to add one or more helper nodes: %w", err)
+	}
+
+	/*
+	 * Additional Cluster Preparation (post start)
+	 */
+
+	postStartErrgrp, postStartErrgrpCtx := errgroup.WithContext(ctx)
+
+	/*** DNS ***/
+
+	// add host.k3d.internal record to /etc/hosts in all nodes
+	postStartErrgrp.Go(func() error {
+		return prepInjectHostIP(postStartErrgrpCtx, runtime, cluster, &clusterStartOpts)
+	})
+
+	postStartErrgrp.Go(func() error {
+
+		hosts := fmt.Sprintf("%s %s\n", clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord)
+
+		net, err := runtime.GetNetwork(ctx, &cluster.Network)
+		if err != nil {
+			return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
+		}
+		for _, member := range net.Members {
+			hosts += fmt.Sprintf("%s %s\n", member.IP.String(), member.Name)
+		}
+
+		l.Log().Infof("Injecting records for host.k3d.internal and for %d network members into CoreDNS configmap...", len(net.Members))
+		act := actions.RewriteFileAction{
+			Runtime: runtime,
+			Path:    "/var/lib/rancher/k3s/server/manifests/coredns.yaml",
+			Mode:    0744,
+			RewriteFunc: func(input []byte) ([]byte, error) {
+				split, err := util.SplitYAML(input)
+				if err != nil {
+					return nil, fmt.Errorf("error splitting yaml: %w", err)
+				}
+
+				var outputBuf bytes.Buffer
+				outputEncoder := yaml.NewEncoder(&outputBuf)
+
+				for _, d := range split {
+					var doc map[string]interface{}
+					if err := yaml.Unmarshal(d, &doc); err != nil {
+						return nil, err
+					}
+					if kind, ok := doc["kind"]; ok {
+						if strings.ToLower(kind.(string)) == "configmap" {
+							configmapData := doc["data"].(map[interface{}]interface{})
+							configmapData["NodeHosts"] = hosts
+						}
+					}
+					if err := outputEncoder.Encode(doc); err != nil {
+						return nil, err
+					}
+				}
+				outputEncoder.Close()
+				return outputBuf.Bytes(), nil
+			},
+		}
+
+		// get the first server in the list and run action on it once it's ready for it
+		for _, n := range cluster.Nodes {
+			if n.Role == k3d.ServerRole {
+				ts, err := time.Parse("2006-01-02T15:04:05.999999999Z", n.State.Started)
+				if err != nil {
+					return err
+				}
+				if err := NodeWaitForLogMessage(ctx, runtime, n, "Cluster dns configmap", ts.Truncate(time.Second)); err != nil {
+					return err
+				}
+				return act.Run(ctx, n)
+			}
+		}
+		return nil
+	})
+
+	if err := postStartErrgrp.Wait(); err != nil {
+		return fmt.Errorf("error during post-start cluster preparation: %w", err)
 	}
 
 	return nil
@@ -931,12 +1017,12 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
 
 // ClusterStop stops a whole cluster (i.e. all nodes of the cluster)
 func ClusterStop(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
-	log.Infof("Stopping cluster '%s'", cluster.Name)
+	l.Log().Infof("Stopping cluster '%s'", cluster.Name)
 
 	failed := 0
 	for _, node := range cluster.Nodes {
 		if err := runtime.StopNode(ctx, node); err != nil {
-			log.Warningf("Failed to stop node '%s': Try to stop it manually", node.Name)
+			l.Log().Warningf("Failed to stop node '%s': Try to stop it manually", node.Name)
 			failed++
 			continue
 		}
@@ -945,6 +1031,8 @@ func ClusterStop(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluste
 	if failed > 0 {
 		return fmt.Errorf("Failed to stop %d nodes: Try to stop them manually", failed)
 	}
+
+	l.Log().Infof("Stopped cluster '%s'", cluster.Name)
 	return nil
 }
 
@@ -956,60 +1044,85 @@ func SortClusters(clusters []*k3d.Cluster) []*k3d.Cluster {
 	return clusters
 }
 
-// prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
-func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) {
-	log.Infoln("(Optional) Trying to get IP of the docker host and inject it into the cluster as 'host.k3d.internal' for easy access")
-	hostIP, err := GetHostIP(ctx, runtime, cluster)
-	if err != nil {
-		log.Warnf("Failed to get HostIP: %+v", err)
-	}
-	if hostIP != nil {
-		hostRecordSuccessMessage := ""
-		etcHostsFailureCount := 0
-		hostsEntry := fmt.Sprintf("%s %s", hostIP, k3d.DefaultK3dInternalHostRecord)
-		log.Debugf("Adding extra host entry '%s'...", hostsEntry)
-		for _, node := range cluster.Nodes {
-			if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)}); err != nil {
-				log.Warnf("Failed to add extra entry '%s' to /etc/hosts in node '%s'", hostsEntry, node.Name)
-				etcHostsFailureCount++
-			}
-		}
-		if etcHostsFailureCount < len(cluster.Nodes) {
-			hostRecordSuccessMessage += fmt.Sprintf("Successfully added host record to /etc/hosts in %d/%d nodes", (len(cluster.Nodes) - etcHostsFailureCount), len(cluster.Nodes))
-		}
-
-		patchCmd := `patch=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s+host\.k3d\.internal$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | busybox xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch"`
-		successInjectCoreDNSEntry := false
-		for _, node := range cluster.Nodes {
-
-			if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole {
-				logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd})
-				if err == nil {
-					successInjectCoreDNSEntry = true
-					break
-				} else {
-					msg := fmt.Sprintf("error patching the CoreDNS ConfigMap to include entry '%s': %+v", hostsEntry, err)
-					readlogs, err := ioutil.ReadAll(logreader)
-					if err != nil {
-						log.Debugf("error reading the logs from failed CoreDNS patch exec process in node %s: %v", node.Name, err)
-					} else {
-						msg += fmt.Sprintf("\nLogs: %s", string(readlogs))
-					}
-					log.Debugln(msg)
-				}
-			}
-		}
-		if successInjectCoreDNSEntry == false {
-			log.Warnf("Failed to patch CoreDNS ConfigMap to include entry '%s' (see debug logs)", hostsEntry)
+// corednsAddHost adds a host entry to the CoreDNS configmap if it doesn't exist (a host entry is a single line of the form "IP HOST")
+func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, ip string, name string) error {
+	retries := 3
+	if v, ok := os.LookupEnv(k3d.K3dEnvDebugCorednsRetries); ok && v != "" {
+		l.Log().Debugf("Running with %s=%s", k3d.K3dEnvDebugCorednsRetries, v)
+		if r, err := strconv.Atoi(v); err == nil {
+			retries = r
 		} else {
-			hostRecordSuccessMessage += " and to the CoreDNS ConfigMap"
+			return fmt.Errorf("Invalid value set for env var %s (%s): %w", k3d.K3dEnvDebugCorednsRetries, v, err)
 		}
-
-		if hostRecordSuccessMessage != "" {
-			log.Infoln(hostRecordSuccessMessage)
-		}
-
 	}
+
+	// select any server node
+	var node *k3d.Node
+	for _, n := range cluster.Nodes {
+		if n.Role == k3d.ServerRole {
+			node = n
+		}
+	}
+
+	hostsEntry := fmt.Sprintf("%s %s", ip, name)
+	patchCmd := `patch=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s` + name + `$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | busybox xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch"`
+	successInjectCoreDNSEntry := false
+
+	// try 3 (or K3D_DEBUG_COREDNS_RETRIES value) times, as e.g. on cluster startup it may take some time for the Configmap to be available and the server to be responsive
+	for i := 0; i < retries; i++ {
+		l.Log().Debugf("Running CoreDNS patch in node %s to add %s (try %d/%d)...", node.Name, hostsEntry, i, retries)
+		logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd})
+		if err == nil {
+			successInjectCoreDNSEntry = true
+			break
+		} else {
+			msg := fmt.Sprintf("(try %d/%d) error patching the CoreDNS ConfigMap to include entry '%s': %+v", i, retries, hostsEntry, err)
+			if logreader != nil {
+				readlogs, err := io.ReadAll(logreader)
+				if err != nil {
+					l.Log().Debugf("(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: %v", i, retries, node.Name, err)
+				} else {
+					msg += fmt.Sprintf("\nLogs: %s", string(readlogs))
+				}
+			} else {
+				l.Log().Debugf("(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: no logreader returned for exec process", i, retries, node.Name)
+			}
+			l.Log().Debugln(msg)
+			time.Sleep(1 * time.Second)
+		}
+	}
+	if !successInjectCoreDNSEntry {
+		return fmt.Errorf("failed to patch CoreDNS ConfigMap to include entry '%s' (%d tries, see debug logs)", hostsEntry, retries)
+	}
+	l.Log().Debugf("Successfully patched CoreDNS Configmap with record '%s'", hostsEntry)
+	return nil
+}
+
+// prepInjectHostIP adds /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
+func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts *k3d.ClusterStartOpts) error {
+	if cluster.Network.Name == "host" {
+		l.Log().Tracef("Not injecting hostIP as clusternetwork is 'host'")
+		return nil
+	}
+
+	hostIP := clusterStartOpts.EnvironmentInfo.HostGateway
+	hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
+	l.Log().Infof("Injecting '%s' into /etc/hosts of all nodes...", hostsEntry)
+
+	// entry in /etc/hosts
+	errgrp, errgrpctx := errgroup.WithContext(ctx)
+	for _, node := range cluster.Nodes {
+		n := node
+		errgrp.Go(func() error {
+			return runtime.ExecInNode(errgrpctx, n, []string{"sh", "-c", fmt.Sprintf("echo '%s' >> /etc/hosts", hostsEntry)})
+		})
+	}
+	if err := errgrp.Wait(); err != nil {
+		return fmt.Errorf("failed to add hosts entry %s: %w", hostsEntry, err)
+	}
+	l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
+
+	return nil
 }
 
 func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
@@ -1021,12 +1134,12 @@ func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.
 				success = true
 				break
 			} else {
-				log.Debugf("Failed to create LocalRegistryHosting ConfigMap in node %s: %+v", node.Name, err)
+				l.Log().Debugf("Failed to create LocalRegistryHosting ConfigMap in node %s: %+v", node.Name, err)
 			}
 		}
 	}
 	if success == false {
-		log.Warnf("Failed to create LocalRegistryHosting ConfigMap")
+		l.Log().Warnf("Failed to create LocalRegistryHosting ConfigMap")
 	}
 	return nil
 }
@@ -1063,7 +1176,7 @@ func ClusterEditChangesetSimple(ctx context.Context, runtime k3drt.Runtime, clus
 		for _, portWithNodeFilters := range changeset.Ports {
 			filteredNodes, err := util.FilterNodesWithSuffix(nodeList, portWithNodeFilters.NodeFilters)
 			if err != nil {
-				return err
+				return fmt.Errorf("failed to filter nodes: %w", err)
 			}
 
 			for suffix := range filteredNodes {
@@ -1083,12 +1196,12 @@ func ClusterEditChangesetSimple(ctx context.Context, runtime k3drt.Runtime, clus
 		}
 	}
 
-	log.Debugf("ORIGINAL:\n> Ports: %+v\n> Config: %+v\nCHANGESET:\n> Ports: %+v\n> Config: %+v", existingLB.Node.Ports, existingLB.Config, lbChangeset.Node.Ports, lbChangeset.Config)
+	l.Log().Debugf("ORIGINAL:\n> Ports: %+v\n> Config: %+v\nCHANGESET:\n> Ports: %+v\n> Config: %+v", existingLB.Node.Ports, existingLB.Config, lbChangeset.Node.Ports, lbChangeset.Config)
 
 	// prepare to write config to lb container
 	configyaml, err := yaml.Marshal(lbChangeset.Config)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to marshal loadbalancer config changeset: %w", err)
 	}
 	writeLbConfigAction := k3d.NodeHook{
 		Stage: k3d.LifecycleStagePreStart,

pkg/client/clusterName.go

@@ -24,7 +24,7 @@ package client
 import (
 	"fmt"
 
-	"github.com/rancher/k3d/v4/pkg/types"
+	"github.com/rancher/k3d/v5/pkg/types"
 )
 
 // CheckName ensures that a cluster name is also a valid host name according to RFC 1123.