From 02b80fecfc6a3340667057ee0b802beb0e118cd6 Mon Sep 17 00:00:00 2001 From: Jamie van Dyke Date: Tue, 29 Oct 2019 21:07:51 +0000 Subject: [PATCH 01/14] Update the documentation with an up to date template for the insecure registry example --- docs/examples.md | 52 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 14 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index c80a3360..3c6d92a4 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -108,25 +108,49 @@ First we need a place to store the config template: `mkdir -p /home/${USER}/.k3d Create a file named `config.toml.tmpl` in `/home/${USER}/.k3d`, with following content:
-# Original section: no changes
 [plugins.opt]
-path = "{{ .NodeConfig.Containerd.Opt }}"
+  path = "{{ .NodeConfig.Containerd.Opt }}"
 [plugins.cri]
-stream_server_address = "{{ .NodeConfig.AgentConfig.NodeName }}"
-stream_server_port = "10010"
+  stream_server_address = "127.0.0.1"
+  stream_server_port = "10010"
 {{- if .IsRunningInUserNS }}
-disable_cgroup = true
-disable_apparmor = true
-restrict_oom_score_adj = true
-{{ end -}}
+  disable_cgroup = true
+  disable_apparmor = true
+  restrict_oom_score_adj = true
+{{end}}
 {{- if .NodeConfig.AgentConfig.PauseImage }}
-sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-{{ end -}}
+  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{end}}
 {{- if not .NodeConfig.NoFlannel }}
-  [plugins.cri.cni]
-    bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-    conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-{{ end -}}
+[plugins.cri.cni]
+  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
+{{end}}
+[plugins.cri.containerd.runtimes.runc]
+  runtime_type = "io.containerd.runc.v2"
+{{ if .PrivateRegistryConfig }}
+{{ if .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors]{{end}}
+{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
+[plugins.cri.registry.mirrors."{{$k}}"]
+  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
+{{end}}
+{{range $k, $v := .PrivateRegistryConfig.Configs }}
+{{ if $v.Auth }}
+[plugins.cri.registry.configs."{{$k}}".auth]
+  {{ if $v.Auth.Username }}username = "{{ $v.Auth.Username }}"{{end}}
+  {{ if $v.Auth.Password }}password = "{{ $v.Auth.Password }}"{{end}}
+  {{ if $v.Auth.Auth }}auth = "{{ $v.Auth.Auth }}"{{end}}
+  {{ if $v.Auth.IdentityToken }}identity_token = "{{ $v.Auth.IdentityToken }}"{{end}}
+{{end}}
+{{ if $v.TLS }}
+[plugins.cri.registry.configs."{{$k}}".tls]
+  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
+  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
+  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
+{{end}}
+{{end}}
+{{end}}
 
 # Added section: additional registries and the endpoints
 [plugins.cri.registry.mirrors]

From e79c3a98d8ce9a1f49e2118c6d8ebd00b6ca1a92 Mon Sep 17 00:00:00 2001
From: iwilltry42 
Date: Mon, 4 Nov 2019 09:49:22 +0100
Subject: [PATCH 02/14] add init to worker as well (thanks for the hint
 @cbandy)

---
 cli/container.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cli/container.go b/cli/container.go
index 4268b1d3..50d6426e 100644
--- a/cli/container.go
+++ b/cli/container.go
@@ -180,6 +180,7 @@ func createWorker(spec *ClusterSpec, postfix int) (string, error) {
 		},
 		PortBindings: workerPublishedPorts.PortBindings,
 		Privileged:   true,
+		Init:         &[]bool{true}[0],
 	}
 
 	if spec.AutoRestart {

From dc19eb71398745e0cd6b1f12db6c912e6f200399 Mon Sep 17 00:00:00 2001
From: iwilltry42 
Date: Mon, 4 Nov 2019 10:00:24 +0100
Subject: [PATCH 03/14] add section about 'NodeHasDiskPressure' issues to FAQ

---
 docs/faq.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/faq.md b/docs/faq.md
index 5f4835dc..c652f5f5 100644
--- a/docs/faq.md
+++ b/docs/faq.md
@@ -3,3 +3,11 @@
 - As [@jaredallard](https://github.com/jaredallard) [pointed out](https://github.com/rancher/k3d/pull/48), people running `k3d` on a system with **btrfs**, may need to mount `/dev/mapper` into the nodes for the setup to work.
   - This will do: `k3d create -v /dev/mapper:/dev/mapper`
   - An additional solution proposed by [@zer0def](https://github.com/zer0def) can be found in the [examples section](examples.md) (_Running on filesystems k3s doesn't like (btrfs, tmpfs, …)_)
+
+- Pods go to evicted state after doing X
+  - Related issues: [#133 - Pods evicted due to `NodeHasDiskPressure`](https://github.com/rancher/k3d/issues/133) (collection of #119 and #130)
+  - Background: somehow docker runs out of space for the k3d node containers, which triggers a hard eviction in the kubelet
+  - Possible [fix/workaround by @zer0def](https://github.com/rancher/k3d/issues/133#issuecomment-549065666):
+    - use a docker storage driver which cleans up properly (e.g. overlay2)
+    - clean up or expand docker root filesystem
+    - change the kubelet's eviction thresholds upon cluster creation: `k3d create --agent-arg '--eviction-hard=imagefs.available<1%,nodefs.available<1%' --agent-arg '--eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%'`

From 3b0c0957659e06dd85c498d56f79d040492684ed Mon Sep 17 00:00:00 2001
From: iwilltry42 
Date: Mon, 4 Nov 2019 10:16:58 +0100
Subject: [PATCH 04/14] add examples for all k3s versions to use a private
 registry

---
 docs/examples.md | 61 +++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 52 insertions(+), 9 deletions(-)

diff --git a/docs/examples.md b/docs/examples.md
index 3c6d92a4..fb94012d 100644
--- a/docs/examples.md
+++ b/docs/examples.md
@@ -93,7 +93,7 @@ This guide takes you through setting up a local insecure (http) registry and int
 
 The registry will be named `registry.local` and run on port `5000`.
 
-### Create the registry
+### Step 1: Create the registry
 
 
 docker volume create local_registry
@@ -101,12 +101,21 @@ docker volume create local_registry
 docker container run -d --name registry.local -v local_registry:/var/lib/registry --restart always -p 5000:5000 registry:2
 
-### Create the cluster with k3d +### Step 2: Prepare configuration to connect to the registry First we need a place to store the config template: `mkdir -p /home/${USER}/.k3d` +#### Step 2 - Option 1: use `registries.yaml` (for k3s >= v0.10.0) + +Create a file named `registries.yaml` in `/home/${USER}/.k3d` with following content: + + +#### Step 2 - Option 2: use `config.toml.tmpl` to directly modify the containerd config (all versions) + Create a file named `config.toml.tmpl` in `/home/${USER}/.k3d`, with following content: +##### Step 2 - Option 2.1 -> for k3s >= v0.10.0 +
 [plugins.opt]
   path = "{{ .NodeConfig.Containerd.Opt }}"
@@ -158,23 +167,57 @@ Create a file named `config.toml.tmpl` in `/home/${USER}/.k3d`, with following c
     endpoint = ["http://registry.local:5000"]
 
-Finally start a cluster with k3d, passing-in the config template: +##### Step 2 - Option 2.2 -> for k3s <= v0.9.1 + +
+# Original section: no changes
+[plugins.opt]
+path = "{{ .NodeConfig.Containerd.Opt }}"
+[plugins.cri]
+stream_server_address = "{{ .NodeConfig.AgentConfig.NodeName }}"
+stream_server_port = "10010"
+{{- if .IsRunningInUserNS }}
+disable_cgroup = true
+disable_apparmor = true
+restrict_oom_score_adj = true
+{{ end -}}
+{{- if .NodeConfig.AgentConfig.PauseImage }}
+sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{ end -}}
+{{- if not .NodeConfig.NoFlannel }}
+  [plugins.cri.cni]
+    bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+    conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
+{{ end -}}
+
+# Added section: additional registries and the endpoints
+[plugins.cri.registry.mirrors]
+  [plugins.cri.registry.mirrors."registry.local:5000"]
+    endpoint = ["http://registry.local:5000"]
+
+ +### Step 3 - Start the cluster + +Finally start a cluster with k3d, passing-in the `registries.yaml` or `config.toml.tmpl`: + +```bash +k3d create \ + --volume /home/${USER}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml +``` + +or ```bash -CLUSTER_NAME=k3s-default k3d create \ - --name ${CLUSTER_NAME} \ - --wait 0 \ - --auto-restart \ --volume /home/${USER}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl ``` -### Wire them up +### Step 4 - Wire them up - Connect the registry to the cluster network: `docker network connect k3d-k3s-default registry.local` - Add `127.0.0.1 registry.local` to your `/etc/hosts` -### Test +### Step 5 - Test Push an image to the registry: From 2205264a11896c1ef799e827caac2615d0456e3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Lutz=20Bru=CC=88ggen?= Date: Mon, 4 Nov 2019 17:29:39 +0100 Subject: [PATCH 05/14] Fix kubeconfig not pointing to correct host The kubeconfig generated by docker.io/rancher/k3s:v0.10.2 or earlier sets the cluster.server address to 127.0.0.1. Previously this seems to have been localhost. And we only replace localhost with the correct address for our local kubeconfig. The error this can lead to: ``` $ k3d --version k3d version v1.3.4 $ docker-machine --version docker-machine version 0.16.1, build cce350d7 $ docker --version Docker version 19.03.1, build 74b1e89 $ k3d create INFO[0000] Created cluster network with ID 649d6f34b84a4df16d2524f0ea0ce69cd4f964a79ae56e2a07bb1ee11d1fce50 INFO[0001] Add TLS SAN for 192.168.99.100 INFO[0001] Created docker volume k3d-k3s-default-images INFO[0001] Creating cluster [k3s-default] INFO[0001] Creating server using docker.io/rancher/k3s:v0.10.2... INFO[0001] Pulling image docker.io/rancher/k3s:v0.10.2... INFO[0018] SUCCESS: created cluster [k3s-default] INFO[0018] You can now use the cluster with: export KUBECONFIG="$(k3d get-kubeconfig --name='k3s-default')" kubectl cluster-info $ export KUBECONFIG="$(k3d get-kubeconfig --name='k3s-default')" $ kubectl cluster-info To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'. The connection to the server 127.0.0.1:6443 was refused - did you specify the right host or port? ``` --- cli/cluster.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/cluster.go b/cli/cluster.go index 722ea67d..4db61fe5 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -161,6 +161,7 @@ func createKubeConfigFile(cluster string) error { if apiHost != "" { s := string(trimBytes) s = strings.Replace(s, "localhost", apiHost, 1) + s = strings.Replace(s, "127.0.0.1", apiHost, 1) trimBytes = []byte(s) } _, err = kubeconfigfile.Write(trimBytes) From 8079d7d2be7292944a3255bd9fb3cec4515db08f Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 7 Nov 2019 14:41:57 +0100 Subject: [PATCH 06/14] exclude .local --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index a35b5300..a74cc4a6 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ _dist/ *.out # Editors -.vscode/ \ No newline at end of file +.vscode/ +.local/ \ No newline at end of file From 10179f8555c454ab43154a8b64a451bcbbc8be9b Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 7 Nov 2019 15:28:14 +0100 Subject: [PATCH 07/14] add warning for when agent arg is supplied without agents being created --- cli/commands.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cli/commands.go b/cli/commands.go index 5508cf60..11874be9 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -131,6 +131,9 @@ func CreateCluster(c *cli.Context) error { } if c.IsSet("agent-arg") { + if c.Int("workers") < 1 { + log.Warnln("--agent-arg supplied, but --workers is 0, so no agents will be created") + } k3AgentArgs = append(k3AgentArgs, c.StringSlice("agent-arg")...) } From fbe93eb039536d71fbac014fbe0314471bee0923 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 7 Nov 2019 15:39:31 +0100 Subject: [PATCH 08/14] add more verbose error messages for getKubeconfig and Delete if no flag was set --- cli/commands.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cli/commands.go b/cli/commands.go index 11874be9..9c0d460c 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -243,6 +243,7 @@ kubectl cluster-info`, os.Args[0], c.String("name")) // DeleteCluster removes the containers belonging to a cluster and its local directory func DeleteCluster(c *cli.Context) error { + clusters, err := getClusters(c.Bool("all"), c.String("name")) if err != nil { @@ -250,6 +251,9 @@ func DeleteCluster(c *cli.Context) error { } if len(clusters) == 0 { + if !c.IsSet("all") && !c.IsSet("name") { + return fmt.Errorf("No cluster with name '%s' found (You can add `--all` and `--name ` to delete other clusters)", c.String("name")) + } return fmt.Errorf("No cluster(s) found") } @@ -386,6 +390,9 @@ func GetKubeConfig(c *cli.Context) error { } if len(clusters) == 0 { + if !c.IsSet("all") && !c.IsSet("name") { + return fmt.Errorf("No cluster with name '%s' found (You can add `--all` and `--name ` to check other clusters)", c.String("name")) + } return fmt.Errorf("No cluster(s) found") } From 576ac040cc90cc0d0e705609ab6fb09f2e09a824 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Fri, 8 Nov 2019 10:27:26 +0100 Subject: [PATCH 09/14] add note about kubectl --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9998c9c2..0c0630af 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,7 @@ or... Check out what you can do via `k3d help` Example Workflow: Create a new cluster and use it with `kubectl` +(*Note:* `kubectl` is not part of `k3d`, so you have to [install it first if needed](https://kubernetes.io/docs/tasks/tools/install-kubectl/)) 1. `k3d create` to create a new single-node cluster (docker container) 2. `export KUBECONFIG=$(k3d get-kubeconfig)` to make `kubectl` to use the kubeconfig for that cluster From 469b56c253d7fd59def127a7b2568ac75f74c7d9 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Mon, 11 Nov 2019 11:53:22 +0100 Subject: [PATCH 10/14] replace 'default' with clustername in kubeconfig --- cli/cluster.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/cli/cluster.go b/cli/cluster.go index 4db61fe5..ab2e0a32 100644 --- a/cli/cluster.go +++ b/cli/cluster.go @@ -156,17 +156,21 @@ func createKubeConfigFile(cluster string) error { // set the host name to remote docker machine's IP address. // // Otherwise, the hostname remains as 'localhost' + // + // Additionally, we replace every occurence of 'default' in the kubeconfig with the actual cluster name apiHost := server[0].Labels["apihost"] + s := string(trimBytes) + s = strings.ReplaceAll(s, "default", cluster) if apiHost != "" { - s := string(trimBytes) s = strings.Replace(s, "localhost", apiHost, 1) s = strings.Replace(s, "127.0.0.1", apiHost, 1) - trimBytes = []byte(s) } + trimBytes = []byte(s) + _, err = kubeconfigfile.Write(trimBytes) if err != nil { - return fmt.Errorf(" Couldn't write to kubeconfig.yaml\n%+v", err) + return fmt.Errorf("Couldn't write to kubeconfig.yaml\n%+v", err) } return nil From c3b799c6e26f4b43aaecb4c207e26eb4552fd02a Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Fri, 22 Nov 2019 08:01:08 +0100 Subject: [PATCH 11/14] add registry.yaml to docs (thanks @pojntfx) --- docs/examples.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/examples.md b/docs/examples.md index fb94012d..7be4eb03 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -109,6 +109,12 @@ First we need a place to store the config template: `mkdir -p /home/${USER}/.k3d Create a file named `registries.yaml` in `/home/${USER}/.k3d` with following content: +```yaml +mirrors: + "registry.local:5000": + endpoint: + - http://registry.local:5000 +``` #### Step 2 - Option 2: use `config.toml.tmpl` to directly modify the containerd config (all versions) From b1b7e957bcfdd7693a5ba8154ec9cd8d0c792c52 Mon Sep 17 00:00:00 2001 From: Chris Johnson Date: Mon, 23 Dec 2019 20:33:29 -0500 Subject: [PATCH 12/14] Add example for non-publicly-trusted CA --- docs/examples.md | 52 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 7be4eb03..61a2be9d 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -84,9 +84,9 @@ Therefore, we have to create the cluster in a way, that the internal port 80 (wh `curl localhost:8082/` -## Connect with a local insecure registry +## Connect with a private insecure registry -This guide takes you through setting up a local insecure (http) registry and integrating it into your workflow so that: +This guide takes you through setting up a private insecure (http) registry and integrating it into your workflow so that: - you can push to the registry from your host - the cluster managed by k3d can pull from that registry @@ -202,7 +202,7 @@ sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}" endpoint = ["http://registry.local:5000"]
-### Step 3 - Start the cluster +### Step 3: Start the cluster Finally start a cluster with k3d, passing-in the `registries.yaml` or `config.toml.tmpl`: @@ -218,12 +218,12 @@ k3d create \ --volume /home/${USER}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl ``` -### Step 4 - Wire them up +### Step 4: Wire them up - Connect the registry to the cluster network: `docker network connect k3d-k3s-default registry.local` - Add `127.0.0.1 registry.local` to your `/etc/hosts` -### Step 5 - Test +### Step 5: Test Push an image to the registry: @@ -263,6 +263,48 @@ EOF ... and check that the pod is running: `kubectl get pods -l "app=nginx-test-registry"` +## Connect with a private secure registry + +This guide takes you through setting up a private secure (https) registry with a non-publicly-trusted CA and integrating it into your workflow so that: + +- you can push to the registry from your host +- the cluster managed by k3d can pull from that registry + +The registry will be named `registry.companyinternal.net` and run on port `5000`. + +### Step 1: Prepare configuration to connect to the registry + +First we need a place to store the config template: `mkdir -p /home/${USER}/.k3d` + +### Step 2: Configure `registries.yaml` (for k3s >= v0.10.0) to point to your root CA + +Create a file named `registries.yaml` in `/home/${USER}/.k3d` with following content: + +```yaml +mirrors: + registry.companyinternal.net: + endpoint: + - https://registry.companyinternal.net +configs: + registry.companyinternal.net: + tls: + ca_file: "/etc/ssl/certs/companycaroot.pem" +``` + +### Step 3: Get a copy of the root CA + +Download it to `/home/${USER}/.k3d/companycaroot.pem` + +### Step 4: Start the cluster + +Finally start a cluster with k3d, passing-in the `registries.yaml` and root CA cert: + +```bash +k3d create \ + --volume /home/${USER}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml \ + --volume /home/${USER}/.k3d/companycaroot.pem:/etc/ssl/certs/companycaroot.pem +``` + ## Running on filesystems k3s doesn't like (btrfs, tmpfs, …) The following script leverages a [Docker loopback volume plugin](https://github.com/ashald/docker-volume-loopback) to mask the problematic filesystem away from k3s by providing a small ext4 filesystem underneath `/var/lib/rancher/k3s` (k3s' data dir). From f9fe8ef0c1fb52718b71ae269228b0ae8628e3fb Mon Sep 17 00:00:00 2001 From: Chris Johnson Date: Tue, 24 Dec 2019 09:39:27 -0500 Subject: [PATCH 13/14] Fix up examples, remove unnecessary port reference and use $HOME so the examples are mac compatible --- docs/examples.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 61a2be9d..b1aa0a28 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -103,11 +103,11 @@ docker container run -d --name registry.local -v local_registry:/var/lib/ ### Step 2: Prepare configuration to connect to the registry -First we need a place to store the config template: `mkdir -p /home/${USER}/.k3d` +First we need a place to store the config template: `mkdir -p ${HOME}/.k3d` #### Step 2 - Option 1: use `registries.yaml` (for k3s >= v0.10.0) -Create a file named `registries.yaml` in `/home/${USER}/.k3d` with following content: +Create a file named `registries.yaml` in `${HOME}/.k3d` with following content: ```yaml mirrors: @@ -118,7 +118,7 @@ mirrors: #### Step 2 - Option 2: use `config.toml.tmpl` to directly modify the containerd config (all versions) -Create a file named `config.toml.tmpl` in `/home/${USER}/.k3d`, with following content: +Create a file named `config.toml.tmpl` in `${HOME}/.k3d`, with following content: ##### Step 2 - Option 2.1 -> for k3s >= v0.10.0 @@ -208,14 +208,14 @@ Finally start a cluster with k3d, passing-in the `registries.yaml` or `config.to ```bash k3d create \ - --volume /home/${USER}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml + --volume ${HOME}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml ``` or ```bash k3d create \ - --volume /home/${USER}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl + --volume ${HOME}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl ``` ### Step 4: Wire them up @@ -265,20 +265,20 @@ EOF ## Connect with a private secure registry -This guide takes you through setting up a private secure (https) registry with a non-publicly-trusted CA and integrating it into your workflow so that: +This guide takes you through setting up a private secure (https) registry with a non-publicly trusted CA and integrating it into your workflow so that: -- you can push to the registry from your host +- you can push to the registry - the cluster managed by k3d can pull from that registry -The registry will be named `registry.companyinternal.net` and run on port `5000`. +The registry will be named `registry.companyinternal.net` and it is assumed to already be set up, with a non-publicly trusted cert. ### Step 1: Prepare configuration to connect to the registry -First we need a place to store the config template: `mkdir -p /home/${USER}/.k3d` +First we need a place to store the config template: `mkdir -p ${HOME}/.k3d` ### Step 2: Configure `registries.yaml` (for k3s >= v0.10.0) to point to your root CA -Create a file named `registries.yaml` in `/home/${USER}/.k3d` with following content: +Create a file named `registries.yaml` in `${HOME}/.k3d` with following content: ```yaml mirrors: @@ -293,7 +293,7 @@ configs: ### Step 3: Get a copy of the root CA -Download it to `/home/${USER}/.k3d/companycaroot.pem` +Download it to `${HOME}/.k3d/companycaroot.pem` ### Step 4: Start the cluster @@ -301,8 +301,8 @@ Finally start a cluster with k3d, passing-in the `registries.yaml` and root CA c ```bash k3d create \ - --volume /home/${USER}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml \ - --volume /home/${USER}/.k3d/companycaroot.pem:/etc/ssl/certs/companycaroot.pem + --volume ${HOME}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml \ + --volume ${HOME}/.k3d/companycaroot.pem:/etc/ssl/certs/companycaroot.pem ``` ## Running on filesystems k3s doesn't like (btrfs, tmpfs, …) From 122ea4637af56a0f7d6b8ae238feb766c670cfc4 Mon Sep 17 00:00:00 2001 From: iwilltry42 Date: Thu, 2 Jan 2020 14:37:27 +0100 Subject: [PATCH 14/14] wait for kubeconfig to be written when --wait is set --- cli/commands.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/commands.go b/cli/commands.go index 9c0d460c..30c4ed3c 100644 --- a/cli/commands.go +++ b/cli/commands.go @@ -211,7 +211,7 @@ func CreateCluster(c *cli.Context) error { nRead, _ := buf.ReadFrom(out) out.Close() output := buf.String() - if nRead > 0 && strings.Contains(string(output), "Running kubelet") { + if nRead > 0 && strings.Contains(string(output), "Wrote kubeconfig") { break }