diff --git a/README.md b/README.md index 0c0630af..32737fe2 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,7 @@ Check out the [examples here](docs/examples.md). Find more details under the following Links: - [Further documentation](docs/documentation.md) +- [Using registries](docs/registries.md) - [Usage examples](docs/examples.md) - [Frequently asked questions and nice-to-know facts](docs/faq.md) diff --git a/docs/examples.md b/docs/examples.md index b1aa0a28..594db748 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -84,226 +84,6 @@ Therefore, we have to create the cluster in a way, that the internal port 80 (wh `curl localhost:8082/` -## Connect with a private insecure registry - -This guide takes you through setting up a private insecure (http) registry and integrating it into your workflow so that: - -- you can push to the registry from your host -- the cluster managed by k3d can pull from that registry - -The registry will be named `registry.local` and run on port `5000`. - -### Step 1: Create the registry - -
-docker volume create local_registry
-
-docker container run -d --name registry.local -v local_registry:/var/lib/registry --restart always -p 5000:5000 registry:2
-
- -### Step 2: Prepare configuration to connect to the registry - -First we need a place to store the config template: `mkdir -p ${HOME}/.k3d` - -#### Step 2 - Option 1: use `registries.yaml` (for k3s >= v0.10.0) - -Create a file named `registries.yaml` in `${HOME}/.k3d` with following content: - -```yaml -mirrors: - "registry.local:5000": - endpoint: - - http://registry.local:5000 -``` - -#### Step 2 - Option 2: use `config.toml.tmpl` to directly modify the containerd config (all versions) - -Create a file named `config.toml.tmpl` in `${HOME}/.k3d`, with following content: - -##### Step 2 - Option 2.1 -> for k3s >= v0.10.0 - -
-[plugins.opt]
-  path = "{{ .NodeConfig.Containerd.Opt }}"
-[plugins.cri]
-  stream_server_address = "127.0.0.1"
-  stream_server_port = "10010"
-{{- if .IsRunningInUserNS }}
-  disable_cgroup = true
-  disable_apparmor = true
-  restrict_oom_score_adj = true
-{{end}}
-{{- if .NodeConfig.AgentConfig.PauseImage }}
-  sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-{{end}}
-{{- if not .NodeConfig.NoFlannel }}
-[plugins.cri.cni]
-  bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-  conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-{{end}}
-[plugins.cri.containerd.runtimes.runc]
-  runtime_type = "io.containerd.runc.v2"
-{{ if .PrivateRegistryConfig }}
-{{ if .PrivateRegistryConfig.Mirrors }}
-[plugins.cri.registry.mirrors]{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Mirrors }}
-[plugins.cri.registry.mirrors."{{$k}}"]
-  endpoint = [{{range $i, $j := $v.Endpoints}}{{if $i}}, {{end}}{{printf "%q" .}}{{end}}]
-{{end}}
-{{range $k, $v := .PrivateRegistryConfig.Configs }}
-{{ if $v.Auth }}
-[plugins.cri.registry.configs."{{$k}}".auth]
-  {{ if $v.Auth.Username }}username = "{{ $v.Auth.Username }}"{{end}}
-  {{ if $v.Auth.Password }}password = "{{ $v.Auth.Password }}"{{end}}
-  {{ if $v.Auth.Auth }}auth = "{{ $v.Auth.Auth }}"{{end}}
-  {{ if $v.Auth.IdentityToken }}identity_token = "{{ $v.Auth.IdentityToken }}"{{end}}
-{{end}}
-{{ if $v.TLS }}
-[plugins.cri.registry.configs."{{$k}}".tls]
-  {{ if $v.TLS.CAFile }}ca_file = "{{ $v.TLS.CAFile }}"{{end}}
-  {{ if $v.TLS.CertFile }}cert_file = "{{ $v.TLS.CertFile }}"{{end}}
-  {{ if $v.TLS.KeyFile }}key_file = "{{ $v.TLS.KeyFile }}"{{end}}
-{{end}}
-{{end}}
-{{end}}
-
-# Added section: additional registries and the endpoints
-[plugins.cri.registry.mirrors]
-  [plugins.cri.registry.mirrors."registry.local:5000"]
-    endpoint = ["http://registry.local:5000"]
-
- -##### Step 2 - Option 2.2 -> for k3s <= v0.9.1 - -
-# Original section: no changes
-[plugins.opt]
-path = "{{ .NodeConfig.Containerd.Opt }}"
-[plugins.cri]
-stream_server_address = "{{ .NodeConfig.AgentConfig.NodeName }}"
-stream_server_port = "10010"
-{{- if .IsRunningInUserNS }}
-disable_cgroup = true
-disable_apparmor = true
-restrict_oom_score_adj = true
-{{ end -}}
-{{- if .NodeConfig.AgentConfig.PauseImage }}
-sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
-{{ end -}}
-{{- if not .NodeConfig.NoFlannel }}
-  [plugins.cri.cni]
-    bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
-    conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
-{{ end -}}
-
-# Added section: additional registries and the endpoints
-[plugins.cri.registry.mirrors]
-  [plugins.cri.registry.mirrors."registry.local:5000"]
-    endpoint = ["http://registry.local:5000"]
-
- -### Step 3: Start the cluster - -Finally start a cluster with k3d, passing-in the `registries.yaml` or `config.toml.tmpl`: - -```bash -k3d create \ - --volume ${HOME}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml -``` - -or - -```bash -k3d create \ - --volume ${HOME}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl -``` - -### Step 4: Wire them up - -- Connect the registry to the cluster network: `docker network connect k3d-k3s-default registry.local` -- Add `127.0.0.1 registry.local` to your `/etc/hosts` - -### Step 5: Test - -Push an image to the registry: - -```bash -docker pull nginx:latest -docker tag nginx:latest registry.local:5000/nginx:latest -docker push registry.local:5000/nginx:latest -``` - -Deploy a pod referencing this image to your cluster: - -```bash -cat <= v0.10.0) to point to your root CA - -Create a file named `registries.yaml` in `${HOME}/.k3d` with following content: - -```yaml -mirrors: - registry.companyinternal.net: - endpoint: - - https://registry.companyinternal.net -configs: - registry.companyinternal.net: - tls: - ca_file: "/etc/ssl/certs/companycaroot.pem" -``` - -### Step 3: Get a copy of the root CA - -Download it to `${HOME}/.k3d/companycaroot.pem` - -### Step 4: Start the cluster - -Finally start a cluster with k3d, passing-in the `registries.yaml` and root CA cert: - -```bash -k3d create \ - --volume ${HOME}/.k3d/registries.yaml:/etc/rancher/k3s/registries.yaml \ - --volume ${HOME}/.k3d/companycaroot.pem:/etc/ssl/certs/companycaroot.pem -``` ## Running on filesystems k3s doesn't like (btrfs, tmpfs, …) diff --git a/docs/registries.md b/docs/registries.md new file mode 100644 index 00000000..d6a1f957 --- /dev/null +++ b/docs/registries.md @@ -0,0 +1,223 @@ +# Using registries with k3d + +## Registries configuration file + +You can add registries by specifying them in a `registries.yaml` in your `$HOME/.k3d` directory. +This file will be loaded automatically by k3d if present and will be shared between all your +k3d clusters, but you can also use a specific file for a new cluster with the +`--registries-file` argument. + +This file is a regular [k3s registries configuration file](https://rancher.com/docs/k3s/latest/en/installation/airgap/#create-registry-yaml), +and looks like this: + +```yaml +mirrors: + "my.company.registry:5000": + endpoint: + - http://my.company.registry:5000 +``` + +In this example, an image with a name like `my.company.registry:5000/nginx:latest` would be +_pulled_ from the registry running at `http://my.company.registry:5000`. + +Note well there is an important limitation: **this configuration file will only work with +k3s >= v0.10.0**. It will fail silently with previous versions of k3s, but you find in the +[section below](#k3s-old) an alternative solution. + +This file can also be used for providing additional information necessary for accessing +some registries, like [authentication](#auth) and [certificates](#certs). + +### Authenticated registries + +When using authenticated registries, we can add the _username_ and _password_ in a +`configs` section in the `registries.yaml`, like this: + +```yaml +mirrors: + my.company.registry: + endpoint: + - http://my.company.registry + +configs: + my.company.registry: + auth: + username: aladin + password: abracadabra +``` + +### Secure registries + +When using secure registries, the [`registries.yaml` file](#registries-file) must include information +about the certificates. For example, if you want to use images from the secure registry +running at `https://my.company.registry`, you must first download a CA file valid for that server +and store it in some well-known directory like `${HOME}/.k3d/my-company-root.pem`. + +Then you have to mount the CA file in some directory in the nodes in the cluster and +include that mounted file in a `configs` section in the [`registries.yaml` file](#registries-file). +For example, if we mount the CA file in `/etc/ssl/certs/my-company-root.pem`, the `registries.yaml` +will look like: + +```yaml +mirrors: + my.company.registry: + endpoint: + - https://my.company.registry + +configs: + my.company.registry: + tls: + # we will mount "my-company-root.pem" in the /etc/ssl/certs/ directory. + ca_file: "/etc/ssl/certs/my-company-root.pem" +``` + +Finally, we can create the cluster, mounting the CA file in the path we +specified in `ca_file`: + +```shell script +k3d create --volume ${HOME}/.k3d/my-company-root.pem:/etc/ssl/certs/my-company-root.pem ... +``` + +## Using a local registry + +### Using the k3d registry + +k3d can manage a local registry that you can use for pushing your images to, and your k3d nodes +will be able to use those images automatically. k3d will create the registry for you and connect +it to your k3d cluster. It is important to note that this registry will be shared between all +your k3d clusters, and it will be released when the last of the k3d clusters that was using it +is deleted. + +In order to enable the k3d registry when creating a new cluster, you must run k3d with the +`--enable-registry` argument + +```shell script +k3d create --enable-registry ... +``` + +Then you must add an entry in `/etc/hosts` as described in [the next section](#etc-hosts). And +then you should [check you local registry](#testing). + +### Using your own local registry + +If you don't want k3d to manage your registry, you can start it with some `docker` commands, like: + +```shell script +docker volume create local_registry +docker container run -d --name registry.local -v local_registry:/var/lib/registry --restart always -p 5000:5000 registry:2 +``` + +These commands will start you registry in `registry.local:5000`. In order to push to this registry, you will +need to add the line at `/etc/hosts` as we described in [the previous section ](#etc-hosts). Once your +registry is up and running, we will need to add it to your [`registries.yaml` configuration file](#registries-file). +Finally, you must connect the registry network to the k3d cluster network: +`docker network connect k3d-k3s-default registry.local`. And then you can +[check you local registry](#testing). + +### Pushing to your local registry address + +The registry will be located, by default, at `registry.local:5000` (customizable with the `--registry-name` +and `--registry-port` parameters). All the nodes in your k3d cluster can resolve this hostname (thanks to the +DNS server provided by the Docker daemon) but, in order to be able to push to this registry, this hostname +but also be resolved from your host. + +The easiest solution for this is to add an entry in your `/etc/hosts` file like this: + +```shell script +127.0.0.1 registry.local +``` + +Once again, this will only work with k3s >= v0.10.0 (see the [section below](#k3s-old) +when using k3s <= v0.9.1) + +## Testing your registry + +You should test that you can +* push to your registry from your local development machine. +* use images from that registry in `Deployments` in your k3d cluster. + +We will verify these two things for a local registry (located at `registry.local:5000`) running +in your development machine. Things would be basically the same for checking an external +registry, but some additional configuration could be necessary in your local machine when +using an authenticated or secure registry (please refer to Docker's documentation for this). + +Firstly, we can download some image (like `nginx`) and push it to our local registry with: + +```shell script +docker pull nginx:latest +docker tag nginx:latest registry.local:5000/nginx:latest +docker push registry.local:5000/nginx:latest +``` + +Then we can deploy a pod referencing this image to your cluster: + +```shell script +cat <Configuring registries for k3s <= v0.9.1 + +k3s servers below v0.9.1 do not recognize the `registries.yaml` file as we described in +the [previous section](#registries-file), so you will need to embed the contents of that +file in a `containerd` configuration file. You will have to create your own `containerd` +configuration file at some well-known path like `${HOME}/.k3d/config.toml.tmpl`, like this: + +
+# Original section: no changes
+[plugins.opt]
+path = "{{ .NodeConfig.Containerd.Opt }}"
+[plugins.cri]
+stream_server_address = "{{ .NodeConfig.AgentConfig.NodeName }}"
+stream_server_port = "10010"
+{{- if .IsRunningInUserNS }}
+disable_cgroup = true
+disable_apparmor = true
+restrict_oom_score_adj = true
+{{ end -}}
+{{- if .NodeConfig.AgentConfig.PauseImage }}
+sandbox_image = "{{ .NodeConfig.AgentConfig.PauseImage }}"
+{{ end -}}
+{{- if not .NodeConfig.NoFlannel }}
+  [plugins.cri.cni]
+    bin_dir = "{{ .NodeConfig.AgentConfig.CNIBinDir }}"
+    conf_dir = "{{ .NodeConfig.AgentConfig.CNIConfDir }}"
+{{ end -}}
+
+# Added section: additional registries and the endpoints
+[plugins.cri.registry.mirrors]
+  [plugins.cri.registry.mirrors."registry.local:5000"]
+    endpoint = ["http://registry.local:5000"]
+
+ +and then mount it at `/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl` (where +the `containerd` in your k3d nodes will load it) when creating the k3d cluster: + +```bash +k3d create \ + --volume ${HOME}/.k3d/config.toml.tmpl:/var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl +``` + +