Compare commits

..

2 Commits

Author SHA1 Message Date
Thorsten Klein
18a8ee1821
Merge branch 'main' into dependabot/go_modules/github.com/containerd/containerd-1.5.7 2021-10-05 12:19:08 +02:00
dependabot[bot]
0c8500118e
build(deps): bump github.com/containerd/containerd from 1.5.5 to 1.5.7
Bumps [github.com/containerd/containerd](https://github.com/containerd/containerd) from 1.5.5 to 1.5.7.
- [Release notes](https://github.com/containerd/containerd/releases)
- [Changelog](https://github.com/containerd/containerd/blob/main/RELEASES.md)
- [Commits](https://github.com/containerd/containerd/compare/v1.5.5...v1.5.7)

---
updated-dependencies:
- dependency-name: github.com/containerd/containerd
  dependency-type: direct:production
...

Signed-off-by: dependabot[bot] <support@github.com>
2021-10-05 09:32:42 +00:00
67 changed files with 562 additions and 1212 deletions

View File

@ -130,15 +130,6 @@
"contributions": [
"code"
]
},
{
"login": "benjaminjb",
"name": "Benjamin Blattberg",
"avatar_url": "https://avatars.githubusercontent.com/u/4651855?v=4",
"profile": "https://github.com/benjaminjb",
"contributions": [
"code"
]
}
],
"contributorsPerLine": 7,

View File

@ -1,48 +1,5 @@
# Changelog
## v5.0.3
### Enhancements & Fixes
- simplified way of getting a Docker API Client that works with Docker Contexts and `DOCKER_*` environment variable configuration (#829, @dragonflylee)
- fix: didn't honor `DOCKER_TLS` environment variables before
## v5.0.2
### Enhancements
- CoreDNS Configmap is now edited in the auto-deploy manifest on disk instead of relying on `kubectl patch` command (#814)
- refactor: add cmd subcommands in a single function call (#819, @moeryomenko)
- handle ready-log-messages by type and intent & check them in single log streams instead of checking whole chunks every time (#818)
### Fixes
- fix: config file check failing with env var expansion because unexpanded input file was checked
### Misc
- cleanup: ensure that connections/streams are closed once unused (#818)
- cleanup: split type definitions across multiple files to increase readability (#818)
- docs: clarify `node create` help text about cluster reference (#808, @losinggeneration)
- refactor: move from io/ioutil (deprecated) to io and os packages (#827, @Juneezee)
## v5.0.1
### Enhancement
- add `HostFromClusterNetwork` field to `LocalRegistryHosting` configmap as per KEP-1755 (#754)
### Fixes
- fix: nilpointer exception on failed exec process with no returned logreader
- make post-create cluster preparation (DNS stuff mostly) more resilient (#780)
- fix v1alpha2 -> v1alpha3 config migration (and other related issues) (#799)
### Misc
- docs: fix typo (#784)
- docs: fix usage of legacy `--k3s-agent/server-arg` flag
## v5.0.0
This release contains a whole lot of new features, breaking changes as well as smaller fixes and improvements.

View File

@ -9,7 +9,7 @@
[![Go Report Card](https://goreportcard.com/badge/github.com/rancher/k3d?style=flat-square)](https://goreportcard.com/report/github.com/rancher/k3d)
<!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
[![All Contributors](https://img.shields.io/badge/all_contributors-14-orange.svg?style=flat-square)](#contributors-)
[![All Contributors](https://img.shields.io/badge/all_contributors-13-orange.svg?style=flat-square)](#contributors-)
<!-- ALL-CONTRIBUTORS-BADGE:END -->
[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md)
@ -141,7 +141,6 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
<td align="center"><a href="https://github.com/erwinkersten"><img src="https://avatars0.githubusercontent.com/u/4391121?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Erwin Kersten</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=erwinkersten" title="Documentation">📖</a></td>
<td align="center"><a href="http://www.alexsears.com"><img src="https://avatars.githubusercontent.com/u/3712883?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Alex Sears</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=searsaw" title="Documentation">📖</a></td>
<td align="center"><a href="http://shanduur.github.io"><img src="https://avatars.githubusercontent.com/u/32583062?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Mateusz Urbanek</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=Shanduur" title="Code">💻</a></td>
<td align="center"><a href="https://github.com/benjaminjb"><img src="https://avatars.githubusercontent.com/u/4651855?v=4?s=100" width="100px;" alt=""/><br /><sub><b>Benjamin Blattberg</b></sub></a><br /><a href="https://github.com/rancher/k3d/commits?author=benjaminjb" title="Code">💻</a></td>
</tr>
</table>

View File

@ -44,12 +44,12 @@ func NewCmdCluster() *cobra.Command {
}
// add subcommands
cmd.AddCommand(NewCmdClusterCreate(),
NewCmdClusterStart(),
NewCmdClusterStop(),
NewCmdClusterDelete(),
NewCmdClusterList(),
NewCmdClusterEdit())
cmd.AddCommand(NewCmdClusterCreate())
cmd.AddCommand(NewCmdClusterStart())
cmd.AddCommand(NewCmdClusterStop())
cmd.AddCommand(NewCmdClusterDelete())
cmd.AddCommand(NewCmdClusterList())
cmd.AddCommand(NewCmdClusterEdit())
// add flags

View File

@ -191,7 +191,7 @@ func NewCmdClusterCreate() *cobra.Command {
* Kubeconfig *
**************/
if !clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && clusterConfig.KubeconfigOpts.SwitchCurrentContext {
if clusterConfig.KubeconfigOpts.UpdateDefaultKubeconfig && clusterConfig.KubeconfigOpts.SwitchCurrentContext {
l.Log().Infoln("--kubeconfig-update-default=false --> sets --kubeconfig-switch-context=false")
clusterConfig.KubeconfigOpts.SwitchCurrentContext = false
}

View File

@ -37,9 +37,7 @@ import (
// NewCmdClusterStart returns a new cobra command
func NewCmdClusterStart() *cobra.Command {
startClusterOpts := types.ClusterStartOpts{
Intent: k3d.IntentClusterStart,
}
startClusterOpts := types.ClusterStartOpts{}
// create new command
cmd := &cobra.Command{
@ -61,7 +59,6 @@ func NewCmdClusterStart() *cobra.Command {
if err := client.ClusterStart(cmd.Context(), runtimes.SelectedRuntime, c, startClusterOpts); err != nil {
l.Log().Fatalln(err)
}
l.Log().Infof("Started cluster '%s'", c.Name)
}
}
},

View File

@ -41,7 +41,8 @@ func NewCmdConfig() *cobra.Command {
},
}
cmd.AddCommand(NewCmdConfigInit(), NewCmdConfigMigrate())
cmd.AddCommand(NewCmdConfigInit())
cmd.AddCommand(NewCmdConfigMigrate())
return cmd
}

View File

@ -43,7 +43,8 @@ func NewCmdKubeconfig() *cobra.Command {
}
// add subcommands
cmd.AddCommand(NewCmdKubeconfigGet(), NewCmdKubeconfigMerge())
cmd.AddCommand(NewCmdKubeconfigGet())
cmd.AddCommand(NewCmdKubeconfigMerge())
// add flags

View File

@ -43,12 +43,12 @@ func NewCmdNode() *cobra.Command {
}
// add subcommands
cmd.AddCommand(NewCmdNodeCreate(),
NewCmdNodeStart(),
NewCmdNodeStop(),
NewCmdNodeDelete(),
NewCmdNodeList(),
NewCmdNodeEdit())
cmd.AddCommand(NewCmdNodeCreate())
cmd.AddCommand(NewCmdNodeStart())
cmd.AddCommand(NewCmdNodeStop())
cmd.AddCommand(NewCmdNodeDelete())
cmd.AddCommand(NewCmdNodeList())
cmd.AddCommand(NewCmdNodeEdit())
// add flags

View File

@ -72,7 +72,7 @@ func NewCmdNodeCreate() *cobra.Command {
if err := cmd.RegisterFlagCompletionFunc("role", util.ValidArgsNodeRoles); err != nil {
l.Log().Fatalln("Failed to register flag completion for '--role'", err)
}
cmd.Flags().StringP("cluster", "c", k3d.DefaultClusterName, "Cluster URL or k3d cluster name to connect to.")
cmd.Flags().StringP("cluster", "c", k3d.DefaultClusterName, "Select the cluster that the node shall connect to.")
if err := cmd.RegisterFlagCompletionFunc("cluster", util.ValidArgsAvailableClusters); err != nil {
l.Log().Fatalln("Failed to register flag completion for '--cluster'", err)
}

View File

@ -44,11 +44,11 @@ func NewCmdRegistry() *cobra.Command {
}
// add subcommands
cmd.AddCommand(NewCmdRegistryCreate(),
NewCmdRegistryStart(),
NewCmdRegistryStop(),
NewCmdRegistryDelete(),
NewCmdRegistryList())
cmd.AddCommand(NewCmdRegistryCreate())
cmd.AddCommand(NewCmdRegistryStart())
cmd.AddCommand(NewCmdRegistryStop())
cmd.AddCommand(NewCmdRegistryDelete())
cmd.AddCommand(NewCmdRegistryList())
// add flags

View File

@ -25,6 +25,7 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
@ -85,38 +86,40 @@ All Nodes of a k3d cluster are part of the same docker network.`,
rootCmd.Flags().BoolVar(&flags.version, "version", false, "Show k3d and default k3s version")
// add subcommands
rootCmd.AddCommand(NewCmdCompletion(rootCmd),
cluster.NewCmdCluster(),
kubeconfig.NewCmdKubeconfig(),
node.NewCmdNode(),
image.NewCmdImage(),
cfg.NewCmdConfig(),
registry.NewCmdRegistry(),
debug.NewCmdDebug(),
&cobra.Command{
Use: "version",
Short: "Show k3d and default k3s version",
Long: "Show k3d and default k3s version",
Run: func(cmd *cobra.Command, args []string) {
printVersion()
},
rootCmd.AddCommand(NewCmdCompletion(rootCmd))
rootCmd.AddCommand(cluster.NewCmdCluster())
rootCmd.AddCommand(kubeconfig.NewCmdKubeconfig())
rootCmd.AddCommand(node.NewCmdNode())
rootCmd.AddCommand(image.NewCmdImage())
rootCmd.AddCommand(cfg.NewCmdConfig())
rootCmd.AddCommand(registry.NewCmdRegistry())
rootCmd.AddCommand(debug.NewCmdDebug())
rootCmd.AddCommand(&cobra.Command{
Use: "version",
Short: "Show k3d and default k3s version",
Long: "Show k3d and default k3s version",
Run: func(cmd *cobra.Command, args []string) {
printVersion()
},
&cobra.Command{
Use: "runtime-info",
Short: "Show runtime information",
Long: "Show some information about the runtime environment (e.g. docker info)",
Run: func(cmd *cobra.Command, args []string) {
info, err := runtimes.SelectedRuntime.Info()
if err != nil {
l.Log().Fatalln(err)
}
err = yaml.NewEncoder(os.Stdout).Encode(info)
if err != nil {
l.Log().Fatalln(err)
}
},
Hidden: true,
})
})
rootCmd.AddCommand(&cobra.Command{
Use: "runtime-info",
Short: "Show runtime information",
Long: "Show some information about the runtime environment (e.g. docker info)",
Run: func(cmd *cobra.Command, args []string) {
info, err := runtimes.SelectedRuntime.Info()
if err != nil {
l.Log().Fatalln(err)
}
err = yaml.NewEncoder(os.Stdout).Encode(info)
if err != nil {
l.Log().Fatalln(err)
}
},
Hidden: true,
})
// Init
cobra.OnInitialize(initLogging, initRuntime)
@ -166,7 +169,7 @@ func initLogging() {
l.Log().SetLevel(logrus.InfoLevel)
}
}
l.Log().SetOutput(io.Discard)
l.Log().SetOutput(ioutil.Discard)
l.Log().AddHook(&writer.Hook{
Writer: os.Stderr,
LogLevels: []logrus.Level{

View File

@ -23,6 +23,7 @@ package config
import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"
@ -57,7 +58,7 @@ func InitViperWithConfigFile(cfgViper *viper.Viper, configFile string) error {
}
defer tmpfile.Close()
originalcontent, err := os.ReadFile(configFile)
originalcontent, err := ioutil.ReadFile(configFile)
if err != nil {
l.Log().Fatalf("error reading config file %s: %v", configFile, err)
}
@ -83,7 +84,7 @@ func InitViperWithConfigFile(cfgViper *viper.Viper, configFile string) error {
l.Log().Fatalf("Cannot validate config file %s: %+v", configFile, err)
}
if err := config.ValidateSchemaFile(tmpfile.Name(), schema); err != nil {
if err := config.ValidateSchemaFile(configFile, schema); err != nil {
l.Log().Fatalf("Schema Validation failed for config file %s: %+v", configFile, err)
}

View File

@ -28,8 +28,8 @@
```bash
k3d cluster create \
--k3s-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%@agent:*' \
--k3s-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%@agent:*'
--k3s-agent-arg '--kubelet-arg=eviction-hard=imagefs.available<1%,nodefs.available<1%' \
--k3s-agent-arg '--kubelet-arg=eviction-minimum-reclaim=imagefs.available=1%,nodefs.available=1%'
```
## Restarting a multi-server cluster or the initializing server node fails
@ -44,7 +44,7 @@
- The Problem: Passing a feature flag to the Kubernetes API Server running inside k3s.
- Example: you want to enable the EphemeralContainers feature flag in Kubernetes
- Solution: `#!bash k3d cluster create --k3s-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true@server:*'`
- Solution: `#!bash k3d cluster create --k3s-server-arg '--kube-apiserver-arg=feature-gates=EphemeralContainers=true'`
- **Note**: Be aware of where the flags require dashes (`--`) and where not.
- the k3s flag (`--kube-apiserver-arg`) has the dashes
- the kube-apiserver flag `feature-gates` doesn't have them (k3s adds them internally)
@ -53,10 +53,10 @@
```bash
k3d cluster create k3d-one \
--k3s-arg "--cluster-cidr=10.118.0.0/17@server:*" \
--k3s-arg "--service-cidr=10.118.128.0/17@server:*" \
--k3s-arg "--disable=servicelb@server:*" \
--k3s-arg "--disable=traefik@server:*" \
--k3s-server-arg --cluster-cidr="10.118.0.0/17" \
--k3s-server-arg --service-cidr="10.118.128.0/17" \
--k3s-server-arg --disable=servicelb \
--k3s-server-arg --disable=traefik \
--verbose
```
@ -105,8 +105,8 @@ Some can be fixed by passing the `HTTP_PROXY` environment variables to k3d, some
```bash
k3d cluster create \
--k3s-arg "--kube-proxy-arg=conntrack-max-per-core=0@server:*" \
--k3s-arg "--kube-proxy-arg=conntrack-max-per-core=0@agent:*" \
--k3s-server-arg "--kube-proxy-arg=conntrack-max-per-core=0" \
--k3s-agent-arg "--kube-proxy-arg=conntrack-max-per-core=0" \
--image rancher/k3s:v1.20.6-k3s
```

View File

@ -20,14 +20,14 @@ Or you can directly use this [calico.yaml](calico.yaml) manifest
On the k3s cluster creation :
- add the flag `--flannel-backend=none`. For this, on k3d you need to forward this flag to k3s with the option `--k3s-arg`.
- add the flag `--flannel-backend=none`. For this, on k3d you need to forward this flag to k3s with the option `--k3s-server-arg`.
- mount (`--volume`) the calico descriptor in the auto deploy manifest directory of k3s `/var/lib/rancher/k3s/server/manifests/`
So the command of the cluster creation is (when you are at root of the k3d repository)
```bash
k3d cluster create "${clustername}" \
--k3s-arg '--flannel-backend=none@server:*' \
--k3s-server-arg '--flannel-backend=none' \
--volume "$(pwd)/docs/usage/guides/calico.yaml:/var/lib/rancher/k3s/server/manifests/calico.yaml"
```

View File

@ -16,7 +16,8 @@ k3d
-e, --env # add environment variables to the nodes (quoted string, format: 'KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]', use flag multiple times)
--gpus # [from docker CLI] add GPU devices to the node containers (string, e.g. 'all')
-i, --image # specify which k3s image should be used for the nodes (string, default: 'docker.io/rancher/k3s:v1.20.0-k3s2', tag changes per build)
--k3s-arg # add additional arguments to the k3s server/agent (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help & https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/#k3s-agent-cli-help)
--k3s-agent-arg # add additional arguments to the k3s agent (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/agent-config/#k3s-agent-cli-help)
--k3s-server-arg # add additional arguments to the k3s server (quoted string, use flag multiple times) (see https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#k3s-server-cli-help)
--kubeconfig-switch-context # (implies --kubeconfig-update-default) automatically sets the current-context of your default kubeconfig to the new cluster's context (default: true)
--kubeconfig-update-default # enable the automated update of the default kubeconfig with the details of the newly created cluster (also sets '--wait=true') (default: true)
-l, --label # add (docker) labels to the node containers (format: 'KEY[=VALUE][@NODEFILTER[;NODEFILTER...]]', use flag multiple times)

View File

@ -13,7 +13,7 @@ k3d node create NAME [flags]
### Options
```
-c, --cluster string Cluster URL or k3d cluster name to connect to. (default "k3s-default")
-c, --cluster string Select the cluster that the node shall connect to. (default "k3s-default")
-h, --help help for create
-i, --image string Specify k3s image used for the node(s) (default "docker.io/rancher/k3s:v1.21.4-k3s2")
--k3s-node-label strings Specify k3s node labels in format "foo=bar"

View File

@ -104,7 +104,7 @@ k3d cluster create \
#### Create a customized k3d-managed registry
1. `#!bash k3d registry create myregistry.localhost --port 12345` creates a new registry called `k3d-myregistry.localhost` (could be used with automatic resolution of `*.localhost`, see next section - also, **note the `k3d-` prefix** that k3d adds to all resources it creates)
2. `#!bash k3d cluster create newcluster --registry-use k3d-myregistry.localhost:12345` (make sure you use the **`k3d-` prefix** here) creates a new cluster set up to use that registry
2. `#!bash k3d cluster create newcluster --registry-use k3d-myregistry.localhost:12345` (make sure you use the **`k3d-` prefix** here) creates a new cluster set up to us that registry
3. continue with step 3 and 4 from the last section for testing
<!-- Admonition to describe usage of a non-k3d-managed registry -->

9
go.mod
View File

@ -4,9 +4,9 @@ go 1.17
require (
github.com/Microsoft/go-winio v0.4.17 // indirect
github.com/Microsoft/hcsshim v0.8.18 // indirect
github.com/Microsoft/hcsshim v0.8.21 // indirect
github.com/containerd/cgroups v1.0.1 // indirect
github.com/containerd/containerd v1.5.5
github.com/containerd/containerd v1.5.7
github.com/docker/cli v20.10.8+incompatible
github.com/docker/docker v20.10.8+incompatible
github.com/docker/docker-credential-helpers v0.6.4 // indirect
@ -23,7 +23,7 @@ require (
github.com/moby/sys/mount v0.2.0 // indirect
github.com/moby/term v0.0.0-20201110203204-bea5bbe245bf // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/runc v1.0.1 // indirect
github.com/opencontainers/runc v1.0.2 // indirect
github.com/pkg/errors v0.9.1
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.2.1
@ -41,8 +41,6 @@ require (
sigs.k8s.io/yaml v1.2.0
)
require github.com/spf13/pflag v1.0.5
require (
github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@ -80,6 +78,7 @@ require (
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/cast v1.3.1 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.2.0 // indirect
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
go.opencensus.io v0.23.0 // indirect

12
go.sum
View File

@ -72,8 +72,8 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3
github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg=
github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00=
github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600=
github.com/Microsoft/hcsshim v0.8.18 h1:cYnKADiM1869gvBpos3YCteeT6sZLB48lB5dmMMs8Tg=
github.com/Microsoft/hcsshim v0.8.18/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
github.com/Microsoft/hcsshim v0.8.21 h1:btRfUDThBE5IKcvI8O8jOiIkujUsAMBSRsYDYmEi6oM=
github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4=
github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU=
github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY=
github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
@ -175,8 +175,8 @@ github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo
github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI=
github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s=
github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
github.com/containerd/containerd v1.5.5 h1:q1gxsZsGZ8ddVe98yO6pR21b5xQSMiR61lD0W96pgQo=
github.com/containerd/containerd v1.5.5/go.mod h1:oSTh0QpT1w6jYcGmbiSbxv9OSQYaa88mPyWIuU79zyo=
github.com/containerd/containerd v1.5.7 h1:rQyoYtj4KddB3bxG6SAqd4+08gePNyJjRqvOIfV3rkM=
github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@ -686,8 +686,8 @@ github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59P
github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0=
github.com/opencontainers/runc v1.0.1 h1:G18PGckGdAm3yVQRWDVQ1rLSLntiniKJ0cNRT2Tm5gs=
github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg=
github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0=
github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=

View File

@ -22,16 +22,11 @@ THE SOFTWARE.
package actions
import (
"bytes"
"context"
"fmt"
"io"
"os"
"github.com/rancher/k3d/v5/pkg/runtimes"
k3d "github.com/rancher/k3d/v5/pkg/types"
l "github.com/rancher/k3d/v5/pkg/logger"
)
type WriteFileAction struct {
@ -44,35 +39,3 @@ type WriteFileAction struct {
func (act WriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
return act.Runtime.WriteToNode(ctx, act.Content, act.Dest, act.Mode, node)
}
type RewriteFileAction struct {
Runtime runtimes.Runtime
Path string
RewriteFunc func([]byte) ([]byte, error)
Mode os.FileMode
}
func (act RewriteFileAction) Run(ctx context.Context, node *k3d.Node) error {
reader, err := act.Runtime.ReadFromNode(ctx, act.Path, node)
if err != nil {
return fmt.Errorf("runtime failed to read '%s' from node '%s': %w", act.Path, node.Name, err)
}
defer reader.Close()
file, err := io.ReadAll(reader)
if err != nil {
return fmt.Errorf("failed to read file: %w", err)
}
file = bytes.Trim(file[512:], "\x00") // trim control characters, etc.
file, err = act.RewriteFunc(file)
if err != nil {
return fmt.Errorf("error while rewriting %s in %s: %w", act.Path, node.Name, err)
}
l.Log().Tracef("Rewritten:\n%s", string(file))
return act.Runtime.WriteToNode(ctx, file, act.Path, act.Mode, node)
}

View File

@ -22,16 +22,13 @@ THE SOFTWARE.
package client
import (
"bytes"
"context"
_ "embed"
"errors"
"fmt"
"io"
"os"
"io/ioutil"
"sort"
"strconv"
"strings"
"time"
gort "runtime"
@ -89,7 +86,6 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi
Timeout: clusterConfig.ClusterCreateOpts.Timeout, // TODO: here we should consider the time used so far
NodeHooks: clusterConfig.ClusterCreateOpts.NodeHooks,
EnvironmentInfo: envInfo,
Intent: k3d.IntentClusterCreate,
}); err != nil {
return fmt.Errorf("Failed Cluster Start: %+v", err)
}
@ -380,7 +376,7 @@ ClusterCreatOpts:
// connection url is always the name of the first server node (index 0) // TODO: change this to the server loadbalancer
connectionURL := fmt.Sprintf("https://%s:%s", GenerateNodeName(cluster.Name, k3d.ServerRole, 0), k3d.DefaultAPIPort)
clusterCreateOpts.GlobalLabels[k3d.LabelClusterURL] = connectionURL
clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("%s=%s", k3s.EnvClusterToken, cluster.Token))
clusterCreateOpts.GlobalEnv = append(clusterCreateOpts.GlobalEnv, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, cluster.Token))
nodeSetup := func(node *k3d.Node) error {
// cluster specific settings
@ -414,12 +410,12 @@ ClusterCreatOpts:
// the cluster has an init server node, but its not this one, so connect it to the init node
if cluster.InitNode != nil && !node.ServerOpts.IsInit {
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, connectionURL))
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, connectionURL))
node.RuntimeLabels[k3d.LabelServerIsInit] = "false" // set label, that this server node is not the init server
}
} else if node.Role == k3d.AgentRole {
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, connectionURL))
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, connectionURL))
}
node.Networks = []string{cluster.Network.Name}
@ -823,10 +819,6 @@ func GenerateNodeName(cluster string, role k3d.Role, suffix int) string {
func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, clusterStartOpts types.ClusterStartOpts) error {
l.Log().Infof("Starting cluster '%s'", cluster.Name)
if clusterStartOpts.Intent == "" {
clusterStartOpts.Intent = k3d.IntentClusterStart
}
if clusterStartOpts.Timeout > 0*time.Second {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, clusterStartOpts.Timeout)
@ -852,10 +844,18 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
}
}
// sort list of servers for properly ordered sequential start
// TODO: remove trace logs below
l.Log().Traceln("Servers before sort:")
for i, n := range servers {
l.Log().Tracef("Server %d - %s", i, n.Name)
}
sort.Slice(servers, func(i, j int) bool {
return servers[i].Name < servers[j].Name
})
l.Log().Traceln("Servers after sort:")
for i, n := range servers {
l.Log().Tracef("Server %d - %s", i, n.Name)
}
/*
* Init Node
@ -865,7 +865,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
if err := NodeStart(ctx, runtime, initNode, &k3d.NodeStartOpts{
Wait: true, // always wait for the init node
NodeHooks: clusterStartOpts.NodeHooks,
ReadyLogMessage: types.GetReadyLogMessage(initNode, clusterStartOpts.Intent), // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
ReadyLogMessage: "Running kube-apiserver", // initNode means, that we're using etcd -> this will need quorum, so "k3s is up and running" won't happen right now
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start initializing server node: %+v", err)
@ -879,7 +879,7 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
for _, serverNode := range servers {
if err := NodeStart(ctx, runtime, serverNode, &k3d.NodeStartOpts{
Wait: true,
NodeHooks: append(clusterStartOpts.NodeHooks, serverNode.HookActions...),
NodeHooks: clusterStartOpts.NodeHooks,
EnvironmentInfo: clusterStartOpts.EnvironmentInfo,
}); err != nil {
return fmt.Errorf("Failed to start server %s: %+v", serverNode.Name, err)
@ -934,82 +934,19 @@ func ClusterStart(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Clust
}
/*
* Additional Cluster Preparation (post start)
* Additional Cluster Preparation
*/
postStartErrgrp, postStartErrgrpCtx := errgroup.WithContext(ctx)
/*** DNS ***/
// add host.k3d.internal record to /etc/hosts in all nodes
postStartErrgrp.Go(func() error {
return prepInjectHostIP(postStartErrgrpCtx, runtime, cluster, &clusterStartOpts)
})
// add /etc/hosts and CoreDNS entry for host.k3d.internal, referring to the host system
if err := prepInjectHostIP(ctx, runtime, cluster, &clusterStartOpts); err != nil {
return fmt.Errorf("failed to inject host IP: %w", err)
}
postStartErrgrp.Go(func() error {
hosts := fmt.Sprintf("%s %s\n", clusterStartOpts.EnvironmentInfo.HostGateway.String(), k3d.DefaultK3dInternalHostRecord)
net, err := runtime.GetNetwork(ctx, &cluster.Network)
if err != nil {
return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %w", cluster.Network.Name, err)
}
for _, member := range net.Members {
hosts += fmt.Sprintf("%s %s\n", member.IP.String(), member.Name)
}
l.Log().Infof("Injecting records for host.k3d.internal and for %d network members into CoreDNS configmap...", len(net.Members))
act := actions.RewriteFileAction{
Runtime: runtime,
Path: "/var/lib/rancher/k3s/server/manifests/coredns.yaml",
Mode: 0744,
RewriteFunc: func(input []byte) ([]byte, error) {
split, err := util.SplitYAML(input)
if err != nil {
return nil, fmt.Errorf("error splitting yaml: %w", err)
}
var outputBuf bytes.Buffer
outputEncoder := yaml.NewEncoder(&outputBuf)
for _, d := range split {
var doc map[string]interface{}
if err := yaml.Unmarshal(d, &doc); err != nil {
return nil, err
}
if kind, ok := doc["kind"]; ok {
if strings.ToLower(kind.(string)) == "configmap" {
configmapData := doc["data"].(map[interface{}]interface{})
configmapData["NodeHosts"] = hosts
}
}
if err := outputEncoder.Encode(doc); err != nil {
return nil, err
}
}
outputEncoder.Close()
return outputBuf.Bytes(), nil
},
}
// get the first server in the list and run action on it once it's ready for it
for _, n := range cluster.Nodes {
if n.Role == k3d.ServerRole {
ts, err := time.Parse("2006-01-02T15:04:05.999999999Z", n.State.Started)
if err != nil {
return err
}
if err := NodeWaitForLogMessage(ctx, runtime, n, "Cluster dns configmap", ts.Truncate(time.Second)); err != nil {
return err
}
return act.Run(ctx, n)
}
}
return nil
})
if err := postStartErrgrp.Wait(); err != nil {
return fmt.Errorf("error during post-start cluster preparation: %w", err)
// create host records in CoreDNS for external registries
if err := prepCoreDNSInjectNetworkMembers(ctx, runtime, cluster); err != nil {
return fmt.Errorf("failed to patch CoreDNS with network members: %w", err)
}
return nil
@ -1046,55 +983,30 @@ func SortClusters(clusters []*k3d.Cluster) []*k3d.Cluster {
// corednsAddHost adds a host entry to the CoreDNS configmap if it doesn't exist (a host entry is a single line of the form "IP HOST")
func corednsAddHost(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster, ip string, name string) error {
retries := 3
if v, ok := os.LookupEnv(k3d.K3dEnvDebugCorednsRetries); ok && v != "" {
l.Log().Debugf("Running with %s=%s", k3d.K3dEnvDebugCorednsRetries, v)
if r, err := strconv.Atoi(v); err == nil {
retries = r
} else {
return fmt.Errorf("Invalid value set for env var %s (%s): %w", k3d.K3dEnvDebugCorednsRetries, v, err)
}
}
// select any server node
var node *k3d.Node
for _, n := range cluster.Nodes {
if n.Role == k3d.ServerRole {
node = n
}
}
hostsEntry := fmt.Sprintf("%s %s", ip, name)
patchCmd := `patch=$(kubectl get cm coredns -n kube-system --template='{{.data.NodeHosts}}' | sed -n -E -e '/[0-9\.]{4,12}\s` + name + `$/!p' -e '$a` + hostsEntry + `' | tr '\n' '^' | busybox xargs -0 printf '{"data": {"NodeHosts":"%s"}}'| sed -E 's%\^%\\n%g') && kubectl patch cm coredns -n kube-system -p="$patch"`
successInjectCoreDNSEntry := false
// try 3 (or K3D_DEBUG_COREDNS_RETRIES value) times, as e.g. on cluster startup it may take some time for the Configmap to be available and the server to be responsive
for i := 0; i < retries; i++ {
l.Log().Debugf("Running CoreDNS patch in node %s to add %s (try %d/%d)...", node.Name, hostsEntry, i, retries)
logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd})
if err == nil {
successInjectCoreDNSEntry = true
break
} else {
msg := fmt.Sprintf("(try %d/%d) error patching the CoreDNS ConfigMap to include entry '%s': %+v", i, retries, hostsEntry, err)
if logreader != nil {
readlogs, err := io.ReadAll(logreader)
for _, node := range cluster.Nodes {
if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole {
logreader, err := runtime.ExecInNodeGetLogs(ctx, node, []string{"sh", "-c", patchCmd})
if err == nil {
successInjectCoreDNSEntry = true
break
} else {
msg := fmt.Sprintf("error patching the CoreDNS ConfigMap to include entry '%s': %+v", hostsEntry, err)
readlogs, err := ioutil.ReadAll(logreader)
if err != nil {
l.Log().Debugf("(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: %v", i, retries, node.Name, err)
l.Log().Debugf("error reading the logs from failed CoreDNS patch exec process in node %s: %v", node.Name, err)
} else {
msg += fmt.Sprintf("\nLogs: %s", string(readlogs))
}
} else {
l.Log().Debugf("(try %d/%d) error reading the logs from failed CoreDNS patch exec process in node %s: no logreader returned for exec process", i, retries, node.Name)
l.Log().Debugln(msg)
}
l.Log().Debugln(msg)
time.Sleep(1 * time.Second)
}
}
if !successInjectCoreDNSEntry {
return fmt.Errorf("failed to patch CoreDNS ConfigMap to include entry '%s' (%d tries, see debug logs)", hostsEntry, retries)
return fmt.Errorf("Failed to patch CoreDNS ConfigMap to include entry '%s' (see debug logs)", hostsEntry)
}
l.Log().Debugf("Successfully patched CoreDNS Configmap with record '%s'", hostsEntry)
return nil
}
@ -1107,7 +1019,7 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C
hostIP := clusterStartOpts.EnvironmentInfo.HostGateway
hostsEntry := fmt.Sprintf("%s %s", hostIP.String(), k3d.DefaultK3dInternalHostRecord)
l.Log().Infof("Injecting '%s' into /etc/hosts of all nodes...", hostsEntry)
l.Log().Infof("Injecting record '%s'...", hostsEntry)
// entry in /etc/hosts
errgrp, errgrpctx := errgroup.WithContext(ctx)
@ -1122,6 +1034,26 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C
}
l.Log().Debugf("Successfully added host record \"%s\" to /etc/hosts in all nodes", hostsEntry)
err := corednsAddHost(ctx, runtime, cluster, hostIP.String(), k3d.DefaultK3dInternalHostRecord)
if err != nil {
return fmt.Errorf("failed to inject host record \"%s\" into CoreDNS ConfigMap: %w", hostsEntry, err)
}
l.Log().Debugf("Successfully added host record \"%s\" to the CoreDNS ConfigMap ", hostsEntry)
return nil
}
func prepCoreDNSInjectNetworkMembers(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error {
net, err := runtime.GetNetwork(ctx, &cluster.Network)
if err != nil {
return fmt.Errorf("failed to get cluster network %s to inject host records into CoreDNS: %v", cluster.Network.Name, err)
}
l.Log().Debugf("Adding %d network members to coredns", len(net.Members))
for _, member := range net.Members {
hostsEntry := fmt.Sprintf("%s %s", member.IP.String(), member.Name)
if err := corednsAddHost(ctx, runtime, cluster, member.IP.String(), member.Name); err != nil {
return fmt.Errorf("failed to add host entry \"%s\" into CoreDNS: %v", hostsEntry, err)
}
}
return nil
}

View File

@ -25,7 +25,7 @@ import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"time"
@ -147,7 +147,7 @@ func KubeconfigGet(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.C
}
defer reader.Close()
readBytes, err := io.ReadAll(reader)
readBytes, err := ioutil.ReadAll(reader)
if err != nil {
return nil, fmt.Errorf("failed to read kubeconfig file: %w", err)
}

View File

@ -26,7 +26,7 @@ import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"strings"
"time"
@ -86,7 +86,7 @@ func UpdateLoadbalancerConfig(ctx context.Context, runtime runtimes.Runtime, clu
successCtx, successCtxCancel := context.WithDeadline(ctx, time.Now().Add(5*time.Second))
defer successCtxCancel()
err = NodeWaitForLogMessage(successCtx, runtime, cluster.ServerLoadBalancer.Node, k3d.GetReadyLogMessage(cluster.ServerLoadBalancer.Node, k3d.IntentAny), startTime)
err = NodeWaitForLogMessage(successCtx, runtime, cluster.ServerLoadBalancer.Node, k3d.ReadyLogMessageByRole[k3d.LoadBalancerRole], startTime)
if err != nil {
if errors.Is(err, context.DeadlineExceeded) {
failureCtx, failureCtxCancel := context.WithDeadline(ctx, time.Now().Add(5*time.Second))
@ -134,7 +134,7 @@ func GetLoadbalancerConfig(ctx context.Context, runtime runtimes.Runtime, cluste
}
defer reader.Close()
file, err := io.ReadAll(reader)
file, err := ioutil.ReadAll(reader)
if err != nil {
return cfg, fmt.Errorf("failed to read loadbalancer config file: %w", err)
}

View File

@ -23,19 +23,18 @@ THE SOFTWARE.
package client
import (
"bufio"
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"reflect"
"strconv"
"strings"
"time"
copystruct "github.com/mitchellh/copystructure"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/docker/go-connections/nat"
@ -45,12 +44,9 @@ import (
l "github.com/rancher/k3d/v5/pkg/logger"
"github.com/rancher/k3d/v5/pkg/runtimes"
"github.com/rancher/k3d/v5/pkg/runtimes/docker"
runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types"
runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/rancher/k3d/v5/pkg/types/fixes"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/rancher/k3d/v5/pkg/util"
"golang.org/x/sync/errgroup"
)
@ -156,7 +152,7 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
defer registryConfigReader.Close()
var err error
registryConfigBytes, err = io.ReadAll(registryConfigReader)
registryConfigBytes, err = ioutil.ReadAll(registryConfigReader)
if err != nil {
l.Log().Warnf("Failed to read registry config from node %s: %+v", node.Name, err)
}
@ -176,23 +172,23 @@ func NodeAddToCluster(ctx context.Context, runtime runtimes.Runtime, node *k3d.N
k3sURLEnvFound := false
k3sTokenEnvFoundIndex := -1
for index, envVar := range node.Env {
if strings.HasPrefix(envVar, k3s.EnvClusterConnectURL) {
if strings.HasPrefix(envVar, k3d.K3sEnvClusterConnectURL) {
k3sURLEnvFound = true
}
if strings.HasPrefix(envVar, k3s.EnvClusterToken) {
if strings.HasPrefix(envVar, k3d.K3sEnvClusterToken) {
k3sTokenEnvFoundIndex = index
}
}
if !k3sURLEnvFound {
if url, ok := node.RuntimeLabels[k3d.LabelClusterURL]; ok {
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, url))
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, url))
} else {
l.Log().Warnln("Failed to find K3S_URL value!")
}
}
if k3sTokenEnvFoundIndex != -1 && createNodeOpts.ClusterToken != "" {
l.Log().Debugln("Overriding copied cluster token with value from nodeCreateOpts...")
node.Env[k3sTokenEnvFoundIndex] = fmt.Sprintf("%s=%s", k3s.EnvClusterToken, createNodeOpts.ClusterToken)
node.Env[k3sTokenEnvFoundIndex] = fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, createNodeOpts.ClusterToken)
node.RuntimeLabels[k3d.LabelClusterToken] = createNodeOpts.ClusterToken
}
@ -250,8 +246,8 @@ func NodeAddToClusterRemote(ctx context.Context, runtime runtimes.Runtime, node
node.Env = []string{}
}
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterConnectURL, clusterRef))
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3s.EnvClusterToken, createNodeOpts.ClusterToken))
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterConnectURL, clusterRef))
node.Env = append(node.Env, fmt.Sprintf("%s=%s", k3d.K3sEnvClusterToken, createNodeOpts.ClusterToken))
if err := NodeRun(ctx, runtime, node, createNodeOpts); err != nil {
return fmt.Errorf("failed to run node '%s': %w", node.Name, err)
@ -320,7 +316,7 @@ func NodeCreateMulti(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d
currentNode := node
nodeWaitGroup.Go(func() error {
l.Log().Debugf("Starting to wait for node '%s'", currentNode.Name)
readyLogMessage := k3d.GetReadyLogMessage(currentNode, k3d.IntentNodeCreate)
readyLogMessage := k3d.ReadyLogMessageByRole[currentNode.Role]
if readyLogMessage != "" {
return NodeWaitForLogMessage(ctx, runtime, currentNode, readyLogMessage, time.Time{})
}
@ -331,7 +327,9 @@ func NodeCreateMulti(ctx context.Context, runtime runtimes.Runtime, nodes []*k3d
}
if err := nodeWaitGroup.Wait(); err != nil {
return fmt.Errorf("failed to create nodes: %w", err)
l.Log().Errorln("Failed to bring up all nodes in time. Check the logs:")
l.Log().Errorf(">>> %+v", err)
return fmt.Errorf("Failed to create nodes")
}
return nil
@ -348,7 +346,6 @@ func NodeRun(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, node
Timeout: nodeCreateOpts.Timeout,
NodeHooks: nodeCreateOpts.NodeHooks,
EnvironmentInfo: nodeCreateOpts.EnvironmentInfo,
Intent: k3d.IntentNodeCreate,
}); err != nil {
return fmt.Errorf("failed to start node '%s': %w", node.Name, err)
}
@ -400,7 +397,7 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
if nodeStartOpts.Wait {
if nodeStartOpts.ReadyLogMessage == "" {
nodeStartOpts.ReadyLogMessage = k3d.GetReadyLogMessage(node, nodeStartOpts.Intent)
nodeStartOpts.ReadyLogMessage = k3d.ReadyLogMessageByRole[node.Role]
}
if nodeStartOpts.ReadyLogMessage != "" {
l.Log().Debugf("Waiting for node %s to get ready (Log: '%s')", node.Name, nodeStartOpts.ReadyLogMessage)
@ -412,16 +409,6 @@ func NodeStart(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, no
}
}
// execute lifecycle hook actions
for _, hook := range nodeStartOpts.NodeHooks {
if hook.Stage == k3d.LifecycleStagePostStart {
l.Log().Tracef("Node %s: Executing postStartAction '%s'", node.Name, reflect.TypeOf(hook))
if err := hook.Action.Run(ctx, node); err != nil {
l.Log().Errorf("Node %s: Failed executing postStartAction '%+v': %+v", node.Name, hook, err)
}
}
}
return nil
}
@ -457,7 +444,7 @@ func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node,
// DNS Fix
if fixes.FixEnabled(fixes.EnvFixDNS) {
l.Log().Debugln(">>> enabling dns magic")
l.Log().Debugf("ENABLING DNS MAGIC!!!")
if nodeStartOpts.EnvironmentInfo == nil || nodeStartOpts.EnvironmentInfo.HostGateway == nil {
return fmt.Errorf("Cannot enable DNS fix, as Host Gateway IP is missing!")
@ -478,7 +465,7 @@ func enableFixes(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node,
// CGroupsV2Fix
if fixes.FixEnabled(fixes.EnvFixCgroupV2) {
l.Log().Debugf(">>> enabling cgroupsv2 magic")
l.Log().Debugf("ENABLING CGROUPSV2 MAGIC!!!")
if nodeStartOpts.NodeHooks == nil {
nodeStartOpts.NodeHooks = []k3d.NodeHook{}
@ -672,100 +659,61 @@ func NodeGet(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node) (*k3
// NodeWaitForLogMessage follows the logs of a node container and returns if it finds a specific line in there (or timeout is reached)
func NodeWaitForLogMessage(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, message string, since time.Time) error {
l.Log().Tracef("NodeWaitForLogMessage: Node '%s' waiting for log message '%s' since '%+v'", node.Name, message, since)
// specify max number of retries if container is in crashloop (as defined by last seen message being a fatal log)
backOffLimit := k3d.DefaultNodeWaitForLogMessageCrashLoopBackOffLimit
if l, ok := os.LookupEnv(k3d.K3dEnvDebugNodeWaitBackOffLimit); ok {
limit, err := strconv.Atoi(l)
if err == nil {
backOffLimit = limit
}
}
// start a goroutine to print a warning continuously if a node is restarting for quite some time already
donechan := make(chan struct{})
defer close(donechan)
go func(ctx context.Context, runtime runtimes.Runtime, node *k3d.Node, since time.Time, donechan chan struct{}) {
for {
select {
case <-ctx.Done():
return
case <-donechan:
return
default:
for {
select {
case <-ctx.Done():
if ctx.Err() == context.DeadlineExceeded {
d, ok := ctx.Deadline()
if ok {
l.Log().Debugf("NodeWaitForLogMessage: Context Deadline (%s) > Current Time (%s)", d, time.Now())
}
return fmt.Errorf("Context deadline exceeded while waiting for log message '%s' of node %s: %w", message, node.Name, ctx.Err())
}
// check if the container is restarting
running, status, _ := runtime.GetNodeStatus(ctx, node)
if running && status == k3d.NodeStatusRestarting && time.Now().Sub(since) > k3d.NodeWaitForLogMessageRestartWarnTime {
l.Log().Warnf("Node '%s' is restarting for more than %s now. Possibly it will recover soon (e.g. when it's waiting to join). Consider using a creation timeout to avoid waiting forever in a Restart Loop.", node.Name, k3d.NodeWaitForLogMessageRestartWarnTime)
}
time.Sleep(500 * time.Millisecond)
return ctx.Err()
default:
}
}(ctx, runtime, node, since, donechan)
// Start loop to check log stream for specified log message.
// We're looping here, as sometimes the containers run into a crash loop, but *may* recover from that
// e.g. when a new server is joining an existing cluster and has to wait for another member to finish learning.
// The logstream returned by docker ends everytime the container restarts, so we have to start from the beginning.
for i := 0; i < backOffLimit; i++ {
// get the log stream (reader is following the logstream)
out, err := runtime.GetNodeLogs(ctx, node, since, &runtimeTypes.NodeLogsOpts{Follow: true})
if out != nil {
defer out.Close()
}
// read the logs
out, err := runtime.GetNodeLogs(ctx, node, since)
if err != nil {
if out != nil {
out.Close()
}
return fmt.Errorf("Failed waiting for log message '%s' from node '%s': %w", message, node.Name, err)
}
defer out.Close()
// We're scanning the logstream continuously line-by-line
scanner := bufio.NewScanner(out)
var previousline string
for scanner.Scan() {
select {
case <-ctx.Done():
if ctx.Err() == context.DeadlineExceeded {
d, ok := ctx.Deadline()
if ok {
l.Log().Debugf("NodeWaitForLogMessage: Context Deadline (%s) > Current Time (%s)", d, time.Now())
}
return fmt.Errorf("Context deadline exceeded while waiting for log message '%s' of node %s: %w", message, node.Name, ctx.Err())
}
return ctx.Err()
default:
}
if strings.Contains(os.Getenv(k3d.K3dEnvLogNodeWaitLogs), string(node.Role)) {
l.Log().Tracef(">>> Parsing log line: `%s`", scanner.Text())
}
// check if we can find the specified line in the log
if strings.Contains(scanner.Text(), message) {
l.Log().Tracef("Found target message `%s` in log line `%s`", message, scanner.Text())
l.Log().Debugf("Finished waiting for log message '%s' from node '%s'", message, node.Name)
return nil
}
previousline = scanner.Text()
buf := new(bytes.Buffer)
nRead, _ := buf.ReadFrom(out)
out.Close()
output := buf.String()
if nRead > 0 && strings.Contains(os.Getenv("K3D_LOG_NODE_WAIT_LOGS"), string(node.Role)) {
l.Log().Tracef("=== Read logs since %s ===\n%s\n", since, output)
}
out.Close() // no more input on scanner, but target log not yet found -> close current logreader (precautionary)
// we got here, because the logstream ended (no more input on scanner), so we check if maybe the container crashed
if strings.Contains(previousline, "level=fatal") {
// case 1: last log line we saw contained a fatal error, so probably it crashed and we want to retry on restart
l.Log().Warnf("warning: encountered fatal log from node %s (retrying %d/%d): %s", node.Name, i, backOffLimit, previousline)
out.Close()
time.Sleep(500 * time.Millisecond)
continue
} else {
// case 2: last log line we saw did not contain a fatal error, so we break the loop here and return a generic error
// check if we can find the specified line in the log
if nRead > 0 && strings.Contains(output, message) {
if l.Log().GetLevel() >= logrus.TraceLevel {
temp := strings.Split(output, "\n")
for _, t := range temp {
if strings.Contains(t, message) {
l.Log().Tracef("Found target log line: `%s`", t)
}
}
}
break
}
// check if the container is restarting
running, status, _ := runtime.GetNodeStatus(ctx, node)
if running && status == k3d.NodeStatusRestarting && time.Now().Sub(since) > k3d.NodeWaitForLogMessageRestartWarnTime {
l.Log().Warnf("Node '%s' is restarting for more than a minute now. Possibly it will recover soon (e.g. when it's waiting to join). Consider using a creation timeout to avoid waiting forever in a Restart Loop.", node.Name)
}
time.Sleep(500 * time.Millisecond) // wait for half a second to avoid overloading docker (error `socket: too many open files`)
}
return fmt.Errorf("error waiting for log line `%s` from node '%s': stopped returning log lines", message, node.Name)
l.Log().Debugf("Finished waiting for log message '%s' from node '%s'", message, node.Name)
return nil
}
// NodeFilterByRoles filters a list of nodes by their roles

View File

@ -100,9 +100,8 @@ func TransformPorts(ctx context.Context, runtime runtimes.Runtime, cluster *k3d.
}
// print generated loadbalancer config if exists
// (avoid segmentation fault if loadbalancer is disabled)
if l.Log().GetLevel() >= logrus.DebugLevel && cluster.ServerLoadBalancer != nil {
// print generated loadbalancer config
if l.Log().GetLevel() >= logrus.DebugLevel {
yamlized, err := yaml.Marshal(cluster.ServerLoadBalancer.Config)
if err != nil {
l.Log().Errorf("error printing loadbalancer config: %v", err)

View File

@ -310,7 +310,6 @@ func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, runt
k8s.LocalRegistryHostingV1{
Host: fmt.Sprintf("%s:%s", host, registries[0].ExposureOpts.Binding.HostPort),
HostFromContainerRuntime: fmt.Sprintf("%s:%s", registries[0].Host, registries[0].ExposureOpts.Port.Port()),
HostFromClusterNetwork: fmt.Sprintf("%s:%s", registries[0].Host, registries[0].ExposureOpts.Port.Port()),
Help: "https://k3d.io/usage/guides/registries/#using-a-local-registry",
},
)

View File

@ -42,7 +42,6 @@ metadata:
data:
localRegistryHosting.v1: |
host: test-host:5432
hostFromClusterNetwork: test-host:1234
hostFromContainerRuntime: test-host:1234
help: https://k3d.io/usage/guides/registries/#using-a-local-registry
`

View File

@ -24,6 +24,7 @@ package client
import (
"context"
"io/ioutil"
"os"
"testing"
@ -177,7 +178,7 @@ func Test_findRuntimeImage(T *testing.T) {
func Test_findImages(t *testing.T) {
// given
tarImage, err := os.CreateTemp("", "images.tgz")
tarImage, err := ioutil.TempFile("", "images.tgz")
if err != nil {
t.Fatal("Failed to create temporary file")
}

View File

@ -25,7 +25,7 @@ import (
"bytes"
"errors"
"fmt"
"os"
"io/ioutil"
"strings"
"sigs.k8s.io/yaml"
@ -39,7 +39,7 @@ import (
func ValidateSchemaFile(filepath string, schema []byte) error {
l.Log().Debugf("Validating file %s against default JSONSchema...", filepath)
fileContents, err := os.ReadFile(filepath)
fileContents, err := ioutil.ReadFile(filepath)
if err != nil {
return fmt.Errorf("Failed to read file %s: %+v", filepath, err)
}
@ -53,7 +53,7 @@ func ValidateSchemaFile(filepath string, schema []byte) error {
}
// ValidateSchema validates a YAML construct (non-struct representation) against a JSON Schema
func ValidateSchema(content interface{}, schemaJSON []byte) error {
func ValidateSchema(content map[string]interface{}, schemaJSON []byte) error {
contentYaml, err := yaml.Marshal(content)
if err != nil {
@ -64,11 +64,6 @@ func ValidateSchema(content interface{}, schemaJSON []byte) error {
return err
}
return ValidateSchemaJSON(contentJSON, schemaJSON)
}
func ValidateSchemaJSON(contentJSON []byte, schemaJSON []byte) error {
if bytes.Equal(contentJSON, []byte("null")) {
contentJSON = []byte("{}") // non-json yaml struct
}

View File

@ -35,20 +35,6 @@ func Migrate(config types.Config, targetVersion string) (types.Config, error) {
return nil, fmt.Errorf("no migration possible from '%s' to '%s'", config.GetAPIVersion(), targetVersion)
}
cfg, err := migration(config)
if err != nil {
return nil, fmt.Errorf("error migrating config: %w", err)
}
schema, err := GetSchemaByVersion(cfg.GetAPIVersion())
if err != nil {
return nil, fmt.Errorf("error getting schema for config apiVersion %s: %w", cfg.GetAPIVersion(), err)
}
if err := ValidateSchema(cfg, schema); err != nil {
return config, fmt.Errorf("post-migrate schema validation failed: %w", err)
}
return cfg, err
return migration(config)
}

View File

@ -1,76 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package config
import (
"testing"
"github.com/go-test/deep"
l "github.com/rancher/k3d/v5/pkg/logger"
"github.com/spf13/viper"
)
func TestMigrateV1Alpha2ToV1Alpha3(t *testing.T) {
actualPath := "test_assets/config_test_simple_migration_v1alpha2.yaml"
expectedPath := "test_assets/config_test_simple_migration_v1alpha3.yaml"
actualViper := viper.New()
expectedViper := viper.New()
actualViper.SetConfigType("yaml")
expectedViper.SetConfigType("yaml")
actualViper.SetConfigFile(actualPath)
expectedViper.SetConfigFile(expectedPath)
if err := actualViper.ReadInConfig(); err != nil {
t.Fatal(err)
}
if err := expectedViper.ReadInConfig(); err != nil {
t.Fatal(err)
}
actualCfg, err := FromViper(actualViper)
if err != nil {
t.Fatal(err)
}
if actualCfg.GetAPIVersion() != DefaultConfigApiVersion {
actualCfg, err = Migrate(actualCfg, DefaultConfigApiVersion)
if err != nil {
l.Log().Fatalln(err)
}
}
expectedCfg, err := FromViper(expectedViper)
if err != nil {
t.Fatal(err)
}
if diff := deep.Equal(actualCfg, expectedCfg); diff != nil {
t.Fatalf("Actual\n%#v\ndoes not match expected\n%+v\nDiff:\n%#v", actualCfg, expectedCfg, diff)
}
}

View File

@ -1,51 +0,0 @@
apiVersion: k3d.io/v1alpha2
kind: Simple
name: test
servers: 3
agents: 2
kubeAPI:
hostIP: "0.0.0.0"
hostPort: "6446"
#image: rancher/k3s:latest
volumes:
- volume: /my/path:/some/path
nodeFilters:
- all
ports:
- port: 80:80
nodeFilters:
- loadbalancer
- port: 0.0.0.0:443:443
nodeFilters:
- loadbalancer
env:
- envVar: bar=baz,bob
nodeFilters:
- all
labels:
- label: foo=bar
nodeFilters:
- server[0]
- loadbalancer
registries:
create: true
use: []
config: |
mirrors:
"my.company.registry":
endpoint:
- http://my.company.registry:5000
options:
k3d:
wait: true
timeout: "360s" # should be pretty high for multi-server clusters to allow for a proper startup routine
disableLoadbalancer: false
disableImageVolume: false
k3s:
extraServerArgs:
- --tls-san=127.0.0.1
extraAgentArgs: []
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true

View File

@ -1,55 +0,0 @@
apiVersion: k3d.io/v1alpha3
kind: Simple
name: test
servers: 3
agents: 2
kubeAPI:
hostIP: "0.0.0.0"
hostPort: "6446"
#image: rancher/k3s:latest
volumes:
- volume: /my/path:/some/path
nodeFilters:
- all
ports:
- port: 80:80
nodeFilters:
- loadbalancer
- port: 0.0.0.0:443:443
nodeFilters:
- loadbalancer
env:
- envVar: bar=baz,bob
nodeFilters:
- all
registries:
create:
name: k3d-test-registry
host: "0.0.0.0"
hostPort: random
config: |
mirrors:
"my.company.registry":
endpoint:
- http://my.company.registry:5000
options:
k3d:
wait: true
timeout: "360s" # should be pretty high for multi-server clusters to allow for a proper startup routine
disableLoadbalancer: false
disableImageVolume: false
k3s:
extraArgs:
- arg: --tls-san=127.0.0.1
nodeFilters:
- server:*
kubeconfig:
updateDefaultKubeconfig: true
switchCurrentContext: true
runtime:
labels:
- label: foo=bar
nodeFilters:
- server:0
- loadbalancer

View File

@ -25,7 +25,7 @@ package config
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
@ -336,7 +336,7 @@ func TransformSimpleToClusterConfig(ctx context.Context, runtime runtimes.Runtim
if err != nil {
return nil, fmt.Errorf("failed to open registry config file at %s: %w", simpleConfig.Registries.Config, err)
}
configBytes, err := io.ReadAll(registryConfigFile)
configBytes, err := ioutil.ReadAll(registryConfigFile)
if err != nil {
return nil, fmt.Errorf("failed to read registry config file at %s: %w", registryConfigFile.Name(), err)
}

View File

@ -57,23 +57,23 @@ var DefaultConfig = fmt.Sprintf(
)
type VolumeWithNodeFilters struct {
Volume string `mapstructure:"volume" yaml:"volume,omitempty" json:"volume,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Volume string `mapstructure:"volume" yaml:"volume" json:"volume,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type PortWithNodeFilters struct {
Port string `mapstructure:"port" yaml:"port,omitempty" json:"port,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Port string `mapstructure:"port" yaml:"port" json:"port,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type LabelWithNodeFilters struct {
Label string `mapstructure:"label" yaml:"label,omitempty" json:"label,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Label string `mapstructure:"label" yaml:"label" json:"label,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type EnvVarWithNodeFilters struct {
EnvVar string `mapstructure:"envVar" yaml:"envVar,omitempty" json:"envVar,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
EnvVar string `mapstructure:"envVar" yaml:"envVar" json:"envVar,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
// SimpleConfigOptionsKubeconfig describes the set of options referring to the kubeconfig during cluster creation.
@ -83,49 +83,49 @@ type SimpleConfigOptionsKubeconfig struct {
}
type SimpleConfigOptions struct {
K3dOptions SimpleConfigOptionsK3d `mapstructure:"k3d" yaml:"k3d" json:"k3d"`
K3sOptions SimpleConfigOptionsK3s `mapstructure:"k3s" yaml:"k3s" json:"k3s"`
KubeconfigOptions SimpleConfigOptionsKubeconfig `mapstructure:"kubeconfig" yaml:"kubeconfig" json:"kubeconfig"`
Runtime SimpleConfigOptionsRuntime `mapstructure:"runtime" yaml:"runtime" json:"runtime"`
K3dOptions SimpleConfigOptionsK3d `mapstructure:"k3d" yaml:"k3d"`
K3sOptions SimpleConfigOptionsK3s `mapstructure:"k3s" yaml:"k3s"`
KubeconfigOptions SimpleConfigOptionsKubeconfig `mapstructure:"kubeconfig" yaml:"kubeconfig"`
Runtime SimpleConfigOptionsRuntime `mapstructure:"runtime" yaml:"runtime"`
}
type SimpleConfigOptionsRuntime struct {
GPURequest string `mapstructure:"gpuRequest,omitempty" yaml:"gpuRequest,omitempty" json:"gpuRequest,omitempty"`
ServersMemory string `mapstructure:"serversMemory,omitempty" yaml:"serversMemory,omitempty" json:"serversMemory,omitempty"`
AgentsMemory string `mapstructure:"agentsMemory,omitempty" yaml:"agentsMemory,omitempty" json:"agentsMemory,omitempty"`
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"`
AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"`
}
type SimpleConfigOptionsK3d struct {
Wait bool `mapstructure:"wait" yaml:"wait" json:"wait"`
Timeout time.Duration `mapstructure:"timeout" yaml:"timeout,omitempty" json:"timeout,omitempty"`
DisableLoadbalancer bool `mapstructure:"disableLoadbalancer" yaml:"disableLoadbalancer" json:"disableLoadbalancer"`
DisableImageVolume bool `mapstructure:"disableImageVolume" yaml:"disableImageVolume" json:"disableImageVolume"`
NoRollback bool `mapstructure:"disableRollback" yaml:"disableRollback" json:"disableRollback"`
PrepDisableHostIPInjection bool `mapstructure:"disableHostIPInjection" json:"disableHostIPInjection"`
NodeHookActions []k3d.NodeHookAction `mapstructure:"nodeHookActions" yaml:"nodeHookActions,omitempty" json:"nodeHookActions,omitempty"`
Wait bool `mapstructure:"wait" yaml:"wait"`
Timeout time.Duration `mapstructure:"timeout" yaml:"timeout"`
DisableLoadbalancer bool `mapstructure:"disableLoadbalancer" yaml:"disableLoadbalancer"`
DisableImageVolume bool `mapstructure:"disableImageVolume" yaml:"disableImageVolume"`
NoRollback bool `mapstructure:"disableRollback" yaml:"disableRollback"`
PrepDisableHostIPInjection bool `mapstructure:"disableHostIPInjection" yaml:"disableHostIPInjection"`
NodeHookActions []k3d.NodeHookAction `mapstructure:"nodeHookActions" yaml:"nodeHookActions,omitempty"`
}
type SimpleConfigOptionsK3s struct {
ExtraServerArgs []string `mapstructure:"extraServerArgs,omitempty" yaml:"extraServerArgs,omitempty" json:"extraServerArgs,omitempty"`
ExtraAgentArgs []string `mapstructure:"extraAgentArgs,omitempty" yaml:"extraAgentArgs,omitempty" json:"extraAgentArgs,omitempty"`
ExtraServerArgs []string `mapstructure:"extraServerArgs" yaml:"extraServerArgs"`
ExtraAgentArgs []string `mapstructure:"extraAgentArgs" yaml:"extraAgentArgs"`
}
// SimpleConfig describes the toplevel k3d configuration file.
type SimpleConfig struct {
configtypes.TypeMeta `mapstructure:",squash" yaml:",inline"`
Name string `mapstructure:"name" yaml:"name,omitempty" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers,omitempty" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents,omitempty" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI,omitempty" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image,omitempty" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network,omitempty" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet,omitempty" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken,omitempty" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes,omitempty" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports,omitempty" json:"ports,omitempty"`
Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels,omitempty" json:"labels,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options,omitempty" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env,omitempty" json:"env,omitempty"`
Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels" json:"labels,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries struct {
Use []string `mapstructure:"use" yaml:"use,omitempty" json:"use,omitempty"`
Create bool `mapstructure:"create" yaml:"create,omitempty" json:"create,omitempty"`

View File

@ -119,6 +119,7 @@
"default": true
},
"timeout": {
"type": "string",
"examples": [
"60s",
"1m",

View File

@ -57,34 +57,34 @@ var DefaultConfig = fmt.Sprintf(
)
type VolumeWithNodeFilters struct {
Volume string `mapstructure:"volume" yaml:"volume,omitempty" json:"volume,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Volume string `mapstructure:"volume" yaml:"volume" json:"volume,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type PortWithNodeFilters struct {
Port string `mapstructure:"port" yaml:"port,omitempty" json:"port,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Port string `mapstructure:"port" yaml:"port" json:"port,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type LabelWithNodeFilters struct {
Label string `mapstructure:"label" yaml:"label,omitempty" json:"label,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Label string `mapstructure:"label" yaml:"label" json:"label,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type EnvVarWithNodeFilters struct {
EnvVar string `mapstructure:"envVar" yaml:"envVar,omitempty" json:"envVar,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
EnvVar string `mapstructure:"envVar" yaml:"envVar" json:"envVar,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type K3sArgWithNodeFilters struct {
Arg string `mapstructure:"arg" yaml:"arg,omitempty" json:"arg,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters,omitempty" json:"nodeFilters,omitempty"`
Arg string `mapstructure:"arg" yaml:"arg" json:"arg,omitempty"`
NodeFilters []string `mapstructure:"nodeFilters" yaml:"nodeFilters" json:"nodeFilters,omitempty"`
}
type SimpleConfigRegistryCreateConfig struct {
Name string `mapstructure:"name" yaml:"name,omitempty" json:"name,omitempty"`
Host string `mapstructure:"host" yaml:"host,omitempty" json:"host,omitempty"`
HostPort string `mapstructure:"hostPort" yaml:"hostPort,omitempty" json:"hostPort,omitempty"`
Name string `mapstructure:"name" yaml:"name" json:"name"`
Host string `mapstructure:"host" yaml:"host" json:"host"`
HostPort string `mapstructure:"hostPort" yaml:"hostPort" json:"hostPort"`
}
// SimpleConfigOptionsKubeconfig describes the set of options referring to the kubeconfig during cluster creation.
@ -94,36 +94,36 @@ type SimpleConfigOptionsKubeconfig struct {
}
type SimpleConfigOptions struct {
K3dOptions SimpleConfigOptionsK3d `mapstructure:"k3d" yaml:"k3d" json:"k3d"`
K3sOptions SimpleConfigOptionsK3s `mapstructure:"k3s" yaml:"k3s" json:"k3s"`
KubeconfigOptions SimpleConfigOptionsKubeconfig `mapstructure:"kubeconfig" yaml:"kubeconfig" json:"kubeconfig"`
Runtime SimpleConfigOptionsRuntime `mapstructure:"runtime" yaml:"runtime" json:"runtime"`
K3dOptions SimpleConfigOptionsK3d `mapstructure:"k3d" yaml:"k3d"`
K3sOptions SimpleConfigOptionsK3s `mapstructure:"k3s" yaml:"k3s"`
KubeconfigOptions SimpleConfigOptionsKubeconfig `mapstructure:"kubeconfig" yaml:"kubeconfig"`
Runtime SimpleConfigOptionsRuntime `mapstructure:"runtime" yaml:"runtime"`
}
type SimpleConfigOptionsRuntime struct {
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest,omitempty" json:"gpuRequest,omitempty"`
ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory,omitempty" json:"serversMemory,omitempty"`
AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory,omitempty" json:"agentsMemory,omitempty"`
Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels,omitempty" json:"labels,omitempty"`
GPURequest string `mapstructure:"gpuRequest" yaml:"gpuRequest"`
ServersMemory string `mapstructure:"serversMemory" yaml:"serversMemory"`
AgentsMemory string `mapstructure:"agentsMemory" yaml:"agentsMemory"`
Labels []LabelWithNodeFilters `mapstructure:"labels" yaml:"labels"`
}
type SimpleConfigOptionsK3d struct {
Wait bool `mapstructure:"wait" yaml:"wait" json:"wait"`
Timeout time.Duration `mapstructure:"timeout" yaml:"timeout,omitempty" json:"timeout,omitempty"`
DisableLoadbalancer bool `mapstructure:"disableLoadbalancer" yaml:"disableLoadbalancer" json:"disableLoadbalancer"`
DisableImageVolume bool `mapstructure:"disableImageVolume" yaml:"disableImageVolume" json:"disableImageVolume"`
NoRollback bool `mapstructure:"disableRollback" yaml:"disableRollback" json:"disableRollback"`
NodeHookActions []k3d.NodeHookAction `mapstructure:"nodeHookActions" yaml:"nodeHookActions,omitempty" json:"nodeHookActions,omitempty"`
Loadbalancer SimpleConfigOptionsK3dLoadbalancer `mapstructure:"loadbalancer" yaml:"loadbalancer,omitempty" json:"loadbalancer,omitempty"`
Wait bool `mapstructure:"wait" yaml:"wait"`
Timeout time.Duration `mapstructure:"timeout" yaml:"timeout"`
DisableLoadbalancer bool `mapstructure:"disableLoadbalancer" yaml:"disableLoadbalancer"`
DisableImageVolume bool `mapstructure:"disableImageVolume" yaml:"disableImageVolume"`
NoRollback bool `mapstructure:"disableRollback" yaml:"disableRollback"`
NodeHookActions []k3d.NodeHookAction `mapstructure:"nodeHookActions" yaml:"nodeHookActions,omitempty"`
Loadbalancer SimpleConfigOptionsK3dLoadbalancer `mapstructure:"loadbalancer" yaml:"loadbalancer,omitempty"`
}
type SimpleConfigOptionsK3dLoadbalancer struct {
ConfigOverrides []string `mapstructure:"configOverrides" yaml:"configOverrides,omitempty" json:"configOverrides,omitempty"`
ConfigOverrides []string `mapstructure:"configOverrides" yaml:"configOverrides,omitempty"`
}
type SimpleConfigOptionsK3s struct {
ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs,omitempty" json:"extraArgs,omitempty"`
NodeLabels []LabelWithNodeFilters `mapstructure:"nodeLabels" yaml:"nodeLabels,omitempty" json:"nodeLabels,omitempty"`
ExtraArgs []K3sArgWithNodeFilters `mapstructure:"extraArgs" yaml:"extraArgs"`
NodeLabels []LabelWithNodeFilters `mapstructure:"nodeLabels" yaml:"nodeLabels"`
}
type SimpleConfigRegistries struct {
@ -141,35 +141,35 @@ type SimpleConfigRegistriesIntermediateV1alpha2 struct {
// SimpleConfig describes the toplevel k3d configuration file.
type SimpleConfig struct {
config.TypeMeta `mapstructure:",squash" yaml:",inline"`
Name string `mapstructure:"name" yaml:"name,omitempty" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers,omitempty" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents,omitempty" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI,omitempty" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image,omitempty" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network,omitempty" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet,omitempty" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken,omitempty" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes,omitempty" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports,omitempty" json:"ports,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options,omitempty" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env,omitempty" json:"env,omitempty"`
Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries SimpleConfigRegistries `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
}
type SimpleConfigIntermediateV1alpha2 struct {
config.TypeMeta `mapstructure:",squash" yaml:",inline"`
Name string `mapstructure:"name" yaml:"name,omitempty" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers,omitempty" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents,omitempty" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI,omitempty" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image,omitempty" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network,omitempty" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet,omitempty" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken,omitempty" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes,omitempty" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports,omitempty" json:"ports,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options,omitempty" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env,omitempty" json:"env,omitempty"`
Name string `mapstructure:"name" yaml:"name" json:"name,omitempty"`
Servers int `mapstructure:"servers" yaml:"servers" json:"servers,omitempty"` //nolint:lll // default 1
Agents int `mapstructure:"agents" yaml:"agents" json:"agents,omitempty"` //nolint:lll // default 0
ExposeAPI SimpleExposureOpts `mapstructure:"kubeAPI" yaml:"kubeAPI" json:"kubeAPI,omitempty"`
Image string `mapstructure:"image" yaml:"image" json:"image,omitempty"`
Network string `mapstructure:"network" yaml:"network" json:"network,omitempty"`
Subnet string `mapstructure:"subnet" yaml:"subnet" json:"subnet,omitempty"`
ClusterToken string `mapstructure:"token" yaml:"clusterToken" json:"clusterToken,omitempty"` // default: auto-generated
Volumes []VolumeWithNodeFilters `mapstructure:"volumes" yaml:"volumes" json:"volumes,omitempty"`
Ports []PortWithNodeFilters `mapstructure:"ports" yaml:"ports" json:"ports,omitempty"`
Options SimpleConfigOptions `mapstructure:"options" yaml:"options" json:"options,omitempty"`
Env []EnvVarWithNodeFilters `mapstructure:"env" yaml:"env" json:"env,omitempty"`
Registries SimpleConfigRegistriesIntermediateV1alpha2 `mapstructure:"registries" yaml:"registries,omitempty" json:"registries,omitempty"`
}

View File

@ -26,6 +26,7 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"github.com/docker/docker/api/types"
@ -107,7 +108,7 @@ func removeContainer(ctx context.Context, ID string) error {
}
// pullImage pulls a container image and outputs progress if --verbose flag is set
func pullImage(ctx context.Context, docker client.APIClient, image string) error {
func pullImage(ctx context.Context, docker *client.Client, image string) error {
resp, err := docker.ImagePull(ctx, image, types.ImagePullOptions{})
if err != nil {
@ -118,7 +119,7 @@ func pullImage(ctx context.Context, docker client.APIClient, image string) error
l.Log().Infof("Pulling image '%s'", image)
// in debug mode (--verbose flag set), output pull progress
var writer io.Writer = io.Discard
var writer io.Writer = ioutil.Discard
if l.Log().GetLevel() == logrus.DebugLevel {
writer = os.Stdout
}

View File

@ -28,14 +28,13 @@ import (
"errors"
"fmt"
"io"
"io/ioutil"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
l "github.com/rancher/k3d/v5/pkg/logger"
runtimeErr "github.com/rancher/k3d/v5/pkg/runtimes/errors"
runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types"
k3d "github.com/rancher/k3d/v5/pkg/types"
)
@ -272,7 +271,7 @@ func (d Docker) NodeIsRunning(ctx context.Context, node *k3d.Node) (bool, error)
}
// GetNodeLogs returns the logs from a given node
func (d Docker) GetNodeLogs(ctx context.Context, node *k3d.Node, since time.Time, opts *runtimeTypes.NodeLogsOpts) (io.ReadCloser, error) {
func (d Docker) GetNodeLogs(ctx context.Context, node *k3d.Node, since time.Time) (io.ReadCloser, error) {
// get the container for the given node
container, err := getNodeContainer(ctx, node)
if err != nil {
@ -299,7 +298,7 @@ func (d Docker) GetNodeLogs(ctx context.Context, node *k3d.Node, since time.Time
if !since.IsZero() {
sinceStr = since.Format("2006-01-02T15:04:05.999999999Z")
}
logreader, err := docker.ContainerLogs(ctx, container.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Since: sinceStr, Follow: opts.Follow})
logreader, err := docker.ContainerLogs(ctx, container.ID, types.ContainerLogsOptions{ShowStdout: true, ShowStderr: true, Since: sinceStr})
if err != nil {
return nil, fmt.Errorf("docker failed to get logs from node '%s' (container '%s'): %w", node.Name, container.ID, err)
}
@ -310,9 +309,6 @@ func (d Docker) GetNodeLogs(ctx context.Context, node *k3d.Node, since time.Time
// ExecInNodeGetLogs executes a command inside a node and returns the logs to the caller, e.g. to parse them
func (d Docker) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []string) (*bufio.Reader, error) {
resp, err := executeInNode(ctx, node, cmd)
if resp != nil {
defer resp.Close()
}
if err != nil {
if resp != nil && resp.Reader != nil { // sometimes the exec process returns with a non-zero exit code, but we still have the logs we
return resp.Reader, err
@ -325,12 +321,9 @@ func (d Docker) ExecInNodeGetLogs(ctx context.Context, node *k3d.Node, cmd []str
// ExecInNode execs a command inside a node
func (d Docker) ExecInNode(ctx context.Context, node *k3d.Node, cmd []string) error {
execConnection, err := executeInNode(ctx, node, cmd)
if execConnection != nil {
defer execConnection.Close()
}
if err != nil {
if execConnection != nil && execConnection.Reader != nil {
logs, err := io.ReadAll(execConnection.Reader)
logs, err := ioutil.ReadAll(execConnection.Reader)
if err != nil {
return fmt.Errorf("failed to get logs from errored exec process in node '%s': %w", node.Name, err)
}

View File

@ -47,7 +47,7 @@ import (
// TranslateNodeToContainer translates a k3d node specification to a docker container representation
func TranslateNodeToContainer(node *k3d.Node) (*NodeInDocker, error) {
init := true
if disableInit, err := strconv.ParseBool(os.Getenv(k3d.K3dEnvDebugDisableDockerInit)); err == nil && disableInit {
if disableInit, err := strconv.ParseBool(os.Getenv("K3D_DEBUG_DISABLE_DOCKER_INIT")); err == nil && disableInit {
l.Log().Traceln("docker-init disabled for all containers")
init = false
}

View File

@ -60,7 +60,7 @@ func TestTranslateNodeToContainer(t *testing.T) {
}
init := true
if disableInit, err := strconv.ParseBool(os.Getenv(k3d.K3dEnvDebugDisableDockerInit)); err == nil && disableInit {
if disableInit, err := strconv.ParseBool(os.Getenv("K3D_DEBUG_DISABLE_DOCKER_INIT")); err == nil && disableInit {
init = false
}

View File

@ -28,6 +28,7 @@ import (
"fmt"
"io"
"os"
"path"
"github.com/docker/cli/cli/command"
"github.com/docker/cli/cli/flags"
@ -39,7 +40,6 @@ import (
l "github.com/rancher/k3d/v5/pkg/logger"
runtimeErrors "github.com/rancher/k3d/v5/pkg/runtimes/errors"
k3d "github.com/rancher/k3d/v5/pkg/types"
"github.com/spf13/pflag"
)
// GetDefaultObjectLabelsFilter returns docker type filters created from k3d labels
@ -149,7 +149,6 @@ func (d Docker) ReadFromNode(ctx context.Context, path string, node *k3d.Node) (
if err != nil {
return nil, fmt.Errorf("failed to get docker client: %w", err)
}
defer docker.Close()
reader, _, err := docker.CopyFromContainer(ctx, nodeContainer.ID, path)
if err != nil {
@ -163,7 +162,7 @@ func (d Docker) ReadFromNode(ctx context.Context, path string, node *k3d.Node) (
}
// GetDockerClient returns a docker client
func GetDockerClient() (client.APIClient, error) {
func GetDockerClient() (*client.Client, error) {
dockerCli, err := command.NewDockerCli(command.WithStandardStreams())
if err != nil {
return nil, fmt.Errorf("failed to create new docker CLI with standard streams: %w", err)
@ -172,16 +171,49 @@ func GetDockerClient() (client.APIClient, error) {
newClientOpts := flags.NewClientOptions()
newClientOpts.Common.LogLevel = l.Log().GetLevel().String() // this is needed, as the following Initialize() call will set a new log level on the global logrus instance
flagset := pflag.NewFlagSet("docker", pflag.ContinueOnError)
newClientOpts.Common.InstallFlags(flagset)
newClientOpts.Common.SetDefaultOptions(flagset)
err = dockerCli.Initialize(newClientOpts)
if err != nil {
return nil, fmt.Errorf("failed to initialize docker CLI: %w", err)
}
return dockerCli.Client(), nil
// check for TLS Files used for protected connections
currentContext := dockerCli.CurrentContext()
storageInfo := dockerCli.ContextStore().GetStorageInfo(currentContext)
tlsFilesMap, err := dockerCli.ContextStore().ListTLSFiles(currentContext)
if err != nil {
return nil, fmt.Errorf("docker CLI failed to list TLS files for context '%s': %w", currentContext, err)
}
endpointDriver := "docker"
tlsFiles := tlsFilesMap[endpointDriver]
// get client by endpoint configuration
// inspired by https://github.com/docker/cli/blob/a32cd16160f1b41c1c4ae7bee4dac929d1484e59/cli/command/cli.go#L296-L308
ep := dockerCli.DockerEndpoint()
if ep.Host != "" {
clientopts, err := ep.ClientOpts()
if err != nil {
return nil, fmt.Errorf("failed to get client opts for docker endpoint: %w", err)
}
headers := make(map[string]string, 1)
headers["User-Agent"] = command.UserAgent()
clientopts = append(clientopts, client.WithHTTPHeaders(headers))
// only set TLS config if present
if len(tlsFiles) >= 3 {
clientopts = append(clientopts,
client.WithTLSClientConfig(
path.Join(storageInfo.TLSPath, endpointDriver, tlsFiles[0]),
path.Join(storageInfo.TLSPath, endpointDriver, tlsFiles[1]),
path.Join(storageInfo.TLSPath, endpointDriver, tlsFiles[2]),
),
)
}
return client.NewClientWithOpts(clientopts...)
}
// fallback default client
return client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
}
// isAttachedToNetwork return true if node is attached to network

View File

@ -68,7 +68,7 @@ type Runtime interface {
GetRuntimePath() string // returns e.g. '/var/run/docker.sock' for a default docker setup
ExecInNode(context.Context, *k3d.Node, []string) error
ExecInNodeGetLogs(context.Context, *k3d.Node, []string) (*bufio.Reader, error)
GetNodeLogs(context.Context, *k3d.Node, time.Time, *runtimeTypes.NodeLogsOpts) (io.ReadCloser, error)
GetNodeLogs(context.Context, *k3d.Node, time.Time) (io.ReadCloser, error)
GetImages(context.Context) ([]string, error)
CopyToNode(context.Context, string, string, *k3d.Node) error // @param context, source, destination, node
WriteToNode(context.Context, []byte, string, os.FileMode, *k3d.Node) error // @param context, content, destination, filemode, node

View File

@ -32,7 +32,3 @@ type RuntimeInfo struct {
CgroupDriver string `yaml:",omitempty" json:",omitempty"`
Filesystem string `yaml:",omitempty" json:",omitempty"`
}
type NodeLogsOpts struct {
Follow bool
}

View File

@ -1,96 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package types
import (
"fmt"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/rancher/k3d/v5/version"
)
// DefaultClusterName specifies the default name used for newly created clusters
const DefaultClusterName = "k3s-default"
// DefaultClusterNameMaxLength specifies the maximal length of a passed in cluster name
// This restriction allows us to construct an name consisting of
// <DefaultObjectNamePrefix[3]>-<ClusterName>-<TypeSuffix[5-10]>-<Counter[1-3]>
// ... and still stay within the 64 character limit (e.g. of docker)
const DefaultClusterNameMaxLength = 32
// DefaultObjectNamePrefix defines the name prefix for every object created by k3d
const DefaultObjectNamePrefix = "k3d"
// DefaultRuntimeLabels specifies a set of labels that will be attached to k3d runtime objects by default
var DefaultRuntimeLabels = map[string]string{
"app": "k3d",
}
// DefaultRuntimeLabelsVar specifies a set of labels that will be attached to k3d runtime objects by default but are not static (e.g. across k3d versions)
var DefaultRuntimeLabelsVar = map[string]string{
"k3d.version": version.GetVersion(),
}
// DefaultRoleCmds maps the node roles to their respective default commands
var DefaultRoleCmds = map[Role][]string{
ServerRole: {"server"},
AgentRole: {"agent"},
}
// DefaultTmpfsMounts specifies tmpfs mounts that are required for all k3d nodes
var DefaultTmpfsMounts = []string{
"/run",
"/var/run",
}
// DefaultNodeEnv defines some default environment variables that should be set on every node
var DefaultNodeEnv = []string{
fmt.Sprintf("%s=/output/kubeconfig.yaml", k3s.EnvKubeconfigOutput),
}
// DefaultK3dInternalHostRecord defines the default /etc/hosts entry for the k3d host
const DefaultK3dInternalHostRecord = "host.k3d.internal"
// DefaultImageVolumeMountPath defines the mount path inside k3d nodes where we will mount the shared image volume by default
const DefaultImageVolumeMountPath = "/k3d/images"
// DefaultConfigDirName defines the name of the config directory (where we'll e.g. put the kubeconfigs)
const DefaultConfigDirName = ".k3d" // should end up in $HOME/
// DefaultKubeconfigPrefix defines the default prefix for kubeconfig files
const DefaultKubeconfigPrefix = DefaultObjectNamePrefix + "-kubeconfig"
// DefaultAPIPort defines the default Kubernetes API Port
const DefaultAPIPort = "6443"
// DefaultAPIHost defines the default host (IP) for the Kubernetes API
const DefaultAPIHost = "0.0.0.0"
// GetDefaultObjectName prefixes the passed name with the default prefix
func GetDefaultObjectName(name string) string {
return fmt.Sprintf("%s-%s", DefaultObjectNamePrefix, name)
}
// DefaultNodeWaitForLogMessageCrashLoopBackOffLimit defines the maximum number of retries to find the target log message, if the
// container is in a crash loop.
// This makes sense e.g. when a new server is waiting to join an existing cluster and has to wait for other learners to finish.
const DefaultNodeWaitForLogMessageCrashLoopBackOffLimit = 10

View File

@ -1,42 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package types
// k3d config environment variables for options that don't have a place in the config file or CLI
const (
// Log config
K3dEnvLogNodeWaitLogs = "K3D_LOG_NODE_WAIT_LOGS"
// Images
K3dEnvImageLoadbalancer = "K3D_IMAGE_LOADBALANCER"
K3dEnvImageTools = "K3D_IMAGE_TOOLS"
K3dEnvImageHelperTag = "K3D_HELPER_IMAGE_TAG"
// Debug options
K3dEnvDebugCorednsRetries = "K3D_DEBUG_COREDNS_RETRIES"
K3dEnvDebugDisableDockerInit = "K3D_DEBUG_DISABLE_DOCKER_INIT"
K3dEnvDebugNodeWaitBackOffLimit = "K3D_DEBUG_NODE_WAIT_BACKOFF_LIMIT"
// Fixes
K3dEnvFixCgroupV2 = "K3D_FIX_CGROUPV2"
K3dEnvFixDNS = "K3D_FIX_DNS"
)

View File

@ -25,8 +25,6 @@ import (
_ "embed"
"os"
"strconv"
k3d "github.com/rancher/k3d/v5/pkg/types"
)
/* NOTE
@ -42,8 +40,8 @@ import (
type K3DFixEnv string
const (
EnvFixCgroupV2 K3DFixEnv = k3d.K3dEnvFixCgroupV2 // EnvFixCgroupV2 is the environment variable that k3d will check for to enable/disable the cgroupv2 workaround
EnvFixDNS K3DFixEnv = k3d.K3dEnvFixDNS // EnvFixDNS is the environment variable that check for to enable/disable the application of network magic related to DNS
EnvFixCgroupV2 K3DFixEnv = "K3D_FIX_CGROUPV2" // EnvFixCgroupV2 is the environment variable that k3d will check for to enable/disable the cgroupv2 workaround
EnvFixDNS K3DFixEnv = "K3D_FIX_DNS" // EnvFixDNS is the environment variable that check for to enable/disable the application of network magic related to DNS
)
var FixEnvs []K3DFixEnv = []K3DFixEnv{

View File

@ -24,7 +24,6 @@ package types
import (
"fmt"
"os"
"strings"
l "github.com/rancher/k3d/v5/pkg/logger"
"github.com/rancher/k3d/v5/version"
@ -46,34 +45,19 @@ const DefaultRegistryImageRepo = "docker.io/library/registry"
const DefaultRegistryImageTag = "2"
func GetLoadbalancerImage() string {
if img := os.Getenv(K3dEnvImageLoadbalancer); img != "" {
l.Log().Infof("Loadbalancer image set from env var $%s: %s", K3dEnvImageLoadbalancer, img)
if img := os.Getenv("K3D_IMAGE_LOADBALANCER"); img != "" {
l.Log().Infof("Loadbalancer image set from env var $K3D_IMAGE_LOADBALANCER: %s", img)
return img
}
return fmt.Sprintf("%s:%s", DefaultLBImageRepo, GetHelperImageVersion())
return fmt.Sprintf("%s:%s", DefaultLBImageRepo, version.GetHelperImageVersion())
}
func GetToolsImage() string {
if img := os.Getenv(K3dEnvImageTools); img != "" {
l.Log().Infof("Tools image set from env var $%s: %s", K3dEnvImageTools, img)
if img := os.Getenv("K3D_IMAGE_TOOLS"); img != "" {
l.Log().Infof("Tools image set from env var $K3D_IMAGE_TOOLS: %s", img)
return img
}
return fmt.Sprintf("%s:%s", DefaultToolsImageRepo, GetHelperImageVersion())
}
// GetHelperImageVersion returns the CLI version or 'latest'
func GetHelperImageVersion() string {
if tag := os.Getenv(K3dEnvImageHelperTag); tag != "" {
l.Log().Infoln("Helper image tag set from env var")
return tag
}
if len(version.HelperVersionOverride) > 0 {
return version.HelperVersionOverride
}
if len(version.Version) == 0 {
return "latest"
}
return strings.TrimPrefix(version.Version, "v")
return fmt.Sprintf("%s:%s", DefaultToolsImageRepo, version.GetHelperImageVersion())
}

View File

@ -1,32 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package types
type Intent string
const (
IntentClusterCreate Intent = "cluster-create"
IntentClusterStart Intent = "cluster-start"
IntentNodeCreate Intent = "node-create"
IntentNodeStart Intent = "node-start"
IntentAny Intent = ""
)

View File

@ -1,29 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package k3s
// k3s environment variables
const (
EnvClusterToken string = "K3S_TOKEN"
EnvClusterConnectURL string = "K3S_URL"
EnvKubeconfigOutput string = "K3S_KUBECONFIG_OUTPUT"
)

View File

@ -1,69 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package types
import (
"time"
l "github.com/rancher/k3d/v5/pkg/logger"
)
// NodeWaitForLogMessageRestartWarnTime is the time after which to warn about a restarting container
const NodeWaitForLogMessageRestartWarnTime = 2 * time.Minute
var ReadyLogMessagesByRoleAndIntent = map[Role]map[Intent]string{
Role(InternalRoleInitServer): {
IntentClusterCreate: "Containerd is now running",
IntentClusterStart: "Running kube-apiserver",
IntentAny: "Running kube-apiserver",
},
ServerRole: {
IntentAny: "k3s is up and running",
},
AgentRole: {
IntentAny: "Successfully registered node",
},
LoadBalancerRole: {
IntentAny: "start worker processes",
},
RegistryRole: {
IntentAny: "listening on",
},
}
func GetReadyLogMessage(node *Node, intent Intent) string {
role := node.Role
if node.Role == ServerRole && node.ServerOpts.IsInit {
role = Role(InternalRoleInitServer)
}
if _, ok := ReadyLogMessagesByRoleAndIntent[role]; ok {
if msg, ok := ReadyLogMessagesByRoleAndIntent[role][intent]; ok {
return msg
} else {
if msg, ok := ReadyLogMessagesByRoleAndIntent[role][IntentAny]; ok {
return msg
}
}
}
l.Log().Warnf("error looking up ready log message for role %s and intent %s: not defined", role, intent)
return ""
}

View File

@ -1,59 +0,0 @@
/*
Copyright © 2020-2021 The k3d Author(s)
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/
package types
// Registry Defaults
const (
DefaultRegistryPort = "5000"
DefaultRegistryName = DefaultObjectNamePrefix + "-registry"
DefaultRegistriesFilePath = "/etc/rancher/k3s/registries.yaml"
DefaultRegistryMountPath = "/var/lib/registry"
DefaultDockerHubAddress = "registry-1.docker.io"
// Default temporary path for the LocalRegistryHosting configmap, from where it will be applied via kubectl
DefaultLocalRegistryHostingConfigmapTempPath = "/tmp/localRegistryHostingCM.yaml"
)
// Registry describes a k3d-managed registry
type Registry struct {
ClusterRef string // filled automatically -> if created with a cluster
Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
Host string `yaml:"host" json:"host"`
Image string `yaml:"image,omitempty" json:"image,omitempty"`
ExposureOpts ExposureOpts `yaml:"expose" json:"expose"`
Options struct {
ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"`
Proxy struct {
RemoteURL string `yaml:"remoteURL" json:"remoteURL"`
Username string `yaml:"username,omitempty" json:"username,omitempty"`
Password string `yaml:"password,omitempty" json:"password,omitempty"`
} `yaml:"proxy,omitempty" json:"proxy,omitempty"`
} `yaml:"options,omitempty" json:"options,omitempty"`
}
// RegistryExternal describes a minimal spec for an "external" registry
// "external" meaning, that it's unrelated to the current cluster
// e.g. used for the --registry-use flag registry reference
type RegistryExternal struct {
Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
Host string `yaml:"host" json:"host"`
Port string `yaml:"port" json:"port"`
}

View File

@ -23,15 +23,40 @@ package types
import (
"context"
"fmt"
"net"
"time"
"github.com/docker/go-connections/nat"
runtimeTypes "github.com/rancher/k3d/v5/pkg/runtimes/types"
"github.com/rancher/k3d/v5/pkg/types/k3s"
"github.com/rancher/k3d/v5/version"
"inet.af/netaddr"
)
// DefaultClusterName specifies the default name used for newly created clusters
const DefaultClusterName = "k3s-default"
// DefaultClusterNameMaxLength specifies the maximal length of a passed in cluster name
// This restriction allows us to construct an name consisting of
// <DefaultObjectNamePrefix[3]>-<ClusterName>-<TypeSuffix[5-10]>-<Counter[1-3]>
// ... and still stay within the 64 character limit (e.g. of docker)
const DefaultClusterNameMaxLength = 32
// DefaultObjectNamePrefix defines the name prefix for every object created by k3d
const DefaultObjectNamePrefix = "k3d"
// ReadyLogMessageByRole defines the log messages we wait for until a server node is considered ready
var ReadyLogMessageByRole = map[Role]string{
ServerRole: "k3s is up and running",
AgentRole: "Successfully registered node",
LoadBalancerRole: "start worker processes",
RegistryRole: "listening on",
}
// NodeWaitForLogMessageRestartWarnTime is the time after which to warn about a restarting container
const NodeWaitForLogMessageRestartWarnTime = 2 * time.Minute
// NodeStatusRestarting defines the status string that signals the node container is restarting
const NodeStatusRestarting = "restarting"
@ -47,12 +72,6 @@ const (
RegistryRole Role = "registry"
)
type InternalRole Role
const (
InternalRoleInitServer InternalRole = "initServer"
)
// NodeRoles defines the roles available for nodes
var NodeRoles = map[string]Role{
string(ServerRole): ServerRole,
@ -73,6 +92,16 @@ var ClusterExternalNodeRoles = []Role{
RegistryRole,
}
// DefaultRuntimeLabels specifies a set of labels that will be attached to k3d runtime objects by default
var DefaultRuntimeLabels = map[string]string{
"app": "k3d",
}
// DefaultRuntimeLabelsVar specifies a set of labels that will be attached to k3d runtime objects by default but are not static (e.g. across k3d versions)
var DefaultRuntimeLabelsVar = map[string]string{
"k3d.version": version.GetVersion(),
}
// List of k3d technical label name
const (
LabelClusterName string = "k3d.cluster"
@ -96,6 +125,48 @@ const (
LabelNodeStaticIP string = "k3d.node.staticIP"
)
// DefaultRoleCmds maps the node roles to their respective default commands
var DefaultRoleCmds = map[Role][]string{
ServerRole: {"server"},
AgentRole: {"agent"},
}
// DefaultTmpfsMounts specifies tmpfs mounts that are required for all k3d nodes
var DefaultTmpfsMounts = []string{
"/run",
"/var/run",
}
// DefaultNodeEnv defines some default environment variables that should be set on every node
var DefaultNodeEnv = []string{
fmt.Sprintf("%s=/output/kubeconfig.yaml", K3sEnvKubeconfigOutput),
}
// k3s environment variables
const (
K3sEnvClusterToken string = "K3S_TOKEN"
K3sEnvClusterConnectURL string = "K3S_URL"
K3sEnvKubeconfigOutput string = "K3S_KUBECONFIG_OUTPUT"
)
// DefaultK3dInternalHostRecord defines the default /etc/hosts entry for the k3d host
const DefaultK3dInternalHostRecord = "host.k3d.internal"
// DefaultImageVolumeMountPath defines the mount path inside k3d nodes where we will mount the shared image volume by default
const DefaultImageVolumeMountPath = "/k3d/images"
// DefaultConfigDirName defines the name of the config directory (where we'll e.g. put the kubeconfigs)
const DefaultConfigDirName = ".k3d" // should end up in $HOME/
// DefaultKubeconfigPrefix defines the default prefix for kubeconfig files
const DefaultKubeconfigPrefix = DefaultObjectNamePrefix + "-kubeconfig"
// DefaultAPIPort defines the default Kubernetes API Port
const DefaultAPIPort = "6443"
// DefaultAPIHost defines the default host (IP) for the Kubernetes API
const DefaultAPIHost = "0.0.0.0"
// DoNotCopyServerFlags defines a list of commands/args that shouldn't be copied from an existing node when adding a similar node to a cluster
var DoNotCopyServerFlags = []string{
"--cluster-init",
@ -141,7 +212,6 @@ type ClusterStartOpts struct {
Timeout time.Duration
NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"`
EnvironmentInfo *EnvironmentInfo
Intent Intent
}
// ClusterDeleteOpts describe a set of options one can set when deleting a cluster
@ -165,7 +235,6 @@ type NodeStartOpts struct {
NodeHooks []NodeHook `yaml:"nodeHooks,omitempty" json:"nodeHooks,omitempty"`
ReadyLogMessage string
EnvironmentInfo *EnvironmentInfo
Intent Intent
}
// NodeDeleteOpts describes a set of options one can set when deleting a node
@ -301,6 +370,11 @@ type ExternalDatastore struct {
// AgentOpts describes some additional agent role specific opts
type AgentOpts struct{}
// GetDefaultObjectName prefixes the passed name with the default prefix
func GetDefaultObjectName(name string) string {
return fmt.Sprintf("%s-%s", DefaultObjectNamePrefix, name)
}
// NodeState describes the current state of a node
type NodeState struct {
Running bool
@ -308,6 +382,47 @@ type NodeState struct {
Started string
}
/*
* Registry
*/
// Registry Defaults
const (
DefaultRegistryPort = "5000"
DefaultRegistryName = DefaultObjectNamePrefix + "-registry"
DefaultRegistriesFilePath = "/etc/rancher/k3s/registries.yaml"
DefaultRegistryMountPath = "/var/lib/registry"
DefaultDockerHubAddress = "registry-1.docker.io"
// Default temporary path for the LocalRegistryHosting configmap, from where it will be applied via kubectl
DefaultLocalRegistryHostingConfigmapTempPath = "/tmp/localRegistryHostingCM.yaml"
)
// Registry describes a k3d-managed registry
type Registry struct {
ClusterRef string // filled automatically -> if created with a cluster
Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
Host string `yaml:"host" json:"host"`
Image string `yaml:"image,omitempty" json:"image,omitempty"`
ExposureOpts ExposureOpts `yaml:"expose" json:"expose"`
Options struct {
ConfigFile string `yaml:"configFile,omitempty" json:"configFile,omitempty"`
Proxy struct {
RemoteURL string `yaml:"remoteURL" json:"remoteURL"`
Username string `yaml:"username,omitempty" json:"username,omitempty"`
Password string `yaml:"password,omitempty" json:"password,omitempty"`
} `yaml:"proxy,omitempty" json:"proxy,omitempty"`
} `yaml:"options,omitempty" json:"options,omitempty"`
}
// RegistryExternal describes a minimal spec for an "external" registry
// "external" meaning, that it's unrelated to the current cluster
// e.g. used for the --registry-use flag registry reference
type RegistryExternal struct {
Protocol string `yaml:"protocol,omitempty" json:"protocol,omitempty"` // default: http
Host string `yaml:"host" json:"host"`
Port string `yaml:"port" json:"port"`
}
type EnvironmentInfo struct {
HostGateway net.IP
RuntimeInfo runtimeTypes.RuntimeInfo

View File

@ -22,13 +22,7 @@ THE SOFTWARE.
package util
import (
"bytes"
"io"
"strings"
"gopkg.in/yaml.v2"
)
import "strings"
func RemoveElementFromStringSlice(slice []string, index int) []string {
slice[index] = slice[len(slice)-1]
@ -41,26 +35,3 @@ func ReplaceInAllElements(replacer *strings.Replacer, arr []string) []string {
}
return arr
}
func SplitYAML(resources []byte) ([][]byte, error) {
dec := yaml.NewDecoder(bytes.NewReader(resources))
var res [][]byte
for {
var value interface{}
err := dec.Decode(&value)
if err == io.EOF {
break
}
if err != nil {
return nil, err
}
valueBytes, err := yaml.Marshal(value)
if err != nil {
return nil, err
}
res = append(res, valueBytes)
}
return res, nil
}

View File

@ -1,4 +0,0 @@
apiVersion: k3d.io/v1alpha3
kind: Simple
name: testenvexpand
servers: ${K3D_TEST_SERVERS}

View File

@ -31,6 +31,7 @@ registries:
"my.company.registry":
endpoint:
- http://my.company.registry:5000
options:
k3d:
wait: true

View File

@ -27,6 +27,7 @@ registries:
name: k3d-test-registry
host: "0.0.0.0"
hostPort: random
use: []
config: |
mirrors:
"my.company.registry":

View File

@ -181,4 +181,4 @@ k3s_assert_node_label() {
# $1 = node name
# $2 = label to assert
kubectl get node "$1" --output go-template='{{ range $k, $v := .metadata.labels }}{{ printf "%s=%s\n" $k $v }}{{ end }}' | grep -qE "^$2$"
}
}

View File

@ -1,51 +0,0 @@
#!/bin/bash
CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
[ -d "$CURR_DIR" ] || { echo "FATAL: no current dir (maybe running in zsh?)"; exit 1; }
# shellcheck source=./common.sh
source "$CURR_DIR/common.sh"
export CURRENT_STAGE="local | remote_docker"
info "Starting dind with TLS (sleeping for 10s to give it time to get ready)"
docker run -d -p 3376:2376 -e DOCKER_TLS_CERTDIR=/certs -v /tmp/dockercerts:/certs --privileged --rm --name k3dlocaltestdindsec docker:20.10-dind
sleep 10
info "Setting Docker Context"
docker context create k3dlocaltestdindsec --description "dind local secure" --docker "host=tcp://127.0.0.1:3376,ca=/tmp/dockercerts/client/ca.pem,cert=/tmp/dockercerts/client/cert.pem,key=/tmp/dockercerts/client/key.pem"
docker context use k3dlocaltestdindsec
docker context list
info "Running k3d"
k3d cluster create test1
k3d cluster list
info "Switching to default context"
docker context list
docker ps
docker context use default
docker ps
info "Checking DOCKER_TLS env var based setting"
export DOCKER_HOST=tcp://127.0.0.1:3376
export DOCKER_TLS_VERIFY=1
export DOCKER_CERT_PATH=/tmp/dockercerts/client
docker context list
docker ps
k3d cluster create test2
k3d cluster list
docker ps
info "Cleaning up"
unset DOCKER_HOST
unset DOCKER_TLS_VERIFY
unset DOCKER_CERT_PATH
k3d cluster rm -a
docker context use default
docker rm -f k3dlocaltestdindsec
docker context rm k3dlocaltestdindsec
info ">>> DONE <<<"

View File

@ -49,8 +49,6 @@ $EXE cluster start "$clustername" --wait --timeout 360s || failed "cluster didn'
info "Checking that we have access to the cluster..."
check_clusters "$clustername" || failed "error checking cluster"
kubectl delete pod -n kube-system -l k8s-app=kube-dns > /dev/null 2>&1 # delete coredns to force reload of config (reload plugin uses default 30s, which will make tests below fail)
info "Checking that we have 2 nodes online..."
check_multi_node "$clustername" 2 || failed "failed to verify number of nodes"
@ -79,8 +77,7 @@ wait_for_pod_running_by_label "k8s-app=kube-dns" "kube-system"
sleep 5
# 6. test host.k3d.internal
info "Checking DNS Lookup for host.k3d.internal..."
kubectl describe cm coredns -n kube-system | grep "host.k3d.internal" > /dev/null 2>&1 || failed "Couldn't find host.k3d.internal in CoreDNS configmap"
info "Checking DNS Lookup for host.k3d.internal"
wait_for_pod_exec "testimage" "nslookup host.k3d.internal" 15 || failed "DNS Lookup for host.k3d.internal failed"
# Cleanup

View File

@ -59,7 +59,7 @@ var (
// ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation
ErrVmcomputeOperationInvalidState = hcs.ErrVmcomputeOperationInvalidState
// ErrProcNotFound is an error encountered when the the process cannot be found
// ErrProcNotFound is an error encountered when a procedure look up fails.
ErrProcNotFound = hcs.ErrProcNotFound
// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
@ -159,7 +159,7 @@ func (e *ProcessError) Error() string {
// IsNotExist checks if an error is caused by the Container or Process not existing.
// Note: Currently, ErrElementNotFound can mean that a Process has either
// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
// will currently return true when the error is ErrElementNotFound.
func IsNotExist(err error) bool {
if _, ok := err.(EndpointNotFoundError); ok {
return true
@ -192,7 +192,7 @@ func IsTimeout(err error) bool {
// a Container or Process being already stopped.
// Note: Currently, ErrElementNotFound can mean that a Process has either
// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
// will currently return true when the error is ErrElementNotFound.
func IsAlreadyStopped(err error) bool {
return hcs.IsAlreadyStopped(getInnerError(err))
}

View File

@ -7,6 +7,9 @@ import (
// HNSEndpoint represents a network endpoint in HNS
type HNSEndpoint = hns.HNSEndpoint
// HNSEndpointStats represent the stats for an networkendpoint in HNS
type HNSEndpointStats = hns.EndpointStats
// Namespace represents a Compartment.
type Namespace = hns.Namespace
@ -108,3 +111,8 @@ func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) {
func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) {
return hns.GetHNSEndpointByName(endpointName)
}
// GetHNSEndpointStats gets the endpoint stats by ID
func GetHNSEndpointStats(endpointName string) (*HNSEndpointStats, error) {
return hns.GetHNSEndpointStats(endpointName)
}

View File

@ -60,7 +60,7 @@ var (
// ErrVmcomputeOperationInvalidState is an error encountered when the compute system is not in a valid state for the requested operation
ErrVmcomputeOperationInvalidState = syscall.Errno(0xc0370105)
// ErrProcNotFound is an error encountered when the the process cannot be found
// ErrProcNotFound is an error encountered when a procedure look up fails.
ErrProcNotFound = syscall.Errno(0x7f)
// ErrVmcomputeOperationAccessIsDenied is an error which can be encountered when enumerating compute systems in RS1/RS2
@ -242,12 +242,11 @@ func makeProcessError(process *Process, op string, err error, events []ErrorEven
// IsNotExist checks if an error is caused by the Container or Process not existing.
// Note: Currently, ErrElementNotFound can mean that a Process has either
// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
// will currently return true when the error is ErrElementNotFound.
func IsNotExist(err error) bool {
err = getInnerError(err)
return err == ErrComputeSystemDoesNotExist ||
err == ErrElementNotFound ||
err == ErrProcNotFound
err == ErrElementNotFound
}
// IsAlreadyClosed checks if an error is caused by the Container or Process having been
@ -278,12 +277,11 @@ func IsTimeout(err error) bool {
// a Container or Process being already stopped.
// Note: Currently, ErrElementNotFound can mean that a Process has either
// already exited, or does not exist. Both IsAlreadyStopped and IsNotExist
// will currently return true when the error is ErrElementNotFound or ErrProcNotFound.
// will currently return true when the error is ErrElementNotFound.
func IsAlreadyStopped(err error) bool {
err = getInnerError(err)
return err == ErrVmcomputeAlreadyStopped ||
err == ErrElementNotFound ||
err == ErrProcNotFound
err == ErrElementNotFound
}
// IsNotSupported returns a boolean indicating whether the error is caused by

View File

@ -30,6 +30,7 @@ type HNSEndpoint struct {
EnableLowMetric bool `json:",omitempty"`
Namespace *Namespace `json:",omitempty"`
EncapOverhead uint16 `json:",omitempty"`
SharedContainers []string `json:",omitempty"`
}
//SystemType represents the type of the system on which actions are done
@ -57,6 +58,18 @@ type EndpointResquestResponse struct {
Error string
}
// EndpointStats is the object that has stats for a given endpoint
type EndpointStats struct {
BytesReceived uint64 `json:"BytesReceived"`
BytesSent uint64 `json:"BytesSent"`
DroppedPacketsIncoming uint64 `json:"DroppedPacketsIncoming"`
DroppedPacketsOutgoing uint64 `json:"DroppedPacketsOutgoing"`
EndpointID string `json:"EndpointId"`
InstanceID string `json:"InstanceId"`
PacketsReceived uint64 `json:"PacketsReceived"`
PacketsSent uint64 `json:"PacketsSent"`
}
// HNSEndpointRequest makes a HNS call to modify/query a network endpoint
func HNSEndpointRequest(method, path, request string) (*HNSEndpoint, error) {
endpoint := &HNSEndpoint{}
@ -79,11 +92,27 @@ func HNSListEndpointRequest() ([]HNSEndpoint, error) {
return endpoint, nil
}
// hnsEndpointStatsRequest makes a HNS call to query the stats for a given endpoint ID
func hnsEndpointStatsRequest(id string) (*EndpointStats, error) {
var stats EndpointStats
err := hnsCall("GET", "/endpointstats/"+id, "", &stats)
if err != nil {
return nil, err
}
return &stats, nil
}
// GetHNSEndpointByID get the Endpoint by ID
func GetHNSEndpointByID(endpointID string) (*HNSEndpoint, error) {
return HNSEndpointRequest("GET", endpointID, "")
}
// GetHNSEndpointStats get the stats for a n Endpoint by ID
func GetHNSEndpointStats(endpointID string) (*EndpointStats, error) {
return hnsEndpointStatsRequest(endpointID)
}
// GetHNSEndpointByName gets the endpoint filtered by Name
func GetHNSEndpointByName(endpointName string) (*HNSEndpoint, error) {
hnsResponse, err := HNSListEndpointRequest()

6
vendor/modules.txt vendored
View File

@ -8,7 +8,7 @@ github.com/Microsoft/go-winio
github.com/Microsoft/go-winio/pkg/guid
github.com/Microsoft/go-winio/pkg/security
github.com/Microsoft/go-winio/vhd
# github.com/Microsoft/hcsshim v0.8.18
# github.com/Microsoft/hcsshim v0.8.21
## explicit; go 1.13
github.com/Microsoft/hcsshim
github.com/Microsoft/hcsshim/computestorage
@ -39,7 +39,7 @@ github.com/cespare/xxhash/v2
# github.com/containerd/cgroups v1.0.1
## explicit; go 1.13
github.com/containerd/cgroups/stats/v1
# github.com/containerd/containerd v1.5.5
# github.com/containerd/containerd v1.5.7
## explicit; go 1.16
github.com/containerd/containerd/errdefs
github.com/containerd/containerd/log
@ -258,7 +258,7 @@ github.com/opencontainers/go-digest
## explicit
github.com/opencontainers/image-spec/specs-go
github.com/opencontainers/image-spec/specs-go/v1
# github.com/opencontainers/runc v1.0.1
# github.com/opencontainers/runc v1.0.2
## explicit; go 1.13
github.com/opencontainers/runc/libcontainer/user
# github.com/pelletier/go-toml v1.9.3

View File

@ -23,6 +23,8 @@ package version
import (
"fmt"
"os"
"strings"
"github.com/heroku/docker-registry-client/registry"
l "github.com/rancher/k3d/v5/pkg/logger"
@ -45,6 +47,21 @@ func GetVersion() string {
return Version
}
// GetHelperImageVersion returns the CLI version or 'latest'
func GetHelperImageVersion() string {
if tag := os.Getenv("K3D_HELPER_IMAGE_TAG"); tag != "" {
l.Log().Infoln("Helper image tag set from env var")
return tag
}
if len(HelperVersionOverride) > 0 {
return HelperVersionOverride
}
if len(Version) == 0 {
return "latest"
}
return strings.TrimPrefix(Version, "v")
}
// GetK3sVersion returns the version string for K3s
func GetK3sVersion(latest bool) string {
if latest {