diff --git a/CHANGELOG.md b/CHANGELOG.md index d8317d0d..38067c81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ - Option 2: customized, managed stand-alone - `k3d registry [create/start/stop/delete]` - Check the documentation, help text and tutorials for more details + - Communicate managed registry using the LocalRegistryHostingV1 spec from [KEP-1755](https://github.com/kubernetes/enhancements/blob/0d69f7cea6fbe73a7d70fab569c6898f5ccb7be0/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry/README.md) + - interesting especially for tools that reload images, like Tilt or Skaffold - Config File Support - Put all your CLI-Arguments/Flags into a more readable config file and re-use it everywhere (keep it in your repo) diff --git a/pkg/client/cluster.go b/pkg/client/cluster.go index 74426925..c5e5d999 100644 --- a/pkg/client/cluster.go +++ b/pkg/client/cluster.go @@ -96,6 +96,11 @@ func ClusterRun(ctx context.Context, runtime k3drt.Runtime, clusterConfig *confi prepInjectHostIP(ctx, runtime, &clusterConfig.Cluster) } + // create the registry hosting configmap + if err := prepCreateLocalRegistryHostingConfigMap(ctx, runtime, &clusterConfig.Cluster); err != nil { + log.Warnf("Failed to create LocalRegistryHosting ConfigMap: %+v", err) + } + return nil } @@ -196,6 +201,21 @@ func ClusterPrep(ctx context.Context, runtime k3drt.Runtime, clusterConfig *conf }, }) + // generate the LocalRegistryHosting configmap + regCm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx, clusterConfig.ClusterCreateOpts.Registries.Use) + if err != nil { + return fmt.Errorf("Failed to generate LocalRegistryHosting configmap: %+v", err) + } + log.Errorf("Writing YAML: %s", string(regCm)) + clusterConfig.ClusterCreateOpts.NodeHooks = append(clusterConfig.ClusterCreateOpts.NodeHooks, k3d.NodeHook{ + Stage: k3d.LifecycleStagePreStart, + Action: actions.WriteFileAction{ + Runtime: runtime, + Content: regCm, + Dest: "/tmp/reg.yaml", + }, + }) + } return nil @@ -890,3 +910,14 @@ func prepInjectHostIP(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.C } } + +func prepCreateLocalRegistryHostingConfigMap(ctx context.Context, runtime k3drt.Runtime, cluster *k3d.Cluster) error { + for _, node := range cluster.Nodes { + if node.Role == k3d.AgentRole || node.Role == k3d.ServerRole { + if err := runtime.ExecInNode(ctx, node, []string{"sh", "-c", "kubectl apply -f /tmp/reg.yaml"}); err != nil { + log.Warnf("Failed to create cm in node '%s'", node.Name) + } + } + } + return nil +} diff --git a/pkg/client/registry.go b/pkg/client/registry.go index 8726c810..e4674184 100644 --- a/pkg/client/registry.go +++ b/pkg/client/registry.go @@ -29,7 +29,9 @@ import ( "github.com/rancher/k3d/v4/pkg/runtimes" k3d "github.com/rancher/k3d/v4/pkg/types" "github.com/rancher/k3d/v4/pkg/types/k3s" + "github.com/rancher/k3d/v4/pkg/types/k8s" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" ) func RegistryRun(ctx context.Context, runtime runtimes.Runtime, reg *k3d.Registry) (*k3d.Node, error) { @@ -236,3 +238,69 @@ func RegistryFromNode(node *k3d.Node) (*k3d.Registry, error) { return registry, nil } + +// RegistryGenerateLocalRegistryHostingConfigMapYAML generates a ConfigMap used to advertise the registries in the cluster +func RegistryGenerateLocalRegistryHostingConfigMapYAML(ctx context.Context, registries []*k3d.Registry) ([]byte, error) { + + type cmMetadata struct { + Name string `yaml:"name"` + Namespace string `yaml:"namespace"` + } + + type cmData struct { + RegHostV1 string `yaml:"localRegistryHosting.v1"` + } + + type configmap struct { + APIVersion string `yaml:"apiVersion"` + Kind string `yaml:"kind"` + Metadata cmMetadata `yaml:"metadata"` + Data cmData `yaml:"data"` + } + + if len(registries) > 1 { + log.Warnf("More than one registry specified, but the LocalRegistryHostingV1 spec only supports one -> Selecting the first one: %s", registries[0].Host) + } + + if len(registries) < 1 { + log.Debugln("No registry specified, not generating local registry hosting configmap") + return nil, nil + } + + host := registries[0].ExposureOpts.Host + if host == "" { + host = registries[0].ExposureOpts.Binding.HostIP + } + + dat, err := yaml.Marshal( + k8s.LocalRegistryHostingV1{ + Host: fmt.Sprintf("%s:%s", host, registries[0].ExposureOpts.Binding.HostPort), + HostFromContainerRuntime: fmt.Sprintf("%s:%s", registries[0].Host, registries[0].ExposureOpts.Port.Port()), + Help: "https://k3d.io/usage/guides/registries/#using-a-local-registry", + }, + ) + if err != nil { + return nil, err + } + + cm := configmap{ + APIVersion: "v1", + Kind: "ConfigMap", + Metadata: cmMetadata{ + Name: "local-registry-hosting", + Namespace: "kube-public", + }, + Data: cmData{ + RegHostV1: string(dat), + }, + } + + cmYaml, err := yaml.Marshal(cm) + if err != nil { + return nil, err + } + + log.Tracef("LocalRegistryHostingConfigMapYaml: %s", string(cmYaml)) + + return cmYaml, nil +} diff --git a/pkg/client/registry_test.go b/pkg/client/registry_test.go new file mode 100644 index 00000000..59012719 --- /dev/null +++ b/pkg/client/registry_test.go @@ -0,0 +1,66 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ +package client + +import ( + "context" + "strings" + "testing" + + "github.com/docker/go-connections/nat" + k3d "github.com/rancher/k3d/v4/pkg/types" +) + +func TestRegistryGenerateLocalRegistryHostingConfigMapYAML(t *testing.T) { + var err error + + expectedYAMLString := `apiVersion: v1 +kind: ConfigMap +metadata: + name: local-registry-hosting + namespace: kube-public +data: + localRegistryHosting.v1: | + host: test-host:5432 + hostFromContainerRuntime: test-host:1234 + help: https://k3d.io/usage/guides/registries/#using-a-local-registry +` + + reg := &k3d.Registry{ + Host: "test-host", + } + reg.ExposureOpts.Host = "test-host" + reg.ExposureOpts.Port = nat.Port("1234/tcp") + reg.ExposureOpts.Binding.HostPort = "5432" + + regs := []*k3d.Registry{reg} + + cm, err := RegistryGenerateLocalRegistryHostingConfigMapYAML(context.Background(), regs) + if err != nil { + t.Error(err) + } + + if !(strings.TrimSpace(string(cm)) == strings.TrimSpace(expectedYAMLString)) { + t.Errorf("Computed configmap\n-> Actual: %s\n does not match expected YAML\n-> Expected: %s", strings.TrimSpace(string(cm)), strings.TrimSpace(expectedYAMLString)) + } + +} diff --git a/pkg/client/test.json b/pkg/client/test.json new file mode 100644 index 00000000..e7a0ced3 --- /dev/null +++ b/pkg/client/test.json @@ -0,0 +1 @@ +{"host":"test-host:5432","hostFromContainerRuntime":"test-host:1234","help":"https://k3d.io/usage/guides/registries/#using-a-local-registry"} \ No newline at end of file diff --git a/pkg/types/k8s/registry.go b/pkg/types/k8s/registry.go new file mode 100644 index 00000000..3229acc6 --- /dev/null +++ b/pkg/types/k8s/registry.go @@ -0,0 +1,94 @@ +/* +Copyright © 2020 The k3d Author(s) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +*/ + +package k8s + +/* + * Source: https://github.com/kubernetes/enhancements/blob/0d69f7cea6fbe73a7d70fab569c6898f5ccb7be0/keps/sig-cluster-lifecycle/generic/1755-communicating-a-local-registry/README.md#specification-for-localregistryhosting-v1 + * Copied over: 07.01.2020 + * Original License + * > Copyright 2020 The Kubernetes Authors + * > + * > Licensed under the Apache License, Version 2.0 (the "License"); + * > you may not use this file except in compliance with the License. + * > You may obtain a copy of the License at + * > + * > http://www.apache.org/licenses/LICENSE-2.0 + * > + * > Unless required by applicable law or agreed to in writing, software + * > distributed under the License is distributed on an "AS IS" BASIS, + * > WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * > See the License for the specific language governing permissions and + * > limitations under the License. + */ + +// LocalRegistryHostingV1 describes a local registry that developer tools can +// connect to. A local registry allows clients to load images into the local +// cluster by pushing to this registry. +type LocalRegistryHostingV1 struct { + // Host documents the host (hostname and port) of the registry, as seen from + // outside the cluster. + // + // This is the registry host that tools outside the cluster should push images + // to. + Host string `yaml:"host,omitempty" json:"host,omitempty"` + + // HostFromClusterNetwork documents the host (hostname and port) of the + // registry, as seen from networking inside the container pods. + // + // This is the registry host that tools running on pods inside the cluster + // should push images to. If not set, then tools inside the cluster should + // assume the local registry is not available to them. + HostFromClusterNetwork string `yaml:"hostFromClusterNetwork,omitempty" json:"hostFromClusterNetwork,omitempty"` + + // HostFromContainerRuntime documents the host (hostname and port) of the + // registry, as seen from the cluster's container runtime. + // + // When tools apply Kubernetes objects to the cluster, this host should be + // used for image name fields. If not set, users of this field should use the + // value of Host instead. + // + // Note that it doesn't make sense semantically to define this field, but not + // define Host or HostFromClusterNetwork. That would imply a way to pull + // images without a way to push images. + HostFromContainerRuntime string `yaml:"hostFromContainerRuntime,omitempty" json:"hostFromContainerRuntime,omitempty"` + + // Help contains a URL pointing to documentation for users on how to set + // up and configure a local registry. + // + // Tools can use this to nudge users to enable the registry. When possible, + // the writer should use as permanent a URL as possible to prevent drift + // (e.g., a version control SHA). + // + // When image pushes to a registry host specified in one of the other fields + // fail, the tool should display this help URL to the user. The help URL + // should contain instructions on how to diagnose broken or misconfigured + // registries. + Help string `yaml:"help,omitempty" json:"help,omitempty"` +} + +// LocalRegistryHosting defaults +const ( + LocalRegistryHostingNamespace = "kube-public" + LocalRegistryHostingName = "local-registry-hosting" + LocalRegistryHostingData = "localRegistryHosting.v1" +) diff --git a/tests/test_registry.sh b/tests/test_registry.sh index 9344ee3f..f323d6db 100755 --- a/tests/test_registry.sh +++ b/tests/test_registry.sh @@ -33,15 +33,19 @@ check_clusters "$clustername" || failed "error checking cluster" info "Checking that we have 2 nodes online..." check_multi_node "$clustername" 2 || failed "failed to verify number of nodes" -# 4. load an image into the cluster -info "Importing an image into the cluster..." +# 2. Check that we can discover the LocalRegistryHosting Configmap that points to our registry +info "Checking that we can discover the LocalRegistryHosting Configmap..." +kubectl get configmap -n kube-public local-registry-hosting -o go-template='{{index .data "localRegistryHosting.v1"}}' | grep -q 'host' || failed "failed to discover LocalRegistryHosting Configmap" + +# 3. load an image into the registry +info "Pushing an image to the registry..." registryPort=$(docker inspect k3d-$clustername-registry | jq '.[0].NetworkSettings.Ports["5000/tcp"][0].HostPort' | sed -E 's/"//g') docker pull alpine:latest > /dev/null docker tag alpine:latest k3d-$clustername-registry:$registryPort/alpine:local > /dev/null docker push k3d-$clustername-registry:$registryPort/alpine:local || fail "Failed to push image to managed registry" -# 5. use imported image -info "Spawning a pod using the imported image..." +# 4. use imported image +info "Spawning a pod using the pushed image..." kubectl run --image k3d-$clustername-registry:$registryPort/alpine:local testimage --command -- tail -f /dev/null info "Waiting for a bit for the pod to start..." sleep 5