From b2b86a622eed51cb6d94e16e0971a4bb70d01bef Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 9 Nov 2020 23:10:19 +0300 Subject: [PATCH] fix: remove 'token creds' from maintenance service This fixes the reverse Go dependency from `pkg/machinery` to `talos` package. Add a check to `Dockerfile` to prevent `pkg/machinery/go.mod` getting out of sync, this should prevent problems in the future. Fix potential security issue in `token` authorizer to deny requests without grpc metadata. In provisioner, add support for launching nodes without the config (config is not delivered to the provisioned nodes). Breaking change in `pkg/provision`: now `NodeRequest.Type` should be set to the node type (as config can be missing now). In `talosctl cluster create` add a flag to skip providing config to the nodes so that they enter maintenance mode, while the generated configs are written down to disk (so they can be tweaked and applied easily). Signed-off-by: Andrey Smirnov --- Dockerfile | 6 ++- cmd/talosctl/cmd/mgmt/cluster/create.go | 38 ++++++++++++---- cmd/talosctl/cmd/mgmt/config.go | 31 +------------ cmd/talosctl/cmd/talos/apply-config.go | 7 +-- internal/app/maintenance/main.go | 5 --- internal/integration/provision/upgrade.go | 2 + pkg/grpc/middleware/auth/basic/token.go | 4 +- pkg/machinery/client/client.go | 19 -------- .../v1alpha1/v1alpha1_configurator_bundle.go | 45 +++++++++++++++++++ pkg/provision/providers/docker/node.go | 24 +++++----- pkg/provision/providers/firecracker/node.go | 19 +++++--- pkg/provision/providers/qemu/node.go | 11 ++--- pkg/provision/request.go | 13 ++---- website/content/docs/v0.7/Reference/cli.md | 1 + 14 files changed, 122 insertions(+), 103 deletions(-) diff --git a/Dockerfile b/Dockerfile index 55ee7c78b..b0ab6ab3e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -123,6 +123,10 @@ COPY --from=generate /pkg/machinery/api ./pkg/machinery/api COPY --from=generate /pkg/machinery/config ./pkg/machinery/config RUN go list -mod=readonly all >/dev/null RUN ! go mod tidy -v 2>&1 | grep . +WORKDIR /src/pkg/machinery +RUN go list -mod=readonly all >/dev/null +RUN ! go mod tidy -v 2>&1 | grep . +WORKDIR /src # The init target builds the init binary. @@ -579,7 +583,7 @@ RUN --mount=type=cache,target=/.cache/go-build --mount=type=cache,target=/.cache WORKDIR /src/pkg/machinery RUN --mount=type=cache,target=/.cache/go-build --mount=type=cache,target=/.cache/golangci-lint golangci-lint run --config ../../.golangci.yml WORKDIR /src -# RUN --mount=type=cache,target=/.cache/go-build importvet github.com/talos-systems/talos/... +RUN --mount=type=cache,target=/.cache/go-build importvet github.com/talos-systems/talos/... RUN find . -name '*.pb.go' | xargs rm RUN FILES="$(gofumports -l -local github.com/talos-systems/talos .)" && test -z "${FILES}" || (echo -e "Source code is not formatted with 'gofumports -w -local github.com/talos-systems/talos .':\n${FILES}"; exit 1) diff --git a/cmd/talosctl/cmd/mgmt/cluster/create.go b/cmd/talosctl/cmd/mgmt/cluster/create.go index 1175e11a0..2e2afa604 100644 --- a/cmd/talosctl/cmd/mgmt/cluster/create.go +++ b/cmd/talosctl/cmd/mgmt/cluster/create.go @@ -33,6 +33,7 @@ import ( "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1" "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/bundle" "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/generate" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/machinery/constants" "github.com/talos-systems/talos/pkg/provision" "github.com/talos-systems/talos/pkg/provision/access" @@ -78,6 +79,7 @@ var ( customCNIUrl string crashdumpOnFailure bool skipKubeconfig bool + skipInjectingConfig bool ) // createCmd represents the cluster up command. @@ -286,6 +288,18 @@ func create(ctx context.Context) (err error) { return err } + if skipInjectingConfig { + types := []machine.Type{machine.TypeControlPlane, machine.TypeJoin} + + if withInitNode { + types = append([]machine.Type{machine.TypeInit}, types...) + } + + if err = configBundle.Write(".", types...); err != nil { + return err + } + } + // Add talosconfig to provision options so we'll have it to parse there provisionOptions = append(provisionOptions, provision.WithTalosConfig(configBundle.TalosConfig())) @@ -295,6 +309,7 @@ func create(ctx context.Context) (err error) { nodeReq := provision.NodeRequest{ Name: fmt.Sprintf("%s-master-%d", clusterName, i+1), + Type: machine.TypeControlPlane, IP: ips[i], Memory: memory, NanoCPUs: nanoCPUs, @@ -305,17 +320,16 @@ func create(ctx context.Context) (err error) { nodeReq.Ports = []string{"50000:50000/tcp", "6443:6443/tcp"} } - if withInitNode { - if i == 0 { - cfg = configBundle.Init() - } else { - cfg = configBundle.ControlPlane() - } + if withInitNode && i == 0 { + cfg = configBundle.Init() + nodeReq.Type = machine.TypeInit } else { cfg = configBundle.ControlPlane() } - nodeReq.Config = cfg + if !skipInjectingConfig { + nodeReq.Config = cfg + } request.Nodes = append(request.Nodes, nodeReq) } @@ -323,14 +337,21 @@ func create(ctx context.Context) (err error) { for i := 1; i <= workers; i++ { name := fmt.Sprintf("%s-worker-%d", clusterName, i) + var cfg config.Provider + + if !skipInjectingConfig { + cfg = configBundle.Join() + } + request.Nodes = append(request.Nodes, provision.NodeRequest{ Name: name, + Type: machine.TypeJoin, IP: ips[masters+i-1], Memory: memory, NanoCPUs: nanoCPUs, Disks: disks, - Config: configBundle.Join(), + Config: cfg, }) } @@ -574,5 +595,6 @@ func init() { createCmd.Flags().StringVar(&dnsDomain, "dns-domain", "cluster.local", "the dns domain to use for cluster") createCmd.Flags().BoolVar(&crashdumpOnFailure, "crashdump", false, "print debug crashdump to stderr when cluster startup fails") createCmd.Flags().BoolVar(&skipKubeconfig, "skip-kubeconfig", false, "skip merging kubeconfig from the created cluster") + createCmd.Flags().BoolVar(&skipInjectingConfig, "skip-injecting-config", false, "skip injecting config from embedded metadata server, write config files to current directory") Cmd.AddCommand(createCmd) } diff --git a/cmd/talosctl/cmd/mgmt/config.go b/cmd/talosctl/cmd/mgmt/config.go index 0210a62f8..da66e7839 100644 --- a/cmd/talosctl/cmd/mgmt/config.go +++ b/cmd/talosctl/cmd/mgmt/config.go @@ -113,35 +113,8 @@ func genV1Alpha1Config(args []string) error { return fmt.Errorf("failed to generate config bundle: %w", err) } - for _, t := range []machine.Type{machine.TypeInit, machine.TypeControlPlane, machine.TypeJoin} { - name := strings.ToLower(t.String()) + ".yaml" - fullFilePath := filepath.Join(outputDir, name) - - var configString string - - switch t { //nolint: exhaustive - case machine.TypeInit: - configString, err = configBundle.Init().String() - if err != nil { - return err - } - case machine.TypeControlPlane: - configString, err = configBundle.ControlPlane().String() - if err != nil { - return err - } - case machine.TypeJoin: - configString, err = configBundle.Join().String() - if err != nil { - return err - } - } - - if err = ioutil.WriteFile(fullFilePath, []byte(configString), 0o644); err != nil { - return err - } - - fmt.Printf("created %s\n", fullFilePath) + if err = configBundle.Write(outputDir, machine.TypeInit, machine.TypeControlPlane, machine.TypeJoin); err != nil { + return err } // We set the default endpoint to localhost for configs generated, with expectation user will tweak later diff --git a/cmd/talosctl/cmd/talos/apply-config.go b/cmd/talosctl/cmd/talos/apply-config.go index 02fccd00e..7ef3a9056 100644 --- a/cmd/talosctl/cmd/talos/apply-config.go +++ b/cmd/talosctl/cmd/talos/apply-config.go @@ -6,6 +6,7 @@ package talos import ( "context" + "crypto/tls" "fmt" "io/ioutil" @@ -47,9 +48,9 @@ var applyConfigCmd = &cobra.Command{ return fmt.Errorf("insecure mode requires one and only one node, got %d", len(Nodes)) } - addr := Nodes[0] - - c, err := client.NewInsecureTokenClient(ctx, addr) + c, err := client.New(ctx, client.WithTLSConfig(&tls.Config{ + InsecureSkipVerify: true, + }), client.WithEndpoints(Nodes...)) if err != nil { return err } diff --git a/internal/app/maintenance/main.go b/internal/app/maintenance/main.go index 6fd7536f1..8b5b43f3a 100644 --- a/internal/app/maintenance/main.go +++ b/internal/app/maintenance/main.go @@ -21,7 +21,6 @@ import ( "github.com/talos-systems/talos/internal/app/maintenance/server" "github.com/talos-systems/talos/pkg/grpc/factory" "github.com/talos-systems/talos/pkg/grpc/gen" - "github.com/talos-systems/talos/pkg/grpc/middleware/auth/basic" "github.com/talos-systems/talos/pkg/machinery/constants" ) @@ -42,13 +41,9 @@ func Run(ctx context.Context, logger *log.Logger, r runtime.Runtime) ([]byte, er s := server.New(r, logger, cfgCh) // Start the server. - - creds := basic.NewTokenCredentials("") - server := factory.NewServer( s, factory.WithDefaultLog(), - factory.WithUnaryInterceptor(creds.UnaryInterceptor()), factory.ServerOptions( grpc.Creds( credentials.NewTLS(tlsConfig), diff --git a/internal/integration/provision/upgrade.go b/internal/integration/provision/upgrade.go index 69b972cda..9e34488bf 100644 --- a/internal/integration/provision/upgrade.go +++ b/internal/integration/provision/upgrade.go @@ -312,6 +312,7 @@ func (suite *UpgradeSuite) setupCluster() { request.Nodes = append(request.Nodes, provision.NodeRequest{ Name: fmt.Sprintf("master-%d", i+1), + Type: machine.TypeControlPlane, IP: ips[i], Memory: DefaultSettings.MemMB * 1024 * 1024, NanoCPUs: DefaultSettings.CPUs * 1000 * 1000 * 1000, @@ -328,6 +329,7 @@ func (suite *UpgradeSuite) setupCluster() { request.Nodes = append(request.Nodes, provision.NodeRequest{ Name: fmt.Sprintf("worker-%d", i), + Type: machine.TypeJoin, IP: ips[suite.spec.MasterNodes+i-1], Memory: DefaultSettings.MemMB * 1024 * 1024, NanoCPUs: DefaultSettings.CPUs * 1000 * 1000 * 1000, diff --git a/pkg/grpc/middleware/auth/basic/token.go b/pkg/grpc/middleware/auth/basic/token.go index 178a38d6e..c5887ba0d 100644 --- a/pkg/grpc/middleware/auth/basic/token.go +++ b/pkg/grpc/middleware/auth/basic/token.go @@ -48,11 +48,9 @@ func (b *TokenCredentials) authorize(ctx context.Context) error { if len(md["token"]) > 0 && md["token"][0] == b.Token { return nil } - - return fmt.Errorf("%s", codes.Unauthenticated.String()) } - return nil + return fmt.Errorf("%s", codes.Unauthenticated.String()) } // UnaryInterceptor sets the UnaryServerInterceptor for the server and enforces diff --git a/pkg/machinery/client/client.go b/pkg/machinery/client/client.go index 428e52edd..23e0abc79 100644 --- a/pkg/machinery/client/client.go +++ b/pkg/machinery/client/client.go @@ -27,7 +27,6 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/durationpb" - "github.com/talos-systems/talos/pkg/grpc/middleware/auth/basic" clusterapi "github.com/talos-systems/talos/pkg/machinery/api/cluster" "github.com/talos-systems/talos/pkg/machinery/api/common" machineapi "github.com/talos-systems/talos/pkg/machinery/api/machine" @@ -144,24 +143,6 @@ func New(ctx context.Context, opts ...OptionFunc) (c *Client, err error) { c.TimeClient = timeapi.NewTimeServiceClient(c.conn) c.NetworkClient = networkapi.NewNetworkServiceClient(c.conn) c.ClusterClient = clusterapi.NewClusterServiceClient(c.conn) - - return c, nil -} - -// NewInsecureTokenClient returns a new Client configured with an empty basic -// token credentials. -func NewInsecureTokenClient(ctx context.Context, addr string, opts ...grpc.DialOption) (c *Client, err error) { - c = new(Client) - - addr = net.FormatAddress(addr) - - creds := basic.NewTokenCredentials("") - - c.conn, err = basic.NewConnection(addr, constants.ApidPort, creds) - if err != nil { - return nil, fmt.Errorf("failed to create client connection: %w", err) - } - c.MaintenanceServiceClient = machineapi.NewMaintenanceServiceClient(c.conn) return c, nil diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_configurator_bundle.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_configurator_bundle.go index 6accdbcc0..fbef1827f 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_configurator_bundle.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_configurator_bundle.go @@ -5,8 +5,14 @@ package v1alpha1 import ( + "fmt" + "io/ioutil" + "path/filepath" + "strings" + clientconfig "github.com/talos-systems/talos/pkg/machinery/client/config" "github.com/talos-systems/talos/pkg/machinery/config" + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" ) // ConfigBundle defines the group of v1alpha1 config files. @@ -37,3 +43,42 @@ func (c *ConfigBundle) Join() config.Provider { func (c *ConfigBundle) TalosConfig() *clientconfig.Config { return c.TalosCfg } + +// Write config files to output directory. +func (c *ConfigBundle) Write(outputDir string, types ...machine.Type) error { + for _, t := range types { + name := strings.ToLower(t.String()) + ".yaml" + fullFilePath := filepath.Join(outputDir, name) + + var ( + configString string + err error + ) + + switch t { //nolint: exhaustive + case machine.TypeInit: + configString, err = c.Init().String() + if err != nil { + return err + } + case machine.TypeControlPlane: + configString, err = c.ControlPlane().String() + if err != nil { + return err + } + case machine.TypeJoin: + configString, err = c.Join().String() + if err != nil { + return err + } + } + + if err = ioutil.WriteFile(fullFilePath, []byte(configString), 0o644); err != nil { + return err + } + + fmt.Printf("created %s\n", fullFilePath) + } + + return nil +} diff --git a/pkg/provision/providers/docker/node.go b/pkg/provision/providers/docker/node.go index ef24d0dac..5936c2aa7 100644 --- a/pkg/provision/providers/docker/node.go +++ b/pkg/provision/providers/docker/node.go @@ -63,20 +63,26 @@ func (p *provisioner) createNodes(ctx context.Context, clusterReq provision.Clus //nolint: gocyclo func (p *provisioner) createNode(ctx context.Context, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest, options *provision.Options) (provision.NodeInfo, error) { - cfg, err := nodeReq.Config.String() - if err != nil { - return provision.NodeInfo{}, err + env := []string{"PLATFORM=container"} + + if nodeReq.Config != nil { + cfg, err := nodeReq.Config.String() + if err != nil { + return provision.NodeInfo{}, err + } + + env = append(env, "USERDATA="+base64.StdEncoding.EncodeToString([]byte(cfg))) } // Create the container config. containerConfig := &container.Config{ Hostname: nodeReq.Name, Image: clusterReq.Image, - Env: []string{"PLATFORM=container", "USERDATA=" + base64.StdEncoding.EncodeToString([]byte(cfg))}, + Env: env, Labels: map[string]string{ "talos.owned": "true", "talos.cluster.name": clusterReq.Name, - "talos.type": nodeReq.Config.Machine().Type().String(), + "talos.type": nodeReq.Type.String(), }, Volumes: map[string]struct{}{ "/var/lib/containerd": {}, @@ -110,16 +116,14 @@ func (p *provisioner) createNode(ctx context.Context, clusterReq provision.Clust // Mutate the container configurations based on the node type. - if nodeReq.Config.Machine().Type() == machine.TypeInit || nodeReq.Config.Machine().Type() == machine.TypeControlPlane { + if nodeReq.Type == machine.TypeInit || nodeReq.Type == machine.TypeControlPlane { portsToOpen := nodeReq.Ports if len(options.DockerPorts) > 0 { portsToOpen = append(portsToOpen, options.DockerPorts...) } - var generatedPortMap portMap - - generatedPortMap, err = genPortMap(portsToOpen, options.DockerPortsHostIP) + generatedPortMap, err := genPortMap(portsToOpen, options.DockerPortsHostIP) if err != nil { return provision.NodeInfo{}, err } @@ -160,7 +164,7 @@ func (p *provisioner) createNode(ctx context.Context, clusterReq provision.Clust nodeInfo := provision.NodeInfo{ ID: info.ID, Name: info.Name, - Type: nodeReq.Config.Machine().Type(), + Type: nodeReq.Type, NanoCPUs: nodeReq.NanoCPUs, Memory: nodeReq.Memory, diff --git a/pkg/provision/providers/firecracker/node.go b/pkg/provision/providers/firecracker/node.go index 58ed41b7f..aa4ca5dbd 100644 --- a/pkg/provision/providers/firecracker/node.go +++ b/pkg/provision/providers/firecracker/node.go @@ -91,9 +91,19 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe // Talos config cmdline.Append("talos.platform", "metal") - cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher cmdline.Append("talos.hostname", nodeReq.Name) + var nodeConfig string + + if nodeReq.Config != nil { + cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher + + nodeConfig, err = nodeReq.Config.String() + if err != nil { + return provision.NodeInfo{}, err + } + } + ones, _ := clusterReq.Network.CIDR.Mask.Size() drives := make([]models.Drive, len(diskPaths)) @@ -144,11 +154,6 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe defer logFile.Close() //nolint: errcheck - nodeConfig, err := nodeReq.Config.String() - if err != nil { - return provision.NodeInfo{}, err - } - launchConfig := LaunchConfig{ FirecrackerConfig: cfg, Config: nodeConfig, @@ -192,7 +197,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe nodeInfo := provision.NodeInfo{ ID: pidPath, Name: nodeReq.Name, - Type: nodeReq.Config.Machine().Type(), + Type: nodeReq.Type, NanoCPUs: nodeReq.NanoCPUs, Memory: nodeReq.Memory, diff --git a/pkg/provision/providers/qemu/node.go b/pkg/provision/providers/qemu/node.go index c797832d2..c12a73e8c 100644 --- a/pkg/provision/providers/qemu/node.go +++ b/pkg/provision/providers/qemu/node.go @@ -22,7 +22,6 @@ import ( multierror "github.com/hashicorp/go-multierror" "github.com/talos-systems/go-procfs/procfs" - "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/machinery/constants" "github.com/talos-systems/talos/pkg/provision" "github.com/talos-systems/talos/pkg/provision/providers/vm" @@ -74,11 +73,12 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe // Talos config cmdline.Append("talos.platform", "metal") - cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher var nodeConfig string if nodeReq.Config != nil { + cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher + nodeConfig, err = nodeReq.Config.String() if err != nil { return provision.NodeInfo{}, err @@ -160,16 +160,11 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe // no need to wait here, as cmd has all the Stdin/out/err via *os.File - nodeType := machine.TypeUnknown - if nodeReq.Config != nil { - nodeType = nodeReq.Config.Machine().Type() - } - nodeInfo := provision.NodeInfo{ ID: pidPath, UUID: nodeUUID, Name: nodeReq.Name, - Type: nodeType, + Type: nodeReq.Type, NanoCPUs: nodeReq.NanoCPUs, Memory: nodeReq.Memory, diff --git a/pkg/provision/request.go b/pkg/provision/request.go index ca558c1cd..31ea58aca 100644 --- a/pkg/provision/request.go +++ b/pkg/provision/request.go @@ -86,11 +86,7 @@ func (reqs NodeRequests) FindInitNode() (req NodeRequest, err error) { // MasterNodes returns subset of nodes which are Init/ControlPlane type. func (reqs NodeRequests) MasterNodes() (nodes []NodeRequest) { for i := range reqs { - if reqs[i].Config == nil { - continue - } - - if reqs[i].Config.Machine().Type() == machine.TypeInit || reqs[i].Config.Machine().Type() == machine.TypeControlPlane { + if reqs[i].Type == machine.TypeInit || reqs[i].Type == machine.TypeControlPlane { nodes = append(nodes, reqs[i]) } } @@ -101,11 +97,7 @@ func (reqs NodeRequests) MasterNodes() (nodes []NodeRequest) { // WorkerNodes returns subset of nodes which are Init/ControlPlane type. func (reqs NodeRequests) WorkerNodes() (nodes []NodeRequest) { for i := range reqs { - if reqs[i].Config == nil { - continue - } - - if reqs[i].Config.Machine().Type() == machine.TypeJoin { + if reqs[i].Type == machine.TypeJoin { nodes = append(nodes, reqs[i]) } } @@ -137,6 +129,7 @@ type NodeRequest struct { Name string IP net.IP Config config.Provider + Type machine.Type // Share of CPUs, in 1e-9 fractions NanoCPUs int64 diff --git a/website/content/docs/v0.7/Reference/cli.md b/website/content/docs/v0.7/Reference/cli.md index c7e781420..10c3c3de8 100644 --- a/website/content/docs/v0.7/Reference/cli.md +++ b/website/content/docs/v0.7/Reference/cli.md @@ -98,6 +98,7 @@ talosctl cluster create [flags] --nameservers strings list of nameservers to use (default [8.8.8.8,1.1.1.1]) --registry-insecure-skip-verify strings list of registry hostnames to skip TLS verification for --registry-mirror strings list of registry mirrors to use in format: = + --skip-injecting-config skip injecting config from embedded metadata server, write config files to current directory --skip-kubeconfig skip merging kubeconfig from the created cluster --user-disk strings list of disks to create for each VM in format: ::: --vmlinuz-path string the compressed kernel image to use (default "_out/vmlinuz-${ARCH}")