talos/internal/test-framework/cmd/basic-integration.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package cmd

import (
	"context"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"runtime"
	"strconv"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/container"
	"github.com/docker/docker/api/types/mount"
	"github.com/docker/docker/client"
	"github.com/spf13/cobra"

	"github.com/talos-systems/talos/cmd/osctl/pkg/helpers"
	"github.com/talos-systems/talos/internal/test-framework/internal/pkg/runner"
	"github.com/talos-systems/talos/pkg/constants"
)

const (
	timeoutSeconds = 300
	tmpPath        = "/tmp/e2e"
)

var (
	cleanup    bool
	talosImage = "docker.io/autonomy/talos:" + os.Getenv("TAG")
	kubeImage  = constants.KubernetesImage + ":v" + constants.DefaultKubernetesVersion
)

// Add basic-integration command
var basicIntegrationCmd = &cobra.Command{
	Use:   "basic-integration",
	Short: "Runs the docker-based basic integration test",
	Run: func(cmd *cobra.Command, args []string) {
		helpers.Should(basicIntegration())
	},
}

// Stub out the basics for our runner configuration
var runnerConfig = &runner.ContainerConfigs{
	ContainerConfig: &container.Config{
		Image: kubeImage,
		Tty:   true,
		Env: []string{
			"TALOSCONFIG=" + filepath.Join(tmpPath, "talosconfig"),
			"KUBECONFIG=" + filepath.Join(tmpPath, "kubeconfig"),
		},
	},
	HostConfig: &container.HostConfig{
		Mounts: []mount.Mount{
			{
				Type:   mount.TypeBind,
				Source: tmpPath,
				Target: tmpPath,
			},
		},
	},
}

func init() {
	basicIntegrationCmd.Flags().BoolVar(&cleanup, "cleanup", false, "Cleanup the created cluster after completion")
	rootCmd.AddCommand(basicIntegrationCmd)
}

// basicIntegration creates a docker-based talos cluster
// nolint: gocyclo
func basicIntegration() error {
	// Ensure tmp dir and set env vars
	if err := os.MkdirAll("/tmp/e2e", os.ModePerm); err != nil {
		return err
	}

	if err := os.Setenv("TALOSCONFIG", filepath.Join(tmpPath, "talosconfig")); err != nil {
		return err
	}

	if err := os.Setenv("KUBECONFIG", filepath.Join(tmpPath, "kubeconfig")); err != nil {
		return err
	}

	// Create docker client and pull down hyperkube
	ctx := context.Background()

	cli, err := client.NewEnvClient()
	if err != nil {
		return err
	}

	var reader io.ReadCloser

	if reader, err = cli.ImagePull(ctx, kubeImage, types.ImagePullOptions{}); err != nil {
		return err
	}

	// nolint: errcheck
	defer reader.Close()

	if _, err = io.Copy(ioutil.Discard, reader); err != nil {
		return err
	}

	// Discover proper path to osctl build and add it to runner mounts
	osctlBinary := "osctl-darwin-amd64"
	if runtime.GOOS == "linux" {
		osctlBinary = "osctl-linux-amd64"
	}

	currentDir, err := os.Getwd()
	if err != nil {
		return err
	}

	osctlBinPath := filepath.Join(currentDir, "build", osctlBinary)
	integrationBinPath := filepath.Join(currentDir, "bin", "integration-test")

	runnerConfig.HostConfig.Mounts = append(runnerConfig.HostConfig.Mounts,
		mount.Mount{
			Type:   mount.TypeBind,
			Source: osctlBinPath,
			Target: "/bin/osctl",
		},
		mount.Mount{
			Type:   mount.TypeBind,
			Source: integrationBinPath,
			Target: "/bin/integration-test",
		},
	)

	// Create basic integration cluster and defer cleaning it up if the cleanup flag is set
	log.Println("deploying basic integration cluster")

	if err = runner.CommandLocal(osctlBinPath + " cluster create --name integration --masters=3 --mtu 1440 --cpus 4.0 --image " + talosImage); err != nil {
		return err
	}

	if cleanup {
		// nolint: errcheck
		defer runner.CommandLocal(osctlBinPath + " cluster destroy --name integration")
	}

	// Set osctl to talk to master-1
	log.Println("targeting master-1 in talosconfig")

	if err = runner.CommandLocal(osctlBinPath + " config endpoint 10.5.0.2"); err != nil {
		return err
	}

	// Wait for bootkube completion
	log.Println("waiting for bootkube completion")

	if err = runner.CommandInContainerWithTimeout(ctx, cli, runnerConfig, "osctl service bootkube | grep Finished", timeoutSeconds); err != nil {
		return err
	}

	// Wait for kubeconfig and target master-1
	log.Println("waiting for kubeconfig")

	if err = runner.CommandInContainerWithTimeout(ctx, cli, runnerConfig, "osctl kubeconfig /tmp/e2e -f", timeoutSeconds); err != nil {
		return err
	}

	log.Println("targeting master-1 in kubeconfig")

	if err = runner.CommandInContainer(
		ctx,
		cli,
		runnerConfig,
		"kubectl --kubeconfig ${KUBECONFIG} config set-cluster local --server https://10.5.0.2:6443",
	); err != nil {
		return err
	}

	// Wait for all nodes to report in
	log.Println("waiting for nodes to report in")

	if err = runner.CommandInContainerWithTimeout(
		ctx,
		cli,
		runnerConfig,
		"kubectl get nodes -o go-template='{{ len .items }}' | grep 4 >/dev/null",
		timeoutSeconds,
	); err != nil {
		return err
	}

	// Check all nodes ready
	log.Println("waiting for all nodes to be ready")

	if err = runner.CommandInContainerWithTimeout(
		ctx,
		cli,
		runnerConfig,
		"kubectl wait --timeout="+strconv.Itoa(timeoutSeconds)+"s --for=condition=ready=true --all nodes",
		timeoutSeconds,
	); err != nil {
		return err
	}

	// Verify HA control plane
	log.Println("Waiting for all masters to become ready")

	if err = runner.CommandInContainerWithTimeout(
		ctx,
		cli,
		runnerConfig,
		"kubectl get nodes -l node-role.kubernetes.io/master='' -o go-template='{{ len .items }}' | grep 3 >/dev/null",
		timeoutSeconds,
	); err != nil {
		return err
	}

	// Show etcd running
	for _, ip := range []string{"10.5.0.2", "10.5.0.3", "10.5.0.4"} {
		if err = runner.CommandInContainer(
			ctx,
			cli,
			runnerConfig,
			"osctl -e "+ip+" service etcd | grep Running",
		); err != nil {
			return err
		}
	}

	// Show containers and svcs
	for _, desiredCommand := range []string{"containers", "services"} {
		if err = runner.CommandInContainer(
			ctx,
			cli,
			runnerConfig,
			"osctl --nodes 10.5.0.2,10.5.0.3,10.5.0.4,10.5.0.5 "+desiredCommand,
		); err != nil {
			return err
		}
	}

	// Run integration tests
	if err = runner.CommandInContainer(
		ctx,
		cli,
		runnerConfig,
		"osctl config endpoint 10.5.0.2 && integration-test -test.v",
	); err != nil {
		return err
	}

	return nil
}