talos/internal/pkg/provision/providers/firecracker/node.go

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package firecracker

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"syscall"

	"github.com/firecracker-microvm/firecracker-go-sdk"
	models "github.com/firecracker-microvm/firecracker-go-sdk/client/models"
	"github.com/hashicorp/go-multierror"
	"k8s.io/apimachinery/pkg/util/json"

	"github.com/talos-systems/go-procfs/procfs"

	"github.com/talos-systems/talos/internal/pkg/provision"
)

func (p *provisioner) createDisk(state *state, nodeReq provision.NodeRequest) (diskPath string, err error) {
	diskPath = filepath.Join(state.statePath, fmt.Sprintf("%s.disk", nodeReq.Name))

	var diskF *os.File

	diskF, err = os.Create(diskPath)
	if err != nil {
		return
	}

	defer diskF.Close() //nolint: errcheck

	err = diskF.Truncate(nodeReq.DiskSize)

	return
}

func (p *provisioner) createNodes(state *state, clusterReq provision.ClusterRequest, nodeReqs []provision.NodeRequest, opts *provision.Options) ([]provision.NodeInfo, error) {
	errCh := make(chan error)
	nodeCh := make(chan provision.NodeInfo, len(nodeReqs))

	for _, nodeReq := range nodeReqs {
		go func(nodeReq provision.NodeRequest) {
			nodeInfo, err := p.createNode(state, clusterReq, nodeReq, opts)
			if err == nil {
				nodeCh <- nodeInfo
			}

			errCh <- err
		}(nodeReq)
	}

	var multiErr *multierror.Error

	for range nodeReqs {
		multiErr = multierror.Append(multiErr, <-errCh)
	}

	close(nodeCh)

	nodesInfo := make([]provision.NodeInfo, 0, len(nodeReqs))

	for nodeInfo := range nodeCh {
		nodesInfo = append(nodesInfo, nodeInfo)
	}

	return nodesInfo, multiErr.ErrorOrNil()
}

//nolint: gocyclo
func (p *provisioner) createNode(state *state, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest, opts *provision.Options) (provision.NodeInfo, error) {
	socketPath := filepath.Join(state.statePath, fmt.Sprintf("%s.sock", nodeReq.Name))
	pidPath := filepath.Join(state.statePath, fmt.Sprintf("%s.pid", nodeReq.Name))

	vcpuCount := int64(math.RoundToEven(float64(nodeReq.NanoCPUs) / 1000 / 1000 / 1000))
	if vcpuCount < 2 {
		vcpuCount = 1
	}

	memSize := nodeReq.Memory / 1024 / 1024

	diskPath, err := p.createDisk(state, nodeReq)
	if err != nil {
		return provision.NodeInfo{}, err
	}

	cmdline := procfs.NewDefaultCmdline()

	// required to get kernel console
	cmdline.Append("console", "ttyS0")

	// reboot configuration
	cmdline.Append("reboot", "k")
	cmdline.Append("panic", "1")

	// disable stuff we don't need
	cmdline.Append("pci", "off")
	cmdline.Append("acpi", "off")
	cmdline.Append("i8042.noaux", "")

	// Talos config
	cmdline.Append("talos.platform", "metal")
	cmdline.Append("talos.config", "{TALOS_CONFIG_URL}") // to be patched by launcher
	cmdline.Append("talos.hostname", nodeReq.Name)

	ones, _ := clusterReq.Network.CIDR.Mask.Size()

	cfg := firecracker.Config{
		SocketPath:      socketPath,
		KernelImagePath: clusterReq.KernelPath,
		KernelArgs:      cmdline.String(),
		InitrdPath:      clusterReq.InitramfsPath,
		ForwardSignals:  []os.Signal{}, // don't forward any signals
		MachineCfg: models.MachineConfiguration{
			HtEnabled:  firecracker.Bool(false),
			VcpuCount:  firecracker.Int64(vcpuCount),
			MemSizeMib: firecracker.Int64(memSize),
		},
		NetworkInterfaces: firecracker.NetworkInterfaces{
			firecracker.NetworkInterface{
				CNIConfiguration: &firecracker.CNIConfiguration{
					BinPath:       clusterReq.Network.CNI.BinPath,
					ConfDir:       clusterReq.Network.CNI.ConfDir,
					CacheDir:      clusterReq.Network.CNI.CacheDir,
					NetworkConfig: state.vmCNIConfig,
					Args: [][2]string{
						{"IP", fmt.Sprintf("%s/%d", nodeReq.IP, ones)},
						{"GATEWAY", clusterReq.Network.GatewayAddr.String()},
					},
					IfName:   "veth0",
					VMIfName: "eth0",
				},
			},
		},
		Drives: []models.Drive{
			{
				DriveID:      firecracker.String("disk"),
				IsRootDevice: firecracker.Bool(false),
				IsReadOnly:   firecracker.Bool(false),
				PathOnHost:   firecracker.String(diskPath),
			},
		},
	}

	logFile, err := os.OpenFile(filepath.Join(state.statePath, fmt.Sprintf("%s.log", nodeReq.Name)), os.O_APPEND|os.O_CREATE|os.O_RDWR, 0666)
	if err != nil {
		return provision.NodeInfo{}, err
	}

	defer logFile.Close() //nolint: errcheck

	nodeConfig, err := nodeReq.Config.String()
	if err != nil {
		return provision.NodeInfo{}, err
	}

	launchConfig := LaunchConfig{
		FirecrackerConfig:   cfg,
		Config:              nodeConfig,
		GatewayAddr:         clusterReq.Network.GatewayAddr.String(),
		BootloaderEmulation: opts.BootloaderEmulation,
	}

	launchConfigFile, err := os.Create(filepath.Join(state.statePath, fmt.Sprintf("%s.config", nodeReq.Name)))
	if err != nil {
		return provision.NodeInfo{}, err
	}

	if err = json.NewEncoder(launchConfigFile).Encode(&launchConfig); err != nil {
		return provision.NodeInfo{}, err
	}

	if _, err = launchConfigFile.Seek(0, io.SeekStart); err != nil {
		return provision.NodeInfo{}, err
	}

	defer launchConfigFile.Close() //nolint: errcheck

	cmd := exec.Command(clusterReq.SelfExecutable, "firecracker-launch")
	cmd.Stdout = logFile
	cmd.Stderr = logFile
	cmd.Stdin = launchConfigFile
	cmd.SysProcAttr = &syscall.SysProcAttr{
		Setsid: true, // daemonize
	}

	if err = cmd.Start(); err != nil {
		return provision.NodeInfo{}, err
	}

	if err = ioutil.WriteFile(pidPath, []byte(strconv.Itoa(cmd.Process.Pid)), os.ModePerm); err != nil {
		return provision.NodeInfo{}, fmt.Errorf("error writing PID file: %w", err)
	}

	// no need to wait here, as cmd has all the Stdin/out/err via *os.File

	nodeInfo := provision.NodeInfo{
		ID:   pidPath,
		Name: nodeReq.Name,
		Type: nodeReq.Config.Machine().Type(),

		NanoCPUs: nodeReq.NanoCPUs,
		Memory:   nodeReq.Memory,
		DiskSize: nodeReq.DiskSize,

		PrivateIP: nodeReq.IP,
	}

	return nodeInfo, nil
}

func (p *provisioner) destroyNodes(cluster provision.ClusterInfo, options *provision.Options) error {
	errCh := make(chan error)

	for _, node := range cluster.Nodes {
		go func(node provision.NodeInfo) {
			fmt.Fprintln(options.LogWriter, "stopping VM", node.Name)

			errCh <- p.destroyNode(node)
		}(node)
	}

	var multiErr *multierror.Error

	for range cluster.Nodes {
		multiErr = multierror.Append(multiErr, <-errCh)
	}

	return multiErr.ErrorOrNil()
}

func (p *provisioner) destroyNode(node provision.NodeInfo) error {
	return stopProcessByPidfile(node.ID) // node.ID stores PID path for control process
}