mirror of
https://github.com/siderolabs/talos.git
synced 2026-05-12 00:11:23 +02:00
chore: add darwin vmnet qemu support
* split the networking config into common and platform specific structs * add logic to get the config file server for darwin * generate a random mac address for the darwin qemu machines Signed-off-by: Orzelius <33936483+Orzelius@users.noreply.github.com>
This commit is contained in:
Orzelius
parent
fc1237343f
commit
7fcb89ee38
@ -18,11 +18,9 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/containernetworking/cni/libcni"
|
||||
"github.com/google/uuid"
|
||||
"github.com/siderolabs/go-blockdevice/v2/blkid"
|
||||
|
||||
"github.com/siderolabs/talos/pkg/provision"
|
||||
"github.com/siderolabs/talos/pkg/provision/providers/vm"
|
||||
)
|
||||
|
||||
@ -57,18 +55,6 @@ type LaunchConfig struct {
|
||||
// Talos config
|
||||
Config string
|
||||
|
||||
// Network
|
||||
BridgeName string
|
||||
NetworkConfig *libcni.NetworkConfigList
|
||||
CNI provision.CNIConfig
|
||||
IPs []netip.Addr
|
||||
CIDRs []netip.Prefix
|
||||
NoMasqueradeCIDRs []netip.Prefix
|
||||
Hostname string
|
||||
GatewayAddrs []netip.Addr
|
||||
MTU int
|
||||
Nameservers []netip.Addr
|
||||
|
||||
// PXE
|
||||
TFTPServer string
|
||||
BootFilename string
|
||||
@ -81,10 +67,10 @@ type LaunchConfig struct {
|
||||
sdStubExtraCmdline string
|
||||
sdStubExtraCmdlineConfig string
|
||||
|
||||
// filled by CNI invocation
|
||||
tapName string
|
||||
VMMac string
|
||||
nsPath string
|
||||
// platform specific Network configuration
|
||||
Network networkConfig
|
||||
|
||||
VMMac string
|
||||
|
||||
// signals
|
||||
c chan os.Signal
|
||||
@ -93,6 +79,16 @@ type LaunchConfig struct {
|
||||
controller *Controller
|
||||
}
|
||||
|
||||
type networkConfigBase struct {
|
||||
BridgeName string
|
||||
IPs []netip.Addr
|
||||
CIDRs []netip.Prefix
|
||||
GatewayAddrs []netip.Addr
|
||||
Hostname string
|
||||
MTU int
|
||||
Nameservers []netip.Addr
|
||||
}
|
||||
|
||||
type tpmConfig struct {
|
||||
NodeName string
|
||||
StateDir string
|
||||
@ -121,7 +117,7 @@ func launchVM(config *LaunchConfig) error {
|
||||
"-smp", fmt.Sprintf("cpus=%d", config.VCPUCount),
|
||||
"-cpu", cpuArg,
|
||||
"-nographic",
|
||||
"-netdev", fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", config.tapName),
|
||||
"-netdev", getNetdevParams(config.Network, "net0"),
|
||||
"-device", fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", config.VMMac),
|
||||
// TODO: uncomment the following line to get another eth interface not connected to anything
|
||||
// "-nic", "tap,model=virtio-net-pci",
|
||||
@ -131,19 +127,18 @@ func launchVM(config *LaunchConfig) error {
|
||||
"-no-reboot",
|
||||
"-boot", fmt.Sprintf("order=%s,reboot-timeout=5000", bootOrder),
|
||||
"-smbios", fmt.Sprintf("type=1,uuid=%s", config.NodeUUID),
|
||||
"-chardev", fmt.Sprintf("socket,path=%s/%s.sock,server=on,wait=off,id=qga0", config.StatePath, config.Hostname),
|
||||
"-chardev", fmt.Sprintf("socket,path=%s/%s.sock,server=on,wait=off,id=qga0", config.StatePath, config.Network.Hostname),
|
||||
"-device", "virtio-serial",
|
||||
"-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0",
|
||||
"-device", "i6300esb,id=watchdog0",
|
||||
"-watchdog-action",
|
||||
"pause",
|
||||
"-watchdog-action", "pause",
|
||||
}
|
||||
|
||||
if config.WithDebugShell {
|
||||
args = append(
|
||||
args,
|
||||
"-serial",
|
||||
fmt.Sprintf("unix:%s/%s.serial,server,nowait", config.StatePath, config.Hostname),
|
||||
fmt.Sprintf("unix:%s/%s.serial,server,nowait", config.StatePath, config.Network.Hostname),
|
||||
)
|
||||
}
|
||||
|
||||
@ -392,6 +387,8 @@ func launchVM(config *LaunchConfig) error {
|
||||
// logfile in state directory.
|
||||
//
|
||||
// When signals SIGINT, SIGTERM are received, control process stops qemu and exits.
|
||||
//
|
||||
//nolint:gocyclo
|
||||
func Launch() error {
|
||||
var config LaunchConfig
|
||||
|
||||
@ -417,7 +414,9 @@ func Launch() error {
|
||||
httpServer.Serve()
|
||||
defer httpServer.Shutdown(ctx) //nolint:errcheck
|
||||
|
||||
patchKernelArgs(&config, httpServer.GetAddr().String())
|
||||
if err := patchKernelArgs(&config, httpServer.GetAddr()); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return withNetworkContext(ctx, &config, func(config *LaunchConfig) error {
|
||||
err = dumpIpam(*config)
|
||||
@ -444,13 +443,20 @@ func Launch() error {
|
||||
})
|
||||
}
|
||||
|
||||
func patchKernelArgs(config *LaunchConfig, httpServerAddr string) {
|
||||
func patchKernelArgs(config *LaunchConfig, httpServerAddr net.Addr) error {
|
||||
configServerAddr, err := getConfigServerAddr(httpServerAddr, *config)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
config.sdStubExtraCmdline = "console=ttyS0"
|
||||
|
||||
if strings.Contains(config.KernelArgs, "{TALOS_CONFIG_URL}") {
|
||||
config.KernelArgs = strings.ReplaceAll(config.KernelArgs, "{TALOS_CONFIG_URL}", fmt.Sprintf("http://%s/config.yaml", httpServerAddr))
|
||||
config.KernelArgs = strings.ReplaceAll(config.KernelArgs, "{TALOS_CONFIG_URL}", fmt.Sprintf("http://%s/config.yaml", configServerAddr))
|
||||
config.sdStubExtraCmdlineConfig = fmt.Sprintf(" talos.config=http://%s/config.yaml", httpServerAddr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func waitForFileToExist(path string, timeout time.Duration) error {
|
||||
@ -472,30 +478,30 @@ func waitForFileToExist(path string, timeout time.Duration) error {
|
||||
}
|
||||
|
||||
func dumpIpam(config LaunchConfig) error {
|
||||
for j := range config.CIDRs {
|
||||
nameservers := make([]netip.Addr, 0, len(config.Nameservers))
|
||||
for j := range config.Network.CIDRs {
|
||||
nameservers := make([]netip.Addr, 0, len(config.Network.Nameservers))
|
||||
|
||||
// filter nameservers by IPv4/IPv6 matching IPs
|
||||
for i := range config.Nameservers {
|
||||
if config.IPs[j].Is6() {
|
||||
if config.Nameservers[i].Is6() {
|
||||
nameservers = append(nameservers, config.Nameservers[i])
|
||||
for i := range config.Network.Nameservers {
|
||||
if config.Network.IPs[j].Is6() {
|
||||
if config.Network.Nameservers[i].Is6() {
|
||||
nameservers = append(nameservers, config.Network.Nameservers[i])
|
||||
}
|
||||
} else {
|
||||
if config.Nameservers[i].Is4() {
|
||||
nameservers = append(nameservers, config.Nameservers[i])
|
||||
if config.Network.Nameservers[i].Is4() {
|
||||
nameservers = append(nameservers, config.Network.Nameservers[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// dump node IP/mac/hostname for dhcp
|
||||
if err := vm.DumpIPAMRecord(config.StatePath, vm.IPAMRecord{
|
||||
IP: config.IPs[j],
|
||||
Netmask: byte(config.CIDRs[j].Bits()),
|
||||
IP: config.Network.IPs[j],
|
||||
Netmask: byte(config.Network.CIDRs[j].Bits()),
|
||||
MAC: config.VMMac,
|
||||
Hostname: config.Hostname,
|
||||
Gateway: config.GatewayAddrs[j],
|
||||
MTU: config.MTU,
|
||||
Hostname: config.Network.Hostname,
|
||||
Gateway: config.Network.GatewayAddrs[j],
|
||||
MTU: config.Network.MTU,
|
||||
Nameservers: nameservers,
|
||||
TFTPServer: config.TFTPServer,
|
||||
IPXEBootFilename: config.IPXEBootFileName,
|
||||
|
||||
@ -6,9 +6,55 @@ package qemu
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os/exec"
|
||||
|
||||
"github.com/siderolabs/talos/pkg/provision"
|
||||
"github.com/siderolabs/talos/pkg/provision/providers/vm"
|
||||
)
|
||||
|
||||
type networkConfig struct {
|
||||
networkConfigBase
|
||||
StartAddr netip.Addr
|
||||
EndAddr netip.Addr
|
||||
}
|
||||
|
||||
func getLaunchNetworkConfig(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest) networkConfig {
|
||||
// This ip will be assigned to the bridge
|
||||
// The following ips will be assigned to the vms
|
||||
startAddr := clusterReq.Nodes[0].IPs[0].Prev()
|
||||
endAddr := clusterReq.Nodes[len(clusterReq.Nodes)-1].IPs[0].Next()
|
||||
|
||||
return networkConfig{
|
||||
networkConfigBase: getLaunchNetworkConfigBase(state, clusterReq, nodeReq),
|
||||
StartAddr: startAddr,
|
||||
EndAddr: endAddr,
|
||||
}
|
||||
}
|
||||
|
||||
func getNetdevParams(networkConfig networkConfig, id string) string {
|
||||
netDevArg := "vmnet-shared,id=" + id
|
||||
cidr := networkConfig.CIDRs[0]
|
||||
m := net.CIDRMask(cidr.Bits(), 32)
|
||||
subnetMask := fmt.Sprintf("%d.%d.%d.%d", m[0], m[1], m[2], m[3])
|
||||
netDevArg += fmt.Sprintf(",start-address=%s,end-address=%s,subnet-mask=%s", networkConfig.StartAddr, networkConfig.EndAddr, subnetMask)
|
||||
|
||||
return netDevArg
|
||||
}
|
||||
|
||||
// getConfigServerAddr returns the ip accessible to the VM that will route to the config server.
|
||||
// hostAddrs is the address on which the server is accessible from the host network.
|
||||
func getConfigServerAddr(hostAddrs net.Addr, config LaunchConfig) (netip.AddrPort, error) {
|
||||
addrPort, err := netip.ParseAddrPort(hostAddrs.String())
|
||||
if err != nil {
|
||||
return netip.AddrPort{}, err
|
||||
}
|
||||
|
||||
return netip.AddrPortFrom(config.Network.GatewayAddrs[0], addrPort.Port()), nil
|
||||
}
|
||||
|
||||
// withNetworkContext runs the f on the host network on darwin.
|
||||
func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config *LaunchConfig) error) error {
|
||||
return f(config)
|
||||
|
||||
@ -9,6 +9,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/netip"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@ -26,9 +27,41 @@ import (
|
||||
"github.com/siderolabs/gen/xslices"
|
||||
sideronet "github.com/siderolabs/net"
|
||||
|
||||
"github.com/siderolabs/talos/pkg/provision"
|
||||
"github.com/siderolabs/talos/pkg/provision/internal/cniutils"
|
||||
"github.com/siderolabs/talos/pkg/provision/providers/vm"
|
||||
)
|
||||
|
||||
type networkConfig struct {
|
||||
networkConfigBase
|
||||
|
||||
// TODO: rename field to cniNetworkConfig
|
||||
CniNetworkConfig *libcni.NetworkConfigList
|
||||
CNI provision.CNIConfig
|
||||
NoMasqueradeCIDRs []netip.Prefix
|
||||
|
||||
// filled by CNI invocation
|
||||
tapName string
|
||||
ns ns.NetNS
|
||||
}
|
||||
|
||||
func getLaunchNetworkConfig(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest) networkConfig {
|
||||
return networkConfig{
|
||||
networkConfigBase: getLaunchNetworkConfigBase(state, clusterReq, nodeReq),
|
||||
CniNetworkConfig: state.VMCNIConfig,
|
||||
CNI: clusterReq.Network.CNI,
|
||||
NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs,
|
||||
}
|
||||
}
|
||||
|
||||
func getNetdevParams(networkConfig networkConfig, id string) string {
|
||||
return fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", id, networkConfig.tapName)
|
||||
}
|
||||
|
||||
func getConfigServerAddr(hostAddrs net.Addr, _ LaunchConfig) (net.Addr, error) {
|
||||
return hostAddrs, nil
|
||||
}
|
||||
|
||||
// withCNIOperationLocked ensures that CNI operations don't run concurrently.
|
||||
//
|
||||
// There are race conditions in the CNI plugins that can cause a failure if called concurrently.
|
||||
@ -70,7 +103,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
// random ID for the CNI, maps to single VM
|
||||
containerID := uuid.New().String()
|
||||
|
||||
cniConfig := libcni.NewCNIConfigWithCacheDir(config.CNI.BinPath, config.CNI.CacheDir, nil)
|
||||
cniConfig := libcni.NewCNIConfigWithCacheDir(config.Network.CNI.BinPath, config.Network.CNI.CacheDir, nil)
|
||||
|
||||
// create a network namespace
|
||||
ns, err := testutils.NewNS()
|
||||
@ -83,12 +116,12 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
testutils.UnmountNS(ns) //nolint:errcheck
|
||||
}()
|
||||
|
||||
ips := make([]string, len(config.IPs))
|
||||
ips := make([]string, len(config.Network.IPs))
|
||||
for j := range ips {
|
||||
ips[j] = sideronet.FormatCIDR(config.IPs[j], config.CIDRs[j])
|
||||
ips[j] = sideronet.FormatCIDR(config.Network.IPs[j], config.Network.CIDRs[j])
|
||||
}
|
||||
|
||||
gatewayAddrs := xslices.Map(config.GatewayAddrs, netip.Addr.String)
|
||||
gatewayAddrs := xslices.Map(config.Network.GatewayAddrs, netip.Addr.String)
|
||||
|
||||
runtimeConf := libcni.RuntimeConf{
|
||||
ContainerID: containerID,
|
||||
@ -105,7 +138,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
err = withCNIOperationLockedNoResult(
|
||||
config,
|
||||
func() error {
|
||||
return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf)
|
||||
return cniConfig.DelNetworkList(ctx, config.Network.CniNetworkConfig, &runtimeConf)
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@ -115,7 +148,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
res, err := withCNIOperationLocked(
|
||||
config,
|
||||
func() (types.Result, error) {
|
||||
return cniConfig.AddNetworkList(ctx, config.NetworkConfig, &runtimeConf)
|
||||
return cniConfig.AddNetworkList(ctx, config.Network.CniNetworkConfig, &runtimeConf)
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
@ -126,7 +159,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
if e := withCNIOperationLockedNoResult(
|
||||
config,
|
||||
func() error {
|
||||
return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf)
|
||||
return cniConfig.DelNetworkList(ctx, config.Network.CniNetworkConfig, &runtimeConf)
|
||||
},
|
||||
); e != nil {
|
||||
log.Printf("error cleaning up CNI: %s", e)
|
||||
@ -145,7 +178,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
"that supports automatic VM network configuration such as tc-redirect-tap")
|
||||
}
|
||||
|
||||
cniChain := utils.FormatChainName(config.NetworkConfig.Name, containerID)
|
||||
cniChain := utils.FormatChainName(config.Network.CniNetworkConfig.Name, containerID)
|
||||
|
||||
ipt, err := iptables.New()
|
||||
if err != nil {
|
||||
@ -159,21 +192,21 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config
|
||||
return fmt.Errorf("failed to insert iptables rule to allow broadcast traffic: %w", err)
|
||||
}
|
||||
|
||||
for _, cidr := range config.NoMasqueradeCIDRs {
|
||||
for _, cidr := range config.Network.NoMasqueradeCIDRs {
|
||||
if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", cidr.String(), "-j", "ACCEPT"); err != nil {
|
||||
return fmt.Errorf("failed to insert iptables rule to allow non-masquerade traffic to cidr %q: %w", cidr.String(), err)
|
||||
}
|
||||
}
|
||||
|
||||
config.tapName = tapIface.Name
|
||||
config.Network.tapName = tapIface.Name
|
||||
config.VMMac = vmIface.Mac
|
||||
config.nsPath = ns.Path()
|
||||
config.Network.ns = ns
|
||||
|
||||
return f(config)
|
||||
}
|
||||
|
||||
func startQemuCmd(config *LaunchConfig, cmd *exec.Cmd) error {
|
||||
if err := ns.WithNetNSPath(config.nsPath, func(_ ns.NetNS) error {
|
||||
if err := ns.WithNetNSPath(config.Network.ns.Path(), func(_ ns.NetNS) error {
|
||||
return cmd.Start()
|
||||
}); err != nil {
|
||||
return err
|
||||
|
||||
@ -5,10 +5,12 @@
|
||||
package qemu
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@ -173,20 +175,16 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe
|
||||
BootloaderEnabled: opts.BootloaderEnabled,
|
||||
NodeUUID: nodeUUID,
|
||||
Config: nodeConfig,
|
||||
BridgeName: state.BridgeName,
|
||||
NetworkConfig: state.VMCNIConfig,
|
||||
CNI: clusterReq.Network.CNI,
|
||||
CIDRs: clusterReq.Network.CIDRs,
|
||||
NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs,
|
||||
IPs: nodeReq.IPs,
|
||||
GatewayAddrs: clusterReq.Network.GatewayAddrs,
|
||||
MTU: clusterReq.Network.MTU,
|
||||
Nameservers: clusterReq.Network.Nameservers,
|
||||
TFTPServer: nodeReq.TFTPServer,
|
||||
IPXEBootFileName: nodeReq.IPXEBootFilename,
|
||||
APIBindAddress: apiBind,
|
||||
WithDebugShell: opts.WithDebugShell,
|
||||
IOMMUEnabled: opts.IOMMUEnabled,
|
||||
Network: getLaunchNetworkConfig(state, clusterReq, nodeReq),
|
||||
|
||||
// Generate a random MAC address.
|
||||
// On linux this is later overridden to the interface mac.
|
||||
VMMac: getRandomMacAddress(),
|
||||
}
|
||||
|
||||
if clusterReq.IPXEBootScript != "" {
|
||||
@ -220,7 +218,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe
|
||||
}
|
||||
|
||||
if !clusterReq.Network.DHCPSkipHostname {
|
||||
launchConfig.Hostname = nodeReq.Name
|
||||
launchConfig.Network.Hostname = nodeReq.Name
|
||||
}
|
||||
|
||||
if !nodeReq.PXEBooted && launchConfig.IPXEBootFileName == "" {
|
||||
@ -368,3 +366,30 @@ func (p *provisioner) createMetalConfigISO(state *vm.State, nodeName, config str
|
||||
|
||||
return isoPath, nil
|
||||
}
|
||||
|
||||
func getLaunchNetworkConfigBase(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest) networkConfigBase {
|
||||
return networkConfigBase{
|
||||
BridgeName: state.BridgeName,
|
||||
CIDRs: clusterReq.Network.CIDRs,
|
||||
IPs: nodeReq.IPs,
|
||||
GatewayAddrs: clusterReq.Network.GatewayAddrs,
|
||||
MTU: clusterReq.Network.MTU,
|
||||
Nameservers: clusterReq.Network.Nameservers,
|
||||
}
|
||||
}
|
||||
|
||||
// getRandomMacAddress generates a random local MAC address
|
||||
// https://stackoverflow.com/a/21027407/10938317
|
||||
func getRandomMacAddress() string {
|
||||
const (
|
||||
local = 0b10
|
||||
multicast = 0b1
|
||||
)
|
||||
|
||||
buf := make([]byte, 6)
|
||||
rand.Read(buf) //nolint:errcheck
|
||||
// clear multicast bit (&^), ensure local bit (|)
|
||||
buf[0] = buf[0]&^multicast | local
|
||||
|
||||
return net.HardwareAddr(buf).String()
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user