From 7fcb89ee385fdbf47dae4a8308299c00488df84a Mon Sep 17 00:00:00 2001 From: Orzelius Date: Wed, 21 May 2025 20:44:21 +0900 Subject: [PATCH] chore: add darwin vmnet qemu support * split the networking config into common and platform specific structs * add logic to get the config file server for darwin * generate a random mac address for the darwin qemu machines Signed-off-by: Orzelius <33936483+Orzelius@users.noreply.github.com> --- pkg/provision/providers/qemu/launch.go | 84 ++++++++++--------- pkg/provision/providers/qemu/launch_darwin.go | 46 ++++++++++ pkg/provision/providers/qemu/launch_linux.go | 57 ++++++++++--- pkg/provision/providers/qemu/node.go | 45 +++++++--- 4 files changed, 171 insertions(+), 61 deletions(-) diff --git a/pkg/provision/providers/qemu/launch.go b/pkg/provision/providers/qemu/launch.go index 031f9db7d..82fd8eac8 100644 --- a/pkg/provision/providers/qemu/launch.go +++ b/pkg/provision/providers/qemu/launch.go @@ -18,11 +18,9 @@ import ( "strings" "time" - "github.com/containernetworking/cni/libcni" "github.com/google/uuid" "github.com/siderolabs/go-blockdevice/v2/blkid" - "github.com/siderolabs/talos/pkg/provision" "github.com/siderolabs/talos/pkg/provision/providers/vm" ) @@ -57,18 +55,6 @@ type LaunchConfig struct { // Talos config Config string - // Network - BridgeName string - NetworkConfig *libcni.NetworkConfigList - CNI provision.CNIConfig - IPs []netip.Addr - CIDRs []netip.Prefix - NoMasqueradeCIDRs []netip.Prefix - Hostname string - GatewayAddrs []netip.Addr - MTU int - Nameservers []netip.Addr - // PXE TFTPServer string BootFilename string @@ -81,10 +67,10 @@ type LaunchConfig struct { sdStubExtraCmdline string sdStubExtraCmdlineConfig string - // filled by CNI invocation - tapName string - VMMac string - nsPath string + // platform specific Network configuration + Network networkConfig + + VMMac string // signals c chan os.Signal @@ -93,6 +79,16 @@ type LaunchConfig struct { controller *Controller } +type networkConfigBase struct { + BridgeName string + IPs []netip.Addr + CIDRs []netip.Prefix + GatewayAddrs []netip.Addr + Hostname string + MTU int + Nameservers []netip.Addr +} + type tpmConfig struct { NodeName string StateDir string @@ -121,7 +117,7 @@ func launchVM(config *LaunchConfig) error { "-smp", fmt.Sprintf("cpus=%d", config.VCPUCount), "-cpu", cpuArg, "-nographic", - "-netdev", fmt.Sprintf("tap,id=net0,ifname=%s,script=no,downscript=no", config.tapName), + "-netdev", getNetdevParams(config.Network, "net0"), "-device", fmt.Sprintf("virtio-net-pci,netdev=net0,mac=%s", config.VMMac), // TODO: uncomment the following line to get another eth interface not connected to anything // "-nic", "tap,model=virtio-net-pci", @@ -131,19 +127,18 @@ func launchVM(config *LaunchConfig) error { "-no-reboot", "-boot", fmt.Sprintf("order=%s,reboot-timeout=5000", bootOrder), "-smbios", fmt.Sprintf("type=1,uuid=%s", config.NodeUUID), - "-chardev", fmt.Sprintf("socket,path=%s/%s.sock,server=on,wait=off,id=qga0", config.StatePath, config.Hostname), + "-chardev", fmt.Sprintf("socket,path=%s/%s.sock,server=on,wait=off,id=qga0", config.StatePath, config.Network.Hostname), "-device", "virtio-serial", "-device", "virtserialport,chardev=qga0,name=org.qemu.guest_agent.0", "-device", "i6300esb,id=watchdog0", - "-watchdog-action", - "pause", + "-watchdog-action", "pause", } if config.WithDebugShell { args = append( args, "-serial", - fmt.Sprintf("unix:%s/%s.serial,server,nowait", config.StatePath, config.Hostname), + fmt.Sprintf("unix:%s/%s.serial,server,nowait", config.StatePath, config.Network.Hostname), ) } @@ -392,6 +387,8 @@ func launchVM(config *LaunchConfig) error { // logfile in state directory. // // When signals SIGINT, SIGTERM are received, control process stops qemu and exits. +// +//nolint:gocyclo func Launch() error { var config LaunchConfig @@ -417,7 +414,9 @@ func Launch() error { httpServer.Serve() defer httpServer.Shutdown(ctx) //nolint:errcheck - patchKernelArgs(&config, httpServer.GetAddr().String()) + if err := patchKernelArgs(&config, httpServer.GetAddr()); err != nil { + return err + } return withNetworkContext(ctx, &config, func(config *LaunchConfig) error { err = dumpIpam(*config) @@ -444,13 +443,20 @@ func Launch() error { }) } -func patchKernelArgs(config *LaunchConfig, httpServerAddr string) { +func patchKernelArgs(config *LaunchConfig, httpServerAddr net.Addr) error { + configServerAddr, err := getConfigServerAddr(httpServerAddr, *config) + if err != nil { + return err + } + config.sdStubExtraCmdline = "console=ttyS0" if strings.Contains(config.KernelArgs, "{TALOS_CONFIG_URL}") { - config.KernelArgs = strings.ReplaceAll(config.KernelArgs, "{TALOS_CONFIG_URL}", fmt.Sprintf("http://%s/config.yaml", httpServerAddr)) + config.KernelArgs = strings.ReplaceAll(config.KernelArgs, "{TALOS_CONFIG_URL}", fmt.Sprintf("http://%s/config.yaml", configServerAddr)) config.sdStubExtraCmdlineConfig = fmt.Sprintf(" talos.config=http://%s/config.yaml", httpServerAddr) } + + return nil } func waitForFileToExist(path string, timeout time.Duration) error { @@ -472,30 +478,30 @@ func waitForFileToExist(path string, timeout time.Duration) error { } func dumpIpam(config LaunchConfig) error { - for j := range config.CIDRs { - nameservers := make([]netip.Addr, 0, len(config.Nameservers)) + for j := range config.Network.CIDRs { + nameservers := make([]netip.Addr, 0, len(config.Network.Nameservers)) // filter nameservers by IPv4/IPv6 matching IPs - for i := range config.Nameservers { - if config.IPs[j].Is6() { - if config.Nameservers[i].Is6() { - nameservers = append(nameservers, config.Nameservers[i]) + for i := range config.Network.Nameservers { + if config.Network.IPs[j].Is6() { + if config.Network.Nameservers[i].Is6() { + nameservers = append(nameservers, config.Network.Nameservers[i]) } } else { - if config.Nameservers[i].Is4() { - nameservers = append(nameservers, config.Nameservers[i]) + if config.Network.Nameservers[i].Is4() { + nameservers = append(nameservers, config.Network.Nameservers[i]) } } } // dump node IP/mac/hostname for dhcp if err := vm.DumpIPAMRecord(config.StatePath, vm.IPAMRecord{ - IP: config.IPs[j], - Netmask: byte(config.CIDRs[j].Bits()), + IP: config.Network.IPs[j], + Netmask: byte(config.Network.CIDRs[j].Bits()), MAC: config.VMMac, - Hostname: config.Hostname, - Gateway: config.GatewayAddrs[j], - MTU: config.MTU, + Hostname: config.Network.Hostname, + Gateway: config.Network.GatewayAddrs[j], + MTU: config.Network.MTU, Nameservers: nameservers, TFTPServer: config.TFTPServer, IPXEBootFilename: config.IPXEBootFileName, diff --git a/pkg/provision/providers/qemu/launch_darwin.go b/pkg/provision/providers/qemu/launch_darwin.go index 9e1faaeb2..85238bab0 100644 --- a/pkg/provision/providers/qemu/launch_darwin.go +++ b/pkg/provision/providers/qemu/launch_darwin.go @@ -6,9 +6,55 @@ package qemu import ( "context" + "fmt" + "net" + "net/netip" "os/exec" + + "github.com/siderolabs/talos/pkg/provision" + "github.com/siderolabs/talos/pkg/provision/providers/vm" ) +type networkConfig struct { + networkConfigBase + StartAddr netip.Addr + EndAddr netip.Addr +} + +func getLaunchNetworkConfig(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest) networkConfig { + // This ip will be assigned to the bridge + // The following ips will be assigned to the vms + startAddr := clusterReq.Nodes[0].IPs[0].Prev() + endAddr := clusterReq.Nodes[len(clusterReq.Nodes)-1].IPs[0].Next() + + return networkConfig{ + networkConfigBase: getLaunchNetworkConfigBase(state, clusterReq, nodeReq), + StartAddr: startAddr, + EndAddr: endAddr, + } +} + +func getNetdevParams(networkConfig networkConfig, id string) string { + netDevArg := "vmnet-shared,id=" + id + cidr := networkConfig.CIDRs[0] + m := net.CIDRMask(cidr.Bits(), 32) + subnetMask := fmt.Sprintf("%d.%d.%d.%d", m[0], m[1], m[2], m[3]) + netDevArg += fmt.Sprintf(",start-address=%s,end-address=%s,subnet-mask=%s", networkConfig.StartAddr, networkConfig.EndAddr, subnetMask) + + return netDevArg +} + +// getConfigServerAddr returns the ip accessible to the VM that will route to the config server. +// hostAddrs is the address on which the server is accessible from the host network. +func getConfigServerAddr(hostAddrs net.Addr, config LaunchConfig) (netip.AddrPort, error) { + addrPort, err := netip.ParseAddrPort(hostAddrs.String()) + if err != nil { + return netip.AddrPort{}, err + } + + return netip.AddrPortFrom(config.Network.GatewayAddrs[0], addrPort.Port()), nil +} + // withNetworkContext runs the f on the host network on darwin. func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config *LaunchConfig) error) error { return f(config) diff --git a/pkg/provision/providers/qemu/launch_linux.go b/pkg/provision/providers/qemu/launch_linux.go index 7eba75c7c..af35cc7ce 100644 --- a/pkg/provision/providers/qemu/launch_linux.go +++ b/pkg/provision/providers/qemu/launch_linux.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "log" + "net" "net/netip" "os/exec" "path/filepath" @@ -26,9 +27,41 @@ import ( "github.com/siderolabs/gen/xslices" sideronet "github.com/siderolabs/net" + "github.com/siderolabs/talos/pkg/provision" "github.com/siderolabs/talos/pkg/provision/internal/cniutils" + "github.com/siderolabs/talos/pkg/provision/providers/vm" ) +type networkConfig struct { + networkConfigBase + + // TODO: rename field to cniNetworkConfig + CniNetworkConfig *libcni.NetworkConfigList + CNI provision.CNIConfig + NoMasqueradeCIDRs []netip.Prefix + + // filled by CNI invocation + tapName string + ns ns.NetNS +} + +func getLaunchNetworkConfig(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest) networkConfig { + return networkConfig{ + networkConfigBase: getLaunchNetworkConfigBase(state, clusterReq, nodeReq), + CniNetworkConfig: state.VMCNIConfig, + CNI: clusterReq.Network.CNI, + NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs, + } +} + +func getNetdevParams(networkConfig networkConfig, id string) string { + return fmt.Sprintf("tap,id=%s,ifname=%s,script=no,downscript=no", id, networkConfig.tapName) +} + +func getConfigServerAddr(hostAddrs net.Addr, _ LaunchConfig) (net.Addr, error) { + return hostAddrs, nil +} + // withCNIOperationLocked ensures that CNI operations don't run concurrently. // // There are race conditions in the CNI plugins that can cause a failure if called concurrently. @@ -70,7 +103,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config // random ID for the CNI, maps to single VM containerID := uuid.New().String() - cniConfig := libcni.NewCNIConfigWithCacheDir(config.CNI.BinPath, config.CNI.CacheDir, nil) + cniConfig := libcni.NewCNIConfigWithCacheDir(config.Network.CNI.BinPath, config.Network.CNI.CacheDir, nil) // create a network namespace ns, err := testutils.NewNS() @@ -83,12 +116,12 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config testutils.UnmountNS(ns) //nolint:errcheck }() - ips := make([]string, len(config.IPs)) + ips := make([]string, len(config.Network.IPs)) for j := range ips { - ips[j] = sideronet.FormatCIDR(config.IPs[j], config.CIDRs[j]) + ips[j] = sideronet.FormatCIDR(config.Network.IPs[j], config.Network.CIDRs[j]) } - gatewayAddrs := xslices.Map(config.GatewayAddrs, netip.Addr.String) + gatewayAddrs := xslices.Map(config.Network.GatewayAddrs, netip.Addr.String) runtimeConf := libcni.RuntimeConf{ ContainerID: containerID, @@ -105,7 +138,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config err = withCNIOperationLockedNoResult( config, func() error { - return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf) + return cniConfig.DelNetworkList(ctx, config.Network.CniNetworkConfig, &runtimeConf) }, ) if err != nil { @@ -115,7 +148,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config res, err := withCNIOperationLocked( config, func() (types.Result, error) { - return cniConfig.AddNetworkList(ctx, config.NetworkConfig, &runtimeConf) + return cniConfig.AddNetworkList(ctx, config.Network.CniNetworkConfig, &runtimeConf) }, ) if err != nil { @@ -126,7 +159,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config if e := withCNIOperationLockedNoResult( config, func() error { - return cniConfig.DelNetworkList(ctx, config.NetworkConfig, &runtimeConf) + return cniConfig.DelNetworkList(ctx, config.Network.CniNetworkConfig, &runtimeConf) }, ); e != nil { log.Printf("error cleaning up CNI: %s", e) @@ -145,7 +178,7 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config "that supports automatic VM network configuration such as tc-redirect-tap") } - cniChain := utils.FormatChainName(config.NetworkConfig.Name, containerID) + cniChain := utils.FormatChainName(config.Network.CniNetworkConfig.Name, containerID) ipt, err := iptables.New() if err != nil { @@ -159,21 +192,21 @@ func withNetworkContext(ctx context.Context, config *LaunchConfig, f func(config return fmt.Errorf("failed to insert iptables rule to allow broadcast traffic: %w", err) } - for _, cidr := range config.NoMasqueradeCIDRs { + for _, cidr := range config.Network.NoMasqueradeCIDRs { if err = ipt.InsertUnique("nat", cniChain, 1, "--destination", cidr.String(), "-j", "ACCEPT"); err != nil { return fmt.Errorf("failed to insert iptables rule to allow non-masquerade traffic to cidr %q: %w", cidr.String(), err) } } - config.tapName = tapIface.Name + config.Network.tapName = tapIface.Name config.VMMac = vmIface.Mac - config.nsPath = ns.Path() + config.Network.ns = ns return f(config) } func startQemuCmd(config *LaunchConfig, cmd *exec.Cmd) error { - if err := ns.WithNetNSPath(config.nsPath, func(_ ns.NetNS) error { + if err := ns.WithNetNSPath(config.Network.ns.Path(), func(_ ns.NetNS) error { return cmd.Start() }); err != nil { return err diff --git a/pkg/provision/providers/qemu/node.go b/pkg/provision/providers/qemu/node.go index df7bc6582..3c823ef5e 100644 --- a/pkg/provision/providers/qemu/node.go +++ b/pkg/provision/providers/qemu/node.go @@ -5,10 +5,12 @@ package qemu import ( + "crypto/rand" "encoding/json" "fmt" "io" "math" + "net" "os" "os/exec" "path/filepath" @@ -173,20 +175,16 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe BootloaderEnabled: opts.BootloaderEnabled, NodeUUID: nodeUUID, Config: nodeConfig, - BridgeName: state.BridgeName, - NetworkConfig: state.VMCNIConfig, - CNI: clusterReq.Network.CNI, - CIDRs: clusterReq.Network.CIDRs, - NoMasqueradeCIDRs: clusterReq.Network.NoMasqueradeCIDRs, - IPs: nodeReq.IPs, - GatewayAddrs: clusterReq.Network.GatewayAddrs, - MTU: clusterReq.Network.MTU, - Nameservers: clusterReq.Network.Nameservers, TFTPServer: nodeReq.TFTPServer, IPXEBootFileName: nodeReq.IPXEBootFilename, APIBindAddress: apiBind, WithDebugShell: opts.WithDebugShell, IOMMUEnabled: opts.IOMMUEnabled, + Network: getLaunchNetworkConfig(state, clusterReq, nodeReq), + + // Generate a random MAC address. + // On linux this is later overridden to the interface mac. + VMMac: getRandomMacAddress(), } if clusterReq.IPXEBootScript != "" { @@ -220,7 +218,7 @@ func (p *provisioner) createNode(state *vm.State, clusterReq provision.ClusterRe } if !clusterReq.Network.DHCPSkipHostname { - launchConfig.Hostname = nodeReq.Name + launchConfig.Network.Hostname = nodeReq.Name } if !nodeReq.PXEBooted && launchConfig.IPXEBootFileName == "" { @@ -368,3 +366,30 @@ func (p *provisioner) createMetalConfigISO(state *vm.State, nodeName, config str return isoPath, nil } + +func getLaunchNetworkConfigBase(state *vm.State, clusterReq provision.ClusterRequest, nodeReq provision.NodeRequest) networkConfigBase { + return networkConfigBase{ + BridgeName: state.BridgeName, + CIDRs: clusterReq.Network.CIDRs, + IPs: nodeReq.IPs, + GatewayAddrs: clusterReq.Network.GatewayAddrs, + MTU: clusterReq.Network.MTU, + Nameservers: clusterReq.Network.Nameservers, + } +} + +// getRandomMacAddress generates a random local MAC address +// https://stackoverflow.com/a/21027407/10938317 +func getRandomMacAddress() string { + const ( + local = 0b10 + multicast = 0b1 + ) + + buf := make([]byte, 6) + rand.Read(buf) //nolint:errcheck + // clear multicast bit (&^), ensure local bit (|) + buf[0] = buf[0]&^multicast | local + + return net.HardwareAddr(buf).String() +}