From 7bb86f2c162b0d77b5b370b4711f5e5ff8956642 Mon Sep 17 00:00:00 2001 From: Kristoffer Dalby Date: Wed, 29 Apr 2026 12:25:40 +0000 Subject: [PATCH] integration: HA cable-pull lifecycle test Add DisconnectFromNetwork/ReconnectToNetwork on TailscaleClient backed by pool.Client.DisconnectNetwork. Exercise single-router fail+recover either side, sequential dual failure, and simultaneous dual failure. The dual-failure legs assert no flap to a known-bad primary; the single-router-return legs check traffic only because docker network disconnect transiently fails probes on sibling routers. Fails on parent; passes after the fix. Updates #3203 --- integration/dockertestutil/network.go | 41 +++++ integration/route_test.go | 252 ++++++++++++++++++++++++++ integration/tailscale.go | 2 + integration/tsic/tsic.go | 15 ++ 4 files changed, 310 insertions(+) diff --git a/integration/dockertestutil/network.go b/integration/dockertestutil/network.go index 68d472f4..d224c43c 100644 --- a/integration/dockertestutil/network.go +++ b/integration/dockertestutil/network.go @@ -91,6 +91,47 @@ func AddContainerToNetwork( return nil } +// DisconnectContainerFromNetwork removes the container from network at +// the docker daemon level. Mirrors a physical cable pull: the +// container's network interface for that network disappears and any +// in-flight TCP connections are left half-open, exactly the failure +// mode iptables-based simulations cannot reproduce. +func DisconnectContainerFromNetwork( + pool *dockertest.Pool, + network *dockertest.Network, + testContainer string, +) error { + containers, err := pool.Client.ListContainers(docker.ListContainersOptions{ + All: true, + Filters: map[string][]string{ + "name": {testContainer}, + }, + }) + if err != nil { + return err + } + + if len(containers) == 0 { + return fmt.Errorf("%w: %s", ErrContainerNotFound, testContainer) + } + + return pool.Client.DisconnectNetwork(network.Network.ID, docker.NetworkConnectionOptions{ + Container: containers[0].ID, + Force: true, + }) +} + +// ReconnectContainerToNetwork is the inverse of +// DisconnectContainerFromNetwork — re-attaches the container to the +// network so traffic can flow again. +func ReconnectContainerToNetwork( + pool *dockertest.Pool, + network *dockertest.Network, + testContainer string, +) error { + return AddContainerToNetwork(pool, network, testContainer) +} + // RandomFreeHostPort asks the kernel for a free open port that is ready to use. // (from https://github.com/phayes/freeport) func RandomFreeHostPort() (int, error) { diff --git a/integration/route_test.go b/integration/route_test.go index 6cc47c30..a9b1f7f5 100644 --- a/integration/route_test.go +++ b/integration/route_test.go @@ -4297,3 +4297,255 @@ func TestHASubnetRouterFailoverBothOfflineCablePull(t *testing.T) { assert.Len(c, result, 13) }, propagationTime, 1*time.Second, "client reaches webservice via r2 after recovery") } + +// TestHASubnetRouterFailoverDockerDisconnect drives a multi-phase +// up/down/up/down lifecycle of two HA subnet routers using real +// docker network disconnects — the same failure primitive nblock +// observed when pulling a Proxmox interface in issue #3203. +// iptables-based simulations cannot reproduce this because the +// container's kernel still owns the socket; only daemon-level +// disconnect leaves the long-poll TCP half-open at the peer. +// +// Phases: +// 1. r1 starts as primary (lowest NodeID). +// 2. r1 alone fails and recovers — failover to r2, then traffic +// resumes when r1 returns. +// 3. r2 alone fails and recovers — failover, then traffic resumes. +// 4. Sequential dual failure — the issue #3203 bug. +// 4a. r1 down → r2 promoted. +// 4b. r2 down → primary must NOT flap to offline r1. +// 4c. r2 up → r2 primary again, traffic resumes. +// 5. Simultaneous dual failure. +// 5a. r1 + r2 down → primary must NOT flap to offline r1. +// 5b. both up → primary stays r2, traffic resumes. +// +// The no-flap assertions in 4b and 5a are the regression barriers +// for #3203. Phases 2/3 are functional checks (failover works, +// traffic recovers) without strict identity assertions on the +// "return" leg, since `docker network disconnect` triggers bridge +// reconfiguration that can transiently affect probing of OTHER +// containers on the same network — a test-infrastructure quirk +// that does not occur with a real cable pull. +func TestHASubnetRouterFailoverDockerDisconnect(t *testing.T) { + IntegrationSkip(t) + + propagationTime := integrationutil.ScaledTimeout(120 * time.Second) + flapWindow := integrationutil.ScaledTimeout(40 * time.Second) + + spec := ScenarioSpec{ + NodesPerUser: 2, + Users: []string{"user1", "user2"}, + Networks: map[string]NetworkSpec{ + "usernet1": {Users: []string{"user1"}}, + "usernet2": {Users: []string{"user2"}}, + }, + ExtraService: map[string][]extraServiceFunc{ + "usernet1": {Webservice}, + }, + Versions: []string{"head"}, + } + + scenario, err := NewScenario(spec) + require.NoErrorf(t, err, "failed to create scenario: %s", err) + + err = scenario.CreateHeadscaleEnv( + []tsic.Option{tsic.WithAcceptRoutes()}, + hsic.WithTestName("rt-hadocker"), + ) + requireNoErrHeadscaleEnv(t, err) + + allClients, err := scenario.ListTailscaleClients() + requireNoErrListClients(t, err) + + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) + + headscale, err := scenario.Headscale() + requireNoErrGetHeadscale(t, err) + + prefp, err := scenario.SubnetOfNetwork("usernet1") + require.NoError(t, err) + + pref := *prefp + + usernet1, err := scenario.Network("usernet1") + require.NoError(t, err) + + services, err := scenario.Services("usernet1") + require.NoError(t, err) + require.Len(t, services, 1) + + web := services[0] + webip := netip.MustParseAddr(web.GetIPInNetwork(usernet1)) + weburl := fmt.Sprintf("http://%s/etc/hostname", webip) + + sort.SliceStable(allClients, func(i, j int) bool { + return allClients[i].MustStatus().Self.ID < allClients[j].MustStatus().Self.ID + }) + + subRouter1 := allClients[0] + subRouter2 := allClients[1] + client := allClients[2] + + for _, r := range []TailscaleClient{subRouter1, subRouter2} { + _, _, err = r.Execute([]string{ + "tailscale", "set", "--advertise-routes=" + pref.String(), + }) + require.NoErrorf(t, err, "advertise route on %s", r.Hostname()) + } + + err = scenario.WaitForTailscaleSync() + requireNoErrSync(t, err) + + var nodes []*v1.Node + + assert.EventuallyWithT(t, func(c *assert.CollectT) { + nodes, err = headscale.ListNodes() + assert.NoError(c, err) + assert.Len(c, nodes, 4) + }, propagationTime, 200*time.Millisecond, "nodes registered") + + _, err = headscale.ApproveRoutes( + MustFindNode(subRouter1.Hostname(), nodes).GetId(), + []netip.Prefix{pref}, + ) + require.NoError(t, err) + + _, err = headscale.ApproveRoutes( + MustFindNode(subRouter2.Hostname(), nodes).GetId(), + []netip.Prefix{pref}, + ) + require.NoError(t, err) + + nodeID1 := types.NodeID(MustFindNode(subRouter1.Hostname(), nodes).GetId()) + nodeID2 := types.NodeID(MustFindNode(subRouter2.Hostname(), nodes).GetId()) + + // requirePrimary blocks until headscale reports want as the + // primary advertiser for pref. + requirePrimary := func(want types.NodeID, msg string) { + t.Helper() + assert.EventuallyWithT(t, func(c *assert.CollectT) { + pr, err := headscale.PrimaryRoutes() + assert.NoError(c, err) + assert.Equal(c, map[string]types.NodeID{ + pref.String(): want, + }, pr.PrimaryRoutes, msg) + }, propagationTime, 1*time.Second, msg) + } + + // requireTrafficWorks asserts the client can reach the webservice + // across the tailnet (i.e. via whichever router is primary). + requireTrafficWorks := func(msg string) { + t.Helper() + assert.EventuallyWithT(t, func(c *assert.CollectT) { + result, err := client.Curl(weburl) + assert.NoError(c, err) + assert.Len(c, result, 13) + }, propagationTime, 1*time.Second, msg) + } + + // requirePrimaryStable asserts primary == want for the entire + // window. Catches transient flaps and verifies anti-flap on + // prev-primary return. + requirePrimaryStable := func(want types.NodeID, window time.Duration, msg string) { + t.Helper() + require.Never(t, func() bool { + pr, err := headscale.PrimaryRoutes() + if err != nil { + return false + } + + owner, ok := pr.PrimaryRoutes[pref.String()] + + return !ok || owner != want + }, window, 1*time.Second, msg) + } + + // ============================================================ + // Phase 1: initial state — r1 (lowest NodeID) is primary. + // ============================================================ + t.Log("=== Phase 1: initial state — r1 should be primary. ===") + requirePrimary(nodeID1, "phase 1: r1 primary at start") + requireTrafficWorks("phase 1: client reaches webservice via r1") + + // ============================================================ + // Phase 2: r1 alone fails and returns. Failover to r2, traffic + // resumes; reconnect r1 and verify traffic still flows. We do + // not assert primary identity across the r1-return leg because + // docker bridge reconfiguration can transiently fail probes on + // r2 (real cable pulls do not have this side effect). + // ============================================================ + t.Log("=== Phase 2a: cable-pull r1, expect failover to r2. ===") + require.NoError(t, subRouter1.DisconnectFromNetwork(usernet1), + "phase 2a: docker disconnect r1") + requirePrimary(nodeID2, "phase 2a: r2 promoted after r1 down") + requireTrafficWorks("phase 2a: client reaches webservice via r2") + + t.Log("=== Phase 2b: reconnect r1, traffic should still flow. ===") + require.NoError(t, subRouter1.ReconnectToNetwork(usernet1), + "phase 2b: docker reconnect r1") + requireTrafficWorks("phase 2b: client still reaches webservice") + + // ============================================================ + // Phase 3: r2 alone fails and returns. Same caveats as phase 2 + // on identity assertions during the return leg. + // ============================================================ + t.Log("=== Phase 3a: cable-pull r2, traffic should fail over. ===") + require.NoError(t, subRouter2.DisconnectFromNetwork(usernet1), + "phase 3a: docker disconnect r2") + requireTrafficWorks("phase 3a: client reaches webservice via remaining router") + + t.Log("=== Phase 3b: reconnect r2, traffic should still flow. ===") + require.NoError(t, subRouter2.ReconnectToNetwork(usernet1), + "phase 3b: docker reconnect r2") + requireTrafficWorks("phase 3b: client still reaches webservice") + + // ============================================================ + // Phase 4: sequential dual failure — the issue #3203 bug. The + // flap target is r1 because under cable-pull both routers + // linger as IsOnline=true (half-open TCP), both go Unhealthy, + // and electPrimaryRoutes' all-unhealthy fallback selects the + // lowest NodeID regardless of who was prev primary. + // ============================================================ + t.Log("=== Phase 4a: cable-pull r1, expect failover to r2. ===") + require.NoError(t, subRouter1.DisconnectFromNetwork(usernet1), + "phase 4a: docker disconnect r1") + requirePrimary(nodeID2, "phase 4a: r2 promoted after r1 down") + + t.Log("=== Phase 4b: cable-pull r2, primary must NOT flap to offline r1. ===") + require.NoError(t, subRouter2.DisconnectFromNetwork(usernet1), + "phase 4b: docker disconnect r2") + requirePrimaryStable(nodeID2, flapWindow, + "phase 4b: primary must not flap to offline r1 (issue #3203)") + + t.Log("=== Phase 4c: reconnect r2, r2 should resume as primary. ===") + require.NoError(t, subRouter2.ReconnectToNetwork(usernet1), + "phase 4c: docker reconnect r2") + requirePrimary(nodeID2, "phase 4c: r2 primary after reconnect") + requireTrafficWorks("phase 4c: client reaches webservice via r2 after recovery") + + t.Log("=== Phase 4d: reconnect r1, traffic should still flow. ===") + require.NoError(t, subRouter1.ReconnectToNetwork(usernet1), + "phase 4d: docker reconnect r1") + requireTrafficWorks("phase 4d: client still reaches webservice") + + // ============================================================ + // Phase 5: simultaneous dual failure (whole-segment outage). + // prev going in is r2 — primary must not flap to offline r1. + // ============================================================ + t.Log("=== Phase 5a: cable-pull r1 and r2 simultaneously. ===") + require.NoError(t, subRouter1.DisconnectFromNetwork(usernet1), + "phase 5a: docker disconnect r1") + require.NoError(t, subRouter2.DisconnectFromNetwork(usernet1), + "phase 5a: docker disconnect r2") + requirePrimaryStable(nodeID2, flapWindow, + "phase 5a: primary must not flap to offline r1 (issue #3203)") + + t.Log("=== Phase 5b: reconnect both, r2 should remain primary. ===") + require.NoError(t, subRouter1.ReconnectToNetwork(usernet1), + "phase 5b: docker reconnect r1") + require.NoError(t, subRouter2.ReconnectToNetwork(usernet1), + "phase 5b: docker reconnect r2") + requirePrimary(nodeID2, "phase 5b: r2 primary after both reconnect") + requireTrafficWorks("phase 5b: client reaches webservice via r2") +} diff --git a/integration/tailscale.go b/integration/tailscale.go index 4c9a761b..9f5ea2f6 100644 --- a/integration/tailscale.go +++ b/integration/tailscale.go @@ -58,6 +58,8 @@ type TailscaleClient interface { ReadFile(path string) ([]byte, error) PacketFilter() ([]filter.Match, error) ConnectToNetwork(network *dockertest.Network) error + DisconnectFromNetwork(network *dockertest.Network) error + ReconnectToNetwork(network *dockertest.Network) error // FailingPeersAsString returns a formatted-ish multi-line-string of peers in the client // and a bool indicating if the clients online count and peer count is equal. diff --git a/integration/tsic/tsic.go b/integration/tsic/tsic.go index c0ea8174..4a558a04 100644 --- a/integration/tsic/tsic.go +++ b/integration/tsic/tsic.go @@ -807,6 +807,21 @@ func (t *TailscaleInContainer) Down() error { return nil } +// DisconnectFromNetwork detaches the container from network at the +// docker daemon level. The container's network interface for that +// network disappears and any in-flight TCP connection is left +// half-open at the peer — the same failure mode a real cable pull +// produces, which iptables-based simulations cannot reproduce. +func (t *TailscaleInContainer) DisconnectFromNetwork(network *dockertest.Network) error { + return dockertestutil.DisconnectContainerFromNetwork(t.pool, network, t.hostname) +} + +// ReconnectToNetwork is the inverse of DisconnectFromNetwork: it +// re-attaches the container to network so traffic can flow again. +func (t *TailscaleInContainer) ReconnectToNetwork(network *dockertest.Network) error { + return dockertestutil.ReconnectContainerToNetwork(t.pool, network, t.hostname) +} + // IPs returns the netip.Addr of the Tailscale instance. func (t *TailscaleInContainer) IPs() ([]netip.Addr, error) { if len(t.ips) != 0 {