perf(bridge): attach and configure a TAP in one netlink message

network/bridge/bridge_linux.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/projecteru2/core/log"
 	"github.com/vishvananda/netlink"
+	"github.com/vishvananda/netlink/nl"
+	"golang.org/x/sys/unix"
 
 	"github.com/cocoonstack/cocoon/config"
 	"github.com/cocoonstack/cocoon/gc"
@@ -112,36 +114,14 @@ func (b *Bridge) Add(ctx context.Context, vmID string, vmCfg *types.VMConfig, sp
 			}
 		}
 		queues := network.ResolveQueues(spec.Queues, vmCfg.CPU)
-		if cErr := network.CreateTAP(name, queues); cErr != nil {
+		tapIndex, cErr := network.CreateTAP(name, queues)
+		if cErr != nil {
 			return nil, cErr
 		}
 		added = append(added, spec.Index)
 
-		tap, lErr := netlink.LinkByName(name)
-		if lErr != nil {
-			return nil, fmt.Errorf("find tap %s: %w", name, lErr)
-		}
-
-		if mErr := netlink.LinkSetMaster(tap, br); mErr != nil {
-			return nil, fmt.Errorf("add %s to %s: %w", name, b.bridgeDev, mErr)
-		}
-
-		// Best-effort tuning, but leave a trace: a silently failed MTU sync
-		// surfaces later as a connectivity symptom with no trail back here.
-		if lErr := netlink.LinkSetLearning(tap, false); lErr != nil {
-			logger.Debugf(ctx, "disable learning on %s: %v", name, lErr)
-		}
-		if mtu := br.Attrs().MTU; mtu > 0 {
-			if mErr := netlink.LinkSetMTU(tap, mtu); mErr != nil {
-				logger.Debugf(ctx, "sync mtu %d to %s: %v", mtu, name, mErr)
-			}
-		}
-		if tErr := network.TuneTAP(tap); tErr != nil {
-			logger.Debugf(ctx, "tune tap %s: %v", name, tErr)
-		}
-
-		if uErr := netlink.LinkSetUp(tap); uErr != nil {
-			return nil, fmt.Errorf("set %s up: %w", name, uErr)
+		if aErr := attachBridgeUp(tapIndex, b.bridgeIdx, br.Attrs().MTU); aErr != nil {
+			return nil, aErr
 		}
 
 		configs = append(configs, &types.NetworkConfig{
@@ -201,6 +181,26 @@ func CleanupTAPs(vmIDs []string) []string {
 	return cleaned
 }
 
+// attachBridgeUp enslaves a TAP to the bridge, applies MTU/txqlen/GRO tuning and brings it up in one RTM_SETLINK, paying the node-wide rtnl lock once.
+func attachBridgeUp(tapIndex, bridgeIndex, mtu int) error {
+	req := nl.NewNetlinkRequest(unix.RTM_SETLINK, unix.NLM_F_ACK)
+	msg := nl.NewIfInfomsg(unix.AF_UNSPEC)
+	msg.Index = int32(tapIndex) //nolint:gosec // kernel-issued ifindex fits int32
+	msg.Flags = unix.IFF_UP
+	msg.Change = unix.IFF_UP
+	req.AddData(msg)
+	req.AddData(nl.NewRtAttr(unix.IFLA_MASTER, nl.Uint32Attr(uint32(bridgeIndex)))) //nolint:gosec // kernel-issued ifindex fits uint32
+	req.AddData(nl.NewRtAttr(unix.IFLA_TXQLEN, nl.Uint32Attr(network.TAPTxQueueLen)))
+	req.AddData(nl.NewRtAttr(unix.IFLA_GRO_MAX_SIZE, nl.Uint32Attr(network.GROMaxSize)))
+	if mtu > 0 {
+		req.AddData(nl.NewRtAttr(unix.IFLA_MTU, nl.Uint32Attr(uint32(mtu)))) //nolint:gosec // guarded > 0; kernel MTU fits uint32
+	}
+	if _, err := req.Execute(unix.NETLINK_ROUTE, 0); err != nil {
+		return fmt.Errorf("attach tap %d to bridge %d: %w", tapIndex, bridgeIndex, err)
+	}
+	return nil
+}
+
 func tearDownTAPs(vmID string, indices []int, bestEffort bool) error {
 	for _, i := range indices {
 		name := tapName(vmID, i)

network/cni/lifecycle_linux.go

@@ -148,7 +148,7 @@ func tcRedirectInNS(ifName, tapName string, queues int, overrideMAC string) (str
 		}
 	}
 
-	if tapErr := network.CreateTAP(tapName, queues); tapErr != nil {
+	if _, tapErr := network.CreateTAP(tapName, queues); tapErr != nil {
 		return "", tapErr
 	}
 	tapLink, err := netlink.LinkByName(tapName)

network/tap_linux.go

@@ -6,18 +6,19 @@ import (
 	"fmt"
 
 	"github.com/vishvananda/netlink"
+	"golang.org/x/sys/unix"
 )
 
 const (
-	// tapTxQueueLen absorbs traffic bursts (especially UDP) without dropping; the kernel default of 1000 is too small for VM workloads.
-	tapTxQueueLen = 10000
+	// TAPTxQueueLen absorbs traffic bursts (especially UDP) without dropping; the kernel default of 1000 is too small for VM workloads.
+	TAPTxQueueLen = 10000
 
-	// groMaxSize matches the maximum virtio-net segment size so the kernel aggregates inbound packets before CH reads them.
-	groMaxSize = 65536
+	// GROMaxSize matches the maximum virtio-net segment size so the kernel aggregates inbound packets before CH reads them.
+	GROMaxSize = 65536
 )
 
-// CreateTAP adds a multi-queue TAP sized for numQueues virtio-net queues, then closes the kernel fds (CH/QEMU reopen it by name).
-func CreateTAP(name string, numQueues int) error {
+// CreateTAP adds a multi-queue TAP sized for numQueues virtio-net queues and returns its index, then closes the kernel fds (CH/QEMU reopen it by name).
+func CreateTAP(name string, numQueues int) (int, error) {
 	// queue_pairs = num_queues / 2 (TX+RX pair); multi-queue needs >1 and must match the VMM's IFF_MULTI_QUEUE expectation.
 	queuePairs := max(1, numQueues/2) //nolint:mnd
 	flags := netlink.TUNTAP_VNET_HDR | netlink.TUNTAP_NO_PI
@@ -33,18 +34,26 @@ func CreateTAP(name string, numQueues int) error {
 		Flags:     flags,
 	}
 	if err := netlink.LinkAdd(tap); err != nil {
-		return fmt.Errorf("add tap %s: %w", name, err)
+		return 0, fmt.Errorf("add tap %s: %w", name, err)
+	}
+	// netlink's post-TUNSETIFF index lookup drops its error; on a zero index drop persistence so the device dies with the fds instead of outliving the failure.
+	index := tap.Attrs().Index
+	if index == 0 {
+		_ = unix.IoctlSetInt(int(tap.Fds[0].Fd()), unix.TUNSETPERSIST, 0)
 	}
 	for _, fd := range tap.Fds {
 		_ = fd.Close()
 	}
-	return nil
+	if index == 0 {
+		return 0, fmt.Errorf("resolve index of tap %s", name)
+	}
+	return index, nil
 }
 
 // TuneTAP applies best-effort performance tuning to a TAP device.
 func TuneTAP(link netlink.Link) error {
-	if err := netlink.LinkSetTxQLen(link, tapTxQueueLen); err != nil {
+	if err := netlink.LinkSetTxQLen(link, TAPTxQueueLen); err != nil {
 		return err
 	}
-	return netlink.LinkSetGROMaxSize(link, groMaxSize)
+	return netlink.LinkSetGROMaxSize(link, GROMaxSize)
 }