diff --git a/network/bridge/bridge_linux.go b/network/bridge/bridge_linux.go index c4912b17..7cdd12db 100644 --- a/network/bridge/bridge_linux.go +++ b/network/bridge/bridge_linux.go @@ -10,6 +10,8 @@ import ( "github.com/projecteru2/core/log" "github.com/vishvananda/netlink" + "github.com/vishvananda/netlink/nl" + "golang.org/x/sys/unix" "github.com/cocoonstack/cocoon/config" "github.com/cocoonstack/cocoon/gc" @@ -112,36 +114,14 @@ func (b *Bridge) Add(ctx context.Context, vmID string, vmCfg *types.VMConfig, sp } } queues := network.ResolveQueues(spec.Queues, vmCfg.CPU) - if cErr := network.CreateTAP(name, queues); cErr != nil { + tapIndex, cErr := network.CreateTAP(name, queues) + if cErr != nil { return nil, cErr } added = append(added, spec.Index) - tap, lErr := netlink.LinkByName(name) - if lErr != nil { - return nil, fmt.Errorf("find tap %s: %w", name, lErr) - } - - if mErr := netlink.LinkSetMaster(tap, br); mErr != nil { - return nil, fmt.Errorf("add %s to %s: %w", name, b.bridgeDev, mErr) - } - - // Best-effort tuning, but leave a trace: a silently failed MTU sync - // surfaces later as a connectivity symptom with no trail back here. - if lErr := netlink.LinkSetLearning(tap, false); lErr != nil { - logger.Debugf(ctx, "disable learning on %s: %v", name, lErr) - } - if mtu := br.Attrs().MTU; mtu > 0 { - if mErr := netlink.LinkSetMTU(tap, mtu); mErr != nil { - logger.Debugf(ctx, "sync mtu %d to %s: %v", mtu, name, mErr) - } - } - if tErr := network.TuneTAP(tap); tErr != nil { - logger.Debugf(ctx, "tune tap %s: %v", name, tErr) - } - - if uErr := netlink.LinkSetUp(tap); uErr != nil { - return nil, fmt.Errorf("set %s up: %w", name, uErr) + if aErr := attachBridgeUp(tapIndex, b.bridgeIdx, br.Attrs().MTU); aErr != nil { + return nil, aErr } configs = append(configs, &types.NetworkConfig{ @@ -201,6 +181,26 @@ func CleanupTAPs(vmIDs []string) []string { return cleaned } +// attachBridgeUp enslaves a TAP to the bridge, applies MTU/txqlen/GRO tuning and brings it up in one RTM_SETLINK, paying the node-wide rtnl lock once. +func attachBridgeUp(tapIndex, bridgeIndex, mtu int) error { + req := nl.NewNetlinkRequest(unix.RTM_SETLINK, unix.NLM_F_ACK) + msg := nl.NewIfInfomsg(unix.AF_UNSPEC) + msg.Index = int32(tapIndex) //nolint:gosec // kernel-issued ifindex fits int32 + msg.Flags = unix.IFF_UP + msg.Change = unix.IFF_UP + req.AddData(msg) + req.AddData(nl.NewRtAttr(unix.IFLA_MASTER, nl.Uint32Attr(uint32(bridgeIndex)))) //nolint:gosec // kernel-issued ifindex fits uint32 + req.AddData(nl.NewRtAttr(unix.IFLA_TXQLEN, nl.Uint32Attr(network.TAPTxQueueLen))) + req.AddData(nl.NewRtAttr(unix.IFLA_GRO_MAX_SIZE, nl.Uint32Attr(network.GROMaxSize))) + if mtu > 0 { + req.AddData(nl.NewRtAttr(unix.IFLA_MTU, nl.Uint32Attr(uint32(mtu)))) //nolint:gosec // guarded > 0; kernel MTU fits uint32 + } + if _, err := req.Execute(unix.NETLINK_ROUTE, 0); err != nil { + return fmt.Errorf("attach tap %d to bridge %d: %w", tapIndex, bridgeIndex, err) + } + return nil +} + func tearDownTAPs(vmID string, indices []int, bestEffort bool) error { for _, i := range indices { name := tapName(vmID, i) diff --git a/network/cni/lifecycle_linux.go b/network/cni/lifecycle_linux.go index b986a11f..14994846 100644 --- a/network/cni/lifecycle_linux.go +++ b/network/cni/lifecycle_linux.go @@ -148,7 +148,7 @@ func tcRedirectInNS(ifName, tapName string, queues int, overrideMAC string) (str } } - if tapErr := network.CreateTAP(tapName, queues); tapErr != nil { + if _, tapErr := network.CreateTAP(tapName, queues); tapErr != nil { return "", tapErr } tapLink, err := netlink.LinkByName(tapName) diff --git a/network/tap_linux.go b/network/tap_linux.go index 45f4831a..733b856d 100644 --- a/network/tap_linux.go +++ b/network/tap_linux.go @@ -6,18 +6,19 @@ import ( "fmt" "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" ) const ( - // tapTxQueueLen absorbs traffic bursts (especially UDP) without dropping; the kernel default of 1000 is too small for VM workloads. - tapTxQueueLen = 10000 + // TAPTxQueueLen absorbs traffic bursts (especially UDP) without dropping; the kernel default of 1000 is too small for VM workloads. + TAPTxQueueLen = 10000 - // groMaxSize matches the maximum virtio-net segment size so the kernel aggregates inbound packets before CH reads them. - groMaxSize = 65536 + // GROMaxSize matches the maximum virtio-net segment size so the kernel aggregates inbound packets before CH reads them. + GROMaxSize = 65536 ) -// CreateTAP adds a multi-queue TAP sized for numQueues virtio-net queues, then closes the kernel fds (CH/QEMU reopen it by name). -func CreateTAP(name string, numQueues int) error { +// CreateTAP adds a multi-queue TAP sized for numQueues virtio-net queues and returns its index, then closes the kernel fds (CH/QEMU reopen it by name). +func CreateTAP(name string, numQueues int) (int, error) { // queue_pairs = num_queues / 2 (TX+RX pair); multi-queue needs >1 and must match the VMM's IFF_MULTI_QUEUE expectation. queuePairs := max(1, numQueues/2) //nolint:mnd flags := netlink.TUNTAP_VNET_HDR | netlink.TUNTAP_NO_PI @@ -33,18 +34,26 @@ func CreateTAP(name string, numQueues int) error { Flags: flags, } if err := netlink.LinkAdd(tap); err != nil { - return fmt.Errorf("add tap %s: %w", name, err) + return 0, fmt.Errorf("add tap %s: %w", name, err) + } + // netlink's post-TUNSETIFF index lookup drops its error; on a zero index drop persistence so the device dies with the fds instead of outliving the failure. + index := tap.Attrs().Index + if index == 0 { + _ = unix.IoctlSetInt(int(tap.Fds[0].Fd()), unix.TUNSETPERSIST, 0) } for _, fd := range tap.Fds { _ = fd.Close() } - return nil + if index == 0 { + return 0, fmt.Errorf("resolve index of tap %s", name) + } + return index, nil } // TuneTAP applies best-effort performance tuning to a TAP device. func TuneTAP(link netlink.Link) error { - if err := netlink.LinkSetTxQLen(link, tapTxQueueLen); err != nil { + if err := netlink.LinkSetTxQLen(link, TAPTxQueueLen); err != nil { return err } - return netlink.LinkSetGROMaxSize(link, groMaxSize) + return netlink.LinkSetGROMaxSize(link, GROMaxSize) }