DebTik/internal/aerotun/device.go

package aerotun

import (
	"encoding/binary"
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"
)

type VirtualDevice interface {
	Read(buf []byte) (int, error)
	Write(buf []byte) (int, error)
	Close() error
	Name() string
}

// CreateDevice creates either a native Linux TUN device, or falls back to a high-speed
// veth pair with AF_PACKET raw socket if running in an unprivileged container without /dev/net/tun.
func CreateDevice(name string, mtu int, localCIDR string) (VirtualDevice, error) {
	if _, err := net.InterfaceByName(name); err == nil {
		return nil, fmt.Errorf("interface %s already exists", name)
	}
	if mtu <= 0 {
		mtu = DefaultMTU
	}

	if _, err := os.Stat("/dev/net/tun"); err == nil {
		dev, err := createNativeTun(name, mtu, localCIDR)
		if err == nil {
			return dev, nil
		}
	}

	// Universal container / Proxmox LXC fallback:
	return createVethTun(name, mtu, localCIDR)
}

// ---------------------------------------------------------------------------
// 1. Native Linux TUN Implementation (/dev/net/tun)
// ---------------------------------------------------------------------------

type nativeTun struct {
	file *os.File
	name string
}

func createNativeTun(name string, mtu int, localCIDR string) (*nativeTun, error) {
	fd, err := syscall.Open("/dev/net/tun", syscall.O_RDWR|syscall.O_NONBLOCK|syscall.O_CLOEXEC, 0)
	if err != nil {
		return nil, err
	}

	type ifreq struct {
		name  [16]byte
		flags uint16
		_pad  [22]byte
	}

	var ifr ifreq
	copy(ifr.name[:], name)
	// IFF_TUN = 0x0001, IFF_NO_PI = 0x1000
	ifr.flags = 0x0001 | 0x1000 | 0x8000 // IFF_TUN_EXCL: never attach an existing device

	_, _, errno := syscall.Syscall(
		syscall.SYS_IOCTL,
		uintptr(fd),
		uintptr(0x400454ca), // TUNSETIFF
		uintptr(unsafe.Pointer(&ifr)),
	)
	if errno != 0 {
		_ = syscall.Close(fd)
		return nil, errno
	}

	// Configure MTU and IP
	if _, err := runIP("link", "set", name, "mtu", fmt.Sprintf("%d", mtu)); err != nil {
		_ = syscall.Close(fd)
		return nil, err
	}
	if _, err := runIP("link", "set", name, "up"); err != nil {
		_ = syscall.Close(fd)
		return nil, err
	}

	if localCIDR != "" {
		if _, err := runIP("addr", "add", localCIDR, "dev", name); err != nil {
			_ = syscall.Close(fd)
			return nil, err
		}
	}

	return &nativeTun{
		file: os.NewFile(uintptr(fd), name),
		name: name,
	}, nil
}

func (t *nativeTun) Read(buf []byte) (int, error) {
	return t.file.Read(buf)
}

func (t *nativeTun) Write(buf []byte) (int, error) {
	return t.file.Write(buf)
}

func (t *nativeTun) Close() error {
	// Non-persistent TUN interfaces disappear when the descriptor closes.
	return t.file.Close()
}

func (t *nativeTun) Name() string {
	return t.name
}

// ---------------------------------------------------------------------------
// 2. High-Performance Container Fallback (VETH + AF_PACKET)
// ---------------------------------------------------------------------------

type vethTun struct {
	mu        sync.RWMutex
	closed    bool
	name      string
	epName    string
	sockFd    int
	ifIndex   int
	hwAddr    [6]byte
	epHwAddr  [6]byte
	packetBuf []byte
	writeBuf  []byte
}

func createVethTun(name string, mtu int, localCIDR string) (*vethTun, error) {
	epName := fmt.Sprintf("%s_ep", name)
	if len(epName) > 15 {
		epName = epName[:15]
	}

	// Existing interfaces may belong to another tunnel: creation must fail safely.

	// 1. Create veth pair
	out, err := exec.Command("ip", "link", "add", name, "type", "veth", "peer", "name", epName).CombinedOutput()
	if err != nil {
		return nil, fmt.Errorf("failed to create veth pair %s: %s: %w", name, string(out), err)
	}

	// 2. Set Point-to-Point / ARP off behavior
	if _, err := runIP("link", "set", name, "arp", "off"); err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, err
	}
	if _, err := runIP("link", "set", epName, "arp", "off"); err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, err
	}
	if _, err := runIP("link", "set", name, "mtu", fmt.Sprintf("%d", mtu)); err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, err
	}
	if _, err := runIP("link", "set", epName, "mtu", fmt.Sprintf("%d", mtu)); err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, err
	}
	if _, err := runIP("link", "set", name, "up"); err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, err
	}
	if _, err := runIP("link", "set", epName, "up"); err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, err
	}

	// 3. Assign IP to main device
	if localCIDR != "" {
		if _, err := runIP("addr", "add", localCIDR, "dev", name); err != nil {
			_ = exec.Command("ip", "link", "delete", name).Run()
			return nil, err
		}
	}

	// Raw sockets do not complete CHECKSUM_PARTIAL or segment GSO packets.
	// Disable these features on both ends before any traffic can be forwarded.
	for _, iface := range []string{name, epName} {
		if err := disableOffloads(iface); err != nil {
			_ = exec.Command("ip", "link", "delete", name).Run()
			return nil, err
		}
	}

	// 4. Get endpoint interface index
	epIface, err := net.InterfaceByName(epName)
	if err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, fmt.Errorf("failed to lookup %s: %w", epName, err)
	}

	// 5. Open AF_PACKET raw socket bound to endpoint
	// htons(ETH_P_ALL) = 0x0300
	sockFd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW|syscall.SOCK_CLOEXEC, int(htons(syscall.ETH_P_ALL)))
	if err != nil {
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, fmt.Errorf("failed to open raw AF_PACKET socket: %w", err)
	}

	// PACKET_IGNORE_OUTGOING avoids injecting locally sent frames back into VPN.
	_ = syscall.SetsockoptInt(sockFd, 263, 23, 1)
	// A receive timeout lets Close reliably stop a blocked raw-socket reader.
	_ = syscall.SetsockoptTimeval(sockFd, syscall.SOL_SOCKET, syscall.SO_RCVTIMEO, &syscall.Timeval{Sec: 1})
	addr := syscall.SockaddrLinklayer{
		Protocol: htons(syscall.ETH_P_ALL),
		Ifindex:  epIface.Index,
	}
	if err := syscall.Bind(sockFd, &addr); err != nil {
		_ = syscall.Close(sockFd)
		_ = exec.Command("ip", "link", "delete", name).Run()
		return nil, fmt.Errorf("failed to bind raw socket to %s: %w", epName, err)
	}

	dev := &vethTun{
		packetBuf: make([]byte, mtu+14), writeBuf: make([]byte, mtu+14),
		name:    name,
		epName:  epName,
		sockFd:  sockFd,
		ifIndex: epIface.Index,
	}

	if len(epIface.HardwareAddr) >= 6 {
		copy(dev.epHwAddr[:], epIface.HardwareAddr[:6])
	}
	if mainIface, err := net.InterfaceByName(name); err == nil && len(mainIface.HardwareAddr) >= 6 {
		copy(dev.hwAddr[:], mainIface.HardwareAddr[:6])
	}

	return dev, nil
}

func (v *vethTun) Read(buf []byte) (int, error) {
	for {
		v.mu.RLock()
		if v.closed {
			v.mu.RUnlock()
			return 0, net.ErrClosed
		}
		n, addr, err := syscall.Recvfrom(v.sockFd, v.packetBuf[:], 0)
		v.mu.RUnlock()
		if errors.Is(err, syscall.EAGAIN) || errors.Is(err, syscall.EINTR) {
			continue
		}
		if err != nil {
			return 0, err
		}
		if ll, ok := addr.(*syscall.SockaddrLinklayer); ok && ll.Pkttype == syscall.PACKET_OUTGOING {
			continue
		}
		if n <= 14 {
			continue // Too small for ethernet frame
		}

		// Ethernet frame header: [Dst MAC (6)][Src MAC (6)][EtherType (2)]
		etherType := binary.BigEndian.Uint16(v.packetBuf[12:14])
		// Check for IPv4 (0x0800) or IPv6 (0x86DD)
		if etherType != 0x0800 && etherType != 0x86DD {
			continue // Ignore ARP, STP, etc.
		}

		ipPayload := v.packetBuf[14:n]
		copied := copy(buf, ipPayload)
		return copied, nil
	}
}

func (v *vethTun) Write(buf []byte) (int, error) {
	if len(buf) == 0 {
		return 0, nil
	}

	// Build raw Ethernet frame to inject into endpoint
	frameLen := 14 + len(buf)
	if frameLen > len(v.writeBuf) {
		return 0, fmt.Errorf("packet exceeds configured veth MTU")
	}
	frame := v.writeBuf[:frameLen] // Separate reusable RX/TX storage; one writer per device.

	// Dst MAC: main interface, Src MAC: endpoint
	copy(frame[0:6], v.hwAddr[:])
	copy(frame[6:12], v.epHwAddr[:])

	// Detect IP version
	var etherType uint16 = 0x0800 // IPv4 default
	if (buf[0] >> 4) == 6 {
		etherType = 0x86DD // IPv6
	}
	binary.BigEndian.PutUint16(frame[12:14], etherType)
	copy(frame[14:], buf)

	addr := syscall.SockaddrLinklayer{
		Protocol: htons(etherType),
		Ifindex:  v.ifIndex,
		Halen:    6,
	}
	copy(addr.Addr[:], v.hwAddr[:])

	v.mu.RLock()
	defer v.mu.RUnlock()
	if v.closed {
		return 0, net.ErrClosed
	}
	err := syscall.Sendto(v.sockFd, frame, 0, &addr)
	if err != nil {
		return 0, err
	}
	return len(buf), nil
}

func (v *vethTun) Close() error {
	v.mu.Lock()
	if v.closed {
		v.mu.Unlock()
		return nil
	}
	v.closed = true
	_ = syscall.Close(v.sockFd)
	v.mu.Unlock()
	_ = exec.Command("ip", "link", "delete", v.name).Run()
	return nil
}

func (v *vethTun) Name() string {
	return v.name
}

func htons(n uint16) uint16 {
	return (n<<8)&0xff00 | (n>>8)&0x00ff
}

// Linux ethtool ioctls avoid requiring an external ethtool executable in LXC.
func disableOffloads(name string) error {
	fd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
	if err != nil {
		return err
	}
	defer unix.Close(fd)
	for _, cmd := range []uint32{0x1f, 0x24, 0x2c, 0x17} { // TSO, GSO, GRO, TX checksum
		value := struct{ Command, Data uint32 }{Command: cmd}
		ifr := struct {
			Name [16]byte
			Data unsafe.Pointer
			Pad  [16]byte
		}{Data: unsafe.Pointer(&value)}
		copy(ifr.Name[:], name)
		_, _, errno := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), 0x8946, uintptr(unsafe.Pointer(&ifr)))
		var err error
		if errno != 0 {
			err = errno
		}
		runtime.KeepAlive(&value)
		// Unsupported offloads are already unavailable on this device.
		if err != nil && !errors.Is(err, unix.EOPNOTSUPP) {
			return fmt.Errorf("disable offload %#x on %s: %w", cmd, name, err)
		}
	}
	return nil
}