eoip.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624
  1. package vpn
  2. import (
  3. "crypto/rand"
  4. "encoding/binary"
  5. "encoding/json"
  6. "fmt"
  7. "log"
  8. "net"
  9. "os"
  10. "os/exec"
  11. "path/filepath"
  12. "strconv"
  13. "strings"
  14. "sync"
  15. "sync/atomic"
  16. "syscall"
  17. "time"
  18. )
  19. // EoIPInstance represents a MikroTik Ethernet-over-IP tunnel instance
  20. type EoIPInstance struct {
  21. ID string `json:"id"`
  22. Name string `json:"name"` // e.g. "Goldix" or "eoip-tunnel1"
  23. TunnelID int `json:"tunnel_id"` // 1 - 65535
  24. RemoteAddress string `json:"remote_address"` // Remote MikroTik IPv4
  25. LocalAddress string `json:"local_address"` // Local router IPv4 (optional)
  26. IPAddress string `json:"ip_address,omitempty"` // Overlay IP/CIDR (e.g. "172.16.33.254/30")
  27. MACAddress string `json:"mac_address"` // e.g. "02:64:83:6b:46:44"
  28. MTU int `json:"mtu"` // default 1500
  29. ClampMSS bool `json:"clamp_mss"` // Clamp TCP MSS for PMTUD
  30. Comment string `json:"comment,omitempty"`
  31. Disabled bool `json:"disabled"`
  32. Running bool `json:"running"` // MikroTik 'R' flag
  33. RxPackets uint64 `json:"rx_packets"`
  34. TxPackets uint64 `json:"tx_packets"`
  35. RxBytes uint64 `json:"rx_bytes"`
  36. TxBytes uint64 `json:"tx_bytes"`
  37. RxHuman string `json:"rx_human,omitempty"`
  38. TxHuman string `json:"tx_human,omitempty"`
  39. LastError string `json:"last_error,omitempty"`
  40. }
  41. type EoIPManager struct {
  42. mu sync.RWMutex
  43. storePath string
  44. tunnels map[string]*EoIPInstance
  45. workers map[string]*eoipTunnelWorker
  46. }
  47. func NewEoIPManager() *EoIPManager {
  48. mgr := &EoIPManager{
  49. storePath: "/etc/proxmox-router/vpn/eoip.json",
  50. tunnels: make(map[string]*EoIPInstance),
  51. workers: make(map[string]*eoipTunnelWorker),
  52. }
  53. mgr.load()
  54. mgr.syncAll()
  55. return mgr
  56. }
  57. func (m *EoIPManager) load() {
  58. if data, err := os.ReadFile(m.storePath); err == nil {
  59. var list []*EoIPInstance
  60. if err := json.Unmarshal(data, &list); err == nil {
  61. for _, t := range list {
  62. m.tunnels[t.ID] = t
  63. }
  64. }
  65. }
  66. }
  67. func (m *EoIPManager) save() error {
  68. _ = os.MkdirAll(filepath.Dir(m.storePath), 0755)
  69. var list []*EoIPInstance
  70. for _, t := range m.tunnels {
  71. list = append(list, t)
  72. }
  73. data, err := json.MarshalIndent(list, "", " ")
  74. if err != nil {
  75. return err
  76. }
  77. return os.WriteFile(m.storePath, data, 0644)
  78. }
  79. func (m *EoIPManager) List() []*EoIPInstance {
  80. m.mu.Lock()
  81. defer m.mu.Unlock()
  82. list := make([]*EoIPInstance, 0, len(m.tunnels))
  83. for _, t := range m.tunnels {
  84. m.updateStats(t)
  85. list = append(list, t)
  86. }
  87. return list
  88. }
  89. func (m *EoIPManager) Get(id string) (*EoIPInstance, bool) {
  90. m.mu.Lock()
  91. defer m.mu.Unlock()
  92. t, ok := m.tunnels[id]
  93. if ok {
  94. m.updateStats(t)
  95. }
  96. return t, ok
  97. }
  98. func (m *EoIPManager) Save(inst *EoIPInstance) error {
  99. m.mu.Lock()
  100. defer m.mu.Unlock()
  101. if inst.Name == "" {
  102. inst.Name = fmt.Sprintf("eoip-tunnel%d", inst.TunnelID)
  103. }
  104. if inst.TunnelID <= 0 || inst.TunnelID > 65535 {
  105. return fmt.Errorf("tunnel_id must be between 1 and 65535")
  106. }
  107. if inst.RemoteAddress == "" {
  108. return fmt.Errorf("remote_address is required")
  109. }
  110. if inst.MTU <= 0 {
  111. inst.MTU = 1500
  112. }
  113. if inst.MACAddress == "" {
  114. inst.MACAddress = generateMAC()
  115. }
  116. if inst.ID == "" {
  117. inst.ID = fmt.Sprintf("eoip-%d", time.Now().UnixNano())
  118. }
  119. // Smart detection: if user put CIDR into LocalAddress by mistake, move it to IPAddress
  120. if strings.Contains(inst.LocalAddress, "/") && inst.IPAddress == "" {
  121. inst.IPAddress = inst.LocalAddress
  122. inst.LocalAddress = ""
  123. }
  124. // Stop previous worker if exists
  125. if w, exists := m.workers[inst.ID]; exists {
  126. w.stop()
  127. delete(m.workers, inst.ID)
  128. }
  129. m.tunnels[inst.ID] = inst
  130. _ = m.save()
  131. if !inst.Disabled {
  132. worker, err := newEoIPTunnelWorker(inst)
  133. if err != nil {
  134. inst.LastError = err.Error()
  135. inst.Running = false
  136. return err
  137. }
  138. m.workers[inst.ID] = worker
  139. go worker.start()
  140. } else {
  141. inst.Running = false
  142. }
  143. return nil
  144. }
  145. func (m *EoIPManager) SetState(id string, disabled bool) error {
  146. m.mu.Lock()
  147. defer m.mu.Unlock()
  148. t, ok := m.tunnels[id]
  149. if !ok {
  150. return fmt.Errorf("tunnel not found")
  151. }
  152. t.Disabled = disabled
  153. _ = m.save()
  154. if w, exists := m.workers[id]; exists {
  155. w.stop()
  156. delete(m.workers, id)
  157. }
  158. if disabled {
  159. t.Running = false
  160. return nil
  161. }
  162. worker, err := newEoIPTunnelWorker(t)
  163. if err != nil {
  164. t.LastError = err.Error()
  165. t.Running = false
  166. return err
  167. }
  168. m.workers[id] = worker
  169. go worker.start()
  170. return nil
  171. }
  172. func (m *EoIPManager) Delete(id string) error {
  173. m.mu.Lock()
  174. defer m.mu.Unlock()
  175. if _, ok := m.tunnels[id]; !ok {
  176. return fmt.Errorf("tunnel not found")
  177. }
  178. if w, exists := m.workers[id]; exists {
  179. w.stop()
  180. delete(m.workers, id)
  181. }
  182. delete(m.tunnels, id)
  183. return m.save()
  184. }
  185. func (m *EoIPManager) syncAll() {
  186. m.mu.Lock()
  187. defer m.mu.Unlock()
  188. for id, t := range m.tunnels {
  189. if !t.Disabled {
  190. if w, exists := m.workers[id]; exists {
  191. w.stop()
  192. delete(m.workers, id)
  193. }
  194. worker, err := newEoIPTunnelWorker(t)
  195. if err != nil {
  196. t.LastError = err.Error()
  197. t.Running = false
  198. continue
  199. }
  200. m.workers[id] = worker
  201. go worker.start()
  202. } else {
  203. t.Running = false
  204. }
  205. }
  206. }
  207. func (m *EoIPManager) updateStats(t *EoIPInstance) {
  208. if w, exists := m.workers[t.ID]; exists {
  209. t.Running = w.isRunning()
  210. t.RxBytes = atomic.LoadUint64(&w.rxBytes)
  211. t.RxPackets = atomic.LoadUint64(&w.rxPackets)
  212. t.TxBytes = atomic.LoadUint64(&w.txBytes)
  213. t.TxPackets = atomic.LoadUint64(&w.txPackets)
  214. } else {
  215. t.Running = false
  216. }
  217. t.RxHuman = formatBytes(t.RxBytes)
  218. t.TxHuman = formatBytes(t.TxBytes)
  219. // Detect any bound IPv4 if not explicitly saved in JSON
  220. if t.IPAddress == "" {
  221. if ifi, err := net.InterfaceByName(t.Name); err == nil {
  222. if addrs, err := ifi.Addrs(); err == nil {
  223. for _, a := range addrs {
  224. if ipNet, ok := a.(*net.IPNet); ok && ipNet.IP.To4() != nil {
  225. t.IPAddress = ipNet.String()
  226. break
  227. }
  228. }
  229. }
  230. }
  231. }
  232. }
  233. // -------------------------------------------------------------
  234. // Native MikroTik EoIP Engine Worker
  235. // -------------------------------------------------------------
  236. type eoipTunnelWorker struct {
  237. inst *EoIPInstance
  238. peerIfName string
  239. peerIfIndex int
  240. greFd int
  241. pktFd int
  242. stopChan chan struct{}
  243. lastRx int64 // unix nano
  244. rxBytes uint64
  245. rxPackets uint64
  246. txBytes uint64
  247. txPackets uint64
  248. remoteAddrInet syscall.SockaddrInet4
  249. }
  250. func newEoIPTunnelWorker(inst *EoIPInstance) (*eoipTunnelWorker, error) {
  251. // Parse Remote IPv4
  252. remoteIP := net.ParseIP(inst.RemoteAddress).To4()
  253. if remoteIP == nil {
  254. return nil, fmt.Errorf("invalid remote address: %s", inst.RemoteAddress)
  255. }
  256. // Internal peer interface name (max 15 chars for Linux ifnames)
  257. peerName := fmt.Sprintf("ep_%s", inst.Name)
  258. if len(peerName) > 15 {
  259. peerName = peerName[:15]
  260. }
  261. // 1. Clean up existing links if present
  262. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  263. _ = exec.Command("ip", "link", "del", peerName).Run()
  264. // 2. Create veth pair: inst.Name <--> peerName
  265. cmdAdd := exec.Command("ip", "link", "add", inst.Name, "type", "veth", "peer", "name", peerName)
  266. if out, err := cmdAdd.CombinedOutput(); err != nil {
  267. return nil, fmt.Errorf("failed to create veth pair: %s - %w", string(out), err)
  268. }
  269. // 3. Set MAC address and MTU
  270. if inst.MACAddress != "" {
  271. _ = exec.Command("ip", "link", "set", inst.Name, "address", inst.MACAddress).Run()
  272. }
  273. mtuStr := strconv.Itoa(inst.MTU)
  274. _ = exec.Command("ip", "link", "set", inst.Name, "mtu", mtuStr).Run()
  275. _ = exec.Command("ip", "link", "set", peerName, "mtu", mtuStr).Run()
  276. // 4. Bring both interfaces UP
  277. _ = exec.Command("ip", "link", "set", inst.Name, "up").Run()
  278. _ = exec.Command("ip", "link", "set", peerName, "up").Run()
  279. // 4b. Assign Overlay IP address if specified
  280. if inst.IPAddress != "" {
  281. _ = exec.Command("ip", "addr", "add", inst.IPAddress, "dev", inst.Name).Run()
  282. }
  283. // 5. TCP MSS Clamping
  284. if inst.ClampMSS {
  285. _ = exec.Command("iptables", "-t", "mangle", "-A", "FORWARD", "-o", inst.Name, "-p", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", "--clamp-mss-to-pmtu").Run()
  286. }
  287. // 6. Find interface index of peerName
  288. peerIf, err := net.InterfaceByName(peerName)
  289. if err != nil {
  290. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  291. return nil, fmt.Errorf("cannot find peer interface %s: %w", peerName, err)
  292. }
  293. // 7. Open AF_PACKET raw socket on peer interface
  294. pktFd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(htons(syscall.ETH_P_ALL)))
  295. if err != nil {
  296. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  297. return nil, fmt.Errorf("cannot create AF_PACKET socket: %w", err)
  298. }
  299. pktAddr := syscall.SockaddrLinklayer{
  300. Protocol: htons(syscall.ETH_P_ALL),
  301. Ifindex: peerIf.Index,
  302. }
  303. if err := syscall.Bind(pktFd, &pktAddr); err != nil {
  304. syscall.Close(pktFd)
  305. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  306. return nil, fmt.Errorf("cannot bind AF_PACKET socket to %s: %w", peerName, err)
  307. }
  308. // 8. Open raw GRE socket (IPPROTO_GRE = 47)
  309. greFd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_GRE)
  310. if err != nil {
  311. syscall.Close(pktFd)
  312. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  313. return nil, fmt.Errorf("cannot create raw GRE socket: %w", err)
  314. }
  315. // Bind GRE socket to LocalAddress if specified
  316. if inst.LocalAddress != "" {
  317. localIP := net.ParseIP(inst.LocalAddress).To4()
  318. if localIP != nil {
  319. localAddr := &syscall.SockaddrInet4{
  320. Addr: [4]byte{localIP[0], localIP[1], localIP[2], localIP[3]},
  321. }
  322. _ = syscall.Bind(greFd, localAddr)
  323. }
  324. }
  325. w := &eoipTunnelWorker{
  326. inst: inst,
  327. peerIfName: peerName,
  328. peerIfIndex: peerIf.Index,
  329. greFd: greFd,
  330. pktFd: pktFd,
  331. stopChan: make(chan struct{}),
  332. remoteAddrInet: syscall.SockaddrInet4{
  333. Addr: [4]byte{remoteIP[0], remoteIP[1], remoteIP[2], remoteIP[3]},
  334. },
  335. rxBytes: inst.RxBytes,
  336. rxPackets: inst.RxPackets,
  337. txBytes: inst.TxBytes,
  338. txPackets: inst.TxPackets,
  339. }
  340. return w, nil
  341. }
  342. func (w *eoipTunnelWorker) start() {
  343. log.Printf("[EoIP] Started tunnel '%s' (ID: %d, Remote: %s)", w.inst.Name, w.inst.TunnelID, w.inst.RemoteAddress)
  344. // Goroutine 1: Forward packets from Local veth -> Remote MikroTik via GRE
  345. go w.txLoop()
  346. // Goroutine 2: Receive packets from Remote MikroTik via GRE -> Inject to Local veth
  347. go w.rxLoop()
  348. // Goroutine 3: Periodic MikroTik EoIP Keepalive
  349. go w.keepaliveLoop()
  350. }
  351. func (w *eoipTunnelWorker) isRunning() bool {
  352. last := atomic.LoadInt64(&w.lastRx)
  353. if last == 0 {
  354. return false
  355. }
  356. return time.Since(time.Unix(0, last)) < 15*time.Second
  357. }
  358. func (w *eoipTunnelWorker) sendKeepalive() {
  359. // MikroTik EoIP Keepalive packet:
  360. // GRE Flags: 0x2001
  361. // Protocol: 0x6400
  362. // Length: 0x0000
  363. // Tunnel ID: Little-Endian
  364. tid := uint16(w.inst.TunnelID)
  365. pkt := []byte{
  366. 0x20, 0x01,
  367. 0x64, 0x00,
  368. 0x00, 0x00,
  369. byte(tid & 0xff), byte(tid >> 8),
  370. }
  371. _ = syscall.Sendto(w.greFd, pkt, 0, &w.remoteAddrInet)
  372. }
  373. func (w *eoipTunnelWorker) keepaliveLoop() {
  374. ticker := time.NewTicker(5 * time.Second)
  375. defer ticker.Stop()
  376. // Send immediate initial keepalive
  377. w.sendKeepalive()
  378. for {
  379. select {
  380. case <-w.stopChan:
  381. return
  382. case <-ticker.C:
  383. w.sendKeepalive()
  384. }
  385. }
  386. }
  387. func (w *eoipTunnelWorker) txLoop() {
  388. buf := make([]byte, 2048)
  389. tid := uint16(w.inst.TunnelID)
  390. for {
  391. select {
  392. case <-w.stopChan:
  393. return
  394. default:
  395. }
  396. n, from, err := syscall.Recvfrom(w.pktFd, buf, 0)
  397. if err != nil {
  398. select {
  399. case <-w.stopChan:
  400. return
  401. default:
  402. time.Sleep(10 * time.Millisecond)
  403. continue
  404. }
  405. }
  406. // Filter out outgoing packets to prevent reflection
  407. if fromLink, ok := from.(*syscall.SockaddrLinklayer); ok {
  408. if fromLink.Pkttype == syscall.PACKET_OUTGOING {
  409. continue
  410. }
  411. }
  412. // Must have at least minimum Ethernet header length (14 bytes)
  413. if n < 14 {
  414. continue
  415. }
  416. // Construct MikroTik EoIP Header (8 bytes) + Ethernet Frame
  417. // [0..1] = 0x2001 (Flags)
  418. // [2..3] = 0x6400 (Protocol)
  419. // [4..5] = payload length (Big-Endian)
  420. // [6..7] = tunnel ID (Little-Endian)
  421. grePkt := make([]byte, 8+n)
  422. grePkt[0] = 0x20
  423. grePkt[1] = 0x01
  424. grePkt[2] = 0x64
  425. grePkt[3] = 0x00
  426. binary.BigEndian.PutUint16(grePkt[4:6], uint16(n))
  427. binary.LittleEndian.PutUint16(grePkt[6:8], tid)
  428. copy(grePkt[8:], buf[:n])
  429. // Send over raw GRE socket
  430. if err := syscall.Sendto(w.greFd, grePkt, 0, &w.remoteAddrInet); err == nil {
  431. atomic.AddUint64(&w.txPackets, 1)
  432. atomic.AddUint64(&w.txBytes, uint64(n))
  433. }
  434. }
  435. }
  436. func (w *eoipTunnelWorker) rxLoop() {
  437. buf := make([]byte, 4096)
  438. expectedTid := uint16(w.inst.TunnelID)
  439. remoteIP := net.ParseIP(w.inst.RemoteAddress).To4()
  440. injectAddr := syscall.SockaddrLinklayer{
  441. Protocol: htons(syscall.ETH_P_ALL),
  442. Ifindex: w.peerIfIndex,
  443. Halen: 6,
  444. }
  445. for {
  446. select {
  447. case <-w.stopChan:
  448. return
  449. default:
  450. }
  451. n, from, err := syscall.Recvfrom(w.greFd, buf, 0)
  452. if err != nil {
  453. select {
  454. case <-w.stopChan:
  455. return
  456. default:
  457. time.Sleep(10 * time.Millisecond)
  458. continue
  459. }
  460. }
  461. // RAW IPv4 socket Recvfrom returns [IPv4 Header (20+ bytes)] + [GRE Header (8 bytes)] + [Payload]
  462. if n < 28 {
  463. continue
  464. }
  465. // Verify remote sender IP
  466. if fromIP, ok := from.(*syscall.SockaddrInet4); ok {
  467. if fromIP.Addr[0] != remoteIP[0] || fromIP.Addr[1] != remoteIP[1] ||
  468. fromIP.Addr[2] != remoteIP[2] || fromIP.Addr[3] != remoteIP[3] {
  469. continue
  470. }
  471. }
  472. // Calculate IPv4 Header Length
  473. ipHdrLen := int(buf[0]&0x0f) * 4
  474. if n < ipHdrLen+8 {
  475. continue
  476. }
  477. gre := buf[ipHdrLen:n]
  478. // Verify MikroTik EoIP GRE Header
  479. // Flags: 0x2001 (or 0x2000)
  480. // Proto: 0x6400
  481. if gre[0] != 0x20 || (gre[1] != 0x01 && gre[1] != 0x00) {
  482. continue
  483. }
  484. if gre[2] != 0x64 || gre[3] != 0x00 {
  485. continue
  486. }
  487. frameLen := binary.BigEndian.Uint16(gre[4:6])
  488. tunnelID := binary.LittleEndian.Uint16(gre[6:8])
  489. if tunnelID != expectedTid {
  490. continue
  491. }
  492. // Record successful packet reception
  493. now := time.Now().UnixNano()
  494. atomic.StoreInt64(&w.lastRx, now)
  495. if frameLen == 0 {
  496. // Keepalive probe from MikroTik: reply with keepalive
  497. w.sendKeepalive()
  498. continue
  499. }
  500. // Valid Ethernet frame received from MikroTik EoIP tunnel
  501. ethFrame := gre[8:]
  502. if len(ethFrame) < int(frameLen) {
  503. continue
  504. }
  505. payload := ethFrame[:frameLen]
  506. // Inject Ethernet frame into veth peer -> kernel delivers it to user interface (w.inst.Name)
  507. if err := syscall.Sendto(w.pktFd, payload, 0, &injectAddr); err == nil {
  508. atomic.AddUint64(&w.rxPackets, 1)
  509. atomic.AddUint64(&w.rxBytes, uint64(frameLen))
  510. }
  511. }
  512. }
  513. func (w *eoipTunnelWorker) stop() {
  514. close(w.stopChan)
  515. _ = syscall.Close(w.pktFd)
  516. _ = syscall.Close(w.greFd)
  517. // Clean up interfaces & iptables
  518. if w.inst.ClampMSS {
  519. _ = exec.Command("iptables", "-t", "mangle", "-D", "FORWARD", "-o", w.inst.Name, "-p", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", "--clamp-mss-to-pmtu").Run()
  520. }
  521. _ = exec.Command("ip", "link", "del", w.inst.Name).Run()
  522. log.Printf("[EoIP] Stopped tunnel '%s'", w.inst.Name)
  523. }
  524. func htons(n uint16) uint16 {
  525. return (n<<8)&0xff00 | (n>>8)&0x00ff
  526. }
  527. func generateMAC() string {
  528. buf := make([]byte, 3)
  529. _, _ = rand.Read(buf)
  530. return fmt.Sprintf("02:64:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], time.Now().UnixNano()%256)
  531. }
  532. func formatBytes(b uint64) string {
  533. const unit = 1024
  534. if b < unit {
  535. return fmt.Sprintf("%d B", b)
  536. }
  537. div, exp := uint64(unit), 0
  538. for n := b / unit; n >= unit; n /= unit {
  539. div *= unit
  540. exp++
  541. }
  542. return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
  543. }