eoip.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633
  1. package vpn
  2. import (
  3. "crypto/rand"
  4. "encoding/binary"
  5. "encoding/json"
  6. "fmt"
  7. "log"
  8. "net"
  9. "os"
  10. "os/exec"
  11. "path/filepath"
  12. "sort"
  13. "strconv"
  14. "strings"
  15. "sync"
  16. "sync/atomic"
  17. "syscall"
  18. "time"
  19. )
  20. // EoIPInstance represents a MikroTik Ethernet-over-IP tunnel instance
  21. type EoIPInstance struct {
  22. ID string `json:"id"`
  23. Name string `json:"name"` // e.g. "Goldix" or "eoip-tunnel1"
  24. TunnelID int `json:"tunnel_id"` // 1 - 65535
  25. RemoteAddress string `json:"remote_address"` // Remote MikroTik IPv4
  26. LocalAddress string `json:"local_address"` // Local router IPv4 (optional)
  27. IPAddress string `json:"ip_address,omitempty"` // Overlay IP/CIDR (e.g. "172.16.33.254/30")
  28. MACAddress string `json:"mac_address"` // e.g. "02:64:83:6b:46:44"
  29. MTU int `json:"mtu"` // default 1500
  30. ClampMSS bool `json:"clamp_mss"` // Clamp TCP MSS for PMTUD
  31. Comment string `json:"comment,omitempty"`
  32. Disabled bool `json:"disabled"`
  33. Running bool `json:"running"` // MikroTik 'R' flag
  34. RxPackets uint64 `json:"rx_packets"`
  35. TxPackets uint64 `json:"tx_packets"`
  36. RxBytes uint64 `json:"rx_bytes"`
  37. TxBytes uint64 `json:"tx_bytes"`
  38. RxHuman string `json:"rx_human,omitempty"`
  39. TxHuman string `json:"tx_human,omitempty"`
  40. LastError string `json:"last_error,omitempty"`
  41. }
  42. type EoIPManager struct {
  43. mu sync.RWMutex
  44. storePath string
  45. tunnels map[string]*EoIPInstance
  46. workers map[string]*eoipTunnelWorker
  47. }
  48. func NewEoIPManager() *EoIPManager {
  49. mgr := &EoIPManager{
  50. storePath: "/etc/proxmox-router/vpn/eoip.json",
  51. tunnels: make(map[string]*EoIPInstance),
  52. workers: make(map[string]*eoipTunnelWorker),
  53. }
  54. mgr.load()
  55. mgr.syncAll()
  56. return mgr
  57. }
  58. func (m *EoIPManager) load() {
  59. if data, err := os.ReadFile(m.storePath); err == nil {
  60. var list []*EoIPInstance
  61. if err := json.Unmarshal(data, &list); err == nil {
  62. for _, t := range list {
  63. m.tunnels[t.ID] = t
  64. }
  65. }
  66. }
  67. }
  68. func (m *EoIPManager) save() error {
  69. _ = os.MkdirAll(filepath.Dir(m.storePath), 0755)
  70. var list []*EoIPInstance
  71. for _, t := range m.tunnels {
  72. list = append(list, t)
  73. }
  74. data, err := json.MarshalIndent(list, "", " ")
  75. if err != nil {
  76. return err
  77. }
  78. return os.WriteFile(m.storePath, data, 0644)
  79. }
  80. func (m *EoIPManager) List() []*EoIPInstance {
  81. m.mu.Lock()
  82. defer m.mu.Unlock()
  83. list := make([]*EoIPInstance, 0, len(m.tunnels))
  84. for _, t := range m.tunnels {
  85. m.updateStats(t)
  86. list = append(list, t)
  87. }
  88. sort.Slice(list, func(i, j int) bool {
  89. if list[i].TunnelID != list[j].TunnelID {
  90. return list[i].TunnelID < list[j].TunnelID
  91. }
  92. return list[i].Name < list[j].Name
  93. })
  94. return list
  95. }
  96. func (m *EoIPManager) Get(id string) (*EoIPInstance, bool) {
  97. m.mu.Lock()
  98. defer m.mu.Unlock()
  99. t, ok := m.tunnels[id]
  100. if ok {
  101. m.updateStats(t)
  102. }
  103. return t, ok
  104. }
  105. func (m *EoIPManager) Save(inst *EoIPInstance) error {
  106. m.mu.Lock()
  107. defer m.mu.Unlock()
  108. if inst.Name == "" {
  109. inst.Name = fmt.Sprintf("eoip-tunnel%d", inst.TunnelID)
  110. }
  111. if inst.TunnelID <= 0 || inst.TunnelID > 65535 {
  112. return fmt.Errorf("tunnel_id must be between 1 and 65535")
  113. }
  114. if inst.RemoteAddress == "" {
  115. return fmt.Errorf("remote_address is required")
  116. }
  117. if inst.MTU <= 0 {
  118. inst.MTU = 1500
  119. }
  120. if inst.MACAddress == "" {
  121. inst.MACAddress = generateMAC()
  122. }
  123. if inst.ID == "" {
  124. inst.ID = fmt.Sprintf("eoip-%d", time.Now().UnixNano())
  125. }
  126. // Smart detection: if user put CIDR into LocalAddress by mistake, move it to IPAddress
  127. if strings.Contains(inst.LocalAddress, "/") && inst.IPAddress == "" {
  128. inst.IPAddress = inst.LocalAddress
  129. inst.LocalAddress = ""
  130. }
  131. // Stop previous worker if exists
  132. if w, exists := m.workers[inst.ID]; exists {
  133. w.stop()
  134. delete(m.workers, inst.ID)
  135. }
  136. m.tunnels[inst.ID] = inst
  137. _ = m.save()
  138. if !inst.Disabled {
  139. worker, err := newEoIPTunnelWorker(inst)
  140. if err != nil {
  141. inst.LastError = err.Error()
  142. inst.Running = false
  143. return err
  144. }
  145. m.workers[inst.ID] = worker
  146. go worker.start()
  147. } else {
  148. inst.Running = false
  149. }
  150. return nil
  151. }
  152. func (m *EoIPManager) SetState(id string, disabled bool) error {
  153. m.mu.Lock()
  154. defer m.mu.Unlock()
  155. t, ok := m.tunnels[id]
  156. if !ok {
  157. return fmt.Errorf("tunnel not found")
  158. }
  159. t.Disabled = disabled
  160. _ = m.save()
  161. if w, exists := m.workers[id]; exists {
  162. w.stop()
  163. delete(m.workers, id)
  164. }
  165. if disabled {
  166. t.Running = false
  167. return nil
  168. }
  169. worker, err := newEoIPTunnelWorker(t)
  170. if err != nil {
  171. t.LastError = err.Error()
  172. t.Running = false
  173. return err
  174. }
  175. m.workers[id] = worker
  176. go worker.start()
  177. return nil
  178. }
  179. func (m *EoIPManager) Delete(id string) error {
  180. m.mu.Lock()
  181. defer m.mu.Unlock()
  182. if _, ok := m.tunnels[id]; !ok {
  183. return fmt.Errorf("tunnel not found")
  184. }
  185. if w, exists := m.workers[id]; exists {
  186. w.stop()
  187. delete(m.workers, id)
  188. }
  189. delete(m.tunnels, id)
  190. return m.save()
  191. }
  192. func (m *EoIPManager) syncAll() {
  193. m.mu.Lock()
  194. defer m.mu.Unlock()
  195. for id, t := range m.tunnels {
  196. if !t.Disabled {
  197. if w, exists := m.workers[id]; exists {
  198. w.stop()
  199. delete(m.workers, id)
  200. }
  201. worker, err := newEoIPTunnelWorker(t)
  202. if err != nil {
  203. t.LastError = err.Error()
  204. t.Running = false
  205. continue
  206. }
  207. m.workers[id] = worker
  208. go worker.start()
  209. } else {
  210. t.Running = false
  211. }
  212. }
  213. }
  214. func (m *EoIPManager) updateStats(t *EoIPInstance) {
  215. if w, exists := m.workers[t.ID]; exists {
  216. t.Running = w.isRunning()
  217. t.RxBytes = atomic.LoadUint64(&w.rxBytes)
  218. t.RxPackets = atomic.LoadUint64(&w.rxPackets)
  219. t.TxBytes = atomic.LoadUint64(&w.txBytes)
  220. t.TxPackets = atomic.LoadUint64(&w.txPackets)
  221. } else {
  222. t.Running = false
  223. }
  224. t.RxHuman = formatBytes(t.RxBytes)
  225. t.TxHuman = formatBytes(t.TxBytes)
  226. // Detect any bound IPv4 if not explicitly saved in JSON
  227. if t.IPAddress == "" {
  228. if ifi, err := net.InterfaceByName(t.Name); err == nil {
  229. if addrs, err := ifi.Addrs(); err == nil {
  230. for _, a := range addrs {
  231. if ipNet, ok := a.(*net.IPNet); ok && ipNet.IP.To4() != nil {
  232. t.IPAddress = ipNet.String()
  233. break
  234. }
  235. }
  236. }
  237. }
  238. }
  239. }
  240. // -------------------------------------------------------------
  241. // Native MikroTik EoIP Engine Worker
  242. // -------------------------------------------------------------
  243. type eoipTunnelWorker struct {
  244. inst *EoIPInstance
  245. peerIfName string
  246. peerIfIndex int
  247. greFd int
  248. pktFd int
  249. stopChan chan struct{}
  250. lastRx int64 // unix nano
  251. rxBytes uint64
  252. rxPackets uint64
  253. txBytes uint64
  254. txPackets uint64
  255. remoteAddrInet syscall.SockaddrInet4
  256. }
  257. func newEoIPTunnelWorker(inst *EoIPInstance) (*eoipTunnelWorker, error) {
  258. // Parse Remote IPv4
  259. remoteIP := net.ParseIP(inst.RemoteAddress).To4()
  260. if remoteIP == nil {
  261. return nil, fmt.Errorf("invalid remote address: %s", inst.RemoteAddress)
  262. }
  263. // Internal peer interface name (max 15 chars for Linux ifnames)
  264. peerName := fmt.Sprintf("ep_%s", inst.Name)
  265. if len(peerName) > 15 {
  266. peerName = peerName[:15]
  267. }
  268. // 1. Clean up existing links if present
  269. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  270. _ = exec.Command("ip", "link", "del", peerName).Run()
  271. // 2. Create veth pair: inst.Name <--> peerName
  272. cmdAdd := exec.Command("ip", "link", "add", inst.Name, "type", "veth", "peer", "name", peerName)
  273. if out, err := cmdAdd.CombinedOutput(); err != nil {
  274. return nil, fmt.Errorf("failed to create veth pair: %s - %w", string(out), err)
  275. }
  276. // 3. Set MAC address and MTU
  277. if inst.MACAddress != "" {
  278. _ = exec.Command("ip", "link", "set", inst.Name, "address", inst.MACAddress).Run()
  279. }
  280. mtuStr := strconv.Itoa(inst.MTU)
  281. _ = exec.Command("ip", "link", "set", inst.Name, "mtu", mtuStr).Run()
  282. _ = exec.Command("ip", "link", "set", peerName, "mtu", mtuStr).Run()
  283. // 4. Bring both interfaces UP
  284. _ = exec.Command("ip", "link", "set", inst.Name, "up").Run()
  285. _ = exec.Command("ip", "link", "set", peerName, "up").Run()
  286. // 4b. Assign Overlay IP address if specified
  287. if inst.IPAddress != "" {
  288. _ = exec.Command("ip", "addr", "add", inst.IPAddress, "dev", inst.Name).Run()
  289. }
  290. // 5. TCP MSS Clamping
  291. if inst.ClampMSS {
  292. _ = exec.Command("iptables", "-t", "mangle", "-A", "FORWARD", "-o", inst.Name, "-p", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", "--clamp-mss-to-pmtu").Run()
  293. }
  294. // 6. Find interface index of peerName
  295. peerIf, err := net.InterfaceByName(peerName)
  296. if err != nil {
  297. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  298. return nil, fmt.Errorf("cannot find peer interface %s: %w", peerName, err)
  299. }
  300. // 7. Open AF_PACKET raw socket on peer interface
  301. pktFd, err := syscall.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, int(htons(syscall.ETH_P_ALL)))
  302. if err != nil {
  303. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  304. return nil, fmt.Errorf("cannot create AF_PACKET socket: %w", err)
  305. }
  306. pktAddr := syscall.SockaddrLinklayer{
  307. Protocol: htons(syscall.ETH_P_ALL),
  308. Ifindex: peerIf.Index,
  309. }
  310. if err := syscall.Bind(pktFd, &pktAddr); err != nil {
  311. syscall.Close(pktFd)
  312. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  313. return nil, fmt.Errorf("cannot bind AF_PACKET socket to %s: %w", peerName, err)
  314. }
  315. // 8. Open raw GRE socket (IPPROTO_GRE = 47)
  316. greFd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_GRE)
  317. if err != nil {
  318. syscall.Close(pktFd)
  319. _ = exec.Command("ip", "link", "del", inst.Name).Run()
  320. return nil, fmt.Errorf("cannot create raw GRE socket: %w", err)
  321. }
  322. // Bind GRE socket to LocalAddress if specified
  323. if inst.LocalAddress != "" {
  324. localIP := net.ParseIP(inst.LocalAddress).To4()
  325. if localIP != nil {
  326. localAddr := &syscall.SockaddrInet4{
  327. Addr: [4]byte{localIP[0], localIP[1], localIP[2], localIP[3]},
  328. }
  329. _ = syscall.Bind(greFd, localAddr)
  330. }
  331. }
  332. w := &eoipTunnelWorker{
  333. inst: inst,
  334. peerIfName: peerName,
  335. peerIfIndex: peerIf.Index,
  336. greFd: greFd,
  337. pktFd: pktFd,
  338. stopChan: make(chan struct{}),
  339. remoteAddrInet: syscall.SockaddrInet4{
  340. Addr: [4]byte{remoteIP[0], remoteIP[1], remoteIP[2], remoteIP[3]},
  341. },
  342. rxBytes: inst.RxBytes,
  343. rxPackets: inst.RxPackets,
  344. txBytes: inst.TxBytes,
  345. txPackets: inst.TxPackets,
  346. }
  347. return w, nil
  348. }
  349. func (w *eoipTunnelWorker) start() {
  350. log.Printf("[EoIP] Started tunnel '%s' (ID: %d, Remote: %s)", w.inst.Name, w.inst.TunnelID, w.inst.RemoteAddress)
  351. // Goroutine 1: Forward packets from Local veth -> Remote MikroTik via GRE
  352. go w.txLoop()
  353. // Goroutine 2: Receive packets from Remote MikroTik via GRE -> Inject to Local veth
  354. go w.rxLoop()
  355. // Goroutine 3: Periodic MikroTik EoIP Keepalive
  356. go w.keepaliveLoop()
  357. }
  358. func (w *eoipTunnelWorker) isRunning() bool {
  359. last := atomic.LoadInt64(&w.lastRx)
  360. if last == 0 {
  361. return false
  362. }
  363. return time.Since(time.Unix(0, last)) < 15*time.Second
  364. }
  365. func (w *eoipTunnelWorker) sendKeepalive() {
  366. // MikroTik EoIP Keepalive packet:
  367. // GRE Flags: 0x2001
  368. // Protocol: 0x6400
  369. // Length: 0x0000
  370. // Tunnel ID: Little-Endian
  371. tid := uint16(w.inst.TunnelID)
  372. pkt := []byte{
  373. 0x20, 0x01,
  374. 0x64, 0x00,
  375. 0x00, 0x00,
  376. byte(tid & 0xff), byte(tid >> 8),
  377. }
  378. _ = syscall.Sendto(w.greFd, pkt, 0, &w.remoteAddrInet)
  379. }
  380. func (w *eoipTunnelWorker) keepaliveLoop() {
  381. ticker := time.NewTicker(5 * time.Second)
  382. defer ticker.Stop()
  383. // Send immediate initial keepalive
  384. w.sendKeepalive()
  385. for {
  386. select {
  387. case <-w.stopChan:
  388. return
  389. case <-ticker.C:
  390. w.sendKeepalive()
  391. }
  392. }
  393. }
  394. func (w *eoipTunnelWorker) txLoop() {
  395. buf := make([]byte, 2048)
  396. tid := uint16(w.inst.TunnelID)
  397. for {
  398. select {
  399. case <-w.stopChan:
  400. return
  401. default:
  402. }
  403. n, from, err := syscall.Recvfrom(w.pktFd, buf, 0)
  404. if err != nil {
  405. select {
  406. case <-w.stopChan:
  407. return
  408. default:
  409. time.Sleep(10 * time.Millisecond)
  410. continue
  411. }
  412. }
  413. // Filter out outgoing packets to prevent reflection
  414. if fromLink, ok := from.(*syscall.SockaddrLinklayer); ok {
  415. if fromLink.Pkttype == syscall.PACKET_OUTGOING {
  416. continue
  417. }
  418. }
  419. // Must have at least minimum Ethernet header length (14 bytes)
  420. if n < 14 {
  421. continue
  422. }
  423. // Construct MikroTik EoIP Header (8 bytes) + Ethernet Frame
  424. // [0..1] = 0x2001 (Flags)
  425. // [2..3] = 0x6400 (Protocol)
  426. // [4..5] = payload length (Big-Endian)
  427. // [6..7] = tunnel ID (Little-Endian)
  428. grePkt := make([]byte, 8+n)
  429. grePkt[0] = 0x20
  430. grePkt[1] = 0x01
  431. grePkt[2] = 0x64
  432. grePkt[3] = 0x00
  433. binary.BigEndian.PutUint16(grePkt[4:6], uint16(n))
  434. binary.LittleEndian.PutUint16(grePkt[6:8], tid)
  435. copy(grePkt[8:], buf[:n])
  436. // Send over raw GRE socket
  437. if err := syscall.Sendto(w.greFd, grePkt, 0, &w.remoteAddrInet); err == nil {
  438. atomic.AddUint64(&w.txPackets, 1)
  439. atomic.AddUint64(&w.txBytes, uint64(n))
  440. }
  441. }
  442. }
  443. func (w *eoipTunnelWorker) rxLoop() {
  444. buf := make([]byte, 4096)
  445. expectedTid := uint16(w.inst.TunnelID)
  446. remoteIP := net.ParseIP(w.inst.RemoteAddress).To4()
  447. injectAddr := syscall.SockaddrLinklayer{
  448. Protocol: htons(syscall.ETH_P_ALL),
  449. Ifindex: w.peerIfIndex,
  450. Halen: 6,
  451. }
  452. for {
  453. select {
  454. case <-w.stopChan:
  455. return
  456. default:
  457. }
  458. n, from, err := syscall.Recvfrom(w.greFd, buf, 0)
  459. if err != nil {
  460. select {
  461. case <-w.stopChan:
  462. return
  463. default:
  464. time.Sleep(10 * time.Millisecond)
  465. continue
  466. }
  467. }
  468. // RAW IPv4 socket Recvfrom returns [IPv4 Header (20+ bytes)] + [GRE Header (8 bytes)] + [Payload]
  469. if n < 28 {
  470. continue
  471. }
  472. // Verify remote sender IP
  473. if fromIP, ok := from.(*syscall.SockaddrInet4); ok {
  474. if fromIP.Addr[0] != remoteIP[0] || fromIP.Addr[1] != remoteIP[1] ||
  475. fromIP.Addr[2] != remoteIP[2] || fromIP.Addr[3] != remoteIP[3] {
  476. continue
  477. }
  478. }
  479. // Calculate IPv4 Header Length
  480. ipHdrLen := int(buf[0]&0x0f) * 4
  481. if n < ipHdrLen+8 {
  482. continue
  483. }
  484. gre := buf[ipHdrLen:n]
  485. // Verify MikroTik EoIP GRE Header
  486. // Flags: 0x2001 (or 0x2000)
  487. // Proto: 0x6400
  488. if gre[0] != 0x20 || (gre[1] != 0x01 && gre[1] != 0x00) {
  489. continue
  490. }
  491. if gre[2] != 0x64 || gre[3] != 0x00 {
  492. continue
  493. }
  494. frameLen := binary.BigEndian.Uint16(gre[4:6])
  495. tunnelID := binary.LittleEndian.Uint16(gre[6:8])
  496. if tunnelID != expectedTid {
  497. continue
  498. }
  499. // Record successful packet reception
  500. now := time.Now().UnixNano()
  501. atomic.StoreInt64(&w.lastRx, now)
  502. if frameLen == 0 {
  503. // Keepalive probe from MikroTik: reply with keepalive
  504. w.sendKeepalive()
  505. continue
  506. }
  507. // Valid Ethernet frame received from MikroTik EoIP tunnel
  508. ethFrame := gre[8:]
  509. if len(ethFrame) < int(frameLen) {
  510. continue
  511. }
  512. payload := ethFrame[:frameLen]
  513. // Inject Ethernet frame into veth peer -> kernel delivers it to user interface (w.inst.Name)
  514. if err := syscall.Sendto(w.pktFd, payload, 0, &injectAddr); err == nil {
  515. atomic.AddUint64(&w.rxPackets, 1)
  516. atomic.AddUint64(&w.rxBytes, uint64(frameLen))
  517. }
  518. }
  519. }
  520. func (w *eoipTunnelWorker) stop() {
  521. close(w.stopChan)
  522. _ = syscall.Close(w.pktFd)
  523. _ = syscall.Close(w.greFd)
  524. // Clean up interfaces & iptables
  525. if w.inst.ClampMSS {
  526. _ = exec.Command("iptables", "-t", "mangle", "-D", "FORWARD", "-o", w.inst.Name, "-p", "tcp", "--tcp-flags", "SYN,RST", "SYN", "-j", "TCPMSS", "--clamp-mss-to-pmtu").Run()
  527. }
  528. _ = exec.Command("ip", "link", "del", w.inst.Name).Run()
  529. log.Printf("[EoIP] Stopped tunnel '%s'", w.inst.Name)
  530. }
  531. func htons(n uint16) uint16 {
  532. return (n<<8)&0xff00 | (n>>8)&0x00ff
  533. }
  534. func generateMAC() string {
  535. buf := make([]byte, 3)
  536. _, _ = rand.Read(buf)
  537. return fmt.Sprintf("02:64:%02x:%02x:%02x:%02x", buf[0], buf[1], buf[2], time.Now().UnixNano()%256)
  538. }
  539. func formatBytes(b uint64) string {
  540. const unit = 1024
  541. if b < unit {
  542. return fmt.Sprintf("%d B", b)
  543. }
  544. div, exp := uint64(unit), 0
  545. for n := b / unit; n >= unit; n /= unit {
  546. div *= unit
  547. exp++
  548. }
  549. return fmt.Sprintf("%.1f %cB", float64(b)/float64(div), "KMGTPE"[exp])
  550. }