DebTik/internal/routing/health.go

package routing

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"debtik/internal/sdwan"
	"github.com/vishvananda/netlink"
)

var (
	pingLossRegex = regexp.MustCompile(`(\d+)% packet loss`)
	pingRttRegex  = regexp.MustCompile(`rtt min/avg/max/mdev = [\d.]+/([\d.]+)/`)
)

// resolveGatewayInterface determines which network device routes to the gateway IP
func resolveGatewayInterface(gwIP string) string {
	ip := net.ParseIP(strings.TrimSpace(gwIP))
	if ip == nil {
		return ""
	}

	routes, err := netlink.RouteGet(ip)
	if err == nil && len(routes) > 0 {
		link, err := netlink.LinkByIndex(routes[0].LinkIndex)
		if err == nil && link != nil {
			return link.Attrs().Name
		}
	}

	// Fallback using `ip route get`
	out, err := exec.Command("ip", "route", "get", gwIP).Output()
	if err == nil {
		fields := strings.Fields(string(out))
		for i, f := range fields {
			if f == "dev" && i+1 < len(fields) {
				return fields[i+1]
			}
		}
	}

	return ""
}

func probeRouteHealthOnce(target string, iface string) (bool, float64, float64) {
	var args []string
	if iface != "" {
		args = []string{"-c", "3", "-W", "2", "-i", "0.3", "-I", iface, target}
	} else {
		args = []string{"-c", "3", "-W", "2", "-i", "0.3", target}
	}

	cmd := exec.Command("ping", args...)
	var outBuf bytes.Buffer
	cmd.Stdout = &outBuf
	cmd.Stderr = &outBuf

	_ = cmd.Run()
	output := outBuf.String()

	lossPct := 100.0
	if match := pingLossRegex.FindStringSubmatch(output); len(match) > 1 {
		if val, err := strconv.ParseFloat(match[1], 64); err == nil {
			lossPct = val
		}
	}

	var avgRtt float64
	if match := pingRttRegex.FindStringSubmatch(output); len(match) > 1 {
		if val, err := strconv.ParseFloat(match[1], 64); err == nil {
			avgRtt = val
		}
	}

	isUp := (lossPct < 100.0)
	return isUp, avgRtt, lossPct
}

// probeRouteHealth tests reachability of a gateway or public internet host bound to a specific interface
func probeRouteHealth(target string, iface string) (bool, float64, float64, error) {
	target = strings.TrimSpace(target)
	if target == "" {
		return false, 0, 100, fmt.Errorf("empty probe target")
	}

	isUp, avgRtt, lossPct := probeRouteHealthOnce(target, iface)
	if isUp {
		return true, avgRtt, lossPct, nil
	}

	// Secondary fallback target to avoid false failovers from CDN ICMP drops
	fallbackTarget := "8.8.8.8"
	if target == "8.8.8.8" || target == "8.8.4.4" {
		fallbackTarget = "1.1.1.1"
	}

	fallbackUp, fallbackRtt, fallbackLoss := probeRouteHealthOnce(fallbackTarget, iface)
	if fallbackUp {
		return true, fallbackRtt, fallbackLoss, nil
	}

	return false, avgRtt, lossPct, nil
}

// startHealthCheckWorker initiates background internet connectivity probing for static routes
func (m *RoutingManager) startHealthCheckWorker() {
	ticker := time.NewTicker(2 * time.Second)
	go func() {
		for {
			select {
			case <-m.stopChan:
				ticker.Stop()
				return
			case now := <-ticker.C:
				m.checkAllRoutesHealth(now)
			}
		}
	}()
}

func (m *RoutingManager) checkAllRoutesHealth(now time.Time) {
	m.mu.Lock()
	defer m.mu.Unlock()

	for i := range m.staticRoutes {
		r := &m.staticRoutes[i]
		if r.Disabled {
			continue
		}

		mode := strings.ToLower(strings.TrimSpace(r.CheckGateway))
		if mode != "ping" && mode != "internet" {
			continue
		}

		interval := r.CheckInterval
		if interval <= 0 {
			interval = 5
		}

		// Check if it's time to probe
		if r.HealthLastCheck != "" {
			if lastT, err := time.Parse(time.RFC3339, r.HealthLastCheck); err == nil {
				if now.Sub(lastT) < time.Duration(interval)*time.Second {
					continue
				}
			}
		}

		// Determine probe target
		target := r.CheckTarget
		if mode == "ping" {
			target = r.Gateway
		} else if mode == "internet" {
			if target == "" {
				target = "1.1.1.1"
			}
		}

		// Determine egress interface
		iface := r.Interface
		if iface == "" && r.Gateway != "" {
			iface = resolveGatewayInterface(r.Gateway)
		}

		// If this route is bound to an SD-WAN group, resolve egress interface to the group's active member
		if sdwan.IsSDWANGroup(iface) || sdwan.IsSDWANGroup(r.Gateway) {
			targetSDWAN := iface
			if !sdwan.IsSDWANGroup(targetSDWAN) {
				targetSDWAN = r.Gateway
			}
			resolved, ok := sdwan.ResolveGroupForRouting(targetSDWAN)
			if ok && len(resolved) > 0 {
				iface = resolved[0].Interface
			}
		}

		// Threshold
		threshold := r.CheckThreshold
		if threshold < 5 {
			threshold = 5
		}

		// Execute probe asynchronously with timeout to prevent blocking routing manager lock
		go func(routeID string, probeTarget string, probeIface string, checkThreshold int) {
			isUp, rtt, loss, _ := probeRouteHealth(probeTarget, probeIface)

			m.mu.Lock()
			defer m.mu.Unlock()

			// Find route by ID again
			var targetRoute *RouteItem
			for idx := range m.staticRoutes {
				if m.staticRoutes[idx].ID == routeID {
					targetRoute = &m.staticRoutes[idx]
					break
				}
			}
			if targetRoute == nil || targetRoute.Disabled {
				return
			}

			targetRoute.HealthRTT = rtt
			targetRoute.HealthLossPct = loss
			targetRoute.HealthLastCheck = time.Now().Format(time.RFC3339)

			if isUp {
				wasOffline := targetRoute.HealthStatus == "offline"
				targetRoute.HealthStatus = "online"
				targetRoute.HealthFailCount = 0

				// If it was offline, restore route in kernel
				if wasOffline || !targetRoute.Active {
					targetRoute.Active = true
					_ = m.applyRouteToKernel(*targetRoute)
					log.Printf("[RouteHealth] Route %s via %s (target %s) Internet RESTORED (RTT: %.1fms) -> Re-installed in kernel",
						targetRoute.DstAddress, targetRoute.Gateway, probeTarget, rtt)
					_ = m.save()
				}
			} else {
				targetRoute.HealthFailCount++
				if targetRoute.HealthFailCount >= checkThreshold {
					wasOnline := targetRoute.HealthStatus != "offline"
					targetRoute.HealthStatus = "offline"

					// Safety guard: if this is a default route and the ONLY active default route remaining,
					// do NOT withdraw it from the kernel to prevent a total internet blackout!
					hasOtherActiveDefault := false
					for _, other := range m.staticRoutes {
						if other.ID != targetRoute.ID && normalizeDst(other.DstAddress) == "default" && !other.Disabled && other.HealthStatus != "offline" {
							hasOtherActiveDefault = true
							break
						}
					}

					if !hasOtherActiveDefault && normalizeDst(targetRoute.DstAddress) == "default" {
						log.Printf("[RouteHealth] Route %s via %s (target %s) failed health check, but keeping active as LAST RESORT default route to prevent total internet blackout",
							targetRoute.DstAddress, targetRoute.Gateway, probeTarget)
						if !targetRoute.Active {
							targetRoute.Active = true
							_ = m.applyRouteToKernel(*targetRoute)
							_ = m.save()
						}
					} else {
						// If it was online, withdraw route from kernel to allow failover
						if wasOnline || targetRoute.Active {
							targetRoute.Active = false
							_ = m.removeRouteFromKernel(*targetRoute)
							log.Printf("[RouteHealth] Route %s via %s (target %s) Internet DOWN (threshold %d reached) -> Removed from kernel for failover",
								targetRoute.DstAddress, targetRoute.Gateway, probeTarget, checkThreshold)
							_ = m.save()
						}
					}
				}
			}
		}(r.ID, target, iface, threshold)
	}
}