package routing import ( "bytes" "fmt" "log" "net" "os/exec" "regexp" "strconv" "strings" "time" "debtik/internal/sdwan" "github.com/vishvananda/netlink" ) var ( pingLossRegex = regexp.MustCompile(`(\d+)% packet loss`) pingRttRegex = regexp.MustCompile(`rtt min/avg/max/mdev = [\d.]+/([\d.]+)/`) ) // resolveGatewayInterface determines which network device routes to the gateway IP func resolveGatewayInterface(gwIP string) string { ip := net.ParseIP(strings.TrimSpace(gwIP)) if ip == nil { return "" } routes, err := netlink.RouteGet(ip) if err == nil && len(routes) > 0 { link, err := netlink.LinkByIndex(routes[0].LinkIndex) if err == nil && link != nil { return link.Attrs().Name } } // Fallback using `ip route get` out, err := exec.Command("ip", "route", "get", gwIP).Output() if err == nil { fields := strings.Fields(string(out)) for i, f := range fields { if f == "dev" && i+1 < len(fields) { return fields[i+1] } } } return "" } func probeRouteHealthOnce(target string, iface string) (bool, float64, float64) { var args []string if iface != "" { args = []string{"-c", "3", "-W", "2", "-i", "0.3", "-I", iface, target} } else { args = []string{"-c", "3", "-W", "2", "-i", "0.3", target} } cmd := exec.Command("ping", args...) var outBuf bytes.Buffer cmd.Stdout = &outBuf cmd.Stderr = &outBuf _ = cmd.Run() output := outBuf.String() lossPct := 100.0 if match := pingLossRegex.FindStringSubmatch(output); len(match) > 1 { if val, err := strconv.ParseFloat(match[1], 64); err == nil { lossPct = val } } var avgRtt float64 if match := pingRttRegex.FindStringSubmatch(output); len(match) > 1 { if val, err := strconv.ParseFloat(match[1], 64); err == nil { avgRtt = val } } isUp := (lossPct < 100.0) return isUp, avgRtt, lossPct } // probeRouteHealth tests reachability of a gateway or public internet host bound to a specific interface func probeRouteHealth(target string, iface string) (bool, float64, float64, error) { target = strings.TrimSpace(target) if target == "" { return false, 0, 100, fmt.Errorf("empty probe target") } isUp, avgRtt, lossPct := probeRouteHealthOnce(target, iface) if isUp { return true, avgRtt, lossPct, nil } // Secondary fallback target to avoid false failovers from CDN ICMP drops fallbackTarget := "8.8.8.8" if target == "8.8.8.8" || target == "8.8.4.4" { fallbackTarget = "1.1.1.1" } fallbackUp, fallbackRtt, fallbackLoss := probeRouteHealthOnce(fallbackTarget, iface) if fallbackUp { return true, fallbackRtt, fallbackLoss, nil } return false, avgRtt, lossPct, nil } // startHealthCheckWorker initiates background internet connectivity probing for static routes func (m *RoutingManager) startHealthCheckWorker() { ticker := time.NewTicker(2 * time.Second) go func() { for { select { case <-m.stopChan: ticker.Stop() return case now := <-ticker.C: m.checkAllRoutesHealth(now) } } }() } func (m *RoutingManager) checkAllRoutesHealth(now time.Time) { m.mu.Lock() defer m.mu.Unlock() for i := range m.staticRoutes { r := &m.staticRoutes[i] if r.Disabled { continue } mode := strings.ToLower(strings.TrimSpace(r.CheckGateway)) if mode != "ping" && mode != "internet" { continue } interval := r.CheckInterval if interval <= 0 { interval = 5 } // Check if it's time to probe if r.HealthLastCheck != "" { if lastT, err := time.Parse(time.RFC3339, r.HealthLastCheck); err == nil { if now.Sub(lastT) < time.Duration(interval)*time.Second { continue } } } // Determine probe target target := r.CheckTarget if mode == "ping" { target = r.Gateway } else if mode == "internet" { if target == "" { target = "1.1.1.1" } } // Determine egress interface iface := r.Interface if iface == "" && r.Gateway != "" { iface = resolveGatewayInterface(r.Gateway) } // If this route is bound to an SD-WAN group, resolve egress interface to the group's active member if sdwan.IsSDWANGroup(iface) || sdwan.IsSDWANGroup(r.Gateway) { targetSDWAN := iface if !sdwan.IsSDWANGroup(targetSDWAN) { targetSDWAN = r.Gateway } resolved, ok := sdwan.ResolveGroupForRouting(targetSDWAN) if ok && len(resolved) > 0 { iface = resolved[0].Interface } } // Threshold threshold := r.CheckThreshold if threshold < 5 { threshold = 5 } // Execute probe asynchronously with timeout to prevent blocking routing manager lock go func(routeID string, probeTarget string, probeIface string, checkThreshold int) { isUp, rtt, loss, _ := probeRouteHealth(probeTarget, probeIface) m.mu.Lock() defer m.mu.Unlock() // Find route by ID again var targetRoute *RouteItem for idx := range m.staticRoutes { if m.staticRoutes[idx].ID == routeID { targetRoute = &m.staticRoutes[idx] break } } if targetRoute == nil || targetRoute.Disabled { return } targetRoute.HealthRTT = rtt targetRoute.HealthLossPct = loss targetRoute.HealthLastCheck = time.Now().Format(time.RFC3339) if isUp { wasOffline := targetRoute.HealthStatus == "offline" targetRoute.HealthStatus = "online" targetRoute.HealthFailCount = 0 // If it was offline, restore route in kernel if wasOffline || !targetRoute.Active { targetRoute.Active = true _ = m.applyRouteToKernel(*targetRoute) log.Printf("[RouteHealth] Route %s via %s (target %s) Internet RESTORED (RTT: %.1fms) -> Re-installed in kernel", targetRoute.DstAddress, targetRoute.Gateway, probeTarget, rtt) _ = m.save() } } else { targetRoute.HealthFailCount++ if targetRoute.HealthFailCount >= checkThreshold { wasOnline := targetRoute.HealthStatus != "offline" targetRoute.HealthStatus = "offline" // Safety guard: if this is a default route and the ONLY active default route remaining, // do NOT withdraw it from the kernel to prevent a total internet blackout! hasOtherActiveDefault := false for _, other := range m.staticRoutes { if other.ID != targetRoute.ID && normalizeDst(other.DstAddress) == "default" && !other.Disabled && other.HealthStatus != "offline" { hasOtherActiveDefault = true break } } if !hasOtherActiveDefault && normalizeDst(targetRoute.DstAddress) == "default" { log.Printf("[RouteHealth] Route %s via %s (target %s) failed health check, but keeping active as LAST RESORT default route to prevent total internet blackout", targetRoute.DstAddress, targetRoute.Gateway, probeTarget) if !targetRoute.Active { targetRoute.Active = true _ = m.applyRouteToKernel(*targetRoute) _ = m.save() } } else { // If it was online, withdraw route from kernel to allow failover if wasOnline || targetRoute.Active { targetRoute.Active = false _ = m.removeRouteFromKernel(*targetRoute) log.Printf("[RouteHealth] Route %s via %s (target %s) Internet DOWN (threshold %d reached) -> Removed from kernel for failover", targetRoute.DstAddress, targetRoute.Gateway, probeTarget, checkThreshold) _ = m.save() } } } } }(r.ID, target, iface, threshold) } }