DebTik/internal/sdwan/health.go

package sdwan

import (
	"context"
	"log"
	"math"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	rttRegex  = regexp.MustCompile(`time=([0-9.]+)\s*ms`)
	lossRegex = regexp.MustCompile(`([0-9.]+)%\s*packet\s*loss`)
)

type probeSample struct {
	latency float64
	success bool
	t       time.Time
}

// HealthMonitor periodically probes SD-WAN member interfaces and updates SLA telemetry.
type HealthMonitor struct {
	mu            sync.RWMutex
	samples       map[string][]probeSample // memberID -> rolling probe samples
	cancel        context.CancelFunc
	onStateChange func()
}

func NewHealthMonitor(onStateChange func()) *HealthMonitor {
	return &HealthMonitor{
		samples:       make(map[string][]probeSample),
		onStateChange: onStateChange,
	}
}

// Start begins periodic background health checking for active SD-WAN members.
func (h *HealthMonitor) Start(getGroups func() []*SDWANGroup) {
	ctx, cancel := context.WithCancel(context.Background())
	h.cancel = cancel

	go func() {
		ticker := time.NewTicker(3 * time.Second)
		defer ticker.Stop()

		for {
			select {
			case <-ctx.Done():
				return
			case <-ticker.C:
				groups := getGroups()
				h.runProbeIteration(groups)
			}
		}
	}()
}

// Stop terminates the probing goroutine.
func (h *HealthMonitor) Stop() {
	if h.cancel != nil {
		h.cancel()
	}
}

func (h *HealthMonitor) runProbeIteration(groups []*SDWANGroup) {
	stateChanged := false

	for _, g := range groups {
		if g.Disabled {
			continue
		}

		for _, m := range g.Members {
			if m.Disabled || m.Interface == "" {
				m.Status = StatusDisabled
				continue
			}

			probeTarget := m.TargetProbe
			if probeTarget == "" {
				probeTarget = "1.1.1.1"
			}

			// Run ICMP ping probe
			latency, success := h.executeProbe(m.Interface, probeTarget)

			h.mu.Lock()
			history := h.samples[m.ID]
			history = append(history, probeSample{
				latency: latency,
				success: success,
				t:       time.Now(),
			})
			if len(history) > 10 {
				history = history[len(history)-10:]
			}
			h.samples[m.ID] = history
			h.mu.Unlock()

			prevStatus := m.Status

			// Calculate telemetry
			m.LastChecked = time.Now()
			if success {
				m.ConsecutiveSuccesses++
				m.ConsecutiveFails = 0
			} else {
				m.ConsecutiveFails++
				m.ConsecutiveSuccesses = 0
			}

			h.calculateMetrics(m, history)

			// Determine status based on SLA thresholds with anti-flapping hysteresis
			if m.ConsecutiveFails >= 5 {
				m.Status = StatusDown
			} else if prevStatus == StatusDown && m.ConsecutiveSuccesses < 2 {
				// Require 2 consecutive successful probe cycles before recovering from DOWN
				m.Status = StatusDown
			} else {
				maxLat := float64(m.MaxLatency)
				if maxLat <= 0 {
					maxLat = 250.0
				}
				maxLoss := float64(m.MaxLoss)
				if maxLoss <= 0 {
					maxLoss = 20.0
				}

				if m.PacketLoss > maxLoss || m.LiveLatency > maxLat {
					m.Status = StatusDegraded
				} else {
					m.Status = StatusHealthy
				}
			}

			// Check if link transitioned between UP and DOWN
			if (prevStatus == StatusDown && m.Status != StatusDown) ||
				(prevStatus != StatusDown && m.Status == StatusDown) {
				stateChanged = true
				log.Printf("[SD-WAN] Health alert: %s status changed from %s to %s (latency: %.1fms, loss: %.1f%%)",
					m.Interface, prevStatus, m.Status, m.LiveLatency, m.PacketLoss)
			}
		}
	}

	if stateChanged && h.onStateChange != nil {
		h.onStateChange()
	}
}

func (h *HealthMonitor) executeProbe(iface, target string) (float64, bool) {
	// Sanitize probe target: fallback if empty, ending in .0, or known rate-limited anycast target
	cleanTarget := strings.TrimSpace(target)
	if cleanTarget == "" || cleanTarget == "188.114.97.0" || strings.HasSuffix(cleanTarget, ".0") {
		cleanTarget = "1.1.1.1"
	}

	// ping -c 2 -W 2 -i 0.2 -I <iface> <cleanTarget>
	cmd := exec.Command("ping", "-c", "2", "-W", "2", "-i", "0.2", "-I", iface, cleanTarget)
	out, err := cmd.CombinedOutput()
	if err != nil {
		return 0, false
	}

	str := string(out)
	if strings.Contains(str, "100% packet loss") || strings.Contains(str, " 0 received") || strings.Contains(str, "0 packets received") {
		return 0, false
	}

	matches := rttRegex.FindStringSubmatch(str)
	if len(matches) > 1 {
		if val, err := strconv.ParseFloat(matches[1], 64); err == nil {
			return val, true
		}
	}

	return 1.0, true
}

func (h *HealthMonitor) calculateMetrics(m *SDWANMember, history []probeSample) {
	if len(history) == 0 {
		return
	}

	total := len(history)
	failed := 0
	var sumLat float64
	var countSuccess int
	var latencies []float64

	for _, s := range history {
		if !s.success {
			failed++
		} else {
			sumLat += s.latency
			countSuccess++
			latencies = append(latencies, s.latency)
		}
	}

	m.PacketLoss = (float64(failed) / float64(total)) * 100.0

	if countSuccess > 0 {
		avg := sumLat / float64(countSuccess)
		m.LiveLatency = math.Round(avg*10) / 10

		// Jitter calculation (average difference between consecutive probes)
		if len(latencies) >= 2 {
			var diffSum float64
			for i := 1; i < len(latencies); i++ {
				diffSum += math.Abs(latencies[i] - latencies[i-1])
			}
			m.Jitter = math.Round((diffSum/float64(len(latencies)-1))*10) / 10
		}
	} else {
		m.LiveLatency = 0
		m.Jitter = 0
	}
}