DebTik/internal/sdwan/health_concurrency_test.go

package sdwan

import (
	"sync"
	"testing"
	"time"
)

func TestProbeSnapshotsAndStaleResult(t *testing.T) {
	original := &SDWANMember{ID: "member", Interface: "eth0", TargetProbe: "192.0.2.1", Status: StatusHealthy}
	mgr := &Manager{groups: map[string]*SDWANGroup{"group": {ID: "group", Members: []*SDWANMember{original}}}}
	snap := mgr.List()
	snap[0].Members[0].Status = StatusDown
	if original.Status != StatusHealthy {
		t.Fatal("snapshot mutated live state")
	}
	mgr.mu.Lock()
	original.TargetProbe = "192.0.2.2"
	mgr.mu.Unlock()
	if mgr.applyProbeResults(snap) || original.Status != StatusHealthy {
		t.Fatal("stale probe applied")
	}
	snap = mgr.List()
	snap[0].Members[0].Status = StatusDown
	if !mgr.applyProbeResults(snap) || original.Status != StatusDown {
		t.Fatal("valid health transition not applied")
	}
}
func TestProbeRunsInParallelAndMergesSafely(t *testing.T) {
	mgr := &Manager{groups: map[string]*SDWANGroup{"g": {ID: "g", Members: []*SDWANMember{{ID: "a", Interface: "a"}, {ID: "b", Interface: "b"}}}}}
	h := NewHealthMonitor(nil)
	started := make(chan string, 2)
	release := make(chan struct{})
	h.probe = func(iface, target string) (float64, bool) { started <- iface; <-release; return 5, true }
	snapshot := mgr.List()
	done := make(chan struct{})
	go func() { defer close(done); h.runProbeIteration(snapshot); mgr.applyProbeResults(snapshot) }()
	for i := 0; i < 2; i++ {
		select {
		case <-started:
		case <-time.After(time.Second):
			close(release)
			t.Fatal("probes did not run concurrently")
		}
	}
	var wg sync.WaitGroup
	for i := 0; i < 4; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for j := 0; j < 1000; j++ {
				mgr.List()
				mgr.GetTelemetrySummary()
			}
		}()
	}
	close(release)
	<-done
	wg.Wait()
	for _, m := range mgr.List()[0].Members {
		if m.Status != StatusHealthy || m.LastChecked.IsZero() {
			t.Fatalf("probe missing: %+v", m)
		}
	}
}

func TestFailoverProbeHysteresisAndRecovery(t *testing.T) {
	mgr := &Manager{groups: map[string]*SDWANGroup{"g": {ID: "g", Mode: ModeFailover, Members: []*SDWANMember{{ID: "a", Interface: "a", Gateway: "192.0.2.1", Priority: 1, Status: StatusHealthy}, {ID: "b", Interface: "b", Gateway: "192.0.2.2", Priority: 2, Status: StatusHealthy}}}}, engine: NewKernelEngine()}
	h := NewHealthMonitor(nil)
	primaryUp := false
	h.probe = func(iface, target string) (float64, bool) { return 5, iface != "a" || primaryUp }
	cycle := func() { snapshot := mgr.List(); h.runProbeIteration(snapshot); mgr.applyProbeResults(snapshot) }
	for i := 0; i < 4; i++ {
		cycle()
		hops, ok := mgr.ResolveGroupForRouting("g")
		if !ok || len(hops) != 1 || hops[0].Interface != "a" {
			t.Fatalf("failover too early at %d: %+v", i, hops)
		}
	}
	cycle()
	hops, ok := mgr.ResolveGroupForRouting("g")
	if !ok || len(hops) != 1 || hops[0].Interface != "b" {
		t.Fatalf("backup not selected: %+v", hops)
	}
	primaryUp = true
	cycle()
	hops, _ = mgr.ResolveGroupForRouting("g")
	if len(hops) != 1 || hops[0].Interface != "b" {
		t.Fatalf("recovered too early: %+v", hops)
	}
	cycle()
	hops, _ = mgr.ResolveGroupForRouting("g")
	if len(hops) != 1 || hops[0].Interface != "a" {
		t.Fatalf("primary not restored: %+v", hops)
	}
}

func TestProbeIntervalsPerGroup(t *testing.T) {
	now := time.Unix(1000, 0)
	last := map[string]time.Time{}
	groups := []*SDWANGroup{{ID: "fast", ProbeInterval: 3}, {ID: "slow", ProbeInterval: 10}, {ID: "disabled", Disabled: true}}
	if due := dueProbeGroups(groups, last, now); len(due) != 2 {
		t.Fatalf("initial probes: %d", len(due))
	}
	if due := dueProbeGroups(groups, last, now.Add(2*time.Second)); len(due) != 0 {
		t.Fatalf("probed too early: %d", len(due))
	}
	if due := dueProbeGroups(groups, last, now.Add(3*time.Second)); len(due) != 1 || due[0].ID != "fast" {
		t.Fatalf("3-second interval: %+v", due)
	}
	if due := dueProbeGroups(groups, last, now.Add(10*time.Second)); len(due) != 2 {
		t.Fatalf("10-second interval: %+v", due)
	}
	dueProbeGroups(groups[:1], last, now.Add(11*time.Second))
	if _, ok := last["slow"]; ok {
		t.Fatal("removed group retained schedule")
	}
}