package aerotun import ( "encoding/json" "fmt" "os" "path/filepath" "sync" "time" "golang.org/x/sys/unix" ) const healthHistoryLimit = 500 type HealthEvent struct { Time time.Time `json:"time"` TunnelID string `json:"tunnel_id"` Name string `json:"name"` Type string `json:"type"` Status string `json:"status"` Previous string `json:"previous,omitempty"` PreviousDurationSeconds float64 `json:"previous_duration_seconds,omitempty"` Detail string `json:"detail,omitempty"` } type healthState struct { name, status string since time.Time } type healthMonitor struct { mu sync.Mutex checkMu sync.Mutex path string events []HealthEvent states map[string]healthState done, stopped chan struct{} once sync.Once lastCPUSeconds float64 lastCPUAt time.Time processCPU *float64 } func newHealthMonitor(path string) *healthMonitor { h := &healthMonitor{path: path, states: make(map[string]healthState), done: make(chan struct{}), stopped: make(chan struct{})} if data, err := os.ReadFile(path); err == nil && len(data) <= 1024*1024 { _ = json.Unmarshal(data, &h.events) } if len(h.events) > healthHistoryLimit { h.events = h.events[len(h.events)-healthHistoryLimit:] } return h } func (h *healthMonitor) stop() { h.once.Do(func() { close(h.done) }); <-h.stopped } func tunnelHealthStatus(t TunnelInstance) string { if t.Disabled { return "disabled" } if !t.Running { if t.LastError != "" { return "error" } return "stopped" } if t.Connected { return "connected" } return "waiting" } func (h *healthMonitor) addLocked(e HealthEvent) { h.events = append(h.events, e) if len(h.events) > healthHistoryLimit { h.events = append([]HealthEvent(nil), h.events[len(h.events)-healthHistoryLimit:]...) } } func (h *healthMonitor) saveLocked() error { data, err := json.Marshal(h.events) if err != nil { return err } if err = os.MkdirAll(filepath.Dir(h.path), 0700); err != nil { return err } f, err := os.CreateTemp(filepath.Dir(h.path), ".aerotun-health-*") if err != nil { return err } defer os.Remove(f.Name()) if err = f.Chmod(0600); err == nil { _, err = f.Write(data) } if err == nil { err = f.Sync() } if closeErr := f.Close(); err == nil { err = closeErr } if err != nil { return err } return os.Rename(f.Name(), h.path) } func (h *healthMonitor) observe(rows []TunnelInstance, now time.Time) { h.mu.Lock() defer h.mu.Unlock() var usage unix.Rusage if unix.Getrusage(unix.RUSAGE_SELF, &usage) == nil { seconds := float64(usage.Utime.Sec) + float64(usage.Utime.Usec)/1e6 + float64(usage.Stime.Sec) + float64(usage.Stime.Usec)/1e6 elapsed := now.Sub(h.lastCPUAt).Seconds() if !h.lastCPUAt.IsZero() && elapsed > 0 && seconds >= h.lastCPUSeconds { v := (seconds - h.lastCPUSeconds) / elapsed * 100 h.processCPU = &v } h.lastCPUAt = now h.lastCPUSeconds = seconds } seen := make(map[string]bool) changed := false for _, row := range rows { seen[row.ID] = true status := tunnelHealthStatus(row) old, ok := h.states[row.ID] if !ok || old.status != status { e := HealthEvent{Time: now, TunnelID: row.ID, Name: row.Name, Type: "state", Status: status, Detail: row.LastError} if ok { e.Previous = old.status e.PreviousDurationSeconds = now.Sub(old.since).Seconds() } h.addLocked(e) h.states[row.ID] = healthState{name: row.Name, status: status, since: now} changed = true } else if old.name != row.Name { old.name = row.Name h.states[row.ID] = old } } for id, old := range h.states { if !seen[id] { h.addLocked(HealthEvent{Time: now, TunnelID: id, Name: old.name, Type: "state", Status: "removed", Previous: old.status, PreviousDurationSeconds: now.Sub(old.since).Seconds()}) delete(h.states, id) changed = true } } if changed { if err := h.saveLocked(); err != nil { fmt.Fprintf(os.Stderr, "AeroTUN health history: %v\n", err) } } } func (h *healthMonitor) history(id string) []HealthEvent { h.mu.Lock() defer h.mu.Unlock() out := make([]HealthEvent, 0, len(h.events)) for i := len(h.events) - 1; i >= 0; i-- { if id == "" || h.events[i].TunnelID == id { out = append(out, h.events[i]) } } return out } func (h *healthMonitor) status(id string) (string, time.Time) { h.mu.Lock() defer h.mu.Unlock() s := h.states[id] return s.status, s.since } func (m *Manager) monitorHealth() { defer close(m.health.stopped) tick := time.NewTicker(2 * time.Second) defer tick.Stop() for { m.health.observe(m.List(), time.Now()) select { case <-m.health.done: return case <-tick.C: } } } func (m *Manager) HealthHistory(id string) []HealthEvent { if m.health == nil { return []HealthEvent{} } return m.health.history(id) } func (m *Manager) ProcessCPUPercent() *float64 { if m.health == nil { return nil } m.health.mu.Lock() defer m.health.mu.Unlock() if m.health.processCPU == nil { return nil } v := *m.health.processCPU return &v } type HealthSummary struct { TunnelID string `json:"tunnel_id"` Status string `json:"status"` Since time.Time `json:"since"` } func (m *Manager) HealthSummaries() []HealthSummary { rows := m.List() out := make([]HealthSummary, 0, len(rows)) for _, t := range rows { status, since := tunnelHealthStatus(t), time.Time{} if m.health != nil { if recorded, at := m.health.status(t.ID); recorded == status { since = at } } out = append(out, HealthSummary{TunnelID: t.ID, Status: status, Since: since}) } return out }