WiSearch/dnscrawl.go

package main

import (
	"bytes"
	"crypto/tls"
	"database/sql"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"net/url"
	"os"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/PuerkitoBio/goquery"
	_ "github.com/lib/pq"
	"github.com/miekg/dns"
	"golang.org/x/net/html/charset"
)

// --- CONFIGURATION STRUCTURES ---
type DBConfig struct {
	Host     string `json:"host"`
	Port     string `json:"port"`
	User     string `json:"user"`
	Password string `json:"password"`
	DBName   string `json:"dbname"`
	SSLMode  string `json:"sslmode"`
}

type CFCredential struct {
	Comment   string `json:"comment"`
	AccountID string `json:"account_id"`
	APIToken  string `json:"api_token"`
	Model     string `json:"model"`
}

type AppConfig struct {
	Database        DBConfig       `json:"database"`
	DBConnLegacy    string         `json:"db_conn,omitempty"`
	StartServers    []string       `json:"start_servers"`
	ZoneNames       []string       `json:"zone_names"`
	MaxDepth        int            `json:"max_depth"`
	MaxWorkers      int            `json:"max_workers"`
	MaxPagesPerHost int            `json:"max_pages_per_host"`
	WebUIPort       string         `json:"web_ui_port"`
	ScheduleHours   int            `json:"schedule_hours"`
	ScheduleMinutes int            `json:"schedule_minutes"`
	ScanOnStart     bool           `json:"scan_on_start"`
	CoreServices    []string       `json:"core_services"`
	CFCredentials   []CFCredential `json:"cf_credentials"`
	AIPrompt        string         `json:"ai_prompt"`
}

func (c DBConfig) GetConnString() string {
	return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", c.Host, c.Port, c.User, c.Password, c.DBName, c.SSLMode)
}

// --- GLOBAL VARIABLES ---
var (
	cfg       AppConfig
	cfgMutex  sync.RWMutex
	dbGlobal  *sql.DB
	
	daemonEnabled bool = true
	isScanning    bool
	stopRequested int32
	lastRun       time.Time
	nextRunTime   time.Time
	stateMutex    sync.Mutex

	visitedZones   sync.Map
	visitedWeb     sync.Map
	hostVisitCount sync.Map

	sitesOffline uint64
	sitesActive  uint64
	sitesSkipped uint64
	mediaFound   uint64

	spiderPending int32
	aiPending     int32
	
	spiderQueue = make(chan SpiderJob, 300000)
	aiQueue     = make(chan AIJob, 100000)

	globalStats = &StatsManager{CategoryCounts: make(map[string]int)}

	forceLookups = []string{
		"minecraft.maiden2", // ΝΕΟ: Αναγκάζει να ψάξει αυτό πρώτο!
		"routemap", "activeservices", "router", "stats", "www", "node", "auth",
		"camera", "weather", "voip", "irc", "matrix", "speedtest", "proxy",
		"mirror", "archive", "git", "cloud", "files", "game", "mc", "ts3",
		"mumble", "ntp", "dns", "mail", "iot", "sensor", "radio", "tv", "stream",
		"torrent", "tracker", "portal", "p2p", "dcplusplus", "hub", "search",
		"forum", "board", "bb", "vbulletin", "smf", "phpbb", "discuss",
	}

	httpClient = &http.Client{
		Timeout: 8 * time.Second,
		Transport: &http.Transport{
			TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
			MaxIdleConns:        1000,
			MaxIdleConnsPerHost: 100,
		},
	}
	aiClient = &http.Client{Timeout: 30 * time.Second}

	logBuffer []string
	logMutex  sync.Mutex
)

type SpiderJob struct {
	RecordID   int
	CurrentURL string
	BaseHost   string
	Depth      int
}

type AIJob struct {
	URL       string
	Title     string
	MetaDesc  string
	Generator string
	H1        string
	Preview   string
}

type AIAnalysis struct {
	Category string `json:"category"`
	Summary  string `json:"summary"`
	Status   string `json:"status"`
}

type StatsManager struct {
	sync.Mutex
	CategoryCounts map[string]int
}

func (s *StatsManager) Update(category string) {
	s.Lock()
	defer s.Unlock()
	s.CategoryCounts[category]++
}

func (s *StatsManager) Reset() {
	s.Lock()
	defer s.Unlock()
	s.CategoryCounts = make(map[string]int)
}

func logMsg(format string, a ...interface{}) {
	msg := fmt.Sprintf("[%s] %s", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...))
	
	// 1. Τυπώνει στην οθόνη (Terminal)
	fmt.Println(msg)
	
	logMutex.Lock()
	
	// 2. Το κρατάει στη μνήμη για το Web UI (Dashboard)
	logBuffer = append(logBuffer, msg)
	if len(logBuffer) > 300 { logBuffer = logBuffer[1:] }
	
	// 3. ΝΕΟ: Το γράφει μόνιμα στο αρχείο /var/log/wisearch-crawler.log
	f, err := os.OpenFile("/var/log/wisearch-crawler.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
	if err == nil {
		f.WriteString(msg + "\n")
		f.Close()
	} else {
		// Αν δεν έχει δικαιώματα, το τυπώνει στην οθόνη για να το δεις
		fmt.Printf("[⚠️ FILE ERROR] Δεν μπορώ να γράψω στο log file: %v\n", err)
	}

	logMutex.Unlock()
}

func loadConfig() {
	cfgMutex.Lock()
	defer cfgMutex.Unlock()

	defaultPrompt := "You are an AWMN web classifier. Return ONLY JSON. Do not include markdown.\nCategories: \"Router & Admin\", \"Network Tools & Stats\", \"Media & Streaming\", \"P2P & Torrents\", \"Storage & FTP\", \"Forums & Communities\", \"Communication & VoIP\", \"Gaming\", \"IoT & Sensors\", \"Web Services & Blogs\", \"Default Page\", \"Other\".\nFormat: {\"category\": \"Exact category\", \"summary\": \"Short Greek summary, max 8 words\", \"status\": \"Active\"}"

	data, err := os.ReadFile("config.json")
	if err != nil {
		logMsg("[⚠️ SYSTEM] config.json not found. Creating default configuration.")
		cfg = AppConfig{
			Database:        DBConfig{Host: "192.168.131.124", Port: "5432", User: "postgres", Password: "postpass", DBName: "wisearch", SSLMode: "disable"},
			StartServers:    []string{"192.168.131.98"},
			ZoneNames:       []string{"awmn."},
			MaxDepth:        50,
			MaxWorkers:      60,
			MaxPagesPerHost: 10000,
			WebUIPort:       ":8080",
			ScheduleHours:   12,
			ScheduleMinutes: 0,
			ScanOnStart:     false,
			AIPrompt:        defaultPrompt,
			CoreServices:    []string{}, // Άδειο από default
			CFCredentials:   []CFCredential{}, // Άδειο από default
		}
		saveConfigInternal()
		return
	}
	json.Unmarshal(data, &cfg)
	if cfg.AIPrompt == "" {
		cfg.AIPrompt = defaultPrompt
	}
	logMsg("[🔧 SYSTEM] Configuration loaded.")
}

func saveConfigInternal() {
	cfg.DBConnLegacy = ""
	data, _ := json.MarshalIndent(cfg, "", "  ")
	os.WriteFile("config.json", data, 0644)
}

func isCoreService(hostname string) bool {
	cfgMutex.RLock()
	defer cfgMutex.RUnlock()
	for _, svc := range cfg.CoreServices {
		if svc == hostname { return true }
	}
	return false
}

func main() {
	loadConfig()
	logMsg("[🚀 SYSTEM] WiSearch Daemon Initialized | C&C Mode")

	var err error
	dbGlobal, err = sql.Open("postgres", cfg.Database.GetConnString())
	if err != nil { log.Fatal("[❌ FATAL] Database connection string failed:", err) }
	
	if err = dbGlobal.Ping(); err != nil {
		logMsg("[❌ FATAL DB ERROR] Η Βάση Δεδομένων δεν απαντάει! Έλεγξε IP/Κωδικό: %v", err)
	} else {
		logMsg("[✅ SYSTEM] Database connection successful.")
	}

	defer dbGlobal.Close()
	dbGlobal.SetMaxOpenConns(150)
	dbGlobal.SetMaxIdleConns(75)

	go startWebUI()

	cfgMutex.RLock()
	workers := cfg.MaxWorkers
	onStart := cfg.ScanOnStart
	cfgMutex.RUnlock()

	for i := 0; i < workers; i++ { go spiderWorker() }
	for i := 0; i < 4; i++ { go aiWorker() }

	calcNextRun()
	go schedulerLoop()
	go watchdogLoop()

	if onStart {
		logMsg("[▶️ SYSTEM] ScanOnStart is enabled. Triggering immediate scan.")
		triggerScan()
	}

	select {}
}

func calcNextRun() {
	cfgMutex.RLock()
	h, m := cfg.ScheduleHours, cfg.ScheduleMinutes
	cfgMutex.RUnlock()
	
	stateMutex.Lock()
	defer stateMutex.Unlock()
	
	if h == 0 && m == 0 {
		nextRunTime = time.Time{}
		return
	}
	
	interval := time.Duration(h)*time.Hour + time.Duration(m)*time.Minute
	now := time.Now()
	
	if lastRun.IsZero() { lastRun = now }
	nextRunTime = lastRun.Add(interval)
	if nextRunTime.Before(now) {
		lastRun = now
		nextRunTime = now.Add(interval)
	}
}

func triggerScan() {
	stateMutex.Lock()
	if isScanning {
		stateMutex.Unlock()
		logMsg("[⚠️ SYSTEM] Scan is already running. Ignoring trigger.")
		return
	}
	isScanning = true
	lastRun = time.Now()
	stateMutex.Unlock()
	
	calcNextRun()

	atomic.StoreInt32(&stopRequested, 0)
	logMsg("[🔄 SYSTEM] Initiating New Multi-Zone Scan Cycle...")

	visitedZones = sync.Map{}
	visitedWeb = sync.Map{}
	hostVisitCount = sync.Map{}
	
	atomic.StoreUint64(&sitesActive, 0)
	atomic.StoreUint64(&sitesOffline, 0)
	atomic.StoreUint64(&sitesSkipped, 0)
	atomic.StoreUint64(&mediaFound, 0)
	globalStats.Reset()

	cfgMutex.RLock()
	zones := cfg.ZoneNames
	servers := cfg.StartServers
	cfgMutex.RUnlock()

	for _, z := range zones {
		for _, s := range servers {
			atomic.AddInt32(&spiderPending, 1)
			go fetchAndStoreDNS(z, s)
		}
	}

	go processSubmittedURLs()
}

func stopScan() {
	atomic.StoreInt32(&stopRequested, 1)
	logMsg("[🛑 SYSTEM] Emergency Stop Requested! Draining queues...")
}

func schedulerLoop() {
	for {
		time.Sleep(10 * time.Second)
		stateMutex.Lock()
		scanning := isScanning
		enabled := daemonEnabled
		target := nextRunTime
		stateMutex.Unlock()

		if !enabled || scanning || target.IsZero() { continue }

		if time.Now().After(target) {
			logMsg("[⏰ SCHEDULE] Scheduled time reached. Starting scan.")
			triggerScan()
		}
	}
}

func watchdogLoop() {
	idleSeconds := 0
	for {
		time.Sleep(5 * time.Second)
		
		stateMutex.Lock()
		scanning := isScanning
		stateMutex.Unlock()

		if !scanning { continue }

		sPend := atomic.LoadInt32(&spiderPending)
		aPend := atomic.LoadInt32(&aiPending)
		stopped := atomic.LoadInt32(&stopRequested) == 1
		
		if sPend == 0 && aPend == 0 && len(spiderQueue) == 0 && len(aiQueue) == 0 {
			idleSeconds += 5
			if idleSeconds >= 15 || stopped {
				if stopped {
					logMsg("[🛑 SYSTEM] Scan aborted successfully.")
				} else {
					logMsg("[✅ SYSTEM] Scan Cycle Complete!")
					logMsg("[🔄 SYSTEM] Updating PageRank scores...")
					updatePageRank()
				}
				
				stateMutex.Lock()
				isScanning = false
				stateMutex.Unlock()
				
				idleSeconds = 0
			}
		} else {
			idleSeconds = 0
		}
	}
}

func processSubmittedURLs() {
	rows, err := dbGlobal.Query(`SELECT id, url FROM submitted_urls`)
	if err != nil { return }
	defer rows.Close()

	for rows.Next() {
		var id int
		var rawURL string
		if err := rows.Scan(&id, &rawURL); err == nil {
			u, pErr := url.Parse(rawURL)
			if pErr == nil && u.Hostname() != "" {
				baseHost := strings.ToLower(u.Hostname())

				var zID int
				dbGlobal.QueryRow(`INSERT INTO zones (zone_name, dns_server_ip) VALUES ('custom_submissions', '0.0.0.0') ON CONFLICT (zone_name) DO UPDATE SET last_synced = CURRENT_TIMESTAMP RETURNING id`).Scan(&zID)

				var recID int
				dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, 'A', 'Χειροκίνητο') ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zID, baseHost).Scan(&recID)

				if recID > 0 {
					atomic.AddInt32(&spiderPending, 1)
					spiderQueue <- SpiderJob{RecordID: recID, CurrentURL: rawURL, BaseHost: baseHost, Depth: 0}
					dbGlobal.Exec(`DELETE FROM submitted_urls WHERE id = $1`, id)
					logMsg("[🔗 SUBMIT] Βρέθηκε χειροκίνητο URL: %s", rawURL)
				}
			}
		}
	}
}

func spiderWorker() {
	for job := range spiderQueue {
		if atomic.LoadInt32(&stopRequested) == 0 {
			processSpider(job)
		}
		atomic.AddInt32(&spiderPending, -1)
	}
}

func aiWorker() {
	for job := range aiQueue {
		if atomic.LoadInt32(&stopRequested) == 0 {
			analysis := getDeepAnalysis(job.URL, job.Title, job.MetaDesc, job.Generator, job.H1, job.Preview)
			u, err := url.Parse(job.URL)
			if err == nil {
				hostname := strings.ToLower(strings.TrimSuffix(u.Hostname(), "."))
				if isCoreService(hostname) { analysis.Category = "Core Services" }
			}

			globalStats.Update(analysis.Category)
			logMsg("[🤖 AI-TAG] %-25s -> [%s]", u.Hostname(), analysis.Category)

			dbGlobal.Exec(`UPDATE web_health SET category = $1, ai_summary = $2, activity_status = $3 WHERE url = $4`,
				analysis.Category, analysis.Summary, analysis.Status, job.URL)
			time.Sleep(1000 * time.Millisecond)
		}
		atomic.AddInt32(&aiPending, -1)
	}
}

func processSpider(job SpiderJob) {
	cfgMutex.RLock()
	mDepth := cfg.MaxDepth
	mPages := cfg.MaxPagesPerHost
	cfgMutex.RUnlock()

	if job.Depth > mDepth { return }
	
	cleanURL := cleanURLForDB(job.CurrentURL)
	if _, loaded := visitedWeb.LoadOrStore(cleanURL, true); loaded { return }

	lowerURL := strings.ToLower(cleanURL)
	skipExt := []string{".jpg", ".jpeg", ".png", ".gif", ".pdf", ".zip", ".php", "graphs", "?print=", "&print=", ".iso", ".rar", ".torrent", ".mkv", ".mp4", ".mp3"}
	for _, ext := range skipExt {
		if strings.Contains(lowerURL, ext) {
			atomic.AddUint64(&sitesSkipped, 1)
			return
		}
	}

	vCountRaw, _ := hostVisitCount.LoadOrStore(job.BaseHost, 0)
	vCount := vCountRaw.(int)
	if vCount >= mPages { return }
	hostVisitCount.Store(job.BaseHost, vCount+1)

	// --- HTTP REQUEST (ΜΕ FAKE USER AGENT) ---
	req, _ := http.NewRequest("GET", cleanURL, nil)
	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 WiSearchBot/1.0")
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
	
	resp, err := httpClient.Do(req)
	if err != nil { 
		atomic.AddUint64(&sitesOffline, 1)
		logMsg("[❌ DEAD] Σφάλμα σύνδεσης στο %s: %v", cleanURL, err)
		dbGlobal.Exec(`UPDATE web_health SET is_up = false, activity_status = 'Down', last_crawled = CURRENT_TIMESTAMP WHERE url = $1`, cleanURL)
		return 
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 { 
		atomic.AddUint64(&sitesOffline, 1)
		logMsg("[⚠️ HTTP ERROR] Το %s γύρισε κωδικό %d", cleanURL, resp.StatusCode)
		return 
	}

	contentType := resp.Header.Get("Content-Type")
	if !strings.Contains(contentType, "text/html") {
		atomic.AddUint64(&sitesSkipped, 1)
		logMsg("[⚠️ SKIPPED-TYPE] URL: %s | Ήρθε με Content-Type: %s", cleanURL, contentType)
		return
	}

	utf8Body, _ := decodeToUTF8(resp.Body, contentType)
	doc, err := goquery.NewDocumentFromReader(utf8Body)
	if err != nil { return }

	doc.Find("img").Each(func(i int, s *goquery.Selection) {
		src, exists := s.Attr("src")
		alt, _ := s.Attr("alt")
		if exists && len(src) > 2 && !strings.HasPrefix(src, "data:image") {
			u, err := url.Parse(src)
			if err == nil {
				base, _ := url.Parse(job.CurrentURL)
				absImg := base.ResolveReference(u).String()
				dbGlobal.Exec(`INSERT INTO media_files (source_url, media_url, media_type, alt_text) VALUES ($1, $2, 'image', $3) ON CONFLICT DO NOTHING`, cleanURL, absImg, strings.TrimSpace(alt))
				atomic.AddUint64(&mediaFound, 1)
			}
		}
	})

	title := strings.TrimSpace(doc.Find("title").Text())
	metaDesc, _ := doc.Find("meta[name=description]").Attr("content")
	generator, _ := doc.Find("meta[name=generator]").Attr("content")
	h1 := strings.TrimSpace(doc.Find("h1").First().Text())

	doc.Find("script, style, nav, footer, header, aside, noscript, iframe, svg, canvas").Remove()
	bodyText := strings.Join(strings.Fields(doc.Find("body").Text()), " ")

	// --- ΔΙΟΡΘΩΣΗ: ΠΙΟ ΕΛΑΣΤΙΚΟΣ ΚΟΦΤΗΣ ΧΑΡΑΚΤΗΡΩΝ ---
	if len(strings.TrimSpace(bodyText)) < 5 && title == "" {
		atomic.AddUint64(&sitesSkipped, 1)
		logMsg("[⚠️ SKIPPED-EMPTY] URL: %s | Βρήκε κείμενο μήκους: %d | Τίτλος: '%s'", cleanURL, len(strings.TrimSpace(bodyText)), title)
		return
	}

	atomic.AddUint64(&sitesActive, 1)
	logMsg("[✅ ALIVE] Active Node: %s", cleanURL)

	dbGlobal.Exec(`INSERT INTO web_health (dns_record_id, url, status_code, page_title, is_up, last_crawled, full_content)
		VALUES ($1, $2, $3, $4, $5, CURRENT_TIMESTAMP, $6)
		ON CONFLICT (url) DO UPDATE SET page_title = EXCLUDED.page_title, is_up = true, last_crawled = CURRENT_TIMESTAMP, full_content = EXCLUDED.full_content`,
		job.RecordID, cleanURL, resp.StatusCode, title, true, bodyText)

	runes := []rune(bodyText)
	preview := ""
	if len(runes) > 400 { preview = string(runes[:400]) } else { preview = string(runes) }

	atomic.AddInt32(&aiPending, 1)
	aiQueue <- AIJob{URL: cleanURL, Title: title, MetaDesc: metaDesc, Generator: generator, H1: h1, Preview: preview}

	// --- ΕΞΑΓΩΓΗ LINKS (SPIDER) ---
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		if atomic.LoadInt32(&stopRequested) == 1 { return }
		href, _ := s.Attr("href")
		u, err := url.Parse(href)
		if err == nil {
			base, _ := url.Parse(job.CurrentURL)
			abs := base.ResolveReference(u)
			
			absHost := strings.ToLower(abs.Hostname())
			
			isAllowed := false
			
			// 1. Είναι το ίδιο site ή subdomain (πχ wind.awmn -> stats.wind.awmn)
			if absHost == job.BaseHost || strings.HasSuffix(absHost, "."+job.BaseHost) {
				isAllowed = true
			} else if strings.HasSuffix(absHost, ".awmn") || strings.HasSuffix(absHost, ".local") {
				// 2. Είναι οποιοδήποτε άλλο site στο δίκτυο (.awmn ή .local)
				isAllowed = true
			} else {
				// 3. Ή ταιριάζει με κάτι που έχουμε βάλει στα Zones στο Admin Panel
				cfgMutex.RLock()
				for _, z := range cfg.ZoneNames {
					cz := strings.ToLower(strings.TrimSuffix(z, "."))
					if strings.HasSuffix(absHost, "."+cz) || absHost == cz {
						isAllowed = true
						break
					}
				}
				cfgMutex.RUnlock()
			}
			
			if isAllowed && (abs.Scheme == "http" || abs.Scheme == "https") {
				target := cleanURLForDB(abs.String())
				if _, loaded := visitedWeb.Load(target); !loaded {
					dbGlobal.Exec(`INSERT INTO web_links (source_url, target_url) VALUES ($1, $2) ON CONFLICT DO NOTHING`, cleanURL, target)
					
					targetRecordID := job.RecordID
					
					if absHost != job.BaseHost {
						var zID int
						dbGlobal.QueryRow(`INSERT INTO zones (zone_name, dns_server_ip) VALUES ('auto_discovered', '0.0.0.0') ON CONFLICT (zone_name) DO UPDATE SET last_synced = CURRENT_TIMESTAMP RETURNING id`).Scan(&zID)
						dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, 'A', 'Scraped') ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zID, absHost).Scan(&targetRecordID)
					}

					if targetRecordID > 0 {
						atomic.AddInt32(&spiderPending, 1)
						spiderQueue <- SpiderJob{RecordID: targetRecordID, CurrentURL: target, BaseHost: absHost, Depth: job.Depth + 1}
					}
				}
			}
		}
	})
}

func getDeepAnalysis(targetURL, title, metaDesc, generator, h1, body string) AIAnalysis {
	if len(strings.TrimSpace(body)) < 30 && strings.TrimSpace(title) == "" {
		return AIAnalysis{Category: "Other", Summary: "Insufficient content.", Status: "Active"}
	}

	cfgMutex.RLock()
	systemPrompt := cfg.AIPrompt
	creds := cfg.CFCredentials
	cfgMutex.RUnlock()
	
	if systemPrompt == "" {
		systemPrompt = "You are an AWMN web classifier. Return ONLY JSON."
	}
	
	userPrompt := fmt.Sprintf("URL: %s\nTitle: %s\nMeta: %s\nContent: %s", targetURL, title, metaDesc, body)
	payload := map[string]interface{}{"messages": []map[string]string{{"role": "system", "content": systemPrompt}, {"role": "user", "content": userPrompt}}}
	bodyJSON, _ := json.Marshal(payload)

	for _, cred := range creds {
		if atomic.LoadInt32(&stopRequested) == 1 { break }
		apiURL := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/ai/run/%s", cred.AccountID, cred.Model)
		req, _ := http.NewRequest("POST", apiURL, bytes.NewBuffer(bodyJSON))
		req.Header.Set("Authorization", "Bearer "+cred.APIToken) 
		req.Header.Set("Content-Type", "application/json")
		
		resp, err := aiClient.Do(req)
		if err != nil { continue }
		if resp.StatusCode != 200 { resp.Body.Close(); continue }

		var cfRes struct { Result struct { Response string `json:"response"` } `json:"result"`; Success bool `json:"success"` }
		json.NewDecoder(resp.Body).Decode(&cfRes)
		resp.Body.Close()

		cleanJSON := cfRes.Result.Response
		if start := strings.Index(cleanJSON, "{"); start != -1 {
			if end := strings.LastIndex(cleanJSON, "}"); end != -1 && start <= end {
				cleanJSON = cleanJSON[start : end+1]
			}
		}

		var analysis AIAnalysis
		if err := json.Unmarshal([]byte(cleanJSON), &analysis); err == nil {
			if analysis.Category == "" { analysis.Category = "Other" }
			return analysis
		}
	}
	return AIAnalysis{Category: "Other", Summary: "AI Classification Failed", Status: "Active"}
}

func cleanURLForDB(inputURL string) string {
	u, err := url.Parse(inputURL)
	if err != nil { return inputURL }
	q := u.Query()
	
	// --- ΔΙΟΡΘΩΣΗ ΛΑΒΥΡΙΝΘΟΥ: Αφαιρέσαμε τα junk parameters (node, page, id κλπ) ---
	junk := []string{
		"sid", "s", "osid", "sort", "order", "phpsessid", "view", "lang", 
		"search", "highlight", "mode", "start", "form_search_nodes_search", 
		"limit", "offset", "direction", "node", "page", "table_nodes_showpage", 
		"action", "id", "p", "t",
	}
	for _, k := range junk { q.Del(k) }
	
	u.RawQuery = q.Encode()
	u.Fragment = ""
	return u.String()
}

func decodeToUTF8(r io.Reader, contentType string) (io.Reader, error) {
	buf, err := io.ReadAll(r)
	if err != nil { return nil, err }
	determineEncoding, _, _ := charset.DetermineEncoding(buf, contentType)
	utf8Reader := determineEncoding.NewDecoder().Reader(bytes.NewReader(buf))
	finalBuf, _ := io.ReadAll(utf8Reader)
	cleanStr := strings.Map(func(r rune) rune {
		if r == '\uFFFD' || r < 0 { return -1 }
		return r
	}, string(finalBuf))
	return strings.NewReader(cleanStr), nil
}

func updatePageRank() {
	dbGlobal.Exec(`UPDATE web_health wh SET pagerank_score = (1.0 + (SELECT COUNT(*) FROM web_links wl WHERE wl.target_url = wh.url)) * (CASE WHEN wh.category = 'Core Services' THEN 10.0 WHEN wh.category IN ('Gaming', 'Media & Streaming', 'P2P & Torrents', 'Web Services & Blogs', 'Forums & Communities', 'Communication & VoIP') THEN 3.0 WHEN wh.category IN ('Storage & FTP') THEN 2.0 WHEN wh.category IN ('Network Tools & Stats', 'IoT & Sensors') THEN 1.0 ELSE 1.0 END)`)
}

// --- DNS LOGIC ---
func fetchAndStoreDNS(zoneName string, dnsServer string) {
	defer atomic.AddInt32(&spiderPending, -1)
	if atomic.LoadInt32(&stopRequested) == 1 { return }

	zoneName = dns.Fqdn(strings.ToLower(zoneName))
	if _, loaded := visitedZones.LoadOrStore(zoneName, true); loaded { return }

	var zoneID int
	err := dbGlobal.QueryRow(`INSERT INTO zones (zone_name, dns_server_ip) VALUES ($1, $2) ON CONFLICT (zone_name) DO UPDATE SET last_synced = CURRENT_TIMESTAMP RETURNING id`, strings.TrimSuffix(zoneName, "."), dnsServer).Scan(&zoneID)
	if err != nil {
		logMsg("[❌ DB ERROR] Αποτυχία αποθήκευσης Ζώνης '%s': %v", zoneName, err)
		return
	}

	atomic.AddInt32(&spiderPending, 1)
	go directLookup(zoneID, zoneName, dnsServer)

	for _, sub := range forceLookups {
		atomic.AddInt32(&spiderPending, 1)
		go directLookup(zoneID, fmt.Sprintf("%s.%s", sub, zoneName), dnsServer)
	}

	m := new(dns.Msg); m.SetAxfr(zoneName)
	tr := &dns.Transfer{DialTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second}
	
	channel, err := tr.In(m, net.JoinHostPort(dnsServer, "53"))
	if err != nil {
		logMsg("[⚠️ DNS] Zone Transfer (AXFR) Failed for %s: %v", zoneName, err)
		return 
	}
	
	for env := range channel {
		if env.Error != nil { 
			logMsg("[⚠️ DNS] Ενδιάμεσο σφάλμα AXFR για %s: %v", zoneName, env.Error)
			break 
		}

		for _, rr := range env.RR {
			if atomic.LoadInt32(&stopRequested) == 1 { return }
			hName := strings.ToLower(rr.Header().Name)
			
			switch t := rr.(type) {
			case *dns.A:
				var lastID int
				err := dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, $3, $4) ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zoneID, hName, "A", t.A.String()).Scan(&lastID)
				
				if err != nil {
					logMsg("[❌ DB ERROR] Αποτυχία αποθήκευσης DNS ρεκόρ '%s': %v", hName, err)
				} else if lastID > 0 {
					atomic.AddInt32(&spiderPending, 1)
					spiderQueue <- SpiderJob{RecordID: lastID, CurrentURL: "http://" + strings.TrimSuffix(hName, "."), BaseHost: strings.TrimSuffix(hName, "."), Depth: 0}
				}
			
			case *dns.CNAME:
				var lastID int
				err := dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, $3, $4) ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zoneID, hName, "CNAME", t.Target).Scan(&lastID)
				if err == nil && lastID > 0 {
					atomic.AddInt32(&spiderPending, 1)
					spiderQueue <- SpiderJob{RecordID: lastID, CurrentURL: "http://" + strings.TrimSuffix(hName, "."), BaseHost: strings.TrimSuffix(hName, "."), Depth: 0}
				}
			
			case *dns.NS:
				if dns.Fqdn(hName) != zoneName {
					atomic.AddInt32(&spiderPending, 1)
					go func(sz, nt string) {
						defer atomic.AddInt32(&spiderPending, -1)
						c := new(dns.Client); c.Timeout = 2 * time.Second
						m2 := new(dns.Msg); m2.SetQuestion(dns.Fqdn(nt), dns.TypeA)
						r, _, _ := c.Exchange(m2, net.JoinHostPort(dnsServer, "53"))
						if r != nil && len(r.Answer) > 0 {
							if a, ok := r.Answer[0].(*dns.A); ok {
								atomic.AddInt32(&spiderPending, 1)
								fetchAndStoreDNS(sz, a.A.String())
							}
						}
					}(hName, t.Ns)
				}
			}
		}
	}
}

func directLookup(zoneID int, hostname string, server string) {
	defer atomic.AddInt32(&spiderPending, -1)
	if atomic.LoadInt32(&stopRequested) == 1 { return }

	m := new(dns.Msg); m.SetQuestion(dns.Fqdn(hostname), dns.TypeA)
	c := new(dns.Client); c.Timeout = 2 * time.Second
	r, _, err := c.Exchange(m, net.JoinHostPort(server, "53"))
	
	if err == nil && r != nil {
		for _, ans := range r.Answer {
			if t, ok := ans.(*dns.A); ok {
				var lastID int
				h := strings.ToLower(t.Header().Name)
				errDB := dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, $3, $4) ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zoneID, h, "A", t.A.String()).Scan(&lastID)
				
				if errDB != nil {
					logMsg("[❌ DB ERROR] Αποτυχία αποθήκευσης Force Lookup '%s': %v", h, errDB)
				} else if lastID > 0 {
					atomic.AddInt32(&spiderPending, 1)
					spiderQueue <- SpiderJob{RecordID: lastID, CurrentURL: "http://" + strings.TrimSuffix(h, "."), BaseHost: strings.TrimSuffix(h, "."), Depth: 0}
				}
			}
		}
	}
}

func startWebUI() {
	mux := http.NewServeMux()

	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != "/" { http.NotFound(w, r); return }
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		w.Write([]byte(htmlDashboard))
	})

	mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) {
		globalStats.Lock()
		defer globalStats.Unlock()
		stateMutex.Lock()
		scanStat := isScanning
		enabled := daemonEnabled
		target := nextRunTime
		stateMutex.Unlock()
		
		nextRunStr := "Daemon Stopped"
		if enabled {
			if target.IsZero() {
				nextRunStr = "Manual Only"
			} else {
				rem := time.Until(target)
				if rem > 0 {
					nextRunStr = fmt.Sprintf("%dh %dm", int(rem.Hours()), int(rem.Minutes())%60)
				} else {
					nextRunStr = "Imminent"
				}
			}
		}

		json.NewEncoder(w).Encode(map[string]interface{}{
			"isScanning":    scanStat,
			"daemonEnabled": enabled,
			"nextRun":       nextRunStr,
			"spiderQueue":   len(spiderQueue),
			"aiQueue":       len(aiQueue),
			"active":        atomic.LoadUint64(&sitesActive),
			"dead":          atomic.LoadUint64(&sitesOffline),
			"skipped":       atomic.LoadUint64(&sitesSkipped),
			"media":         atomic.LoadUint64(&mediaFound),
			"categories":    globalStats.CategoryCounts,
		})
	})

	mux.HandleFunc("/api/logs", func(w http.ResponseWriter, r *http.Request) {
		logMutex.Lock()
		defer logMutex.Unlock()
		json.NewEncoder(w).Encode(logBuffer)
	})

	mux.HandleFunc("/api/config", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "GET" {
			cfgMutex.RLock()
			defer cfgMutex.RUnlock()
			json.NewEncoder(w).Encode(cfg)
		} else if r.Method == "POST" {
			var newCfg AppConfig
			if err := json.NewDecoder(r.Body).Decode(&newCfg); err == nil {
				cfgMutex.Lock()
				cfg = newCfg
				saveConfigInternal()
				cfgMutex.Unlock()
				calcNextRun()
				logMsg("[🔧 SYSTEM] Config saved via Web UI.")
				w.WriteHeader(http.StatusOK)
			} else {
				http.Error(w, err.Error(), http.StatusBadRequest)
			}
		}
	})
	
	mux.HandleFunc("/api/config/reload", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			loadConfig()
			calcNextRun()
			w.WriteHeader(http.StatusOK)
		}
	})

	mux.HandleFunc("/api/daemon/start", func(w http.ResponseWriter, r *http.Request) {
		stateMutex.Lock()
		daemonEnabled = true
		stateMutex.Unlock()
		calcNextRun()
		logMsg("[▶️ SYSTEM] Daemon Started (Scheduler active).")
		w.WriteHeader(http.StatusOK)
	})

	mux.HandleFunc("/api/daemon/stop", func(w http.ResponseWriter, r *http.Request) {
		stateMutex.Lock()
		daemonEnabled = false
		stateMutex.Unlock()
		stopScan()
		w.WriteHeader(http.StatusOK)
	})

	mux.HandleFunc("/api/scan/force", func(w http.ResponseWriter, r *http.Request) {
		if r.Method == "POST" {
			go triggerScan()
			w.WriteHeader(http.StatusOK)
		}
	})

	cfgMutex.RLock()
	port := cfg.WebUIPort
	cfgMutex.RUnlock()
	if !strings.Contains(port, ":") { port = ":" + port }

	logMsg("[🌍 WEB-UI] Daemon C&C Dashboard running on 0.0.0.0%s", port)
	err := http.ListenAndServe(port, mux)
	if err != nil { logMsg("[❌ FATAL] Web UI Failed: %v", err) }
}

const htmlDashboard = `
<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <title>WiSearch C&C</title>
    <style>
        body { background: #121212; color: #e0e0e0; font-family: 'Segoe UI', Tahoma, sans-serif; margin: 0; padding: 20px; }
        .header { border-bottom: 2px solid #333; padding-bottom: 15px; margin-bottom: 20px; display: flex; justify-content: space-between; align-items: center;}
        .title { color: #bb86fc; margin: 0; font-size: 24px; }
        .nav-tabs { margin-bottom: 10px; }
        .nav-tabs button { background: #1e1e1e; color: #e0e0e0; border: 1px solid #333; padding: 10px 20px; cursor: pointer; border-radius: 4px; margin-right: 10px; font-weight: bold; }
        .nav-tabs button.active { background: #bb86fc; color: #121212; }
        .sub-tabs { margin-bottom: 20px; background: #1a1a1a; padding: 10px; border-radius: 8px; border: 1px solid #2d2d2d; display:flex; gap:10px; }
        .sub-tabs button { background: transparent; color: #9e9e9e; border: none; padding: 8px 15px; cursor: pointer; border-radius: 4px; font-weight: bold; transition: 0.2s;}
        .sub-tabs button:hover { background: #333; color: #fff;}
        .sub-tabs button.active { background: #03dac6; color: #000; }
        
        .grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 15px; margin-bottom: 20px; }
        .card { background: #1e1e1e; padding: 20px; border-radius: 8px; border: 1px solid #333; text-align: center; }
        .card h3 { margin: 0 0 10px 0; font-size: 13px; color: #9e9e9e; text-transform: uppercase; }
        .card p { margin: 0; font-size: 24px; font-weight: bold; color: #03dac6; }
        .card.red p { color: #cf6679; }
        .card.purple p { color: #bb86fc; }
        .main-container { display: flex; gap: 20px; height: calc(100vh - 270px); }
        .terminal { flex: 2; background: #000; border-radius: 8px; padding: 15px; overflow-y: auto; font-family: 'Consolas', monospace; font-size: 13px; line-height: 1.5; border: 1px solid #333; box-shadow: inset 0 0 10px rgba(0,0,0,0.8); }
        .log-line { margin-bottom: 4px; word-wrap: break-word; }
        .log-info { color: #4af626; }
        .log-err { color: #ff5555; font-weight: bold; }
        .log-warn { color: #ffb86c; }
        .log-ai { color: #bd93f9; }
        .sidebar { flex: 1; background: #1e1e1e; border-radius: 8px; padding: 15px; border: 1px solid #333; overflow-y: auto; }
        .cat-list { list-style: none; padding: 0; margin: 0; }
        .cat-list li { display: flex; justify-content: space-between; padding: 8px 0; border-bottom: 1px solid #2d2d2d; font-size: 14px; }
        .cat-list li span { color: #03dac6; font-weight: bold; }
        
        .btn { border: none; padding: 8px 15px; border-radius: 4px; cursor: pointer; font-weight: bold; color:#fff; }
        .btn-start { background: #03dac6; color:#000; }
        .btn-stop { background: #cf6679; }
        .btn-force { background: #bb86fc; color:#000; }
        .btn-reload { background: #3700b3; }
        .btn-add { background: #03dac6; color: #000; font-size:12px; padding: 4px 8px; margin-left: 10px; }
        .btn-del { background: #cf6679; color: #fff; font-size:12px; padding: 4px 8px; margin-left: 5px; }
        
        #settings-panel { display: none; background: #1e1e1e; padding: 20px; border-radius: 8px; border: 1px solid #333; height: calc(100vh - 180px); display:flex; flex-direction:column; }
        .set-section { display: none; flex: 1; overflow-y: auto; }
        .set-section.active { display: block; }
        
        .form-group { display: flex; flex-direction: column; margin-bottom: 15px; background: rgba(255,255,255,0.02); padding: 15px; border-radius: 8px; border: 1px solid #2d2d2d;}
        .form-group.inline { flex-direction: row; gap:10px; align-items:center; background:none; padding:0; border:none; margin-bottom: 10px;}
        label { font-size: 12px; color: #bb86fc; margin-bottom: 5px; font-weight:bold; text-transform:uppercase; }
        input, select, textarea { background: #121212; color: #e0e0e0; border: 1px solid #333; padding: 10px; border-radius: 4px; font-family: monospace; width: 100%; box-sizing: border-box;}
        
        .list-item { display: flex; gap: 10px; margin-bottom: 5px; }
        .cf-item { background: #121212; padding: 10px; border-radius: 4px; border: 1px solid #333; margin-bottom: 10px; position:relative; }
    </style>
</head>
<body>
    <div class="header">
        <h1 class="title">WiSearch <span>C&C</span></h1>
        <div style="display:flex; gap:10px; align-items:center;">
            <span id="status-badge" style="font-family: monospace; padding: 5px 10px; border-radius: 4px;">● STATUS</span>
            <span style="font-family: monospace; color: #9e9e9e; margin-right:15px;">Next Run: <b id="next-run" style="color:#bb86fc;">--</b></span>
            <button class="btn btn-start" onclick="action('start')">▶ Start</button>
            <button class="btn btn-stop" onclick="action('stop')">⏸ Stop</button>
            <button class="btn btn-force" onclick="action('force')">⚡ Force Scan</button>
            <button class="btn btn-reload" onclick="action('reload')">🔄 Reload Config</button>
        </div>
    </div>

    <div class="nav-tabs">
        <button id="tab-dash" class="active" onclick="switchMainTab('dash')">Dashboard</button>
        <button id="tab-set" onclick="switchMainTab('set')">Settings Form</button>
    </div>

    <div id="dash-panel">
        <div class="grid">
            <div class="card purple"><h3>Spider Queue</h3><p id="sq">0</p></div>
            <div class="card purple"><h3>AI Queue</h3><p id="aq">0</p></div>
            <div class="card"><h3>Active Nodes</h3><p id="ac">0</p></div>
            <div class="card red"><h3>Dead Nodes</h3><p id="de">0</p></div>
            <div class="card"><h3>Skipped</h3><p id="sk">0</p></div>
            <div class="card"><h3>Media</h3><p id="me">0</p></div>
        </div>
        <div class="main-container">
            <div class="terminal" id="terminal"></div>
            <div class="sidebar">
                <h3 style="margin-top:0; color: #bb86fc; border-bottom: 1px solid #333; padding-bottom: 10px;">AI Classifier</h3>
                <ul class="cat-list" id="cat-list"></ul>
            </div>
        </div>
    </div>

    <div id="settings-panel" style="display:none;">
        <div class="sub-tabs">
            <button id="sub-db" class="active" onclick="switchSetTab('db')">Database</button>
            <button id="sub-crawl" onclick="switchSetTab('crawl')">Crawler & Schedule</button>
            <button id="sub-targets" onclick="switchSetTab('targets')">Targets (DNS/Zones)</button>
            <button id="sub-ai" onclick="switchSetTab('ai')">AI Accounts</button>
            <button id="sub-prompt" onclick="switchSetTab('prompt')">AI Prompt & Whitelist</button>
        </div>

        <div id="set-db" class="set-section active">
            <div class="form-group">
                <label>Database Connection Details</label>
                <div class="form-group inline"><label style="width:60px;">Host:</label><input type="text" id="db-host"></div>
                <div class="form-group inline"><label style="width:60px;">Port:</label><input type="text" id="db-port"></div>
                <div class="form-group inline"><label style="width:60px;">User:</label><input type="text" id="db-user"></div>
                <div class="form-group inline"><label style="width:60px;">Pass:</label><input type="password" id="db-pass"></div>
                <div class="form-group inline"><label style="width:60px;">DB Name:</label><input type="text" id="db-name"></div>
                <div class="form-group inline"><label style="width:60px;">SSL Mode:</label><input type="text" id="db-ssl"></div>
            </div>
        </div>

        <div id="set-crawl" class="set-section">
            <div class="form-group">
                <label>Crawler Limits</label>
                <div class="form-group inline"><label style="width:120px;">Web UI Port:</label><input type="text" id="cfg-port"></div>
                <div class="form-group inline"><label style="width:120px;">Max Depth:</label><input type="number" id="cfg-depth"></div>
                <div class="form-group inline"><label style="width:120px;">Workers:</label><input type="number" id="cfg-workers"></div>
                <div class="form-group inline"><label style="width:120px;">Pages/Host:</label><input type="number" id="cfg-pages"></div>
            </div>
            <div class="form-group">
                <label>Scheduler <span style="color:#e0e0e0; font-weight:normal; text-transform:none;">(0 to disable)</span></label>
                <div class="form-group inline"><label style="width:120px;">Run Every (Hours):</label><input type="number" id="cfg-sh"></div>
                <div class="form-group inline"><label style="width:120px;">Run Every (Mins):</label><input type="number" id="cfg-sm"></div>
                <label style="display:flex; align-items:center; gap:10px; color:#e0e0e0; margin-top:10px; cursor:pointer;">
                    <input type="checkbox" id="cfg-onstart" style="width:auto; transform:scale(1.5);"> Scan immediately on daemon start
                </label>
            </div>
        </div>

        <div id="set-targets" class="set-section">
            <div class="form-group">
                <label>DNS Servers <button class="btn btn-add" onclick="addListInput('dns-list')">+ Add</button></label>
                <div id="dns-list"></div>
            </div>
            <div class="form-group">
                <label>DNS Zones <button class="btn btn-add" onclick="addListInput('zones-list')">+ Add</button></label>
                <div id="zones-list"></div>
            </div>
        </div>

        <div id="set-ai" class="set-section">
            <div class="form-group">
                <label>Cloudflare AI Accounts <button class="btn btn-add" onclick="addCFInput()">+ Add Account</button></label>
                <div id="cf-list" style="max-height: 400px; overflow-y:auto; padding-right:10px;"></div>
            </div>
        </div>

        <div id="set-prompt" class="set-section">
            <div class="form-group">
                <label>System Prompt for AI Classifier</label>
                <textarea id="cfg-prompt" style="height:200px; resize:vertical; font-size:13px;" placeholder="Enter system prompt here..."></textarea>
                <span style="font-size:11px; color:#9e9e9e; margin-top:5px;">This prompt tells the LLM how to format the JSON response and what categories to use.</span>
            </div>
            <div class="form-group">
                <label>Core Services Whitelist <button class="btn btn-add" onclick="addListInput('core-list')">+ Add</button></label>
                <div id="core-list" style="max-height: 200px; overflow-y:auto; padding-right:10px;"></div>
            </div>
        </div>

        <div style="margin-top: 15px; border-top: 1px solid #333; padding-top: 15px;">
            <button class="btn btn-start" style="padding:15px 30px; font-size:16px; box-shadow: 0 4px 6px rgba(0,0,0,0.3);" onclick="saveConfig()">💾 Save Configuration</button>
        </div>
    </div>

    <script>
        function colorizeLog(text) {
            if (text.includes("ERR") || text.includes("DEAD") || text.includes("FATAL")) return '<span class="log-err">' + text + '</span>';
            if (text.includes("RATE LIMIT") || text.includes("[🛑")) return '<span class="log-err">' + text + '</span>';
            if (text.includes("[🤖")) return '<span class="log-ai">' + text + '</span>';
            if (text.includes("[🌐") || text.includes("[🚀") || text.includes("[⏰") || text.includes("[🔄") || text.includes("[🌍") || text.includes("[▶️")) return '<span class="log-warn">' + text + '</span>';
            if (text.includes("[✅") || text.includes("[🔧")) return '<span class="log-info">' + text + '</span>';
            return text;
        }

        async function fetchStats() {
            try {
                let res = await fetch('/api/stats');
                let data = await res.json();
                
                let badge = document.getElementById('status-badge');
                if(!data.daemonEnabled) {
                    badge.innerText = "● STOPPED"; badge.style.color = "#cf6679"; badge.style.background = "rgba(207,102,121,0.1)";
                } else if(data.isScanning) {
                    badge.innerText = "● SCANNING"; badge.style.color = "#03dac6"; badge.style.background = "rgba(3,218,198,0.1)";
                } else {
                    badge.innerText = "● IDLE"; badge.style.color = "#9e9e9e"; badge.style.background = "rgba(158,158,158,0.1)";
                }

                document.getElementById('next-run').innerText = data.nextRun;
                document.getElementById('sq').innerText = data.spiderQueue;
                document.getElementById('aq').innerText = data.aiQueue;
                document.getElementById('ac').innerText = data.active;
                document.getElementById('de').innerText = data.dead;
                document.getElementById('sk').innerText = data.skipped;
                document.getElementById('me').innerText = data.media;

                let catHtml = '';
                for (const [key, value] of Object.entries(data.categories)) {
                    catHtml += '<li>' + key + ' <span>' + value + '</span></li>';
                }
                document.getElementById('cat-list').innerHTML = catHtml || '<li>Awaiting Data...</li>';
            } catch (e) {}
        }

        async function fetchLogs() {
            try {
                let res = await fetch('/api/logs');
                let logs = await res.json();
                let term = document.getElementById('terminal');
                let isScrolledToBottom = term.scrollHeight - term.clientHeight <= term.scrollTop + 10;
                term.innerHTML = logs.map(l => '<div class="log-line">' + colorizeLog(l) + '</div>').join('');
                if (isScrolledToBottom) term.scrollTop = term.scrollHeight;
            } catch (e) {}
        }

        async function action(type) {
            if(type === 'start') await fetch('/api/daemon/start', {method: 'POST'});
            if(type === 'stop') await fetch('/api/daemon/stop', {method: 'POST'});
            if(type === 'force') await fetch('/api/scan/force', {method: 'POST'});
            if(type === 'reload') {
                await fetch('/api/config/reload', {method: 'POST'});
                alert("Config Reloaded from Disk!");
            }
        }

        function addListInput(containerId, val = "") {
            let div = document.createElement('div');
            div.className = "list-item";
            div.innerHTML = '<input type="text" value="' + val + '"><button class="btn btn-del" onclick="this.parentElement.remove()">X</button>';
            document.getElementById(containerId).appendChild(div);
        }
        
        function getListValues(containerId) {
            return Array.from(document.getElementById(containerId).querySelectorAll('input')).map(inp => inp.value.trim()).filter(v => v);
        }

        function addCFInput(comment="", acc="", token="", model="@cf/meta/llama-3-8b-instruct") {
            let div = document.createElement('div');
            div.className = "cf-item";
            let sel1 = (model==='@cf/meta/llama-3-8b-instruct') ? 'selected' : '';
            let sel2 = (model==='@cf/meta/llama-2-7b-chat-int8') ? 'selected' : '';
            let sel3 = (model==='@cf/mistral/mistral-7b-instruct-v0.1') ? 'selected' : '';

            div.innerHTML = '<button class="btn btn-del" style="position:absolute; top:5px; right:5px;" onclick="this.parentElement.remove()">X</button>' +
                '<div class="form-group inline"><label style="width:60px;">Tag:</label><input type="text" class="cf-comment" value="' + comment + '" placeholder="e.g. email@gmail.com"></div>' +
                '<div class="form-group inline"><label style="width:60px;">Acc ID:</label><input type="text" class="cf-acc" value="' + acc + '"></div>' +
                '<div class="form-group inline"><label style="width:60px;">Token:</label><input type="password" class="cf-token" value="' + token + '"></div>' +
                '<div class="form-group inline"><label style="width:60px;">Model:</label>' +
                    '<select class="cf-model">' +
                        '<option value="@cf/meta/llama-3-8b-instruct" ' + sel1 + '>LLaMA-3 8B (Fast)</option>' +
                        '<option value="@cf/meta/llama-2-7b-chat-int8" ' + sel2 + '>LLaMA-2 7B</option>' +
                        '<option value="@cf/mistral/mistral-7b-instruct-v0.1" ' + sel3 + '>Mistral 7B</option>' +
                    '</select>' +
                '</div>';
            document.getElementById('cf-list').appendChild(div);
        }

        function getCFValues() {
            return Array.from(document.querySelectorAll('.cf-item')).map(item => ({
                comment: item.querySelector('.cf-comment').value.trim(),
                account_id: item.querySelector('.cf-acc').value.trim(),
                api_token: item.querySelector('.cf-token').value.trim(),
                model: item.querySelector('.cf-model').value
            })).filter(c => c.account_id && c.api_token);
        }

        async function loadConfigUI() {
            let res = await fetch('/api/config');
            let j = await res.json();
            
            let db = j.database || {};
            if(j.db_conn && !db.host) {
                let parts = j.db_conn.split(' ');
                parts.forEach(p => {
                    let kv = p.split('=');
                    if(kv.length === 2) {
                        if(kv[0]==='host') db.host = kv[1];
                        if(kv[0]==='port') db.port = kv[1];
                        if(kv[0]==='user') db.user = kv[1];
                        if(kv[0]==='password') db.password = kv[1];
                        if(kv[0]==='dbname') db.dbname = kv[1];
                        if(kv[0]==='sslmode') db.sslmode = kv[1];
                    }
                });
            }

            document.getElementById('db-host').value = db.host || '';
            document.getElementById('db-port').value = db.port || '';
            document.getElementById('db-user').value = db.user || '';
            document.getElementById('db-pass').value = db.password || '';
            document.getElementById('db-name').value = db.dbname || '';
            document.getElementById('db-ssl').value = db.sslmode || '';

            document.getElementById('cfg-port').value = j.web_ui_port || '';
            document.getElementById('cfg-depth').value = j.max_depth || 50;
            document.getElementById('cfg-workers').value = j.max_workers || 60;
            document.getElementById('cfg-pages').value = j.max_pages_per_host || 10000;
            document.getElementById('cfg-sh').value = j.schedule_hours || 0;
            document.getElementById('cfg-sm').value = j.schedule_minutes || 0;
            document.getElementById('cfg-onstart').checked = j.scan_on_start || false;
            
            document.getElementById('cfg-prompt').value = j.ai_prompt || '';

            document.getElementById('dns-list').innerHTML = '';
            document.getElementById('zones-list').innerHTML = '';
            document.getElementById('core-list').innerHTML = '';
            document.getElementById('cf-list').innerHTML = '';

            (j.start_servers || []).forEach(v => addListInput('dns-list', v));
            (j.zone_names || []).forEach(v => addListInput('zones-list', v));
            (j.core_services || []).forEach(v => addListInput('core-list', v));
            (j.cf_credentials || []).forEach(c => addCFInput(c.comment, c.account_id, c.api_token, c.model));
            
            if((j.start_servers||[]).length===0) addListInput('dns-list');
            if((j.zone_names||[]).length===0) addListInput('zones-list');
            if((j.core_services||[]).length===0) addListInput('core-list');
            if((j.cf_credentials||[]).length===0) addCFInput();
        }

        async function saveConfig() {
            let payload = {
                database: {
                    host: document.getElementById('db-host').value,
                    port: document.getElementById('db-port').value,
                    user: document.getElementById('db-user').value,
                    password: document.getElementById('db-pass').value,
                    dbname: document.getElementById('db-name').value,
                    sslmode: document.getElementById('db-ssl').value,
                },
                web_ui_port: document.getElementById('cfg-port').value,
                max_depth: parseInt(document.getElementById('cfg-depth').value),
                max_workers: parseInt(document.getElementById('cfg-workers').value),
                max_pages_per_host: parseInt(document.getElementById('cfg-pages').value),
                schedule_hours: parseInt(document.getElementById('cfg-sh').value),
                schedule_minutes: parseInt(document.getElementById('cfg-sm').value),
                scan_on_start: document.getElementById('cfg-onstart').checked,
                ai_prompt: document.getElementById('cfg-prompt').value,
                start_servers: getListValues('dns-list'),
                zone_names: getListValues('zones-list'),
                core_services: getListValues('core-list'),
                cf_credentials: getCFValues()
            };

            let res = await fetch('/api/config', {
                method: 'POST',
                headers: {'Content-Type': 'application/json'},
                body: JSON.stringify(payload)
            });
            if(res.ok) alert("Configuration Saved Successfully! Changes apply to next run.");
            else alert("Failed to save configuration.");
        }

        function switchMainTab(t) {
            document.getElementById('tab-dash').classList.remove('active');
            document.getElementById('tab-set').classList.remove('active');
            document.getElementById('dash-panel').style.display = 'none';
            document.getElementById('settings-panel').style.display = 'none';
            if(t === 'dash') {
                document.getElementById('tab-dash').classList.add('active');
                document.getElementById('dash-panel').style.display = 'block';
            } else {
                document.getElementById('tab-set').classList.add('active');
                document.getElementById('settings-panel').style.display = 'flex';
                loadConfigUI();
            }
        }

        function switchSetTab(t) {
            document.querySelectorAll('.sub-tabs button').forEach(b => b.classList.remove('active'));
            document.getElementById('sub-'+t).classList.add('active');
            document.querySelectorAll('.set-section').forEach(s => s.classList.remove('active'));
            document.getElementById('set-'+t).classList.add('active');
        }

        setInterval(fetchStats, 2000);
        setInterval(fetchLogs, 1000);
        fetchStats(); fetchLogs();
    </script>
</body>
</html>
`