package main import ( "bytes" "crypto/tls" "database/sql" "encoding/json" "fmt" "io" "log" "net" "net/http" "net/url" "os" "strings" "sync" "sync/atomic" "time" "github.com/PuerkitoBio/goquery" _ "github.com/lib/pq" "github.com/miekg/dns" "golang.org/x/net/html/charset" ) // --- CONFIGURATION STRUCTURES --- type DBConfig struct { Host string `json:"host"` Port string `json:"port"` User string `json:"user"` Password string `json:"password"` DBName string `json:"dbname"` SSLMode string `json:"sslmode"` } type CFCredential struct { Comment string `json:"comment"` AccountID string `json:"account_id"` APIToken string `json:"api_token"` Model string `json:"model"` } type AppConfig struct { Database DBConfig `json:"database"` DBConnLegacy string `json:"db_conn,omitempty"` StartServers []string `json:"start_servers"` ZoneNames []string `json:"zone_names"` MaxDepth int `json:"max_depth"` MaxWorkers int `json:"max_workers"` MaxPagesPerHost int `json:"max_pages_per_host"` WebUIPort string `json:"web_ui_port"` ScheduleHours int `json:"schedule_hours"` ScheduleMinutes int `json:"schedule_minutes"` ScanOnStart bool `json:"scan_on_start"` CoreServices []string `json:"core_services"` CFCredentials []CFCredential `json:"cf_credentials"` AIPrompt string `json:"ai_prompt"` } func (c DBConfig) GetConnString() string { return fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s", c.Host, c.Port, c.User, c.Password, c.DBName, c.SSLMode) } // --- GLOBAL VARIABLES --- var ( cfg AppConfig cfgMutex sync.RWMutex dbGlobal *sql.DB daemonEnabled bool = true isScanning bool stopRequested int32 lastRun time.Time nextRunTime time.Time stateMutex sync.Mutex visitedZones sync.Map visitedWeb sync.Map hostVisitCount sync.Map sitesOffline uint64 sitesActive uint64 sitesSkipped uint64 mediaFound uint64 spiderPending int32 aiPending int32 spiderQueue = make(chan SpiderJob, 300000) aiQueue = make(chan AIJob, 100000) globalStats = &StatsManager{CategoryCounts: make(map[string]int)} forceLookups = []string{ "minecraft.maiden2", // ΝΕΟ: Αναγκάζει να ψάξει αυτό πρώτο! "routemap", "activeservices", "router", "stats", "www", "node", "auth", "camera", "weather", "voip", "irc", "matrix", "speedtest", "proxy", "mirror", "archive", "git", "cloud", "files", "game", "mc", "ts3", "mumble", "ntp", "dns", "mail", "iot", "sensor", "radio", "tv", "stream", "torrent", "tracker", "portal", "p2p", "dcplusplus", "hub", "search", "forum", "board", "bb", "vbulletin", "smf", "phpbb", "discuss", } httpClient = &http.Client{ Timeout: 8 * time.Second, Transport: &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, MaxIdleConns: 1000, MaxIdleConnsPerHost: 100, }, } aiClient = &http.Client{Timeout: 30 * time.Second} logBuffer []string logMutex sync.Mutex ) type SpiderJob struct { RecordID int CurrentURL string BaseHost string Depth int } type AIJob struct { URL string Title string MetaDesc string Generator string H1 string Preview string } type AIAnalysis struct { Category string `json:"category"` Summary string `json:"summary"` Status string `json:"status"` } type StatsManager struct { sync.Mutex CategoryCounts map[string]int } func (s *StatsManager) Update(category string) { s.Lock() defer s.Unlock() s.CategoryCounts[category]++ } func (s *StatsManager) Reset() { s.Lock() defer s.Unlock() s.CategoryCounts = make(map[string]int) } func logMsg(format string, a ...interface{}) { msg := fmt.Sprintf("[%s] %s", time.Now().Format("15:04:05"), fmt.Sprintf(format, a...)) // 1. Τυπώνει στην οθόνη (Terminal) fmt.Println(msg) logMutex.Lock() // 2. Το κρατάει στη μνήμη για το Web UI (Dashboard) logBuffer = append(logBuffer, msg) if len(logBuffer) > 300 { logBuffer = logBuffer[1:] } // 3. ΝΕΟ: Το γράφει μόνιμα στο αρχείο /var/log/wisearch-crawler.log f, err := os.OpenFile("/var/log/wisearch-crawler.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err == nil { f.WriteString(msg + "\n") f.Close() } else { // Αν δεν έχει δικαιώματα, το τυπώνει στην οθόνη για να το δεις fmt.Printf("[⚠️ FILE ERROR] Δεν μπορώ να γράψω στο log file: %v\n", err) } logMutex.Unlock() } func loadConfig() { cfgMutex.Lock() defer cfgMutex.Unlock() defaultPrompt := "You are an AWMN web classifier. Return ONLY JSON. Do not include markdown.\nCategories: \"Router & Admin\", \"Network Tools & Stats\", \"Media & Streaming\", \"P2P & Torrents\", \"Storage & FTP\", \"Forums & Communities\", \"Communication & VoIP\", \"Gaming\", \"IoT & Sensors\", \"Web Services & Blogs\", \"Default Page\", \"Other\".\nFormat: {\"category\": \"Exact category\", \"summary\": \"Short Greek summary, max 8 words\", \"status\": \"Active\"}" data, err := os.ReadFile("config.json") if err != nil { logMsg("[⚠️ SYSTEM] config.json not found. Creating default configuration.") cfg = AppConfig{ Database: DBConfig{Host: "192.168.131.124", Port: "5432", User: "postgres", Password: "postpass", DBName: "wisearch", SSLMode: "disable"}, StartServers: []string{"192.168.131.98"}, ZoneNames: []string{"awmn."}, MaxDepth: 50, MaxWorkers: 60, MaxPagesPerHost: 10000, WebUIPort: ":8080", ScheduleHours: 12, ScheduleMinutes: 0, ScanOnStart: false, AIPrompt: defaultPrompt, CoreServices: []string{}, // Άδειο από default CFCredentials: []CFCredential{}, // Άδειο από default } saveConfigInternal() return } json.Unmarshal(data, &cfg) if cfg.AIPrompt == "" { cfg.AIPrompt = defaultPrompt } logMsg("[🔧 SYSTEM] Configuration loaded.") } func saveConfigInternal() { cfg.DBConnLegacy = "" data, _ := json.MarshalIndent(cfg, "", " ") os.WriteFile("config.json", data, 0644) } func isCoreService(hostname string) bool { cfgMutex.RLock() defer cfgMutex.RUnlock() for _, svc := range cfg.CoreServices { if svc == hostname { return true } } return false } func main() { loadConfig() logMsg("[🚀 SYSTEM] WiSearch Daemon Initialized | C&C Mode") var err error dbGlobal, err = sql.Open("postgres", cfg.Database.GetConnString()) if err != nil { log.Fatal("[❌ FATAL] Database connection string failed:", err) } if err = dbGlobal.Ping(); err != nil { logMsg("[❌ FATAL DB ERROR] Η Βάση Δεδομένων δεν απαντάει! Έλεγξε IP/Κωδικό: %v", err) } else { logMsg("[✅ SYSTEM] Database connection successful.") } defer dbGlobal.Close() dbGlobal.SetMaxOpenConns(150) dbGlobal.SetMaxIdleConns(75) go startWebUI() cfgMutex.RLock() workers := cfg.MaxWorkers onStart := cfg.ScanOnStart cfgMutex.RUnlock() for i := 0; i < workers; i++ { go spiderWorker() } for i := 0; i < 4; i++ { go aiWorker() } calcNextRun() go schedulerLoop() go watchdogLoop() if onStart { logMsg("[▶️ SYSTEM] ScanOnStart is enabled. Triggering immediate scan.") triggerScan() } select {} } func calcNextRun() { cfgMutex.RLock() h, m := cfg.ScheduleHours, cfg.ScheduleMinutes cfgMutex.RUnlock() stateMutex.Lock() defer stateMutex.Unlock() if h == 0 && m == 0 { nextRunTime = time.Time{} return } interval := time.Duration(h)*time.Hour + time.Duration(m)*time.Minute now := time.Now() if lastRun.IsZero() { lastRun = now } nextRunTime = lastRun.Add(interval) if nextRunTime.Before(now) { lastRun = now nextRunTime = now.Add(interval) } } func triggerScan() { stateMutex.Lock() if isScanning { stateMutex.Unlock() logMsg("[⚠️ SYSTEM] Scan is already running. Ignoring trigger.") return } isScanning = true lastRun = time.Now() stateMutex.Unlock() calcNextRun() atomic.StoreInt32(&stopRequested, 0) logMsg("[🔄 SYSTEM] Initiating New Multi-Zone Scan Cycle...") visitedZones = sync.Map{} visitedWeb = sync.Map{} hostVisitCount = sync.Map{} atomic.StoreUint64(&sitesActive, 0) atomic.StoreUint64(&sitesOffline, 0) atomic.StoreUint64(&sitesSkipped, 0) atomic.StoreUint64(&mediaFound, 0) globalStats.Reset() cfgMutex.RLock() zones := cfg.ZoneNames servers := cfg.StartServers cfgMutex.RUnlock() for _, z := range zones { for _, s := range servers { atomic.AddInt32(&spiderPending, 1) go fetchAndStoreDNS(z, s) } } go processSubmittedURLs() } func stopScan() { atomic.StoreInt32(&stopRequested, 1) logMsg("[🛑 SYSTEM] Emergency Stop Requested! Draining queues...") } func schedulerLoop() { for { time.Sleep(10 * time.Second) stateMutex.Lock() scanning := isScanning enabled := daemonEnabled target := nextRunTime stateMutex.Unlock() if !enabled || scanning || target.IsZero() { continue } if time.Now().After(target) { logMsg("[⏰ SCHEDULE] Scheduled time reached. Starting scan.") triggerScan() } } } func watchdogLoop() { idleSeconds := 0 for { time.Sleep(5 * time.Second) stateMutex.Lock() scanning := isScanning stateMutex.Unlock() if !scanning { continue } sPend := atomic.LoadInt32(&spiderPending) aPend := atomic.LoadInt32(&aiPending) stopped := atomic.LoadInt32(&stopRequested) == 1 if sPend == 0 && aPend == 0 && len(spiderQueue) == 0 && len(aiQueue) == 0 { idleSeconds += 5 if idleSeconds >= 15 || stopped { if stopped { logMsg("[🛑 SYSTEM] Scan aborted successfully.") } else { logMsg("[✅ SYSTEM] Scan Cycle Complete!") logMsg("[🔄 SYSTEM] Updating PageRank scores...") updatePageRank() } stateMutex.Lock() isScanning = false stateMutex.Unlock() idleSeconds = 0 } } else { idleSeconds = 0 } } } func processSubmittedURLs() { rows, err := dbGlobal.Query(`SELECT id, url FROM submitted_urls`) if err != nil { return } defer rows.Close() for rows.Next() { var id int var rawURL string if err := rows.Scan(&id, &rawURL); err == nil { u, pErr := url.Parse(rawURL) if pErr == nil && u.Hostname() != "" { baseHost := strings.ToLower(u.Hostname()) var zID int dbGlobal.QueryRow(`INSERT INTO zones (zone_name, dns_server_ip) VALUES ('custom_submissions', '0.0.0.0') ON CONFLICT (zone_name) DO UPDATE SET last_synced = CURRENT_TIMESTAMP RETURNING id`).Scan(&zID) var recID int dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, 'A', 'Χειροκίνητο') ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zID, baseHost).Scan(&recID) if recID > 0 { atomic.AddInt32(&spiderPending, 1) spiderQueue <- SpiderJob{RecordID: recID, CurrentURL: rawURL, BaseHost: baseHost, Depth: 0} dbGlobal.Exec(`DELETE FROM submitted_urls WHERE id = $1`, id) logMsg("[🔗 SUBMIT] Βρέθηκε χειροκίνητο URL: %s", rawURL) } } } } } func spiderWorker() { for job := range spiderQueue { if atomic.LoadInt32(&stopRequested) == 0 { processSpider(job) } atomic.AddInt32(&spiderPending, -1) } } func aiWorker() { for job := range aiQueue { if atomic.LoadInt32(&stopRequested) == 0 { analysis := getDeepAnalysis(job.URL, job.Title, job.MetaDesc, job.Generator, job.H1, job.Preview) u, err := url.Parse(job.URL) if err == nil { hostname := strings.ToLower(strings.TrimSuffix(u.Hostname(), ".")) if isCoreService(hostname) { analysis.Category = "Core Services" } } globalStats.Update(analysis.Category) logMsg("[🤖 AI-TAG] %-25s -> [%s]", u.Hostname(), analysis.Category) dbGlobal.Exec(`UPDATE web_health SET category = $1, ai_summary = $2, activity_status = $3 WHERE url = $4`, analysis.Category, analysis.Summary, analysis.Status, job.URL) time.Sleep(1000 * time.Millisecond) } atomic.AddInt32(&aiPending, -1) } } func processSpider(job SpiderJob) { cfgMutex.RLock() mDepth := cfg.MaxDepth mPages := cfg.MaxPagesPerHost cfgMutex.RUnlock() if job.Depth > mDepth { return } cleanURL := cleanURLForDB(job.CurrentURL) if _, loaded := visitedWeb.LoadOrStore(cleanURL, true); loaded { return } lowerURL := strings.ToLower(cleanURL) skipExt := []string{".jpg", ".jpeg", ".png", ".gif", ".pdf", ".zip", ".php", "graphs", "?print=", "&print=", ".iso", ".rar", ".torrent", ".mkv", ".mp4", ".mp3"} for _, ext := range skipExt { if strings.Contains(lowerURL, ext) { atomic.AddUint64(&sitesSkipped, 1) return } } vCountRaw, _ := hostVisitCount.LoadOrStore(job.BaseHost, 0) vCount := vCountRaw.(int) if vCount >= mPages { return } hostVisitCount.Store(job.BaseHost, vCount+1) // --- HTTP REQUEST (ΜΕ FAKE USER AGENT) --- req, _ := http.NewRequest("GET", cleanURL, nil) req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 WiSearchBot/1.0") req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") resp, err := httpClient.Do(req) if err != nil { atomic.AddUint64(&sitesOffline, 1) logMsg("[❌ DEAD] Σφάλμα σύνδεσης στο %s: %v", cleanURL, err) dbGlobal.Exec(`UPDATE web_health SET is_up = false, activity_status = 'Down', last_crawled = CURRENT_TIMESTAMP WHERE url = $1`, cleanURL) return } defer resp.Body.Close() if resp.StatusCode != 200 { atomic.AddUint64(&sitesOffline, 1) logMsg("[⚠️ HTTP ERROR] Το %s γύρισε κωδικό %d", cleanURL, resp.StatusCode) return } contentType := resp.Header.Get("Content-Type") if !strings.Contains(contentType, "text/html") { atomic.AddUint64(&sitesSkipped, 1) logMsg("[⚠️ SKIPPED-TYPE] URL: %s | Ήρθε με Content-Type: %s", cleanURL, contentType) return } utf8Body, _ := decodeToUTF8(resp.Body, contentType) doc, err := goquery.NewDocumentFromReader(utf8Body) if err != nil { return } doc.Find("img").Each(func(i int, s *goquery.Selection) { src, exists := s.Attr("src") alt, _ := s.Attr("alt") if exists && len(src) > 2 && !strings.HasPrefix(src, "data:image") { u, err := url.Parse(src) if err == nil { base, _ := url.Parse(job.CurrentURL) absImg := base.ResolveReference(u).String() dbGlobal.Exec(`INSERT INTO media_files (source_url, media_url, media_type, alt_text) VALUES ($1, $2, 'image', $3) ON CONFLICT DO NOTHING`, cleanURL, absImg, strings.TrimSpace(alt)) atomic.AddUint64(&mediaFound, 1) } } }) title := strings.TrimSpace(doc.Find("title").Text()) metaDesc, _ := doc.Find("meta[name=description]").Attr("content") generator, _ := doc.Find("meta[name=generator]").Attr("content") h1 := strings.TrimSpace(doc.Find("h1").First().Text()) doc.Find("script, style, nav, footer, header, aside, noscript, iframe, svg, canvas").Remove() bodyText := strings.Join(strings.Fields(doc.Find("body").Text()), " ") // --- ΔΙΟΡΘΩΣΗ: ΠΙΟ ΕΛΑΣΤΙΚΟΣ ΚΟΦΤΗΣ ΧΑΡΑΚΤΗΡΩΝ --- if len(strings.TrimSpace(bodyText)) < 5 && title == "" { atomic.AddUint64(&sitesSkipped, 1) logMsg("[⚠️ SKIPPED-EMPTY] URL: %s | Βρήκε κείμενο μήκους: %d | Τίτλος: '%s'", cleanURL, len(strings.TrimSpace(bodyText)), title) return } atomic.AddUint64(&sitesActive, 1) logMsg("[✅ ALIVE] Active Node: %s", cleanURL) dbGlobal.Exec(`INSERT INTO web_health (dns_record_id, url, status_code, page_title, is_up, last_crawled, full_content) VALUES ($1, $2, $3, $4, $5, CURRENT_TIMESTAMP, $6) ON CONFLICT (url) DO UPDATE SET page_title = EXCLUDED.page_title, is_up = true, last_crawled = CURRENT_TIMESTAMP, full_content = EXCLUDED.full_content`, job.RecordID, cleanURL, resp.StatusCode, title, true, bodyText) runes := []rune(bodyText) preview := "" if len(runes) > 400 { preview = string(runes[:400]) } else { preview = string(runes) } atomic.AddInt32(&aiPending, 1) aiQueue <- AIJob{URL: cleanURL, Title: title, MetaDesc: metaDesc, Generator: generator, H1: h1, Preview: preview} // --- ΕΞΑΓΩΓΗ LINKS (SPIDER) --- doc.Find("a").Each(func(i int, s *goquery.Selection) { if atomic.LoadInt32(&stopRequested) == 1 { return } href, _ := s.Attr("href") u, err := url.Parse(href) if err == nil { base, _ := url.Parse(job.CurrentURL) abs := base.ResolveReference(u) absHost := strings.ToLower(abs.Hostname()) isAllowed := false // 1. Είναι το ίδιο site ή subdomain (πχ wind.awmn -> stats.wind.awmn) if absHost == job.BaseHost || strings.HasSuffix(absHost, "."+job.BaseHost) { isAllowed = true } else if strings.HasSuffix(absHost, ".awmn") || strings.HasSuffix(absHost, ".local") { // 2. Είναι οποιοδήποτε άλλο site στο δίκτυο (.awmn ή .local) isAllowed = true } else { // 3. Ή ταιριάζει με κάτι που έχουμε βάλει στα Zones στο Admin Panel cfgMutex.RLock() for _, z := range cfg.ZoneNames { cz := strings.ToLower(strings.TrimSuffix(z, ".")) if strings.HasSuffix(absHost, "."+cz) || absHost == cz { isAllowed = true break } } cfgMutex.RUnlock() } if isAllowed && (abs.Scheme == "http" || abs.Scheme == "https") { target := cleanURLForDB(abs.String()) if _, loaded := visitedWeb.Load(target); !loaded { dbGlobal.Exec(`INSERT INTO web_links (source_url, target_url) VALUES ($1, $2) ON CONFLICT DO NOTHING`, cleanURL, target) targetRecordID := job.RecordID if absHost != job.BaseHost { var zID int dbGlobal.QueryRow(`INSERT INTO zones (zone_name, dns_server_ip) VALUES ('auto_discovered', '0.0.0.0') ON CONFLICT (zone_name) DO UPDATE SET last_synced = CURRENT_TIMESTAMP RETURNING id`).Scan(&zID) dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, 'A', 'Scraped') ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zID, absHost).Scan(&targetRecordID) } if targetRecordID > 0 { atomic.AddInt32(&spiderPending, 1) spiderQueue <- SpiderJob{RecordID: targetRecordID, CurrentURL: target, BaseHost: absHost, Depth: job.Depth + 1} } } } } }) } func getDeepAnalysis(targetURL, title, metaDesc, generator, h1, body string) AIAnalysis { if len(strings.TrimSpace(body)) < 30 && strings.TrimSpace(title) == "" { return AIAnalysis{Category: "Other", Summary: "Insufficient content.", Status: "Active"} } cfgMutex.RLock() systemPrompt := cfg.AIPrompt creds := cfg.CFCredentials cfgMutex.RUnlock() if systemPrompt == "" { systemPrompt = "You are an AWMN web classifier. Return ONLY JSON." } userPrompt := fmt.Sprintf("URL: %s\nTitle: %s\nMeta: %s\nContent: %s", targetURL, title, metaDesc, body) payload := map[string]interface{}{"messages": []map[string]string{{"role": "system", "content": systemPrompt}, {"role": "user", "content": userPrompt}}} bodyJSON, _ := json.Marshal(payload) for _, cred := range creds { if atomic.LoadInt32(&stopRequested) == 1 { break } apiURL := fmt.Sprintf("https://api.cloudflare.com/client/v4/accounts/%s/ai/run/%s", cred.AccountID, cred.Model) req, _ := http.NewRequest("POST", apiURL, bytes.NewBuffer(bodyJSON)) req.Header.Set("Authorization", "Bearer "+cred.APIToken) req.Header.Set("Content-Type", "application/json") resp, err := aiClient.Do(req) if err != nil { continue } if resp.StatusCode != 200 { resp.Body.Close(); continue } var cfRes struct { Result struct { Response string `json:"response"` } `json:"result"`; Success bool `json:"success"` } json.NewDecoder(resp.Body).Decode(&cfRes) resp.Body.Close() cleanJSON := cfRes.Result.Response if start := strings.Index(cleanJSON, "{"); start != -1 { if end := strings.LastIndex(cleanJSON, "}"); end != -1 && start <= end { cleanJSON = cleanJSON[start : end+1] } } var analysis AIAnalysis if err := json.Unmarshal([]byte(cleanJSON), &analysis); err == nil { if analysis.Category == "" { analysis.Category = "Other" } return analysis } } return AIAnalysis{Category: "Other", Summary: "AI Classification Failed", Status: "Active"} } func cleanURLForDB(inputURL string) string { u, err := url.Parse(inputURL) if err != nil { return inputURL } q := u.Query() // --- ΔΙΟΡΘΩΣΗ ΛΑΒΥΡΙΝΘΟΥ: Αφαιρέσαμε τα junk parameters (node, page, id κλπ) --- junk := []string{ "sid", "s", "osid", "sort", "order", "phpsessid", "view", "lang", "search", "highlight", "mode", "start", "form_search_nodes_search", "limit", "offset", "direction", "node", "page", "table_nodes_showpage", "action", "id", "p", "t", } for _, k := range junk { q.Del(k) } u.RawQuery = q.Encode() u.Fragment = "" return u.String() } func decodeToUTF8(r io.Reader, contentType string) (io.Reader, error) { buf, err := io.ReadAll(r) if err != nil { return nil, err } determineEncoding, _, _ := charset.DetermineEncoding(buf, contentType) utf8Reader := determineEncoding.NewDecoder().Reader(bytes.NewReader(buf)) finalBuf, _ := io.ReadAll(utf8Reader) cleanStr := strings.Map(func(r rune) rune { if r == '\uFFFD' || r < 0 { return -1 } return r }, string(finalBuf)) return strings.NewReader(cleanStr), nil } func updatePageRank() { dbGlobal.Exec(`UPDATE web_health wh SET pagerank_score = (1.0 + (SELECT COUNT(*) FROM web_links wl WHERE wl.target_url = wh.url)) * (CASE WHEN wh.category = 'Core Services' THEN 10.0 WHEN wh.category IN ('Gaming', 'Media & Streaming', 'P2P & Torrents', 'Web Services & Blogs', 'Forums & Communities', 'Communication & VoIP') THEN 3.0 WHEN wh.category IN ('Storage & FTP') THEN 2.0 WHEN wh.category IN ('Network Tools & Stats', 'IoT & Sensors') THEN 1.0 ELSE 1.0 END)`) } // --- DNS LOGIC --- func fetchAndStoreDNS(zoneName string, dnsServer string) { defer atomic.AddInt32(&spiderPending, -1) if atomic.LoadInt32(&stopRequested) == 1 { return } zoneName = dns.Fqdn(strings.ToLower(zoneName)) if _, loaded := visitedZones.LoadOrStore(zoneName, true); loaded { return } var zoneID int err := dbGlobal.QueryRow(`INSERT INTO zones (zone_name, dns_server_ip) VALUES ($1, $2) ON CONFLICT (zone_name) DO UPDATE SET last_synced = CURRENT_TIMESTAMP RETURNING id`, strings.TrimSuffix(zoneName, "."), dnsServer).Scan(&zoneID) if err != nil { logMsg("[❌ DB ERROR] Αποτυχία αποθήκευσης Ζώνης '%s': %v", zoneName, err) return } atomic.AddInt32(&spiderPending, 1) go directLookup(zoneID, zoneName, dnsServer) for _, sub := range forceLookups { atomic.AddInt32(&spiderPending, 1) go directLookup(zoneID, fmt.Sprintf("%s.%s", sub, zoneName), dnsServer) } m := new(dns.Msg); m.SetAxfr(zoneName) tr := &dns.Transfer{DialTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second} channel, err := tr.In(m, net.JoinHostPort(dnsServer, "53")) if err != nil { logMsg("[⚠️ DNS] Zone Transfer (AXFR) Failed for %s: %v", zoneName, err) return } for env := range channel { if env.Error != nil { logMsg("[⚠️ DNS] Ενδιάμεσο σφάλμα AXFR για %s: %v", zoneName, env.Error) break } for _, rr := range env.RR { if atomic.LoadInt32(&stopRequested) == 1 { return } hName := strings.ToLower(rr.Header().Name) switch t := rr.(type) { case *dns.A: var lastID int err := dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, $3, $4) ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zoneID, hName, "A", t.A.String()).Scan(&lastID) if err != nil { logMsg("[❌ DB ERROR] Αποτυχία αποθήκευσης DNS ρεκόρ '%s': %v", hName, err) } else if lastID > 0 { atomic.AddInt32(&spiderPending, 1) spiderQueue <- SpiderJob{RecordID: lastID, CurrentURL: "http://" + strings.TrimSuffix(hName, "."), BaseHost: strings.TrimSuffix(hName, "."), Depth: 0} } case *dns.CNAME: var lastID int err := dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, $3, $4) ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zoneID, hName, "CNAME", t.Target).Scan(&lastID) if err == nil && lastID > 0 { atomic.AddInt32(&spiderPending, 1) spiderQueue <- SpiderJob{RecordID: lastID, CurrentURL: "http://" + strings.TrimSuffix(hName, "."), BaseHost: strings.TrimSuffix(hName, "."), Depth: 0} } case *dns.NS: if dns.Fqdn(hName) != zoneName { atomic.AddInt32(&spiderPending, 1) go func(sz, nt string) { defer atomic.AddInt32(&spiderPending, -1) c := new(dns.Client); c.Timeout = 2 * time.Second m2 := new(dns.Msg); m2.SetQuestion(dns.Fqdn(nt), dns.TypeA) r, _, _ := c.Exchange(m2, net.JoinHostPort(dnsServer, "53")) if r != nil && len(r.Answer) > 0 { if a, ok := r.Answer[0].(*dns.A); ok { atomic.AddInt32(&spiderPending, 1) fetchAndStoreDNS(sz, a.A.String()) } } }(hName, t.Ns) } } } } } func directLookup(zoneID int, hostname string, server string) { defer atomic.AddInt32(&spiderPending, -1) if atomic.LoadInt32(&stopRequested) == 1 { return } m := new(dns.Msg); m.SetQuestion(dns.Fqdn(hostname), dns.TypeA) c := new(dns.Client); c.Timeout = 2 * time.Second r, _, err := c.Exchange(m, net.JoinHostPort(server, "53")) if err == nil && r != nil { for _, ans := range r.Answer { if t, ok := ans.(*dns.A); ok { var lastID int h := strings.ToLower(t.Header().Name) errDB := dbGlobal.QueryRow(`INSERT INTO dns_records (zone_id, hostname, record_type, value) VALUES ($1, $2, $3, $4) ON CONFLICT (zone_id, hostname, record_type, value) DO UPDATE SET last_seen = CURRENT_TIMESTAMP RETURNING id`, zoneID, h, "A", t.A.String()).Scan(&lastID) if errDB != nil { logMsg("[❌ DB ERROR] Αποτυχία αποθήκευσης Force Lookup '%s': %v", h, errDB) } else if lastID > 0 { atomic.AddInt32(&spiderPending, 1) spiderQueue <- SpiderJob{RecordID: lastID, CurrentURL: "http://" + strings.TrimSuffix(h, "."), BaseHost: strings.TrimSuffix(h, "."), Depth: 0} } } } } } func startWebUI() { mux := http.NewServeMux() mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/" { http.NotFound(w, r); return } w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Write([]byte(htmlDashboard)) }) mux.HandleFunc("/api/stats", func(w http.ResponseWriter, r *http.Request) { globalStats.Lock() defer globalStats.Unlock() stateMutex.Lock() scanStat := isScanning enabled := daemonEnabled target := nextRunTime stateMutex.Unlock() nextRunStr := "Daemon Stopped" if enabled { if target.IsZero() { nextRunStr = "Manual Only" } else { rem := time.Until(target) if rem > 0 { nextRunStr = fmt.Sprintf("%dh %dm", int(rem.Hours()), int(rem.Minutes())%60) } else { nextRunStr = "Imminent" } } } json.NewEncoder(w).Encode(map[string]interface{}{ "isScanning": scanStat, "daemonEnabled": enabled, "nextRun": nextRunStr, "spiderQueue": len(spiderQueue), "aiQueue": len(aiQueue), "active": atomic.LoadUint64(&sitesActive), "dead": atomic.LoadUint64(&sitesOffline), "skipped": atomic.LoadUint64(&sitesSkipped), "media": atomic.LoadUint64(&mediaFound), "categories": globalStats.CategoryCounts, }) }) mux.HandleFunc("/api/logs", func(w http.ResponseWriter, r *http.Request) { logMutex.Lock() defer logMutex.Unlock() json.NewEncoder(w).Encode(logBuffer) }) mux.HandleFunc("/api/config", func(w http.ResponseWriter, r *http.Request) { if r.Method == "GET" { cfgMutex.RLock() defer cfgMutex.RUnlock() json.NewEncoder(w).Encode(cfg) } else if r.Method == "POST" { var newCfg AppConfig if err := json.NewDecoder(r.Body).Decode(&newCfg); err == nil { cfgMutex.Lock() cfg = newCfg saveConfigInternal() cfgMutex.Unlock() calcNextRun() logMsg("[🔧 SYSTEM] Config saved via Web UI.") w.WriteHeader(http.StatusOK) } else { http.Error(w, err.Error(), http.StatusBadRequest) } } }) mux.HandleFunc("/api/config/reload", func(w http.ResponseWriter, r *http.Request) { if r.Method == "POST" { loadConfig() calcNextRun() w.WriteHeader(http.StatusOK) } }) mux.HandleFunc("/api/daemon/start", func(w http.ResponseWriter, r *http.Request) { stateMutex.Lock() daemonEnabled = true stateMutex.Unlock() calcNextRun() logMsg("[▶️ SYSTEM] Daemon Started (Scheduler active).") w.WriteHeader(http.StatusOK) }) mux.HandleFunc("/api/daemon/stop", func(w http.ResponseWriter, r *http.Request) { stateMutex.Lock() daemonEnabled = false stateMutex.Unlock() stopScan() w.WriteHeader(http.StatusOK) }) mux.HandleFunc("/api/scan/force", func(w http.ResponseWriter, r *http.Request) { if r.Method == "POST" { go triggerScan() w.WriteHeader(http.StatusOK) } }) cfgMutex.RLock() port := cfg.WebUIPort cfgMutex.RUnlock() if !strings.Contains(port, ":") { port = ":" + port } logMsg("[🌍 WEB-UI] Daemon C&C Dashboard running on 0.0.0.0%s", port) err := http.ListenAndServe(port, mux) if err != nil { logMsg("[❌ FATAL] Web UI Failed: %v", err) } } const htmlDashboard = ` WiSearch C&C

WiSearch C&C

● STATUS Next Run: --

Spider Queue

0

AI Queue

0

Active Nodes

0

Dead Nodes

0

Skipped

0

Media

0

AI Classifier

    `