parental_loader.go

/*
File:    parental_loader.go
Version: 1.3.0 (Split)
Updated: 21-Apr-2026 14:42 CEST

Description:
  Data orchestration, HTTP loading, and caching for Parental Categories.

Changes:
  1.3.0 - [FEAT] Integrated `ipVersionSupport` check into `loadAllCategoryLists` 
          manual `Add` overrides to selectively discard IP versions.
  1.2.0 - [FEAT] Added support for `forceRefreshStartup`.
  ...
*/

package main

import (
	"bufio"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"log"
	"net/http"
	"net/netip"
	"os"
	"path/filepath"
	"strings"
	"time"
)

const catCacheFile = "parental-catcache.json"
const catMetaFile = "parental-catmeta.json"

type catListHeaders struct {
	LastModified string `json:"last_modified"`
	ETag         string `json:"etag"`
}

type catMetaState struct {
	ConfigHash string                    `json:"config_hash"`
	HTTPMeta   map[string]catListHeaders `json:"http_meta"`
	FileMeta   map[string]int64          `json:"file_meta"`
}

func loadCategoryMeta() catMetaState {
	var meta catMetaState
	b, err := os.ReadFile(filepath.Join(snapshotDir(), catMetaFile))
	if err == nil {
		json.Unmarshal(b, &meta)
	}
	if meta.HTTPMeta == nil {
		meta.HTTPMeta = make(map[string]catListHeaders)
	}
	if meta.FileMeta == nil {
		meta.FileMeta = make(map[string]int64)
	}
	return meta
}

func saveCategoryMeta(meta catMetaState) {
	if err := os.MkdirAll(snapshotDir(), 0755); err != nil {
		return
	}
	b, err := json.Marshal(meta)
	if err == nil {
		os.WriteFile(filepath.Join(snapshotDir(), catMetaFile), b, 0644)
	}
}

func computeConfigHash() string {
	type hashData struct {
		Categories   map[string]CategoryConfig
		Consol       *bool
		SynthParents *bool
		RemoveSub    *bool
		StripLabels  *bool
		ParentThresh int
		HomogPct     int
		IPVersion    string
	}
	hd := hashData{
		Categories:   cfg.Parental.Categories,
		Consol:       cfg.Parental.Consolidation,
		SynthParents: cfg.Parental.SynthesiseParents,
		RemoveSub:    cfg.Parental.RemoveRedundantSubdomains,
		StripLabels:  cfg.Parental.StripServiceLabels,
		ParentThresh: cfg.Parental.ParentConsolidationThreshold,
		HomogPct:     cfg.Parental.ConsolidationHomogeneityPct,
		IPVersion:    ipVersionSupport,
	}
	b, _ := json.Marshal(hd)
	h := sha256.Sum256(b)
	return hex.EncodeToString(h[:])
}

func getURLCachePath(url string) string {
	h := sha256.Sum256([]byte(url))
	return filepath.Join(snapshotDir(), "src-"+hex.EncodeToString(h[:8])+".txt")
}

func loadURLCache(path string) []string {
	f, err := os.Open(path)
	if err != nil {
		return nil
	}
	defer f.Close()
	var lines []string
	sc := bufio.NewScanner(f)
	for sc.Scan() {
		if txt := strings.TrimSpace(sc.Text()); txt != "" {
			lines = append(lines, txt)
		}
	}
	return lines
}

func saveURLCache(path string, lines []string) {
	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
		return
	}
	f, err := os.Create(path)
	if err != nil {
		return
	}
	defer f.Close()
	w := bufio.NewWriter(f)
	for _, l := range lines {
		w.WriteString(l)
		w.WriteByte('\n')
	}
	w.Flush()
}

var catHTTPClient = &http.Client{Timeout: 30 * time.Second}

func loadAllCategoryLists(force bool) {
	needed := make(map[string]bool)
	for _, grp := range cfg.Groups {
		for key := range grp.Budget {
			if key != "total" {
				needed[key] = true
			}
		}
	}

	meta := loadCategoryMeta()
	needsRebuild := false

	if force {
		log.Printf("[PARENTAL] Force-refresh enabled. Bypassing metadata caches.")
		needsRebuild = true
		meta.HTTPMeta = make(map[string]catListHeaders)
		meta.FileMeta = make(map[string]int64)
	}

	currentHash := computeConfigHash()
	if meta.ConfigHash != currentHash {
		needsRebuild = true
		meta.ConfigHash = currentHash
		if !force {
			log.Printf("[PARENTAL] Category configuration changed, full rebuild triggered")
		}
	}

	data := catMap.Load()
	if data == nil || len(data.apex) == 0 {
		needsRebuild = true
		if !force {
			log.Printf("[PARENTAL] In-memory category cache missing, full rebuild triggered")
		}
	}

	activeURLs := make(map[string]bool)
	urlData := make(map[string][]string)
	currentFileMeta := make(map[string]int64)

	for cat, cc := range cfg.Parental.Categories {
		if !needed[cat] {
			continue
		}
		for _, src := range cc.ListURLs {
			if !isSourceURL(src) {
				info, err := os.Stat(src)
				if err == nil {
					mtime := info.ModTime().UnixNano()
					currentFileMeta[src] = mtime
					if meta.FileMeta[src] != mtime {
						needsRebuild = true
					}
				} else {
					if _, ok := meta.FileMeta[src]; ok {
						needsRebuild = true
					}
				}
			}
		}
	}

	var delayedCacheLoad []string
	for cat, cc := range cfg.Parental.Categories {
		if !needed[cat] {
			continue
		}
		for _, src := range cc.ListURLs {
			if isSourceURL(src) {
				activeURLs[src] = true
				cachePath := getURLCachePath(src)

				req, err := http.NewRequest(http.MethodGet, src, nil)
				if err != nil {
					log.Printf("[PARENTAL] Category %q: request build error %s: %v", cat, src, err)
					continue
				}
				req.Header.Set("User-Agent", "sdproxy-parental/1.6")

				hasCacheFile := false
				if _, err := os.Stat(cachePath); err == nil {
					hasCacheFile = true
				}

				if !hasCacheFile {
					needsRebuild = true
				} else if !force {
					if m, ok := meta.HTTPMeta[src]; ok {
						if m.LastModified != "" {
							req.Header.Set("If-Modified-Since", m.LastModified)
						}
						if m.ETag != "" {
							req.Header.Set("If-None-Match", m.ETag)
						}
					}
				}

				resp, err := catHTTPClient.Do(req)
				if err != nil {
					cacheFile := filepath.Base(cachePath)
					if needsRebuild && hasCacheFile {
						urlData[src] = loadURLCache(cachePath)
						log.Printf("[PARENTAL] Category %q: list fetch error %s: %v — loaded from local cache (%s)", cat, src, err, cacheFile)
					} else if !needsRebuild && hasCacheFile {
						delayedCacheLoad = append(delayedCacheLoad, src)
						log.Printf("[PARENTAL] Category %q: list fetch error %s: %v — queued local cache (%s)", cat, src, err, cacheFile)
					} else {
						log.Printf("[PARENTAL] Category %q: list fetch error %s: %v", cat, src, err)
					}
					continue
				}

				if resp.StatusCode == http.StatusNotModified {
					resp.Body.Close()
					cacheFile := filepath.Base(cachePath)
					if needsRebuild && hasCacheFile {
						lines := loadURLCache(cachePath)
						urlData[src] = lines
						log.Printf("[PARENTAL] Category %q: not modified (304) — loaded %d entries from local cache (%s) for %s", cat, len(lines), cacheFile, src)
					} else if !needsRebuild {
						delayedCacheLoad = append(delayedCacheLoad, src)
						log.Printf("[PARENTAL] Category %q: not modified (304) for %s — queued local cache (%s)", cat, src, cacheFile)
					}
				} else if resp.StatusCode == http.StatusOK {
					if !needsRebuild {
						needsRebuild = true
						for _, delayedSrc := range delayedCacheLoad {
							cPath := getURLCachePath(delayedSrc)
							urlData[delayedSrc] = loadURLCache(cPath)
							log.Printf("[PARENTAL] Delayed load: fetched %d entries from local cache (%s) for %s", len(urlData[delayedSrc]), filepath.Base(cPath), delayedSrc)
						}
						delayedCacheLoad = nil
					}
					lines, eTLDs := parseSourceBody(resp.Body)
					resp.Body.Close()
					saveURLCache(cachePath, lines)
					urlData[src] = lines
					meta.HTTPMeta[src] = catListHeaders{
						LastModified: resp.Header.Get("Last-Modified"),
						ETag:         resp.Header.Get("ETag"),
					}
					if eTLDs > 0 {
						log.Printf("[PARENTAL] Category %q: WARNING — loaded %d eTLD/bare-TLD entries from %s", cat, eTLDs, src)
					}
					cacheFile := filepath.Base(cachePath)
					log.Printf("[PARENTAL] Category %q: fetched %d entries from %s — saved to cache (%s)", cat, len(lines), src, cacheFile)
				} else {
					resp.Body.Close()
					cacheFile := filepath.Base(cachePath)
					if needsRebuild && hasCacheFile {
						urlData[src] = loadURLCache(cachePath)
						log.Printf("[PARENTAL] Category %q: HTTP %d for %s — loaded from local cache (%s)", cat, resp.StatusCode, src, cacheFile)
					} else if !needsRebuild && hasCacheFile {
						delayedCacheLoad = append(delayedCacheLoad, src)
						log.Printf("[PARENTAL] Category %q: HTTP %d for %s — queued local cache (%s)", cat, resp.StatusCode, src, cacheFile)
					} else {
						log.Printf("[PARENTAL] Category %q: HTTP %d for %s", cat, resp.StatusCode, src)
					}
				}
			}
		}
	}

	for url := range meta.HTTPMeta {
		if !activeURLs[url] {
			delete(meta.HTTPMeta, url)
			os.Remove(getURLCachePath(url))
			needsRebuild = true
			log.Printf("[PARENTAL] Garbage collected stale cache for removed URL: %s", url)
		}
	}
	for file := range meta.FileMeta {
		if _, exists := currentFileMeta[file]; !exists {
			delete(meta.FileMeta, file)
			needsRebuild = true
		}
	}

	if needsRebuild && len(delayedCacheLoad) > 0 {
		for _, delayedSrc := range delayedCacheLoad {
			cPath := getURLCachePath(delayedSrc)
			urlData[delayedSrc] = loadURLCache(cPath)
			log.Printf("[PARENTAL] Delayed load: fetched %d entries from local cache (%s) for %s", len(urlData[delayedSrc]), filepath.Base(cPath), delayedSrc)
		}
		delayedCacheLoad = nil
	}

	if !needsRebuild {
		log.Printf("[PARENTAL] All lists and configs up to date. Processing skipped.")
		return
	}

	meta.FileMeta = currentFileMeta
	saveCategoryMeta(meta)

	newApex := make(map[string]string, 4096)
	var newPatterns []wildcardCatEntry
	newIPs := make(map[netip.Addr]string)
	var newCIDRs []cidrCatEntry

	doStrip := false
	if cfg.Parental.StripServiceLabels != nil {
		doStrip = *cfg.Parental.StripServiceLabels
	}

	for cat, cc := range cfg.Parental.Categories {
		if !needed[cat] {
			continue
		}

		for _, d := range cc.Add {
			d = strings.ToLower(strings.TrimSuffix(d, "."))
			if ip, err := netip.ParseAddr(d); err == nil {
				unmapped := ip.Unmap()
				if ipVersionSupport != "both" {
					if ipVersionSupport == "ipv4" && !unmapped.Is4() { continue }
					if ipVersionSupport == "ipv6" && !unmapped.Is6() { continue }
				}
				newIPs[unmapped] = cat
				continue
			}
			if prefix, err := netip.ParsePrefix(d); err == nil {
				if ipVersionSupport != "both" {
					if ipVersionSupport == "ipv4" && !prefix.Addr().Is4() { continue }
					if ipVersionSupport == "ipv6" && !prefix.Addr().Is6() { continue }
				}
				newCIDRs = append(newCIDRs, cidrCatEntry{prefix: prefix, cat: cat})
				continue
			}
			if isWildcard(d) {
				newPatterns = append(newPatterns, wildcardCatEntry{pattern: d, cat: cat})
			} else if d != "" {
				if !isIPOrCIDR(d) && isPublicSuffix(d) {
					log.Printf("[PARENTAL] Category %q: WARNING — manual 'add:' entry includes eTLD/bare-TLD %q", cat, d)
				}
				if doStrip {
					if stripped, ok := stripServiceLabel(d); ok {
						log.Printf("[PARENTAL] Category %q: stripped service label %q → %q (add: entry)", cat, d, stripped)
						d = stripped
					}
				}
				newApex[d] = cat
			}
		}

		for _, src := range cc.ListURLs {
			var lines []string
			if isSourceURL(src) {
				lines = urlData[src]
			} else {
				fLines, eTLDs := parseLocalFile(src)
				if eTLDs > 0 {
					log.Printf("[PARENTAL] Category %q: WARNING — loaded %d eTLD/bare-TLD entries from %s (override with 'remove:' if unintended)", cat, eTLDs, src)
				}
				lines = fLines
			}

			for _, d := range lines {
				if ip, err := netip.ParseAddr(d); err == nil {
					unmapped := ip.Unmap()
					if ipVersionSupport != "both" {
						if ipVersionSupport == "ipv4" && !unmapped.Is4() { continue }
						if ipVersionSupport == "ipv6" && !unmapped.Is6() { continue }
					}
					if _, exists := newIPs[unmapped]; !exists {
						newIPs[unmapped] = cat
					}
					continue
				}
				if prefix, err := netip.ParsePrefix(d); err == nil {
					if ipVersionSupport != "both" {
						if ipVersionSupport == "ipv4" && !prefix.Addr().Is4() { continue }
						if ipVersionSupport == "ipv6" && !prefix.Addr().Is6() { continue }
					}
					newCIDRs = append(newCIDRs, cidrCatEntry{prefix: prefix, cat: cat})
					continue
				}

				if doStrip {
					if stripped, ok := stripServiceLabel(d); ok {
						log.Printf("[PARENTAL] Category %q: stripped service label %q → %q", cat, d, stripped)
						d = stripped
					}
				}
				if _, exists := newApex[d]; !exists {
					newApex[d] = cat
				}
			}
		}

		for _, d := range cc.Remove {
			d = strings.ToLower(strings.TrimSuffix(d, "."))
			if ip, err := netip.ParseAddr(d); err == nil {
				unmapped := ip.Unmap()
				if newIPs[unmapped] == cat {
					delete(newIPs, unmapped)
				}
				continue
			}
			if prefix, err := netip.ParsePrefix(d); err == nil {
				var kept []cidrCatEntry
				for _, c := range newCIDRs {
					if c.cat == cat && c.prefix == prefix {
						continue
					}
					kept = append(kept, c)
				}
				newCIDRs = kept
				continue
			}
			if isWildcard(d) {
				for k, c := range newApex {
					if c == cat && matchGlob(d, k) {
						delete(newApex, k)
					}
				}
				var kept []wildcardCatEntry
				for _, p := range newPatterns {
					if p.cat == cat && p.pattern == d {
						continue
					}
					kept = append(kept, p)
				}
				newPatterns = kept
			} else if d != "" {
				if newApex[d] == cat {
					delete(newApex, d)
				}
			}
		}
	}

	doConsolidation := true
	if cfg.Parental.Consolidation != nil {
		doConsolidation = *cfg.Parental.Consolidation
	}

	if doConsolidation {
		doSynth := false
		if cfg.Parental.SynthesiseParents != nil {
			doSynth = *cfg.Parental.SynthesiseParents
		}

		if doSynth {
			threshold := cfg.Parental.ParentConsolidationThreshold
			if threshold <= 0 {
				threshold = 10
			}
			homogPct := cfg.Parental.ConsolidationHomogeneityPct
			if homogPct <= 0 || homogPct > 100 {
				homogPct = 90
			}
			if n := consolidateParentDomains(newApex, threshold, homogPct); n > 0 {
				log.Printf("[PARENTAL] Synthesised %d parent domain(s) via consolidation", n)
			}
		}

		doRemove := true
		if cfg.Parental.RemoveRedundantSubdomains != nil {
			doRemove = *cfg.Parental.RemoveRedundantSubdomains
		}

		if doRemove {
			catCountsBefore := make(map[string]int)
			for _, cat := range newApex {
				catCountsBefore[cat]++
			}

			if n := dedupeApex(newApex); n > 0 {
				catCountsAfter := make(map[string]int)
				for _, cat := range newApex {
					catCountsAfter[cat]++
				}
				for cat, before := range catCountsBefore {
					after := catCountsAfter[cat]
					if removed := before - after; removed > 0 {
						log.Printf("[PARENTAL] Category %q: dedupped from %d to %d (removed %d redundant sub-domain apex entries)", cat, before, after, removed)
					}
				}
			}
		}
	}

	catMap.Store(&categoryData{apex: newApex, patterns: newPatterns, ips: newIPs, cidrs: newCIDRs})
	computeCatLabelBounds(newApex)
	saveCatCache(newApex, newPatterns, newIPs, newCIDRs)
	log.Printf("[PARENTAL] Category maps ready: %d apex entries, %d wildcard pattern(s), %d IPs, %d CIDRs", len(newApex), len(newPatterns), len(newIPs), len(newCIDRs))
}

func computeCatLabelBounds(apex map[string]string) {
	if len(apex) == 0 {
		catMinLabels = 1
		catMaxLabels = 128
		return
	}
	minL, maxL := int(^uint(0)>>1), 0
	for k := range apex {
		n := countDomainLabels(k)
		if n < minL {
			minL = n
		}
		if n > maxL {
			maxL = n
		}
	}
	catMinLabels = minL
	catMaxLabels = maxL
	log.Printf("[PARENTAL] Category walk bounds: [%d..%d] labels (%d apex entries)", catMinLabels, catMaxLabels, len(apex))
}

type catCacheData struct {
	Apex     map[string]string  `json:"apex"`
	Patterns []wildcardCatEntry `json:"patterns"`
	IPs      map[string]string  `json:"ips"`
	CIDRs    []cidrCatEntryStr  `json:"cidrs"`
}

type cidrCatEntryStr struct {
	Prefix string `json:"prefix"`
	Cat    string `json:"cat"`
}

func loadCatCache() {
	path := filepath.Join(snapshotDir(), catCacheFile)
	b, err := os.ReadFile(path)
	if err != nil {
		return
	}
	var data catCacheData
	if err := json.Unmarshal(b, &data); err != nil {
		var apex map[string]string
		if err := json.Unmarshal(b, &apex); err != nil {
			log.Printf("[PARENTAL] Cat-cache parse error: %v", err)
			return
		}
		data.Apex = apex
	}

	var patterns []wildcardCatEntry
	if len(data.Patterns) == 0 {
		for cat, cc := range cfg.Parental.Categories {
			for _, d := range cc.Add {
				d = strings.ToLower(strings.TrimSuffix(d, "."))
				if isWildcard(d) {
					patterns = append(patterns, wildcardCatEntry{pattern: d, cat: cat})
				}
			}
		}
	} else {
		patterns = data.Patterns
	}

	ips := make(map[netip.Addr]string, len(data.IPs))
	for k, v := range data.IPs {
		if ip, err := netip.ParseAddr(k); err == nil {
			ips[ip] = v
		}
	}

	var cidrs []cidrCatEntry
	for _, c := range data.CIDRs {
		if prefix, err := netip.ParsePrefix(c.Prefix); err == nil {
			cidrs = append(cidrs, cidrCatEntry{prefix: prefix, cat: c.Cat})
		}
	}

	catMap.Store(&categoryData{
		apex:     data.Apex,
		patterns: patterns,
		ips:      ips,
		cidrs:    cidrs,
	})
	computeCatLabelBounds(data.Apex)
	log.Printf("[PARENTAL] Category cache loaded: %d apex, %d patterns, %d IPs, %d CIDRs", len(data.Apex), len(patterns), len(ips), len(cidrs))
}

func saveCatCache(apex map[string]string, patterns []wildcardCatEntry, ips map[netip.Addr]string, cidrs []cidrCatEntry) {
	path := filepath.Join(snapshotDir(), catCacheFile)
	if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
		return
	}

	data := catCacheData{
		Apex:     apex,
		Patterns: patterns,
		IPs:      make(map[string]string, len(ips)),
		CIDRs:    make([]cidrCatEntryStr, len(cidrs)),
	}
	for k, v := range ips {
		data.IPs[k.String()] = v
	}
	for i, c := range cidrs {
		data.CIDRs[i] = cidrCatEntryStr{Prefix: c.prefix.String(), Cat: c.cat}
	}

	b, err := json.Marshal(data)
	if err != nil {
		return
	}
	_ = os.WriteFile(path, b, 0644)
}