#!/usr/bin/env python3
"""
Interactief script om mailinglijsten te beoordelen.
h = houden  |  a = afmelden  |  s = overslaan  |  q = stoppen

Eerste run: scant mailbox en slaat cache op in mailing_lists_cache.json.
Volgende runs: laadt direct uit cache.
Beslissingen worden opgeslagen in decisions.json (herstart-safe).
"""

import email
import email.header
import email.utils
import re
import sys
import tty
import termios
import json
from pathlib import Path
from collections import defaultdict, Counter
from datetime import timezone

MAILBOX_DIR    = Path(__file__).parent / "mailbox"
CACHE_FILE     = Path(__file__).parent / "mailing_lists_cache.json"
DECISIONS_FILE = Path(__file__).parent / "decisions.json"

# Domeinen die door meerdere afzenders worden gedeeld → groeperen op naam, niet domein
MULTITENANT_DOMAINS = {
    "substack.com", "mcsv.net", "list-manage.com",
    "sparkpostmail.com", "eu.sparkpostmail.com",
    "convertkit-mail.com", "convertkit-mail2.com", "convertkit-mail4.com",
    "sendgrid.net", "mailgun.org", "dripemail.com", "dripemail3.com",
    "mailchimpapp.net", "rsgsv.net",
}

MIN_COUNT = 3  # lijsten met minder dan dit aantal mails worden overgeslagen


# ── helpers ──────────────────────────────────────────────────────────────────

def decode_hdr(value) -> str:
    if not value:
        return ""
    try:
        parts = email.header.decode_header(value)
        out = []
        for raw, cs in parts:
            if isinstance(raw, bytes):
                out.append(raw.decode(cs or "utf-8", errors="replace"))
            else:
                out.append(str(raw))
        return " ".join(out).strip()
    except Exception:
        return str(value or "")


def parse_dt(s):
    if not s:
        return None
    try:
        return email.utils.parsedate_to_datetime(s)
    except Exception:
        return None


def to_utc_ts(d) -> float | None:
    if d is None:
        return None
    if d.tzinfo is None:
        d = d.replace(tzinfo=timezone.utc)
    return d.timestamp()


_EMAIL_SUBDOMAINS = re.compile(
    r"^(email|mail|e|m|news|newsletter|noreply|no-reply|lists?|"
    r"mailing|bounce|notifications?|updates?|info|reply|send|smtp|"
    r"mg|sg|em|learn|go|click|edm|cdn|app|get)\.",
    re.I,
)

def sender_domain(from_addr: str) -> str:
    m = re.search(r"@([\w.\-]+)", from_addr)
    if not m:
        return "onbekend"
    domain = m.group(1).lower()
    # Strip bekende e-mail-subdomeinen zodat bijv. email.coursera.org → coursera.org
    parts = domain.split(".")
    while len(parts) > 2 and _EMAIL_SUBDOMAINS.match(parts[0] + "."):
        parts = parts[1:]
    return ".".join(parts)


def display_name(from_addr: str) -> str:
    """Haal leesbare naam op; val terug op domein."""
    m = re.match(r'^"?([^"<@\n]{3,})"?\s*<', from_addr)
    if m:
        name = m.group(1).strip().strip('"').strip()
        if len(name) > 2:
            return name
    return sender_domain(from_addr)


def normalize_name(name: str) -> str:
    """Normaliseer naam voor groepering (lowercase, alleen woorden)."""
    return " ".join(re.findall(r"[a-z0-9]+", name.lower()))


def fmt_date(ts: float | None) -> str:
    if ts is None:
        return "?"
    from datetime import datetime
    return datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d")


def getch() -> str:
    fd = sys.stdin.fileno()
    old = termios.tcgetattr(fd)
    try:
        tty.setraw(fd)
        return sys.stdin.read(1)
    finally:
        termios.tcsetattr(fd, termios.TCSADRAIN, old)


# ── cache bouwen ─────────────────────────────────────────────────────────────

def build_cache() -> list[dict]:
    """
    Scan mailbox headers en groepeer op weergavenaam.
    Elke entry in de cache is één logische mailinglijst.
    """
    eml_files = list(MAILBOX_DIR.rglob("*.eml"))
    total = len(eml_files)
    print(f"Scannen van {total} e-mails (alleen headers)...", flush=True)

    # Per weergavenaam (genormaliseerd) verzamelen we alle info
    by_name: dict[str, dict] = defaultdict(lambda: {
        "names": [],          # alle aangetroffen weergavenamen
        "domains": [],        # alle aangetroffen domeinen
        "unsubscribe": [],    # List-Unsubscribe waarden
        "count": 0,
        "timestamps": [],     # als float (UTC)
        "subjects": [],
        "folders": [],
    })

    for i, path in enumerate(eml_files, 1):
        if i % 2000 == 0:
            print(f"  {i}/{total}...", flush=True)

        try:
            with open(path, "rb") as f:
                chunk = f.read(8192)
            msg = email.message_from_bytes(chunk)
        except Exception:
            continue

        from_raw = decode_hdr(msg.get("From", ""))
        subject  = decode_hdr(msg.get("Subject", ""))
        date     = parse_dt(msg.get("Date"))
        list_id  = (msg.get("List-ID") or "").strip()
        list_us  = (msg.get("List-Unsubscribe") or "").strip()
        prec     = (msg.get("Precedence") or "").lower()
        folder   = path.parent.name

        is_ml = bool(
            list_id or list_us
            or prec in ("bulk", "list")
            or "mailinglist" in folder.lower()
        )
        if not is_ml:
            continue

        name   = display_name(from_raw)
        domain = sender_domain(from_raw)
        # Multi-tenant domeinen: groepeer op genormaliseerde naam
        # Gewone domeinen: groepeer op domein (voorkomt splits zoals "LinkedIn" vs "LinkedIn Job Alerts")
        if domain in MULTITENANT_DOMAINS:
            key = normalize_name(name) or domain
        else:
            key = domain

        e = by_name[key]
        e["names"].append(name)
        e["domains"].append(domain)
        e["count"] += 1
        e["folders"].append(folder)
        ts = to_utc_ts(date)
        if ts:
            e["timestamps"].append(ts)
        if len(e["subjects"]) < 3:
            e["subjects"].append(subject)
        if list_us and list_us not in e["unsubscribe"]:
            e["unsubscribe"].append(list_us)

    print(f"  {total}/{total} klaar.\n")

    # Omzetten naar gesorteerde lijst met mooie namen
    result = []
    for key, e in by_name.items():
        best_name = Counter(e["names"]).most_common(1)[0][0]
        best_domain = Counter(e["domains"]).most_common(1)[0][0]
        best_folder = Counter(e["folders"]).most_common(1)[0][0]
        ts_sorted = sorted(e["timestamps"])

        result.append({
            "key":        key,
            "name":       best_name,
            "domain":     best_domain,
            "folder":     best_folder,
            "count":      e["count"],
            "oldest":     ts_sorted[0]  if ts_sorted else None,
            "newest":     ts_sorted[-1] if ts_sorted else None,
            "subjects":   e["subjects"],
            "unsubscribe": e["unsubscribe"],
            "domains":    list(set(e["domains"])),
        })

    result.sort(key=lambda x: -x["count"])
    result = [e for e in result if e["count"] >= MIN_COUNT]

    CACHE_FILE.write_text(
        json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
    )
    print(f"{len(result)} unieke mailinglijsten opgeslagen in cache.\n")
    return result


def load_cache() -> list[dict]:
    return json.loads(CACHE_FILE.read_text(encoding="utf-8"))


# ── beslissingen ─────────────────────────────────────────────────────────────

def load_decisions() -> dict:
    if not DECISIONS_FILE.exists():
        return {}
    try:
        data = json.loads(DECISIONS_FILE.read_text(encoding="utf-8"))
        return {k: (v["keuze"] if isinstance(v, dict) else v) for k, v in data.items()}
    except Exception:
        return {}


def save_decisions(decisions: dict, lists: list[dict]):
    by_key = {e["key"]: e for e in lists}
    output = {}
    for key, keuze in decisions.items():
        e = by_key.get(key, {})
        output[key] = {
            "keuze":       keuze,
            "naam":        e.get("name", key),
            "count":       e.get("count", 0),
            "domains":     e.get("domains", []),
            "unsubscribe": e.get("unsubscribe", []),
        }
    DECISIONS_FILE.write_text(
        json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8"
    )


# ── interactieve review ───────────────────────────────────────────────────────

KEYS   = {"h", "a", "s", "q"}
LABELS = {
    "h": ("houden",     "\033[32m"),
    "a": ("afmelden",   "\033[31m"),
    "s": ("overslaan",  "\033[33m"),
}


def run_review(lists: list[dict], decisions: dict):
    total     = len(lists)
    remaining = [e for e in lists if e["key"] not in decisions]

    print(f"{'─'*64}")
    print(f"  Totaal: {total}  |  Al beoordeeld: {len(decisions)}  |  Te gaan: {len(remaining)}")
    print(f"  \033[32mh\033[0m houden   \033[31ma\033[0m afmelden   \033[33ms\033[0m overslaan   \033[90mq\033[0m stoppen")
    print(f"{'─'*64}\n")

    done = sum(1 for e in lists if e["key"] in decisions)

    for entry in remaining:
        unsub = "✓ beschikbaar" if entry["unsubscribe"] else "✗ handmatig opzoeken"
        domains_str = ", ".join(entry["domains"][:3])
        if len(entry["domains"]) > 3:
            domains_str += f" (+{len(entry['domains'])-3})"

        print(f"\033[1m[{done+1}/{total}]  {entry['name']}\033[0m")
        print(f"  \033[90mmails  :\033[0m {entry['count']}   "
              f"({fmt_date(entry['oldest'])} – {fmt_date(entry['newest'])})")
        print(f"  \033[90mdomein :\033[0m {domains_str}")
        print(f"  \033[90mmap    :\033[0m {entry['folder']}")
        print(f"  \033[90mafmeld :\033[0m {unsub}")
        if entry["subjects"]:
            print(f"  \033[90mvoorbeeld:\033[0m {entry['subjects'][0][:68]}")
        print()
        print("  > ", end="", flush=True)

        ch = ""
        while ch not in KEYS:
            ch = getch().lower()

        if ch == "q":
            print("\n\nGestopt. Voortgang opgeslagen.\n")
            save_decisions(decisions, lists)
            print_summary(decisions)
            return

        label, color = LABELS[ch]
        print(f"{color}{label}\033[0m\n")
        decisions[entry["key"]] = ch
        done += 1
        save_decisions(decisions, lists)

    print(f"{'─'*64}")
    print(f"Klaar! Alle {total} lijsten beoordeeld.")
    save_decisions(decisions, lists)
    print_summary(decisions)


def print_summary(decisions: dict):
    a = sum(1 for c in decisions.values() if c == "a")
    h = sum(1 for c in decisions.values() if c == "h")
    s = sum(1 for c in decisions.values() if c == "s")
    print(f"\n  Afmelden    : {a}")
    print(f"  Houden      : {h}")
    print(f"  Overgeslagen: {s}")
    print(f"\nOpgeslagen in: {DECISIONS_FILE}")
    if a:
        print("Draai daarna:  python3 unsubscribe.py")


# ── main ──────────────────────────────────────────────────────────────────────

if __name__ == "__main__":
    if CACHE_FILE.exists():
        print(f"Cache gevonden, laden...\n")
        lists = load_cache()
    else:
        lists = build_cache()

    decisions = load_decisions()
    if decisions:
        active_keys = {e["key"] for e in lists}
        active = {k: v for k, v in decisions.items() if k in active_keys}
        a = sum(1 for c in active.values() if c == "a")
        h = sum(1 for c in active.values() if c == "h")
        s = sum(1 for c in active.values() if c == "s")
        stale = len(decisions) - len(active)
        msg = f"Vorige sessie: {a} afmelden, {h} houden, {s} overgeslagen"
        if stale:
            msg += f" ({stale} verouderde beslissingen genegeerd)"
        print(msg + ".\n")

    run_review(lists, decisions)