mailcat
357 lines
12 KiB
Python
357 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Interactief script om mailinglijsten te beoordelen.
|
||
h = houden | a = afmelden | s = overslaan | q = stoppen
|
||
|
||
Eerste run: scant mailbox en slaat cache op in mailing_lists_cache.json.
|
||
Volgende runs: laadt direct uit cache.
|
||
Beslissingen worden opgeslagen in decisions.json (herstart-safe).
|
||
"""
|
||
|
||
import email
|
||
import email.header
|
||
import email.utils
|
||
import re
|
||
import sys
|
||
import tty
|
||
import termios
|
||
import json
|
||
from pathlib import Path
|
||
from collections import defaultdict, Counter
|
||
from datetime import timezone
|
||
|
||
MAILBOX_DIR = Path(__file__).parent / "mailbox"
|
||
CACHE_FILE = Path(__file__).parent / "mailing_lists_cache.json"
|
||
DECISIONS_FILE = Path(__file__).parent / "decisions.json"
|
||
|
||
# Domeinen die door meerdere afzenders worden gedeeld → groeperen op naam, niet domein
|
||
MULTITENANT_DOMAINS = {
|
||
"substack.com", "mcsv.net", "list-manage.com",
|
||
"sparkpostmail.com", "eu.sparkpostmail.com",
|
||
"convertkit-mail.com", "convertkit-mail2.com", "convertkit-mail4.com",
|
||
"sendgrid.net", "mailgun.org", "dripemail.com", "dripemail3.com",
|
||
"mailchimpapp.net", "rsgsv.net",
|
||
}
|
||
|
||
MIN_COUNT = 3 # lijsten met minder dan dit aantal mails worden overgeslagen
|
||
|
||
|
||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
def decode_hdr(value) -> str:
|
||
if not value:
|
||
return ""
|
||
try:
|
||
parts = email.header.decode_header(value)
|
||
out = []
|
||
for raw, cs in parts:
|
||
if isinstance(raw, bytes):
|
||
out.append(raw.decode(cs or "utf-8", errors="replace"))
|
||
else:
|
||
out.append(str(raw))
|
||
return " ".join(out).strip()
|
||
except Exception:
|
||
return str(value or "")
|
||
|
||
|
||
def parse_dt(s):
|
||
if not s:
|
||
return None
|
||
try:
|
||
return email.utils.parsedate_to_datetime(s)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def to_utc_ts(d) -> float | None:
|
||
if d is None:
|
||
return None
|
||
if d.tzinfo is None:
|
||
d = d.replace(tzinfo=timezone.utc)
|
||
return d.timestamp()
|
||
|
||
|
||
_EMAIL_SUBDOMAINS = re.compile(
|
||
r"^(email|mail|e|m|news|newsletter|noreply|no-reply|lists?|"
|
||
r"mailing|bounce|notifications?|updates?|info|reply|send|smtp|"
|
||
r"mg|sg|em|learn|go|click|edm|cdn|app|get)\.",
|
||
re.I,
|
||
)
|
||
|
||
def sender_domain(from_addr: str) -> str:
|
||
m = re.search(r"@([\w.\-]+)", from_addr)
|
||
if not m:
|
||
return "onbekend"
|
||
domain = m.group(1).lower()
|
||
# Strip bekende e-mail-subdomeinen zodat bijv. email.coursera.org → coursera.org
|
||
parts = domain.split(".")
|
||
while len(parts) > 2 and _EMAIL_SUBDOMAINS.match(parts[0] + "."):
|
||
parts = parts[1:]
|
||
return ".".join(parts)
|
||
|
||
|
||
def display_name(from_addr: str) -> str:
|
||
"""Haal leesbare naam op; val terug op domein."""
|
||
m = re.match(r'^"?([^"<@\n]{3,})"?\s*<', from_addr)
|
||
if m:
|
||
name = m.group(1).strip().strip('"').strip()
|
||
if len(name) > 2:
|
||
return name
|
||
return sender_domain(from_addr)
|
||
|
||
|
||
def normalize_name(name: str) -> str:
|
||
"""Normaliseer naam voor groepering (lowercase, alleen woorden)."""
|
||
return " ".join(re.findall(r"[a-z0-9]+", name.lower()))
|
||
|
||
|
||
def fmt_date(ts: float | None) -> str:
|
||
if ts is None:
|
||
return "?"
|
||
from datetime import datetime
|
||
return datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d")
|
||
|
||
|
||
def getch() -> str:
|
||
fd = sys.stdin.fileno()
|
||
old = termios.tcgetattr(fd)
|
||
try:
|
||
tty.setraw(fd)
|
||
return sys.stdin.read(1)
|
||
finally:
|
||
termios.tcsetattr(fd, termios.TCSADRAIN, old)
|
||
|
||
|
||
# ── cache bouwen ─────────────────────────────────────────────────────────────
|
||
|
||
def build_cache() -> list[dict]:
|
||
"""
|
||
Scan mailbox headers en groepeer op weergavenaam.
|
||
Elke entry in de cache is één logische mailinglijst.
|
||
"""
|
||
eml_files = list(MAILBOX_DIR.rglob("*.eml"))
|
||
total = len(eml_files)
|
||
print(f"Scannen van {total} e-mails (alleen headers)...", flush=True)
|
||
|
||
# Per weergavenaam (genormaliseerd) verzamelen we alle info
|
||
by_name: dict[str, dict] = defaultdict(lambda: {
|
||
"names": [], # alle aangetroffen weergavenamen
|
||
"domains": [], # alle aangetroffen domeinen
|
||
"unsubscribe": [], # List-Unsubscribe waarden
|
||
"count": 0,
|
||
"timestamps": [], # als float (UTC)
|
||
"subjects": [],
|
||
"folders": [],
|
||
})
|
||
|
||
for i, path in enumerate(eml_files, 1):
|
||
if i % 2000 == 0:
|
||
print(f" {i}/{total}...", flush=True)
|
||
|
||
try:
|
||
with open(path, "rb") as f:
|
||
chunk = f.read(8192)
|
||
msg = email.message_from_bytes(chunk)
|
||
except Exception:
|
||
continue
|
||
|
||
from_raw = decode_hdr(msg.get("From", ""))
|
||
subject = decode_hdr(msg.get("Subject", ""))
|
||
date = parse_dt(msg.get("Date"))
|
||
list_id = (msg.get("List-ID") or "").strip()
|
||
list_us = (msg.get("List-Unsubscribe") or "").strip()
|
||
prec = (msg.get("Precedence") or "").lower()
|
||
folder = path.parent.name
|
||
|
||
is_ml = bool(
|
||
list_id or list_us
|
||
or prec in ("bulk", "list")
|
||
or "mailinglist" in folder.lower()
|
||
)
|
||
if not is_ml:
|
||
continue
|
||
|
||
name = display_name(from_raw)
|
||
domain = sender_domain(from_raw)
|
||
# Multi-tenant domeinen: groepeer op genormaliseerde naam
|
||
# Gewone domeinen: groepeer op domein (voorkomt splits zoals "LinkedIn" vs "LinkedIn Job Alerts")
|
||
if domain in MULTITENANT_DOMAINS:
|
||
key = normalize_name(name) or domain
|
||
else:
|
||
key = domain
|
||
|
||
e = by_name[key]
|
||
e["names"].append(name)
|
||
e["domains"].append(domain)
|
||
e["count"] += 1
|
||
e["folders"].append(folder)
|
||
ts = to_utc_ts(date)
|
||
if ts:
|
||
e["timestamps"].append(ts)
|
||
if len(e["subjects"]) < 3:
|
||
e["subjects"].append(subject)
|
||
if list_us and list_us not in e["unsubscribe"]:
|
||
e["unsubscribe"].append(list_us)
|
||
|
||
print(f" {total}/{total} klaar.\n")
|
||
|
||
# Omzetten naar gesorteerde lijst met mooie namen
|
||
result = []
|
||
for key, e in by_name.items():
|
||
best_name = Counter(e["names"]).most_common(1)[0][0]
|
||
best_domain = Counter(e["domains"]).most_common(1)[0][0]
|
||
best_folder = Counter(e["folders"]).most_common(1)[0][0]
|
||
ts_sorted = sorted(e["timestamps"])
|
||
|
||
result.append({
|
||
"key": key,
|
||
"name": best_name,
|
||
"domain": best_domain,
|
||
"folder": best_folder,
|
||
"count": e["count"],
|
||
"oldest": ts_sorted[0] if ts_sorted else None,
|
||
"newest": ts_sorted[-1] if ts_sorted else None,
|
||
"subjects": e["subjects"],
|
||
"unsubscribe": e["unsubscribe"],
|
||
"domains": list(set(e["domains"])),
|
||
})
|
||
|
||
result.sort(key=lambda x: -x["count"])
|
||
result = [e for e in result if e["count"] >= MIN_COUNT]
|
||
|
||
CACHE_FILE.write_text(
|
||
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
|
||
)
|
||
print(f"{len(result)} unieke mailinglijsten opgeslagen in cache.\n")
|
||
return result
|
||
|
||
|
||
def load_cache() -> list[dict]:
|
||
return json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
||
|
||
|
||
# ── beslissingen ─────────────────────────────────────────────────────────────
|
||
|
||
def load_decisions() -> dict:
|
||
if not DECISIONS_FILE.exists():
|
||
return {}
|
||
try:
|
||
data = json.loads(DECISIONS_FILE.read_text(encoding="utf-8"))
|
||
return {k: (v["keuze"] if isinstance(v, dict) else v) for k, v in data.items()}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_decisions(decisions: dict, lists: list[dict]):
|
||
by_key = {e["key"]: e for e in lists}
|
||
output = {}
|
||
for key, keuze in decisions.items():
|
||
e = by_key.get(key, {})
|
||
output[key] = {
|
||
"keuze": keuze,
|
||
"naam": e.get("name", key),
|
||
"count": e.get("count", 0),
|
||
"domains": e.get("domains", []),
|
||
"unsubscribe": e.get("unsubscribe", []),
|
||
}
|
||
DECISIONS_FILE.write_text(
|
||
json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8"
|
||
)
|
||
|
||
|
||
# ── interactieve review ───────────────────────────────────────────────────────
|
||
|
||
KEYS = {"h", "a", "s", "q"}
|
||
LABELS = {
|
||
"h": ("houden", "\033[32m"),
|
||
"a": ("afmelden", "\033[31m"),
|
||
"s": ("overslaan", "\033[33m"),
|
||
}
|
||
|
||
|
||
def run_review(lists: list[dict], decisions: dict):
|
||
total = len(lists)
|
||
remaining = [e for e in lists if e["key"] not in decisions]
|
||
|
||
print(f"{'─'*64}")
|
||
print(f" Totaal: {total} | Al beoordeeld: {len(decisions)} | Te gaan: {len(remaining)}")
|
||
print(f" \033[32mh\033[0m houden \033[31ma\033[0m afmelden \033[33ms\033[0m overslaan \033[90mq\033[0m stoppen")
|
||
print(f"{'─'*64}\n")
|
||
|
||
done = sum(1 for e in lists if e["key"] in decisions)
|
||
|
||
for entry in remaining:
|
||
unsub = "✓ beschikbaar" if entry["unsubscribe"] else "✗ handmatig opzoeken"
|
||
domains_str = ", ".join(entry["domains"][:3])
|
||
if len(entry["domains"]) > 3:
|
||
domains_str += f" (+{len(entry['domains'])-3})"
|
||
|
||
print(f"\033[1m[{done+1}/{total}] {entry['name']}\033[0m")
|
||
print(f" \033[90mmails :\033[0m {entry['count']} "
|
||
f"({fmt_date(entry['oldest'])} – {fmt_date(entry['newest'])})")
|
||
print(f" \033[90mdomein :\033[0m {domains_str}")
|
||
print(f" \033[90mmap :\033[0m {entry['folder']}")
|
||
print(f" \033[90mafmeld :\033[0m {unsub}")
|
||
if entry["subjects"]:
|
||
print(f" \033[90mvoorbeeld:\033[0m {entry['subjects'][0][:68]}")
|
||
print()
|
||
print(" > ", end="", flush=True)
|
||
|
||
ch = ""
|
||
while ch not in KEYS:
|
||
ch = getch().lower()
|
||
|
||
if ch == "q":
|
||
print("\n\nGestopt. Voortgang opgeslagen.\n")
|
||
save_decisions(decisions, lists)
|
||
print_summary(decisions)
|
||
return
|
||
|
||
label, color = LABELS[ch]
|
||
print(f"{color}{label}\033[0m\n")
|
||
decisions[entry["key"]] = ch
|
||
done += 1
|
||
save_decisions(decisions, lists)
|
||
|
||
print(f"{'─'*64}")
|
||
print(f"Klaar! Alle {total} lijsten beoordeeld.")
|
||
save_decisions(decisions, lists)
|
||
print_summary(decisions)
|
||
|
||
|
||
def print_summary(decisions: dict):
|
||
a = sum(1 for c in decisions.values() if c == "a")
|
||
h = sum(1 for c in decisions.values() if c == "h")
|
||
s = sum(1 for c in decisions.values() if c == "s")
|
||
print(f"\n Afmelden : {a}")
|
||
print(f" Houden : {h}")
|
||
print(f" Overgeslagen: {s}")
|
||
print(f"\nOpgeslagen in: {DECISIONS_FILE}")
|
||
if a:
|
||
print("Draai daarna: python3 unsubscribe.py")
|
||
|
||
|
||
# ── main ──────────────────────────────────────────────────────────────────────
|
||
|
||
if __name__ == "__main__":
|
||
if CACHE_FILE.exists():
|
||
print(f"Cache gevonden, laden...\n")
|
||
lists = load_cache()
|
||
else:
|
||
lists = build_cache()
|
||
|
||
decisions = load_decisions()
|
||
if decisions:
|
||
active_keys = {e["key"] for e in lists}
|
||
active = {k: v for k, v in decisions.items() if k in active_keys}
|
||
a = sum(1 for c in active.values() if c == "a")
|
||
h = sum(1 for c in active.values() if c == "h")
|
||
s = sum(1 for c in active.values() if c == "s")
|
||
stale = len(decisions) - len(active)
|
||
msg = f"Vorige sessie: {a} afmelden, {h} houden, {s} overgeslagen"
|
||
if stale:
|
||
msg += f" ({stale} verouderde beslissingen genegeerd)"
|
||
print(msg + ".\n")
|
||
|
||
run_review(lists, decisions)
|