wienen
4780c18d70
Claude has done a lot of work, but there are still some errors which cause the mails to be distributed to wrong mail boxes. I am now going to let codex and claude compete. They will have their own branches and merge to main as soon as we are somewhere where we can switch AI.
357 lines
12 KiB
Python
357 lines
12 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Interactief script om mailinglijsten te beoordelen.
|
||
h = houden | a = afmelden | s = overslaan | q = stoppen
|
||
|
||
Eerste run: scant mailbox en slaat cache op in mailing_lists_cache.json.
|
||
Volgende runs: laadt direct uit cache.
|
||
Beslissingen worden opgeslagen in decisions.json (herstart-safe).
|
||
"""
|
||
|
||
import email
|
||
import email.header
|
||
import email.utils
|
||
import re
|
||
import sys
|
||
import tty
|
||
import termios
|
||
import json
|
||
from pathlib import Path
|
||
from collections import defaultdict, Counter
|
||
from datetime import timezone
|
||
|
||
MAILBOX_DIR = Path(__file__).parent / "mailbox"
|
||
CACHE_FILE = Path(__file__).parent / "mailing_lists_cache.json"
|
||
DECISIONS_FILE = Path(__file__).parent / "decisions.json"
|
||
|
||
# Domeinen die door meerdere afzenders worden gedeeld → groeperen op naam, niet domein
|
||
MULTITENANT_DOMAINS = {
|
||
"substack.com", "mcsv.net", "list-manage.com",
|
||
"sparkpostmail.com", "eu.sparkpostmail.com",
|
||
"convertkit-mail.com", "convertkit-mail2.com", "convertkit-mail4.com",
|
||
"sendgrid.net", "mailgun.org", "dripemail.com", "dripemail3.com",
|
||
"mailchimpapp.net", "rsgsv.net",
|
||
}
|
||
|
||
MIN_COUNT = 3 # lijsten met minder dan dit aantal mails worden overgeslagen
|
||
|
||
|
||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
def decode_hdr(value) -> str:
|
||
if not value:
|
||
return ""
|
||
try:
|
||
parts = email.header.decode_header(value)
|
||
out = []
|
||
for raw, cs in parts:
|
||
if isinstance(raw, bytes):
|
||
out.append(raw.decode(cs or "utf-8", errors="replace"))
|
||
else:
|
||
out.append(str(raw))
|
||
return " ".join(out).strip()
|
||
except Exception:
|
||
return str(value or "")
|
||
|
||
|
||
def parse_dt(s):
|
||
if not s:
|
||
return None
|
||
try:
|
||
return email.utils.parsedate_to_datetime(s)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def to_utc_ts(d) -> float | None:
|
||
if d is None:
|
||
return None
|
||
if d.tzinfo is None:
|
||
d = d.replace(tzinfo=timezone.utc)
|
||
return d.timestamp()
|
||
|
||
|
||
_EMAIL_SUBDOMAINS = re.compile(
|
||
r"^(email|mail|e|m|news|newsletter|noreply|no-reply|lists?|"
|
||
r"mailing|bounce|notifications?|updates?|info|reply|send|smtp|"
|
||
r"mg|sg|em|learn|go|click|edm|cdn|app|get)\.",
|
||
re.I,
|
||
)
|
||
|
||
def sender_domain(from_addr: str) -> str:
|
||
m = re.search(r"@([\w.\-]+)", from_addr)
|
||
if not m:
|
||
return "onbekend"
|
||
domain = m.group(1).lower()
|
||
# Strip bekende e-mail-subdomeinen zodat bijv. email.coursera.org → coursera.org
|
||
parts = domain.split(".")
|
||
while len(parts) > 2 and _EMAIL_SUBDOMAINS.match(parts[0] + "."):
|
||
parts = parts[1:]
|
||
return ".".join(parts)
|
||
|
||
|
||
def display_name(from_addr: str) -> str:
|
||
"""Haal leesbare naam op; val terug op domein."""
|
||
m = re.match(r'^"?([^"<@\n]{3,})"?\s*<', from_addr)
|
||
if m:
|
||
name = m.group(1).strip().strip('"').strip()
|
||
if len(name) > 2:
|
||
return name
|
||
return sender_domain(from_addr)
|
||
|
||
|
||
def normalize_name(name: str) -> str:
|
||
"""Normaliseer naam voor groepering (lowercase, alleen woorden)."""
|
||
return " ".join(re.findall(r"[a-z0-9]+", name.lower()))
|
||
|
||
|
||
def fmt_date(ts: float | None) -> str:
|
||
if ts is None:
|
||
return "?"
|
||
from datetime import datetime
|
||
return datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d")
|
||
|
||
|
||
def getch() -> str:
|
||
fd = sys.stdin.fileno()
|
||
old = termios.tcgetattr(fd)
|
||
try:
|
||
tty.setraw(fd)
|
||
return sys.stdin.read(1)
|
||
finally:
|
||
termios.tcsetattr(fd, termios.TCSADRAIN, old)
|
||
|
||
|
||
# ── cache bouwen ─────────────────────────────────────────────────────────────
|
||
|
||
def build_cache() -> list[dict]:
|
||
"""
|
||
Scan mailbox headers en groepeer op weergavenaam.
|
||
Elke entry in de cache is één logische mailinglijst.
|
||
"""
|
||
eml_files = list(MAILBOX_DIR.rglob("*.eml"))
|
||
total = len(eml_files)
|
||
print(f"Scannen van {total} e-mails (alleen headers)...", flush=True)
|
||
|
||
# Per weergavenaam (genormaliseerd) verzamelen we alle info
|
||
by_name: dict[str, dict] = defaultdict(lambda: {
|
||
"names": [], # alle aangetroffen weergavenamen
|
||
"domains": [], # alle aangetroffen domeinen
|
||
"unsubscribe": [], # List-Unsubscribe waarden
|
||
"count": 0,
|
||
"timestamps": [], # als float (UTC)
|
||
"subjects": [],
|
||
"folders": [],
|
||
})
|
||
|
||
for i, path in enumerate(eml_files, 1):
|
||
if i % 2000 == 0:
|
||
print(f" {i}/{total}...", flush=True)
|
||
|
||
try:
|
||
with open(path, "rb") as f:
|
||
chunk = f.read(8192)
|
||
msg = email.message_from_bytes(chunk)
|
||
except Exception:
|
||
continue
|
||
|
||
from_raw = decode_hdr(msg.get("From", ""))
|
||
subject = decode_hdr(msg.get("Subject", ""))
|
||
date = parse_dt(msg.get("Date"))
|
||
list_id = (msg.get("List-ID") or "").strip()
|
||
list_us = (msg.get("List-Unsubscribe") or "").strip()
|
||
prec = (msg.get("Precedence") or "").lower()
|
||
folder = path.parent.name
|
||
|
||
is_ml = bool(
|
||
list_id or list_us
|
||
or prec in ("bulk", "list")
|
||
or "mailinglist" in folder.lower()
|
||
)
|
||
if not is_ml:
|
||
continue
|
||
|
||
name = display_name(from_raw)
|
||
domain = sender_domain(from_raw)
|
||
# Multi-tenant domeinen: groepeer op genormaliseerde naam
|
||
# Gewone domeinen: groepeer op domein (voorkomt splits zoals "LinkedIn" vs "LinkedIn Job Alerts")
|
||
if domain in MULTITENANT_DOMAINS:
|
||
key = normalize_name(name) or domain
|
||
else:
|
||
key = domain
|
||
|
||
e = by_name[key]
|
||
e["names"].append(name)
|
||
e["domains"].append(domain)
|
||
e["count"] += 1
|
||
e["folders"].append(folder)
|
||
ts = to_utc_ts(date)
|
||
if ts:
|
||
e["timestamps"].append(ts)
|
||
if len(e["subjects"]) < 3:
|
||
e["subjects"].append(subject)
|
||
if list_us and list_us not in e["unsubscribe"]:
|
||
e["unsubscribe"].append(list_us)
|
||
|
||
print(f" {total}/{total} klaar.\n")
|
||
|
||
# Omzetten naar gesorteerde lijst met mooie namen
|
||
result = []
|
||
for key, e in by_name.items():
|
||
best_name = Counter(e["names"]).most_common(1)[0][0]
|
||
best_domain = Counter(e["domains"]).most_common(1)[0][0]
|
||
best_folder = Counter(e["folders"]).most_common(1)[0][0]
|
||
ts_sorted = sorted(e["timestamps"])
|
||
|
||
result.append({
|
||
"key": key,
|
||
"name": best_name,
|
||
"domain": best_domain,
|
||
"folder": best_folder,
|
||
"count": e["count"],
|
||
"oldest": ts_sorted[0] if ts_sorted else None,
|
||
"newest": ts_sorted[-1] if ts_sorted else None,
|
||
"subjects": e["subjects"],
|
||
"unsubscribe": e["unsubscribe"],
|
||
"domains": list(set(e["domains"])),
|
||
})
|
||
|
||
result.sort(key=lambda x: -x["count"])
|
||
result = [e for e in result if e["count"] >= MIN_COUNT]
|
||
|
||
CACHE_FILE.write_text(
|
||
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
|
||
)
|
||
print(f"{len(result)} unieke mailinglijsten opgeslagen in cache.\n")
|
||
return result
|
||
|
||
|
||
def load_cache() -> list[dict]:
|
||
return json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
||
|
||
|
||
# ── beslissingen ─────────────────────────────────────────────────────────────
|
||
|
||
def load_decisions() -> dict:
|
||
if not DECISIONS_FILE.exists():
|
||
return {}
|
||
try:
|
||
data = json.loads(DECISIONS_FILE.read_text(encoding="utf-8"))
|
||
return {k: (v["keuze"] if isinstance(v, dict) else v) for k, v in data.items()}
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def save_decisions(decisions: dict, lists: list[dict]):
|
||
by_key = {e["key"]: e for e in lists}
|
||
output = {}
|
||
for key, keuze in decisions.items():
|
||
e = by_key.get(key, {})
|
||
output[key] = {
|
||
"keuze": keuze,
|
||
"naam": e.get("name", key),
|
||
"count": e.get("count", 0),
|
||
"domains": e.get("domains", []),
|
||
"unsubscribe": e.get("unsubscribe", []),
|
||
}
|
||
DECISIONS_FILE.write_text(
|
||
json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8"
|
||
)
|
||
|
||
|
||
# ── interactieve review ───────────────────────────────────────────────────────
|
||
|
||
KEYS = {"h", "a", "s", "q"}
|
||
LABELS = {
|
||
"h": ("houden", "\033[32m"),
|
||
"a": ("afmelden", "\033[31m"),
|
||
"s": ("overslaan", "\033[33m"),
|
||
}
|
||
|
||
|
||
def run_review(lists: list[dict], decisions: dict):
|
||
total = len(lists)
|
||
remaining = [e for e in lists if e["key"] not in decisions]
|
||
|
||
print(f"{'─'*64}")
|
||
print(f" Totaal: {total} | Al beoordeeld: {len(decisions)} | Te gaan: {len(remaining)}")
|
||
print(f" \033[32mh\033[0m houden \033[31ma\033[0m afmelden \033[33ms\033[0m overslaan \033[90mq\033[0m stoppen")
|
||
print(f"{'─'*64}\n")
|
||
|
||
done = sum(1 for e in lists if e["key"] in decisions)
|
||
|
||
for entry in remaining:
|
||
unsub = "✓ beschikbaar" if entry["unsubscribe"] else "✗ handmatig opzoeken"
|
||
domains_str = ", ".join(entry["domains"][:3])
|
||
if len(entry["domains"]) > 3:
|
||
domains_str += f" (+{len(entry['domains'])-3})"
|
||
|
||
print(f"\033[1m[{done+1}/{total}] {entry['name']}\033[0m")
|
||
print(f" \033[90mmails :\033[0m {entry['count']} "
|
||
f"({fmt_date(entry['oldest'])} – {fmt_date(entry['newest'])})")
|
||
print(f" \033[90mdomein :\033[0m {domains_str}")
|
||
print(f" \033[90mmap :\033[0m {entry['folder']}")
|
||
print(f" \033[90mafmeld :\033[0m {unsub}")
|
||
if entry["subjects"]:
|
||
print(f" \033[90mvoorbeeld:\033[0m {entry['subjects'][0][:68]}")
|
||
print()
|
||
print(" > ", end="", flush=True)
|
||
|
||
ch = ""
|
||
while ch not in KEYS:
|
||
ch = getch().lower()
|
||
|
||
if ch == "q":
|
||
print("\n\nGestopt. Voortgang opgeslagen.\n")
|
||
save_decisions(decisions, lists)
|
||
print_summary(decisions)
|
||
return
|
||
|
||
label, color = LABELS[ch]
|
||
print(f"{color}{label}\033[0m\n")
|
||
decisions[entry["key"]] = ch
|
||
done += 1
|
||
save_decisions(decisions, lists)
|
||
|
||
print(f"{'─'*64}")
|
||
print(f"Klaar! Alle {total} lijsten beoordeeld.")
|
||
save_decisions(decisions, lists)
|
||
print_summary(decisions)
|
||
|
||
|
||
def print_summary(decisions: dict):
|
||
a = sum(1 for c in decisions.values() if c == "a")
|
||
h = sum(1 for c in decisions.values() if c == "h")
|
||
s = sum(1 for c in decisions.values() if c == "s")
|
||
print(f"\n Afmelden : {a}")
|
||
print(f" Houden : {h}")
|
||
print(f" Overgeslagen: {s}")
|
||
print(f"\nOpgeslagen in: {DECISIONS_FILE}")
|
||
if a:
|
||
print("Draai daarna: python3 unsubscribe.py")
|
||
|
||
|
||
# ── main ──────────────────────────────────────────────────────────────────────
|
||
|
||
if __name__ == "__main__":
|
||
if CACHE_FILE.exists():
|
||
print(f"Cache gevonden, laden...\n")
|
||
lists = load_cache()
|
||
else:
|
||
lists = build_cache()
|
||
|
||
decisions = load_decisions()
|
||
if decisions:
|
||
active_keys = {e["key"] for e in lists}
|
||
active = {k: v for k, v in decisions.items() if k in active_keys}
|
||
a = sum(1 for c in active.values() if c == "a")
|
||
h = sum(1 for c in active.values() if c == "h")
|
||
s = sum(1 for c in active.values() if c == "s")
|
||
stale = len(decisions) - len(active)
|
||
msg = f"Vorige sessie: {a} afmelden, {h} houden, {s} overgeslagen"
|
||
if stale:
|
||
msg += f" ({stale} verouderde beslissingen genegeerd)"
|
||
print(msg + ".\n")
|
||
|
||
run_review(lists, decisions)
|