Initial status for developed files on dagda
This commit is contained in:
mailcat
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Interactief script om mailinglijsten te beoordelen.
|
||||
h = houden | a = afmelden | s = overslaan | q = stoppen
|
||||
|
||||
Eerste run: scant mailbox en slaat cache op in mailing_lists_cache.json.
|
||||
Volgende runs: laadt direct uit cache.
|
||||
Beslissingen worden opgeslagen in decisions.json (herstart-safe).
|
||||
"""
|
||||
|
||||
import email
|
||||
import email.header
|
||||
import email.utils
|
||||
import re
|
||||
import sys
|
||||
import tty
|
||||
import termios
|
||||
import json
|
||||
from pathlib import Path
|
||||
from collections import defaultdict, Counter
|
||||
from datetime import timezone
|
||||
|
||||
MAILBOX_DIR = Path(__file__).parent / "mailbox"
|
||||
CACHE_FILE = Path(__file__).parent / "mailing_lists_cache.json"
|
||||
DECISIONS_FILE = Path(__file__).parent / "decisions.json"
|
||||
|
||||
# Domeinen die door meerdere afzenders worden gedeeld → groeperen op naam, niet domein
|
||||
MULTITENANT_DOMAINS = {
|
||||
"substack.com", "mcsv.net", "list-manage.com",
|
||||
"sparkpostmail.com", "eu.sparkpostmail.com",
|
||||
"convertkit-mail.com", "convertkit-mail2.com", "convertkit-mail4.com",
|
||||
"sendgrid.net", "mailgun.org", "dripemail.com", "dripemail3.com",
|
||||
"mailchimpapp.net", "rsgsv.net",
|
||||
}
|
||||
|
||||
MIN_COUNT = 3 # lijsten met minder dan dit aantal mails worden overgeslagen
|
||||
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
def decode_hdr(value) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
try:
|
||||
parts = email.header.decode_header(value)
|
||||
out = []
|
||||
for raw, cs in parts:
|
||||
if isinstance(raw, bytes):
|
||||
out.append(raw.decode(cs or "utf-8", errors="replace"))
|
||||
else:
|
||||
out.append(str(raw))
|
||||
return " ".join(out).strip()
|
||||
except Exception:
|
||||
return str(value or "")
|
||||
|
||||
|
||||
def parse_dt(s):
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return email.utils.parsedate_to_datetime(s)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def to_utc_ts(d) -> float | None:
|
||||
if d is None:
|
||||
return None
|
||||
if d.tzinfo is None:
|
||||
d = d.replace(tzinfo=timezone.utc)
|
||||
return d.timestamp()
|
||||
|
||||
|
||||
_EMAIL_SUBDOMAINS = re.compile(
|
||||
r"^(email|mail|e|m|news|newsletter|noreply|no-reply|lists?|"
|
||||
r"mailing|bounce|notifications?|updates?|info|reply|send|smtp|"
|
||||
r"mg|sg|em|learn|go|click|edm|cdn|app|get)\.",
|
||||
re.I,
|
||||
)
|
||||
|
||||
def sender_domain(from_addr: str) -> str:
|
||||
m = re.search(r"@([\w.\-]+)", from_addr)
|
||||
if not m:
|
||||
return "onbekend"
|
||||
domain = m.group(1).lower()
|
||||
# Strip bekende e-mail-subdomeinen zodat bijv. email.coursera.org → coursera.org
|
||||
parts = domain.split(".")
|
||||
while len(parts) > 2 and _EMAIL_SUBDOMAINS.match(parts[0] + "."):
|
||||
parts = parts[1:]
|
||||
return ".".join(parts)
|
||||
|
||||
|
||||
def display_name(from_addr: str) -> str:
|
||||
"""Haal leesbare naam op; val terug op domein."""
|
||||
m = re.match(r'^"?([^"<@\n]{3,})"?\s*<', from_addr)
|
||||
if m:
|
||||
name = m.group(1).strip().strip('"').strip()
|
||||
if len(name) > 2:
|
||||
return name
|
||||
return sender_domain(from_addr)
|
||||
|
||||
|
||||
def normalize_name(name: str) -> str:
|
||||
"""Normaliseer naam voor groepering (lowercase, alleen woorden)."""
|
||||
return " ".join(re.findall(r"[a-z0-9]+", name.lower()))
|
||||
|
||||
|
||||
def fmt_date(ts: float | None) -> str:
|
||||
if ts is None:
|
||||
return "?"
|
||||
from datetime import datetime
|
||||
return datetime.utcfromtimestamp(ts).strftime("%Y-%m-%d")
|
||||
|
||||
|
||||
def getch() -> str:
|
||||
fd = sys.stdin.fileno()
|
||||
old = termios.tcgetattr(fd)
|
||||
try:
|
||||
tty.setraw(fd)
|
||||
return sys.stdin.read(1)
|
||||
finally:
|
||||
termios.tcsetattr(fd, termios.TCSADRAIN, old)
|
||||
|
||||
|
||||
# ── cache bouwen ─────────────────────────────────────────────────────────────
|
||||
|
||||
def build_cache() -> list[dict]:
|
||||
"""
|
||||
Scan mailbox headers en groepeer op weergavenaam.
|
||||
Elke entry in de cache is één logische mailinglijst.
|
||||
"""
|
||||
eml_files = list(MAILBOX_DIR.rglob("*.eml"))
|
||||
total = len(eml_files)
|
||||
print(f"Scannen van {total} e-mails (alleen headers)...", flush=True)
|
||||
|
||||
# Per weergavenaam (genormaliseerd) verzamelen we alle info
|
||||
by_name: dict[str, dict] = defaultdict(lambda: {
|
||||
"names": [], # alle aangetroffen weergavenamen
|
||||
"domains": [], # alle aangetroffen domeinen
|
||||
"unsubscribe": [], # List-Unsubscribe waarden
|
||||
"count": 0,
|
||||
"timestamps": [], # als float (UTC)
|
||||
"subjects": [],
|
||||
"folders": [],
|
||||
})
|
||||
|
||||
for i, path in enumerate(eml_files, 1):
|
||||
if i % 2000 == 0:
|
||||
print(f" {i}/{total}...", flush=True)
|
||||
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
chunk = f.read(8192)
|
||||
msg = email.message_from_bytes(chunk)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
from_raw = decode_hdr(msg.get("From", ""))
|
||||
subject = decode_hdr(msg.get("Subject", ""))
|
||||
date = parse_dt(msg.get("Date"))
|
||||
list_id = (msg.get("List-ID") or "").strip()
|
||||
list_us = (msg.get("List-Unsubscribe") or "").strip()
|
||||
prec = (msg.get("Precedence") or "").lower()
|
||||
folder = path.parent.name
|
||||
|
||||
is_ml = bool(
|
||||
list_id or list_us
|
||||
or prec in ("bulk", "list")
|
||||
or "mailinglist" in folder.lower()
|
||||
)
|
||||
if not is_ml:
|
||||
continue
|
||||
|
||||
name = display_name(from_raw)
|
||||
domain = sender_domain(from_raw)
|
||||
# Multi-tenant domeinen: groepeer op genormaliseerde naam
|
||||
# Gewone domeinen: groepeer op domein (voorkomt splits zoals "LinkedIn" vs "LinkedIn Job Alerts")
|
||||
if domain in MULTITENANT_DOMAINS:
|
||||
key = normalize_name(name) or domain
|
||||
else:
|
||||
key = domain
|
||||
|
||||
e = by_name[key]
|
||||
e["names"].append(name)
|
||||
e["domains"].append(domain)
|
||||
e["count"] += 1
|
||||
e["folders"].append(folder)
|
||||
ts = to_utc_ts(date)
|
||||
if ts:
|
||||
e["timestamps"].append(ts)
|
||||
if len(e["subjects"]) < 3:
|
||||
e["subjects"].append(subject)
|
||||
if list_us and list_us not in e["unsubscribe"]:
|
||||
e["unsubscribe"].append(list_us)
|
||||
|
||||
print(f" {total}/{total} klaar.\n")
|
||||
|
||||
# Omzetten naar gesorteerde lijst met mooie namen
|
||||
result = []
|
||||
for key, e in by_name.items():
|
||||
best_name = Counter(e["names"]).most_common(1)[0][0]
|
||||
best_domain = Counter(e["domains"]).most_common(1)[0][0]
|
||||
best_folder = Counter(e["folders"]).most_common(1)[0][0]
|
||||
ts_sorted = sorted(e["timestamps"])
|
||||
|
||||
result.append({
|
||||
"key": key,
|
||||
"name": best_name,
|
||||
"domain": best_domain,
|
||||
"folder": best_folder,
|
||||
"count": e["count"],
|
||||
"oldest": ts_sorted[0] if ts_sorted else None,
|
||||
"newest": ts_sorted[-1] if ts_sorted else None,
|
||||
"subjects": e["subjects"],
|
||||
"unsubscribe": e["unsubscribe"],
|
||||
"domains": list(set(e["domains"])),
|
||||
})
|
||||
|
||||
result.sort(key=lambda x: -x["count"])
|
||||
result = [e for e in result if e["count"] >= MIN_COUNT]
|
||||
|
||||
CACHE_FILE.write_text(
|
||||
json.dumps(result, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
print(f"{len(result)} unieke mailinglijsten opgeslagen in cache.\n")
|
||||
return result
|
||||
|
||||
|
||||
def load_cache() -> list[dict]:
|
||||
return json.loads(CACHE_FILE.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
# ── beslissingen ─────────────────────────────────────────────────────────────
|
||||
|
||||
def load_decisions() -> dict:
|
||||
if not DECISIONS_FILE.exists():
|
||||
return {}
|
||||
try:
|
||||
data = json.loads(DECISIONS_FILE.read_text(encoding="utf-8"))
|
||||
return {k: (v["keuze"] if isinstance(v, dict) else v) for k, v in data.items()}
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def save_decisions(decisions: dict, lists: list[dict]):
|
||||
by_key = {e["key"]: e for e in lists}
|
||||
output = {}
|
||||
for key, keuze in decisions.items():
|
||||
e = by_key.get(key, {})
|
||||
output[key] = {
|
||||
"keuze": keuze,
|
||||
"naam": e.get("name", key),
|
||||
"count": e.get("count", 0),
|
||||
"domains": e.get("domains", []),
|
||||
"unsubscribe": e.get("unsubscribe", []),
|
||||
}
|
||||
DECISIONS_FILE.write_text(
|
||||
json.dumps(output, indent=2, ensure_ascii=False), encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
# ── interactieve review ───────────────────────────────────────────────────────
|
||||
|
||||
KEYS = {"h", "a", "s", "q"}
|
||||
LABELS = {
|
||||
"h": ("houden", "\033[32m"),
|
||||
"a": ("afmelden", "\033[31m"),
|
||||
"s": ("overslaan", "\033[33m"),
|
||||
}
|
||||
|
||||
|
||||
def run_review(lists: list[dict], decisions: dict):
|
||||
total = len(lists)
|
||||
remaining = [e for e in lists if e["key"] not in decisions]
|
||||
|
||||
print(f"{'─'*64}")
|
||||
print(f" Totaal: {total} | Al beoordeeld: {len(decisions)} | Te gaan: {len(remaining)}")
|
||||
print(f" \033[32mh\033[0m houden \033[31ma\033[0m afmelden \033[33ms\033[0m overslaan \033[90mq\033[0m stoppen")
|
||||
print(f"{'─'*64}\n")
|
||||
|
||||
done = sum(1 for e in lists if e["key"] in decisions)
|
||||
|
||||
for entry in remaining:
|
||||
unsub = "✓ beschikbaar" if entry["unsubscribe"] else "✗ handmatig opzoeken"
|
||||
domains_str = ", ".join(entry["domains"][:3])
|
||||
if len(entry["domains"]) > 3:
|
||||
domains_str += f" (+{len(entry['domains'])-3})"
|
||||
|
||||
print(f"\033[1m[{done+1}/{total}] {entry['name']}\033[0m")
|
||||
print(f" \033[90mmails :\033[0m {entry['count']} "
|
||||
f"({fmt_date(entry['oldest'])} – {fmt_date(entry['newest'])})")
|
||||
print(f" \033[90mdomein :\033[0m {domains_str}")
|
||||
print(f" \033[90mmap :\033[0m {entry['folder']}")
|
||||
print(f" \033[90mafmeld :\033[0m {unsub}")
|
||||
if entry["subjects"]:
|
||||
print(f" \033[90mvoorbeeld:\033[0m {entry['subjects'][0][:68]}")
|
||||
print()
|
||||
print(" > ", end="", flush=True)
|
||||
|
||||
ch = ""
|
||||
while ch not in KEYS:
|
||||
ch = getch().lower()
|
||||
|
||||
if ch == "q":
|
||||
print("\n\nGestopt. Voortgang opgeslagen.\n")
|
||||
save_decisions(decisions, lists)
|
||||
print_summary(decisions)
|
||||
return
|
||||
|
||||
label, color = LABELS[ch]
|
||||
print(f"{color}{label}\033[0m\n")
|
||||
decisions[entry["key"]] = ch
|
||||
done += 1
|
||||
save_decisions(decisions, lists)
|
||||
|
||||
print(f"{'─'*64}")
|
||||
print(f"Klaar! Alle {total} lijsten beoordeeld.")
|
||||
save_decisions(decisions, lists)
|
||||
print_summary(decisions)
|
||||
|
||||
|
||||
def print_summary(decisions: dict):
|
||||
a = sum(1 for c in decisions.values() if c == "a")
|
||||
h = sum(1 for c in decisions.values() if c == "h")
|
||||
s = sum(1 for c in decisions.values() if c == "s")
|
||||
print(f"\n Afmelden : {a}")
|
||||
print(f" Houden : {h}")
|
||||
print(f" Overgeslagen: {s}")
|
||||
print(f"\nOpgeslagen in: {DECISIONS_FILE}")
|
||||
if a:
|
||||
print("Draai daarna: python3 unsubscribe.py")
|
||||
|
||||
|
||||
# ── main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
if __name__ == "__main__":
|
||||
if CACHE_FILE.exists():
|
||||
print(f"Cache gevonden, laden...\n")
|
||||
lists = load_cache()
|
||||
else:
|
||||
lists = build_cache()
|
||||
|
||||
decisions = load_decisions()
|
||||
if decisions:
|
||||
active_keys = {e["key"] for e in lists}
|
||||
active = {k: v for k, v in decisions.items() if k in active_keys}
|
||||
a = sum(1 for c in active.values() if c == "a")
|
||||
h = sum(1 for c in active.values() if c == "h")
|
||||
s = sum(1 for c in active.values() if c == "s")
|
||||
stale = len(decisions) - len(active)
|
||||
msg = f"Vorige sessie: {a} afmelden, {h} houden, {s} overgeslagen"
|
||||
if stale:
|
||||
msg += f" ({stale} verouderde beslissingen genegeerd)"
|
||||
print(msg + ".\n")
|
||||
|
||||
run_review(lists, decisions)
|
||||
Reference in New Issue
Block a user