commit 4780c18d7028685e6bd96423d3cb4c74cfa7d307 Author: Hans Wienen Date: Sat Jul 4 11:18:05 2026 +0200 Initial commit Claude has done a lot of work, but there are still some errors which cause the mails to be distributed to wrong mail boxes. I am now going to let codex and claude compete. They will have their own branches and merge to main as soon as we are somewhere where we can switch AI. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..9b67a16 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +__pycache__/ +*.py[cod] +.DS_Store +config.json +**/*.eml diff --git a/analyse_mailbox.py b/analyse_mailbox.py new file mode 100644 index 0000000..31dab3e --- /dev/null +++ b/analyse_mailbox.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +""" +Fase 1: Analyseer lokale mailbox-kopie en genereer overzichten. +Output: reports/mailinglijsten.md, facturen_mailbox.md, facturen_links.md, prullenbak.md +""" + +import email +import email.header +import email.utils +import re +import sys +from pathlib import Path +from collections import defaultdict +from datetime import datetime, timezone + +MAILBOX_DIR = Path(__file__).parent / "mailbox" +REPORTS_DIR = Path(__file__).parent / "reports" + +INVOICE_SUBJECT_KW = [ + "factuur", "invoice", "rekening", "nota", "kwitantie", "receipt", + "betaalverzoek", "betalingsverzoek", "betaling ontvangen", + "payment received", "order confirmation", "orderbevestiging", + "your order", "je bestelling", "proforma", +] +INVOICE_ATTACH_KW = [ + "factuur", "invoice", "rekening", "nota", "receipt", "bill", +] +INVOICE_LINK_KW = [ + "bekijk je factuur", "download je factuur", "download your invoice", + "view your invoice", "your invoice is ready", "je factuur staat klaar", + "factuur is beschikbaar", "invoice is available", "factuur online", + "invoice online", "mijn facturen", "my invoices", "portaal", "portal", + "mijn account", "my account", "log in to view", "inloggen om te bekijken", +] + +TRASH_SENDER_RE = re.compile( + r"(noreply|no-reply|donotreply|do-not-reply|notifications?|newsletter|" + r"nieuwsbrief|alerts?|updates?|mailer-daemon|postmaster|bounce|" + r"auto-?reply|automatisch)@", + re.I, +) +TRASH_SUBJECT_RE = re.compile( + r"(out of office|automatisch antwoord|afwezig|buiten kantoor|" + r"delivery (status notification|failed|failure)|undeliverable|" + r"mail delivery|mailer-daemon|bounced|returned mail)", + re.I, +) + + +# ── helpers ───────────────────────────────────────────────────────────────── + +def decode_hdr(value: str | None) -> str: + if not value: + return "" + try: + parts = email.header.decode_header(value) + out = [] + for raw, cs in parts: + if isinstance(raw, bytes): + out.append(raw.decode(cs or "utf-8", errors="replace")) + else: + out.append(str(raw)) + return " ".join(out).strip() + except Exception: + return str(value or "") + + +def parse_dt(date_str: str | None) -> datetime | None: + if not date_str: + return None + try: + return email.utils.parsedate_to_datetime(date_str) + except Exception: + return None + + +def sender_domain(from_addr: str) -> str: + m = re.search(r"@([\w.\-]+)", from_addr) + return m.group(1).lower() if m else "onbekend" + + +def get_body(msg) -> str: + parts = [] + try: + for part in (msg.walk() if msg.is_multipart() else [msg]): + if part.get_content_type() in ("text/plain", "text/html"): + cs = part.get_content_charset() or "utf-8" + try: + parts.append(part.get_payload(decode=True).decode(cs, errors="replace")) + except Exception: + pass + except Exception: + pass + return "\n".join(parts)[:4000] + + +def get_attachments(msg) -> list[dict]: + result = [] + try: + for part in msg.walk(): + disp = (part.get("Content-Disposition") or "").lower() + ct = part.get_content_type() or "" + fn = decode_hdr(part.get_filename() or "") + if "attachment" in disp or (fn and ct == "application/pdf"): + result.append({"filename": fn, "type": ct}) + except Exception: + pass + return result + + +def extract_urls(text: str) -> list[str]: + return list(dict.fromkeys(re.findall(r"https?://[^\s<>\"')]+", text))) + + +def url_domain(url: str) -> str: + m = re.match(r"https?://([^/]+)", url) + return m.group(1).lower() if m else "" + + +# ── main analysis ──────────────────────────────────────────────────────────── + +def analyze(): + REPORTS_DIR.mkdir(exist_ok=True) + + eml_files = list(MAILBOX_DIR.rglob("*.eml")) + total = len(eml_files) + print(f"Analyseren van {total} e-mails...\n") + + # mailinglijsten: key → {count, list_id, unsubscribe, senders, dates, folder} + ml: dict[str, dict] = defaultdict(lambda: { + "list_id": "", "unsubscribe": [], "senders": set(), + "count": 0, "dates": [], "folder": "", + }) + + invoices_in_mb: list[dict] = [] + invoice_links: list[dict] = [] + trash: list[dict] = [] + + now_utc = datetime.now(timezone.utc) + + for i, path in enumerate(eml_files, 1): + if i % 1000 == 0: + print(f" {i}/{total}...", flush=True) + + try: + with open(path, "rb") as f: + msg = email.message_from_bytes(f.read()) + except Exception: + continue + + folder = path.parent.name + from_raw = decode_hdr(msg.get("From", "")) + subject = decode_hdr(msg.get("Subject", "")) + date = parse_dt(msg.get("Date")) + list_id = (msg.get("List-ID") or "").strip() + list_us = (msg.get("List-Unsubscribe") or "").strip() + prec = (msg.get("Precedence") or "").lower() + from_lo = from_raw.lower() + subj_lo = subject.lower() + domain = sender_domain(from_raw) + + attachments = get_attachments(msg) + body = get_body(msg) + body_lo = body.lower() + + # ── mailinglijsten ──────────────────────────────────────────────── + ml_key = None + if list_id: + ml_key = re.sub(r"[<>]", "", list_id).strip() + elif "mailinglists" in folder.lower() or "mailinglist" in folder.lower(): + ml_key = folder # use the subfolder name as key + + if ml_key: + entry = ml[ml_key] + entry["list_id"] = list_id or ml_key + entry["count"] += 1 + entry["folder"] = folder + entry["senders"].add(from_raw) + if date: + entry["dates"].append(date) + if list_us and list_us not in entry["unsubscribe"]: + entry["unsubscribe"].append(list_us) + + # ── facturen in mailbox ─────────────────────────────────────────── + invoice_reasons: list[str] = [] + + for att in attachments: + fn_lo = att["filename"].lower() + if att["type"] == "application/pdf" and any(kw in fn_lo for kw in INVOICE_ATTACH_KW): + invoice_reasons.append(f"PDF: {att['filename']}") + + if any(kw in subj_lo for kw in INVOICE_SUBJECT_KW): + invoice_reasons.append("factuurtermen in onderwerp") + + # always include what's already in the Facturen folders + if "facturen" in folder.lower(): + invoice_reasons.append(f"staat in map {folder}") + + if invoice_reasons: + invoices_in_mb.append({ + "from": from_raw, + "domain": domain, + "subject": subject, + "date": date, + "folder": folder, + "reasons": invoice_reasons, + "attachments": [a["filename"] for a in attachments if a["type"] == "application/pdf"], + }) + + # ── factuurlinks ────────────────────────────────────────────────── + has_link_kw = any(kw in body_lo for kw in INVOICE_LINK_KW) + has_no_pdf = not any(a["type"] == "application/pdf" for a in attachments) + + if has_link_kw and has_no_pdf: + urls = extract_urls(body) + if urls: + invoice_links.append({ + "from": from_raw, + "domain": domain, + "subject": subject, + "date": date, + "folder": folder, + "urls": urls[:5], + "url_domains": list(dict.fromkeys(url_domain(u) for u in urls[:5])), + }) + + # ── prullenbak ──────────────────────────────────────────────────── + trash_reasons: list[str] = [] + + if TRASH_SENDER_RE.search(from_lo): + trash_reasons.append("automatisch afzenderadres") + + if TRASH_SUBJECT_RE.search(subj_lo): + trash_reasons.append("automatisch/fout-onderwerp") + + if prec in ("bulk", "junk"): + trash_reasons.append(f"Precedence: {prec}") + + # Old bulk mail sitting in INBOX + if folder == "INBOX" and prec in ("bulk", "list", "junk") and date: + age = (now_utc - date.astimezone(timezone.utc)).days + if age > 180: + trash_reasons.append(f"bulk in INBOX, {age} dagen oud") + + if trash_reasons: + trash.append({ + "from": from_raw, + "domain": domain, + "subject": subject, + "date": date, + "folder": folder, + "reasons": trash_reasons, + "path": str(path), + }) + + print(f" {total}/{total} klaar.\n") + + write_mailinglist_report(ml) + write_invoice_report(invoices_in_mb) + write_invoice_links_report(invoice_links) + write_trash_report(trash) + + print(f"\nRapporten opgeslagen in: {REPORTS_DIR.resolve()}") + + +# ── report writers ─────────────────────────────────────────────────────────── + +def fmt_date(d: datetime | None) -> str: + return d.strftime("%Y-%m-%d") if d else "?" + + +def write_mailinglist_report(ml: dict): + lines = [ + "# Mailinglijsten\n", + f"Totaal gevonden: **{len(ml)}** lijsten.\n", + "_Advies: > 30 mails = hoog volume, overweeg afmelden._\n", + ] + + for key, entry in sorted(ml.items(), key=lambda x: -x[1]["count"]): + dates = sorted(entry["dates"]) + oldest = fmt_date(dates[0]) if dates else "?" + newest = fmt_date(dates[-1]) if dates else "?" + senders = sorted(entry["senders"])[:2] + unsub = entry["unsubscribe"] + + if entry["count"] > 30: + advice = "⚠️ Hoog volume – overweeg afmelden" + elif entry["count"] > 10: + advice = "Gemiddeld volume – beoordeel zelf" + else: + advice = "Laag volume – waarschijnlijk houden" + + can_unsub = "Ja" if unsub else "Nee (handmatig opzoeken)" + + lines += [ + f"## {key}", + f"- **Mails:** {entry['count']} | **Periode:** {oldest} – {newest}", + f"- **Map:** {entry['folder']}", + f"- **Afzender:** {', '.join(senders)}", + f"- **Afmelden mogelijk:** {can_unsub}", + ] + if unsub: + lines.append(f"- **Afmeldlink:** `{unsub[0][:120]}`") + lines += [f"- **Advies:** {advice}", ""] + + path = REPORTS_DIR / "mailinglijsten.md" + path.write_text("\n".join(lines), encoding="utf-8") + print(f"✓ mailinglijsten.md ({len(ml)} lijsten)") + + +def write_invoice_report(invoices: list[dict]): + lines = [ + "# Facturen in mailbox\n", + f"Totaal: **{len(invoices)}** factuurmails gevonden.\n", + ] + + by_domain: dict[str, list] = defaultdict(list) + for inv in invoices: + by_domain[inv["domain"]].append(inv) + + for domain, items in sorted(by_domain.items(), key=lambda x: -len(x[1])): + items_sorted = sorted(items, key=lambda x: x["date"] or datetime.min.replace(tzinfo=timezone.utc)) + lines.append(f"## {domain} ({len(items)} mails)") + for item in items_sorted[:25]: + atts = f" → {', '.join(item['attachments'])}" if item["attachments"] else "" + lines.append(f"- {fmt_date(item['date'])} | {item['subject'][:80]}{atts}") + if len(items) > 25: + lines.append(f" _...en {len(items) - 25} meer_") + lines.append("") + + (REPORTS_DIR / "facturen_mailbox.md").write_text("\n".join(lines), encoding="utf-8") + print(f"✓ facturen_mailbox.md ({len(invoices)} facturen)") + + +def write_invoice_links_report(invoice_links: list[dict]): + lines = [ + "# Factuurlinks – verwijzingen naar portalen\n", + f"Totaal: **{len(invoice_links)}** mails met factuurverwijzingen (zonder PDF-bijlage).\n", + "_Dit zijn mails die zeggen 'bekijk je factuur online' – de factuur staat op een website._\n", + ] + + by_url_domain: dict[str, list] = defaultdict(list) + for item in invoice_links: + for ud in (item["url_domains"] or ["onbekend"]): + by_url_domain[ud].append(item) + + for ud, items in sorted(by_url_domain.items(), key=lambda x: -len(x[1])): + unique_senders = sorted({i["domain"] for i in items}) + lines.append(f"## {ud} ({len(items)} mails)") + lines.append(f" _Afzenders: {', '.join(unique_senders[:5])}_") + for item in sorted(items, key=lambda x: x["date"] or datetime.min.replace(tzinfo=timezone.utc))[:10]: + lines.append(f"- {fmt_date(item['date'])} | {item['subject'][:80]}") + for url in item["urls"][:2]: + lines.append(f" `{url[:120]}`") + if len(items) > 10: + lines.append(f" _...en {len(items) - 10} meer_") + lines.append("") + + (REPORTS_DIR / "facturen_links.md").write_text("\n".join(lines), encoding="utf-8") + print(f"✓ facturen_links.md ({len(invoice_links)} mails)") + + +def write_trash_report(trash: list[dict]): + lines = [ + "# Prullenbak-kandidaten\n", + f"Totaal: **{len(trash)}** kandidaten gevonden.\n", + ] + + by_reason: dict[str, list] = defaultdict(list) + for item in trash: + by_reason[item["reasons"][0]].append(item) + + for reason, items in sorted(by_reason.items(), key=lambda x: -len(x[1])): + lines.append(f"## {reason} ({len(items)} mails)") + + by_domain: dict[str, list] = defaultdict(list) + for item in items: + by_domain[item["domain"]].append(item) + + def _sort_key(x): + d = x["date"] + if d is None: + return datetime.min.replace(tzinfo=timezone.utc) + return d if d.tzinfo else d.replace(tzinfo=timezone.utc) + + for domain, ditems in sorted(by_domain.items(), key=lambda x: -len(x[1])): + lines.append(f"### {domain} ({len(ditems)})") + for item in sorted(ditems, key=_sort_key)[:10]: + lines.append(f"- {fmt_date(item['date'])} | {item['folder']} | {item['subject'][:70]}") + if len(ditems) > 10: + lines.append(f" _...en {len(ditems) - 10} meer_") + lines.append("") + + (REPORTS_DIR / "prullenbak.md").write_text("\n".join(lines), encoding="utf-8") + print(f"✓ prullenbak.md ({len(trash)} kandidaten)") + + +if __name__ == "__main__": + analyze() diff --git a/backup_log.json b/backup_log.json new file mode 100644 index 0000000..515d86f --- /dev/null +++ b/backup_log.json @@ -0,0 +1,40 @@ +{ + "voltooide_mappen": [ + "INBOX.Archief", + "INBOX.Archief.2021.inkomend", + "INBOX.Archief.2022.inkomend", + "INBOX.Facturen - te verwerken", + "INBOX.Mailinglists.Storytelling with Data", + "INBOX.Archief.2022.verzonden", + "INBOX.Mailinglists.Ben Tiggelaar", + "INBOX.Technisch", + "INBOX.Mailinglists", + "INBOX.Kennis", + "INBOX.Archief.2022", + "INBOX.Mailinglists.Lendahand", + "INBOX.Kennis.Coursera", + "INBOX.Hulp", + "INBOX.Sent", + "INBOX.Facturen - verwerkt", + "INBOX.Mailinglists.BASB", + "INBOX.Mailinglists.Formspree", + "INBOX.Archief.2020.inkomend", + "INBOX.Kennis.Leeslijst", + "INBOX.Hulp.Gespreksverslagen", + "INBOX.Archief.2020.verzonden", + "INBOX.Mailinglists.MIT Technology Review", + "INBOX.Drafts", + "INBOX.Mailinglists.Brian Keating", + "INBOX.Mailinglists.Calmer Notes", + "INBOX.Technisch.dmarc", + "INBOX.Archief.2021", + "INBOX.Mailinglists.Eva Keiffenheim", + "INBOX.Archief.2021.verzonden", + "INBOX.Mailinglists.LYT - Nick Milo", + "INBOX.Archief.2020", + "INBOX.Notes", + "INBOX" + ], + "totaal_gekopieerd": 11188, + "laatste_run": "2026-07-04T00:08:25.493919" +} \ No newline at end of file diff --git a/config.py b/config.py new file mode 100644 index 0000000..c49b013 --- /dev/null +++ b/config.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +""" +Centrale configuratie voor alle mailcat-scripts. + +Setup: python3 config.py --setup +Gebruik in andere scripts: + import config + cfg = config.load() # leest --account uit sys.argv, anders actief account + mail.login(cfg.username, cfg.password) + +Wisselen van account zonder config te wijzigen: + python3 maak_mappen.py --account backup + python3 verplaats_bestaand.py --account backup --uitvoeren +""" + +import json +import sys +import getpass +from pathlib import Path +from dataclasses import dataclass + +CONFIG_FILE = Path(__file__).parent / "config.json" + + +@dataclass +class Account: + name: str + username: str + password: str + imap_host: str + imap_port: int + smtp_host: str + smtp_port: int + sieve_host: str + sieve_port: int + + +def load(account_name: str | None = None) -> Account: + """ + Laad het opgegeven account (of het actieve account uit config.json). + account_name kan ook via --account op de commandoregel worden opgegeven. + """ + # --account vlag heeft prioriteit boven argument + args = sys.argv[1:] + if "--account" in args: + idx = args.index("--account") + if idx + 1 < len(args): + account_name = args[idx + 1] + + if not CONFIG_FILE.exists(): + print("Geen config gevonden. Draai eerst: python3 config.py --setup") + sys.exit(1) + + raw = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + + name = account_name or raw.get("active", "") + if not name: + print("Geen actief account ingesteld. Gebruik --account of stel 'active' in config.json in.") + sys.exit(1) + + accounts = raw.get("accounts", {}) + if name not in accounts: + beschikbaar = ", ".join(accounts.keys()) + print(f"Account '{name}' niet gevonden. Beschikbaar: {beschikbaar}") + sys.exit(1) + + acc = accounts[name] + return Account( + name = name, + username = acc["username"], + password = acc["password"], + imap_host = raw.get("imap_host", "australius.nl"), + imap_port = raw.get("imap_port", 993), + smtp_host = raw.get("smtp_host", "australius.nl"), + smtp_port = raw.get("smtp_port", 587), + sieve_host = raw.get("sieve_host", "australius.nl"), + sieve_port = raw.get("sieve_port", 4190), + ) + + +def active_account_name() -> str: + """Geef de naam van het actief te gebruiken account (voor weergave).""" + args = sys.argv[1:] + if "--account" in args: + idx = args.index("--account") + if idx + 1 < len(args): + return args[idx + 1] + if CONFIG_FILE.exists(): + raw = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + return raw.get("active", "?") + return "?" + + +# ── setup ───────────────────────────────────────────────────────────────────── + +def setup(): + """Interactief: voeg een account toe of wijzig een bestaand account.""" + raw: dict = {} + if CONFIG_FILE.exists(): + raw = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + + raw.setdefault("imap_host", "australius.nl") + raw.setdefault("imap_port", 993) + raw.setdefault("smtp_host", "australius.nl") + raw.setdefault("smtp_port", 587) + raw.setdefault("sieve_host", "australius.nl") + raw.setdefault("sieve_port", 4190) + raw.setdefault("accounts", {}) + + print("─" * 50) + print("Mailcat account setup") + print("─" * 50) + + bestaande = list(raw["accounts"].keys()) + if bestaande: + print(f"Bestaande accounts: {', '.join(bestaande)}") + + naam = input("\nAccountnaam (bijv. hans of backup): ").strip() + if not naam: + print("Afgebroken.") + return + + huidig = raw["accounts"].get(naam, {}) + huidig_user = huidig.get("username", "") + + username = input(f"E-mailadres [{huidig_user}]: ").strip() or huidig_user + password = getpass.getpass("Wachtwoord: ") + + raw["accounts"][naam] = {"username": username, "password": password} + + # Vraag of dit het actieve account moet zijn + huidig_actief = raw.get("active", "") + instellen = input(f"\nDit als actief account instellen? (huidig: {huidig_actief or 'geen'}) [J/n]: ").strip().lower() + if instellen != "n": + raw["active"] = naam + + CONFIG_FILE.write_text(json.dumps(raw, indent=2, ensure_ascii=False), encoding="utf-8") + CONFIG_FILE.chmod(0o600) + print(f"\n✓ Account '{naam}' opgeslagen. Actief: {raw['active']}") + print(f" Config: {CONFIG_FILE}") + + if len(raw["accounts"]) > 1: + print(f"\nWisselen: voeg --account toe aan elk script.") + print(f" Bijv: python3 maak_mappen.py --account backup") + + +if __name__ == "__main__": + if "--setup" in sys.argv: + setup() + else: + # Toon huidige status + if not CONFIG_FILE.exists(): + print("Geen config. Draai: python3 config.py --setup") + else: + raw = json.loads(CONFIG_FILE.read_text(encoding="utf-8")) + actief = raw.get("active", "?") + accounts = list(raw.get("accounts", {}).keys()) + print(f"Actief account : {actief}") + print(f"Alle accounts : {', '.join(accounts)}") + print(f"IMAP : {raw.get('imap_host')}:{raw.get('imap_port')}") + print(f"\nSetup: python3 config.py --setup") + print(f"Wisselen in script: python3