Harden mail move audit logging

The prompts used to arrive at this change since the previous commit

Read restart_prompt.md first, then continue from there.

Any special observations that may be relevant for version management for this version. Be brief.

No mailbox-affecting scripts were run by Codex. A generated dry-run transcript appeared locally as verplaats_bestaand.log and is now ignored via *.log.

.gitignore

@@ -1,5 +1,6 @@
 __pycache__/
 *.py[cod]
+*.log
 .DS_Store
 config.json
 **/*.eml

kopieer_naar_backup.py

@@ -88,39 +88,69 @@ def kopieer_map(
     doel: imaplib.IMAP4_SSL,
     folder: str,
     log: dict,
-) -> tuple[int, int]:
+) -> tuple[int, int, int]:
     """
     Kopieer berichten van bron-folder naar doel-folder.
-    Geeft (gekopieerd, overgeslagen) terug.
+    Geeft (gekopieerd, overgeslagen, mislukt) terug.
     """
     # Selecteer bronmap
     status, data = bron.select(quote_mailbox(folder), readonly=True)
     if status != "OK":
         print(f"  Kan bronmap niet openen: {folder}")
-        return 0, 0
+        log.setdefault("fouten", []).append({
+            "folder": folder,
+            "actie": "select_bron",
+            "status": status,
+            "response": repr(data),
+        })
+        return 0, 0, 1
 
     totaal = int(data[0])
     if totaal == 0:
-        return 0, 0
+        return 0, 0, 0
 
     # Haal alle bron-UIDs op
     status, data = bron.search(None, "ALL")
     if status != "OK" or not data[0]:
-        return 0, 0
+        log.setdefault("fouten", []).append({
+            "folder": folder,
+            "actie": "search_bron",
+            "status": status,
+            "response": repr(data),
+        })
+        return 0, 0, 1
     ids = data[0].split()
 
     # Zorg dat doelmap bestaat
-    doel.create(quote_mailbox(folder))   # negeer fout als al bestaat
+    create_status, create_data = doel.create(quote_mailbox(folder))
+    if create_status not in {"OK", "NO"}:
+        log.setdefault("fouten", []).append({
+            "folder": folder,
+            "actie": "create_doel",
+            "status": create_status,
+            "response": repr(create_data),
+        })
+        return 0, 0, 1
+    select_status, select_data = doel.select(quote_mailbox(folder), readonly=True)
+    if select_status != "OK":
+        log.setdefault("fouten", []).append({
+            "folder": folder,
+            "actie": "select_doel_na_create",
+            "status": select_status,
+            "response": repr(select_data),
+        })
+        return 0, 0, 1
 
     # Welke Message-IDs staan al in de doelmap?
     bestaande = haal_bestaande_message_ids(doel, folder)
 
     gekopieerd = 0
     overgeslagen = 0
+    mislukt = 0
 
     for i, uid in enumerate(ids, 1):
         # Voortgang op één regel
-        print(f"\r  {folder}: {i}/{totaal}  ({gekopieerd} gekopieerd, {overgeslagen} skip)   ",
+        print(f"\r  {folder}: {i}/{totaal}  ({gekopieerd} gekopieerd, {overgeslagen} skip, {mislukt} fout)   ",
               end="", flush=True)
 
         # Haal volledige bericht op + vlaggen + datum
@@ -128,6 +158,14 @@ def kopieer_map(
             uid, "(FLAGS INTERNALDATE BODY.PEEK[])"
         )
         if status != "OK" or not data:
+            mislukt += 1
+            log.setdefault("fouten", []).append({
+                "folder": folder,
+                "uid": uid.decode(errors="replace"),
+                "actie": "fetch_bron",
+                "status": status,
+                "response": repr(data),
+            })
             continue
 
         # Parse fetch-response
@@ -144,6 +182,14 @@ def kopieer_map(
                 internaldate = parse_internaldate(item[0])
 
         if not raw_message:
+            mislukt += 1
+            log.setdefault("fouten", []).append({
+                "folder": folder,
+                "uid": uid.decode(errors="replace"),
+                "actie": "fetch_lege_body",
+                "status": status,
+                "response": repr(data),
+            })
             continue
 
         # Controleer Message-ID voor deduplicatie
@@ -159,7 +205,7 @@ def kopieer_map(
 
         # Schrijf naar doelmap
         flag_str = "(" + " ".join(flags) + ")" if flags else "()"
-        status, _ = doel.append(
+        status, append_data = doel.append(
             quote_mailbox(folder),
             flag_str,
             internaldate,
@@ -170,14 +216,22 @@ def kopieer_map(
             if mid:
                 bestaande.add(mid)
         else:
-            pass  # stil doorgaan bij incidentele fout
+            mislukt += 1
+            log.setdefault("fouten", []).append({
+                "folder": folder,
+                "uid": uid.decode(errors="replace"),
+                "actie": "append_doel",
+                "message_id": mid,
+                "status": status,
+                "response": repr(append_data),
+            })
 
         # Even pauzeren elke 50 berichten om server te ontlasten
         if gekopieerd % 50 == 0 and gekopieerd > 0:
             time.sleep(0.2)
 
-    print(f"\r  {folder}: {totaal}/{totaal}  → {gekopieerd} gekopieerd, {overgeslagen} al aanwezig   ")
+    print(f"\r  {folder}: {totaal}/{totaal}  → {gekopieerd} gekopieerd, {overgeslagen} al aanwezig, {mislukt} fout   ")
-    return gekopieerd, overgeslagen
+    return gekopieerd, overgeslagen, mislukt
 
 
 # ── log ───────────────────────────────────────────────────────────────────────
@@ -247,6 +301,7 @@ def run():
 
     totaal_gekopieerd = log.get("totaal_gekopieerd", 0)
     totaal_overgeslagen = 0
+    totaal_mislukt = 0
 
     for i, folder in enumerate(alle_mappen, 1):
         print(f"[{i}/{len(alle_mappen)}] {folder}")
@@ -255,12 +310,17 @@ def run():
             print(f"  ✓ al volledig gekopieerd, overgeslagen.\n")
             continue
 
-        gekopieerd, overgeslagen = kopieer_map(bron, doel, folder, log)
+        gekopieerd, overgeslagen, mislukt = kopieer_map(bron, doel, folder, log)
         totaal_gekopieerd   += gekopieerd
         totaal_overgeslagen += overgeslagen
+        totaal_mislukt      += mislukt
 
-        log["voltooide_mappen"] = list(al_klaar | {folder})
+        if mislukt == 0:
-        al_klaar.add(folder)
+            log["voltooide_mappen"] = list(al_klaar | {folder})
+            al_klaar.add(folder)
+        else:
+            log["voltooide_mappen"] = list(al_klaar)
+            print(f"  ! niet als voltooid gemarkeerd door {mislukt} fout(en)")
         log["totaal_gekopieerd"] = totaal_gekopieerd
         log["laatste_run"] = datetime.now().isoformat()
         save_log(log)
@@ -273,6 +333,7 @@ def run():
     print(f"Klaar.")
     print(f"  Gekopieerd   : {totaal_gekopieerd}")
     print(f"  Al aanwezig  : {totaal_overgeslagen}")
+    print(f"  Fouten       : {totaal_mislukt}")
     print(f"  Log          : {LOG_FILE}")
 
 

restart_prompt.md

@@ -21,6 +21,7 @@ Git workflow:
 - The repo has branches `main`, `claude`, and `codex`.
 - Work should continue on `codex` unless the user says otherwise.
 - Follow the user workflow: initialize Git when needed, branch before work in existing repos, and commit after each user-prompted change using the required three-section commit message format.
+- Before every commit, rewrite this file from scratch so it fully reconstructs the current project state.
 
 Ignored local data includes:
 
@@ -29,6 +30,7 @@ Ignored local data includes:
 - `config.json`
 - `__pycache__/`
 - bytecode files
+- `*.log`
 - `.DS_Store`
 
 Do not commit email `.eml` files or credentials.
@@ -61,7 +63,7 @@ Sieve:
 Shared modules:
 
 - `imap_utils.py` centralizes IMAP modified UTF-7 folder encoding/decoding, quoted mailbox names, LIST parsing, and folder listing.
-- `mail_routes.py` centralizes `PREFIX`, domain routes, invoice keywords, source-folder exclusions, already-sorted folders, invoice quarter routing, and generated destination folders.
+- `mail_routes.py` centralizes `PREFIX`, domain routes, invoice keywords, source-folder exclusions, already-sorted folders, invoice quarter routing, generated destination folders, and normalized mailing-list routing.
 
 Scripts using shared IMAP folder handling:
 
@@ -73,13 +75,16 @@ Scripts using shared IMAP folder handling:
 
 `verplaats_bestaand.py` current behavior:
 
+- Default mode is dry-run; real moves require `--uitvoeren` and an interactive `JA` confirmation.
+- `--audit` is available only in dry-run mode. It suppresses per-message dry-run lines and prints planned move counts grouped by source folder and destination folder for step 2 and step 3.
 - `MAP_RENAMES` is intentionally empty.
 - Source folders are selected by shared `list_folders()` plus `mail_routes.is_source_folder()`.
 - Step 2 routes all eligible source folders, not just `INBOX`.
-- Step 2 now reports counters per folder and in total: planned moves, actual moves when executing, no-match messages, skipped messages, and fetch/move failures.
+- Step 2 reports counters per folder and in total: planned moves, actual moves when executing, no-match messages, skipped messages, and fetch/move failures.
-- Dry runs now report `Gepland` counters instead of misleadingly showing zero moved.
+- Dry runs report `Gepland` counters instead of misleadingly showing zero moved.
-- Step 3 invoice-quarter sorting also reports planned/moved/failure counters.
+- Step 3 invoice-quarter sorting also reports planned/moved/failure counters and supports audit summaries.
-- Move success requires both `UID COPY` and `UID STORE +FLAGS \Deleted` to return `OK`.
+- Move success requires source reselect, `UID COPY`, and `UID STORE +FLAGS \Deleted` to return `OK`.
+- Move and fetch failures are recorded in `verplaats_log.json` under `fouten` with step, source folder, UID, destination when known, action, status, and server response.
 - Step 3 no longer treats `INBOX.Facturen - verwerkt` as a source.
 - IMAP mailbox names are consistently quoted and encoded through `imap_utils.quote_mailbox()`.
 
@@ -97,6 +102,9 @@ Scripts using shared IMAP folder handling:
 - Deduplicates by `Message-ID` within each destination folder.
 - Uses shared folder listing and mailbox quoting/encoding for select/create/append operations.
 - Uses `backup_log.json` to record completed folders and the copied-message total.
+- Records select/search/fetch/append failures in `backup_log.json` under `fouten`.
+- Verifies the destination folder is selectable before dedupe and append work.
+- Does not mark a folder complete if any failure occurs while copying that folder.
 
 ## Current Backup Test Status
 
@@ -112,7 +120,7 @@ After resetting the copy status, `python3 kopieer_naar_backup.py --van hans --na
 
 - Script total copied: 11,190.
 - Script duplicate skips: 13.
-- `backup_log.json` now marks 34 folders complete and records `totaal_gekopieerd: 11190`.
+- `backup_log.json` marked 34 folders complete and recorded `totaal_gekopieerd: 11190`.
 
 Server-side IMAP verification after the mirror:
 
@@ -134,23 +142,7 @@ Empty source folders not visible as selectable backup folders after the mirror a
 - `INBOX.Notes`
 - `INBOX.Technisch.dmarc`
 
-Do not run the actual sorter on backup without explicit user confirmation. A dry run is the next safe diagnostic step.
+Do not run mailbox-affecting scripts yourself in this project. The user wants to run scripts personally when instructed. A sorter dry-run audit on `backup@australius.nl` is the next safe diagnostic, but ask the user to run it rather than running it yourself.
-
-## Last Verification
-
-Latest code verification before the backup mirror test:
-
-- `python3 -m py_compile *.py`
-- route/folder consistency check returned:
-  - `routes 72`
-  - `folders 127`
-  - `missing_targets []`
-  - `archief_targets []`
-- `git diff --check`
-
-Latest live mailbox verification:
-
-- Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror.
 
 ## Reports and Findings
 
@@ -164,47 +156,53 @@ That report interprets `decisions.json` as advisory intent:
 - `keuze: "a"` means route below `INBOX.Afmelden.*` for unsubscribe/review.
 - `keuze: "s"` is ambiguous and should be confirmed before automation.
 - Obvious aliases from the same sender or organization should be combined, including Coursera, MIT Technology Review, Eva Keiffenheim/Substack, PostNL, STRATO, Forte Labs, CIONET, Gusti, Nord, Proton, Vonage/Nexmo, and Visme.
-- The normalized policy is now in `mailinglist_routes.json`; `mail_routes.py` loads it before falling back to legacy `DOMAIN_ROUTES`.
+- The normalized policy is in `mailinglist_routes.json`; `mail_routes.py` loads it before falling back to legacy `DOMAIN_ROUTES`.
 - `mailinglist_routes.json` has 153 grouped rules and covers every key from `decisions.json` through `source_decision_keys`.
-- `destination_folders()` now includes functional keep folders plus `Afmelden.*` folders from `mailinglist_routes.json`.
+- `destination_folders()` includes functional keep folders plus `Afmelden.*` folders from `mailinglist_routes.json`.
 - Invoice keyword routing still takes precedence over mailing-list routing.
-- Do not run mailbox-affecting scripts yourself in this project. The user wants to run scripts personally when instructed.
-
-Key earlier findings:
-
-- The old redistribution script selected some source folders read-only and then tried to delete messages.
-- The old logging could mark partial moves as complete.
-- The old implementation only routed `INBOX`.
-- Folder creation and route targets were inconsistent before recent fixes.
-
-Known remaining risk:
-
-- `kopieer_naar_backup.py` should be hardened so append failures cannot be silently marked as folder completion.
-- `verplaats_bestaand.py` should log failures with source folder, UID, destination, and server response.
 - Mailing-list routing currently supports domain and `From` substring policy matching. It does not yet fetch or match `List-ID` headers.
+
+Notable policy risks:
+
 - `circle8.nl` still has ambiguous decision value `s`; keep it as review/functional routing until the user confirms the meaning.
 - `alfen.com`, `vodafone.nl`, `dhlecommerce.nl`, and Proton splits are implemented according to `decisions.json` precedence, but remain notable policy changes compared with older hardcoded domain routes.
 
+## Last Verification
+
+Latest local code verification:
+
+- `python3 -m py_compile *.py`
+- `git diff --check`
+
+Earlier route/folder consistency check returned:
+
+- `routes 72`
+- `folders 127`
+- `missing_targets []`
+- `archief_targets []`
+
+Latest live mailbox verification:
+
+- Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror.
+- No live mailbox script was run while adding the latest auditability changes.
+
 ## What To Do Next
 
 Recommended next work:
 
-1. Run a non-destructive dry-run sorter on `backup@australius.nl` and review planned moves by source and destination.
+1. Ask the user to run a non-destructive dry-run sorter audit on `backup@australius.nl`, for example `python3 verplaats_bestaand.py --audit` with config pointed at the backup account, then review planned moves by source and destination.
-2. Harden move and mirror auditability:
+2. Review the audit output for suspicious high-volume destinations, missing matches, and unexpected `Afmelden.*` or invoice routes before approving any actual sorting.
-   - `kopieer_naar_backup.py` should not mark a folder complete if append failures occur.
-   - `verplaats_bestaand.py` should log failures with source folder, UID, destination, and server response.
-   - Add a dry-run audit mode showing planned moves by source and destination.
 3. Add shared operation helpers where useful:
    - UID fetch wrappers.
    - safe copy/delete/expunge helper.
    - Message-ID dedupe helper reused by live automation.
-4. Add VPS automation:
+4. Add `List-ID` header support to mailing-list routing if the audit shows sender/domain matching is too coarse.
+5. Add VPS automation only after dry-run and actual backup sorting behavior are approved:
    - create an IMAP IDLE daemon for `backup@australius.nl`
    - deploy via SSH to `vps.austalius.nl`
    - use a non-root sudo user
    - install as a systemd service
    - keep secrets in an ignored `config.json` with restrictive permissions
-5. Test actual sorting only after the dry run is approved.
 
 ## Persistent File Rule
 

verplaats_bestaand.py

@@ -25,6 +25,7 @@ import sys
 import json
 import config
 from pathlib import Path
+from collections import Counter
 from imap_utils import list_folders, quote_mailbox
 from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject
 
@@ -32,6 +33,7 @@ from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_sub
 LOG_FILE  = Path(__file__).parent / "verplaats_log.json"
 
 DRY_RUN   = "--uitvoeren" not in sys.argv
+AUDIT     = "--audit" in sys.argv
 
 
 # Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw).
@@ -65,20 +67,56 @@ def list_source_folders(mail) -> list[str]:
     return [folder for folder in list_folders(mail) if is_source_folder(folder)]
 
 
-def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool) -> bool:
+def log_failure(
+    log: dict,
+    stap: str,
+    src: str,
+    uid: bytes | str,
+    dst: str | None,
+    actie: str,
+    status: str | None,
+    response,
+):
+    uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid)
+    log.setdefault("fouten", []).append({
+        "stap": stap,
+        "bron": src,
+        "uid": uid_str,
+        "doel": dst,
+        "actie": actie,
+        "status": status,
+        "response": repr(response),
+    })
+
+
+def print_audit(title: str, by_source: Counter, by_destination: Counter):
+    print(f"\n  Audit: {title}")
+    print("  Per bronmap:")
+    for folder, count in by_source.most_common():
+        print(f"    {count:5d}  {folder}")
+    print("  Per doelmap:")
+    for folder, count in by_destination.most_common():
+        print(f"    {count:5d}  {folder}")
+
+
+def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
     """Kopieer bericht naar dst en markeer als verwijderd in src."""
     if dry:
-        return True
+        return True, None
     try:
-        mail.select(quote_mailbox(src))
+        select_res, select_data = mail.select(quote_mailbox(src))
-        res, _ = mail.uid("COPY", uid, quote_mailbox(dst_full))
+        if select_res != "OK":
+            return False, ("select_source", select_res, select_data)
+        res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full))
         if res != "OK":
-            return False
+            return False, ("copy", res, copy_data)
-        store_res, _ = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
+        store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
-        return store_res == "OK"
+        if store_res != "OK":
+            return False, ("store_deleted", store_res, store_data)
+        return True, None
     except Exception as e:
         print(f"    FOUT bij verplaatsen: {e}")
-        return False
+        return False, ("exception", None, str(e))
 
 
 # ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────
@@ -130,6 +168,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
     bron_mappen = list_source_folders(mail)
     print(f"  {len(bron_mappen)} bronmappen")
 
+    audit_by_source = Counter()
+    audit_by_destination = Counter()
     totaal_verplaatst = 0
     totaal_gepland = 0
     totaal_geen_match = 0
@@ -171,6 +211,7 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
             status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])")
             if status != "OK" or not data or not data[0]:
                 mislukt += 1
+                log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
                 continue
 
             msg = email.message_from_bytes(data[0][1])
@@ -193,10 +234,13 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
                 continue
 
             gepland += 1
+            audit_by_source[bron] += 1
+            audit_by_destination[doel] += 1
             if dry:
-                print(f"    [DRY {verwerkt}/{len(ids)}] {bron} → {doel}  |  {subject[:60]}")
+                if not AUDIT:
+                    print(f"    [DRY {verwerkt}/{len(ids)}] {bron} → {doel}  |  {subject[:60]}")
             else:
-                ok = move_message(mail, uid, bron, doel, dry=False)
+                ok, failure = move_message(mail, uid, bron, doel, dry=False)
                 if ok:
                     verplaatst += 1
                     log.setdefault("stap2_verwerkt", []).append(log_key)
@@ -205,6 +249,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
                         _save_log(log)
                 else:
                     mislukt += 1
+                    actie, status, response = failure
+                    log_failure(log, "stap2", bron, uid, doel, actie, status, response)
 
         if not dry:
             mail.expunge()
@@ -232,6 +278,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
             f"  Totaal gepland: {totaal_gepland}  |  Totaal geen match: {totaal_geen_match}"
             f"  |  Totaal overgeslagen: {totaal_overgeslagen}  |  Totaal fouten: {totaal_mislukt}"
         )
+        if AUDIT:
+            print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination)
     else:
         print(
             f"  Totaal gepland: {totaal_gepland}  |  Totaal verplaatst: {totaal_verplaatst}"
@@ -248,6 +296,8 @@ def stap3_facturen(mail, log: dict, dry: bool):
         "INBOX.Facturen - te verwerken",
         "INBOX.Financieel.Facturen",   # als ze er al in zitten
     ]
+    audit_by_source = Counter()
+    audit_by_destination = Counter()
 
     for bron in bron_mappen:
         status, data = mail.select(quote_mailbox(bron), readonly=dry)
@@ -272,29 +322,39 @@ def stap3_facturen(mail, log: dict, dry: bool):
             status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])")
             if status != "OK" or not data or not data[0]:
                 mislukt += 1
+                log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
                 continue
 
             msg      = email.message_from_bytes(data[0][1])
             date_str = msg.get("Date", "")
             doel     = PREFIX + quarter_folder(date_str)
             gepland += 1
+            audit_by_source[bron] += 1
+            audit_by_destination[doel] += 1
 
             if dry:
                 dt_kort = date_str[:16] if date_str else "?"
-                print(f"    [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
+                if not AUDIT:
+                    print(f"    [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
             else:
-                ok = move_message(mail, uid, bron, doel, dry=False)
+                ok, failure = move_message(mail, uid, bron, doel, dry=False)
                 if ok:
                     verplaatst += 1
                 else:
                     mislukt += 1
+                    actie, status, response = failure
+                    log_failure(log, "stap3", bron, uid, doel, actie, status, response)
 
         if not dry:
             mail.expunge()
             print(f"    Gepland: {gepland}  |  Verplaatst: {verplaatst}  |  Fouten: {mislukt}")
+            _save_log(log)
         else:
             print(f"    Gepland: {gepland}  |  Fouten: {mislukt}")
 
+    if dry and AUDIT:
+        print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination)
+
 
 # ── log ──────────────────────────────────────────────────────────────────────
 
@@ -314,9 +374,15 @@ def load_log() -> dict:
 # ── main ──────────────────────────────────────────────────────────────────────
 
 def run():
+    if AUDIT and not DRY_RUN:
+        print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.")
+        sys.exit(1)
+
     if DRY_RUN:
         print("DRY RUN — er wordt niets verplaatst.")
         print("Voeg --uitvoeren toe om echt uit te voeren.\n")
+        if AUDIT:
+            print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n")
     else:
         print("LET OP: dit verplaatst berichten op de echte server.")
         bevestig = input("Typ JA om door te gaan: ").strip()