From f44c3630250580f5bece650aa442c106fd358e74 Mon Sep 17 00:00:00 2001 From: Hans Wienen Date: Sat, 4 Jul 2026 16:33:58 +0200 Subject: [PATCH] Harden mail move audit logging The prompts used to arrive at this change since the previous commit Read restart_prompt.md first, then continue from there. Any special observations that may be relevant for version management for this version. Be brief. No mailbox-affecting scripts were run by Codex. A generated dry-run transcript appeared locally as verplaats_bestaand.log and is now ignored via *.log. --- .gitignore | 1 + kopieer_naar_backup.py | 89 ++++++++++++++++++++++++++++++++++------- restart_prompt.md | 88 ++++++++++++++++++++--------------------- verplaats_bestaand.py | 90 ++++++++++++++++++++++++++++++++++++------ 4 files changed, 197 insertions(+), 71 deletions(-) diff --git a/.gitignore b/.gitignore index 9b67a16..b5385a0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ __pycache__/ *.py[cod] +*.log .DS_Store config.json **/*.eml diff --git a/kopieer_naar_backup.py b/kopieer_naar_backup.py index 9eca7cf..c8e8a4d 100644 --- a/kopieer_naar_backup.py +++ b/kopieer_naar_backup.py @@ -88,39 +88,69 @@ def kopieer_map( doel: imaplib.IMAP4_SSL, folder: str, log: dict, -) -> tuple[int, int]: +) -> tuple[int, int, int]: """ Kopieer berichten van bron-folder naar doel-folder. - Geeft (gekopieerd, overgeslagen) terug. + Geeft (gekopieerd, overgeslagen, mislukt) terug. """ # Selecteer bronmap status, data = bron.select(quote_mailbox(folder), readonly=True) if status != "OK": print(f" Kan bronmap niet openen: {folder}") - return 0, 0 + log.setdefault("fouten", []).append({ + "folder": folder, + "actie": "select_bron", + "status": status, + "response": repr(data), + }) + return 0, 0, 1 totaal = int(data[0]) if totaal == 0: - return 0, 0 + return 0, 0, 0 # Haal alle bron-UIDs op status, data = bron.search(None, "ALL") if status != "OK" or not data[0]: - return 0, 0 + log.setdefault("fouten", []).append({ + "folder": folder, + "actie": "search_bron", + "status": status, + "response": repr(data), + }) + return 0, 0, 1 ids = data[0].split() # Zorg dat doelmap bestaat - doel.create(quote_mailbox(folder)) # negeer fout als al bestaat + create_status, create_data = doel.create(quote_mailbox(folder)) + if create_status not in {"OK", "NO"}: + log.setdefault("fouten", []).append({ + "folder": folder, + "actie": "create_doel", + "status": create_status, + "response": repr(create_data), + }) + return 0, 0, 1 + select_status, select_data = doel.select(quote_mailbox(folder), readonly=True) + if select_status != "OK": + log.setdefault("fouten", []).append({ + "folder": folder, + "actie": "select_doel_na_create", + "status": select_status, + "response": repr(select_data), + }) + return 0, 0, 1 # Welke Message-IDs staan al in de doelmap? bestaande = haal_bestaande_message_ids(doel, folder) gekopieerd = 0 overgeslagen = 0 + mislukt = 0 for i, uid in enumerate(ids, 1): # Voortgang op één regel - print(f"\r {folder}: {i}/{totaal} ({gekopieerd} gekopieerd, {overgeslagen} skip) ", + print(f"\r {folder}: {i}/{totaal} ({gekopieerd} gekopieerd, {overgeslagen} skip, {mislukt} fout) ", end="", flush=True) # Haal volledige bericht op + vlaggen + datum @@ -128,6 +158,14 @@ def kopieer_map( uid, "(FLAGS INTERNALDATE BODY.PEEK[])" ) if status != "OK" or not data: + mislukt += 1 + log.setdefault("fouten", []).append({ + "folder": folder, + "uid": uid.decode(errors="replace"), + "actie": "fetch_bron", + "status": status, + "response": repr(data), + }) continue # Parse fetch-response @@ -144,6 +182,14 @@ def kopieer_map( internaldate = parse_internaldate(item[0]) if not raw_message: + mislukt += 1 + log.setdefault("fouten", []).append({ + "folder": folder, + "uid": uid.decode(errors="replace"), + "actie": "fetch_lege_body", + "status": status, + "response": repr(data), + }) continue # Controleer Message-ID voor deduplicatie @@ -159,7 +205,7 @@ def kopieer_map( # Schrijf naar doelmap flag_str = "(" + " ".join(flags) + ")" if flags else "()" - status, _ = doel.append( + status, append_data = doel.append( quote_mailbox(folder), flag_str, internaldate, @@ -170,14 +216,22 @@ def kopieer_map( if mid: bestaande.add(mid) else: - pass # stil doorgaan bij incidentele fout + mislukt += 1 + log.setdefault("fouten", []).append({ + "folder": folder, + "uid": uid.decode(errors="replace"), + "actie": "append_doel", + "message_id": mid, + "status": status, + "response": repr(append_data), + }) # Even pauzeren elke 50 berichten om server te ontlasten if gekopieerd % 50 == 0 and gekopieerd > 0: time.sleep(0.2) - print(f"\r {folder}: {totaal}/{totaal} → {gekopieerd} gekopieerd, {overgeslagen} al aanwezig ") - return gekopieerd, overgeslagen + print(f"\r {folder}: {totaal}/{totaal} → {gekopieerd} gekopieerd, {overgeslagen} al aanwezig, {mislukt} fout ") + return gekopieerd, overgeslagen, mislukt # ── log ─────────────────────────────────────────────────────────────────────── @@ -247,6 +301,7 @@ def run(): totaal_gekopieerd = log.get("totaal_gekopieerd", 0) totaal_overgeslagen = 0 + totaal_mislukt = 0 for i, folder in enumerate(alle_mappen, 1): print(f"[{i}/{len(alle_mappen)}] {folder}") @@ -255,12 +310,17 @@ def run(): print(f" ✓ al volledig gekopieerd, overgeslagen.\n") continue - gekopieerd, overgeslagen = kopieer_map(bron, doel, folder, log) + gekopieerd, overgeslagen, mislukt = kopieer_map(bron, doel, folder, log) totaal_gekopieerd += gekopieerd totaal_overgeslagen += overgeslagen + totaal_mislukt += mislukt - log["voltooide_mappen"] = list(al_klaar | {folder}) - al_klaar.add(folder) + if mislukt == 0: + log["voltooide_mappen"] = list(al_klaar | {folder}) + al_klaar.add(folder) + else: + log["voltooide_mappen"] = list(al_klaar) + print(f" ! niet als voltooid gemarkeerd door {mislukt} fout(en)") log["totaal_gekopieerd"] = totaal_gekopieerd log["laatste_run"] = datetime.now().isoformat() save_log(log) @@ -273,6 +333,7 @@ def run(): print(f"Klaar.") print(f" Gekopieerd : {totaal_gekopieerd}") print(f" Al aanwezig : {totaal_overgeslagen}") + print(f" Fouten : {totaal_mislukt}") print(f" Log : {LOG_FILE}") diff --git a/restart_prompt.md b/restart_prompt.md index 13283ba..2c13dcc 100644 --- a/restart_prompt.md +++ b/restart_prompt.md @@ -21,6 +21,7 @@ Git workflow: - The repo has branches `main`, `claude`, and `codex`. - Work should continue on `codex` unless the user says otherwise. - Follow the user workflow: initialize Git when needed, branch before work in existing repos, and commit after each user-prompted change using the required three-section commit message format. +- Before every commit, rewrite this file from scratch so it fully reconstructs the current project state. Ignored local data includes: @@ -29,6 +30,7 @@ Ignored local data includes: - `config.json` - `__pycache__/` - bytecode files +- `*.log` - `.DS_Store` Do not commit email `.eml` files or credentials. @@ -61,7 +63,7 @@ Sieve: Shared modules: - `imap_utils.py` centralizes IMAP modified UTF-7 folder encoding/decoding, quoted mailbox names, LIST parsing, and folder listing. -- `mail_routes.py` centralizes `PREFIX`, domain routes, invoice keywords, source-folder exclusions, already-sorted folders, invoice quarter routing, and generated destination folders. +- `mail_routes.py` centralizes `PREFIX`, domain routes, invoice keywords, source-folder exclusions, already-sorted folders, invoice quarter routing, generated destination folders, and normalized mailing-list routing. Scripts using shared IMAP folder handling: @@ -73,13 +75,16 @@ Scripts using shared IMAP folder handling: `verplaats_bestaand.py` current behavior: +- Default mode is dry-run; real moves require `--uitvoeren` and an interactive `JA` confirmation. +- `--audit` is available only in dry-run mode. It suppresses per-message dry-run lines and prints planned move counts grouped by source folder and destination folder for step 2 and step 3. - `MAP_RENAMES` is intentionally empty. - Source folders are selected by shared `list_folders()` plus `mail_routes.is_source_folder()`. - Step 2 routes all eligible source folders, not just `INBOX`. -- Step 2 now reports counters per folder and in total: planned moves, actual moves when executing, no-match messages, skipped messages, and fetch/move failures. -- Dry runs now report `Gepland` counters instead of misleadingly showing zero moved. -- Step 3 invoice-quarter sorting also reports planned/moved/failure counters. -- Move success requires both `UID COPY` and `UID STORE +FLAGS \Deleted` to return `OK`. +- Step 2 reports counters per folder and in total: planned moves, actual moves when executing, no-match messages, skipped messages, and fetch/move failures. +- Dry runs report `Gepland` counters instead of misleadingly showing zero moved. +- Step 3 invoice-quarter sorting also reports planned/moved/failure counters and supports audit summaries. +- Move success requires source reselect, `UID COPY`, and `UID STORE +FLAGS \Deleted` to return `OK`. +- Move and fetch failures are recorded in `verplaats_log.json` under `fouten` with step, source folder, UID, destination when known, action, status, and server response. - Step 3 no longer treats `INBOX.Facturen - verwerkt` as a source. - IMAP mailbox names are consistently quoted and encoded through `imap_utils.quote_mailbox()`. @@ -97,6 +102,9 @@ Scripts using shared IMAP folder handling: - Deduplicates by `Message-ID` within each destination folder. - Uses shared folder listing and mailbox quoting/encoding for select/create/append operations. - Uses `backup_log.json` to record completed folders and the copied-message total. +- Records select/search/fetch/append failures in `backup_log.json` under `fouten`. +- Verifies the destination folder is selectable before dedupe and append work. +- Does not mark a folder complete if any failure occurs while copying that folder. ## Current Backup Test Status @@ -112,7 +120,7 @@ After resetting the copy status, `python3 kopieer_naar_backup.py --van hans --na - Script total copied: 11,190. - Script duplicate skips: 13. -- `backup_log.json` now marks 34 folders complete and records `totaal_gekopieerd: 11190`. +- `backup_log.json` marked 34 folders complete and recorded `totaal_gekopieerd: 11190`. Server-side IMAP verification after the mirror: @@ -134,23 +142,7 @@ Empty source folders not visible as selectable backup folders after the mirror a - `INBOX.Notes` - `INBOX.Technisch.dmarc` -Do not run the actual sorter on backup without explicit user confirmation. A dry run is the next safe diagnostic step. - -## Last Verification - -Latest code verification before the backup mirror test: - -- `python3 -m py_compile *.py` -- route/folder consistency check returned: - - `routes 72` - - `folders 127` - - `missing_targets []` - - `archief_targets []` -- `git diff --check` - -Latest live mailbox verification: - -- Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror. +Do not run mailbox-affecting scripts yourself in this project. The user wants to run scripts personally when instructed. A sorter dry-run audit on `backup@australius.nl` is the next safe diagnostic, but ask the user to run it rather than running it yourself. ## Reports and Findings @@ -164,47 +156,53 @@ That report interprets `decisions.json` as advisory intent: - `keuze: "a"` means route below `INBOX.Afmelden.*` for unsubscribe/review. - `keuze: "s"` is ambiguous and should be confirmed before automation. - Obvious aliases from the same sender or organization should be combined, including Coursera, MIT Technology Review, Eva Keiffenheim/Substack, PostNL, STRATO, Forte Labs, CIONET, Gusti, Nord, Proton, Vonage/Nexmo, and Visme. -- The normalized policy is now in `mailinglist_routes.json`; `mail_routes.py` loads it before falling back to legacy `DOMAIN_ROUTES`. +- The normalized policy is in `mailinglist_routes.json`; `mail_routes.py` loads it before falling back to legacy `DOMAIN_ROUTES`. - `mailinglist_routes.json` has 153 grouped rules and covers every key from `decisions.json` through `source_decision_keys`. -- `destination_folders()` now includes functional keep folders plus `Afmelden.*` folders from `mailinglist_routes.json`. +- `destination_folders()` includes functional keep folders plus `Afmelden.*` folders from `mailinglist_routes.json`. - Invoice keyword routing still takes precedence over mailing-list routing. -- Do not run mailbox-affecting scripts yourself in this project. The user wants to run scripts personally when instructed. - -Key earlier findings: - -- The old redistribution script selected some source folders read-only and then tried to delete messages. -- The old logging could mark partial moves as complete. -- The old implementation only routed `INBOX`. -- Folder creation and route targets were inconsistent before recent fixes. - -Known remaining risk: - -- `kopieer_naar_backup.py` should be hardened so append failures cannot be silently marked as folder completion. -- `verplaats_bestaand.py` should log failures with source folder, UID, destination, and server response. - Mailing-list routing currently supports domain and `From` substring policy matching. It does not yet fetch or match `List-ID` headers. + +Notable policy risks: + - `circle8.nl` still has ambiguous decision value `s`; keep it as review/functional routing until the user confirms the meaning. - `alfen.com`, `vodafone.nl`, `dhlecommerce.nl`, and Proton splits are implemented according to `decisions.json` precedence, but remain notable policy changes compared with older hardcoded domain routes. +## Last Verification + +Latest local code verification: + +- `python3 -m py_compile *.py` +- `git diff --check` + +Earlier route/folder consistency check returned: + +- `routes 72` +- `folders 127` +- `missing_targets []` +- `archief_targets []` + +Latest live mailbox verification: + +- Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror. +- No live mailbox script was run while adding the latest auditability changes. + ## What To Do Next Recommended next work: -1. Run a non-destructive dry-run sorter on `backup@australius.nl` and review planned moves by source and destination. -2. Harden move and mirror auditability: - - `kopieer_naar_backup.py` should not mark a folder complete if append failures occur. - - `verplaats_bestaand.py` should log failures with source folder, UID, destination, and server response. - - Add a dry-run audit mode showing planned moves by source and destination. +1. Ask the user to run a non-destructive dry-run sorter audit on `backup@australius.nl`, for example `python3 verplaats_bestaand.py --audit` with config pointed at the backup account, then review planned moves by source and destination. +2. Review the audit output for suspicious high-volume destinations, missing matches, and unexpected `Afmelden.*` or invoice routes before approving any actual sorting. 3. Add shared operation helpers where useful: - UID fetch wrappers. - safe copy/delete/expunge helper. - Message-ID dedupe helper reused by live automation. -4. Add VPS automation: +4. Add `List-ID` header support to mailing-list routing if the audit shows sender/domain matching is too coarse. +5. Add VPS automation only after dry-run and actual backup sorting behavior are approved: - create an IMAP IDLE daemon for `backup@australius.nl` - deploy via SSH to `vps.austalius.nl` - use a non-root sudo user - install as a systemd service - keep secrets in an ignored `config.json` with restrictive permissions -5. Test actual sorting only after the dry run is approved. ## Persistent File Rule diff --git a/verplaats_bestaand.py b/verplaats_bestaand.py index 377c12a..82851ea 100644 --- a/verplaats_bestaand.py +++ b/verplaats_bestaand.py @@ -25,6 +25,7 @@ import sys import json import config from pathlib import Path +from collections import Counter from imap_utils import list_folders, quote_mailbox from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject @@ -32,6 +33,7 @@ from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_sub LOG_FILE = Path(__file__).parent / "verplaats_log.json" DRY_RUN = "--uitvoeren" not in sys.argv +AUDIT = "--audit" in sys.argv # Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw). @@ -65,20 +67,56 @@ def list_source_folders(mail) -> list[str]: return [folder for folder in list_folders(mail) if is_source_folder(folder)] -def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool) -> bool: +def log_failure( + log: dict, + stap: str, + src: str, + uid: bytes | str, + dst: str | None, + actie: str, + status: str | None, + response, +): + uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid) + log.setdefault("fouten", []).append({ + "stap": stap, + "bron": src, + "uid": uid_str, + "doel": dst, + "actie": actie, + "status": status, + "response": repr(response), + }) + + +def print_audit(title: str, by_source: Counter, by_destination: Counter): + print(f"\n Audit: {title}") + print(" Per bronmap:") + for folder, count in by_source.most_common(): + print(f" {count:5d} {folder}") + print(" Per doelmap:") + for folder, count in by_destination.most_common(): + print(f" {count:5d} {folder}") + + +def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool): """Kopieer bericht naar dst en markeer als verwijderd in src.""" if dry: - return True + return True, None try: - mail.select(quote_mailbox(src)) - res, _ = mail.uid("COPY", uid, quote_mailbox(dst_full)) + select_res, select_data = mail.select(quote_mailbox(src)) + if select_res != "OK": + return False, ("select_source", select_res, select_data) + res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full)) if res != "OK": - return False - store_res, _ = mail.uid("STORE", uid, "+FLAGS", "\\Deleted") - return store_res == "OK" + return False, ("copy", res, copy_data) + store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted") + if store_res != "OK": + return False, ("store_deleted", store_res, store_data) + return True, None except Exception as e: print(f" FOUT bij verplaatsen: {e}") - return False + return False, ("exception", None, str(e)) # ── stap 1: vaste mapverplaatsingen ───────────────────────────────────────── @@ -130,6 +168,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): bron_mappen = list_source_folders(mail) print(f" {len(bron_mappen)} bronmappen") + audit_by_source = Counter() + audit_by_destination = Counter() totaal_verplaatst = 0 totaal_gepland = 0 totaal_geen_match = 0 @@ -171,6 +211,7 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])") if status != "OK" or not data or not data[0]: mislukt += 1 + log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data) continue msg = email.message_from_bytes(data[0][1]) @@ -193,10 +234,13 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): continue gepland += 1 + audit_by_source[bron] += 1 + audit_by_destination[doel] += 1 if dry: - print(f" [DRY {verwerkt}/{len(ids)}] {bron} → {doel} | {subject[:60]}") + if not AUDIT: + print(f" [DRY {verwerkt}/{len(ids)}] {bron} → {doel} | {subject[:60]}") else: - ok = move_message(mail, uid, bron, doel, dry=False) + ok, failure = move_message(mail, uid, bron, doel, dry=False) if ok: verplaatst += 1 log.setdefault("stap2_verwerkt", []).append(log_key) @@ -205,6 +249,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): _save_log(log) else: mislukt += 1 + actie, status, response = failure + log_failure(log, "stap2", bron, uid, doel, actie, status, response) if not dry: mail.expunge() @@ -232,6 +278,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): f" Totaal gepland: {totaal_gepland} | Totaal geen match: {totaal_geen_match}" f" | Totaal overgeslagen: {totaal_overgeslagen} | Totaal fouten: {totaal_mislukt}" ) + if AUDIT: + print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination) else: print( f" Totaal gepland: {totaal_gepland} | Totaal verplaatst: {totaal_verplaatst}" @@ -248,6 +296,8 @@ def stap3_facturen(mail, log: dict, dry: bool): "INBOX.Facturen - te verwerken", "INBOX.Financieel.Facturen", # als ze er al in zitten ] + audit_by_source = Counter() + audit_by_destination = Counter() for bron in bron_mappen: status, data = mail.select(quote_mailbox(bron), readonly=dry) @@ -272,29 +322,39 @@ def stap3_facturen(mail, log: dict, dry: bool): status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])") if status != "OK" or not data or not data[0]: mislukt += 1 + log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data) continue msg = email.message_from_bytes(data[0][1]) date_str = msg.get("Date", "") doel = PREFIX + quarter_folder(date_str) gepland += 1 + audit_by_source[bron] += 1 + audit_by_destination[doel] += 1 if dry: dt_kort = date_str[:16] if date_str else "?" - print(f" [DRY {index}/{len(ids)}] {dt_kort} → {doel}") + if not AUDIT: + print(f" [DRY {index}/{len(ids)}] {dt_kort} → {doel}") else: - ok = move_message(mail, uid, bron, doel, dry=False) + ok, failure = move_message(mail, uid, bron, doel, dry=False) if ok: verplaatst += 1 else: mislukt += 1 + actie, status, response = failure + log_failure(log, "stap3", bron, uid, doel, actie, status, response) if not dry: mail.expunge() print(f" Gepland: {gepland} | Verplaatst: {verplaatst} | Fouten: {mislukt}") + _save_log(log) else: print(f" Gepland: {gepland} | Fouten: {mislukt}") + if dry and AUDIT: + print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination) + # ── log ────────────────────────────────────────────────────────────────────── @@ -314,9 +374,15 @@ def load_log() -> dict: # ── main ────────────────────────────────────────────────────────────────────── def run(): + if AUDIT and not DRY_RUN: + print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.") + sys.exit(1) + if DRY_RUN: print("DRY RUN — er wordt niets verplaatst.") print("Voeg --uitvoeren toe om echt uit te voeren.\n") + if AUDIT: + print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n") else: print("LET OP: dit verplaatst berichten op de echte server.") bevestig = input("Typ JA om door te gaan: ").strip()