diff --git a/restart_prompt.md b/restart_prompt.md index 2c13dcc..5145baf 100644 --- a/restart_prompt.md +++ b/restart_prompt.md @@ -77,6 +77,7 @@ Scripts using shared IMAP folder handling: - Default mode is dry-run; real moves require `--uitvoeren` and an interactive `JA` confirmation. - `--audit` is available only in dry-run mode. It suppresses per-message dry-run lines and prints planned move counts grouped by source folder and destination folder for step 2 and step 3. +- IMAP `LIST`, `SELECT`, `UID SEARCH`, and `UID FETCH` aborts are logged and retried once after reconnecting. If reconnect also fails, the script records an `ABORT`/`ERROR` in `verplaats_log.json` instead of printing a Python traceback. - `MAP_RENAMES` is intentionally empty. - Source folders are selected by shared `list_folders()` plus `mail_routes.is_source_folder()`. - Step 2 routes all eligible source folders, not just `INBOX`. @@ -184,7 +185,8 @@ Earlier route/folder consistency check returned: Latest live mailbox verification: - Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror. -- No live mailbox script was run while adding the latest auditability changes. +- The user ran the previous dry-run sorter version against `backup@australius.nl`. It reached `INBOX.Archief.2022.verzonden` after completing `INBOX.Archief.2022.inkomend`, then the IMAP server closed the connection with `Server shutting down.`. The current code has reconnect/retry handling for that failure mode. +- No live mailbox script was run by Codex while adding the latest reconnect changes. ## What To Do Next diff --git a/verplaats_bestaand.py b/verplaats_bestaand.py index 82851ea..6b341b1 100644 --- a/verplaats_bestaand.py +++ b/verplaats_bestaand.py @@ -24,6 +24,7 @@ import imaplib import sys import json import config +import time from pathlib import Path from collections import Counter from imap_utils import list_folders, quote_mailbox @@ -67,6 +68,31 @@ def list_source_folders(mail) -> list[str]: return [folder for folder in list_folders(mail) if is_source_folder(folder)] +class ImapSession: + """IMAP connection wrapper with conservative reconnect support.""" + + def __init__(self, cfg): + self.cfg = cfg + self.mail = None + + def connect(self): + self.mail = imaplib.IMAP4_SSL(self.cfg.imap_host, self.cfg.imap_port) + self.mail.login(self.cfg.username, self.cfg.password) + + def reconnect(self): + if self.mail is not None: + try: + self.mail.logout() + except Exception: + pass + time.sleep(2) + self.connect() + + def logout(self): + if self.mail is not None: + self.mail.logout() + + def log_failure( log: dict, stap: str, @@ -99,6 +125,102 @@ def print_audit(title: str, by_source: Counter, by_destination: Counter): print(f" {count:5d} {folder}") +def safe_select(session: ImapSession, log: dict, stap: str, folder: str, readonly: bool): + for attempt in range(2): + try: + return session.mail.select(quote_mailbox(folder), readonly=readonly) + except imaplib.IMAP4.abort as e: + log_failure(log, stap, folder, "", None, "select_abort", "ABORT", str(e)) + if attempt == 1: + return "ABORT", [str(e).encode()] + print(f" IMAP-verbinding verbroken bij openen van {folder}; opnieuw verbinden ...") + try: + session.reconnect() + except Exception as reconnect_error: + log_failure( + log, + stap, + folder, + "", + None, + "reconnect_failed", + "ERROR", + str(reconnect_error), + ) + return "ABORT", [str(reconnect_error).encode()] + + +def safe_uid( + session: ImapSession, + log: dict, + stap: str, + folder: str, + uid: bytes | str, + command: str, + *args, + readonly: bool, +): + for attempt in range(2): + try: + return session.mail.uid(command, *args) + except imaplib.IMAP4.abort as e: + log_failure( + log, + stap, + folder, + uid, + None, + f"uid_{command.lower()}_abort", + "ABORT", + str(e), + ) + if attempt == 1: + return "ABORT", [str(e).encode()] + print(f" IMAP-verbinding verbroken in {folder}; opnieuw verbinden en opnieuw proberen ...") + try: + session.reconnect() + except Exception as reconnect_error: + log_failure( + log, + stap, + folder, + uid, + None, + "reconnect_failed", + "ERROR", + str(reconnect_error), + ) + return "ABORT", [str(reconnect_error).encode()] + status, data = safe_select(session, log, stap, folder, readonly=readonly) + if status != "OK": + return status, data + + +def safe_list_source_folders(session: ImapSession, log: dict) -> list[str]: + for attempt in range(2): + try: + return list_source_folders(session.mail) + except imaplib.IMAP4.abort as e: + log_failure(log, "stap2", "", "", None, "list_abort", "ABORT", str(e)) + if attempt == 1: + return [] + print(" IMAP-verbinding verbroken bij ophalen van mappen; opnieuw verbinden ...") + try: + session.reconnect() + except Exception as reconnect_error: + log_failure( + log, + "stap2", + "", + "", + None, + "reconnect_failed", + "ERROR", + str(reconnect_error), + ) + return [] + + def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool): """Kopieer bericht naar dst en markeer als verwijderd in src.""" if dry: @@ -121,7 +243,8 @@ def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool): # ── stap 1: vaste mapverplaatsingen ───────────────────────────────────────── -def stap1_mapverplaatsingen(mail, log: dict, dry: bool): +def stap1_mapverplaatsingen(session: ImapSession, log: dict, dry: bool): + mail = session.mail print("\n── Stap 1: vaste mapverplaatsingen ──") for oud, nieuw in MAP_RENAMES: if nieuw == "__FACTUREN__": @@ -132,7 +255,7 @@ def stap1_mapverplaatsingen(mail, log: dict, dry: bool): continue # Controleer of bronmap bestaat - status, data = mail.select(quote_mailbox(oud), readonly=True) + status, data = safe_select(session, log, "stap1", oud, readonly=True) if status != "OK": print(f" – bestaat niet: {oud}") continue @@ -163,9 +286,10 @@ def stap1_mapverplaatsingen(mail, log: dict, dry: bool): # ── stap 2: per-bericht routing (INBOX) ───────────────────────────────────── -def stap2_bronmappen_routing(mail, log: dict, dry: bool): +def stap2_bronmappen_routing(session: ImapSession, log: dict, dry: bool): + mail = session.mail print("\n── Stap 2: bronmappen per-bericht routing ──") - bron_mappen = list_source_folders(mail) + bron_mappen = safe_list_source_folders(session, log) print(f" {len(bron_mappen)} bronmappen") audit_by_source = Counter() @@ -178,7 +302,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): al_verwerkt = set(log.get("stap2_verwerkt", [])) for bron in bron_mappen: - status, data = mail.select(quote_mailbox(bron), readonly=dry) + status, data = safe_select(session, log, "stap2", bron, readonly=dry) + mail = session.mail if status != "OK": print(f" Kan map niet openen: {bron}") continue @@ -188,7 +313,10 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): continue print(f" {bron}: {n} berichten") - status, data = mail.uid("SEARCH", "ALL") + status, data = safe_uid( + session, log, "stap2", bron, "", "SEARCH", "ALL", readonly=dry + ) + mail = session.mail if status != "OK" or not data or not data[0]: continue ids = [uid for uid in data[0].split() if uid] @@ -208,7 +336,18 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): overgeslagen += 1 continue - status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])") + status, data = safe_uid( + session, + log, + "stap2", + bron, + uid, + "FETCH", + uid, + "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])", + readonly=dry, + ) + mail = session.mail if status != "OK" or not data or not data[0]: mislukt += 1 log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data) @@ -290,7 +429,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool): # ── stap 3: facturen per kwartaal ──────────────────────────────────────────── -def stap3_facturen(mail, log: dict, dry: bool): +def stap3_facturen(session: ImapSession, log: dict, dry: bool): + mail = session.mail print("\n── Stap 3: facturen per kwartaal ──") bron_mappen = [ "INBOX.Facturen - te verwerken", @@ -300,7 +440,8 @@ def stap3_facturen(mail, log: dict, dry: bool): audit_by_destination = Counter() for bron in bron_mappen: - status, data = mail.select(quote_mailbox(bron), readonly=dry) + status, data = safe_select(session, log, "stap3", bron, readonly=dry) + mail = session.mail if status != "OK": continue @@ -310,7 +451,10 @@ def stap3_facturen(mail, log: dict, dry: bool): continue print(f" {bron}: {n} berichten") - status, data = mail.uid("SEARCH", "ALL") + status, data = safe_uid( + session, log, "stap3", bron, "", "SEARCH", "ALL", readonly=dry + ) + mail = session.mail if status != "OK" or not data or not data[0]: continue ids = [uid for uid in data[0].split() if uid] @@ -319,7 +463,18 @@ def stap3_facturen(mail, log: dict, dry: bool): mislukt = 0 for index, uid in enumerate(ids, start=1): - status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])") + status, data = safe_uid( + session, + log, + "stap3", + bron, + uid, + "FETCH", + uid, + "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])", + readonly=dry, + ) + mail = session.mail if status != "OK" or not data or not data[0]: mislukt += 1 log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data) @@ -394,20 +549,26 @@ def run(): print(f"Account: {cfg.username}\n") print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...") + session = ImapSession(cfg) try: - mail = imaplib.IMAP4_SSL(cfg.imap_host, cfg.imap_port) - mail.login(cfg.username, cfg.password) + session.connect() except Exception as e: print(f"Verbinding mislukt: {e}") sys.exit(1) log = load_log() - stap1_mapverplaatsingen(mail, log, DRY_RUN) - stap2_bronmappen_routing(mail, log, DRY_RUN) - stap3_facturen(mail, log, DRY_RUN) + try: + stap1_mapverplaatsingen(session, log, DRY_RUN) + stap2_bronmappen_routing(session, log, DRY_RUN) + stap3_facturen(session, log, DRY_RUN) + finally: + _save_log(log) + try: + session.logout() + except Exception as e: + print(f"Afmelden mislukt: {e}") - mail.logout() print("\nKlaar.") if DRY_RUN: print("Draai met --uitvoeren om de wijzigingen door te voeren.")