Handle IMAP aborts during sorter dry runs

The prompts used to arrive at this change since the previous commit

I have run the previous version of verplaats_bestaand. The results are in verplaats_bestaand.log and I got the following error message: imaplib.IMAP4.abort: command: UID => Server shutting down.

Any special observations that may be relevant for version management for this version. Be brief.

The user-run dry-run reached INBOX.Archief.2022.verzonden before the server closed the IMAP connection. Codex did not run live mailbox scripts; reconnect handling was verified with a fake local IMAP object.

restart_prompt.md

@@ -77,6 +77,7 @@ Scripts using shared IMAP folder handling:
 
 - Default mode is dry-run; real moves require `--uitvoeren` and an interactive `JA` confirmation.
 - `--audit` is available only in dry-run mode. It suppresses per-message dry-run lines and prints planned move counts grouped by source folder and destination folder for step 2 and step 3.
+- IMAP `LIST`, `SELECT`, `UID SEARCH`, and `UID FETCH` aborts are logged and retried once after reconnecting. If reconnect also fails, the script records an `ABORT`/`ERROR` in `verplaats_log.json` instead of printing a Python traceback.
 - `MAP_RENAMES` is intentionally empty.
 - Source folders are selected by shared `list_folders()` plus `mail_routes.is_source_folder()`.
 - Step 2 routes all eligible source folders, not just `INBOX`.
@@ -184,7 +185,8 @@ Earlier route/folder consistency check returned:
 Latest live mailbox verification:
 
 - Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror.
-- No live mailbox script was run while adding the latest auditability changes.
+- The user ran the previous dry-run sorter version against `backup@australius.nl`. It reached `INBOX.Archief.2022.verzonden` after completing `INBOX.Archief.2022.inkomend`, then the IMAP server closed the connection with `Server shutting down.`. The current code has reconnect/retry handling for that failure mode.
+- No live mailbox script was run by Codex while adding the latest reconnect changes.
 
 ## What To Do Next
 

verplaats_bestaand.py

@@ -24,6 +24,7 @@ import imaplib
 import sys
 import json
 import config
+import time
 from pathlib import Path
 from collections import Counter
 from imap_utils import list_folders, quote_mailbox
@@ -67,6 +68,31 @@ def list_source_folders(mail) -> list[str]:
     return [folder for folder in list_folders(mail) if is_source_folder(folder)]
 
 
+class ImapSession:
+    """IMAP connection wrapper with conservative reconnect support."""
+
+    def __init__(self, cfg):
+        self.cfg = cfg
+        self.mail = None
+
+    def connect(self):
+        self.mail = imaplib.IMAP4_SSL(self.cfg.imap_host, self.cfg.imap_port)
+        self.mail.login(self.cfg.username, self.cfg.password)
+
+    def reconnect(self):
+        if self.mail is not None:
+            try:
+                self.mail.logout()
+            except Exception:
+                pass
+        time.sleep(2)
+        self.connect()
+
+    def logout(self):
+        if self.mail is not None:
+            self.mail.logout()
+
+
 def log_failure(
     log: dict,
     stap: str,
@@ -99,6 +125,102 @@ def print_audit(title: str, by_source: Counter, by_destination: Counter):
         print(f"    {count:5d}  {folder}")
 
 
+def safe_select(session: ImapSession, log: dict, stap: str, folder: str, readonly: bool):
+    for attempt in range(2):
+        try:
+            return session.mail.select(quote_mailbox(folder), readonly=readonly)
+        except imaplib.IMAP4.abort as e:
+            log_failure(log, stap, folder, "", None, "select_abort", "ABORT", str(e))
+            if attempt == 1:
+                return "ABORT", [str(e).encode()]
+            print(f"  IMAP-verbinding verbroken bij openen van {folder}; opnieuw verbinden ...")
+            try:
+                session.reconnect()
+            except Exception as reconnect_error:
+                log_failure(
+                    log,
+                    stap,
+                    folder,
+                    "",
+                    None,
+                    "reconnect_failed",
+                    "ERROR",
+                    str(reconnect_error),
+                )
+                return "ABORT", [str(reconnect_error).encode()]
+
+
+def safe_uid(
+    session: ImapSession,
+    log: dict,
+    stap: str,
+    folder: str,
+    uid: bytes | str,
+    command: str,
+    *args,
+    readonly: bool,
+):
+    for attempt in range(2):
+        try:
+            return session.mail.uid(command, *args)
+        except imaplib.IMAP4.abort as e:
+            log_failure(
+                log,
+                stap,
+                folder,
+                uid,
+                None,
+                f"uid_{command.lower()}_abort",
+                "ABORT",
+                str(e),
+            )
+            if attempt == 1:
+                return "ABORT", [str(e).encode()]
+            print(f"  IMAP-verbinding verbroken in {folder}; opnieuw verbinden en opnieuw proberen ...")
+            try:
+                session.reconnect()
+            except Exception as reconnect_error:
+                log_failure(
+                    log,
+                    stap,
+                    folder,
+                    uid,
+                    None,
+                    "reconnect_failed",
+                    "ERROR",
+                    str(reconnect_error),
+                )
+                return "ABORT", [str(reconnect_error).encode()]
+            status, data = safe_select(session, log, stap, folder, readonly=readonly)
+            if status != "OK":
+                return status, data
+
+
+def safe_list_source_folders(session: ImapSession, log: dict) -> list[str]:
+    for attempt in range(2):
+        try:
+            return list_source_folders(session.mail)
+        except imaplib.IMAP4.abort as e:
+            log_failure(log, "stap2", "", "", None, "list_abort", "ABORT", str(e))
+            if attempt == 1:
+                return []
+            print("  IMAP-verbinding verbroken bij ophalen van mappen; opnieuw verbinden ...")
+            try:
+                session.reconnect()
+            except Exception as reconnect_error:
+                log_failure(
+                    log,
+                    "stap2",
+                    "",
+                    "",
+                    None,
+                    "reconnect_failed",
+                    "ERROR",
+                    str(reconnect_error),
+                )
+                return []
+
+
 def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
     """Kopieer bericht naar dst en markeer als verwijderd in src."""
     if dry:
@@ -121,7 +243,8 @@ def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
 
 # ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────
 
-def stap1_mapverplaatsingen(mail, log: dict, dry: bool):
+def stap1_mapverplaatsingen(session: ImapSession, log: dict, dry: bool):
+    mail = session.mail
     print("\n── Stap 1: vaste mapverplaatsingen ──")
     for oud, nieuw in MAP_RENAMES:
         if nieuw == "__FACTUREN__":
@@ -132,7 +255,7 @@ def stap1_mapverplaatsingen(mail, log: dict, dry: bool):
             continue
 
         # Controleer of bronmap bestaat
-        status, data = mail.select(quote_mailbox(oud), readonly=True)
+        status, data = safe_select(session, log, "stap1", oud, readonly=True)
         if status != "OK":
             print(f"  – bestaat niet: {oud}")
             continue
@@ -163,9 +286,10 @@ def stap1_mapverplaatsingen(mail, log: dict, dry: bool):
 
 # ── stap 2: per-bericht routing (INBOX) ─────────────────────────────────────
 
-def stap2_bronmappen_routing(mail, log: dict, dry: bool):
+def stap2_bronmappen_routing(session: ImapSession, log: dict, dry: bool):
+    mail = session.mail
     print("\n── Stap 2: bronmappen per-bericht routing ──")
-    bron_mappen = list_source_folders(mail)
+    bron_mappen = safe_list_source_folders(session, log)
     print(f"  {len(bron_mappen)} bronmappen")
 
     audit_by_source = Counter()
@@ -178,7 +302,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
     al_verwerkt = set(log.get("stap2_verwerkt", []))
 
     for bron in bron_mappen:
-        status, data = mail.select(quote_mailbox(bron), readonly=dry)
+        status, data = safe_select(session, log, "stap2", bron, readonly=dry)
+        mail = session.mail
         if status != "OK":
             print(f"  Kan map niet openen: {bron}")
             continue
@@ -188,7 +313,10 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
             continue
         print(f"  {bron}: {n} berichten")
 
-        status, data = mail.uid("SEARCH", "ALL")
+        status, data = safe_uid(
+            session, log, "stap2", bron, "", "SEARCH", "ALL", readonly=dry
+        )
+        mail = session.mail
         if status != "OK" or not data or not data[0]:
             continue
         ids = [uid for uid in data[0].split() if uid]
@@ -208,7 +336,18 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
                 overgeslagen += 1
                 continue
 
-            status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])")
+            status, data = safe_uid(
+                session,
+                log,
+                "stap2",
+                bron,
+                uid,
+                "FETCH",
+                uid,
+                "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])",
+                readonly=dry,
+            )
+            mail = session.mail
             if status != "OK" or not data or not data[0]:
                 mislukt += 1
                 log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
@@ -290,7 +429,8 @@ def stap2_bronmappen_routing(mail, log: dict, dry: bool):
 
 # ── stap 3: facturen per kwartaal ────────────────────────────────────────────
 
-def stap3_facturen(mail, log: dict, dry: bool):
+def stap3_facturen(session: ImapSession, log: dict, dry: bool):
+    mail = session.mail
     print("\n── Stap 3: facturen per kwartaal ──")
     bron_mappen = [
         "INBOX.Facturen - te verwerken",
@@ -300,7 +440,8 @@ def stap3_facturen(mail, log: dict, dry: bool):
     audit_by_destination = Counter()
 
     for bron in bron_mappen:
-        status, data = mail.select(quote_mailbox(bron), readonly=dry)
+        status, data = safe_select(session, log, "stap3", bron, readonly=dry)
+        mail = session.mail
         if status != "OK":
             continue
 
@@ -310,7 +451,10 @@ def stap3_facturen(mail, log: dict, dry: bool):
             continue
 
         print(f"  {bron}: {n} berichten")
-        status, data = mail.uid("SEARCH", "ALL")
+        status, data = safe_uid(
+            session, log, "stap3", bron, "", "SEARCH", "ALL", readonly=dry
+        )
+        mail = session.mail
         if status != "OK" or not data or not data[0]:
             continue
         ids = [uid for uid in data[0].split() if uid]
@@ -319,7 +463,18 @@ def stap3_facturen(mail, log: dict, dry: bool):
         mislukt = 0
 
         for index, uid in enumerate(ids, start=1):
-            status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])")
+            status, data = safe_uid(
+                session,
+                log,
+                "stap3",
+                bron,
+                uid,
+                "FETCH",
+                uid,
+                "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])",
+                readonly=dry,
+            )
+            mail = session.mail
             if status != "OK" or not data or not data[0]:
                 mislukt += 1
                 log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
@@ -394,20 +549,26 @@ def run():
     print(f"Account: {cfg.username}\n")
 
     print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...")
+    session = ImapSession(cfg)
     try:
-        mail = imaplib.IMAP4_SSL(cfg.imap_host, cfg.imap_port)
+        session.connect()
-        mail.login(cfg.username, cfg.password)
     except Exception as e:
         print(f"Verbinding mislukt: {e}")
         sys.exit(1)
 
     log = load_log()
 
-    stap1_mapverplaatsingen(mail, log, DRY_RUN)
+    try:
-    stap2_bronmappen_routing(mail, log, DRY_RUN)
+        stap1_mapverplaatsingen(session, log, DRY_RUN)
-    stap3_facturen(mail, log, DRY_RUN)
+        stap2_bronmappen_routing(session, log, DRY_RUN)
+        stap3_facturen(session, log, DRY_RUN)
+    finally:
+        _save_log(log)
+        try:
+            session.logout()
+        except Exception as e:
+            print(f"Afmelden mislukt: {e}")
 
-    mail.logout()
     print("\nKlaar.")
     if DRY_RUN:
         print("Draai met --uitvoeren om de wijzigingen door te voeren.")