wienen
edcad4560c
The prompts used to arrive at this change since the previous commit User asked to check the audit log in the current directory before reordering the real mailbox. Any special observations that may be relevant for version management for this version. Be brief. The inspected log was a partial real backup run, not a hans audit. Header parsing now skips non-bytes FETCH payloads instead of aborting. verplaats_log.json and plan.md were left unstaged.
642 lines
21 KiB
Python
642 lines
21 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Herindeling van bestaande mail naar de nieuwe mappenstructuur.
|
||
|
||
Stap 1 – Vaste mapverplaatsingen:
|
||
Uitgeschakeld. Alle niet-uitgesloten mappen worden als bron behandeld.
|
||
|
||
Stap 2 – Per-bericht routing:
|
||
Op basis van afzenderdomein en onderwerpkeywords.
|
||
Bronmappen: alles behalve Sent, Drafts, Trash, Spam en
|
||
INBOX.Facturen - verwerkt.
|
||
|
||
Stap 3 – Facturen per kwartaal sorteren (Facturen - te verwerken).
|
||
|
||
Standaard: DRY RUN (laat zien wat er zou gebeuren, verplaatst niets).
|
||
Account kiezen: python3 verplaats_bestaand.py --account backup --audit
|
||
Echt uitvoeren: python3 verplaats_bestaand.py --account backup --uitvoeren
|
||
|
||
Log: verplaats_log.json
|
||
"""
|
||
|
||
import email
|
||
import email.header
|
||
import imaplib
|
||
import sys
|
||
import json
|
||
import config
|
||
import time
|
||
from pathlib import Path
|
||
from collections import Counter
|
||
from imap_utils import list_folders, quote_mailbox
|
||
from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject
|
||
|
||
|
||
LOG_FILE = Path(__file__).parent / "verplaats_log.json"
|
||
|
||
DRY_RUN = "--uitvoeren" not in sys.argv
|
||
AUDIT = "--audit" in sys.argv
|
||
PROGRESS_INTERVAL = 250
|
||
|
||
|
||
# Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw).
|
||
#
|
||
# Bewust leeg: oude mappen, archiefmappen en mailinglistmappen zijn bronnen.
|
||
# Alleen Sent, Drafts, Trash, Spam en INBOX.Facturen - verwerkt worden
|
||
# uitgesloten van routering.
|
||
MAP_RENAMES: list[tuple[str, str]] = [
|
||
]
|
||
|
||
|
||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
def decode_hdr(value) -> str:
|
||
if not value:
|
||
return ""
|
||
try:
|
||
parts = email.header.decode_header(value)
|
||
out = []
|
||
for raw, cs in parts:
|
||
if isinstance(raw, bytes):
|
||
out.append(raw.decode(cs or "utf-8", errors="replace"))
|
||
else:
|
||
out.append(str(raw))
|
||
return " ".join(out).strip()
|
||
except Exception:
|
||
return str(value or "")
|
||
|
||
|
||
def list_source_folders(mail) -> list[str]:
|
||
return [folder for folder in list_folders(mail) if is_source_folder(folder)]
|
||
|
||
|
||
class ImapSession:
|
||
"""IMAP connection wrapper with conservative reconnect support."""
|
||
|
||
def __init__(self, cfg):
|
||
self.cfg = cfg
|
||
self.mail = None
|
||
|
||
def connect(self):
|
||
self.mail = imaplib.IMAP4_SSL(self.cfg.imap_host, self.cfg.imap_port)
|
||
self.mail.login(self.cfg.username, self.cfg.password)
|
||
|
||
def reconnect(self):
|
||
if self.mail is not None:
|
||
try:
|
||
self.mail.logout()
|
||
except Exception:
|
||
pass
|
||
time.sleep(2)
|
||
self.connect()
|
||
|
||
def logout(self):
|
||
if self.mail is not None:
|
||
self.mail.logout()
|
||
|
||
|
||
def log_failure(
|
||
log: dict,
|
||
stap: str,
|
||
src: str,
|
||
uid: bytes | str,
|
||
dst: str | None,
|
||
actie: str,
|
||
status: str | None,
|
||
response,
|
||
):
|
||
uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid)
|
||
log.setdefault("fouten", []).append({
|
||
"stap": stap,
|
||
"bron": src,
|
||
"uid": uid_str,
|
||
"doel": dst,
|
||
"actie": actie,
|
||
"status": status,
|
||
"response": repr(response),
|
||
})
|
||
|
||
|
||
def print_audit(title: str, by_source: Counter, by_destination: Counter):
|
||
print(f"\n Audit: {title}")
|
||
print(" Per bronmap:")
|
||
for folder, count in by_source.most_common():
|
||
print(f" {count:5d} {folder}")
|
||
print(" Per doelmap:")
|
||
for folder, count in by_destination.most_common():
|
||
print(f" {count:5d} {folder}")
|
||
|
||
|
||
def print_progress(folder: str, processed: int, total: int):
|
||
if processed == total or processed % PROGRESS_INTERVAL == 0:
|
||
print(f" Gescand: {processed}/{total} | {folder}")
|
||
|
||
|
||
def fetch_body(data) -> bytes | None:
|
||
"""Return the first bytes payload from an IMAP FETCH response."""
|
||
if not data:
|
||
return None
|
||
for item in data:
|
||
if isinstance(item, tuple) and len(item) >= 2 and isinstance(item[1], bytes):
|
||
return item[1]
|
||
return None
|
||
|
||
|
||
def safe_select(session: ImapSession, log: dict, stap: str, folder: str, readonly: bool):
|
||
for attempt in range(2):
|
||
try:
|
||
return session.mail.select(quote_mailbox(folder), readonly=readonly)
|
||
except imaplib.IMAP4.abort as e:
|
||
log_failure(log, stap, folder, "", None, "select_abort", "ABORT", str(e))
|
||
if attempt == 1:
|
||
return "ABORT", [str(e).encode()]
|
||
print(f" IMAP-verbinding verbroken bij openen van {folder}; opnieuw verbinden ...")
|
||
try:
|
||
session.reconnect()
|
||
except Exception as reconnect_error:
|
||
log_failure(
|
||
log,
|
||
stap,
|
||
folder,
|
||
"",
|
||
None,
|
||
"reconnect_failed",
|
||
"ERROR",
|
||
str(reconnect_error),
|
||
)
|
||
return "ABORT", [str(reconnect_error).encode()]
|
||
|
||
|
||
def safe_uid(
|
||
session: ImapSession,
|
||
log: dict,
|
||
stap: str,
|
||
folder: str,
|
||
uid: bytes | str,
|
||
command: str,
|
||
*args,
|
||
readonly: bool,
|
||
):
|
||
for attempt in range(2):
|
||
try:
|
||
return session.mail.uid(command, *args)
|
||
except imaplib.IMAP4.abort as e:
|
||
log_failure(
|
||
log,
|
||
stap,
|
||
folder,
|
||
uid,
|
||
None,
|
||
f"uid_{command.lower()}_abort",
|
||
"ABORT",
|
||
str(e),
|
||
)
|
||
if attempt == 1:
|
||
return "ABORT", [str(e).encode()]
|
||
print(f" IMAP-verbinding verbroken in {folder}; opnieuw verbinden en opnieuw proberen ...")
|
||
try:
|
||
session.reconnect()
|
||
except Exception as reconnect_error:
|
||
log_failure(
|
||
log,
|
||
stap,
|
||
folder,
|
||
uid,
|
||
None,
|
||
"reconnect_failed",
|
||
"ERROR",
|
||
str(reconnect_error),
|
||
)
|
||
return "ABORT", [str(reconnect_error).encode()]
|
||
status, data = safe_select(session, log, stap, folder, readonly=readonly)
|
||
if status != "OK":
|
||
return status, data
|
||
|
||
|
||
def safe_list_source_folders(session: ImapSession, log: dict) -> list[str]:
|
||
for attempt in range(2):
|
||
try:
|
||
return list_source_folders(session.mail)
|
||
except imaplib.IMAP4.abort as e:
|
||
log_failure(log, "stap2", "", "", None, "list_abort", "ABORT", str(e))
|
||
if attempt == 1:
|
||
return []
|
||
print(" IMAP-verbinding verbroken bij ophalen van mappen; opnieuw verbinden ...")
|
||
try:
|
||
session.reconnect()
|
||
except Exception as reconnect_error:
|
||
log_failure(
|
||
log,
|
||
"stap2",
|
||
"",
|
||
"",
|
||
None,
|
||
"reconnect_failed",
|
||
"ERROR",
|
||
str(reconnect_error),
|
||
)
|
||
return []
|
||
|
||
|
||
def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
|
||
"""Kopieer bericht naar dst en markeer als verwijderd in src."""
|
||
if dry:
|
||
return True, None
|
||
try:
|
||
select_res, select_data = mail.select(quote_mailbox(src))
|
||
if select_res != "OK":
|
||
return False, ("select_source", select_res, select_data)
|
||
res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full))
|
||
if res != "OK":
|
||
return False, ("copy", res, copy_data)
|
||
store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
|
||
if store_res != "OK":
|
||
return False, ("store_deleted", store_res, store_data)
|
||
return True, None
|
||
except Exception as e:
|
||
print(f" FOUT bij verplaatsen: {e}")
|
||
return False, ("exception", None, str(e))
|
||
|
||
|
||
# ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────
|
||
|
||
def stap1_mapverplaatsingen(session: ImapSession, log: dict, dry: bool):
|
||
mail = session.mail
|
||
print("\n── Stap 1: vaste mapverplaatsingen ──")
|
||
for oud, nieuw in MAP_RENAMES:
|
||
if nieuw == "__FACTUREN__":
|
||
continue # wordt in stap 3 verwerkt
|
||
|
||
if oud in log.get("stap1_klaar", []):
|
||
print(f" ✓ al gedaan: {oud}")
|
||
continue
|
||
|
||
# Controleer of bronmap bestaat
|
||
status, data = safe_select(session, log, "stap1", oud, readonly=True)
|
||
if status != "OK":
|
||
print(f" – bestaat niet: {oud}")
|
||
continue
|
||
|
||
n = int(data[0])
|
||
nieuw_full = nieuw if nieuw.startswith("INBOX.") else PREFIX + nieuw
|
||
|
||
if dry:
|
||
print(f" [DRY] {oud} → {nieuw} ({n} berichten)")
|
||
continue
|
||
|
||
# Berichten kopiëren
|
||
status, data = mail.search(None, "ALL")
|
||
ids = data[0].split()
|
||
gekopieerd = 0
|
||
for uid in ids:
|
||
res, _ = mail.copy(uid.decode(), quote_mailbox(nieuw_full))
|
||
if res == "OK":
|
||
mail.store(uid.decode(), "+FLAGS", "\\Deleted")
|
||
gekopieerd += 1
|
||
|
||
mail.expunge()
|
||
print(f" ✓ {oud} → {nieuw} ({gekopieerd}/{n})")
|
||
|
||
log.setdefault("stap1_klaar", []).append(oud)
|
||
_save_log(log)
|
||
|
||
|
||
# ── stap 2: per-bericht routing (INBOX) ─────────────────────────────────────
|
||
|
||
def stap2_bronmappen_routing(session: ImapSession, log: dict, dry: bool):
|
||
mail = session.mail
|
||
print("\n── Stap 2: bronmappen per-bericht routing ──")
|
||
bron_mappen = safe_list_source_folders(session, log)
|
||
print(f" {len(bron_mappen)} bronmappen")
|
||
|
||
audit_by_source = Counter()
|
||
audit_by_destination = Counter()
|
||
totaal_verplaatst = 0
|
||
totaal_gepland = 0
|
||
totaal_geen_match = 0
|
||
totaal_overgeslagen = 0
|
||
totaal_mislukt = 0
|
||
al_verwerkt = set(log.get("stap2_verwerkt", []))
|
||
|
||
for folder_index, bron in enumerate(bron_mappen, start=1):
|
||
status, data = safe_select(session, log, "stap2", bron, readonly=dry)
|
||
mail = session.mail
|
||
if status != "OK":
|
||
print(f" Kan map niet openen: {bron}")
|
||
continue
|
||
|
||
n = int(data[0])
|
||
if n == 0:
|
||
continue
|
||
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: {n} berichten")
|
||
|
||
status, data = safe_uid(
|
||
session, log, "stap2", bron, "", "SEARCH", "ALL", readonly=dry
|
||
)
|
||
mail = session.mail
|
||
if status != "OK" or not data or not data[0]:
|
||
continue
|
||
ids = [uid for uid in data[0].split() if uid]
|
||
|
||
verplaatst = 0
|
||
gepland = 0
|
||
geen_match = 0
|
||
overgeslagen = 0
|
||
mislukt = 0
|
||
verwerkt = 0
|
||
|
||
for uid in ids:
|
||
verwerkt += 1
|
||
if AUDIT:
|
||
print_progress(bron, verwerkt, len(ids))
|
||
uid_str = uid.decode()
|
||
log_key = f"{bron}:{uid_str}"
|
||
if log_key in al_verwerkt:
|
||
overgeslagen += 1
|
||
continue
|
||
|
||
status, data = safe_uid(
|
||
session,
|
||
log,
|
||
"stap2",
|
||
bron,
|
||
uid,
|
||
"FETCH",
|
||
uid,
|
||
"(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])",
|
||
readonly=dry,
|
||
)
|
||
mail = session.mail
|
||
raw_header = fetch_body(data)
|
||
if status != "OK" or raw_header is None:
|
||
mislukt += 1
|
||
log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
|
||
continue
|
||
if not isinstance(raw_header, bytes):
|
||
mislukt += 1
|
||
log_failure(
|
||
log,
|
||
"stap2",
|
||
bron,
|
||
uid,
|
||
None,
|
||
"fetch_header_type",
|
||
status,
|
||
f"expected bytes, got {type(raw_header).__name__}",
|
||
)
|
||
continue
|
||
|
||
msg = email.message_from_bytes(raw_header)
|
||
from_raw = decode_hdr(msg.get("From", ""))
|
||
subject = decode_hdr(msg.get("Subject", ""))
|
||
date_str = msg.get("Date", "")
|
||
|
||
doel = route_from_subject(from_raw, subject)
|
||
|
||
if doel == "__FACTUUR_DATE__":
|
||
doel = PREFIX + quarter_folder(date_str)
|
||
elif doel:
|
||
doel = PREFIX + doel
|
||
else:
|
||
geen_match += 1
|
||
continue
|
||
|
||
if bron == doel:
|
||
overgeslagen += 1
|
||
continue
|
||
|
||
gepland += 1
|
||
audit_by_source[bron] += 1
|
||
audit_by_destination[doel] += 1
|
||
if dry:
|
||
if not AUDIT:
|
||
print(f" [DRY {verwerkt}/{len(ids)}] {bron} → {doel} | {subject[:60]}")
|
||
else:
|
||
ok, failure = move_message(mail, uid, bron, doel, dry=False)
|
||
if ok:
|
||
verplaatst += 1
|
||
log.setdefault("stap2_verwerkt", []).append(log_key)
|
||
if verplaatst % 50 == 0:
|
||
mail.expunge()
|
||
_save_log(log)
|
||
else:
|
||
mislukt += 1
|
||
actie, status, response = failure
|
||
log_failure(log, "stap2", bron, uid, doel, actie, status, response)
|
||
|
||
if not dry:
|
||
mail.expunge()
|
||
_save_log(log)
|
||
|
||
totaal_verplaatst += verplaatst
|
||
totaal_gepland += gepland
|
||
totaal_geen_match += geen_match
|
||
totaal_overgeslagen += overgeslagen
|
||
totaal_mislukt += mislukt
|
||
if dry:
|
||
print(
|
||
f" Gepland: {gepland} | Geen match: {geen_match}"
|
||
f" | Overgeslagen: {overgeslagen} | Fouten: {mislukt}"
|
||
)
|
||
else:
|
||
print(
|
||
f" Gepland: {gepland} | Verplaatst: {verplaatst}"
|
||
f" | Geen match: {geen_match} | Overgeslagen: {overgeslagen}"
|
||
f" | Fouten: {mislukt}"
|
||
)
|
||
|
||
if dry:
|
||
print(
|
||
f" Totaal gepland: {totaal_gepland} | Totaal geen match: {totaal_geen_match}"
|
||
f" | Totaal overgeslagen: {totaal_overgeslagen} | Totaal fouten: {totaal_mislukt}"
|
||
)
|
||
if AUDIT:
|
||
print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination)
|
||
else:
|
||
print(
|
||
f" Totaal gepland: {totaal_gepland} | Totaal verplaatst: {totaal_verplaatst}"
|
||
f" | Totaal geen match: {totaal_geen_match} | Totaal overgeslagen: {totaal_overgeslagen}"
|
||
f" | Totaal fouten: {totaal_mislukt}"
|
||
)
|
||
|
||
|
||
# ── stap 3: facturen per kwartaal ────────────────────────────────────────────
|
||
|
||
def stap3_facturen(session: ImapSession, log: dict, dry: bool):
|
||
mail = session.mail
|
||
print("\n── Stap 3: facturen per kwartaal ──")
|
||
bron_mappen = [
|
||
"INBOX.Facturen - te verwerken",
|
||
"INBOX.Financieel.Facturen", # als ze er al in zitten
|
||
]
|
||
audit_by_source = Counter()
|
||
audit_by_destination = Counter()
|
||
|
||
for folder_index, bron in enumerate(bron_mappen, start=1):
|
||
status, data = safe_select(session, log, "stap3", bron, readonly=dry)
|
||
mail = session.mail
|
||
if status != "OK":
|
||
continue
|
||
|
||
n = int(data[0])
|
||
if n == 0:
|
||
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: leeg")
|
||
continue
|
||
|
||
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: {n} berichten")
|
||
status, data = safe_uid(
|
||
session, log, "stap3", bron, "", "SEARCH", "ALL", readonly=dry
|
||
)
|
||
mail = session.mail
|
||
if status != "OK" or not data or not data[0]:
|
||
continue
|
||
ids = [uid for uid in data[0].split() if uid]
|
||
verplaatst = 0
|
||
gepland = 0
|
||
mislukt = 0
|
||
|
||
for index, uid in enumerate(ids, start=1):
|
||
if AUDIT:
|
||
print_progress(bron, index, len(ids))
|
||
status, data = safe_uid(
|
||
session,
|
||
log,
|
||
"stap3",
|
||
bron,
|
||
uid,
|
||
"FETCH",
|
||
uid,
|
||
"(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])",
|
||
readonly=dry,
|
||
)
|
||
mail = session.mail
|
||
raw_header = fetch_body(data)
|
||
if status != "OK" or raw_header is None:
|
||
mislukt += 1
|
||
log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
|
||
continue
|
||
if not isinstance(raw_header, bytes):
|
||
mislukt += 1
|
||
log_failure(
|
||
log,
|
||
"stap3",
|
||
bron,
|
||
uid,
|
||
None,
|
||
"fetch_header_type",
|
||
status,
|
||
f"expected bytes, got {type(raw_header).__name__}",
|
||
)
|
||
continue
|
||
|
||
msg = email.message_from_bytes(raw_header)
|
||
date_str = msg.get("Date", "")
|
||
doel = PREFIX + quarter_folder(date_str)
|
||
gepland += 1
|
||
audit_by_source[bron] += 1
|
||
audit_by_destination[doel] += 1
|
||
|
||
if dry:
|
||
dt_kort = date_str[:16] if date_str else "?"
|
||
if not AUDIT:
|
||
print(f" [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
|
||
else:
|
||
ok, failure = move_message(mail, uid, bron, doel, dry=False)
|
||
if ok:
|
||
verplaatst += 1
|
||
else:
|
||
mislukt += 1
|
||
actie, status, response = failure
|
||
log_failure(log, "stap3", bron, uid, doel, actie, status, response)
|
||
|
||
if not dry:
|
||
mail.expunge()
|
||
print(f" Gepland: {gepland} | Verplaatst: {verplaatst} | Fouten: {mislukt}")
|
||
_save_log(log)
|
||
else:
|
||
print(f" Gepland: {gepland} | Fouten: {mislukt}")
|
||
|
||
if dry and AUDIT:
|
||
print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination)
|
||
|
||
|
||
# ── log ──────────────────────────────────────────────────────────────────────
|
||
|
||
def _save_log(log: dict):
|
||
LOG_FILE.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
def load_log() -> dict:
|
||
if LOG_FILE.exists():
|
||
try:
|
||
return json.loads(LOG_FILE.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
|
||
def account_name_from_args() -> str | None:
|
||
args = sys.argv[1:]
|
||
if "--account" not in args:
|
||
return None
|
||
|
||
idx = args.index("--account")
|
||
if idx + 1 >= len(args) or args[idx + 1].startswith("--"):
|
||
print("Gebruik: python3 verplaats_bestaand.py --account <naam> [--audit|--uitvoeren]")
|
||
sys.exit(1)
|
||
|
||
return args[idx + 1]
|
||
|
||
|
||
# ── main ──────────────────────────────────────────────────────────────────────
|
||
|
||
def run():
|
||
if AUDIT and not DRY_RUN:
|
||
print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.")
|
||
sys.exit(1)
|
||
|
||
if DRY_RUN:
|
||
print("DRY RUN — er wordt niets verplaatst.")
|
||
print("Voeg --uitvoeren toe om echt uit te voeren.\n")
|
||
if AUDIT:
|
||
print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n")
|
||
else:
|
||
print("LET OP: dit verplaatst berichten op de echte server.")
|
||
bevestig = input("Typ JA om door te gaan: ").strip()
|
||
if bevestig != "JA":
|
||
print("Afgebroken.")
|
||
sys.exit(0)
|
||
|
||
account_name = account_name_from_args()
|
||
cfg = config.load(account_name)
|
||
print(f"Account: {cfg.name} ({cfg.username})\n")
|
||
|
||
print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...")
|
||
session = ImapSession(cfg)
|
||
try:
|
||
session.connect()
|
||
except Exception as e:
|
||
print(f"Verbinding mislukt: {e}")
|
||
sys.exit(1)
|
||
|
||
log = load_log()
|
||
|
||
try:
|
||
stap1_mapverplaatsingen(session, log, DRY_RUN)
|
||
stap2_bronmappen_routing(session, log, DRY_RUN)
|
||
stap3_facturen(session, log, DRY_RUN)
|
||
finally:
|
||
_save_log(log)
|
||
try:
|
||
session.logout()
|
||
except Exception as e:
|
||
print(f"Afmelden mislukt: {e}")
|
||
|
||
print("\nKlaar.")
|
||
if DRY_RUN:
|
||
print("Draai met --uitvoeren om de wijzigingen door te voeren.")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|