mailcat
616 lines
20 KiB
Python
616 lines
20 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Herindeling van bestaande mail naar de nieuwe mappenstructuur.
|
||
|
||
Stap 1 – Vaste mapverplaatsingen:
|
||
Uitgeschakeld. Alle niet-uitgesloten mappen worden als bron behandeld.
|
||
|
||
Stap 2 – Per-bericht routing:
|
||
Op basis van afzenderdomein en onderwerpkeywords.
|
||
Bronmappen: alles behalve Sent, Drafts, Trash, Spam en
|
||
INBOX.Facturen - verwerkt.
|
||
|
||
Stap 3 – Facturen per kwartaal sorteren (Facturen - te verwerken).
|
||
|
||
Standaard: DRY RUN (laat zien wat er zou gebeuren, verplaatst niets).
|
||
Account kiezen: python3 verplaats_bestaand.py --account backup --audit
|
||
Echt uitvoeren: python3 verplaats_bestaand.py --account backup --uitvoeren
|
||
|
||
Log: verplaats_log.json
|
||
"""
|
||
|
||
import email
|
||
import email.header
|
||
import imaplib
|
||
import sys
|
||
import json
|
||
import config
|
||
import time
|
||
from pathlib import Path
|
||
from collections import Counter
|
||
from imap_utils import list_folders, quote_mailbox
|
||
from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject
|
||
|
||
|
||
LOG_FILE = Path(__file__).parent / "verplaats_log.json"
|
||
|
||
DRY_RUN = "--uitvoeren" not in sys.argv
|
||
AUDIT = "--audit" in sys.argv
|
||
PROGRESS_INTERVAL = 250
|
||
|
||
|
||
# Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw).
|
||
#
|
||
# Bewust leeg: oude mappen, archiefmappen en mailinglistmappen zijn bronnen.
|
||
# Alleen Sent, Drafts, Trash, Spam en INBOX.Facturen - verwerkt worden
|
||
# uitgesloten van routering.
|
||
MAP_RENAMES: list[tuple[str, str]] = [
|
||
]
|
||
|
||
|
||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
def decode_hdr(value) -> str:
|
||
if not value:
|
||
return ""
|
||
try:
|
||
parts = email.header.decode_header(value)
|
||
out = []
|
||
for raw, cs in parts:
|
||
if isinstance(raw, bytes):
|
||
out.append(raw.decode(cs or "utf-8", errors="replace"))
|
||
else:
|
||
out.append(str(raw))
|
||
return " ".join(out).strip()
|
||
except Exception:
|
||
return str(value or "")
|
||
|
||
|
||
def list_source_folders(mail) -> list[str]:
|
||
return [folder for folder in list_folders(mail) if is_source_folder(folder)]
|
||
|
||
|
||
class ImapSession:
|
||
"""IMAP connection wrapper with conservative reconnect support."""
|
||
|
||
def __init__(self, cfg):
|
||
self.cfg = cfg
|
||
self.mail = None
|
||
|
||
def connect(self):
|
||
self.mail = imaplib.IMAP4_SSL(self.cfg.imap_host, self.cfg.imap_port)
|
||
self.mail.login(self.cfg.username, self.cfg.password)
|
||
|
||
def reconnect(self):
|
||
if self.mail is not None:
|
||
try:
|
||
self.mail.logout()
|
||
except Exception:
|
||
pass
|
||
time.sleep(2)
|
||
self.connect()
|
||
|
||
def logout(self):
|
||
if self.mail is not None:
|
||
self.mail.logout()
|
||
|
||
|
||
def log_failure(
|
||
log: dict,
|
||
stap: str,
|
||
src: str,
|
||
uid: bytes | str,
|
||
dst: str | None,
|
||
actie: str,
|
||
status: str | None,
|
||
response,
|
||
):
|
||
uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid)
|
||
log.setdefault("fouten", []).append({
|
||
"stap": stap,
|
||
"bron": src,
|
||
"uid": uid_str,
|
||
"doel": dst,
|
||
"actie": actie,
|
||
"status": status,
|
||
"response": repr(response),
|
||
})
|
||
|
||
|
||
def print_audit(title: str, by_source: Counter, by_destination: Counter):
|
||
print(f"\n Audit: {title}")
|
||
print(" Per bronmap:")
|
||
for folder, count in by_source.most_common():
|
||
print(f" {count:5d} {folder}")
|
||
print(" Per doelmap:")
|
||
for folder, count in by_destination.most_common():
|
||
print(f" {count:5d} {folder}")
|
||
|
||
|
||
def print_progress(folder: str, processed: int, total: int):
|
||
if processed == total or processed % PROGRESS_INTERVAL == 0:
|
||
print(f" Gescand: {processed}/{total} | {folder}")
|
||
|
||
|
||
def fetch_body(data) -> bytes | None:
|
||
"""Return the first bytes payload from an IMAP FETCH response."""
|
||
if not data:
|
||
return None
|
||
for item in data:
|
||
if isinstance(item, tuple) and len(item) >= 2 and isinstance(item[1], bytes):
|
||
return item[1]
|
||
return None
|
||
|
||
|
||
def safe_select(session: ImapSession, log: dict, stap: str, folder: str, readonly: bool):
|
||
for attempt in range(2):
|
||
try:
|
||
return session.mail.select(quote_mailbox(folder), readonly=readonly)
|
||
except imaplib.IMAP4.abort as e:
|
||
log_failure(log, stap, folder, "", None, "select_abort", "ABORT", str(e))
|
||
if attempt == 1:
|
||
return "ABORT", [str(e).encode()]
|
||
print(f" IMAP-verbinding verbroken bij openen van {folder}; opnieuw verbinden ...")
|
||
try:
|
||
session.reconnect()
|
||
except Exception as reconnect_error:
|
||
log_failure(
|
||
log,
|
||
stap,
|
||
folder,
|
||
"",
|
||
None,
|
||
"reconnect_failed",
|
||
"ERROR",
|
||
str(reconnect_error),
|
||
)
|
||
return "ABORT", [str(reconnect_error).encode()]
|
||
|
||
|
||
def safe_uid(
|
||
session: ImapSession,
|
||
log: dict,
|
||
stap: str,
|
||
folder: str,
|
||
uid: bytes | str,
|
||
command: str,
|
||
*args,
|
||
readonly: bool,
|
||
):
|
||
for attempt in range(2):
|
||
try:
|
||
return session.mail.uid(command, *args)
|
||
except imaplib.IMAP4.abort as e:
|
||
log_failure(
|
||
log,
|
||
stap,
|
||
folder,
|
||
uid,
|
||
None,
|
||
f"uid_{command.lower()}_abort",
|
||
"ABORT",
|
||
str(e),
|
||
)
|
||
if attempt == 1:
|
||
return "ABORT", [str(e).encode()]
|
||
print(f" IMAP-verbinding verbroken in {folder}; opnieuw verbinden en opnieuw proberen ...")
|
||
try:
|
||
session.reconnect()
|
||
except Exception as reconnect_error:
|
||
log_failure(
|
||
log,
|
||
stap,
|
||
folder,
|
||
uid,
|
||
None,
|
||
"reconnect_failed",
|
||
"ERROR",
|
||
str(reconnect_error),
|
||
)
|
||
return "ABORT", [str(reconnect_error).encode()]
|
||
status, data = safe_select(session, log, stap, folder, readonly=readonly)
|
||
if status != "OK":
|
||
return status, data
|
||
|
||
|
||
def safe_list_source_folders(session: ImapSession, log: dict) -> list[str]:
|
||
for attempt in range(2):
|
||
try:
|
||
return list_source_folders(session.mail)
|
||
except imaplib.IMAP4.abort as e:
|
||
log_failure(log, "stap2", "", "", None, "list_abort", "ABORT", str(e))
|
||
if attempt == 1:
|
||
return []
|
||
print(" IMAP-verbinding verbroken bij ophalen van mappen; opnieuw verbinden ...")
|
||
try:
|
||
session.reconnect()
|
||
except Exception as reconnect_error:
|
||
log_failure(
|
||
log,
|
||
"stap2",
|
||
"",
|
||
"",
|
||
None,
|
||
"reconnect_failed",
|
||
"ERROR",
|
||
str(reconnect_error),
|
||
)
|
||
return []
|
||
|
||
|
||
def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
|
||
"""Kopieer bericht naar dst en markeer als verwijderd in src."""
|
||
if dry:
|
||
return True, None
|
||
try:
|
||
select_res, select_data = mail.select(quote_mailbox(src))
|
||
if select_res != "OK":
|
||
return False, ("select_source", select_res, select_data)
|
||
res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full))
|
||
if res != "OK":
|
||
return False, ("copy", res, copy_data)
|
||
store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
|
||
if store_res != "OK":
|
||
return False, ("store_deleted", store_res, store_data)
|
||
return True, None
|
||
except Exception as e:
|
||
print(f" FOUT bij verplaatsen: {e}")
|
||
return False, ("exception", None, str(e))
|
||
|
||
|
||
# ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────
|
||
|
||
def stap1_mapverplaatsingen(session: ImapSession, log: dict, dry: bool):
|
||
mail = session.mail
|
||
print("\n── Stap 1: vaste mapverplaatsingen ──")
|
||
for oud, nieuw in MAP_RENAMES:
|
||
if nieuw == "__FACTUREN__":
|
||
continue # wordt in stap 3 verwerkt
|
||
|
||
if oud in log.get("stap1_klaar", []):
|
||
print(f" ✓ al gedaan: {oud}")
|
||
continue
|
||
|
||
# Controleer of bronmap bestaat
|
||
status, data = safe_select(session, log, "stap1", oud, readonly=True)
|
||
if status != "OK":
|
||
print(f" – bestaat niet: {oud}")
|
||
continue
|
||
|
||
n = int(data[0])
|
||
nieuw_full = nieuw if nieuw.startswith("INBOX.") else PREFIX + nieuw
|
||
|
||
if dry:
|
||
print(f" [DRY] {oud} → {nieuw} ({n} berichten)")
|
||
continue
|
||
|
||
# Berichten kopiëren
|
||
status, data = mail.search(None, "ALL")
|
||
ids = data[0].split()
|
||
gekopieerd = 0
|
||
for uid in ids:
|
||
res, _ = mail.copy(uid.decode(), quote_mailbox(nieuw_full))
|
||
if res == "OK":
|
||
mail.store(uid.decode(), "+FLAGS", "\\Deleted")
|
||
gekopieerd += 1
|
||
|
||
mail.expunge()
|
||
print(f" ✓ {oud} → {nieuw} ({gekopieerd}/{n})")
|
||
|
||
log.setdefault("stap1_klaar", []).append(oud)
|
||
_save_log(log)
|
||
|
||
|
||
# ── stap 2: per-bericht routing (INBOX) ─────────────────────────────────────
|
||
|
||
def stap2_bronmappen_routing(session: ImapSession, log: dict, dry: bool):
|
||
mail = session.mail
|
||
print("\n── Stap 2: bronmappen per-bericht routing ──")
|
||
bron_mappen = safe_list_source_folders(session, log)
|
||
print(f" {len(bron_mappen)} bronmappen")
|
||
|
||
audit_by_source = Counter()
|
||
audit_by_destination = Counter()
|
||
totaal_verplaatst = 0
|
||
totaal_gepland = 0
|
||
totaal_geen_match = 0
|
||
totaal_overgeslagen = 0
|
||
totaal_mislukt = 0
|
||
al_verwerkt = set(log.get("stap2_verwerkt", []))
|
||
|
||
for folder_index, bron in enumerate(bron_mappen, start=1):
|
||
status, data = safe_select(session, log, "stap2", bron, readonly=dry)
|
||
mail = session.mail
|
||
if status != "OK":
|
||
print(f" Kan map niet openen: {bron}")
|
||
continue
|
||
|
||
n = int(data[0])
|
||
if n == 0:
|
||
continue
|
||
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: {n} berichten")
|
||
|
||
status, data = safe_uid(
|
||
session, log, "stap2", bron, "", "SEARCH", "ALL", readonly=dry
|
||
)
|
||
mail = session.mail
|
||
if status != "OK" or not data or not data[0]:
|
||
continue
|
||
ids = [uid for uid in data[0].split() if uid]
|
||
|
||
verplaatst = 0
|
||
gepland = 0
|
||
geen_match = 0
|
||
overgeslagen = 0
|
||
mislukt = 0
|
||
verwerkt = 0
|
||
|
||
for uid in ids:
|
||
verwerkt += 1
|
||
if AUDIT:
|
||
print_progress(bron, verwerkt, len(ids))
|
||
uid_str = uid.decode()
|
||
log_key = f"{bron}:{uid_str}"
|
||
if log_key in al_verwerkt:
|
||
overgeslagen += 1
|
||
continue
|
||
|
||
status, data = safe_uid(
|
||
session,
|
||
log,
|
||
"stap2",
|
||
bron,
|
||
uid,
|
||
"FETCH",
|
||
uid,
|
||
"(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])",
|
||
readonly=dry,
|
||
)
|
||
mail = session.mail
|
||
raw_header = fetch_body(data)
|
||
if status != "OK" or raw_header is None:
|
||
mislukt += 1
|
||
log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
|
||
continue
|
||
|
||
msg = email.message_from_bytes(raw_header)
|
||
from_raw = decode_hdr(msg.get("From", ""))
|
||
subject = decode_hdr(msg.get("Subject", ""))
|
||
date_str = msg.get("Date", "")
|
||
|
||
doel = route_from_subject(from_raw, subject)
|
||
|
||
if doel == "__FACTUUR_DATE__":
|
||
doel = PREFIX + quarter_folder(date_str)
|
||
elif doel:
|
||
doel = PREFIX + doel
|
||
else:
|
||
geen_match += 1
|
||
continue
|
||
|
||
if bron == doel:
|
||
overgeslagen += 1
|
||
continue
|
||
|
||
gepland += 1
|
||
audit_by_source[bron] += 1
|
||
audit_by_destination[doel] += 1
|
||
if dry:
|
||
if not AUDIT:
|
||
print(f" [DRY {verwerkt}/{len(ids)}] {bron} → {doel} | {subject[:60]}")
|
||
else:
|
||
ok, failure = move_message(mail, uid, bron, doel, dry=False)
|
||
if ok:
|
||
verplaatst += 1
|
||
log.setdefault("stap2_verwerkt", []).append(log_key)
|
||
if verplaatst % 50 == 0:
|
||
mail.expunge()
|
||
_save_log(log)
|
||
else:
|
||
mislukt += 1
|
||
actie, status, response = failure
|
||
log_failure(log, "stap2", bron, uid, doel, actie, status, response)
|
||
|
||
if not dry:
|
||
mail.expunge()
|
||
_save_log(log)
|
||
|
||
totaal_verplaatst += verplaatst
|
||
totaal_gepland += gepland
|
||
totaal_geen_match += geen_match
|
||
totaal_overgeslagen += overgeslagen
|
||
totaal_mislukt += mislukt
|
||
if dry:
|
||
print(
|
||
f" Gepland: {gepland} | Geen match: {geen_match}"
|
||
f" | Overgeslagen: {overgeslagen} | Fouten: {mislukt}"
|
||
)
|
||
else:
|
||
print(
|
||
f" Gepland: {gepland} | Verplaatst: {verplaatst}"
|
||
f" | Geen match: {geen_match} | Overgeslagen: {overgeslagen}"
|
||
f" | Fouten: {mislukt}"
|
||
)
|
||
|
||
if dry:
|
||
print(
|
||
f" Totaal gepland: {totaal_gepland} | Totaal geen match: {totaal_geen_match}"
|
||
f" | Totaal overgeslagen: {totaal_overgeslagen} | Totaal fouten: {totaal_mislukt}"
|
||
)
|
||
if AUDIT:
|
||
print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination)
|
||
else:
|
||
print(
|
||
f" Totaal gepland: {totaal_gepland} | Totaal verplaatst: {totaal_verplaatst}"
|
||
f" | Totaal geen match: {totaal_geen_match} | Totaal overgeslagen: {totaal_overgeslagen}"
|
||
f" | Totaal fouten: {totaal_mislukt}"
|
||
)
|
||
|
||
|
||
# ── stap 3: facturen per kwartaal ────────────────────────────────────────────
|
||
|
||
def stap3_facturen(session: ImapSession, log: dict, dry: bool):
|
||
mail = session.mail
|
||
print("\n── Stap 3: facturen per kwartaal ──")
|
||
bron_mappen = [
|
||
"INBOX.Facturen - te verwerken",
|
||
"INBOX.Financieel.Facturen", # als ze er al in zitten
|
||
]
|
||
audit_by_source = Counter()
|
||
audit_by_destination = Counter()
|
||
|
||
for folder_index, bron in enumerate(bron_mappen, start=1):
|
||
status, data = safe_select(session, log, "stap3", bron, readonly=dry)
|
||
mail = session.mail
|
||
if status != "OK":
|
||
continue
|
||
|
||
n = int(data[0])
|
||
if n == 0:
|
||
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: leeg")
|
||
continue
|
||
|
||
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: {n} berichten")
|
||
status, data = safe_uid(
|
||
session, log, "stap3", bron, "", "SEARCH", "ALL", readonly=dry
|
||
)
|
||
mail = session.mail
|
||
if status != "OK" or not data or not data[0]:
|
||
continue
|
||
ids = [uid for uid in data[0].split() if uid]
|
||
verplaatst = 0
|
||
gepland = 0
|
||
mislukt = 0
|
||
|
||
for index, uid in enumerate(ids, start=1):
|
||
if AUDIT:
|
||
print_progress(bron, index, len(ids))
|
||
status, data = safe_uid(
|
||
session,
|
||
log,
|
||
"stap3",
|
||
bron,
|
||
uid,
|
||
"FETCH",
|
||
uid,
|
||
"(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])",
|
||
readonly=dry,
|
||
)
|
||
mail = session.mail
|
||
raw_header = fetch_body(data)
|
||
if status != "OK" or raw_header is None:
|
||
mislukt += 1
|
||
log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
|
||
continue
|
||
|
||
msg = email.message_from_bytes(raw_header)
|
||
date_str = msg.get("Date", "")
|
||
doel = PREFIX + quarter_folder(date_str)
|
||
gepland += 1
|
||
audit_by_source[bron] += 1
|
||
audit_by_destination[doel] += 1
|
||
|
||
if dry:
|
||
dt_kort = date_str[:16] if date_str else "?"
|
||
if not AUDIT:
|
||
print(f" [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
|
||
else:
|
||
ok, failure = move_message(mail, uid, bron, doel, dry=False)
|
||
if ok:
|
||
verplaatst += 1
|
||
else:
|
||
mislukt += 1
|
||
actie, status, response = failure
|
||
log_failure(log, "stap3", bron, uid, doel, actie, status, response)
|
||
|
||
if not dry:
|
||
mail.expunge()
|
||
print(f" Gepland: {gepland} | Verplaatst: {verplaatst} | Fouten: {mislukt}")
|
||
_save_log(log)
|
||
else:
|
||
print(f" Gepland: {gepland} | Fouten: {mislukt}")
|
||
|
||
if dry and AUDIT:
|
||
print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination)
|
||
|
||
|
||
# ── log ──────────────────────────────────────────────────────────────────────
|
||
|
||
def _save_log(log: dict):
|
||
LOG_FILE.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
def load_log() -> dict:
|
||
if LOG_FILE.exists():
|
||
try:
|
||
return json.loads(LOG_FILE.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
|
||
def account_name_from_args() -> str | None:
|
||
args = sys.argv[1:]
|
||
if "--account" not in args:
|
||
return None
|
||
|
||
idx = args.index("--account")
|
||
if idx + 1 >= len(args) or args[idx + 1].startswith("--"):
|
||
print("Gebruik: python3 verplaats_bestaand.py --account <naam> [--audit|--uitvoeren]")
|
||
sys.exit(1)
|
||
|
||
return args[idx + 1]
|
||
|
||
|
||
# ── main ──────────────────────────────────────────────────────────────────────
|
||
|
||
def run():
|
||
if AUDIT and not DRY_RUN:
|
||
print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.")
|
||
sys.exit(1)
|
||
|
||
if DRY_RUN:
|
||
print("DRY RUN — er wordt niets verplaatst.")
|
||
print("Voeg --uitvoeren toe om echt uit te voeren.\n")
|
||
if AUDIT:
|
||
print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n")
|
||
else:
|
||
print("LET OP: dit verplaatst berichten op de echte server.")
|
||
bevestig = input("Typ JA om door te gaan: ").strip()
|
||
if bevestig != "JA":
|
||
print("Afgebroken.")
|
||
sys.exit(0)
|
||
|
||
account_name = account_name_from_args()
|
||
cfg = config.load(account_name)
|
||
print(f"Account: {cfg.name} ({cfg.username})\n")
|
||
|
||
print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...")
|
||
session = ImapSession(cfg)
|
||
try:
|
||
session.connect()
|
||
except Exception as e:
|
||
print(f"Verbinding mislukt: {e}")
|
||
sys.exit(1)
|
||
|
||
log = load_log()
|
||
|
||
try:
|
||
stap1_mapverplaatsingen(session, log, DRY_RUN)
|
||
stap2_bronmappen_routing(session, log, DRY_RUN)
|
||
stap3_facturen(session, log, DRY_RUN)
|
||
finally:
|
||
_save_log(log)
|
||
try:
|
||
session.logout()
|
||
except Exception as e:
|
||
print(f"Afmelden mislukt: {e}")
|
||
|
||
print("\nKlaar.")
|
||
if DRY_RUN:
|
||
print("Draai met --uitvoeren om de wijzigingen door te voeren.")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|