wienen
f44c363025
The prompts used to arrive at this change since the previous commit Read restart_prompt.md first, then continue from there. Any special observations that may be relevant for version management for this version. Be brief. No mailbox-affecting scripts were run by Codex. A generated dry-run transcript appeared locally as verplaats_bestaand.log and is now ignored via *.log.
418 lines
14 KiB
Python
418 lines
14 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
Herindeling van bestaande mail naar de nieuwe mappenstructuur.
|
||
|
||
Stap 1 – Vaste mapverplaatsingen:
|
||
Uitgeschakeld. Alle niet-uitgesloten mappen worden als bron behandeld.
|
||
|
||
Stap 2 – Per-bericht routing:
|
||
Op basis van afzenderdomein en onderwerpkeywords.
|
||
Bronmappen: alles behalve Sent, Drafts, Trash, Spam en
|
||
INBOX.Facturen - verwerkt.
|
||
|
||
Stap 3 – Facturen per kwartaal sorteren (Facturen - te verwerken).
|
||
|
||
Standaard: DRY RUN (laat zien wat er zou gebeuren, verplaatst niets).
|
||
Echt uitvoeren: python3 verplaats_bestaand.py --uitvoeren
|
||
|
||
Log: verplaats_log.json
|
||
"""
|
||
|
||
import email
|
||
import email.header
|
||
import imaplib
|
||
import sys
|
||
import json
|
||
import config
|
||
from pathlib import Path
|
||
from collections import Counter
|
||
from imap_utils import list_folders, quote_mailbox
|
||
from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject
|
||
|
||
|
||
LOG_FILE = Path(__file__).parent / "verplaats_log.json"
|
||
|
||
DRY_RUN = "--uitvoeren" not in sys.argv
|
||
AUDIT = "--audit" in sys.argv
|
||
|
||
|
||
# Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw).
|
||
#
|
||
# Bewust leeg: oude mappen, archiefmappen en mailinglistmappen zijn bronnen.
|
||
# Alleen Sent, Drafts, Trash, Spam en INBOX.Facturen - verwerkt worden
|
||
# uitgesloten van routering.
|
||
MAP_RENAMES: list[tuple[str, str]] = [
|
||
]
|
||
|
||
|
||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
def decode_hdr(value) -> str:
|
||
if not value:
|
||
return ""
|
||
try:
|
||
parts = email.header.decode_header(value)
|
||
out = []
|
||
for raw, cs in parts:
|
||
if isinstance(raw, bytes):
|
||
out.append(raw.decode(cs or "utf-8", errors="replace"))
|
||
else:
|
||
out.append(str(raw))
|
||
return " ".join(out).strip()
|
||
except Exception:
|
||
return str(value or "")
|
||
|
||
|
||
def list_source_folders(mail) -> list[str]:
|
||
return [folder for folder in list_folders(mail) if is_source_folder(folder)]
|
||
|
||
|
||
def log_failure(
|
||
log: dict,
|
||
stap: str,
|
||
src: str,
|
||
uid: bytes | str,
|
||
dst: str | None,
|
||
actie: str,
|
||
status: str | None,
|
||
response,
|
||
):
|
||
uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid)
|
||
log.setdefault("fouten", []).append({
|
||
"stap": stap,
|
||
"bron": src,
|
||
"uid": uid_str,
|
||
"doel": dst,
|
||
"actie": actie,
|
||
"status": status,
|
||
"response": repr(response),
|
||
})
|
||
|
||
|
||
def print_audit(title: str, by_source: Counter, by_destination: Counter):
|
||
print(f"\n Audit: {title}")
|
||
print(" Per bronmap:")
|
||
for folder, count in by_source.most_common():
|
||
print(f" {count:5d} {folder}")
|
||
print(" Per doelmap:")
|
||
for folder, count in by_destination.most_common():
|
||
print(f" {count:5d} {folder}")
|
||
|
||
|
||
def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
|
||
"""Kopieer bericht naar dst en markeer als verwijderd in src."""
|
||
if dry:
|
||
return True, None
|
||
try:
|
||
select_res, select_data = mail.select(quote_mailbox(src))
|
||
if select_res != "OK":
|
||
return False, ("select_source", select_res, select_data)
|
||
res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full))
|
||
if res != "OK":
|
||
return False, ("copy", res, copy_data)
|
||
store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
|
||
if store_res != "OK":
|
||
return False, ("store_deleted", store_res, store_data)
|
||
return True, None
|
||
except Exception as e:
|
||
print(f" FOUT bij verplaatsen: {e}")
|
||
return False, ("exception", None, str(e))
|
||
|
||
|
||
# ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────
|
||
|
||
def stap1_mapverplaatsingen(mail, log: dict, dry: bool):
|
||
print("\n── Stap 1: vaste mapverplaatsingen ──")
|
||
for oud, nieuw in MAP_RENAMES:
|
||
if nieuw == "__FACTUREN__":
|
||
continue # wordt in stap 3 verwerkt
|
||
|
||
if oud in log.get("stap1_klaar", []):
|
||
print(f" ✓ al gedaan: {oud}")
|
||
continue
|
||
|
||
# Controleer of bronmap bestaat
|
||
status, data = mail.select(quote_mailbox(oud), readonly=True)
|
||
if status != "OK":
|
||
print(f" – bestaat niet: {oud}")
|
||
continue
|
||
|
||
n = int(data[0])
|
||
nieuw_full = nieuw if nieuw.startswith("INBOX.") else PREFIX + nieuw
|
||
|
||
if dry:
|
||
print(f" [DRY] {oud} → {nieuw} ({n} berichten)")
|
||
continue
|
||
|
||
# Berichten kopiëren
|
||
status, data = mail.search(None, "ALL")
|
||
ids = data[0].split()
|
||
gekopieerd = 0
|
||
for uid in ids:
|
||
res, _ = mail.copy(uid.decode(), quote_mailbox(nieuw_full))
|
||
if res == "OK":
|
||
mail.store(uid.decode(), "+FLAGS", "\\Deleted")
|
||
gekopieerd += 1
|
||
|
||
mail.expunge()
|
||
print(f" ✓ {oud} → {nieuw} ({gekopieerd}/{n})")
|
||
|
||
log.setdefault("stap1_klaar", []).append(oud)
|
||
_save_log(log)
|
||
|
||
|
||
# ── stap 2: per-bericht routing (INBOX) ─────────────────────────────────────
|
||
|
||
def stap2_bronmappen_routing(mail, log: dict, dry: bool):
|
||
print("\n── Stap 2: bronmappen per-bericht routing ──")
|
||
bron_mappen = list_source_folders(mail)
|
||
print(f" {len(bron_mappen)} bronmappen")
|
||
|
||
audit_by_source = Counter()
|
||
audit_by_destination = Counter()
|
||
totaal_verplaatst = 0
|
||
totaal_gepland = 0
|
||
totaal_geen_match = 0
|
||
totaal_overgeslagen = 0
|
||
totaal_mislukt = 0
|
||
al_verwerkt = set(log.get("stap2_verwerkt", []))
|
||
|
||
for bron in bron_mappen:
|
||
status, data = mail.select(quote_mailbox(bron), readonly=dry)
|
||
if status != "OK":
|
||
print(f" Kan map niet openen: {bron}")
|
||
continue
|
||
|
||
n = int(data[0])
|
||
if n == 0:
|
||
continue
|
||
print(f" {bron}: {n} berichten")
|
||
|
||
status, data = mail.uid("SEARCH", "ALL")
|
||
if status != "OK" or not data or not data[0]:
|
||
continue
|
||
ids = [uid for uid in data[0].split() if uid]
|
||
|
||
verplaatst = 0
|
||
gepland = 0
|
||
geen_match = 0
|
||
overgeslagen = 0
|
||
mislukt = 0
|
||
verwerkt = 0
|
||
|
||
for uid in ids:
|
||
verwerkt += 1
|
||
uid_str = uid.decode()
|
||
log_key = f"{bron}:{uid_str}"
|
||
if log_key in al_verwerkt:
|
||
overgeslagen += 1
|
||
continue
|
||
|
||
status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])")
|
||
if status != "OK" or not data or not data[0]:
|
||
mislukt += 1
|
||
log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
|
||
continue
|
||
|
||
msg = email.message_from_bytes(data[0][1])
|
||
from_raw = decode_hdr(msg.get("From", ""))
|
||
subject = decode_hdr(msg.get("Subject", ""))
|
||
date_str = msg.get("Date", "")
|
||
|
||
doel = route_from_subject(from_raw, subject)
|
||
|
||
if doel == "__FACTUUR_DATE__":
|
||
doel = PREFIX + quarter_folder(date_str)
|
||
elif doel:
|
||
doel = PREFIX + doel
|
||
else:
|
||
geen_match += 1
|
||
continue
|
||
|
||
if bron == doel:
|
||
overgeslagen += 1
|
||
continue
|
||
|
||
gepland += 1
|
||
audit_by_source[bron] += 1
|
||
audit_by_destination[doel] += 1
|
||
if dry:
|
||
if not AUDIT:
|
||
print(f" [DRY {verwerkt}/{len(ids)}] {bron} → {doel} | {subject[:60]}")
|
||
else:
|
||
ok, failure = move_message(mail, uid, bron, doel, dry=False)
|
||
if ok:
|
||
verplaatst += 1
|
||
log.setdefault("stap2_verwerkt", []).append(log_key)
|
||
if verplaatst % 50 == 0:
|
||
mail.expunge()
|
||
_save_log(log)
|
||
else:
|
||
mislukt += 1
|
||
actie, status, response = failure
|
||
log_failure(log, "stap2", bron, uid, doel, actie, status, response)
|
||
|
||
if not dry:
|
||
mail.expunge()
|
||
_save_log(log)
|
||
|
||
totaal_verplaatst += verplaatst
|
||
totaal_gepland += gepland
|
||
totaal_geen_match += geen_match
|
||
totaal_overgeslagen += overgeslagen
|
||
totaal_mislukt += mislukt
|
||
if dry:
|
||
print(
|
||
f" Gepland: {gepland} | Geen match: {geen_match}"
|
||
f" | Overgeslagen: {overgeslagen} | Fouten: {mislukt}"
|
||
)
|
||
else:
|
||
print(
|
||
f" Gepland: {gepland} | Verplaatst: {verplaatst}"
|
||
f" | Geen match: {geen_match} | Overgeslagen: {overgeslagen}"
|
||
f" | Fouten: {mislukt}"
|
||
)
|
||
|
||
if dry:
|
||
print(
|
||
f" Totaal gepland: {totaal_gepland} | Totaal geen match: {totaal_geen_match}"
|
||
f" | Totaal overgeslagen: {totaal_overgeslagen} | Totaal fouten: {totaal_mislukt}"
|
||
)
|
||
if AUDIT:
|
||
print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination)
|
||
else:
|
||
print(
|
||
f" Totaal gepland: {totaal_gepland} | Totaal verplaatst: {totaal_verplaatst}"
|
||
f" | Totaal geen match: {totaal_geen_match} | Totaal overgeslagen: {totaal_overgeslagen}"
|
||
f" | Totaal fouten: {totaal_mislukt}"
|
||
)
|
||
|
||
|
||
# ── stap 3: facturen per kwartaal ────────────────────────────────────────────
|
||
|
||
def stap3_facturen(mail, log: dict, dry: bool):
|
||
print("\n── Stap 3: facturen per kwartaal ──")
|
||
bron_mappen = [
|
||
"INBOX.Facturen - te verwerken",
|
||
"INBOX.Financieel.Facturen", # als ze er al in zitten
|
||
]
|
||
audit_by_source = Counter()
|
||
audit_by_destination = Counter()
|
||
|
||
for bron in bron_mappen:
|
||
status, data = mail.select(quote_mailbox(bron), readonly=dry)
|
||
if status != "OK":
|
||
continue
|
||
|
||
n = int(data[0])
|
||
if n == 0:
|
||
print(f" {bron}: leeg")
|
||
continue
|
||
|
||
print(f" {bron}: {n} berichten")
|
||
status, data = mail.uid("SEARCH", "ALL")
|
||
if status != "OK" or not data or not data[0]:
|
||
continue
|
||
ids = [uid for uid in data[0].split() if uid]
|
||
verplaatst = 0
|
||
gepland = 0
|
||
mislukt = 0
|
||
|
||
for index, uid in enumerate(ids, start=1):
|
||
status, data = mail.uid("FETCH", uid, "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])")
|
||
if status != "OK" or not data or not data[0]:
|
||
mislukt += 1
|
||
log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
|
||
continue
|
||
|
||
msg = email.message_from_bytes(data[0][1])
|
||
date_str = msg.get("Date", "")
|
||
doel = PREFIX + quarter_folder(date_str)
|
||
gepland += 1
|
||
audit_by_source[bron] += 1
|
||
audit_by_destination[doel] += 1
|
||
|
||
if dry:
|
||
dt_kort = date_str[:16] if date_str else "?"
|
||
if not AUDIT:
|
||
print(f" [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
|
||
else:
|
||
ok, failure = move_message(mail, uid, bron, doel, dry=False)
|
||
if ok:
|
||
verplaatst += 1
|
||
else:
|
||
mislukt += 1
|
||
actie, status, response = failure
|
||
log_failure(log, "stap3", bron, uid, doel, actie, status, response)
|
||
|
||
if not dry:
|
||
mail.expunge()
|
||
print(f" Gepland: {gepland} | Verplaatst: {verplaatst} | Fouten: {mislukt}")
|
||
_save_log(log)
|
||
else:
|
||
print(f" Gepland: {gepland} | Fouten: {mislukt}")
|
||
|
||
if dry and AUDIT:
|
||
print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination)
|
||
|
||
|
||
# ── log ──────────────────────────────────────────────────────────────────────
|
||
|
||
def _save_log(log: dict):
|
||
LOG_FILE.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
||
|
||
|
||
def load_log() -> dict:
|
||
if LOG_FILE.exists():
|
||
try:
|
||
return json.loads(LOG_FILE.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
pass
|
||
return {}
|
||
|
||
|
||
# ── main ──────────────────────────────────────────────────────────────────────
|
||
|
||
def run():
|
||
if AUDIT and not DRY_RUN:
|
||
print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.")
|
||
sys.exit(1)
|
||
|
||
if DRY_RUN:
|
||
print("DRY RUN — er wordt niets verplaatst.")
|
||
print("Voeg --uitvoeren toe om echt uit te voeren.\n")
|
||
if AUDIT:
|
||
print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n")
|
||
else:
|
||
print("LET OP: dit verplaatst berichten op de echte server.")
|
||
bevestig = input("Typ JA om door te gaan: ").strip()
|
||
if bevestig != "JA":
|
||
print("Afgebroken.")
|
||
sys.exit(0)
|
||
|
||
cfg = config.load()
|
||
print(f"Account: {cfg.username}\n")
|
||
|
||
print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...")
|
||
try:
|
||
mail = imaplib.IMAP4_SSL(cfg.imap_host, cfg.imap_port)
|
||
mail.login(cfg.username, cfg.password)
|
||
except Exception as e:
|
||
print(f"Verbinding mislukt: {e}")
|
||
sys.exit(1)
|
||
|
||
log = load_log()
|
||
|
||
stap1_mapverplaatsingen(mail, log, DRY_RUN)
|
||
stap2_bronmappen_routing(mail, log, DRY_RUN)
|
||
stap3_facturen(mail, log, DRY_RUN)
|
||
|
||
mail.logout()
|
||
print("\nKlaar.")
|
||
if DRY_RUN:
|
||
print("Draai met --uitvoeren om de wijzigingen door te voeren.")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
run()
|