mailcat/verplaats_bestaand.py

#!/usr/bin/env python3
"""
Herindeling van bestaande mail naar de nieuwe mappenstructuur.

Stap 1 – Vaste mapverplaatsingen:
         Uitgeschakeld. Alle niet-uitgesloten mappen worden als bron behandeld.

Stap 2 – Per-bericht routing:
         Op basis van afzenderdomein en onderwerpkeywords.
         Bronmappen: alles behalve Sent, Drafts, Trash, Spam en
         INBOX.Facturen - verwerkt.

Stap 3 – Facturen per kwartaal sorteren (Facturen - te verwerken).

Standaard: DRY RUN (laat zien wat er zou gebeuren, verplaatst niets).
Account kiezen: python3 verplaats_bestaand.py --account backup --audit
Echt uitvoeren: python3 verplaats_bestaand.py --account backup --uitvoeren

Log: verplaats_log.json
"""

import email
import email.header
import imaplib
import sys
import json
import config
import time
from pathlib import Path
from collections import Counter
from imap_utils import list_folders, quote_mailbox
from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject


LOG_FILE  = Path(__file__).parent / "verplaats_log.json"

DRY_RUN   = "--uitvoeren" not in sys.argv
AUDIT     = "--audit" in sys.argv


# Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw).
#
# Bewust leeg: oude mappen, archiefmappen en mailinglistmappen zijn bronnen.
# Alleen Sent, Drafts, Trash, Spam en INBOX.Facturen - verwerkt worden
# uitgesloten van routering.
MAP_RENAMES: list[tuple[str, str]] = [
]


# ── helpers ──────────────────────────────────────────────────────────────────

def decode_hdr(value) -> str:
    if not value:
        return ""
    try:
        parts = email.header.decode_header(value)
        out = []
        for raw, cs in parts:
            if isinstance(raw, bytes):
                out.append(raw.decode(cs or "utf-8", errors="replace"))
            else:
                out.append(str(raw))
        return " ".join(out).strip()
    except Exception:
        return str(value or "")


def list_source_folders(mail) -> list[str]:
    return [folder for folder in list_folders(mail) if is_source_folder(folder)]


class ImapSession:
    """IMAP connection wrapper with conservative reconnect support."""

    def __init__(self, cfg):
        self.cfg = cfg
        self.mail = None

    def connect(self):
        self.mail = imaplib.IMAP4_SSL(self.cfg.imap_host, self.cfg.imap_port)
        self.mail.login(self.cfg.username, self.cfg.password)

    def reconnect(self):
        if self.mail is not None:
            try:
                self.mail.logout()
            except Exception:
                pass
        time.sleep(2)
        self.connect()

    def logout(self):
        if self.mail is not None:
            self.mail.logout()


def log_failure(
    log: dict,
    stap: str,
    src: str,
    uid: bytes | str,
    dst: str | None,
    actie: str,
    status: str | None,
    response,
):
    uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid)
    log.setdefault("fouten", []).append({
        "stap": stap,
        "bron": src,
        "uid": uid_str,
        "doel": dst,
        "actie": actie,
        "status": status,
        "response": repr(response),
    })


def print_audit(title: str, by_source: Counter, by_destination: Counter):
    print(f"\n  Audit: {title}")
    print("  Per bronmap:")
    for folder, count in by_source.most_common():
        print(f"    {count:5d}  {folder}")
    print("  Per doelmap:")
    for folder, count in by_destination.most_common():
        print(f"    {count:5d}  {folder}")


def safe_select(session: ImapSession, log: dict, stap: str, folder: str, readonly: bool):
    for attempt in range(2):
        try:
            return session.mail.select(quote_mailbox(folder), readonly=readonly)
        except imaplib.IMAP4.abort as e:
            log_failure(log, stap, folder, "", None, "select_abort", "ABORT", str(e))
            if attempt == 1:
                return "ABORT", [str(e).encode()]
            print(f"  IMAP-verbinding verbroken bij openen van {folder}; opnieuw verbinden ...")
            try:
                session.reconnect()
            except Exception as reconnect_error:
                log_failure(
                    log,
                    stap,
                    folder,
                    "",
                    None,
                    "reconnect_failed",
                    "ERROR",
                    str(reconnect_error),
                )
                return "ABORT", [str(reconnect_error).encode()]


def safe_uid(
    session: ImapSession,
    log: dict,
    stap: str,
    folder: str,
    uid: bytes | str,
    command: str,
    *args,
    readonly: bool,
):
    for attempt in range(2):
        try:
            return session.mail.uid(command, *args)
        except imaplib.IMAP4.abort as e:
            log_failure(
                log,
                stap,
                folder,
                uid,
                None,
                f"uid_{command.lower()}_abort",
                "ABORT",
                str(e),
            )
            if attempt == 1:
                return "ABORT", [str(e).encode()]
            print(f"  IMAP-verbinding verbroken in {folder}; opnieuw verbinden en opnieuw proberen ...")
            try:
                session.reconnect()
            except Exception as reconnect_error:
                log_failure(
                    log,
                    stap,
                    folder,
                    uid,
                    None,
                    "reconnect_failed",
                    "ERROR",
                    str(reconnect_error),
                )
                return "ABORT", [str(reconnect_error).encode()]
            status, data = safe_select(session, log, stap, folder, readonly=readonly)
            if status != "OK":
                return status, data


def safe_list_source_folders(session: ImapSession, log: dict) -> list[str]:
    for attempt in range(2):
        try:
            return list_source_folders(session.mail)
        except imaplib.IMAP4.abort as e:
            log_failure(log, "stap2", "", "", None, "list_abort", "ABORT", str(e))
            if attempt == 1:
                return []
            print("  IMAP-verbinding verbroken bij ophalen van mappen; opnieuw verbinden ...")
            try:
                session.reconnect()
            except Exception as reconnect_error:
                log_failure(
                    log,
                    "stap2",
                    "",
                    "",
                    None,
                    "reconnect_failed",
                    "ERROR",
                    str(reconnect_error),
                )
                return []


def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
    """Kopieer bericht naar dst en markeer als verwijderd in src."""
    if dry:
        return True, None
    try:
        select_res, select_data = mail.select(quote_mailbox(src))
        if select_res != "OK":
            return False, ("select_source", select_res, select_data)
        res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full))
        if res != "OK":
            return False, ("copy", res, copy_data)
        store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
        if store_res != "OK":
            return False, ("store_deleted", store_res, store_data)
        return True, None
    except Exception as e:
        print(f"    FOUT bij verplaatsen: {e}")
        return False, ("exception", None, str(e))


# ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────

def stap1_mapverplaatsingen(session: ImapSession, log: dict, dry: bool):
    mail = session.mail
    print("\n── Stap 1: vaste mapverplaatsingen ──")
    for oud, nieuw in MAP_RENAMES:
        if nieuw == "__FACTUREN__":
            continue  # wordt in stap 3 verwerkt

        if oud in log.get("stap1_klaar", []):
            print(f"  ✓ al gedaan: {oud}")
            continue

        # Controleer of bronmap bestaat
        status, data = safe_select(session, log, "stap1", oud, readonly=True)
        if status != "OK":
            print(f"  – bestaat niet: {oud}")
            continue

        n = int(data[0])
        nieuw_full = nieuw if nieuw.startswith("INBOX.") else PREFIX + nieuw

        if dry:
            print(f"  [DRY] {oud} → {nieuw}  ({n} berichten)")
            continue

        # Berichten kopiëren
        status, data = mail.search(None, "ALL")
        ids = data[0].split()
        gekopieerd = 0
        for uid in ids:
            res, _ = mail.copy(uid.decode(), quote_mailbox(nieuw_full))
            if res == "OK":
                mail.store(uid.decode(), "+FLAGS", "\\Deleted")
                gekopieerd += 1

        mail.expunge()
        print(f"  ✓ {oud} → {nieuw}  ({gekopieerd}/{n})")

        log.setdefault("stap1_klaar", []).append(oud)
        _save_log(log)


# ── stap 2: per-bericht routing (INBOX) ─────────────────────────────────────

def stap2_bronmappen_routing(session: ImapSession, log: dict, dry: bool):
    mail = session.mail
    print("\n── Stap 2: bronmappen per-bericht routing ──")
    bron_mappen = safe_list_source_folders(session, log)
    print(f"  {len(bron_mappen)} bronmappen")

    audit_by_source = Counter()
    audit_by_destination = Counter()
    totaal_verplaatst = 0
    totaal_gepland = 0
    totaal_geen_match = 0
    totaal_overgeslagen = 0
    totaal_mislukt = 0
    al_verwerkt = set(log.get("stap2_verwerkt", []))

    for bron in bron_mappen:
        status, data = safe_select(session, log, "stap2", bron, readonly=dry)
        mail = session.mail
        if status != "OK":
            print(f"  Kan map niet openen: {bron}")
            continue

        n = int(data[0])
        if n == 0:
            continue
        print(f"  {bron}: {n} berichten")

        status, data = safe_uid(
            session, log, "stap2", bron, "", "SEARCH", "ALL", readonly=dry
        )
        mail = session.mail
        if status != "OK" or not data or not data[0]:
            continue
        ids = [uid for uid in data[0].split() if uid]

        verplaatst = 0
        gepland = 0
        geen_match = 0
        overgeslagen = 0
        mislukt = 0
        verwerkt = 0

        for uid in ids:
            verwerkt += 1
            uid_str = uid.decode()
            log_key = f"{bron}:{uid_str}"
            if log_key in al_verwerkt:
                overgeslagen += 1
                continue

            status, data = safe_uid(
                session,
                log,
                "stap2",
                bron,
                uid,
                "FETCH",
                uid,
                "(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])",
                readonly=dry,
            )
            mail = session.mail
            if status != "OK" or not data or not data[0]:
                mislukt += 1
                log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
                continue

            msg = email.message_from_bytes(data[0][1])
            from_raw = decode_hdr(msg.get("From", ""))
            subject  = decode_hdr(msg.get("Subject", ""))
            date_str = msg.get("Date", "")

            doel = route_from_subject(from_raw, subject)

            if doel == "__FACTUUR_DATE__":
                doel = PREFIX + quarter_folder(date_str)
            elif doel:
                doel = PREFIX + doel
            else:
                geen_match += 1
                continue

            if bron == doel:
                overgeslagen += 1
                continue

            gepland += 1
            audit_by_source[bron] += 1
            audit_by_destination[doel] += 1
            if dry:
                if not AUDIT:
                    print(f"    [DRY {verwerkt}/{len(ids)}] {bron} → {doel}  |  {subject[:60]}")
            else:
                ok, failure = move_message(mail, uid, bron, doel, dry=False)
                if ok:
                    verplaatst += 1
                    log.setdefault("stap2_verwerkt", []).append(log_key)
                    if verplaatst % 50 == 0:
                        mail.expunge()
                        _save_log(log)
                else:
                    mislukt += 1
                    actie, status, response = failure
                    log_failure(log, "stap2", bron, uid, doel, actie, status, response)

        if not dry:
            mail.expunge()
            _save_log(log)

        totaal_verplaatst += verplaatst
        totaal_gepland += gepland
        totaal_geen_match += geen_match
        totaal_overgeslagen += overgeslagen
        totaal_mislukt += mislukt
        if dry:
            print(
                f"    Gepland: {gepland}  |  Geen match: {geen_match}"
                f"  |  Overgeslagen: {overgeslagen}  |  Fouten: {mislukt}"
            )
        else:
            print(
                f"    Gepland: {gepland}  |  Verplaatst: {verplaatst}"
                f"  |  Geen match: {geen_match}  |  Overgeslagen: {overgeslagen}"
                f"  |  Fouten: {mislukt}"
            )

    if dry:
        print(
            f"  Totaal gepland: {totaal_gepland}  |  Totaal geen match: {totaal_geen_match}"
            f"  |  Totaal overgeslagen: {totaal_overgeslagen}  |  Totaal fouten: {totaal_mislukt}"
        )
        if AUDIT:
            print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination)
    else:
        print(
            f"  Totaal gepland: {totaal_gepland}  |  Totaal verplaatst: {totaal_verplaatst}"
            f"  |  Totaal geen match: {totaal_geen_match}  |  Totaal overgeslagen: {totaal_overgeslagen}"
            f"  |  Totaal fouten: {totaal_mislukt}"
        )


# ── stap 3: facturen per kwartaal ────────────────────────────────────────────

def stap3_facturen(session: ImapSession, log: dict, dry: bool):
    mail = session.mail
    print("\n── Stap 3: facturen per kwartaal ──")
    bron_mappen = [
        "INBOX.Facturen - te verwerken",
        "INBOX.Financieel.Facturen",   # als ze er al in zitten
    ]
    audit_by_source = Counter()
    audit_by_destination = Counter()

    for bron in bron_mappen:
        status, data = safe_select(session, log, "stap3", bron, readonly=dry)
        mail = session.mail
        if status != "OK":
            continue

        n = int(data[0])
        if n == 0:
            print(f"  {bron}: leeg")
            continue

        print(f"  {bron}: {n} berichten")
        status, data = safe_uid(
            session, log, "stap3", bron, "", "SEARCH", "ALL", readonly=dry
        )
        mail = session.mail
        if status != "OK" or not data or not data[0]:
            continue
        ids = [uid for uid in data[0].split() if uid]
        verplaatst = 0
        gepland = 0
        mislukt = 0

        for index, uid in enumerate(ids, start=1):
            status, data = safe_uid(
                session,
                log,
                "stap3",
                bron,
                uid,
                "FETCH",
                uid,
                "(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])",
                readonly=dry,
            )
            mail = session.mail
            if status != "OK" or not data or not data[0]:
                mislukt += 1
                log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
                continue

            msg      = email.message_from_bytes(data[0][1])
            date_str = msg.get("Date", "")
            doel     = PREFIX + quarter_folder(date_str)
            gepland += 1
            audit_by_source[bron] += 1
            audit_by_destination[doel] += 1

            if dry:
                dt_kort = date_str[:16] if date_str else "?"
                if not AUDIT:
                    print(f"    [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
            else:
                ok, failure = move_message(mail, uid, bron, doel, dry=False)
                if ok:
                    verplaatst += 1
                else:
                    mislukt += 1
                    actie, status, response = failure
                    log_failure(log, "stap3", bron, uid, doel, actie, status, response)

        if not dry:
            mail.expunge()
            print(f"    Gepland: {gepland}  |  Verplaatst: {verplaatst}  |  Fouten: {mislukt}")
            _save_log(log)
        else:
            print(f"    Gepland: {gepland}  |  Fouten: {mislukt}")

    if dry and AUDIT:
        print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination)


# ── log ──────────────────────────────────────────────────────────────────────

def _save_log(log: dict):
    LOG_FILE.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")


def load_log() -> dict:
    if LOG_FILE.exists():
        try:
            return json.loads(LOG_FILE.read_text(encoding="utf-8"))
        except Exception:
            pass
    return {}


def account_name_from_args() -> str | None:
    args = sys.argv[1:]
    if "--account" not in args:
        return None

    idx = args.index("--account")
    if idx + 1 >= len(args) or args[idx + 1].startswith("--"):
        print("Gebruik: python3 verplaats_bestaand.py --account <naam> [--audit|--uitvoeren]")
        sys.exit(1)

    return args[idx + 1]


# ── main ──────────────────────────────────────────────────────────────────────

def run():
    if AUDIT and not DRY_RUN:
        print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.")
        sys.exit(1)

    if DRY_RUN:
        print("DRY RUN — er wordt niets verplaatst.")
        print("Voeg --uitvoeren toe om echt uit te voeren.\n")
        if AUDIT:
            print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n")
    else:
        print("LET OP: dit verplaatst berichten op de echte server.")
        bevestig = input("Typ JA om door te gaan: ").strip()
        if bevestig != "JA":
            print("Afgebroken.")
            sys.exit(0)

    account_name = account_name_from_args()
    cfg = config.load(account_name)
    print(f"Account: {cfg.name} ({cfg.username})\n")

    print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...")
    session = ImapSession(cfg)
    try:
        session.connect()
    except Exception as e:
        print(f"Verbinding mislukt: {e}")
        sys.exit(1)

    log = load_log()

    try:
        stap1_mapverplaatsingen(session, log, DRY_RUN)
        stap2_bronmappen_routing(session, log, DRY_RUN)
        stap3_facturen(session, log, DRY_RUN)
    finally:
        _save_log(log)
        try:
            session.logout()
        except Exception as e:
            print(f"Afmelden mislukt: {e}")

    print("\nKlaar.")
    if DRY_RUN:
        print("Draai met --uitvoeren om de wijzigingen door te voeren.")


if __name__ == "__main__":
    run()