wienen/mailcat
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial status for developed files on dagda

Browse Source
This commit is contained in:
mailcat
2026-07-20 20:49:36 +02:00
commit 337378a362
102 changed files with 37778 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
verplaats_bestaand.py
+615
View File
@@ -0,0 +1,615 @@
#!/usr/bin/env python3
"""
Herindeling van bestaande mail naar de nieuwe mappenstructuur.
Stap 1 – Vaste mapverplaatsingen:
Uitgeschakeld. Alle niet-uitgesloten mappen worden als bron behandeld.
Stap 2 – Per-bericht routing:
Op basis van afzenderdomein en onderwerpkeywords.
Bronmappen: alles behalve Sent, Drafts, Trash, Spam en
INBOX.Facturen - verwerkt.
Stap 3 – Facturen per kwartaal sorteren (Facturen - te verwerken).
Standaard: DRY RUN (laat zien wat er zou gebeuren, verplaatst niets).
Account kiezen: python3 verplaats_bestaand.py --account backup --audit
Echt uitvoeren: python3 verplaats_bestaand.py --account backup --uitvoeren
Log: verplaats_log.json
"""
import email
import email.header
import imaplib
import sys
import json
import config
import time
from pathlib import Path
from collections import Counter
from imap_utils import list_folders, quote_mailbox
from mail_routes import PREFIX, is_source_folder, quarter_folder, route_from_subject
LOG_FILE = Path(__file__).parent / "verplaats_log.json"
DRY_RUN = "--uitvoeren" not in sys.argv
AUDIT = "--audit" in sys.argv
PROGRESS_INTERVAL = 250
# Mappen die in één keer worden hernoemd/verplaatst (oud → nieuw).
#
# Bewust leeg: oude mappen, archiefmappen en mailinglistmappen zijn bronnen.
# Alleen Sent, Drafts, Trash, Spam en INBOX.Facturen - verwerkt worden
# uitgesloten van routering.
MAP_RENAMES: list[tuple[str, str]] = [
]
# ── helpers ──────────────────────────────────────────────────────────────────
def decode_hdr(value) -> str:
if not value:
return ""
try:
parts = email.header.decode_header(value)
out = []
for raw, cs in parts:
if isinstance(raw, bytes):
out.append(raw.decode(cs or "utf-8", errors="replace"))
else:
out.append(str(raw))
return " ".join(out).strip()
except Exception:
return str(value or "")
def list_source_folders(mail) -> list[str]:
return [folder for folder in list_folders(mail) if is_source_folder(folder)]
class ImapSession:
"""IMAP connection wrapper with conservative reconnect support."""
def __init__(self, cfg):
self.cfg = cfg
self.mail = None
def connect(self):
self.mail = imaplib.IMAP4_SSL(self.cfg.imap_host, self.cfg.imap_port)
self.mail.login(self.cfg.username, self.cfg.password)
def reconnect(self):
if self.mail is not None:
try:
self.mail.logout()
except Exception:
pass
time.sleep(2)
self.connect()
def logout(self):
if self.mail is not None:
self.mail.logout()
def log_failure(
log: dict,
stap: str,
src: str,
uid: bytes | str,
dst: str | None,
actie: str,
status: str | None,
response,
):
uid_str = uid.decode(errors="replace") if isinstance(uid, bytes) else str(uid)
log.setdefault("fouten", []).append({
"stap": stap,
"bron": src,
"uid": uid_str,
"doel": dst,
"actie": actie,
"status": status,
"response": repr(response),
})
def print_audit(title: str, by_source: Counter, by_destination: Counter):
print(f"\n Audit: {title}")
print(" Per bronmap:")
for folder, count in by_source.most_common():
print(f" {count:5d} {folder}")
print(" Per doelmap:")
for folder, count in by_destination.most_common():
print(f" {count:5d} {folder}")
def print_progress(folder: str, processed: int, total: int):
if processed == total or processed % PROGRESS_INTERVAL == 0:
print(f" Gescand: {processed}/{total} | {folder}")
def fetch_body(data) -> bytes | None:
"""Return the first bytes payload from an IMAP FETCH response."""
if not data:
return None
for item in data:
if isinstance(item, tuple) and len(item) >= 2 and isinstance(item[1], bytes):
return item[1]
return None
def safe_select(session: ImapSession, log: dict, stap: str, folder: str, readonly: bool):
for attempt in range(2):
try:
return session.mail.select(quote_mailbox(folder), readonly=readonly)
except imaplib.IMAP4.abort as e:
log_failure(log, stap, folder, "", None, "select_abort", "ABORT", str(e))
if attempt == 1:
return "ABORT", [str(e).encode()]
print(f" IMAP-verbinding verbroken bij openen van {folder}; opnieuw verbinden ...")
try:
session.reconnect()
except Exception as reconnect_error:
log_failure(
log,
stap,
folder,
"",
None,
"reconnect_failed",
"ERROR",
str(reconnect_error),
)
return "ABORT", [str(reconnect_error).encode()]
def safe_uid(
session: ImapSession,
log: dict,
stap: str,
folder: str,
uid: bytes | str,
command: str,
*args,
readonly: bool,
):
for attempt in range(2):
try:
return session.mail.uid(command, *args)
except imaplib.IMAP4.abort as e:
log_failure(
log,
stap,
folder,
uid,
None,
f"uid_{command.lower()}_abort",
"ABORT",
str(e),
)
if attempt == 1:
return "ABORT", [str(e).encode()]
print(f" IMAP-verbinding verbroken in {folder}; opnieuw verbinden en opnieuw proberen ...")
try:
session.reconnect()
except Exception as reconnect_error:
log_failure(
log,
stap,
folder,
uid,
None,
"reconnect_failed",
"ERROR",
str(reconnect_error),
)
return "ABORT", [str(reconnect_error).encode()]
status, data = safe_select(session, log, stap, folder, readonly=readonly)
if status != "OK":
return status, data
def safe_list_source_folders(session: ImapSession, log: dict) -> list[str]:
for attempt in range(2):
try:
return list_source_folders(session.mail)
except imaplib.IMAP4.abort as e:
log_failure(log, "stap2", "", "", None, "list_abort", "ABORT", str(e))
if attempt == 1:
return []
print(" IMAP-verbinding verbroken bij ophalen van mappen; opnieuw verbinden ...")
try:
session.reconnect()
except Exception as reconnect_error:
log_failure(
log,
"stap2",
"",
"",
None,
"reconnect_failed",
"ERROR",
str(reconnect_error),
)
return []
def move_message(mail, uid: bytes, src: str, dst_full: str, dry: bool):
"""Kopieer bericht naar dst en markeer als verwijderd in src."""
if dry:
return True, None
try:
select_res, select_data = mail.select(quote_mailbox(src))
if select_res != "OK":
return False, ("select_source", select_res, select_data)
res, copy_data = mail.uid("COPY", uid, quote_mailbox(dst_full))
if res != "OK":
return False, ("copy", res, copy_data)
store_res, store_data = mail.uid("STORE", uid, "+FLAGS", "\\Deleted")
if store_res != "OK":
return False, ("store_deleted", store_res, store_data)
return True, None
except Exception as e:
print(f" FOUT bij verplaatsen: {e}")
return False, ("exception", None, str(e))
# ── stap 1: vaste mapverplaatsingen ─────────────────────────────────────────
def stap1_mapverplaatsingen(session: ImapSession, log: dict, dry: bool):
mail = session.mail
print("\n── Stap 1: vaste mapverplaatsingen ──")
for oud, nieuw in MAP_RENAMES:
if nieuw == "__FACTUREN__":
continue # wordt in stap 3 verwerkt
if oud in log.get("stap1_klaar", []):
print(f" ✓ al gedaan: {oud}")
continue
# Controleer of bronmap bestaat
status, data = safe_select(session, log, "stap1", oud, readonly=True)
if status != "OK":
print(f" – bestaat niet: {oud}")
continue
n = int(data[0])
nieuw_full = nieuw if nieuw.startswith("INBOX.") else PREFIX + nieuw
if dry:
print(f" [DRY] {oud} → {nieuw} ({n} berichten)")
continue
# Berichten kopiëren
status, data = mail.search(None, "ALL")
ids = data[0].split()
gekopieerd = 0
for uid in ids:
res, _ = mail.copy(uid.decode(), quote_mailbox(nieuw_full))
if res == "OK":
mail.store(uid.decode(), "+FLAGS", "\\Deleted")
gekopieerd += 1
mail.expunge()
print(f" ✓ {oud} → {nieuw} ({gekopieerd}/{n})")
log.setdefault("stap1_klaar", []).append(oud)
_save_log(log)
# ── stap 2: per-bericht routing (INBOX) ─────────────────────────────────────
def stap2_bronmappen_routing(session: ImapSession, log: dict, dry: bool):
mail = session.mail
print("\n── Stap 2: bronmappen per-bericht routing ──")
bron_mappen = safe_list_source_folders(session, log)
print(f" {len(bron_mappen)} bronmappen")
audit_by_source = Counter()
audit_by_destination = Counter()
totaal_verplaatst = 0
totaal_gepland = 0
totaal_geen_match = 0
totaal_overgeslagen = 0
totaal_mislukt = 0
al_verwerkt = set(log.get("stap2_verwerkt", []))
for folder_index, bron in enumerate(bron_mappen, start=1):
status, data = safe_select(session, log, "stap2", bron, readonly=dry)
mail = session.mail
if status != "OK":
print(f" Kan map niet openen: {bron}")
continue
n = int(data[0])
if n == 0:
continue
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: {n} berichten")
status, data = safe_uid(
session, log, "stap2", bron, "", "SEARCH", "ALL", readonly=dry
)
mail = session.mail
if status != "OK" or not data or not data[0]:
continue
ids = [uid for uid in data[0].split() if uid]
verplaatst = 0
gepland = 0
geen_match = 0
overgeslagen = 0
mislukt = 0
verwerkt = 0
for uid in ids:
verwerkt += 1
if AUDIT:
print_progress(bron, verwerkt, len(ids))
uid_str = uid.decode()
log_key = f"{bron}:{uid_str}"
if log_key in al_verwerkt:
overgeslagen += 1
continue
status, data = safe_uid(
session,
log,
"stap2",
bron,
uid,
"FETCH",
uid,
"(BODY.PEEK[HEADER.FIELDS (FROM SUBJECT DATE)])",
readonly=dry,
)
mail = session.mail
raw_header = fetch_body(data)
if status != "OK" or raw_header is None:
mislukt += 1
log_failure(log, "stap2", bron, uid, None, "fetch_header", status, data)
continue
msg = email.message_from_bytes(raw_header)
from_raw = decode_hdr(msg.get("From", ""))
subject = decode_hdr(msg.get("Subject", ""))
date_str = msg.get("Date", "")
doel = route_from_subject(from_raw, subject)
if doel == "__FACTUUR_DATE__":
doel = PREFIX + quarter_folder(date_str)
elif doel:
doel = PREFIX + doel
else:
geen_match += 1
continue
if bron == doel:
overgeslagen += 1
continue
gepland += 1
audit_by_source[bron] += 1
audit_by_destination[doel] += 1
if dry:
if not AUDIT:
print(f" [DRY {verwerkt}/{len(ids)}] {bron} → {doel} | {subject[:60]}")
else:
ok, failure = move_message(mail, uid, bron, doel, dry=False)
if ok:
verplaatst += 1
log.setdefault("stap2_verwerkt", []).append(log_key)
if verplaatst % 50 == 0:
mail.expunge()
_save_log(log)
else:
mislukt += 1
actie, status, response = failure
log_failure(log, "stap2", bron, uid, doel, actie, status, response)
if not dry:
mail.expunge()
_save_log(log)
totaal_verplaatst += verplaatst
totaal_gepland += gepland
totaal_geen_match += geen_match
totaal_overgeslagen += overgeslagen
totaal_mislukt += mislukt
if dry:
print(
f" Gepland: {gepland} | Geen match: {geen_match}"
f" | Overgeslagen: {overgeslagen} | Fouten: {mislukt}"
)
else:
print(
f" Gepland: {gepland} | Verplaatst: {verplaatst}"
f" | Geen match: {geen_match} | Overgeslagen: {overgeslagen}"
f" | Fouten: {mislukt}"
)
if dry:
print(
f" Totaal gepland: {totaal_gepland} | Totaal geen match: {totaal_geen_match}"
f" | Totaal overgeslagen: {totaal_overgeslagen} | Totaal fouten: {totaal_mislukt}"
)
if AUDIT:
print_audit("stap 2 geplande verplaatsingen", audit_by_source, audit_by_destination)
else:
print(
f" Totaal gepland: {totaal_gepland} | Totaal verplaatst: {totaal_verplaatst}"
f" | Totaal geen match: {totaal_geen_match} | Totaal overgeslagen: {totaal_overgeslagen}"
f" | Totaal fouten: {totaal_mislukt}"
)
# ── stap 3: facturen per kwartaal ────────────────────────────────────────────
def stap3_facturen(session: ImapSession, log: dict, dry: bool):
mail = session.mail
print("\n── Stap 3: facturen per kwartaal ──")
bron_mappen = [
"INBOX.Facturen - te verwerken",
"INBOX.Financieel.Facturen", # als ze er al in zitten
]
audit_by_source = Counter()
audit_by_destination = Counter()
for folder_index, bron in enumerate(bron_mappen, start=1):
status, data = safe_select(session, log, "stap3", bron, readonly=dry)
mail = session.mail
if status != "OK":
continue
n = int(data[0])
if n == 0:
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: leeg")
continue
print(f" [{folder_index}/{len(bron_mappen)}] {bron}: {n} berichten")
status, data = safe_uid(
session, log, "stap3", bron, "", "SEARCH", "ALL", readonly=dry
)
mail = session.mail
if status != "OK" or not data or not data[0]:
continue
ids = [uid for uid in data[0].split() if uid]
verplaatst = 0
gepland = 0
mislukt = 0
for index, uid in enumerate(ids, start=1):
if AUDIT:
print_progress(bron, index, len(ids))
status, data = safe_uid(
session,
log,
"stap3",
bron,
uid,
"FETCH",
uid,
"(BODY.PEEK[HEADER.FIELDS (DATE SUBJECT)])",
readonly=dry,
)
mail = session.mail
raw_header = fetch_body(data)
if status != "OK" or raw_header is None:
mislukt += 1
log_failure(log, "stap3", bron, uid, None, "fetch_header", status, data)
continue
msg = email.message_from_bytes(raw_header)
date_str = msg.get("Date", "")
doel = PREFIX + quarter_folder(date_str)
gepland += 1
audit_by_source[bron] += 1
audit_by_destination[doel] += 1
if dry:
dt_kort = date_str[:16] if date_str else "?"
if not AUDIT:
print(f" [DRY {index}/{len(ids)}] {dt_kort} → {doel}")
else:
ok, failure = move_message(mail, uid, bron, doel, dry=False)
if ok:
verplaatst += 1
else:
mislukt += 1
actie, status, response = failure
log_failure(log, "stap3", bron, uid, doel, actie, status, response)
if not dry:
mail.expunge()
print(f" Gepland: {gepland} | Verplaatst: {verplaatst} | Fouten: {mislukt}")
_save_log(log)
else:
print(f" Gepland: {gepland} | Fouten: {mislukt}")
if dry and AUDIT:
print_audit("stap 3 geplande factuurverplaatsingen", audit_by_source, audit_by_destination)
# ── log ──────────────────────────────────────────────────────────────────────
def _save_log(log: dict):
LOG_FILE.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
def load_log() -> dict:
if LOG_FILE.exists():
try:
return json.loads(LOG_FILE.read_text(encoding="utf-8"))
except Exception:
pass
return {}
def account_name_from_args() -> str | None:
args = sys.argv[1:]
if "--account" not in args:
return None
idx = args.index("--account")
if idx + 1 >= len(args) or args[idx + 1].startswith("--"):
print("Gebruik: python3 verplaats_bestaand.py --account <naam> [--audit|--uitvoeren]")
sys.exit(1)
return args[idx + 1]
# ── main ──────────────────────────────────────────────────────────────────────
def run():
if AUDIT and not DRY_RUN:
print("--audit is alleen bedoeld voor dry-runs. Laat --uitvoeren weg.")
sys.exit(1)
if DRY_RUN:
print("DRY RUN — er wordt niets verplaatst.")
print("Voeg --uitvoeren toe om echt uit te voeren.\n")
if AUDIT:
print("Auditmodus: per-bericht dry-runregels worden samengevat per bron en doel.\n")
else:
print("LET OP: dit verplaatst berichten op de echte server.")
bevestig = input("Typ JA om door te gaan: ").strip()
if bevestig != "JA":
print("Afgebroken.")
sys.exit(0)
account_name = account_name_from_args()
cfg = config.load(account_name)
print(f"Account: {cfg.name} ({cfg.username})\n")
print(f"Verbinden met {cfg.imap_host}:{cfg.imap_port} ...")
session = ImapSession(cfg)
try:
session.connect()
except Exception as e:
print(f"Verbinding mislukt: {e}")
sys.exit(1)
log = load_log()
try:
stap1_mapverplaatsingen(session, log, DRY_RUN)
stap2_bronmappen_routing(session, log, DRY_RUN)
stap3_facturen(session, log, DRY_RUN)
finally:
_save_log(log)
try:
session.logout()
except Exception as e:
print(f"Afmelden mislukt: {e}")
print("\nKlaar.")
if DRY_RUN:
print("Draai met --uitvoeren om de wijzigingen door te voeren.")
if __name__ == "__main__":
run()