Initial status for developed files on dagda
This commit is contained in:
mailcat
@@ -0,0 +1,341 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Kopieert alle mail van een bronaccount naar een doelaccount.
|
||||
|
||||
Gebruik: python3 kopieer_naar_backup.py --van hans --naar backup
|
||||
|
||||
Bron en doel worden opgezocht in config.json (aanmaken via python3 config.py --setup).
|
||||
|
||||
- Mappenstructuur wordt gespiegeld
|
||||
- Vlaggen (gelezen, gemarkeerd, etc.) en ontvangstdatum worden bewaard
|
||||
- Herstart-safe: berichten die al in het doel staan (zelfde Message-ID)
|
||||
worden overgeslagen
|
||||
- Niets wordt verwijderd of gewijzigd in de bronmailbox
|
||||
"""
|
||||
|
||||
import imaplib
|
||||
import email
|
||||
import email.header
|
||||
import email.utils
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import config
|
||||
from pathlib import Path
|
||||
from datetime import datetime, timezone
|
||||
from imap_utils import list_folders, quote_mailbox
|
||||
|
||||
LOG_FILE = Path(__file__).parent / "backup_log.json"
|
||||
|
||||
# Mappen die overgeslagen worden (Trash en Spam zijn optioneel)
|
||||
SKIP_MAPPEN = {"INBOX.Trash", "INBOX.Spam"}
|
||||
|
||||
|
||||
# ── helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def parse_flags(flag_str: str) -> list[str]:
|
||||
"""Haal standaard IMAP-vlaggen op uit fetch-response."""
|
||||
return re.findall(r'\\[A-Za-z]+', flag_str)
|
||||
|
||||
|
||||
def parse_internaldate(response: bytes) -> str | None:
|
||||
"""Haal INTERNALDATE op uit fetch-response voor gebruik in APPEND."""
|
||||
m = re.search(rb'INTERNALDATE "([^"]+)"', response)
|
||||
if m:
|
||||
try:
|
||||
dt = email.utils.parsedate_to_datetime(m.group(1).decode())
|
||||
return imaplib.Time2Internaldate(dt.timestamp())
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def haal_bestaande_message_ids(mail: imaplib.IMAP4_SSL, folder: str) -> set[str]:
|
||||
"""Geef alle Message-IDs terug die al in een doelmap staan."""
|
||||
status, data = mail.select(quote_mailbox(folder), readonly=True)
|
||||
if status != "OK" or int(data[0]) == 0:
|
||||
return set()
|
||||
|
||||
status, data = mail.search(None, "ALL")
|
||||
if status != "OK" or not data[0]:
|
||||
return set()
|
||||
|
||||
ids = data[0].split()
|
||||
msg_ids: set[str] = set()
|
||||
|
||||
# Batch ophalen in stukken van 100
|
||||
for i in range(0, len(ids), 100):
|
||||
batch = b",".join(ids[i:i+100])
|
||||
status, data = mail.fetch(batch, "(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])")
|
||||
if status != "OK":
|
||||
continue
|
||||
for item in data:
|
||||
if isinstance(item, tuple):
|
||||
msg = email.message_from_bytes(item[1])
|
||||
mid = (msg.get("Message-ID") or "").strip()
|
||||
if mid:
|
||||
msg_ids.add(mid)
|
||||
|
||||
return msg_ids
|
||||
|
||||
|
||||
# ── kopieerlogica ─────────────────────────────────────────────────────────────
|
||||
|
||||
def kopieer_map(
|
||||
bron: imaplib.IMAP4_SSL,
|
||||
doel: imaplib.IMAP4_SSL,
|
||||
folder: str,
|
||||
log: dict,
|
||||
) -> tuple[int, int, int]:
|
||||
"""
|
||||
Kopieer berichten van bron-folder naar doel-folder.
|
||||
Geeft (gekopieerd, overgeslagen, mislukt) terug.
|
||||
"""
|
||||
# Selecteer bronmap
|
||||
status, data = bron.select(quote_mailbox(folder), readonly=True)
|
||||
if status != "OK":
|
||||
print(f" Kan bronmap niet openen: {folder}")
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"actie": "select_bron",
|
||||
"status": status,
|
||||
"response": repr(data),
|
||||
})
|
||||
return 0, 0, 1
|
||||
|
||||
totaal = int(data[0])
|
||||
if totaal == 0:
|
||||
return 0, 0, 0
|
||||
|
||||
# Haal alle bron-UIDs op
|
||||
status, data = bron.search(None, "ALL")
|
||||
if status != "OK" or not data[0]:
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"actie": "search_bron",
|
||||
"status": status,
|
||||
"response": repr(data),
|
||||
})
|
||||
return 0, 0, 1
|
||||
ids = data[0].split()
|
||||
|
||||
# Zorg dat doelmap bestaat
|
||||
create_status, create_data = doel.create(quote_mailbox(folder))
|
||||
if create_status not in {"OK", "NO"}:
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"actie": "create_doel",
|
||||
"status": create_status,
|
||||
"response": repr(create_data),
|
||||
})
|
||||
return 0, 0, 1
|
||||
select_status, select_data = doel.select(quote_mailbox(folder), readonly=True)
|
||||
if select_status != "OK":
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"actie": "select_doel_na_create",
|
||||
"status": select_status,
|
||||
"response": repr(select_data),
|
||||
})
|
||||
return 0, 0, 1
|
||||
|
||||
# Welke Message-IDs staan al in de doelmap?
|
||||
bestaande = haal_bestaande_message_ids(doel, folder)
|
||||
|
||||
gekopieerd = 0
|
||||
overgeslagen = 0
|
||||
mislukt = 0
|
||||
|
||||
for i, uid in enumerate(ids, 1):
|
||||
# Voortgang op één regel
|
||||
print(f"\r {folder}: {i}/{totaal} ({gekopieerd} gekopieerd, {overgeslagen} skip, {mislukt} fout) ",
|
||||
end="", flush=True)
|
||||
|
||||
# Haal volledige bericht op + vlaggen + datum
|
||||
status, data = bron.fetch(
|
||||
uid, "(FLAGS INTERNALDATE BODY.PEEK[])"
|
||||
)
|
||||
if status != "OK" or not data:
|
||||
mislukt += 1
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"uid": uid.decode(errors="replace"),
|
||||
"actie": "fetch_bron",
|
||||
"status": status,
|
||||
"response": repr(data),
|
||||
})
|
||||
continue
|
||||
|
||||
# Parse fetch-response
|
||||
raw_response = b""
|
||||
raw_message = b""
|
||||
flags = []
|
||||
internaldate = None
|
||||
|
||||
for item in data:
|
||||
if isinstance(item, tuple):
|
||||
header_part = item[0].decode("utf-8", errors="replace")
|
||||
raw_message = item[1]
|
||||
flags = parse_flags(header_part)
|
||||
internaldate = parse_internaldate(item[0])
|
||||
|
||||
if not raw_message:
|
||||
mislukt += 1
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"uid": uid.decode(errors="replace"),
|
||||
"actie": "fetch_lege_body",
|
||||
"status": status,
|
||||
"response": repr(data),
|
||||
})
|
||||
continue
|
||||
|
||||
# Controleer Message-ID voor deduplicatie
|
||||
try:
|
||||
msg = email.message_from_bytes(raw_message[:2048])
|
||||
mid = (msg.get("Message-ID") or "").strip()
|
||||
except Exception:
|
||||
mid = ""
|
||||
|
||||
if mid and mid in bestaande:
|
||||
overgeslagen += 1
|
||||
continue
|
||||
|
||||
# Schrijf naar doelmap
|
||||
flag_str = "(" + " ".join(flags) + ")" if flags else "()"
|
||||
status, append_data = doel.append(
|
||||
quote_mailbox(folder),
|
||||
flag_str,
|
||||
internaldate,
|
||||
raw_message,
|
||||
)
|
||||
if status == "OK":
|
||||
gekopieerd += 1
|
||||
if mid:
|
||||
bestaande.add(mid)
|
||||
else:
|
||||
mislukt += 1
|
||||
log.setdefault("fouten", []).append({
|
||||
"folder": folder,
|
||||
"uid": uid.decode(errors="replace"),
|
||||
"actie": "append_doel",
|
||||
"message_id": mid,
|
||||
"status": status,
|
||||
"response": repr(append_data),
|
||||
})
|
||||
|
||||
# Even pauzeren elke 50 berichten om server te ontlasten
|
||||
if gekopieerd % 50 == 0 and gekopieerd > 0:
|
||||
time.sleep(0.2)
|
||||
|
||||
print(f"\r {folder}: {totaal}/{totaal} → {gekopieerd} gekopieerd, {overgeslagen} al aanwezig, {mislukt} fout ")
|
||||
return gekopieerd, overgeslagen, mislukt
|
||||
|
||||
|
||||
# ── log ───────────────────────────────────────────────────────────────────────
|
||||
|
||||
def load_log() -> dict:
|
||||
if LOG_FILE.exists():
|
||||
try:
|
||||
return json.loads(LOG_FILE.read_text(encoding="utf-8"))
|
||||
except Exception:
|
||||
pass
|
||||
return {"voltooide_mappen": [], "totaal_gekopieerd": 0}
|
||||
|
||||
|
||||
def save_log(log: dict):
|
||||
LOG_FILE.write_text(json.dumps(log, indent=2, ensure_ascii=False), encoding="utf-8")
|
||||
|
||||
|
||||
# ── main ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
def _account_name(flag: str, default: str) -> str:
|
||||
args = sys.argv[1:]
|
||||
if flag in args:
|
||||
idx = args.index(flag)
|
||||
if idx + 1 < len(args):
|
||||
return args[idx + 1]
|
||||
return default
|
||||
|
||||
|
||||
def run():
|
||||
bron_naam = _account_name("--van", "hans")
|
||||
doel_naam = _account_name("--naar", "backup")
|
||||
|
||||
bron_cfg = config.load(bron_naam)
|
||||
doel_cfg = config.load(doel_naam)
|
||||
|
||||
print(f"Bron : {bron_cfg.username}")
|
||||
print(f"Doel : {doel_cfg.username}")
|
||||
print(f"Host : {bron_cfg.imap_host}:{bron_cfg.imap_port}\n")
|
||||
|
||||
print("Verbinden ...")
|
||||
try:
|
||||
bron = imaplib.IMAP4_SSL(bron_cfg.imap_host, bron_cfg.imap_port)
|
||||
bron.login(bron_cfg.username, bron_cfg.password)
|
||||
print(f" ✓ {bron_cfg.username}")
|
||||
except Exception as e:
|
||||
print(f" ✗ Bron: {e}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
doel = imaplib.IMAP4_SSL(doel_cfg.imap_host, doel_cfg.imap_port)
|
||||
doel.login(doel_cfg.username, doel_cfg.password)
|
||||
print(f" ✓ {doel_cfg.username}")
|
||||
except Exception as e:
|
||||
print(f" ✗ Doel: {e}")
|
||||
bron.logout()
|
||||
sys.exit(1)
|
||||
|
||||
# Mappenlijst van bron
|
||||
alle_mappen = [folder for folder in list_folders(bron) if folder not in SKIP_MAPPEN]
|
||||
alle_mappen.sort()
|
||||
|
||||
log = load_log()
|
||||
al_klaar = set(log.get("voltooide_mappen", []))
|
||||
|
||||
print(f"\n{len(alle_mappen)} mappen gevonden "
|
||||
f"({len(al_klaar)} al eerder voltooid)\n")
|
||||
|
||||
totaal_gekopieerd = log.get("totaal_gekopieerd", 0)
|
||||
totaal_overgeslagen = 0
|
||||
totaal_mislukt = 0
|
||||
|
||||
for i, folder in enumerate(alle_mappen, 1):
|
||||
print(f"[{i}/{len(alle_mappen)}] {folder}")
|
||||
|
||||
if folder in al_klaar:
|
||||
print(f" ✓ al volledig gekopieerd, overgeslagen.\n")
|
||||
continue
|
||||
|
||||
gekopieerd, overgeslagen, mislukt = kopieer_map(bron, doel, folder, log)
|
||||
totaal_gekopieerd += gekopieerd
|
||||
totaal_overgeslagen += overgeslagen
|
||||
totaal_mislukt += mislukt
|
||||
|
||||
if mislukt == 0:
|
||||
log["voltooide_mappen"] = list(al_klaar | {folder})
|
||||
al_klaar.add(folder)
|
||||
else:
|
||||
log["voltooide_mappen"] = list(al_klaar)
|
||||
print(f" ! niet als voltooid gemarkeerd door {mislukt} fout(en)")
|
||||
log["totaal_gekopieerd"] = totaal_gekopieerd
|
||||
log["laatste_run"] = datetime.now().isoformat()
|
||||
save_log(log)
|
||||
print()
|
||||
|
||||
bron.logout()
|
||||
doel.logout()
|
||||
|
||||
print("─" * 50)
|
||||
print(f"Klaar.")
|
||||
print(f" Gekopieerd : {totaal_gekopieerd}")
|
||||
print(f" Al aanwezig : {totaal_overgeslagen}")
|
||||
print(f" Fouten : {totaal_mislukt}")
|
||||
print(f" Log : {LOG_FILE}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run()
|
||||
Reference in New Issue
Block a user