Add route management web interface

Prompts used since previous commit:
- Kun je ook een webinterface maken waarmee ik de regels kan beheren?
- Ik wil dat de webinterface benaderbaar is vanaf mijn hele interne netwerk. En ik wil alle regels kunnen beheren. Dus ook de bestaande ingebouwde routes.
- 1. basic auth
- 2. 4321

Special observations:
- Domain routes moved from Python source to domain_routes.json so the web UI can manage all previous built-in domain routes.
- web_config.json remains ignored; live sorter processes must be restarted after route edits to pick up changed rules.
This commit is contained in:
2026-07-05 00:19:53 +02:00
parent cfb96bc610
commit ae0813d519
8 changed files with 1462 additions and 308 deletions
+1
View File
@@ -3,6 +3,7 @@ __pycache__/
*.log
.DS_Store
config.json
web_config.json
**/*.eml
sort_mail_daemon_state.json
verplaats_log_*.json
+110 -3
View File
@@ -15,6 +15,8 @@ APP_USER="${APP_USER:-mailcat}"
SERVICE_NAME="${SERVICE_NAME:-mailcat-sort-backup}"
ACCOUNT="${ACCOUNT:-backup}"
DAILY_REPORT_TO="${DAILY_REPORT_TO:-hans@australius.nl}"
WEB_HOST="${WEB_HOST:-0.0.0.0}"
WEB_PORT="${WEB_PORT:-4321}"
REMOTE_PATH="${REMOTE_PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}"
SERVICE_MANAGER="${SERVICE_MANAGER:-systemd}"
MAILCAT_HOME="${MAILCAT_HOME:-/share/homes/$APP_USER}"
@@ -40,12 +42,16 @@ if [ "$SERVICE_MANAGER" = "plain" ]; then
LOG_DIR="${LOG_DIR:-$APP_DIR/log}"
RUN_DIR="${RUN_DIR:-$APP_DIR/run}"
PID_FILE="${PID_FILE:-$RUN_DIR/mailcat.pid}"
WEB_PID_FILE="${WEB_PID_FILE:-$RUN_DIR/mailcat_web.pid}"
WEB_CONFIG="${WEB_CONFIG:-$APP_DIR/web_config.json}"
PYTHON="${PYTHON:-/opt/bin/python3}"
else
STATE_DIR="${STATE_DIR:-/var/lib/mailcat}"
LOG_DIR="${LOG_DIR:-/var/log/mailcat}"
RUN_DIR="${RUN_DIR:-/run/mailcat}"
PID_FILE="${PID_FILE:-$RUN_DIR/mailcat.pid}"
WEB_PID_FILE="${WEB_PID_FILE:-$RUN_DIR/mailcat_web.pid}"
WEB_CONFIG="${WEB_CONFIG:-$APP_DIR/web_config.json}"
PYTHON="${PYTHON:-/usr/bin/python3}"
fi
@@ -64,6 +70,7 @@ tar \
--exclude "__pycache__" \
--exclude "mailbox" \
--exclude "config.json" \
--exclude "web_config.json" \
--exclude "*.log" \
--exclude "backup_log.json" \
--exclude "verplaats_log*.json" \
@@ -118,6 +125,11 @@ APP_GROUP="$(id -gn "$APP_USER")"
if [ ! -f "$APP_DIR/config.json" ]; then
echo "Missing $APP_DIR/config.json on remote host. Create it with mode 600 before starting the service." >&2
fi
"$SUDO" "$CHMOD" 600 "$APP_DIR/web_config.json" 2>/dev/null || true
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/web_config.json" 2>/dev/null || true
if [ ! -f "$APP_DIR/web_config.json" ]; then
echo "Missing $APP_DIR/web_config.json on remote host. Create it with mode 600 before starting the web UI." >&2
fi
"$SUDO" "$CHMOD" 600 "$APP_DIR/config.json" 2>/dev/null || true
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/config.json" 2>/dev/null || true
@@ -218,12 +230,105 @@ cd "\$APP_DIR"
"\$PYTHON" "\$APP_DIR/dagelijks_overzicht.py" --account "\$ACCOUNT" --mail-to "\$MAIL_TO" >> "\$LOG_DIR/daily_report.out" 2>&1
REPORT_SCRIPT
cat > /tmp/mailcat-web-start.sh <<WEB_START_SCRIPT
#!/bin/sh
set -eu
APP_DIR="$APP_DIR"
PYTHON="$PYTHON"
LOG_DIR="$LOG_DIR"
RUN_DIR="$RUN_DIR"
WEB_PID_FILE="$WEB_PID_FILE"
WEB_HOST="$WEB_HOST"
WEB_PORT="$WEB_PORT"
WEB_CONFIG="$WEB_CONFIG"
MKDIR="$MKDIR"
if [ ! -f "\$WEB_CONFIG" ]; then
echo "missing web config: \$WEB_CONFIG"
exit 1
fi
"\$MKDIR" -p "\$LOG_DIR" "\$RUN_DIR"
if [ -f "\$WEB_PID_FILE" ]; then
old_pid=""
IFS= read -r old_pid < "\$WEB_PID_FILE" || true
if [ -n "\$old_pid" ] && kill -0 "\$old_pid" 2>/dev/null; then
echo "mailcat web already running: \$old_pid"
exit 0
fi
fi
cd "\$APP_DIR"
"\$PYTHON" "\$APP_DIR/manage_routes_web.py" --host "\$WEB_HOST" --port "\$WEB_PORT" --config "\$WEB_CONFIG" >> "\$LOG_DIR/web.out" 2>&1 &
pid="\$!"
echo "\$pid" > "\$WEB_PID_FILE"
echo "mailcat web started: \$pid"
WEB_START_SCRIPT
cat > /tmp/mailcat-web-stop.sh <<WEB_STOP_SCRIPT
#!/bin/sh
set -eu
WEB_PID_FILE="$WEB_PID_FILE"
RM="$RM"
if [ ! -f "\$WEB_PID_FILE" ]; then
echo "mailcat web not running: no pid file"
exit 0
fi
pid=""
IFS= read -r pid < "\$WEB_PID_FILE" || true
if [ -z "\$pid" ]; then
"\$RM" -f "\$WEB_PID_FILE"
echo "mailcat web not running: empty pid file removed"
exit 0
fi
if kill -0 "\$pid" 2>/dev/null; then
kill "\$pid"
sleep 2
if kill -0 "\$pid" 2>/dev/null; then
echo "mailcat web still running after SIGTERM: \$pid"
exit 1
fi
fi
"\$RM" -f "\$WEB_PID_FILE"
echo "mailcat web stopped"
WEB_STOP_SCRIPT
cat > /tmp/mailcat-web-status.sh <<WEB_STATUS_SCRIPT
#!/bin/sh
set -eu
WEB_PID_FILE="$WEB_PID_FILE"
WEB_HOST="$WEB_HOST"
WEB_PORT="$WEB_PORT"
LOG_DIR="$LOG_DIR"
if [ -f "\$WEB_PID_FILE" ]; then
pid=""
IFS= read -r pid < "\$WEB_PID_FILE" || true
if [ -n "\$pid" ] && kill -0 "\$pid" 2>/dev/null; then
echo "mailcat web running: \$pid"
echo "url: http://\$WEB_HOST:\$WEB_PORT/"
exit 0
fi
echo "mailcat web not running: stale pid file (\$pid)"
exit 1
fi
echo "mailcat web not running"
echo "logs: \$LOG_DIR/web.out"
exit 1
WEB_STATUS_SCRIPT
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-start.sh "$APP_DIR/bin/start_mailcat.sh"
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-stop.sh "$APP_DIR/bin/stop_mailcat.sh"
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-status.sh "$APP_DIR/bin/status_mailcat.sh"
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-daily-report.sh "$APP_DIR/bin/run_daily_report.sh"
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/bin/start_mailcat.sh" "$APP_DIR/bin/stop_mailcat.sh" "$APP_DIR/bin/status_mailcat.sh" "$APP_DIR/bin/run_daily_report.sh"
"$RM" -f /tmp/mailcat-start.sh /tmp/mailcat-stop.sh /tmp/mailcat-status.sh /tmp/mailcat-daily-report.sh
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-web-start.sh "$APP_DIR/bin/start_web.sh"
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-web-stop.sh "$APP_DIR/bin/stop_web.sh"
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-web-status.sh "$APP_DIR/bin/status_web.sh"
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/bin/start_mailcat.sh" "$APP_DIR/bin/stop_mailcat.sh" "$APP_DIR/bin/status_mailcat.sh" "$APP_DIR/bin/run_daily_report.sh" "$APP_DIR/bin/start_web.sh" "$APP_DIR/bin/stop_web.sh" "$APP_DIR/bin/status_web.sh"
"$RM" -f /tmp/mailcat-start.sh /tmp/mailcat-stop.sh /tmp/mailcat-status.sh /tmp/mailcat-daily-report.sh /tmp/mailcat-web-start.sh /tmp/mailcat-web-stop.sh /tmp/mailcat-web-status.sh
else
"$SUDO" "$INSTALL" -m 0644 "$APP_DIR/systemd/mailcat-sort.service" "/etc/systemd/system/$SERVICE_NAME.service"
"$SUDO" "$SED" -i \
@@ -247,6 +352,8 @@ if [ "$SERVICE_MANAGER" = "plain" ]; then
echo " sudo -u $APP_USER $APP_DIR/bin/start_mailcat.sh"
echo " sudo -u $APP_USER $APP_DIR/bin/status_mailcat.sh"
echo " sudo -u $APP_USER $APP_DIR/bin/run_daily_report.sh"
echo " sudo -u $APP_USER $APP_DIR/bin/start_web.sh"
echo " sudo -u $APP_USER $APP_DIR/bin/status_web.sh"
echo " tail -f $LOG_DIR/sort_mail_daemon.log"
else
echo " sudo systemctl start $SERVICE_NAME.service"
@@ -257,5 +364,5 @@ REMOTE
scp "$ARCHIVE" "$REMOTE_USER@$REMOTE_HOST:/tmp/mailcat-deploy.tar.gz"
scp "$REMOTE_SCRIPT" "$REMOTE_USER@$REMOTE_HOST:/tmp/mailcat-remote-deploy.sh"
REMOTE_COMMAND=$(printf "APP_USER=%q APP_DIR=%q SERVICE_NAME=%q ACCOUNT=%q DAILY_REPORT_TO=%q REMOTE_PATH=%q SERVICE_MANAGER=%q MAILCAT_HOME=%q STATE_DIR=%q LOG_DIR=%q RUN_DIR=%q PID_FILE=%q PYTHON=%q SUDO=%q USERADD=%q INSTALL=%q SED=%q TAR=%q MKDIR=%q CHOWN=%q CHMOD=%q RM=%q SYSTEMCTL=%q /bin/sh /tmp/mailcat-remote-deploy.sh" "$APP_USER" "$APP_DIR" "$SERVICE_NAME" "$ACCOUNT" "$DAILY_REPORT_TO" "$REMOTE_PATH" "$SERVICE_MANAGER" "$MAILCAT_HOME" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" "$PID_FILE" "$PYTHON" "$SUDO" "$USERADD" "$INSTALL" "$SED" "$TAR" "$MKDIR" "$CHOWN" "$CHMOD" "$RM" "$SYSTEMCTL")
REMOTE_COMMAND=$(printf "APP_USER=%q APP_DIR=%q SERVICE_NAME=%q ACCOUNT=%q DAILY_REPORT_TO=%q WEB_HOST=%q WEB_PORT=%q REMOTE_PATH=%q SERVICE_MANAGER=%q MAILCAT_HOME=%q STATE_DIR=%q LOG_DIR=%q RUN_DIR=%q PID_FILE=%q WEB_PID_FILE=%q WEB_CONFIG=%q PYTHON=%q SUDO=%q USERADD=%q INSTALL=%q SED=%q TAR=%q MKDIR=%q CHOWN=%q CHMOD=%q RM=%q SYSTEMCTL=%q /bin/sh /tmp/mailcat-remote-deploy.sh" "$APP_USER" "$APP_DIR" "$SERVICE_NAME" "$ACCOUNT" "$DAILY_REPORT_TO" "$WEB_HOST" "$WEB_PORT" "$REMOTE_PATH" "$SERVICE_MANAGER" "$MAILCAT_HOME" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" "$PID_FILE" "$WEB_PID_FILE" "$WEB_CONFIG" "$PYTHON" "$SUDO" "$USERADD" "$INSTALL" "$SED" "$TAR" "$MKDIR" "$CHOWN" "$CHMOD" "$RM" "$SYSTEMCTL")
ssh -tt "$REMOTE_USER@$REMOTE_HOST" "$REMOTE_COMMAND"
+601
View File
@@ -0,0 +1,601 @@
[
{
"domains": [
"nl.abnamro.com",
"abnamro.nl"
],
"mailbox": "Financieel.Bank.ABN AMRO"
},
{
"domains": [
"icscards.nl",
"icsmarketing.icscards.nl",
"service.icscards.nl"
],
"mailbox": "Financieel.Bank.ICS"
},
{
"domains": [
"npex.nl"
],
"mailbox": "Financieel.Bank.NPEX"
},
{
"domains": [
"n26.com"
],
"mailbox": "Financieel.Bank.N26"
},
{
"domains": [
"update.bunq.com",
"hello.bunq.com"
],
"mailbox": "Financieel.Bank.bunq"
},
{
"domains": [
"snelstart.nl"
],
"mailbox": "Financieel.Bank.SnelStart"
},
{
"domains": [
"tellow.nl"
],
"mailbox": "Financieel.Bank.Tellow"
},
{
"domains": [
"mijnkantoorapp.nl",
"dijkman-ac.nl"
],
"mailbox": "Financieel.Boekhouding.Dijkman"
},
{
"domains": [
"unive.nl",
"nieuwsbrief.unive.nl"
],
"mailbox": "Financieel.Verzekering.Univé"
},
{
"domains": [
"heinenoord.nl"
],
"mailbox": "Financieel.Verzekering.VEZA"
},
{
"domains": [
"flextender.nl"
],
"mailbox": "Werk.Opdrachten.Flextender"
},
{
"domains": [
"onestopsourcing.nl"
],
"mailbox": "Werk.Opdrachten.OneStopSourcing"
},
{
"domains": [
"immens.nu"
],
"mailbox": "Werk.Opdrachten.IMMENS"
},
{
"domains": [
"circle8.nl"
],
"mailbox": "Werk.Opdrachten.Circle8"
},
{
"domains": [
"getthere.nl"
],
"mailbox": "Werk.Opdrachten.Get There"
},
{
"domains": [
"hey.meetjamie.ai",
"notifications.meetjamie.ai"
],
"mailbox": "Diensten.AI.Jamie"
},
{
"domains": [
"huckr.ai"
],
"mailbox": "Diensten.AI.Huckr"
},
{
"domains": [
"maandag.com"
],
"mailbox": "Werk.Opdrachten.Maandag"
},
{
"domains": [
"futurexl.nl"
],
"mailbox": "Werk.Opdrachten.FutureXL"
},
{
"domains": [
"drenthe.nl"
],
"mailbox": "Werk.Opdrachten.Provincie Drenthe"
},
{
"domains": [
"rdw.nl"
],
"mailbox": "Werk.Opdrachten.RDW"
},
{
"domains": [
"plasbossinade.nl"
],
"mailbox": "Werk.Opdrachten.PlasBossinade"
},
{
"domains": [
"boip.int"
],
"mailbox": "Werk.Overig.BOIP"
},
{
"domains": [
"groningen.nl",
"groningenbereikbaar.nl"
],
"mailbox": "Werk.Overig.Gemeente Groningen"
},
{
"domains": [
"kvk.nl"
],
"mailbox": "Administratie.KvK"
},
{
"domains": [
"eneco-emobility.com",
"eneco.nl"
],
"mailbox": "Mobiliteit.EV.Eneco eMobility"
},
{
"domains": [
"shellrecharge.com"
],
"mailbox": "Mobiliteit.EV.Shell Recharge"
},
{
"domains": [
"fastned.nl"
],
"mailbox": "Mobiliteit.EV.FastNed"
},
{
"domains": [
"opngo.com"
],
"mailbox": "Mobiliteit.EV.OPnGO"
},
{
"domains": [
"tesla.com"
],
"mailbox": "Mobiliteit.EV.Tesla"
},
{
"domains": [
"mmsn.nl"
],
"mailbox": "Mobiliteit.EV.Mitsubishi"
},
{
"domains": [
"alfen.com"
],
"mailbox": "Mobiliteit.EV.Alfen"
},
{
"domains": [
"ns.nl",
"email.ns.nl"
],
"mailbox": "Mobiliteit.OV & Reizen.NS"
},
{
"domains": [
"booking.com"
],
"mailbox": "Mobiliteit.OV & Reizen.Booking"
},
{
"domains": [
"flitsmeister.nl"
],
"mailbox": "Mobiliteit.OV & Reizen.Flitsmeister"
},
{
"domains": [
"postnl.nl",
"edm.postnl.nl",
"notificatie.postnl.nl"
],
"mailbox": "Bestellingen.PostNL"
},
{
"domains": [
"dhlparcel.nl",
"dhlecommerce.nl"
],
"mailbox": "Bestellingen.DHL"
},
{
"domains": [
"gls-netherlands.com"
],
"mailbox": "Bestellingen.GLS"
},
{
"domains": [
"bol.com",
"feedback.bol.com"
],
"mailbox": "Bestellingen.bol"
},
{
"domains": [
"coolblue.nl",
"coolblue.eu"
],
"mailbox": "Bestellingen.Coolblue"
},
{
"domains": [
"allekabels.nl"
],
"mailbox": "Bestellingen.Allekabels"
},
{
"domains": [
"vikingdirect.nl",
"delivery.vikingdirect.nl",
"online.vikingdirect.nl"
],
"mailbox": "Bestellingen.Viking"
},
{
"domains": [
"123inkt.nl",
"m.123inkt.nl"
],
"mailbox": "Bestellingen.123inkt"
},
{
"domains": [
"officecentre.nl"
],
"mailbox": "Bestellingen.Office Centre"
},
{
"domains": [
"sligro.nl"
],
"mailbox": "Bestellingen.Sligro"
},
{
"domains": [
"makro.nl"
],
"mailbox": "Bestellingen.Makro"
},
{
"domains": [
"smartphonehoesjes.nl"
],
"mailbox": "Bestellingen.Smartphonehoesjes"
},
{
"domains": [
"strato.com",
"news.strato.com",
"marketingradar.strato.com"
],
"mailbox": "Diensten.Hosting.STRATO"
},
{
"domains": [
"transip.nl"
],
"mailbox": "Diensten.Hosting.TransIP"
},
{
"domains": [
"cloud86.io",
"cloud86.zendesk.com"
],
"mailbox": "Diensten.Hosting.Cloud86"
},
{
"domains": [
"godaddy.com"
],
"mailbox": "Diensten.Hosting.GoDaddy"
},
{
"domains": [
"letsencrypt.org"
],
"mailbox": "Diensten.Hosting.Lets Encrypt"
},
{
"domains": [
"backblaze.com"
],
"mailbox": "Diensten.Hosting.Backblaze"
},
{
"domains": [
"kpn.com",
"campagnes.kpn.com"
],
"mailbox": "Diensten.Telecom.KPN"
},
{
"domains": [
"zakelijk.vodafone.nl",
"vodafone.nl"
],
"mailbox": "Diensten.Telecom.Vodafone"
},
{
"domains": [
"nordaccount.com"
],
"mailbox": "Diensten.Beveiliging.Nord"
},
{
"domains": [
"info.expressvpn.com"
],
"mailbox": "Diensten.Beveiliging.ExpressVPN"
},
{
"domains": [
"email.remarkable.com"
],
"mailbox": "Diensten.Apparaten.reMarkable"
},
{
"domains": [
"1password.com"
],
"mailbox": "Diensten.Software.1Password"
},
{
"domains": [
"home.sophos.com",
"cleverbridge.com"
],
"mailbox": "Diensten.Software.Sophos"
},
{
"domains": [
"cogsciapps.com"
],
"mailbox": "Diensten.Software.CogSci Apps"
},
{
"domains": [
"claude.com",
"mail.anthropic.com"
],
"mailbox": "Diensten.AI.Claude"
},
{
"domains": [
"cursor.com",
"mail.cursor.com"
],
"mailbox": "Diensten.AI.Cursor"
},
{
"domains": [
"mistral.ai",
"mistralai.intercom-mail.eu"
],
"mailbox": "Diensten.AI.Mistral"
},
{
"domains": [
"openai.com",
"chatgpt.com",
"tm.openai.com"
],
"mailbox": "Diensten.AI.OpenAI"
},
{
"domains": [
"perplexity.ai"
],
"mailbox": "Diensten.AI.Perplexity"
},
{
"domains": [
"canva.com",
"account.canva.com"
],
"mailbox": "Diensten.AI.Canva"
},
{
"domains": [
"microsoft.com"
],
"mailbox": "Diensten.Software.Microsoft"
},
{
"domains": [
"microsoft365.com"
],
"mailbox": "Diensten.Software.Microsoft Teams"
},
{
"domains": [
"market.envato.com"
],
"mailbox": "Diensten.Software.Envato"
},
{
"domains": [
"getmatter.com"
],
"mailbox": "Diensten.Software.Matter"
},
{
"domains": [
"mindnode.com"
],
"mailbox": "Diensten.Software.MindNode"
},
{
"domains": [
"papersapp.com"
],
"mailbox": "Diensten.Software.Papers"
},
{
"domains": [
"team.kahoot.com"
],
"mailbox": "Diensten.Software.Kahoot"
},
{
"domains": [
"coursera.org",
"email.coursera.org",
"m.mail.coursera.org",
"m.learn.coursera.org"
],
"mailbox": "Nieuwsbrieven.Leren.Coursera"
},
{
"domains": [
"technologyreview.com",
"fulfillment.technologyreview.com"
],
"mailbox": "Nieuwsbrieven.Leren.MIT Technology Review"
},
{
"domains": [
"mit.edu"
],
"mailbox": "Nieuwsbrieven.Leren.MIT Open Learning"
},
{
"domains": [
"storytellingwithdata.com"
],
"mailbox": "Nieuwsbrieven.Leren.Storytelling with Data"
},
{
"domains": [
"linkingyourthinking.com"
],
"mailbox": "Nieuwsbrieven.Leren.Nick Milo"
},
{
"domains": [
"evakeiffenheim.com"
],
"mailbox": "Nieuwsbrieven.Leren.Eva Keiffenheim"
},
{
"domains": [
"tiggelaar.nl"
],
"mailbox": "Nieuwsbrieven.Leren.Ben Tiggelaar"
},
{
"domains": [
"briankeating.com"
],
"mailbox": "Nieuwsbrieven.Leren.Brian Keating"
},
{
"domains": [
"beehiiv.com",
"mail.beehiiv.com"
],
"mailbox": "Nieuwsbrieven.Leren.AI Report"
},
{
"domains": [
"elizabethbutlermd.com"
],
"mailbox": "Nieuwsbrieven.Leren.Elizabeth Butler"
},
{
"domains": [
"notify.orcid.org"
],
"mailbox": "Nieuwsbrieven.Leren.ORCID"
},
{
"domains": [
"crisismanager.nl"
],
"mailbox": "Nieuwsbrieven.Nieuws & Vakbladen.Vakblad Crisismanager"
},
{
"domains": [
"jaarbeurs.nl"
],
"mailbox": "Nieuwsbrieven.Nieuws & Vakbladen.Computable"
},
{
"domains": [
"presentation-guru.com"
],
"mailbox": "Nieuwsbrieven.Nieuws & Vakbladen.The Presentation Guru"
},
{
"domains": [
"lendahand.com"
],
"mailbox": "Nieuwsbrieven.Overig.Lendahand"
},
{
"domains": [
"thegoodroll.nl"
],
"mailbox": "Nieuwsbrieven.Overig.The Good Roll"
},
{
"domains": [
"tasks.clickup.com"
],
"mailbox": "Afmelden.Software.ClickUp"
},
{
"domains": [
"gemeenteprojecten.talent-pool.com"
],
"mailbox": "Afmelden.Werk.Gemeenteprojecten"
},
{
"domains": [
"engage.istockphoto.com"
],
"mailbox": "Afmelden.Creatief.iStock"
},
{
"domains": [
"dmarc.postmarkapp.com"
],
"mailbox": "Technisch.DMARC"
}
]
+58 -103
View File
@@ -9,109 +9,64 @@ from pathlib import Path
PREFIX = "INBOX."
MAILINGLIST_ROUTES_FILE = Path(__file__).parent / "mailinglist_routes.json"
DOMAIN_ROUTES_FILE = Path(__file__).parent / "domain_routes.json"
DOMAIN_ROUTES: list[tuple[list[str], str]] = [
(["nl.abnamro.com", "abnamro.nl"], "Financieel.Bank.ABN AMRO"),
(["icscards.nl", "icsmarketing.icscards.nl", "service.icscards.nl"], "Financieel.Bank.ICS"),
(["npex.nl"], "Financieel.Bank.NPEX"),
(["n26.com"], "Financieel.Bank.N26"),
(["update.bunq.com", "hello.bunq.com"], "Financieel.Bank.bunq"),
(["snelstart.nl"], "Financieel.Bank.SnelStart"),
(["tellow.nl"], "Financieel.Bank.Tellow"),
(["mijnkantoorapp.nl", "dijkman-ac.nl"], "Financieel.Boekhouding.Dijkman"),
(["unive.nl", "nieuwsbrief.unive.nl"], "Financieel.Verzekering.Univé"),
(["heinenoord.nl"], "Financieel.Verzekering.VEZA"),
(["flextender.nl"], "Werk.Opdrachten.Flextender"),
(["onestopsourcing.nl"], "Werk.Opdrachten.OneStopSourcing"),
(["immens.nu"], "Werk.Opdrachten.IMMENS"),
(["circle8.nl"], "Werk.Opdrachten.Circle8"),
(["getthere.nl"], "Werk.Opdrachten.Get There"),
(["hey.meetjamie.ai", "notifications.meetjamie.ai"], "Diensten.AI.Jamie"),
(["huckr.ai"], "Diensten.AI.Huckr"),
(["maandag.com"], "Werk.Opdrachten.Maandag"),
(["futurexl.nl"], "Werk.Opdrachten.FutureXL"),
(["drenthe.nl"], "Werk.Opdrachten.Provincie Drenthe"),
(["rdw.nl"], "Werk.Opdrachten.RDW"),
(["plasbossinade.nl"], "Werk.Opdrachten.PlasBossinade"),
(["boip.int"], "Werk.Overig.BOIP"),
(["groningen.nl", "groningenbereikbaar.nl"], "Werk.Overig.Gemeente Groningen"),
(["kvk.nl"], "Administratie.KvK"),
(["eneco-emobility.com", "eneco.nl"], "Mobiliteit.EV.Eneco eMobility"),
(["shellrecharge.com"], "Mobiliteit.EV.Shell Recharge"),
(["fastned.nl"], "Mobiliteit.EV.FastNed"),
(["opngo.com"], "Mobiliteit.EV.OPnGO"),
(["tesla.com"], "Mobiliteit.EV.Tesla"),
(["mmsn.nl"], "Mobiliteit.EV.Mitsubishi"),
(["alfen.com"], "Mobiliteit.EV.Alfen"),
(["ns.nl", "email.ns.nl"], "Mobiliteit.OV & Reizen.NS"),
(["booking.com"], "Mobiliteit.OV & Reizen.Booking"),
(["flitsmeister.nl"], "Mobiliteit.OV & Reizen.Flitsmeister"),
(["postnl.nl", "edm.postnl.nl", "notificatie.postnl.nl"], "Bestellingen.PostNL"),
(["dhlparcel.nl", "dhlecommerce.nl"], "Bestellingen.DHL"),
(["gls-netherlands.com"], "Bestellingen.GLS"),
(["bol.com", "feedback.bol.com"], "Bestellingen.bol"),
(["coolblue.nl", "coolblue.eu"], "Bestellingen.Coolblue"),
(["allekabels.nl"], "Bestellingen.Allekabels"),
(["vikingdirect.nl", "delivery.vikingdirect.nl", "online.vikingdirect.nl"], "Bestellingen.Viking"),
(["123inkt.nl", "m.123inkt.nl"], "Bestellingen.123inkt"),
(["officecentre.nl"], "Bestellingen.Office Centre"),
(["sligro.nl"], "Bestellingen.Sligro"),
(["makro.nl"], "Bestellingen.Makro"),
(["smartphonehoesjes.nl"], "Bestellingen.Smartphonehoesjes"),
(["strato.com", "news.strato.com", "marketingradar.strato.com"], "Diensten.Hosting.STRATO"),
(["transip.nl"], "Diensten.Hosting.TransIP"),
(["cloud86.io", "cloud86.zendesk.com"], "Diensten.Hosting.Cloud86"),
(["godaddy.com"], "Diensten.Hosting.GoDaddy"),
(["letsencrypt.org"], "Diensten.Hosting.Lets Encrypt"),
(["backblaze.com"], "Diensten.Hosting.Backblaze"),
(["kpn.com", "campagnes.kpn.com"], "Diensten.Telecom.KPN"),
(["zakelijk.vodafone.nl", "vodafone.nl"], "Diensten.Telecom.Vodafone"),
(["nordaccount.com"], "Diensten.Beveiliging.Nord"),
(["info.expressvpn.com"], "Diensten.Beveiliging.ExpressVPN"),
(["email.remarkable.com"], "Diensten.Apparaten.reMarkable"),
(["1password.com"], "Diensten.Software.1Password"),
(["home.sophos.com", "cleverbridge.com"], "Diensten.Software.Sophos"),
(["cogsciapps.com"], "Diensten.Software.CogSci Apps"),
(["claude.com", "mail.anthropic.com"], "Diensten.AI.Claude"),
(["cursor.com", "mail.cursor.com"], "Diensten.AI.Cursor"),
(["mistral.ai", "mistralai.intercom-mail.eu"], "Diensten.AI.Mistral"),
(["openai.com", "chatgpt.com", "tm.openai.com"], "Diensten.AI.OpenAI"),
(["perplexity.ai"], "Diensten.AI.Perplexity"),
(["canva.com", "account.canva.com"], "Diensten.AI.Canva"),
(["microsoft.com"], "Diensten.Software.Microsoft"),
(["microsoft365.com"], "Diensten.Software.Microsoft Teams"),
(["market.envato.com"], "Diensten.Software.Envato"),
(["getmatter.com"], "Diensten.Software.Matter"),
(["mindnode.com"], "Diensten.Software.MindNode"),
(["papersapp.com"], "Diensten.Software.Papers"),
(["team.kahoot.com"], "Diensten.Software.Kahoot"),
(["coursera.org", "email.coursera.org", "m.mail.coursera.org", "m.learn.coursera.org"], "Nieuwsbrieven.Leren.Coursera"),
(["technologyreview.com", "fulfillment.technologyreview.com"], "Nieuwsbrieven.Leren.MIT Technology Review"),
(["mit.edu"], "Nieuwsbrieven.Leren.MIT Open Learning"),
(["storytellingwithdata.com"], "Nieuwsbrieven.Leren.Storytelling with Data"),
(["linkingyourthinking.com"], "Nieuwsbrieven.Leren.Nick Milo"),
(["evakeiffenheim.com"], "Nieuwsbrieven.Leren.Eva Keiffenheim"),
(["tiggelaar.nl"], "Nieuwsbrieven.Leren.Ben Tiggelaar"),
(["briankeating.com"], "Nieuwsbrieven.Leren.Brian Keating"),
(["beehiiv.com", "mail.beehiiv.com"], "Nieuwsbrieven.Leren.AI Report"),
(["elizabethbutlermd.com"], "Nieuwsbrieven.Leren.Elizabeth Butler"),
(["notify.orcid.org"], "Nieuwsbrieven.Leren.ORCID"),
(["crisismanager.nl"], "Nieuwsbrieven.Nieuws & Vakbladen.Vakblad Crisismanager"),
(["jaarbeurs.nl"], "Nieuwsbrieven.Nieuws & Vakbladen.Computable"),
(["presentation-guru.com"], "Nieuwsbrieven.Nieuws & Vakbladen.The Presentation Guru"),
(["lendahand.com"], "Nieuwsbrieven.Overig.Lendahand"),
(["thegoodroll.nl"], "Nieuwsbrieven.Overig.The Good Roll"),
(["tasks.clickup.com"], "Afmelden.Software.ClickUp"),
(["gemeenteprojecten.talent-pool.com"], "Afmelden.Werk.Gemeenteprojecten"),
(["engage.istockphoto.com"], "Afmelden.Creatief.iStock"),
(["dmarc.postmarkapp.com"], "Technisch.DMARC"),
]
DOMAIN_LOOKUP: dict[str, str] = {
domain.lower(): target
for domains, target in DOMAIN_ROUTES
for domain in domains
}
def _normalize_domain(value: str) -> str:
domain = value.strip().lower()
if not domain or "@" in domain or "/" in domain or any(ch.isspace() for ch in domain):
raise ValueError(f"Invalid domain route domain: {value!r}")
return domain
def _normalize_mailbox(value: str) -> str:
mailbox = value.strip()
if not mailbox or mailbox.startswith(PREFIX) or mailbox.startswith(".") or mailbox.endswith(".") or ".." in mailbox:
raise ValueError(f"Invalid domain route mailbox: {value!r}")
return mailbox
def normalize_domain_route(rule: dict) -> tuple[list[str], str]:
"""Return a validated domain route as (domains, mailbox)."""
if not isinstance(rule, dict):
raise ValueError(f"Invalid domain route entry: {rule!r}")
raw_domains = rule.get("domains")
if not isinstance(raw_domains, list):
raise ValueError(f"Domain route domains must be a list: {rule!r}")
domains = [_normalize_domain(str(item)) for item in raw_domains]
domains = list(dict.fromkeys(domains))
if not domains:
raise ValueError(f"Domain route must contain at least one domain: {rule!r}")
mailbox = _normalize_mailbox(str(rule.get("mailbox", "")))
return domains, mailbox
@cache
def domain_routes() -> list[tuple[list[str], str]]:
"""Return validated domain routing rules from JSON policy."""
raw = json.loads(DOMAIN_ROUTES_FILE.read_text(encoding="utf-8"))
if not isinstance(raw, list):
raise ValueError("domain_routes.json must contain a list")
return [normalize_domain_route(rule) for rule in raw]
@cache
def domain_lookup() -> dict[str, str]:
"""Return domain-to-mailbox lookup built from domain_routes.json."""
lookup: dict[str, str] = {}
for domains, target in domain_routes():
for domain in domains:
lookup[domain] = target
return lookup
# Compatibility for existing scripts/tests that imported DOMAIN_ROUTES directly.
DOMAIN_ROUTES = domain_routes()
DOMAIN_LOOKUP = domain_lookup()
FACTUUR_KEYWORDS = [
"factuur", "invoice", "rekening", "nota", "betaalverzoek",
@@ -179,7 +134,7 @@ def route_from_subject(from_addr: str, subject: str) -> str | None:
return "__FACTUUR_DATE__"
if destination := mailinglist_destination(from_addr):
return destination
return DOMAIN_LOOKUP.get(sender_domain(from_addr))
return domain_lookup().get(sender_domain(from_addr))
def is_source_folder(folder: str) -> bool:
@@ -189,7 +144,7 @@ def is_source_folder(folder: str) -> bool:
def destination_folders(start_year: int = 2020, end_year: int = 2026) -> list[str]:
"""Return all destination folders without the INBOX. prefix."""
folders = {target for _, target in DOMAIN_ROUTES}
folders = {target for _, target in domain_routes()}
folders.update(
rule["mailbox"]
for rule in mailinglist_routes()
+453
View File
@@ -0,0 +1,453 @@
"""Small Basic Auth web UI for managing mailcat domain routes."""
from __future__ import annotations
import argparse
import base64
import html
import json
import os
import secrets
import shutil
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import urlparse
from mail_routes import DOMAIN_ROUTES_FILE, domain_lookup, domain_routes, normalize_domain_route
DEFAULT_CONFIG_FILE = Path(__file__).parent / "web_config.json"
DEFAULT_HOST = "0.0.0.0"
DEFAULT_PORT = 4321
def validate_routes(raw_routes: list[dict]) -> list[dict]:
"""Return normalized route dictionaries and reject duplicate domains."""
if not isinstance(raw_routes, list):
raise ValueError("Routes must be a list")
normalized: list[dict] = []
seen: dict[str, int] = {}
for index, raw_rule in enumerate(raw_routes):
domains, mailbox = normalize_domain_route(raw_rule)
for domain in domains:
if domain in seen:
other = seen[domain] + 1
raise ValueError(f"Domain {domain!r} is already used by route {other}")
seen[domain] = index
normalized.append({"domains": domains, "mailbox": mailbox})
return normalized
def load_routes(path: Path = DOMAIN_ROUTES_FILE) -> list[dict]:
raw = json.loads(path.read_text(encoding="utf-8"))
return validate_routes(raw)
def save_routes(routes: list[dict], path: Path = DOMAIN_ROUTES_FILE) -> list[dict]:
normalized = validate_routes(routes)
tmp_path = path.with_name(f".{path.name}.tmp")
backup_path = path.with_suffix(f"{path.suffix}.bak")
tmp_path.write_text(json.dumps(normalized, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
if path.exists():
shutil.copy2(path, backup_path)
os.replace(tmp_path, path)
domain_routes.cache_clear()
domain_lookup.cache_clear()
return normalized
def load_web_config(path: Path) -> tuple[str, str]:
config = {}
if path.exists():
config = json.loads(path.read_text(encoding="utf-8"))
username = os.environ.get("MAILCAT_WEB_USER") or config.get("username")
password = os.environ.get("MAILCAT_WEB_PASSWORD") or config.get("password")
if not username or not password:
raise SystemExit(
f"Missing web credentials. Create {path} or set MAILCAT_WEB_USER and MAILCAT_WEB_PASSWORD."
)
return str(username), str(password)
def basic_auth_ok(header: str | None, username: str, password: str) -> bool:
if not header or not header.startswith("Basic "):
return False
try:
decoded = base64.b64decode(header[6:], validate=True).decode("utf-8")
supplied_user, supplied_password = decoded.split(":", 1)
except Exception:
return False
return secrets.compare_digest(supplied_user, username) and secrets.compare_digest(
supplied_password, password
)
def page_html() -> bytes:
return f"""<!doctype html>
<html lang="nl">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Mailcat routes</title>
<style>
:root {{
color-scheme: light;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
background: #f5f6f8;
color: #20242a;
}}
body {{ margin: 0; }}
header {{
display: flex;
align-items: center;
justify-content: space-between;
gap: 16px;
padding: 14px 20px;
background: #17324d;
color: #fff;
}}
h1 {{ font-size: 18px; line-height: 1.2; margin: 0; font-weight: 650; }}
main {{ padding: 18px 20px 28px; max-width: 1280px; margin: 0 auto; }}
.toolbar {{
display: grid;
grid-template-columns: minmax(180px, 1fr) auto auto;
gap: 10px;
align-items: center;
margin-bottom: 14px;
}}
input, textarea {{
width: 100%;
box-sizing: border-box;
font: inherit;
border: 1px solid #b9c1cc;
border-radius: 6px;
padding: 8px 10px;
background: #fff;
color: #20242a;
}}
textarea {{ min-height: 72px; resize: vertical; }}
button {{
border: 1px solid #245782;
background: #245782;
color: #fff;
border-radius: 6px;
padding: 8px 12px;
font: inherit;
cursor: pointer;
white-space: nowrap;
}}
button.secondary {{ background: #fff; color: #245782; }}
button.danger {{ background: #8c2634; border-color: #8c2634; }}
button:disabled {{ opacity: .55; cursor: default; }}
.layout {{ display: grid; grid-template-columns: 1fr 360px; gap: 18px; align-items: start; }}
table {{ width: 100%; border-collapse: collapse; background: #fff; border: 1px solid #d7dce2; }}
th, td {{ text-align: left; vertical-align: top; border-bottom: 1px solid #e2e6eb; padding: 8px 10px; }}
th {{ background: #edf1f5; font-size: 13px; color: #34495e; }}
tr.selected {{ background: #eaf4ff; }}
td.index {{ width: 56px; color: #596775; }}
td.actions {{ width: 96px; text-align: right; }}
.panel {{ background: #fff; border: 1px solid #d7dce2; padding: 14px; }}
.panel h2 {{ margin: 0 0 12px; font-size: 15px; }}
label {{ display: block; margin: 10px 0 6px; font-size: 13px; color: #34495e; }}
.form-actions {{ display: flex; gap: 8px; margin-top: 12px; flex-wrap: wrap; }}
.status {{ min-height: 20px; font-size: 13px; color: #34495e; }}
.error {{ color: #8c2634; }}
.domains {{ line-height: 1.45; }}
.mailbox {{ font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 13px; }}
@media (max-width: 860px) {{
.layout {{ grid-template-columns: 1fr; }}
.toolbar {{ grid-template-columns: 1fr; }}
main {{ padding: 14px; }}
}}
</style>
</head>
<body>
<header>
<h1>Mailcat routes</h1>
<div id="count"></div>
</header>
<main>
<div class="toolbar">
<input id="search" type="search" placeholder="Zoek domein of map">
<button id="newButton" type="button" class="secondary">Nieuwe regel</button>
<button id="reloadButton" type="button" class="secondary">Herladen</button>
</div>
<div class="layout">
<table>
<thead><tr><th>#</th><th>Domeinen</th><th>Map</th><th></th></tr></thead>
<tbody id="routesBody"></tbody>
</table>
<section class="panel">
<h2 id="formTitle">Nieuwe regel</h2>
<form id="routeForm">
<input id="routeIndex" type="hidden">
<label for="domains">Domeinen, een per regel of komma-gescheiden</label>
<textarea id="domains" required></textarea>
<label for="mailbox">Doelmap zonder INBOX.</label>
<input id="mailbox" required placeholder="Diensten.AI.OpenAI">
<div class="form-actions">
<button id="saveButton" type="submit">Opslaan</button>
<button id="deleteButton" type="button" class="danger" disabled>Verwijderen</button>
<button id="clearButton" type="button" class="secondary">Leegmaken</button>
</div>
</form>
<p id="status" class="status"></p>
</section>
</div>
</main>
<script>
let routes = [];
let selectedIndex = null;
const $ = (id) => document.getElementById(id);
function splitDomains(value) {{
return value.split(/[\\n,]/).map((item) => item.trim()).filter(Boolean);
}}
function setStatus(text, isError = false) {{
$("status").textContent = text;
$("status").className = isError ? "status error" : "status";
}}
function clearForm() {{
selectedIndex = null;
$("routeIndex").value = "";
$("domains").value = "";
$("mailbox").value = "";
$("formTitle").textContent = "Nieuwe regel";
$("deleteButton").disabled = true;
renderRoutes();
}}
function editRoute(index) {{
const route = routes[index];
selectedIndex = index;
$("routeIndex").value = String(index);
$("domains").value = route.domains.join("\\n");
$("mailbox").value = route.mailbox;
$("formTitle").textContent = `Regel ${{index + 1}} bewerken`;
$("deleteButton").disabled = false;
renderRoutes();
}}
async function requestJson(url, options = {{}}) {{
const response = await fetch(url, {{
headers: {{ "Content-Type": "application/json", ...(options.headers || {{}}) }},
...options,
}});
const payload = await response.json().catch(() => ({{ error: response.statusText }}));
if (!response.ok) throw new Error(payload.error || response.statusText);
return payload;
}}
async function loadRoutes() {{
const payload = await requestJson("/api/routes");
routes = payload.routes;
$("count").textContent = `${{routes.length}} regels`;
renderRoutes();
setStatus("Geladen.");
}}
function renderRoutes() {{
const needle = $("search").value.trim().toLowerCase();
const rows = routes
.map((route, index) => ({{ route, index }}))
.filter(({{ route }}) => !needle || route.mailbox.toLowerCase().includes(needle) || route.domains.join(" ").toLowerCase().includes(needle))
.map(({{ route, index }}) => `
<tr class="${{selectedIndex === index ? "selected" : ""}}">
<td class="index">${{index + 1}}</td>
<td class="domains">${{route.domains.map(escapeHtml).join("<br>")}}</td>
<td class="mailbox">${{escapeHtml(route.mailbox)}}</td>
<td class="actions"><button type="button" class="secondary" onclick="editRoute(${{index}})">Bewerk</button></td>
</tr>`);
$("routesBody").innerHTML = rows.join("");
}}
function escapeHtml(value) {{
return value.replace(/[&<>"']/g, (ch) => ({{ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;", "'": "&#39;" }}[ch]));
}}
$("routeForm").addEventListener("submit", async (event) => {{
event.preventDefault();
const route = {{ domains: splitDomains($("domains").value), mailbox: $("mailbox").value.trim() }};
const indexValue = $("routeIndex").value;
try {{
const method = indexValue === "" ? "POST" : "PUT";
const url = indexValue === "" ? "/api/routes" : `/api/routes/${{indexValue}}`;
const payload = await requestJson(url, {{ method, body: JSON.stringify(route) }});
routes = payload.routes;
clearForm();
setStatus("Opgeslagen.");
}} catch (error) {{
setStatus(error.message, true);
}}
}});
$("deleteButton").addEventListener("click", async () => {{
if (selectedIndex === null) return;
if (!confirm(`Regel ${{selectedIndex + 1}} verwijderen?`)) return;
try {{
const payload = await requestJson(`/api/routes/${{selectedIndex}}`, {{ method: "DELETE" }});
routes = payload.routes;
clearForm();
setStatus("Verwijderd.");
}} catch (error) {{
setStatus(error.message, true);
}}
}});
$("search").addEventListener("input", renderRoutes);
$("newButton").addEventListener("click", clearForm);
$("clearButton").addEventListener("click", clearForm);
$("reloadButton").addEventListener("click", loadRoutes);
loadRoutes().catch((error) => setStatus(error.message, true));
</script>
</body>
</html>
""".encode("utf-8")
class RouteHandler(BaseHTTPRequestHandler):
routes_file = DOMAIN_ROUTES_FILE
username = ""
password = ""
def log_message(self, fmt: str, *args: object) -> None:
print(f"{self.address_string()} - {fmt % args}")
def _require_auth(self) -> bool:
if basic_auth_ok(self.headers.get("Authorization"), self.username, self.password):
return True
self.send_response(HTTPStatus.UNAUTHORIZED)
self.send_header("WWW-Authenticate", 'Basic realm="mailcat"')
self.send_header("Content-Length", "0")
self.end_headers()
return False
def _send(self, status: HTTPStatus, body: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
def _json(self, status: HTTPStatus, payload: dict) -> None:
self._send(
status,
json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8"),
"application/json; charset=utf-8",
)
def _error(self, status: HTTPStatus, message: str) -> None:
self._json(status, {"error": message})
def _read_route_body(self) -> dict:
length = int(self.headers.get("Content-Length", "0"))
body = self.rfile.read(length)
payload = json.loads(body.decode("utf-8"))
if not isinstance(payload, dict):
raise ValueError("Request body must contain one route object")
return payload
def _route_index(self) -> int | None:
parts = [part for part in urlparse(self.path).path.split("/") if part]
if len(parts) != 3 or parts[:2] != ["api", "routes"]:
return None
try:
return int(parts[2])
except ValueError:
return None
def do_GET(self) -> None:
if not self._require_auth():
return
path = urlparse(self.path).path
if path == "/":
self._send(HTTPStatus.OK, page_html(), "text/html; charset=utf-8")
return
if path == "/api/routes":
try:
self._json(HTTPStatus.OK, {"routes": load_routes(self.routes_file)})
except Exception as exc:
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
return
self._error(HTTPStatus.NOT_FOUND, "Not found")
def do_POST(self) -> None:
if not self._require_auth():
return
if urlparse(self.path).path != "/api/routes":
self._error(HTTPStatus.NOT_FOUND, "Not found")
return
try:
routes = load_routes(self.routes_file)
routes.append(self._read_route_body())
self._json(HTTPStatus.CREATED, {"routes": save_routes(routes, self.routes_file)})
except json.JSONDecodeError:
self._error(HTTPStatus.BAD_REQUEST, "Invalid JSON")
except ValueError as exc:
self._error(HTTPStatus.BAD_REQUEST, str(exc))
except Exception as exc:
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
def do_PUT(self) -> None:
if not self._require_auth():
return
index = self._route_index()
if index is None:
self._error(HTTPStatus.NOT_FOUND, "Not found")
return
try:
routes = load_routes(self.routes_file)
if index < 0 or index >= len(routes):
raise ValueError("Route index out of range")
routes[index] = self._read_route_body()
self._json(HTTPStatus.OK, {"routes": save_routes(routes, self.routes_file)})
except json.JSONDecodeError:
self._error(HTTPStatus.BAD_REQUEST, "Invalid JSON")
except ValueError as exc:
self._error(HTTPStatus.BAD_REQUEST, str(exc))
except Exception as exc:
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
def do_DELETE(self) -> None:
if not self._require_auth():
return
index = self._route_index()
if index is None:
self._error(HTTPStatus.NOT_FOUND, "Not found")
return
try:
routes = load_routes(self.routes_file)
if index < 0 or index >= len(routes):
raise ValueError("Route index out of range")
del routes[index]
self._json(HTTPStatus.OK, {"routes": save_routes(routes, self.routes_file)})
except ValueError as exc:
self._error(HTTPStatus.BAD_REQUEST, str(exc))
except Exception as exc:
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
def main() -> None:
parser = argparse.ArgumentParser(description="Run the mailcat route management web UI.")
parser.add_argument("--host", default=DEFAULT_HOST)
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_FILE)
parser.add_argument("--routes-file", type=Path, default=DOMAIN_ROUTES_FILE)
args = parser.parse_args()
username, password = load_web_config(args.config)
RouteHandler.routes_file = args.routes_file
RouteHandler.username = username
RouteHandler.password = password
server = ThreadingHTTPServer((args.host, args.port), RouteHandler)
print(f"mailcat route web listening on http://{html.escape(args.host)}:{args.port}/")
server.serve_forever()
if __name__ == "__main__":
main()
+184 -202
View File
@@ -6,36 +6,36 @@ You are continuing the `mailcat` project in `/Users/hanswienen/Documents/Develop
Build a reliable mailbox cleanup and automation toolkit for `australius.nl`.
The project started as an offline analysis of a local mailbox export and is being moved toward an IMAP-based sorter because Sieve does not work with the email provider. The intended production model is:
The operational direction is Python over IMAP/SMTP because Sieve is not viable with the provider. The current test target is `backup@australius.nl`. The user wants to test always-on automation first on a local QNAP-like NAS named `hades`, not on `vps.australius.nl`.
- Email is hosted at `australius.nl`.
- A VPS is available at `vps.austalius.nl`, but the user now wants to test automation first on a local home NAS instead of the VPS.
- The test mailbox is `backup@australius.nl`.
- The future automation should run on an always-on remote host, preferably via SSH deployment and an IMAP IDLE daemon. The NAS is QNAP-like BusyBox Linux without systemd, so test deployment should use `SERVICE_MANAGER=plain`.
- The current end-to-end test uses the `backup` account as a disposable test target.
Do not run mailbox-affecting scripts yourself unless the user explicitly authorizes it. The user prefers to run those scripts personally.
## Repository State
## Collaboration Rules
Git workflow:
- Follow the git workflow from `AGENTS.md`.
- Work on branch `codex` unless the user says otherwise.
- Before every commit, rewrite this file from scratch as the current reconstruction prompt.
- Commit after each user-prompted code change using the required three-section commit message format.
- The user asked: first provide a plan and wait for approval before doing future work.
- Do not stage local runtime output such as `verplaats_log.json` unless the user explicitly asks.
- The repo has branches `main`, `claude`, and `codex`.
- Work should continue on `codex` unless the user says otherwise.
- Follow the user workflow: initialize Git when needed, branch before work in existing repos, and commit after each user-prompted change using the required three-section commit message format.
- Before every commit, rewrite this file from scratch so it fully reconstructs the current project state.
## Local Data And Secrets
Ignored local data includes:
Ignored or local-only data includes:
- `mailbox/`
- `**/*.eml`
- `config.json`
- `web_config.json`
- `__pycache__/`
- bytecode files
- `*.log`
- `.DS_Store`
- generated runtime state/log JSON files
Do not commit email `.eml` files or credentials.
Do not commit credentials.
## Current Decisions
## Routing Policy
Sorting policy:
@@ -43,232 +43,214 @@ Sorting policy:
- `Sent`, `Drafts`, `Trash`, and `Spam` are excluded from sorting.
- Every other mailbox folder is treated as a source.
- Do not use `Archief` as a final destination.
- Existing `Archief.*` folders are source folders; route matching messages out of them into functional destination folders.
- Existing `Archief.*` folders are source folders. Matching messages should move out into functional folders.
- If an archived/source message does not match a rule, leave it in place.
Destination policy:
- Final destination folders are functional folders such as `Financieel`, `Werk`, `Diensten`, `Nieuwsbrieven`, `Bestellingen`, `Mobiliteit`, and `Technisch`.
- Final destination folders under `Archief` should not be created.
- `Technisch` and `Technisch.DMARC` are valid destination folders.
- `Administratie` is a valid destination root; KvK messages route to `Administratie.KvK`.
- Former assignment senders should be routed into separate folders under `Werk.Opdrachten.*`.
- `zuiderzee.net` and generic `gmail.com` messages should remain unmatched for now.
- Final destination folders are functional folders such as `Financieel`, `Werk`, `Diensten`, `Nieuwsbrieven`, `Bestellingen`, `Mobiliteit`, `Administratie`, and `Technisch`.
- `Technisch` and `Technisch.DMARC` are valid destinations.
- `Administratie.KvK` is the destination for KvK messages.
- Former assignment senders route under `Werk.Opdrachten.*`.
- `zuiderzee.net` and generic `gmail.com` intentionally remain unmatched.
- AI providers route under `Diensten.AI`; confirmed AI providers include Canva, Claude/Anthropic, Cursor, Huckr, Jamie, Mistral, OpenAI, and Perplexity.
- Microsoft, Zapier, Envato, reMarkable, Nord, and ExpressVPN are not treated as AI providers unless the user reclassifies them.
Sieve:
- `mailrules.sieve` exists as historical/source material.
- Sieve deployment is not viable with the provider.
- Future work should move operational automation to Python over IMAP/SMTP.
## Current Implementation Notes
## Current Implementation
Shared modules:
- `imap_utils.py` centralizes IMAP modified UTF-7 folder encoding/decoding, quoted mailbox names, LIST parsing, and folder listing.
- `mail_routes.py` centralizes `PREFIX`, domain routes, invoice keywords, source-folder exclusions, already-sorted folders, invoice quarter routing, generated destination folders, and normalized mailing-list routing.
- `mail_routes.py` centralizes routing behavior: invoice keyword detection, mailing-list routing, domain routing, source-folder exclusions, already-sorted folders, invoice quarter routing, and destination folder generation.
- `mail_imap_ops.py` centralizes message header extraction, destination computation, destination mailbox creation, and UID COPY plus UID STORE move semantics.
Current INBOX candidate domain routing additions:
Domain routes:
- `mijnkantoorapp.nl` and `dijkman-ac.nl` route to `Financieel.Boekhouding.Dijkman`.
- `drenthe.nl`, `rdw.nl`, and `plasbossinade.nl` route to separate folders under `Werk.Opdrachten`.
- `kvk.nl` routes to `Administratie.KvK`.
- `godaddy.com`, `cloud86.io`, and `cloud86.zendesk.com` route under `Diensten.Hosting`.
- Clear AI providers route under `Diensten.AI`: Canva, Claude/Anthropic, Cursor, Huckr, Jamie, Mistral, OpenAI, and Perplexity.
- `microsoft.com`, `market.envato.com`, `email.remarkable.com`, `nordaccount.com`, and `info.expressvpn.com` route under non-AI `Diensten` folders unless the user explicitly reclassifies them. Microsoft and Zapier should not be treated as AI providers for now.
- `maandag.com` routes under `Werk.Opdrachten`.
- `tasks.clickup.com`, `gemeenteprojecten.talent-pool.com`, and `engage.istockphoto.com` route under `Afmelden`.
- `n26.com`, `update.bunq.com`, and `hello.bunq.com` route under `Financieel.Bank`.
- `notify.orcid.org` routes to `Nieuwsbrieven.Leren.ORCID`.
- `zuiderzee.net` and `gmail.com` are intentionally not routed.
- Hardcoded domain routes have been moved out of Python into `domain_routes.json`.
- `mail_routes.domain_routes()` loads and validates `domain_routes.json`.
- `mail_routes.domain_lookup()` builds the cached domain lookup.
- `DOMAIN_ROUTES` and `DOMAIN_LOOKUP` still exist for compatibility, but active routing uses `domain_lookup()`.
- If a long-running sorter process is already running, route edits on disk require restarting that process before it sees the new rules.
Scripts using shared IMAP folder handling:
Mailing-list routes:
- `verplaats_bestaand.py`
- `maak_mappen.py`
- `kopieer_naar_backup.py`
- `download_mailbox.py`
- `dagelijks_overzicht.py`
- `sort_mail_daemon.py`
- `mailinglist_routes.json` is still separate from domain routes.
- `mail_routes.py` checks mailing-list routes before domain routes.
- `mailinglist_routes.json` covers normalized policy from `decisions.json`, including `Afmelden.*` destinations.
- Mailing-list routing supports domain and `From` substring matching; it does not yet fetch or match `List-ID`.
`dagelijks_overzicht.py` current behavior:
Historical sorting:
- Builds an HTML digest of messages that arrived on the selected date, defaulting to yesterday.
- Uses central `config.py`; select the mailbox account with `--account`, for example `--account backup`.
- Sends the digest via SMTP using the selected account credentials.
- Supports `--mail-to <address>`; the NAS daily report target is `hans@australius.nl`.
- Sends the report even when zero messages are found, so scheduled runs are visible.
- `verplaats_bestaand.py` defaults to dry-run. Real moves require `--uitvoeren` and interactive `JA`.
- Account selection is explicit with `--account <naam>`, for example `--account backup`.
- `--audit` is available in dry-run mode.
- It logs to `verplaats_log.json`, retries IMAP aborts where implemented, and parses FETCH responses defensively.
Shared operation helpers:
Folder creation:
- `mail_imap_ops.py` centralizes header decoding, FETCH payload extraction, routing fetched headers to full IMAP destinations, destination mailbox creation, and copy/delete move semantics for live sorting.
- `verplaats_bestaand.py` still contains its own historical-sorter retry/logging flow; do not refactor it casually because it has already been exercised against the backup mailbox.
- `maak_mappen.py` creates destination folders generated from `mail_routes.destination_folders()`.
- It creates parent folders as needed and avoids `Archief.*` destinations.
`verplaats_bestaand.py` current behavior:
Backup mirror:
- Default mode is dry-run; real moves require `--uitvoeren` and an interactive `JA` confirmation.
- Account selection is explicit with `--account <naam>`; use `--account backup` for the disposable backup mailbox. The script prints both the config account name and mailbox address before connecting.
- `--audit` is available only in dry-run mode. It suppresses per-message dry-run lines and prints planned move counts grouped by source folder and destination folder for step 2 and step 3.
- Long scans show progress counters: source folders are printed as `[current/total]`, and audit mode prints `Gescand: processed/total` every 250 messages and at folder completion.
- IMAP `LIST`, `SELECT`, `UID SEARCH`, and `UID FETCH` aborts are logged and retried once after reconnecting. If reconnect also fails, the script records an `ABORT`/`ERROR` in `verplaats_log.json` instead of printing a Python traceback.
- IMAP `FETCH` responses are parsed by selecting the first tuple bytes payload. This avoids crashes when `imaplib` returns extra response items before/after the actual header payload.
- `MAP_RENAMES` is intentionally empty.
- Source folders are selected by shared `list_folders()` plus `mail_routes.is_source_folder()`.
- Step 2 routes all eligible source folders, not just `INBOX`.
- Step 2 reports counters per folder and in total: planned moves, actual moves when executing, no-match messages, skipped messages, and fetch/move failures.
- Dry runs report `Gepland` counters instead of misleadingly showing zero moved.
- Step 3 invoice-quarter sorting also reports planned/moved/failure counters and supports audit summaries.
- Move success requires source reselect, `UID COPY`, and `UID STORE +FLAGS \Deleted` to return `OK`.
- Move and fetch failures are recorded in `verplaats_log.json` under `fouten` with step, source folder, UID, destination when known, action, status, and server response.
- Step 3 no longer treats `INBOX.Facturen - verwerkt` as a source.
- IMAP mailbox names are consistently quoted and encoded through `imap_utils.quote_mailbox()`.
- `kopieer_naar_backup.py` mirrors configured IMAP accounts, deduplicating by `Message-ID` per destination folder.
- It uses `backup_log.json` for progress and does not mark a folder complete if failures occurred.
`maak_mappen.py` current behavior:
Live daemon:
- Destination folders are generated from `mail_routes.destination_folders()`.
- It no longer creates `Archief.*` destination folders.
- It creates all parent folders needed for route targets, including `Technisch` and `Technisch.DMARC`.
- IMAP mailbox names are consistently quoted and encoded through `imap_utils.quote_mailbox()`.
- `sort_mail_daemon.py` defaults to `--account backup --folder INBOX`.
- It uses shared routing through `mail_imap_ops.destination_from_header()`.
- It maintains processed UID state and resets when UIDVALIDITY changes.
- It supports `--once` and `--dry-run`.
- It uses IMAP IDLE where available, falls back conservatively, reconnects on failures, and logs to `sort_mail_daemon.log` by default.
`kopieer_naar_backup.py` current behavior:
Daily report:
- Mirrors folders from one configured IMAP account to another.
- Skips `INBOX.Trash` and `INBOX.Spam`.
- Deduplicates by `Message-ID` within each destination folder.
- Uses shared folder listing and mailbox quoting/encoding for select/create/append operations.
- Uses `backup_log.json` to record completed folders and the copied-message total.
- Records select/search/fetch/append failures in `backup_log.json` under `fouten`.
- Verifies the destination folder is selectable before dedupe and append work.
- Does not mark a folder complete if any failure occurs while copying that folder.
- `dagelijks_overzicht.py` builds an HTML digest for a selected date, defaulting to yesterday.
- It supports `--account` and `--mail-to`.
- The NAS daily report target is `hans@australius.nl`.
- It sends a report even when there are zero messages.
`sort_mail_daemon.py` current behavior:
## Web Route Management
- Intended remote-host automation entry point for IMAP sorting.
- Defaults to `--account backup` and `--folder INBOX`.
- Uses the same shared routing policy through `mail_imap_ops.destination_from_header()`.
- Maintains a JSON state file of processed UIDs per folder and resets that state when UIDVALIDITY changes.
- Supports `--once` for local/non-daemon smoke tests and `--dry-run` for non-mutating checks.
- Watches for new mail with IMAP IDLE. On Python versions with public `imaplib.IMAP4.idle()` it uses that API; otherwise it uses a conservative private-IDLE fallback and reconnects on failures.
- Logs to `sort_mail_daemon.log` by default, or to the path passed with `--log-file`.
- Ensures the destination mailbox exists before moving a message, then moves by UID COPY plus UID STORE `\Deleted`, followed by expunge after a scan.
The route-management web interface is implemented in `manage_routes_web.py`.
Remote/NAS deployment files:
Current decisions:
- `deploy_vps.sh` is host-neutral despite its historical filename. It packages the repo excluding `.git`, `config.json`, logs, runtime JSON state/log files, bytecode, and local mailbox data; uploads to the SSH host in `REMOTE_HOST`; installs under `/opt/mailcat` by default; creates/uses a non-root system user named `mailcat` by default; installs a systemd service; enables but does not start the service.
- The deploy script requires `REMOTE_HOST` and `REMOTE_USER`. The old `VPS_HOST` and `VPS_USER` names still work as aliases. Optional overrides: `APP_USER`, `APP_DIR`, `SERVICE_NAME`, and `ACCOUNT`.
- The deploy script uploads a temporary remote shell script and runs it with `ssh -tt`, so remote `sudo` can prompt for a password on NAS systems that require a terminal. The remote script starts with `sudo -v`.
- The deploy script exports an explicit remote PATH before running install commands: `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` by default, overrideable with `REMOTE_PATH`. It searches both PATH and common absolute paths such as `/usr/local/bin/useradd`, `/usr/sbin/useradd`, and `/sbin/useradd`. If `useradd` is genuinely unavailable, it falls back to the existing SSH user and that user's primary group for service ownership.
- For the local NAS, use `SERVICE_MANAGER=plain`, `MAILCAT_HOME=/share/homes/mailcat`, and `PYTHON=/opt/bin/python3`. In plain mode, default paths are `APP_DIR=$MAILCAT_HOME/mailcat`, `STATE_DIR=$APP_DIR/state`, `LOG_DIR=$APP_DIR/log`, `RUN_DIR=$APP_DIR/run`, and `PID_FILE=$RUN_DIR/mailcat.pid`.
- QNAP path facts supplied by the user: `SUDO=/usr/bin/sudo`, `USERADD=/usr/local/bin/useradd`, `PYTHON=/opt/bin/python3`, `INSTALL=/usr/bin/install`, `SED=/bin/sed`, `TAR=/bin/tar`, `MKDIR=/bin/mkdir`, `CHOWN=/bin/chown`, `CHMOD=/bin/chmod`, and `RM=/bin/rm`. `nologin`, `python3`, `systemctl`, and `nohup` are not in the user's default PATH.
- The deploy script accepts absolute tool overrides for those commands and passes them into the remote script. In plain mode, the generated start/stop/status scripts also bake in absolute `MKDIR` and `RM` paths.
- In plain mode, the deploy script does not install systemd files. It writes executable scripts to `$APP_DIR/bin/start_mailcat.sh`, `$APP_DIR/bin/stop_mailcat.sh`, `$APP_DIR/bin/status_mailcat.sh`, and `$APP_DIR/bin/run_daily_report.sh`.
- `DAILY_REPORT_TO` controls the report recipient during deploy and defaults to `hans@australius.nl`. The daily report wrapper appends output to `$LOG_DIR/daily_report.out`.
- `systemd/mailcat-sort.service` is a template consumed by `deploy_vps.sh`; after placeholder replacement it runs `sort_mail_daemon.py --account backup --folder INBOX` and stores state/logs under `/var/lib/mailcat` and `/var/log/mailcat`.
- `config.json` remains ignored and must be created manually on the remote host with mode `600` before starting the service.
- LAN binding: `0.0.0.0`
- Port: `4321`
- Authentication: HTTP Basic Auth
- Credentials come from ignored `web_config.json` or environment variables `MAILCAT_WEB_USER` and `MAILCAT_WEB_PASSWORD`.
- Example config is in `web_config.example.json`.
Scope:
- The web UI manages all domain routes in `domain_routes.json`, including the previous built-in routes.
- Mailing-list routes remain in `mailinglist_routes.json` and are not yet managed by the web UI.
Behavior:
- `GET /` serves a dense table/form UI.
- `GET /api/routes` returns all routes.
- `POST /api/routes` appends a route.
- `PUT /api/routes/<index>` updates a route.
- `DELETE /api/routes/<index>` deletes a route.
- Domains and mailbox names are validated server-side.
- Duplicate domains across routes are rejected.
- Writes are atomic and create `domain_routes.json.bak`.
Security note:
- This is Basic Auth on the internal network. It is not HTTPS by itself. If exposed beyond the LAN, put it behind TLS or a VPN.
## NAS Deployment
`deploy_vps.sh` is host-neutral despite its historical filename.
For the QNAP-like NAS, use:
```sh
SERVICE_MANAGER=plain
MAILCAT_HOME=/share/homes/mailcat
PYTHON=/opt/bin/python3
APP_USER=mailcat
```
Known NAS command paths supplied by the user:
- `SUDO=/usr/bin/sudo`
- `USERADD=/usr/local/bin/useradd`
- `PYTHON=/opt/bin/python3`
- `INSTALL=/usr/bin/install`
- `SED=/bin/sed`
- `TAR=/bin/tar`
- `MKDIR=/bin/mkdir`
- `CHOWN=/bin/chown`
- `CHMOD=/bin/chmod`
- `RM=/bin/rm`
In `SERVICE_MANAGER=plain` mode:
- Default `APP_DIR` is `/share/homes/mailcat/mailcat`.
- Default `STATE_DIR` is `$APP_DIR/state`.
- Default `LOG_DIR` is `$APP_DIR/log`.
- Default `RUN_DIR` is `$APP_DIR/run`.
- The sorter PID file is `$RUN_DIR/mailcat.pid`.
- The web PID file is `$RUN_DIR/mailcat_web.pid`.
- The deploy script installs:
- `$APP_DIR/bin/start_mailcat.sh`
- `$APP_DIR/bin/stop_mailcat.sh`
- `$APP_DIR/bin/status_mailcat.sh`
- `$APP_DIR/bin/run_daily_report.sh`
- `$APP_DIR/bin/start_web.sh`
- `$APP_DIR/bin/stop_web.sh`
- `$APP_DIR/bin/status_web.sh`
- The sorter appends wrapper stdout/stderr to `$LOG_DIR/daemon.out`.
- The web UI appends stdout/stderr to `$LOG_DIR/web.out`.
- The daily report wrapper appends stdout/stderr to `$LOG_DIR/daily_report.out`.
Before starting services on the NAS:
- Create `/share/homes/mailcat/mailcat/config.json` with mode `600`.
- Create `/share/homes/mailcat/mailcat/web_config.json` with mode `600`, for example:
```json
{
"username": "mailcat",
"password": "use-a-real-password"
}
```
After route edits through the web UI, restart `start_mailcat.sh`/`stop_mailcat.sh` if the live sorter should use the new rules.
Systemd mode still exists for a future VPS deployment using `systemd/mailcat-sort.service`.
## Current Backup Test Status
The user reported that `backup@australius.nl` was empty except for folder structure, so the stale `backup_log.json` state was reset and the real mailbox state was inspected.
The backup mirror completed earlier:
Observed before the mirror rerun:
- Source eligible mirror scope: 34 folders, 11,203 messages.
- Script copied: 11,190 messages.
- Duplicate skips: 13.
- Backup account after mirror: 159 folders, 11,190 messages.
- Source `hans` account: 36 folders, 11,857 total messages.
- Eligible mirror scope: 34 folders, 11,203 messages, excluding only `INBOX.Spam` and `INBOX.Trash`.
- Backup account: 132 folders, 0 messages.
The user later ran the backup historical sorter successfully and reported:
After resetting the copy status, `python3 kopieer_naar_backup.py --van hans --naar backup` completed:
```text
Totaal gepland: 0 | Totaal geen match: 2218 | Totaal overgeslagen: 8219 | Totaal fouten: 0
```
- Script total copied: 11,190.
- Script duplicate skips: 13.
- `backup_log.json` marked 34 folders complete and recorded `totaal_gekopieerd: 11190`.
Treat that as a successful backup historical-sorter run with no immediate recovery work needed.
Server-side IMAP verification after the mirror:
## Verification Commands
- Source account still has 36 folders and 11,857 total messages.
- Eligible mirror scope is still 34 folders and 11,203 messages.
- Backup account has 159 folders and 11,190 total messages.
- Backup has 24 non-empty folders.
- The 13-message difference equals the script's duplicate `Message-ID` skips.
- Count deltas caused by duplicate skips:
- `INBOX.Archief.2020.verzonden`: source 162, backup 161.
- `INBOX.Archief.2021.inkomend`: source 894, backup 890.
- `INBOX.Archief.2022.verzonden`: source 174, backup 173.
- `INBOX.Facturen - verwerkt`: source 153, backup 151.
- `INBOX.Sent`: source 836, backup 831.
Latest local verification should include:
Empty source folders not visible as selectable backup folders after the mirror audit:
```sh
python3 -m py_compile *.py tests/*.py
bash -n deploy_vps.sh
python3 -m unittest discover -s tests
git diff --check
```
- `INBOX.Facturen - te verwerken`
- `INBOX.Notes`
- `INBOX.Technisch.dmarc`
Useful routing sanity check:
Do not run mailbox-affecting scripts yourself in this project. The user wants to run scripts personally when instructed. A sorter dry-run audit on `backup@australius.nl` is the next safe diagnostic, but ask the user to run it rather than running it yourself.
```sh
python3 - <<'PY'
from mail_routes import destination_folders, domain_routes, route_from_subject
print("routes", len(domain_routes()))
print("folders", len(destination_folders()))
print("openai", route_from_subject("noreply@openai.com", "update"))
print("huckr", route_from_subject("hello@huckr.ai", "update"))
PY
```
## Reports and Findings
## Next Expected Work
The prior script review is in `reports/script_review_findings.md`.
Likely next steps:
The decisions-based mailing-list routing proposal is in `reports/decisions_mailinglist_routing_report.md`.
That report interprets `decisions.json` as advisory intent:
- `keuze: "h"` means keep and route to an appropriate functional mailbox.
- `keuze: "a"` means route below `INBOX.Afmelden.*` for unsubscribe/review.
- `keuze: "s"` is ambiguous and should be confirmed before automation.
- Obvious aliases from the same sender or organization should be combined, including Coursera, MIT Technology Review, Eva Keiffenheim/Substack, PostNL, STRATO, Forte Labs, CIONET, Gusti, Nord, Proton, Vonage/Nexmo, and Visme.
- The normalized policy is in `mailinglist_routes.json`; `mail_routes.py` loads it before falling back to legacy `DOMAIN_ROUTES`.
- `mailinglist_routes.json` has 153 grouped rules and covers every key from `decisions.json` through `source_decision_keys`.
- `destination_folders()` includes functional keep folders plus `Afmelden.*` folders from `mailinglist_routes.json`.
- Invoice keyword routing still takes precedence over mailing-list routing.
- Mailing-list routing currently supports domain and `From` substring policy matching. It does not yet fetch or match `List-ID` headers.
Notable policy risks:
- `circle8.nl` still has ambiguous decision value `s`; keep it as review/functional routing until the user confirms the meaning.
- `alfen.com`, `vodafone.nl`, `dhlecommerce.nl`, and Proton splits are implemented according to `decisions.json` precedence, but remain notable policy changes compared with older hardcoded domain routes.
## Last Verification
Latest local code verification:
- `python3 -m py_compile *.py`
- `python3 -m py_compile *.py tests/*.py`
- `python3 -m unittest discover -s tests`
- `bash -n deploy_vps.sh`
- Explicit `route_from_subject()` checks for the latest INBOX candidate domains, including negative checks for `zuiderzee.net` and `gmail.com`.
- `git diff --check`
Earlier route/folder consistency check returned:
- `routes 72`
- `folders 127`
- `missing_targets []`
- `archief_targets []`
Latest live mailbox verification:
- Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror.
- The user ran the previous dry-run sorter version against `backup@australius.nl`. It reached `INBOX.Archief.2022.verzonden` after completing `INBOX.Archief.2022.inkomend`, then the IMAP server closed the connection with `Server shutting down.`. The current code has reconnect/retry handling for that failure mode.
- The user started an actual sorter run on `backup@australius.nl`; it began moving messages from `INBOX` and then crashed on an unexpected IMAP `FETCH` response shape: `AttributeError: 'int' object has no attribute 'decode'`. The local `verplaats_log.json` contains processed `INBOX:<uid>` entries from that run and should be preserved if the user resumes sorting. The current code fixes this parser crash.
- A later read-only INBOX inspection of the `backup` account found 1,908 unmatched messages and identified additional candidate sender domains. No mailbox-affecting script was run by Codex.
- After the latest routing refinements, the user reported the backup sorter ended with `Totaal gepland: 0 | Totaal geen match: 2218 | Totaal overgeslagen: 8219 | Totaal fouten: 0`. Treat this as a successful backup historical-sorter run with no immediate recovery work needed.
- Remote NAS/systemd deployment and daemon live tests have not yet been run by Codex.
## What To Do Next
Recommended next work:
1. Deploy to the QNAP NAS with a command like `REMOTE_HOST=hades REMOTE_USER=Hel SERVICE_MANAGER=plain MAILCAT_HOME=/share/homes/mailcat ./deploy_vps.sh`.
2. Before starting the daemon, ensure `/share/homes/mailcat/mailcat/config.json` on the NAS contains the `backup` account credentials and has mode `600`.
3. On the NAS, start and inspect the daemon:
- `sudo -u mailcat /share/homes/mailcat/mailcat/bin/start_mailcat.sh`
- `sudo -u mailcat /share/homes/mailcat/mailcat/bin/status_mailcat.sh`
- `tail -f /share/homes/mailcat/mailcat/log/sort_mail_daemon.log`
4. Test the daily report manually: `sudo -u mailcat /share/homes/mailcat/mailcat/bin/run_daily_report.sh`, then check `tail -100 /share/homes/mailcat/mailcat/log/daily_report.out` and confirm mail delivery to `hans@australius.nl`.
5. Add the daily report wrapper to QNAP scheduler after the manual test succeeds.
6. Send controlled test messages to `backup@australius.nl` for invoices, clear AI providers, a newsletter/service route, and unmatched mail; verify expected folder moves and daemon logs.
7. Add `List-ID` header support to mailing-list routing if live daemon tests show sender/domain matching is too coarse.
## Persistent File Rule
Before every future commit in this project, rewrite this `restart_prompt.md` file so it describes the current state at that commit. Do not append. Replace the content with a fresh, accurate reconstruction prompt.
1. Deploy the updated code to the NAS in `SERVICE_MANAGER=plain` mode.
2. Create remote `web_config.json`.
3. Start the web UI with `/share/homes/mailcat/mailcat/bin/start_web.sh`.
4. Open `http://<nas-host-or-ip>:4321/` from the internal network and verify Basic Auth.
5. Make a harmless route edit, verify `domain_routes.json` and `domain_routes.json.bak`, then restart the sorter if the live daemon is running.
+51
View File
@@ -0,0 +1,51 @@
import base64
import json
import tempfile
import unittest
from pathlib import Path
from manage_routes_web import basic_auth_ok, load_routes, save_routes, validate_routes
class ManageRoutesWebTests(unittest.TestCase):
def test_validate_routes_normalizes_domains_and_mailbox(self):
routes = validate_routes([
{"domains": [" Example.COM ", "example.com"], "mailbox": " Diensten.Test "}
])
self.assertEqual(routes, [{"domains": ["example.com"], "mailbox": "Diensten.Test"}])
def test_validate_routes_rejects_duplicate_domain_across_routes(self):
with self.assertRaisesRegex(ValueError, "already used"):
validate_routes([
{"domains": ["example.com"], "mailbox": "Diensten.Een"},
{"domains": ["EXAMPLE.com"], "mailbox": "Diensten.Twee"},
])
def test_save_routes_writes_backup_and_normalized_json(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / "domain_routes.json"
path.write_text(
json.dumps([{"domains": ["old.example"], "mailbox": "Diensten.Oud"}]) + "\n",
encoding="utf-8",
)
saved = save_routes(
[{"domains": [" New.EXAMPLE "], "mailbox": " Diensten.Nieuw "}],
path,
)
self.assertEqual(saved, [{"domains": ["new.example"], "mailbox": "Diensten.Nieuw"}])
self.assertEqual(load_routes(path), saved)
backup = path.with_suffix(".json.bak")
self.assertTrue(backup.exists())
self.assertIn("old.example", backup.read_text(encoding="utf-8"))
def test_basic_auth_ok_accepts_exact_credentials(self):
token = base64.b64encode(b"mailcat:secret").decode("ascii")
self.assertTrue(basic_auth_ok(f"Basic {token}", "mailcat", "secret"))
self.assertFalse(basic_auth_ok(f"Basic {token}", "mailcat", "wrong"))
self.assertFalse(basic_auth_ok(None, "mailcat", "secret"))
if __name__ == "__main__":
unittest.main()
+4
View File
@@ -0,0 +1,4 @@
{
"username": "mailcat",
"password": "change-me"
}