Add route management web interface
Prompts used since previous commit: - Kun je ook een webinterface maken waarmee ik de regels kan beheren? - Ik wil dat de webinterface benaderbaar is vanaf mijn hele interne netwerk. En ik wil alle regels kunnen beheren. Dus ook de bestaande ingebouwde routes. - 1. basic auth - 2. 4321 Special observations: - Domain routes moved from Python source to domain_routes.json so the web UI can manage all previous built-in domain routes. - web_config.json remains ignored; live sorter processes must be restarted after route edits to pick up changed rules.
This commit is contained in:
@@ -3,6 +3,7 @@ __pycache__/
|
||||
*.log
|
||||
.DS_Store
|
||||
config.json
|
||||
web_config.json
|
||||
**/*.eml
|
||||
sort_mail_daemon_state.json
|
||||
verplaats_log_*.json
|
||||
|
||||
+110
-3
@@ -15,6 +15,8 @@ APP_USER="${APP_USER:-mailcat}"
|
||||
SERVICE_NAME="${SERVICE_NAME:-mailcat-sort-backup}"
|
||||
ACCOUNT="${ACCOUNT:-backup}"
|
||||
DAILY_REPORT_TO="${DAILY_REPORT_TO:-hans@australius.nl}"
|
||||
WEB_HOST="${WEB_HOST:-0.0.0.0}"
|
||||
WEB_PORT="${WEB_PORT:-4321}"
|
||||
REMOTE_PATH="${REMOTE_PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}"
|
||||
SERVICE_MANAGER="${SERVICE_MANAGER:-systemd}"
|
||||
MAILCAT_HOME="${MAILCAT_HOME:-/share/homes/$APP_USER}"
|
||||
@@ -40,12 +42,16 @@ if [ "$SERVICE_MANAGER" = "plain" ]; then
|
||||
LOG_DIR="${LOG_DIR:-$APP_DIR/log}"
|
||||
RUN_DIR="${RUN_DIR:-$APP_DIR/run}"
|
||||
PID_FILE="${PID_FILE:-$RUN_DIR/mailcat.pid}"
|
||||
WEB_PID_FILE="${WEB_PID_FILE:-$RUN_DIR/mailcat_web.pid}"
|
||||
WEB_CONFIG="${WEB_CONFIG:-$APP_DIR/web_config.json}"
|
||||
PYTHON="${PYTHON:-/opt/bin/python3}"
|
||||
else
|
||||
STATE_DIR="${STATE_DIR:-/var/lib/mailcat}"
|
||||
LOG_DIR="${LOG_DIR:-/var/log/mailcat}"
|
||||
RUN_DIR="${RUN_DIR:-/run/mailcat}"
|
||||
PID_FILE="${PID_FILE:-$RUN_DIR/mailcat.pid}"
|
||||
WEB_PID_FILE="${WEB_PID_FILE:-$RUN_DIR/mailcat_web.pid}"
|
||||
WEB_CONFIG="${WEB_CONFIG:-$APP_DIR/web_config.json}"
|
||||
PYTHON="${PYTHON:-/usr/bin/python3}"
|
||||
fi
|
||||
|
||||
@@ -64,6 +70,7 @@ tar \
|
||||
--exclude "__pycache__" \
|
||||
--exclude "mailbox" \
|
||||
--exclude "config.json" \
|
||||
--exclude "web_config.json" \
|
||||
--exclude "*.log" \
|
||||
--exclude "backup_log.json" \
|
||||
--exclude "verplaats_log*.json" \
|
||||
@@ -118,6 +125,11 @@ APP_GROUP="$(id -gn "$APP_USER")"
|
||||
if [ ! -f "$APP_DIR/config.json" ]; then
|
||||
echo "Missing $APP_DIR/config.json on remote host. Create it with mode 600 before starting the service." >&2
|
||||
fi
|
||||
"$SUDO" "$CHMOD" 600 "$APP_DIR/web_config.json" 2>/dev/null || true
|
||||
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/web_config.json" 2>/dev/null || true
|
||||
if [ ! -f "$APP_DIR/web_config.json" ]; then
|
||||
echo "Missing $APP_DIR/web_config.json on remote host. Create it with mode 600 before starting the web UI." >&2
|
||||
fi
|
||||
"$SUDO" "$CHMOD" 600 "$APP_DIR/config.json" 2>/dev/null || true
|
||||
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/config.json" 2>/dev/null || true
|
||||
|
||||
@@ -218,12 +230,105 @@ cd "\$APP_DIR"
|
||||
"\$PYTHON" "\$APP_DIR/dagelijks_overzicht.py" --account "\$ACCOUNT" --mail-to "\$MAIL_TO" >> "\$LOG_DIR/daily_report.out" 2>&1
|
||||
REPORT_SCRIPT
|
||||
|
||||
cat > /tmp/mailcat-web-start.sh <<WEB_START_SCRIPT
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
APP_DIR="$APP_DIR"
|
||||
PYTHON="$PYTHON"
|
||||
LOG_DIR="$LOG_DIR"
|
||||
RUN_DIR="$RUN_DIR"
|
||||
WEB_PID_FILE="$WEB_PID_FILE"
|
||||
WEB_HOST="$WEB_HOST"
|
||||
WEB_PORT="$WEB_PORT"
|
||||
WEB_CONFIG="$WEB_CONFIG"
|
||||
MKDIR="$MKDIR"
|
||||
|
||||
if [ ! -f "\$WEB_CONFIG" ]; then
|
||||
echo "missing web config: \$WEB_CONFIG"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
"\$MKDIR" -p "\$LOG_DIR" "\$RUN_DIR"
|
||||
if [ -f "\$WEB_PID_FILE" ]; then
|
||||
old_pid=""
|
||||
IFS= read -r old_pid < "\$WEB_PID_FILE" || true
|
||||
if [ -n "\$old_pid" ] && kill -0 "\$old_pid" 2>/dev/null; then
|
||||
echo "mailcat web already running: \$old_pid"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
cd "\$APP_DIR"
|
||||
"\$PYTHON" "\$APP_DIR/manage_routes_web.py" --host "\$WEB_HOST" --port "\$WEB_PORT" --config "\$WEB_CONFIG" >> "\$LOG_DIR/web.out" 2>&1 &
|
||||
pid="\$!"
|
||||
echo "\$pid" > "\$WEB_PID_FILE"
|
||||
echo "mailcat web started: \$pid"
|
||||
WEB_START_SCRIPT
|
||||
|
||||
cat > /tmp/mailcat-web-stop.sh <<WEB_STOP_SCRIPT
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
WEB_PID_FILE="$WEB_PID_FILE"
|
||||
RM="$RM"
|
||||
|
||||
if [ ! -f "\$WEB_PID_FILE" ]; then
|
||||
echo "mailcat web not running: no pid file"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
pid=""
|
||||
IFS= read -r pid < "\$WEB_PID_FILE" || true
|
||||
if [ -z "\$pid" ]; then
|
||||
"\$RM" -f "\$WEB_PID_FILE"
|
||||
echo "mailcat web not running: empty pid file removed"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if kill -0 "\$pid" 2>/dev/null; then
|
||||
kill "\$pid"
|
||||
sleep 2
|
||||
if kill -0 "\$pid" 2>/dev/null; then
|
||||
echo "mailcat web still running after SIGTERM: \$pid"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
"\$RM" -f "\$WEB_PID_FILE"
|
||||
echo "mailcat web stopped"
|
||||
WEB_STOP_SCRIPT
|
||||
|
||||
cat > /tmp/mailcat-web-status.sh <<WEB_STATUS_SCRIPT
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
WEB_PID_FILE="$WEB_PID_FILE"
|
||||
WEB_HOST="$WEB_HOST"
|
||||
WEB_PORT="$WEB_PORT"
|
||||
LOG_DIR="$LOG_DIR"
|
||||
|
||||
if [ -f "\$WEB_PID_FILE" ]; then
|
||||
pid=""
|
||||
IFS= read -r pid < "\$WEB_PID_FILE" || true
|
||||
if [ -n "\$pid" ] && kill -0 "\$pid" 2>/dev/null; then
|
||||
echo "mailcat web running: \$pid"
|
||||
echo "url: http://\$WEB_HOST:\$WEB_PORT/"
|
||||
exit 0
|
||||
fi
|
||||
echo "mailcat web not running: stale pid file (\$pid)"
|
||||
exit 1
|
||||
fi
|
||||
echo "mailcat web not running"
|
||||
echo "logs: \$LOG_DIR/web.out"
|
||||
exit 1
|
||||
WEB_STATUS_SCRIPT
|
||||
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-start.sh "$APP_DIR/bin/start_mailcat.sh"
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-stop.sh "$APP_DIR/bin/stop_mailcat.sh"
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-status.sh "$APP_DIR/bin/status_mailcat.sh"
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-daily-report.sh "$APP_DIR/bin/run_daily_report.sh"
|
||||
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/bin/start_mailcat.sh" "$APP_DIR/bin/stop_mailcat.sh" "$APP_DIR/bin/status_mailcat.sh" "$APP_DIR/bin/run_daily_report.sh"
|
||||
"$RM" -f /tmp/mailcat-start.sh /tmp/mailcat-stop.sh /tmp/mailcat-status.sh /tmp/mailcat-daily-report.sh
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-web-start.sh "$APP_DIR/bin/start_web.sh"
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-web-stop.sh "$APP_DIR/bin/stop_web.sh"
|
||||
"$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-web-status.sh "$APP_DIR/bin/status_web.sh"
|
||||
"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/bin/start_mailcat.sh" "$APP_DIR/bin/stop_mailcat.sh" "$APP_DIR/bin/status_mailcat.sh" "$APP_DIR/bin/run_daily_report.sh" "$APP_DIR/bin/start_web.sh" "$APP_DIR/bin/stop_web.sh" "$APP_DIR/bin/status_web.sh"
|
||||
"$RM" -f /tmp/mailcat-start.sh /tmp/mailcat-stop.sh /tmp/mailcat-status.sh /tmp/mailcat-daily-report.sh /tmp/mailcat-web-start.sh /tmp/mailcat-web-stop.sh /tmp/mailcat-web-status.sh
|
||||
else
|
||||
"$SUDO" "$INSTALL" -m 0644 "$APP_DIR/systemd/mailcat-sort.service" "/etc/systemd/system/$SERVICE_NAME.service"
|
||||
"$SUDO" "$SED" -i \
|
||||
@@ -247,6 +352,8 @@ if [ "$SERVICE_MANAGER" = "plain" ]; then
|
||||
echo " sudo -u $APP_USER $APP_DIR/bin/start_mailcat.sh"
|
||||
echo " sudo -u $APP_USER $APP_DIR/bin/status_mailcat.sh"
|
||||
echo " sudo -u $APP_USER $APP_DIR/bin/run_daily_report.sh"
|
||||
echo " sudo -u $APP_USER $APP_DIR/bin/start_web.sh"
|
||||
echo " sudo -u $APP_USER $APP_DIR/bin/status_web.sh"
|
||||
echo " tail -f $LOG_DIR/sort_mail_daemon.log"
|
||||
else
|
||||
echo " sudo systemctl start $SERVICE_NAME.service"
|
||||
@@ -257,5 +364,5 @@ REMOTE
|
||||
scp "$ARCHIVE" "$REMOTE_USER@$REMOTE_HOST:/tmp/mailcat-deploy.tar.gz"
|
||||
scp "$REMOTE_SCRIPT" "$REMOTE_USER@$REMOTE_HOST:/tmp/mailcat-remote-deploy.sh"
|
||||
|
||||
REMOTE_COMMAND=$(printf "APP_USER=%q APP_DIR=%q SERVICE_NAME=%q ACCOUNT=%q DAILY_REPORT_TO=%q REMOTE_PATH=%q SERVICE_MANAGER=%q MAILCAT_HOME=%q STATE_DIR=%q LOG_DIR=%q RUN_DIR=%q PID_FILE=%q PYTHON=%q SUDO=%q USERADD=%q INSTALL=%q SED=%q TAR=%q MKDIR=%q CHOWN=%q CHMOD=%q RM=%q SYSTEMCTL=%q /bin/sh /tmp/mailcat-remote-deploy.sh" "$APP_USER" "$APP_DIR" "$SERVICE_NAME" "$ACCOUNT" "$DAILY_REPORT_TO" "$REMOTE_PATH" "$SERVICE_MANAGER" "$MAILCAT_HOME" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" "$PID_FILE" "$PYTHON" "$SUDO" "$USERADD" "$INSTALL" "$SED" "$TAR" "$MKDIR" "$CHOWN" "$CHMOD" "$RM" "$SYSTEMCTL")
|
||||
REMOTE_COMMAND=$(printf "APP_USER=%q APP_DIR=%q SERVICE_NAME=%q ACCOUNT=%q DAILY_REPORT_TO=%q WEB_HOST=%q WEB_PORT=%q REMOTE_PATH=%q SERVICE_MANAGER=%q MAILCAT_HOME=%q STATE_DIR=%q LOG_DIR=%q RUN_DIR=%q PID_FILE=%q WEB_PID_FILE=%q WEB_CONFIG=%q PYTHON=%q SUDO=%q USERADD=%q INSTALL=%q SED=%q TAR=%q MKDIR=%q CHOWN=%q CHMOD=%q RM=%q SYSTEMCTL=%q /bin/sh /tmp/mailcat-remote-deploy.sh" "$APP_USER" "$APP_DIR" "$SERVICE_NAME" "$ACCOUNT" "$DAILY_REPORT_TO" "$WEB_HOST" "$WEB_PORT" "$REMOTE_PATH" "$SERVICE_MANAGER" "$MAILCAT_HOME" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" "$PID_FILE" "$WEB_PID_FILE" "$WEB_CONFIG" "$PYTHON" "$SUDO" "$USERADD" "$INSTALL" "$SED" "$TAR" "$MKDIR" "$CHOWN" "$CHMOD" "$RM" "$SYSTEMCTL")
|
||||
ssh -tt "$REMOTE_USER@$REMOTE_HOST" "$REMOTE_COMMAND"
|
||||
|
||||
@@ -0,0 +1,601 @@
|
||||
[
|
||||
{
|
||||
"domains": [
|
||||
"nl.abnamro.com",
|
||||
"abnamro.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.ABN AMRO"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"icscards.nl",
|
||||
"icsmarketing.icscards.nl",
|
||||
"service.icscards.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.ICS"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"npex.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.NPEX"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"n26.com"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.N26"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"update.bunq.com",
|
||||
"hello.bunq.com"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.bunq"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"snelstart.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.SnelStart"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"tellow.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Bank.Tellow"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"mijnkantoorapp.nl",
|
||||
"dijkman-ac.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Boekhouding.Dijkman"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"unive.nl",
|
||||
"nieuwsbrief.unive.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Verzekering.Univé"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"heinenoord.nl"
|
||||
],
|
||||
"mailbox": "Financieel.Verzekering.VEZA"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"flextender.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.Flextender"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"onestopsourcing.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.OneStopSourcing"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"immens.nu"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.IMMENS"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"circle8.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.Circle8"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"getthere.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.Get There"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"hey.meetjamie.ai",
|
||||
"notifications.meetjamie.ai"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Jamie"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"huckr.ai"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Huckr"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"maandag.com"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.Maandag"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"futurexl.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.FutureXL"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"drenthe.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.Provincie Drenthe"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"rdw.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.RDW"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"plasbossinade.nl"
|
||||
],
|
||||
"mailbox": "Werk.Opdrachten.PlasBossinade"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"boip.int"
|
||||
],
|
||||
"mailbox": "Werk.Overig.BOIP"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"groningen.nl",
|
||||
"groningenbereikbaar.nl"
|
||||
],
|
||||
"mailbox": "Werk.Overig.Gemeente Groningen"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"kvk.nl"
|
||||
],
|
||||
"mailbox": "Administratie.KvK"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"eneco-emobility.com",
|
||||
"eneco.nl"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.Eneco eMobility"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"shellrecharge.com"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.Shell Recharge"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"fastned.nl"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.FastNed"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"opngo.com"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.OPnGO"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"tesla.com"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.Tesla"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"mmsn.nl"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.Mitsubishi"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"alfen.com"
|
||||
],
|
||||
"mailbox": "Mobiliteit.EV.Alfen"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"ns.nl",
|
||||
"email.ns.nl"
|
||||
],
|
||||
"mailbox": "Mobiliteit.OV & Reizen.NS"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"booking.com"
|
||||
],
|
||||
"mailbox": "Mobiliteit.OV & Reizen.Booking"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"flitsmeister.nl"
|
||||
],
|
||||
"mailbox": "Mobiliteit.OV & Reizen.Flitsmeister"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"postnl.nl",
|
||||
"edm.postnl.nl",
|
||||
"notificatie.postnl.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.PostNL"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"dhlparcel.nl",
|
||||
"dhlecommerce.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.DHL"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"gls-netherlands.com"
|
||||
],
|
||||
"mailbox": "Bestellingen.GLS"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"bol.com",
|
||||
"feedback.bol.com"
|
||||
],
|
||||
"mailbox": "Bestellingen.bol"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"coolblue.nl",
|
||||
"coolblue.eu"
|
||||
],
|
||||
"mailbox": "Bestellingen.Coolblue"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"allekabels.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.Allekabels"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"vikingdirect.nl",
|
||||
"delivery.vikingdirect.nl",
|
||||
"online.vikingdirect.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.Viking"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"123inkt.nl",
|
||||
"m.123inkt.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.123inkt"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"officecentre.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.Office Centre"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"sligro.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.Sligro"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"makro.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.Makro"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"smartphonehoesjes.nl"
|
||||
],
|
||||
"mailbox": "Bestellingen.Smartphonehoesjes"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"strato.com",
|
||||
"news.strato.com",
|
||||
"marketingradar.strato.com"
|
||||
],
|
||||
"mailbox": "Diensten.Hosting.STRATO"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"transip.nl"
|
||||
],
|
||||
"mailbox": "Diensten.Hosting.TransIP"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"cloud86.io",
|
||||
"cloud86.zendesk.com"
|
||||
],
|
||||
"mailbox": "Diensten.Hosting.Cloud86"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"godaddy.com"
|
||||
],
|
||||
"mailbox": "Diensten.Hosting.GoDaddy"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"letsencrypt.org"
|
||||
],
|
||||
"mailbox": "Diensten.Hosting.Lets Encrypt"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"backblaze.com"
|
||||
],
|
||||
"mailbox": "Diensten.Hosting.Backblaze"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"kpn.com",
|
||||
"campagnes.kpn.com"
|
||||
],
|
||||
"mailbox": "Diensten.Telecom.KPN"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"zakelijk.vodafone.nl",
|
||||
"vodafone.nl"
|
||||
],
|
||||
"mailbox": "Diensten.Telecom.Vodafone"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"nordaccount.com"
|
||||
],
|
||||
"mailbox": "Diensten.Beveiliging.Nord"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"info.expressvpn.com"
|
||||
],
|
||||
"mailbox": "Diensten.Beveiliging.ExpressVPN"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"email.remarkable.com"
|
||||
],
|
||||
"mailbox": "Diensten.Apparaten.reMarkable"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"1password.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.1Password"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"home.sophos.com",
|
||||
"cleverbridge.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Sophos"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"cogsciapps.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.CogSci Apps"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"claude.com",
|
||||
"mail.anthropic.com"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Claude"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"cursor.com",
|
||||
"mail.cursor.com"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Cursor"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"mistral.ai",
|
||||
"mistralai.intercom-mail.eu"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Mistral"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"openai.com",
|
||||
"chatgpt.com",
|
||||
"tm.openai.com"
|
||||
],
|
||||
"mailbox": "Diensten.AI.OpenAI"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"perplexity.ai"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Perplexity"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"canva.com",
|
||||
"account.canva.com"
|
||||
],
|
||||
"mailbox": "Diensten.AI.Canva"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"microsoft.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Microsoft"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"microsoft365.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Microsoft Teams"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"market.envato.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Envato"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"getmatter.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Matter"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"mindnode.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.MindNode"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"papersapp.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Papers"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"team.kahoot.com"
|
||||
],
|
||||
"mailbox": "Diensten.Software.Kahoot"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"coursera.org",
|
||||
"email.coursera.org",
|
||||
"m.mail.coursera.org",
|
||||
"m.learn.coursera.org"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Coursera"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"technologyreview.com",
|
||||
"fulfillment.technologyreview.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.MIT Technology Review"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"mit.edu"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.MIT Open Learning"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"storytellingwithdata.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Storytelling with Data"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"linkingyourthinking.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Nick Milo"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"evakeiffenheim.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Eva Keiffenheim"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"tiggelaar.nl"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Ben Tiggelaar"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"briankeating.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Brian Keating"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"beehiiv.com",
|
||||
"mail.beehiiv.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.AI Report"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"elizabethbutlermd.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.Elizabeth Butler"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"notify.orcid.org"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Leren.ORCID"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"crisismanager.nl"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Nieuws & Vakbladen.Vakblad Crisismanager"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"jaarbeurs.nl"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Nieuws & Vakbladen.Computable"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"presentation-guru.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Nieuws & Vakbladen.The Presentation Guru"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"lendahand.com"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Overig.Lendahand"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"thegoodroll.nl"
|
||||
],
|
||||
"mailbox": "Nieuwsbrieven.Overig.The Good Roll"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"tasks.clickup.com"
|
||||
],
|
||||
"mailbox": "Afmelden.Software.ClickUp"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"gemeenteprojecten.talent-pool.com"
|
||||
],
|
||||
"mailbox": "Afmelden.Werk.Gemeenteprojecten"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"engage.istockphoto.com"
|
||||
],
|
||||
"mailbox": "Afmelden.Creatief.iStock"
|
||||
},
|
||||
{
|
||||
"domains": [
|
||||
"dmarc.postmarkapp.com"
|
||||
],
|
||||
"mailbox": "Technisch.DMARC"
|
||||
}
|
||||
]
|
||||
+58
-103
@@ -9,109 +9,64 @@ from pathlib import Path
|
||||
|
||||
PREFIX = "INBOX."
|
||||
MAILINGLIST_ROUTES_FILE = Path(__file__).parent / "mailinglist_routes.json"
|
||||
DOMAIN_ROUTES_FILE = Path(__file__).parent / "domain_routes.json"
|
||||
|
||||
DOMAIN_ROUTES: list[tuple[list[str], str]] = [
|
||||
(["nl.abnamro.com", "abnamro.nl"], "Financieel.Bank.ABN AMRO"),
|
||||
(["icscards.nl", "icsmarketing.icscards.nl", "service.icscards.nl"], "Financieel.Bank.ICS"),
|
||||
(["npex.nl"], "Financieel.Bank.NPEX"),
|
||||
(["n26.com"], "Financieel.Bank.N26"),
|
||||
(["update.bunq.com", "hello.bunq.com"], "Financieel.Bank.bunq"),
|
||||
(["snelstart.nl"], "Financieel.Bank.SnelStart"),
|
||||
(["tellow.nl"], "Financieel.Bank.Tellow"),
|
||||
(["mijnkantoorapp.nl", "dijkman-ac.nl"], "Financieel.Boekhouding.Dijkman"),
|
||||
(["unive.nl", "nieuwsbrief.unive.nl"], "Financieel.Verzekering.Univé"),
|
||||
(["heinenoord.nl"], "Financieel.Verzekering.VEZA"),
|
||||
(["flextender.nl"], "Werk.Opdrachten.Flextender"),
|
||||
(["onestopsourcing.nl"], "Werk.Opdrachten.OneStopSourcing"),
|
||||
(["immens.nu"], "Werk.Opdrachten.IMMENS"),
|
||||
(["circle8.nl"], "Werk.Opdrachten.Circle8"),
|
||||
(["getthere.nl"], "Werk.Opdrachten.Get There"),
|
||||
(["hey.meetjamie.ai", "notifications.meetjamie.ai"], "Diensten.AI.Jamie"),
|
||||
(["huckr.ai"], "Diensten.AI.Huckr"),
|
||||
(["maandag.com"], "Werk.Opdrachten.Maandag"),
|
||||
(["futurexl.nl"], "Werk.Opdrachten.FutureXL"),
|
||||
(["drenthe.nl"], "Werk.Opdrachten.Provincie Drenthe"),
|
||||
(["rdw.nl"], "Werk.Opdrachten.RDW"),
|
||||
(["plasbossinade.nl"], "Werk.Opdrachten.PlasBossinade"),
|
||||
(["boip.int"], "Werk.Overig.BOIP"),
|
||||
(["groningen.nl", "groningenbereikbaar.nl"], "Werk.Overig.Gemeente Groningen"),
|
||||
(["kvk.nl"], "Administratie.KvK"),
|
||||
(["eneco-emobility.com", "eneco.nl"], "Mobiliteit.EV.Eneco eMobility"),
|
||||
(["shellrecharge.com"], "Mobiliteit.EV.Shell Recharge"),
|
||||
(["fastned.nl"], "Mobiliteit.EV.FastNed"),
|
||||
(["opngo.com"], "Mobiliteit.EV.OPnGO"),
|
||||
(["tesla.com"], "Mobiliteit.EV.Tesla"),
|
||||
(["mmsn.nl"], "Mobiliteit.EV.Mitsubishi"),
|
||||
(["alfen.com"], "Mobiliteit.EV.Alfen"),
|
||||
(["ns.nl", "email.ns.nl"], "Mobiliteit.OV & Reizen.NS"),
|
||||
(["booking.com"], "Mobiliteit.OV & Reizen.Booking"),
|
||||
(["flitsmeister.nl"], "Mobiliteit.OV & Reizen.Flitsmeister"),
|
||||
(["postnl.nl", "edm.postnl.nl", "notificatie.postnl.nl"], "Bestellingen.PostNL"),
|
||||
(["dhlparcel.nl", "dhlecommerce.nl"], "Bestellingen.DHL"),
|
||||
(["gls-netherlands.com"], "Bestellingen.GLS"),
|
||||
(["bol.com", "feedback.bol.com"], "Bestellingen.bol"),
|
||||
(["coolblue.nl", "coolblue.eu"], "Bestellingen.Coolblue"),
|
||||
(["allekabels.nl"], "Bestellingen.Allekabels"),
|
||||
(["vikingdirect.nl", "delivery.vikingdirect.nl", "online.vikingdirect.nl"], "Bestellingen.Viking"),
|
||||
(["123inkt.nl", "m.123inkt.nl"], "Bestellingen.123inkt"),
|
||||
(["officecentre.nl"], "Bestellingen.Office Centre"),
|
||||
(["sligro.nl"], "Bestellingen.Sligro"),
|
||||
(["makro.nl"], "Bestellingen.Makro"),
|
||||
(["smartphonehoesjes.nl"], "Bestellingen.Smartphonehoesjes"),
|
||||
(["strato.com", "news.strato.com", "marketingradar.strato.com"], "Diensten.Hosting.STRATO"),
|
||||
(["transip.nl"], "Diensten.Hosting.TransIP"),
|
||||
(["cloud86.io", "cloud86.zendesk.com"], "Diensten.Hosting.Cloud86"),
|
||||
(["godaddy.com"], "Diensten.Hosting.GoDaddy"),
|
||||
(["letsencrypt.org"], "Diensten.Hosting.Lets Encrypt"),
|
||||
(["backblaze.com"], "Diensten.Hosting.Backblaze"),
|
||||
(["kpn.com", "campagnes.kpn.com"], "Diensten.Telecom.KPN"),
|
||||
(["zakelijk.vodafone.nl", "vodafone.nl"], "Diensten.Telecom.Vodafone"),
|
||||
(["nordaccount.com"], "Diensten.Beveiliging.Nord"),
|
||||
(["info.expressvpn.com"], "Diensten.Beveiliging.ExpressVPN"),
|
||||
(["email.remarkable.com"], "Diensten.Apparaten.reMarkable"),
|
||||
(["1password.com"], "Diensten.Software.1Password"),
|
||||
(["home.sophos.com", "cleverbridge.com"], "Diensten.Software.Sophos"),
|
||||
(["cogsciapps.com"], "Diensten.Software.CogSci Apps"),
|
||||
(["claude.com", "mail.anthropic.com"], "Diensten.AI.Claude"),
|
||||
(["cursor.com", "mail.cursor.com"], "Diensten.AI.Cursor"),
|
||||
(["mistral.ai", "mistralai.intercom-mail.eu"], "Diensten.AI.Mistral"),
|
||||
(["openai.com", "chatgpt.com", "tm.openai.com"], "Diensten.AI.OpenAI"),
|
||||
(["perplexity.ai"], "Diensten.AI.Perplexity"),
|
||||
(["canva.com", "account.canva.com"], "Diensten.AI.Canva"),
|
||||
(["microsoft.com"], "Diensten.Software.Microsoft"),
|
||||
(["microsoft365.com"], "Diensten.Software.Microsoft Teams"),
|
||||
(["market.envato.com"], "Diensten.Software.Envato"),
|
||||
(["getmatter.com"], "Diensten.Software.Matter"),
|
||||
(["mindnode.com"], "Diensten.Software.MindNode"),
|
||||
(["papersapp.com"], "Diensten.Software.Papers"),
|
||||
(["team.kahoot.com"], "Diensten.Software.Kahoot"),
|
||||
(["coursera.org", "email.coursera.org", "m.mail.coursera.org", "m.learn.coursera.org"], "Nieuwsbrieven.Leren.Coursera"),
|
||||
(["technologyreview.com", "fulfillment.technologyreview.com"], "Nieuwsbrieven.Leren.MIT Technology Review"),
|
||||
(["mit.edu"], "Nieuwsbrieven.Leren.MIT Open Learning"),
|
||||
(["storytellingwithdata.com"], "Nieuwsbrieven.Leren.Storytelling with Data"),
|
||||
(["linkingyourthinking.com"], "Nieuwsbrieven.Leren.Nick Milo"),
|
||||
(["evakeiffenheim.com"], "Nieuwsbrieven.Leren.Eva Keiffenheim"),
|
||||
(["tiggelaar.nl"], "Nieuwsbrieven.Leren.Ben Tiggelaar"),
|
||||
(["briankeating.com"], "Nieuwsbrieven.Leren.Brian Keating"),
|
||||
(["beehiiv.com", "mail.beehiiv.com"], "Nieuwsbrieven.Leren.AI Report"),
|
||||
(["elizabethbutlermd.com"], "Nieuwsbrieven.Leren.Elizabeth Butler"),
|
||||
(["notify.orcid.org"], "Nieuwsbrieven.Leren.ORCID"),
|
||||
(["crisismanager.nl"], "Nieuwsbrieven.Nieuws & Vakbladen.Vakblad Crisismanager"),
|
||||
(["jaarbeurs.nl"], "Nieuwsbrieven.Nieuws & Vakbladen.Computable"),
|
||||
(["presentation-guru.com"], "Nieuwsbrieven.Nieuws & Vakbladen.The Presentation Guru"),
|
||||
(["lendahand.com"], "Nieuwsbrieven.Overig.Lendahand"),
|
||||
(["thegoodroll.nl"], "Nieuwsbrieven.Overig.The Good Roll"),
|
||||
(["tasks.clickup.com"], "Afmelden.Software.ClickUp"),
|
||||
(["gemeenteprojecten.talent-pool.com"], "Afmelden.Werk.Gemeenteprojecten"),
|
||||
(["engage.istockphoto.com"], "Afmelden.Creatief.iStock"),
|
||||
(["dmarc.postmarkapp.com"], "Technisch.DMARC"),
|
||||
]
|
||||
|
||||
DOMAIN_LOOKUP: dict[str, str] = {
|
||||
domain.lower(): target
|
||||
for domains, target in DOMAIN_ROUTES
|
||||
for domain in domains
|
||||
}
|
||||
def _normalize_domain(value: str) -> str:
|
||||
domain = value.strip().lower()
|
||||
if not domain or "@" in domain or "/" in domain or any(ch.isspace() for ch in domain):
|
||||
raise ValueError(f"Invalid domain route domain: {value!r}")
|
||||
return domain
|
||||
|
||||
|
||||
def _normalize_mailbox(value: str) -> str:
|
||||
mailbox = value.strip()
|
||||
if not mailbox or mailbox.startswith(PREFIX) or mailbox.startswith(".") or mailbox.endswith(".") or ".." in mailbox:
|
||||
raise ValueError(f"Invalid domain route mailbox: {value!r}")
|
||||
return mailbox
|
||||
|
||||
|
||||
def normalize_domain_route(rule: dict) -> tuple[list[str], str]:
|
||||
"""Return a validated domain route as (domains, mailbox)."""
|
||||
if not isinstance(rule, dict):
|
||||
raise ValueError(f"Invalid domain route entry: {rule!r}")
|
||||
|
||||
raw_domains = rule.get("domains")
|
||||
if not isinstance(raw_domains, list):
|
||||
raise ValueError(f"Domain route domains must be a list: {rule!r}")
|
||||
|
||||
domains = [_normalize_domain(str(item)) for item in raw_domains]
|
||||
domains = list(dict.fromkeys(domains))
|
||||
if not domains:
|
||||
raise ValueError(f"Domain route must contain at least one domain: {rule!r}")
|
||||
|
||||
mailbox = _normalize_mailbox(str(rule.get("mailbox", "")))
|
||||
return domains, mailbox
|
||||
|
||||
|
||||
@cache
|
||||
def domain_routes() -> list[tuple[list[str], str]]:
|
||||
"""Return validated domain routing rules from JSON policy."""
|
||||
raw = json.loads(DOMAIN_ROUTES_FILE.read_text(encoding="utf-8"))
|
||||
if not isinstance(raw, list):
|
||||
raise ValueError("domain_routes.json must contain a list")
|
||||
return [normalize_domain_route(rule) for rule in raw]
|
||||
|
||||
|
||||
@cache
|
||||
def domain_lookup() -> dict[str, str]:
|
||||
"""Return domain-to-mailbox lookup built from domain_routes.json."""
|
||||
lookup: dict[str, str] = {}
|
||||
for domains, target in domain_routes():
|
||||
for domain in domains:
|
||||
lookup[domain] = target
|
||||
return lookup
|
||||
|
||||
|
||||
# Compatibility for existing scripts/tests that imported DOMAIN_ROUTES directly.
|
||||
DOMAIN_ROUTES = domain_routes()
|
||||
DOMAIN_LOOKUP = domain_lookup()
|
||||
|
||||
|
||||
FACTUUR_KEYWORDS = [
|
||||
"factuur", "invoice", "rekening", "nota", "betaalverzoek",
|
||||
@@ -179,7 +134,7 @@ def route_from_subject(from_addr: str, subject: str) -> str | None:
|
||||
return "__FACTUUR_DATE__"
|
||||
if destination := mailinglist_destination(from_addr):
|
||||
return destination
|
||||
return DOMAIN_LOOKUP.get(sender_domain(from_addr))
|
||||
return domain_lookup().get(sender_domain(from_addr))
|
||||
|
||||
|
||||
def is_source_folder(folder: str) -> bool:
|
||||
@@ -189,7 +144,7 @@ def is_source_folder(folder: str) -> bool:
|
||||
|
||||
def destination_folders(start_year: int = 2020, end_year: int = 2026) -> list[str]:
|
||||
"""Return all destination folders without the INBOX. prefix."""
|
||||
folders = {target for _, target in DOMAIN_ROUTES}
|
||||
folders = {target for _, target in domain_routes()}
|
||||
folders.update(
|
||||
rule["mailbox"]
|
||||
for rule in mailinglist_routes()
|
||||
|
||||
@@ -0,0 +1,453 @@
|
||||
"""Small Basic Auth web UI for managing mailcat domain routes."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from mail_routes import DOMAIN_ROUTES_FILE, domain_lookup, domain_routes, normalize_domain_route
|
||||
|
||||
DEFAULT_CONFIG_FILE = Path(__file__).parent / "web_config.json"
|
||||
DEFAULT_HOST = "0.0.0.0"
|
||||
DEFAULT_PORT = 4321
|
||||
|
||||
|
||||
def validate_routes(raw_routes: list[dict]) -> list[dict]:
|
||||
"""Return normalized route dictionaries and reject duplicate domains."""
|
||||
if not isinstance(raw_routes, list):
|
||||
raise ValueError("Routes must be a list")
|
||||
|
||||
normalized: list[dict] = []
|
||||
seen: dict[str, int] = {}
|
||||
for index, raw_rule in enumerate(raw_routes):
|
||||
domains, mailbox = normalize_domain_route(raw_rule)
|
||||
for domain in domains:
|
||||
if domain in seen:
|
||||
other = seen[domain] + 1
|
||||
raise ValueError(f"Domain {domain!r} is already used by route {other}")
|
||||
seen[domain] = index
|
||||
normalized.append({"domains": domains, "mailbox": mailbox})
|
||||
return normalized
|
||||
|
||||
|
||||
def load_routes(path: Path = DOMAIN_ROUTES_FILE) -> list[dict]:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
return validate_routes(raw)
|
||||
|
||||
|
||||
def save_routes(routes: list[dict], path: Path = DOMAIN_ROUTES_FILE) -> list[dict]:
|
||||
normalized = validate_routes(routes)
|
||||
tmp_path = path.with_name(f".{path.name}.tmp")
|
||||
backup_path = path.with_suffix(f"{path.suffix}.bak")
|
||||
tmp_path.write_text(json.dumps(normalized, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
if path.exists():
|
||||
shutil.copy2(path, backup_path)
|
||||
os.replace(tmp_path, path)
|
||||
domain_routes.cache_clear()
|
||||
domain_lookup.cache_clear()
|
||||
return normalized
|
||||
|
||||
|
||||
def load_web_config(path: Path) -> tuple[str, str]:
|
||||
config = {}
|
||||
if path.exists():
|
||||
config = json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
username = os.environ.get("MAILCAT_WEB_USER") or config.get("username")
|
||||
password = os.environ.get("MAILCAT_WEB_PASSWORD") or config.get("password")
|
||||
if not username or not password:
|
||||
raise SystemExit(
|
||||
f"Missing web credentials. Create {path} or set MAILCAT_WEB_USER and MAILCAT_WEB_PASSWORD."
|
||||
)
|
||||
return str(username), str(password)
|
||||
|
||||
|
||||
def basic_auth_ok(header: str | None, username: str, password: str) -> bool:
|
||||
if not header or not header.startswith("Basic "):
|
||||
return False
|
||||
try:
|
||||
decoded = base64.b64decode(header[6:], validate=True).decode("utf-8")
|
||||
supplied_user, supplied_password = decoded.split(":", 1)
|
||||
except Exception:
|
||||
return False
|
||||
return secrets.compare_digest(supplied_user, username) and secrets.compare_digest(
|
||||
supplied_password, password
|
||||
)
|
||||
|
||||
|
||||
def page_html() -> bytes:
|
||||
return f"""<!doctype html>
|
||||
<html lang="nl">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Mailcat routes</title>
|
||||
<style>
|
||||
:root {{
|
||||
color-scheme: light;
|
||||
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
|
||||
background: #f5f6f8;
|
||||
color: #20242a;
|
||||
}}
|
||||
body {{ margin: 0; }}
|
||||
header {{
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 16px;
|
||||
padding: 14px 20px;
|
||||
background: #17324d;
|
||||
color: #fff;
|
||||
}}
|
||||
h1 {{ font-size: 18px; line-height: 1.2; margin: 0; font-weight: 650; }}
|
||||
main {{ padding: 18px 20px 28px; max-width: 1280px; margin: 0 auto; }}
|
||||
.toolbar {{
|
||||
display: grid;
|
||||
grid-template-columns: minmax(180px, 1fr) auto auto;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
margin-bottom: 14px;
|
||||
}}
|
||||
input, textarea {{
|
||||
width: 100%;
|
||||
box-sizing: border-box;
|
||||
font: inherit;
|
||||
border: 1px solid #b9c1cc;
|
||||
border-radius: 6px;
|
||||
padding: 8px 10px;
|
||||
background: #fff;
|
||||
color: #20242a;
|
||||
}}
|
||||
textarea {{ min-height: 72px; resize: vertical; }}
|
||||
button {{
|
||||
border: 1px solid #245782;
|
||||
background: #245782;
|
||||
color: #fff;
|
||||
border-radius: 6px;
|
||||
padding: 8px 12px;
|
||||
font: inherit;
|
||||
cursor: pointer;
|
||||
white-space: nowrap;
|
||||
}}
|
||||
button.secondary {{ background: #fff; color: #245782; }}
|
||||
button.danger {{ background: #8c2634; border-color: #8c2634; }}
|
||||
button:disabled {{ opacity: .55; cursor: default; }}
|
||||
.layout {{ display: grid; grid-template-columns: 1fr 360px; gap: 18px; align-items: start; }}
|
||||
table {{ width: 100%; border-collapse: collapse; background: #fff; border: 1px solid #d7dce2; }}
|
||||
th, td {{ text-align: left; vertical-align: top; border-bottom: 1px solid #e2e6eb; padding: 8px 10px; }}
|
||||
th {{ background: #edf1f5; font-size: 13px; color: #34495e; }}
|
||||
tr.selected {{ background: #eaf4ff; }}
|
||||
td.index {{ width: 56px; color: #596775; }}
|
||||
td.actions {{ width: 96px; text-align: right; }}
|
||||
.panel {{ background: #fff; border: 1px solid #d7dce2; padding: 14px; }}
|
||||
.panel h2 {{ margin: 0 0 12px; font-size: 15px; }}
|
||||
label {{ display: block; margin: 10px 0 6px; font-size: 13px; color: #34495e; }}
|
||||
.form-actions {{ display: flex; gap: 8px; margin-top: 12px; flex-wrap: wrap; }}
|
||||
.status {{ min-height: 20px; font-size: 13px; color: #34495e; }}
|
||||
.error {{ color: #8c2634; }}
|
||||
.domains {{ line-height: 1.45; }}
|
||||
.mailbox {{ font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: 13px; }}
|
||||
@media (max-width: 860px) {{
|
||||
.layout {{ grid-template-columns: 1fr; }}
|
||||
.toolbar {{ grid-template-columns: 1fr; }}
|
||||
main {{ padding: 14px; }}
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>Mailcat routes</h1>
|
||||
<div id="count"></div>
|
||||
</header>
|
||||
<main>
|
||||
<div class="toolbar">
|
||||
<input id="search" type="search" placeholder="Zoek domein of map">
|
||||
<button id="newButton" type="button" class="secondary">Nieuwe regel</button>
|
||||
<button id="reloadButton" type="button" class="secondary">Herladen</button>
|
||||
</div>
|
||||
<div class="layout">
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>Domeinen</th><th>Map</th><th></th></tr></thead>
|
||||
<tbody id="routesBody"></tbody>
|
||||
</table>
|
||||
<section class="panel">
|
||||
<h2 id="formTitle">Nieuwe regel</h2>
|
||||
<form id="routeForm">
|
||||
<input id="routeIndex" type="hidden">
|
||||
<label for="domains">Domeinen, een per regel of komma-gescheiden</label>
|
||||
<textarea id="domains" required></textarea>
|
||||
<label for="mailbox">Doelmap zonder INBOX.</label>
|
||||
<input id="mailbox" required placeholder="Diensten.AI.OpenAI">
|
||||
<div class="form-actions">
|
||||
<button id="saveButton" type="submit">Opslaan</button>
|
||||
<button id="deleteButton" type="button" class="danger" disabled>Verwijderen</button>
|
||||
<button id="clearButton" type="button" class="secondary">Leegmaken</button>
|
||||
</div>
|
||||
</form>
|
||||
<p id="status" class="status"></p>
|
||||
</section>
|
||||
</div>
|
||||
</main>
|
||||
<script>
|
||||
let routes = [];
|
||||
let selectedIndex = null;
|
||||
const $ = (id) => document.getElementById(id);
|
||||
|
||||
function splitDomains(value) {{
|
||||
return value.split(/[\\n,]/).map((item) => item.trim()).filter(Boolean);
|
||||
}}
|
||||
|
||||
function setStatus(text, isError = false) {{
|
||||
$("status").textContent = text;
|
||||
$("status").className = isError ? "status error" : "status";
|
||||
}}
|
||||
|
||||
function clearForm() {{
|
||||
selectedIndex = null;
|
||||
$("routeIndex").value = "";
|
||||
$("domains").value = "";
|
||||
$("mailbox").value = "";
|
||||
$("formTitle").textContent = "Nieuwe regel";
|
||||
$("deleteButton").disabled = true;
|
||||
renderRoutes();
|
||||
}}
|
||||
|
||||
function editRoute(index) {{
|
||||
const route = routes[index];
|
||||
selectedIndex = index;
|
||||
$("routeIndex").value = String(index);
|
||||
$("domains").value = route.domains.join("\\n");
|
||||
$("mailbox").value = route.mailbox;
|
||||
$("formTitle").textContent = `Regel ${{index + 1}} bewerken`;
|
||||
$("deleteButton").disabled = false;
|
||||
renderRoutes();
|
||||
}}
|
||||
|
||||
async function requestJson(url, options = {{}}) {{
|
||||
const response = await fetch(url, {{
|
||||
headers: {{ "Content-Type": "application/json", ...(options.headers || {{}}) }},
|
||||
...options,
|
||||
}});
|
||||
const payload = await response.json().catch(() => ({{ error: response.statusText }}));
|
||||
if (!response.ok) throw new Error(payload.error || response.statusText);
|
||||
return payload;
|
||||
}}
|
||||
|
||||
async function loadRoutes() {{
|
||||
const payload = await requestJson("/api/routes");
|
||||
routes = payload.routes;
|
||||
$("count").textContent = `${{routes.length}} regels`;
|
||||
renderRoutes();
|
||||
setStatus("Geladen.");
|
||||
}}
|
||||
|
||||
function renderRoutes() {{
|
||||
const needle = $("search").value.trim().toLowerCase();
|
||||
const rows = routes
|
||||
.map((route, index) => ({{ route, index }}))
|
||||
.filter(({{ route }}) => !needle || route.mailbox.toLowerCase().includes(needle) || route.domains.join(" ").toLowerCase().includes(needle))
|
||||
.map(({{ route, index }}) => `
|
||||
<tr class="${{selectedIndex === index ? "selected" : ""}}">
|
||||
<td class="index">${{index + 1}}</td>
|
||||
<td class="domains">${{route.domains.map(escapeHtml).join("<br>")}}</td>
|
||||
<td class="mailbox">${{escapeHtml(route.mailbox)}}</td>
|
||||
<td class="actions"><button type="button" class="secondary" onclick="editRoute(${{index}})">Bewerk</button></td>
|
||||
</tr>`);
|
||||
$("routesBody").innerHTML = rows.join("");
|
||||
}}
|
||||
|
||||
function escapeHtml(value) {{
|
||||
return value.replace(/[&<>"']/g, (ch) => ({{ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" }}[ch]));
|
||||
}}
|
||||
|
||||
$("routeForm").addEventListener("submit", async (event) => {{
|
||||
event.preventDefault();
|
||||
const route = {{ domains: splitDomains($("domains").value), mailbox: $("mailbox").value.trim() }};
|
||||
const indexValue = $("routeIndex").value;
|
||||
try {{
|
||||
const method = indexValue === "" ? "POST" : "PUT";
|
||||
const url = indexValue === "" ? "/api/routes" : `/api/routes/${{indexValue}}`;
|
||||
const payload = await requestJson(url, {{ method, body: JSON.stringify(route) }});
|
||||
routes = payload.routes;
|
||||
clearForm();
|
||||
setStatus("Opgeslagen.");
|
||||
}} catch (error) {{
|
||||
setStatus(error.message, true);
|
||||
}}
|
||||
}});
|
||||
|
||||
$("deleteButton").addEventListener("click", async () => {{
|
||||
if (selectedIndex === null) return;
|
||||
if (!confirm(`Regel ${{selectedIndex + 1}} verwijderen?`)) return;
|
||||
try {{
|
||||
const payload = await requestJson(`/api/routes/${{selectedIndex}}`, {{ method: "DELETE" }});
|
||||
routes = payload.routes;
|
||||
clearForm();
|
||||
setStatus("Verwijderd.");
|
||||
}} catch (error) {{
|
||||
setStatus(error.message, true);
|
||||
}}
|
||||
}});
|
||||
|
||||
$("search").addEventListener("input", renderRoutes);
|
||||
$("newButton").addEventListener("click", clearForm);
|
||||
$("clearButton").addEventListener("click", clearForm);
|
||||
$("reloadButton").addEventListener("click", loadRoutes);
|
||||
loadRoutes().catch((error) => setStatus(error.message, true));
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
""".encode("utf-8")
|
||||
|
||||
|
||||
class RouteHandler(BaseHTTPRequestHandler):
|
||||
routes_file = DOMAIN_ROUTES_FILE
|
||||
username = ""
|
||||
password = ""
|
||||
|
||||
def log_message(self, fmt: str, *args: object) -> None:
|
||||
print(f"{self.address_string()} - {fmt % args}")
|
||||
|
||||
def _require_auth(self) -> bool:
|
||||
if basic_auth_ok(self.headers.get("Authorization"), self.username, self.password):
|
||||
return True
|
||||
self.send_response(HTTPStatus.UNAUTHORIZED)
|
||||
self.send_header("WWW-Authenticate", 'Basic realm="mailcat"')
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return False
|
||||
|
||||
def _send(self, status: HTTPStatus, body: bytes, content_type: str) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def _json(self, status: HTTPStatus, payload: dict) -> None:
|
||||
self._send(
|
||||
status,
|
||||
json.dumps(payload, indent=2, ensure_ascii=False).encode("utf-8"),
|
||||
"application/json; charset=utf-8",
|
||||
)
|
||||
|
||||
def _error(self, status: HTTPStatus, message: str) -> None:
|
||||
self._json(status, {"error": message})
|
||||
|
||||
def _read_route_body(self) -> dict:
|
||||
length = int(self.headers.get("Content-Length", "0"))
|
||||
body = self.rfile.read(length)
|
||||
payload = json.loads(body.decode("utf-8"))
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError("Request body must contain one route object")
|
||||
return payload
|
||||
|
||||
def _route_index(self) -> int | None:
|
||||
parts = [part for part in urlparse(self.path).path.split("/") if part]
|
||||
if len(parts) != 3 or parts[:2] != ["api", "routes"]:
|
||||
return None
|
||||
try:
|
||||
return int(parts[2])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
def do_GET(self) -> None:
|
||||
if not self._require_auth():
|
||||
return
|
||||
path = urlparse(self.path).path
|
||||
if path == "/":
|
||||
self._send(HTTPStatus.OK, page_html(), "text/html; charset=utf-8")
|
||||
return
|
||||
if path == "/api/routes":
|
||||
try:
|
||||
self._json(HTTPStatus.OK, {"routes": load_routes(self.routes_file)})
|
||||
except Exception as exc:
|
||||
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
|
||||
return
|
||||
self._error(HTTPStatus.NOT_FOUND, "Not found")
|
||||
|
||||
def do_POST(self) -> None:
|
||||
if not self._require_auth():
|
||||
return
|
||||
if urlparse(self.path).path != "/api/routes":
|
||||
self._error(HTTPStatus.NOT_FOUND, "Not found")
|
||||
return
|
||||
try:
|
||||
routes = load_routes(self.routes_file)
|
||||
routes.append(self._read_route_body())
|
||||
self._json(HTTPStatus.CREATED, {"routes": save_routes(routes, self.routes_file)})
|
||||
except json.JSONDecodeError:
|
||||
self._error(HTTPStatus.BAD_REQUEST, "Invalid JSON")
|
||||
except ValueError as exc:
|
||||
self._error(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
except Exception as exc:
|
||||
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
|
||||
|
||||
def do_PUT(self) -> None:
|
||||
if not self._require_auth():
|
||||
return
|
||||
index = self._route_index()
|
||||
if index is None:
|
||||
self._error(HTTPStatus.NOT_FOUND, "Not found")
|
||||
return
|
||||
try:
|
||||
routes = load_routes(self.routes_file)
|
||||
if index < 0 or index >= len(routes):
|
||||
raise ValueError("Route index out of range")
|
||||
routes[index] = self._read_route_body()
|
||||
self._json(HTTPStatus.OK, {"routes": save_routes(routes, self.routes_file)})
|
||||
except json.JSONDecodeError:
|
||||
self._error(HTTPStatus.BAD_REQUEST, "Invalid JSON")
|
||||
except ValueError as exc:
|
||||
self._error(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
except Exception as exc:
|
||||
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
|
||||
|
||||
def do_DELETE(self) -> None:
|
||||
if not self._require_auth():
|
||||
return
|
||||
index = self._route_index()
|
||||
if index is None:
|
||||
self._error(HTTPStatus.NOT_FOUND, "Not found")
|
||||
return
|
||||
try:
|
||||
routes = load_routes(self.routes_file)
|
||||
if index < 0 or index >= len(routes):
|
||||
raise ValueError("Route index out of range")
|
||||
del routes[index]
|
||||
self._json(HTTPStatus.OK, {"routes": save_routes(routes, self.routes_file)})
|
||||
except ValueError as exc:
|
||||
self._error(HTTPStatus.BAD_REQUEST, str(exc))
|
||||
except Exception as exc:
|
||||
self._error(HTTPStatus.INTERNAL_SERVER_ERROR, str(exc))
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run the mailcat route management web UI.")
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
parser.add_argument("--config", type=Path, default=DEFAULT_CONFIG_FILE)
|
||||
parser.add_argument("--routes-file", type=Path, default=DOMAIN_ROUTES_FILE)
|
||||
args = parser.parse_args()
|
||||
|
||||
username, password = load_web_config(args.config)
|
||||
RouteHandler.routes_file = args.routes_file
|
||||
RouteHandler.username = username
|
||||
RouteHandler.password = password
|
||||
|
||||
server = ThreadingHTTPServer((args.host, args.port), RouteHandler)
|
||||
print(f"mailcat route web listening on http://{html.escape(args.host)}:{args.port}/")
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+184
-202
@@ -6,36 +6,36 @@ You are continuing the `mailcat` project in `/Users/hanswienen/Documents/Develop
|
||||
|
||||
Build a reliable mailbox cleanup and automation toolkit for `australius.nl`.
|
||||
|
||||
The project started as an offline analysis of a local mailbox export and is being moved toward an IMAP-based sorter because Sieve does not work with the email provider. The intended production model is:
|
||||
The operational direction is Python over IMAP/SMTP because Sieve is not viable with the provider. The current test target is `backup@australius.nl`. The user wants to test always-on automation first on a local QNAP-like NAS named `hades`, not on `vps.australius.nl`.
|
||||
|
||||
- Email is hosted at `australius.nl`.
|
||||
- A VPS is available at `vps.austalius.nl`, but the user now wants to test automation first on a local home NAS instead of the VPS.
|
||||
- The test mailbox is `backup@australius.nl`.
|
||||
- The future automation should run on an always-on remote host, preferably via SSH deployment and an IMAP IDLE daemon. The NAS is QNAP-like BusyBox Linux without systemd, so test deployment should use `SERVICE_MANAGER=plain`.
|
||||
- The current end-to-end test uses the `backup` account as a disposable test target.
|
||||
Do not run mailbox-affecting scripts yourself unless the user explicitly authorizes it. The user prefers to run those scripts personally.
|
||||
|
||||
## Repository State
|
||||
## Collaboration Rules
|
||||
|
||||
Git workflow:
|
||||
- Follow the git workflow from `AGENTS.md`.
|
||||
- Work on branch `codex` unless the user says otherwise.
|
||||
- Before every commit, rewrite this file from scratch as the current reconstruction prompt.
|
||||
- Commit after each user-prompted code change using the required three-section commit message format.
|
||||
- The user asked: first provide a plan and wait for approval before doing future work.
|
||||
- Do not stage local runtime output such as `verplaats_log.json` unless the user explicitly asks.
|
||||
|
||||
- The repo has branches `main`, `claude`, and `codex`.
|
||||
- Work should continue on `codex` unless the user says otherwise.
|
||||
- Follow the user workflow: initialize Git when needed, branch before work in existing repos, and commit after each user-prompted change using the required three-section commit message format.
|
||||
- Before every commit, rewrite this file from scratch so it fully reconstructs the current project state.
|
||||
## Local Data And Secrets
|
||||
|
||||
Ignored local data includes:
|
||||
Ignored or local-only data includes:
|
||||
|
||||
- `mailbox/`
|
||||
- `**/*.eml`
|
||||
- `config.json`
|
||||
- `web_config.json`
|
||||
- `__pycache__/`
|
||||
- bytecode files
|
||||
- `*.log`
|
||||
- `.DS_Store`
|
||||
- generated runtime state/log JSON files
|
||||
|
||||
Do not commit email `.eml` files or credentials.
|
||||
Do not commit credentials.
|
||||
|
||||
## Current Decisions
|
||||
## Routing Policy
|
||||
|
||||
Sorting policy:
|
||||
|
||||
@@ -43,232 +43,214 @@ Sorting policy:
|
||||
- `Sent`, `Drafts`, `Trash`, and `Spam` are excluded from sorting.
|
||||
- Every other mailbox folder is treated as a source.
|
||||
- Do not use `Archief` as a final destination.
|
||||
- Existing `Archief.*` folders are source folders; route matching messages out of them into functional destination folders.
|
||||
- Existing `Archief.*` folders are source folders. Matching messages should move out into functional folders.
|
||||
- If an archived/source message does not match a rule, leave it in place.
|
||||
|
||||
Destination policy:
|
||||
|
||||
- Final destination folders are functional folders such as `Financieel`, `Werk`, `Diensten`, `Nieuwsbrieven`, `Bestellingen`, `Mobiliteit`, and `Technisch`.
|
||||
- Final destination folders under `Archief` should not be created.
|
||||
- `Technisch` and `Technisch.DMARC` are valid destination folders.
|
||||
- `Administratie` is a valid destination root; KvK messages route to `Administratie.KvK`.
|
||||
- Former assignment senders should be routed into separate folders under `Werk.Opdrachten.*`.
|
||||
- `zuiderzee.net` and generic `gmail.com` messages should remain unmatched for now.
|
||||
- Final destination folders are functional folders such as `Financieel`, `Werk`, `Diensten`, `Nieuwsbrieven`, `Bestellingen`, `Mobiliteit`, `Administratie`, and `Technisch`.
|
||||
- `Technisch` and `Technisch.DMARC` are valid destinations.
|
||||
- `Administratie.KvK` is the destination for KvK messages.
|
||||
- Former assignment senders route under `Werk.Opdrachten.*`.
|
||||
- `zuiderzee.net` and generic `gmail.com` intentionally remain unmatched.
|
||||
- AI providers route under `Diensten.AI`; confirmed AI providers include Canva, Claude/Anthropic, Cursor, Huckr, Jamie, Mistral, OpenAI, and Perplexity.
|
||||
- Microsoft, Zapier, Envato, reMarkable, Nord, and ExpressVPN are not treated as AI providers unless the user reclassifies them.
|
||||
|
||||
Sieve:
|
||||
|
||||
- `mailrules.sieve` exists as historical/source material.
|
||||
- Sieve deployment is not viable with the provider.
|
||||
- Future work should move operational automation to Python over IMAP/SMTP.
|
||||
|
||||
## Current Implementation Notes
|
||||
## Current Implementation
|
||||
|
||||
Shared modules:
|
||||
|
||||
- `imap_utils.py` centralizes IMAP modified UTF-7 folder encoding/decoding, quoted mailbox names, LIST parsing, and folder listing.
|
||||
- `mail_routes.py` centralizes `PREFIX`, domain routes, invoice keywords, source-folder exclusions, already-sorted folders, invoice quarter routing, generated destination folders, and normalized mailing-list routing.
|
||||
- `mail_routes.py` centralizes routing behavior: invoice keyword detection, mailing-list routing, domain routing, source-folder exclusions, already-sorted folders, invoice quarter routing, and destination folder generation.
|
||||
- `mail_imap_ops.py` centralizes message header extraction, destination computation, destination mailbox creation, and UID COPY plus UID STORE move semantics.
|
||||
|
||||
Current INBOX candidate domain routing additions:
|
||||
Domain routes:
|
||||
|
||||
- `mijnkantoorapp.nl` and `dijkman-ac.nl` route to `Financieel.Boekhouding.Dijkman`.
|
||||
- `drenthe.nl`, `rdw.nl`, and `plasbossinade.nl` route to separate folders under `Werk.Opdrachten`.
|
||||
- `kvk.nl` routes to `Administratie.KvK`.
|
||||
- `godaddy.com`, `cloud86.io`, and `cloud86.zendesk.com` route under `Diensten.Hosting`.
|
||||
- Clear AI providers route under `Diensten.AI`: Canva, Claude/Anthropic, Cursor, Huckr, Jamie, Mistral, OpenAI, and Perplexity.
|
||||
- `microsoft.com`, `market.envato.com`, `email.remarkable.com`, `nordaccount.com`, and `info.expressvpn.com` route under non-AI `Diensten` folders unless the user explicitly reclassifies them. Microsoft and Zapier should not be treated as AI providers for now.
|
||||
- `maandag.com` routes under `Werk.Opdrachten`.
|
||||
- `tasks.clickup.com`, `gemeenteprojecten.talent-pool.com`, and `engage.istockphoto.com` route under `Afmelden`.
|
||||
- `n26.com`, `update.bunq.com`, and `hello.bunq.com` route under `Financieel.Bank`.
|
||||
- `notify.orcid.org` routes to `Nieuwsbrieven.Leren.ORCID`.
|
||||
- `zuiderzee.net` and `gmail.com` are intentionally not routed.
|
||||
- Hardcoded domain routes have been moved out of Python into `domain_routes.json`.
|
||||
- `mail_routes.domain_routes()` loads and validates `domain_routes.json`.
|
||||
- `mail_routes.domain_lookup()` builds the cached domain lookup.
|
||||
- `DOMAIN_ROUTES` and `DOMAIN_LOOKUP` still exist for compatibility, but active routing uses `domain_lookup()`.
|
||||
- If a long-running sorter process is already running, route edits on disk require restarting that process before it sees the new rules.
|
||||
|
||||
Scripts using shared IMAP folder handling:
|
||||
Mailing-list routes:
|
||||
|
||||
- `verplaats_bestaand.py`
|
||||
- `maak_mappen.py`
|
||||
- `kopieer_naar_backup.py`
|
||||
- `download_mailbox.py`
|
||||
- `dagelijks_overzicht.py`
|
||||
- `sort_mail_daemon.py`
|
||||
- `mailinglist_routes.json` is still separate from domain routes.
|
||||
- `mail_routes.py` checks mailing-list routes before domain routes.
|
||||
- `mailinglist_routes.json` covers normalized policy from `decisions.json`, including `Afmelden.*` destinations.
|
||||
- Mailing-list routing supports domain and `From` substring matching; it does not yet fetch or match `List-ID`.
|
||||
|
||||
`dagelijks_overzicht.py` current behavior:
|
||||
Historical sorting:
|
||||
|
||||
- Builds an HTML digest of messages that arrived on the selected date, defaulting to yesterday.
|
||||
- Uses central `config.py`; select the mailbox account with `--account`, for example `--account backup`.
|
||||
- Sends the digest via SMTP using the selected account credentials.
|
||||
- Supports `--mail-to <address>`; the NAS daily report target is `hans@australius.nl`.
|
||||
- Sends the report even when zero messages are found, so scheduled runs are visible.
|
||||
- `verplaats_bestaand.py` defaults to dry-run. Real moves require `--uitvoeren` and interactive `JA`.
|
||||
- Account selection is explicit with `--account <naam>`, for example `--account backup`.
|
||||
- `--audit` is available in dry-run mode.
|
||||
- It logs to `verplaats_log.json`, retries IMAP aborts where implemented, and parses FETCH responses defensively.
|
||||
|
||||
Shared operation helpers:
|
||||
Folder creation:
|
||||
|
||||
- `mail_imap_ops.py` centralizes header decoding, FETCH payload extraction, routing fetched headers to full IMAP destinations, destination mailbox creation, and copy/delete move semantics for live sorting.
|
||||
- `verplaats_bestaand.py` still contains its own historical-sorter retry/logging flow; do not refactor it casually because it has already been exercised against the backup mailbox.
|
||||
- `maak_mappen.py` creates destination folders generated from `mail_routes.destination_folders()`.
|
||||
- It creates parent folders as needed and avoids `Archief.*` destinations.
|
||||
|
||||
`verplaats_bestaand.py` current behavior:
|
||||
Backup mirror:
|
||||
|
||||
- Default mode is dry-run; real moves require `--uitvoeren` and an interactive `JA` confirmation.
|
||||
- Account selection is explicit with `--account <naam>`; use `--account backup` for the disposable backup mailbox. The script prints both the config account name and mailbox address before connecting.
|
||||
- `--audit` is available only in dry-run mode. It suppresses per-message dry-run lines and prints planned move counts grouped by source folder and destination folder for step 2 and step 3.
|
||||
- Long scans show progress counters: source folders are printed as `[current/total]`, and audit mode prints `Gescand: processed/total` every 250 messages and at folder completion.
|
||||
- IMAP `LIST`, `SELECT`, `UID SEARCH`, and `UID FETCH` aborts are logged and retried once after reconnecting. If reconnect also fails, the script records an `ABORT`/`ERROR` in `verplaats_log.json` instead of printing a Python traceback.
|
||||
- IMAP `FETCH` responses are parsed by selecting the first tuple bytes payload. This avoids crashes when `imaplib` returns extra response items before/after the actual header payload.
|
||||
- `MAP_RENAMES` is intentionally empty.
|
||||
- Source folders are selected by shared `list_folders()` plus `mail_routes.is_source_folder()`.
|
||||
- Step 2 routes all eligible source folders, not just `INBOX`.
|
||||
- Step 2 reports counters per folder and in total: planned moves, actual moves when executing, no-match messages, skipped messages, and fetch/move failures.
|
||||
- Dry runs report `Gepland` counters instead of misleadingly showing zero moved.
|
||||
- Step 3 invoice-quarter sorting also reports planned/moved/failure counters and supports audit summaries.
|
||||
- Move success requires source reselect, `UID COPY`, and `UID STORE +FLAGS \Deleted` to return `OK`.
|
||||
- Move and fetch failures are recorded in `verplaats_log.json` under `fouten` with step, source folder, UID, destination when known, action, status, and server response.
|
||||
- Step 3 no longer treats `INBOX.Facturen - verwerkt` as a source.
|
||||
- IMAP mailbox names are consistently quoted and encoded through `imap_utils.quote_mailbox()`.
|
||||
- `kopieer_naar_backup.py` mirrors configured IMAP accounts, deduplicating by `Message-ID` per destination folder.
|
||||
- It uses `backup_log.json` for progress and does not mark a folder complete if failures occurred.
|
||||
|
||||
`maak_mappen.py` current behavior:
|
||||
Live daemon:
|
||||
|
||||
- Destination folders are generated from `mail_routes.destination_folders()`.
|
||||
- It no longer creates `Archief.*` destination folders.
|
||||
- It creates all parent folders needed for route targets, including `Technisch` and `Technisch.DMARC`.
|
||||
- IMAP mailbox names are consistently quoted and encoded through `imap_utils.quote_mailbox()`.
|
||||
- `sort_mail_daemon.py` defaults to `--account backup --folder INBOX`.
|
||||
- It uses shared routing through `mail_imap_ops.destination_from_header()`.
|
||||
- It maintains processed UID state and resets when UIDVALIDITY changes.
|
||||
- It supports `--once` and `--dry-run`.
|
||||
- It uses IMAP IDLE where available, falls back conservatively, reconnects on failures, and logs to `sort_mail_daemon.log` by default.
|
||||
|
||||
`kopieer_naar_backup.py` current behavior:
|
||||
Daily report:
|
||||
|
||||
- Mirrors folders from one configured IMAP account to another.
|
||||
- Skips `INBOX.Trash` and `INBOX.Spam`.
|
||||
- Deduplicates by `Message-ID` within each destination folder.
|
||||
- Uses shared folder listing and mailbox quoting/encoding for select/create/append operations.
|
||||
- Uses `backup_log.json` to record completed folders and the copied-message total.
|
||||
- Records select/search/fetch/append failures in `backup_log.json` under `fouten`.
|
||||
- Verifies the destination folder is selectable before dedupe and append work.
|
||||
- Does not mark a folder complete if any failure occurs while copying that folder.
|
||||
- `dagelijks_overzicht.py` builds an HTML digest for a selected date, defaulting to yesterday.
|
||||
- It supports `--account` and `--mail-to`.
|
||||
- The NAS daily report target is `hans@australius.nl`.
|
||||
- It sends a report even when there are zero messages.
|
||||
|
||||
`sort_mail_daemon.py` current behavior:
|
||||
## Web Route Management
|
||||
|
||||
- Intended remote-host automation entry point for IMAP sorting.
|
||||
- Defaults to `--account backup` and `--folder INBOX`.
|
||||
- Uses the same shared routing policy through `mail_imap_ops.destination_from_header()`.
|
||||
- Maintains a JSON state file of processed UIDs per folder and resets that state when UIDVALIDITY changes.
|
||||
- Supports `--once` for local/non-daemon smoke tests and `--dry-run` for non-mutating checks.
|
||||
- Watches for new mail with IMAP IDLE. On Python versions with public `imaplib.IMAP4.idle()` it uses that API; otherwise it uses a conservative private-IDLE fallback and reconnects on failures.
|
||||
- Logs to `sort_mail_daemon.log` by default, or to the path passed with `--log-file`.
|
||||
- Ensures the destination mailbox exists before moving a message, then moves by UID COPY plus UID STORE `\Deleted`, followed by expunge after a scan.
|
||||
The route-management web interface is implemented in `manage_routes_web.py`.
|
||||
|
||||
Remote/NAS deployment files:
|
||||
Current decisions:
|
||||
|
||||
- `deploy_vps.sh` is host-neutral despite its historical filename. It packages the repo excluding `.git`, `config.json`, logs, runtime JSON state/log files, bytecode, and local mailbox data; uploads to the SSH host in `REMOTE_HOST`; installs under `/opt/mailcat` by default; creates/uses a non-root system user named `mailcat` by default; installs a systemd service; enables but does not start the service.
|
||||
- The deploy script requires `REMOTE_HOST` and `REMOTE_USER`. The old `VPS_HOST` and `VPS_USER` names still work as aliases. Optional overrides: `APP_USER`, `APP_DIR`, `SERVICE_NAME`, and `ACCOUNT`.
|
||||
- The deploy script uploads a temporary remote shell script and runs it with `ssh -tt`, so remote `sudo` can prompt for a password on NAS systems that require a terminal. The remote script starts with `sudo -v`.
|
||||
- The deploy script exports an explicit remote PATH before running install commands: `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` by default, overrideable with `REMOTE_PATH`. It searches both PATH and common absolute paths such as `/usr/local/bin/useradd`, `/usr/sbin/useradd`, and `/sbin/useradd`. If `useradd` is genuinely unavailable, it falls back to the existing SSH user and that user's primary group for service ownership.
|
||||
- For the local NAS, use `SERVICE_MANAGER=plain`, `MAILCAT_HOME=/share/homes/mailcat`, and `PYTHON=/opt/bin/python3`. In plain mode, default paths are `APP_DIR=$MAILCAT_HOME/mailcat`, `STATE_DIR=$APP_DIR/state`, `LOG_DIR=$APP_DIR/log`, `RUN_DIR=$APP_DIR/run`, and `PID_FILE=$RUN_DIR/mailcat.pid`.
|
||||
- QNAP path facts supplied by the user: `SUDO=/usr/bin/sudo`, `USERADD=/usr/local/bin/useradd`, `PYTHON=/opt/bin/python3`, `INSTALL=/usr/bin/install`, `SED=/bin/sed`, `TAR=/bin/tar`, `MKDIR=/bin/mkdir`, `CHOWN=/bin/chown`, `CHMOD=/bin/chmod`, and `RM=/bin/rm`. `nologin`, `python3`, `systemctl`, and `nohup` are not in the user's default PATH.
|
||||
- The deploy script accepts absolute tool overrides for those commands and passes them into the remote script. In plain mode, the generated start/stop/status scripts also bake in absolute `MKDIR` and `RM` paths.
|
||||
- In plain mode, the deploy script does not install systemd files. It writes executable scripts to `$APP_DIR/bin/start_mailcat.sh`, `$APP_DIR/bin/stop_mailcat.sh`, `$APP_DIR/bin/status_mailcat.sh`, and `$APP_DIR/bin/run_daily_report.sh`.
|
||||
- `DAILY_REPORT_TO` controls the report recipient during deploy and defaults to `hans@australius.nl`. The daily report wrapper appends output to `$LOG_DIR/daily_report.out`.
|
||||
- `systemd/mailcat-sort.service` is a template consumed by `deploy_vps.sh`; after placeholder replacement it runs `sort_mail_daemon.py --account backup --folder INBOX` and stores state/logs under `/var/lib/mailcat` and `/var/log/mailcat`.
|
||||
- `config.json` remains ignored and must be created manually on the remote host with mode `600` before starting the service.
|
||||
- LAN binding: `0.0.0.0`
|
||||
- Port: `4321`
|
||||
- Authentication: HTTP Basic Auth
|
||||
- Credentials come from ignored `web_config.json` or environment variables `MAILCAT_WEB_USER` and `MAILCAT_WEB_PASSWORD`.
|
||||
- Example config is in `web_config.example.json`.
|
||||
|
||||
Scope:
|
||||
|
||||
- The web UI manages all domain routes in `domain_routes.json`, including the previous built-in routes.
|
||||
- Mailing-list routes remain in `mailinglist_routes.json` and are not yet managed by the web UI.
|
||||
|
||||
Behavior:
|
||||
|
||||
- `GET /` serves a dense table/form UI.
|
||||
- `GET /api/routes` returns all routes.
|
||||
- `POST /api/routes` appends a route.
|
||||
- `PUT /api/routes/<index>` updates a route.
|
||||
- `DELETE /api/routes/<index>` deletes a route.
|
||||
- Domains and mailbox names are validated server-side.
|
||||
- Duplicate domains across routes are rejected.
|
||||
- Writes are atomic and create `domain_routes.json.bak`.
|
||||
|
||||
Security note:
|
||||
|
||||
- This is Basic Auth on the internal network. It is not HTTPS by itself. If exposed beyond the LAN, put it behind TLS or a VPN.
|
||||
|
||||
## NAS Deployment
|
||||
|
||||
`deploy_vps.sh` is host-neutral despite its historical filename.
|
||||
|
||||
For the QNAP-like NAS, use:
|
||||
|
||||
```sh
|
||||
SERVICE_MANAGER=plain
|
||||
MAILCAT_HOME=/share/homes/mailcat
|
||||
PYTHON=/opt/bin/python3
|
||||
APP_USER=mailcat
|
||||
```
|
||||
|
||||
Known NAS command paths supplied by the user:
|
||||
|
||||
- `SUDO=/usr/bin/sudo`
|
||||
- `USERADD=/usr/local/bin/useradd`
|
||||
- `PYTHON=/opt/bin/python3`
|
||||
- `INSTALL=/usr/bin/install`
|
||||
- `SED=/bin/sed`
|
||||
- `TAR=/bin/tar`
|
||||
- `MKDIR=/bin/mkdir`
|
||||
- `CHOWN=/bin/chown`
|
||||
- `CHMOD=/bin/chmod`
|
||||
- `RM=/bin/rm`
|
||||
|
||||
In `SERVICE_MANAGER=plain` mode:
|
||||
|
||||
- Default `APP_DIR` is `/share/homes/mailcat/mailcat`.
|
||||
- Default `STATE_DIR` is `$APP_DIR/state`.
|
||||
- Default `LOG_DIR` is `$APP_DIR/log`.
|
||||
- Default `RUN_DIR` is `$APP_DIR/run`.
|
||||
- The sorter PID file is `$RUN_DIR/mailcat.pid`.
|
||||
- The web PID file is `$RUN_DIR/mailcat_web.pid`.
|
||||
- The deploy script installs:
|
||||
- `$APP_DIR/bin/start_mailcat.sh`
|
||||
- `$APP_DIR/bin/stop_mailcat.sh`
|
||||
- `$APP_DIR/bin/status_mailcat.sh`
|
||||
- `$APP_DIR/bin/run_daily_report.sh`
|
||||
- `$APP_DIR/bin/start_web.sh`
|
||||
- `$APP_DIR/bin/stop_web.sh`
|
||||
- `$APP_DIR/bin/status_web.sh`
|
||||
- The sorter appends wrapper stdout/stderr to `$LOG_DIR/daemon.out`.
|
||||
- The web UI appends stdout/stderr to `$LOG_DIR/web.out`.
|
||||
- The daily report wrapper appends stdout/stderr to `$LOG_DIR/daily_report.out`.
|
||||
|
||||
Before starting services on the NAS:
|
||||
|
||||
- Create `/share/homes/mailcat/mailcat/config.json` with mode `600`.
|
||||
- Create `/share/homes/mailcat/mailcat/web_config.json` with mode `600`, for example:
|
||||
|
||||
```json
|
||||
{
|
||||
"username": "mailcat",
|
||||
"password": "use-a-real-password"
|
||||
}
|
||||
```
|
||||
|
||||
After route edits through the web UI, restart `start_mailcat.sh`/`stop_mailcat.sh` if the live sorter should use the new rules.
|
||||
|
||||
Systemd mode still exists for a future VPS deployment using `systemd/mailcat-sort.service`.
|
||||
|
||||
## Current Backup Test Status
|
||||
|
||||
The user reported that `backup@australius.nl` was empty except for folder structure, so the stale `backup_log.json` state was reset and the real mailbox state was inspected.
|
||||
The backup mirror completed earlier:
|
||||
|
||||
Observed before the mirror rerun:
|
||||
- Source eligible mirror scope: 34 folders, 11,203 messages.
|
||||
- Script copied: 11,190 messages.
|
||||
- Duplicate skips: 13.
|
||||
- Backup account after mirror: 159 folders, 11,190 messages.
|
||||
|
||||
- Source `hans` account: 36 folders, 11,857 total messages.
|
||||
- Eligible mirror scope: 34 folders, 11,203 messages, excluding only `INBOX.Spam` and `INBOX.Trash`.
|
||||
- Backup account: 132 folders, 0 messages.
|
||||
The user later ran the backup historical sorter successfully and reported:
|
||||
|
||||
After resetting the copy status, `python3 kopieer_naar_backup.py --van hans --naar backup` completed:
|
||||
```text
|
||||
Totaal gepland: 0 | Totaal geen match: 2218 | Totaal overgeslagen: 8219 | Totaal fouten: 0
|
||||
```
|
||||
|
||||
- Script total copied: 11,190.
|
||||
- Script duplicate skips: 13.
|
||||
- `backup_log.json` marked 34 folders complete and recorded `totaal_gekopieerd: 11190`.
|
||||
Treat that as a successful backup historical-sorter run with no immediate recovery work needed.
|
||||
|
||||
Server-side IMAP verification after the mirror:
|
||||
## Verification Commands
|
||||
|
||||
- Source account still has 36 folders and 11,857 total messages.
|
||||
- Eligible mirror scope is still 34 folders and 11,203 messages.
|
||||
- Backup account has 159 folders and 11,190 total messages.
|
||||
- Backup has 24 non-empty folders.
|
||||
- The 13-message difference equals the script's duplicate `Message-ID` skips.
|
||||
- Count deltas caused by duplicate skips:
|
||||
- `INBOX.Archief.2020.verzonden`: source 162, backup 161.
|
||||
- `INBOX.Archief.2021.inkomend`: source 894, backup 890.
|
||||
- `INBOX.Archief.2022.verzonden`: source 174, backup 173.
|
||||
- `INBOX.Facturen - verwerkt`: source 153, backup 151.
|
||||
- `INBOX.Sent`: source 836, backup 831.
|
||||
Latest local verification should include:
|
||||
|
||||
Empty source folders not visible as selectable backup folders after the mirror audit:
|
||||
```sh
|
||||
python3 -m py_compile *.py tests/*.py
|
||||
bash -n deploy_vps.sh
|
||||
python3 -m unittest discover -s tests
|
||||
git diff --check
|
||||
```
|
||||
|
||||
- `INBOX.Facturen - te verwerken`
|
||||
- `INBOX.Notes`
|
||||
- `INBOX.Technisch.dmarc`
|
||||
Useful routing sanity check:
|
||||
|
||||
Do not run mailbox-affecting scripts yourself in this project. The user wants to run scripts personally when instructed. A sorter dry-run audit on `backup@australius.nl` is the next safe diagnostic, but ask the user to run it rather than running it yourself.
|
||||
```sh
|
||||
python3 - <<'PY'
|
||||
from mail_routes import destination_folders, domain_routes, route_from_subject
|
||||
print("routes", len(domain_routes()))
|
||||
print("folders", len(destination_folders()))
|
||||
print("openai", route_from_subject("noreply@openai.com", "update"))
|
||||
print("huckr", route_from_subject("hello@huckr.ai", "update"))
|
||||
PY
|
||||
```
|
||||
|
||||
## Reports and Findings
|
||||
## Next Expected Work
|
||||
|
||||
The prior script review is in `reports/script_review_findings.md`.
|
||||
Likely next steps:
|
||||
|
||||
The decisions-based mailing-list routing proposal is in `reports/decisions_mailinglist_routing_report.md`.
|
||||
|
||||
That report interprets `decisions.json` as advisory intent:
|
||||
|
||||
- `keuze: "h"` means keep and route to an appropriate functional mailbox.
|
||||
- `keuze: "a"` means route below `INBOX.Afmelden.*` for unsubscribe/review.
|
||||
- `keuze: "s"` is ambiguous and should be confirmed before automation.
|
||||
- Obvious aliases from the same sender or organization should be combined, including Coursera, MIT Technology Review, Eva Keiffenheim/Substack, PostNL, STRATO, Forte Labs, CIONET, Gusti, Nord, Proton, Vonage/Nexmo, and Visme.
|
||||
- The normalized policy is in `mailinglist_routes.json`; `mail_routes.py` loads it before falling back to legacy `DOMAIN_ROUTES`.
|
||||
- `mailinglist_routes.json` has 153 grouped rules and covers every key from `decisions.json` through `source_decision_keys`.
|
||||
- `destination_folders()` includes functional keep folders plus `Afmelden.*` folders from `mailinglist_routes.json`.
|
||||
- Invoice keyword routing still takes precedence over mailing-list routing.
|
||||
- Mailing-list routing currently supports domain and `From` substring policy matching. It does not yet fetch or match `List-ID` headers.
|
||||
|
||||
Notable policy risks:
|
||||
|
||||
- `circle8.nl` still has ambiguous decision value `s`; keep it as review/functional routing until the user confirms the meaning.
|
||||
- `alfen.com`, `vodafone.nl`, `dhlecommerce.nl`, and Proton splits are implemented according to `decisions.json` precedence, but remain notable policy changes compared with older hardcoded domain routes.
|
||||
|
||||
## Last Verification
|
||||
|
||||
Latest local code verification:
|
||||
|
||||
- `python3 -m py_compile *.py`
|
||||
- `python3 -m py_compile *.py tests/*.py`
|
||||
- `python3 -m unittest discover -s tests`
|
||||
- `bash -n deploy_vps.sh`
|
||||
- Explicit `route_from_subject()` checks for the latest INBOX candidate domains, including negative checks for `zuiderzee.net` and `gmail.com`.
|
||||
- `git diff --check`
|
||||
|
||||
Earlier route/folder consistency check returned:
|
||||
|
||||
- `routes 72`
|
||||
- `folders 127`
|
||||
- `missing_targets []`
|
||||
- `archief_targets []`
|
||||
|
||||
Latest live mailbox verification:
|
||||
|
||||
- Direct read-only IMAP count audit of both `hans` and `backup` accounts after the mirror.
|
||||
- The user ran the previous dry-run sorter version against `backup@australius.nl`. It reached `INBOX.Archief.2022.verzonden` after completing `INBOX.Archief.2022.inkomend`, then the IMAP server closed the connection with `Server shutting down.`. The current code has reconnect/retry handling for that failure mode.
|
||||
- The user started an actual sorter run on `backup@australius.nl`; it began moving messages from `INBOX` and then crashed on an unexpected IMAP `FETCH` response shape: `AttributeError: 'int' object has no attribute 'decode'`. The local `verplaats_log.json` contains processed `INBOX:<uid>` entries from that run and should be preserved if the user resumes sorting. The current code fixes this parser crash.
|
||||
- A later read-only INBOX inspection of the `backup` account found 1,908 unmatched messages and identified additional candidate sender domains. No mailbox-affecting script was run by Codex.
|
||||
- After the latest routing refinements, the user reported the backup sorter ended with `Totaal gepland: 0 | Totaal geen match: 2218 | Totaal overgeslagen: 8219 | Totaal fouten: 0`. Treat this as a successful backup historical-sorter run with no immediate recovery work needed.
|
||||
- Remote NAS/systemd deployment and daemon live tests have not yet been run by Codex.
|
||||
|
||||
## What To Do Next
|
||||
|
||||
Recommended next work:
|
||||
|
||||
1. Deploy to the QNAP NAS with a command like `REMOTE_HOST=hades REMOTE_USER=Hel SERVICE_MANAGER=plain MAILCAT_HOME=/share/homes/mailcat ./deploy_vps.sh`.
|
||||
2. Before starting the daemon, ensure `/share/homes/mailcat/mailcat/config.json` on the NAS contains the `backup` account credentials and has mode `600`.
|
||||
3. On the NAS, start and inspect the daemon:
|
||||
- `sudo -u mailcat /share/homes/mailcat/mailcat/bin/start_mailcat.sh`
|
||||
- `sudo -u mailcat /share/homes/mailcat/mailcat/bin/status_mailcat.sh`
|
||||
- `tail -f /share/homes/mailcat/mailcat/log/sort_mail_daemon.log`
|
||||
4. Test the daily report manually: `sudo -u mailcat /share/homes/mailcat/mailcat/bin/run_daily_report.sh`, then check `tail -100 /share/homes/mailcat/mailcat/log/daily_report.out` and confirm mail delivery to `hans@australius.nl`.
|
||||
5. Add the daily report wrapper to QNAP scheduler after the manual test succeeds.
|
||||
6. Send controlled test messages to `backup@australius.nl` for invoices, clear AI providers, a newsletter/service route, and unmatched mail; verify expected folder moves and daemon logs.
|
||||
7. Add `List-ID` header support to mailing-list routing if live daemon tests show sender/domain matching is too coarse.
|
||||
|
||||
## Persistent File Rule
|
||||
|
||||
Before every future commit in this project, rewrite this `restart_prompt.md` file so it describes the current state at that commit. Do not append. Replace the content with a fresh, accurate reconstruction prompt.
|
||||
1. Deploy the updated code to the NAS in `SERVICE_MANAGER=plain` mode.
|
||||
2. Create remote `web_config.json`.
|
||||
3. Start the web UI with `/share/homes/mailcat/mailcat/bin/start_web.sh`.
|
||||
4. Open `http://<nas-host-or-ip>:4321/` from the internal network and verify Basic Auth.
|
||||
5. Make a harmless route edit, verify `domain_routes.json` and `domain_routes.json.bak`, then restart the sorter if the live daemon is running.
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import base64
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from manage_routes_web import basic_auth_ok, load_routes, save_routes, validate_routes
|
||||
|
||||
|
||||
class ManageRoutesWebTests(unittest.TestCase):
|
||||
def test_validate_routes_normalizes_domains_and_mailbox(self):
|
||||
routes = validate_routes([
|
||||
{"domains": [" Example.COM ", "example.com"], "mailbox": " Diensten.Test "}
|
||||
])
|
||||
self.assertEqual(routes, [{"domains": ["example.com"], "mailbox": "Diensten.Test"}])
|
||||
|
||||
def test_validate_routes_rejects_duplicate_domain_across_routes(self):
|
||||
with self.assertRaisesRegex(ValueError, "already used"):
|
||||
validate_routes([
|
||||
{"domains": ["example.com"], "mailbox": "Diensten.Een"},
|
||||
{"domains": ["EXAMPLE.com"], "mailbox": "Diensten.Twee"},
|
||||
])
|
||||
|
||||
def test_save_routes_writes_backup_and_normalized_json(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / "domain_routes.json"
|
||||
path.write_text(
|
||||
json.dumps([{"domains": ["old.example"], "mailbox": "Diensten.Oud"}]) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
saved = save_routes(
|
||||
[{"domains": [" New.EXAMPLE "], "mailbox": " Diensten.Nieuw "}],
|
||||
path,
|
||||
)
|
||||
|
||||
self.assertEqual(saved, [{"domains": ["new.example"], "mailbox": "Diensten.Nieuw"}])
|
||||
self.assertEqual(load_routes(path), saved)
|
||||
backup = path.with_suffix(".json.bak")
|
||||
self.assertTrue(backup.exists())
|
||||
self.assertIn("old.example", backup.read_text(encoding="utf-8"))
|
||||
|
||||
def test_basic_auth_ok_accepts_exact_credentials(self):
|
||||
token = base64.b64encode(b"mailcat:secret").decode("ascii")
|
||||
self.assertTrue(basic_auth_ok(f"Basic {token}", "mailcat", "secret"))
|
||||
self.assertFalse(basic_auth_ok(f"Basic {token}", "mailcat", "wrong"))
|
||||
self.assertFalse(basic_auth_ok(None, "mailcat", "secret"))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"username": "mailcat",
|
||||
"password": "change-me"
|
||||
}
|
||||
Reference in New Issue
Block a user