From 90d97efa6c58c7deb3dff0e42e79e15a08f7b296 Mon Sep 17 00:00:00 2001 From: Hans Wienen Date: Sat, 4 Jul 2026 20:26:25 +0200 Subject: [PATCH] Add QNAP plain deployment mode Prompts used since the previous commit: - Doe maar - De homedirectory voor mailcat moet zijn /share/homes/mailcat Special observations: - Added SERVICE_MANAGER=plain for QNAP-style NAS hosts without systemd. - Plain mode installs start/stop/status scripts under /share/homes/mailcat/mailcat/bin and uses /opt/bin/python3 by default. - Remote install now uses explicit absolute tool path variables supplied for the NAS. - verplaats_log.json remains modified from the user's run and was intentionally not staged. --- deploy_vps.sh | 181 ++++++++++++++++++++++++++++++----- restart_prompt.md | 23 +++-- systemd/mailcat-sort.service | 4 +- 3 files changed, 170 insertions(+), 38 deletions(-) diff --git a/deploy_vps.sh b/deploy_vps.sh index f497758..aac8e9e 100755 --- a/deploy_vps.sh +++ b/deploy_vps.sh @@ -12,10 +12,41 @@ if [ -z "$REMOTE_USER" ]; then exit 2 fi APP_USER="${APP_USER:-mailcat}" -APP_DIR="${APP_DIR:-/opt/mailcat}" SERVICE_NAME="${SERVICE_NAME:-mailcat-sort-backup}" ACCOUNT="${ACCOUNT:-backup}" REMOTE_PATH="${REMOTE_PATH:-/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin}" +SERVICE_MANAGER="${SERVICE_MANAGER:-systemd}" +MAILCAT_HOME="${MAILCAT_HOME:-/share/homes/$APP_USER}" +SUDO="${SUDO:-/usr/bin/sudo}" +USERADD="${USERADD:-/usr/local/bin/useradd}" +INSTALL="${INSTALL:-/usr/bin/install}" +SED="${SED:-/bin/sed}" +TAR="${TAR:-/bin/tar}" +MKDIR="${MKDIR:-/bin/mkdir}" +CHOWN="${CHOWN:-/bin/chown}" +CHMOD="${CHMOD:-/bin/chmod}" +RM="${RM:-/bin/rm}" +SYSTEMCTL="${SYSTEMCTL:-/bin/systemctl}" +if [ -z "${APP_DIR:-}" ]; then + if [ "$SERVICE_MANAGER" = "plain" ]; then + APP_DIR="$MAILCAT_HOME/mailcat" + else + APP_DIR="/opt/mailcat" + fi +fi +if [ "$SERVICE_MANAGER" = "plain" ]; then + STATE_DIR="${STATE_DIR:-$APP_DIR/state}" + LOG_DIR="${LOG_DIR:-$APP_DIR/log}" + RUN_DIR="${RUN_DIR:-$APP_DIR/run}" + PID_FILE="${PID_FILE:-$RUN_DIR/mailcat.pid}" + PYTHON="${PYTHON:-/opt/bin/python3}" +else + STATE_DIR="${STATE_DIR:-/var/lib/mailcat}" + LOG_DIR="${LOG_DIR:-/var/log/mailcat}" + RUN_DIR="${RUN_DIR:-/run/mailcat}" + PID_FILE="${PID_FILE:-$RUN_DIR/mailcat.pid}" + PYTHON="${PYTHON:-/usr/bin/python3}" +fi ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ARCHIVE="$(mktemp -t mailcat-deploy.XXXXXX.tar.gz)" @@ -42,13 +73,13 @@ tar \ -czf "$ARCHIVE" . cat > "$REMOTE_SCRIPT" <<'REMOTE' -set -euo pipefail +set -eu export PATH="$REMOTE_PATH:$PATH" -sudo -v +"$SUDO" -v find_command() { - local name="$1" + name="$1" shift if command -v "$name" >/dev/null 2>&1; then command -v "$name" @@ -64,12 +95,12 @@ find_command() { } if ! id "$APP_USER" >/dev/null 2>&1; then - if USERADD="$(find_command useradd /usr/local/bin/useradd /usr/local/sbin/useradd /usr/sbin/useradd /sbin/useradd)"; then + if [ -x "$USERADD" ] || USERADD="$(find_command useradd /usr/local/bin/useradd /usr/local/sbin/useradd /usr/sbin/useradd /sbin/useradd)"; then NOLOGIN="$(find_command nologin /usr/local/bin/nologin /usr/local/sbin/nologin /usr/sbin/nologin /sbin/nologin || true)" if [ -n "$NOLOGIN" ]; then - sudo "$USERADD" --system --home "$APP_DIR" --shell "$NOLOGIN" "$APP_USER" + "$SUDO" "$USERADD" -d "$MAILCAT_HOME" -s "$NOLOGIN" "$APP_USER" else - sudo "$USERADD" --system --home "$APP_DIR" "$APP_USER" + "$SUDO" "$USERADD" -d "$MAILCAT_HOME" "$APP_USER" fi else APP_USER="$(id -un)" @@ -78,37 +109,135 @@ if ! id "$APP_USER" >/dev/null 2>&1; then fi APP_GROUP="$(id -gn "$APP_USER")" -sudo mkdir -p "$APP_DIR" /var/lib/mailcat /var/log/mailcat -sudo tar -xzf /tmp/mailcat-deploy.tar.gz -C "$APP_DIR" -sudo chown -R "$APP_USER:$APP_GROUP" "$APP_DIR" /var/lib/mailcat /var/log/mailcat -sudo chmod 750 "$APP_DIR" +"$SUDO" "$MKDIR" -p "$APP_DIR" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" "$APP_DIR/bin" +"$SUDO" "$TAR" -xzf /tmp/mailcat-deploy.tar.gz -C "$APP_DIR" +"$SUDO" "$CHOWN" -R "$APP_USER:$APP_GROUP" "$MAILCAT_HOME" "$APP_DIR" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" +"$SUDO" "$CHMOD" 750 "$APP_DIR" if [ ! -f "$APP_DIR/config.json" ]; then echo "Missing $APP_DIR/config.json on remote host. Create it with mode 600 before starting the service." >&2 fi -sudo chmod 600 "$APP_DIR/config.json" 2>/dev/null || true -sudo chown "$APP_USER:$APP_GROUP" "$APP_DIR/config.json" 2>/dev/null || true +"$SUDO" "$CHMOD" 600 "$APP_DIR/config.json" 2>/dev/null || true +"$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/config.json" 2>/dev/null || true -sudo install -m 0644 "$APP_DIR/systemd/mailcat-sort.service" "/etc/systemd/system/$SERVICE_NAME.service" -sudo sed -i \ - -e "s#__APP_USER__#$APP_USER#g" \ - -e "s#__APP_GROUP__#$APP_GROUP#g" \ - -e "s#__APP_DIR__#$APP_DIR#g" \ - -e "s#__ACCOUNT__#$ACCOUNT#g" \ - "/etc/systemd/system/$SERVICE_NAME.service" +if [ "$SERVICE_MANAGER" = "plain" ]; then + cat > /tmp/mailcat-start.sh </dev/null; then + echo "mailcat already running: \$old_pid" + exit 0 + fi +fi + +cd "\$APP_DIR" +"\$PYTHON" "\$APP_DIR/sort_mail_daemon.py" --account "\$ACCOUNT" --folder INBOX --state-file "\$STATE_DIR/sort_mail_daemon_state.json" --log-file "\$LOG_DIR/sort_mail_daemon.log" >> "\$LOG_DIR/daemon.out" 2>&1 & +pid="\$!" +echo "\$pid" > "\$PID_FILE" +echo "mailcat started: \$pid" +START_SCRIPT + + cat > /tmp/mailcat-stop.sh </dev/null; then + kill "\$pid" + sleep 2 + if kill -0 "\$pid" 2>/dev/null; then + echo "mailcat still running after SIGTERM: \$pid" + exit 1 + fi +fi +"\$RM" -f "\$PID_FILE" +echo "mailcat stopped" +STOP_SCRIPT + + cat > /tmp/mailcat-status.sh </dev/null; then + echo "mailcat running: \$pid" + exit 0 + fi + echo "mailcat not running: stale pid file (\$pid)" + exit 1 +fi +echo "mailcat not running" +echo "logs: \$LOG_DIR" +exit 1 +STATUS_SCRIPT + + "$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-start.sh "$APP_DIR/bin/start_mailcat.sh" + "$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-stop.sh "$APP_DIR/bin/stop_mailcat.sh" + "$SUDO" "$INSTALL" -m 0755 /tmp/mailcat-status.sh "$APP_DIR/bin/status_mailcat.sh" + "$SUDO" "$CHOWN" "$APP_USER:$APP_GROUP" "$APP_DIR/bin/start_mailcat.sh" "$APP_DIR/bin/stop_mailcat.sh" "$APP_DIR/bin/status_mailcat.sh" + "$RM" -f /tmp/mailcat-start.sh /tmp/mailcat-stop.sh /tmp/mailcat-status.sh +else + "$SUDO" "$INSTALL" -m 0644 "$APP_DIR/systemd/mailcat-sort.service" "/etc/systemd/system/$SERVICE_NAME.service" + "$SUDO" "$SED" -i \ + -e "s#__APP_USER__#$APP_USER#g" \ + -e "s#__APP_GROUP__#$APP_GROUP#g" \ + -e "s#__APP_DIR__#$APP_DIR#g" \ + -e "s#__ACCOUNT__#$ACCOUNT#g" \ + -e "s#__PYTHON__#$PYTHON#g" \ + -e "s#__STATE_DIR__#$STATE_DIR#g" \ + -e "s#__LOG_DIR__#$LOG_DIR#g" \ + "/etc/systemd/system/$SERVICE_NAME.service" + + "$SUDO" "$SYSTEMCTL" daemon-reload + "$SUDO" "$SYSTEMCTL" enable "$SERVICE_NAME.service" +fi +"$RM" -f /tmp/mailcat-deploy.tar.gz /tmp/mailcat-remote-deploy.sh echo "Deployment complete." echo "Create/check $APP_DIR/config.json, then run:" -echo " sudo systemctl start $SERVICE_NAME.service" -echo " sudo journalctl -u $SERVICE_NAME.service -f" +if [ "$SERVICE_MANAGER" = "plain" ]; then + echo " sudo -u $APP_USER $APP_DIR/bin/start_mailcat.sh" + echo " sudo -u $APP_USER $APP_DIR/bin/status_mailcat.sh" + echo " tail -f $LOG_DIR/sort_mail_daemon.log" +else + echo " sudo systemctl start $SERVICE_NAME.service" + echo " sudo journalctl -u $SERVICE_NAME.service -f" +fi REMOTE scp "$ARCHIVE" "$REMOTE_USER@$REMOTE_HOST:/tmp/mailcat-deploy.tar.gz" scp "$REMOTE_SCRIPT" "$REMOTE_USER@$REMOTE_HOST:/tmp/mailcat-remote-deploy.sh" -REMOTE_COMMAND=$(printf "APP_USER=%q APP_DIR=%q SERVICE_NAME=%q ACCOUNT=%q REMOTE_PATH=%q bash /tmp/mailcat-remote-deploy.sh" "$APP_USER" "$APP_DIR" "$SERVICE_NAME" "$ACCOUNT" "$REMOTE_PATH") +REMOTE_COMMAND=$(printf "APP_USER=%q APP_DIR=%q SERVICE_NAME=%q ACCOUNT=%q REMOTE_PATH=%q SERVICE_MANAGER=%q MAILCAT_HOME=%q STATE_DIR=%q LOG_DIR=%q RUN_DIR=%q PID_FILE=%q PYTHON=%q SUDO=%q USERADD=%q INSTALL=%q SED=%q TAR=%q MKDIR=%q CHOWN=%q CHMOD=%q RM=%q SYSTEMCTL=%q /bin/sh /tmp/mailcat-remote-deploy.sh" "$APP_USER" "$APP_DIR" "$SERVICE_NAME" "$ACCOUNT" "$REMOTE_PATH" "$SERVICE_MANAGER" "$MAILCAT_HOME" "$STATE_DIR" "$LOG_DIR" "$RUN_DIR" "$PID_FILE" "$PYTHON" "$SUDO" "$USERADD" "$INSTALL" "$SED" "$TAR" "$MKDIR" "$CHOWN" "$CHMOD" "$RM" "$SYSTEMCTL") ssh -tt "$REMOTE_USER@$REMOTE_HOST" "$REMOTE_COMMAND" diff --git a/restart_prompt.md b/restart_prompt.md index 39102d4..fa554ac 100644 --- a/restart_prompt.md +++ b/restart_prompt.md @@ -11,7 +11,7 @@ The project started as an offline analysis of a local mailbox export and is bein - Email is hosted at `australius.nl`. - A VPS is available at `vps.austalius.nl`, but the user now wants to test automation first on a local home NAS instead of the VPS. - The test mailbox is `backup@australius.nl`. -- The future automation should run on an always-on remote host, preferably via SSH deployment and an IMAP IDLE daemon. The current deploy path assumes a Linux/systemd target with sudo; if the NAS does not support that, adapt the run method to the NAS scheduler or container tooling. +- The future automation should run on an always-on remote host, preferably via SSH deployment and an IMAP IDLE daemon. The NAS is QNAP-like BusyBox Linux without systemd, so test deployment should use `SERVICE_MANAGER=plain`. - The current end-to-end test uses the `backup` account as a disposable test target. ## Repository State @@ -150,6 +150,10 @@ Remote/NAS deployment files: - The deploy script requires `REMOTE_HOST` and `REMOTE_USER`. The old `VPS_HOST` and `VPS_USER` names still work as aliases. Optional overrides: `APP_USER`, `APP_DIR`, `SERVICE_NAME`, and `ACCOUNT`. - The deploy script uploads a temporary remote shell script and runs it with `ssh -tt`, so remote `sudo` can prompt for a password on NAS systems that require a terminal. The remote script starts with `sudo -v`. - The deploy script exports an explicit remote PATH before running install commands: `/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin` by default, overrideable with `REMOTE_PATH`. It searches both PATH and common absolute paths such as `/usr/local/bin/useradd`, `/usr/sbin/useradd`, and `/sbin/useradd`. If `useradd` is genuinely unavailable, it falls back to the existing SSH user and that user's primary group for service ownership. +- For the local NAS, use `SERVICE_MANAGER=plain`, `MAILCAT_HOME=/share/homes/mailcat`, and `PYTHON=/opt/bin/python3`. In plain mode, default paths are `APP_DIR=$MAILCAT_HOME/mailcat`, `STATE_DIR=$APP_DIR/state`, `LOG_DIR=$APP_DIR/log`, `RUN_DIR=$APP_DIR/run`, and `PID_FILE=$RUN_DIR/mailcat.pid`. +- QNAP path facts supplied by the user: `SUDO=/usr/bin/sudo`, `USERADD=/usr/local/bin/useradd`, `PYTHON=/opt/bin/python3`, `INSTALL=/usr/bin/install`, `SED=/bin/sed`, `TAR=/bin/tar`, `MKDIR=/bin/mkdir`, `CHOWN=/bin/chown`, `CHMOD=/bin/chmod`, and `RM=/bin/rm`. `nologin`, `python3`, `systemctl`, and `nohup` are not in the user's default PATH. +- The deploy script accepts absolute tool overrides for those commands and passes them into the remote script. In plain mode, the generated start/stop/status scripts also bake in absolute `MKDIR` and `RM` paths. +- In plain mode, the deploy script does not install systemd files. It writes executable scripts to `$APP_DIR/bin/start_mailcat.sh`, `$APP_DIR/bin/stop_mailcat.sh`, and `$APP_DIR/bin/status_mailcat.sh`. - `systemd/mailcat-sort.service` is a template consumed by `deploy_vps.sh`; after placeholder replacement it runs `sort_mail_daemon.py --account backup --folder INBOX` and stores state/logs under `/var/lib/mailcat` and `/var/log/mailcat`. - `config.json` remains ignored and must be created manually on the remote host with mode `600` before starting the service. @@ -245,15 +249,14 @@ Latest live mailbox verification: Recommended next work: -1. Ask the user for the NAS SSH hostname/IP, SSH username, and whether the NAS has Linux/systemd with sudo. If it is a Synology/QNAP-style NAS without systemd, adapt the run method to its scheduler or container tooling before deploying. -2. Before starting the service, ensure `/opt/mailcat/config.json` on the NAS contains the `backup` account credentials and has mode `600`. -3. Deploy to a Linux/systemd NAS with a command like `REMOTE_HOST= REMOTE_USER= ./deploy_vps.sh`. -4. On the NAS, start and inspect the service: - - `sudo systemctl start mailcat-sort-backup.service` - - `sudo systemctl status mailcat-sort-backup.service` - - `sudo journalctl -u mailcat-sort-backup.service -f` -5. Send controlled test messages to `backup@australius.nl` for invoices, clear AI providers, a newsletter/service route, and unmatched mail; verify expected folder moves and daemon logs. -6. Add `List-ID` header support to mailing-list routing if live daemon tests show sender/domain matching is too coarse. +1. Deploy to the QNAP NAS with a command like `REMOTE_HOST=hades REMOTE_USER=Hel SERVICE_MANAGER=plain MAILCAT_HOME=/share/homes/mailcat ./deploy_vps.sh`. +2. Before starting the daemon, ensure `/share/homes/mailcat/mailcat/config.json` on the NAS contains the `backup` account credentials and has mode `600`. +3. On the NAS, start and inspect the daemon: + - `sudo -u mailcat /share/homes/mailcat/mailcat/bin/start_mailcat.sh` + - `sudo -u mailcat /share/homes/mailcat/mailcat/bin/status_mailcat.sh` + - `tail -f /share/homes/mailcat/mailcat/log/sort_mail_daemon.log` +4. Send controlled test messages to `backup@australius.nl` for invoices, clear AI providers, a newsletter/service route, and unmatched mail; verify expected folder moves and daemon logs. +5. Add `List-ID` header support to mailing-list routing if live daemon tests show sender/domain matching is too coarse. ## Persistent File Rule diff --git a/systemd/mailcat-sort.service b/systemd/mailcat-sort.service index 7b1bf6d..1ee3bce 100644 --- a/systemd/mailcat-sort.service +++ b/systemd/mailcat-sort.service @@ -8,14 +8,14 @@ Type=simple User=__APP_USER__ Group=__APP_GROUP__ WorkingDirectory=__APP_DIR__ -ExecStart=/usr/bin/python3 __APP_DIR__/sort_mail_daemon.py --account __ACCOUNT__ --folder INBOX --state-file /var/lib/mailcat/sort_mail_daemon_state.json --log-file /var/log/mailcat/sort_mail_daemon.log +ExecStart=__PYTHON__ __APP_DIR__/sort_mail_daemon.py --account __ACCOUNT__ --folder INBOX --state-file __STATE_DIR__/sort_mail_daemon_state.json --log-file __LOG_DIR__/sort_mail_daemon.log Restart=always RestartSec=30 NoNewPrivileges=true PrivateTmp=true ProtectSystem=full ProtectHome=true -ReadWritePaths=__APP_DIR__ /var/lib/mailcat /var/log/mailcat +ReadWritePaths=__APP_DIR__ __STATE_DIR__ __LOG_DIR__ [Install] WantedBy=multi-user.target