#!/usr/bin/env bash # # pull-domains.sh — fetch the rspamd domains CDB from the agent host and # install it for migadu_counters, the way rcptd's build.sh pulls its maps. # # The agent already produces the finished CDB (extract.sh: cdb -m -> # /usr/local/etc/agent/build/rspamd/rspamd_domains_db), so this is # pull → compare → install: # # 1. rsync the file from the agent into $MAPS_DIR (ssh forced to IPv4, see # rcptd: agenthost also has an IPv6 address not reachable from every host). # 2. cmp against the installed baseline $DOMAINS_CDB.orig; identical → done. # Most runs end here: the agent only rewrites the file when its content # changed, so the 15-second cron cycle is a no-op cmp. # 3. Changed: copy to a dotfile next to the target, verify it opens and has # records (cdb -s), then mv it over $DOMAINS_CDB — atomic, so the daemon # never sees a half-written file — and keep the pulled copy as .orig. # The daemon's fsnotify watcher reloads on the mv; the explicit reload is # belt and braces. # # Never rewrite $DOMAINS_CDB in place: the daemon keeps the file open and an # in-place rewrite truncates the inode under it. This script only ever mv's. # # Environment (templated by the Ansible role; defaults below are the # production paths). Tests override RSYNC_SRC with a local path. # set -u AGENT_HOST=${AGENT_HOST:-agent@agenthost} AGENT_PATH=${AGENT_PATH:-/usr/local/etc/agent/build/rspamd/rspamd_domains_db} RSYNC_SRC=${RSYNC_SRC:-$AGENT_HOST:$AGENT_PATH} # a local path skips ssh (tests, co-located agent) RSYNC_RSH=${RSYNC_RSH:-ssh -4} MAPS_DIR=${MAPS_DIR:-/usr/local/etc/migadu_counters/maps} DOMAINS_DIR=${DOMAINS_DIR:-/usr/local/etc/migadu_counters/db} DOMAINS_CDB=${DOMAINS_CDB:-$DOMAINS_DIR/rspamd_domains_db} LOG_FILE=${LOG_FILE:-/var/log/migadu_counters/pull.log} LOCK_FILE=${LOCK_FILE:-$DOMAINS_DIR/.pull-lock} LOCK_MAX_AGE=${LOCK_MAX_AGE:-120} # seconds; older locks are stale (crashed run) RELOAD_CMD=${RELOAD_CMD:-service migadu_counters reload} NAME=$(basename "$DOMAINS_CDB") PULLED=$MAPS_DIR/$NAME TMP=$DOMAINS_DIR/.$NAME.tmp log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $*" | tee -a "$LOG_FILE"; } mtime() { stat -f %m "$1" 2>/dev/null || stat -c %Y "$1"; } cleanup() { rm -f "$TMP" "$LOCK_FILE"; } fail() { log "FAIL: $*"; cleanup; exit 1; } mkdir -p "$MAPS_DIR" "$DOMAINS_DIR" "$(dirname "$LOG_FILE")" || { echo "cannot create directories" >&2; exit 1; } # Lock: cron (@15) cannot overlap this job with itself; the lock guards a # manual or ansible-triggered run racing the scheduled one. if [ -f "$LOCK_FILE" ] && [ "$(mtime "$LOCK_FILE")" -ge $(( $(date +%s) - LOCK_MAX_AGE )) ]; then log "previous instance is still running, skipping" exit 0 fi touch "$LOCK_FILE" || { echo "cannot create $LOCK_FILE" >&2; exit 1; } trap 'rm -f "$TMP" "$LOCK_FILE"' EXIT # 1. pull. --checksum: the agent can rebuild the file with the same size # within the same second, which rsync's size+mtime quick check would miss; # checksumming one ~9 MB file every cycle is cheap. if ! rsync -a --checksum -e "$RSYNC_RSH" "$RSYNC_SRC" "$PULLED" >>"$LOG_FILE" 2>&1; then fail "rsync from $RSYNC_SRC failed" fi [ -s "$PULLED" ] || fail "pulled $PULLED is missing or empty" # 2. compare with the installed baseline if [ -f "$DOMAINS_CDB" ] && cmp -s "$PULLED" "$DOMAINS_CDB.orig"; then log "$NAME unchanged" exit 0 fi # 3. verify, then install atomically rm -f "$TMP" cp "$PULLED" "$TMP" || fail "copy to $TMP failed" records=$(cdb -s "$TMP" 2>/dev/null | awk '/number of records/ { print $4 }') if [ -z "$records" ]; then fail "$PULLED is not a valid CDB (cdb -s failed), keeping the installed file" fi if [ "$records" -eq 0 ]; then fail "$PULLED holds no records, keeping the installed file" fi chmod 0644 "$TMP" mv "$TMP" "$DOMAINS_CDB" || fail "mv into place failed" cp "$PULLED" "$DOMAINS_CDB.orig" || log "warning: could not update $DOMAINS_CDB.orig (next run will reinstall)" log "$NAME installed ($records records)" # The fsnotify watcher already picked up the mv; make sure anyway. $RELOAD_CMD >>"$LOG_FILE" 2>&1 || true exit 0