Add 80% limit warning, stop repeat notifications, fix review findings

Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
+12 -4
View File
@@ -2,10 +2,10 @@
# Install as /usr/local/etc/migadu_counters/config.toml (or pass -config PATH).
addr = ":5555" # API: POST /in, POST /out, GET /counters
admin_addr = ":9095" # /health and /metrics, no auth
admin_addr = "127.0.0.1:9095" # /health and /metrics, no auth: keep it private
auth_token = "…" # REQUIRED (or MIGADU_COUNTERS_AUTH_TOKEN); same value in mailqueuer's config
allowed_cidrs = [] # optional source-IP allowlist for the API, e.g. ["10.0.0.0/8"]
domains_cdb = "/usr/local/etc/rspamd/db/rspamd_domains_db"
domains_cdb = "/usr/local/etc/migadu_counters/db/rspamd_domains_db" # where contrib/pull-domains.sh installs it
exempt = ["support@migadu.com", "contact@migadu.com"] # mail to/from these never counts
count_junk = false # count /in requests flagged junk by mizu
@@ -33,7 +33,8 @@ counting_format = "text" # Lua-like lines (default) or "json"
[redis]
db = 0 # must stay 0: migadu_web reads message counters from DB 0
password = "…" # or MIGADU_COUNTERS_REDIS_PASSWORD
password = "…" # or MIGADU_COUNTERS_REDIS_PASSWORD; required with sentinels
# password_optional = true # development only: allow Sentinel without a password
pool_size = 32
timeout_ms = 500
# Sentinel mode (production): writes -> master, reads -> replicas
@@ -50,6 +51,13 @@ sentinel_password = ""
org_url = "https://admin.migadu.com/api/limits?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT"
address_url = "https://admin.migadu.com/api/limits?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT"
domain_url = "" # web app's D branch is still a TODO
# Early warning: one call when a counter reaches warn_percent of its limit
# (0 = off). Empty URLs disable a level. These must not be the /api/limits
# URLs above: the web app treats every call there as a breach.
warn_percent = 80
warn_org_url = "" # e.g. https://admin.migadu.com/api/limits/warning?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT&limit=$LIMIT&percent=$PERCENT
warn_address_url = ""
warn_domain_url = ""
user = "…"
password = "…" # or MIGADU_COUNTERS_NOTIFY_PASSWORD
timeout_seconds = 5
timeout_seconds = 15 # per attempt; a request sent but unanswered is not retried