Add 80% limit warning, stop repeat notifications, fix review findings

Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
+52 -11
View File
@@ -17,6 +17,7 @@ import (
"errors"
"fmt"
"io"
"net/url"
"sort"
"strings"
"time"
@@ -102,8 +103,12 @@ func (l *Logger) Outcome(out *counting.Outcome, dispatched []bool) {
b.tracked(r, out.Unknown != nil)
}
for i, r := range out.Results {
sent := i < len(dispatched) && dispatched[i]
if r.Reached {
b.reached(r, i < len(dispatched) && dispatched[i])
b.reached(r, sent)
}
if r.Warned {
b.warned(r, sent)
}
}
if out.Skipped != "" {
@@ -128,6 +133,13 @@ func (l *Logger) Dropped(route, traceID string) {
b.flush()
}
// Rejected logs a request refused by the validation layer.
func (l *Logger) Rejected(route, traceID string, status int, reason string) {
b := l.begin(traceID, route)
b.line("rejected", fmt.Sprintf("rejected: %d %s (not counted)", status, reason), map[string]any{"status": status, "reason": reason})
b.flush()
}
// Error logs a request that failed before a plan existed (domain lookup error).
func (l *Logger) Error(route, traceID string, err error) {
b := l.begin(traceID, route)
@@ -136,28 +148,39 @@ func (l *Logger) Error(route, traceID string, err error) {
}
// NotificationSent logs a delivered web app notification (notify hook).
func (l *Logger) NotificationSent(ev notify.Event, url string, attempt int) {
func (l *Logger) NotificationSent(ev notify.Event, rawURL string, attempt int) {
u := redact(rawURL)
b := l.begin(ev.TraceID, ev.Route)
b.line("notification_sent", fmt.Sprintf("sent limit exceeded notification to url %q", url),
map[string]any{"url": url, "attempt": attempt, "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.line("notification_sent", fmt.Sprintf("sent %s notification to url %q", kindWords(ev), u),
map[string]any{"url": u, "attempt": attempt, "kind": ev.KindName(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.flush()
}
// NotificationFailed logs a notification given up on (notify hook).
func (l *Logger) NotificationFailed(ev notify.Event, url string, attempts int, err error) {
func (l *Logger) NotificationFailed(ev notify.Event, rawURL string, attempts int, err error) {
u := redact(rawURL)
b := l.begin(ev.TraceID, ev.Route)
b.line("notification_failed", fmt.Sprintf("failed limit exceeded notification to url %q after %d attempts: %v", url, attempts, err),
map[string]any{"url": url, "attempts": attempts, "error": err.Error(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.line("notification_failed", fmt.Sprintf("failed %s notification to url %q after %d attempts: %v", kindWords(ev), u, attempts, err),
map[string]any{"url": u, "attempts": attempts, "error": err.Error(), "kind": ev.KindName(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.flush()
}
// redact hides a password embedded in a notification URL.
func redact(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return "<unparseable url>"
}
return u.Redacted()
}
func (b *block) request(p *counting.Plan) {
if p.Route == counting.RouteOut {
msg := fmt.Sprintf("request auth_user=%s recipient=%s mail_from=%s", p.AuthUser, p.Recipient, p.MailFrom)
fields := map[string]any{"auth_user": p.AuthUser, "recipient": p.Recipient, "mail_from": p.MailFrom}
if p.LoginSuffix != "" {
msg += " login_suffix=" + p.LoginSuffix
fields["login_suffix"] = p.LoginSuffix
if p.TokenLogin {
msg += " token_login=true"
fields["token_login"] = true
}
b.line("request", msg, fields)
} else {
@@ -235,7 +258,7 @@ func (b *block) tracked(r counting.Result, reread bool) {
}
func (b *block) reached(r counting.Result, dispatched bool) {
name := notify.URLName(r.Level)
name := notify.URLName(notify.KindLimit, r.Level)
verdict := "notifying " + name
if !dispatched {
verdict = "no " + name + " configured, not notifying"
@@ -245,6 +268,24 @@ func (b *block) reached(r counting.Result, dispatched bool) {
map[string]any{"level": r.Level, "id": r.ID, "flow": r.Flow, "by": r.By, "count": r.Count, "limit": r.Limit, "notifying": dispatched, "url_name": name})
}
func (b *block) warned(r counting.Result, dispatched bool) {
name := notify.URLName(notify.KindWarning, r.Level)
verdict := "notifying " + name
if !dispatched {
verdict = "no " + name + " configured, not notifying"
}
b.line("warning_reached",
fmt.Sprintf("warning reached (level=[%s] id=[%s] flow=[%s] by=[%s]) => %s", r.Level, r.ID, r.Flow, r.By, verdict),
map[string]any{"level": r.Level, "id": r.ID, "flow": r.Flow, "by": r.By, "count": r.Count, "warn_at": r.WarnAt, "limit": r.Limit, "notifying": dispatched, "url_name": name})
}
func kindWords(ev notify.Event) string {
if ev.KindName() == notify.KindWarning {
return "limit warning"
}
return "limit exceeded"
}
// summarize picks the limit keys worth printing for one address: the base
// keys and the address's own overrides. Some domains carry thousands of
// per-mailbox overrides (boxmailhere.com: 9 798); dumping them all would make