Add 80% limit warning, stop repeat notifications, fix review findings
Notifications: - Warn once when a counter reaches warn_percent of its limit, through separate warn_*_url endpoints. - Write ntf/ markers so a lost Redis reply cannot re-send a breach. - Retry only when the request never left or the web app answered 429/5xx. Fixes: - Retry transactions Redis rejected; treat pool timeouts as definite. - Install the signal handler before startup work. - Stop logging the login suffix; redact notify URL credentials. - Reject corrupt or empty CDBs, truncated hash tables included. - Require a Sentinel password, refuse example placeholders, and default admin_addr to localhost. - Normalise /counters domains, count on the arrival day, log rejections, hide internals in errors and exit 1 on a failed drain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c43ff17627
commit
54f9626121
+52
-11
@@ -17,6 +17,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
@@ -102,8 +103,12 @@ func (l *Logger) Outcome(out *counting.Outcome, dispatched []bool) {
|
||||
b.tracked(r, out.Unknown != nil)
|
||||
}
|
||||
for i, r := range out.Results {
|
||||
sent := i < len(dispatched) && dispatched[i]
|
||||
if r.Reached {
|
||||
b.reached(r, i < len(dispatched) && dispatched[i])
|
||||
b.reached(r, sent)
|
||||
}
|
||||
if r.Warned {
|
||||
b.warned(r, sent)
|
||||
}
|
||||
}
|
||||
if out.Skipped != "" {
|
||||
@@ -128,6 +133,13 @@ func (l *Logger) Dropped(route, traceID string) {
|
||||
b.flush()
|
||||
}
|
||||
|
||||
// Rejected logs a request refused by the validation layer.
|
||||
func (l *Logger) Rejected(route, traceID string, status int, reason string) {
|
||||
b := l.begin(traceID, route)
|
||||
b.line("rejected", fmt.Sprintf("rejected: %d %s (not counted)", status, reason), map[string]any{"status": status, "reason": reason})
|
||||
b.flush()
|
||||
}
|
||||
|
||||
// Error logs a request that failed before a plan existed (domain lookup error).
|
||||
func (l *Logger) Error(route, traceID string, err error) {
|
||||
b := l.begin(traceID, route)
|
||||
@@ -136,28 +148,39 @@ func (l *Logger) Error(route, traceID string, err error) {
|
||||
}
|
||||
|
||||
// NotificationSent logs a delivered web app notification (notify hook).
|
||||
func (l *Logger) NotificationSent(ev notify.Event, url string, attempt int) {
|
||||
func (l *Logger) NotificationSent(ev notify.Event, rawURL string, attempt int) {
|
||||
u := redact(rawURL)
|
||||
b := l.begin(ev.TraceID, ev.Route)
|
||||
b.line("notification_sent", fmt.Sprintf("sent limit exceeded notification to url %q", url),
|
||||
map[string]any{"url": url, "attempt": attempt, "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
|
||||
b.line("notification_sent", fmt.Sprintf("sent %s notification to url %q", kindWords(ev), u),
|
||||
map[string]any{"url": u, "attempt": attempt, "kind": ev.KindName(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
|
||||
b.flush()
|
||||
}
|
||||
|
||||
// NotificationFailed logs a notification given up on (notify hook).
|
||||
func (l *Logger) NotificationFailed(ev notify.Event, url string, attempts int, err error) {
|
||||
func (l *Logger) NotificationFailed(ev notify.Event, rawURL string, attempts int, err error) {
|
||||
u := redact(rawURL)
|
||||
b := l.begin(ev.TraceID, ev.Route)
|
||||
b.line("notification_failed", fmt.Sprintf("failed limit exceeded notification to url %q after %d attempts: %v", url, attempts, err),
|
||||
map[string]any{"url": url, "attempts": attempts, "error": err.Error(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
|
||||
b.line("notification_failed", fmt.Sprintf("failed %s notification to url %q after %d attempts: %v", kindWords(ev), u, attempts, err),
|
||||
map[string]any{"url": u, "attempts": attempts, "error": err.Error(), "kind": ev.KindName(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
|
||||
b.flush()
|
||||
}
|
||||
|
||||
// redact hides a password embedded in a notification URL.
|
||||
func redact(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return "<unparseable url>"
|
||||
}
|
||||
return u.Redacted()
|
||||
}
|
||||
|
||||
func (b *block) request(p *counting.Plan) {
|
||||
if p.Route == counting.RouteOut {
|
||||
msg := fmt.Sprintf("request auth_user=%s recipient=%s mail_from=%s", p.AuthUser, p.Recipient, p.MailFrom)
|
||||
fields := map[string]any{"auth_user": p.AuthUser, "recipient": p.Recipient, "mail_from": p.MailFrom}
|
||||
if p.LoginSuffix != "" {
|
||||
msg += " login_suffix=" + p.LoginSuffix
|
||||
fields["login_suffix"] = p.LoginSuffix
|
||||
if p.TokenLogin {
|
||||
msg += " token_login=true"
|
||||
fields["token_login"] = true
|
||||
}
|
||||
b.line("request", msg, fields)
|
||||
} else {
|
||||
@@ -235,7 +258,7 @@ func (b *block) tracked(r counting.Result, reread bool) {
|
||||
}
|
||||
|
||||
func (b *block) reached(r counting.Result, dispatched bool) {
|
||||
name := notify.URLName(r.Level)
|
||||
name := notify.URLName(notify.KindLimit, r.Level)
|
||||
verdict := "notifying " + name
|
||||
if !dispatched {
|
||||
verdict = "no " + name + " configured, not notifying"
|
||||
@@ -245,6 +268,24 @@ func (b *block) reached(r counting.Result, dispatched bool) {
|
||||
map[string]any{"level": r.Level, "id": r.ID, "flow": r.Flow, "by": r.By, "count": r.Count, "limit": r.Limit, "notifying": dispatched, "url_name": name})
|
||||
}
|
||||
|
||||
func (b *block) warned(r counting.Result, dispatched bool) {
|
||||
name := notify.URLName(notify.KindWarning, r.Level)
|
||||
verdict := "notifying " + name
|
||||
if !dispatched {
|
||||
verdict = "no " + name + " configured, not notifying"
|
||||
}
|
||||
b.line("warning_reached",
|
||||
fmt.Sprintf("warning reached (level=[%s] id=[%s] flow=[%s] by=[%s]) => %s", r.Level, r.ID, r.Flow, r.By, verdict),
|
||||
map[string]any{"level": r.Level, "id": r.ID, "flow": r.Flow, "by": r.By, "count": r.Count, "warn_at": r.WarnAt, "limit": r.Limit, "notifying": dispatched, "url_name": name})
|
||||
}
|
||||
|
||||
func kindWords(ev notify.Event) string {
|
||||
if ev.KindName() == notify.KindWarning {
|
||||
return "limit warning"
|
||||
}
|
||||
return "limit exceeded"
|
||||
}
|
||||
|
||||
// summarize picks the limit keys worth printing for one address: the base
|
||||
// keys and the address's own overrides. Some domains carry thousands of
|
||||
// per-mailbox overrides (boxmailhere.com: 9 798); dumping them all would make
|
||||
|
||||
Reference in New Issue
Block a user