Add 80% limit warning, stop repeat notifications, fix review findings

Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/server/middleware.go

@@ -48,6 +48,17 @@ func (s *Server) reject(w http.ResponseWriter, r *http.Request, route string, e
 	s.d.Logger.Warn("request rejected", "route", route, "remote", r.RemoteAddr, "status", e.status, "reason", e.msg)
 }
 
+// rejectBody is reject for a request whose body was read: the trace id, when
+// there is one, ties the rejection to the message, in both logs.
+func (s *Server) rejectBody(w http.ResponseWriter, r *http.Request, route, traceID string, e *httpError) {
+	writeError(w, e.status, e.msg)
+	id := api.SanitizeTraceID(traceID)
+	s.d.Logger.Warn("request rejected", "route", route, "trace_id", id, "remote", r.RemoteAddr, "status", e.status, "reason", e.msg)
+	if s.d.TxLog != nil {
+		s.d.TxLog.Rejected(strings.TrimPrefix(route, "/"), id, e.status, api.SanitizeForLog(e.msg))
+	}
+}
+
 // statusWriter captures the status code for metrics.
 type statusWriter struct {
 	http.ResponseWriter

internal/server/notifications_test.go

@@ -0,0 +1,173 @@
+package server
+
+import (
+	"context"
+	"net/url"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+
+	"migadu_counters/internal/store"
+)
+
+// Real CDB: yagolivas.com has A/o=20 and O/o=20, so 80% is 16.
+func TestOut_WarnsAt80PercentOnceAndBreachesOnce(t *testing.T) {
+	h := newHarness(t, options{warnPercent: 80})
+	for i := 0; i < 22; i++ {
+		h.accept("/out", `{"trace_id":"w","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
+	}
+	if err := h.notifier.Shutdown(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if h.warnAddr.count() != 1 || h.warnOrg.count() != 1 {
+		t.Errorf("warnings addr=%d org=%d, want 1 each", h.warnAddr.count(), h.warnOrg.count())
+	}
+	if h.addrURL.count() != 1 || h.orgURL.count() != 1 {
+		t.Errorf("breaches addr=%d org=%d, want 1 each", h.addrURL.count(), h.orgURL.count())
+	}
+	u, err := url.Parse(h.warnAddr.urls[0])
+	if err != nil {
+		t.Fatal(err)
+	}
+	q := u.Query()
+	if u.Path != "/api/limits/warning" || q.Get("count") != "16" || q.Get("limit") != "20" || q.Get("percent") != "80" || q.Get("id") != "a@yagolivas.com" {
+		t.Errorf("warning url = %s", h.warnAddr.urls[0])
+	}
+	if h.metric(h.m.WarningReached.WithLabelValues("A", "o")) != 1 {
+		t.Error("warning_reached_total not recorded")
+	}
+	if h.metric(h.m.Notifications.WithLabelValues("warning", "A", "sent")) != 1 {
+		t.Error("notifications_total{warning,A,sent} not recorded")
+	}
+	if !strings.Contains(h.tx.String(), "warning reached (level=[A] id=[a@yagolivas.com] flow=[o] by=[a@yagolivas.com]) => notifying warn_address_url") {
+		t.Errorf("transaction log:\n%s", h.tx.String())
+	}
+}
+
+func TestOut_ConcurrentIncrementsNotifyOnce(t *testing.T) {
+	h := newHarness(t, options{warnPercent: 80, workers: 8})
+	var wg sync.WaitGroup
+	for i := 0; i < 40; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			h.post("/out", `{"trace_id":"c","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
+		}()
+	}
+	wg.Wait()
+	h.settle()
+	if err := h.notifier.Shutdown(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if h.redisInt(store.Key("A", "a@yagolivas.com", day, "o")) != 40 {
+		t.Fatalf("counter = %d, want 40", h.redisInt(store.Key("A", "a@yagolivas.com", day, "o")))
+	}
+	if h.warnAddr.count() != 1 || h.addrURL.count() != 1 {
+		t.Errorf("warnings=%d breaches=%d, want 1 each", h.warnAddr.count(), h.addrURL.count())
+	}
+}
+
+func TestOut_WarningsOffSendsNone(t *testing.T) {
+	h := newHarness(t, options{})
+	for i := 0; i < 20; i++ {
+		h.accept("/out", `{"trace_id":"w","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
+	}
+	h.notifier.Shutdown(context.Background())
+	if h.warnAddr.count() != 0 || h.addrURL.count() != 1 {
+		t.Errorf("warnings=%d breaches=%d, want 0 and 1", h.warnAddr.count(), h.addrURL.count())
+	}
+	if strings.Contains(h.tx.String(), "warning reached") {
+		t.Errorf("transaction log mentions a warning:\n%s", h.tx.String())
+	}
+}
+
+// Once a breach was notified, a lost EXEC reply on a later message sends nothing.
+func TestLostReplyAfterBreachSendsNothing(t *testing.T) {
+	backend := miniredis.RunT(t)
+	proxy := delayProxy(t, backend.Addr(), 3*time.Second)
+	h := newHarness(t, options{writeAddr: proxy})
+	key := store.Key("A", "a@yagolivas.com", day, "o") // A/o=20
+	backend.Set(key, "40")
+	backend.Set(store.MarkerKey("A", "a@yagolivas.com", day, "o", store.MarkerLimit), "1")
+	h.accept("/out", `{"trace_id":"lost2","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
+	if err := h.notifier.Shutdown(context.Background()); err != nil {
+		t.Fatal(err)
+	}
+	if h.addrURL.count() != 0 {
+		t.Errorf("address notifications = %d, want 0", h.addrURL.count())
+	}
+	if !strings.Contains(h.tx.String(), "count=41 limit=20 reached=false (re-read)") {
+		t.Errorf("transaction log:\n%s", h.tx.String())
+	}
+}
+
+// --- review round 2 ------------------------------------------------------------
+
+func TestOut_LoginSuffixNeverLogged(t *testing.T) {
+	h := newHarness(t, options{})
+	h.accept("/out", `{"trace_id":"m2","auth_user":"x@yagolivas.com@SECRET\\nFORGED LINE","recipient":"y@yagolivas.com"}`)
+	h.accept("/out", `{"trace_id":"m3","auth_user":"x@yagolivas.com@SECRET\nFORGED LINE","recipient":"y@yagolivas.com"}`)
+	if tx := h.tx.String(); strings.Contains(tx, "SECRET") || strings.Contains(tx, "FORGED") {
+		t.Errorf("transaction log contains the login suffix:\n%s", tx)
+	}
+}
+
+func TestReject_LeavesTraceInBothLogs(t *testing.T) {
+	h := newHarness(t, options{})
+	if st, _ := h.post("/in", `{"trace_id":"TRACE-XYZ","recipient":"nobody"}`); st != 400 {
+		t.Fatal(st)
+	}
+	if !strings.Contains(h.log.String(), "TRACE-XYZ") {
+		t.Errorf("app log lacks the trace id:\n%s", h.log.String())
+	}
+	if !strings.Contains(h.tx.String(), "<TRACE-XYZ> in; rejected: 400 ") {
+		t.Errorf("counting log lacks the rejection:\n%s", h.tx.String())
+	}
+}
+
+func TestErrors_DoNotExposeInternals(t *testing.T) {
+	h := newHarness(t, options{})
+	readAddr, writeAddr := h.read.Addr(), h.write.Addr()
+	h.read.Close()
+	st, body := h.get("/counters?address=a@yagolivas.com")
+	if st != 503 || strings.Contains(body, readAddr) || strings.Contains(body, "dial") {
+		t.Errorf("/counters = %d %s", st, body)
+	}
+	if !strings.Contains(h.log.String(), readAddr) {
+		t.Errorf("detail should stay in the app log:\n%s", h.log.String())
+	}
+	h.write.Close()
+	st, body = h.adminGet("/health")
+	if st != 503 || strings.Contains(body, writeAddr) || strings.Contains(body, "dial") {
+		t.Errorf("/health = %d %s", st, body)
+	}
+}
+
+func TestPanic_CountedInRequestMetrics(t *testing.T) {
+	h := newHarness(t, options{})
+	h.srv.panicHook = func() { panic("boom") }
+	if st, _ := h.post("/in", `{"recipient":"b@yagolivas.com"}`); st != 500 {
+		t.Fatal(st)
+	}
+	if got := h.metric(h.m.Requests.WithLabelValues("/in", "500")); got != 1 {
+		t.Errorf("requests_total{/in,500} = %v, want 1", got)
+	}
+}
+
+// Real CDB: xn-- domains are hosted under their punycode key.
+func TestCounters_UnicodeDomainIsNormalised(t *testing.T) {
+	h := newHarness(t, options{})
+	if st, _ := h.get("/counters?domain=host_name.example.com"); st != 400 {
+		t.Errorf("invalid domain status = %d, want 400", st)
+	}
+	st, body := h.get("/counters?domain=" + url.QueryEscape("Bücher.example."))
+	if st != 200 {
+		t.Fatalf("status %d: %s", st, body)
+	}
+	if r := decodeCounters(t, body); r.Query != "xn--bcher-kva.example" {
+		t.Errorf("query = %q, want the punycode form", r.Query)
+	}
+}

internal/server/server.go

@@ -92,7 +92,7 @@ func NotifyHooks(tx *txlog.Logger, m *metrics.Metrics) notify.Hooks {
 	return notify.Hooks{
 		Sent: func(ev notify.Event, url string, attempt int) {
 			if m != nil {
-				m.RecordNotification(ev.Level, "sent")
+				m.RecordNotification(ev.KindName(), ev.Level, "sent")
 			}
 			if tx != nil {
 				tx.NotificationSent(ev, url, attempt)
@@ -100,7 +100,7 @@ func NotifyHooks(tx *txlog.Logger, m *metrics.Metrics) notify.Hooks {
 		},
 		Failed: func(ev notify.Event, url string, attempts int, err error) {
 			if m != nil {
-				m.RecordNotification(ev.Level, "failed")
+				m.RecordNotification(ev.KindName(), ev.Level, "failed")
 			}
 			if tx != nil {
 				tx.NotificationFailed(ev, url, attempts, err)
@@ -109,7 +109,7 @@ func NotifyHooks(tx *txlog.Logger, m *metrics.Metrics) notify.Hooks {
 	}
 }
 
-// Handler returns the API handler: recover → metrics → timeout → routes.
+// Handler returns the API handler: metrics → recover → timeout → routes.
 func (s *Server) Handler() http.Handler {
 	mux := http.NewServeMux()
 	mux.Handle("/out", s.route("/out", http.MethodPost, s.handleOut))
@@ -119,7 +119,7 @@ func (s *Server) Handler() http.Handler {
 		writeError(w, http.StatusNotFound, "not found")
 	})
 	timed := http.TimeoutHandler(mux, s.d.HandlerTimeout, `{"error":"request timeout"}`)
-	return s.recoverer(s.observe(timed))
+	return s.observe(s.recoverer(timed))
 }
 
 // AdminHandler returns /health and /metrics (no auth; bind admin_addr locally).
@@ -135,17 +135,17 @@ func (s *Server) AdminHandler() http.Handler {
 func (s *Server) handleOut(w http.ResponseWriter, r *http.Request) {
 	var req api.OutRequest
 	if e := decodeJSON(r, &req); e != nil {
-		s.reject(w, r, "/out", e)
+		s.rejectBody(w, r, "/out", req.TraceID, e)
 		return
 	}
 	v, err := api.ValidateOut(req)
 	if err != nil {
-		s.reject(w, r, "/out", &httpError{http.StatusBadRequest, err.Error()})
+		s.rejectBody(w, r, "/out", req.TraceID, &httpError{http.StatusBadRequest, err.Error()})
 		return
 	}
 	job := counting.Job{
 		Route: counting.RouteOut, TraceID: v.TraceID, Received: s.d.Now(),
-		AuthUser: v.AuthUser.Value, LoginSuffix: v.LoginSuffix,
+		AuthUser: v.AuthUser.Value, TokenLogin: v.TokenLogin,
 		MailFrom: v.MailFrom, MailFromUsable: v.MailFromUsable,
 		Recipient: v.Recipient.Value,
 	}
@@ -161,12 +161,12 @@ func (s *Server) handleOut(w http.ResponseWriter, r *http.Request) {
 func (s *Server) handleIn(w http.ResponseWriter, r *http.Request) {
 	var req api.InRequest
 	if e := decodeJSON(r, &req); e != nil {
-		s.reject(w, r, "/in", e)
+		s.rejectBody(w, r, "/in", req.TraceID, e)
 		return
 	}
 	v, err := api.ValidateIn(req)
 	if err != nil {
-		s.reject(w, r, "/in", &httpError{http.StatusBadRequest, err.Error()})
+		s.rejectBody(w, r, "/in", req.TraceID, &httpError{http.StatusBadRequest, err.Error()})
 		return
 	}
 	s.enqueue(w, r, "/in", counting.Job{
@@ -242,16 +242,23 @@ func (s *Server) process(ctx context.Context, job counting.Job) {
 		if s.d.Metrics != nil {
 			s.d.Metrics.RecordIncrement(res.Level, res.Flow)
 		}
-		if res.Reached {
+		if res.MarkerErr != nil {
+			if s.d.Metrics != nil {
+				s.d.Metrics.RecordRedisError("marker")
+			}
+			s.d.Logger.Error("notification marker failed", "route", plan.Route, "trace_id", plan.TraceID, "key", res.Key, "error", res.MarkerErr)
+		}
+		switch {
+		case res.Reached:
 			if s.d.Metrics != nil {
 				s.d.Metrics.RecordLimitReached(res.Level, res.Flow)
 			}
-			ev := notify.Event{TraceID: plan.TraceID, Route: plan.Route, Level: res.Level, ID: res.ID,
-				Flow: res.Flow, Address: res.By, Count: res.Count, Limit: res.Limit}
-			dispatched[i] = s.d.Notifier != nil && s.d.Notifier.Notify(ev)
-			if !dispatched[i] && s.d.Metrics != nil {
-				s.d.Metrics.RecordNotification(res.Level, "disabled")
+			dispatched[i] = s.notify(plan, res, notify.KindLimit)
+		case res.Warned:
+			if s.d.Metrics != nil {
+				s.d.Metrics.RecordWarningReached(res.Level, res.Flow)
 			}
+			dispatched[i] = s.notify(plan, res, notify.KindWarning)
 		}
 	}
 	if s.d.TxLog != nil {
@@ -259,11 +266,22 @@ func (s *Server) process(ctx context.Context, job counting.Job) {
 	}
 }
 
+// notify dispatches one notification and reports whether it was handed over.
+func (s *Server) notify(plan *counting.Plan, res counting.Result, kind string) bool {
+	ev := notify.Event{Kind: kind, TraceID: plan.TraceID, Route: plan.Route, Level: res.Level, ID: res.ID,
+		Flow: res.Flow, Address: res.By, Count: res.Count, Limit: res.Limit, Percent: s.d.Counter.Planner.WarnPercent}
+	sent := s.d.Notifier != nil && s.d.Notifier.Notify(ev)
+	if !sent && s.d.Metrics != nil {
+		s.d.Metrics.RecordNotification(kind, res.Level, "disabled")
+	}
+	return sent
+}
+
 // handleCounters answers GET /counters?address=|domain=[&date=] from the
 // read side.
 func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
 	q := r.URL.Query()
-	address, domain := strings.TrimSpace(q.Get("address")), strings.ToLower(strings.TrimSpace(q.Get("domain")))
+	address, domain := strings.TrimSpace(q.Get("address")), strings.TrimSpace(q.Get("domain"))
 	if (address == "") == (domain == "") {
 		s.reject(w, r, "/counters", &httpError{http.StatusBadRequest, "exactly one of address or domain is required"})
 		return
@@ -287,6 +305,12 @@ func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
 		at := strings.LastIndex(addr, "@")
 		local, domain = addr[:at], addr[at+1:]
 	} else {
+		dom, err := api.NormalizeDomain(domain)
+		if err != nil {
+			s.reject(w, r, "/counters", &httpError{http.StatusBadRequest, "domain: invalid domain"})
+			return
+		}
+		domain = dom
 		resp.Query, resp.Mode = domain, "domain"
 	}
 
@@ -299,7 +323,8 @@ func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
 	if errors.Is(err, domains.ErrMalformed) {
 		s.d.Logger.Warn("malformed row in domains db, domain treated as not hosted", "domain", domain, "route", "/counters", "error", err)
 	} else if err != nil && !errors.Is(err, domains.ErrNotFound) {
-		writeError(w, http.StatusServiceUnavailable, "domain lookup failed: "+err.Error())
+		s.d.Logger.Error("domain lookup failed", "domain", domain, "route", "/counters", "error", err)
+		writeError(w, http.StatusServiceUnavailable, "domain lookup failed")
 		return
 	}
 	managed := err == nil
@@ -326,7 +351,8 @@ func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
 		if s.d.Metrics != nil {
 			s.d.Metrics.RecordRedisError("mget")
 		}
-		writeError(w, http.StatusServiceUnavailable, "redis unavailable: "+err.Error())
+		s.d.Logger.Error("redis read failed", "route", "/counters", "error", err)
+		writeError(w, http.StatusServiceUnavailable, "redis unavailable")
 		return
 	}
 	limit := func(level, flow string) *int64 {
@@ -355,7 +381,8 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if err := db.Ping(); err != nil {
-		writeError(w, http.StatusServiceUnavailable, "domains db not readable: "+err.Error())
+		s.d.Logger.Warn("health: domains db not readable", "error", err)
+		writeError(w, http.StatusServiceUnavailable, "domains db not readable")
 		return
 	}
 	ctx, cancel := context.WithTimeout(r.Context(), 200*time.Millisecond)
@@ -364,7 +391,8 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
 		if s.d.Metrics != nil {
 			s.d.Metrics.RecordRedisError("ping")
 		}
-		writeError(w, http.StatusServiceUnavailable, fmt.Sprintf("redis: %v", err))
+		s.d.Logger.Warn("health: redis not reachable", "error", err)
+		writeError(w, http.StatusServiceUnavailable, "redis unavailable")
 		return
 	}
 	writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})

internal/server/server_test.go

@@ -87,6 +87,7 @@ type options struct {
 	exempt         map[string]bool
 	queueSize      int
 	workers        int
+	warnPercent    int64 // > 0: warnings on, sent to warnAddr / warnOrg
 }
 
 type harness struct {
@@ -102,6 +103,8 @@ type harness struct {
 	addrURL  *capture
 	orgURL   *capture
 	domURL   *capture
+	warnAddr *capture
+	warnOrg  *capture
 	log      *syncBuffer
 	tx       *syncBuffer
 	reg      *prometheus.Registry
@@ -154,6 +157,14 @@ func newHarness(t *testing.T, o options) *harness {
 	if o.domainURL {
 		ncfg.DomainURL = sd.URL + q
 	}
+	h.warnAddr, h.warnOrg = &capture{}, &capture{}
+	if o.warnPercent > 0 {
+		swa, swo := httptest.NewServer(h.warnAddr), httptest.NewServer(h.warnOrg)
+		t.Cleanup(swa.Close)
+		t.Cleanup(swo.Close)
+		wq := "/api/limits/warning?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT&limit=$LIMIT&percent=$PERCENT"
+		ncfg.WarnAddressURL, ncfg.WarnOrgURL = swa.URL+wq, swo.URL+wq
+	}
 
 	h.reg = prometheus.NewRegistry()
 	h.m = metrics.New(h.reg)
@@ -164,7 +175,7 @@ func newHarness(t *testing.T, o options) *harness {
 	logger := slog.New(slog.NewTextHandler(h.log, &slog.HandlerOptions{Level: slog.LevelDebug}))
 	planner := &counting.Planner{
 		Lookup: counting.LookupFunc(func(d string) (*domains.Domain, error) { return (*h.lookup.Load())(d) }),
-		Exempt: o.exempt, Now: func() time.Time { return clock },
+		Exempt: o.exempt, Now: func() time.Time { return clock }, WarnPercent: o.warnPercent,
 	}
 	s := New(Deps{
 		AuthToken:      o.token,
@@ -349,8 +360,8 @@ func TestOut_ReachedNotifiesConfiguredLevelsOnce(t *testing.T) {
 			t.Errorf("transaction log lacks %q:\n%s", want, tx)
 		}
 	}
-	if h.metric(h.m.LimitReached.WithLabelValues("A", "o")) != 1 || h.metric(h.m.Notifications.WithLabelValues("A", "sent")) != 1 ||
-		h.metric(h.m.Notifications.WithLabelValues("D", "disabled")) != 1 {
+	if h.metric(h.m.LimitReached.WithLabelValues("A", "o")) != 1 || h.metric(h.m.Notifications.WithLabelValues("limit", "A", "sent")) != 1 ||
+		h.metric(h.m.Notifications.WithLabelValues("limit", "D", "disabled")) != 1 {
 		t.Error("limit/notification metrics not recorded")
 	}
 }
@@ -569,7 +580,7 @@ func TestOut_LoginSuffixCountsBaseAddress(t *testing.T) {
 	if h.redisInt(store.Key("A", "x@yagolivas.com", day, "o")) != 1 || h.redisInt(store.Key("A", "y@yagolivas.com", day, "i")) != 1 {
 		t.Errorf("base address not counted: %v", h.write.Keys())
 	}
-	if !strings.Contains(h.tx.String(), "<m1> out; request auth_user=x@yagolivas.com recipient=y@yagolivas.com mail_from=x@yagolivas.com login_suffix=SUPPORTTOKEN") {
+	if !strings.Contains(h.tx.String(), "<m1> out; request auth_user=x@yagolivas.com recipient=y@yagolivas.com mail_from=x@yagolivas.com token_login=true") {
 		t.Errorf("transaction log:\n%s", h.tx.String())
 	}
 }
@@ -1046,7 +1057,7 @@ func TestNotifyHooks(t *testing.T) {
 	ev := notify.Event{TraceID: "t", Route: "out", Level: "O"}
 	hooks.Sent(ev, "http://x", 1)
 	hooks.Failed(ev, "http://x", 5, io.EOF)
-	if testutil.ToFloat64(m.Notifications.WithLabelValues("O", "sent")) != 1 || testutil.ToFloat64(m.Notifications.WithLabelValues("O", "failed")) != 1 {
+	if testutil.ToFloat64(m.Notifications.WithLabelValues("limit", "O", "sent")) != 1 || testutil.ToFloat64(m.Notifications.WithLabelValues("limit", "O", "failed")) != 1 {
 		t.Error("hooks did not record metrics")
 	}
 	if s := buf.String(); !strings.Contains(s, "sent limit exceeded") || !strings.Contains(s, "failed limit exceeded") {