Add 80% limit warning, stop repeat notifications, fix review findings

Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/notify/notify_test.go

@@ -36,10 +36,15 @@ func TestBuildURL(t *testing.T) {
 }
 
 func TestURLFor(t *testing.T) {
-	cfg := Config{OrgURL: "o", AddressURL: "a", DomainURL: "d"}
+	cfg := Config{OrgURL: "o", AddressURL: "a", DomainURL: "d", WarnOrgURL: "wo", WarnAddressURL: "wa", WarnDomainURL: "wd"}
 	for level, want := range map[string]string{"O": "o", "A": "a", "D": "d", "X": ""} {
-		if got := cfg.URLFor(level); got != want {
-			t.Errorf("URLFor(%s) = %q, want %q", level, got, want)
+		if got := cfg.URLFor(KindLimit, level); got != want {
+			t.Errorf("URLFor(limit, %s) = %q, want %q", level, got, want)
+		}
+	}
+	for level, want := range map[string]string{"O": "wo", "A": "wa", "D": "wd", "X": ""} {
+		if got := cfg.URLFor(KindWarning, level); got != want {
+			t.Errorf("URLFor(warning, %s) = %q, want %q", level, got, want)
 		}
 	}
 }
@@ -260,20 +265,146 @@ func TestNotify_NoRetryOn4xx(t *testing.T) {
 	}
 }
 
-func TestNotify_TimeoutIsRetried(t *testing.T) {
+// The request reached the web app and only the answer is missing: it may have
+// acted, so a retry could send the customer a second email.
+func TestNotify_TimeoutAfterWriteIsNotRetried(t *testing.T) {
 	c := &capture{delay: 300 * time.Millisecond}
 	s := httptest.NewServer(c)
 	defer s.Close()
 	h := newHooks()
 	cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
 	cfg.Timeout = 50 * time.Millisecond
-	cfg.Attempts = 2
 	n := New(cfg, h.Hooks())
 	n.Notify(Event{Level: "O", ID: "7"})
 	h.wait(t, 1)
 	n.Shutdown(context.Background())
-	if len(h.failed) != 1 || h.failed[0].attempts != 2 {
-		t.Errorf("failed=%+v; want failure after 2 timed-out attempts", h.failed)
+	if len(h.failed) != 1 || h.failed[0].attempts != 1 {
+		t.Fatalf("failed=%+v; want one failure after a single attempt", h.failed)
+	}
+	if !strings.Contains(h.failed[0].err.Error(), "outcome unknown") {
+		t.Errorf("error should say the outcome is unknown: %v", h.failed[0].err)
+	}
+	waitCalls(t, c, 1)
+	time.Sleep(100 * time.Millisecond)
+	if c.calls.Load() != 1 {
+		t.Errorf("web app calls = %d, want 1", c.calls.Load())
+	}
+}
+
+// Nothing was delivered, so retrying cannot duplicate anything.
+func TestNotify_RefusedConnectionIsRetried(t *testing.T) {
+	s := httptest.NewServer(&capture{})
+	addr := s.URL
+	s.Close()
+	h := newHooks()
+	cfg := fastConfig(addr+"/l?level=$LEVEL", "", "")
+	cfg.Attempts = 3
+	n := New(cfg, h.Hooks())
+	n.Notify(Event{Level: "O", ID: "7"})
+	h.wait(t, 1)
+	n.Shutdown(context.Background())
+	if len(h.failed) != 1 || h.failed[0].attempts != 3 {
+		t.Errorf("failed=%+v; want failure after 3 attempts", h.failed)
+	}
+}
+
+// A notification waiting out its backoff must not hold a concurrency slot.
+func TestNotify_BackoffReleasesSlot(t *testing.T) {
+	c := &capture{status: []int{500, 200, 200}}
+	s := httptest.NewServer(c)
+	defer s.Close()
+	h := newHooks()
+	cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
+	cfg.MaxConcurrent = 1
+	cfg.Backoff = 400 * time.Millisecond
+	n := New(cfg, h.Hooks())
+	defer n.Shutdown(context.Background())
+	n.Notify(Event{Level: "O", ID: "retrying"})
+	waitCalls(t, c, 1)
+	n.Notify(Event{Level: "O", ID: "second"})
+	select {
+	case <-h.done:
+	case <-time.After(250 * time.Millisecond):
+		t.Fatal("the second notification waited for the first one's backoff")
+	}
+	h.done <- struct{}{}
+	h.wait(t, 2)
+}
+
+func waitCalls(t *testing.T, c *capture, n int32) {
+	t.Helper()
+	deadline := time.Now().Add(2 * time.Second)
+	for c.calls.Load() < n {
+		if time.Now().After(deadline) {
+			t.Fatalf("calls = %d, want %d", c.calls.Load(), n)
+		}
+		time.Sleep(time.Millisecond)
+	}
+}
+
+// Shutdown lets a pending retry finish while its deadline allows.
+func TestNotify_ShutdownFinishesPendingRetries(t *testing.T) {
+	c := &capture{status: []int{500, 200}}
+	s := httptest.NewServer(c)
+	defer s.Close()
+	h := newHooks()
+	cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
+	cfg.Backoff = 100 * time.Millisecond
+	n := New(cfg, h.Hooks())
+	n.Notify(Event{Level: "O", ID: "7"})
+	waitCalls(t, c, 1)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+	if err := n.Shutdown(ctx); err != nil {
+		t.Fatal(err)
+	}
+	if len(h.sent) != 1 || h.sent[0].attempt != 2 || len(h.failed) != 0 {
+		t.Errorf("sent=%+v failed=%+v; want delivery on attempt 2", h.sent, h.failed)
+	}
+}
+
+// When the deadline cuts a retry short the failure reports the real attempt count.
+func TestNotify_ShutdownDeadlineReportsRealAttempts(t *testing.T) {
+	c := &capture{status: []int{500}}
+	s := httptest.NewServer(c)
+	defer s.Close()
+	h := newHooks()
+	cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
+	cfg.Backoff = 5 * time.Second
+	n := New(cfg, h.Hooks())
+	n.Notify(Event{Level: "O", ID: "7"})
+	waitCalls(t, c, 1)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
+	defer cancel()
+	_ = n.Shutdown(ctx)
+	h.wait(t, 1)
+	if len(h.failed) != 1 || h.failed[0].attempts != 1 {
+		t.Errorf("failed=%+v; want 1 real attempt", h.failed)
+	}
+}
+
+func TestNotify_WarningUsesWarningURL(t *testing.T) {
+	breach, warn := &capture{}, &capture{}
+	sb, sw := httptest.NewServer(breach), httptest.NewServer(warn)
+	defer sb.Close()
+	defer sw.Close()
+	h := newHooks()
+	cfg := fastConfig(sb.URL+"/l?level=$LEVEL", "", "")
+	cfg.WarnOrgURL = sw.URL + "/w?level=$LEVEL&limit=$LIMIT&percent=$PERCENT"
+	n := New(cfg, h.Hooks())
+	defer n.Shutdown(context.Background())
+	if !n.Notify(Event{Kind: KindWarning, Level: "O", ID: "7", Limit: 100, Count: 80, Percent: 80}) {
+		t.Fatal("warning refused")
+	}
+	h.wait(t, 1)
+	if breach.calls.Load() != 0 || warn.calls.Load() != 1 {
+		t.Fatalf("breach=%d warn=%d, want the warning URL only", breach.calls.Load(), warn.calls.Load())
+	}
+	if q := warn.requests[0].URL.Query(); q.Get("percent") != "80" || q.Get("limit") != "100" {
+		t.Errorf("query = %v", q)
+	}
+	if n.Notify(Event{Kind: KindWarning, Level: "A"}) {
+		t.Error("a level with no warning URL must be refused")
 	}
 }
 
@@ -326,7 +457,7 @@ func TestNotify_ShutdownDrainsAndRespectsContext(t *testing.T) {
 func TestNew_Defaults(t *testing.T) {
 	n := New(Config{OrgURL: "https://x/?level=$LEVEL"}, Hooks{})
 	defer n.Shutdown(context.Background())
-	if n.cfg.Attempts != 5 || n.cfg.Backoff != time.Second || n.cfg.MaxConcurrent != 8 || n.cfg.Timeout != 5*time.Second {
+	if n.cfg.Attempts != 5 || n.cfg.Backoff != time.Second || n.cfg.MaxConcurrent != 8 || n.cfg.Timeout != 15*time.Second {
 		t.Errorf("defaults not applied: %+v", n.cfg)
 	}
 }