Add 80% limit warning, stop repeat notifications, fix review findings

Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/store/markers_test.go

@@ -0,0 +1,132 @@
+package store
+
+import (
+	"context"
+	"errors"
+	"net"
+	"testing"
+	"time"
+
+	"migadu_counters/internal/config"
+)
+
+func TestMarkerKey(t *testing.T) {
+	got := MarkerKey("A", "x@example.com", "2026-10-13", "o", MarkerLimit)
+	if want := "ntf/A/x@example.com/d/2026-10-13/o/limit"; got != want {
+		t.Errorf("MarkerKey = %q, want %q", got, want)
+	}
+	if got := MarkerKey("O", "7", "2026-10-13", "i", MarkerWarn); got != "ntf/O/7/d/2026-10-13/i/warn" {
+		t.Errorf("MarkerKey = %q", got)
+	}
+}
+
+func TestMarkAndClaimNotified(t *testing.T) {
+	s, w, _ := testStore(t)
+	ctx := context.Background()
+	if err := s.MarkNotified(ctx, "ntf/a"); err != nil {
+		t.Fatal(err)
+	}
+	if ttl := w.TTL("ntf/a"); ttl != MarkerTTL {
+		t.Errorf("TTL = %v, want %v", ttl, MarkerTTL)
+	}
+	// Marking again is not an error: a reset counter may cross again.
+	if err := s.MarkNotified(ctx, "ntf/a"); err != nil {
+		t.Errorf("second mark: %v", err)
+	}
+	ok, err := s.ClaimNotified(ctx, "ntf/a")
+	if err != nil || ok {
+		t.Errorf("claim of a marked key = %v, %v; want false", ok, err)
+	}
+	ok, err = s.ClaimNotified(ctx, "ntf/b")
+	if err != nil || !ok {
+		t.Errorf("first claim = %v, %v; want true", ok, err)
+	}
+	if ttl := w.TTL("ntf/b"); ttl != MarkerTTL {
+		t.Errorf("claim TTL = %v, want %v", ttl, MarkerTTL)
+	}
+	ok, err = s.ClaimNotified(ctx, "ntf/b")
+	if err != nil || ok {
+		t.Errorf("second claim = %v, %v; want false", ok, err)
+	}
+}
+
+func TestClaimNotified_RedisDown(t *testing.T) {
+	s, w, _ := testStore(t)
+	w.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
+	if ok, err := s.ClaimNotified(ctx, "ntf/a"); err == nil || ok {
+		t.Errorf("claim with redis down = %v, %v; want an error and false", ok, err)
+	}
+}
+
+// No connection could be had from the pool: no command was ever sent.
+func TestOutcomeUnknown_PoolTimeoutIsDefinite(t *testing.T) {
+	if OutcomeUnknown(errors.New("redis incr: redis: connection pool timeout")) {
+		t.Error("a pool timeout never reached Redis and must not be an unknown outcome")
+	}
+}
+
+func TestOutcomeUnknown_NotAppliedIsDefinite(t *testing.T) {
+	if OutcomeUnknown(ErrNotApplied) {
+		t.Error("ErrNotApplied must not be an unknown outcome")
+	}
+}
+
+// Transaction-level rejection: every command failed, nothing was applied.
+func TestIncr_RejectedTransactionIsNotApplied(t *testing.T) {
+	s, w, _ := testStore(t)
+	w.SetError("READONLY You can't write against a read only replica.")
+	defer w.SetError("")
+	_, err := s.Incr(context.Background(), []string{"a", "b"})
+	if !errors.Is(err, ErrNotApplied) {
+		t.Fatalf("err = %v, want ErrNotApplied", err)
+	}
+}
+
+// All INCRs failing on non-numbers is still per-key: the EXPIREs succeeded and
+// EXEC ran, so it must not be mistaken for a rejected transaction.
+func TestIncr_AllKeysNonNumericIsPerKey(t *testing.T) {
+	s, w, _ := testStore(t)
+	w.Set("a", "x")
+	w.Set("b", "y")
+	res, err := s.Incr(context.Background(), []string{"a", "b"})
+	if err != nil {
+		t.Fatalf("per-key failures must not fail the transaction: %v", err)
+	}
+	if res[0].Err == nil || res[1].Err == nil {
+		t.Errorf("results = %+v", res)
+	}
+}
+
+// A context deadline must bound socket time, not only the configured timeout.
+func TestPing_HonoursContextDeadline(t *testing.T) {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ln.Close()
+	go func() { // accepts and never answers
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			defer c.Close()
+		}
+	}()
+	s, err := New(config.RedisConfig{WriteAddr: ln.Addr().String(), PoolSize: 2, TimeoutMS: 3000})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer s.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond)
+	defer cancel()
+	start := time.Now()
+	if err := s.Ping(ctx); err == nil {
+		t.Fatal("expected a timeout")
+	}
+	if d := time.Since(start); d > time.Second {
+		t.Errorf("Ping took %v with a 100ms deadline", d)
+	}
+}

internal/store/store.go

@@ -32,13 +32,30 @@ const (
 	Prefix = "cnt"
 	// TTL is refreshed on every increment, like the Lua plugin did.
 	TTL = 31 * 24 * time.Hour
+
+	// MarkerLimit and MarkerWarn name the two notification markers of a counter.
+	MarkerLimit = "limit"
+	MarkerWarn  = "warn"
+	// MarkerTTL outlives the UTC day the marker belongs to.
+	MarkerTTL = 48 * time.Hour
 )
 
+// ErrNotApplied wraps an Incr failure where Redis answered and rejected the
+// whole transaction (EXECABORT, READONLY, LOADING, ...): nothing was applied,
+// so running it again cannot double count.
+var ErrNotApplied = errors.New("transaction not applied")
+
 // Key builds a counter key: Key("A", "x@example.com", "2026-10-13", "i").
 func Key(level, id, date, flow string) string {
 	return Prefix + "/" + level + "/" + id + "/d/" + date + "/" + flow
 }
 
+// MarkerKey builds the key recording that a counter's notification went out.
+// It uses its own prefix because the web app scans cnt/*<address>*.
+func MarkerKey(level, id, date, flow, kind string) string {
+	return "ntf/" + level + "/" + id + "/d/" + date + "/" + flow + "/" + kind
+}
+
 // DayKey returns the UTC calendar day of t, as used in the keys. The web app
 // builds its keys with Date.utc_today(), so this must never use local time.
 func DayKey(t time.Time) string { return t.UTC().Format("2006-01-02") }
@@ -69,6 +86,8 @@ func New(cfg config.RedisConfig) (*Store, error) {
 			DialTimeout:      timeout,
 			ReadTimeout:      timeout,
 			WriteTimeout:     timeout,
+
+			ContextTimeoutEnabled: true,
 		}
 		write := redis.NewFailoverClient(&base)
 		// ReplicaOnly picks a random replica and falls back to the master when
@@ -91,6 +110,8 @@ func New(cfg config.RedisConfig) (*Store, error) {
 			DialTimeout:  timeout,
 			ReadTimeout:  timeout,
 			WriteTimeout: timeout,
+
+			ContextTimeoutEnabled: true,
 		}
 	}
 	addrs := cfg.ReadAddrs
@@ -137,7 +158,7 @@ func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
 		return nil, nil
 	}
 	cmds := make([]*redis.IntCmd, len(keys))
-	_, err := s.write.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
+	all, err := s.write.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
 		for i, k := range keys {
 			cmds[i] = pipe.Incr(ctx, k)
 			pipe.Expire(ctx, k, TTL)
@@ -154,6 +175,9 @@ func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
 		if !errors.As(err, &rerr) {
 			return nil, fmt.Errorf("redis incr: %w", err)
 		}
+		if allFailed(all) {
+			return nil, fmt.Errorf("redis incr: %w: %w", ErrNotApplied, err)
+		}
 	}
 	for i, c := range cmds {
 		if cerr := c.Err(); cerr != nil {
@@ -165,12 +189,29 @@ func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
 	return out, nil
 }
 
+// allFailed tells a rejected transaction from per-key failures: an INCR on a
+// non-number fails alone and its EXPIRE still succeeds, so only a transaction
+// Redis refused as a whole leaves every command with an error.
+func allFailed(cmds []redis.Cmder) bool {
+	for _, c := range cmds {
+		if c.Err() == nil {
+			return false
+		}
+	}
+	return len(cmds) > 0
+}
+
 // OutcomeUnknown reports whether a failed Incr may still have been applied
 // by Redis: the request may have been written but the reply lost (timeout,
 // EOF, reset). A refused connection, a dial failure or a cancelled context
 // never reached Redis and is a definite failure.
 func OutcomeUnknown(err error) bool {
-	if err == nil || errors.Is(err, context.Canceled) {
+	if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, ErrNotApplied) {
+		return false
+	}
+	// No connection could be had from the pool, so no command was sent. go-redis
+	// keeps this error in an internal package; only its text is available.
+	if strings.Contains(err.Error(), "connection pool timeout") {
 		return false
 	}
 	var opErr *net.OpError
@@ -188,6 +229,18 @@ func OutcomeUnknown(err error) bool {
 	return strings.Contains(msg, "timeout") || strings.Contains(msg, "reset") || strings.Contains(msg, "broken pipe") || strings.Contains(msg, "EOF")
 }
 
+// MarkNotified records that a counter's notification was sent. It overwrites
+// any older marker: a counter the web app reset may legitimately cross again.
+func (s *Store) MarkNotified(ctx context.Context, key string) error {
+	return s.write.Set(ctx, key, "1", MarkerTTL).Err()
+}
+
+// ClaimNotified atomically takes the right to send a notification. It returns
+// true for exactly one caller while the marker exists.
+func (s *Store) ClaimNotified(ctx context.Context, key string) (bool, error) {
+	return s.write.SetNX(ctx, key, "1", MarkerTTL).Result()
+}
+
 // GetMaster reads the keys from the write side (the master) with one MGET;
 // used to establish the outcome of an Incr whose reply was lost, when a
 // replica might not have the increment yet. A missing key reads as 0.