Add 80% limit warning, stop repeat notifications, fix review findings

Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/store/store.go

@@ -32,13 +32,30 @@ const (
 	Prefix = "cnt"
 	// TTL is refreshed on every increment, like the Lua plugin did.
 	TTL = 31 * 24 * time.Hour
+
+	// MarkerLimit and MarkerWarn name the two notification markers of a counter.
+	MarkerLimit = "limit"
+	MarkerWarn  = "warn"
+	// MarkerTTL outlives the UTC day the marker belongs to.
+	MarkerTTL = 48 * time.Hour
 )
 
+// ErrNotApplied wraps an Incr failure where Redis answered and rejected the
+// whole transaction (EXECABORT, READONLY, LOADING, ...): nothing was applied,
+// so running it again cannot double count.
+var ErrNotApplied = errors.New("transaction not applied")
+
 // Key builds a counter key: Key("A", "x@example.com", "2026-10-13", "i").
 func Key(level, id, date, flow string) string {
 	return Prefix + "/" + level + "/" + id + "/d/" + date + "/" + flow
 }
 
+// MarkerKey builds the key recording that a counter's notification went out.
+// It uses its own prefix because the web app scans cnt/*<address>*.
+func MarkerKey(level, id, date, flow, kind string) string {
+	return "ntf/" + level + "/" + id + "/d/" + date + "/" + flow + "/" + kind
+}
+
 // DayKey returns the UTC calendar day of t, as used in the keys. The web app
 // builds its keys with Date.utc_today(), so this must never use local time.
 func DayKey(t time.Time) string { return t.UTC().Format("2006-01-02") }
@@ -69,6 +86,8 @@ func New(cfg config.RedisConfig) (*Store, error) {
 			DialTimeout:      timeout,
 			ReadTimeout:      timeout,
 			WriteTimeout:     timeout,
+
+			ContextTimeoutEnabled: true,
 		}
 		write := redis.NewFailoverClient(&base)
 		// ReplicaOnly picks a random replica and falls back to the master when
@@ -91,6 +110,8 @@ func New(cfg config.RedisConfig) (*Store, error) {
 			DialTimeout:  timeout,
 			ReadTimeout:  timeout,
 			WriteTimeout: timeout,
+
+			ContextTimeoutEnabled: true,
 		}
 	}
 	addrs := cfg.ReadAddrs
@@ -137,7 +158,7 @@ func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
 		return nil, nil
 	}
 	cmds := make([]*redis.IntCmd, len(keys))
-	_, err := s.write.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
+	all, err := s.write.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
 		for i, k := range keys {
 			cmds[i] = pipe.Incr(ctx, k)
 			pipe.Expire(ctx, k, TTL)
@@ -154,6 +175,9 @@ func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
 		if !errors.As(err, &rerr) {
 			return nil, fmt.Errorf("redis incr: %w", err)
 		}
+		if allFailed(all) {
+			return nil, fmt.Errorf("redis incr: %w: %w", ErrNotApplied, err)
+		}
 	}
 	for i, c := range cmds {
 		if cerr := c.Err(); cerr != nil {
@@ -165,12 +189,29 @@ func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
 	return out, nil
 }
 
+// allFailed tells a rejected transaction from per-key failures: an INCR on a
+// non-number fails alone and its EXPIRE still succeeds, so only a transaction
+// Redis refused as a whole leaves every command with an error.
+func allFailed(cmds []redis.Cmder) bool {
+	for _, c := range cmds {
+		if c.Err() == nil {
+			return false
+		}
+	}
+	return len(cmds) > 0
+}
+
 // OutcomeUnknown reports whether a failed Incr may still have been applied
 // by Redis: the request may have been written but the reply lost (timeout,
 // EOF, reset). A refused connection, a dial failure or a cancelled context
 // never reached Redis and is a definite failure.
 func OutcomeUnknown(err error) bool {
-	if err == nil || errors.Is(err, context.Canceled) {
+	if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, ErrNotApplied) {
+		return false
+	}
+	// No connection could be had from the pool, so no command was sent. go-redis
+	// keeps this error in an internal package; only its text is available.
+	if strings.Contains(err.Error(), "connection pool timeout") {
 		return false
 	}
 	var opErr *net.OpError
@@ -188,6 +229,18 @@ func OutcomeUnknown(err error) bool {
 	return strings.Contains(msg, "timeout") || strings.Contains(msg, "reset") || strings.Contains(msg, "broken pipe") || strings.Contains(msg, "EOF")
 }
 
+// MarkNotified records that a counter's notification was sent. It overwrites
+// any older marker: a counter the web app reset may legitimately cross again.
+func (s *Store) MarkNotified(ctx context.Context, key string) error {
+	return s.write.Set(ctx, key, "1", MarkerTTL).Err()
+}
+
+// ClaimNotified atomically takes the right to send a notification. It returns
+// true for exactly one caller while the marker exists.
+func (s *Store) ClaimNotified(ctx context.Context, key string) (bool, error) {
+	return s.write.SetNX(ctx, key, "1", MarkerTTL).Result()
+}
+
 // GetMaster reads the keys from the write side (the master) with one MGET;
 // used to establish the outcome of an Incr whose reply was lost, when a
 // replica might not have the increment yet. A missing key reads as 0.