dimitrije/counters
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 80% limit warning, stop repeat notifications, fix review findings

Browse Source 
Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/counting/counting.go
+120 -23
View File
@@ -56,6 +56,8 @@ type Planner struct {
Exempt map[string]bool // lowercased addresses that never count
CountJunk bool // count /in requests flagged junk
Now func() time.Time // nil = time.Now; the UTC day of Now() is the counter day
// WarnPercent is the share of a limit that triggers an early warning; 0 = off.
WarnPercent int64
}
// Job is one validated request handed to the queue: the fields the counting
@@ -66,9 +68,9 @@ type Job struct {
TraceID string
Received time.Time // when the request arrived (queue wait metric)
AuthUser string // /out: normalised SMTP AUTH user, "" when not usable
LoginSuffix string // /out: "SUPPORTTOKEN" of a user@dom@SUPPORTTOKEN login
MailFrom string // normalised, or the sanitised input when not an address
AuthUser string // /out: normalised SMTP AUTH user, "" when not usable
TokenLogin bool // /out: a user@dom@SUFFIX login
MailFrom string // normalised, or the sanitised input when not an address
// MailFromUsable tells whether MailFrom is a normalised address (and may
// match the exempt list); an unusable value is never exempt.
MailFromUsable bool
@@ -107,20 +109,21 @@ type Increment struct {
By string // address that caused it (sender for o, recipient for i); $ADDRESS in notifications
Limit int64 // configured limit, 0 when unlimited
Limited bool // false when no limit is configured for this level/flow
WarnAt int64 // counter value that triggers the early warning, 0 = none
Domain *domains.Domain
}
// Plan is the result of applying the rules to one request.
type Plan struct {
Route string
TraceID string
Date string // UTC day used in the keys
Skipped string // "" or one of the Skip* reasons
AuthUser string
LoginSuffix string
MailFrom string
Recipient string
Junk bool // /in only: the message was flagged junk by mizu
Route string
TraceID string
Date string // UTC day used in the keys
Skipped string // "" or one of the Skip* reasons
AuthUser string
TokenLogin bool
MailFrom string
Recipient string
Junk bool // /in only: the message was flagged junk by mizu
Unusable []Unusable // sides that were not counted because they failed validation
Lookups []Lookup
@@ -166,8 +169,12 @@ func (p *Planner) Plan(job Job) (*Plan, error) {
if job.MailFromUsable {
mailFrom = normalize(mailFrom)
}
plan := &Plan{Route: job.Route, TraceID: job.TraceID, Date: store.DayKey(p.now()),
AuthUser: authUser, LoginSuffix: job.LoginSuffix, MailFrom: mailFrom, Recipient: recipient, Junk: job.Junk}
counted := job.Received
if counted.IsZero() {
counted = p.now()
}
plan := &Plan{Route: job.Route, TraceID: job.TraceID, Date: store.DayKey(counted),
AuthUser: authUser, TokenLogin: job.TokenLogin, MailFrom: mailFrom, Recipient: recipient, Junk: job.Junk}
if job.SenderUnusable.Set() {
plan.Unusable = append(plan.Unusable, job.SenderUnusable)
}
@@ -234,11 +241,33 @@ func (p *Planner) add(plan *Plan, role, address, flow string) error {
Level: l.level, ID: l.id, Flow: flow,
Key: store.Key(l.level, l.id, plan.Date, flow),
By: address, Limit: limit, Limited: limited, Domain: d,
WarnAt: warnAt(limit, limited, p.WarnPercent),
})
}
return nil
}
func warnAt(limit int64, limited bool, percent int64) int64 {
if !limited {
return 0
}
return WarnThreshold(limit, percent)
}
// WarnThreshold returns the counter value at which percent of limit is used,
// rounded up, or 0 when no warning applies: warnings are off, there is no
// positive limit, or the threshold is not below the limit itself.
func WarnThreshold(limit, percent int64) int64 {
if limit <= 0 || percent <= 0 {
return 0
}
t := limit/100*percent + (limit%100*percent+99)/100
if t >= limit {
return 0
}
return t
}
// Crossed reports whether an increment of amount that produced newValue is
// the one that reached limit: old < limit <= new. Because INCRBY is atomic
// exactly one increment can cross, so this is the once-per-crossing trigger
@@ -251,8 +280,12 @@ func Crossed(newValue, amount, limit int64) bool {
type Result struct {
Increment
Count int64 // value after the increment (from EXEC, or re-read after a lost reply)
Reached bool // this increment crossed the limit
Reached bool // this increment crossed the limit and is the one to notify
Warned bool // this increment crossed the warning threshold and is the one to notify
Err error // this key's own INCR failed (e.g. it holds a non-number); Count is 0
// MarkerErr is set when the notification marker could not be read or
// written. After a lost reply the notification is then withheld.
MarkerErr error
}
// Outcome is an executed Plan.
@@ -269,6 +302,9 @@ type Counter struct {
Planner *Planner
Store *store.Store
ReReadTimeout time.Duration // budget for the master re-read after a lost reply (default 2s)
// AbortRetryDelay is the pause before the single retry of a transaction
// Redis rejected, e.g. during a failover (default 200ms).
AbortRetryDelay time.Duration
}
// CountOut plans and executes an /out request.
@@ -293,12 +329,17 @@ func (c *Counter) CountIn(ctx context.Context, traceID, mailFrom, recipient stri
// not touch Redis.
//
// Failure handling (review #4): a key whose own INCR failed is reported in
// its Result.Err while the other keys are counted normally. When the whole
// reply was lost (timeout, reset) after the write may have been applied, the
// keys are re-read once from the master and the outcome is flagged Unknown;
// crossing detection then uses count >= limit so a crossing that happened in
// the lost transaction is still notified. A definite failure (connection
// refused, cancelled) is returned as an error: nothing was counted.
// its Result.Err while the other keys are counted normally. A transaction
// Redis rejected as a whole applied nothing and is retried once. When the
// whole reply was lost (timeout, reset) after the write may have been applied,
// the keys are re-read once from the master and the outcome is flagged
// Unknown. A definite failure (connection refused, cancelled) is returned as
// an error: nothing was counted.
//
// Notifications go out once per counter: a confirmed crossing is exact, so it
// always fires and writes a marker. After a lost reply the crossing cannot be
// pinned to this increment, so it fires only for the caller that wins the
// marker, and a counter that already notified stays silent.
func (c *Counter) Execute(ctx context.Context, plan *Plan) (*Outcome, error) {
out := &Outcome{Plan: plan}
if len(plan.Increments) == 0 {
@@ -309,13 +350,15 @@ func (c *Counter) Execute(ctx context.Context, plan *Plan) (*Outcome, error) {
keys[i] = inc.Key
}
out.Results = make([]Result, len(plan.Increments))
results, err := c.Store.Incr(ctx, keys)
results, err := c.incr(ctx, keys)
if err == nil {
for i, inc := range plan.Increments {
r := Result{Increment: inc, Err: results[i].Err}
if r.Err == nil {
r.Count = results[i].Count
r.Reached = inc.Limited && Crossed(r.Count, 1, inc.Limit)
r.Warned = Crossed(r.Count, 1, inc.WarnAt)
c.mark(ctx, plan.Date, &r)
}
out.Results[i] = r
}
@@ -332,11 +375,65 @@ func (c *Counter) Execute(ctx context.Context, plan *Plan) (*Outcome, error) {
}
out.Unknown = err
for i, inc := range plan.Increments {
out.Results[i] = Result{Increment: inc, Count: counts[i], Reached: inc.Limited && counts[i] >= inc.Limit}
r := Result{Increment: inc, Count: counts[i]}
switch {
case inc.Limited && inc.Limit > 0 && r.Count >= inc.Limit:
r.Reached = c.claim(rctx, plan.Date, &r, store.MarkerLimit)
case inc.WarnAt > 0 && r.Count >= inc.WarnAt:
r.Warned = c.claim(rctx, plan.Date, &r, store.MarkerWarn)
}
out.Results[i] = r
}
return out, nil
}
// incr runs the transaction and retries it once when Redis rejected it.
func (c *Counter) incr(ctx context.Context, keys []string) ([]store.IncrResult, error) {
results, err := c.Store.Incr(ctx, keys)
if !errors.Is(err, store.ErrNotApplied) {
return results, err
}
select {
case <-time.After(c.abortRetryDelay()):
case <-ctx.Done():
return nil, err
}
return c.Store.Incr(ctx, keys)
}
// mark records a confirmed crossing. A failure is reported but the
// notification still goes out, because this crossing is certain.
func (c *Counter) mark(ctx context.Context, date string, r *Result) {
if !r.Reached && !r.Warned {
return
}
kind := store.MarkerLimit
if r.Warned {
kind = store.MarkerWarn
}
mctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), c.reReadTimeout())
defer cancel()
r.MarkerErr = c.Store.MarkNotified(mctx, store.MarkerKey(r.Level, r.ID, date, r.Flow, kind))
}
// claim takes the marker after a lost reply. If that fails nothing is sent:
// a missed notification is preferred to a repeated one.
func (c *Counter) claim(ctx context.Context, date string, r *Result, kind string) bool {
won, err := c.Store.ClaimNotified(ctx, store.MarkerKey(r.Level, r.ID, date, r.Flow, kind))
if err != nil {
r.MarkerErr = err
return false
}
return won
}
func (c *Counter) abortRetryDelay() time.Duration {
if c.AbortRetryDelay > 0 {
return c.AbortRetryDelay
}
return 200 * time.Millisecond
}
func (c *Counter) reReadTimeout() time.Duration {
if c.ReReadTimeout > 0 {
return c.ReReadTimeout