Add 80% limit warning, stop repeat notifications, fix review findings
Notifications: - Warn once when a counter reaches warn_percent of its limit, through separate warn_*_url endpoints. - Write ntf/ markers so a lost Redis reply cannot re-send a breach. - Retry only when the request never left or the web app answered 429/5xx. Fixes: - Retry transactions Redis rejected; treat pool timeouts as definite. - Install the signal handler before startup work. - Stop logging the login suffix; redact notify URL credentials. - Reject corrupt or empty CDBs, truncated hash tables included. - Require a Sentinel password, refuse example placeholders, and default admin_addr to localhost. - Normalise /counters domains, count on the arrival day, log rejections, hide internals in errors and exit 1 on a failed drain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
c43ff17627
commit
54f9626121
+120
-23
@@ -56,6 +56,8 @@ type Planner struct {
|
||||
Exempt map[string]bool // lowercased addresses that never count
|
||||
CountJunk bool // count /in requests flagged junk
|
||||
Now func() time.Time // nil = time.Now; the UTC day of Now() is the counter day
|
||||
// WarnPercent is the share of a limit that triggers an early warning; 0 = off.
|
||||
WarnPercent int64
|
||||
}
|
||||
|
||||
// Job is one validated request handed to the queue: the fields the counting
|
||||
@@ -66,9 +68,9 @@ type Job struct {
|
||||
TraceID string
|
||||
Received time.Time // when the request arrived (queue wait metric)
|
||||
|
||||
AuthUser string // /out: normalised SMTP AUTH user, "" when not usable
|
||||
LoginSuffix string // /out: "SUPPORTTOKEN" of a user@dom@SUPPORTTOKEN login
|
||||
MailFrom string // normalised, or the sanitised input when not an address
|
||||
AuthUser string // /out: normalised SMTP AUTH user, "" when not usable
|
||||
TokenLogin bool // /out: a user@dom@SUFFIX login
|
||||
MailFrom string // normalised, or the sanitised input when not an address
|
||||
// MailFromUsable tells whether MailFrom is a normalised address (and may
|
||||
// match the exempt list); an unusable value is never exempt.
|
||||
MailFromUsable bool
|
||||
@@ -107,20 +109,21 @@ type Increment struct {
|
||||
By string // address that caused it (sender for o, recipient for i); $ADDRESS in notifications
|
||||
Limit int64 // configured limit, 0 when unlimited
|
||||
Limited bool // false when no limit is configured for this level/flow
|
||||
WarnAt int64 // counter value that triggers the early warning, 0 = none
|
||||
Domain *domains.Domain
|
||||
}
|
||||
|
||||
// Plan is the result of applying the rules to one request.
|
||||
type Plan struct {
|
||||
Route string
|
||||
TraceID string
|
||||
Date string // UTC day used in the keys
|
||||
Skipped string // "" or one of the Skip* reasons
|
||||
AuthUser string
|
||||
LoginSuffix string
|
||||
MailFrom string
|
||||
Recipient string
|
||||
Junk bool // /in only: the message was flagged junk by mizu
|
||||
Route string
|
||||
TraceID string
|
||||
Date string // UTC day used in the keys
|
||||
Skipped string // "" or one of the Skip* reasons
|
||||
AuthUser string
|
||||
TokenLogin bool
|
||||
MailFrom string
|
||||
Recipient string
|
||||
Junk bool // /in only: the message was flagged junk by mizu
|
||||
|
||||
Unusable []Unusable // sides that were not counted because they failed validation
|
||||
Lookups []Lookup
|
||||
@@ -166,8 +169,12 @@ func (p *Planner) Plan(job Job) (*Plan, error) {
|
||||
if job.MailFromUsable {
|
||||
mailFrom = normalize(mailFrom)
|
||||
}
|
||||
plan := &Plan{Route: job.Route, TraceID: job.TraceID, Date: store.DayKey(p.now()),
|
||||
AuthUser: authUser, LoginSuffix: job.LoginSuffix, MailFrom: mailFrom, Recipient: recipient, Junk: job.Junk}
|
||||
counted := job.Received
|
||||
if counted.IsZero() {
|
||||
counted = p.now()
|
||||
}
|
||||
plan := &Plan{Route: job.Route, TraceID: job.TraceID, Date: store.DayKey(counted),
|
||||
AuthUser: authUser, TokenLogin: job.TokenLogin, MailFrom: mailFrom, Recipient: recipient, Junk: job.Junk}
|
||||
if job.SenderUnusable.Set() {
|
||||
plan.Unusable = append(plan.Unusable, job.SenderUnusable)
|
||||
}
|
||||
@@ -234,11 +241,33 @@ func (p *Planner) add(plan *Plan, role, address, flow string) error {
|
||||
Level: l.level, ID: l.id, Flow: flow,
|
||||
Key: store.Key(l.level, l.id, plan.Date, flow),
|
||||
By: address, Limit: limit, Limited: limited, Domain: d,
|
||||
WarnAt: warnAt(limit, limited, p.WarnPercent),
|
||||
})
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func warnAt(limit int64, limited bool, percent int64) int64 {
|
||||
if !limited {
|
||||
return 0
|
||||
}
|
||||
return WarnThreshold(limit, percent)
|
||||
}
|
||||
|
||||
// WarnThreshold returns the counter value at which percent of limit is used,
|
||||
// rounded up, or 0 when no warning applies: warnings are off, there is no
|
||||
// positive limit, or the threshold is not below the limit itself.
|
||||
func WarnThreshold(limit, percent int64) int64 {
|
||||
if limit <= 0 || percent <= 0 {
|
||||
return 0
|
||||
}
|
||||
t := limit/100*percent + (limit%100*percent+99)/100
|
||||
if t >= limit {
|
||||
return 0
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// Crossed reports whether an increment of amount that produced newValue is
|
||||
// the one that reached limit: old < limit <= new. Because INCRBY is atomic
|
||||
// exactly one increment can cross, so this is the once-per-crossing trigger
|
||||
@@ -251,8 +280,12 @@ func Crossed(newValue, amount, limit int64) bool {
|
||||
type Result struct {
|
||||
Increment
|
||||
Count int64 // value after the increment (from EXEC, or re-read after a lost reply)
|
||||
Reached bool // this increment crossed the limit
|
||||
Reached bool // this increment crossed the limit and is the one to notify
|
||||
Warned bool // this increment crossed the warning threshold and is the one to notify
|
||||
Err error // this key's own INCR failed (e.g. it holds a non-number); Count is 0
|
||||
// MarkerErr is set when the notification marker could not be read or
|
||||
// written. After a lost reply the notification is then withheld.
|
||||
MarkerErr error
|
||||
}
|
||||
|
||||
// Outcome is an executed Plan.
|
||||
@@ -269,6 +302,9 @@ type Counter struct {
|
||||
Planner *Planner
|
||||
Store *store.Store
|
||||
ReReadTimeout time.Duration // budget for the master re-read after a lost reply (default 2s)
|
||||
// AbortRetryDelay is the pause before the single retry of a transaction
|
||||
// Redis rejected, e.g. during a failover (default 200ms).
|
||||
AbortRetryDelay time.Duration
|
||||
}
|
||||
|
||||
// CountOut plans and executes an /out request.
|
||||
@@ -293,12 +329,17 @@ func (c *Counter) CountIn(ctx context.Context, traceID, mailFrom, recipient stri
|
||||
// not touch Redis.
|
||||
//
|
||||
// Failure handling (review #4): a key whose own INCR failed is reported in
|
||||
// its Result.Err while the other keys are counted normally. When the whole
|
||||
// reply was lost (timeout, reset) after the write may have been applied, the
|
||||
// keys are re-read once from the master and the outcome is flagged Unknown;
|
||||
// crossing detection then uses count >= limit so a crossing that happened in
|
||||
// the lost transaction is still notified. A definite failure (connection
|
||||
// refused, cancelled) is returned as an error: nothing was counted.
|
||||
// its Result.Err while the other keys are counted normally. A transaction
|
||||
// Redis rejected as a whole applied nothing and is retried once. When the
|
||||
// whole reply was lost (timeout, reset) after the write may have been applied,
|
||||
// the keys are re-read once from the master and the outcome is flagged
|
||||
// Unknown. A definite failure (connection refused, cancelled) is returned as
|
||||
// an error: nothing was counted.
|
||||
//
|
||||
// Notifications go out once per counter: a confirmed crossing is exact, so it
|
||||
// always fires and writes a marker. After a lost reply the crossing cannot be
|
||||
// pinned to this increment, so it fires only for the caller that wins the
|
||||
// marker, and a counter that already notified stays silent.
|
||||
func (c *Counter) Execute(ctx context.Context, plan *Plan) (*Outcome, error) {
|
||||
out := &Outcome{Plan: plan}
|
||||
if len(plan.Increments) == 0 {
|
||||
@@ -309,13 +350,15 @@ func (c *Counter) Execute(ctx context.Context, plan *Plan) (*Outcome, error) {
|
||||
keys[i] = inc.Key
|
||||
}
|
||||
out.Results = make([]Result, len(plan.Increments))
|
||||
results, err := c.Store.Incr(ctx, keys)
|
||||
results, err := c.incr(ctx, keys)
|
||||
if err == nil {
|
||||
for i, inc := range plan.Increments {
|
||||
r := Result{Increment: inc, Err: results[i].Err}
|
||||
if r.Err == nil {
|
||||
r.Count = results[i].Count
|
||||
r.Reached = inc.Limited && Crossed(r.Count, 1, inc.Limit)
|
||||
r.Warned = Crossed(r.Count, 1, inc.WarnAt)
|
||||
c.mark(ctx, plan.Date, &r)
|
||||
}
|
||||
out.Results[i] = r
|
||||
}
|
||||
@@ -332,11 +375,65 @@ func (c *Counter) Execute(ctx context.Context, plan *Plan) (*Outcome, error) {
|
||||
}
|
||||
out.Unknown = err
|
||||
for i, inc := range plan.Increments {
|
||||
out.Results[i] = Result{Increment: inc, Count: counts[i], Reached: inc.Limited && counts[i] >= inc.Limit}
|
||||
r := Result{Increment: inc, Count: counts[i]}
|
||||
switch {
|
||||
case inc.Limited && inc.Limit > 0 && r.Count >= inc.Limit:
|
||||
r.Reached = c.claim(rctx, plan.Date, &r, store.MarkerLimit)
|
||||
case inc.WarnAt > 0 && r.Count >= inc.WarnAt:
|
||||
r.Warned = c.claim(rctx, plan.Date, &r, store.MarkerWarn)
|
||||
}
|
||||
out.Results[i] = r
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// incr runs the transaction and retries it once when Redis rejected it.
|
||||
func (c *Counter) incr(ctx context.Context, keys []string) ([]store.IncrResult, error) {
|
||||
results, err := c.Store.Incr(ctx, keys)
|
||||
if !errors.Is(err, store.ErrNotApplied) {
|
||||
return results, err
|
||||
}
|
||||
select {
|
||||
case <-time.After(c.abortRetryDelay()):
|
||||
case <-ctx.Done():
|
||||
return nil, err
|
||||
}
|
||||
return c.Store.Incr(ctx, keys)
|
||||
}
|
||||
|
||||
// mark records a confirmed crossing. A failure is reported but the
|
||||
// notification still goes out, because this crossing is certain.
|
||||
func (c *Counter) mark(ctx context.Context, date string, r *Result) {
|
||||
if !r.Reached && !r.Warned {
|
||||
return
|
||||
}
|
||||
kind := store.MarkerLimit
|
||||
if r.Warned {
|
||||
kind = store.MarkerWarn
|
||||
}
|
||||
mctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), c.reReadTimeout())
|
||||
defer cancel()
|
||||
r.MarkerErr = c.Store.MarkNotified(mctx, store.MarkerKey(r.Level, r.ID, date, r.Flow, kind))
|
||||
}
|
||||
|
||||
// claim takes the marker after a lost reply. If that fails nothing is sent:
|
||||
// a missed notification is preferred to a repeated one.
|
||||
func (c *Counter) claim(ctx context.Context, date string, r *Result, kind string) bool {
|
||||
won, err := c.Store.ClaimNotified(ctx, store.MarkerKey(r.Level, r.ID, date, r.Flow, kind))
|
||||
if err != nil {
|
||||
r.MarkerErr = err
|
||||
return false
|
||||
}
|
||||
return won
|
||||
}
|
||||
|
||||
func (c *Counter) abortRetryDelay() time.Duration {
|
||||
if c.AbortRetryDelay > 0 {
|
||||
return c.AbortRetryDelay
|
||||
}
|
||||
return 200 * time.Millisecond
|
||||
}
|
||||
|
||||
func (c *Counter) reReadTimeout() time.Duration {
|
||||
if c.ReReadTimeout > 0 {
|
||||
return c.ReReadTimeout
|
||||
|
||||
@@ -450,7 +450,7 @@ func TestPlan_OutSenderUnusableCountsRecipient(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestPlan_OutRecipientUnusableCountsSender(t *testing.T) {
|
||||
job := Job{Route: RouteOut, TraceID: "u2", AuthUser: "x@example.com", LoginSuffix: "SUPPORTTOKEN", MailFrom: "x@example.com",
|
||||
job := Job{Route: RouteOut, TraceID: "u2", AuthUser: "x@example.com", TokenLogin: true, MailFrom: "x@example.com",
|
||||
RecipientUnusable: Unusable{Role: "recipient", Value: "u@[192.0.2.1]", Reason: "invalid e-mail address"}}
|
||||
plan, err := newPlanner(false).Plan(job)
|
||||
if err != nil {
|
||||
@@ -461,8 +461,8 @@ func TestPlan_OutRecipientUnusableCountsSender(t *testing.T) {
|
||||
{"D", "example.com", "o", "x@example.com", 2200, true},
|
||||
{"O", "7", "o", "x@example.com", 2200, true},
|
||||
})
|
||||
if plan.LoginSuffix != "SUPPORTTOKEN" || plan.AuthUser != "x@example.com" {
|
||||
t.Errorf("login suffix not carried: %+v", plan)
|
||||
if !plan.TokenLogin || plan.AuthUser != "x@example.com" {
|
||||
t.Errorf("token login not carried: %+v", plan)
|
||||
}
|
||||
if len(plan.Unusable) != 1 || plan.Unusable[0].Role != "recipient" {
|
||||
t.Errorf("unusable = %+v", plan.Unusable)
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
package counting
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"migadu_counters/internal/store"
|
||||
)
|
||||
|
||||
func TestWarnThreshold(t *testing.T) {
|
||||
tests := []struct {
|
||||
limit, percent, want int64
|
||||
}{
|
||||
{1, 80, 0}, // ceil(0.8) = 1 is the limit itself: no warning
|
||||
{2, 80, 0}, // ceil(1.6) = 2 is the limit itself: no warning
|
||||
{5, 80, 4},
|
||||
{100, 80, 80},
|
||||
{101, 80, 81},
|
||||
{100, 0, 0}, // disabled
|
||||
{0, 80, 0}, // no limit, no warning
|
||||
{-5, 80, 0},
|
||||
{1 << 62, 50, 1 << 61}, // no overflow
|
||||
}
|
||||
for _, tt := range tests {
|
||||
if got := WarnThreshold(tt.limit, tt.percent); got != tt.want {
|
||||
t.Errorf("WarnThreshold(%d, %d) = %d, want %d", tt.limit, tt.percent, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func warnCounter(t *testing.T) (*Counter, *miniredis.Miniredis) {
|
||||
t.Helper()
|
||||
c, m := newCounter(t)
|
||||
c.Planner.WarnPercent = 80
|
||||
return c, m
|
||||
}
|
||||
|
||||
func TestPlan_SetsWarnAt(t *testing.T) {
|
||||
p := newPlanner(false)
|
||||
p.WarnPercent = 80
|
||||
plan, err := p.PlanOut("w", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := []int64{plan.Increments[0].WarnAt, plan.Increments[1].WarnAt, plan.Increments[2].WarnAt}
|
||||
if got[0] != 4 || got[1] != 1760 || got[2] != 1760 {
|
||||
t.Errorf("WarnAt = %v, want [4 1760 1760]", got)
|
||||
}
|
||||
p.WarnPercent = 0
|
||||
plan, _ = p.PlanOut("w", "noreply@example.com", "", "ext@gmail.com")
|
||||
if plan.Increments[0].WarnAt != 0 {
|
||||
t.Errorf("WarnAt with warnings off = %d, want 0", plan.Increments[0].WarnAt)
|
||||
}
|
||||
}
|
||||
|
||||
// noreply@example.com has A/o=5, so 80% is the 4th message.
|
||||
func TestCounter_WarnsOnceThenBreachesOnce(t *testing.T) {
|
||||
c, _ := warnCounter(t)
|
||||
ctx := context.Background()
|
||||
for i := 1; i <= 7; i++ {
|
||||
out, err := c.CountOut(ctx, "tr", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := out.Results[0]
|
||||
if a.Warned != (i == 4) {
|
||||
t.Errorf("message %d: warned = %v", i, a.Warned)
|
||||
}
|
||||
if a.Reached != (i == 5) {
|
||||
t.Errorf("message %d: reached = %v", i, a.Reached)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Markers must stay out of the cnt/ namespace: the web app scans cnt/*<address>*.
|
||||
func TestCounter_ConfirmedCrossingWritesMarker(t *testing.T) {
|
||||
c, m := warnCounter(t)
|
||||
ctx := context.Background()
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := c.CountOut(ctx, "tr", "noreply@example.com", "", "ext@gmail.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, kind := range []string{store.MarkerLimit, store.MarkerWarn} {
|
||||
key := store.MarkerKey("A", "noreply@example.com", day, "o", kind)
|
||||
if !strings.HasPrefix(key, "ntf/") {
|
||||
t.Errorf("marker key %q must not share the cnt/ namespace", key)
|
||||
}
|
||||
if !m.Exists(key) {
|
||||
t.Errorf("marker %s not written; keys = %v", key, m.Keys())
|
||||
}
|
||||
if ttl := m.TTL(key); ttl != store.MarkerTTL {
|
||||
t.Errorf("marker %s TTL = %v, want %v", key, ttl, store.MarkerTTL)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// lostReplyCounter counts through a proxy that delays the next EXEC reply past
|
||||
// the client's read timeout, so Redis applies the transaction but the reply is lost.
|
||||
func lostReplyCounter(t *testing.T, m *miniredis.Miniredis) *Counter {
|
||||
t.Helper()
|
||||
proxy := delayProxy(t, m.Addr(), 1, 800*time.Millisecond)
|
||||
st := store.NewWithClients(redis.NewClient(&redis.Options{Addr: proxy, ReadTimeout: 200 * time.Millisecond, PoolSize: 2}), nil)
|
||||
t.Cleanup(func() { st.Close() })
|
||||
p := newPlanner(false)
|
||||
p.WarnPercent = 80
|
||||
return &Counter{Planner: p, Store: st}
|
||||
}
|
||||
|
||||
// A counter that already crossed and notified must stay silent on every later
|
||||
// lost reply.
|
||||
func TestCounter_LostReplyAfterBreachDoesNotNotifyAgain(t *testing.T) {
|
||||
c, m := warnCounter(t)
|
||||
ctx := context.Background()
|
||||
for i := 0; i < 5; i++ {
|
||||
if _, err := c.CountOut(ctx, "tr", "noreply@example.com", "", "ext@gmail.com"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
out, err := lostReplyCounter(t, m).CountOut(ctx, "lost", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Unknown == nil {
|
||||
t.Fatal("reply should have been lost")
|
||||
}
|
||||
if a := out.Results[0]; a.Count != 6 || a.Reached || a.Warned {
|
||||
t.Errorf("A = %+v, want count 6, no reached, no warned", a)
|
||||
}
|
||||
}
|
||||
|
||||
// When the crossing increment itself loses its reply it notifies exactly once,
|
||||
// however many later replies are lost too.
|
||||
func TestCounter_LostReplyOnCrossingNotifiesOnce(t *testing.T) {
|
||||
m := miniredis.RunT(t)
|
||||
key := store.Key("A", "noreply@example.com", day, "o")
|
||||
m.Set(key, "4") // A/o=5: the next increment crosses
|
||||
ctx := context.Background()
|
||||
reached := 0
|
||||
for i := 0; i < 2; i++ {
|
||||
out, err := lostReplyCounter(t, m).CountOut(ctx, "lost", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Unknown == nil {
|
||||
t.Fatalf("round %d: reply should have been lost", i)
|
||||
}
|
||||
if out.Results[0].Reached {
|
||||
reached++
|
||||
}
|
||||
}
|
||||
if reached != 1 {
|
||||
t.Errorf("reached %d times over two lost replies, want 1", reached)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCounter_LostReplyWarnsOnce(t *testing.T) {
|
||||
m := miniredis.RunT(t)
|
||||
key := store.Key("A", "noreply@example.com", day, "o")
|
||||
m.Set(key, "3") // 80% of A/o=5 is 4: the next increment warns
|
||||
ctx := context.Background()
|
||||
out, err := lostReplyCounter(t, m).CountOut(ctx, "lost", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := out.Results[0]; a.Count != 4 || !a.Warned || a.Reached {
|
||||
t.Errorf("A = %+v, want count 4 warned", a)
|
||||
}
|
||||
m.Set(key, "3") // second lost reply lands on the same value
|
||||
out, err = lostReplyCounter(t, m).CountOut(ctx, "lost", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if a := out.Results[0]; a.Warned {
|
||||
t.Errorf("A = %+v, must not warn twice", a)
|
||||
}
|
||||
}
|
||||
|
||||
// Crossed says a limit of 0 never fires; the lost-reply path must agree.
|
||||
func TestCounter_LostReplyZeroLimitNeverFires(t *testing.T) {
|
||||
m := miniredis.RunT(t)
|
||||
c := lostReplyCounter(t, m)
|
||||
plan := &Plan{Route: RouteOut, Date: day, Increments: []Increment{
|
||||
{Level: "A", ID: "x@example.com", Flow: "o", Key: store.Key("A", "x@example.com", day, "o"), Limited: true, Limit: 0},
|
||||
}}
|
||||
out, err := c.Execute(context.Background(), plan)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if out.Unknown == nil {
|
||||
t.Fatal("reply should have been lost")
|
||||
}
|
||||
if out.Results[0].Reached {
|
||||
t.Errorf("limit 0 fired after a lost reply: %+v", out.Results[0])
|
||||
}
|
||||
}
|
||||
|
||||
// A transaction Redis aborted (READONLY during a failover) applied nothing, so
|
||||
// it is safe to run it once more.
|
||||
func TestCounter_AbortedTransactionIsRetried(t *testing.T) {
|
||||
c, m := newCounter(t)
|
||||
c.AbortRetryDelay = 150 * time.Millisecond
|
||||
m.SetError("READONLY You can't write against a read only replica.")
|
||||
go func() {
|
||||
time.Sleep(30 * time.Millisecond)
|
||||
m.SetError("")
|
||||
}()
|
||||
out, err := c.CountOut(context.Background(), "fo", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err != nil {
|
||||
t.Fatalf("retry should have counted: %v", err)
|
||||
}
|
||||
if out.Unknown != nil || out.Results[0].Err != nil || out.Results[0].Count != 1 {
|
||||
t.Errorf("outcome = %+v unknown=%v", out.Results[0], out.Unknown)
|
||||
}
|
||||
if v, _ := m.Get(store.Key("A", "noreply@example.com", day, "o")); v != "1" {
|
||||
t.Errorf("redis A/o = %q, want exactly 1", v)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCounter_AbortedTwiceIsAWholeFailure(t *testing.T) {
|
||||
c, m := newCounter(t)
|
||||
c.AbortRetryDelay = time.Millisecond
|
||||
m.SetError("READONLY You can't write against a read only replica.")
|
||||
out, err := c.CountOut(context.Background(), "fo", "noreply@example.com", "", "ext@gmail.com")
|
||||
if err == nil || !errors.Is(err, store.ErrNotApplied) {
|
||||
t.Fatalf("err = %v, outcome = %+v; want ErrNotApplied", err, out)
|
||||
}
|
||||
m.SetError("")
|
||||
if keys := m.Keys(); len(keys) != 0 {
|
||||
t.Errorf("nothing should be written, keys = %v", keys)
|
||||
}
|
||||
}
|
||||
|
||||
// A job that waited in the queue across midnight UTC counts on the day it arrived.
|
||||
func TestPlan_DayIsWhenTheRequestArrived(t *testing.T) {
|
||||
p := newPlanner(false)
|
||||
received := fixedNow.Add(-24 * time.Hour)
|
||||
plan, err := p.Plan(Job{Route: RouteIn, Recipient: "y@example.com", Received: received})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if want := store.DayKey(received); plan.Date != want {
|
||||
t.Errorf("plan date = %s, want %s", plan.Date, want)
|
||||
}
|
||||
plan, _ = p.Plan(Job{Route: RouteIn, Recipient: "y@example.com"})
|
||||
if plan.Date != day {
|
||||
t.Errorf("plan date without Received = %s, want %s", plan.Date, day)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user