Add 80% limit warning, stop repeat notifications, fix review findings
Notifications: - Warn once when a counter reaches warn_percent of its limit, through separate warn_*_url endpoints. - Write ntf/ markers so a lost Redis reply cannot re-send a breach. - Retry only when the request never left or the web app answered 429/5xx. Fixes: - Retry transactions Redis rejected; treat pool timeouts as definite. - Install the signal handler before startup work. - Stop logging the login suffix; redact notify URL credentials. - Reject corrupt or empty CDBs, truncated hash tables included. - Require a Sentinel password, refuse example placeholders, and default admin_addr to localhost. - Normalise /counters domains, count on the arrival day, log rejections, hide internals in errors and exit 1 on a failed drain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
c43ff17627
commit
54f9626121
+110
-40
@@ -12,55 +12,85 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Event kinds: a limit breach, or the early warning before it.
|
||||
const (
|
||||
KindLimit = "limit"
|
||||
KindWarning = "warning"
|
||||
)
|
||||
|
||||
// Config holds the notification settings.
|
||||
type Config struct {
|
||||
OrgURL string // $LEVEL=O
|
||||
AddressURL string // $LEVEL=A
|
||||
DomainURL string // $LEVEL=D
|
||||
User string // basic auth; empty = no Authorization header
|
||||
Password string
|
||||
|
||||
Timeout time.Duration // per attempt (default 5s)
|
||||
// Early-warning URLs, per level. They must not be the breach URLs: the
|
||||
// web app treats every call to those as a breach.
|
||||
WarnOrgURL string
|
||||
WarnAddressURL string
|
||||
WarnDomainURL string
|
||||
|
||||
User string // basic auth; empty = no Authorization header
|
||||
Password string
|
||||
|
||||
Timeout time.Duration // per attempt (default 15s)
|
||||
Attempts int // total attempts (default 5)
|
||||
Backoff time.Duration // first retry delay, doubled each retry (default 1s)
|
||||
MaxConcurrent int // notifications in flight (default 8)
|
||||
}
|
||||
|
||||
// URLFor returns the template for a level, "" when that level is disabled.
|
||||
func (c Config) URLFor(level string) string {
|
||||
// URLFor returns the template for a kind and level, "" when disabled.
|
||||
func (c Config) URLFor(kind, level string) string {
|
||||
warn := kind == KindWarning
|
||||
switch level {
|
||||
case "O":
|
||||
if warn {
|
||||
return c.WarnOrgURL
|
||||
}
|
||||
return c.OrgURL
|
||||
case "A":
|
||||
if warn {
|
||||
return c.WarnAddressURL
|
||||
}
|
||||
return c.AddressURL
|
||||
case "D":
|
||||
if warn {
|
||||
return c.WarnDomainURL
|
||||
}
|
||||
return c.DomainURL
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// URLName returns the config key that serves a level, for log lines.
|
||||
func URLName(level string) string {
|
||||
// URLName returns the config key that serves a kind and level, for log lines.
|
||||
func URLName(kind, level string) string {
|
||||
prefix := ""
|
||||
if kind == KindWarning {
|
||||
prefix = "warn_"
|
||||
}
|
||||
switch level {
|
||||
case "O":
|
||||
return "org_url"
|
||||
return prefix + "org_url"
|
||||
case "A":
|
||||
return "address_url"
|
||||
return prefix + "address_url"
|
||||
case "D":
|
||||
return "domain_url"
|
||||
return prefix + "domain_url"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Event is one "limit reached" occurrence.
|
||||
// Event is one "limit reached" or early-warning occurrence.
|
||||
type Event struct {
|
||||
Kind string // KindLimit (default) or KindWarning
|
||||
TraceID string
|
||||
Route string // in / out, for log correlation
|
||||
Level string // A, D or O
|
||||
@@ -69,6 +99,15 @@ type Event struct {
|
||||
Address string // the address that triggered it ($ADDRESS / by=)
|
||||
Count int64 // counter value after the increment ($COUNT)
|
||||
Limit int64 // configured limit ($LIMIT)
|
||||
Percent int64 // warning threshold in percent ($PERCENT)
|
||||
}
|
||||
|
||||
// KindName returns the event's kind, defaulting to a limit breach.
|
||||
func (e Event) KindName() string {
|
||||
if e.Kind == "" {
|
||||
return KindLimit
|
||||
}
|
||||
return e.Kind
|
||||
}
|
||||
|
||||
// BuildURL fills the template placeholders with URL-escaped event values.
|
||||
@@ -81,6 +120,7 @@ func BuildURL(tmpl string, ev Event) string {
|
||||
"$ID", esc(ev.ID),
|
||||
"$FLOW", esc(ev.Flow),
|
||||
"$PERIOD", "d",
|
||||
"$PERCENT", strconv.FormatInt(ev.Percent, 10),
|
||||
"$ADDRESS", esc(ev.Address),
|
||||
"$COUNT", strconv.FormatInt(ev.Count, 10),
|
||||
).Replace(tmpl)
|
||||
@@ -102,14 +142,15 @@ type Notifier struct {
|
||||
wg sync.WaitGroup
|
||||
quit chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
quitOnce sync.Once
|
||||
}
|
||||
|
||||
// New applies defaults and returns a ready Notifier.
|
||||
func New(cfg Config, hooks Hooks) *Notifier {
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 5 * time.Second
|
||||
cfg.Timeout = 15 * time.Second
|
||||
}
|
||||
if cfg.Attempts <= 0 {
|
||||
cfg.Attempts = 5
|
||||
@@ -132,7 +173,7 @@ func New(cfg Config, hooks Hooks) *Notifier {
|
||||
// Notify dispatches ev in the background. It returns false, without doing
|
||||
// anything, when no URL is configured for the level or after Shutdown.
|
||||
func (n *Notifier) Notify(ev Event) bool {
|
||||
tmpl := n.cfg.URLFor(ev.Level)
|
||||
tmpl := n.cfg.URLFor(ev.KindName(), ev.Level)
|
||||
if tmpl == "" {
|
||||
return false
|
||||
}
|
||||
@@ -147,57 +188,87 @@ func (n *Notifier) Notify(ev Event) bool {
|
||||
u := BuildURL(tmpl, ev)
|
||||
go func() {
|
||||
defer n.wg.Done()
|
||||
n.sem <- struct{}{}
|
||||
defer func() { <-n.sem }()
|
||||
n.send(ev, u)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// send delivers ev, holding a concurrency slot only while a request is in
|
||||
// flight, never during a backoff.
|
||||
func (n *Notifier) send(ev Event, u string) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= n.cfg.Attempts; attempt++ {
|
||||
attempts := 0
|
||||
for attempts < n.cfg.Attempts {
|
||||
if attempts > 0 && !n.wait(n.cfg.Backoff<<(attempts-1)) {
|
||||
lastErr = fmt.Errorf("shutting down after attempt %d: %w", attempts, lastErr)
|
||||
break
|
||||
}
|
||||
select {
|
||||
case n.sem <- struct{}{}:
|
||||
case <-n.quit:
|
||||
n.failed(ev, u, attempts, errors.New("shutting down before the request was sent"))
|
||||
return
|
||||
}
|
||||
err, retry := n.attempt(u)
|
||||
<-n.sem
|
||||
attempts++
|
||||
if err == nil {
|
||||
if n.hooks.Sent != nil {
|
||||
n.hooks.Sent(ev, u, attempt)
|
||||
n.hooks.Sent(ev, u, attempts)
|
||||
}
|
||||
return
|
||||
}
|
||||
lastErr = err
|
||||
if !retry {
|
||||
if n.hooks.Failed != nil {
|
||||
n.hooks.Failed(ev, u, attempt, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if attempt < n.cfg.Attempts {
|
||||
select {
|
||||
case <-time.After(n.cfg.Backoff << (attempt - 1)):
|
||||
case <-n.quit:
|
||||
lastErr = fmt.Errorf("shutting down after attempt %d: %w", attempt, err)
|
||||
attempt = n.cfg.Attempts // stop retrying
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
n.failed(ev, u, attempts, lastErr)
|
||||
}
|
||||
|
||||
func (n *Notifier) failed(ev Event, u string, attempts int, err error) {
|
||||
if n.hooks.Failed != nil {
|
||||
n.hooks.Failed(ev, u, n.cfg.Attempts, lastErr)
|
||||
n.hooks.Failed(ev, u, attempts, err)
|
||||
}
|
||||
}
|
||||
|
||||
// attempt performs one GET. retry is false for 4xx: wrong credentials or a
|
||||
// wrong URL will not fix themselves.
|
||||
// wait sleeps for d and reports false when the shutdown deadline passed first.
|
||||
func (n *Notifier) wait(d time.Duration) bool {
|
||||
select {
|
||||
case <-time.After(d):
|
||||
return true
|
||||
case <-n.quit:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// attempt performs one GET. retry is true only when a repeat cannot duplicate
|
||||
// the notification or the web app said it failed: the request never left, or
|
||||
// the answer was 429 or 5xx. A request that was sent but got no answer may
|
||||
// have been acted on, and 4xx (wrong credentials, wrong URL) will not fix
|
||||
// itself.
|
||||
func (n *Notifier) attempt(u string) (err error, retry bool) {
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err, false
|
||||
}
|
||||
var wrote atomic.Bool
|
||||
req = req.WithContext(httptrace.WithClientTrace(req.Context(), &httptrace.ClientTrace{
|
||||
WroteRequest: func(info httptrace.WroteRequestInfo) {
|
||||
if info.Err == nil {
|
||||
wrote.Store(true)
|
||||
}
|
||||
},
|
||||
}))
|
||||
if n.cfg.User != "" {
|
||||
req.SetBasicAuth(n.cfg.User, n.cfg.Password)
|
||||
}
|
||||
req.Header.Set("User-Agent", "migadu_counters")
|
||||
resp, err := n.client.Do(req)
|
||||
if err != nil {
|
||||
if wrote.Load() {
|
||||
return fmt.Errorf("outcome unknown, request was sent: %w", err), false
|
||||
}
|
||||
return err, true
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
@@ -212,14 +283,12 @@ func (n *Notifier) attempt(u string) (err error, retry bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown refuses new events, stops pending retries and waits for in-flight
|
||||
// requests until ctx expires. Safe to call more than once.
|
||||
// Shutdown refuses new events and waits for in-flight notifications and their
|
||||
// pending retries until ctx expires; then remaining retries are abandoned.
|
||||
// Safe to call more than once.
|
||||
func (n *Notifier) Shutdown(ctx context.Context) error {
|
||||
n.mu.Lock()
|
||||
if !n.closed {
|
||||
n.closed = true
|
||||
close(n.quit)
|
||||
}
|
||||
n.closed = true
|
||||
n.mu.Unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
@@ -231,6 +300,7 @@ func (n *Notifier) Shutdown(ctx context.Context) error {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
n.quitOnce.Do(func() { close(n.quit) })
|
||||
return errors.Join(ctx.Err(), errors.New("notifications still in flight"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,10 +36,15 @@ func TestBuildURL(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestURLFor(t *testing.T) {
|
||||
cfg := Config{OrgURL: "o", AddressURL: "a", DomainURL: "d"}
|
||||
cfg := Config{OrgURL: "o", AddressURL: "a", DomainURL: "d", WarnOrgURL: "wo", WarnAddressURL: "wa", WarnDomainURL: "wd"}
|
||||
for level, want := range map[string]string{"O": "o", "A": "a", "D": "d", "X": ""} {
|
||||
if got := cfg.URLFor(level); got != want {
|
||||
t.Errorf("URLFor(%s) = %q, want %q", level, got, want)
|
||||
if got := cfg.URLFor(KindLimit, level); got != want {
|
||||
t.Errorf("URLFor(limit, %s) = %q, want %q", level, got, want)
|
||||
}
|
||||
}
|
||||
for level, want := range map[string]string{"O": "wo", "A": "wa", "D": "wd", "X": ""} {
|
||||
if got := cfg.URLFor(KindWarning, level); got != want {
|
||||
t.Errorf("URLFor(warning, %s) = %q, want %q", level, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -260,20 +265,146 @@ func TestNotify_NoRetryOn4xx(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotify_TimeoutIsRetried(t *testing.T) {
|
||||
// The request reached the web app and only the answer is missing: it may have
|
||||
// acted, so a retry could send the customer a second email.
|
||||
func TestNotify_TimeoutAfterWriteIsNotRetried(t *testing.T) {
|
||||
c := &capture{delay: 300 * time.Millisecond}
|
||||
s := httptest.NewServer(c)
|
||||
defer s.Close()
|
||||
h := newHooks()
|
||||
cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
|
||||
cfg.Timeout = 50 * time.Millisecond
|
||||
cfg.Attempts = 2
|
||||
n := New(cfg, h.Hooks())
|
||||
n.Notify(Event{Level: "O", ID: "7"})
|
||||
h.wait(t, 1)
|
||||
n.Shutdown(context.Background())
|
||||
if len(h.failed) != 1 || h.failed[0].attempts != 2 {
|
||||
t.Errorf("failed=%+v; want failure after 2 timed-out attempts", h.failed)
|
||||
if len(h.failed) != 1 || h.failed[0].attempts != 1 {
|
||||
t.Fatalf("failed=%+v; want one failure after a single attempt", h.failed)
|
||||
}
|
||||
if !strings.Contains(h.failed[0].err.Error(), "outcome unknown") {
|
||||
t.Errorf("error should say the outcome is unknown: %v", h.failed[0].err)
|
||||
}
|
||||
waitCalls(t, c, 1)
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if c.calls.Load() != 1 {
|
||||
t.Errorf("web app calls = %d, want 1", c.calls.Load())
|
||||
}
|
||||
}
|
||||
|
||||
// Nothing was delivered, so retrying cannot duplicate anything.
|
||||
func TestNotify_RefusedConnectionIsRetried(t *testing.T) {
|
||||
s := httptest.NewServer(&capture{})
|
||||
addr := s.URL
|
||||
s.Close()
|
||||
h := newHooks()
|
||||
cfg := fastConfig(addr+"/l?level=$LEVEL", "", "")
|
||||
cfg.Attempts = 3
|
||||
n := New(cfg, h.Hooks())
|
||||
n.Notify(Event{Level: "O", ID: "7"})
|
||||
h.wait(t, 1)
|
||||
n.Shutdown(context.Background())
|
||||
if len(h.failed) != 1 || h.failed[0].attempts != 3 {
|
||||
t.Errorf("failed=%+v; want failure after 3 attempts", h.failed)
|
||||
}
|
||||
}
|
||||
|
||||
// A notification waiting out its backoff must not hold a concurrency slot.
|
||||
func TestNotify_BackoffReleasesSlot(t *testing.T) {
|
||||
c := &capture{status: []int{500, 200, 200}}
|
||||
s := httptest.NewServer(c)
|
||||
defer s.Close()
|
||||
h := newHooks()
|
||||
cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
|
||||
cfg.MaxConcurrent = 1
|
||||
cfg.Backoff = 400 * time.Millisecond
|
||||
n := New(cfg, h.Hooks())
|
||||
defer n.Shutdown(context.Background())
|
||||
n.Notify(Event{Level: "O", ID: "retrying"})
|
||||
waitCalls(t, c, 1)
|
||||
n.Notify(Event{Level: "O", ID: "second"})
|
||||
select {
|
||||
case <-h.done:
|
||||
case <-time.After(250 * time.Millisecond):
|
||||
t.Fatal("the second notification waited for the first one's backoff")
|
||||
}
|
||||
h.done <- struct{}{}
|
||||
h.wait(t, 2)
|
||||
}
|
||||
|
||||
func waitCalls(t *testing.T, c *capture, n int32) {
|
||||
t.Helper()
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for c.calls.Load() < n {
|
||||
if time.Now().After(deadline) {
|
||||
t.Fatalf("calls = %d, want %d", c.calls.Load(), n)
|
||||
}
|
||||
time.Sleep(time.Millisecond)
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown lets a pending retry finish while its deadline allows.
|
||||
func TestNotify_ShutdownFinishesPendingRetries(t *testing.T) {
|
||||
c := &capture{status: []int{500, 200}}
|
||||
s := httptest.NewServer(c)
|
||||
defer s.Close()
|
||||
h := newHooks()
|
||||
cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
|
||||
cfg.Backoff = 100 * time.Millisecond
|
||||
n := New(cfg, h.Hooks())
|
||||
n.Notify(Event{Level: "O", ID: "7"})
|
||||
waitCalls(t, c, 1)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
if err := n.Shutdown(ctx); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(h.sent) != 1 || h.sent[0].attempt != 2 || len(h.failed) != 0 {
|
||||
t.Errorf("sent=%+v failed=%+v; want delivery on attempt 2", h.sent, h.failed)
|
||||
}
|
||||
}
|
||||
|
||||
// When the deadline cuts a retry short the failure reports the real attempt count.
|
||||
func TestNotify_ShutdownDeadlineReportsRealAttempts(t *testing.T) {
|
||||
c := &capture{status: []int{500}}
|
||||
s := httptest.NewServer(c)
|
||||
defer s.Close()
|
||||
h := newHooks()
|
||||
cfg := fastConfig(s.URL+"/l?level=$LEVEL", "", "")
|
||||
cfg.Backoff = 5 * time.Second
|
||||
n := New(cfg, h.Hooks())
|
||||
n.Notify(Event{Level: "O", ID: "7"})
|
||||
waitCalls(t, c, 1)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Millisecond)
|
||||
defer cancel()
|
||||
_ = n.Shutdown(ctx)
|
||||
h.wait(t, 1)
|
||||
if len(h.failed) != 1 || h.failed[0].attempts != 1 {
|
||||
t.Errorf("failed=%+v; want 1 real attempt", h.failed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNotify_WarningUsesWarningURL(t *testing.T) {
|
||||
breach, warn := &capture{}, &capture{}
|
||||
sb, sw := httptest.NewServer(breach), httptest.NewServer(warn)
|
||||
defer sb.Close()
|
||||
defer sw.Close()
|
||||
h := newHooks()
|
||||
cfg := fastConfig(sb.URL+"/l?level=$LEVEL", "", "")
|
||||
cfg.WarnOrgURL = sw.URL + "/w?level=$LEVEL&limit=$LIMIT&percent=$PERCENT"
|
||||
n := New(cfg, h.Hooks())
|
||||
defer n.Shutdown(context.Background())
|
||||
if !n.Notify(Event{Kind: KindWarning, Level: "O", ID: "7", Limit: 100, Count: 80, Percent: 80}) {
|
||||
t.Fatal("warning refused")
|
||||
}
|
||||
h.wait(t, 1)
|
||||
if breach.calls.Load() != 0 || warn.calls.Load() != 1 {
|
||||
t.Fatalf("breach=%d warn=%d, want the warning URL only", breach.calls.Load(), warn.calls.Load())
|
||||
}
|
||||
if q := warn.requests[0].URL.Query(); q.Get("percent") != "80" || q.Get("limit") != "100" {
|
||||
t.Errorf("query = %v", q)
|
||||
}
|
||||
if n.Notify(Event{Kind: KindWarning, Level: "A"}) {
|
||||
t.Error("a level with no warning URL must be refused")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -326,7 +457,7 @@ func TestNotify_ShutdownDrainsAndRespectsContext(t *testing.T) {
|
||||
func TestNew_Defaults(t *testing.T) {
|
||||
n := New(Config{OrgURL: "https://x/?level=$LEVEL"}, Hooks{})
|
||||
defer n.Shutdown(context.Background())
|
||||
if n.cfg.Attempts != 5 || n.cfg.Backoff != time.Second || n.cfg.MaxConcurrent != 8 || n.cfg.Timeout != 5*time.Second {
|
||||
if n.cfg.Attempts != 5 || n.cfg.Backoff != time.Second || n.cfg.MaxConcurrent != 8 || n.cfg.Timeout != 15*time.Second {
|
||||
t.Errorf("defaults not applied: %+v", n.cfg)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user