Add 80% limit warning, stop repeat notifications, fix review findings
Notifications: - Warn once when a counter reaches warn_percent of its limit, through separate warn_*_url endpoints. - Write ntf/ markers so a lost Redis reply cannot re-send a breach. - Retry only when the request never left or the web app answered 429/5xx. Fixes: - Retry transactions Redis rejected; treat pool timeouts as definite. - Install the signal handler before startup work. - Stop logging the login suffix; redact notify URL credentials. - Reject corrupt or empty CDBs, truncated hash tables included. - Require a Sentinel password, refuse example placeholders, and default admin_addr to localhost. - Normalise /counters domains, count on the arrival day, log rejections, hide internals in errors and exit 1 on a failed drain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
c43ff17627
commit
54f9626121
+110
-40
@@ -12,55 +12,85 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptrace"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Event kinds: a limit breach, or the early warning before it.
|
||||
const (
|
||||
KindLimit = "limit"
|
||||
KindWarning = "warning"
|
||||
)
|
||||
|
||||
// Config holds the notification settings.
|
||||
type Config struct {
|
||||
OrgURL string // $LEVEL=O
|
||||
AddressURL string // $LEVEL=A
|
||||
DomainURL string // $LEVEL=D
|
||||
User string // basic auth; empty = no Authorization header
|
||||
Password string
|
||||
|
||||
Timeout time.Duration // per attempt (default 5s)
|
||||
// Early-warning URLs, per level. They must not be the breach URLs: the
|
||||
// web app treats every call to those as a breach.
|
||||
WarnOrgURL string
|
||||
WarnAddressURL string
|
||||
WarnDomainURL string
|
||||
|
||||
User string // basic auth; empty = no Authorization header
|
||||
Password string
|
||||
|
||||
Timeout time.Duration // per attempt (default 15s)
|
||||
Attempts int // total attempts (default 5)
|
||||
Backoff time.Duration // first retry delay, doubled each retry (default 1s)
|
||||
MaxConcurrent int // notifications in flight (default 8)
|
||||
}
|
||||
|
||||
// URLFor returns the template for a level, "" when that level is disabled.
|
||||
func (c Config) URLFor(level string) string {
|
||||
// URLFor returns the template for a kind and level, "" when disabled.
|
||||
func (c Config) URLFor(kind, level string) string {
|
||||
warn := kind == KindWarning
|
||||
switch level {
|
||||
case "O":
|
||||
if warn {
|
||||
return c.WarnOrgURL
|
||||
}
|
||||
return c.OrgURL
|
||||
case "A":
|
||||
if warn {
|
||||
return c.WarnAddressURL
|
||||
}
|
||||
return c.AddressURL
|
||||
case "D":
|
||||
if warn {
|
||||
return c.WarnDomainURL
|
||||
}
|
||||
return c.DomainURL
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// URLName returns the config key that serves a level, for log lines.
|
||||
func URLName(level string) string {
|
||||
// URLName returns the config key that serves a kind and level, for log lines.
|
||||
func URLName(kind, level string) string {
|
||||
prefix := ""
|
||||
if kind == KindWarning {
|
||||
prefix = "warn_"
|
||||
}
|
||||
switch level {
|
||||
case "O":
|
||||
return "org_url"
|
||||
return prefix + "org_url"
|
||||
case "A":
|
||||
return "address_url"
|
||||
return prefix + "address_url"
|
||||
case "D":
|
||||
return "domain_url"
|
||||
return prefix + "domain_url"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Event is one "limit reached" occurrence.
|
||||
// Event is one "limit reached" or early-warning occurrence.
|
||||
type Event struct {
|
||||
Kind string // KindLimit (default) or KindWarning
|
||||
TraceID string
|
||||
Route string // in / out, for log correlation
|
||||
Level string // A, D or O
|
||||
@@ -69,6 +99,15 @@ type Event struct {
|
||||
Address string // the address that triggered it ($ADDRESS / by=)
|
||||
Count int64 // counter value after the increment ($COUNT)
|
||||
Limit int64 // configured limit ($LIMIT)
|
||||
Percent int64 // warning threshold in percent ($PERCENT)
|
||||
}
|
||||
|
||||
// KindName returns the event's kind, defaulting to a limit breach.
|
||||
func (e Event) KindName() string {
|
||||
if e.Kind == "" {
|
||||
return KindLimit
|
||||
}
|
||||
return e.Kind
|
||||
}
|
||||
|
||||
// BuildURL fills the template placeholders with URL-escaped event values.
|
||||
@@ -81,6 +120,7 @@ func BuildURL(tmpl string, ev Event) string {
|
||||
"$ID", esc(ev.ID),
|
||||
"$FLOW", esc(ev.Flow),
|
||||
"$PERIOD", "d",
|
||||
"$PERCENT", strconv.FormatInt(ev.Percent, 10),
|
||||
"$ADDRESS", esc(ev.Address),
|
||||
"$COUNT", strconv.FormatInt(ev.Count, 10),
|
||||
).Replace(tmpl)
|
||||
@@ -102,14 +142,15 @@ type Notifier struct {
|
||||
wg sync.WaitGroup
|
||||
quit chan struct{}
|
||||
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
mu sync.Mutex
|
||||
closed bool
|
||||
quitOnce sync.Once
|
||||
}
|
||||
|
||||
// New applies defaults and returns a ready Notifier.
|
||||
func New(cfg Config, hooks Hooks) *Notifier {
|
||||
if cfg.Timeout <= 0 {
|
||||
cfg.Timeout = 5 * time.Second
|
||||
cfg.Timeout = 15 * time.Second
|
||||
}
|
||||
if cfg.Attempts <= 0 {
|
||||
cfg.Attempts = 5
|
||||
@@ -132,7 +173,7 @@ func New(cfg Config, hooks Hooks) *Notifier {
|
||||
// Notify dispatches ev in the background. It returns false, without doing
|
||||
// anything, when no URL is configured for the level or after Shutdown.
|
||||
func (n *Notifier) Notify(ev Event) bool {
|
||||
tmpl := n.cfg.URLFor(ev.Level)
|
||||
tmpl := n.cfg.URLFor(ev.KindName(), ev.Level)
|
||||
if tmpl == "" {
|
||||
return false
|
||||
}
|
||||
@@ -147,57 +188,87 @@ func (n *Notifier) Notify(ev Event) bool {
|
||||
u := BuildURL(tmpl, ev)
|
||||
go func() {
|
||||
defer n.wg.Done()
|
||||
n.sem <- struct{}{}
|
||||
defer func() { <-n.sem }()
|
||||
n.send(ev, u)
|
||||
}()
|
||||
return true
|
||||
}
|
||||
|
||||
// send delivers ev, holding a concurrency slot only while a request is in
|
||||
// flight, never during a backoff.
|
||||
func (n *Notifier) send(ev Event, u string) {
|
||||
var lastErr error
|
||||
for attempt := 1; attempt <= n.cfg.Attempts; attempt++ {
|
||||
attempts := 0
|
||||
for attempts < n.cfg.Attempts {
|
||||
if attempts > 0 && !n.wait(n.cfg.Backoff<<(attempts-1)) {
|
||||
lastErr = fmt.Errorf("shutting down after attempt %d: %w", attempts, lastErr)
|
||||
break
|
||||
}
|
||||
select {
|
||||
case n.sem <- struct{}{}:
|
||||
case <-n.quit:
|
||||
n.failed(ev, u, attempts, errors.New("shutting down before the request was sent"))
|
||||
return
|
||||
}
|
||||
err, retry := n.attempt(u)
|
||||
<-n.sem
|
||||
attempts++
|
||||
if err == nil {
|
||||
if n.hooks.Sent != nil {
|
||||
n.hooks.Sent(ev, u, attempt)
|
||||
n.hooks.Sent(ev, u, attempts)
|
||||
}
|
||||
return
|
||||
}
|
||||
lastErr = err
|
||||
if !retry {
|
||||
if n.hooks.Failed != nil {
|
||||
n.hooks.Failed(ev, u, attempt, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if attempt < n.cfg.Attempts {
|
||||
select {
|
||||
case <-time.After(n.cfg.Backoff << (attempt - 1)):
|
||||
case <-n.quit:
|
||||
lastErr = fmt.Errorf("shutting down after attempt %d: %w", attempt, err)
|
||||
attempt = n.cfg.Attempts // stop retrying
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
n.failed(ev, u, attempts, lastErr)
|
||||
}
|
||||
|
||||
func (n *Notifier) failed(ev Event, u string, attempts int, err error) {
|
||||
if n.hooks.Failed != nil {
|
||||
n.hooks.Failed(ev, u, n.cfg.Attempts, lastErr)
|
||||
n.hooks.Failed(ev, u, attempts, err)
|
||||
}
|
||||
}
|
||||
|
||||
// attempt performs one GET. retry is false for 4xx: wrong credentials or a
|
||||
// wrong URL will not fix themselves.
|
||||
// wait sleeps for d and reports false when the shutdown deadline passed first.
|
||||
func (n *Notifier) wait(d time.Duration) bool {
|
||||
select {
|
||||
case <-time.After(d):
|
||||
return true
|
||||
case <-n.quit:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// attempt performs one GET. retry is true only when a repeat cannot duplicate
|
||||
// the notification or the web app said it failed: the request never left, or
|
||||
// the answer was 429 or 5xx. A request that was sent but got no answer may
|
||||
// have been acted on, and 4xx (wrong credentials, wrong URL) will not fix
|
||||
// itself.
|
||||
func (n *Notifier) attempt(u string) (err error, retry bool) {
|
||||
req, err := http.NewRequest(http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return err, false
|
||||
}
|
||||
var wrote atomic.Bool
|
||||
req = req.WithContext(httptrace.WithClientTrace(req.Context(), &httptrace.ClientTrace{
|
||||
WroteRequest: func(info httptrace.WroteRequestInfo) {
|
||||
if info.Err == nil {
|
||||
wrote.Store(true)
|
||||
}
|
||||
},
|
||||
}))
|
||||
if n.cfg.User != "" {
|
||||
req.SetBasicAuth(n.cfg.User, n.cfg.Password)
|
||||
}
|
||||
req.Header.Set("User-Agent", "migadu_counters")
|
||||
resp, err := n.client.Do(req)
|
||||
if err != nil {
|
||||
if wrote.Load() {
|
||||
return fmt.Errorf("outcome unknown, request was sent: %w", err), false
|
||||
}
|
||||
return err, true
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
@@ -212,14 +283,12 @@ func (n *Notifier) attempt(u string) (err error, retry bool) {
|
||||
}
|
||||
}
|
||||
|
||||
// Shutdown refuses new events, stops pending retries and waits for in-flight
|
||||
// requests until ctx expires. Safe to call more than once.
|
||||
// Shutdown refuses new events and waits for in-flight notifications and their
|
||||
// pending retries until ctx expires; then remaining retries are abandoned.
|
||||
// Safe to call more than once.
|
||||
func (n *Notifier) Shutdown(ctx context.Context) error {
|
||||
n.mu.Lock()
|
||||
if !n.closed {
|
||||
n.closed = true
|
||||
close(n.quit)
|
||||
}
|
||||
n.closed = true
|
||||
n.mu.Unlock()
|
||||
|
||||
done := make(chan struct{})
|
||||
@@ -231,6 +300,7 @@ func (n *Notifier) Shutdown(ctx context.Context) error {
|
||||
case <-done:
|
||||
return nil
|
||||
case <-ctx.Done():
|
||||
n.quitOnce.Do(func() { close(n.quit) })
|
||||
return errors.Join(ctx.Err(), errors.New("notifications still in flight"))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user