dimitrije/counters
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 80% limit warning, stop repeat notifications, fix review findings

Browse Source 
Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/server_test.go
+16 -5
View File
@@ -87,6 +87,7 @@ type options struct {
exempt map[string]bool
queueSize int
workers int
warnPercent int64 // > 0: warnings on, sent to warnAddr / warnOrg
}
type harness struct {
@@ -102,6 +103,8 @@ type harness struct {
addrURL *capture
orgURL *capture
domURL *capture
warnAddr *capture
warnOrg *capture
log *syncBuffer
tx *syncBuffer
reg *prometheus.Registry
@@ -154,6 +157,14 @@ func newHarness(t *testing.T, o options) *harness {
if o.domainURL {
ncfg.DomainURL = sd.URL + q
}
h.warnAddr, h.warnOrg = &capture{}, &capture{}
if o.warnPercent > 0 {
swa, swo := httptest.NewServer(h.warnAddr), httptest.NewServer(h.warnOrg)
t.Cleanup(swa.Close)
t.Cleanup(swo.Close)
wq := "/api/limits/warning?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT&limit=$LIMIT&percent=$PERCENT"
ncfg.WarnAddressURL, ncfg.WarnOrgURL = swa.URL+wq, swo.URL+wq
}
h.reg = prometheus.NewRegistry()
h.m = metrics.New(h.reg)
@@ -164,7 +175,7 @@ func newHarness(t *testing.T, o options) *harness {
logger := slog.New(slog.NewTextHandler(h.log, &slog.HandlerOptions{Level: slog.LevelDebug}))
planner := &counting.Planner{
Lookup: counting.LookupFunc(func(d string) (*domains.Domain, error) { return (*h.lookup.Load())(d) }),
Exempt: o.exempt, Now: func() time.Time { return clock },
Exempt: o.exempt, Now: func() time.Time { return clock }, WarnPercent: o.warnPercent,
}
s := New(Deps{
AuthToken: o.token,
@@ -349,8 +360,8 @@ func TestOut_ReachedNotifiesConfiguredLevelsOnce(t *testing.T) {
t.Errorf("transaction log lacks %q:\n%s", want, tx)
}
}
if h.metric(h.m.LimitReached.WithLabelValues("A", "o")) != 1 || h.metric(h.m.Notifications.WithLabelValues("A", "sent")) != 1 ||
h.metric(h.m.Notifications.WithLabelValues("D", "disabled")) != 1 {
if h.metric(h.m.LimitReached.WithLabelValues("A", "o")) != 1 || h.metric(h.m.Notifications.WithLabelValues("limit", "A", "sent")) != 1 ||
h.metric(h.m.Notifications.WithLabelValues("limit", "D", "disabled")) != 1 {
t.Error("limit/notification metrics not recorded")
}
}
@@ -569,7 +580,7 @@ func TestOut_LoginSuffixCountsBaseAddress(t *testing.T) {
if h.redisInt(store.Key("A", "x@yagolivas.com", day, "o")) != 1 || h.redisInt(store.Key("A", "y@yagolivas.com", day, "i")) != 1 {
t.Errorf("base address not counted: %v", h.write.Keys())
}
if !strings.Contains(h.tx.String(), "<m1> out; request auth_user=x@yagolivas.com recipient=y@yagolivas.com mail_from=x@yagolivas.com login_suffix=SUPPORTTOKEN") {
if !strings.Contains(h.tx.String(), "<m1> out; request auth_user=x@yagolivas.com recipient=y@yagolivas.com mail_from=x@yagolivas.com token_login=true") {
t.Errorf("transaction log:\n%s", h.tx.String())
}
}
@@ -1046,7 +1057,7 @@ func TestNotifyHooks(t *testing.T) {
ev := notify.Event{TraceID: "t", Route: "out", Level: "O"}
hooks.Sent(ev, "http://x", 1)
hooks.Failed(ev, "http://x", 5, io.EOF)
if testutil.ToFloat64(m.Notifications.WithLabelValues("O", "sent")) != 1 || testutil.ToFloat64(m.Notifications.WithLabelValues("O", "failed")) != 1 {
if testutil.ToFloat64(m.Notifications.WithLabelValues("limit", "O", "sent")) != 1 || testutil.ToFloat64(m.Notifications.WithLabelValues("limit", "O", "failed")) != 1 {
t.Error("hooks did not record metrics")
}
if s := buf.String(); !strings.Contains(s, "sent limit exceeded") || !strings.Contains(s, "failed limit exceeded") {