dimitrije/counters
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 80% limit warning, stop repeat notifications, fix review findings

Browse Source 
Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/txlog/txlog.go
+52 -11
View File
@@ -17,6 +17,7 @@ import (
"errors"
"fmt"
"io"
"net/url"
"sort"
"strings"
"time"
@@ -102,8 +103,12 @@ func (l *Logger) Outcome(out *counting.Outcome, dispatched []bool) {
b.tracked(r, out.Unknown != nil)
}
for i, r := range out.Results {
sent := i < len(dispatched) && dispatched[i]
if r.Reached {
b.reached(r, i < len(dispatched) && dispatched[i])
b.reached(r, sent)
}
if r.Warned {
b.warned(r, sent)
}
}
if out.Skipped != "" {
@@ -128,6 +133,13 @@ func (l *Logger) Dropped(route, traceID string) {
b.flush()
}
// Rejected logs a request refused by the validation layer.
func (l *Logger) Rejected(route, traceID string, status int, reason string) {
b := l.begin(traceID, route)
b.line("rejected", fmt.Sprintf("rejected: %d %s (not counted)", status, reason), map[string]any{"status": status, "reason": reason})
b.flush()
}
// Error logs a request that failed before a plan existed (domain lookup error).
func (l *Logger) Error(route, traceID string, err error) {
b := l.begin(traceID, route)
@@ -136,28 +148,39 @@ func (l *Logger) Error(route, traceID string, err error) {
}
// NotificationSent logs a delivered web app notification (notify hook).
func (l *Logger) NotificationSent(ev notify.Event, url string, attempt int) {
func (l *Logger) NotificationSent(ev notify.Event, rawURL string, attempt int) {
u := redact(rawURL)
b := l.begin(ev.TraceID, ev.Route)
b.line("notification_sent", fmt.Sprintf("sent limit exceeded notification to url %q", url),
map[string]any{"url": url, "attempt": attempt, "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.line("notification_sent", fmt.Sprintf("sent %s notification to url %q", kindWords(ev), u),
map[string]any{"url": u, "attempt": attempt, "kind": ev.KindName(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.flush()
}
// NotificationFailed logs a notification given up on (notify hook).
func (l *Logger) NotificationFailed(ev notify.Event, url string, attempts int, err error) {
func (l *Logger) NotificationFailed(ev notify.Event, rawURL string, attempts int, err error) {
u := redact(rawURL)
b := l.begin(ev.TraceID, ev.Route)
b.line("notification_failed", fmt.Sprintf("failed limit exceeded notification to url %q after %d attempts: %v", url, attempts, err),
map[string]any{"url": url, "attempts": attempts, "error": err.Error(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.line("notification_failed", fmt.Sprintf("failed %s notification to url %q after %d attempts: %v", kindWords(ev), u, attempts, err),
map[string]any{"url": u, "attempts": attempts, "error": err.Error(), "kind": ev.KindName(), "level": ev.Level, "id": ev.ID, "flow": ev.Flow})
b.flush()
}
// redact hides a password embedded in a notification URL.
func redact(raw string) string {
u, err := url.Parse(raw)
if err != nil {
return "<unparseable url>"
}
return u.Redacted()
}
func (b *block) request(p *counting.Plan) {
if p.Route == counting.RouteOut {
msg := fmt.Sprintf("request auth_user=%s recipient=%s mail_from=%s", p.AuthUser, p.Recipient, p.MailFrom)
fields := map[string]any{"auth_user": p.AuthUser, "recipient": p.Recipient, "mail_from": p.MailFrom}
if p.LoginSuffix != "" {
msg += " login_suffix=" + p.LoginSuffix
fields["login_suffix"] = p.LoginSuffix
if p.TokenLogin {
msg += " token_login=true"
fields["token_login"] = true
}
b.line("request", msg, fields)
} else {
@@ -235,7 +258,7 @@ func (b *block) tracked(r counting.Result, reread bool) {
}
func (b *block) reached(r counting.Result, dispatched bool) {
name := notify.URLName(r.Level)
name := notify.URLName(notify.KindLimit, r.Level)
verdict := "notifying " + name
if !dispatched {
verdict = "no " + name + " configured, not notifying"
@@ -245,6 +268,24 @@ func (b *block) reached(r counting.Result, dispatched bool) {
map[string]any{"level": r.Level, "id": r.ID, "flow": r.Flow, "by": r.By, "count": r.Count, "limit": r.Limit, "notifying": dispatched, "url_name": name})
}
func (b *block) warned(r counting.Result, dispatched bool) {
name := notify.URLName(notify.KindWarning, r.Level)
verdict := "notifying " + name
if !dispatched {
verdict = "no " + name + " configured, not notifying"
}
b.line("warning_reached",
fmt.Sprintf("warning reached (level=[%s] id=[%s] flow=[%s] by=[%s]) => %s", r.Level, r.ID, r.Flow, r.By, verdict),
map[string]any{"level": r.Level, "id": r.ID, "flow": r.Flow, "by": r.By, "count": r.Count, "warn_at": r.WarnAt, "limit": r.Limit, "notifying": dispatched, "url_name": name})
}
func kindWords(ev notify.Event) string {
if ev.KindName() == notify.KindWarning {
return "limit warning"
}
return "limit exceeded"
}
// summarize picks the limit keys worth printing for one address: the base
// keys and the address's own overrides. Some domains carry thousands of
// per-mailbox overrides (boxmailhere.com: 9 798); dumping them all would make
internal/txlog/txlog_test.go
+21 -3
View File
@@ -295,7 +295,7 @@ func TestErrorLine(t *testing.T) {
// --- review round 1 lines -----------------------------------------------------
func TestRequestLine_LoginSuffixAndUnusable(t *testing.T) {
plan, err := planner().Plan(counting.Job{Route: counting.RouteOut, TraceID: "m1", AuthUser: "x@wearemiraclesnow.com", LoginSuffix: "SUPPORTTOKEN",
plan, err := planner().Plan(counting.Job{Route: counting.RouteOut, TraceID: "m1", AuthUser: "x@wearemiraclesnow.com", TokenLogin: true,
MailFrom: "x@wearemiraclesnow.com", MailFromUsable: true,
RecipientUnusable: counting.Unusable{Role: "recipient", Value: "u@[192.0.2.1]", Reason: "invalid e-mail address"}})
if err != nil {
@@ -304,7 +304,7 @@ func TestRequestLine_LoginSuffixAndUnusable(t *testing.T) {
l, buf := newLogger("text")
l.Outcome(outcome(plan, 1, 1, 1), nil)
got := buf.String()
if !strings.Contains(got, ts+" <m1> out; request auth_user=x@wearemiraclesnow.com recipient= mail_from=x@wearemiraclesnow.com login_suffix=SUPPORTTOKEN\n") {
if !strings.Contains(got, ts+" <m1> out; request auth_user=x@wearemiraclesnow.com recipient= mail_from=x@wearemiraclesnow.com token_login=true\n") {
t.Errorf("request line should show the login suffix:\n%s", got)
}
if !strings.Contains(got, ts+" <m1> out; recipient address not usable: u@[192.0.2.1] (invalid e-mail address, not counted)\n") {
@@ -318,7 +318,7 @@ func TestRequestLine_LoginSuffixAndUnusable(t *testing.T) {
SenderUnusable: counting.Unusable{Role: "sender", Reason: "missing"}})
l, buf = newLogger("text")
l.Outcome(outcome(plan, 1, 1, 1), nil)
if got := buf.String(); !strings.Contains(got, " <m2> out; sender address not usable: <empty> (missing, not counted)\n") || strings.Contains(got, "login_suffix") {
if got := buf.String(); !strings.Contains(got, " <m2> out; sender address not usable: <empty> (missing, not counted)\n") || strings.Contains(got, "token_login") {
t.Errorf("missing sender line / unexpected suffix:\n%s", got)
}
@@ -411,3 +411,21 @@ func (m malformedTable) Lookup(d string) (*domains.Domain, error) {
}
return m.table.Lookup(d)
}
func TestNotificationLines_RedactCredentials(t *testing.T) {
l, buf := newLogger("text")
ev := notify.Event{TraceID: "t", Route: "out", Level: "O"}
l.NotificationSent(ev, "https://api:s3cret@admin.example/api/limits?level=O", 1)
l.NotificationFailed(ev, "https://api:s3cret@admin.example/api/limits?level=O", 2, errors.New("boom"))
if got := buf.String(); strings.Contains(got, "s3cret") || !strings.Contains(got, "api:xxxxx@admin.example/api/limits?level=O") {
t.Errorf("log:\n%s", got)
}
}
func TestRejected(t *testing.T) {
l, buf := newLogger("text")
l.Rejected("in", "tr9", 400, "recipient: missing")
if got, want := buf.String(), ts+" <tr9> in; rejected: 400 recipient: missing (not counted)\n"; got != want {
t.Errorf("got %q, want %q", got, want)
}
}