Notifications: - Warn once when a counter reaches warn_percent of its limit, through separate warn_*_url endpoints. - Write ntf/ markers so a lost Redis reply cannot re-send a breach. - Retry only when the request never left or the web app answered 429/5xx. Fixes: - Retry transactions Redis rejected; treat pool timeouts as definite. - Install the signal handler before startup work. - Stop logging the login suffix; redact notify URL credentials. - Reject corrupt or empty CDBs, truncated hash tables included. - Require a Sentinel password, refuse example placeholders, and default admin_addr to localhost. - Normalise /counters domains, count on the arrival day, log rejections, hide internals in errors and exit 1 on a failed drain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
174 lines
6.2 KiB
Go
174 lines
6.2 KiB
Go
package server
|
|
|
|
import (
|
|
"context"
|
|
"net/url"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/alicebob/miniredis/v2"
|
|
|
|
"migadu_counters/internal/store"
|
|
)
|
|
|
|
// Real CDB: yagolivas.com has A/o=20 and O/o=20, so 80% is 16.
|
|
func TestOut_WarnsAt80PercentOnceAndBreachesOnce(t *testing.T) {
|
|
h := newHarness(t, options{warnPercent: 80})
|
|
for i := 0; i < 22; i++ {
|
|
h.accept("/out", `{"trace_id":"w","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
|
|
}
|
|
if err := h.notifier.Shutdown(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.warnAddr.count() != 1 || h.warnOrg.count() != 1 {
|
|
t.Errorf("warnings addr=%d org=%d, want 1 each", h.warnAddr.count(), h.warnOrg.count())
|
|
}
|
|
if h.addrURL.count() != 1 || h.orgURL.count() != 1 {
|
|
t.Errorf("breaches addr=%d org=%d, want 1 each", h.addrURL.count(), h.orgURL.count())
|
|
}
|
|
u, err := url.Parse(h.warnAddr.urls[0])
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
q := u.Query()
|
|
if u.Path != "/api/limits/warning" || q.Get("count") != "16" || q.Get("limit") != "20" || q.Get("percent") != "80" || q.Get("id") != "a@yagolivas.com" {
|
|
t.Errorf("warning url = %s", h.warnAddr.urls[0])
|
|
}
|
|
if h.metric(h.m.WarningReached.WithLabelValues("A", "o")) != 1 {
|
|
t.Error("warning_reached_total not recorded")
|
|
}
|
|
if h.metric(h.m.Notifications.WithLabelValues("warning", "A", "sent")) != 1 {
|
|
t.Error("notifications_total{warning,A,sent} not recorded")
|
|
}
|
|
if !strings.Contains(h.tx.String(), "warning reached (level=[A] id=[a@yagolivas.com] flow=[o] by=[a@yagolivas.com]) => notifying warn_address_url") {
|
|
t.Errorf("transaction log:\n%s", h.tx.String())
|
|
}
|
|
}
|
|
|
|
func TestOut_ConcurrentIncrementsNotifyOnce(t *testing.T) {
|
|
h := newHarness(t, options{warnPercent: 80, workers: 8})
|
|
var wg sync.WaitGroup
|
|
for i := 0; i < 40; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
h.post("/out", `{"trace_id":"c","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
|
|
}()
|
|
}
|
|
wg.Wait()
|
|
h.settle()
|
|
if err := h.notifier.Shutdown(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.redisInt(store.Key("A", "a@yagolivas.com", day, "o")) != 40 {
|
|
t.Fatalf("counter = %d, want 40", h.redisInt(store.Key("A", "a@yagolivas.com", day, "o")))
|
|
}
|
|
if h.warnAddr.count() != 1 || h.addrURL.count() != 1 {
|
|
t.Errorf("warnings=%d breaches=%d, want 1 each", h.warnAddr.count(), h.addrURL.count())
|
|
}
|
|
}
|
|
|
|
func TestOut_WarningsOffSendsNone(t *testing.T) {
|
|
h := newHarness(t, options{})
|
|
for i := 0; i < 20; i++ {
|
|
h.accept("/out", `{"trace_id":"w","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
|
|
}
|
|
h.notifier.Shutdown(context.Background())
|
|
if h.warnAddr.count() != 0 || h.addrURL.count() != 1 {
|
|
t.Errorf("warnings=%d breaches=%d, want 0 and 1", h.warnAddr.count(), h.addrURL.count())
|
|
}
|
|
if strings.Contains(h.tx.String(), "warning reached") {
|
|
t.Errorf("transaction log mentions a warning:\n%s", h.tx.String())
|
|
}
|
|
}
|
|
|
|
// Once a breach was notified, a lost EXEC reply on a later message sends nothing.
|
|
func TestLostReplyAfterBreachSendsNothing(t *testing.T) {
|
|
backend := miniredis.RunT(t)
|
|
proxy := delayProxy(t, backend.Addr(), 3*time.Second)
|
|
h := newHarness(t, options{writeAddr: proxy})
|
|
key := store.Key("A", "a@yagolivas.com", day, "o") // A/o=20
|
|
backend.Set(key, "40")
|
|
backend.Set(store.MarkerKey("A", "a@yagolivas.com", day, "o", store.MarkerLimit), "1")
|
|
h.accept("/out", `{"trace_id":"lost2","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
|
|
if err := h.notifier.Shutdown(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if h.addrURL.count() != 0 {
|
|
t.Errorf("address notifications = %d, want 0", h.addrURL.count())
|
|
}
|
|
if !strings.Contains(h.tx.String(), "count=41 limit=20 reached=false (re-read)") {
|
|
t.Errorf("transaction log:\n%s", h.tx.String())
|
|
}
|
|
}
|
|
|
|
// --- review round 2 ------------------------------------------------------------
|
|
|
|
func TestOut_LoginSuffixNeverLogged(t *testing.T) {
|
|
h := newHarness(t, options{})
|
|
h.accept("/out", `{"trace_id":"m2","auth_user":"x@yagolivas.com@SECRET\\nFORGED LINE","recipient":"y@yagolivas.com"}`)
|
|
h.accept("/out", `{"trace_id":"m3","auth_user":"x@yagolivas.com@SECRET\nFORGED LINE","recipient":"y@yagolivas.com"}`)
|
|
if tx := h.tx.String(); strings.Contains(tx, "SECRET") || strings.Contains(tx, "FORGED") {
|
|
t.Errorf("transaction log contains the login suffix:\n%s", tx)
|
|
}
|
|
}
|
|
|
|
func TestReject_LeavesTraceInBothLogs(t *testing.T) {
|
|
h := newHarness(t, options{})
|
|
if st, _ := h.post("/in", `{"trace_id":"TRACE-XYZ","recipient":"nobody"}`); st != 400 {
|
|
t.Fatal(st)
|
|
}
|
|
if !strings.Contains(h.log.String(), "TRACE-XYZ") {
|
|
t.Errorf("app log lacks the trace id:\n%s", h.log.String())
|
|
}
|
|
if !strings.Contains(h.tx.String(), "<TRACE-XYZ> in; rejected: 400 ") {
|
|
t.Errorf("counting log lacks the rejection:\n%s", h.tx.String())
|
|
}
|
|
}
|
|
|
|
func TestErrors_DoNotExposeInternals(t *testing.T) {
|
|
h := newHarness(t, options{})
|
|
readAddr, writeAddr := h.read.Addr(), h.write.Addr()
|
|
h.read.Close()
|
|
st, body := h.get("/counters?address=a@yagolivas.com")
|
|
if st != 503 || strings.Contains(body, readAddr) || strings.Contains(body, "dial") {
|
|
t.Errorf("/counters = %d %s", st, body)
|
|
}
|
|
if !strings.Contains(h.log.String(), readAddr) {
|
|
t.Errorf("detail should stay in the app log:\n%s", h.log.String())
|
|
}
|
|
h.write.Close()
|
|
st, body = h.adminGet("/health")
|
|
if st != 503 || strings.Contains(body, writeAddr) || strings.Contains(body, "dial") {
|
|
t.Errorf("/health = %d %s", st, body)
|
|
}
|
|
}
|
|
|
|
func TestPanic_CountedInRequestMetrics(t *testing.T) {
|
|
h := newHarness(t, options{})
|
|
h.srv.panicHook = func() { panic("boom") }
|
|
if st, _ := h.post("/in", `{"recipient":"b@yagolivas.com"}`); st != 500 {
|
|
t.Fatal(st)
|
|
}
|
|
if got := h.metric(h.m.Requests.WithLabelValues("/in", "500")); got != 1 {
|
|
t.Errorf("requests_total{/in,500} = %v, want 1", got)
|
|
}
|
|
}
|
|
|
|
// Real CDB: xn-- domains are hosted under their punycode key.
|
|
func TestCounters_UnicodeDomainIsNormalised(t *testing.T) {
|
|
h := newHarness(t, options{})
|
|
if st, _ := h.get("/counters?domain=host_name.example.com"); st != 400 {
|
|
t.Errorf("invalid domain status = %d, want 400", st)
|
|
}
|
|
st, body := h.get("/counters?domain=" + url.QueryEscape("Bücher.example."))
|
|
if st != 200 {
|
|
t.Fatalf("status %d: %s", st, body)
|
|
}
|
|
if r := decodeCounters(t, body); r.Query != "xn--bcher-kva.example" {
|
|
t.Errorf("query = %q, want the punycode form", r.Query)
|
|
}
|
|
}
|