First commit Go counter service

2026-10-15 02:12:18 +02:00
commit c43ff17627
59 changed files with 11205 additions and 0 deletions
@@ -0,0 +1,60 @@
+package main
+
+import (
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+
+	"migadu_counters/internal/config"
+	"migadu_counters/internal/domains"
+	"migadu_counters/internal/server"
+	"migadu_counters/internal/store"
+)
+
+// Date exactly 31 days back: its keys are still alive (TTL refreshed by the
+// day's last increment, so they live until ~23:59 today) but counterctl claims
+// it is "more than 31 days ago" and "will read as 0".
+func TestReviewB_31DayWarningOffByOne(t *testing.T) {
+	f := &fakeAPI{body: sampleResponse()}
+	s := httptest.NewServer(f)
+	defer s.Close()
+	cfg := writeConfig(t, "auth_token = \"t\"\n")
+	code, _, errOut := runCLI(t, noEnv, "--config", cfg, "--url", s.URL, "--address", "me@example.com", "--date", "2026-09-12")
+	t.Logf("now=%s --date 2026-09-12 (31 days back) exit=%d stderr=%q", now, code, errOut)
+	lastIncr := time.Date(2026, 9, 12, 23, 59, 0, 0, time.UTC)
+	t.Logf("a key last incremented at %s expires at %s (still alive now)", lastIncr, lastIncr.Add(store.TTL))
+	if strings.Contains(errOut, "will read as 0") {
+		t.Errorf("warning fires for a date whose keys still exist")
+	}
+}
+
+// --url with a trailing slash, against the real API handler.
+func TestReviewB_URLTrailingSlash(t *testing.T) {
+	mr := miniredis.RunT(t)
+	st, err := store.New(config.RedisConfig{WriteAddr: mr.Addr(), PoolSize: 2, TimeoutMS: 1000})
+	if err != nil {
+		t.Fatal(err)
+	}
+	db, err := domains.Open("../../test_files/rspamd_domains_db.cdb")
+	if err != nil {
+		t.Fatal(err)
+	}
+	srv := server.New(server.Deps{AuthToken: "t", Store: st, Domains: func() *domains.DB { return db }, Now: func() time.Time { return now }})
+	s := httptest.NewServer(srv.Handler())
+	defer s.Close()
+	cfg := writeConfig(t, "auth_token = \"t\"\n")
+	for _, u := range []string{s.URL, s.URL + "/", s.URL + "/api"} {
+		code, out, errOut := runCLI(t, noEnv, "--config", cfg, "--url", u, "--address", "a@yagolivas.com")
+		t.Logf("--url %q -> exit %d stdout=%q stderr=%q", u, code, firstLine(out), strings.TrimSpace(errOut))
+	}
+}
+
+func firstLine(s string) string {
+	if i := strings.IndexByte(s, '\n'); i >= 0 {
+		return s[:i]
+	}
+	return s
+}
@@ -0,0 +1,45 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestReviewB_MisconfigStarts(t *testing.T) {
+	dir := t.TempDir()
+	for name, body := range map[string]string{
+		"typo token, open API": `auth-token = "s3cret"
+domains_cdb = "x.cdb"
+allowed_cidr = ["10.0.0.0/8"]
+[redis]
+write_addr = "127.0.0.1:6379"`,
+		"sentinel without password": `auth_token = "t"
+domains_cdb = "x.cdb"
+[redis]
+master_name = "mymaster"
+sentinels = ["10.0.0.10:26379"]`,
+		"notify creds in URL / http basic": `auth_token = "t"
+domains_cdb = "x.cdb"
+[redis]
+write_addr = "127.0.0.1:6379"
+[notify]
+org_url = "http://admin:hunter2@admin.migadu.com/api/limits?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT"
+user = "u"
+password = "p"`,
+		"public admin": `auth_token = "t"
+domains_cdb = "x.cdb"
+admin_addr = "0.0.0.0:9095"
+[redis]
+write_addr = "127.0.0.1:6379"`,
+	} {
+		p := filepath.Join(dir, "c.toml")
+		os.WriteFile(p, []byte(body), 0o600)
+		cfg, err := Load(p)
+		if err != nil {
+			t.Logf("%s: Load error: %v", name, err)
+			continue
+		}
+		t.Logf("%s: Load OK; AuthToken=%q prefixes=%v redis.password=%q warnings=%q", name, cfg.AuthToken, cfg.AllowedPrefixes(), cfg.Redis.Password, cfg.Warnings())
+	}
+}
@@ -0,0 +1,87 @@
+package server
+
+import (
+	"io"
+	"net"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+
+	"migadu_counters/internal/store"
+)
+
+// delayProxy forwards to backend; replies are held back by *delay.
+func delayProxy(t *testing.T, backend string, delay *atomic.Int64) string {
+	ln, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { ln.Close() })
+	go func() {
+		for {
+			c, err := ln.Accept()
+			if err != nil {
+				return
+			}
+			b, err := net.Dial("tcp", backend)
+			if err != nil {
+				c.Close()
+				continue
+			}
+			go func() { io.Copy(b, c); b.Close() }()
+			go func() {
+				buf := make([]byte, 32*1024)
+				for {
+					n, err := b.Read(buf)
+					if n > 0 {
+						time.Sleep(time.Duration(delay.Load()))
+						if _, werr := c.Write(buf[:n]); werr != nil {
+							return
+						}
+					}
+					if err != nil {
+						c.Close()
+						return
+					}
+				}
+			}()
+		}
+	}()
+	return ln.Addr().String()
+}
+
+// F: the EXEC reply is lost (read timeout) after Redis applied the increments:
+// the service answers 503 "nothing counted", the crossing increment is never
+// reported, and the web app is never notified for that day.
+func TestReviewB_LostCrossingOnReplyTimeout(t *testing.T) {
+	backend := miniredis.RunT(t)
+	var delay atomic.Int64
+	proxy := delayProxy(t, backend.Addr(), &delay)
+	h := newHarness(t, options{writeAddr: proxy, handlerTimeout: 10 * time.Second})
+
+	key := store.Key("A", "a@yagolivas.com", day, "o") // A/o = 20
+	backend.Set(key, "19")
+
+	// warm the pool so the handshake is done before replies are delayed
+	if st, body := h.post("/in", `{"recipient":"w@sttml.ru"}`); st != 200 {
+		t.Fatalf("warmup %d %s", st, body)
+	}
+	delay.Store(int64(2500 * time.Millisecond)) // > timeout_ms 2000 of the harness
+	st, body := h.post("/out", `{"trace_id":"t1","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
+	t.Logf("request 1: %d %s", st, strings.TrimSpace(body))
+	v, _ := backend.Get(key)
+	t.Logf("redis value after 503: %s", v)
+	t.Logf("txlog:\n%s", h.tx.String())
+
+	delay.Store(0)
+	time.Sleep(3 * time.Second)
+	st, body = h.post("/out", `{"trace_id":"t2","auth_user":"a@yagolivas.com","recipient":"x@gmail.com"}`)
+	t.Logf("request 2: %d %s", st, strings.TrimSpace(body))
+	time.Sleep(200 * time.Millisecond)
+	if h.addrURL.count() == 0 {
+		t.Errorf("A/o limit 20 was crossed (value %s -> now >20) but no notification was ever sent", v)
+	}
+}
@@ -0,0 +1,32 @@
+package server
+
+import (
+	"context"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"migadu_counters/internal/notify"
+	"migadu_counters/internal/txlog"
+)
+
+func TestReviewB_NotifyURLCredentialsLogged(t *testing.T) {
+	var gotAuth string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotAuth = r.Header.Get("Authorization")
+	}))
+	defer ts.Close()
+	u := strings.Replace(ts.URL, "http://", "http://admin:hunter2@", 1) + "/api/limits?level=$LEVEL&id=$ID&flow=$FLOW&period=$PERIOD&by=$ADDRESS&count=$COUNT"
+	buf := &syncBuffer{}
+	tx := txlog.New(buf, "text", nil)
+	n := notify.New(notify.Config{OrgURL: u, Timeout: time.Second}, NotifyHooks(tx, nil))
+	n.Notify(notify.Event{TraceID: "t", Route: "out", Level: "O", ID: "1049", Flow: "o", Address: "a@yagolivas.com", Count: 20, Limit: 20})
+	n.Shutdown(context.Background())
+	t.Logf("server saw Authorization=%q", gotAuth)
+	t.Logf("counting log: %s", buf.String())
+	if strings.Contains(buf.String(), "hunter2") {
+		t.Errorf("notify credentials written to the counting log")
+	}
+}
@@ -0,0 +1,27 @@
+package server
+
+import (
+	"strings"
+	"testing"
+
+	"migadu_counters/internal/counting"
+	"migadu_counters/internal/domains"
+)
+
+func TestReviewB_MalformedOrgIDDropsOtherParty(t *testing.T) {
+	h := newHarness(t, options{})
+	db, err := domains.Open("../../review/_proof_tests/b/bad.cdb")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	lf := counting.LookupFunc(db.Lookup)
+	h.lookup.Store(&lf)
+	st, body := h.post("/out", `{"trace_id":"t","auth_user":"a@good.com","recipient":"b@bad.com"}`)
+	t.Logf("/out good.com -> bad.com: %d %s keys=%v", st, strings.TrimSpace(body), h.write.Keys())
+	st, body = h.post("/in", `{"trace_id":"t","recipient":"b@bad.com"}`)
+	t.Logf("/in bad.com: %d %s", st, strings.TrimSpace(body))
+	if len(h.write.Keys()) == 0 {
+		t.Errorf("sender a@good.com outgoing not counted because the recipient's CDB row is malformed")
+	}
+}
@@ -0,0 +1,142 @@
+package server
+
+import (
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"migadu_counters/internal/counting"
+	"migadu_counters/internal/domains"
+	"migadu_counters/internal/store"
+)
+
+// F1: an /in request whose (never counted) mail_from is unusual-but-SMTP-valid
+// is rejected wholesale, so the recipient's incoming counters are not bumped.
+func TestReviewB_InMailFromDropsRecipientCount(t *testing.T) {
+	h := newHarness(t, options{})
+	for _, mf := range []string{
+		"john smith@example.com",         // go-smtp unquotes MAIL FROM:<"john smith"@example.com>
+		"a@b@example.com",                // MAIL FROM:<"a@b"@example.com>
+		"info@bücher.de",                 // SMTPUTF8 U-label sender domain
+		"postmaster@[192.0.2.1]",         // address literal
+		"bounce@mail_relay.example.com",  // underscore host
+		strings.Repeat("v", 250) + "@example.com", // > 254 chars
+	} {
+		st, body := h.post("/in", `{"trace_id":"t","mail_from":"`+mf+`","recipient":"b@yagolivas.com"}`)
+		t.Logf("mail_from=%.40q -> %d %s", mf, st, strings.TrimSpace(body))
+	}
+	t.Logf("keys written: %v", h.write.Keys())
+	if !h.write.Exists(store.Key("A", "b@yagolivas.com", day, "i")) {
+		t.Errorf("recipient b@yagolivas.com never counted although every message was accepted by mizu")
+	}
+}
+
+// F1b: /out where one recipient has a domain the validator dislikes drops the
+// sender's outgoing increment too.
+func TestReviewB_OutRecipientDropsSenderCount(t *testing.T) {
+	h := newHarness(t, options{})
+	for _, rc := range []string{"x@[192.0.2.1]", "x@bücher.de", "x@host_name.example.com"} {
+		st, body := h.post("/out", `{"trace_id":"t","auth_user":"a@yagolivas.com","mail_from":"a@yagolivas.com","recipient":"`+rc+`"}`)
+		t.Logf("recipient=%q -> %d %s", rc, st, strings.TrimSpace(body))
+	}
+	if !h.write.Exists(store.Key("A", "a@yagolivas.com", day, "o")) {
+		t.Errorf("sender a@yagolivas.com outgoing never counted")
+	}
+}
+
+// F2: plus-addressing creates a separate A counter per tag.
+func TestReviewB_PlusTagSeparateCounters(t *testing.T) {
+	h := newHarness(t, options{})
+	for _, r := range []string{"b+1@yagolivas.com", "b+2@yagolivas.com", "B+3@yagolivas.com"} {
+		if st, body := h.post("/in", `{"recipient":"`+r+`"}`); st != 200 {
+			t.Fatalf("%d %s", st, body)
+		}
+	}
+	t.Logf("keys: %v", h.write.Keys())
+	if !h.write.Exists(store.Key("A", "b@yagolivas.com", day, "i")) {
+		t.Errorf("mailbox b@yagolivas.com A/i counter (what the web app reads) never incremented")
+	}
+}
+
+// F5: a recovered panic is answered 500 but not counted in requests_total.
+func TestReviewB_PanicNotInMetrics(t *testing.T) {
+	h := newHarness(t, options{})
+	boom := counting.LookupFunc(func(string) (*domains.Domain, error) { panic("boom") })
+	h.lookup.Store(&boom)
+	st, _ := h.post("/in", `{"recipient":"b@yagolivas.com"}`)
+	got := h.metric(h.m.Requests.WithLabelValues("/in", "500"))
+	t.Logf("status %d, requests_total{/in,500}=%v", st, got)
+	if got != 1 {
+		t.Errorf("500 from recovered panic not counted in requests_total")
+	}
+}
+
+// F6: rejection log has no trace_id.
+func TestReviewB_RejectLogHasNoTraceID(t *testing.T) {
+	h := newHarness(t, options{})
+	st, _ := h.post("/in", `{"trace_id":"TRACE-XYZ","recipient":"nobody"}`)
+	if st != 400 {
+		t.Fatal(st)
+	}
+	if !strings.Contains(h.log.String(), "TRACE-XYZ") {
+		t.Errorf("rejection log lacks trace_id:\n%s", h.log.String())
+	}
+}
+
+// Leakage: 503 body / health carry raw redis errors with internal addresses.
+func TestReviewB_ErrorLeak(t *testing.T) {
+	h := newHarness(t, options{})
+	addr := h.write.Addr()
+	h.write.Close()
+	st, body := h.post("/in", `{"recipient":"b@yagolivas.com"}`)
+	t.Logf("api: %d %s", st, strings.TrimSpace(body))
+	resp, err := http.Get(h.admin.URL + "/health")
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	t.Logf("health: %d %s", resp.StatusCode, strings.TrimSpace(string(b)))
+	if strings.Contains(body, addr) {
+		t.Logf("API 503 body leaks redis address %s", addr)
+	}
+}
+
+// Chunked body over 64 KB is rejected 413.
+func TestReviewB_ChunkedTooLarge(t *testing.T) {
+	h := newHarness(t, options{})
+	pr, pw := io.Pipe()
+	go func() {
+		pw.Write([]byte(`{"recipient":"b@yagolivas.com","pad":"`))
+		for i := 0; i < 100; i++ {
+			pw.Write([]byte(strings.Repeat("x", 1024)))
+		}
+		pw.Write([]byte(`"}`))
+		pw.Close()
+	}()
+	req, _ := http.NewRequest("POST", h.api.URL+"/in", pr)
+	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	t.Logf("chunked 100KB: %d %s (cl=%d)", resp.StatusCode, b, req.ContentLength)
+	if resp.StatusCode != 413 {
+		t.Errorf("want 413")
+	}
+}
+
+// Empty configured token = no auth at all.
+func TestReviewB_EmptyTokenOpen(t *testing.T) {
+	h := newHarness(t, options{token: "x"})
+	_ = h
+}
+
+var _ = net.Listen
+var _ = time.Second