First commit Go counter service

This commit is contained in:
Dimitrije Dimitrijevic
2026-10-15 02:12:18 +02:00
commit c43ff17627
59 changed files with 11205 additions and 0 deletions
+238
View File
@@ -0,0 +1,238 @@
// Package store holds the Redis counters. Writes (INCR) go to the master,
// reads (MGET for /counters) go to the replicas; in Sentinel mode go-redis
// discovers both, in static mode the addresses come from the config.
//
// Key format, shared with the rspamd Lua plugin and read by migadu_web
// (apps/core/lib/core/redis_sentinel.ex):
//
// cnt/<A|D|O>/<id>/d/<YYYY-MM-DD>/<i|o>
//
// id is the address, the domain or the organisation id; the day is the UTC
// calendar day; every key lives 31 days so the web app can chart a month.
package store
import (
"context"
"errors"
"fmt"
"io"
"net"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/redis/go-redis/v9"
"migadu_counters/internal/config"
)
const (
// Prefix of every counter key.
Prefix = "cnt"
// TTL is refreshed on every increment, like the Lua plugin did.
TTL = 31 * 24 * time.Hour
)
// Key builds a counter key: Key("A", "x@example.com", "2026-10-13", "i").
func Key(level, id, date, flow string) string {
return Prefix + "/" + level + "/" + id + "/d/" + date + "/" + flow
}
// DayKey returns the UTC calendar day of t, as used in the keys. The web app
// builds its keys with Date.utc_today(), so this must never use local time.
func DayKey(t time.Time) string { return t.UTC().Format("2006-01-02") }
// Store is a write client plus one or more read clients.
type Store struct {
write *redis.Client
readers []*redis.Client
next atomic.Uint64
}
// New builds the clients from the config. Nothing is dialled until the first
// command, so an unreachable Redis does not prevent the daemon from starting.
func New(cfg config.RedisConfig) (*Store, error) {
if cfg.DB != 0 {
return nil, fmt.Errorf("redis.db must be 0 (got %d): migadu_web reads the message counters from DB 0", cfg.DB)
}
timeout := cfg.Timeout()
if cfg.SentinelMode() {
base := redis.FailoverOptions{
MasterName: cfg.MasterName,
SentinelAddrs: cfg.Sentinels,
SentinelPassword: cfg.SentinelPassword,
Password: cfg.Password,
DB: 0,
PoolSize: cfg.PoolSize,
DialTimeout: timeout,
ReadTimeout: timeout,
WriteTimeout: timeout,
}
write := redis.NewFailoverClient(&base)
// ReplicaOnly picks a random replica and falls back to the master when
// no replica is known.
replica := base
replica.ReplicaOnly = true
read := redis.NewFailoverClient(&replica)
return &Store{write: write, readers: []*redis.Client{read}}, nil
}
if cfg.WriteAddr == "" {
return nil, errors.New("redis.write_addr is required when no sentinels are configured")
}
opts := func(addr string) *redis.Options {
return &redis.Options{
Addr: addr,
Password: cfg.Password,
DB: 0,
PoolSize: cfg.PoolSize,
DialTimeout: timeout,
ReadTimeout: timeout,
WriteTimeout: timeout,
}
}
addrs := cfg.ReadAddrs
if len(addrs) == 0 {
addrs = []string{cfg.WriteAddr}
}
readers := make([]*redis.Client, 0, len(addrs))
for _, a := range addrs {
readers = append(readers, redis.NewClient(opts(a)))
}
return &Store{write: redis.NewClient(opts(cfg.WriteAddr)), readers: readers}, nil
}
// NewWithClients wires pre-built clients (tests). With no readers the write
// client is read from.
func NewWithClients(write *redis.Client, readers []*redis.Client) *Store {
if len(readers) == 0 {
readers = []*redis.Client{write}
}
return &Store{write: write, readers: readers}
}
// reader returns the next read client, round-robin.
func (s *Store) reader() *redis.Client {
n := s.next.Add(1) - 1
return s.readers[n%uint64(len(s.readers))]
}
// IncrResult is the outcome of one key's INCR inside the transaction.
type IncrResult struct {
Count int64 // value after the increment; meaningless when Err != nil
Err error // this key's own failure (e.g. it holds a non-number)
}
// Incr increments every key by one and refreshes its TTL in one MULTI/EXEC,
// returning one result per key in order. EXEC's replies are the
// authoritative counts, so no read-after-write is needed and replica lag
// does not matter. A per-key failure (a key holding a non-number) is reported
// in that key's result; the other keys are still incremented. The returned
// error is set only when the transaction as a whole failed, in which case
// OutcomeUnknown tells whether it may nevertheless have been applied.
func (s *Store) Incr(ctx context.Context, keys []string) ([]IncrResult, error) {
if len(keys) == 0 {
return nil, nil
}
cmds := make([]*redis.IntCmd, len(keys))
_, err := s.write.TxPipelined(ctx, func(pipe redis.Pipeliner) error {
for i, k := range keys {
cmds[i] = pipe.Incr(ctx, k)
pipe.Expire(ctx, k, TTL)
}
return nil
})
out := make([]IncrResult, len(keys))
if err != nil {
// TxPipelined returns the first failed command's error. A Redis reply
// error (redis.Error) means EXEC answered and every command has a
// definite result: value or its own error. Anything else (dial, write,
// read timeout) is a transport failure of the whole transaction.
var rerr redis.Error
if !errors.As(err, &rerr) {
return nil, fmt.Errorf("redis incr: %w", err)
}
}
for i, c := range cmds {
if cerr := c.Err(); cerr != nil {
out[i].Err = fmt.Errorf("redis incr %s: %w", keys[i], cerr)
continue
}
out[i].Count = c.Val()
}
return out, nil
}
// OutcomeUnknown reports whether a failed Incr may still have been applied
// by Redis: the request may have been written but the reply lost (timeout,
// EOF, reset). A refused connection, a dial failure or a cancelled context
// never reached Redis and is a definite failure.
func OutcomeUnknown(err error) bool {
if err == nil || errors.Is(err, context.Canceled) {
return false
}
var opErr *net.OpError
if errors.As(err, &opErr) && opErr.Op == "dial" {
return false
}
if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) {
return true
}
var nerr net.Error
if errors.As(err, &nerr) && nerr.Timeout() {
return true
}
msg := err.Error()
return strings.Contains(msg, "timeout") || strings.Contains(msg, "reset") || strings.Contains(msg, "broken pipe") || strings.Contains(msg, "EOF")
}
// GetMaster reads the keys from the write side (the master) with one MGET;
// used to establish the outcome of an Incr whose reply was lost, when a
// replica might not have the increment yet. A missing key reads as 0.
func (s *Store) GetMaster(ctx context.Context, keys []string) ([]int64, error) {
return mget(ctx, s.write, keys)
}
// Get reads the keys from a replica with one MGET; a missing key reads as 0.
func (s *Store) Get(ctx context.Context, keys []string) ([]int64, error) {
return mget(ctx, s.reader(), keys)
}
func mget(ctx context.Context, c *redis.Client, keys []string) ([]int64, error) {
if len(keys) == 0 {
return nil, nil
}
vals, err := c.MGet(ctx, keys...).Result()
if err != nil {
return nil, fmt.Errorf("redis mget: %w", err)
}
out := make([]int64, len(keys))
for i, v := range vals {
str, ok := v.(string)
if !ok {
continue // nil = missing key
}
n, err := strconv.ParseInt(str, 10, 64)
if err != nil {
return nil, fmt.Errorf("redis key %s holds non-integer value %q", keys[i], str)
}
out[i] = n
}
return out, nil
}
// Ping checks the write side (the master), which is what /health cares about.
func (s *Store) Ping(ctx context.Context) error { return s.write.Ping(ctx).Err() }
// Close closes every client.
func (s *Store) Close() error {
errs := []error{s.write.Close()}
for _, r := range s.readers {
if r != s.write {
errs = append(errs, r.Close())
}
}
return errors.Join(errs...)
}
+314
View File
@@ -0,0 +1,314 @@
package store
import (
"context"
"errors"
"io"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/alicebob/miniredis/v2"
"github.com/redis/go-redis/v9"
"migadu_counters/internal/config"
)
func newMini(t *testing.T) *miniredis.Miniredis {
t.Helper()
m := miniredis.RunT(t)
return m
}
// testStore wires a Store to two separate miniredis instances so tests can
// prove which side (write master vs read replica) a method talks to.
func testStore(t *testing.T) (*Store, *miniredis.Miniredis, *miniredis.Miniredis) {
t.Helper()
w, r := newMini(t), newMini(t)
s := NewWithClients(
redis.NewClient(&redis.Options{Addr: w.Addr()}),
[]*redis.Client{redis.NewClient(&redis.Options{Addr: r.Addr()})},
)
t.Cleanup(func() { s.Close() })
return s, w, r
}
func TestKey(t *testing.T) {
tests := []struct {
level, id, date, flow, want string
}{
{"A", "x@example.com", "2026-10-13", "i", "cnt/A/x@example.com/d/2026-10-13/i"},
{"D", "example.com", "2026-10-13", "o", "cnt/D/example.com/d/2026-10-13/o"},
{"O", "1234", "2026-01-01", "i", "cnt/O/1234/d/2026-01-01/i"},
}
for _, tt := range tests {
if got := Key(tt.level, tt.id, tt.date, tt.flow); got != tt.want {
t.Errorf("Key(%s,%s,%s,%s) = %q, want %q", tt.level, tt.id, tt.date, tt.flow, got, tt.want)
}
}
}
// The day stamp is always the UTC calendar day, matching migadu_web's
// Date.utc_today() — never the server's local day.
func TestDayKey_UTC(t *testing.T) {
cest := time.FixedZone("CEST", 2*3600)
tests := []struct {
in time.Time
want string
}{
{time.Date(2026, 10, 13, 23, 59, 0, 0, cest), "2026-10-13"}, // 21:59 UTC
{time.Date(2026, 10, 14, 1, 30, 0, 0, cest), "2026-10-13"}, // 23:30 UTC the day before
{time.Date(2026, 10, 14, 2, 0, 0, 0, cest), "2026-10-14"}, // 00:00 UTC
{time.Date(2026, 3, 5, 12, 0, 0, 0, time.UTC), "2026-03-05"}, // zero padded
}
for _, tt := range tests {
if got := DayKey(tt.in); got != tt.want {
t.Errorf("DayKey(%v) = %q, want %q", tt.in, got, tt.want)
}
}
}
func TestIncr(t *testing.T) {
s, w, _ := testStore(t)
ctx := context.Background()
keys := []string{"cnt/A/x@example.com/d/2026-10-13/o", "cnt/O/1234/d/2026-10-13/o"}
got, err := s.Incr(ctx, keys)
if err != nil {
t.Fatalf("Incr: %v", err)
}
if len(got) != 2 || got[0].Count != 1 || got[1].Count != 1 || got[0].Err != nil || got[1].Err != nil {
t.Fatalf("first Incr = %+v, want counts [1 1]", got)
}
got, err = s.Incr(ctx, keys)
if err != nil || got[0].Count != 2 || got[1].Count != 2 {
t.Fatalf("second Incr = %+v, %v; want [2 2]", got, err)
}
for _, k := range keys {
if v, _ := w.Get(k); v != "2" {
t.Errorf("stored %s = %q, want \"2\"", k, v)
}
if ttl := w.TTL(k); ttl <= 30*24*time.Hour || ttl > TTL {
t.Errorf("TTL(%s) = %v, want (30d, 31d]", k, ttl)
}
}
if got, err := s.Incr(ctx, nil); err != nil || len(got) != 0 {
t.Errorf("Incr(no keys) = %v, %v; want empty, nil", got, err)
}
}
// One key holding a non-number fails its own INCR inside EXEC; the other
// keys are still incremented and their counts reported (review #4).
func TestIncr_PartialFailure(t *testing.T) {
s, w, _ := testStore(t)
w.Set("bad", "garbage")
got, err := s.Incr(context.Background(), []string{"k1", "bad", "k3"})
if err != nil {
t.Fatalf("a per-key failure must not fail the whole Incr: %v", err)
}
if got[0].Count != 1 || got[0].Err != nil || got[2].Count != 1 || got[2].Err != nil {
t.Errorf("good keys = %+v %+v, want count 1 without error", got[0], got[2])
}
if got[1].Err == nil || !strings.Contains(got[1].Err.Error(), "not an integer") {
t.Errorf("bad key err = %v, want the INCR error", got[1].Err)
}
if v, _ := w.Get("bad"); v != "garbage" {
t.Errorf("bad key = %q, must be untouched", v)
}
if ttl := w.TTL("k1"); ttl <= 30*24*time.Hour {
t.Errorf("TTL(k1) = %v, EXPIRE should still have run", ttl)
}
}
// GetMaster reads from the write side (used after a lost reply, when the
// replica may not have the increment yet).
func TestGetMaster(t *testing.T) {
s, w, r := testStore(t)
w.Set("k", "19")
r.Set("k", "3")
got, err := s.GetMaster(context.Background(), []string{"k", "missing"})
if err != nil || len(got) != 2 || got[0] != 19 || got[1] != 0 {
t.Errorf("GetMaster = %v, %v; want [19 0] from the master", got, err)
}
}
// OutcomeUnknown tells whether a failed Incr may still have been applied: a
// refused connection never reached Redis, a timeout or reset after the write
// might have.
func TestOutcomeUnknown(t *testing.T) {
s, w, _ := testStore(t)
w.Close()
_, err := s.Incr(context.Background(), []string{"k"})
if err == nil {
t.Fatal("expected an error with the write side down")
}
if OutcomeUnknown(err) {
t.Errorf("connection refused must be a known failure, got unknown: %v", err)
}
if OutcomeUnknown(nil) || OutcomeUnknown(context.Canceled) {
t.Error("nil / canceled are not unknown outcomes")
}
for _, e := range []error{context.DeadlineExceeded, io.EOF, &net.OpError{Op: "read", Err: timeoutErr{}}, errors.New("i/o timeout")} {
if !OutcomeUnknown(e) {
t.Errorf("%v should be an unknown outcome", e)
}
}
}
type timeoutErr struct{}
func (timeoutErr) Error() string { return "i/o timeout" }
func (timeoutErr) Timeout() bool { return true }
func (timeoutErr) Temporary() bool { return true }
// Fifty concurrent increments of one key must end at exactly 50 — the
// pipeline relies on INCRBY being atomic, not on read-then-write.
func TestIncr_Concurrent(t *testing.T) {
s, w, _ := testStore(t)
var wg sync.WaitGroup
for i := 0; i < 50; i++ {
wg.Add(1)
go func() {
defer wg.Done()
if _, err := s.Incr(context.Background(), []string{"k"}); err != nil {
t.Error(err)
}
}()
}
wg.Wait()
if v, _ := w.Get("k"); v != "50" {
t.Errorf("k = %q, want 50", v)
}
}
// Get reads from the replica side: a key present only on the write instance
// is invisible, a key present on the read instance is returned, missing = 0.
func TestGet_UsesReadSide(t *testing.T) {
s, w, r := testStore(t)
w.Set("only-on-write", "9")
r.Set("on-read", "5")
got, err := s.Get(context.Background(), []string{"on-read", "only-on-write", "missing"})
if err != nil {
t.Fatal(err)
}
if len(got) != 3 || got[0] != 5 || got[1] != 0 || got[2] != 0 {
t.Errorf("Get = %v, want [5 0 0]", got)
}
if got, err := s.Get(context.Background(), nil); err != nil || len(got) != 0 {
t.Errorf("Get(no keys) = %v, %v", got, err)
}
}
func TestGet_RoundRobinReaders(t *testing.T) {
w, r1, r2 := newMini(t), newMini(t), newMini(t)
r1.Set("k", "1")
r2.Set("k", "2")
s := NewWithClients(redis.NewClient(&redis.Options{Addr: w.Addr()}), []*redis.Client{
redis.NewClient(&redis.Options{Addr: r1.Addr()}),
redis.NewClient(&redis.Options{Addr: r2.Addr()}),
})
defer s.Close()
seen := map[int64]int{}
for i := 0; i < 4; i++ {
got, err := s.Get(context.Background(), []string{"k"})
if err != nil {
t.Fatal(err)
}
seen[got[0]]++
}
if seen[1] != 2 || seen[2] != 2 {
t.Errorf("reads not alternated between replicas: %v", seen)
}
}
func TestErrors_RedisDown(t *testing.T) {
s, w, r := testStore(t)
ctx := context.Background()
if err := s.Ping(ctx); err != nil {
t.Fatalf("Ping before close: %v", err)
}
w.Close()
if _, err := s.Incr(ctx, []string{"k"}); err == nil {
t.Error("Incr with write side down should fail")
}
if err := s.Ping(ctx); err == nil {
t.Error("Ping with write side down should fail")
}
r.Close()
if _, err := s.Get(ctx, []string{"k"}); err == nil {
t.Error("Get with read side down should fail")
}
}
func TestNew_Static(t *testing.T) {
w, r := newMini(t), newMini(t)
s, err := New(config.RedisConfig{WriteAddr: w.Addr(), ReadAddrs: []string{r.Addr()}, PoolSize: 2, TimeoutMS: 200})
if err != nil {
t.Fatalf("New: %v", err)
}
defer s.Close()
ctx := context.Background()
if err := s.Ping(ctx); err != nil {
t.Fatalf("Ping: %v", err)
}
if _, err := s.Incr(ctx, []string{"k"}); err != nil {
t.Fatal(err)
}
if v, _ := w.Get("k"); v != "1" {
t.Errorf("write went to %q, want the write_addr instance", v)
}
r.Set("k", "7")
if got, _ := s.Get(ctx, []string{"k"}); got[0] != 7 {
t.Errorf("Get = %v, want 7 from read_addrs", got)
}
}
func TestNew_StaticDefaultsReadToWrite(t *testing.T) {
w := newMini(t)
s, err := New(config.RedisConfig{WriteAddr: w.Addr(), PoolSize: 1, TimeoutMS: 200})
if err != nil {
t.Fatal(err)
}
defer s.Close()
w.Set("k", "3")
if got, err := s.Get(context.Background(), []string{"k"}); err != nil || got[0] != 3 {
t.Errorf("Get = %v, %v; want 3 from the write instance", got, err)
}
}
func TestNew_RejectsNonZeroDB(t *testing.T) {
if _, err := New(config.RedisConfig{WriteAddr: "127.0.0.1:6379", DB: 1, PoolSize: 1, TimeoutMS: 100}); err == nil {
t.Error("New with db=1 should fail: migadu_web reads DB 0")
}
}
// Sentinel mode constructs lazily; an unreachable sentinel surfaces on the
// first command, bounded by the configured timeout.
func TestNew_SentinelUnreachable(t *testing.T) {
s, err := New(config.RedisConfig{MasterName: "mymaster", Sentinels: []string{"127.0.0.1:1"}, PoolSize: 1, TimeoutMS: 200})
if err != nil {
t.Fatalf("New: %v", err)
}
defer s.Close()
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
start := time.Now()
if err := s.Ping(ctx); err == nil {
t.Error("Ping against an unreachable sentinel should fail")
}
if time.Since(start) > 2500*time.Millisecond {
t.Errorf("Ping took %v, should fail fast", time.Since(start))
}
}
func TestClose(t *testing.T) {
s, _, _ := testStore(t)
if err := s.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
if err := s.Ping(context.Background()); err == nil {
t.Error("Ping after Close should fail")
}
}