dimitrije/counters
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 80% limit warning, stop repeat notifications, fix review findings

Browse Source 
Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/server/server.go
+48 -20
View File
@@ -92,7 +92,7 @@ func NotifyHooks(tx *txlog.Logger, m *metrics.Metrics) notify.Hooks {
return notify.Hooks{
Sent: func(ev notify.Event, url string, attempt int) {
if m != nil {
m.RecordNotification(ev.Level, "sent")
m.RecordNotification(ev.KindName(), ev.Level, "sent")
}
if tx != nil {
tx.NotificationSent(ev, url, attempt)
@@ -100,7 +100,7 @@ func NotifyHooks(tx *txlog.Logger, m *metrics.Metrics) notify.Hooks {
},
Failed: func(ev notify.Event, url string, attempts int, err error) {
if m != nil {
m.RecordNotification(ev.Level, "failed")
m.RecordNotification(ev.KindName(), ev.Level, "failed")
}
if tx != nil {
tx.NotificationFailed(ev, url, attempts, err)
@@ -109,7 +109,7 @@ func NotifyHooks(tx *txlog.Logger, m *metrics.Metrics) notify.Hooks {
}
}
// Handler returns the API handler: recover → metrics → timeout → routes.
// Handler returns the API handler: metrics → recover → timeout → routes.
func (s *Server) Handler() http.Handler {
mux := http.NewServeMux()
mux.Handle("/out", s.route("/out", http.MethodPost, s.handleOut))
@@ -119,7 +119,7 @@ func (s *Server) Handler() http.Handler {
writeError(w, http.StatusNotFound, "not found")
})
timed := http.TimeoutHandler(mux, s.d.HandlerTimeout, `{"error":"request timeout"}`)
return s.recoverer(s.observe(timed))
return s.observe(s.recoverer(timed))
}
// AdminHandler returns /health and /metrics (no auth; bind admin_addr locally).
@@ -135,17 +135,17 @@ func (s *Server) AdminHandler() http.Handler {
func (s *Server) handleOut(w http.ResponseWriter, r *http.Request) {
var req api.OutRequest
if e := decodeJSON(r, &req); e != nil {
s.reject(w, r, "/out", e)
s.rejectBody(w, r, "/out", req.TraceID, e)
return
}
v, err := api.ValidateOut(req)
if err != nil {
s.reject(w, r, "/out", &httpError{http.StatusBadRequest, err.Error()})
s.rejectBody(w, r, "/out", req.TraceID, &httpError{http.StatusBadRequest, err.Error()})
return
}
job := counting.Job{
Route: counting.RouteOut, TraceID: v.TraceID, Received: s.d.Now(),
AuthUser: v.AuthUser.Value, LoginSuffix: v.LoginSuffix,
AuthUser: v.AuthUser.Value, TokenLogin: v.TokenLogin,
MailFrom: v.MailFrom, MailFromUsable: v.MailFromUsable,
Recipient: v.Recipient.Value,
}
@@ -161,12 +161,12 @@ func (s *Server) handleOut(w http.ResponseWriter, r *http.Request) {
func (s *Server) handleIn(w http.ResponseWriter, r *http.Request) {
var req api.InRequest
if e := decodeJSON(r, &req); e != nil {
s.reject(w, r, "/in", e)
s.rejectBody(w, r, "/in", req.TraceID, e)
return
}
v, err := api.ValidateIn(req)
if err != nil {
s.reject(w, r, "/in", &httpError{http.StatusBadRequest, err.Error()})
s.rejectBody(w, r, "/in", req.TraceID, &httpError{http.StatusBadRequest, err.Error()})
return
}
s.enqueue(w, r, "/in", counting.Job{
@@ -242,16 +242,23 @@ func (s *Server) process(ctx context.Context, job counting.Job) {
if s.d.Metrics != nil {
s.d.Metrics.RecordIncrement(res.Level, res.Flow)
}
if res.Reached {
if res.MarkerErr != nil {
if s.d.Metrics != nil {
s.d.Metrics.RecordRedisError("marker")
}
s.d.Logger.Error("notification marker failed", "route", plan.Route, "trace_id", plan.TraceID, "key", res.Key, "error", res.MarkerErr)
}
switch {
case res.Reached:
if s.d.Metrics != nil {
s.d.Metrics.RecordLimitReached(res.Level, res.Flow)
}
ev := notify.Event{TraceID: plan.TraceID, Route: plan.Route, Level: res.Level, ID: res.ID,
Flow: res.Flow, Address: res.By, Count: res.Count, Limit: res.Limit}
dispatched[i] = s.d.Notifier != nil && s.d.Notifier.Notify(ev)
if !dispatched[i] && s.d.Metrics != nil {
s.d.Metrics.RecordNotification(res.Level, "disabled")
dispatched[i] = s.notify(plan, res, notify.KindLimit)
case res.Warned:
if s.d.Metrics != nil {
s.d.Metrics.RecordWarningReached(res.Level, res.Flow)
}
dispatched[i] = s.notify(plan, res, notify.KindWarning)
}
}
if s.d.TxLog != nil {
@@ -259,11 +266,22 @@ func (s *Server) process(ctx context.Context, job counting.Job) {
}
}
// notify dispatches one notification and reports whether it was handed over.
func (s *Server) notify(plan *counting.Plan, res counting.Result, kind string) bool {
ev := notify.Event{Kind: kind, TraceID: plan.TraceID, Route: plan.Route, Level: res.Level, ID: res.ID,
Flow: res.Flow, Address: res.By, Count: res.Count, Limit: res.Limit, Percent: s.d.Counter.Planner.WarnPercent}
sent := s.d.Notifier != nil && s.d.Notifier.Notify(ev)
if !sent && s.d.Metrics != nil {
s.d.Metrics.RecordNotification(kind, res.Level, "disabled")
}
return sent
}
// handleCounters answers GET /counters?address=|domain=[&date=] from the
// read side.
func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
address, domain := strings.TrimSpace(q.Get("address")), strings.ToLower(strings.TrimSpace(q.Get("domain")))
address, domain := strings.TrimSpace(q.Get("address")), strings.TrimSpace(q.Get("domain"))
if (address == "") == (domain == "") {
s.reject(w, r, "/counters", &httpError{http.StatusBadRequest, "exactly one of address or domain is required"})
return
@@ -287,6 +305,12 @@ func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
at := strings.LastIndex(addr, "@")
local, domain = addr[:at], addr[at+1:]
} else {
dom, err := api.NormalizeDomain(domain)
if err != nil {
s.reject(w, r, "/counters", &httpError{http.StatusBadRequest, "domain: invalid domain"})
return
}
domain = dom
resp.Query, resp.Mode = domain, "domain"
}
@@ -299,7 +323,8 @@ func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
if errors.Is(err, domains.ErrMalformed) {
s.d.Logger.Warn("malformed row in domains db, domain treated as not hosted", "domain", domain, "route", "/counters", "error", err)
} else if err != nil && !errors.Is(err, domains.ErrNotFound) {
writeError(w, http.StatusServiceUnavailable, "domain lookup failed: "+err.Error())
s.d.Logger.Error("domain lookup failed", "domain", domain, "route", "/counters", "error", err)
writeError(w, http.StatusServiceUnavailable, "domain lookup failed")
return
}
managed := err == nil
@@ -326,7 +351,8 @@ func (s *Server) handleCounters(w http.ResponseWriter, r *http.Request) {
if s.d.Metrics != nil {
s.d.Metrics.RecordRedisError("mget")
}
writeError(w, http.StatusServiceUnavailable, "redis unavailable: "+err.Error())
s.d.Logger.Error("redis read failed", "route", "/counters", "error", err)
writeError(w, http.StatusServiceUnavailable, "redis unavailable")
return
}
limit := func(level, flow string) *int64 {
@@ -355,7 +381,8 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
return
}
if err := db.Ping(); err != nil {
writeError(w, http.StatusServiceUnavailable, "domains db not readable: "+err.Error())
s.d.Logger.Warn("health: domains db not readable", "error", err)
writeError(w, http.StatusServiceUnavailable, "domains db not readable")
return
}
ctx, cancel := context.WithTimeout(r.Context(), 200*time.Millisecond)
@@ -364,7 +391,8 @@ func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
if s.d.Metrics != nil {
s.d.Metrics.RecordRedisError("ping")
}
writeError(w, http.StatusServiceUnavailable, fmt.Sprintf("redis: %v", err))
s.d.Logger.Warn("health: redis not reachable", "error", err)
writeError(w, http.StatusServiceUnavailable, "redis unavailable")
return
}
writeJSON(w, http.StatusOK, map[string]string{"status": "ok"})