dimitrije/counters
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add 80% limit warning, stop repeat notifications, fix review findings

Browse Source 
Notifications:
- Warn once when a counter reaches warn_percent of its limit, through
  separate warn_*_url endpoints.
- Write ntf/ markers so a lost Redis reply cannot re-send a breach.
- Retry only when the request never left or the web app answered 429/5xx.

Fixes:
- Retry transactions Redis rejected; treat pool timeouts as definite.
- Install the signal handler before startup work.
- Stop logging the login suffix; redact notify URL credentials.
- Reject corrupt or empty CDBs, truncated hash tables included.
- Require a Sentinel password, refuse example placeholders, and default
  admin_addr to localhost.
- Normalise /counters domains, count on the arrival day, log rejections,
  hide internals in errors and exit 1 on a failed drain.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Dimitrije Dimitrijevic
2026-11-09 09:01:56 +01:00
co-authored by Claude Opus 5.5
parent c43ff17627
commit 54f9626121
29 changed files with 1673 additions and 245 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/metrics/metrics.go
+27 -17
View File
@@ -7,17 +7,18 @@ import (
// Metrics holds every collector; fields are exported for tests.
type Metrics struct {
Requests *prometheus.CounterVec // route, status
Duration *prometheus.HistogramVec // route
Increments *prometheus.CounterVec // level, flow
LimitReached *prometheus.CounterVec // level, flow
Notifications *prometheus.CounterVec // level, result (sent|failed|disabled)
RedisErrors *prometheus.CounterVec // op
CDBDomains prometheus.Gauge
CDBReloads *prometheus.CounterVec // result (success|failure)
Dropped *prometheus.CounterVec // route: requests refused because the counting queue was full
QueueDepth prometheus.Gauge
QueueWait prometheus.Histogram
Requests *prometheus.CounterVec // route, status
Duration *prometheus.HistogramVec // route
Increments *prometheus.CounterVec // level, flow
LimitReached *prometheus.CounterVec // level, flow
WarningReached *prometheus.CounterVec // level, flow
Notifications *prometheus.CounterVec // kind (limit|warning), level, result (sent|failed|disabled)
RedisErrors *prometheus.CounterVec // op
CDBDomains prometheus.Gauge
CDBReloads *prometheus.CounterVec // result (success|failure)
Dropped *prometheus.CounterVec // route: requests refused because the counting queue was full
QueueDepth prometheus.Gauge
QueueWait prometheus.Histogram
}
// New creates the collectors and registers them with reg.
@@ -40,13 +41,17 @@ func New(reg prometheus.Registerer) *Metrics {
Name: "migadu_counters_limit_reached_total",
Help: "Increments that crossed a configured limit, by level and flow.",
}, []string{"level", "flow"}),
WarningReached: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "migadu_counters_warning_reached_total",
Help: "Increments that crossed the early-warning threshold, by level and flow.",
}, []string{"level", "flow"}),
Notifications: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "migadu_counters_notifications_total",
Help: "Web app notifications by level and result (sent, failed, disabled).",
}, []string{"level", "result"}),
Help: "Web app notifications by kind (limit, warning), level and result (sent, failed, disabled).",
}, []string{"kind", "level", "result"}),
RedisErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "migadu_counters_redis_errors_total",
Help: "Failed Redis operations by op (incr, mget, ping).",
Help: "Failed Redis operations by op (incr, incr_unknown, mget, ping, marker).",
}, []string{"op"}),
CDBDomains: prometheus.NewGauge(prometheus.GaugeOpts{
Name: "migadu_counters_cdb_domains",
@@ -70,7 +75,7 @@ func New(reg prometheus.Registerer) *Metrics {
Buckets: []float64{.0001, .0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5},
}),
}
reg.MustRegister(m.Requests, m.Duration, m.Increments, m.LimitReached, m.Notifications, m.RedisErrors, m.CDBDomains, m.CDBReloads,
reg.MustRegister(m.Requests, m.Duration, m.Increments, m.LimitReached, m.WarningReached, m.Notifications, m.RedisErrors, m.CDBDomains, m.CDBReloads,
m.Dropped, m.QueueDepth, m.QueueWait)
return m
}
@@ -101,8 +106,13 @@ func (m *Metrics) RecordLimitReached(level, flow string) {
}
// RecordNotification counts one notification outcome.
func (m *Metrics) RecordNotification(level, result string) {
m.Notifications.WithLabelValues(level, result).Inc()
func (m *Metrics) RecordNotification(kind, level, result string) {
m.Notifications.WithLabelValues(kind, level, result).Inc()
}
// RecordWarningReached counts one early-warning crossing.
func (m *Metrics) RecordWarningReached(level, flow string) {
m.WarningReached.WithLabelValues(level, flow).Inc()
}
// RecordRedisError counts one failed Redis operation.