Notifications: - Warn once when a counter reaches warn_percent of its limit, through separate warn_*_url endpoints. - Write ntf/ markers so a lost Redis reply cannot re-send a breach. - Retry only when the request never left or the web app answered 429/5xx. Fixes: - Retry transactions Redis rejected; treat pool timeouts as definite. - Install the signal handler before startup work. - Stop logging the login suffix; redact notify URL credentials. - Reject corrupt or empty CDBs, truncated hash tables included. - Require a Sentinel password, refuse example placeholders, and default admin_addr to localhost. - Normalise /counters domains, count on the arrival day, log rejections, hide internals in errors and exit 1 on a failed drain. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
132 lines
5.6 KiB
Go
132 lines
5.6 KiB
Go
// Package metrics defines the Prometheus metrics served on the admin port.
|
|
package metrics
|
|
|
|
import (
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
)
|
|
|
|
// Metrics holds every collector; fields are exported for tests.
|
|
type Metrics struct {
|
|
Requests *prometheus.CounterVec // route, status
|
|
Duration *prometheus.HistogramVec // route
|
|
Increments *prometheus.CounterVec // level, flow
|
|
LimitReached *prometheus.CounterVec // level, flow
|
|
WarningReached *prometheus.CounterVec // level, flow
|
|
Notifications *prometheus.CounterVec // kind (limit|warning), level, result (sent|failed|disabled)
|
|
RedisErrors *prometheus.CounterVec // op
|
|
CDBDomains prometheus.Gauge
|
|
CDBReloads *prometheus.CounterVec // result (success|failure)
|
|
Dropped *prometheus.CounterVec // route: requests refused because the counting queue was full
|
|
QueueDepth prometheus.Gauge
|
|
QueueWait prometheus.Histogram
|
|
}
|
|
|
|
// New creates the collectors and registers them with reg.
|
|
func New(reg prometheus.Registerer) *Metrics {
|
|
m := &Metrics{
|
|
Requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_requests_total",
|
|
Help: "HTTP requests by route and status code.",
|
|
}, []string{"route", "status"}),
|
|
Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
|
Name: "migadu_counters_request_duration_seconds",
|
|
Help: "Request latency by route.",
|
|
Buckets: []float64{.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5},
|
|
}, []string{"route"}),
|
|
Increments: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_increments_total",
|
|
Help: "Counter increments by level (A, D, O) and flow (i, o).",
|
|
}, []string{"level", "flow"}),
|
|
LimitReached: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_limit_reached_total",
|
|
Help: "Increments that crossed a configured limit, by level and flow.",
|
|
}, []string{"level", "flow"}),
|
|
WarningReached: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_warning_reached_total",
|
|
Help: "Increments that crossed the early-warning threshold, by level and flow.",
|
|
}, []string{"level", "flow"}),
|
|
Notifications: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_notifications_total",
|
|
Help: "Web app notifications by kind (limit, warning), level and result (sent, failed, disabled).",
|
|
}, []string{"kind", "level", "result"}),
|
|
RedisErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_redis_errors_total",
|
|
Help: "Failed Redis operations by op (incr, incr_unknown, mget, ping, marker).",
|
|
}, []string{"op"}),
|
|
CDBDomains: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "migadu_counters_cdb_domains",
|
|
Help: "Domains in the currently loaded domains CDB.",
|
|
}),
|
|
CDBReloads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_cdb_reloads_total",
|
|
Help: "Domains CDB reload attempts by result (success, failure).",
|
|
}, []string{"result"}),
|
|
Dropped: prometheus.NewCounterVec(prometheus.CounterOpts{
|
|
Name: "migadu_counters_dropped_total",
|
|
Help: "Validated requests refused with 503 because the counting queue was full, by route.",
|
|
}, []string{"route"}),
|
|
QueueDepth: prometheus.NewGauge(prometheus.GaugeOpts{
|
|
Name: "migadu_counters_queue_depth",
|
|
Help: "Jobs waiting in the counting queue.",
|
|
}),
|
|
QueueWait: prometheus.NewHistogram(prometheus.HistogramOpts{
|
|
Name: "migadu_counters_queue_wait_seconds",
|
|
Help: "Time a job waited in the counting queue before a worker took it.",
|
|
Buckets: []float64{.0001, .0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5},
|
|
}),
|
|
}
|
|
reg.MustRegister(m.Requests, m.Duration, m.Increments, m.LimitReached, m.WarningReached, m.Notifications, m.RedisErrors, m.CDBDomains, m.CDBReloads,
|
|
m.Dropped, m.QueueDepth, m.QueueWait)
|
|
return m
|
|
}
|
|
|
|
// RecordDropped counts a request refused because the queue was full.
|
|
func (m *Metrics) RecordDropped(route string) { m.Dropped.WithLabelValues(route).Inc() }
|
|
|
|
// SetQueueDepth publishes the current queue depth.
|
|
func (m *Metrics) SetQueueDepth(n int) { m.QueueDepth.Set(float64(n)) }
|
|
|
|
// ObserveQueueWait records how long a job waited for a worker.
|
|
func (m *Metrics) ObserveQueueWait(seconds float64) { m.QueueWait.Observe(seconds) }
|
|
|
|
// RecordRequest counts one HTTP request and observes its latency.
|
|
func (m *Metrics) RecordRequest(route, status string, seconds float64) {
|
|
m.Requests.WithLabelValues(route, status).Inc()
|
|
m.Duration.WithLabelValues(route).Observe(seconds)
|
|
}
|
|
|
|
// RecordIncrement counts one counter increment.
|
|
func (m *Metrics) RecordIncrement(level, flow string) {
|
|
m.Increments.WithLabelValues(level, flow).Inc()
|
|
}
|
|
|
|
// RecordLimitReached counts one limit crossing.
|
|
func (m *Metrics) RecordLimitReached(level, flow string) {
|
|
m.LimitReached.WithLabelValues(level, flow).Inc()
|
|
}
|
|
|
|
// RecordNotification counts one notification outcome.
|
|
func (m *Metrics) RecordNotification(kind, level, result string) {
|
|
m.Notifications.WithLabelValues(kind, level, result).Inc()
|
|
}
|
|
|
|
// RecordWarningReached counts one early-warning crossing.
|
|
func (m *Metrics) RecordWarningReached(level, flow string) {
|
|
m.WarningReached.WithLabelValues(level, flow).Inc()
|
|
}
|
|
|
|
// RecordRedisError counts one failed Redis operation.
|
|
func (m *Metrics) RecordRedisError(op string) { m.RedisErrors.WithLabelValues(op).Inc() }
|
|
|
|
// SetCDBDomains publishes the record count of the loaded CDB.
|
|
func (m *Metrics) SetCDBDomains(n int) { m.CDBDomains.Set(float64(n)) }
|
|
|
|
// RecordReload counts one CDB reload attempt.
|
|
func (m *Metrics) RecordReload(ok bool) {
|
|
result := "failure"
|
|
if ok {
|
|
result = "success"
|
|
}
|
|
m.CDBReloads.WithLabelValues(result).Inc()
|
|
}
|