First commit Go counter service
This commit is contained in:
Dimitrije Dimitrijevic
@@ -0,0 +1,121 @@
|
||||
// Package metrics defines the Prometheus metrics served on the admin port.
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// Metrics holds every collector; fields are exported for tests.
|
||||
type Metrics struct {
|
||||
Requests *prometheus.CounterVec // route, status
|
||||
Duration *prometheus.HistogramVec // route
|
||||
Increments *prometheus.CounterVec // level, flow
|
||||
LimitReached *prometheus.CounterVec // level, flow
|
||||
Notifications *prometheus.CounterVec // level, result (sent|failed|disabled)
|
||||
RedisErrors *prometheus.CounterVec // op
|
||||
CDBDomains prometheus.Gauge
|
||||
CDBReloads *prometheus.CounterVec // result (success|failure)
|
||||
Dropped *prometheus.CounterVec // route: requests refused because the counting queue was full
|
||||
QueueDepth prometheus.Gauge
|
||||
QueueWait prometheus.Histogram
|
||||
}
|
||||
|
||||
// New creates the collectors and registers them with reg.
|
||||
func New(reg prometheus.Registerer) *Metrics {
|
||||
m := &Metrics{
|
||||
Requests: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_requests_total",
|
||||
Help: "HTTP requests by route and status code.",
|
||||
}, []string{"route", "status"}),
|
||||
Duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Name: "migadu_counters_request_duration_seconds",
|
||||
Help: "Request latency by route.",
|
||||
Buckets: []float64{.0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5},
|
||||
}, []string{"route"}),
|
||||
Increments: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_increments_total",
|
||||
Help: "Counter increments by level (A, D, O) and flow (i, o).",
|
||||
}, []string{"level", "flow"}),
|
||||
LimitReached: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_limit_reached_total",
|
||||
Help: "Increments that crossed a configured limit, by level and flow.",
|
||||
}, []string{"level", "flow"}),
|
||||
Notifications: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_notifications_total",
|
||||
Help: "Web app notifications by level and result (sent, failed, disabled).",
|
||||
}, []string{"level", "result"}),
|
||||
RedisErrors: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_redis_errors_total",
|
||||
Help: "Failed Redis operations by op (incr, mget, ping).",
|
||||
}, []string{"op"}),
|
||||
CDBDomains: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "migadu_counters_cdb_domains",
|
||||
Help: "Domains in the currently loaded domains CDB.",
|
||||
}),
|
||||
CDBReloads: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_cdb_reloads_total",
|
||||
Help: "Domains CDB reload attempts by result (success, failure).",
|
||||
}, []string{"result"}),
|
||||
Dropped: prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "migadu_counters_dropped_total",
|
||||
Help: "Validated requests refused with 503 because the counting queue was full, by route.",
|
||||
}, []string{"route"}),
|
||||
QueueDepth: prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "migadu_counters_queue_depth",
|
||||
Help: "Jobs waiting in the counting queue.",
|
||||
}),
|
||||
QueueWait: prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "migadu_counters_queue_wait_seconds",
|
||||
Help: "Time a job waited in the counting queue before a worker took it.",
|
||||
Buckets: []float64{.0001, .0005, .001, .0025, .005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5},
|
||||
}),
|
||||
}
|
||||
reg.MustRegister(m.Requests, m.Duration, m.Increments, m.LimitReached, m.Notifications, m.RedisErrors, m.CDBDomains, m.CDBReloads,
|
||||
m.Dropped, m.QueueDepth, m.QueueWait)
|
||||
return m
|
||||
}
|
||||
|
||||
// RecordDropped counts a request refused because the queue was full.
|
||||
func (m *Metrics) RecordDropped(route string) { m.Dropped.WithLabelValues(route).Inc() }
|
||||
|
||||
// SetQueueDepth publishes the current queue depth.
|
||||
func (m *Metrics) SetQueueDepth(n int) { m.QueueDepth.Set(float64(n)) }
|
||||
|
||||
// ObserveQueueWait records how long a job waited for a worker.
|
||||
func (m *Metrics) ObserveQueueWait(seconds float64) { m.QueueWait.Observe(seconds) }
|
||||
|
||||
// RecordRequest counts one HTTP request and observes its latency.
|
||||
func (m *Metrics) RecordRequest(route, status string, seconds float64) {
|
||||
m.Requests.WithLabelValues(route, status).Inc()
|
||||
m.Duration.WithLabelValues(route).Observe(seconds)
|
||||
}
|
||||
|
||||
// RecordIncrement counts one counter increment.
|
||||
func (m *Metrics) RecordIncrement(level, flow string) {
|
||||
m.Increments.WithLabelValues(level, flow).Inc()
|
||||
}
|
||||
|
||||
// RecordLimitReached counts one limit crossing.
|
||||
func (m *Metrics) RecordLimitReached(level, flow string) {
|
||||
m.LimitReached.WithLabelValues(level, flow).Inc()
|
||||
}
|
||||
|
||||
// RecordNotification counts one notification outcome.
|
||||
func (m *Metrics) RecordNotification(level, result string) {
|
||||
m.Notifications.WithLabelValues(level, result).Inc()
|
||||
}
|
||||
|
||||
// RecordRedisError counts one failed Redis operation.
|
||||
func (m *Metrics) RecordRedisError(op string) { m.RedisErrors.WithLabelValues(op).Inc() }
|
||||
|
||||
// SetCDBDomains publishes the record count of the loaded CDB.
|
||||
func (m *Metrics) SetCDBDomains(n int) { m.CDBDomains.Set(float64(n)) }
|
||||
|
||||
// RecordReload counts one CDB reload attempt.
|
||||
func (m *Metrics) RecordReload(ok bool) {
|
||||
result := "failure"
|
||||
if ok {
|
||||
result = "success"
|
||||
}
|
||||
m.CDBReloads.WithLabelValues(result).Inc()
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/testutil"
|
||||
)
|
||||
|
||||
func TestMetrics(t *testing.T) {
|
||||
reg := prometheus.NewRegistry()
|
||||
m := New(reg)
|
||||
|
||||
m.RecordRequest("out", "200", 0.001)
|
||||
m.RecordRequest("out", "200", 0.002)
|
||||
m.RecordRequest("in", "401", 0.0005)
|
||||
m.RecordIncrement("A", "o")
|
||||
m.RecordIncrement("A", "o")
|
||||
m.RecordIncrement("O", "i")
|
||||
m.RecordLimitReached("O", "o")
|
||||
m.RecordNotification("O", "sent")
|
||||
m.RecordNotification("A", "failed")
|
||||
m.RecordRedisError("incr")
|
||||
m.SetCDBDomains(93549)
|
||||
m.RecordReload(true)
|
||||
m.RecordReload(false)
|
||||
m.RecordReload(false)
|
||||
m.RecordDropped("/in")
|
||||
m.RecordDropped("/in")
|
||||
m.SetQueueDepth(7)
|
||||
m.ObserveQueueWait(0.01)
|
||||
|
||||
checks := []struct {
|
||||
name string
|
||||
c prometheus.Collector
|
||||
want float64
|
||||
}{
|
||||
{"requests out 200", m.Requests.WithLabelValues("out", "200"), 2},
|
||||
{"requests in 401", m.Requests.WithLabelValues("in", "401"), 1},
|
||||
{"increments A o", m.Increments.WithLabelValues("A", "o"), 2},
|
||||
{"increments O i", m.Increments.WithLabelValues("O", "i"), 1},
|
||||
{"limit reached O o", m.LimitReached.WithLabelValues("O", "o"), 1},
|
||||
{"notifications O sent", m.Notifications.WithLabelValues("O", "sent"), 1},
|
||||
{"notifications A failed", m.Notifications.WithLabelValues("A", "failed"), 1},
|
||||
{"redis errors incr", m.RedisErrors.WithLabelValues("incr"), 1},
|
||||
{"cdb domains", m.CDBDomains, 93549},
|
||||
{"reload success", m.CDBReloads.WithLabelValues("success"), 1},
|
||||
{"reload failure", m.CDBReloads.WithLabelValues("failure"), 2},
|
||||
{"dropped /in", m.Dropped.WithLabelValues("/in"), 2},
|
||||
{"queue depth", m.QueueDepth, 7},
|
||||
}
|
||||
for _, c := range checks {
|
||||
if got := testutil.ToFloat64(c.c); got != c.want {
|
||||
t.Errorf("%s = %v, want %v", c.name, got, c.want)
|
||||
}
|
||||
}
|
||||
|
||||
families, err := reg.Gather()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, f := range families {
|
||||
seen[f.GetName()] = true
|
||||
if f.GetName() == "migadu_counters_request_duration_seconds" {
|
||||
for _, metric := range f.GetMetric() {
|
||||
for _, l := range metric.GetLabel() {
|
||||
if l.GetName() == "route" && l.GetValue() == "out" && metric.GetHistogram().GetSampleCount() != 2 {
|
||||
t.Errorf("duration histogram for out has %d samples, want 2", metric.GetHistogram().GetSampleCount())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, name := range []string{
|
||||
"migadu_counters_requests_total",
|
||||
"migadu_counters_request_duration_seconds",
|
||||
"migadu_counters_increments_total",
|
||||
"migadu_counters_limit_reached_total",
|
||||
"migadu_counters_notifications_total",
|
||||
"migadu_counters_redis_errors_total",
|
||||
"migadu_counters_cdb_domains",
|
||||
"migadu_counters_cdb_reloads_total",
|
||||
"migadu_counters_dropped_total",
|
||||
"migadu_counters_queue_depth",
|
||||
"migadu_counters_queue_wait_seconds",
|
||||
} {
|
||||
if !seen[name] {
|
||||
t.Errorf("metric %s not registered (have %v)", name, keys(seen))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func keys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Two daemons' worth of metrics (tests, counterctl) must not collide: each
|
||||
// registry gets its own collectors.
|
||||
func TestNew_SeparateRegistries(t *testing.T) {
|
||||
a := New(prometheus.NewRegistry())
|
||||
b := New(prometheus.NewRegistry())
|
||||
a.RecordRequest("in", "200", 0.1)
|
||||
if testutil.ToFloat64(b.Requests.WithLabelValues("in", "200")) != 0 {
|
||||
t.Error("registries share state")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user