First commit Go counter service
This commit is contained in:
Dimitrije Dimitrijevic
@@ -0,0 +1,369 @@
|
||||
// Package config loads and validates the migadu_counters TOML configuration,
|
||||
// shared by the daemon (cmd/migadu_counters) and the CLI (cmd/counterctl).
|
||||
package config
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/BurntSushi/toml"
|
||||
)
|
||||
|
||||
// DefaultPaths are tried in order when no -config flag is given.
|
||||
var DefaultPaths = []string{
|
||||
"/etc/migadu_counters/config.toml",
|
||||
"/usr/local/etc/migadu_counters/config.toml",
|
||||
}
|
||||
|
||||
// Environment variables that override the secrets from the file.
|
||||
const (
|
||||
EnvAuthToken = "MIGADU_COUNTERS_AUTH_TOKEN"
|
||||
EnvRedisPassword = "MIGADU_COUNTERS_REDIS_PASSWORD"
|
||||
EnvNotifyPassword = "MIGADU_COUNTERS_NOTIFY_PASSWORD"
|
||||
)
|
||||
|
||||
// NotifyPlaceholders are the template variables every notify URL is expected
|
||||
// to carry: the web app's /api/limits action pattern-matches all six query
|
||||
// parameters and answers 500 when one is missing.
|
||||
var NotifyPlaceholders = []string{"$LEVEL", "$ID", "$FLOW", "$PERIOD", "$ADDRESS", "$COUNT"}
|
||||
|
||||
// Config is the full daemon configuration.
|
||||
type Config struct {
|
||||
Addr string `toml:"addr"`
|
||||
AdminAddr string `toml:"admin_addr"`
|
||||
AuthToken string `toml:"auth_token"`
|
||||
AllowedCIDRs []string `toml:"allowed_cidrs"`
|
||||
DomainsCDB string `toml:"domains_cdb"`
|
||||
Exempt []string `toml:"exempt"`
|
||||
CountJunk bool `toml:"count_junk"`
|
||||
|
||||
// Counting runs off the request path: /in and /out enqueue a job and
|
||||
// answer 202; Workers goroutines drain a queue of QueueSize jobs.
|
||||
QueueSize int `toml:"queue_size"`
|
||||
Workers int `toml:"workers"`
|
||||
// Parsed CDB records are cached per loaded table up to this many
|
||||
// domains; beyond it lookups parse without caching. 0 disables the cache.
|
||||
MaxCachedDomains int `toml:"max_cached_domains"`
|
||||
|
||||
// HTTP server timeouts in seconds.
|
||||
ReadHeaderTimeout int `toml:"read_header_timeout"`
|
||||
WriteTimeout int `toml:"write_timeout"`
|
||||
IdleTimeout int `toml:"idle_timeout"`
|
||||
HandlerTimeout int `toml:"handler_timeout"`
|
||||
|
||||
Logging LoggingConfig `toml:"logging"`
|
||||
Redis RedisConfig `toml:"redis"`
|
||||
Notify NotifyConfig `toml:"notify"`
|
||||
|
||||
prefixes []netip.Prefix
|
||||
exempt map[string]bool
|
||||
}
|
||||
|
||||
// LoggingConfig holds the application log and the counting transaction log
|
||||
// settings (the first three fields match the rcptd/mailqueuer/mizu pattern).
|
||||
type LoggingConfig struct {
|
||||
Output string `toml:"output"` // "stderr", "stdout", or a file path
|
||||
Format string `toml:"format"` // "console" or "json"
|
||||
Level string `toml:"level"` // "debug", "info", "warn", "error"
|
||||
|
||||
CountingOutput string `toml:"counting_output"` // default: same as Output; "off" disables
|
||||
CountingFormat string `toml:"counting_format"` // "text" (Lua-like lines) or "json"
|
||||
}
|
||||
|
||||
// CountingEnabled reports whether the counting transaction log is written.
|
||||
func (l LoggingConfig) CountingEnabled() bool { return l.CountingOutput != "off" }
|
||||
|
||||
// RedisConfig describes the write (master) and read (replica) sides.
|
||||
// Sentinel mode is used when Sentinels is non-empty; otherwise the static
|
||||
// WriteAddr/ReadAddrs are used (development, tests).
|
||||
type RedisConfig struct {
|
||||
DB int `toml:"db"`
|
||||
Password string `toml:"password"`
|
||||
PoolSize int `toml:"pool_size"`
|
||||
TimeoutMS int `toml:"timeout_ms"`
|
||||
|
||||
MasterName string `toml:"master_name"`
|
||||
Sentinels []string `toml:"sentinels"`
|
||||
SentinelPassword string `toml:"sentinel_password"`
|
||||
|
||||
WriteAddr string `toml:"write_addr"`
|
||||
ReadAddrs []string `toml:"read_addrs"`
|
||||
}
|
||||
|
||||
// SentinelMode reports whether addresses are discovered through Sentinel.
|
||||
func (r RedisConfig) SentinelMode() bool { return len(r.Sentinels) > 0 }
|
||||
|
||||
// Timeout is the dial/read/write timeout for every Redis command.
|
||||
func (r RedisConfig) Timeout() time.Duration { return time.Duration(r.TimeoutMS) * time.Millisecond }
|
||||
|
||||
// NotifyConfig holds the per-level web app notification URLs. An empty URL
|
||||
// disables notifications for that level.
|
||||
type NotifyConfig struct {
|
||||
OrgURL string `toml:"org_url"`
|
||||
AddressURL string `toml:"address_url"`
|
||||
DomainURL string `toml:"domain_url"`
|
||||
User string `toml:"user"`
|
||||
Password string `toml:"password"`
|
||||
TimeoutSeconds int `toml:"timeout_seconds"`
|
||||
}
|
||||
|
||||
// Timeout is the per-attempt HTTP timeout for a notification.
|
||||
func (n NotifyConfig) Timeout() time.Duration { return time.Duration(n.TimeoutSeconds) * time.Second }
|
||||
|
||||
// Default returns the configuration with every default applied.
|
||||
func Default() Config {
|
||||
return Config{
|
||||
Addr: ":5555",
|
||||
AdminAddr: ":9095",
|
||||
Exempt: []string{"support@migadu.com", "contact@migadu.com"},
|
||||
QueueSize: 10000,
|
||||
Workers: 8,
|
||||
MaxCachedDomains: 20000,
|
||||
ReadHeaderTimeout: 5,
|
||||
WriteTimeout: 30,
|
||||
IdleTimeout: 60,
|
||||
HandlerTimeout: 4,
|
||||
Logging: LoggingConfig{Output: "stderr", Format: "console", Level: "info", CountingFormat: "text"},
|
||||
Redis: RedisConfig{PoolSize: 32, TimeoutMS: 500},
|
||||
Notify: NotifyConfig{TimeoutSeconds: 5},
|
||||
}
|
||||
}
|
||||
|
||||
// Discover returns flagPath when given, otherwise the first existing entry of
|
||||
// DefaultPaths, or an error naming every path that was tried.
|
||||
func Discover(flagPath string) (string, error) {
|
||||
if flagPath != "" {
|
||||
return flagPath, nil
|
||||
}
|
||||
for _, p := range DefaultPaths {
|
||||
if _, err := os.Stat(p); err == nil {
|
||||
return p, nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("no config found at %s; specify -config PATH", strings.Join(DefaultPaths, " or "))
|
||||
}
|
||||
|
||||
// Load reads the TOML file at path, applies defaults, environment overrides
|
||||
// and normalisation, and validates the result. Unknown keys are fatal: a typo
|
||||
// such as "auth-token" must not start the daemon with the default in place.
|
||||
func Load(path string) (*Config, error) {
|
||||
cfg := Default()
|
||||
md, err := toml.DecodeFile(path, &cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read config %s: %w", path, err)
|
||||
}
|
||||
var unknown []error
|
||||
for _, k := range md.Undecoded() {
|
||||
unknown = append(unknown, fmt.Errorf("unknown config key %s", k.String()))
|
||||
}
|
||||
if len(unknown) > 0 {
|
||||
return nil, fmt.Errorf("config %s: %w", path, errors.Join(unknown...))
|
||||
}
|
||||
|
||||
if v := os.Getenv(EnvAuthToken); v != "" {
|
||||
cfg.AuthToken = v
|
||||
}
|
||||
if v := os.Getenv(EnvRedisPassword); v != "" {
|
||||
cfg.Redis.Password = v
|
||||
}
|
||||
if v := os.Getenv(EnvNotifyPassword); v != "" {
|
||||
cfg.Notify.Password = v
|
||||
}
|
||||
|
||||
// A relative CDB path is resolved against the config file's directory,
|
||||
// like rcptd's cdb_tables_dir.
|
||||
if cfg.DomainsCDB != "" && !filepath.IsAbs(cfg.DomainsCDB) {
|
||||
cfg.DomainsCDB = filepath.Join(filepath.Dir(path), cfg.DomainsCDB)
|
||||
}
|
||||
|
||||
cfg.normalize()
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("config %s: %w", path, err)
|
||||
}
|
||||
return &cfg, nil
|
||||
}
|
||||
|
||||
func (c *Config) normalize() {
|
||||
c.exempt = make(map[string]bool, len(c.Exempt))
|
||||
for _, a := range c.Exempt {
|
||||
if a = strings.ToLower(strings.TrimSpace(a)); a != "" {
|
||||
c.exempt[a] = true
|
||||
}
|
||||
}
|
||||
|
||||
c.Logging.Format = strings.ToLower(strings.TrimSpace(c.Logging.Format))
|
||||
c.Logging.Level = strings.ToLower(strings.TrimSpace(c.Logging.Level))
|
||||
c.Logging.CountingFormat = strings.ToLower(strings.TrimSpace(c.Logging.CountingFormat))
|
||||
c.Logging.CountingOutput = strings.TrimSpace(c.Logging.CountingOutput)
|
||||
switch {
|
||||
case c.Logging.CountingOutput == "":
|
||||
c.Logging.CountingOutput = c.Logging.Output
|
||||
case strings.EqualFold(c.Logging.CountingOutput, "off"):
|
||||
c.Logging.CountingOutput = "off"
|
||||
}
|
||||
|
||||
if !c.Redis.SentinelMode() && len(c.Redis.ReadAddrs) == 0 && c.Redis.WriteAddr != "" {
|
||||
c.Redis.ReadAddrs = []string{c.Redis.WriteAddr}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate checks the configuration and returns every problem found, joined.
|
||||
func (c *Config) Validate() error {
|
||||
var errs []error
|
||||
fail := func(format string, args ...any) { errs = append(errs, fmt.Errorf(format, args...)) }
|
||||
|
||||
if c.DomainsCDB == "" {
|
||||
fail("domains_cdb is required")
|
||||
}
|
||||
if strings.TrimSpace(c.AuthToken) == "" {
|
||||
fail("auth_token is required; set it in the config or %s", EnvAuthToken)
|
||||
}
|
||||
if c.QueueSize <= 0 {
|
||||
fail("queue_size must be > 0")
|
||||
}
|
||||
if c.Workers <= 0 {
|
||||
fail("workers must be > 0")
|
||||
}
|
||||
if c.MaxCachedDomains < 0 {
|
||||
fail("max_cached_domains must be >= 0")
|
||||
}
|
||||
for _, a := range []struct{ name, addr string }{{"addr", c.Addr}, {"admin_addr", c.AdminAddr}} {
|
||||
if _, _, err := net.SplitHostPort(a.addr); err != nil {
|
||||
fail("%s: invalid listen address %q: %v", a.name, a.addr, err)
|
||||
}
|
||||
}
|
||||
for _, d := range []struct {
|
||||
name string
|
||||
v int
|
||||
}{{"read_header_timeout", c.ReadHeaderTimeout}, {"write_timeout", c.WriteTimeout}, {"idle_timeout", c.IdleTimeout}, {"handler_timeout", c.HandlerTimeout}} {
|
||||
if d.v <= 0 {
|
||||
fail("%s must be > 0 (seconds)", d.name)
|
||||
}
|
||||
}
|
||||
|
||||
c.prefixes = c.prefixes[:0]
|
||||
for _, s := range c.AllowedCIDRs {
|
||||
p, err := parsePrefix(s)
|
||||
if err != nil {
|
||||
fail("allowed_cidrs: invalid entry %q: %v", s, err)
|
||||
continue
|
||||
}
|
||||
c.prefixes = append(c.prefixes, p)
|
||||
}
|
||||
|
||||
switch c.Logging.Format {
|
||||
case "console", "json":
|
||||
default:
|
||||
fail("logging.format: %q (must be console or json)", c.Logging.Format)
|
||||
}
|
||||
if _, err := parseLogLevel(c.Logging.Level); err != nil {
|
||||
fail("logging.level: %v", err)
|
||||
}
|
||||
switch c.Logging.CountingFormat {
|
||||
case "text", "json":
|
||||
default:
|
||||
fail("logging.counting_format: %q (must be text or json)", c.Logging.CountingFormat)
|
||||
}
|
||||
|
||||
if c.Redis.DB != 0 {
|
||||
fail("redis.db must be 0: migadu_web reads the message counters from DB 0 (got %d)", c.Redis.DB)
|
||||
}
|
||||
if c.Redis.PoolSize <= 0 {
|
||||
fail("redis.pool_size must be > 0")
|
||||
}
|
||||
if c.Redis.TimeoutMS <= 0 {
|
||||
fail("redis.timeout_ms must be > 0")
|
||||
}
|
||||
if c.Redis.SentinelMode() {
|
||||
if c.Redis.MasterName == "" {
|
||||
fail("redis.master_name is required when redis.sentinels is set")
|
||||
}
|
||||
for _, s := range c.Redis.Sentinels {
|
||||
if _, _, err := net.SplitHostPort(s); err != nil {
|
||||
fail("redis.sentinels: invalid address %q: %v", s, err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if c.Redis.WriteAddr == "" {
|
||||
fail("redis.write_addr is required (or configure redis.sentinels)")
|
||||
} else if _, _, err := net.SplitHostPort(c.Redis.WriteAddr); err != nil {
|
||||
fail("redis.write_addr: invalid address %q: %v", c.Redis.WriteAddr, err)
|
||||
}
|
||||
for _, s := range c.Redis.ReadAddrs {
|
||||
if _, _, err := net.SplitHostPort(s); err != nil {
|
||||
fail("redis.read_addrs: invalid address %q: %v", s, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, u := range c.notifyURLs() {
|
||||
if u.url == "" {
|
||||
continue
|
||||
}
|
||||
parsed, err := url.Parse(u.url)
|
||||
if err != nil || (parsed.Scheme != "http" && parsed.Scheme != "https") || parsed.Host == "" {
|
||||
fail("notify.%s: must be an absolute http(s) URL (got %q)", u.name, u.url)
|
||||
}
|
||||
}
|
||||
if c.Notify.TimeoutSeconds <= 0 {
|
||||
fail("notify.timeout_seconds must be > 0")
|
||||
}
|
||||
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
// Warnings returns non-fatal findings worth logging at startup.
|
||||
func (c *Config) Warnings() []string {
|
||||
var w []string
|
||||
for _, u := range c.notifyURLs() {
|
||||
if u.url == "" {
|
||||
continue
|
||||
}
|
||||
for _, ph := range NotifyPlaceholders {
|
||||
if !strings.Contains(u.url, ph) {
|
||||
w = append(w, fmt.Sprintf("notify.%s: missing %s placeholder (the web app requires all six query parameters)", u.name, ph))
|
||||
}
|
||||
}
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
func (c *Config) notifyURLs() []struct{ name, url string } {
|
||||
return []struct{ name, url string }{
|
||||
{"org_url", c.Notify.OrgURL},
|
||||
{"address_url", c.Notify.AddressURL},
|
||||
{"domain_url", c.Notify.DomainURL},
|
||||
}
|
||||
}
|
||||
|
||||
// ExemptSet returns the normalised exempt addresses.
|
||||
func (c *Config) ExemptSet() map[string]bool { return c.exempt }
|
||||
|
||||
// AllowedPrefixes returns the parsed allowed_cidrs; empty means any source.
|
||||
func (c *Config) AllowedPrefixes() []netip.Prefix { return c.prefixes }
|
||||
|
||||
// HandlerTimeoutDuration is the http.TimeoutHandler budget.
|
||||
func (c *Config) HandlerTimeoutDuration() time.Duration {
|
||||
return time.Duration(c.HandlerTimeout) * time.Second
|
||||
}
|
||||
|
||||
// parsePrefix accepts "10.0.0.0/8" as well as a bare address ("192.168.1.5",
|
||||
// "::1"), which becomes a single-host prefix.
|
||||
func parsePrefix(s string) (netip.Prefix, error) {
|
||||
s = strings.TrimSpace(s)
|
||||
if p, err := netip.ParsePrefix(s); err == nil {
|
||||
return p.Masked(), nil
|
||||
}
|
||||
a, err := netip.ParseAddr(s)
|
||||
if err != nil {
|
||||
return netip.Prefix{}, errors.New("not a CIDR prefix or IP address")
|
||||
}
|
||||
return netip.PrefixFrom(a, a.BitLen()), nil
|
||||
}
|
||||
Reference in New Issue
Block a user