amadan

carlos / platform Public

alerts: the geoip box rule (reason code, 30-minute sustain, clears on zero), its sentences and dashboard text (geoip spec §10.4)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 9f067764ede9e8ad84f37428e35059467e06f850 parent 9c54868
7 files changed, +123 −0
  • internal/console/alertwatch.go +20 −0
  • internal/host/agent.go +16 −0
  • internal/host/alertsink.go +21 −0
  • internal/host/alertsink_geoip_test.go +26 −0
  • internal/host/metrics_loop.go +1 −0
  • internal/metrics/alert.go +12 −0
  • internal/metrics/alert_test.go +27 −0
diff --git a/internal/console/alertwatch.go b/internal/console/alertwatch.go
index 2a2f8fde5..3ec64c0a3 100644
--- a/internal/console/alertwatch.go
+++ b/internal/console/alertwatch.go
@@ -274,6 +274,9 @@ func alertValueText(rule string, value float64) string {
return fmt.Sprintf("%.0f%%", value*100)
case "lag":
return fmt.Sprintf("%.0fs", value)
+ case "geoip":
+ // A reason code, not a measurement (geoip spec §10.4).
+ return geoipReasonText(int(value))
default:
return strconv.FormatFloat(value, 'g', -1, 64)
}
@@ -299,3 +302,20 @@ func alertMessage(box, rule string, rs metrics.RuleState, updated time.Time) (su
"You are getting this because you hold the CARLOS fleet-operator role.\n", rule, box, alertValueText(rule, rs.Value), stamp)
return subject, body
}
+
+// geoipReasonText names a geoip rule reason code (geoip spec §10.4).
+func geoipReasonText(code int) string {
+ switch code {
+ case 1:
+ return "database over 45 days old"
+ case 2:
+ return "GeoLite2 database expires within 9 days"
+ case 3:
+ return "database expired and removed"
+ case 4:
+ return "published database not loaded for over an hour"
+ case 5:
+ return "feed failing"
+ }
+ return "geoip reason " + strconv.Itoa(code)
+}
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 5b934c27e..e6bf35fe5 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -588,6 +588,7 @@ func RunAgent(args []string) {
setUpstreamSource(e.UpstreamStates)
// The GeoIP block for the heartbeat (geoip spec §5.5), the same way.
geoIPSource.Store(e.GeoIPState)
+ geoIPAlertSource.Store(e.GeoIPAlertCode)
// The layout pass reloads routing through this as soon as a move
// commits, instead of waiting for the 2s registry poll.
reload := func() { _ = e.Reload() }
@@ -2791,6 +2792,21 @@ var upstreamSource atomic.Value // func() map[string][]status.UpstreamState
// the edge-ready callback like upstreamSource (geoip spec §5.5).
var geoIPSource atomic.Value // func() edge.GeoIPState
+// geoIPAlertSource is the edge's geoip reason code for the box alert rule
+// (geoip spec §10.4), published from the same ready callback.
+var geoIPAlertSource atomic.Value // func(time.Time) float64
+
+// geoIPAlertReading is the rule's reading, or nil before the edge is ready
+// (no data never fires and never clears).
+func geoIPAlertReading(now time.Time) *float64 {
+ fn, _ := geoIPAlertSource.Load().(func(time.Time) float64)
+ if fn == nil {
+ return nil
+ }
+ v := fn(now)
+ return &v
+}
+
// geoIPState reads it, or nil before the edge is ready.
func geoIPState() *status.GeoIPState {
fn, _ := geoIPSource.Load().(func() edge.GeoIPState)
diff --git a/internal/host/alertsink.go b/internal/host/alertsink.go
index c9aca34d8..cbb7d6e80 100644
--- a/internal/host/alertsink.go
+++ b/internal/host/alertsink.go
@@ -288,6 +288,8 @@ func alertSentence(ev metrics.Event) string {
return fmt.Sprintf("cpu use is %s on %s", pct(ev.Value), ev.Box)
case "lag":
return fmt.Sprintf("mail relay lag is %d seconds on %s", int64(ev.Value), ev.Box)
+ case "geoip":
+ return geoipAlertSentence(ev)
default:
return fmt.Sprintf("%s is firing on %s", ev.Rule, ev.Box)
}
@@ -303,6 +305,8 @@ func alertClearSentence(ev metrics.Event) string {
return fmt.Sprintf("cpu use is back to %s on %s", pct(ev.Value), ev.Box)
case "lag":
return fmt.Sprintf("mail relay lag is back to %d seconds on %s", int64(ev.Value), ev.Box)
+ case "geoip":
+ return fmt.Sprintf("geoip on %s is back to normal", ev.Box)
default:
return fmt.Sprintf("%s has cleared on %s", ev.Rule, ev.Box)
}
@@ -332,5 +336,22 @@ func alertSustain(r metrics.Rules, rule string) time.Duration {
return 0
}
+// geoipAlertSentence reads the geoip rule's reason code (geoip spec §10.4).
+func geoipAlertSentence(ev metrics.Event) string {
+ switch int(ev.Value) {
+ case 1:
+ return fmt.Sprintf("the geoip database on %s is more than 45 days old", ev.Box)
+ case 2:
+ return fmt.Sprintf("the GeoLite2 database on %s expires within 9 days", ev.Box)
+ case 3:
+ return fmt.Sprintf("the geoip database on %s has expired and was removed", ev.Box)
+ case 4:
+ return fmt.Sprintf("%s has not loaded the published geoip database for over an hour", ev.Box)
+ case 5:
+ return fmt.Sprintf("the geoip feed has failed 7 times in a row (seen from %s)", ev.Box)
+ }
+ return fmt.Sprintf("geoip needs attention on %s", ev.Box)
+}
+
// pct renders a 0..1 fraction as a whole percent.
func pct(f float64) string { return fmt.Sprintf("%d%%", int(math.Round(f*100))) }
diff --git a/internal/host/alertsink_geoip_test.go b/internal/host/alertsink_geoip_test.go
new file mode 100644
index 000000000..0025e919b
--- /dev/null
+++ b/internal/host/alertsink_geoip_test.go
@@ -0,0 +1,26 @@
+package host
+
+import (
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/metrics"
+)
+
+func TestGeoIPAlertSentences(t *testing.T) {
+ for code, want := range map[float64]string{
+ 1: "more than 45 days old", 2: "expires within 9 days", 3: "has expired and was removed",
+ 4: "has not loaded the published geoip database", 5: "has failed 7 times in a row",
+ } {
+ if got := alertSentence(metrics.Event{Rule: "geoip", Firing: true, Value: code, Box: "b"}); !strings.Contains(got, want) {
+ t.Errorf("code %v = %q, want %q", code, got, want)
+ }
+ }
+ if got := alertClearSentence(metrics.Event{Rule: "geoip", Box: "b"}); got != "geoip on b is back to normal" {
+ t.Errorf("clear = %q", got)
+ }
+ if geoIPAlertReading(time.Now()) != nil {
+ t.Error("no edge published must mean no reading")
+ }
+}
diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go
index cbd2cc382..f74753a68 100644
--- a/internal/host/metrics_loop.go
+++ b/internal/host/metrics_loop.go
@@ -492,6 +492,7 @@ func alertTick(ctx context.Context, s release.Store, box string, rules metrics.R
MemFrac: latest.Host.MemFrac,
DiskFrac: latest.Host.DiskFrac,
LagSecs: nil, // v1: no lag reading feeds the agent loop yet
+ GeoIP: geoIPAlertReading(latest.At),
}
newState, events := metrics.Evaluate(rules, *st, obs)
diff --git a/internal/metrics/alert.go b/internal/metrics/alert.go
index 9cb788aee..c4a087500 100644
--- a/internal/metrics/alert.go
+++ b/internal/metrics/alert.go
@@ -16,6 +16,9 @@ type Rules struct {
CPUSustain time.Duration // default 15 * time.Minute
LagSecs int64 // default 900, immediate; inert while readings are nil
ClearBy float64 // hysteresis, default 0.05 (fractions); lag clears at LagSecs-300
+ // GeoIPSustain is how long a geoip reason must hold before the rule
+ // fires (geoip spec §10.4); it clears on the first reading of 0.
+ GeoIPSustain time.Duration // default 30 * time.Minute
}
// DefaultRules are the thresholds shipped with the platform.
@@ -28,6 +31,8 @@ func DefaultRules() Rules {
CPUSustain: 15 * time.Minute,
LagSecs: 900,
ClearBy: 0.05,
+ // GeoIPSustain: geoip spec §10.4.
+ GeoIPSustain: 30 * time.Minute,
}
}
@@ -77,6 +82,10 @@ type Observation struct {
MemFrac *float64
DiskFrac *float64
LagSecs *int64
+ // GeoIP is the box's geoip reason code (geoip spec §10.4): 0 none,
+ // 1 stale, 2 expiring, 3 expired, 4 not loaded, 5 feed failing. nil on
+ // a box whose edge has not published one.
+ GeoIP *float64
}
// RuleState is one rule's sustain/hysteresis bookkeeping. Since and FiredAt
@@ -250,6 +259,9 @@ func Evaluate(r Rules, st AlertState, obs Observation) (AlertState, []Event) {
evalFrac("mem", obs.MemFrac, r.MemFrac, r.MemSustain)
evalFrac("cpu", obs.CPUFrac, r.CPUFrac, r.CPUSustain)
evalLagSecs("lag", obs.LagSecs, r.LagSecs, 300, 0)
+ // geoip: a reason code against 0.5, so any reason fires after the
+ // sustain and a reading of 0 clears (geoip spec §10.4).
+ evalFrac("geoip", obs.GeoIP, 0.5, r.GeoIPSustain)
return out, events
}
diff --git a/internal/metrics/alert_test.go b/internal/metrics/alert_test.go
index 60e0acd12..4046d9cf9 100644
--- a/internal/metrics/alert_test.go
+++ b/internal/metrics/alert_test.go
@@ -199,3 +199,30 @@ func TestRulesFromEnv(t *testing.T) {
t.Fatalf("want CPUFrac 0.80, got %v", r.CPUFrac)
}
}
+
+func TestGeoIPRuleFiresAfterItsSustainAndClearsOnZero(t *testing.T) {
+ r := DefaultRules()
+ t0 := time.Date(2026, 10, 6, 0, 0, 0, 0, time.UTC)
+ one, zero := 1.0, 0.0
+ st := AlertState{V: 1, Box: "b"}
+ st, ev := Evaluate(r, st, Observation{At: t0, GeoIP: &one})
+ if len(ev) != 0 {
+ t.Fatalf("fired at once: %v", ev)
+ }
+ st, ev = Evaluate(r, st, Observation{At: t0.Add(29 * time.Minute), GeoIP: &one})
+ if len(ev) != 0 {
+ t.Fatalf("fired before 30 minutes: %v", ev)
+ }
+ st, ev = Evaluate(r, st, Observation{At: t0.Add(30 * time.Minute), GeoIP: &one})
+ if len(ev) != 1 || !ev[0].Firing || ev[0].Rule != "geoip" || ev[0].Value != 1 {
+ t.Fatalf("at 30 minutes: %v", ev)
+ }
+ st, ev = Evaluate(r, st, Observation{At: t0.Add(31 * time.Minute)}) // nil: no change
+ if len(ev) != 0 || !st.Rules["geoip"].Firing {
+ t.Fatalf("a nil reading changed the rule: %v %v", ev, st.Rules["geoip"])
+ }
+ _, ev = Evaluate(r, st, Observation{At: t0.Add(32 * time.Minute), GeoIP: &zero})
+ if len(ev) != 1 || ev[0].Firing {
+ t.Fatalf("a zero reading must clear at once: %v", ev)
+ }
+}