amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package host
2
3import (
4 "context"
5 "errors"
6 "os"
7 "path/filepath"
8 "strings"
9 "testing"
10 "time"
11
12 "github.com/aws/aws-sdk-go-v2/service/sns"
13
14 "github.com/carlosframework/platform/internal/metrics"
15 "github.com/carlosframework/platform/internal/release"
16)
17
18// fakeSNS records every Publish and can be told to fail. It is the only SNS
19// the tests in this file ever talk to.
20type fakeSNS struct {
21 in []*sns.PublishInput
22 err error
23}
24
25func (f *fakeSNS) Publish(_ context.Context, in *sns.PublishInput, _ ...func(*sns.Options)) (*sns.PublishOutput, error) {
26 f.in = append(f.in, in)
27 if f.err != nil {
28 return nil, f.err
29 }
30 return &sns.PublishOutput{}, nil
31}
32
33const testTopicARN = "arn:aws:sns:eu-west-1:271376211898:carlos-flagship-ie-alerts"
34
35var sinkTestAt = time.Date(2026, 10, 5, 12, 0, 0, 0, time.UTC)
36
37func TestFormatAlertFireAndClear(t *testing.T) {
38 rules := metrics.DefaultRules()
39 cases := []struct {
40 name string
41 ev metrics.Event
42 subjectHas []string
43 bodyHas []string
44 subjectNot []string
45 bodyNotHave []string
46 }{
47 {
48 name: "disk fires",
49 ev: metrics.Event{Rule: "disk", Firing: true, Value: 0.917, Box: "ada-stargazing"},
50 subjectHas: []string{"carlos alert", "disk", "92%", "ada-stargazing"},
51 bodyHas: []string{"disk", "92%", "ada-stargazing", "90%", "2026-10-05 12:00 UTC", "clears"},
52 },
53 {
54 name: "disk clears",
55 ev: metrics.Event{Rule: "disk", Firing: false, Value: 0.30, Box: "ada-stargazing"},
56 subjectHas: []string{"carlos alert cleared", "disk", "30%", "ada-stargazing"},
57 bodyHas: []string{"back", "30%", "ada-stargazing"},
58 },
59 {
60 name: "mem fires with its sustain window",
61 ev: metrics.Event{Rule: "mem", Firing: true, Value: 0.94, Box: "box-a"},
62 subjectHas: []string{"memory", "94%", "box-a"},
63 bodyHas: []string{"10 minutes", "90%"},
64 },
65 {
66 name: "cpu fires with its sustain window",
67 ev: metrics.Event{Rule: "cpu", Firing: true, Value: 0.97, Box: "box-a"},
68 subjectHas: []string{"cpu", "97%", "box-a"},
69 bodyHas: []string{"15 minutes", "95%"},
70 },
71 {
72 name: "lag is seconds not a percentage",
73 ev: metrics.Event{Rule: "lag", Firing: true, Value: 1200, Box: "box-a"},
74 subjectHas: []string{"lag", "1200 seconds"},
75 subjectNot: []string{"%"},
76 bodyHas: []string{"900 seconds"},
77 },
78 {
79 name: "an unknown rule still reads",
80 ev: metrics.Event{Rule: "newrule", Firing: true, Value: 0.5, Box: "box-a"},
81 subjectHas: []string{"newrule", "box-a"},
82 },
83 }
84 for _, c := range cases {
85 t.Run(c.name, func(t *testing.T) {
86 subject, body := formatAlert(rules, sinkTestAt, c.ev)
87 for _, w := range c.subjectHas {
88 if !strings.Contains(subject, w) {
89 t.Errorf("subject %q missing %q", subject, w)
90 }
91 }
92 for _, w := range c.subjectNot {
93 if strings.Contains(subject, w) {
94 t.Errorf("subject %q should not contain %q", subject, w)
95 }
96 }
97 for _, w := range c.bodyHas {
98 if !strings.Contains(body, w) {
99 t.Errorf("body missing %q:\n%s", w, body)
100 }
101 }
102 // SNS rejects a subject over 100 chars, or with a newline or
103 // any non-printable-ASCII byte; the mail would never be sent.
104 if len(subject) == 0 || len(subject) > 100 {
105 t.Errorf("subject length %d, want 1..100: %q", len(subject), subject)
106 }
107 for _, r := range subject {
108 if r < 0x20 || r > 0x7e {
109 t.Errorf("subject has a character SNS refuses: %q", subject)
110 break
111 }
112 }
113 for _, s := range []string{subject, body} {
114 if strings.ContainsRune(s, '\u2014') || strings.ContainsRune(s, '\u2013') {
115 t.Errorf("em or en dash in copy: %q", s)
116 }
117 }
118 })
119 }
120}
121
122// TestFormatAlertCarriesNoSecrets: the message is the event, the rule's
123// level and a time. No URL, ARN, credential, or env value can reach it,
124// because nothing but those three inputs is passed in.
125func TestFormatAlertCarriesNoSecrets(t *testing.T) {
126 subject, body := formatAlert(metrics.DefaultRules(), sinkTestAt,
127 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
128 for _, banned := range []string{"http", "arn:", "secret", "token", "key", "password"} {
129 if strings.Contains(strings.ToLower(subject+body), banned) {
130 t.Errorf("message contains %q:\n%s\n%s", banned, subject, body)
131 }
132 }
133}
134
135func TestAlertSinksDeliverSNSOnly(t *testing.T) {
136 pub := &fakeSNS{}
137 counts := &alertCounters{}
138 s := newAlertSinks("", nil, testTopicARN, pub, counts)
139 s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
140 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
141 if len(pub.in) != 1 {
142 t.Fatalf("publishes = %d, want 1", len(pub.in))
143 }
144 in := pub.in[0]
145 if got := deref(in.TopicArn); got != testTopicARN {
146 t.Errorf("TopicArn = %q", got)
147 }
148 if !strings.Contains(deref(in.Subject), "disk") || !strings.Contains(deref(in.Message), "box-a") {
149 t.Errorf("subject/message not readable: %q / %q", deref(in.Subject), deref(in.Message))
150 }
151 if counts.sns.Load() != 0 || counts.webhook.Load() != 0 {
152 t.Errorf("no failures expected, got sns=%d webhook=%d", counts.sns.Load(), counts.webhook.Load())
153 }
154}
155
156func TestAlertSinksDeliverWebhookOnlyIsUnchanged(t *testing.T) {
157 var posts [][]byte
158 var urls []string
159 post := func(url string, body []byte) error {
160 urls = append(urls, url)
161 posts = append(posts, body)
162 return nil
163 }
164 s := newAlertSinks("http://example.test/hook", post, "", nil, &alertCounters{})
165 s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
166 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
167 if len(posts) != 1 || urls[0] != "http://example.test/hook" {
168 t.Fatalf("posts = %d urls = %v", len(posts), urls)
169 }
170 if string(posts[0]) != `{"rule":"disk","firing":true,"value":0.95,"box":"box-a"}` {
171 t.Errorf("webhook payload changed: %s", posts[0])
172 }
173}
174
175func TestAlertSinksBothAreUsed(t *testing.T) {
176 pub := &fakeSNS{}
177 posts := 0
178 post := func(string, []byte) error { posts++; return nil }
179 s := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, &alertCounters{})
180 s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
181 metrics.Event{Rule: "mem", Firing: true, Value: 0.95, Box: "box-a"})
182 if posts != 1 || len(pub.in) != 1 {
183 t.Errorf("webhook posts = %d, sns publishes = %d, want 1 and 1", posts, len(pub.in))
184 }
185}
186
187func TestAlertSinksNoneConfiguredDoesNothing(t *testing.T) {
188 s := newAlertSinks("", nil, "", nil, &alertCounters{})
189 s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
190 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
191 var nilSinks *alertSinks
192 nilSinks.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
193 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
194}
195
196// TestAlertSinksSNSFailureIsCountedAndNeverBlocksTheWebhook: a failing sink
197// is logged and counted, the other sink still runs, and nothing panics or
198// returns an error that could fail the tick.
199func TestAlertSinksSNSFailureIsCountedAndNeverBlocksTheWebhook(t *testing.T) {
200 pub := &fakeSNS{err: errors.New("AccessDenied: not authorized to perform sns:Publish")}
201 posts := 0
202 post := func(string, []byte) error { posts++; return nil }
203 counts := &alertCounters{}
204 s := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, counts)
205 s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
206 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
207 if posts != 1 {
208 t.Errorf("webhook posts = %d, want 1 even though sns failed", posts)
209 }
210 if counts.sns.Load() != 1 {
211 t.Errorf("sns failures = %d, want 1", counts.sns.Load())
212 }
213 if counts.webhook.Load() != 0 {
214 t.Errorf("webhook failures = %d, want 0", counts.webhook.Load())
215 }
216}
217
218func TestAlertSinksWebhookFailureIsCounted(t *testing.T) {
219 pub := &fakeSNS{}
220 post := func(string, []byte) error { return errors.New("boom") }
221 counts := &alertCounters{}
222 s := newAlertSinks("http://example.test/hook", post, testTopicARN, pub, counts)
223 s.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
224 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
225 if counts.webhook.Load() != 1 || len(pub.in) != 1 {
226 t.Errorf("webhook failures = %d (want 1), sns publishes = %d (want 1)", counts.webhook.Load(), len(pub.in))
227 }
228}
229
230// TestAlertTickDeliversThroughSinks: alertTick hands each transition to the
231// sinks exactly once, and a steady state sends nothing more.
232func TestAlertTickDeliversThroughSinks(t *testing.T) {
233 ctx := context.Background()
234 store := release.NewDirStore(t.TempDir())
235 statePath := filepath.Join(t.TempDir(), "monitor", "alerts.json")
236 box := "box-a"
237 rules := metrics.DefaultRules()
238 st := &metrics.AlertState{V: 1, Box: box, Rules: map[string]metrics.RuleState{}}
239 pub := &fakeSNS{}
240 sinks := newAlertSinks("", nil, testTopicARN, pub, &alertCounters{})
241
242 disk := 0.95
243 latest := metrics.Sample{At: sinkTestAt, Host: metrics.HostSample{DiskFrac: &disk}}
244 if err := alertTick(ctx, store, box, rules, st, latest, statePath, sinks); err != nil {
245 t.Fatal(err)
246 }
247 if err := alertTick(ctx, store, box, rules, st, latest, statePath, sinks); err != nil {
248 t.Fatal(err)
249 }
250 if len(pub.in) != 1 {
251 t.Fatalf("publishes = %d, want 1 (second tick has no transition)", len(pub.in))
252 }
253 if !strings.Contains(deref(pub.in[0].Message), box) {
254 t.Errorf("message does not name the box: %q", deref(pub.in[0].Message))
255 }
256
257 // Clearing is a transition too.
258 low := 0.30
259 cleared := metrics.Sample{At: sinkTestAt.Add(time.Minute), Host: metrics.HostSample{DiskFrac: &low}}
260 if err := alertTick(ctx, store, box, rules, st, cleared, statePath, sinks); err != nil {
261 t.Fatal(err)
262 }
263 if len(pub.in) != 2 || !strings.Contains(deref(pub.in[1].Subject), "cleared") {
264 t.Errorf("clear not published: %d publishes", len(pub.in))
265 }
266 if _, err := os.Stat(statePath); err != nil {
267 t.Errorf("state not persisted: %v", err)
268 }
269}
270
271func TestParseAlertTopicARN(t *testing.T) {
272 cases := []struct {
273 in string
274 wantRegion string
275 wantErr bool
276 }{
277 {testTopicARN, "eu-west-1", false},
278 {"arn:aws:sns:ap-southeast-2:271376211898:carlos-flagship-au-alerts", "ap-southeast-2", false},
279 {" " + testTopicARN + "\n", "eu-west-1", false},
280 {"", "", true},
281 {"not-an-arn", "", true},
282 {"arn:aws:s3:::some-bucket", "", true},
283 {"arn:aws:sqs:eu-west-1:271376211898:queue", "", true},
284 {"arn:aws:sns:::topic", "", true},
285 {"arn:aws:sns:eu-west-1:271376211898:", "", true},
286 // A subscription ARN is topic ARN plus ":<subscription-id>". It is
287 // an sns ARN with all the right fields, and publishing to it fails.
288 {testTopicARN + ":8f4c0e7a-1b2c-4d3e-9f50-0123456789ab", "", true},
289 {"arn:aws:sns:eu-west-1:271376211898:topic:subscription-id", "", true},
290 {"arn:aws:sns:eu-west-1:271376211898:a:b", "", true},
291 {"arn:aws:sns:eu-west-1:271376211898:name.fifo", "", true},
292 {"arn:aws:sns:eu-west-1:271376211898:bad name", "", true},
293 {"arn:aws:sns:eu-west-1:27137621189:carlos-alerts", "", true},
294 {"arn:aws:sns:eu-west-1:abcdefghijkl:carlos-alerts", "", true},
295 {"arn:aws:sns:eu-west-1:271376211898:carlos-alerts\nx", "", true},
296 {"arn:aws:sns:eu-west-1:271376211898:*", "", true},
297 }
298 for _, c := range cases {
299 region, err := parseAlertTopicARN(c.in)
300 if (err != nil) != c.wantErr || region != c.wantRegion {
301 t.Errorf("parseAlertTopicARN(%q) = %q, %v; want %q, err=%v", c.in, region, err, c.wantRegion, c.wantErr)
302 }
303 }
304}
305
306func TestAlertDeliveryProbe(t *testing.T) {
307 env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } }
308
309 counts := &alertCounters{}
310 got := alertDeliveryProbe(env(nil), counts)
311 if got.Webhook || got.SNS || got.Delivered() {
312 t.Errorf("nothing set: %+v, want nothing delivered", got)
313 }
314
315 got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_WEBHOOK": "http://x"}), counts)
316 if !got.Webhook || got.SNS || !got.Delivered() {
317 t.Errorf("webhook only: %+v", got)
318 }
319
320 // A valid ARN alone is not a sink until buildAlertSinks has built the
321 // client: before that (a stalled build, or a metrics loop that exited
322 // first) the box must not claim sns.
323 got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts)
324 if got.SNS || got.SNSUnavailable || got.Delivered() {
325 t.Errorf("sink not yet built must not read as sns: %+v", got)
326 }
327
328 counts.snsInit.Store(snsInitOK)
329 got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts)
330 if got.Webhook || !got.SNS || !got.Delivered() {
331 t.Errorf("sns only: %+v", got)
332 }
333
334 // A malformed ARN is not a sink: the box has nowhere to deliver, and
335 // the snapshot must say so rather than claim an sns sink it cannot use.
336 got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": "nonsense"}), counts)
337 if got.SNS || got.Delivered() {
338 t.Errorf("bad arn counted as a sink: %+v", got)
339 }
340
341 counts.sns.Add(3)
342 counts.webhook.Add(2)
343 got = alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts)
344 if got.SNSFailures != 3 || got.WebhookFailures != 2 {
345 t.Errorf("failure counts not carried: %+v", got)
346 }
347}
348
349// fakeClientFactory is the buildAlertSinks seam: it records the region it
350// was asked for and either hands back a fake or fails.
351type fakeClientFactory struct {
352 regions []string
353 err error
354 pub *fakeSNS
355}
356
357func (f *fakeClientFactory) build(_ context.Context, region string) (snsPublisher, error) {
358 f.regions = append(f.regions, region)
359 if f.err != nil {
360 return nil, f.err
361 }
362 return f.pub, nil
363}
364
365func TestBuildAlertSinksInitialisedSNS(t *testing.T) {
366 env := func(m map[string]string) func(string) string { return func(k string) string { return m[k] } }
367 counts := &alertCounters{}
368 f := &fakeClientFactory{pub: &fakeSNS{}}
369 sinks := buildAlertSinks(context.Background(), env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts, f.build)
370 if len(f.regions) != 1 || f.regions[0] != "eu-west-1" {
371 t.Fatalf("client built for %v, want the ARN's region eu-west-1", f.regions)
372 }
373 sinks.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
374 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
375 if len(f.pub.in) != 1 {
376 t.Fatalf("publishes = %d, want 1", len(f.pub.in))
377 }
378 got := alertDeliveryProbe(env(map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}), counts)
379 if !got.SNS || got.SNSUnavailable || got.SNSFailures != 0 {
380 t.Errorf("a working sink must read as sns: %+v", got)
381 }
382}
383
384// TestBuildAlertSinksClientFailureIsVisible is the review's finding: with a
385// valid ARN in the env but a client that cannot be built (no region, bad
386// profile), the sink is off. The status probe must say so, not report the
387// env as if it were a working sink with zero failures.
388func TestBuildAlertSinksClientFailureIsVisible(t *testing.T) {
389 envMap := map[string]string{"CARLOS_ALERT_SNS_TOPIC_ARN": testTopicARN}
390 env := func(k string) string { return envMap[k] }
391 counts := &alertCounters{}
392 f := &fakeClientFactory{err: errors.New("failed to get shared config profile")}
393 sinks := buildAlertSinks(context.Background(), env, counts, f.build)
394 // Delivering must neither panic nor publish.
395 sinks.deliver(context.Background(), metrics.DefaultRules(), sinkTestAt,
396 metrics.Event{Rule: "disk", Firing: true, Value: 0.95, Box: "box-a"})
397
398 got := alertDeliveryProbe(env, counts)
399 if got.SNS || !got.SNSUnavailable || got.Delivered() {
400 t.Errorf("failed init must read as unavailable and not delivered: %+v", got)
401 }
402 if got.SNSFailures < 1 {
403 t.Errorf("failed init must be counted, got %d", got.SNSFailures)
404 }
405
406 // With a webhook as well, the box still delivers, and says what is wrong.
407 envMap["CARLOS_ALERT_WEBHOOK"] = "http://example.test/hook"
408 got = alertDeliveryProbe(env, counts)
409 if !got.Webhook || got.SNS || !got.SNSUnavailable || !got.Delivered() {
410 t.Errorf("webhook alone must still deliver: %+v", got)
411 }
412}
413
414func TestBuildAlertSinksBadARNIsNotASinkAndBuildsNoClient(t *testing.T) {
415 counts := &alertCounters{}
416 f := &fakeClientFactory{pub: &fakeSNS{}}
417 env := func(k string) string {
418 if k == "CARLOS_ALERT_SNS_TOPIC_ARN" {
419 return testTopicARN + ":subscription-id"
420 }
421 return ""
422 }
423 buildAlertSinks(context.Background(), env, counts, f.build)
424 if len(f.regions) != 0 {
425 t.Errorf("a subscription ARN must not build a client: %v", f.regions)
426 }
427 got := alertDeliveryProbe(env, counts)
428 if got.SNS || got.Delivered() {
429 t.Errorf("subscription ARN counted as a sink: %+v", got)
430 }
431}
432
433func TestBuildAlertSinksNothingConfigured(t *testing.T) {
434 counts := &alertCounters{}
435 f := &fakeClientFactory{pub: &fakeSNS{}}
436 buildAlertSinks(context.Background(), func(string) string { return "" }, counts, f.build)
437 if len(f.regions) != 0 {
438 t.Errorf("no ARN set, yet a client was built: %v", f.regions)
439 }
440 got := alertDeliveryProbe(func(string) string { return "" }, counts)
441 if got.Delivered() || got.SNSUnavailable {
442 t.Errorf("nothing configured: %+v", got)
443 }
444}
445
446func deref(s *string) string {
447 if s == nil {
448 return ""
449 }
450 return *s
451}
452