| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "log" |
| 8 | "sort" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/boxchoice" |
| 12 | "github.com/carlosframework/platform/internal/hostops" |
| 13 | "github.com/carlosframework/platform/internal/park" |
| 14 | ) |
| 15 | |
| 16 | // The eligible-capacity metric (region HA spec 2 §4, A1-11): once a |
| 17 | // minute the console counts, for each region its boxes publish, the boxes |
| 18 | // that could take a new instance now (hostops.ExecutorChooser.Capacity: |
| 19 | // enrolled, active, unfenced and placeable), and publishes the counts. An |
| 20 | // alarm on the count catches what the cloud's own group metrics cannot: a |
| 21 | // box that is in service but never enrolled, and a launch loop. Where the |
| 22 | // counts go is an adapter's business (awsregionmetric.go); a console with |
| 23 | // none publishes nothing. |
| 24 | |
| 25 | // capacityInterval is how often the counts are published. |
| 26 | const capacityInterval = time.Minute |
| 27 | |
| 28 | // regionCapacity is one region's count. Placement is the CARLOS placement |
| 29 | // code the metric is keyed by; AWSRegion is where its boxes run. |
| 30 | type regionCapacity struct { |
| 31 | Placement, AWSRegion string |
| 32 | Eligible int |
| 33 | } |
| 34 | |
| 35 | // capacityPublisher sends the counts to a metrics service. |
| 36 | type capacityPublisher interface { |
| 37 | PublishCapacity(ctx context.Context, counts []regionCapacity) error |
| 38 | } |
| 39 | |
| 40 | // regionCapacities counts each placement the snapshot's files name, sorted |
| 41 | // by placement, and takes each one's AWS region from capacity's answer: |
| 42 | // the signed roster's, never the files' own claim, so a stale or |
| 43 | // misconfigured file naming another AWS region cannot send a region's |
| 44 | // count elsewhere (Astra 2a round 2 finding 3). A placement the roster |
| 45 | // does not know (no region entry, so no box there can be chosen) is left |
| 46 | // out. A placement whose count fails, or whose read is incomplete |
| 47 | // (hostops.ErrEvidenceIncomplete), is left out too and its error returned |
| 48 | // with the counts that did succeed: publishing 0 for it would trip the |
| 49 | // eligible-capacity alarm on a transient read failure, whereas a missing |
| 50 | // data point is the alarm's own signal if the failure lasts (spec §4, |
| 51 | // A2-8). Only a complete read that finds no placeable box is a 0. |
| 52 | // |
| 53 | // A home box publishes the home's placement code, "". It is counted under |
| 54 | // home, the home's roster name, as the chooser reads it (Capacity's and |
| 55 | // eligible's mapping of park.HomeRegion to HomeName), so the home region's |
| 56 | // count goes to the AWS region the signed roster gives home (Astra 2a |
| 57 | // final review, P2). With no home name the chooser places nothing in the |
| 58 | // home, and its boxes are left out. |
| 59 | func regionCapacities(ctx context.Context, snap boxchoice.Snapshot, home string, capacity func(context.Context, string) (int, string, error)) ([]regionCapacity, error) { |
| 60 | named := map[string]bool{} |
| 61 | for _, b := range snap.Boxes { |
| 62 | if p := capacityPlacement(home, b.Placement); p != "" { |
| 63 | named[p] = true |
| 64 | } |
| 65 | } |
| 66 | placements := make([]string, 0, len(named)) |
| 67 | for p := range named { |
| 68 | placements = append(placements, p) |
| 69 | } |
| 70 | sort.Strings(placements) |
| 71 | var out []regionCapacity |
| 72 | var errs []error |
| 73 | for _, p := range placements { |
| 74 | n, aws, err := capacity(ctx, p) |
| 75 | if errors.Is(err, hostops.ErrContract) { |
| 76 | continue |
| 77 | } |
| 78 | if err != nil { |
| 79 | errs = append(errs, fmt.Errorf("count %s: %w", p, err)) |
| 80 | continue |
| 81 | } |
| 82 | out = append(out, regionCapacity{Placement: p, AWSRegion: aws, Eligible: n}) |
| 83 | } |
| 84 | return out, errors.Join(errs...) |
| 85 | } |
| 86 | |
| 87 | // capacityPlacement is the region a box's placement code is counted under: |
| 88 | // the code itself, or for the home's "" the home's name, exactly when the |
| 89 | // chooser maps park.HomeRegion to it (home is a placement code). "" is |
| 90 | // counted nowhere. |
| 91 | func capacityPlacement(home, code string) string { |
| 92 | if region := park.PlacementRegion(code); region == park.HomeRegion && home != "" && park.PlacementRegion(home) == home { |
| 93 | return home |
| 94 | } |
| 95 | return code |
| 96 | } |
| 97 | |
| 98 | // publishCapacity counts and publishes once. |
| 99 | func (s *Server) publishCapacity(ctx context.Context, pub capacityPublisher) error { |
| 100 | x := s.placementChooser("") |
| 101 | if x == nil { |
| 102 | return hostops.ErrContract |
| 103 | } |
| 104 | snap, err := s.liveEdges(ctx) |
| 105 | if err != nil { |
| 106 | return err |
| 107 | } |
| 108 | counts, countErr := regionCapacities(ctx, boxSnapshot(snap), x.HomeName, x.Capacity) |
| 109 | if len(counts) == 0 { |
| 110 | return countErr |
| 111 | } |
| 112 | // The regions that counted are published even when another failed: |
| 113 | // the failed one has no data point, which is the alarm's signal. |
| 114 | return errors.Join(pub.PublishCapacity(ctx, counts), countErr) |
| 115 | } |
| 116 | |
| 117 | // runCapacityPass publishes every capacityInterval until ctx ends. A |
| 118 | // failed pass is logged on its first failure and every tenth; the alarm |
| 119 | // treats missing data as breaching, so a console that stops publishing is |
| 120 | // itself an alert. |
| 121 | func (s *Server) runCapacityPass(ctx context.Context, pub capacityPublisher) { |
| 122 | log.Printf("console: eligible-capacity metric armed (every %s)", capacityInterval) |
| 123 | t := time.NewTicker(capacityInterval) |
| 124 | defer t.Stop() |
| 125 | fails := 0 |
| 126 | for { |
| 127 | pass, cancel := context.WithTimeout(ctx, capacityInterval) |
| 128 | err := s.publishCapacity(pass, pub) |
| 129 | cancel() |
| 130 | switch { |
| 131 | case err != nil: |
| 132 | fails++ |
| 133 | if fails == 1 || fails%10 == 0 { |
| 134 | log.Printf("console: eligible-capacity metric (failure #%d): %v", fails, err) |
| 135 | } |
| 136 | case fails > 0: |
| 137 | log.Printf("console: eligible-capacity metric recovered after %d failure(s)", fails) |
| 138 | fails = 0 |
| 139 | } |
| 140 | select { |
| 141 | case <-ctx.Done(): |
| 142 | return |
| 143 | case <-t.C: |
| 144 | } |
| 145 | } |
| 146 | } |
| 147 | |
amadan