| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "log" |
| 8 | "net/http" |
| 9 | "strings" |
| 10 | "sync" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/alerting" |
| 14 | "github.com/carlosframework/platform/internal/hostclaim" |
| 15 | "github.com/carlosframework/platform/internal/hostops" |
| 16 | ) |
| 17 | |
| 18 | // No box has room (region HA spec 2 §7, ruling H27). Every door that |
| 19 | // chooses a box (instance create, domain attach's first route, standalone |
| 20 | // sidecar create, and the move door's destination) refuses with a 503 when |
| 21 | // the placement policy finds no eligible box, and the operator hears of it |
| 22 | // at most once per region an hour per console process: a restart may send |
| 23 | // one extra alert, which H27 accepts rather than persist the limit. A |
| 24 | // create door chooses before it registers anything (chooseExecutorFirst), |
| 25 | // so a refusal leaves nothing registered, nothing written to the claim and |
| 26 | // nothing for recovery to settle. Only a complete placement read with no |
| 27 | // placeable box is "no room": an incomplete one (hostops.ErrEvidenceIncomplete) |
| 28 | // is answered as temporarily unavailable and pages nobody. |
| 29 | |
| 30 | // noCapacityAlertEvery is how often one region's alert may repeat. |
| 31 | const noCapacityAlertEvery = time.Hour |
| 32 | |
| 33 | // noCapacityRetryAfter is the Retry-After (seconds) a no-room 503 carries: |
| 34 | // RHA-C17 tells the caller to try again in a few minutes. Every other |
| 35 | // claim refusal keeps its own. |
| 36 | const noCapacityRetryAfter = "60" |
| 37 | |
| 38 | func noCapacityRefusal(region string) string { |
| 39 | return fmt.Sprintf("No box in %s has room right now. Try again in a few minutes.", region) |
| 40 | } |
| 41 | |
| 42 | func noCapacityAlert(region string) (subject, body string) { |
| 43 | return fmt.Sprintf("No box in %s has room", region), |
| 44 | fmt.Sprintf("A request in %s was refused because no box there can take it. Every box is full, stale or excluded by its role. Add a box, or free memory or disk on one, and check the eligible-capacity alarm for %s.", region, region) |
| 45 | } |
| 46 | |
| 47 | // noCapacityError is a request refused because no box in Region (its |
| 48 | // display name) can take it. It is hostops.ErrNoEligibleBox, and |
| 49 | // claimResponseRefusal answers it with noCapacityRefusal. |
| 50 | type noCapacityError struct{ Region string } |
| 51 | |
| 52 | func (e *noCapacityError) Error() string { |
| 53 | return "no eligible box in " + e.Region + ": " + hostops.ErrNoEligibleBox.Error() |
| 54 | } |
| 55 | |
| 56 | func (e *noCapacityError) Is(target error) bool { return target == hostops.ErrNoEligibleBox } |
| 57 | |
| 58 | // pageNoCapacity sends the alert; tests replace it. |
| 59 | var pageNoCapacity = alerting.Page |
| 60 | |
| 61 | // noCapacityAlerts remembers when each region's alert was last sent, in |
| 62 | // this console process. An alert counts as sent only once its send |
| 63 | // succeeds: due reserves the region's slot so concurrent refusals do not |
| 64 | // page twice while one send is in flight, and failed gives the slot back, |
| 65 | // so the next refusal tries again rather than the region going quiet for |
| 66 | // an hour. The zero value is ready. |
| 67 | type noCapacityAlerts struct { |
| 68 | mu sync.Mutex |
| 69 | last map[string]time.Time |
| 70 | } |
| 71 | |
| 72 | // due reports whether region's alert should be sent at now, and if so |
| 73 | // records it as sent. |
| 74 | func (a *noCapacityAlerts) due(region string, now time.Time) bool { |
| 75 | a.mu.Lock() |
| 76 | defer a.mu.Unlock() |
| 77 | if at, ok := a.last[region]; ok && now.Sub(at) < noCapacityAlertEvery { |
| 78 | return false |
| 79 | } |
| 80 | if a.last == nil { |
| 81 | a.last = map[string]time.Time{} |
| 82 | } |
| 83 | a.last[region] = now |
| 84 | return true |
| 85 | } |
| 86 | |
| 87 | // failed gives back the slot due reserved at for region, because that send |
| 88 | // failed. A later reservation is left alone. |
| 89 | func (a *noCapacityAlerts) failed(region string, at time.Time) { |
| 90 | a.mu.Lock() |
| 91 | defer a.mu.Unlock() |
| 92 | if last, ok := a.last[region]; ok && last.Equal(at) { |
| 93 | delete(a.last, region) |
| 94 | } |
| 95 | } |
| 96 | |
| 97 | // isNoCapacityRefusal reports whether ref is the no-room refusal |
| 98 | // (claimResponseRefusal's answer to a noCapacityError), which the attach |
| 99 | // doors carry as a doorRefusal. |
| 100 | func isNoCapacityRefusal(ref *doorRefusal) bool { |
| 101 | if ref == nil || ref.Status != http.StatusServiceUnavailable { |
| 102 | return false |
| 103 | } |
| 104 | before, after, _ := strings.Cut(noCapacityRefusal("\x00"), "\x00") |
| 105 | region, ok := strings.CutPrefix(ref.Msg, before) |
| 106 | if !ok { |
| 107 | return false |
| 108 | } |
| 109 | region, ok = strings.CutSuffix(region, after) |
| 110 | return ok && region != "" |
| 111 | } |
| 112 | |
| 113 | // setNoCapacityRetryAfter stamps noCapacityRetryAfter on w when ref is the |
| 114 | // no-room refusal, and leaves every other refusal's headers as its door |
| 115 | // writes them. |
| 116 | func setNoCapacityRetryAfter(w http.ResponseWriter, ref *doorRefusal) { |
| 117 | if isNoCapacityRefusal(ref) { |
| 118 | w.Header().Set("Retry-After", noCapacityRetryAfter) |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // noRoom is the refusal for region (record vocabulary, "" for the home): |
| 123 | // it logs, sends the operator alert when one is due, and returns the |
| 124 | // error the door answers with. |
| 125 | func (s *Server) noRoom(ctx context.Context, region string) error { |
| 126 | name := s.homeRegionOut(region) |
| 127 | log.Printf("console: refused: no eligible box in %s", name) |
| 128 | if now := time.Now(); s.noCapacity.due(name, now) { |
| 129 | subject, body := noCapacityAlert(name) |
| 130 | go func() { |
| 131 | ctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) |
| 132 | defer cancel() |
| 133 | if err := pageNoCapacity(ctx, alerting.LevelWarning, subject, body); err != nil { |
| 134 | // Not sent: the next refusal in name tries again. |
| 135 | s.noCapacity.failed(name, now) |
| 136 | log.Printf("console: no-capacity alert for %s: %v", name, err) |
| 137 | } |
| 138 | }() |
| 139 | } |
| 140 | return &noCapacityError{Region: name} |
| 141 | } |
| 142 | |
| 143 | // placementChooser is the create door's chooser scoped as |
| 144 | // ProductionAuthority.SelectExecutor scopes it, for pool ("" the shared |
| 145 | // pool), or nil with region HA off or no claim runtime. |
| 146 | func (s *Server) placementChooser(pool string) *hostops.ExecutorChooser { |
| 147 | a := s.cfg.ClaimAuthority |
| 148 | if !s.cfg.RegionHA || a == nil || a.Executors == nil { |
| 149 | return nil |
| 150 | } |
| 151 | x := *a.Executors |
| 152 | x.Namespace, x.Pool = a.Namespace, pool |
| 153 | return &x |
| 154 | } |
| 155 | |
| 156 | // roomIn is H27's question about region (record vocabulary) for pool, asked |
| 157 | // with region HA on before a door's own placement checks (Astra 2a round 2 |
| 158 | // finding 1): a region the roster places boxes in, whose placeable set is |
| 159 | // empty (no edges file, no public address, stale, full or excluded by |
| 160 | // role), is the no-room refusal, never a 400. It only refuses: it chooses |
| 161 | // nothing and changes nothing the door goes on to do. It answers nil with |
| 162 | // region HA off, for a region the roster does not place (the door's own |
| 163 | // checks answer that), and when a box has room. An incomplete read is |
| 164 | // hostops.ErrEvidenceIncomplete, which the door answers as temporarily |
| 165 | // unavailable, with no alert. |
| 166 | func (s *Server) roomIn(ctx context.Context, region, pool string) error { |
| 167 | x := s.placementChooser(pool) |
| 168 | if x == nil { |
| 169 | return nil |
| 170 | } |
| 171 | boxes, err := x.PlaceableBoxes(ctx, s.cfg.claimPlacementRegion(region)) |
| 172 | switch { |
| 173 | case errors.Is(err, hostops.ErrContract): |
| 174 | return nil |
| 175 | case err != nil: |
| 176 | return err |
| 177 | case len(boxes) == 0: |
| 178 | return s.noRoom(ctx, region) |
| 179 | } |
| 180 | return nil |
| 181 | } |
| 182 | |
| 183 | // errChoiceStale is an early choice whose claim observation is no longer |
| 184 | // the claim: a write landed between the choice and beginClaimCreate's own |
| 185 | // read. It wraps hostclaim.ErrConflict, so errors.Is(errChoiceStale, |
| 186 | // hostclaim.ErrConflict) holds on the sentinel itself and the door answers |
| 187 | // main's retryable busy refusal; nothing was registered or written. |
| 188 | var errChoiceStale = fmt.Errorf("%w: the host's claim changed after its executor was chosen", hostclaim.ErrConflict) |
| 189 | |
| 190 | // chosenOnKey carries the claim observation an early choice was decided |
| 191 | // on, beside the choice itself (hostops.WithChosenExecutor). |
| 192 | type chosenOnKey struct{} |
| 193 | |
| 194 | type chosenOn struct { |
| 195 | host, etag string |
| 196 | absent bool |
| 197 | } |
| 198 | |
| 199 | // chooseExecutorFirst is beginClaimCreate's first step with region HA on, |
| 200 | // given beginClaimCreate's own read of the claim (before). It chooses the |
| 201 | // executor for the record build would make, before anything is |
| 202 | // registered, and carries the choice to the create |
| 203 | // (hostops.WithChosenExecutor). It takes no lease and writes nothing: the |
| 204 | // choice is SelectExecutor's read of the roster and the edges files. No |
| 205 | // eligible box on a complete read is noRoom's refusal; an incomplete read |
| 206 | // (hostops.ErrEvidenceIncomplete) is returned as it is. A choice ctx already carries for host |
| 207 | // (chooseCreateFirst's) is kept only when before is the claim observation |
| 208 | // it was decided on (census ruling region-ha-2a, item 5): otherwise the |
| 209 | // create is refused with errChoiceStale, and the old choice is never used |
| 210 | // and nothing is chosen again. With region HA off it returns ctx |
| 211 | // unchanged, and the create is main's. |
| 212 | func (s *Server) chooseExecutorFirst(ctx context.Context, host string, before hostclaim.Observed, build func(time.Time, string) (hostops.Record, error)) (context.Context, error) { |
| 213 | a := s.cfg.ClaimAuthority |
| 214 | if !s.cfg.RegionHA || a == nil || a.Executors == nil { |
| 215 | return ctx, nil |
| 216 | } |
| 217 | if _, ok := hostops.ChosenExecutor(ctx, host); ok { |
| 218 | on, ok := ctx.Value(chosenOnKey{}).(chosenOn) |
| 219 | if !ok || on.host != host || on.etag != before.ETag || on.absent != before.Absent { |
| 220 | return ctx, fmt.Errorf("%w: %s", errChoiceStale, host) |
| 221 | } |
| 222 | return ctx, nil |
| 223 | } |
| 224 | probe, err := build(time.Now().UTC(), "") |
| 225 | if err != nil { |
| 226 | return ctx, err |
| 227 | } |
| 228 | sel, chosen, err := a.SelectExecutor(ctx, before, probe) |
| 229 | if errors.Is(err, hostops.ErrNoEligibleBox) { |
| 230 | return ctx, s.noRoom(ctx, probe.Holder.RecordRegion) |
| 231 | } |
| 232 | if err != nil || !chosen { |
| 233 | return ctx, err |
| 234 | } |
| 235 | return hostops.WithChosenExecutor(ctx, host, sel), nil |
| 236 | } |
| 237 | |
| 238 | // chooseCreateFirst is the instance create door's choice, made before the |
| 239 | // door's first write (Astra 2a round 2 finding 2): the environment binding |
| 240 | // (bindEnvironmentCreate) can move a seed's store to the new instance, so |
| 241 | // a create refused for want of room must be refused before it. It reads |
| 242 | // host's claim and chooses as chooseExecutorFirst does, and returns the |
| 243 | // choice together with the claim observation it decided on (its ETag, or |
| 244 | // its absence). beginClaimCreate keeps the choice only if its own read is |
| 245 | // that same observation. With region HA off it returns ctx unchanged and |
| 246 | // reads nothing. |
| 247 | func (s *Server) chooseCreateFirst(ctx context.Context, host string, build func(time.Time, string) (hostops.Record, error)) (context.Context, error) { |
| 248 | a := s.cfg.ClaimAuthority |
| 249 | if !s.cfg.RegionHA || a == nil || a.Executors == nil { |
| 250 | return ctx, nil |
| 251 | } |
| 252 | before, err := hostclaim.Read(ctx, s.cfg.Releases, host) |
| 253 | if err != nil { |
| 254 | return ctx, err |
| 255 | } |
| 256 | chosen, err := s.chooseExecutorFirst(ctx, host, before, build) |
| 257 | if err != nil { |
| 258 | return ctx, err |
| 259 | } |
| 260 | if _, ok := hostops.ChosenExecutor(chosen, host); !ok { |
| 261 | return chosen, nil |
| 262 | } |
| 263 | return context.WithValue(chosen, chosenOnKey{}, chosenOn{host: host, etag: before.ETag, absent: before.Absent}), nil |
| 264 | } |
| 265 | |
amadan