| 1 | package hostops |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "time" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/boxchoice" |
| 10 | "github.com/carlosframework/platform/internal/hostactors" |
| 11 | "github.com/carlosframework/platform/internal/hostclaim" |
| 12 | "github.com/carlosframework/platform/internal/park" |
| 13 | ) |
| 14 | |
| 15 | // ErrNoEligibleBox is a create refused because no box in the placement is |
| 16 | // eligible (region HA §5.2). The console maps it to today's refusal. |
| 17 | var ErrNoEligibleBox = errors.New("hostops: region has no box that can take this") |
| 18 | |
| 19 | // ErrEvidenceIncomplete is a placement read that could not judge every |
| 20 | // box: an edges file unread or malformed, the status listing unread, or a |
| 21 | // fresh box with no edges file (boxchoice.Snapshot.Complete false). It is |
| 22 | // never ErrNoEligibleBox: a door answers it as temporarily unavailable, |
| 23 | // and nothing pages "no room" for it (region HA 2a, Task 11 ruling). |
| 24 | var ErrEvidenceIncomplete = errors.New("hostops: the placement read was incomplete, so no box could be judged") |
| 25 | |
| 26 | // ExecutorChooser is create's executor selection: claim P5-D2's choose() |
| 27 | // over the complete eligible set (region HA spec 1 §5.2, H12, L8). It is |
| 28 | // made once, before the create CAS, and recorded in the create intent. |
| 29 | type ExecutorChooser struct { |
| 30 | Directory hostactors.Directory |
| 31 | // Edges is one read of every box's edges file with the caller's |
| 32 | // liveness verdict per box. An incomplete snapshot selects nothing: |
| 33 | // every question answers ErrEvidenceIncomplete (judged). |
| 34 | Edges func(context.Context) (boxchoice.Snapshot, error) |
| 35 | // Loads counts claimed hosts per box and per "acct/app", from the |
| 36 | // claim-recovery listing (claim §5.1 step 5). A stale count only |
| 37 | // affects balance, never safety. |
| 38 | Loads func(context.Context) (map[string]boxchoice.Load, error) |
| 39 | SecretsKeyFP string |
| 40 | HomeName string |
| 41 | Now func() time.Time |
| 42 | // Namespace and Pool are the scope the chooser chooses within, claim |
| 43 | // Plan 5's create scope: only roster boxes in Namespace serving Pool |
| 44 | // are candidates, and only their fences are read. Pool "" is the |
| 45 | // shared pool (the roster's "default"). An empty Namespace matches no |
| 46 | // roster box, so it chooses nothing. ProductionAuthority.SelectExecutor |
| 47 | // sets both from its own namespace and the record's pool. |
| 48 | Namespace, Pool string |
| 49 | } |
| 50 | |
| 51 | // sharedPool is the roster's name for the shared "" pool (claim Plan 5's |
| 52 | // selectExecutor reads a record's empty pool as it). |
| 53 | const sharedPool = "default" |
| 54 | |
| 55 | // Choose returns the box, and its roster enrollment, that a new host in |
| 56 | // r's placement is created on. Identity, placement and AWS region come from |
| 57 | // the roster; a box with a valid termination fence, or not active, is never |
| 58 | // chosen; a box on the claim's lost list, or owing a park-local chore on |
| 59 | // it, is never chosen. Among the rest, region HA spec 2's policy (§7) |
| 60 | // decides: placeable boxes only (placeable), then boxchoice.ChooseRoomiest. |
| 61 | func (x ExecutorChooser) Choose(ctx context.Context, obs hostclaim.Observed, r Record) (hostclaim.ExecutorSelection, error) { |
| 62 | o, cands, err := x.judged(ctx, obs, r) |
| 63 | if err != nil { |
| 64 | return hostclaim.ExecutorSelection{}, err |
| 65 | } |
| 66 | cands = x.placeable(o, cands) |
| 67 | loads, err := x.Loads(ctx) |
| 68 | if err != nil { |
| 69 | return hostclaim.ExecutorSelection{}, err |
| 70 | } |
| 71 | best, ok := boxchoice.ChooseRoomiest(r.Holder.Host, cands, loads, r.Holder.Account+"/"+r.Holder.App) |
| 72 | if !ok { |
| 73 | return hostclaim.ExecutorSelection{}, ErrNoEligibleBox |
| 74 | } |
| 75 | return hostclaim.ExecutorSelection{Box: best.BoxID, Enrollment: o.Roster.Boxes[best.BoxID].Enrollment}, nil |
| 76 | } |
| 77 | |
| 78 | // Permits reports whether box is in the complete eligible set for a host |
| 79 | // in r's placement now (spec 1 §2, §5.8 "The start CAS ... refused ... |
| 80 | // when `to` is not eligible for the host"): the same read Choose makes, |
| 81 | // without choosing. A stale edges file, a missing capability or secrets |
| 82 | // key, a draining box, a fence, or an incomplete snapshot refuses, and so |
| 83 | // does a box spec 2's policy does not place on (placeable): a legacy or |
| 84 | // platform role, too little memory or disk, or stale headroom. |
| 85 | func (x ExecutorChooser) Permits(ctx context.Context, obs hostclaim.Observed, r Record, box string) error { |
| 86 | o, cands, err := x.judged(ctx, obs, r) |
| 87 | if err != nil { |
| 88 | return err |
| 89 | } |
| 90 | for _, c := range x.placeable(o, cands) { |
| 91 | if c.BoxID == box { |
| 92 | return nil |
| 93 | } |
| 94 | } |
| 95 | return fmt.Errorf("%w: %s", ErrNoEligibleBox, box) |
| 96 | } |
| 97 | |
| 98 | // PlaceableBoxes is every box in placement that could take a new |
| 99 | // instance in the chooser's namespace and pool now, sorted: Choose's set, |
| 100 | // before the choice. The console's move door asks it only whether there |
| 101 | // is room (region HA spec 2 §7, H27); it never takes a destination from |
| 102 | // it. An incomplete read is ErrEvidenceIncomplete, never an empty set. |
| 103 | func (x ExecutorChooser) PlaceableBoxes(ctx context.Context, placement string) ([]string, error) { |
| 104 | r := Record{Holder: hostclaim.NewHolder{Region: placement}} |
| 105 | o, cands, err := x.judged(ctx, hostclaim.Observed{Absent: true}, r) |
| 106 | if err != nil { |
| 107 | return nil, err |
| 108 | } |
| 109 | boxes := []string{} |
| 110 | for _, c := range x.placeable(o, cands) { |
| 111 | boxes = append(boxes, c.BoxID) |
| 112 | } |
| 113 | return boxes, nil |
| 114 | } |
| 115 | |
| 116 | // Capacity is how many boxes PlaceableBoxes names, and the AWS region the |
| 117 | // signed roster names for placement. The console publishes the count per |
| 118 | // region as the eligible-capacity metric (region HA spec 2 §4, A1-11), in |
| 119 | // that AWS region: the roster's mapping, the one eligible reads, never an |
| 120 | // edges file's own claim (Astra 2a round 2 finding 3). An incomplete read |
| 121 | // is ErrEvidenceIncomplete, never a count of 0. |
| 122 | func (x ExecutorChooser) Capacity(ctx context.Context, placement string) (int, string, error) { |
| 123 | r := Record{Holder: hostclaim.NewHolder{Region: placement}} |
| 124 | o, cands, err := x.judged(ctx, hostclaim.Observed{Absent: true}, r) |
| 125 | if err != nil { |
| 126 | return 0, "", err |
| 127 | } |
| 128 | // eligible's own home mapping, so the region is the one it read. |
| 129 | if placement == park.HomeRegion && x.HomeName != "" && park.PlacementRegion(x.HomeName) == x.HomeName { |
| 130 | placement = x.HomeName |
| 131 | } |
| 132 | return len(x.placeable(o, cands)), o.Roster.Regions[placement].AWSRegion, nil |
| 133 | } |
| 134 | |
| 135 | // judged is eligible's read, refused with ErrEvidenceIncomplete when the |
| 136 | // edges snapshot it read was incomplete: eligible's empty set then means |
| 137 | // "could not tell", never "no room". eligible itself (an accepted-legacy |
| 138 | // census unit) is unchanged; the snapshot's completeness is taken from |
| 139 | // the same Edges read it makes, through a wrapper on this copy of x. |
| 140 | func (x ExecutorChooser) judged(ctx context.Context, obs hostclaim.Observed, r Record) (hostactors.Observed, []boxchoice.Candidate, error) { |
| 141 | complete := false |
| 142 | if edges := x.Edges; edges != nil { |
| 143 | x.Edges = func(ctx context.Context) (boxchoice.Snapshot, error) { |
| 144 | snap, err := edges(ctx) |
| 145 | complete = err == nil && snap.Complete |
| 146 | return snap, err |
| 147 | } |
| 148 | } |
| 149 | o, cands, err := x.eligible(ctx, obs, r) |
| 150 | if err != nil { |
| 151 | return o, cands, err |
| 152 | } |
| 153 | if !complete { |
| 154 | return hostactors.Observed{}, nil, ErrEvidenceIncomplete |
| 155 | } |
| 156 | return o, cands, nil |
| 157 | } |
| 158 | |
| 159 | // placeable is region HA spec 2's policy over the eligible set (§7): |
| 160 | // each box's roster role, then boxchoice.Placeable's thresholds at |
| 161 | // x.Now(). Identity was Eligible's; nothing here reads a provider. |
| 162 | func (x ExecutorChooser) placeable(o hostactors.Observed, cands []boxchoice.Candidate) []boxchoice.Candidate { |
| 163 | roles := make(map[string]string, len(cands)) |
| 164 | for _, c := range cands { |
| 165 | roles[c.BoxID] = o.Roster.Boxes[c.BoxID].Role |
| 166 | } |
| 167 | return boxchoice.Placeable(cands, roles, x.Now()) |
| 168 | } |
| 169 | |
| 170 | // eligible is one read of the roster, the fences and the edges files, |
| 171 | // reduced to boxchoice.Eligible's candidates. |
| 172 | func (x ExecutorChooser) eligible(ctx context.Context, obs hostclaim.Observed, r Record) (hostactors.Observed, []boxchoice.Candidate, error) { |
| 173 | placement := r.Holder.Region |
| 174 | // A record placed in the home names it park.HomeRegion when no home |
| 175 | // name is configured on the console (main's claimPlacementRegion); the |
| 176 | // roster, its boxes and their edges files name it by the home's roster |
| 177 | // name. The one mapping is main's StartCreate's: park.HomeRegion is |
| 178 | // HomeName when HomeName is a placement code. |
| 179 | if placement == park.HomeRegion && x.HomeName != "" && park.PlacementRegion(x.HomeName) == x.HomeName { |
| 180 | placement = x.HomeName |
| 181 | } |
| 182 | if x.Edges == nil || x.Loads == nil || x.Now == nil || placement == "" { |
| 183 | return hostactors.Observed{}, nil, ErrContract |
| 184 | } |
| 185 | o, err := x.Directory.Read(ctx) |
| 186 | if err != nil { |
| 187 | return hostactors.Observed{}, nil, err |
| 188 | } |
| 189 | region, ok := o.Roster.Regions[placement] |
| 190 | if !ok || region.AWSRegion == "" { |
| 191 | return hostactors.Observed{}, nil, ErrContract |
| 192 | } |
| 193 | pool := x.Pool |
| 194 | if pool == "" { |
| 195 | pool = sharedPool |
| 196 | } |
| 197 | roster := map[string]boxchoice.RosterBox{} |
| 198 | for ref, b := range o.Roster.Boxes { |
| 199 | // Scope first, so a fence is read only for a box this create could |
| 200 | // name: another namespace's unreadable fence never blocks it. |
| 201 | if b.State != "active" || b.Region != placement || b.Namespace != x.Namespace || b.Pool != pool { |
| 202 | continue |
| 203 | } |
| 204 | fenced, err := x.Directory.TerminationFence(ctx, o, ref, x.Now()) |
| 205 | if err != nil { |
| 206 | return hostactors.Observed{}, nil, err // C6: a read error never means "not fenced" |
| 207 | } |
| 208 | if !fenced { |
| 209 | roster[ref] = boxchoice.RosterBox{Placement: b.Region, Cloud: b.Cloud} |
| 210 | } |
| 211 | } |
| 212 | snap, err := x.Edges(ctx) |
| 213 | if err != nil { |
| 214 | return hostactors.Observed{}, nil, err |
| 215 | } |
| 216 | if pool != sharedPool { |
| 217 | // serves_shared_pool is a box's answer for the shared pool only. A |
| 218 | // named pool's membership is the roster's, already applied above, |
| 219 | // so the shared-pool answer is not asked of its boxes. |
| 220 | yes := true |
| 221 | boxes := make([]boxchoice.Box, len(snap.Boxes)) |
| 222 | for i, b := range snap.Boxes { |
| 223 | b.ServesSharedPool = &yes |
| 224 | boxes[i] = b |
| 225 | } |
| 226 | snap.Boxes = boxes |
| 227 | } |
| 228 | // Never a box on the claim's lost list, nor one that still owes a |
| 229 | // park-local chore for this host (spec 1 §2: its old installation is |
| 230 | // not yet retired). |
| 231 | excluded := map[string]bool{} |
| 232 | if !obs.Absent && obs.Claim != nil { |
| 233 | for _, box := range obs.Claim.Lost { |
| 234 | excluded[box] = true |
| 235 | } |
| 236 | for _, box := range hostclaim.LocalChoreBoxes(obs.Claim) { |
| 237 | excluded[box] = true |
| 238 | } |
| 239 | } |
| 240 | return o, boxchoice.Eligible(snap, roster, placement, region.AWSRegion, x.HomeName, x.SecretsKeyFP, excluded), nil |
| 241 | } |
| 242 | |
amadan