| 1 | package edgesfile |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "log" |
| 7 | "strings" |
| 8 | "sync" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/boxchoice" |
| 12 | "github.com/carlosframework/platform/internal/release" |
| 13 | "github.com/carlosframework/platform/internal/status" |
| 14 | ) |
| 15 | |
| 16 | // The deployment-wide placement read: every box's edges file joined with |
| 17 | // its liveness. The console's automatic-eligibility doors and the box |
| 18 | // agent's relocation chooser (region HA stage B Task 10) read through it, |
| 19 | // so a relocation's target and a create's executor come from the same |
| 20 | // evidence. Moved here unchanged from the console (instancesplacement.go, |
| 21 | // boxchoice.go). |
| 22 | |
| 23 | // PlacementView is the reader's half of the edges-file schema this |
| 24 | // join needs: version, box, region, and the host set. Roles are not |
| 25 | // consulted — a host a box names under ANY role is served there in some |
| 26 | // capacity, and "where is this instance" is the question, not "who is its |
| 27 | // write home" (that is the home map's question, internal/edge). |
| 28 | type PlacementView struct { |
| 29 | V int `json:"v"` |
| 30 | Box string `json:"box"` |
| 31 | EIP string `json:"eip"` |
| 32 | Region string `json:"region"` |
| 33 | Placement string `json:"placement"` |
| 34 | // DataBucket is the box's CARLOS_HIBERNATE_BUCKET, verbatim |
| 35 | // (internal/host/edgesfile.go writes it from that env and omits the key |
| 36 | // entirely when it is unset). It carries the move lane's actual |
| 37 | // coordinates, and doubles as positive hosting evidence from a writer |
| 38 | // too old to publish HibernationConfigured. |
| 39 | DataBucket string `json:"data_bucket"` |
| 40 | // DataBucketRegion is that bucket's AWS region, as the box publishes it |
| 41 | // beside DataBucket. An offline move's stage fixes both on the record. |
| 42 | DataBucketRegion string `json:"data_bucket_region"` |
| 43 | // HibernationConfigured is the box's own answer to "is my hibernation |
| 44 | // activator wired" (internal/host/edgesfile.go). A POINTER because |
| 45 | // absent and false are different facts and the difference is the whole |
| 46 | // point: absent is a writer that predates the field, whose capability is |
| 47 | // UNKNOWN, while false is a box that published the field and said no. |
| 48 | // Reading absence as no is what made an earlier version withdraw healthy |
| 49 | // regions mid-roll — see placements. |
| 50 | HibernationConfigured *bool `json:"hibernation_configured"` |
| 51 | // ServesSharedPool and AcceptsInbound: POINTERS, absent = a writer |
| 52 | // predating them (unknown). SecretsKeyFP: absent = no key, or a writer |
| 53 | // predating it; AcceptsInbound's presence tells the two apart |
| 54 | // (nearest-region spec §6.1, R9). |
| 55 | ServesSharedPool *bool `json:"serves_shared_pool"` |
| 56 | AcceptsInbound *bool `json:"accepts_inbound"` |
| 57 | SecretsKeyFP string `json:"secrets_key_fp"` |
| 58 | // Box evidence (region HA spec 1 §5.1). All absent on a writer that |
| 59 | // predates it: BoxID "", Capabilities nil, Draining false, Headroom nil. |
| 60 | BoxID string `json:"box_id"` |
| 61 | Capabilities []string `json:"capabilities"` |
| 62 | Draining bool `json:"draining"` |
| 63 | Headroom *struct { |
| 64 | MemFreeMiB int64 `json:"mem_free_mib"` |
| 65 | DiskFreeMiB int64 `json:"disk_free_mib"` |
| 66 | MemTotalMiB int64 `json:"mem_total_mib"` |
| 67 | DiskTotalMiB int64 `json:"disk_total_mib"` |
| 68 | Awake int `json:"awake"` |
| 69 | } `json:"headroom"` |
| 70 | Hosts map[string]json.RawMessage `json:"hosts"` |
| 71 | |
| 72 | // Generation is when the box last wrote the file (RFC 3339), which |
| 73 | // region HA spec 2's placement policy reads as headroom's age. |
| 74 | Generation string `json:"generation"` |
| 75 | |
| 76 | // The moves protocol (moves spec §11), read only by the Phase 1 move |
| 77 | // door's floor check (selectMoveBoxes); Task 13 adds the writer's side. |
| 78 | // MovesProtocol is the protocol the box speaks (absent: 0). |
| 79 | // MovesFloorMinRelease is the published floor's min_release the box |
| 80 | // has armed locally. PrevRelease is its carlos.prev's release, "none" |
| 81 | // when it keeps none, and PrevAllowedFor the floor's min_release that |
| 82 | // carlos.prev was judged at or above. |
| 83 | MovesProtocol int `json:"moves_protocol"` |
| 84 | MovesFloorMinRelease string `json:"moves_floor_min_release"` |
| 85 | MovesFloorState string `json:"moves_floor_state,omitempty"` |
| 86 | MovesFloorGeneration uint64 `json:"moves_floor_generation,omitempty"` |
| 87 | PrevRelease string `json:"prev_release"` |
| 88 | PrevAllowedFor string `json:"prev_allowed_for"` |
| 89 | // The seeds floor (seeded-instances spec §3.3), read only by the |
| 90 | // console's seedRegionReady. |
| 91 | SeedsFloorState string `json:"seeds_floor_state,omitempty"` |
| 92 | SeedsFloorGeneration uint64 `json:"seeds_floor_generation,omitempty"` |
| 93 | SeedsProtocol int `json:"seeds_protocol"` |
| 94 | SeedsFloorMinRelease string `json:"seeds_floor_min_release"` |
| 95 | PrevSeedsAllowedFor string `json:"prev_seeds_allowed_for"` |
| 96 | // Fresh is not on the wire: ReadLiveSnapshot sets it when the |
| 97 | // box's own status snapshot is still fresh (status.BoxStale). An edges |
| 98 | // file is not liveness, see ReadLiveSnapshot. |
| 99 | Fresh bool `json:"-"` |
| 100 | } |
| 101 | |
| 102 | // livenessSaid throttles ReadLiveSnapshot's two diagnostics to once |
| 103 | // per process, the way noPointSaid does for a code with no point. |
| 104 | var livenessSaid sync.Map |
| 105 | |
| 106 | // ReadLiveSnapshot is ReadSnapshot joined with box liveness, for |
| 107 | // AUTOMATIC eligibility (regionCandidates). An edges file is not evidence |
| 108 | // that its box is alive: nothing deletes a retired box's file, and the |
| 109 | // file's own Generation moves only when the file is rewritten, so a |
| 110 | // healthy box's is arbitrarily old (internal/edgesfile.EdgeIPs's doc |
| 111 | // comment has the whole argument). The status snapshot is written every |
| 112 | // tick, so status.BoxStale is a real liveness signal, and this is the same |
| 113 | // join EdgeIPs makes, over the same period. |
| 114 | // |
| 115 | // A box with no fresh snapshot, and every box when the snapshots cannot be |
| 116 | // loaded, is simply not Fresh. The fold then counts it for nothing, so a |
| 117 | // deployment whose status store is split from its deployment bucket |
| 118 | // (CARLOS_STATUS_BUCKET, which the console does not read) degrades to |
| 119 | // home-only guesses. That is safe but invisible, so both cases are said |
| 120 | // once: the load failure, and files found with no snapshot matching any. |
| 121 | // The explicit door's placements stay on the generous file-only read; a |
| 122 | // person asked for that region and can see it refused later. |
| 123 | func ReadLiveSnapshot(ctx context.Context, s release.Store, now time.Time) (Snapshot, error) { |
| 124 | snap, err := ReadSnapshot(ctx, s) |
| 125 | if err != nil { |
| 126 | return snap, err |
| 127 | } |
| 128 | // No edges file at all is not complete evidence by itself (Task 10 |
| 129 | // review M5): the join still runs, so a fresh status snapshot with no |
| 130 | // edges file (ruling (A) below) makes the evidence incomplete, as it |
| 131 | // does beside any other file. With no file and no fresh snapshot the |
| 132 | // evidence is complete and empty. |
| 133 | // Completeness is carried separately from staleness: LoadBoxesComplete |
| 134 | // returns what LoadBoxes would, plus whether every listed snapshot was |
| 135 | // actually read. A skipped read leaves that box's liveness UNKNOWN, |
| 136 | // and unknown is never "stale". |
| 137 | boxes, boxesComplete, err := status.LoadBoxesComplete(ctx, s) |
| 138 | if err != nil { |
| 139 | if _, said := livenessSaid.LoadOrStore("load", true); !said { |
| 140 | log.Printf("edges: could not load box snapshots (%v), so no box outside the home counts for automatic region placement until they load", err) |
| 141 | } |
| 142 | return snap, nil |
| 143 | } |
| 144 | snap.BoxesComplete = boxesComplete |
| 145 | // known is every box with a status object at all; live is the fresh |
| 146 | // subset. The join key is the status snapshot's Label against the edges |
| 147 | // file's own box field, the pair edgesfile.EdgeIPs joins on; the object |
| 148 | // key names play no part. |
| 149 | known, live := map[string]bool{}, map[string]bool{} |
| 150 | for _, b := range boxes { |
| 151 | known[b.Label] = true |
| 152 | if !status.BoxStale(b, now, status.SnapshotPeriod) { |
| 153 | live[b.Label] = true |
| 154 | } |
| 155 | } |
| 156 | published := make(map[string]bool, len(snap.Files)) |
| 157 | joined := 0 |
| 158 | for i := range snap.Files { |
| 159 | published[snap.Files[i].Box] = true |
| 160 | snap.Files[i].Fresh = live[snap.Files[i].Box] |
| 161 | if snap.Files[i].Fresh { |
| 162 | joined++ |
| 163 | } |
| 164 | // Ruling (B) reversed, re-review r2, on production evidence: every |
| 165 | // box writes both objects and a retired box leaves both behind, so |
| 166 | // an ABSENT snapshot is unknown liveness (incomplete evidence, |
| 167 | // whole-request) and a STALE one is the retired box (not fresh, |
| 168 | // counts for neither side, evidence complete). |
| 169 | if !known[snap.Files[i].Box] { |
| 170 | snap.BoxesComplete = false |
| 171 | if _, said := livenessSaid.LoadOrStore("nostatus", true); !said { |
| 172 | log.Printf("edges: box %q has an edges file under %s but no status snapshot at all, so no region outside the home is automatically eligible until it reports; said once", snap.Files[i].Box, Prefix) |
| 173 | } |
| 174 | } |
| 175 | } |
| 176 | // Ruling (A), re-review r2: a FRESH snapshot whose box has no edges |
| 177 | // file is incomplete evidence, whole-request, like a failed read. |
| 178 | // Reconcile admits a box without its edges file (its publication may |
| 179 | // simply have failed), so a live box the fold cannot see could still |
| 180 | // take the record. A stale snapshot with no file is a retired box and |
| 181 | // is ignored. |
| 182 | for label := range live { |
| 183 | if !published[label] { |
| 184 | snap.BoxesComplete = false |
| 185 | if _, said := livenessSaid.LoadOrStore("unmatched", true); !said { |
| 186 | log.Printf("edges: box %q has a fresh status snapshot but no edges file under %s, so no region outside the home is automatically eligible until it publishes one; said once", label, Prefix) |
| 187 | } |
| 188 | } |
| 189 | } |
| 190 | if joined == 0 && len(snap.Files) > 0 { |
| 191 | if _, said := livenessSaid.LoadOrStore("none", true); !said { |
| 192 | log.Printf("edges: %d edges file(s) under %s but no fresh box snapshot matched any of them, so automatic region placement sees the home alone (are edges and status in different buckets? see CARLOS_STATUS_BUCKET/CARLOS_STATUS_PREFIX)", len(snap.Files), Prefix) |
| 193 | } |
| 194 | } |
| 195 | return snap, nil |
| 196 | } |
| 197 | |
| 198 | // Snapshot is one read of control/edges/. Complete is false when any |
| 199 | // listed file could not be read or parsed: positive facts about the boxes |
| 200 | // that WERE read still hold, but nothing may be concluded about "every |
| 201 | // box" (R9's NoKeyProven, codex plan-R2 1). |
| 202 | type Snapshot struct { |
| 203 | Files []PlacementView |
| 204 | Complete bool |
| 205 | // BoxesComplete: every listed status snapshot was read, and every box |
| 206 | // on either side has its counterpart, so each file's Fresh is a KNOWN |
| 207 | // answer. Set only by ReadLiveSnapshot; a plain ReadSnapshot |
| 208 | // carries no liveness evidence at all. Unknown liveness is not stale |
| 209 | // (re-review r1 hole 2, r2 rulings (A) and (B) reversed): a snapshot |
| 210 | // that could not be read, a fresh snapshot with no edges file, or an |
| 211 | // edges file with no snapshot at all each make the evidence incomplete. |
| 212 | BoxesComplete bool |
| 213 | } |
| 214 | |
| 215 | // EvidenceComplete is the evidence bar for AUTOMATIC non-home eligibility: every |
| 216 | // listed edges file read and parsed, and every listed status snapshot |
| 217 | // read. Short of it a failing sibling may simply be missing from the fold, |
| 218 | // so nothing non-home is eligible for the request (ruling, re-review r1). |
| 219 | func (s Snapshot) EvidenceComplete() bool { return s.Complete && s.BoxesComplete } |
| 220 | |
| 221 | // ReadSnapshot REPORTS a failed listing instead of answering empty: |
| 222 | // the door must tell "could not read" (503, C28) from "no box publishes |
| 223 | // this code" (codex R2-2). A single unreadable or malformed file is |
| 224 | // skipped, as before, and marks the snapshot incomplete. |
| 225 | func ReadSnapshot(ctx context.Context, s release.Store) (Snapshot, error) { |
| 226 | keys, err := s.List(ctx, Prefix) |
| 227 | if err != nil { |
| 228 | return Snapshot{}, err |
| 229 | } |
| 230 | snap := Snapshot{Complete: true} |
| 231 | for _, key := range keys { |
| 232 | name, ok := strings.CutPrefix(key, Prefix) |
| 233 | if !ok || !strings.HasSuffix(name, ".json") || strings.Contains(name, "/") { |
| 234 | continue |
| 235 | } |
| 236 | body, _, err := s.Get(ctx, key) |
| 237 | if err != nil { |
| 238 | log.Printf("edges: could not read %s for the placement join: %v", key, err) |
| 239 | snap.Complete = false |
| 240 | continue |
| 241 | } |
| 242 | var f PlacementView |
| 243 | if err := json.Unmarshal(body, &f); err != nil || f.V < 1 || f.Box == "" { |
| 244 | log.Printf("edges: skipping %s in the placement join: not a versioned edges file", key) |
| 245 | snap.Complete = false |
| 246 | continue |
| 247 | } |
| 248 | snap.Files = append(snap.Files, f) |
| 249 | } |
| 250 | return snap, nil |
| 251 | } |
| 252 | |
| 253 | // BoxChoice is the snapshot as boxchoice's predicate reads it. Every field |
| 254 | // the predicate reads is carried: a dropped field would silently read as |
| 255 | // absent, so the console's eligibility tests run through this conversion. |
| 256 | func (s Snapshot) BoxChoice() boxchoice.Snapshot { |
| 257 | out := boxchoice.Snapshot{Complete: s.EvidenceComplete(), Boxes: make([]boxchoice.Box, 0, len(s.Files))} |
| 258 | for _, f := range s.Files { |
| 259 | b := boxchoice.Box{ |
| 260 | Box: f.Box, BoxID: f.BoxID, EIP: f.EIP, Region: f.Region, Placement: f.Placement, |
| 261 | HibernationConfigured: f.HibernationConfigured, ServesSharedPool: f.ServesSharedPool, |
| 262 | AcceptsInbound: f.AcceptsInbound, SecretsKeyFP: f.SecretsKeyFP, Capabilities: f.Capabilities, |
| 263 | Draining: f.Draining, Generation: f.Generation, Fresh: f.Fresh, |
| 264 | } |
| 265 | if f.Headroom != nil { |
| 266 | b.Headroom = &boxchoice.Headroom{MemFreeMiB: f.Headroom.MemFreeMiB, DiskFreeMiB: f.Headroom.DiskFreeMiB, |
| 267 | MemTotalMiB: f.Headroom.MemTotalMiB, DiskTotalMiB: f.Headroom.DiskTotalMiB, Awake: f.Headroom.Awake} |
| 268 | } |
| 269 | out.Boxes = append(out.Boxes, b) |
| 270 | } |
| 271 | return out |
| 272 | } |
| 273 | |
| 274 | // LiveSnapshot is ReadLiveSnapshot in boxchoice's form: the read |
| 275 | // hostops.ExecutorChooser's Edges takes. |
| 276 | func LiveSnapshot(ctx context.Context, s release.Store, now time.Time) (boxchoice.Snapshot, error) { |
| 277 | snap, err := ReadLiveSnapshot(ctx, s, now) |
| 278 | if err != nil { |
| 279 | return boxchoice.Snapshot{}, err |
| 280 | } |
| 281 | return snap.BoxChoice(), nil |
| 282 | } |
| 283 | |
amadan