| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "log" |
| 7 | "os" |
| 8 | "sort" |
| 9 | "sync" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/geo" |
| 13 | "github.com/carlosframework/platform/internal/provision" |
| 14 | ) |
| 15 | |
| 16 | // disagreeSaid and noPointSaid throttle two diagnostics to once per code. |
| 17 | var disagreeSaid, noPointSaid sync.Map |
| 18 | |
| 19 | // regionCandidate is one region an automatic choice may land in (spec |
| 20 | // §6.1). Code is in the record vocabulary: "" is the home. |
| 21 | type regionCandidate struct { |
| 22 | Code string |
| 23 | Home bool |
| 24 | AWSRegion string |
| 25 | Reachable bool // some FRESH box publishing the code has a dial address (F1: fresh = a current status snapshot) |
| 26 | Hosting bool // ... and its activator configured (unknown counts, as in placements) |
| 27 | KeyOK bool // ... and holds the deployment's secrets key (R9) |
| 28 | Eligible bool // some fresh box has all of it, every fact POSITIVELY true, and no fresh box lacks any of it (F2) |
| 29 | NoKeyProven bool // R9: every reachable hosting box, fresh or stale, is a current writer, and none holds the key |
| 30 | } |
| 31 | |
| 32 | // unavailableReason is "" for an eligible candidate, else the first bar it |
| 33 | // fails (the C16 vocabulary). |
| 34 | func (c regionCandidate) unavailableReason() string { |
| 35 | switch { |
| 36 | case c.Eligible: |
| 37 | return "" |
| 38 | case !c.Reachable: |
| 39 | return "absent" |
| 40 | case !c.Hosting: |
| 41 | return "not-hosting" |
| 42 | case !c.KeyOK: |
| 43 | return "no-secrets-key" |
| 44 | } |
| 45 | return "not-eligible" |
| 46 | } |
| 47 | |
| 48 | func isTrue(b *bool) bool { return b != nil && *b } |
| 49 | |
| 50 | // isFalse is a field the box PUBLISHED as false; absent is unknown, not no. |
| 51 | func isFalse(b *bool) bool { return b != nil && !*b } |
| 52 | |
| 53 | // regionCandidates folds an edges snapshot into candidates. expectedFP is |
| 54 | // the console's own secrets fingerprint; "" means it cannot judge, and then |
| 55 | // no coded region is KeyOK. The home is exempt from every eligibility fact. |
| 56 | // A coded candidate's AWS region comes only from its fully eligible boxes |
| 57 | // (codex plan-R2 2); NoKeyProven needs a complete snapshot (codex plan-R2 |
| 58 | // 1). |
| 59 | // |
| 60 | // A coded region is Eligible only when BOTH hold (final review F2): |
| 61 | // |
| 62 | // - at least one fresh box publishing it passes the whole predicate: |
| 63 | // a dial address, hibernation_configured, serves_shared_pool and |
| 64 | // accepts_inbound all positively true, and the deployment's key; |
| 65 | // - no fresh box publishing it that reconcile would ADMIT for the record |
| 66 | // fails that predicate. Absent and unknown fields fail it. |
| 67 | // |
| 68 | // A record names a region, not a box. internal/provision's reconcile pass |
| 69 | // (inScope, and the CARLOS_INBOUND=none refusal) admits every box whose |
| 70 | // region matches, whose pools serve the shared pool and which takes |
| 71 | // inbound; it never asks whether the box holds the key or has its |
| 72 | // activator wired, so a keyless or activator-less sibling can win the lease |
| 73 | // and wake the instance with nothing. Every box that could serve the record |
| 74 | // must therefore be able to. A box that POSITIVELY publishes |
| 75 | // serves_shared_pool false (a named-pool-only box) or accepts_inbound |
| 76 | // false (CARLOS_INBOUND=none) can never be handed the record, so it neither |
| 77 | // qualifies nor disqualifies the region (Ruling F2, refined). Absent is not |
| 78 | // false: a writer predating the field is on an unknown config, and |
| 79 | // reconcile's defaults (CARLOS_POOLS unset serves the shared pool, |
| 80 | // CARLOS_INBOUND unset accepts) admit it, so it counts as possibly |
| 81 | // admitted and fails. Stale boxes count for neither side (F1): they are |
| 82 | // not evidence that a region works, and not evidence that it is broken. |
| 83 | func regionCandidates(snap edgesSnapshot, homeName, expectedFP string) []regionCandidate { |
| 84 | type acc struct { |
| 85 | c regionCandidate |
| 86 | regions map[string]int |
| 87 | hostingBoxes int |
| 88 | unknownWriter, keyFound bool |
| 89 | anyEligible bool |
| 90 | allEligible bool // over fresh boxes; vacuously true |
| 91 | } |
| 92 | by := map[string]*acc{} |
| 93 | get := func(code string, home bool) *acc { |
| 94 | a, ok := by[code] |
| 95 | if !ok { |
| 96 | a = &acc{c: regionCandidate{Code: code, Home: home}, regions: map[string]int{}, allEligible: true} |
| 97 | by[code] = a |
| 98 | } |
| 99 | return a |
| 100 | } |
| 101 | h := get("", true) |
| 102 | h.c.Reachable, h.c.Hosting, h.c.KeyOK, h.c.Eligible = true, true, true, true |
| 103 | for _, f := range snap.Files { |
| 104 | home := f.Placement == "" || (homeName != "" && f.Placement == homeName) |
| 105 | if !home && !provision.ValidRegion(f.Placement) { |
| 106 | continue // fleet/<name> and malformed codes are never candidates |
| 107 | } |
| 108 | code := f.Placement |
| 109 | if home { |
| 110 | code = "" |
| 111 | } |
| 112 | a := get(code, home) |
| 113 | if home { |
| 114 | if f.EIP != "" && f.Region != "" { |
| 115 | a.regions[f.Region]++ |
| 116 | } |
| 117 | continue |
| 118 | } |
| 119 | reachable := f.EIP != "" |
| 120 | hosting := isTrue(f.HibernationConfigured) || f.HibernationConfigured == nil |
| 121 | keyOK := expectedFP != "" && f.SecretsKeyFP == expectedFP |
| 122 | if reachable && hosting { |
| 123 | // R9's negative proof counts every box with a dial address, |
| 124 | // fresh or not: a lingering keyless file still describes that |
| 125 | // box's configuration, and the explicit door this proof serves |
| 126 | // admits boxes from the files alone (placements), with no |
| 127 | // liveness rule of its own. Liveness changes nothing there. |
| 128 | a.hostingBoxes++ |
| 129 | a.keyFound = a.keyFound || keyOK |
| 130 | a.unknownWriter = a.unknownWriter || f.AcceptsInbound == nil |
| 131 | } |
| 132 | if !f.Fresh { |
| 133 | // F1: a box with no fresh status snapshot is not evidence of |
| 134 | // anything but the code's existence (reason "absent"). Its |
| 135 | // edges file lingers after retirement; see readLiveEdgesSnapshot. |
| 136 | continue |
| 137 | } |
| 138 | // A fresh box with no dial address still fails the predicate (F2): |
| 139 | // reconcile admits it all the same, so it can take the record and |
| 140 | // nothing can reach it. It just does not make the region Reachable. |
| 141 | eligible := reachable && isTrue(f.HibernationConfigured) && isTrue(f.ServesSharedPool) && isTrue(f.AcceptsInbound) && keyOK |
| 142 | a.anyEligible = a.anyEligible || eligible |
| 143 | // Only a box reconcile would admit can take the region out: one |
| 144 | // that does not positively refuse the shared pool or inbound. |
| 145 | // This mirrors poolServed(cfg.Pools, "") and cfg.Inbound != |
| 146 | // InboundNone, the two facts SharedInstanceEligibility publishes. |
| 147 | if !isFalse(f.ServesSharedPool) && !isFalse(f.AcceptsInbound) { |
| 148 | a.allEligible = a.allEligible && eligible |
| 149 | } |
| 150 | if !reachable { |
| 151 | continue |
| 152 | } |
| 153 | a.c.Reachable = true |
| 154 | a.c.Hosting = a.c.Hosting || hosting |
| 155 | if hosting { |
| 156 | a.c.KeyOK = a.c.KeyOK || keyOK |
| 157 | } |
| 158 | if eligible && f.Region != "" { |
| 159 | a.regions[f.Region]++ |
| 160 | } |
| 161 | } |
| 162 | // Complete evidence is the bar for every non-home region (re-review |
| 163 | // r1): with one edges file unread, or one status snapshot unread, a |
| 164 | // failing sibling may be the box that is missing, and the keyed box |
| 165 | // that WAS read would speak for the region alone. Whole-request scope, |
| 166 | // deliberately: simpler, and stricter than per region. |
| 167 | evidence := snap.EvidenceComplete() |
| 168 | if !evidence && len(by) > 1 { |
| 169 | if _, said := livenessSaid.LoadOrStore("incomplete", true); !said { |
| 170 | log.Printf("console: an incomplete read of the edges files or box snapshots (edges complete %v, snapshots complete %v), so no region outside the home is automatically eligible this request; said once", snap.Complete, snap.BoxesComplete) |
| 171 | } |
| 172 | } |
| 173 | out := make([]regionCandidate, 0, len(by)) |
| 174 | for _, a := range by { |
| 175 | if !a.c.Home { |
| 176 | a.c.Eligible = evidence && a.anyEligible && a.allEligible |
| 177 | } |
| 178 | a.c.AWSRegion = majority(a.c.Code, a.regions) |
| 179 | a.c.NoKeyProven = !a.c.Home && snap.Complete && expectedFP != "" && a.hostingBoxes > 0 && !a.unknownWriter && !a.keyFound |
| 180 | if a.c.Eligible && !a.c.Home { // a home without a point yields "default" (spec §6.3), not "left out" |
| 181 | if _, ok := geo.LookupAWSRegion(a.c.AWSRegion); !ok { |
| 182 | if _, said := noPointSaid.LoadOrStore(a.c.Code, true); !said { |
| 183 | log.Printf("console: region %q has no point in the shipped AWS region table (AWS region %q), so region guesses leave it out", a.c.Code, a.c.AWSRegion) |
| 184 | } |
| 185 | } |
| 186 | } |
| 187 | out = append(out, a.c) |
| 188 | } |
| 189 | sort.Slice(out, func(i, j int) bool { return out[i].Code < out[j].Code }) |
| 190 | return out |
| 191 | } |
| 192 | |
| 193 | // regionCands is regionCandidates with this console's home and expected |
| 194 | // secrets fingerprint. The snapshot must come from liveEdges: a plain |
| 195 | // readEdgesSnapshot marks no box fresh, so every coded region would read |
| 196 | // as absent. |
| 197 | func (s *Server) regionCands(snap edgesSnapshot) []regionCandidate { |
| 198 | return regionCandidates(snap, s.cfg.HomeRegion, s.expectedSecretsFP()) |
| 199 | } |
| 200 | |
| 201 | // liveEdges is the one read every automatic-eligibility caller makes: the |
| 202 | // edges files joined with box liveness as of now (F1). |
| 203 | func (s *Server) liveEdges(ctx context.Context) (edgesSnapshot, error) { |
| 204 | return readLiveEdgesSnapshot(ctx, s.cfg.Releases, time.Now()) |
| 205 | } |
| 206 | |
| 207 | // candsOrHome is the candidates from a fresh snapshot, or the home alone |
| 208 | // when the listing fails (or there is no bucket): for displays and guesses |
| 209 | // that must not fail. |
| 210 | func (s *Server) candsOrHome(ctx context.Context) []regionCandidate { |
| 211 | if s.cfg.Releases == nil { |
| 212 | return homeOnlyCandidates() |
| 213 | } |
| 214 | snap, err := s.liveEdges(ctx) |
| 215 | if err != nil { |
| 216 | return homeOnlyCandidates() |
| 217 | } |
| 218 | return s.regionCands(snap) |
| 219 | } |
| 220 | |
| 221 | // majority picks the most common AWS region, ties to the lexically |
| 222 | // smallest, and logs a disagreement once per code. |
| 223 | func majority(code string, regions map[string]int) string { |
| 224 | best, n := "", 0 |
| 225 | for r, c := range regions { |
| 226 | if c > n || (c == n && r < best) { |
| 227 | best, n = r, c |
| 228 | } |
| 229 | } |
| 230 | if len(regions) > 1 { |
| 231 | if _, said := disagreeSaid.LoadOrStore(code, true); !said { |
| 232 | log.Printf("console: boxes for region %q disagree on their AWS region %v; using %s", code, regions, best) |
| 233 | } |
| 234 | } |
| 235 | return best |
| 236 | } |
| 237 | |
| 238 | func homeOnlyCandidates() []regionCandidate { |
| 239 | return []regionCandidate{{Code: "", Home: true, Reachable: true, Hosting: true, KeyOK: true, Eligible: true}} |
| 240 | } |
| 241 | |
| 242 | func findCandidate(cands []regionCandidate, code string) (regionCandidate, bool) { |
| 243 | for _, c := range cands { |
| 244 | if c.Code == code { |
| 245 | return c, true |
| 246 | } |
| 247 | } |
| 248 | return regionCandidate{}, false |
| 249 | } |
| 250 | |
| 251 | func eligibleCount(cands []regionCandidate) int { |
| 252 | n := 0 |
| 253 | for _, c := range cands { |
| 254 | if c.Eligible { |
| 255 | n++ |
| 256 | } |
| 257 | } |
| 258 | return n |
| 259 | } |
| 260 | |
| 261 | type policyState int |
| 262 | |
| 263 | const ( |
| 264 | policyAbsent policyState = iota |
| 265 | policyKnown |
| 266 | policyUnreadable |
| 267 | ) |
| 268 | |
| 269 | // regionPolicy is what the manifest and the fleet placement say about |
| 270 | // where an instance may land (spec §6.1). |
| 271 | type regionPolicy struct { |
| 272 | State policyState |
| 273 | M *provision.Manifest |
| 274 | Placed string |
| 275 | } |
| 276 | |
| 277 | // appPolicy reads the manifest directly (never instanceManifest, which |
| 278 | // answers the caller itself) and the fleet placement. Any failure is |
| 279 | // policyUnreadable: an unrestricted policy is never invented. |
| 280 | func (s *Server) appPolicy(ctx context.Context, acct *Account, app string) regionPolicy { |
| 281 | var p regionPolicy |
| 282 | if s.cfg.Releases == nil { |
| 283 | return regionPolicy{State: policyUnreadable} // no bucket: nothing to read, so nothing is allowed |
| 284 | } |
| 285 | m, err := provision.GetManifest(ctx, s.cfg.Releases, acct.Sqid, app) |
| 286 | switch { |
| 287 | case err == nil: |
| 288 | p.State, p.M = policyKnown, m |
| 289 | case errors.Is(err, os.ErrNotExist): |
| 290 | p.State = policyAbsent |
| 291 | default: |
| 292 | log.Printf("console: region policy %s/%s: read manifest: %v", acct.Sqid, app, err) |
| 293 | return regionPolicy{State: policyUnreadable} |
| 294 | } |
| 295 | reg, err := openRegistry(s.cfg.Registry) |
| 296 | if err != nil { |
| 297 | log.Printf("console: region policy %s/%s: open registry: %v", acct.Sqid, app, err) |
| 298 | return regionPolicy{State: policyUnreadable} |
| 299 | } |
| 300 | placed, err := s.appPlacement(reg, acct, app) |
| 301 | if err != nil { |
| 302 | log.Printf("console: region policy %s/%s: placement: %v", acct.Sqid, app, err) |
| 303 | return regionPolicy{State: policyUnreadable} |
| 304 | } |
| 305 | p.Placed = placed |
| 306 | return p |
| 307 | } |
| 308 | |
| 309 | func (p regionPolicy) pooled() bool { return p.State == policyKnown && p.M.Pool != "" } |
| 310 | |
| 311 | // allows is the door's own allow-list predicate on a record-vocabulary code. |
| 312 | func (p regionPolicy) allows(code string) bool { |
| 313 | switch p.State { |
| 314 | case policyKnown: |
| 315 | return instanceRegionAllowed(p.M.Regions, code) |
| 316 | case policyAbsent: |
| 317 | return true |
| 318 | } |
| 319 | return false |
| 320 | } |
| 321 | |
amadan